@rainbow-o23/n3 1.0.55 → 1.0.57

package/README.md

@@ -492,9 +492,16 @@ Array<TypeOrmEntityToLoad>;
 
 ##### Environment Parameters
 
-| Name                    | Type   | Default Value | Comments    |
-|-------------------------|--------|---------------|-------------|
-| `typeorm.DB.fetch.size` | number | 20            | Fetch size. |
+| Name                              | Type    | Default Value | Comments                               |
+|-----------------------------------|---------|---------------|----------------------------------------|
+| `typeorm.DB.fetch.size`           | number  | 20            | Fetch size.                            |
+| `typeorm.DB.stream.pause.enabled` | boolean | false         | Pause and resume result stream or not. |
+
+##### Constructor Parameters
+
+| Name               | Type    | Default Value | Comments                               |
+|--------------------|---------|---------------|----------------------------------------|
+| pauseStreamEnabled | boolean |               | Pause and resume result stream or not. |
 
 ##### Request and Response
 

package/index.cjs

@@ -2223,6 +2223,7 @@ class TypeOrmLoadManyBySQLUseCursorPipelineStep extends AbstractTypeOrmBySQLPipe
     _streamToSnippet;
     _streamToFunc;
     _stepBuilders;
+    _pauseStreamEnabled;
     constructor(options) {
         super(options);
         const config = this.getConfig();
@@ -2237,6 +2238,7 @@ class TypeOrmLoadManyBySQLUseCursorPipelineStep extends AbstractTypeOrmBySQLPipe
             }
         });
         this._stepBuilders = options.steps;
+        this._pauseStreamEnabled = options.pauseStreamEnabled ?? config.getBoolean(`typeorm.${this.getDataSourceName()}.stream.pause.enabled`, false);
     }
     getFetchSize() {
         return this._fetchSize;
@@ -2250,6 +2252,9 @@ class TypeOrmLoadManyBySQLUseCursorPipelineStep extends AbstractTypeOrmBySQLPipe
     getStepBuilders() {
         return this._stepBuilders ?? [];
     }
+    isPauseStreamEnabled() {
+        return this._pauseStreamEnabled;
+    }
     async doPerform(basis, request) {
         const { sql, params } = this.getSql(basis, basis?.params);
         return await this.autoTrans(async (runner) => {
@@ -2312,7 +2317,9 @@ class TypeOrmLoadManyBySQLUseCursorPipelineStep extends AbstractTypeOrmBySQLPipe
                     reject(e);
                 });
                 readable.on('data', async (data) => {
-                    readable.pause();
+                    if (this.isPauseStreamEnabled()) {
+                        readable.pause();
+                    }
                     rows.push(data);
                     await pipe({
                         resolve, reject: async (e) => {
@@ -2320,7 +2327,9 @@ class TypeOrmLoadManyBySQLUseCursorPipelineStep extends AbstractTypeOrmBySQLPipe
                             reject(e);
                         }, end: false
                     });
-                    readable.resume();
+                    if (this.isPauseStreamEnabled()) {
+                        readable.resume();
+                    }
                 });
             };
             return new Promise((resolve, reject) => read({ resolve, reject }));

package/index.js

@@ -2221,6 +2221,7 @@ class TypeOrmLoadManyBySQLUseCursorPipelineStep extends AbstractTypeOrmBySQLPipe
     _streamToSnippet;
     _streamToFunc;
     _stepBuilders;
+    _pauseStreamEnabled;
     constructor(options) {
         super(options);
         const config = this.getConfig();
@@ -2235,6 +2236,7 @@ class TypeOrmLoadManyBySQLUseCursorPipelineStep extends AbstractTypeOrmBySQLPipe
             }
         });
         this._stepBuilders = options.steps;
+        this._pauseStreamEnabled = options.pauseStreamEnabled ?? config.getBoolean(`typeorm.${this.getDataSourceName()}.stream.pause.enabled`, false);
     }
     getFetchSize() {
         return this._fetchSize;
@@ -2248,6 +2250,9 @@ class TypeOrmLoadManyBySQLUseCursorPipelineStep extends AbstractTypeOrmBySQLPipe
     getStepBuilders() {
         return this._stepBuilders ?? [];
     }
+    isPauseStreamEnabled() {
+        return this._pauseStreamEnabled;
+    }
     async doPerform(basis, request) {
         const { sql, params } = this.getSql(basis, basis?.params);
         return await this.autoTrans(async (runner) => {
@@ -2310,7 +2315,9 @@ class TypeOrmLoadManyBySQLUseCursorPipelineStep extends AbstractTypeOrmBySQLPipe
                     reject(e);
                 });
                 readable.on('data', async (data) => {
-                    readable.pause();
+                    if (this.isPauseStreamEnabled()) {
+                        readable.pause();
+                    }
                     rows.push(data);
                     await pipe({
                         resolve, reject: async (e) => {
@@ -2318,7 +2325,9 @@ class TypeOrmLoadManyBySQLUseCursorPipelineStep extends AbstractTypeOrmBySQLPipe
                             reject(e);
                         }, end: false
                     });
-                    readable.resume();
+                    if (this.isPauseStreamEnabled()) {
+                        readable.resume();
+                    }
                 });
             };
             return new Promise((resolve, reject) => read({ resolve, reject }));

package/lib/typeorm-step/typeorm-load-many-by-sql-use-cursor-step.d.ts

@@ -8,17 +8,20 @@ export interface TypeOrmLoadManyBySQLUseCursorPipelineStepOptions<In = PipelineS
     fetchSize?: number;
     streamTo?: ScriptFuncOrBody<StreamToFunc<In, Item>>;
     steps?: Array<PipelineStepBuilder>;
+    pauseStreamEnabled?: boolean;
 }
 export declare class TypeOrmLoadManyBySQLUseCursorPipelineStep<In = PipelineStepPayload, Out = PipelineStepPayload, InFragment = Undefinable<TypeOrmLoadBasis>, OutFragment = Out, Item = any> extends AbstractTypeOrmBySQLPipelineStep<In, Out, Undefinable<TypeOrmLoadBasis>, OutFragment> {
     private readonly _fetchSize;
     private readonly _streamToSnippet;
     private readonly _streamToFunc;
     private readonly _stepBuilders;
+    private readonly _pauseStreamEnabled;
     constructor(options: TypeOrmLoadManyBySQLUseCursorPipelineStepOptions<In, Out, InFragment, OutFragment>);
     protected getFetchSize(): number;
     getStreamToSnippet(): ScriptFuncOrBody<StreamToFunc<In, Item>>;
     protected generateVariableNames(): Array<string>;
     protected getStepBuilders(): Array<PipelineStepBuilder>;
+    protected isPauseStreamEnabled(): boolean;
     protected doPerform(basis: Undefinable<TypeOrmLoadBasis>, request: PipelineStepData<In>): Promise<Undefinable<OutFragment>>;
     protected getFetchDataVariableName(): string;
     protected getRequestVariableName(): string;

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@rainbow-o23/n3",
-	"version": "1.0.55",
+	"version": "1.0.57",
 	"description": "o23 pipelines",
 	"main": "index.cjs",
 	"module": "index.js",
@@ -21,7 +21,7 @@
 		"url": "https://github.com/InsureMO/rainbow-o23/issues"
 	},
 	"dependencies": {
-		"@rainbow-o23/n1": "1.0.55",
+		"@rainbow-o23/n1": "1.0.57",
 		"node-fetch": "2.6.7",
 		"typeorm": "^0.3.20",
 		"typescript": "5.5.4"

package/src/lib/typeorm-step/typeorm-load-many-by-sql-use-cursor-step.ts

@@ -22,6 +22,7 @@ export interface TypeOrmLoadManyBySQLUseCursorPipelineStepOptions<In = PipelineS
 	fetchSize?: number;
 	streamTo?: ScriptFuncOrBody<StreamToFunc<In, Item>>;
 	steps?: Array<PipelineStepBuilder>;
+	pauseStreamEnabled?: boolean;
 }
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -31,6 +32,7 @@ export class TypeOrmLoadManyBySQLUseCursorPipelineStep<In = PipelineStepPayload,
 	private readonly _streamToSnippet: ScriptFuncOrBody<StreamToFunc<In, Item>>;
 	private readonly _streamToFunc: StreamToFunc<In, Item>;
 	private readonly _stepBuilders: Array<PipelineStepBuilder>;
+	private readonly _pauseStreamEnabled: boolean;
 
 	public constructor(options: TypeOrmLoadManyBySQLUseCursorPipelineStepOptions<In, Out, InFragment, OutFragment>) {
 		super(options);
@@ -47,6 +49,10 @@ export class TypeOrmLoadManyBySQLUseCursorPipelineStep<In = PipelineStepPayload,
 			}
 		});
 		this._stepBuilders = options.steps;
+		// for unknown reason, in some environment if the pause and resume functions of readable invoked,
+		// only the first row are returns and end event invoked immediately
+		// so default disable it.
+		this._pauseStreamEnabled = options.pauseStreamEnabled ?? config.getBoolean(`typeorm.${this.getDataSourceName()}.stream.pause.enabled`, false);
 	}
 
 	protected getFetchSize(): number {
@@ -65,6 +71,10 @@ export class TypeOrmLoadManyBySQLUseCursorPipelineStep<In = PipelineStepPayload,
 		return this._stepBuilders ?? [];
 	}
 
+	protected isPauseStreamEnabled(): boolean {
+		return this._pauseStreamEnabled;
+	}
+
 	protected async doPerform(basis: Undefinable<TypeOrmLoadBasis>, request: PipelineStepData<In>): Promise<Undefinable<OutFragment>> {
 		const {sql, params} = this.getSql(basis, basis?.params);
 		return await this.autoTrans<OutFragment>(async (runner) => {
@@ -136,7 +146,9 @@ export class TypeOrmLoadManyBySQLUseCursorPipelineStep<In = PipelineStepPayload,
 					reject(e);
 				});
 				readable.on('data', async (data) => {
-					readable.pause();
+					if (this.isPauseStreamEnabled()) {
+						readable.pause();
+					}
 					rows.push(data);
 					await pipe({
 						resolve, reject: async (e: Error) => {
@@ -144,7 +156,9 @@ export class TypeOrmLoadManyBySQLUseCursorPipelineStep<In = PipelineStepPayload,
 							reject(e);
 						}, end: false
 					});
-					readable.resume();
+					if (this.isPauseStreamEnabled()) {
+						readable.resume();
+					}
 				});
 			};
 			return new Promise<OutFragment>((resolve, reject) => read({resolve, reject}));

package/test/step/typeorm-by-cursor.test.ts

@@ -9,7 +9,8 @@ describe('TypeORM Cursor Suite', () => {
 		process.env.CFG_TYPEORM_TEST_HOST = 'localhost';
 		process.env.CFG_TYPEORM_TEST_USERNAME = 'o23';
 		process.env.CFG_TYPEORM_TEST_PASSWORD = 'o23';
-		const type = 'pg' + 'sql';
+		// process.env.CFG_TYPEORM_TEST_STREAM_PAUSE_ENABLED = 'true'
+		const type = 'my' + 'sql';
 		if (type === 'mysql') {
 			process.env.CFG_TYPEORM_TEST_TYPE = 'mysql';
 			process.env.CFG_TYPEORM_TEST_PORT = '3306';