@rainbow-o23/n3 1.0.59 → 1.0.61

package/README.md

@@ -492,10 +492,10 @@ Array<TypeOrmEntityToLoad>;
 
 ##### Environment Parameters
 
-| Name                              | Type    | Default Value | Comments                               |
-|-----------------------------------|---------|---------------|----------------------------------------|
-| `typeorm.DB.fetch.size`           | number  | 20            | Fetch size.                            |
-| `typeorm.DB.stream.pause.enabled` | boolean | false         | Pause and resume result stream or not. |
+| Name                              | Type    | Default Value | Comments                                                                                      |
+|-----------------------------------|---------|---------------|-----------------------------------------------------------------------------------------------|
+| `typeorm.DB.fetch.size`           | number  | 20            | Fetch size.                                                                                   |
+| `typeorm.DB.stream.pause.enabled` | boolean | false         | Pause and resume result stream or not. For Mysql, use default `false`. For Pgsql, use `true`. |
 
 ##### Constructor Parameters
 

package/index.cjs

@@ -2401,46 +2401,34 @@ class TypeOrmLoadManyBySQLUseCursorPipelineStep extends AbstractTypeOrmBySQLPipe
         return await this.autoTrans(async (runner) => {
             const results = [];
             const rows = [];
+            const fetchSize = this.getFetchSize();
             let cursorRound = 0;
-            const pipe = async ({ resolve, reject, end }) => {
-                if (!end && rows.length < this.getFetchSize()) {
-                    return;
+            const pipe = async ({ rows, end }) => {
+                const contentForSub = await this._streamToFunc(rows, request, this.getHelpers(), this.getHelpers());
+                let resultContent;
+                if (this.getStepBuilders().length === 0) {
+                    resultContent = contentForSub;
                 }
-                try {
-                    const contentForSub = await this._streamToFunc([...rows], request, this.getHelpers(), this.getHelpers());
-                    rows.length = 0;
-                    let resultContent;
-                    if (this.getStepBuilders().length === 0) {
-                        resultContent = contentForSub;
-                    }
-                    else {
-                        const sets = new PipelineStepSets({
-                            ...this.buildStepOptions(), name: this.getName(), steps: this.getStepBuilders()
-                        });
-                        const { content: _, $context, ...rest } = request;
-                        const contextForSub = $context.temporaryWith({
-                            $typeOrmCursorRound: cursorRound, $typeOrmCursorEnd: end
-                        });
-                        const requestForSub = { ...rest, $context: contextForSub, content: contentForSub };
-                        const result = await sets.perform(requestForSub);
-                        const { content } = result;
-                        resultContent = content;
-                    }
-                    cursorRound = cursorRound + 1;
-                    if (resultContent == null || resultContent == n1.PIPELINE_STEP_RETURN_NULL) {
-                    }
-                    else if (Array.isArray(resultContent)) {
-                        results.push(...resultContent);
-                    }
-                    else {
-                        results.push(resultContent);
-                    }
+                else {
+                    const sets = new PipelineStepSets({
+                        ...this.buildStepOptions(), name: this.getName(), steps: this.getStepBuilders()
+                    });
+                    const { content: _, $context, ...rest } = request;
+                    const contextForSub = $context.temporaryWith({
+                        $typeOrmCursorRound: cursorRound, $typeOrmCursorEnd: end
+                    });
+                    const requestForSub = { ...rest, $context: contextForSub, content: contentForSub };
+                    const result = await sets.perform(requestForSub);
+                    const { content } = result;
+                    resultContent = content;
                 }
-                catch (e) {
-                    reject(e);
+                cursorRound = cursorRound + 1;
+                if (resultContent == null || resultContent == n1.PIPELINE_STEP_RETURN_NULL) ;
+                else if (Array.isArray(resultContent)) {
+                    results.push(...resultContent);
                 }
-                if (end) {
-                    resolve(results);
+                else {
+                    results.push(resultContent);
                 }
             };
             const close = async (readable) => {
@@ -2452,9 +2440,26 @@ class TypeOrmLoadManyBySQLUseCursorPipelineStep extends AbstractTypeOrmBySQLPipe
                 }
             };
             const read = async ({ resolve, reject }) => {
+                const pipes = [];
                 const readable = await runner.stream(sql, params, async () => {
                     await close(readable);
-                    await pipe({ resolve, reject, end: true });
+                    const data = [...rows];
+                    rows.length = 0;
+                    const last = async () => {
+                        try {
+                            await pipe({ rows: data, end: true });
+                            resolve(results);
+                        }
+                        catch (e) {
+                            reject(e);
+                        }
+                    };
+                    if (pipes.length !== 0) {
+                        pipes.push(last);
+                    }
+                    else {
+                        await last();
+                    }
                 }, async (e) => {
                     await close(readable);
                     reject(e);
@@ -2464,12 +2469,28 @@ class TypeOrmLoadManyBySQLUseCursorPipelineStep extends AbstractTypeOrmBySQLPipe
                         readable.pause();
                     }
                     rows.push(data);
-                    await pipe({
-                        resolve, reject: async (e) => {
-                            await close(readable);
-                            reject(e);
-                        }, end: false
-                    });
+                    if (rows.length < fetchSize) ;
+                    else {
+                        const data = [...rows];
+                        rows.length = 0;
+                        pipes.push(async () => {
+                            try {
+                                await pipe({ rows: data, end: false });
+                            }
+                            catch (e) {
+                                await close(readable);
+                                reject(e);
+                                return;
+                            }
+                            pipes.shift();
+                            if (pipes.length !== 0) {
+                                pipes[0]();
+                            }
+                        });
+                        if (pipes.length === 1) {
+                            await pipes[0]();
+                        }
+                    }
                     if (this.isPauseStreamEnabled()) {
                         readable.resume();
                     }

package/index.js

@@ -2399,46 +2399,34 @@ class TypeOrmLoadManyBySQLUseCursorPipelineStep extends AbstractTypeOrmBySQLPipe
         return await this.autoTrans(async (runner) => {
             const results = [];
             const rows = [];
+            const fetchSize = this.getFetchSize();
             let cursorRound = 0;
-            const pipe = async ({ resolve, reject, end }) => {
-                if (!end && rows.length < this.getFetchSize()) {
-                    return;
+            const pipe = async ({ rows, end }) => {
+                const contentForSub = await this._streamToFunc(rows, request, this.getHelpers(), this.getHelpers());
+                let resultContent;
+                if (this.getStepBuilders().length === 0) {
+                    resultContent = contentForSub;
                 }
-                try {
-                    const contentForSub = await this._streamToFunc([...rows], request, this.getHelpers(), this.getHelpers());
-                    rows.length = 0;
-                    let resultContent;
-                    if (this.getStepBuilders().length === 0) {
-                        resultContent = contentForSub;
-                    }
-                    else {
-                        const sets = new PipelineStepSets({
-                            ...this.buildStepOptions(), name: this.getName(), steps: this.getStepBuilders()
-                        });
-                        const { content: _, $context, ...rest } = request;
-                        const contextForSub = $context.temporaryWith({
-                            $typeOrmCursorRound: cursorRound, $typeOrmCursorEnd: end
-                        });
-                        const requestForSub = { ...rest, $context: contextForSub, content: contentForSub };
-                        const result = await sets.perform(requestForSub);
-                        const { content } = result;
-                        resultContent = content;
-                    }
-                    cursorRound = cursorRound + 1;
-                    if (resultContent == null || resultContent == PIPELINE_STEP_RETURN_NULL) {
-                    }
-                    else if (Array.isArray(resultContent)) {
-                        results.push(...resultContent);
-                    }
-                    else {
-                        results.push(resultContent);
-                    }
+                else {
+                    const sets = new PipelineStepSets({
+                        ...this.buildStepOptions(), name: this.getName(), steps: this.getStepBuilders()
+                    });
+                    const { content: _, $context, ...rest } = request;
+                    const contextForSub = $context.temporaryWith({
+                        $typeOrmCursorRound: cursorRound, $typeOrmCursorEnd: end
+                    });
+                    const requestForSub = { ...rest, $context: contextForSub, content: contentForSub };
+                    const result = await sets.perform(requestForSub);
+                    const { content } = result;
+                    resultContent = content;
                 }
-                catch (e) {
-                    reject(e);
+                cursorRound = cursorRound + 1;
+                if (resultContent == null || resultContent == PIPELINE_STEP_RETURN_NULL) ;
+                else if (Array.isArray(resultContent)) {
+                    results.push(...resultContent);
                 }
-                if (end) {
-                    resolve(results);
+                else {
+                    results.push(resultContent);
                 }
             };
             const close = async (readable) => {
@@ -2450,9 +2438,26 @@ class TypeOrmLoadManyBySQLUseCursorPipelineStep extends AbstractTypeOrmBySQLPipe
                 }
             };
             const read = async ({ resolve, reject }) => {
+                const pipes = [];
                 const readable = await runner.stream(sql, params, async () => {
                     await close(readable);
-                    await pipe({ resolve, reject, end: true });
+                    const data = [...rows];
+                    rows.length = 0;
+                    const last = async () => {
+                        try {
+                            await pipe({ rows: data, end: true });
+                            resolve(results);
+                        }
+                        catch (e) {
+                            reject(e);
+                        }
+                    };
+                    if (pipes.length !== 0) {
+                        pipes.push(last);
+                    }
+                    else {
+                        await last();
+                    }
                 }, async (e) => {
                     await close(readable);
                     reject(e);
@@ -2462,12 +2467,28 @@ class TypeOrmLoadManyBySQLUseCursorPipelineStep extends AbstractTypeOrmBySQLPipe
                         readable.pause();
                     }
                     rows.push(data);
-                    await pipe({
-                        resolve, reject: async (e) => {
-                            await close(readable);
-                            reject(e);
-                        }, end: false
-                    });
+                    if (rows.length < fetchSize) ;
+                    else {
+                        const data = [...rows];
+                        rows.length = 0;
+                        pipes.push(async () => {
+                            try {
+                                await pipe({ rows: data, end: false });
+                            }
+                            catch (e) {
+                                await close(readable);
+                                reject(e);
+                                return;
+                            }
+                            pipes.shift();
+                            if (pipes.length !== 0) {
+                                pipes[0]();
+                            }
+                        });
+                        if (pipes.length === 1) {
+                            await pipes[0]();
+                        }
+                    }
                     if (this.isPauseStreamEnabled()) {
                         readable.resume();
                     }

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@rainbow-o23/n3",
-	"version": "1.0.59",
+	"version": "1.0.61",
 	"description": "o23 pipelines",
 	"main": "index.cjs",
 	"module": "index.js",
@@ -21,9 +21,9 @@
 		"url": "https://github.com/InsureMO/rainbow-o23/issues"
 	},
 	"dependencies": {
-		"@rainbow-o23/n1": "1.0.59",
+		"@rainbow-o23/n1": "1.0.61",
 		"node-fetch": "2.6.7",
-		"typeorm": "^0.3.20",
+		"typeorm": "^0.3.28",
 		"typescript": "5.5.4"
 	},
 	"devDependencies": {
@@ -42,7 +42,7 @@
 		"better-sqlite3": "^11.5.0",
 		"eslint": "^9.8.0",
 		"mssql": "^11.0.1",
-		"mysql2": "^3.11.4",
+		"mysql2": "^3.15.3",
 		"oracledb": "^6.6.0",
 		"pg": "^8.13.1",
 		"pg-query-stream": "^4.7.1",

package/src/lib/typeorm-step/typeorm-load-many-by-sql-use-cursor-step.ts

@@ -80,52 +80,39 @@ export class TypeOrmLoadManyBySQLUseCursorPipelineStep<In = PipelineStepPayload,
 		return await this.autoTrans<OutFragment>(async (runner) => {
 			const results = [];
 			const rows = [];
+			const fetchSize = this.getFetchSize();
 			let cursorRound = 0;
-			const pipe = async ({resolve, reject, end}) => {
-				if (!end && rows.length < this.getFetchSize()) {
-					// not end, and size not meet the fresh required
-					// do nothing, wait for next
-					return;
-				}
-				try {
-					// get data from cache
-					const contentForSub = await this._streamToFunc([...rows], request, this.getHelpers(), this.getHelpers());
-					// clear cache
-					rows.length = 0;
-					// eslint-disable-next-line @typescript-eslint/no-explicit-any
-					let resultContent: any;
-					if (this.getStepBuilders().length === 0) {
-						// no sub step, use content as result
-						resultContent = contentForSub;
-					} else {
-						// create a step sets to run
-						const sets = new PipelineStepSets({
-							...this.buildStepOptions(), name: this.getName(), steps: this.getStepBuilders()
-						});
-						// eslint-disable-next-line @typescript-eslint/no-unused-vars
-						const {content: _, $context, ...rest} = request;
-						// pass a cursor end indicator to sub steps
-						const contextForSub = $context.temporaryWith({
-							$typeOrmCursorRound: cursorRound, $typeOrmCursorEnd: end
-						});
-						const requestForSub = {...rest, $context: contextForSub, content: contentForSub};
-						const result = await sets.perform(requestForSub);
-						const {content} = result;
-						resultContent = content;
-					}
-					cursorRound = cursorRound + 1;
-					if (resultContent == null || resultContent == PIPELINE_STEP_RETURN_NULL) {
-						// ignore
-					} else if (Array.isArray(resultContent)) {
-						results.push(...resultContent);
-					} else {
-						results.push(resultContent);
-					}
-				} catch (e) {
-					reject(e);
+			const pipe = async ({rows, end}) => {
+				// get data from cache
+				const contentForSub = await this._streamToFunc(rows, request, this.getHelpers(), this.getHelpers());
+				// eslint-disable-next-line @typescript-eslint/no-explicit-any
+				let resultContent: any;
+				if (this.getStepBuilders().length === 0) {
+					// no sub step, use content as result
+					resultContent = contentForSub;
+				} else {
+					// create a step sets to run
+					const sets = new PipelineStepSets({
+						...this.buildStepOptions(), name: this.getName(), steps: this.getStepBuilders()
+					});
+					// eslint-disable-next-line @typescript-eslint/no-unused-vars
+					const {content: _, $context, ...rest} = request;
+					// pass a cursor end indicator to sub steps
+					const contextForSub = $context.temporaryWith({
+						$typeOrmCursorRound: cursorRound, $typeOrmCursorEnd: end
+					});
+					const requestForSub = {...rest, $context: contextForSub, content: contentForSub};
+					const result = await sets.perform(requestForSub);
+					const {content} = result;
+					resultContent = content;
 				}
-				if (end) {
-					resolve(results);
+				cursorRound = cursorRound + 1;
+				if (resultContent == null || resultContent == PIPELINE_STEP_RETURN_NULL) {
+					// ignore
+				} else if (Array.isArray(resultContent)) {
+					results.push(...resultContent);
+				} else {
+					results.push(resultContent);
 				}
 			};
 			const close = async (readable: ReadStream) => {
@@ -138,10 +125,27 @@ export class TypeOrmLoadManyBySQLUseCursorPipelineStep<In = PipelineStepPayload,
 				}
 			};
 			const read = async ({resolve, reject}) => {
+				const pipes = [];
+
 				const readable = await runner.stream(sql, params, async () => {
 					// on end
 					await close(readable);
-					await pipe({resolve, reject, end: true});
+					const data = [...rows];
+					rows.length = 0;
+					const last = async () => {
+						try {
+							await pipe({rows: data, end: true});
+							resolve(results);
+						} catch (e) {
+							reject(e);
+						}
+					};
+					if (pipes.length !== 0) {
+						// there is not finished
+						pipes.push(last);
+					} else {
+						await last();
+					}
 				}, async (e: Error) => {
 					// on error
 					await close(readable);
@@ -152,12 +156,32 @@ export class TypeOrmLoadManyBySQLUseCursorPipelineStep<In = PipelineStepPayload,
 						readable.pause();
 					}
 					rows.push(data);
-					await pipe({
-						resolve, reject: async (e: Error) => {
-							await close(readable);
-							reject(e);
-						}, end: false
-					});
+					if (rows.length < fetchSize) {
+						// not end, and size not meet the fresh required
+						// do nothing, wait for next
+					} else {
+						const data = [...rows];
+						rows.length = 0;
+						pipes.push(async () => {
+							try {
+								await pipe({rows: data, end: false});
+							} catch (e) {
+								await close(readable);
+								reject(e);
+								return;
+							}
+							// drop the finished
+							pipes.shift();
+							// check there is more or not, if true, run the first one.
+							if (pipes.length !== 0) {
+								pipes[0]();
+							}
+						});
+						// if the only one, run it immediately
+						if (pipes.length === 1) {
+							await pipes[0]();
+						}
+					}
 					if (this.isPauseStreamEnabled()) {
 						readable.resume();
 					}

package/test/step/parallel-step.test.ts

@@ -1,4 +1,11 @@
-import {createConfig, createLogger, PipelineStep, PipelineStepBuilder, PipelineStepOptions} from '@rainbow-o23/n1';
+import {
+	createConfig,
+	createLogger,
+	PipelineExecutionContext,
+	PipelineStep,
+	PipelineStepBuilder,
+	PipelineStepOptions
+} from '@rainbow-o23/n1';
 import {ParallelPipelineStepSets, SnippetPipelineStep} from '../../src';
 
 const logger = createLogger();
@@ -19,7 +26,7 @@ test('Parallel Pipeline Step Test #1, + 100', async () => {
 			}
 		]
 	});
-	const request = {content: {base: 1}};
+	const request = {content: {base: 1}, $context: new PipelineExecutionContext()};
 	const response = await step.perform(request);
 	expect(response.content).toEqual([100, 200]);
 });

package/test/step/snippet-step.test.ts

@@ -1,4 +1,4 @@
-import {createConfig, createLogger} from '@rainbow-o23/n1';
+import {createConfig, createLogger, PipelineExecutionContext} from '@rainbow-o23/n1';
 import {SnippetPipelineStep} from '../../src';
 
 const logger = createLogger();
@@ -7,7 +7,7 @@ const config = createConfig(logger);
 test('Snippet Pipeline Step Test #1, + 100', async () => {
 	const snippet = 'return $factor.base + 100;';
 	const step = new SnippetPipelineStep({config, logger, snippet});
-	const request = {content: {base: 1}};
+	const request = {content: {base: 1}, $context: new PipelineExecutionContext()};
 	const response = await step.perform(request);
 	expect(response.content).toEqual(101);
 });
@@ -15,7 +15,7 @@ test('Snippet Pipeline Step Test #1, + 100', async () => {
 test('Snippet Pipeline Step Test #2, async + 100', async () => {
 	const snippet = 'return await new Promise(resolve => resolve($factor.base + 100));';
 	const step = new SnippetPipelineStep({config, logger, snippet});
-	const request = {content: {base: 1}};
+	const request = {content: {base: 1}, $context: new PipelineExecutionContext()};
 	const response = await step.perform(request);
 	expect(response.content).toEqual(101);
 });

package/test/step/snowflake-step.test.ts

@@ -1,4 +1,4 @@
-import {createConfig, createLogger} from '@rainbow-o23/n1';
+import {createConfig, createLogger, PipelineExecutionContext} from '@rainbow-o23/n1';
 import {SnowflakePipelineStep} from '../../src';
 
 const logger = createLogger();
@@ -6,7 +6,7 @@ const config = createConfig(logger);
 
 test('Snowflake Pipeline Step Test #1, replace content', async () => {
 	const step = new SnowflakePipelineStep({config, logger, mergeRequest: '$id'});
-	const request = {content: (void 0)};
+	const request = {content: (void 0), $context: new PipelineExecutionContext()};
 	const response = await step.perform(request);
 	expect(response.content).not.toBeNull();
 	expect(response.content.$id).not.toBeNull();
@@ -14,7 +14,7 @@ test('Snowflake Pipeline Step Test #1, replace content', async () => {
 
 test('Snowflake Pipeline Step Test #2, merge content', async () => {
 	const step = new SnowflakePipelineStep({config, logger, mergeRequest: '$id'});
-	const request = {content: {base: 1}};
+	const request = {content: {base: 1}, $context: new PipelineExecutionContext()};
 	const response = await step.perform(request);
 	expect(response.content).not.toBeNull();
 	expect(response.content.base).toEqual(1);

package/test/step/typeorm-by-cursor.test.ts

@@ -1,4 +1,4 @@
-import {createConfig, createLogger} from '@rainbow-o23/n1';
+import {createConfig, createLogger, PipelineExecutionContext} from '@rainbow-o23/n1';
 import {TypeOrmDataSourceHelper, TypeOrmLoadManyBySQLUseCursorPipelineStep} from '../../src';
 
 const logger = createLogger();
@@ -44,7 +44,7 @@ describe('TypeORM Cursor Suite', () => {
 			config, logger, dataSourceName: 'TEST', sql: 'SELECT * FROM T_O23_DB_CHANGE_LOG',
 			autonomous: true
 		});
-		const request = {content: (void 0)};
+		const request = {content: (void 0), $context: new PipelineExecutionContext()};
 		const response = await step.perform(request);
 		expect(response.content).not.toBeNull();
 		console.log(response.content);

package/test/step/typeorm-by-sql-autonomous.test.ts

@@ -1,4 +1,4 @@
-import {createConfig, createLogger} from '@rainbow-o23/n1';
+import {createConfig, createLogger, PipelineExecutionContext} from '@rainbow-o23/n1';
 import {Column, Entity, PrimaryColumn} from 'typeorm';
 import {
 	TypeOrmBulkSaveBySQLPipelineStep,
@@ -47,7 +47,7 @@ describe('TypeORM SQL Autonomous Suite', () => {
 			config, logger, dataSourceName: 'TEST', sql: 'SELECT ID id, CONTENT content FROM T_TEST_TABLE WHERE ID = ?',
 			autonomous: true
 		});
-		const request = {content: {params: [1]}};
+		const request = {content: {params: [1]}, $context: new PipelineExecutionContext()};
 		const response = await step.perform(request);
 		expect(response.content).not.toBeNull();
 		expect(response.content.id).toBe(1);
@@ -60,7 +60,7 @@ describe('TypeORM SQL Autonomous Suite', () => {
 			config, logger, dataSourceName: 'TEST', sql: 'INSERT INTO T_TEST_TABLE(ID, CONTENT) VALUES (?, ?)',
 			autonomous: true, mergeRequest: 'id'
 		});
-		const request = {content: {values: [3, 'another world!']}};
+		const request = {content: {values: [3, 'another world!']}, $context: new PipelineExecutionContext()};
 		const response = await step.perform(request);
 		expect(response.content).not.toBeNull();
 		expect(response.content.id).toBe(3);
@@ -72,7 +72,7 @@ describe('TypeORM SQL Autonomous Suite', () => {
 			config, logger, dataSourceName: 'TEST', sql: 'UPDATE T_TEST_TABLE SET CONTENT = ? WHERE ID = ?',
 			autonomous: true, mergeRequest: 'count'
 		});
-		const request = {content: {values: ['world #3!', 3]}};
+		const request = {content: {values: ['world #3!', 3]}, $context: new PipelineExecutionContext()};
 		const response = await step.perform(request);
 		expect(response.content).not.toBeNull();
 		// DON'T KNOW WHY THIS IS 3, SEEMS SHOULD BE 1 ACCORDING TO BETTER-SQLITE3 DOCUMENT
@@ -89,7 +89,8 @@ describe('TypeORM SQL Autonomous Suite', () => {
 		const request = {
 			content: {
 				items: [[4, 'world #4!'], [5, 'world #5!']]
-			}
+			},
+			$context: new PipelineExecutionContext()
 		};
 		const response = await step.perform(request);
 		expect(response.content).not.toBeNull();
@@ -104,7 +105,7 @@ describe('TypeORM SQL Autonomous Suite', () => {
 			config, logger, dataSourceName: 'TEST', sql: 'SELECT ID id, CONTENT content FROM T_TEST_TABLE',
 			autonomous: true
 		});
-		const request = {content: (void 0)};
+		const request = {content: (void 0), $context: new PipelineExecutionContext()};
 		const response = await step.perform(request);
 		expect(response.content).not.toBeNull();
 		expect(Array.isArray(response.content)).toBeTruthy();

package/test/step/typeorm-by-sql-transactional.test.ts

@@ -3,6 +3,7 @@ import {
 	createConfig,
 	createLogger,
 	PipelineCode,
+	PipelineExecutionContext,
 	PipelineStepData,
 	PipelineStepType,
 	Undefinable
@@ -201,7 +202,8 @@ describe('TypeORM SQL Transactional Suite', () => {
 				item3ChangeTo: {id: 3, content: 'world #3!'},
 				item4: {id: 4, content: 'world #4'},
 				item5: {id: 5, content: 'world #5'}
-			}
+			},
+			$context: new PipelineExecutionContext()
 		});
 		expect(response).not.toBeNull();
 	});