@rainbow-o23/n3 1.0.46 → 1.0.48

package/README.md

@@ -526,6 +526,28 @@ export interface TypeOrmLoadBasis extends TypeOrmBasis {
 Array<TypeOrmEntityToLoad>;
 ```
 
+#### Load Many by SQL, Use Cursor
+
+##### Request and Response
+
+```typescript
+// request
+export interface TypeOrmLoadBasis extends TypeOrmBasis {
+	params?: Array<TypeOrmEntityValue> | TypeOrmEntityToSave;
+}
+
+// response
+Array<any>;
+```
+
+By specifying `fetchSize`, each batch of data retrieved will execute sub-steps. Before executing the sub-steps, the data to be passed to it
+will be calculated using the `streamTo` function. If `streamTo` is not specified, the batch of data retrieved itself will be passed to the
+sub-steps. If the sub-steps is not specified, all retrieved data will be merged and returned.
+
+Therefore, the number of times the sub-step is executed is related to the quantity of data and the `fetchSize`. Meanwhile, each time the
+sub-step is invoked, the context will include a `$$typeOrmCursorRound` variable indicating the current batch (starting from 0), and a
+`$typeOrmCursorEnd` variable indicating whether it is the last batch.
+
 #### Save by SQL
 
 ##### Request and Response

package/index.cjs

@@ -25,6 +25,7 @@ const ERR_PIPELINE_SNIPPET_CANNOT_USE_GLOBAL = 'O03-00017';
 const ERR_PIPELINE_SNIPPET_CANNOT_USE_PROCESS = 'O03-00018';
 const ERR_PIPELINE_SNIPPET_CANNOT_USE_EVAL = 'O03-00019';
 const ERR_PIPELINE_SNIPPET_CANNOT_USE_FUNCTION = 'O03-00020';
+const ERR_TYPEORM_STREAM = 'O03-00021';
 n1.ErrorCodes.ERR_PIPELINE_STEP_SNIPPET_NOT_EMPTY = ERR_PIPELINE_STEP_SNIPPET_NOT_EMPTY;
 n1.ErrorCodes.ERR_PIPELINE_STEP_CONDITIONAL_SNIPPET_NOT_EMPTY = ERR_PIPELINE_STEP_CONDITIONAL_SNIPPET_NOT_EMPTY;
 n1.ErrorCodes.ERR_TYPEORM_DATASOURCE_TYPE_NOT_FOUND = ERR_TYPEORM_DATASOURCE_TYPE_NOT_FOUND;
@@ -45,6 +46,7 @@ n1.ErrorCodes.ERR_PIPELINE_SNIPPET_CANNOT_USE_GLOBAL = ERR_PIPELINE_SNIPPET_CANN
 n1.ErrorCodes.ERR_PIPELINE_SNIPPET_CANNOT_USE_PROCESS = ERR_PIPELINE_SNIPPET_CANNOT_USE_PROCESS;
 n1.ErrorCodes.ERR_PIPELINE_SNIPPET_CANNOT_USE_EVAL = ERR_PIPELINE_SNIPPET_CANNOT_USE_EVAL;
 n1.ErrorCodes.ERR_PIPELINE_SNIPPET_CANNOT_USE_FUNCTION = ERR_PIPELINE_SNIPPET_CANNOT_USE_FUNCTION;
+n1.ErrorCodes.ERR_TYPEORM_STREAM = ERR_TYPEORM_STREAM;
 
 class AbstractTypeOrmDataSource {
     _name;
@@ -392,7 +394,7 @@ class Utils {
                         suppressOutputPathCheck: false,
                         skipLibCheck: true,
                         skipDefaultLibCheck: true,
-                        moduleResolution: ts.ModuleResolutionKind.Node16
+                        moduleResolution: ts.ModuleResolutionKind.NodeNext
                     }
                 });
                 snippet = transpiled.outputText;
@@ -2216,6 +2218,124 @@ class TypeOrmLoadManyBySQLPipelineStep extends AbstractTypeOrmLoadBySQLPipelineS
     }
 }
 
+class TypeOrmLoadManyBySQLUseCursorPipelineStep extends AbstractTypeOrmBySQLPipelineStep {
+    _fetchSize;
+    _streamToSnippet;
+    _streamToFunc;
+    _stepBuilders;
+    constructor(options) {
+        super(options);
+        const config = this.getConfig();
+        this._fetchSize = options.fetchSize ?? config.getNumber(`typeorm.${this.getDataSourceName()}.fetch.size`, 20);
+        this._streamToSnippet = options.streamTo;
+        this._streamToFunc = Utils.createAsyncFunction(this.getStreamToSnippet(), {
+            createDefault: () => (async ($factor) => $factor),
+            getVariableNames: () => this.generateVariableNames(),
+            error: (e) => {
+                this.getLogger().error(`Failed on create function for snippet[${this.getStreamToSnippet()}].`);
+                throw e;
+            }
+        });
+        this._stepBuilders = options.steps;
+    }
+    getFetchSize() {
+        return this._fetchSize;
+    }
+    getStreamToSnippet() {
+        return this._streamToSnippet;
+    }
+    generateVariableNames() {
+        return [this.getFetchDataVariableName(), this.getRequestVariableName(), ...this.getHelpersVariableNames()];
+    }
+    getStepBuilders() {
+        return this._stepBuilders ?? [];
+    }
+    async doPerform(basis, request) {
+        const { sql, params } = this.getSql(basis, basis?.params);
+        return await this.autoTrans(async (runner) => {
+            const results = [];
+            const rows = [];
+            let cursorRound = 0;
+            const pipe = async ({ resolve, reject, end }) => {
+                if (!end && rows.length < this.getFetchSize()) {
+                    return;
+                }
+                try {
+                    const contentForSub = await this._streamToFunc([...rows], request, this.getHelpers(), this.getHelpers());
+                    rows.length = 0;
+                    let resultContent;
+                    if (this.getStepBuilders().length === 0) {
+                        resultContent = contentForSub;
+                    }
+                    else {
+                        const sets = new PipelineStepSets({
+                            ...this.buildStepOptions(), name: this.getName(), steps: this.getStepBuilders()
+                        });
+                        const { content: _, $context, ...rest } = request;
+                        const contextForSub = { ...$context, $typeOrmCursorRound: cursorRound, $typeOrmCursorEnd: end };
+                        const requestForSub = { ...rest, $context: contextForSub, content: contentForSub };
+                        const result = await sets.perform(requestForSub);
+                        const { content } = result;
+                        resultContent = content;
+                    }
+                    cursorRound = cursorRound + 1;
+                    if (resultContent == null || resultContent == n1.PIPELINE_STEP_RETURN_NULL) {
+                    }
+                    else if (Array.isArray(resultContent)) {
+                        results.push(...resultContent);
+                    }
+                    else {
+                        results.push(resultContent);
+                    }
+                }
+                catch (e) {
+                    reject(e);
+                }
+                if (end) {
+                    resolve(results);
+                }
+            };
+            const close = async (readable) => {
+                try {
+                    readable?.close((e) => {
+                        this.getLogger().error(e);
+                    });
+                }
+                catch (e) {
+                    this.getLogger().error(e);
+                }
+            };
+            const read = async ({ resolve, reject }) => {
+                const readable = await runner.stream(sql, params, async () => {
+                    await close(readable);
+                    await pipe({ resolve, reject, end: true });
+                }, async (e) => {
+                    await close(readable);
+                    reject(e);
+                });
+                readable.on('data', async (data) => {
+                    rows.push(data);
+                    await pipe({
+                        resolve, reject: async (e) => {
+                            await close(readable);
+                            reject(e);
+                        }, end: false
+                    });
+                    readable.read();
+                });
+                readable.read();
+            };
+            return new Promise((resolve, reject) => read({ resolve, reject }));
+        }, request);
+    }
+    getFetchDataVariableName() {
+        return '$factor';
+    }
+    getRequestVariableName() {
+        return '$request';
+    }
+}
+
 class TypeOrmSaveBySQLPipelineStep extends AbstractTypeOrmBySQLPipelineStep {
     async doPerform(basis, request) {
         const { sql, params } = this.getSql(basis, basis?.values);
@@ -2355,6 +2475,7 @@ exports.ERR_TYPEORM_DATASOURCE_TYPE_NOT_FOUND = ERR_TYPEORM_DATASOURCE_TYPE_NOT_
 exports.ERR_TYPEORM_ENTITY_NOT_FOUND = ERR_TYPEORM_ENTITY_NOT_FOUND;
 exports.ERR_TYPEORM_SQL_NOT_EMPTY = ERR_TYPEORM_SQL_NOT_EMPTY;
 exports.ERR_TYPEORM_STEP_SNIPPET_NOT_EMPTY = ERR_TYPEORM_STEP_SNIPPET_NOT_EMPTY;
+exports.ERR_TYPEORM_STREAM = ERR_TYPEORM_STREAM;
 exports.ERR_TYPEORM_TRANSACTION_NOT_FOUND = ERR_TYPEORM_TRANSACTION_NOT_FOUND;
 exports.EachPipelineStepSets = EachPipelineStepSets;
 exports.FetchPipelineStep = FetchPipelineStep;
@@ -2377,6 +2498,7 @@ exports.TypeOrmBySnippetPipelineStep = TypeOrmBySnippetPipelineStep;
 exports.TypeOrmDataSourceHelper = TypeOrmDataSourceHelper;
 exports.TypeOrmDataSourceManager = TypeOrmDataSourceManager;
 exports.TypeOrmLoadManyBySQLPipelineStep = TypeOrmLoadManyBySQLPipelineStep;
+exports.TypeOrmLoadManyBySQLUseCursorPipelineStep = TypeOrmLoadManyBySQLUseCursorPipelineStep;
 exports.TypeOrmLoadOneBySQLPipelineStep = TypeOrmLoadOneBySQLPipelineStep;
 exports.TypeOrmParsedSQLCache = TypeOrmParsedSQLCache;
 exports.TypeOrmSaveBySQLPipelineStep = TypeOrmSaveBySQLPipelineStep;

package/index.js

@@ -1,4 +1,4 @@
-import { ErrorCodes, UncatchableError, AbstractPipelineStep, CatchableError, ExposedUncatchableError, ERR_UNKNOWN, PipelineRepository, StepHelpersUtils } from '@rainbow-o23/n1';
+import { ErrorCodes, UncatchableError, AbstractPipelineStep, CatchableError, ExposedUncatchableError, ERR_UNKNOWN, PipelineRepository, StepHelpersUtils, PIPELINE_STEP_RETURN_NULL } from '@rainbow-o23/n1';
 import { DataSource } from 'typeorm';
 import ts, { ScriptTarget, JsxEmit, ModuleKind, ModuleResolutionKind } from 'typescript';
 import fetch from 'node-fetch';
@@ -23,6 +23,7 @@ const ERR_PIPELINE_SNIPPET_CANNOT_USE_GLOBAL = 'O03-00017';
 const ERR_PIPELINE_SNIPPET_CANNOT_USE_PROCESS = 'O03-00018';
 const ERR_PIPELINE_SNIPPET_CANNOT_USE_EVAL = 'O03-00019';
 const ERR_PIPELINE_SNIPPET_CANNOT_USE_FUNCTION = 'O03-00020';
+const ERR_TYPEORM_STREAM = 'O03-00021';
 ErrorCodes.ERR_PIPELINE_STEP_SNIPPET_NOT_EMPTY = ERR_PIPELINE_STEP_SNIPPET_NOT_EMPTY;
 ErrorCodes.ERR_PIPELINE_STEP_CONDITIONAL_SNIPPET_NOT_EMPTY = ERR_PIPELINE_STEP_CONDITIONAL_SNIPPET_NOT_EMPTY;
 ErrorCodes.ERR_TYPEORM_DATASOURCE_TYPE_NOT_FOUND = ERR_TYPEORM_DATASOURCE_TYPE_NOT_FOUND;
@@ -43,6 +44,7 @@ ErrorCodes.ERR_PIPELINE_SNIPPET_CANNOT_USE_GLOBAL = ERR_PIPELINE_SNIPPET_CANNOT_
 ErrorCodes.ERR_PIPELINE_SNIPPET_CANNOT_USE_PROCESS = ERR_PIPELINE_SNIPPET_CANNOT_USE_PROCESS;
 ErrorCodes.ERR_PIPELINE_SNIPPET_CANNOT_USE_EVAL = ERR_PIPELINE_SNIPPET_CANNOT_USE_EVAL;
 ErrorCodes.ERR_PIPELINE_SNIPPET_CANNOT_USE_FUNCTION = ERR_PIPELINE_SNIPPET_CANNOT_USE_FUNCTION;
+ErrorCodes.ERR_TYPEORM_STREAM = ERR_TYPEORM_STREAM;
 
 class AbstractTypeOrmDataSource {
     _name;
@@ -390,7 +392,7 @@ class Utils {
                         suppressOutputPathCheck: false,
                         skipLibCheck: true,
                         skipDefaultLibCheck: true,
-                        moduleResolution: ModuleResolutionKind.Node16
+                        moduleResolution: ModuleResolutionKind.NodeNext
                     }
                 });
                 snippet = transpiled.outputText;
@@ -2214,6 +2216,124 @@ class TypeOrmLoadManyBySQLPipelineStep extends AbstractTypeOrmLoadBySQLPipelineS
     }
 }
 
+class TypeOrmLoadManyBySQLUseCursorPipelineStep extends AbstractTypeOrmBySQLPipelineStep {
+    _fetchSize;
+    _streamToSnippet;
+    _streamToFunc;
+    _stepBuilders;
+    constructor(options) {
+        super(options);
+        const config = this.getConfig();
+        this._fetchSize = options.fetchSize ?? config.getNumber(`typeorm.${this.getDataSourceName()}.fetch.size`, 20);
+        this._streamToSnippet = options.streamTo;
+        this._streamToFunc = Utils.createAsyncFunction(this.getStreamToSnippet(), {
+            createDefault: () => (async ($factor) => $factor),
+            getVariableNames: () => this.generateVariableNames(),
+            error: (e) => {
+                this.getLogger().error(`Failed on create function for snippet[${this.getStreamToSnippet()}].`);
+                throw e;
+            }
+        });
+        this._stepBuilders = options.steps;
+    }
+    getFetchSize() {
+        return this._fetchSize;
+    }
+    getStreamToSnippet() {
+        return this._streamToSnippet;
+    }
+    generateVariableNames() {
+        return [this.getFetchDataVariableName(), this.getRequestVariableName(), ...this.getHelpersVariableNames()];
+    }
+    getStepBuilders() {
+        return this._stepBuilders ?? [];
+    }
+    async doPerform(basis, request) {
+        const { sql, params } = this.getSql(basis, basis?.params);
+        return await this.autoTrans(async (runner) => {
+            const results = [];
+            const rows = [];
+            let cursorRound = 0;
+            const pipe = async ({ resolve, reject, end }) => {
+                if (!end && rows.length < this.getFetchSize()) {
+                    return;
+                }
+                try {
+                    const contentForSub = await this._streamToFunc([...rows], request, this.getHelpers(), this.getHelpers());
+                    rows.length = 0;
+                    let resultContent;
+                    if (this.getStepBuilders().length === 0) {
+                        resultContent = contentForSub;
+                    }
+                    else {
+                        const sets = new PipelineStepSets({
+                            ...this.buildStepOptions(), name: this.getName(), steps: this.getStepBuilders()
+                        });
+                        const { content: _, $context, ...rest } = request;
+                        const contextForSub = { ...$context, $typeOrmCursorRound: cursorRound, $typeOrmCursorEnd: end };
+                        const requestForSub = { ...rest, $context: contextForSub, content: contentForSub };
+                        const result = await sets.perform(requestForSub);
+                        const { content } = result;
+                        resultContent = content;
+                    }
+                    cursorRound = cursorRound + 1;
+                    if (resultContent == null || resultContent == PIPELINE_STEP_RETURN_NULL) {
+                    }
+                    else if (Array.isArray(resultContent)) {
+                        results.push(...resultContent);
+                    }
+                    else {
+                        results.push(resultContent);
+                    }
+                }
+                catch (e) {
+                    reject(e);
+                }
+                if (end) {
+                    resolve(results);
+                }
+            };
+            const close = async (readable) => {
+                try {
+                    readable?.close((e) => {
+                        this.getLogger().error(e);
+                    });
+                }
+                catch (e) {
+                    this.getLogger().error(e);
+                }
+            };
+            const read = async ({ resolve, reject }) => {
+                const readable = await runner.stream(sql, params, async () => {
+                    await close(readable);
+                    await pipe({ resolve, reject, end: true });
+                }, async (e) => {
+                    await close(readable);
+                    reject(e);
+                });
+                readable.on('data', async (data) => {
+                    rows.push(data);
+                    await pipe({
+                        resolve, reject: async (e) => {
+                            await close(readable);
+                            reject(e);
+                        }, end: false
+                    });
+                    readable.read();
+                });
+                readable.read();
+            };
+            return new Promise((resolve, reject) => read({ resolve, reject }));
+        }, request);
+    }
+    getFetchDataVariableName() {
+        return '$factor';
+    }
+    getRequestVariableName() {
+        return '$request';
+    }
+}
+
 class TypeOrmSaveBySQLPipelineStep extends AbstractTypeOrmBySQLPipelineStep {
     async doPerform(basis, request) {
         const { sql, params } = this.getSql(basis, basis?.values);
@@ -2324,4 +2444,4 @@ class TypeOrmTransactionalPipelineStepSets extends PipelineStepSets {
     }
 }
 
-export { AbstractFragmentaryPipelineStep, AbstractTypeOrmBySQLPipelineStep, AbstractTypeOrmDataSource, AbstractTypeOrmLoadBySQLPipelineStep, AbstractTypeOrmPipelineStep, AsyncPipelineStepSets, BetterSqlite3TypeOrmDatasource, ConditionalPipelineStepSets, DEFAULT_TRANSACTION_NAME, DeletePropertyPipelineStep, ERR_EACH_FRAGMENT_NOT_ANY_ARRAY, ERR_FETCH_ERROR, ERR_PIPELINE_REF_NOT_EMPTY, ERR_PIPELINE_REF_NOT_FOUND, ERR_PIPELINE_SNIPPET_CANNOT_USE_EVAL, ERR_PIPELINE_SNIPPET_CANNOT_USE_FUNCTION, ERR_PIPELINE_SNIPPET_CANNOT_USE_GLOBAL, ERR_PIPELINE_SNIPPET_CANNOT_USE_PROCESS, ERR_PIPELINE_STEP_CONDITIONAL_SNIPPET_NOT_EMPTY, ERR_PIPELINE_STEP_METHOD_NOT_SUPPORTED, ERR_PIPELINE_STEP_REF_NOT_EMPTY, ERR_PIPELINE_STEP_REF_NOT_FOUND, ERR_PIPELINE_STEP_SNIPPET_NOT_EMPTY, ERR_TYPEORM_DATASOURCE_CREATOR_NOT_FOUND, ERR_TYPEORM_DATASOURCE_NOT_FOUND, ERR_TYPEORM_DATASOURCE_TYPE_NOT_FOUND, ERR_TYPEORM_ENTITY_NOT_FOUND, ERR_TYPEORM_SQL_NOT_EMPTY, ERR_TYPEORM_STEP_SNIPPET_NOT_EMPTY, ERR_TYPEORM_TRANSACTION_NOT_FOUND, EachPipelineStepSets, FetchPipelineStep, GetPropertyPipelineStep, HttpAbortErrorCode, HttpUnknownErrorCode, MssqlTypeOrmDatasource, MysqlTypeOrmDatasource, OracleTypeOrmDatasource, ParallelPipelineStepSets, ParsedSqlSegmentType, PgsqlTypeOrmDatasource, PipelineStepSets, RefPipelinePipelineStep, RefStepPipelineStep, RoutesPipelineStepSets, SnippetPipelineStep, SnowflakePipelineStep, SupportedDataSourceTypes, TypeOrmBulkSaveBySQLPipelineStep, TypeOrmBySnippetPipelineStep, TypeOrmDataSourceHelper, TypeOrmDataSourceManager, TypeOrmLoadManyBySQLPipelineStep, TypeOrmLoadOneBySQLPipelineStep, TypeOrmParsedSQLCache, TypeOrmSaveBySQLPipelineStep, TypeOrmTransactionalPipelineStepSets, Utils };
+export { AbstractFragmentaryPipelineStep, AbstractTypeOrmBySQLPipelineStep, AbstractTypeOrmDataSource, AbstractTypeOrmLoadBySQLPipelineStep, AbstractTypeOrmPipelineStep, AsyncPipelineStepSets, BetterSqlite3TypeOrmDatasource, ConditionalPipelineStepSets, DEFAULT_TRANSACTION_NAME, DeletePropertyPipelineStep, ERR_EACH_FRAGMENT_NOT_ANY_ARRAY, ERR_FETCH_ERROR, ERR_PIPELINE_REF_NOT_EMPTY, ERR_PIPELINE_REF_NOT_FOUND, ERR_PIPELINE_SNIPPET_CANNOT_USE_EVAL, ERR_PIPELINE_SNIPPET_CANNOT_USE_FUNCTION, ERR_PIPELINE_SNIPPET_CANNOT_USE_GLOBAL, ERR_PIPELINE_SNIPPET_CANNOT_USE_PROCESS, ERR_PIPELINE_STEP_CONDITIONAL_SNIPPET_NOT_EMPTY, ERR_PIPELINE_STEP_METHOD_NOT_SUPPORTED, ERR_PIPELINE_STEP_REF_NOT_EMPTY, ERR_PIPELINE_STEP_REF_NOT_FOUND, ERR_PIPELINE_STEP_SNIPPET_NOT_EMPTY, ERR_TYPEORM_DATASOURCE_CREATOR_NOT_FOUND, ERR_TYPEORM_DATASOURCE_NOT_FOUND, ERR_TYPEORM_DATASOURCE_TYPE_NOT_FOUND, ERR_TYPEORM_ENTITY_NOT_FOUND, ERR_TYPEORM_SQL_NOT_EMPTY, ERR_TYPEORM_STEP_SNIPPET_NOT_EMPTY, ERR_TYPEORM_STREAM, ERR_TYPEORM_TRANSACTION_NOT_FOUND, EachPipelineStepSets, FetchPipelineStep, GetPropertyPipelineStep, HttpAbortErrorCode, HttpUnknownErrorCode, MssqlTypeOrmDatasource, MysqlTypeOrmDatasource, OracleTypeOrmDatasource, ParallelPipelineStepSets, ParsedSqlSegmentType, PgsqlTypeOrmDatasource, PipelineStepSets, RefPipelinePipelineStep, RefStepPipelineStep, RoutesPipelineStepSets, SnippetPipelineStep, SnowflakePipelineStep, SupportedDataSourceTypes, TypeOrmBulkSaveBySQLPipelineStep, TypeOrmBySnippetPipelineStep, TypeOrmDataSourceHelper, TypeOrmDataSourceManager, TypeOrmLoadManyBySQLPipelineStep, TypeOrmLoadManyBySQLUseCursorPipelineStep, TypeOrmLoadOneBySQLPipelineStep, TypeOrmParsedSQLCache, TypeOrmSaveBySQLPipelineStep, TypeOrmTransactionalPipelineStepSets, Utils };

package/lib/error-codes.d.ts

@@ -19,3 +19,4 @@ export declare const ERR_PIPELINE_SNIPPET_CANNOT_USE_GLOBAL: O23ReservedErrorCod
 export declare const ERR_PIPELINE_SNIPPET_CANNOT_USE_PROCESS: O23ReservedErrorCode;
 export declare const ERR_PIPELINE_SNIPPET_CANNOT_USE_EVAL: O23ReservedErrorCode;
 export declare const ERR_PIPELINE_SNIPPET_CANNOT_USE_FUNCTION: O23ReservedErrorCode;
+export declare const ERR_TYPEORM_STREAM: O23ReservedErrorCode;

package/lib/typeorm-step/index.d.ts

@@ -5,6 +5,7 @@ export * from './abstract-typeorm-by-sql-step';
 export * from './abstract-typeorm-load-by-sql-step';
 export * from './typeorm-load-one-by-sql-step';
 export * from './typeorm-load-many-by-sql-step';
+export * from './typeorm-load-many-by-sql-use-cursor-step';
 export * from './typeorm-save-by-sql-step';
 export * from './typeorm-bulk-save-by-sql-step';
 export * from './typeorm-transactional-step-sets';

package/lib/typeorm-step/typeorm-load-many-by-sql-use-cursor-step.d.ts

@@ -0,0 +1,25 @@
+import { PipelineStepBuilder, PipelineStepData, PipelineStepHelpers, PipelineStepPayload, Undefinable } from '@rainbow-o23/n1';
+import { ScriptFuncOrBody } from '../step';
+import { AbstractTypeOrmBySQLPipelineStep, TypeOrmBySQLPipelineStepOptions } from './abstract-typeorm-by-sql-step';
+import { TypeOrmLoadBasis } from './abstract-typeorm-load-by-sql-step';
+export type StreamToSubSteps = any;
+export type StreamToFunc<In> = ($factor: Array<any>, $request: PipelineStepData<In>, $helpers: PipelineStepHelpers, $: PipelineStepHelpers) => Promise<StreamToSubSteps>;
+export interface TypeOrmLoadManyBySQLUseCursorPipelineStepOptions<In = PipelineStepPayload, Out = PipelineStepPayload, InFragment = TypeOrmLoadBasis, OutFragment = Out> extends TypeOrmBySQLPipelineStepOptions<In, Out, InFragment, OutFragment> {
+    fetchSize?: number;
+    streamTo?: ScriptFuncOrBody<StreamToFunc<In>>;
+    steps?: Array<PipelineStepBuilder>;
+}
+export declare class TypeOrmLoadManyBySQLUseCursorPipelineStep<In = PipelineStepPayload, Out = PipelineStepPayload, InFragment = Undefinable<TypeOrmLoadBasis>, OutFragment = Out> extends AbstractTypeOrmBySQLPipelineStep<In, Out, Undefinable<TypeOrmLoadBasis>, OutFragment> {
+    private readonly _fetchSize;
+    private readonly _streamToSnippet;
+    private readonly _streamToFunc;
+    private readonly _stepBuilders;
+    constructor(options: TypeOrmLoadManyBySQLUseCursorPipelineStepOptions<In, Out, InFragment, OutFragment>);
+    protected getFetchSize(): number;
+    getStreamToSnippet(): ScriptFuncOrBody<StreamToFunc<In>>;
+    protected generateVariableNames(): Array<string>;
+    protected getStepBuilders(): Array<PipelineStepBuilder>;
+    protected doPerform(basis: Undefinable<TypeOrmLoadBasis>, request: PipelineStepData<In>): Promise<Undefinable<OutFragment>>;
+    protected getFetchDataVariableName(): string;
+    protected getRequestVariableName(): string;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@rainbow-o23/n3",
-	"version": "1.0.46",
+	"version": "1.0.48",
 	"description": "o23 pipelines",
 	"main": "index.cjs",
 	"module": "index.js",
@@ -21,9 +21,9 @@
 		"url": "https://github.com/InsureMO/rainbow-o23/issues"
 	},
 	"dependencies": {
-		"@rainbow-o23/n1": "1.0.46",
+		"@rainbow-o23/n1": "1.0.48",
 		"node-fetch": "2.6.7",
-		"typeorm": "^0.3.17",
+		"typeorm": "^0.3.20",
 		"typescript": "5.5.4"
 	},
 	"devDependencies": {
@@ -36,17 +36,17 @@
 		"@types/events": "^3.0.1",
 		"@types/node": "18.16.12",
 		"@types/node-fetch": "2.6.4",
-		"@typescript-eslint/eslint-plugin": "^7.18.0",
-		"@typescript-eslint/parser": "^7.18.0",
-		"better-sqlite3": "^9.0.0",
+		"@typescript-eslint/eslint-plugin": "^8.13.0",
+		"@typescript-eslint/parser": "^8.13.0",
+		"better-sqlite3": "^11.5.0",
 		"eslint": "^9.8.0",
-		"mssql": "^10.0.1",
-		"mysql2": "^3.9.8",
-		"oracledb": "^6.2.0",
-		"pg": "^8.11.3",
-		"pg-query-stream": "^4.5.3",
+		"mssql": "^11.0.1",
+		"mysql2": "^3.11.4",
+		"oracledb": "^6.6.0",
+		"pg": "^8.13.1",
+		"pg-query-stream": "^4.7.1",
 		"reflect-metadata": "^0.2.2",
-		"rollup": "^3.7.0",
+		"rollup": "^3.29.5",
 		"rollup-plugin-tslint": "^0.2.2",
 		"rollup-plugin-typescript2": "^0.34.1",
 		"tslib": "^2.4.1"
@@ -74,7 +74,6 @@
 		"testEnvironment": "node"
 	},
 	"volta": {
-		"node": "20.17.0",
-		"yarn": "1.22.21"
+		"extends": "../package.json"
 	}
 }

package/src/lib/error-codes.ts

@@ -21,6 +21,7 @@ export const ERR_PIPELINE_SNIPPET_CANNOT_USE_GLOBAL: O23ReservedErrorCode = 'O03
 export const ERR_PIPELINE_SNIPPET_CANNOT_USE_PROCESS: O23ReservedErrorCode = 'O03-00018';
 export const ERR_PIPELINE_SNIPPET_CANNOT_USE_EVAL: O23ReservedErrorCode = 'O03-00019';
 export const ERR_PIPELINE_SNIPPET_CANNOT_USE_FUNCTION: O23ReservedErrorCode = 'O03-00020';
+export const ERR_TYPEORM_STREAM: O23ReservedErrorCode = 'O03-00021';
 
 ErrorCodes.ERR_PIPELINE_STEP_SNIPPET_NOT_EMPTY = ERR_PIPELINE_STEP_SNIPPET_NOT_EMPTY;
 ErrorCodes.ERR_PIPELINE_STEP_CONDITIONAL_SNIPPET_NOT_EMPTY = ERR_PIPELINE_STEP_CONDITIONAL_SNIPPET_NOT_EMPTY;
@@ -43,3 +44,4 @@ ErrorCodes.ERR_PIPELINE_SNIPPET_CANNOT_USE_GLOBAL = ERR_PIPELINE_SNIPPET_CANNOT_
 ErrorCodes.ERR_PIPELINE_SNIPPET_CANNOT_USE_PROCESS = ERR_PIPELINE_SNIPPET_CANNOT_USE_PROCESS;
 ErrorCodes.ERR_PIPELINE_SNIPPET_CANNOT_USE_EVAL = ERR_PIPELINE_SNIPPET_CANNOT_USE_EVAL;
 ErrorCodes.ERR_PIPELINE_SNIPPET_CANNOT_USE_FUNCTION = ERR_PIPELINE_SNIPPET_CANNOT_USE_FUNCTION;
+ErrorCodes.ERR_TYPEORM_STREAM = ERR_TYPEORM_STREAM;

package/src/lib/step/abstract-fragmentary-pipeline-step.ts

@@ -213,7 +213,6 @@ export abstract class AbstractFragmentaryPipelineStep<In = PipelineStepPayload,
 				};
 			}
 		} else if (typeof funcOrSnippet === 'string') {
-			// eslint-disable-next-line @typescript-eslint/ban-types
 			const func = Utils.createAsyncFunction(funcOrSnippet, {
 				createDefault: async () => {
 					throw new UncatchableError(ERR_PIPELINE_STEP_SNIPPET_NOT_EMPTY, 'Cannot create perform func on empty snippet.');

package/src/lib/step/step-sets.ts

@@ -8,6 +8,7 @@ import {
 import {AbstractFragmentaryPipelineStep, FragmentaryPipelineStepOptions} from './abstract-fragmentary-pipeline-step';
 import {Utils} from './utils';
 
+// eslint-disable-next-line @typescript-eslint/no-empty-object-type
 export interface PipelineStepSetsContext {
 }
 

package/src/lib/step/types.ts

@@ -7,7 +7,7 @@ import {
 } from '@rainbow-o23/n1';
 
 export type ScriptFunctionBody = string;
-// eslint-disable-next-line @typescript-eslint/ban-types
+// eslint-disable-next-line @typescript-eslint/no-unsafe-function-type
 export type ScriptFuncOrBody<F = Function> = F | ScriptFunctionBody;
 
 export interface ErrorHandleOptions<In, InFragment, E extends Error = Error> {

package/src/lib/step/utils.ts

@@ -43,7 +43,7 @@ const AvoidProxyObjects = [
 ];
 
 export class Utils {
-	// eslint-disable-next-line @typescript-eslint/ban-types
+	// eslint-disable-next-line @typescript-eslint/no-unsafe-function-type
 	public static createFunction<F = Function>(snippet: ScriptFuncOrBody<F>, creators: {
 		createDefault: () => Undefinable<F> | never;
 		getVariableNames: () => Array<string>;
@@ -70,7 +70,7 @@ export class Utils {
 						suppressOutputPathCheck: false,
 						skipLibCheck: true,
 						skipDefaultLibCheck: true,
-						moduleResolution: ModuleResolutionKind.Node16 // default use node 16
+						moduleResolution: ModuleResolutionKind.NodeNext // default use node next
 					}
 				});
 				snippet = transpiled.outputText;
@@ -105,7 +105,7 @@ export class Utils {
 		}
 	}
 
-	// eslint-disable-next-line @typescript-eslint/ban-types
+	// eslint-disable-next-line @typescript-eslint/no-unsafe-function-type
 	public static createSyncFunction<F = Function>(snippet: ScriptFuncOrBody<F>, creators: {
 		createDefault: () => Undefinable<F> | never;
 		getVariableNames: () => Array<string>;
@@ -114,7 +114,7 @@ export class Utils {
 		return Utils.createFunction(snippet, {...creators});
 	}
 
-	// eslint-disable-next-line @typescript-eslint/ban-types
+	// eslint-disable-next-line @typescript-eslint/no-unsafe-function-type
 	public static createAsyncFunction<F = Function>(snippet: ScriptFuncOrBody<F>, creators: {
 		createDefault: () => Undefinable<F> | never;
 		getVariableNames: () => Array<string>;

package/src/lib/typeorm-step/index.ts

@@ -8,6 +8,7 @@ export * from './abstract-typeorm-by-sql-step';
 export * from './abstract-typeorm-load-by-sql-step';
 export * from './typeorm-load-one-by-sql-step';
 export * from './typeorm-load-many-by-sql-step';
+export * from './typeorm-load-many-by-sql-use-cursor-step';
 
 export * from './typeorm-save-by-sql-step';
 export * from './typeorm-bulk-save-by-sql-step';

package/src/lib/typeorm-step/typeorm-load-many-by-sql-use-cursor-step.ts

@@ -0,0 +1,165 @@
+import {
+	PIPELINE_STEP_RETURN_NULL,
+	PipelineStepBuilder,
+	PipelineStepData,
+	PipelineStepHelpers,
+	PipelineStepPayload,
+	Undefinable
+} from '@rainbow-o23/n1';
+import {ReadStream} from 'fs';
+import {PipelineStepSets, ScriptFuncOrBody, Utils} from '../step';
+import {AbstractTypeOrmBySQLPipelineStep, TypeOrmBySQLPipelineStepOptions} from './abstract-typeorm-by-sql-step';
+import {TypeOrmLoadBasis} from './abstract-typeorm-load-by-sql-step';
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export type StreamToSubSteps = any;
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export type StreamToFunc<In> = ($factor: Array<any>, $request: PipelineStepData<In>, $helpers: PipelineStepHelpers, $: PipelineStepHelpers) => Promise<StreamToSubSteps>;
+
+export interface TypeOrmLoadManyBySQLUseCursorPipelineStepOptions<In = PipelineStepPayload, Out = PipelineStepPayload, InFragment = TypeOrmLoadBasis, OutFragment = Out>
+	extends TypeOrmBySQLPipelineStepOptions<In, Out, InFragment, OutFragment> {
+	fetchSize?: number;
+	streamTo?: ScriptFuncOrBody<StreamToFunc<In>>;
+	steps?: Array<PipelineStepBuilder>;
+}
+
+export class TypeOrmLoadManyBySQLUseCursorPipelineStep<In = PipelineStepPayload, Out = PipelineStepPayload, InFragment = Undefinable<TypeOrmLoadBasis>, OutFragment = Out>
+	extends AbstractTypeOrmBySQLPipelineStep<In, Out, Undefinable<TypeOrmLoadBasis>, OutFragment> {
+	private readonly _fetchSize: number;
+	private readonly _streamToSnippet: ScriptFuncOrBody<StreamToFunc<In>>;
+	private readonly _streamToFunc: StreamToFunc<In>;
+	private readonly _stepBuilders: Array<PipelineStepBuilder>;
+
+	public constructor(options: TypeOrmLoadManyBySQLUseCursorPipelineStepOptions<In, Out, InFragment, OutFragment>) {
+		super(options);
+		const config = this.getConfig();
+		this._fetchSize = options.fetchSize ?? config.getNumber(`typeorm.${this.getDataSourceName()}.fetch.size`, 20);
+		this._streamToSnippet = options.streamTo;
+		this._streamToFunc = Utils.createAsyncFunction(this.getStreamToSnippet(), {
+			// eslint-disable-next-line @typescript-eslint/no-explicit-any
+			createDefault: () => (async ($factor: Array<any>) => $factor),
+			getVariableNames: () => this.generateVariableNames(),
+			error: (e: Error) => {
+				this.getLogger().error(`Failed on create function for snippet[${this.getStreamToSnippet()}].`);
+				throw e;
+			}
+		});
+		this._stepBuilders = options.steps;
+	}
+
+	protected getFetchSize(): number {
+		return this._fetchSize;
+	}
+
+	public getStreamToSnippet(): ScriptFuncOrBody<StreamToFunc<In>> {
+		return this._streamToSnippet;
+	}
+
+	protected generateVariableNames(): Array<string> {
+		return [this.getFetchDataVariableName(), this.getRequestVariableName(), ...this.getHelpersVariableNames()];
+	}
+
+	protected getStepBuilders(): Array<PipelineStepBuilder> {
+		return this._stepBuilders ?? [];
+	}
+
+	protected async doPerform(basis: Undefinable<TypeOrmLoadBasis>, request: PipelineStepData<In>): Promise<Undefinable<OutFragment>> {
+		const {sql, params} = this.getSql(basis, basis?.params);
+		return await this.autoTrans<OutFragment>(async (runner) => {
+			const results = [];
+			const rows = [];
+			let cursorRound = 0;
+			const pipe = async ({resolve, reject, end}) => {
+				if (!end && rows.length < this.getFetchSize()) {
+					// not end, and size not meet the fresh required
+					// do nothing, wait for next
+					return;
+				}
+				try {
+					// get data from cache
+					const contentForSub = await this._streamToFunc([...rows], request, this.getHelpers(), this.getHelpers());
+					// clear cache
+					rows.length = 0;
+					// eslint-disable-next-line @typescript-eslint/no-explicit-any
+					let resultContent: any;
+					if (this.getStepBuilders().length === 0) {
+						// no sub step, use content as result
+						resultContent = contentForSub;
+					} else {
+						// create a step sets to run
+						const sets = new PipelineStepSets({
+							...this.buildStepOptions(), name: this.getName(), steps: this.getStepBuilders()
+						});
+						// eslint-disable-next-line @typescript-eslint/no-unused-vars
+						const {content: _, $context, ...rest} = request;
+						// pass a cursor end indicator to sub steps
+						const contextForSub = {...$context, $typeOrmCursorRound: cursorRound, $typeOrmCursorEnd: end};
+						const requestForSub = {...rest, $context: contextForSub, content: contentForSub};
+						const result = await sets.perform(requestForSub);
+						const {content} = result;
+						resultContent = content;
+					}
+					cursorRound = cursorRound + 1;
+					if (resultContent == null || resultContent == PIPELINE_STEP_RETURN_NULL) {
+						// ignore
+					} else if (Array.isArray(resultContent)) {
+						results.push(...resultContent);
+					} else {
+						results.push(resultContent);
+					}
+				} catch (e) {
+					reject(e);
+				}
+				if (end) {
+					resolve(results);
+				}
+			};
+			const close = async (readable: ReadStream) => {
+				// never throw exception from this function
+				try {
+					readable?.close((e) => {
+						// ignore this error
+						this.getLogger().error(e);
+					});
+				} catch (e) {
+					// ignore this error
+					this.getLogger().error(e);
+				}
+			};
+			const read = async ({resolve, reject}) => {
+				const readable = await runner.stream(sql, params, async () => {
+					// on end
+					await close(readable);
+					await pipe({resolve, reject, end: true});
+				}, async (e: Error) => {
+					// on error
+					await close(readable);
+					reject(e);
+				});
+				readable.on('data', async (data) => {
+					rows.push(data);
+					await pipe({
+						resolve, reject: async (e: Error) => {
+							await close(readable);
+							reject(e);
+						}, end: false
+					});
+					readable.read();
+				});
+				readable.read();
+			};
+			return new Promise<OutFragment>((resolve, reject) => read({resolve, reject}));
+		}, request);
+	}
+
+	/**
+	 * override this method when want to use another variable name rather than "$factor"
+	 */
+	protected getFetchDataVariableName(): string {
+		return '$factor';
+	}
+
+	protected getRequestVariableName(): string {
+		return '$request';
+	}
+}

package/test/step/typeorm-by-cursor.test.ts

@@ -0,0 +1,55 @@
+import {createConfig, createLogger} from '@rainbow-o23/n1';
+import {TypeOrmDataSourceHelper, TypeOrmLoadManyBySQLUseCursorPipelineStep} from '../../src';
+
+const logger = createLogger();
+const config = createConfig(logger);
+
+describe('TypeORM Cursor Suite', () => {
+	beforeAll(async () => {
+		process.env.CFG_TYPEORM_TEST_HOST = 'localhost';
+		process.env.CFG_TYPEORM_TEST_USERNAME = 'o23';
+		process.env.CFG_TYPEORM_TEST_PASSWORD = 'o23';
+		const type = 'pg' + 'sql';
+		if (type === 'mysql') {
+			process.env.CFG_TYPEORM_TEST_TYPE = 'mysql';
+			process.env.CFG_TYPEORM_TEST_PORT = '3306';
+			process.env.CFG_TYPEORM_TEST_DATABASE = 'o23';
+		} else if (type === 'pgsql') {
+			process.env.CFG_TYPEORM_TEST_TYPE = 'pgsql';
+			process.env.CFG_TYPEORM_TEST_PORT = '5432';
+			process.env.CFG_TYPEORM_TEST_DATABASE = 'postgres';
+			process.env.CFG_TYPEORM_TEST_SCHEMA = 'o23';
+		} else if (type === 'mssql') {
+			process.env.CFG_TYPEORM_TEST_TYPE = 'mssql';
+			process.env.CFG_TYPEORM_TEST_PORT = '1433';
+			process.env.CFG_TYPEORM_TEST_PASSWORD = 'o23O23o23!';
+			process.env.CFG_TYPEORM_TEST_DATABASE = 'o23';
+			process.env.CFG_TYPEORM_TEST_SCHEMA = 'dbo';
+			process.env.CFG_TYPEORM_TEST_TRUST_SERVER_CERTIFICATE = 'true';
+		} else {
+			process.env.CFG_TYPEORM_TEST_TYPE = 'oracle';
+			process.env.CFG_TYPEORM_TEST_HOST = '127.0.0.1';
+			process.env.CFG_TYPEORM_TEST_PORT = '1521';
+			process.env.CFG_TYPEORM_TEST_SERVICE_NAME = 'orcl';
+		}
+		// process.env.CFG_TYPEORM_TEST_LOGGING = 'true';
+		await new TypeOrmDataSourceHelper(config).create({});
+		// const repo = (await TypeOrmDataSourceManager.findDataSource('TEST', config)).getDataSource().getRepository(TestTable);
+	});
+
+	test('TypeORM load many using cursor #1', async () => {
+		// noinspection SqlResolve
+		const step = new TypeOrmLoadManyBySQLUseCursorPipelineStep<any, any>({
+			config, logger, dataSourceName: 'TEST', sql: 'SELECT * FROM T_O23_DB_CHANGE_LOG',
+			autonomous: true
+		});
+		const request = {content: (void 0)};
+		const response = await step.perform(request);
+		expect(response.content).not.toBeNull();
+		console.log(response.content);
+	});
+
+	afterAll((done) => {
+		done();
+	});
+});

package/test/step/typeorm-by-sql-transactional.test.ts

@@ -70,10 +70,10 @@ class TransactionalTestStepSets extends TypeOrmTransactionalPipelineStepSets {
 						logger,
 						dataSourceName: 'TEST',
 						sql: 'SELECT ID id, CONTENT content FROM T_TEST_TABLE WHERE ID = ?',
-						fromRequest: ($factor: TransactionalRequest, _$request: PipelineStepData<TransactionalRequest>) => {
+						fromRequest: async ($factor: TransactionalRequest, _$request: PipelineStepData<TransactionalRequest>) => {
 							return {params: [$factor.idToLoad]} as TypeOrmBasis;
 						},
-						toResponse: ($result: TestTable, _$request: PipelineStepData) => {
+						toResponse: async ($result: TestTable, _$request: PipelineStepData) => {
 							expect($result.id).toBe(1);
 							expect($result.content).toBe('hello world!');
 							return {loadedById: $result};
@@ -87,10 +87,10 @@ class TransactionalTestStepSets extends TypeOrmTransactionalPipelineStepSets {
 						logger,
 						dataSourceName: 'TEST',
 						sql: 'INSERT INTO T_TEST_TABLE(ID, CONTENT) VALUES (?, ?)',
-						fromRequest: ($factor: TransactionalRequest, _$request: PipelineStepData<TransactionalRequest>) => {
+						fromRequest: async ($factor: TransactionalRequest, _$request: PipelineStepData<TransactionalRequest>) => {
 							return {values: [$factor.item3.id, $factor.item3.content]};
 						},
-						toResponse: ($result: TypeOrmIdOfInserted, _$request: PipelineStepData) => {
+						toResponse: async ($result: TypeOrmIdOfInserted, _$request: PipelineStepData) => {
 							expect($result).toBe(3);
 							return {insertedId: $result};
 						},
@@ -104,10 +104,10 @@ class TransactionalTestStepSets extends TypeOrmTransactionalPipelineStepSets {
 						dataSourceName: 'TEST',
 						autonomous: true,
 						sql: 'SELECT ID id, CONTENT content FROM T_TEST_TABLE WHERE ID = ?',
-						fromRequest: ($factor: TransactionalRequest, _$request: PipelineStepData<TransactionalRequest>) => {
+						fromRequest: async ($factor: TransactionalRequest, _$request: PipelineStepData<TransactionalRequest>) => {
 							return {params: [$factor.item3.id]};
 						},
-						toResponse: ($result: Undefinable<TestTable>, $request: PipelineStepData) => {
+						toResponse: async ($result: Undefinable<TestTable>, $request: PipelineStepData) => {
 							expect($result).toBeUndefined();
 							return $request.content;
 						}
@@ -116,10 +116,10 @@ class TransactionalTestStepSets extends TypeOrmTransactionalPipelineStepSets {
 				{
 					create: async ({config, logger}) => new TypeOrmSaveBySQLPipelineStep({
 						config, logger, dataSourceName: 'TEST', sql: 'UPDATE T_TEST_TABLE SET CONTENT = ? WHERE ID = ?',
-						fromRequest: ($factor: TransactionalRequest, _$request: PipelineStepData<TransactionalRequest>) => {
+						fromRequest: async ($factor: TransactionalRequest, _$request: PipelineStepData<TransactionalRequest>) => {
 							return {values: [$factor.item3ChangeTo.content, $factor.item3ChangeTo.id]};
 						},
-						toResponse: ($result: TypeOrmCountOfAffected, _$request: PipelineStepData) => {
+						toResponse: async ($result: TypeOrmCountOfAffected, _$request: PipelineStepData) => {
 							// DON'T KNOW WHY THIS IS 3, SEEMS SHOULD BE 1 ACCORDING TO BETTER-SQLITE3 DOCUMENT
 							// BUT CURRENTLY IT RETURNS COUNT OF THIS TABLE, NOT IMPACTED ROW COUNT
 							expect($result).toBe(3);
@@ -134,14 +134,14 @@ class TransactionalTestStepSets extends TypeOrmTransactionalPipelineStepSets {
 						logger,
 						dataSourceName: 'TEST',
 						sql: 'INSERT INTO T_TEST_TABLE(ID, CONTENT) VALUES (?, ?)',
-						fromRequest: ($factor: TransactionalRequest, _$request: PipelineStepData<TransactionalRequest>) => {
+						fromRequest: async ($factor: TransactionalRequest, _$request: PipelineStepData<TransactionalRequest>) => {
 							return {
 								items: [
 									[$factor.item4.id, $factor.item4.content], [$factor.item5.id, $factor.item5.content]
 								]
 							};
 						},
-						toResponse: ($result: TypeOrmIdsOfInserted, _$request: PipelineStepData) => {
+						toResponse: async ($result: TypeOrmIdsOfInserted, _$request: PipelineStepData) => {
 							return {insertedIds: $result};
 						},
 						mergeRequest: true
@@ -150,7 +150,7 @@ class TransactionalTestStepSets extends TypeOrmTransactionalPipelineStepSets {
 				{
 					create: async ({config, logger}) => new TypeOrmLoadManyBySQLPipelineStep({
 						config, logger, dataSourceName: 'TEST', sql: 'SELECT ID id, CONTENT content FROM T_TEST_TABLE',
-						toResponse: ($result: Array<TestTable>, _$request: PipelineStepData) => {
+						toResponse: async ($result: Array<TestTable>, _$request: PipelineStepData) => {
 							return {all: $result};
 						},
 						mergeRequest: true

package/test/step/typeorm-entity.test.ts

@@ -1,50 +0,0 @@
-import {createConfig, createLogger} from '@rainbow-o23/n1';
-import {Column, Entity, PrimaryColumn} from 'typeorm';
-import {TypeOrmDataSourceHelper, TypeOrmDataSourceManager, TypeOrmLoadEntityByIdPipelineStep} from '../../src';
-
-const logger = createLogger();
-const config = createConfig(logger);
-
-// decorators are leading syntax errors in ide, since the test folder is not included in tsconfig.json
-// but the tricky thing is, once test folder included into tsconfig.json, d.ts files will be created in src folder
-// which cause currently use ts-ignore to avoid this syntax errors
-// @ts-ignore
-@Entity({name: 'T_TEST_TABLE'})
-export class TestTable {
-	// @ts-ignore
-	@PrimaryColumn('bigint', {name: 'ID'})
-	id: number;
-	// @ts-ignore
-	@Column('varchar', {name: 'CONTENT'})
-	content: string;
-}
-
-describe('TypeORM Entity Suite', () => {
-	beforeAll(async () => {
-		process.env.CFG_TYPEORM_TEST_TYPE = 'better-sqlite3';
-		process.env.CFG_TYPEORM_TEST_SYNCHRONIZE = 'true';
-		// in memory, kept in global cache, to make sure always use same one.
-		process.env.CFG_TYPEORM_TEST_KEPT_ON_GLOBAL = 'true';
-		await new TypeOrmDataSourceHelper(config).create({
-			'TEST': [TestTable]
-		});
-		const repo = (await TypeOrmDataSourceManager.findDataSource('TEST', config)).getDataSource().getRepository(TestTable);
-		await repo.insert({id: 1, content: 'hello world!'});
-	});
-
-	test('TypeORM load entity by id Pipeline Step Test #1', async () => {
-		const step = new TypeOrmLoadEntityByIdPipelineStep({
-			config, logger, dataSourceName: 'TEST', entityName: 'TestTable',
-			autonomous: true
-		});
-		const request = {content: 1};
-		const response = await step.perform(request);
-		expect(response.content).not.toBeNull();
-		expect(response.content.id).toBe(1);
-		expect(response.content.content).toBe('hello world!');
-	});
-
-	afterAll((done) => {
-		done();
-	});
-});