@rainbow-o23/n3 1.0.47 → 1.0.49

package/README.md

@@ -398,44 +398,6 @@ Autonomous transactions take precedence over the transaction name, meaning that
 specified by the transaction name will be ignored. If you need to use the transaction name, you must nest the pipeline steps within
 transactional step sets, and ensure that the datasource name and transaction name remain the same.
 
-### By Entity (Deprecated, not recommended)
-
-#### Load Entity by ID
-
-##### Constructor Parameters
-
-| Name       | Type   | Default Value | Comments             |
-|------------|--------|---------------|----------------------|
-| entityName | string |               | TypeOrm entity name. |
-
-##### Request and Response
-
-```typescript
-// request
-export type TypeOrmIdType = string | number | bigint;
-// response
-export type TypeOrmEntityToLoad = Undefinable<DeepPartial<ObjectLiteral>>;
-```
-
-#### Save Entity
-
-##### Constructor Parameters
-
-| Name                   | Type                                   | Default Value | Comments             |
-|------------------------|----------------------------------------|---------------|----------------------|
-| entityName             | string                                 |               | TypeOrm entity name. |
-| fillIdBySnowflake      | boolean                                | false         |                      |
-| uniquenessCheckSnippet | ScriptFuncOrBody\<UniquenessCheckFunc> |               |                      |
-
-##### Request and Response
-
-```typescript
-// request
-export type EntityToSave = DeepPartial<ObjectLiteral>;
-// response
-export type EntityToSave = DeepPartial<ObjectLiteral>;
-```
-
 ### By SQL
 
 #### Environment Parameters
@@ -526,6 +488,34 @@ export interface TypeOrmLoadBasis extends TypeOrmBasis {
 Array<TypeOrmEntityToLoad>;
 ```
 
+#### Load Many by SQL, Use Cursor
+
+##### Environment Parameters
+
+| Name                    | Type   | Default Value | Comments    |
+|-------------------------|--------|---------------|-------------|
+| `typeorm.DB.fetch.size` | number | 20            | Fetch size. |
+
+##### Request and Response
+
+```typescript
+// request
+export interface TypeOrmLoadBasis extends TypeOrmBasis {
+	params?: Array<TypeOrmEntityValue> | TypeOrmEntityToSave;
+}
+
+// response
+Array<any>;
+```
+
+By specifying `fetchSize`, each batch of data retrieved will execute sub-steps. Before executing the sub-steps, the data to be passed to it
+will be calculated using the `streamTo` function. If `streamTo` is not specified, the batch of data retrieved itself will be passed to the
+sub-steps. If the sub-steps is not specified, all retrieved data will be merged and returned.
+
+Therefore, the number of times the sub-step is executed is related to the quantity of data and the `fetchSize`. Meanwhile, each time the
+sub-step is invoked, the context will include a `$$typeOrmCursorRound` variable indicating the current batch (starting from 0), and a
+`$typeOrmCursorEnd` variable indicating whether it is the last batch.
+
 #### Save by SQL
 
 ##### Request and Response

package/index.cjs

@@ -25,6 +25,7 @@ const ERR_PIPELINE_SNIPPET_CANNOT_USE_GLOBAL = 'O03-00017';
 const ERR_PIPELINE_SNIPPET_CANNOT_USE_PROCESS = 'O03-00018';
 const ERR_PIPELINE_SNIPPET_CANNOT_USE_EVAL = 'O03-00019';
 const ERR_PIPELINE_SNIPPET_CANNOT_USE_FUNCTION = 'O03-00020';
+const ERR_TYPEORM_STREAM = 'O03-00021';
 n1.ErrorCodes.ERR_PIPELINE_STEP_SNIPPET_NOT_EMPTY = ERR_PIPELINE_STEP_SNIPPET_NOT_EMPTY;
 n1.ErrorCodes.ERR_PIPELINE_STEP_CONDITIONAL_SNIPPET_NOT_EMPTY = ERR_PIPELINE_STEP_CONDITIONAL_SNIPPET_NOT_EMPTY;
 n1.ErrorCodes.ERR_TYPEORM_DATASOURCE_TYPE_NOT_FOUND = ERR_TYPEORM_DATASOURCE_TYPE_NOT_FOUND;
@@ -45,6 +46,7 @@ n1.ErrorCodes.ERR_PIPELINE_SNIPPET_CANNOT_USE_GLOBAL = ERR_PIPELINE_SNIPPET_CANN
 n1.ErrorCodes.ERR_PIPELINE_SNIPPET_CANNOT_USE_PROCESS = ERR_PIPELINE_SNIPPET_CANNOT_USE_PROCESS;
 n1.ErrorCodes.ERR_PIPELINE_SNIPPET_CANNOT_USE_EVAL = ERR_PIPELINE_SNIPPET_CANNOT_USE_EVAL;
 n1.ErrorCodes.ERR_PIPELINE_SNIPPET_CANNOT_USE_FUNCTION = ERR_PIPELINE_SNIPPET_CANNOT_USE_FUNCTION;
+n1.ErrorCodes.ERR_TYPEORM_STREAM = ERR_TYPEORM_STREAM;
 
 class AbstractTypeOrmDataSource {
     _name;
@@ -392,7 +394,7 @@ class Utils {
                         suppressOutputPathCheck: false,
                         skipLibCheck: true,
                         skipDefaultLibCheck: true,
-                        moduleResolution: ts.ModuleResolutionKind.Node16
+                        moduleResolution: ts.ModuleResolutionKind.NodeNext
                     }
                 });
                 snippet = transpiled.outputText;
@@ -2216,6 +2218,122 @@ class TypeOrmLoadManyBySQLPipelineStep extends AbstractTypeOrmLoadBySQLPipelineS
     }
 }
 
+class TypeOrmLoadManyBySQLUseCursorPipelineStep extends AbstractTypeOrmBySQLPipelineStep {
+    _fetchSize;
+    _streamToSnippet;
+    _streamToFunc;
+    _stepBuilders;
+    constructor(options) {
+        super(options);
+        const config = this.getConfig();
+        this._fetchSize = options.fetchSize ?? config.getNumber(`typeorm.${this.getDataSourceName()}.fetch.size`, 20);
+        this._streamToSnippet = options.streamTo;
+        this._streamToFunc = Utils.createAsyncFunction(this.getStreamToSnippet(), {
+            createDefault: () => (async ($factor) => $factor),
+            getVariableNames: () => this.generateVariableNames(),
+            error: (e) => {
+                this.getLogger().error(`Failed on create function for snippet[${this.getStreamToSnippet()}].`);
+                throw e;
+            }
+        });
+        this._stepBuilders = options.steps;
+    }
+    getFetchSize() {
+        return this._fetchSize;
+    }
+    getStreamToSnippet() {
+        return this._streamToSnippet;
+    }
+    generateVariableNames() {
+        return [this.getFetchDataVariableName(), this.getRequestVariableName(), ...this.getHelpersVariableNames()];
+    }
+    getStepBuilders() {
+        return this._stepBuilders ?? [];
+    }
+    async doPerform(basis, request) {
+        const { sql, params } = this.getSql(basis, basis?.params);
+        return await this.autoTrans(async (runner) => {
+            const results = [];
+            const rows = [];
+            let cursorRound = 0;
+            const pipe = async ({ resolve, reject, end }) => {
+                if (!end && rows.length < this.getFetchSize()) {
+                    return;
+                }
+                try {
+                    const contentForSub = await this._streamToFunc([...rows], request, this.getHelpers(), this.getHelpers());
+                    rows.length = 0;
+                    let resultContent;
+                    if (this.getStepBuilders().length === 0) {
+                        resultContent = contentForSub;
+                    }
+                    else {
+                        const sets = new PipelineStepSets({
+                            ...this.buildStepOptions(), name: this.getName(), steps: this.getStepBuilders()
+                        });
+                        const { content: _, $context, ...rest } = request;
+                        const contextForSub = { ...$context, $typeOrmCursorRound: cursorRound, $typeOrmCursorEnd: end };
+                        const requestForSub = { ...rest, $context: contextForSub, content: contentForSub };
+                        const result = await sets.perform(requestForSub);
+                        const { content } = result;
+                        resultContent = content;
+                    }
+                    cursorRound = cursorRound + 1;
+                    if (resultContent == null || resultContent == n1.PIPELINE_STEP_RETURN_NULL) {
+                    }
+                    else if (Array.isArray(resultContent)) {
+                        results.push(...resultContent);
+                    }
+                    else {
+                        results.push(resultContent);
+                    }
+                }
+                catch (e) {
+                    reject(e);
+                }
+                if (end) {
+                    resolve(results);
+                }
+            };
+            const close = async (readable) => {
+                try {
+                    readable?.destroy();
+                }
+                catch (e) {
+                    this.getLogger().error(e);
+                }
+            };
+            const read = async ({ resolve, reject }) => {
+                const readable = await runner.stream(sql, params, async () => {
+                    await close(readable);
+                    await pipe({ resolve, reject, end: true });
+                }, async (e) => {
+                    await close(readable);
+                    reject(e);
+                });
+                readable.on('data', async (data) => {
+                    readable.pause();
+                    rows.push(data);
+                    await pipe({
+                        resolve, reject: async (e) => {
+                            await close(readable);
+                            reject(e);
+                        }, end: false
+                    });
+                    readable.resume();
+                });
+            };
+            return new Promise((resolve, reject) => read({ resolve, reject }));
+        }, request);
+    }
+    getFetchDataVariableName() {
+        return '$factor';
+    }
+    getRequestVariableName() {
+        return '$request';
+    }
+}
+
 class TypeOrmSaveBySQLPipelineStep extends AbstractTypeOrmBySQLPipelineStep {
     async doPerform(basis, request) {
         const { sql, params } = this.getSql(basis, basis?.values);
@@ -2355,6 +2473,7 @@ exports.ERR_TYPEORM_DATASOURCE_TYPE_NOT_FOUND = ERR_TYPEORM_DATASOURCE_TYPE_NOT_
 exports.ERR_TYPEORM_ENTITY_NOT_FOUND = ERR_TYPEORM_ENTITY_NOT_FOUND;
 exports.ERR_TYPEORM_SQL_NOT_EMPTY = ERR_TYPEORM_SQL_NOT_EMPTY;
 exports.ERR_TYPEORM_STEP_SNIPPET_NOT_EMPTY = ERR_TYPEORM_STEP_SNIPPET_NOT_EMPTY;
+exports.ERR_TYPEORM_STREAM = ERR_TYPEORM_STREAM;
 exports.ERR_TYPEORM_TRANSACTION_NOT_FOUND = ERR_TYPEORM_TRANSACTION_NOT_FOUND;
 exports.EachPipelineStepSets = EachPipelineStepSets;
 exports.FetchPipelineStep = FetchPipelineStep;
@@ -2377,6 +2496,7 @@ exports.TypeOrmBySnippetPipelineStep = TypeOrmBySnippetPipelineStep;
 exports.TypeOrmDataSourceHelper = TypeOrmDataSourceHelper;
 exports.TypeOrmDataSourceManager = TypeOrmDataSourceManager;
 exports.TypeOrmLoadManyBySQLPipelineStep = TypeOrmLoadManyBySQLPipelineStep;
+exports.TypeOrmLoadManyBySQLUseCursorPipelineStep = TypeOrmLoadManyBySQLUseCursorPipelineStep;
 exports.TypeOrmLoadOneBySQLPipelineStep = TypeOrmLoadOneBySQLPipelineStep;
 exports.TypeOrmParsedSQLCache = TypeOrmParsedSQLCache;
 exports.TypeOrmSaveBySQLPipelineStep = TypeOrmSaveBySQLPipelineStep;

package/index.js

@@ -1,4 +1,4 @@
-import { ErrorCodes, UncatchableError, AbstractPipelineStep, CatchableError, ExposedUncatchableError, ERR_UNKNOWN, PipelineRepository, StepHelpersUtils } from '@rainbow-o23/n1';
+import { ErrorCodes, UncatchableError, AbstractPipelineStep, CatchableError, ExposedUncatchableError, ERR_UNKNOWN, PipelineRepository, StepHelpersUtils, PIPELINE_STEP_RETURN_NULL } from '@rainbow-o23/n1';
 import { DataSource } from 'typeorm';
 import ts, { ScriptTarget, JsxEmit, ModuleKind, ModuleResolutionKind } from 'typescript';
 import fetch from 'node-fetch';
@@ -23,6 +23,7 @@ const ERR_PIPELINE_SNIPPET_CANNOT_USE_GLOBAL = 'O03-00017';
 const ERR_PIPELINE_SNIPPET_CANNOT_USE_PROCESS = 'O03-00018';
 const ERR_PIPELINE_SNIPPET_CANNOT_USE_EVAL = 'O03-00019';
 const ERR_PIPELINE_SNIPPET_CANNOT_USE_FUNCTION = 'O03-00020';
+const ERR_TYPEORM_STREAM = 'O03-00021';
 ErrorCodes.ERR_PIPELINE_STEP_SNIPPET_NOT_EMPTY = ERR_PIPELINE_STEP_SNIPPET_NOT_EMPTY;
 ErrorCodes.ERR_PIPELINE_STEP_CONDITIONAL_SNIPPET_NOT_EMPTY = ERR_PIPELINE_STEP_CONDITIONAL_SNIPPET_NOT_EMPTY;
 ErrorCodes.ERR_TYPEORM_DATASOURCE_TYPE_NOT_FOUND = ERR_TYPEORM_DATASOURCE_TYPE_NOT_FOUND;
@@ -43,6 +44,7 @@ ErrorCodes.ERR_PIPELINE_SNIPPET_CANNOT_USE_GLOBAL = ERR_PIPELINE_SNIPPET_CANNOT_
 ErrorCodes.ERR_PIPELINE_SNIPPET_CANNOT_USE_PROCESS = ERR_PIPELINE_SNIPPET_CANNOT_USE_PROCESS;
 ErrorCodes.ERR_PIPELINE_SNIPPET_CANNOT_USE_EVAL = ERR_PIPELINE_SNIPPET_CANNOT_USE_EVAL;
 ErrorCodes.ERR_PIPELINE_SNIPPET_CANNOT_USE_FUNCTION = ERR_PIPELINE_SNIPPET_CANNOT_USE_FUNCTION;
+ErrorCodes.ERR_TYPEORM_STREAM = ERR_TYPEORM_STREAM;
 
 class AbstractTypeOrmDataSource {
     _name;
@@ -390,7 +392,7 @@ class Utils {
                         suppressOutputPathCheck: false,
                         skipLibCheck: true,
                         skipDefaultLibCheck: true,
-                        moduleResolution: ModuleResolutionKind.Node16
+                        moduleResolution: ModuleResolutionKind.NodeNext
                     }
                 });
                 snippet = transpiled.outputText;
@@ -2214,6 +2216,122 @@ class TypeOrmLoadManyBySQLPipelineStep extends AbstractTypeOrmLoadBySQLPipelineS
     }
 }
 
+class TypeOrmLoadManyBySQLUseCursorPipelineStep extends AbstractTypeOrmBySQLPipelineStep {
+    _fetchSize;
+    _streamToSnippet;
+    _streamToFunc;
+    _stepBuilders;
+    constructor(options) {
+        super(options);
+        const config = this.getConfig();
+        this._fetchSize = options.fetchSize ?? config.getNumber(`typeorm.${this.getDataSourceName()}.fetch.size`, 20);
+        this._streamToSnippet = options.streamTo;
+        this._streamToFunc = Utils.createAsyncFunction(this.getStreamToSnippet(), {
+            createDefault: () => (async ($factor) => $factor),
+            getVariableNames: () => this.generateVariableNames(),
+            error: (e) => {
+                this.getLogger().error(`Failed on create function for snippet[${this.getStreamToSnippet()}].`);
+                throw e;
+            }
+        });
+        this._stepBuilders = options.steps;
+    }
+    getFetchSize() {
+        return this._fetchSize;
+    }
+    getStreamToSnippet() {
+        return this._streamToSnippet;
+    }
+    generateVariableNames() {
+        return [this.getFetchDataVariableName(), this.getRequestVariableName(), ...this.getHelpersVariableNames()];
+    }
+    getStepBuilders() {
+        return this._stepBuilders ?? [];
+    }
+    async doPerform(basis, request) {
+        const { sql, params } = this.getSql(basis, basis?.params);
+        return await this.autoTrans(async (runner) => {
+            const results = [];
+            const rows = [];
+            let cursorRound = 0;
+            const pipe = async ({ resolve, reject, end }) => {
+                if (!end && rows.length < this.getFetchSize()) {
+                    return;
+                }
+                try {
+                    const contentForSub = await this._streamToFunc([...rows], request, this.getHelpers(), this.getHelpers());
+                    rows.length = 0;
+                    let resultContent;
+                    if (this.getStepBuilders().length === 0) {
+                        resultContent = contentForSub;
+                    }
+                    else {
+                        const sets = new PipelineStepSets({
+                            ...this.buildStepOptions(), name: this.getName(), steps: this.getStepBuilders()
+                        });
+                        const { content: _, $context, ...rest } = request;
+                        const contextForSub = { ...$context, $typeOrmCursorRound: cursorRound, $typeOrmCursorEnd: end };
+                        const requestForSub = { ...rest, $context: contextForSub, content: contentForSub };
+                        const result = await sets.perform(requestForSub);
+                        const { content } = result;
+                        resultContent = content;
+                    }
+                    cursorRound = cursorRound + 1;
+                    if (resultContent == null || resultContent == PIPELINE_STEP_RETURN_NULL) {
+                    }
+                    else if (Array.isArray(resultContent)) {
+                        results.push(...resultContent);
+                    }
+                    else {
+                        results.push(resultContent);
+                    }
+                }
+                catch (e) {
+                    reject(e);
+                }
+                if (end) {
+                    resolve(results);
+                }
+            };
+            const close = async (readable) => {
+                try {
+                    readable?.destroy();
+                }
+                catch (e) {
+                    this.getLogger().error(e);
+                }
+            };
+            const read = async ({ resolve, reject }) => {
+                const readable = await runner.stream(sql, params, async () => {
+                    await close(readable);
+                    await pipe({ resolve, reject, end: true });
+                }, async (e) => {
+                    await close(readable);
+                    reject(e);
+                });
+                readable.on('data', async (data) => {
+                    readable.pause();
+                    rows.push(data);
+                    await pipe({
+                        resolve, reject: async (e) => {
+                            await close(readable);
+                            reject(e);
+                        }, end: false
+                    });
+                    readable.resume();
+                });
+            };
+            return new Promise((resolve, reject) => read({ resolve, reject }));
+        }, request);
+    }
+    getFetchDataVariableName() {
+        return '$factor';
+    }
+    getRequestVariableName() {
+        return '$request';
+    }
+}
+
 class TypeOrmSaveBySQLPipelineStep extends AbstractTypeOrmBySQLPipelineStep {
     async doPerform(basis, request) {
         const { sql, params } = this.getSql(basis, basis?.values);
@@ -2324,4 +2442,4 @@ class TypeOrmTransactionalPipelineStepSets extends PipelineStepSets {
     }
 }
 
-export { AbstractFragmentaryPipelineStep, AbstractTypeOrmBySQLPipelineStep, AbstractTypeOrmDataSource, AbstractTypeOrmLoadBySQLPipelineStep, AbstractTypeOrmPipelineStep, AsyncPipelineStepSets, BetterSqlite3TypeOrmDatasource, ConditionalPipelineStepSets, DEFAULT_TRANSACTION_NAME, DeletePropertyPipelineStep, ERR_EACH_FRAGMENT_NOT_ANY_ARRAY, ERR_FETCH_ERROR, ERR_PIPELINE_REF_NOT_EMPTY, ERR_PIPELINE_REF_NOT_FOUND, ERR_PIPELINE_SNIPPET_CANNOT_USE_EVAL, ERR_PIPELINE_SNIPPET_CANNOT_USE_FUNCTION, ERR_PIPELINE_SNIPPET_CANNOT_USE_GLOBAL, ERR_PIPELINE_SNIPPET_CANNOT_USE_PROCESS, ERR_PIPELINE_STEP_CONDITIONAL_SNIPPET_NOT_EMPTY, ERR_PIPELINE_STEP_METHOD_NOT_SUPPORTED, ERR_PIPELINE_STEP_REF_NOT_EMPTY, ERR_PIPELINE_STEP_REF_NOT_FOUND, ERR_PIPELINE_STEP_SNIPPET_NOT_EMPTY, ERR_TYPEORM_DATASOURCE_CREATOR_NOT_FOUND, ERR_TYPEORM_DATASOURCE_NOT_FOUND, ERR_TYPEORM_DATASOURCE_TYPE_NOT_FOUND, ERR_TYPEORM_ENTITY_NOT_FOUND, ERR_TYPEORM_SQL_NOT_EMPTY, ERR_TYPEORM_STEP_SNIPPET_NOT_EMPTY, ERR_TYPEORM_TRANSACTION_NOT_FOUND, EachPipelineStepSets, FetchPipelineStep, GetPropertyPipelineStep, HttpAbortErrorCode, HttpUnknownErrorCode, MssqlTypeOrmDatasource, MysqlTypeOrmDatasource, OracleTypeOrmDatasource, ParallelPipelineStepSets, ParsedSqlSegmentType, PgsqlTypeOrmDatasource, PipelineStepSets, RefPipelinePipelineStep, RefStepPipelineStep, RoutesPipelineStepSets, SnippetPipelineStep, SnowflakePipelineStep, SupportedDataSourceTypes, TypeOrmBulkSaveBySQLPipelineStep, TypeOrmBySnippetPipelineStep, TypeOrmDataSourceHelper, TypeOrmDataSourceManager, TypeOrmLoadManyBySQLPipelineStep, TypeOrmLoadOneBySQLPipelineStep, TypeOrmParsedSQLCache, TypeOrmSaveBySQLPipelineStep, TypeOrmTransactionalPipelineStepSets, Utils };
+export { AbstractFragmentaryPipelineStep, AbstractTypeOrmBySQLPipelineStep, AbstractTypeOrmDataSource, AbstractTypeOrmLoadBySQLPipelineStep, AbstractTypeOrmPipelineStep, AsyncPipelineStepSets, BetterSqlite3TypeOrmDatasource, ConditionalPipelineStepSets, DEFAULT_TRANSACTION_NAME, DeletePropertyPipelineStep, ERR_EACH_FRAGMENT_NOT_ANY_ARRAY, ERR_FETCH_ERROR, ERR_PIPELINE_REF_NOT_EMPTY, ERR_PIPELINE_REF_NOT_FOUND, ERR_PIPELINE_SNIPPET_CANNOT_USE_EVAL, ERR_PIPELINE_SNIPPET_CANNOT_USE_FUNCTION, ERR_PIPELINE_SNIPPET_CANNOT_USE_GLOBAL, ERR_PIPELINE_SNIPPET_CANNOT_USE_PROCESS, ERR_PIPELINE_STEP_CONDITIONAL_SNIPPET_NOT_EMPTY, ERR_PIPELINE_STEP_METHOD_NOT_SUPPORTED, ERR_PIPELINE_STEP_REF_NOT_EMPTY, ERR_PIPELINE_STEP_REF_NOT_FOUND, ERR_PIPELINE_STEP_SNIPPET_NOT_EMPTY, ERR_TYPEORM_DATASOURCE_CREATOR_NOT_FOUND, ERR_TYPEORM_DATASOURCE_NOT_FOUND, ERR_TYPEORM_DATASOURCE_TYPE_NOT_FOUND, ERR_TYPEORM_ENTITY_NOT_FOUND, ERR_TYPEORM_SQL_NOT_EMPTY, ERR_TYPEORM_STEP_SNIPPET_NOT_EMPTY, ERR_TYPEORM_STREAM, ERR_TYPEORM_TRANSACTION_NOT_FOUND, EachPipelineStepSets, FetchPipelineStep, GetPropertyPipelineStep, HttpAbortErrorCode, HttpUnknownErrorCode, MssqlTypeOrmDatasource, MysqlTypeOrmDatasource, OracleTypeOrmDatasource, ParallelPipelineStepSets, ParsedSqlSegmentType, PgsqlTypeOrmDatasource, PipelineStepSets, RefPipelinePipelineStep, RefStepPipelineStep, RoutesPipelineStepSets, SnippetPipelineStep, SnowflakePipelineStep, SupportedDataSourceTypes, TypeOrmBulkSaveBySQLPipelineStep, TypeOrmBySnippetPipelineStep, TypeOrmDataSourceHelper, TypeOrmDataSourceManager, TypeOrmLoadManyBySQLPipelineStep, TypeOrmLoadManyBySQLUseCursorPipelineStep, TypeOrmLoadOneBySQLPipelineStep, TypeOrmParsedSQLCache, TypeOrmSaveBySQLPipelineStep, TypeOrmTransactionalPipelineStepSets, Utils };

package/lib/error-codes.d.ts

@@ -19,3 +19,4 @@ export declare const ERR_PIPELINE_SNIPPET_CANNOT_USE_GLOBAL: O23ReservedErrorCod
 export declare const ERR_PIPELINE_SNIPPET_CANNOT_USE_PROCESS: O23ReservedErrorCode;
 export declare const ERR_PIPELINE_SNIPPET_CANNOT_USE_EVAL: O23ReservedErrorCode;
 export declare const ERR_PIPELINE_SNIPPET_CANNOT_USE_FUNCTION: O23ReservedErrorCode;
+export declare const ERR_TYPEORM_STREAM: O23ReservedErrorCode;

package/lib/typeorm-step/index.d.ts

@@ -5,6 +5,7 @@ export * from './abstract-typeorm-by-sql-step';
 export * from './abstract-typeorm-load-by-sql-step';
 export * from './typeorm-load-one-by-sql-step';
 export * from './typeorm-load-many-by-sql-step';
+export * from './typeorm-load-many-by-sql-use-cursor-step';
 export * from './typeorm-save-by-sql-step';
 export * from './typeorm-bulk-save-by-sql-step';
 export * from './typeorm-transactional-step-sets';

package/lib/typeorm-step/typeorm-load-many-by-sql-use-cursor-step.d.ts

@@ -0,0 +1,25 @@
+import { PipelineStepBuilder, PipelineStepData, PipelineStepHelpers, PipelineStepPayload, Undefinable } from '@rainbow-o23/n1';
+import { ScriptFuncOrBody } from '../step';
+import { AbstractTypeOrmBySQLPipelineStep, TypeOrmBySQLPipelineStepOptions } from './abstract-typeorm-by-sql-step';
+import { TypeOrmLoadBasis } from './abstract-typeorm-load-by-sql-step';
+export type StreamToSubSteps = any;
+export type StreamToFunc<In> = ($factor: Array<any>, $request: PipelineStepData<In>, $helpers: PipelineStepHelpers, $: PipelineStepHelpers) => Promise<StreamToSubSteps>;
+export interface TypeOrmLoadManyBySQLUseCursorPipelineStepOptions<In = PipelineStepPayload, Out = PipelineStepPayload, InFragment = TypeOrmLoadBasis, OutFragment = Out> extends TypeOrmBySQLPipelineStepOptions<In, Out, InFragment, OutFragment> {
+    fetchSize?: number;
+    streamTo?: ScriptFuncOrBody<StreamToFunc<In>>;
+    steps?: Array<PipelineStepBuilder>;
+}
+export declare class TypeOrmLoadManyBySQLUseCursorPipelineStep<In = PipelineStepPayload, Out = PipelineStepPayload, InFragment = Undefinable<TypeOrmLoadBasis>, OutFragment = Out> extends AbstractTypeOrmBySQLPipelineStep<In, Out, Undefinable<TypeOrmLoadBasis>, OutFragment> {
+    private readonly _fetchSize;
+    private readonly _streamToSnippet;
+    private readonly _streamToFunc;
+    private readonly _stepBuilders;
+    constructor(options: TypeOrmLoadManyBySQLUseCursorPipelineStepOptions<In, Out, InFragment, OutFragment>);
+    protected getFetchSize(): number;
+    getStreamToSnippet(): ScriptFuncOrBody<StreamToFunc<In>>;
+    protected generateVariableNames(): Array<string>;
+    protected getStepBuilders(): Array<PipelineStepBuilder>;
+    protected doPerform(basis: Undefinable<TypeOrmLoadBasis>, request: PipelineStepData<In>): Promise<Undefinable<OutFragment>>;
+    protected getFetchDataVariableName(): string;
+    protected getRequestVariableName(): string;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@rainbow-o23/n3",
-	"version": "1.0.47",
+	"version": "1.0.49",
 	"description": "o23 pipelines",
 	"main": "index.cjs",
 	"module": "index.js",
@@ -21,7 +21,7 @@
 		"url": "https://github.com/InsureMO/rainbow-o23/issues"
 	},
 	"dependencies": {
-		"@rainbow-o23/n1": "1.0.47",
+		"@rainbow-o23/n1": "1.0.49",
 		"node-fetch": "2.6.7",
 		"typeorm": "^0.3.20",
 		"typescript": "5.5.4"

package/src/lib/error-codes.ts

@@ -21,6 +21,7 @@ export const ERR_PIPELINE_SNIPPET_CANNOT_USE_GLOBAL: O23ReservedErrorCode = 'O03
 export const ERR_PIPELINE_SNIPPET_CANNOT_USE_PROCESS: O23ReservedErrorCode = 'O03-00018';
 export const ERR_PIPELINE_SNIPPET_CANNOT_USE_EVAL: O23ReservedErrorCode = 'O03-00019';
 export const ERR_PIPELINE_SNIPPET_CANNOT_USE_FUNCTION: O23ReservedErrorCode = 'O03-00020';
+export const ERR_TYPEORM_STREAM: O23ReservedErrorCode = 'O03-00021';
 
 ErrorCodes.ERR_PIPELINE_STEP_SNIPPET_NOT_EMPTY = ERR_PIPELINE_STEP_SNIPPET_NOT_EMPTY;
 ErrorCodes.ERR_PIPELINE_STEP_CONDITIONAL_SNIPPET_NOT_EMPTY = ERR_PIPELINE_STEP_CONDITIONAL_SNIPPET_NOT_EMPTY;
@@ -43,3 +44,4 @@ ErrorCodes.ERR_PIPELINE_SNIPPET_CANNOT_USE_GLOBAL = ERR_PIPELINE_SNIPPET_CANNOT_
 ErrorCodes.ERR_PIPELINE_SNIPPET_CANNOT_USE_PROCESS = ERR_PIPELINE_SNIPPET_CANNOT_USE_PROCESS;
 ErrorCodes.ERR_PIPELINE_SNIPPET_CANNOT_USE_EVAL = ERR_PIPELINE_SNIPPET_CANNOT_USE_EVAL;
 ErrorCodes.ERR_PIPELINE_SNIPPET_CANNOT_USE_FUNCTION = ERR_PIPELINE_SNIPPET_CANNOT_USE_FUNCTION;
+ErrorCodes.ERR_TYPEORM_STREAM = ERR_TYPEORM_STREAM;

package/src/lib/step/utils.ts

@@ -70,7 +70,7 @@ export class Utils {
 						suppressOutputPathCheck: false,
 						skipLibCheck: true,
 						skipDefaultLibCheck: true,
-						moduleResolution: ModuleResolutionKind.Node16 // default use node 16
+						moduleResolution: ModuleResolutionKind.NodeNext // default use node next
 					}
 				});
 				snippet = transpiled.outputText;

package/src/lib/typeorm-step/index.ts

@@ -8,6 +8,7 @@ export * from './abstract-typeorm-by-sql-step';
 export * from './abstract-typeorm-load-by-sql-step';
 export * from './typeorm-load-one-by-sql-step';
 export * from './typeorm-load-many-by-sql-step';
+export * from './typeorm-load-many-by-sql-use-cursor-step';
 
 export * from './typeorm-save-by-sql-step';
 export * from './typeorm-bulk-save-by-sql-step';

package/src/lib/typeorm-step/typeorm-load-many-by-sql-use-cursor-step.ts

@@ -0,0 +1,162 @@
+import {
+	PIPELINE_STEP_RETURN_NULL,
+	PipelineStepBuilder,
+	PipelineStepData,
+	PipelineStepHelpers,
+	PipelineStepPayload,
+	Undefinable
+} from '@rainbow-o23/n1';
+import {ReadStream} from 'fs';
+import {PipelineStepSets, ScriptFuncOrBody, Utils} from '../step';
+import {AbstractTypeOrmBySQLPipelineStep, TypeOrmBySQLPipelineStepOptions} from './abstract-typeorm-by-sql-step';
+import {TypeOrmLoadBasis} from './abstract-typeorm-load-by-sql-step';
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export type StreamToSubSteps = any;
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export type StreamToFunc<In> = ($factor: Array<any>, $request: PipelineStepData<In>, $helpers: PipelineStepHelpers, $: PipelineStepHelpers) => Promise<StreamToSubSteps>;
+
+export interface TypeOrmLoadManyBySQLUseCursorPipelineStepOptions<In = PipelineStepPayload, Out = PipelineStepPayload, InFragment = TypeOrmLoadBasis, OutFragment = Out>
+	extends TypeOrmBySQLPipelineStepOptions<In, Out, InFragment, OutFragment> {
+	fetchSize?: number;
+	streamTo?: ScriptFuncOrBody<StreamToFunc<In>>;
+	steps?: Array<PipelineStepBuilder>;
+}
+
+export class TypeOrmLoadManyBySQLUseCursorPipelineStep<In = PipelineStepPayload, Out = PipelineStepPayload, InFragment = Undefinable<TypeOrmLoadBasis>, OutFragment = Out>
+	extends AbstractTypeOrmBySQLPipelineStep<In, Out, Undefinable<TypeOrmLoadBasis>, OutFragment> {
+	private readonly _fetchSize: number;
+	private readonly _streamToSnippet: ScriptFuncOrBody<StreamToFunc<In>>;
+	private readonly _streamToFunc: StreamToFunc<In>;
+	private readonly _stepBuilders: Array<PipelineStepBuilder>;
+
+	public constructor(options: TypeOrmLoadManyBySQLUseCursorPipelineStepOptions<In, Out, InFragment, OutFragment>) {
+		super(options);
+		const config = this.getConfig();
+		this._fetchSize = options.fetchSize ?? config.getNumber(`typeorm.${this.getDataSourceName()}.fetch.size`, 20);
+		this._streamToSnippet = options.streamTo;
+		this._streamToFunc = Utils.createAsyncFunction(this.getStreamToSnippet(), {
+			// eslint-disable-next-line @typescript-eslint/no-explicit-any
+			createDefault: () => (async ($factor: Array<any>) => $factor),
+			getVariableNames: () => this.generateVariableNames(),
+			error: (e: Error) => {
+				this.getLogger().error(`Failed on create function for snippet[${this.getStreamToSnippet()}].`);
+				throw e;
+			}
+		});
+		this._stepBuilders = options.steps;
+	}
+
+	protected getFetchSize(): number {
+		return this._fetchSize;
+	}
+
+	public getStreamToSnippet(): ScriptFuncOrBody<StreamToFunc<In>> {
+		return this._streamToSnippet;
+	}
+
+	protected generateVariableNames(): Array<string> {
+		return [this.getFetchDataVariableName(), this.getRequestVariableName(), ...this.getHelpersVariableNames()];
+	}
+
+	protected getStepBuilders(): Array<PipelineStepBuilder> {
+		return this._stepBuilders ?? [];
+	}
+
+	protected async doPerform(basis: Undefinable<TypeOrmLoadBasis>, request: PipelineStepData<In>): Promise<Undefinable<OutFragment>> {
+		const {sql, params} = this.getSql(basis, basis?.params);
+		return await this.autoTrans<OutFragment>(async (runner) => {
+			const results = [];
+			const rows = [];
+			let cursorRound = 0;
+			const pipe = async ({resolve, reject, end}) => {
+				if (!end && rows.length < this.getFetchSize()) {
+					// not end, and size not meet the fresh required
+					// do nothing, wait for next
+					return;
+				}
+				try {
+					// get data from cache
+					const contentForSub = await this._streamToFunc([...rows], request, this.getHelpers(), this.getHelpers());
+					// clear cache
+					rows.length = 0;
+					// eslint-disable-next-line @typescript-eslint/no-explicit-any
+					let resultContent: any;
+					if (this.getStepBuilders().length === 0) {
+						// no sub step, use content as result
+						resultContent = contentForSub;
+					} else {
+						// create a step sets to run
+						const sets = new PipelineStepSets({
+							...this.buildStepOptions(), name: this.getName(), steps: this.getStepBuilders()
+						});
+						// eslint-disable-next-line @typescript-eslint/no-unused-vars
+						const {content: _, $context, ...rest} = request;
+						// pass a cursor end indicator to sub steps
+						const contextForSub = {...$context, $typeOrmCursorRound: cursorRound, $typeOrmCursorEnd: end};
+						const requestForSub = {...rest, $context: contextForSub, content: contentForSub};
+						const result = await sets.perform(requestForSub);
+						const {content} = result;
+						resultContent = content;
+					}
+					cursorRound = cursorRound + 1;
+					if (resultContent == null || resultContent == PIPELINE_STEP_RETURN_NULL) {
+						// ignore
+					} else if (Array.isArray(resultContent)) {
+						results.push(...resultContent);
+					} else {
+						results.push(resultContent);
+					}
+				} catch (e) {
+					reject(e);
+				}
+				if (end) {
+					resolve(results);
+				}
+			};
+			const close = async (readable: ReadStream) => {
+				// never throw exception from this function
+				try {
+					readable?.destroy();
+				} catch (e) {
+					// ignore this error
+					this.getLogger().error(e);
+				}
+			};
+			const read = async ({resolve, reject}) => {
+				const readable = await runner.stream(sql, params, async () => {
+					// on end
+					await close(readable);
+					await pipe({resolve, reject, end: true});
+				}, async (e: Error) => {
+					// on error
+					await close(readable);
+					reject(e);
+				});
+				readable.on('data', async (data) => {
+					readable.pause();
+					rows.push(data);
+					await pipe({
+						resolve, reject: async (e: Error) => {
+							await close(readable);
+							reject(e);
+						}, end: false
+					});
+					readable.resume();
+				});
+			};
+			return new Promise<OutFragment>((resolve, reject) => read({resolve, reject}));
+		}, request);
+	}
+
+	/**
+	 * override this method when want to use another variable name rather than "$factor"
+	 */
+	protected getFetchDataVariableName(): string {
+		return '$factor';
+	}
+
+	protected getRequestVariableName(): string {
+		return '$request';
+	}
+}

package/test/step/typeorm-by-cursor.test.ts

@@ -0,0 +1,55 @@
+import {createConfig, createLogger} from '@rainbow-o23/n1';
+import {TypeOrmDataSourceHelper, TypeOrmLoadManyBySQLUseCursorPipelineStep} from '../../src';
+
+const logger = createLogger();
+const config = createConfig(logger);
+
+describe('TypeORM Cursor Suite', () => {
+	beforeAll(async () => {
+		process.env.CFG_TYPEORM_TEST_HOST = 'localhost';
+		process.env.CFG_TYPEORM_TEST_USERNAME = 'o23';
+		process.env.CFG_TYPEORM_TEST_PASSWORD = 'o23';
+		const type = 'pg' + 'sql';
+		if (type === 'mysql') {
+			process.env.CFG_TYPEORM_TEST_TYPE = 'mysql';
+			process.env.CFG_TYPEORM_TEST_PORT = '3306';
+			process.env.CFG_TYPEORM_TEST_DATABASE = 'o23';
+		} else if (type === 'pgsql') {
+			process.env.CFG_TYPEORM_TEST_TYPE = 'pgsql';
+			process.env.CFG_TYPEORM_TEST_PORT = '5432';
+			process.env.CFG_TYPEORM_TEST_DATABASE = 'postgres';
+			process.env.CFG_TYPEORM_TEST_SCHEMA = 'o23';
+		} else if (type === 'mssql') {
+			process.env.CFG_TYPEORM_TEST_TYPE = 'mssql';
+			process.env.CFG_TYPEORM_TEST_PORT = '1433';
+			process.env.CFG_TYPEORM_TEST_PASSWORD = 'o23O23o23!';
+			process.env.CFG_TYPEORM_TEST_DATABASE = 'o23';
+			process.env.CFG_TYPEORM_TEST_SCHEMA = 'dbo';
+			process.env.CFG_TYPEORM_TEST_TRUST_SERVER_CERTIFICATE = 'true';
+		} else {
+			process.env.CFG_TYPEORM_TEST_TYPE = 'oracle';
+			process.env.CFG_TYPEORM_TEST_HOST = '127.0.0.1';
+			process.env.CFG_TYPEORM_TEST_PORT = '1521';
+			process.env.CFG_TYPEORM_TEST_SERVICE_NAME = 'orcl';
+		}
+		// process.env.CFG_TYPEORM_TEST_LOGGING = 'true';
+		await new TypeOrmDataSourceHelper(config).create({});
+		// const repo = (await TypeOrmDataSourceManager.findDataSource('TEST', config)).getDataSource().getRepository(TestTable);
+	});
+
+	test('TypeORM load many using cursor #1', async () => {
+		// noinspection SqlResolve
+		const step = new TypeOrmLoadManyBySQLUseCursorPipelineStep<any, any>({
+			config, logger, dataSourceName: 'TEST', sql: 'SELECT * FROM T_O23_DB_CHANGE_LOG',
+			autonomous: true
+		});
+		const request = {content: (void 0)};
+		const response = await step.perform(request);
+		expect(response.content).not.toBeNull();
+		console.log(response.content);
+	});
+
+	afterAll((done) => {
+		done();
+	});
+});

package/test/step/typeorm-by-sql-transactional.test.ts

@@ -70,10 +70,10 @@ class TransactionalTestStepSets extends TypeOrmTransactionalPipelineStepSets {
 						logger,
 						dataSourceName: 'TEST',
 						sql: 'SELECT ID id, CONTENT content FROM T_TEST_TABLE WHERE ID = ?',
-						fromRequest: ($factor: TransactionalRequest, _$request: PipelineStepData<TransactionalRequest>) => {
+						fromRequest: async ($factor: TransactionalRequest, _$request: PipelineStepData<TransactionalRequest>) => {
 							return {params: [$factor.idToLoad]} as TypeOrmBasis;
 						},
-						toResponse: ($result: TestTable, _$request: PipelineStepData) => {
+						toResponse: async ($result: TestTable, _$request: PipelineStepData) => {
 							expect($result.id).toBe(1);
 							expect($result.content).toBe('hello world!');
 							return {loadedById: $result};
@@ -87,10 +87,10 @@ class TransactionalTestStepSets extends TypeOrmTransactionalPipelineStepSets {
 						logger,
 						dataSourceName: 'TEST',
 						sql: 'INSERT INTO T_TEST_TABLE(ID, CONTENT) VALUES (?, ?)',
-						fromRequest: ($factor: TransactionalRequest, _$request: PipelineStepData<TransactionalRequest>) => {
+						fromRequest: async ($factor: TransactionalRequest, _$request: PipelineStepData<TransactionalRequest>) => {
 							return {values: [$factor.item3.id, $factor.item3.content]};
 						},
-						toResponse: ($result: TypeOrmIdOfInserted, _$request: PipelineStepData) => {
+						toResponse: async ($result: TypeOrmIdOfInserted, _$request: PipelineStepData) => {
 							expect($result).toBe(3);
 							return {insertedId: $result};
 						},
@@ -104,10 +104,10 @@ class TransactionalTestStepSets extends TypeOrmTransactionalPipelineStepSets {
 						dataSourceName: 'TEST',
 						autonomous: true,
 						sql: 'SELECT ID id, CONTENT content FROM T_TEST_TABLE WHERE ID = ?',
-						fromRequest: ($factor: TransactionalRequest, _$request: PipelineStepData<TransactionalRequest>) => {
+						fromRequest: async ($factor: TransactionalRequest, _$request: PipelineStepData<TransactionalRequest>) => {
 							return {params: [$factor.item3.id]};
 						},
-						toResponse: ($result: Undefinable<TestTable>, $request: PipelineStepData) => {
+						toResponse: async ($result: Undefinable<TestTable>, $request: PipelineStepData) => {
 							expect($result).toBeUndefined();
 							return $request.content;
 						}
@@ -116,10 +116,10 @@ class TransactionalTestStepSets extends TypeOrmTransactionalPipelineStepSets {
 				{
 					create: async ({config, logger}) => new TypeOrmSaveBySQLPipelineStep({
 						config, logger, dataSourceName: 'TEST', sql: 'UPDATE T_TEST_TABLE SET CONTENT = ? WHERE ID = ?',
-						fromRequest: ($factor: TransactionalRequest, _$request: PipelineStepData<TransactionalRequest>) => {
+						fromRequest: async ($factor: TransactionalRequest, _$request: PipelineStepData<TransactionalRequest>) => {
 							return {values: [$factor.item3ChangeTo.content, $factor.item3ChangeTo.id]};
 						},
-						toResponse: ($result: TypeOrmCountOfAffected, _$request: PipelineStepData) => {
+						toResponse: async ($result: TypeOrmCountOfAffected, _$request: PipelineStepData) => {
 							// DON'T KNOW WHY THIS IS 3, SEEMS SHOULD BE 1 ACCORDING TO BETTER-SQLITE3 DOCUMENT
 							// BUT CURRENTLY IT RETURNS COUNT OF THIS TABLE, NOT IMPACTED ROW COUNT
 							expect($result).toBe(3);
@@ -134,14 +134,14 @@ class TransactionalTestStepSets extends TypeOrmTransactionalPipelineStepSets {
 						logger,
 						dataSourceName: 'TEST',
 						sql: 'INSERT INTO T_TEST_TABLE(ID, CONTENT) VALUES (?, ?)',
-						fromRequest: ($factor: TransactionalRequest, _$request: PipelineStepData<TransactionalRequest>) => {
+						fromRequest: async ($factor: TransactionalRequest, _$request: PipelineStepData<TransactionalRequest>) => {
 							return {
 								items: [
 									[$factor.item4.id, $factor.item4.content], [$factor.item5.id, $factor.item5.content]
 								]
 							};
 						},
-						toResponse: ($result: TypeOrmIdsOfInserted, _$request: PipelineStepData) => {
+						toResponse: async ($result: TypeOrmIdsOfInserted, _$request: PipelineStepData) => {
 							return {insertedIds: $result};
 						},
 						mergeRequest: true
@@ -150,7 +150,7 @@ class TransactionalTestStepSets extends TypeOrmTransactionalPipelineStepSets {
 				{
 					create: async ({config, logger}) => new TypeOrmLoadManyBySQLPipelineStep({
 						config, logger, dataSourceName: 'TEST', sql: 'SELECT ID id, CONTENT content FROM T_TEST_TABLE',
-						toResponse: ($result: Array<TestTable>, _$request: PipelineStepData) => {
+						toResponse: async ($result: Array<TestTable>, _$request: PipelineStepData) => {
 							return {all: $result};
 						},
 						mergeRequest: true

package/test/step/typeorm-entity.test.ts

@@ -1,50 +0,0 @@
-import {createConfig, createLogger} from '@rainbow-o23/n1';
-import {Column, Entity, PrimaryColumn} from 'typeorm';
-import {TypeOrmDataSourceHelper, TypeOrmDataSourceManager, TypeOrmLoadEntityByIdPipelineStep} from '../../src';
-
-const logger = createLogger();
-const config = createConfig(logger);
-
-// decorators are leading syntax errors in ide, since the test folder is not included in tsconfig.json
-// but the tricky thing is, once test folder included into tsconfig.json, d.ts files will be created in src folder
-// which cause currently use ts-ignore to avoid this syntax errors
-// @ts-ignore
-@Entity({name: 'T_TEST_TABLE'})
-export class TestTable {
-	// @ts-ignore
-	@PrimaryColumn('bigint', {name: 'ID'})
-	id: number;
-	// @ts-ignore
-	@Column('varchar', {name: 'CONTENT'})
-	content: string;
-}
-
-describe('TypeORM Entity Suite', () => {
-	beforeAll(async () => {
-		process.env.CFG_TYPEORM_TEST_TYPE = 'better-sqlite3';
-		process.env.CFG_TYPEORM_TEST_SYNCHRONIZE = 'true';
-		// in memory, kept in global cache, to make sure always use same one.
-		process.env.CFG_TYPEORM_TEST_KEPT_ON_GLOBAL = 'true';
-		await new TypeOrmDataSourceHelper(config).create({
-			'TEST': [TestTable]
-		});
-		const repo = (await TypeOrmDataSourceManager.findDataSource('TEST', config)).getDataSource().getRepository(TestTable);
-		await repo.insert({id: 1, content: 'hello world!'});
-	});
-
-	test('TypeORM load entity by id Pipeline Step Test #1', async () => {
-		const step = new TypeOrmLoadEntityByIdPipelineStep({
-			config, logger, dataSourceName: 'TEST', entityName: 'TestTable',
-			autonomous: true
-		});
-		const request = {content: 1};
-		const response = await step.perform(request);
-		expect(response.content).not.toBeNull();
-		expect(response.content.id).toBe(1);
-		expect(response.content.content).toBe('hello world!');
-	});
-
-	afterAll((done) => {
-		done();
-	});
-});