@rainbow-o23/n3 0.1.21 → 1.0.23

package/README.md

@@ -171,6 +171,29 @@ const RequestContent = {$content: content, $item: element, $semaphore};
 If you need to terminate the loop prematurely, you just need to return the `$semaphore` signal from the request data. The loop will
 automatically end, and the collected processing results will be gathered and returned as an array.
 
+### Parallel
+
+#### Constructor Parameters
+
+| Name      | Type                                                              | Default Value | Comments                                      |
+|-----------|-------------------------------------------------------------------|---------------|-----------------------------------------------|
+| cloneData | ScriptFuncOrBody\<CloneDataFunc\<In, InFragment, EachInFragment>> |               | Clone request data for each step.             |
+| race      | boolean                                                           | false         | Returns first settled result if race is true. |
+
+The specified set of pipeline steps will be executed parallel, and
+
+- All the execution results will be gathered into an array and returned when `race` is false,
+- Returns the first settled result when `race` is true,
+
+It is important to note that
+
+- Each sub pipeline step will use the same request data, they share the same content memory address. So please be very
+  careful <span style='color: red;'>**NOT**</span> to attempt to modify the request data in the sub pipeline steps. Alternatively, you can
+  use `cloneData` to create a copy of the request data for each sub pipeline steps, so that request data operations can be modified in
+  certain sub pipeline steps without affecting other sub pipeline steps.
+- The error handles of the `For Each` pipeline step will be used for each iteration of executing individual elements, rather than being
+  applied to the execution of the array as a whole.
+
 ### Trigger Pipeline
 
 #### Constructor Parameters

package/index.cjs

@@ -701,7 +701,7 @@ class PipelineStepSets extends AbstractFragmentaryPipelineStep {
                 const request = await promise;
                 return await this.measurePerformance(traceId, 'STEP', step.constructor.name)
                     .execute(async () => {
-                    this.traceStepOut(traceId, step, request);
+                    this.traceStepIn(traceId, step, request);
                     const response = await step.perform({ ...request, $context: { ...context, traceId } });
                     this.traceStepOut(traceId, step, response);
                     return this.returnOrContinueOrClear(request, response);
@@ -719,6 +719,67 @@ class AsyncPipelineStepSets extends PipelineStepSets {
     }
 }
 
+class ParallelPipelineStepSets extends PipelineStepSets {
+    _cloneDataSnippet;
+    _cloneData;
+    _race;
+    constructor(options) {
+        super(options);
+        this._cloneDataSnippet = options.cloneData;
+        this._cloneData = Utils.createSyncFunction(this.getCloneDataSnippet(), {
+            createDefault: () => ($factor, _$request, _$helpers, _$) => {
+                return $factor;
+            },
+            getVariableNames: () => ['$factor', '$request', ...this.getHelpersVariableNames()],
+            error: (e) => {
+                this.getLogger().error(`Failed on create function for clone data, snippet is [${this.getCloneDataSnippet()}].`);
+                throw e;
+            }
+        });
+        this._race = options.race ?? false;
+    }
+    getCloneDataSnippet() {
+        return this._cloneDataSnippet;
+    }
+    raceOne() {
+        return this._race;
+    }
+    cloneDataForEach($factor, $request) {
+        if ($factor == null) {
+            return null;
+        }
+        const $helpers = this.getHelpers();
+        return this._cloneData($factor, $request, $helpers, $helpers);
+    }
+    async doPerform(data, request) {
+        return await this.performWithContext(request, async (request, context) => {
+            const { $context: { traceId } = {} } = request;
+            const steps = await this.createSteps();
+            const execute = () => {
+                return steps.map(async (step) => {
+                    return await this.measurePerformance(traceId, 'STEP', step.constructor.name)
+                        .execute(async () => {
+                        const eachData = this.cloneDataForEach(data, request);
+                        const eachRequest = { content: eachData, $context: { ...context, traceId } };
+                        this.traceStepIn(traceId, step, request);
+                        const response = await step.perform(eachRequest);
+                        this.traceStepOut(traceId, step, response);
+                        return response;
+                    });
+                });
+            };
+            if (this.raceOne()) {
+                const response = await Promise.race(execute());
+                return response.content;
+            }
+            else {
+                const responses = await Promise.all(execute());
+                return responses.map(response => response.content);
+            }
+        });
+    }
+}
+
 class EachPipelineStepSets extends PipelineStepSets {
     _originalContentPropertyName;
     _itemPropertyName;
@@ -2234,6 +2295,7 @@ exports.HttpUnknownErrorCode = HttpUnknownErrorCode;
 exports.MssqlTypeOrmDatasource = MssqlTypeOrmDatasource;
 exports.MysqlTypeOrmDatasource = MysqlTypeOrmDatasource;
 exports.OracleTypeOrmDatasource = OracleTypeOrmDatasource;
+exports.ParallelPipelineStepSets = ParallelPipelineStepSets;
 exports.PgsqlTypeOrmDatasource = PgsqlTypeOrmDatasource;
 exports.PipelineStepSets = PipelineStepSets;
 exports.RefPipelinePipelineStep = RefPipelinePipelineStep;

package/index.js

@@ -699,7 +699,7 @@ class PipelineStepSets extends AbstractFragmentaryPipelineStep {
                 const request = await promise;
                 return await this.measurePerformance(traceId, 'STEP', step.constructor.name)
                     .execute(async () => {
-                    this.traceStepOut(traceId, step, request);
+                    this.traceStepIn(traceId, step, request);
                     const response = await step.perform({ ...request, $context: { ...context, traceId } });
                     this.traceStepOut(traceId, step, response);
                     return this.returnOrContinueOrClear(request, response);
@@ -717,6 +717,67 @@ class AsyncPipelineStepSets extends PipelineStepSets {
     }
 }
 
+class ParallelPipelineStepSets extends PipelineStepSets {
+    _cloneDataSnippet;
+    _cloneData;
+    _race;
+    constructor(options) {
+        super(options);
+        this._cloneDataSnippet = options.cloneData;
+        this._cloneData = Utils.createSyncFunction(this.getCloneDataSnippet(), {
+            createDefault: () => ($factor, _$request, _$helpers, _$) => {
+                return $factor;
+            },
+            getVariableNames: () => ['$factor', '$request', ...this.getHelpersVariableNames()],
+            error: (e) => {
+                this.getLogger().error(`Failed on create function for clone data, snippet is [${this.getCloneDataSnippet()}].`);
+                throw e;
+            }
+        });
+        this._race = options.race ?? false;
+    }
+    getCloneDataSnippet() {
+        return this._cloneDataSnippet;
+    }
+    raceOne() {
+        return this._race;
+    }
+    cloneDataForEach($factor, $request) {
+        if ($factor == null) {
+            return null;
+        }
+        const $helpers = this.getHelpers();
+        return this._cloneData($factor, $request, $helpers, $helpers);
+    }
+    async doPerform(data, request) {
+        return await this.performWithContext(request, async (request, context) => {
+            const { $context: { traceId } = {} } = request;
+            const steps = await this.createSteps();
+            const execute = () => {
+                return steps.map(async (step) => {
+                    return await this.measurePerformance(traceId, 'STEP', step.constructor.name)
+                        .execute(async () => {
+                        const eachData = this.cloneDataForEach(data, request);
+                        const eachRequest = { content: eachData, $context: { ...context, traceId } };
+                        this.traceStepIn(traceId, step, request);
+                        const response = await step.perform(eachRequest);
+                        this.traceStepOut(traceId, step, response);
+                        return response;
+                    });
+                });
+            };
+            if (this.raceOne()) {
+                const response = await Promise.race(execute());
+                return response.content;
+            }
+            else {
+                const responses = await Promise.all(execute());
+                return responses.map(response => response.content);
+            }
+        });
+    }
+}
+
 class EachPipelineStepSets extends PipelineStepSets {
     _originalContentPropertyName;
     _itemPropertyName;
@@ -2198,4 +2259,4 @@ class TypeOrmTransactionalPipelineStepSets extends PipelineStepSets {
     }
 }
 
-export { AbstractFragmentaryPipelineStep, AbstractTypeOrmBySQLPipelineStep, AbstractTypeOrmDataSource, AbstractTypeOrmLoadBySQLPipelineStep, AbstractTypeOrmPipelineStep, AsyncPipelineStepSets, BetterSqlite3TypeOrmDatasource, ConditionalPipelineStepSets, DEFAULT_TRANSACTION_NAME, DeletePropertyPipelineStep, ERR_EACH_FRAGMENT_NOT_ANY_ARRAY, ERR_FETCH_ERROR, ERR_PIPELINE_REF_NOT_EMPTY, ERR_PIPELINE_REF_NOT_FOUND, ERR_PIPELINE_STEP_CONDITIONAL_SNIPPET_NOT_EMPTY, ERR_PIPELINE_STEP_METHOD_NOT_SUPPORTED, ERR_PIPELINE_STEP_REF_NOT_EMPTY, ERR_PIPELINE_STEP_REF_NOT_FOUND, ERR_PIPELINE_STEP_SNIPPET_NOT_EMPTY, ERR_TYPEORM_DATASOURCE_CREATOR_NOT_FOUND, ERR_TYPEORM_DATASOURCE_NOT_FOUND, ERR_TYPEORM_DATASOURCE_TYPE_NOT_FOUND, ERR_TYPEORM_ENTITY_NOT_FOUND, ERR_TYPEORM_SQL_NOT_EMPTY, ERR_TYPEORM_STEP_SNIPPET_NOT_EMPTY, ERR_TYPEORM_TRANSACTION_NOT_FOUND, EachPipelineStepSets, FetchPipelineStep, GetPropertyPipelineStep, HttpAbortErrorCode, HttpUnknownErrorCode, MssqlTypeOrmDatasource, MysqlTypeOrmDatasource, OracleTypeOrmDatasource, ParsedSqlSegmentType, PgsqlTypeOrmDatasource, PipelineStepSets, RefPipelinePipelineStep, RefStepPipelineStep, RoutesPipelineStepSets, SnippetPipelineStep, SnowflakePipelineStep, SupportedDataSourceTypes, TypeOrmBulkSaveBySQLPipelineStep, TypeOrmBySnippetPipelineStep, TypeOrmDataSourceHelper, TypeOrmDataSourceManager, TypeOrmLoadEntityByIdPipelineStep, TypeOrmLoadManyBySQLPipelineStep, TypeOrmLoadOneBySQLPipelineStep, TypeOrmParsedSQLCache, TypeOrmSaveBySQLPipelineStep, TypeOrmSaveEntityPipelineStep, TypeOrmTransactionalPipelineStepSets, Utils };
+export { AbstractFragmentaryPipelineStep, AbstractTypeOrmBySQLPipelineStep, AbstractTypeOrmDataSource, AbstractTypeOrmLoadBySQLPipelineStep, AbstractTypeOrmPipelineStep, AsyncPipelineStepSets, BetterSqlite3TypeOrmDatasource, ConditionalPipelineStepSets, DEFAULT_TRANSACTION_NAME, DeletePropertyPipelineStep, ERR_EACH_FRAGMENT_NOT_ANY_ARRAY, ERR_FETCH_ERROR, ERR_PIPELINE_REF_NOT_EMPTY, ERR_PIPELINE_REF_NOT_FOUND, ERR_PIPELINE_STEP_CONDITIONAL_SNIPPET_NOT_EMPTY, ERR_PIPELINE_STEP_METHOD_NOT_SUPPORTED, ERR_PIPELINE_STEP_REF_NOT_EMPTY, ERR_PIPELINE_STEP_REF_NOT_FOUND, ERR_PIPELINE_STEP_SNIPPET_NOT_EMPTY, ERR_TYPEORM_DATASOURCE_CREATOR_NOT_FOUND, ERR_TYPEORM_DATASOURCE_NOT_FOUND, ERR_TYPEORM_DATASOURCE_TYPE_NOT_FOUND, ERR_TYPEORM_ENTITY_NOT_FOUND, ERR_TYPEORM_SQL_NOT_EMPTY, ERR_TYPEORM_STEP_SNIPPET_NOT_EMPTY, ERR_TYPEORM_TRANSACTION_NOT_FOUND, EachPipelineStepSets, FetchPipelineStep, GetPropertyPipelineStep, HttpAbortErrorCode, HttpUnknownErrorCode, MssqlTypeOrmDatasource, MysqlTypeOrmDatasource, OracleTypeOrmDatasource, ParallelPipelineStepSets, ParsedSqlSegmentType, PgsqlTypeOrmDatasource, PipelineStepSets, RefPipelinePipelineStep, RefStepPipelineStep, RoutesPipelineStepSets, SnippetPipelineStep, SnowflakePipelineStep, SupportedDataSourceTypes, TypeOrmBulkSaveBySQLPipelineStep, TypeOrmBySnippetPipelineStep, TypeOrmDataSourceHelper, TypeOrmDataSourceManager, TypeOrmLoadEntityByIdPipelineStep, TypeOrmLoadManyBySQLPipelineStep, TypeOrmLoadOneBySQLPipelineStep, TypeOrmParsedSQLCache, TypeOrmSaveBySQLPipelineStep, TypeOrmSaveEntityPipelineStep, TypeOrmTransactionalPipelineStepSets, Utils };

package/lib/step/index.d.ts

@@ -3,6 +3,7 @@ export * from './utils';
 export * from './abstract-fragmentary-pipeline-step';
 export * from './step-sets';
 export * from './async-step-sets';
+export * from './parallel-step-sets';
 export * from './each-step-sets';
 export * from './conditional-step-sets';
 export * from './routes-step-sets';

package/lib/step/parallel-step-sets.d.ts

@@ -0,0 +1,18 @@
+import { PipelineStepData, PipelineStepHelpers, PipelineStepPayload } from '@rainbow-o23/n1';
+import { PipelineStepSets, PipelineStepSetsOptions } from './step-sets';
+import { ScriptFuncOrBody } from './types';
+export type CloneDataFunc<In, InFragment, EachInFragment = InFragment> = ($factor: InFragment, $request: PipelineStepData<In>, $helpers: PipelineStepHelpers, $: PipelineStepHelpers) => EachInFragment;
+export interface ParallelPipelineStepSetsOptions<In = PipelineStepPayload, Out = PipelineStepPayload, InFragment = In, OutFragment = Out, EachInFragment = InFragment> extends PipelineStepSetsOptions<In, Out, InFragment, OutFragment> {
+    cloneData?: ScriptFuncOrBody<CloneDataFunc<In, InFragment, EachInFragment>>;
+    race?: boolean;
+}
+export declare class ParallelPipelineStepSets<In = PipelineStepPayload, Out = PipelineStepPayload, InFragment = In, OutFragment = Out, EachInFragment = InFragment> extends PipelineStepSets<In, Out, InFragment, OutFragment> {
+    private readonly _cloneDataSnippet;
+    private readonly _cloneData;
+    private readonly _race;
+    constructor(options: ParallelPipelineStepSetsOptions<In, Out, InFragment, OutFragment, EachInFragment>);
+    getCloneDataSnippet(): ScriptFuncOrBody<CloneDataFunc<In, InFragment, EachInFragment>>;
+    raceOne(): boolean;
+    protected cloneDataForEach($factor: InFragment, $request: PipelineStepData<In>): EachInFragment;
+    protected doPerform(data: InFragment, request: PipelineStepData<In>): Promise<OutFragment>;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@rainbow-o23/n3",
-	"version": "0.1.21",
+	"version": "1.0.23",
 	"description": "o23 pipelines",
 	"main": "index.cjs",
 	"module": "index.js",
@@ -21,7 +21,7 @@
 		"url": "https://github.com/InsureMO/rainbow-o23/issues"
 	},
 	"dependencies": {
-		"@rainbow-o23/n1": "0.1.21",
+		"@rainbow-o23/n1": "1.0.23",
 		"@theinternetfolks/snowflake": "^1.3.0",
 		"node-fetch": "2.6.7",
 		"typeorm": "^0.3.17"

package/src/lib/step/index.ts

@@ -5,6 +5,7 @@ export * from './abstract-fragmentary-pipeline-step';
 
 export * from './step-sets';
 export * from './async-step-sets';
+export * from './parallel-step-sets';
 export * from './each-step-sets';
 export * from './conditional-step-sets';
 export * from './routes-step-sets';

package/src/lib/step/parallel-step-sets.ts

@@ -0,0 +1,92 @@
+import {PipelineStepData, PipelineStepHelpers, PipelineStepPayload} from '@rainbow-o23/n1';
+import {PipelineStepSets, PipelineStepSetsContext, PipelineStepSetsOptions} from './step-sets';
+import {ScriptFuncOrBody} from './types';
+import {Utils} from './utils';
+
+export type CloneDataFunc<In, InFragment, EachInFragment = InFragment> = ($factor: InFragment, $request: PipelineStepData<In>, $helpers: PipelineStepHelpers, $: PipelineStepHelpers) => EachInFragment;
+
+export interface ParallelPipelineStepSetsOptions<In = PipelineStepPayload, Out = PipelineStepPayload, InFragment = In, OutFragment = Out, EachInFragment = InFragment>
+	extends PipelineStepSetsOptions<In, Out, InFragment, OutFragment> {
+	cloneData?: ScriptFuncOrBody<CloneDataFunc<In, InFragment, EachInFragment>>;
+	race?: boolean;
+}
+
+/**
+ * pipeline steps to execute sets of steps parallel.
+ */
+export class ParallelPipelineStepSets<In = PipelineStepPayload, Out = PipelineStepPayload, InFragment = In, OutFragment = Out, EachInFragment = InFragment>
+	extends PipelineStepSets<In, Out, InFragment, OutFragment> {
+	private readonly _cloneDataSnippet: ScriptFuncOrBody<CloneDataFunc<In, InFragment, EachInFragment>>;
+	private readonly _cloneData: CloneDataFunc<In, InFragment, EachInFragment>;
+	private readonly _race: boolean;
+
+	public constructor(options: ParallelPipelineStepSetsOptions<In, Out, InFragment, OutFragment, EachInFragment>) {
+		super(options);
+		this._cloneDataSnippet = options.cloneData;
+		this._cloneData = Utils.createSyncFunction(this.getCloneDataSnippet(), {
+			// eslint-disable-next-line @typescript-eslint/no-unused-vars
+			createDefault: () => ($factor: InFragment, _$request: PipelineStepData<In>, _$helpers: PipelineStepHelpers, _$: PipelineStepHelpers): EachInFragment => {
+				return $factor as unknown as EachInFragment;
+			},
+			getVariableNames: () => ['$factor', '$request', ...this.getHelpersVariableNames()], //this.generateFromRequestVariableNames(),
+			error: (e: Error) => {
+				this.getLogger().error(`Failed on create function for clone data, snippet is [${this.getCloneDataSnippet()}].`);
+				throw e;
+			}
+		});
+		this._race = options.race ?? false;
+	}
+
+	public getCloneDataSnippet(): ScriptFuncOrBody<CloneDataFunc<In, InFragment, EachInFragment>> {
+		return this._cloneDataSnippet;
+	}
+
+	/**
+	 * returns true when only one needs to be retrieved
+	 */
+	public raceOne(): boolean {
+		return this._race;
+	}
+
+	/**
+	 * default get request content
+	 */
+	protected cloneDataForEach($factor: InFragment, $request: PipelineStepData<In>): EachInFragment {
+		if ($factor == null) {
+			return null;
+		}
+		const $helpers = this.getHelpers();
+		return this._cloneData($factor, $request, $helpers, $helpers);
+	}
+
+	protected async doPerform(data: InFragment, request: PipelineStepData<In>): Promise<OutFragment> {
+		return await this.performWithContext(
+			request, async (request: PipelineStepData<In>, context: PipelineStepSetsContext): Promise<OutFragment> => {
+				const {$context: {traceId} = {}} = request;
+				const steps = await this.createSteps();
+				const execute = () => {
+					return steps.map(async step => {
+						return await this.measurePerformance(traceId, 'STEP', step.constructor.name)
+							.execute(async () => {
+								const eachData = this.cloneDataForEach(data, request);
+								const eachRequest = {content: eachData, $context: {...context, traceId}};
+								this.traceStepIn(traceId, step, request);
+								const response = await step.perform(eachRequest);
+								this.traceStepOut(traceId, step, response);
+								// return
+								return response;
+							});
+					});
+				};
+				if (this.raceOne()) {
+					// race
+					const response = await Promise.race(execute());
+					return response.content as OutFragment;
+				} else {
+					// all
+					const responses = await Promise.all(execute());
+					return responses.map(response => response.content) as OutFragment;
+				}
+			});
+	}
+}

package/src/lib/step/step-sets.ts

@@ -85,7 +85,7 @@ export class PipelineStepSets<In = PipelineStepPayload, Out = PipelineStepPayloa
 					const request = await promise;
 					return await this.measurePerformance(traceId, 'STEP', step.constructor.name)
 						.execute(async () => {
-							this.traceStepOut(traceId, step, request);
+							this.traceStepIn(traceId, step, request);
 							const response = await step.perform({...request, $context: {...context, traceId}});
 							this.traceStepOut(traceId, step, response);
 							// if no response returned, keep using request for next

package/test/step/parallel-step.test.ts

@@ -0,0 +1,25 @@
+import {createConfig, createLogger, PipelineStep, PipelineStepBuilder, PipelineStepOptions} from '@rainbow-o23/n1';
+import {ParallelPipelineStepSets, SnippetPipelineStep} from '../../src';
+
+const logger = createLogger();
+const config = createConfig(logger);
+
+test('Parallel Pipeline Step Test #1, + 100', async () => {
+	const step = new ParallelPipelineStepSets({
+		config, logger, steps: [
+			new class implements PipelineStepBuilder {
+				async create(options?: PipelineStepOptions): Promise<PipelineStep> {
+					return new SnippetPipelineStep({config, logger, snippet: 'return $factor.base * 100;'});
+				}
+			},
+			new class implements PipelineStepBuilder {
+				async create(options?: PipelineStepOptions): Promise<PipelineStep> {
+					return new SnippetPipelineStep({config, logger, snippet: 'return $factor.base * 200;'});
+				}
+			}
+		]
+	});
+	const request = {content: {base: 1}};
+	const response = await step.perform(request);
+	expect(response.content).toEqual([100, 200]);
+});