@rainbow-o23/n3 1.0.37 → 1.0.38

package/index.cjs

@@ -2,7 +2,6 @@
 
 var typeorm = require('typeorm');
 var n1 = require('@rainbow-o23/n1');
-var snowflake = require('@theinternetfolks/snowflake');
 var fetch = require('node-fetch');
 
 const ERR_PIPELINE_STEP_SNIPPET_NOT_EMPTY = 'O03-00001';
@@ -703,7 +702,12 @@ class AbstractFragmentaryPipelineStep extends n1.AbstractPipelineStep {
         }
         catch (e) {
             const result = await this.handleError(fragment, request, e);
-            return this.setToOutput(result, request);
+            try {
+                return this.setToOutput(result, request);
+            }
+            catch (e) {
+                return (await this.handleError(fragment, request, e));
+            }
         }
     }
     async perform(request) {
@@ -1061,9 +1065,54 @@ class SnippetPipelineStep extends AbstractFragmentaryPipelineStep {
     }
 }
 
+class Snowflake {
+    static EPOCH = Date.UTC(1970, 0, 1).valueOf();
+    static SHARD_ID = 1;
+    static MAX_SEQUENCE = 4095;
+    static sequence = 0;
+    static lastTimestamp = -1;
+    static generate(options = {}) {
+        let { timestamp = Date.now(), shardId } = options;
+        if (timestamp instanceof Date) {
+            timestamp = timestamp.getTime();
+        }
+        else {
+            timestamp = new Date(timestamp).getTime();
+        }
+        shardId = shardId || Snowflake.SHARD_ID;
+        if (timestamp === Snowflake.lastTimestamp) {
+            Snowflake.sequence = (Snowflake.sequence + 1) % Snowflake.MAX_SEQUENCE;
+            if (Snowflake.sequence === 0) {
+                timestamp = Snowflake.waitNextMillis(timestamp);
+            }
+        }
+        else {
+            Snowflake.sequence = 0;
+        }
+        Snowflake.lastTimestamp = timestamp;
+        const result = (BigInt(timestamp - Snowflake.EPOCH) << BigInt(22)) |
+            (BigInt(shardId % 1024) << BigInt(12)) |
+            BigInt(Snowflake.sequence);
+        return result.toString();
+    }
+    static waitNextMillis(currentTimestamp) {
+        let timestamp = Date.now();
+        while (timestamp <= currentTimestamp) {
+            timestamp = Date.now();
+        }
+        return timestamp;
+    }
+}
+
 class SnowflakePipelineStep extends AbstractFragmentaryPipelineStep {
+    _shardId;
+    constructor(options) {
+        super(options);
+        const config = this.getConfig();
+        this._shardId = config.getNumber(`snowflake.shard.id`);
+    }
     async doPerform(_data, _request) {
-        return snowflake.Snowflake.generate();
+        return Snowflake.generate({ shardId: this._shardId });
     }
 }
 
@@ -1602,12 +1651,15 @@ class TypeOrmLoadEntityByIdPipelineStep extends AbstractTypeOrmPipelineStep {
 class TypeOrmSaveEntityPipelineStep extends AbstractTypeOrmPipelineStep {
     _entityName;
     _fillIdBySnowflake;
+    _snowflakeShardId;
     _uniquenessCheckSnippet;
     _uniquenessCheckFunc;
     constructor(options) {
         super(options);
+        const config = this.getConfig();
         this._entityName = options.entityName;
         this._fillIdBySnowflake = options.fillIdBySnowflake ?? false;
+        this._snowflakeShardId = config.getNumber(`snowflake.shard.id`);
         this._uniquenessCheckSnippet = options.uniquenessCheckSnippet;
         this._uniquenessCheckFunc = Utils.createSyncFunction(this.getUniquenessCheckSnippet(), {
             createDefault: () => (void 0),
@@ -1650,7 +1702,7 @@ class TypeOrmSaveEntityPipelineStep extends AbstractTypeOrmPipelineStep {
         if (this.isFillIdBySnowflake()) {
             const id = entity[column.propertyName];
             if (id == null || `${id ?? ''}`.trim().length === 0) {
-                entity[column.propertyName] = snowflake.Snowflake.generate();
+                entity[column.propertyName] = Snowflake.generate({ shardId: this._snowflakeShardId });
             }
         }
         return await repository.save(entity);

package/index.js

@@ -1,6 +1,5 @@
 import { DataSource } from 'typeorm';
 import { UncatchableError, AbstractPipelineStep, CatchableError, ExposedUncatchableError, ERR_UNKNOWN, PipelineRepository, StepHelpersUtils } from '@rainbow-o23/n1';
-import { Snowflake } from '@theinternetfolks/snowflake';
 import fetch from 'node-fetch';
 
 const ERR_PIPELINE_STEP_SNIPPET_NOT_EMPTY = 'O03-00001';
@@ -701,7 +700,12 @@ class AbstractFragmentaryPipelineStep extends AbstractPipelineStep {
         }
         catch (e) {
             const result = await this.handleError(fragment, request, e);
-            return this.setToOutput(result, request);
+            try {
+                return this.setToOutput(result, request);
+            }
+            catch (e) {
+                return (await this.handleError(fragment, request, e));
+            }
         }
     }
     async perform(request) {
@@ -1059,9 +1063,54 @@ class SnippetPipelineStep extends AbstractFragmentaryPipelineStep {
     }
 }
 
+class Snowflake {
+    static EPOCH = Date.UTC(1970, 0, 1).valueOf();
+    static SHARD_ID = 1;
+    static MAX_SEQUENCE = 4095;
+    static sequence = 0;
+    static lastTimestamp = -1;
+    static generate(options = {}) {
+        let { timestamp = Date.now(), shardId } = options;
+        if (timestamp instanceof Date) {
+            timestamp = timestamp.getTime();
+        }
+        else {
+            timestamp = new Date(timestamp).getTime();
+        }
+        shardId = shardId || Snowflake.SHARD_ID;
+        if (timestamp === Snowflake.lastTimestamp) {
+            Snowflake.sequence = (Snowflake.sequence + 1) % Snowflake.MAX_SEQUENCE;
+            if (Snowflake.sequence === 0) {
+                timestamp = Snowflake.waitNextMillis(timestamp);
+            }
+        }
+        else {
+            Snowflake.sequence = 0;
+        }
+        Snowflake.lastTimestamp = timestamp;
+        const result = (BigInt(timestamp - Snowflake.EPOCH) << BigInt(22)) |
+            (BigInt(shardId % 1024) << BigInt(12)) |
+            BigInt(Snowflake.sequence);
+        return result.toString();
+    }
+    static waitNextMillis(currentTimestamp) {
+        let timestamp = Date.now();
+        while (timestamp <= currentTimestamp) {
+            timestamp = Date.now();
+        }
+        return timestamp;
+    }
+}
+
 class SnowflakePipelineStep extends AbstractFragmentaryPipelineStep {
+    _shardId;
+    constructor(options) {
+        super(options);
+        const config = this.getConfig();
+        this._shardId = config.getNumber(`snowflake.shard.id`);
+    }
     async doPerform(_data, _request) {
-        return Snowflake.generate();
+        return Snowflake.generate({ shardId: this._shardId });
     }
 }
 
@@ -1600,12 +1649,15 @@ class TypeOrmLoadEntityByIdPipelineStep extends AbstractTypeOrmPipelineStep {
 class TypeOrmSaveEntityPipelineStep extends AbstractTypeOrmPipelineStep {
     _entityName;
     _fillIdBySnowflake;
+    _snowflakeShardId;
     _uniquenessCheckSnippet;
     _uniquenessCheckFunc;
     constructor(options) {
         super(options);
+        const config = this.getConfig();
         this._entityName = options.entityName;
         this._fillIdBySnowflake = options.fillIdBySnowflake ?? false;
+        this._snowflakeShardId = config.getNumber(`snowflake.shard.id`);
         this._uniquenessCheckSnippet = options.uniquenessCheckSnippet;
         this._uniquenessCheckFunc = Utils.createSyncFunction(this.getUniquenessCheckSnippet(), {
             createDefault: () => (void 0),
@@ -1648,7 +1700,7 @@ class TypeOrmSaveEntityPipelineStep extends AbstractTypeOrmPipelineStep {
         if (this.isFillIdBySnowflake()) {
             const id = entity[column.propertyName];
             if (id == null || `${id ?? ''}`.trim().length === 0) {
-                entity[column.propertyName] = Snowflake.generate();
+                entity[column.propertyName] = Snowflake.generate({ shardId: this._snowflakeShardId });
             }
         }
         return await repository.save(entity);

package/lib/step/snowflake-step.d.ts

@@ -1,6 +1,8 @@
 import { PipelineStepData, PipelineStepPayload } from '@rainbow-o23/n1';
-import { AbstractFragmentaryPipelineStep } from './abstract-fragmentary-pipeline-step';
-export type SnowflakeId = string;
+import { SnowflakeId } from '../utils';
+import { AbstractFragmentaryPipelineStep, FragmentaryPipelineStepOptions } from './abstract-fragmentary-pipeline-step';
 export declare class SnowflakePipelineStep<In = PipelineStepPayload, Out = PipelineStepPayload, InFragment = In> extends AbstractFragmentaryPipelineStep<In, Out, InFragment, SnowflakeId> {
+    private readonly _shardId;
+    constructor(options: FragmentaryPipelineStepOptions<In, Out, InFragment, SnowflakeId>);
     protected doPerform(_data: InFragment, _request: PipelineStepData<In>): Promise<SnowflakeId>;
 }

package/lib/typeorm-step/typeorm-save-entity-step.d.ts

@@ -20,6 +20,7 @@ export interface TypeOrmSaveEntityPipelineStepOptions<In = PipelineStepPayload,
 export declare class TypeOrmSaveEntityPipelineStep<In = PipelineStepPayload, Out = PipelineStepPayload, InFragment = EntityToSave, OutFragment = EntityToSave> extends AbstractTypeOrmPipelineStep<In, Out, InFragment, OutFragment> {
     private readonly _entityName;
     private readonly _fillIdBySnowflake;
+    private readonly _snowflakeShardId;
     private readonly _uniquenessCheckSnippet?;
     private readonly _uniquenessCheckFunc?;
     constructor(options: TypeOrmSaveEntityPipelineStepOptions<In, Out, InFragment, OutFragment>);

package/lib/utils/index.d.ts

@@ -0,0 +1 @@
+export * from './snowflake';

package/lib/utils/snowflake.d.ts

@@ -0,0 +1,14 @@
+export type SnowflakeId = string;
+export type SnowflakeGenerateOptions = {
+    timestamp?: Date | number;
+    shardId?: number;
+};
+export declare class Snowflake {
+    private static EPOCH;
+    private static SHARD_ID;
+    private static readonly MAX_SEQUENCE;
+    private static sequence;
+    private static lastTimestamp;
+    static generate(options?: SnowflakeGenerateOptions): SnowflakeId;
+    private static waitNextMillis;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@rainbow-o23/n3",
-	"version": "1.0.37",
+	"version": "1.0.38",
 	"description": "o23 pipelines",
 	"main": "index.cjs",
 	"module": "index.js",
@@ -21,8 +21,7 @@
 		"url": "https://github.com/InsureMO/rainbow-o23/issues"
 	},
 	"dependencies": {
-		"@rainbow-o23/n1": "1.0.37",
-		"@theinternetfolks/snowflake": "^1.3.0",
+		"@rainbow-o23/n1": "1.0.38",
 		"node-fetch": "2.6.7",
 		"typeorm": "^0.3.17"
 	},
@@ -41,7 +40,7 @@
 		"better-sqlite3": "^9.0.0",
 		"eslint": "^8.29.0",
 		"mssql": "^10.0.1",
-		"mysql2": "^3.9.7",
+		"mysql2": "^3.9.8",
 		"oracledb": "^6.2.0",
 		"pg": "^8.11.3",
 		"pg-query-stream": "^4.5.3",

package/rollup.config.base.js

@@ -25,8 +25,7 @@ export const buildConfig = (lint) => {
 		external(id) {
 			return ["@rainbow-o23/", "typeorm/"].some(scope => id.startsWith(scope))
 				|| [
-					"typeorm", "reflect-metadata",
-					"@theinternetfolks/snowflake", "node-fetch"
+					"typeorm", "reflect-metadata", "node-fetch"
 				].includes(id);
 		}
 	};

package/src/lib/step/abstract-fragmentary-pipeline-step.ts

@@ -365,7 +365,14 @@ export abstract class AbstractFragmentaryPipelineStep<In = PipelineStepPayload,
 			// eslint-disable-next-line @typescript-eslint/no-explicit-any
 		} catch (e: any) {
 			const result = await this.handleError(fragment, request, e);
-			return this.setToOutput(result, request);
+			try {
+				return this.setToOutput(result, request);
+				// eslint-disable-next-line @typescript-eslint/no-explicit-any
+			} catch (e: any) {
+				// since exception raised during set to output,
+				// therefore never call this function on error handling
+				return (await this.handleError(fragment, request, e)) as PipelineStepData<Out>;
+			}
 		}
 	}
 

package/src/lib/step/snowflake-step.ts

@@ -1,16 +1,22 @@
 import {PipelineStepData, PipelineStepPayload} from '@rainbow-o23/n1';
-import {Snowflake} from '@theinternetfolks/snowflake';
-import {AbstractFragmentaryPipelineStep} from './abstract-fragmentary-pipeline-step';
-
-export type SnowflakeId = string;
+import {Snowflake, SnowflakeId} from '../utils';
+import {AbstractFragmentaryPipelineStep, FragmentaryPipelineStepOptions} from './abstract-fragmentary-pipeline-step';
 
 /**
  * create a snowflake id as out fragment
  */
 export class SnowflakePipelineStep<In = PipelineStepPayload, Out = PipelineStepPayload, InFragment = In>
 	extends AbstractFragmentaryPipelineStep<In, Out, InFragment, SnowflakeId> {
+	private readonly _shardId: number;
+
+	public constructor(options: FragmentaryPipelineStepOptions<In, Out, InFragment, SnowflakeId>) {
+		super(options);
+		const config = this.getConfig();
+		this._shardId = config.getNumber(`snowflake.shard.id`);
+	}
+
 	// eslint-disable-next-line @typescript-eslint/no-unused-vars
 	protected async doPerform(_data: InFragment, _request: PipelineStepData<In>): Promise<SnowflakeId> {
-		return Snowflake.generate();
+		return Snowflake.generate({shardId: this._shardId});
 	}
 }

package/src/lib/typeorm-step/typeorm-save-entity-step.ts

@@ -5,9 +5,9 @@ import {
 	PipelineStepPayload,
 	Undefinable
 } from '@rainbow-o23/n1';
-import {Snowflake} from '@theinternetfolks/snowflake';
 import {DeepPartial, ObjectLiteral} from 'typeorm';
 import {ScriptFuncOrBody, Utils} from '../step';
+import {Snowflake} from '../utils';
 import {AbstractTypeOrmPipelineStep, TypeOrmPipelineStepOptions} from './abstract-typeorm-step';
 import {TypeOrmEntityName} from './types';
 
@@ -33,13 +33,16 @@ export class TypeOrmSaveEntityPipelineStep<In = PipelineStepPayload, Out = Pipel
 	extends AbstractTypeOrmPipelineStep<In, Out, InFragment, OutFragment> {
 	private readonly _entityName: TypeOrmEntityName;
 	private readonly _fillIdBySnowflake: boolean;
+	private readonly _snowflakeShardId: number;
 	private readonly _uniquenessCheckSnippet?: ScriptFuncOrBody<UniquenessCheckFunc<InFragment>>;
 	private readonly _uniquenessCheckFunc?: UniquenessCheckFunc<InFragment>;
 
 	public constructor(options: TypeOrmSaveEntityPipelineStepOptions<In, Out, InFragment, OutFragment>) {
 		super(options);
+		const config = this.getConfig();
 		this._entityName = options.entityName;
 		this._fillIdBySnowflake = options.fillIdBySnowflake ?? false;
+		this._snowflakeShardId = config.getNumber(`snowflake.shard.id`);
 		this._uniquenessCheckSnippet = options.uniquenessCheckSnippet;
 		this._uniquenessCheckFunc = Utils.createSyncFunction(this.getUniquenessCheckSnippet(), {
 			createDefault: () => (void 0),
@@ -100,7 +103,7 @@ export class TypeOrmSaveEntityPipelineStep<In = PipelineStepPayload, Out = Pipel
 			// fill id if not exists
 			const id = entity[column.propertyName];
 			if (id == null || `${id ?? ''}`.trim().length === 0) {
-				entity[column.propertyName] = Snowflake.generate() as unknown as number;
+				entity[column.propertyName] = Snowflake.generate({shardId: this._snowflakeShardId}) as unknown as number;
 			}
 		}
 

package/src/lib/utils/index.ts

@@ -0,0 +1 @@
+export * from './snowflake';

package/src/lib/utils/snowflake.ts

@@ -0,0 +1,58 @@
+export type SnowflakeId = string;
+export type SnowflakeGenerateOptions = { timestamp?: Date | number, shardId?: number };
+
+export class Snowflake {
+	/** The generators epoch timestamp in milliseconds. Defaults to "1st of January 1970, 00:00". */
+	private static EPOCH: number = Date.UTC(1970, 0, 1).valueOf();
+
+	/** The id of the shard running this generator. Defaults to "1". */
+	private static SHARD_ID = 1;
+
+	/**
+	 * Max sequence, 2^12 - 1
+	 */
+	private static readonly MAX_SEQUENCE = 4095;
+	/** current sequence */
+	private static sequence = 0;
+	private static lastTimestamp = -1;
+
+	/**
+	 * Generates a single snowflake.
+	 */
+	static generate(options: SnowflakeGenerateOptions = {}): SnowflakeId {
+		let {timestamp = Date.now(), shardId} = options;
+
+		if (timestamp instanceof Date) {
+			timestamp = timestamp.getTime();
+		} else {
+			timestamp = new Date(timestamp).getTime();
+		}
+
+		shardId = shardId || Snowflake.SHARD_ID;
+
+		if (timestamp === Snowflake.lastTimestamp) {
+			Snowflake.sequence = (Snowflake.sequence + 1) % Snowflake.MAX_SEQUENCE;
+			if (Snowflake.sequence === 0) {
+				timestamp = Snowflake.waitNextMillis(timestamp);
+			}
+		} else {
+			Snowflake.sequence = 0;
+		}
+
+		Snowflake.lastTimestamp = timestamp;
+
+		const result = (BigInt(timestamp - Snowflake.EPOCH) << BigInt(22)) |
+			(BigInt(shardId % 1024) << BigInt(12)) |
+			BigInt(Snowflake.sequence);
+
+		return result.toString();
+	}
+
+	private static waitNextMillis(currentTimestamp: number) {
+		let timestamp = Date.now();
+		while (timestamp <= currentTimestamp) {
+			timestamp = Date.now();
+		}
+		return timestamp;
+	}
+}

package/test/step/typeorm-by-sql-transactional.test.ts

@@ -187,7 +187,7 @@ describe('TypeORM SQL Transactional Suite', () => {
 		await new TypeOrmDataSourceHelper(config).create({
 			'TEST': [TestTable]
 		});
-		const repo = (await TypeOrmDataSourceManager.findDataSource('TEST')).getDataSource().getRepository(TestTable);
+		const repo = (await TypeOrmDataSourceManager.findDataSource('TEST', config)).getDataSource().getRepository(TestTable);
 		await repo.insert({id: 1, content: 'hello world!'});
 		await repo.insert({id: 2, content: 'good-bye world!'});
 	});