@sprucelabs/mercury-chunking-emitter 1.4.2 → 1.5.1

package/build/chunkingEmitter/ChunkingEmitter.d.ts

@@ -6,13 +6,12 @@ export default class ChunkingEmitterImpl {
     protected chunkSize: number;
     private log;
     static Class?: new (options: ChunkingEmitterOptions) => ChunkingEmitter;
+    private emitters;
     private totalErrors;
     protected constructor(options: ChunkingEmitterOptions);
     static Emitter(options: ChunkingEmitterOptions): Promise<ChunkingEmitter>;
     emit(options: ChunkingEmitterEmitOptions): Promise<void>;
-    private emitChunk;
     static reset(): void;
-    private Cursor;
     getTotalErrors(): number;
 }
 interface ChunkingEmitterOptions {
@@ -25,6 +24,8 @@ export type ChunkingEmitterEmitOptions = {
     cursor?: BatchCursor<Record<string, any>>;
     payloadKey: string;
     target?: Record<string, any>;
+    /** will stop any running operations that match the key */
+    uniqueKey?: string;
 };
 export interface ChunkingEmitter {
     emit(options: ChunkingEmitterEmitOptions): Promise<void>;

package/build/chunkingEmitter/ChunkingEmitter.js

@@ -1,18 +1,15 @@
 "use strict";
-var __asyncValues = (this && this.__asyncValues) || function (o) {
-    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
-    var m = o[Symbol.asyncIterator], i;
-    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
-    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
-    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-const data_stores_1 = require("@sprucelabs/data-stores");
 const schema_1 = require("@sprucelabs/schema");
 const spruce_skill_utils_1 = require("@sprucelabs/spruce-skill-utils");
+const SingleChunkEmitter_1 = __importDefault(require("./SingleChunkEmitter"));
 class ChunkingEmitterImpl {
     constructor(options) {
         this.log = (0, spruce_skill_utils_1.buildLog)('ChunkingEmitter');
+        this.emitters = [];
         this.totalErrors = 0;
         const { client, chunkSize } = (0, schema_1.assertOptions)(options, ['client']);
         this.client = client;
@@ -24,8 +21,8 @@ class ChunkingEmitterImpl {
         return new ((_a = this.Class) !== null && _a !== void 0 ? _a : this)(options);
     }
     async emit(options) {
-        var _a, e_1, _b, _c;
-        const { eventName, items, payloadKey, target, cursor } = (0, schema_1.assertOptions)(options, ['eventName', 'payloadKey']);
+        const { items, cursor, uniqueKey: unqiueKey, } = (0, schema_1.assertOptions)(options, ['eventName', 'payloadKey']);
+        this.totalErrors = 0;
         if (!items && !cursor) {
             throw new schema_1.SchemaError({
                 code: 'MISSING_PARAMETERS',
@@ -33,64 +30,20 @@ class ChunkingEmitterImpl {
                 friendlyMessage: `You have to pass either 'items' or a 'cursor' to emit.`,
             });
         }
-        this.totalErrors = 0;
-        let current = 0;
-        let actualCursor = cursor !== null && cursor !== void 0 ? cursor : this.Cursor(items !== null && items !== void 0 ? items : []);
-        const total = await actualCursor.getTotalRecords();
-        try {
-            for (var _d = true, actualCursor_1 = __asyncValues(actualCursor), actualCursor_1_1; actualCursor_1_1 = await actualCursor_1.next(), _a = actualCursor_1_1.done, !_a; _d = true) {
-                _c = actualCursor_1_1.value;
-                _d = false;
-                const chunk = _c;
-                current = await this.emitChunk({
-                    payloadKey,
-                    chunk,
-                    current,
-                    total,
-                    target,
-                    eventName,
-                });
-                current++;
-            }
-        }
-        catch (e_1_1) { e_1 = { error: e_1_1 }; }
-        finally {
-            try {
-                if (!_d && !_a && (_b = actualCursor_1.return)) await _b.call(actualCursor_1);
-            }
-            finally { if (e_1) throw e_1.error; }
+        const match = this.emitters.find((e) => unqiueKey && e.matchesKey(unqiueKey));
+        if (match) {
+            await match.kill();
         }
-    }
-    async emitChunk(options) {
-        const { payloadKey, chunk, current, total, target, eventName } = options;
-        try {
-            let targetAndPayload = {
-                payload: {
-                    [payloadKey]: chunk,
-                    chunk: {
-                        current,
-                        total,
-                    },
-                },
-            };
-            if (target) {
-                targetAndPayload.target = target;
-            }
-            await this.client.emitAndFlattenResponses(eventName, targetAndPayload);
-        }
-        catch (err) {
-            this.log.error('Failed to emit chunk', err);
-            this.totalErrors++;
-        }
-        return current;
+        const emitter = new SingleChunkEmitter_1.default(Object.assign(Object.assign({}, options), { chunkSize: this.chunkSize, client: this.client, log: this.log }));
+        this.emitters.push(emitter);
+        await emitter.emit();
+        this.totalErrors += emitter.getTotalErrors();
+        this.emitters = this.emitters.filter((e) => e !== emitter);
+        return;
     }
     static reset() {
         this.Class = undefined;
     }
-    Cursor(items) {
-        const cursor = new data_stores_1.BatchArrayCursor(items, { batchSize: this.chunkSize });
-        return cursor;
-    }
     getTotalErrors() {
         return this.totalErrors;
     }

package/build/chunkingEmitter/SingleChunkEmitter.d.ts

@@ -0,0 +1,37 @@
+import { BatchCursor } from '@sprucelabs/data-stores';
+import { MercuryClient } from '@sprucelabs/mercury-client';
+import { EventName } from '@sprucelabs/mercury-types';
+import { Log } from '@sprucelabs/spruce-skill-utils';
+export default class SingleChunkEmitter {
+    private cursor?;
+    private items?;
+    private payloadKey;
+    private target?;
+    private eventName;
+    private chunkSize;
+    private client;
+    private total;
+    private uniqueKey?;
+    private log;
+    totalErrors: number;
+    private isKilled;
+    constructor(options: SingleChunkEmitterOptions);
+    matchesKey(key: string): boolean;
+    getTotalErrors(): number;
+    emit(): Promise<void>;
+    kill(): Promise<void>;
+    private emitChunk;
+    private Cursor;
+}
+interface SingleChunkEmitterOptions {
+    cursor?: BatchCursor<Record<string, any>>;
+    items?: Record<string, any>[];
+    payloadKey: string;
+    target?: Record<string, any>;
+    eventName: EventName;
+    chunkSize: number;
+    client: MercuryClient;
+    uniqueKey?: string;
+    log: Log;
+}
+export {};

package/build/chunkingEmitter/SingleChunkEmitter.js

@@ -0,0 +1,92 @@
+"use strict";
+var __asyncValues = (this && this.__asyncValues) || function (o) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var m = o[Symbol.asyncIterator], i;
+    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
+    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
+    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const data_stores_1 = require("@sprucelabs/data-stores");
+class SingleChunkEmitter {
+    constructor(options) {
+        this.totalErrors = 0;
+        this.isKilled = false;
+        const { cursor, items, payloadKey, target, eventName, chunkSize, client, uniqueKey, log, } = options;
+        this.cursor = cursor;
+        this.uniqueKey = uniqueKey;
+        this.items = items;
+        this.payloadKey = payloadKey;
+        this.target = target;
+        this.eventName = eventName;
+        this.chunkSize = chunkSize;
+        this.client = client;
+        this.log = log.buildLog('Emitter');
+    }
+    matchesKey(key) {
+        return this.uniqueKey === key;
+    }
+    getTotalErrors() {
+        return this.totalErrors;
+    }
+    async emit() {
+        var _a, e_1, _b, _c;
+        var _d, _e;
+        let current = 0;
+        const actualCursor = (_d = this.cursor) !== null && _d !== void 0 ? _d : this.Cursor((_e = this.items) !== null && _e !== void 0 ? _e : []);
+        this.total = await actualCursor.getTotalRecords();
+        try {
+            for (var _f = true, actualCursor_1 = __asyncValues(actualCursor), actualCursor_1_1; actualCursor_1_1 = await actualCursor_1.next(), _a = actualCursor_1_1.done, !_a; _f = true) {
+                _c = actualCursor_1_1.value;
+                _f = false;
+                const chunk = _c;
+                try {
+                    current = await this.emitChunk({
+                        chunk,
+                        current,
+                    });
+                    if (this.isKilled) {
+                        return;
+                    }
+                }
+                catch (err) {
+                    this.log.error('Failed to emit chunk', err);
+                    this.totalErrors++;
+                }
+                current++;
+            }
+        }
+        catch (e_1_1) { e_1 = { error: e_1_1 }; }
+        finally {
+            try {
+                if (!_f && !_a && (_b = actualCursor_1.return)) await _b.call(actualCursor_1);
+            }
+            finally { if (e_1) throw e_1.error; }
+        }
+    }
+    async kill() {
+        this.isKilled = true;
+    }
+    async emitChunk(options) {
+        const { chunk, current } = options;
+        let targetAndPayload = {
+            payload: {
+                [this.payloadKey]: chunk,
+                chunk: {
+                    current,
+                    total: this.total,
+                },
+            },
+        };
+        if (this.target) {
+            targetAndPayload.target = this.target;
+        }
+        await this.client.emitAndFlattenResponses(this.eventName, targetAndPayload);
+        return current;
+    }
+    Cursor(items) {
+        const cursor = new data_stores_1.BatchArrayCursor(items, { batchSize: this.chunkSize });
+        return cursor;
+    }
+}
+exports.default = SingleChunkEmitter;

package/build/esm/chunkingEmitter/ChunkingEmitter.d.ts

@@ -6,13 +6,12 @@ export default class ChunkingEmitterImpl {
     protected chunkSize: number;
     private log;
     static Class?: new (options: ChunkingEmitterOptions) => ChunkingEmitter;
+    private emitters;
     private totalErrors;
     protected constructor(options: ChunkingEmitterOptions);
     static Emitter(options: ChunkingEmitterOptions): Promise<ChunkingEmitter>;
     emit(options: ChunkingEmitterEmitOptions): Promise<void>;
-    private emitChunk;
     static reset(): void;
-    private Cursor;
     getTotalErrors(): number;
 }
 interface ChunkingEmitterOptions {
@@ -25,6 +24,8 @@ export type ChunkingEmitterEmitOptions = {
     cursor?: BatchCursor<Record<string, any>>;
     payloadKey: string;
     target?: Record<string, any>;
+    /** will stop any running operations that match the key */
+    uniqueKey?: string;
 };
 export interface ChunkingEmitter {
     emit(options: ChunkingEmitterEmitOptions): Promise<void>;

package/build/esm/chunkingEmitter/ChunkingEmitter.js

@@ -7,19 +7,13 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
-var __asyncValues = (this && this.__asyncValues) || function (o) {
-    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
-    var m = o[Symbol.asyncIterator], i;
-    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
-    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
-    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
-};
-import { BatchArrayCursor } from '@sprucelabs/data-stores';
 import { SchemaError, assertOptions } from '@sprucelabs/schema';
 import { buildLog } from '@sprucelabs/spruce-skill-utils';
+import SingleChunkEmitter from './SingleChunkEmitter.js';
 export default class ChunkingEmitterImpl {
     constructor(options) {
         this.log = buildLog('ChunkingEmitter');
+        this.emitters = [];
         this.totalErrors = 0;
         const { client, chunkSize } = assertOptions(options, ['client']);
         this.client = client;
@@ -33,9 +27,9 @@ export default class ChunkingEmitterImpl {
         });
     }
     emit(options) {
-        var _a, e_1, _b, _c;
         return __awaiter(this, void 0, void 0, function* () {
-            const { eventName, items, payloadKey, target, cursor } = assertOptions(options, ['eventName', 'payloadKey']);
+            const { items, cursor, uniqueKey: unqiueKey, } = assertOptions(options, ['eventName', 'payloadKey']);
+            this.totalErrors = 0;
             if (!items && !cursor) {
                 throw new SchemaError({
                     code: 'MISSING_PARAMETERS',
@@ -43,67 +37,21 @@ export default class ChunkingEmitterImpl {
                     friendlyMessage: `You have to pass either 'items' or a 'cursor' to emit.`,
                 });
             }
-            this.totalErrors = 0;
-            let current = 0;
-            let actualCursor = cursor !== null && cursor !== void 0 ? cursor : this.Cursor(items !== null && items !== void 0 ? items : []);
-            const total = yield actualCursor.getTotalRecords();
-            try {
-                for (var _d = true, actualCursor_1 = __asyncValues(actualCursor), actualCursor_1_1; actualCursor_1_1 = yield actualCursor_1.next(), _a = actualCursor_1_1.done, !_a; _d = true) {
-                    _c = actualCursor_1_1.value;
-                    _d = false;
-                    const chunk = _c;
-                    current = yield this.emitChunk({
-                        payloadKey,
-                        chunk,
-                        current,
-                        total,
-                        target,
-                        eventName,
-                    });
-                    current++;
-                }
-            }
-            catch (e_1_1) { e_1 = { error: e_1_1 }; }
-            finally {
-                try {
-                    if (!_d && !_a && (_b = actualCursor_1.return)) yield _b.call(actualCursor_1);
-                }
-                finally { if (e_1) throw e_1.error; }
-            }
-        });
-    }
-    emitChunk(options) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const { payloadKey, chunk, current, total, target, eventName } = options;
-            try {
-                let targetAndPayload = {
-                    payload: {
-                        [payloadKey]: chunk,
-                        chunk: {
-                            current,
-                            total,
-                        },
-                    },
-                };
-                if (target) {
-                    targetAndPayload.target = target;
-                }
-                yield this.client.emitAndFlattenResponses(eventName, targetAndPayload);
+            const match = this.emitters.find((e) => unqiueKey && e.matchesKey(unqiueKey));
+            if (match) {
+                yield match.kill();
             }
-            catch (err) {
-                this.log.error('Failed to emit chunk', err);
-                this.totalErrors++;
-            }
-            return current;
+            const emitter = new SingleChunkEmitter(Object.assign(Object.assign({}, options), { chunkSize: this.chunkSize, client: this.client, log: this.log }));
+            this.emitters.push(emitter);
+            yield emitter.emit();
+            this.totalErrors += emitter.getTotalErrors();
+            this.emitters = this.emitters.filter((e) => e !== emitter);
+            return;
         });
     }
     static reset() {
         this.Class = undefined;
     }
-    Cursor(items) {
-        const cursor = new BatchArrayCursor(items, { batchSize: this.chunkSize });
-        return cursor;
-    }
     getTotalErrors() {
         return this.totalErrors;
     }

package/build/esm/chunkingEmitter/SingleChunkEmitter.d.ts

@@ -0,0 +1,37 @@
+import { BatchCursor } from '@sprucelabs/data-stores';
+import { MercuryClient } from '@sprucelabs/mercury-client';
+import { EventName } from '@sprucelabs/mercury-types';
+import { Log } from '@sprucelabs/spruce-skill-utils';
+export default class SingleChunkEmitter {
+    private cursor?;
+    private items?;
+    private payloadKey;
+    private target?;
+    private eventName;
+    private chunkSize;
+    private client;
+    private total;
+    private uniqueKey?;
+    private log;
+    totalErrors: number;
+    private isKilled;
+    constructor(options: SingleChunkEmitterOptions);
+    matchesKey(key: string): boolean;
+    getTotalErrors(): number;
+    emit(): Promise<void>;
+    kill(): Promise<void>;
+    private emitChunk;
+    private Cursor;
+}
+interface SingleChunkEmitterOptions {
+    cursor?: BatchCursor<Record<string, any>>;
+    items?: Record<string, any>[];
+    payloadKey: string;
+    target?: Record<string, any>;
+    eventName: EventName;
+    chunkSize: number;
+    client: MercuryClient;
+    uniqueKey?: string;
+    log: Log;
+}
+export {};

package/build/esm/chunkingEmitter/SingleChunkEmitter.js

@@ -0,0 +1,104 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __asyncValues = (this && this.__asyncValues) || function (o) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var m = o[Symbol.asyncIterator], i;
+    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
+    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
+    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
+};
+import { BatchArrayCursor } from '@sprucelabs/data-stores';
+export default class SingleChunkEmitter {
+    constructor(options) {
+        this.totalErrors = 0;
+        this.isKilled = false;
+        const { cursor, items, payloadKey, target, eventName, chunkSize, client, uniqueKey, log, } = options;
+        this.cursor = cursor;
+        this.uniqueKey = uniqueKey;
+        this.items = items;
+        this.payloadKey = payloadKey;
+        this.target = target;
+        this.eventName = eventName;
+        this.chunkSize = chunkSize;
+        this.client = client;
+        this.log = log.buildLog('Emitter');
+    }
+    matchesKey(key) {
+        return this.uniqueKey === key;
+    }
+    getTotalErrors() {
+        return this.totalErrors;
+    }
+    emit() {
+        var _a, e_1, _b, _c;
+        var _d, _e;
+        return __awaiter(this, void 0, void 0, function* () {
+            let current = 0;
+            const actualCursor = (_d = this.cursor) !== null && _d !== void 0 ? _d : this.Cursor((_e = this.items) !== null && _e !== void 0 ? _e : []);
+            this.total = yield actualCursor.getTotalRecords();
+            try {
+                for (var _f = true, actualCursor_1 = __asyncValues(actualCursor), actualCursor_1_1; actualCursor_1_1 = yield actualCursor_1.next(), _a = actualCursor_1_1.done, !_a; _f = true) {
+                    _c = actualCursor_1_1.value;
+                    _f = false;
+                    const chunk = _c;
+                    try {
+                        current = yield this.emitChunk({
+                            chunk,
+                            current,
+                        });
+                        if (this.isKilled) {
+                            return;
+                        }
+                    }
+                    catch (err) {
+                        this.log.error('Failed to emit chunk', err);
+                        this.totalErrors++;
+                    }
+                    current++;
+                }
+            }
+            catch (e_1_1) { e_1 = { error: e_1_1 }; }
+            finally {
+                try {
+                    if (!_f && !_a && (_b = actualCursor_1.return)) yield _b.call(actualCursor_1);
+                }
+                finally { if (e_1) throw e_1.error; }
+            }
+        });
+    }
+    kill() {
+        return __awaiter(this, void 0, void 0, function* () {
+            this.isKilled = true;
+        });
+    }
+    emitChunk(options) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const { chunk, current } = options;
+            let targetAndPayload = {
+                payload: {
+                    [this.payloadKey]: chunk,
+                    chunk: {
+                        current,
+                        total: this.total,
+                    },
+                },
+            };
+            if (this.target) {
+                targetAndPayload.target = this.target;
+            }
+            yield this.client.emitAndFlattenResponses(this.eventName, targetAndPayload);
+            return current;
+        });
+    }
+    Cursor(items) {
+        const cursor = new BatchArrayCursor(items, { batchSize: this.chunkSize });
+        return cursor;
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sprucelabs/mercury-chunking-emitter",
-  "version": "1.4.2",
+  "version": "1.5.1",
   "files": [
     "build/**/*",
     "!build/__tests__",
@@ -39,13 +39,13 @@
     "release": "semantic-release"
   },
   "devDependencies": {
-    "@sprucelabs/esm-postbuild": "^5.0.85",
-    "@sprucelabs/jest-json-reporter": "^7.0.112",
-    "@sprucelabs/resolve-path-aliases": "^1.1.254",
+    "@sprucelabs/esm-postbuild": "^5.0.86",
+    "@sprucelabs/jest-json-reporter": "^7.0.113",
+    "@sprucelabs/resolve-path-aliases": "^1.1.255",
     "@sprucelabs/semantic-release": "^4.0.8",
-    "@sprucelabs/spruce-test-fixtures": "^60.1.13",
-    "@sprucelabs/test": "^8.0.22",
-    "@sprucelabs/test-utils": "^4.0.62",
+    "@sprucelabs/spruce-test-fixtures": "^60.1.14",
+    "@sprucelabs/test": "^8.0.23",
+    "@sprucelabs/test-utils": "^4.0.63",
     "@types/node": "^20.8.9",
     "chokidar-cli": "^3.0.0",
     "concurrently": "^8.2.2",
@@ -86,12 +86,12 @@
     }
   },
   "dependencies": {
-    "@sprucelabs/data-stores": "^25.7.0",
-    "@sprucelabs/mercury-client": "^41.0.117",
-    "@sprucelabs/mercury-types": "^46.0.102",
-    "@sprucelabs/schema": "^29.0.73",
-    "@sprucelabs/spruce-core-schemas": "^39.0.108",
-    "@sprucelabs/spruce-event-utils": "^38.0.122",
-    "@sprucelabs/spruce-skill-utils": "^30.1.6"
+    "@sprucelabs/data-stores": "^25.7.1",
+    "@sprucelabs/mercury-client": "^41.0.118",
+    "@sprucelabs/mercury-types": "^46.0.103",
+    "@sprucelabs/schema": "^29.0.74",
+    "@sprucelabs/spruce-core-schemas": "^39.0.109",
+    "@sprucelabs/spruce-event-utils": "^38.0.123",
+    "@sprucelabs/spruce-skill-utils": "^30.1.7"
   }
 }