@sprucelabs/mercury-chunking-emitter 1.3.13 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (7) hide show
  1. package/build/chunkingEmitter/ChunkingEmitter.d.ts +4 -1
  2. package/build/chunkingEmitter/ChunkingEmitter.js +63 -36
  3. package/build/chunkingEmitter/chunkFieldDefinition.d.ts +2 -2
  4. package/build/esm/chunkingEmitter/ChunkingEmitter.d.ts +4 -1
  5. package/build/esm/chunkingEmitter/ChunkingEmitter.js +66 -37
  6. package/build/esm/chunkingEmitter/chunkFieldDefinition.d.ts +2 -2
  7. package/package.json +4 -3
package/build/chunkingEmitter/ChunkingEmitter.d.ts CHANGED
@@ -1,3 +1,4 @@
1
+ import { BatchCursor } from '@sprucelabs/data-stores';
1
2
  import { MercuryClient } from '@sprucelabs/mercury-client';
2
3
  import { EventName } from '@sprucelabs/mercury-types';
3
4
  export default class ChunkingEmitterImpl {
@@ -9,6 +10,7 @@ export default class ChunkingEmitterImpl {
9
10
  protected constructor(options: ChunkingEmitterOptions);
10
11
  static Emitter(options: ChunkingEmitterOptions): Promise<ChunkingEmitter>;
11
12
  emit(options: ChunkingEmitterEmitOptions): Promise<void>;
13
+ private emitChunk;
12
14
  static reset(): void;
13
15
  private splitItemsIntoChunks;
14
16
  getTotalErrors(): number;
@@ -19,7 +21,8 @@ interface ChunkingEmitterOptions {
19
21
  }
20
22
  export type ChunkingEmitterEmitOptions = {
21
23
  eventName: EventName;
22
- items: Record<string, any>[];
24
+ items?: Record<string, any>[];
25
+ cursor?: BatchCursor<Record<string, any>>;
23
26
  payloadKey: string;
24
27
  target?: Record<string, any>;
25
28
  };
package/build/chunkingEmitter/ChunkingEmitter.js CHANGED
@@ -1,5 +1,13 @@
1
1
  "use strict";
2
+ var __asyncValues = (this && this.__asyncValues) || function (o) {
3
+ if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
4
+ var m = o[Symbol.asyncIterator], i;
5
+ return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
6
+ function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
7
+ function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
8
+ };
2
9
  Object.defineProperty(exports, "__esModule", { value: true });
10
+ const data_stores_1 = require("@sprucelabs/data-stores");
3
11
  const schema_1 = require("@sprucelabs/schema");
4
12
  const spruce_skill_utils_1 = require("@sprucelabs/spruce-skill-utils");
5
13
  class ChunkingEmitterImpl {
@@ -16,53 +24,72 @@ class ChunkingEmitterImpl {
16
24
  return new ((_a = this.Class) !== null && _a !== void 0 ? _a : this)(options);
17
25
  }
18
26
  async emit(options) {
19
- const { eventName, items, payloadKey, target } = (0, schema_1.assertOptions)(options, [
20
- 'eventName',
21
- 'items',
22
- 'payloadKey',
23
- ]);
27
+ var _a, e_1, _b, _c;
28
+ const { eventName, items, payloadKey, target, cursor } = (0, schema_1.assertOptions)(options, ['eventName', 'payloadKey']);
29
+ if (!items && !cursor) {
30
+ throw new schema_1.SchemaError({
31
+ code: 'MISSING_PARAMETERS',
32
+ parameters: ['items', 'cursor'],
33
+ friendlyMessage: `You have to pass either 'items' or a 'cursor' to emit.`,
34
+ });
35
+ }
24
36
  this.totalErrors = 0;
25
- const chunks = this.splitItemsIntoChunks(items);
37
+ let actualCursor = cursor !== null && cursor !== void 0 ? cursor : this.splitItemsIntoChunks(items !== null && items !== void 0 ? items : []);
38
+ const total = await actualCursor.getTotalRecords();
26
39
  let current = 0;
27
- for (const chunk of chunks) {
40
+ try {
41
+ for (var _d = true, actualCursor_1 = __asyncValues(actualCursor), actualCursor_1_1; actualCursor_1_1 = await actualCursor_1.next(), _a = actualCursor_1_1.done, !_a; _d = true) {
42
+ _c = actualCursor_1_1.value;
43
+ _d = false;
44
+ const chunk = _c;
45
+ current = await this.emitChunk({
46
+ payloadKey,
47
+ chunk,
48
+ current,
49
+ total,
50
+ target,
51
+ eventName,
52
+ });
53
+ current++;
54
+ }
55
+ }
56
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
57
+ finally {
28
58
  try {
29
- let targetAndPayload = {
30
- payload: {
31
- [payloadKey]: chunk,
32
- chunk: {
33
- current: current++,
34
- total: chunks.length,
35
- },
36
- },
37
- };
38
- if (target) {
39
- targetAndPayload.target = target;
40
- }
41
- await this.client.emitAndFlattenResponses(eventName, targetAndPayload);
59
+ if (!_d && !_a && (_b = actualCursor_1.return)) await _b.call(actualCursor_1);
42
60
  }
43
- catch (err) {
44
- this.log.error('Failed to emit chunk', err);
45
- this.totalErrors++;
61
+ finally { if (e_1) throw e_1.error; }
62
+ }
63
+ }
64
+ async emitChunk(options) {
65
+ const { payloadKey, chunk, current, total, target, eventName } = options;
66
+ try {
67
+ let targetAndPayload = {
68
+ payload: {
69
+ [payloadKey]: chunk,
70
+ chunk: {
71
+ current,
72
+ total,
73
+ },
74
+ },
75
+ };
76
+ if (target) {
77
+ targetAndPayload.target = target;
46
78
  }
79
+ await this.client.emitAndFlattenResponses(eventName, targetAndPayload);
80
+ }
81
+ catch (err) {
82
+ this.log.error('Failed to emit chunk', err);
83
+ this.totalErrors++;
47
84
  }
85
+ return current;
48
86
  }
49
87
  static reset() {
50
88
  this.Class = undefined;
51
89
  }
52
90
  splitItemsIntoChunks(items) {
53
- const chunks = [];
54
- let index = 0;
55
- for (const item of items) {
56
- if (!chunks[index]) {
57
- chunks[index] = [];
58
- }
59
- if (chunks[index].length === this.chunkSize) {
60
- index++;
61
- chunks[index] = [];
62
- }
63
- chunks[index].push(item);
64
- }
65
- return chunks;
91
+ const cursor = new data_stores_1.BatchArrayCursor(items, { batchSize: this.chunkSize });
92
+ return cursor;
66
93
  }
67
94
  getTotalErrors() {
68
95
  return this.totalErrors;
package/build/chunkingEmitter/chunkFieldDefinition.d.ts CHANGED
@@ -1,2 +1,2 @@
1
- import { FieldDefinitions } from '@sprucelabs/schema';
2
- export declare function chunkFieldDefinition(): FieldDefinitions;
1
+ import { SchemaFieldFieldDefinition } from '@sprucelabs/schema';
2
+ export declare function chunkFieldDefinition(): SchemaFieldFieldDefinition;
package/build/esm/chunkingEmitter/ChunkingEmitter.d.ts CHANGED
@@ -1,3 +1,4 @@
1
+ import { BatchCursor } from '@sprucelabs/data-stores';
1
2
  import { MercuryClient } from '@sprucelabs/mercury-client';
2
3
  import { EventName } from '@sprucelabs/mercury-types';
3
4
  export default class ChunkingEmitterImpl {
@@ -9,6 +10,7 @@ export default class ChunkingEmitterImpl {
9
10
  protected constructor(options: ChunkingEmitterOptions);
10
11
  static Emitter(options: ChunkingEmitterOptions): Promise<ChunkingEmitter>;
11
12
  emit(options: ChunkingEmitterEmitOptions): Promise<void>;
13
+ private emitChunk;
12
14
  static reset(): void;
13
15
  private splitItemsIntoChunks;
14
16
  getTotalErrors(): number;
@@ -19,7 +21,8 @@ interface ChunkingEmitterOptions {
19
21
  }
20
22
  export type ChunkingEmitterEmitOptions = {
21
23
  eventName: EventName;
22
- items: Record<string, any>[];
24
+ items?: Record<string, any>[];
25
+ cursor?: BatchCursor<Record<string, any>>;
23
26
  payloadKey: string;
24
27
  target?: Record<string, any>;
25
28
  };
package/build/esm/chunkingEmitter/ChunkingEmitter.js CHANGED
@@ -7,7 +7,15 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
7
7
  step((generator = generator.apply(thisArg, _arguments || [])).next());
8
8
  });
9
9
  };
10
- import { assertOptions } from '@sprucelabs/schema';
10
+ var __asyncValues = (this && this.__asyncValues) || function (o) {
11
+ if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
12
+ var m = o[Symbol.asyncIterator], i;
13
+ return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
14
+ function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
15
+ function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
16
+ };
17
+ import { BatchArrayCursor } from '@sprucelabs/data-stores';
18
+ import { SchemaError, assertOptions } from '@sprucelabs/schema';
11
19
  import { buildLog } from '@sprucelabs/spruce-skill-utils';
12
20
  export default class ChunkingEmitterImpl {
13
21
  constructor(options) {
@@ -25,55 +33,76 @@ export default class ChunkingEmitterImpl {
25
33
  });
26
34
  }
27
35
  emit(options) {
36
+ var _a, e_1, _b, _c;
28
37
  return __awaiter(this, void 0, void 0, function* () {
29
- const { eventName, items, payloadKey, target } = assertOptions(options, [
30
- 'eventName',
31
- 'items',
32
- 'payloadKey',
33
- ]);
38
+ const { eventName, items, payloadKey, target, cursor } = assertOptions(options, ['eventName', 'payloadKey']);
39
+ if (!items && !cursor) {
40
+ throw new SchemaError({
41
+ code: 'MISSING_PARAMETERS',
42
+ parameters: ['items', 'cursor'],
43
+ friendlyMessage: `You have to pass either 'items' or a 'cursor' to emit.`,
44
+ });
45
+ }
34
46
  this.totalErrors = 0;
35
- const chunks = this.splitItemsIntoChunks(items);
47
+ let actualCursor = cursor !== null && cursor !== void 0 ? cursor : this.splitItemsIntoChunks(items !== null && items !== void 0 ? items : []);
48
+ const total = yield actualCursor.getTotalRecords();
36
49
  let current = 0;
37
- for (const chunk of chunks) {
50
+ try {
51
+ for (var _d = true, actualCursor_1 = __asyncValues(actualCursor), actualCursor_1_1; actualCursor_1_1 = yield actualCursor_1.next(), _a = actualCursor_1_1.done, !_a; _d = true) {
52
+ _c = actualCursor_1_1.value;
53
+ _d = false;
54
+ const chunk = _c;
55
+ current = yield this.emitChunk({
56
+ payloadKey,
57
+ chunk,
58
+ current,
59
+ total,
60
+ target,
61
+ eventName,
62
+ });
63
+ current++;
64
+ }
65
+ }
66
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
67
+ finally {
38
68
  try {
39
- let targetAndPayload = {
40
- payload: {
41
- [payloadKey]: chunk,
42
- chunk: {
43
- current: current++,
44
- total: chunks.length,
45
- },
46
- },
47
- };
48
- if (target) {
49
- targetAndPayload.target = target;
50
- }
51
- yield this.client.emitAndFlattenResponses(eventName, targetAndPayload);
69
+ if (!_d && !_a && (_b = actualCursor_1.return)) yield _b.call(actualCursor_1);
52
70
  }
53
- catch (err) {
54
- this.log.error('Failed to emit chunk', err);
55
- this.totalErrors++;
71
+ finally { if (e_1) throw e_1.error; }
72
+ }
73
+ });
74
+ }
75
+ emitChunk(options) {
76
+ return __awaiter(this, void 0, void 0, function* () {
77
+ const { payloadKey, chunk, current, total, target, eventName } = options;
78
+ try {
79
+ let targetAndPayload = {
80
+ payload: {
81
+ [payloadKey]: chunk,
82
+ chunk: {
83
+ current,
84
+ total,
85
+ },
86
+ },
87
+ };
88
+ if (target) {
89
+ targetAndPayload.target = target;
56
90
  }
91
+ yield this.client.emitAndFlattenResponses(eventName, targetAndPayload);
57
92
  }
93
+ catch (err) {
94
+ this.log.error('Failed to emit chunk', err);
95
+ this.totalErrors++;
96
+ }
97
+ return current;
58
98
  });
59
99
  }
60
100
  static reset() {
61
101
  this.Class = undefined;
62
102
  }
63
103
  splitItemsIntoChunks(items) {
64
- const chunks = [];
65
- let index = 0;
66
- for (const item of items) {
67
- if (!chunks[index]) {
68
- chunks[index] = [];
69
- }
70
- if (chunks[index].length === this.chunkSize) {
71
- index++;
72
- chunks[index] = [];
73
- }
74
- chunks[index].push(item);
75
- }
76
- return chunks;
104
+ const cursor = new BatchArrayCursor(items, { batchSize: this.chunkSize });
105
+ return cursor;
77
106
  }
78
107
  getTotalErrors() {
79
108
  return this.totalErrors;
package/build/esm/chunkingEmitter/chunkFieldDefinition.d.ts CHANGED
@@ -1,2 +1,2 @@
1
- import { FieldDefinitions } from '@sprucelabs/schema';
2
- export declare function chunkFieldDefinition(): FieldDefinitions;
1
+ import { SchemaFieldFieldDefinition } from '@sprucelabs/schema';
2
+ export declare function chunkFieldDefinition(): SchemaFieldFieldDefinition;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@sprucelabs/mercury-chunking-emitter",
3
- "version": "1.3.13",
3
+ "version": "1.4.0",
4
4
  "files": [
5
5
  "build/**/*",
6
6
  "!build/__tests__",
@@ -43,7 +43,7 @@
43
43
  "@sprucelabs/jest-json-reporter": "^7.0.112",
44
44
  "@sprucelabs/resolve-path-aliases": "^1.1.254",
45
45
  "@sprucelabs/semantic-release": "^4.0.8",
46
- "@sprucelabs/spruce-test-fixtures": "^60.1.11",
46
+ "@sprucelabs/spruce-test-fixtures": "^60.1.12",
47
47
  "@sprucelabs/test": "^8.0.22",
48
48
  "@sprucelabs/test-utils": "^4.0.62",
49
49
  "@types/node": "^20.8.9",
@@ -86,7 +86,8 @@
86
86
  }
87
87
  },
88
88
  "dependencies": {
89
- "@sprucelabs/mercury-client": "^41.0.116",
89
+ "@sprucelabs/data-stores": "^25.7.0",
90
+ "@sprucelabs/mercury-client": "^41.0.117",
90
91
  "@sprucelabs/mercury-types": "^46.0.102",
91
92
  "@sprucelabs/schema": "^29.0.73",
92
93
  "@sprucelabs/spruce-core-schemas": "^39.0.108",