@flowcore/data-pump 0.10.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -0
- package/esm/data-pump/data-pump.d.ts +1 -0
- package/esm/data-pump/data-pump.d.ts.map +1 -1
- package/esm/data-pump/data-pump.js +11 -0
- package/esm/data-pump/data-source.d.ts +92 -8
- package/esm/data-pump/data-source.d.ts.map +1 -1
- package/esm/data-pump/data-source.js +75 -0
- package/esm/data-pump/no-op-logger.d.ts +3 -0
- package/esm/data-pump/no-op-logger.d.ts.map +1 -0
- package/esm/data-pump/no-op-logger.js +6 -0
- package/esm/data-pump/notifier.js +1 -1
- package/esm/mod.d.ts +1 -2
- package/esm/mod.d.ts.map +1 -1
- package/esm/mod.js +1 -2
- package/package.json +1 -1
- package/script/data-pump/data-pump.d.ts +1 -0
- package/script/data-pump/data-pump.d.ts.map +1 -1
- package/script/data-pump/data-pump.js +11 -0
- package/script/data-pump/data-source.d.ts +92 -8
- package/script/data-pump/data-source.d.ts.map +1 -1
- package/script/data-pump/data-source.js +75 -0
- package/script/data-pump/no-op-logger.d.ts +3 -0
- package/script/data-pump/no-op-logger.d.ts.map +1 -0
- package/script/data-pump/no-op-logger.js +9 -0
- package/script/data-pump/notifier.js +2 -2
- package/script/mod.d.ts +1 -2
- package/script/mod.d.ts.map +1 -1
- package/script/mod.js +1 -2
- package/esm/lib/data-pump-create.d.ts +0 -46
- package/esm/lib/data-pump-create.d.ts.map +0 -1
- package/esm/lib/data-pump-create.js +0 -64
- package/esm/lib/data-pump.d.ts +0 -80
- package/esm/lib/data-pump.d.ts.map +0 -1
- package/esm/lib/data-pump.js +0 -437
- package/esm/lib/nats-notifier.d.ts +0 -18
- package/esm/lib/nats-notifier.d.ts.map +0 -1
- package/esm/lib/nats-notifier.js +0 -88
- package/esm/lib/ws-notifier.d.ts +0 -37
- package/esm/lib/ws-notifier.d.ts.map +0 -1
- package/esm/lib/ws-notifier.js +0 -86
- package/script/lib/data-pump-create.d.ts +0 -46
- package/script/lib/data-pump-create.d.ts.map +0 -1
- package/script/lib/data-pump-create.js +0 -68
- package/script/lib/data-pump.d.ts +0 -80
- package/script/lib/data-pump.d.ts.map +0 -1
- package/script/lib/data-pump.js +0 -441
- package/script/lib/nats-notifier.d.ts +0 -18
- package/script/lib/nats-notifier.d.ts.map +0 -1
- package/script/lib/nats-notifier.js +0 -115
- package/script/lib/ws-notifier.d.ts +0 -37
- package/script/lib/ws-notifier.d.ts.map +0 -1
- package/script/lib/ws-notifier.js +0 -90
|
@@ -1,9 +1,21 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.FlowcoreDataSource = void 0;
|
|
4
|
+
/**
|
|
5
|
+
* @module data-source
|
|
6
|
+
* @description Provides functionality to interact with Flowcore data sources
|
|
7
|
+
*/
|
|
4
8
|
const mod_js_1 = require("../deps/jsr.io/@flowcore/sdk/1.26.3/src/mod.js");
|
|
5
9
|
const flowcore_client_js_1 = require("./flowcore-client.js");
|
|
10
|
+
/**
|
|
11
|
+
* Provides methods to interact with a Flowcore data source
|
|
12
|
+
* @class FlowcoreDataSource
|
|
13
|
+
*/
|
|
6
14
|
class FlowcoreDataSource {
|
|
15
|
+
/**
|
|
16
|
+
* Creates a new FlowcoreDataSource instance
|
|
17
|
+
* @param options - Configuration options for the data source
|
|
18
|
+
*/
|
|
7
19
|
constructor(options) {
|
|
8
20
|
Object.defineProperty(this, "options", {
|
|
9
21
|
enumerable: true,
|
|
@@ -11,36 +23,42 @@ class FlowcoreDataSource {
|
|
|
11
23
|
writable: true,
|
|
12
24
|
value: options
|
|
13
25
|
});
|
|
26
|
+
/** Flowcore client instance */
|
|
14
27
|
Object.defineProperty(this, "flowcoreClient", {
|
|
15
28
|
enumerable: true,
|
|
16
29
|
configurable: true,
|
|
17
30
|
writable: true,
|
|
18
31
|
value: void 0
|
|
19
32
|
});
|
|
33
|
+
/** Cached tenant ID */
|
|
20
34
|
Object.defineProperty(this, "tenantId", {
|
|
21
35
|
enumerable: true,
|
|
22
36
|
configurable: true,
|
|
23
37
|
writable: true,
|
|
24
38
|
value: void 0
|
|
25
39
|
});
|
|
40
|
+
/** Cached data core ID */
|
|
26
41
|
Object.defineProperty(this, "dataCoreId", {
|
|
27
42
|
enumerable: true,
|
|
28
43
|
configurable: true,
|
|
29
44
|
writable: true,
|
|
30
45
|
value: void 0
|
|
31
46
|
});
|
|
47
|
+
/** Cached flow type ID */
|
|
32
48
|
Object.defineProperty(this, "flowTypeId", {
|
|
33
49
|
enumerable: true,
|
|
34
50
|
configurable: true,
|
|
35
51
|
writable: true,
|
|
36
52
|
value: void 0
|
|
37
53
|
});
|
|
54
|
+
/** Cached event type IDs */
|
|
38
55
|
Object.defineProperty(this, "eventTypeIds", {
|
|
39
56
|
enumerable: true,
|
|
40
57
|
configurable: true,
|
|
41
58
|
writable: true,
|
|
42
59
|
value: void 0
|
|
43
60
|
});
|
|
61
|
+
/** Cached time buckets */
|
|
44
62
|
Object.defineProperty(this, "timeBuckets", {
|
|
45
63
|
enumerable: true,
|
|
46
64
|
configurable: true,
|
|
@@ -49,18 +67,38 @@ class FlowcoreDataSource {
|
|
|
49
67
|
});
|
|
50
68
|
this.flowcoreClient = (0, flowcore_client_js_1.getFlowcoreClient)(this.options.auth, this.options.baseUrlOverride);
|
|
51
69
|
}
|
|
70
|
+
/**
|
|
71
|
+
* Gets the tenant name from the data source configuration
|
|
72
|
+
* @returns Tenant name
|
|
73
|
+
*/
|
|
52
74
|
get tenant() {
|
|
53
75
|
return this.options.dataSource.tenant;
|
|
54
76
|
}
|
|
77
|
+
/**
|
|
78
|
+
* Gets the data core name from the data source configuration
|
|
79
|
+
* @returns Data core name
|
|
80
|
+
*/
|
|
55
81
|
get dataCore() {
|
|
56
82
|
return this.options.dataSource.dataCore;
|
|
57
83
|
}
|
|
84
|
+
/**
|
|
85
|
+
* Gets the flow type name from the data source configuration
|
|
86
|
+
* @returns Flow type name
|
|
87
|
+
*/
|
|
58
88
|
get flowType() {
|
|
59
89
|
return this.options.dataSource.flowType;
|
|
60
90
|
}
|
|
91
|
+
/**
|
|
92
|
+
* Gets the event type names from the data source configuration
|
|
93
|
+
* @returns Array of event type names
|
|
94
|
+
*/
|
|
61
95
|
get eventTypes() {
|
|
62
96
|
return this.options.dataSource.eventTypes;
|
|
63
97
|
}
|
|
98
|
+
/**
|
|
99
|
+
* Gets the tenant ID, translating from name if necessary
|
|
100
|
+
* @returns Promise that resolves to the tenant ID
|
|
101
|
+
*/
|
|
64
102
|
async getTenantId() {
|
|
65
103
|
if (this.tenantId) {
|
|
66
104
|
return this.tenantId;
|
|
@@ -77,6 +115,10 @@ class FlowcoreDataSource {
|
|
|
77
115
|
}
|
|
78
116
|
return this.tenantId;
|
|
79
117
|
}
|
|
118
|
+
/**
|
|
119
|
+
* Gets the data core ID, translating from name if necessary
|
|
120
|
+
* @returns Promise that resolves to the data core ID
|
|
121
|
+
*/
|
|
80
122
|
async getDataCoreId() {
|
|
81
123
|
if (this.dataCoreId) {
|
|
82
124
|
return this.dataCoreId;
|
|
@@ -94,6 +136,10 @@ class FlowcoreDataSource {
|
|
|
94
136
|
}
|
|
95
137
|
return this.dataCoreId;
|
|
96
138
|
}
|
|
139
|
+
/**
|
|
140
|
+
* Gets the flow type ID, translating from name if necessary
|
|
141
|
+
* @returns Promise that resolves to the flow type ID
|
|
142
|
+
*/
|
|
97
143
|
async getFlowTypeId() {
|
|
98
144
|
if (this.flowTypeId) {
|
|
99
145
|
return this.flowTypeId;
|
|
@@ -111,6 +157,11 @@ class FlowcoreDataSource {
|
|
|
111
157
|
}
|
|
112
158
|
return this.flowTypeId;
|
|
113
159
|
}
|
|
160
|
+
/**
|
|
161
|
+
* Gets the event type IDs, translating from names if necessary
|
|
162
|
+
* @returns Promise that resolves to an array of event type IDs
|
|
163
|
+
* @throws Error if an event type is not found
|
|
164
|
+
*/
|
|
114
165
|
async getEventTypeIds() {
|
|
115
166
|
if (this.eventTypeIds) {
|
|
116
167
|
return this.eventTypeIds;
|
|
@@ -135,6 +186,11 @@ class FlowcoreDataSource {
|
|
|
135
186
|
}
|
|
136
187
|
return this.eventTypeIds;
|
|
137
188
|
}
|
|
189
|
+
/**
|
|
190
|
+
* Gets all time buckets for the configured event types
|
|
191
|
+
* @param force - When true, forces a refresh of cached time buckets
|
|
192
|
+
* @returns Promise that resolves to an array of time bucket strings
|
|
193
|
+
*/
|
|
138
194
|
async getTimeBuckets(force = false) {
|
|
139
195
|
if (this.timeBuckets && !force) {
|
|
140
196
|
return this.timeBuckets;
|
|
@@ -154,6 +210,11 @@ class FlowcoreDataSource {
|
|
|
154
210
|
this.timeBuckets = timeBuckets;
|
|
155
211
|
return this.timeBuckets;
|
|
156
212
|
}
|
|
213
|
+
/**
|
|
214
|
+
* Gets the next time bucket after the specified time bucket
|
|
215
|
+
* @param timeBucket - The reference time bucket
|
|
216
|
+
* @returns Promise that resolves to the next time bucket, or null if none exists
|
|
217
|
+
*/
|
|
157
218
|
async getNextTimeBucket(timeBucket) {
|
|
158
219
|
const closestTimeBucket = await this.getClosestTimeBucket(timeBucket);
|
|
159
220
|
if (!closestTimeBucket) {
|
|
@@ -166,6 +227,13 @@ class FlowcoreDataSource {
|
|
|
166
227
|
}
|
|
167
228
|
return timeBuckets[index + 1] ?? null;
|
|
168
229
|
}
|
|
230
|
+
/**
|
|
231
|
+
* Gets the closest time bucket to the specified time bucket
|
|
232
|
+
* @param timeBucket - The reference time bucket
|
|
233
|
+
* @param getBefore - When true, gets the closest time bucket before the reference
|
|
234
|
+
* @returns Promise that resolves to the closest time bucket, or null if none exists
|
|
235
|
+
* @throws Error if the time bucket format is invalid
|
|
236
|
+
*/
|
|
169
237
|
async getClosestTimeBucket(timeBucket, getBefore = false) {
|
|
170
238
|
const timeBuckets = await this.getTimeBuckets();
|
|
171
239
|
if (!timeBucket.match(/^\d{14}$/)) {
|
|
@@ -178,6 +246,13 @@ class FlowcoreDataSource {
|
|
|
178
246
|
return (timeBuckets.find((t) => Number.parseFloat(t) >= Number.parseFloat(timeBucket)) ??
|
|
179
247
|
timeBuckets[timeBuckets.length - 1]);
|
|
180
248
|
}
|
|
249
|
+
/**
|
|
250
|
+
* Gets events from the data source starting from a specific state
|
|
251
|
+
* @param from - The state to start from (includes timeBucket and eventId)
|
|
252
|
+
* @param amount - Maximum number of events to retrieve
|
|
253
|
+
* @param toEventId - Optional ID to stop at when retrieving events
|
|
254
|
+
* @returns Promise that resolves to an array of Flowcore events
|
|
255
|
+
*/
|
|
181
256
|
async getEvents(from, amount, toEventId) {
|
|
182
257
|
const eventTypeIds = await this.getEventTypeIds();
|
|
183
258
|
const result = await this.flowcoreClient.execute(new mod_js_1.EventListCommand({
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"no-op-logger.d.ts","sourceRoot":"","sources":["../../src/data-pump/no-op-logger.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAEhD,eAAO,MAAM,UAAU,EAAE,cAKxB,CAAA"}
|
|
@@ -28,7 +28,7 @@ const mod_js_1 = require("../deps/jsr.io/@flowcore/sdk/1.26.3/src/mod.js");
|
|
|
28
28
|
const Nats = __importStar(require("nats"));
|
|
29
29
|
const rxjs_1 = require("rxjs");
|
|
30
30
|
const data_source_js_1 = require("./data-source.js");
|
|
31
|
-
const
|
|
31
|
+
const no_op_logger_js_1 = require("./no-op-logger.js");
|
|
32
32
|
const DEFAULT_TIMEOUT_MS = 20_000;
|
|
33
33
|
class FlowcoreNotifier {
|
|
34
34
|
constructor(options) {
|
|
@@ -147,7 +147,7 @@ class FlowcoreNotifier {
|
|
|
147
147
|
dataCore: this.dataSource.dataCore,
|
|
148
148
|
flowType: this.dataSource.flowType,
|
|
149
149
|
}, {
|
|
150
|
-
logger: this.options.logger ??
|
|
150
|
+
logger: this.options.logger ?? no_op_logger_js_1.noOpLogger,
|
|
151
151
|
reconnectInterval: 1000,
|
|
152
152
|
});
|
|
153
153
|
await this.notificationClient.connect();
|
package/script/mod.d.ts
CHANGED
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
import "./_dnt.polyfills.js";
|
|
2
|
-
export * from "./lib/data-pump.js";
|
|
3
|
-
export * from "./lib/data-pump-create.js";
|
|
4
2
|
export * from "./data-pump/data-pump.js";
|
|
5
3
|
export * from "./data-pump/data-source.js";
|
|
6
4
|
export * from "./data-pump/metrics.js";
|
|
7
5
|
export * from "./data-pump/types.js";
|
|
6
|
+
export * from "./data-pump/no-op-logger.js";
|
|
8
7
|
//# sourceMappingURL=mod.d.ts.map
|
package/script/mod.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mod.d.ts","sourceRoot":"","sources":["../src/mod.ts"],"names":[],"mappings":"AAAA,OAAO,qBAAqB,CAAC;AAC7B,cAAc,
|
|
1
|
+
{"version":3,"file":"mod.d.ts","sourceRoot":"","sources":["../src/mod.ts"],"names":[],"mappings":"AAAA,OAAO,qBAAqB,CAAC;AAC7B,cAAc,0BAA0B,CAAA;AACxC,cAAc,4BAA4B,CAAA;AAC1C,cAAc,wBAAwB,CAAA;AACtC,cAAc,sBAAsB,CAAA;AACpC,cAAc,6BAA6B,CAAA"}
|
package/script/mod.js
CHANGED
|
@@ -15,9 +15,8 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
15
15
|
};
|
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
17
|
require("./_dnt.polyfills.js");
|
|
18
|
-
__exportStar(require("./lib/data-pump.js"), exports);
|
|
19
|
-
__exportStar(require("./lib/data-pump-create.js"), exports);
|
|
20
18
|
__exportStar(require("./data-pump/data-pump.js"), exports);
|
|
21
19
|
__exportStar(require("./data-pump/data-source.js"), exports);
|
|
22
20
|
__exportStar(require("./data-pump/metrics.js"), exports);
|
|
23
21
|
__exportStar(require("./data-pump/types.js"), exports);
|
|
22
|
+
__exportStar(require("./data-pump/no-op-logger.js"), exports);
|
|
@@ -1,46 +0,0 @@
|
|
|
1
|
-
import { type FlowcoreEvent } from "../deps/jsr.io/@flowcore/sdk/1.26.3/src/mod.js";
|
|
2
|
-
import { DataPump, type DataPumpStateManager, type Logger } from "./data-pump.js";
|
|
3
|
-
export declare const noOpLogger: Logger;
|
|
4
|
-
interface DataPumpClientOptionsAuthOidcClient {
|
|
5
|
-
oidcClient: {
|
|
6
|
-
getToken: () => Promise<{
|
|
7
|
-
accessToken: string;
|
|
8
|
-
}>;
|
|
9
|
-
};
|
|
10
|
-
}
|
|
11
|
-
interface DataPumpClientOptionsAuthApiKey {
|
|
12
|
-
apiKey: string;
|
|
13
|
-
apiKeyId: string;
|
|
14
|
-
}
|
|
15
|
-
type DataPumpClientOptionsAuth = DataPumpClientOptionsAuthOidcClient | DataPumpClientOptionsAuthApiKey;
|
|
16
|
-
/**
|
|
17
|
-
* Create data pump client options
|
|
18
|
-
*/
|
|
19
|
-
export interface DataPumpClientOptions {
|
|
20
|
-
logger?: Logger;
|
|
21
|
-
auth: DataPumpClientOptionsAuth;
|
|
22
|
-
dataSource: {
|
|
23
|
-
tenant: string;
|
|
24
|
-
dataCore: string;
|
|
25
|
-
flowType: string;
|
|
26
|
-
eventTypes: string[];
|
|
27
|
-
};
|
|
28
|
-
buffer?: {
|
|
29
|
-
size?: number;
|
|
30
|
-
threshold?: number;
|
|
31
|
-
maxRedeliveryCount?: number;
|
|
32
|
-
achknowledgeTimeoutMs?: number;
|
|
33
|
-
};
|
|
34
|
-
stateManager?: DataPumpStateManager;
|
|
35
|
-
processor?: {
|
|
36
|
-
concurrency?: number;
|
|
37
|
-
onEvents: (events: FlowcoreEvent[]) => Promise<boolean> | boolean;
|
|
38
|
-
onFailedEvents?: (events: FlowcoreEvent[]) => Promise<void> | void;
|
|
39
|
-
onDone?: () => Promise<void> | void;
|
|
40
|
-
};
|
|
41
|
-
stopAt?: Date;
|
|
42
|
-
natsServers?: string[];
|
|
43
|
-
}
|
|
44
|
-
export declare function createDataPump(options: DataPumpClientOptions): DataPump;
|
|
45
|
-
export {};
|
|
46
|
-
//# sourceMappingURL=data-pump-create.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"data-pump-create.d.ts","sourceRoot":"","sources":["../../src/lib/data-pump-create.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,KAAK,aAAa,EAAE,MAAM,gDAAgD,CAAA;AACnG,OAAO,EAAE,QAAQ,EAAE,KAAK,oBAAoB,EAAE,KAAK,MAAM,EAAE,MAAM,gBAAgB,CAAA;AAIjF,eAAO,MAAM,UAAU,EAAE,MAKxB,CAAA;AAED,UAAU,mCAAmC;IAC3C,UAAU,EAAE;QACV,QAAQ,EAAE,MAAM,OAAO,CAAC;YAAE,WAAW,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KACjD,CAAA;CACF;AAED,UAAU,+BAA+B;IACvC,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;CACjB;AAED,KAAK,yBAAyB,GAAG,mCAAmC,GAAG,+BAA+B,CAAA;AAEtG;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,IAAI,EAAE,yBAAyB,CAAA;IAC/B,UAAU,EAAE;QACV,MAAM,EAAE,MAAM,CAAA;QACd,QAAQ,EAAE,MAAM,CAAA;QAChB,QAAQ,EAAE,MAAM,CAAA;QAChB,UAAU,EAAE,MAAM,EAAE,CAAA;KACrB,CAAA;IACD,MAAM,CAAC,EAAE;QACP,IAAI,CAAC,EAAE,MAAM,CAAA;QACb,SAAS,CAAC,EAAE,MAAM,CAAA;QAClB,kBAAkB,CAAC,EAAE,MAAM,CAAA;QAC3B,qBAAqB,CAAC,EAAE,MAAM,CAAA;KAC/B,CAAA;IACD,YAAY,CAAC,EAAE,oBAAoB,CAAA;IACnC,SAAS,CAAC,EAAE;QACV,WAAW,CAAC,EAAE,MAAM,CAAA;QACpB,QAAQ,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAA;QACjE,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;QAClE,MAAM,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;KACpC,CAAA;IACD,MAAM,CAAC,EAAE,IAAI,CAAA;IACb,WAAW,CAAC,EAAE,MAAM,EAAE,CAAA;CACvB;AAED,wBAAgB,cAAc,CAAC,OAAO,EAAE,qBAAqB,GAAG,QAAQ,CA0DvE"}
|
|
@@ -1,64 +0,0 @@
|
|
|
1
|
-
import { FlowcoreClient } from "../deps/jsr.io/@flowcore/sdk/1.26.3/src/mod.js";
|
|
2
|
-
import { DataPump } from "./data-pump.js";
|
|
3
|
-
import { WebSocketNotifier } from "./ws-notifier.js";
|
|
4
|
-
import { NatsNotifier } from "./nats-notifier.js";
|
|
5
|
-
export const noOpLogger = {
|
|
6
|
-
debug: () => { },
|
|
7
|
-
info: () => { },
|
|
8
|
-
warn: () => { },
|
|
9
|
-
error: console.error,
|
|
10
|
-
};
|
|
11
|
-
export function createDataPump(options) {
|
|
12
|
-
let flowcoreClient;
|
|
13
|
-
if ("oidcClient" in options.auth) {
|
|
14
|
-
const authClient = options.auth.oidcClient;
|
|
15
|
-
flowcoreClient = new FlowcoreClient({
|
|
16
|
-
getBearerToken: async () => (await authClient.getToken()).accessToken ?? null,
|
|
17
|
-
});
|
|
18
|
-
}
|
|
19
|
-
else {
|
|
20
|
-
flowcoreClient = new FlowcoreClient({
|
|
21
|
-
apiKey: options.auth.apiKey,
|
|
22
|
-
apiKeyId: options.auth.apiKeyId,
|
|
23
|
-
});
|
|
24
|
-
}
|
|
25
|
-
const stateManager = options.stateManager ?? {
|
|
26
|
-
getState: () => null,
|
|
27
|
-
setState: () => { },
|
|
28
|
-
};
|
|
29
|
-
const notifier = options.natsServers
|
|
30
|
-
? new NatsNotifier({
|
|
31
|
-
servers: options.natsServers,
|
|
32
|
-
})
|
|
33
|
-
: new WebSocketNotifier({
|
|
34
|
-
auth: options.auth,
|
|
35
|
-
dataSource: options.dataSource,
|
|
36
|
-
logger: options.logger,
|
|
37
|
-
});
|
|
38
|
-
const bufferSize = options.buffer?.size ?? 1000;
|
|
39
|
-
const buffer = {
|
|
40
|
-
size: bufferSize,
|
|
41
|
-
threshold: options.buffer?.threshold ?? Math.round(bufferSize * 0.1),
|
|
42
|
-
maxRedeliveryCount: options.buffer?.maxRedeliveryCount ?? 3,
|
|
43
|
-
achknowledgeTimeoutMs: options.buffer?.achknowledgeTimeoutMs ?? 10_000,
|
|
44
|
-
};
|
|
45
|
-
const processor = options.processor
|
|
46
|
-
? {
|
|
47
|
-
concurrency: options.processor.concurrency ?? 1,
|
|
48
|
-
onEvents: options.processor.onEvents,
|
|
49
|
-
onFailedEvents: options.processor.onFailedEvents,
|
|
50
|
-
onDone: options.processor.onDone,
|
|
51
|
-
}
|
|
52
|
-
: undefined;
|
|
53
|
-
const dataPump = new DataPump({
|
|
54
|
-
flowcoreClient,
|
|
55
|
-
stateManager,
|
|
56
|
-
dataSource: options.dataSource,
|
|
57
|
-
buffer,
|
|
58
|
-
processor,
|
|
59
|
-
notifier: notifier.getNotifier(),
|
|
60
|
-
logger: options.logger,
|
|
61
|
-
stopAt: options.stopAt,
|
|
62
|
-
});
|
|
63
|
-
return dataPump;
|
|
64
|
-
}
|
package/esm/lib/data-pump.d.ts
DELETED
|
@@ -1,80 +0,0 @@
|
|
|
1
|
-
import { type FlowcoreClient, type FlowcoreEvent } from "../deps/jsr.io/@flowcore/sdk/1.26.3/src/mod.js";
|
|
2
|
-
export interface Logger {
|
|
3
|
-
debug(message: string, ...meta: unknown[]): void;
|
|
4
|
-
info(message: string, ...meta: unknown[]): void;
|
|
5
|
-
warn(message: string, ...meta: unknown[]): void;
|
|
6
|
-
error(message: string | Error, ...meta: unknown[]): void;
|
|
7
|
-
}
|
|
8
|
-
interface DataPumpOptions {
|
|
9
|
-
flowcoreClient: FlowcoreClient;
|
|
10
|
-
stateManager: DataPumpStateManager;
|
|
11
|
-
logger?: Logger;
|
|
12
|
-
notifier: DataPumpNotifier;
|
|
13
|
-
dataSource: {
|
|
14
|
-
tenant: string;
|
|
15
|
-
dataCore: string;
|
|
16
|
-
flowType: string;
|
|
17
|
-
eventTypes: string[];
|
|
18
|
-
};
|
|
19
|
-
buffer: {
|
|
20
|
-
size: number;
|
|
21
|
-
threshold: number;
|
|
22
|
-
maxRedeliveryCount: number;
|
|
23
|
-
achknowledgeTimeoutMs: number;
|
|
24
|
-
};
|
|
25
|
-
processor?: {
|
|
26
|
-
concurrency: number;
|
|
27
|
-
onEvents: (events: FlowcoreEvent[]) => Promise<boolean> | boolean;
|
|
28
|
-
onFailedEvents?: (events: FlowcoreEvent[]) => Promise<void> | void;
|
|
29
|
-
onDone?: () => Promise<void> | void;
|
|
30
|
-
};
|
|
31
|
-
stopAt?: Date;
|
|
32
|
-
}
|
|
33
|
-
export interface DataPumpState {
|
|
34
|
-
timeBucket: string;
|
|
35
|
-
eventId?: string;
|
|
36
|
-
}
|
|
37
|
-
export interface DataPumpStateManager {
|
|
38
|
-
getState: () => Promise<DataPumpState | null> | DataPumpState | null;
|
|
39
|
-
setState?: (state: DataPumpState) => Promise<void> | void;
|
|
40
|
-
stopAt?: () => Date;
|
|
41
|
-
}
|
|
42
|
-
export interface DataPumpNotifier {
|
|
43
|
-
wait(dataSource: DataPumpOptions["dataSource"] & {
|
|
44
|
-
dataCoreId: string;
|
|
45
|
-
}, signal?: AbortSignal): Promise<void>;
|
|
46
|
-
}
|
|
47
|
-
export declare class DataPump {
|
|
48
|
-
private options;
|
|
49
|
-
private logger?;
|
|
50
|
-
private state;
|
|
51
|
-
private _timeBuckets;
|
|
52
|
-
private buffer;
|
|
53
|
-
private bufferWaiter;
|
|
54
|
-
private waiter;
|
|
55
|
-
private running;
|
|
56
|
-
private notifierAbortController?;
|
|
57
|
-
private dataCoreId?;
|
|
58
|
-
private stopAt?;
|
|
59
|
-
constructor(options: DataPumpOptions);
|
|
60
|
-
start(): Promise<void>;
|
|
61
|
-
stop(): void;
|
|
62
|
-
private proceccor;
|
|
63
|
-
private eventBufferLoop;
|
|
64
|
-
private reOpen;
|
|
65
|
-
private updateState;
|
|
66
|
-
private get timeBuckets();
|
|
67
|
-
private getClosestTimeBucket;
|
|
68
|
-
private updateTimeBuckets;
|
|
69
|
-
private pullInner;
|
|
70
|
-
private acknowledgeInner;
|
|
71
|
-
private waitForNotifier;
|
|
72
|
-
private waitForEvents;
|
|
73
|
-
private waitForBufferThreshold;
|
|
74
|
-
private getNowTimeBucket;
|
|
75
|
-
private getDataCoreId;
|
|
76
|
-
pull(amount: number): Promise<FlowcoreEvent[]>;
|
|
77
|
-
acknowledge(eventIds: string[]): Promise<void>;
|
|
78
|
-
}
|
|
79
|
-
export {};
|
|
80
|
-
//# sourceMappingURL=data-pump.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"data-pump.d.ts","sourceRoot":"","sources":["../../src/lib/data-pump.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,KAAK,cAAc,EACnB,KAAK,aAAa,EACnB,MAAM,gDAAgD,CAAA;AAiBvD,MAAM,WAAW,MAAM;IACrB,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;IAChD,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;IAC/C,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;IAC/C,KAAK,CAAC,OAAO,EAAE,MAAM,GAAG,KAAK,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;CACzD;AAED,UAAU,eAAe;IACvB,cAAc,EAAE,cAAc,CAAA;IAC9B,YAAY,EAAE,oBAAoB,CAAA;IAClC,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,QAAQ,EAAE,gBAAgB,CAAA;IAC1B,UAAU,EAAE;QACV,MAAM,EAAE,MAAM,CAAA;QACd,QAAQ,EAAE,MAAM,CAAA;QAChB,QAAQ,EAAE,MAAM,CAAA;QAChB,UAAU,EAAE,MAAM,EAAE,CAAA;KACrB,CAAA;IACD,MAAM,EAAE;QACN,IAAI,EAAE,MAAM,CAAA;QACZ,SAAS,EAAE,MAAM,CAAA;QACjB,kBAAkB,EAAE,MAAM,CAAA;QAC1B,qBAAqB,EAAE,MAAM,CAAA;KAC9B,CAAA;IACD,SAAS,CAAC,EAAE;QACV,WAAW,EAAE,MAAM,CAAA;QACnB,QAAQ,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAA;QACjE,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;QAClE,MAAM,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;KACpC,CAAA;IACD,MAAM,CAAC,EAAE,IAAI,CAAA;CACd;AAED,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,MAAM,CAAA;IAClB,OAAO,CAAC,EAAE,MAAM,CAAA;CACjB;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,EAAE,MAAM,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,aAAa,GAAG,IAAI,CAAA;IACpE,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,aAAa,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;IACzD,MAAM,CAAC,EAAE,MAAM,IAAI,CAAA;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,CAAC,UAAU,EAAE,eAAe,CAAC,YAAY,CAAC,GAAG;QAAE,UAAU,EAAE,MAAM,CAAA;KAAE,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;CAC9G;AAED,qBAAa,QAAQ;IAgBP,OAAO,CAAC,OAAO;IAf3B,OAAO,CAAC,MAAM,CAAC,CAAQ;IACvB,OAAO,CAAC,KAAK,CAAe;IAC5B,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,MAAM,CAA4B;IAC1C,OAAO,CAAC,YAAY,CAA8C;IAClE,OAAO,CAAC,MAAM,CAA8C;IAC5D,OAAO,CAAC,OAAO,CAAQ;IACvB,OAAO,CAAC,uBAAuB,CAAC,CAAiB;IACjD,OAAO,CAAC,UAAU,CAAC,CAAQ;IAC3B,OAAO,CAAC,MAAM,CAAC,CAId;gBAEmB,OAAO,EAAE,eAAe;IAO/B,KAAK;IAiDX,IAAI;YAWG,SAAS;YAqBT,eAAe;YAsGf,MAAM;IAsCpB,OAAO,CAAC,WAAW;IAenB,OAAO,KAAK,WAAW,GAMtB;IAED,OAAO,CAAC,oBAAoB;YAgBd,iBAAiB;YAoBjB,SAAS;YA4CT,gBAAgB;YA2BhB,eAAe;YAUf,aAAa;YAab,sBAAsB;IAepC,OAAO,CAAC,gBAAgB;YAIV,aAAa;IAcpB,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IAO9C,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;CAMtD"}
|