@devrev/airsync-sdk 2.0.0-beta.4 → 2.0.0-beta.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/MIGRATION.md +1028 -0
- package/dist/attachments-streaming/attachments-streaming-pool.d.ts +6 -0
- package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map +1 -1
- package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts +2 -2
- package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts.map +1 -1
- package/dist/attachments-streaming/attachments-streaming-pool.js +47 -21
- package/dist/common/constants.d.ts +3 -5
- package/dist/common/constants.d.ts.map +1 -1
- package/dist/common/constants.js +4 -6
- package/dist/common/helpers.d.ts +3 -21
- package/dist/common/helpers.d.ts.map +1 -1
- package/dist/common/helpers.js +6 -15
- package/dist/http/client.d.ts.map +1 -1
- package/dist/http/client.js +4 -8
- package/dist/index.d.ts +11 -11
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +11 -18
- package/dist/logger/logger.d.ts +8 -63
- package/dist/logger/logger.d.ts.map +1 -1
- package/dist/logger/logger.interfaces.d.ts +0 -1
- package/dist/logger/logger.interfaces.d.ts.map +1 -1
- package/dist/logger/logger.js +14 -93
- package/dist/mappers/mappers.d.ts +2 -40
- package/dist/mappers/mappers.d.ts.map +1 -1
- package/dist/mappers/mappers.interfaces.d.ts +35 -154
- package/dist/mappers/mappers.interfaces.d.ts.map +1 -1
- package/dist/mappers/mappers.interfaces.js +1 -10
- package/dist/mappers/mappers.js +2 -40
- package/dist/multithreading/adapters/base-adapter.d.ts +19 -36
- package/dist/multithreading/adapters/base-adapter.d.ts.map +1 -1
- package/dist/multithreading/adapters/base-adapter.js +73 -71
- package/dist/multithreading/adapters/extraction-adapter.d.ts +12 -24
- package/dist/multithreading/adapters/extraction-adapter.d.ts.map +1 -1
- package/dist/multithreading/adapters/extraction-adapter.js +199 -200
- package/dist/multithreading/adapters/loading-adapter.d.ts +1 -6
- package/dist/multithreading/adapters/loading-adapter.d.ts.map +1 -1
- package/dist/multithreading/adapters/loading-adapter.helpers.d.ts +2 -12
- package/dist/multithreading/adapters/loading-adapter.helpers.d.ts.map +1 -1
- package/dist/multithreading/adapters/loading-adapter.helpers.js +2 -12
- package/dist/multithreading/adapters/loading-adapter.js +292 -314
- package/dist/multithreading/emit.d.ts +3 -2
- package/dist/multithreading/emit.d.ts.map +1 -1
- package/dist/multithreading/emit.js +3 -5
- package/dist/multithreading/process-task.d.ts +2 -12
- package/dist/multithreading/process-task.d.ts.map +1 -1
- package/dist/multithreading/process-task.js +45 -59
- package/dist/multithreading/spawn/spawn.d.ts +3 -9
- package/dist/multithreading/spawn/spawn.d.ts.map +1 -1
- package/dist/multithreading/spawn/spawn.helpers.d.ts +6 -25
- package/dist/multithreading/spawn/spawn.helpers.d.ts.map +1 -1
- package/dist/multithreading/spawn/spawn.helpers.js +7 -49
- package/dist/multithreading/spawn/spawn.js +18 -33
- package/dist/multithreading/worker.js +2 -4
- package/dist/repo/repo.d.ts +10 -0
- package/dist/repo/repo.d.ts.map +1 -1
- package/dist/repo/repo.helpers.d.ts +6 -0
- package/dist/repo/repo.helpers.d.ts.map +1 -0
- package/dist/repo/repo.helpers.js +16 -0
- package/dist/repo/repo.interfaces.d.ts +4 -16
- package/dist/repo/repo.interfaces.d.ts.map +1 -1
- package/dist/repo/repo.js +23 -8
- package/dist/state/base-state.d.ts +14 -36
- package/dist/state/base-state.d.ts.map +1 -1
- package/dist/state/base-state.js +16 -38
- package/dist/state/extraction-state.d.ts +4 -19
- package/dist/state/extraction-state.d.ts.map +1 -1
- package/dist/state/extraction-state.js +6 -23
- package/dist/state/install-initial-domain-mapping.d.ts +1 -1
- package/dist/state/install-initial-domain-mapping.d.ts.map +1 -1
- package/dist/state/install-initial-domain-mapping.js +1 -3
- package/dist/state/loading-state.d.ts +1 -11
- package/dist/state/loading-state.d.ts.map +1 -1
- package/dist/state/loading-state.js +2 -12
- package/dist/state/state.d.ts +2 -2
- package/dist/state/state.d.ts.map +1 -1
- package/dist/state/state.interfaces.d.ts +10 -24
- package/dist/state/state.interfaces.d.ts.map +1 -1
- package/dist/state/state.interfaces.js +9 -11
- package/dist/state/state.js +3 -3
- package/dist/state/time-value-resolver.d.ts +7 -36
- package/dist/state/time-value-resolver.d.ts.map +1 -1
- package/dist/state/time-value-resolver.js +7 -40
- package/dist/testing/mock-event.d.ts +2 -10
- package/dist/testing/mock-event.d.ts.map +1 -1
- package/dist/testing/mock-event.js +3 -11
- package/dist/testing/mock-server.d.ts +4 -27
- package/dist/testing/mock-server.d.ts.map +1 -1
- package/dist/testing/mock-server.interfaces.d.ts +10 -35
- package/dist/testing/mock-server.interfaces.d.ts.map +1 -1
- package/dist/testing/mock-server.js +19 -36
- package/dist/types/common.d.ts +1 -12
- package/dist/types/common.d.ts.map +1 -1
- package/dist/types/common.js +1 -6
- package/dist/types/external-domain-metadata.d.ts +30 -171
- package/dist/types/external-domain-metadata.d.ts.map +1 -1
- package/dist/types/extraction.d.ts +35 -120
- package/dist/types/extraction.d.ts.map +1 -1
- package/dist/types/extraction.js +5 -31
- package/dist/types/loading.d.ts.map +1 -1
- package/dist/types/workers.d.ts +18 -116
- package/dist/types/workers.d.ts.map +1 -1
- package/dist/types/workers.js +0 -6
- package/dist/uploader/uploader.d.ts +6 -58
- package/dist/uploader/uploader.d.ts.map +1 -1
- package/dist/uploader/uploader.helpers.d.ts +4 -27
- package/dist/uploader/uploader.helpers.d.ts.map +1 -1
- package/dist/uploader/uploader.helpers.js +64 -30
- package/dist/uploader/uploader.interfaces.d.ts +15 -25
- package/dist/uploader/uploader.interfaces.d.ts.map +1 -1
- package/dist/uploader/uploader.interfaces.js +8 -0
- package/dist/uploader/uploader.js +18 -84
- package/package.json +6 -4
|
@@ -9,6 +9,12 @@ export declare class AttachmentsStreamingPool<ConnectorState> {
|
|
|
9
9
|
private totalProcessedCount;
|
|
10
10
|
private readonly PROGRESS_REPORT_INTERVAL;
|
|
11
11
|
constructor({ adapter, attachments, batchSize, stream, }: AttachmentsStreamingPoolParams<ConnectorState>);
|
|
12
|
+
private recordProcessedAttachment;
|
|
13
|
+
/**
|
|
14
|
+
* Backfills `status` on entries recorded before the field existed (those
|
|
15
|
+
* only ever held successes); state from >= v1.15.2 needs no shape migration.
|
|
16
|
+
*/
|
|
17
|
+
private backfillProcessedAttachmentStatus;
|
|
12
18
|
private updateProgress;
|
|
13
19
|
streamAll(): Promise<ProcessAttachmentReturnType>;
|
|
14
20
|
startPoolStreaming(): Promise<void>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"attachments-streaming-pool.d.ts","sourceRoot":"","sources":["../../src/attachments-streaming/attachments-streaming-pool.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"attachments-streaming-pool.d.ts","sourceRoot":"","sources":["../../src/attachments-streaming/attachments-streaming-pool.ts"],"names":[],"mappings":"AAOA,OAAO,EAEL,2BAA2B,EAC5B,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,8BAA8B,EAAE,MAAM,yCAAyC,CAAC;AAEzF,qBAAa,wBAAwB,CAAC,cAAc;IAClD,OAAO,CAAC,OAAO,CAAoC;IACnD,OAAO,CAAC,WAAW,CAAyB;IAC5C,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,KAAK,CAAqB;IAClC,OAAO,CAAC,MAAM,CAA4C;IAE1D,OAAO,CAAC,mBAAmB,CAAa;IACxC,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAM;gBAEnC,EACV,OAAO,EACP,WAAW,EACX,SAAc,EACd,MAAM,GACP,EAAE,8BAA8B,CAAC,cAAc,CAAC;IAQjD,OAAO,CAAC,yBAAyB;IAyBjC;;;OAGG;IACH,OAAO,CAAC,iCAAiC;YAS3B,cAAc;IAUtB,SAAS,IAAI,OAAO,CAAC,2BAA2B,CAAC;IA6CjD,kBAAkB;CAyFzB"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { ExternalSystemAttachmentStreamingFunction } from '../types/extraction';
|
|
2
|
-
import { NormalizedAttachment } from '../repo/repo.interfaces';
|
|
3
1
|
import { ExtractionAdapter } from '../multithreading/adapters/extraction-adapter';
|
|
2
|
+
import { NormalizedAttachment } from '../repo/repo.interfaces';
|
|
3
|
+
import { ExternalSystemAttachmentStreamingFunction } from '../types/extraction';
|
|
4
4
|
export interface AttachmentsStreamingPoolParams<ConnectorState> {
|
|
5
5
|
adapter: ExtractionAdapter<ConnectorState>;
|
|
6
6
|
attachments: NormalizedAttachment[];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"attachments-streaming-pool.interfaces.d.ts","sourceRoot":"","sources":["../../src/attachments-streaming/attachments-streaming-pool.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"attachments-streaming-pool.interfaces.d.ts","sourceRoot":"","sources":["../../src/attachments-streaming/attachments-streaming-pool.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,+CAA+C,CAAC;AAClF,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,yCAAyC,EAAE,MAAM,qBAAqB,CAAC;AAEhF,MAAM,WAAW,8BAA8B,CAAC,cAAc;IAC5D,OAAO,EAAE,iBAAiB,CAAC,cAAc,CAAC,CAAC;IAC3C,WAAW,EAAE,oBAAoB,EAAE,CAAC;IACpC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,yCAAyC,CAAC;CACnD"}
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.AttachmentsStreamingPool = void 0;
|
|
4
|
+
const helpers_1 = require("../common/helpers");
|
|
5
|
+
const state_interfaces_1 = require("../state/state.interfaces");
|
|
4
6
|
class AttachmentsStreamingPool {
|
|
5
7
|
constructor({ adapter, attachments, batchSize = 10, stream, }) {
|
|
6
8
|
this.totalProcessedCount = 0;
|
|
@@ -11,14 +13,39 @@ class AttachmentsStreamingPool {
|
|
|
11
13
|
this.delay = undefined;
|
|
12
14
|
this.stream = stream;
|
|
13
15
|
}
|
|
16
|
+
recordProcessedAttachment(attachment, status) {
|
|
17
|
+
var _a;
|
|
18
|
+
const attachmentsMetadata = (_a = this.adapter.sdkState.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata;
|
|
19
|
+
if (!(attachmentsMetadata === null || attachmentsMetadata === void 0 ? void 0 : attachmentsMetadata.lastProcessedAttachmentsIdsList)) {
|
|
20
|
+
return;
|
|
21
|
+
}
|
|
22
|
+
const processedAttachmentsIdsList = attachmentsMetadata.lastProcessedAttachmentsIdsList;
|
|
23
|
+
const alreadyRecorded = processedAttachmentsIdsList.some((it) => it.id === attachment.id && it.parent_id === attachment.parent_id);
|
|
24
|
+
if (!alreadyRecorded) {
|
|
25
|
+
processedAttachmentsIdsList.push({
|
|
26
|
+
id: attachment.id,
|
|
27
|
+
parent_id: attachment.parent_id,
|
|
28
|
+
status,
|
|
29
|
+
});
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Backfills `status` on entries recorded before the field existed (those
|
|
34
|
+
* only ever held successes); state from >= v1.15.2 needs no shape migration.
|
|
35
|
+
*/
|
|
36
|
+
backfillProcessedAttachmentStatus(attachments) {
|
|
37
|
+
return attachments.map((it) => {
|
|
38
|
+
var _a;
|
|
39
|
+
return (Object.assign(Object.assign({}, it), { status: (_a = it.status) !== null && _a !== void 0 ? _a : state_interfaces_1.ProcessedAttachmentStatus.Success }));
|
|
40
|
+
});
|
|
41
|
+
}
|
|
14
42
|
async updateProgress() {
|
|
15
43
|
this.totalProcessedCount++;
|
|
16
44
|
if (this.totalProcessedCount % this.PROGRESS_REPORT_INTERVAL === 0) {
|
|
17
45
|
console.info(`Processed ${this.totalProcessedCount} attachments so far.`);
|
|
18
|
-
//
|
|
19
|
-
//
|
|
20
|
-
|
|
21
|
-
await new Promise((resolve) => setImmediate(resolve));
|
|
46
|
+
// Let a pending soft-timeout message (WorkerMessageExit) be delivered so
|
|
47
|
+
// adapter.isTimeout can flip before the next batch is processed.
|
|
48
|
+
await (0, helpers_1.yieldToEventLoop)();
|
|
22
49
|
}
|
|
23
50
|
}
|
|
24
51
|
async streamAll() {
|
|
@@ -28,53 +55,52 @@ class AttachmentsStreamingPool {
|
|
|
28
55
|
console.error(error);
|
|
29
56
|
return { error };
|
|
30
57
|
}
|
|
31
|
-
//
|
|
32
|
-
// If no such list exists, create an empty one.
|
|
58
|
+
// Attachments processed (successfully or not) by a previous, possibly incomplete, run.
|
|
33
59
|
if (!this.adapter.sdkState.toDevRev.attachmentsMetadata
|
|
34
60
|
.lastProcessedAttachmentsIdsList) {
|
|
35
61
|
this.adapter.sdkState.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList =
|
|
36
62
|
[];
|
|
37
63
|
}
|
|
38
|
-
|
|
64
|
+
this.adapter.sdkState.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList =
|
|
65
|
+
this.backfillProcessedAttachmentStatus(this.adapter.sdkState.toDevRev.attachmentsMetadata
|
|
66
|
+
.lastProcessedAttachmentsIdsList);
|
|
39
67
|
const initialBatchSize = Math.min(this.batchSize, this.attachments.length);
|
|
40
68
|
const initialPromises = [];
|
|
41
69
|
for (let i = 0; i < initialBatchSize; i++) {
|
|
42
70
|
initialPromises.push(this.startPoolStreaming());
|
|
43
71
|
}
|
|
44
|
-
|
|
45
|
-
|
|
72
|
+
await Promise.race([
|
|
73
|
+
Promise.all(initialPromises),
|
|
74
|
+
this.adapter.timeoutSignal,
|
|
75
|
+
]);
|
|
46
76
|
if (this.delay) {
|
|
47
77
|
return { delay: this.delay };
|
|
48
78
|
}
|
|
49
79
|
return {};
|
|
50
80
|
}
|
|
51
81
|
async startPoolStreaming() {
|
|
52
|
-
var _a
|
|
53
|
-
// Process attachments until the attachments array is empty
|
|
82
|
+
var _a;
|
|
54
83
|
while (this.attachments.length > 0) {
|
|
55
|
-
// If delay is set, stop streaming
|
|
56
84
|
if (this.delay) {
|
|
57
85
|
break;
|
|
58
86
|
}
|
|
59
|
-
// If timeout is set, stop streaming
|
|
60
87
|
if (this.adapter.isTimeout) {
|
|
61
88
|
console.log('Timeout detected while streaming attachments. Stopping streaming.');
|
|
62
89
|
break;
|
|
63
90
|
}
|
|
64
|
-
// Check if we can process next attachment
|
|
65
91
|
const attachment = this.attachments.shift();
|
|
66
92
|
if (!attachment) {
|
|
67
|
-
break;
|
|
93
|
+
break;
|
|
68
94
|
}
|
|
69
95
|
if (this.adapter.sdkState.toDevRev &&
|
|
70
96
|
((_a = this.adapter.sdkState.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList) === null || _a === void 0 ? void 0 : _a.some((it) => it.id == attachment.id && it.parent_id == attachment.parent_id))) {
|
|
71
|
-
continue; //
|
|
97
|
+
continue; // Already processed in a previous run
|
|
72
98
|
}
|
|
73
99
|
try {
|
|
74
100
|
const response = await this.adapter.processAttachment(attachment, this.stream);
|
|
75
|
-
//
|
|
101
|
+
// Rate limit hit
|
|
76
102
|
if (response === null || response === void 0 ? void 0 : response.delay) {
|
|
77
|
-
this.delay = response.delay;
|
|
103
|
+
this.delay = response.delay;
|
|
78
104
|
return;
|
|
79
105
|
}
|
|
80
106
|
if (response === null || response === void 0 ? void 0 : response.error) {
|
|
@@ -86,12 +112,12 @@ class AttachmentsStreamingPool {
|
|
|
86
112
|
? `and content_type ${attachment.content_type} `
|
|
87
113
|
: '';
|
|
88
114
|
console.warn(`Skipping attachment with ID ${attachment.id} with extension ${fileExtension} ${fileSizeInfo}${contentTypeInfo}due to error returned by the stream function`, response.error.message);
|
|
115
|
+
this.recordProcessedAttachment(attachment, state_interfaces_1.ProcessedAttachmentStatus.Failed);
|
|
89
116
|
await this.updateProgress();
|
|
90
117
|
continue;
|
|
91
118
|
}
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
(_d = this.adapter.sdkState.toDevRev) === null || _d === void 0 ? void 0 : _d.attachmentsMetadata.lastProcessedAttachmentsIdsList.push({ id: attachment.id, parent_id: attachment.parent_id });
|
|
119
|
+
if (!this.adapter.isTimeout) {
|
|
120
|
+
this.recordProcessedAttachment(attachment, state_interfaces_1.ProcessedAttachmentStatus.Success);
|
|
95
121
|
}
|
|
96
122
|
await this.updateProgress();
|
|
97
123
|
}
|
|
@@ -27,11 +27,9 @@ export declare const HARD_TIMEOUT_MULTIPLIER = 1.3;
|
|
|
27
27
|
export declare const MEMORY_LOG_INTERVAL: number;
|
|
28
28
|
export declare const DEFAULT_SLEEP_DELAY_MS: number;
|
|
29
29
|
/**
|
|
30
|
-
* Sentinel
|
|
31
|
-
*
|
|
32
|
-
*
|
|
33
|
-
* no real extraction timestamp can be earlier, preventing accidental overwrites
|
|
34
|
-
* of the boundary by subsequent syncs (e.g. reconciliation with absolute dates).
|
|
30
|
+
* Sentinel for an unbounded extraction timestamp (TimeValueType.UNBOUNDED),
|
|
31
|
+
* stored as workersOldest. The Unix epoch guarantees no real timestamp can be
|
|
32
|
+
* earlier, so subsequent syncs cannot accidentally overwrite the boundary.
|
|
35
33
|
*/
|
|
36
34
|
export declare const UNBOUNDED_DATE_TIME_VALUE = "1970-01-01T00:00:00.000Z";
|
|
37
35
|
//# sourceMappingURL=constants.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/common/constants.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/common/constants.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAIhD,eAAO,MAAM,8BAA8B,aAS1C,CAAC;AAEF,eAAO,MAAM,2BAA2B,aAKvC,CAAC;AAEF,eAAO,MAAM,mBAAmB,aAG/B,CAAC;AAEF,eAAO,MAAM,gCAAgC,aAI5C,CAAC;AAEF,eAAO,MAAM,6BAA6B,aAGzC,CAAC;AAEF,eAAO,MAAM,qBAAqB,aAGjC,CAAC;AAEF,eAAO,MAAM,+BAA+B,aAGzC,CAAC;AAEJ,eAAO,MAAM,4BAA4B,aAExC,CAAC;AAEF,eAAO,MAAM,oBAAoB,aAGhC,CAAC;AAEF,eAAO,MAAM,mBAAmB,OAAO,CAAC;AACxC,eAAO,MAAM,wBAAwB,QAAyB,CAAC;AAC/D,eAAO,MAAM,0BAA0B,MAAM,CAAC;AAC9C,eAAO,MAAM,oCAAoC,KAAK,CAAC;AAGvD,eAAO,MAAM,oBAAoB,SAAU,CAAC;AAE5C,eAAO,MAAM,0BAA0B,QAEtC,CAAC;AAEF,eAAO,MAAM,eAAe,EAAE,MAA0B,CAAC;AACzD,eAAO,MAAM,kBAAkB,uBAAuB,CAAC;AAEvD,oBAAY,uBAAuB;IACjC,wBAAwB,6BAA6B;IACrD,WAAW,gBAAgB;IAC3B,mBAAmB,wBAAwB;CAC5C;AAED,eAAO,MAAM,eAAe,KAAsB,CAAC;AAEnD,eAAO,MAAM,sBAAsB,QAAiB,CAAC;AACrD,eAAO,MAAM,uBAAuB,MAAM,CAAC;AAC3C,eAAO,MAAM,mBAAmB,QAAY,CAAC;AAE7C,eAAO,MAAM,sBAAsB,QAAgB,CAAC;AAEpD;;;;GAIG;AACH,eAAO,MAAM,yBAAyB,6BAA6B,CAAC"}
|
package/dist/common/constants.js
CHANGED
|
@@ -45,7 +45,7 @@ exports.STATEFUL_EVENT_TYPES = [
|
|
|
45
45
|
exports.ARTIFACT_BATCH_SIZE = 2000;
|
|
46
46
|
exports.MAX_DEVREV_ARTIFACT_SIZE = 2 * 1024 * 1024 * 1024; // 2GB
|
|
47
47
|
exports.MAX_DEVREV_FILENAME_LENGTH = 256;
|
|
48
|
-
exports.MAX_DEVREV_FILENAME_EXTENSION_LENGTH = 20;
|
|
48
|
+
exports.MAX_DEVREV_FILENAME_EXTENSION_LENGTH = 20;
|
|
49
49
|
// Max SQS message size is 250KB, we want to leave some room for the other data in the message
|
|
50
50
|
exports.MAX_EVENT_SIZE_BYTES = 200000;
|
|
51
51
|
// We want to leave some room for the other data in the message and process the rest of queued messages
|
|
@@ -64,10 +64,8 @@ exports.HARD_TIMEOUT_MULTIPLIER = 1.3;
|
|
|
64
64
|
exports.MEMORY_LOG_INTERVAL = 30 * 1000; // 30 seconds
|
|
65
65
|
exports.DEFAULT_SLEEP_DELAY_MS = 3 * 60 * 1000; // 3 minutes
|
|
66
66
|
/**
|
|
67
|
-
* Sentinel
|
|
68
|
-
*
|
|
69
|
-
*
|
|
70
|
-
* no real extraction timestamp can be earlier, preventing accidental overwrites
|
|
71
|
-
* of the boundary by subsequent syncs (e.g. reconciliation with absolute dates).
|
|
67
|
+
* Sentinel for an unbounded extraction timestamp (TimeValueType.UNBOUNDED),
|
|
68
|
+
* stored as workersOldest. The Unix epoch guarantees no real timestamp can be
|
|
69
|
+
* earlier, so subsequent syncs cannot accidentally overwrite the boundary.
|
|
72
70
|
*/
|
|
73
71
|
exports.UNBOUNDED_DATE_TIME_VALUE = '1970-01-01T00:00:00.000Z';
|
package/dist/common/helpers.d.ts
CHANGED
|
@@ -1,24 +1,10 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Gets the library version from the package.json file.
|
|
3
|
-
* @returns {string} The library version
|
|
4
|
-
*/
|
|
5
1
|
export declare function getLibraryVersion(): any;
|
|
6
|
-
/**
|
|
7
|
-
* Sleeps for a given number of milliseconds.
|
|
8
|
-
* @param {number} ms - The number of milliseconds to sleep
|
|
9
|
-
* @returns {Promise<void>} A promise that resolves after the given number of milliseconds
|
|
10
|
-
*/
|
|
11
2
|
export declare function sleep(ms: number): Promise<unknown>;
|
|
12
3
|
/**
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
* @property {string} rssUsedMB - The RSS used in MB
|
|
16
|
-
* @property {string} rssUsedPercent - The RSS used percentage
|
|
17
|
-
* @property {string} heapUsedPercent - The heap used percentage
|
|
18
|
-
* @property {string} externalMB - The external memory used in MB
|
|
19
|
-
* @property {string} arrayBuffersMB - The array buffers memory used in MB
|
|
20
|
-
* @property {string} formattedMessage - The formatted message
|
|
4
|
+
* Yields once to the event loop so pending events (e.g. worker messages) can
|
|
5
|
+
* be delivered before continuing.
|
|
21
6
|
*/
|
|
7
|
+
export declare function yieldToEventLoop(): Promise<void>;
|
|
22
8
|
export interface MemoryInfo {
|
|
23
9
|
rssUsedMB: string;
|
|
24
10
|
rssUsedPercent: string;
|
|
@@ -27,9 +13,5 @@ export interface MemoryInfo {
|
|
|
27
13
|
arrayBuffersMB: string;
|
|
28
14
|
formattedMessage: string;
|
|
29
15
|
}
|
|
30
|
-
/**
|
|
31
|
-
* Gets the memory usage information.
|
|
32
|
-
* @returns {MemoryInfo} The memory usage information
|
|
33
|
-
*/
|
|
34
16
|
export declare function getMemoryUsage(): MemoryInfo;
|
|
35
17
|
//# sourceMappingURL=helpers.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"helpers.d.ts","sourceRoot":"","sources":["../../src/common/helpers.ts"],"names":[],"mappings":"AAIA
|
|
1
|
+
{"version":3,"file":"helpers.d.ts","sourceRoot":"","sources":["../../src/common/helpers.ts"],"names":[],"mappings":"AAIA,wBAAgB,iBAAiB,QAiBhC;AAED,wBAAsB,KAAK,CAAC,EAAE,EAAE,MAAM,oBAErC;AAED;;;GAGG;AACH,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC,CAEtD;AAED,MAAM,WAAW,UAAU;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC;IACvB,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,wBAAgB,cAAc,IAAI,UAAU,CA+C3C"}
|
package/dist/common/helpers.js
CHANGED
|
@@ -35,14 +35,11 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
36
|
exports.getLibraryVersion = getLibraryVersion;
|
|
37
37
|
exports.sleep = sleep;
|
|
38
|
+
exports.yieldToEventLoop = yieldToEventLoop;
|
|
38
39
|
exports.getMemoryUsage = getMemoryUsage;
|
|
39
40
|
const fs_1 = require("fs");
|
|
40
41
|
const path = __importStar(require("path"));
|
|
41
42
|
const v8 = __importStar(require("v8"));
|
|
42
|
-
/**
|
|
43
|
-
* Gets the library version from the package.json file.
|
|
44
|
-
* @returns {string} The library version
|
|
45
|
-
*/
|
|
46
43
|
function getLibraryVersion() {
|
|
47
44
|
var _a;
|
|
48
45
|
try {
|
|
@@ -57,18 +54,16 @@ function getLibraryVersion() {
|
|
|
57
54
|
return '';
|
|
58
55
|
}
|
|
59
56
|
}
|
|
60
|
-
/**
|
|
61
|
-
* Sleeps for a given number of milliseconds.
|
|
62
|
-
* @param {number} ms - The number of milliseconds to sleep
|
|
63
|
-
* @returns {Promise<void>} A promise that resolves after the given number of milliseconds
|
|
64
|
-
*/
|
|
65
57
|
async function sleep(ms) {
|
|
66
58
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
67
59
|
}
|
|
68
60
|
/**
|
|
69
|
-
*
|
|
70
|
-
*
|
|
61
|
+
* Yields once to the event loop so pending events (e.g. worker messages) can
|
|
62
|
+
* be delivered before continuing.
|
|
71
63
|
*/
|
|
64
|
+
async function yieldToEventLoop() {
|
|
65
|
+
return new Promise((resolve) => setImmediate(resolve));
|
|
66
|
+
}
|
|
72
67
|
function getMemoryUsage() {
|
|
73
68
|
try {
|
|
74
69
|
const memUsage = process.memoryUsage();
|
|
@@ -76,16 +71,12 @@ function getMemoryUsage() {
|
|
|
76
71
|
const rssUsedMB = memUsage.rss / 1024 / 1024;
|
|
77
72
|
const heapLimitMB = heapStats.heap_size_limit / 1024 / 1024;
|
|
78
73
|
const effectiveMemoryLimitMB = heapLimitMB;
|
|
79
|
-
// Calculate heap values for consistent format
|
|
80
74
|
const heapUsedMB = heapStats.used_heap_size / 1024 / 1024;
|
|
81
75
|
const heapTotalMB = heapStats.heap_size_limit / 1024 / 1024;
|
|
82
|
-
// Calculate external and buffer values (critical for detecting stream leaks)
|
|
83
76
|
const externalMB = memUsage.external / 1024 / 1024;
|
|
84
77
|
const arrayBuffersMB = memUsage.arrayBuffers / 1024 / 1024;
|
|
85
|
-
// Critical percentages for OOM detection
|
|
86
78
|
const rssUsedPercent = ((rssUsedMB / effectiveMemoryLimitMB) * 100).toFixed(2) + '%';
|
|
87
79
|
const heapUsedPercent = ((heapStats.used_heap_size / heapStats.heap_size_limit) * 100).toFixed(2) + '%';
|
|
88
|
-
// Detailed message showing RSS breakdown for leak detection
|
|
89
80
|
const formattedMessage = `Memory: RSS ${rssUsedMB.toFixed(2)}/${effectiveMemoryLimitMB.toFixed(2)}MB (${rssUsedPercent}) [Heap ${heapUsedMB.toFixed(2)}/${heapTotalMB.toFixed(2)}MB (${heapUsedPercent}) + External ${externalMB.toFixed(2)}MB + Buffers ${arrayBuffersMB.toFixed(2)}MB].`;
|
|
90
81
|
return {
|
|
91
82
|
rssUsedMB: rssUsedMB.toFixed(2),
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../src/http/client.ts"],"names":[],"mappings":"AAGA,QAAA,MAAM,WAAW,+BAEf,CAAC;
|
|
1
|
+
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../src/http/client.ts"],"names":[],"mappings":"AAGA,QAAA,MAAM,WAAW,+BAEf,CAAC;AAmFH,OAAO,EAAE,WAAW,EAAE,CAAC"}
|
package/dist/http/client.js
CHANGED
|
@@ -34,11 +34,10 @@ exports.axiosClient = axiosClient;
|
|
|
34
34
|
var _a, _b, _c, _d, _e, _f, _g;
|
|
35
35
|
const retryAfter = ((_b = (_a = error.response) === null || _a === void 0 ? void 0 : _a.headers) === null || _b === void 0 ? void 0 : _b['retry-after']) ||
|
|
36
36
|
((_d = (_c = error.response) === null || _c === void 0 ? void 0 : _c.headers) === null || _d === void 0 ? void 0 : _d['Retry-After']);
|
|
37
|
-
// 5xx errors
|
|
38
37
|
if (((_e = error.response) === null || _e === void 0 ? void 0 : _e.status) && error.response.status >= 500) {
|
|
39
38
|
return true;
|
|
40
39
|
}
|
|
41
|
-
// 429
|
|
40
|
+
// 429 only when a valid non-negative Retry-After header is present
|
|
42
41
|
else if (((_f = error.response) === null || _f === void 0 ? void 0 : _f.status) &&
|
|
43
42
|
error.response.status === 429 &&
|
|
44
43
|
retryAfter &&
|
|
@@ -46,19 +45,16 @@ exports.axiosClient = axiosClient;
|
|
|
46
45
|
Number(retryAfter) >= 0) {
|
|
47
46
|
return true;
|
|
48
47
|
}
|
|
49
|
-
// Network errors for idempotent requests
|
|
48
|
+
// Network errors for idempotent requests; 429 already handled above
|
|
50
49
|
else if (axios_retry_1.default.isNetworkOrIdempotentRequestError(error) &&
|
|
51
50
|
((_g = error.response) === null || _g === void 0 ? void 0 : _g.status) !== 429) {
|
|
52
51
|
return true;
|
|
53
52
|
}
|
|
54
|
-
//
|
|
55
|
-
//
|
|
56
|
-
// Axios only produces ECONNABORTED on client-side timeouts and browser
|
|
57
|
-
// cancellations, never on server responses, so no response guard is needed.
|
|
53
|
+
// axios-retry excludes ECONNABORTED (client-side timeout only, never a
|
|
54
|
+
// server response) from isNetworkError, so handle it here separately.
|
|
58
55
|
else if (error.code === 'ECONNABORTED') {
|
|
59
56
|
return true;
|
|
60
57
|
}
|
|
61
|
-
// all other errors
|
|
62
58
|
else {
|
|
63
59
|
return false;
|
|
64
60
|
}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,27 +1,27 @@
|
|
|
1
|
-
export { processExtractionTask, processLoadingTask, } from './multithreading/process-task';
|
|
2
|
-
export { spawn } from './multithreading/spawn/spawn';
|
|
3
1
|
export { ExtractionAdapter } from './multithreading/adapters/extraction-adapter';
|
|
4
2
|
export { LoadingAdapter } from './multithreading/adapters/loading-adapter';
|
|
5
|
-
export
|
|
3
|
+
export { processExtractionTask, processLoadingTask, } from './multithreading/process-task';
|
|
4
|
+
export { spawn } from './multithreading/spawn/spawn';
|
|
5
|
+
export type { ExtractionScope, ProcessTaskInterface, SpawnFactoryInterface, SpawnInterface, TaskAdapterInterface, TaskResult, TaskStatus, WorkerAdapterInterface, WorkerAdapterOptions, WorkerPathOverrides, } from './types/workers';
|
|
6
6
|
export { AirSyncDefaultItemTypes } from './common/constants';
|
|
7
7
|
export { UNBOUNDED_DATE_TIME_VALUE } from './common/constants';
|
|
8
8
|
export { ExtractionCommonError } from './types/errors';
|
|
9
9
|
export { installInitialDomainMapping } from './state/install-initial-domain-mapping';
|
|
10
10
|
export { serializeError } from './logger/logger';
|
|
11
|
-
export { SyncMode } from './types/common';
|
|
12
11
|
export type { ErrorRecord, InitialDomainMapping } from './types/common';
|
|
13
|
-
export {
|
|
12
|
+
export { SyncMode } from './types/common';
|
|
14
13
|
export type { AirSyncEvent, AirSyncMessage, ConnectionData, EventContext, EventData, ExternalProcessAttachmentFunction, ExternalSyncUnit, ExternalSystemAttachmentIteratorFunction, ExternalSystemAttachmentReducerFunction, ExternalSystemAttachmentStreamingFunction, ExternalSystemAttachmentStreamingParams, ExternalSystemAttachmentStreamingResponse, ExtractorEvent, HttpStreamResponse, ProcessAttachmentReturnType, TimeValue, } from './types/extraction';
|
|
15
|
-
export {
|
|
14
|
+
export { EventType, ExtractorEventType, InitialSyncScope, TimeUnit, TimeValueType, } from './types/extraction';
|
|
16
15
|
export type { ExternalSystemAttachment, ExternalSystemItem, ExternalSystemItemLoadingParams, ExternalSystemItemLoadingResponse, ItemTypeToLoad, } from './types/loading';
|
|
16
|
+
export { LoaderEventType } from './types/loading';
|
|
17
17
|
export type { Item, NormalizedAttachment, NormalizedItem, RepoInterface, } from './repo/repo.interfaces';
|
|
18
18
|
export { Mappers } from './mappers/mappers';
|
|
19
|
-
export { SyncMapperRecordStatus, SyncMapperRecordTargetType, } from './mappers/mappers.interfaces';
|
|
20
19
|
export type { MappersCreateParams, MappersGetByExternalIdParams, MappersGetByTargetIdParams, MappersUpdateParams, } from './mappers/mappers.interfaces';
|
|
20
|
+
export { SyncMapperRecordStatus, SyncMapperRecordTargetType, } from './mappers/mappers.interfaces';
|
|
21
21
|
export type { Artifact, ArtifactsPrepareResponse, SsorAttachment, StreamAttachmentsResponse, StreamResponse, UploadResponse, } from './uploader/uploader.interfaces';
|
|
22
|
-
export type { CollectionData, ConditionalPrivilegeData, CustomLinkData, CustomLinkNames, CustomStage, CustomState, EnumData, EnumValue, EnumValueKey, ExternalDomainMetadata, Field, FieldCondition, FieldConditionComparator, FieldConditionEffect, FieldConditions, FieldKey, FieldPrivilegeData, FieldReferenceData, FieldType, FloatData, IntData, PermissionData, RecordType, RecordTypeCategory, RecordTypeCategoryKey, RecordTypeKey, RecordTypePrivilegeData, RecordTypeScope, ReferenceData, ReferenceDetail, ReferenceType, SchemaVersion,
|
|
23
|
-
export { MockServer, MOCK_SERVER_DEFAULT_URL } from './testing/mock-server';
|
|
24
|
-
export type { RequestInfo, RetryConfig, RouteConfig, } from './testing/mock-server.interfaces';
|
|
25
|
-
export { createMockEvent } from './testing/mock-event';
|
|
22
|
+
export type { CollectionData, ConditionalPrivilegeData, CustomLinkData, CustomLinkNames, CustomStage, CustomState, EnumData, EnumValue, EnumValueKey, ExternalDomainMetadata, Field, FieldCondition, FieldConditionComparator, FieldConditionEffect, FieldConditions, FieldKey, FieldPrivilegeData, FieldReferenceData, FieldType, FloatData, IntData, PermissionData, RecordType, RecordTypeCategory, RecordTypeCategoryKey, RecordTypeKey, RecordTypePrivilegeData, RecordTypeScope, ReferenceData, ReferenceDetail, ReferenceType, SchemaVersion, StageDiagram, StageKey, StateKey, StructData, StructType, StructTypeKey, TargetTypeKeyData, TextData, TypedReferenceData, } from './types/external-domain-metadata';
|
|
26
23
|
export type { DeepPartial } from './testing/mock-event';
|
|
24
|
+
export { createMockEvent } from './testing/mock-event';
|
|
25
|
+
export { MOCK_SERVER_DEFAULT_URL, MockServer } from './testing/mock-server';
|
|
26
|
+
export type { RequestInfo, RetryConfig, RouteConfig, } from './testing/mock-server.interfaces';
|
|
27
27
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,iBAAiB,EAAE,MAAM,8CAA8C,CAAC;AACjF,OAAO,EAAE,cAAc,EAAE,MAAM,2CAA2C,CAAC;AAC3E,OAAO,EACL,qBAAqB,EACrB,kBAAkB,GACnB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,KAAK,EAAE,MAAM,8BAA8B,CAAC;AAGrD,YAAY,EACV,eAAe,EACf,oBAAoB,EACpB,qBAAqB,EACrB,cAAc,EACd,oBAAoB,EACpB,UAAU,EACV,UAAU,EACV,sBAAsB,EACtB,oBAAoB,EACpB,mBAAmB,GACpB,MAAM,iBAAiB,CAAC;AAGzB,OAAO,EAAE,uBAAuB,EAAE,MAAM,oBAAoB,CAAC;AAC7D,OAAO,EAAE,yBAAyB,EAAE,MAAM,oBAAoB,CAAC;AAC/D,OAAO,EAAE,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;AAGvD,OAAO,EAAE,2BAA2B,EAAE,MAAM,wCAAwC,CAAC;AAGrF,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAGjD,YAAY,EAAE,WAAW,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AACxE,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAG1C,YAAY,EACV,YAAY,EACZ,cAAc,EACd,cAAc,EACd,YAAY,EACZ,SAAS,EACT,iCAAiC,EACjC,gBAAgB,EAChB,wCAAwC,EACxC,uCAAuC,EACvC,yCAAyC,EACzC,uCAAuC,EACvC,yCAAyC,EACzC,cAAc,EACd,kBAAkB,EAClB,2BAA2B,EAC3B,SAAS,GACV,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,SAAS,EACT,kBAAkB,EAClB,gBAAgB,EAChB,QAAQ,EACR,aAAa,GACd,MAAM,oBAAoB,CAAC;AAG5B,YAAY,EACV,wBAAwB,EACxB,kBAAkB,EAClB,+BAA+B,EAC/B,iCAAiC,EACjC,cAAc,GACf,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAGlD,YAAY,EACV,IAAI,EACJ,oBAAoB,EACpB,cAAc,EACd,aAAa,GACd,MAAM,wBAAwB,CAAC;AAGhC,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAC5C,YAAY,EACV,mBAAmB,EACnB,4BAA4B,EAC5B,0BAA0B,EAC1B,mBAAmB,GACpB,MAAM,8BAA8B,CAAC;AACtC,OAAO,EACL,sBAAsB,EACtB,0BAA0B,GAC3B,MAAM,8BAA8B,CAAC;AAGtC,YAAY,EACV,QAAQ,EACR,wBAAwB,EACxB,cAAc,EACd,yBAAyB,EACzB,cAAc,EACd,cAAc,GACf,MAAM,gCAAgC,CAAC;AAGxC,YAAY,EACV,cAAc,EACd,wBAAwB,EACxB,cAAc,EACd,eAAe,EACf,WAAW,EACX,WAAW,EACX,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,sBAAsB,EACtB,KAAK,EACL,cAAc,EACd,wBAAwB,EACxB,oBAAoB,EACpB,eAAe,EACf,QAAQ,EACR,kBAAkB,EAClB,kBAAkB,EAClB,SAAS,EACT,SAAS,EACT,OAAO,EACP,cAAc,EACd,UAAU,EACV,kBAAkB,EAClB,qBAAqB,EACrB,aAAa,EACb,uBAAuB,EACvB,eAAe,EACf,aAAa,EACb,eAAe,EACf,aAAa,EACb,aAAa,EACb,YAAY,EACZ,QAAQ,EACR,QAAQ,EACR,UAAU,EACV,UAAU,EACV,aAAa,EACb,iBAAiB,EACjB,QAAQ,EACR,kBAAkB,GACnB,MAAM,kCAAkC,CAAC;AAG1C,YAAY,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,EAAE,uBAAuB,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAC5E,YAAY,EACV,WAAW,EACX,WAAW,EACX,WAAW,GACZ,MAAM,kCAAkC,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -1,24 +1,21 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
// ────────────────────────────────────────────────────────────────────────────
|
|
3
|
-
// Public API barrel for @devrev/airsync-sdk.
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
// named explicitly (no `export *`), so any change to the public API shows up as
|
|
7
|
-
// a diff here. Adding a symbol to the package's public contract means adding it
|
|
8
|
-
// to this file.
|
|
3
|
+
// Public API barrel for @devrev/airsync-sdk. Single source of the public
|
|
4
|
+
// surface: every export is named explicitly (no `export *`), so any public API
|
|
5
|
+
// change shows up as a diff here.
|
|
9
6
|
// ────────────────────────────────────────────────────────────────────────────
|
|
10
7
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
11
|
-
exports.
|
|
8
|
+
exports.MockServer = exports.MOCK_SERVER_DEFAULT_URL = exports.createMockEvent = exports.SyncMapperRecordTargetType = exports.SyncMapperRecordStatus = exports.Mappers = exports.LoaderEventType = exports.TimeValueType = exports.TimeUnit = exports.InitialSyncScope = exports.ExtractorEventType = exports.EventType = exports.SyncMode = exports.serializeError = exports.installInitialDomainMapping = exports.UNBOUNDED_DATE_TIME_VALUE = exports.AirSyncDefaultItemTypes = exports.spawn = exports.processLoadingTask = exports.processExtractionTask = exports.LoadingAdapter = exports.ExtractionAdapter = void 0;
|
|
12
9
|
// ── Entry points & adapters ──
|
|
10
|
+
var extraction_adapter_1 = require("./multithreading/adapters/extraction-adapter");
|
|
11
|
+
Object.defineProperty(exports, "ExtractionAdapter", { enumerable: true, get: function () { return extraction_adapter_1.ExtractionAdapter; } });
|
|
12
|
+
var loading_adapter_1 = require("./multithreading/adapters/loading-adapter");
|
|
13
|
+
Object.defineProperty(exports, "LoadingAdapter", { enumerable: true, get: function () { return loading_adapter_1.LoadingAdapter; } });
|
|
13
14
|
var process_task_1 = require("./multithreading/process-task");
|
|
14
15
|
Object.defineProperty(exports, "processExtractionTask", { enumerable: true, get: function () { return process_task_1.processExtractionTask; } });
|
|
15
16
|
Object.defineProperty(exports, "processLoadingTask", { enumerable: true, get: function () { return process_task_1.processLoadingTask; } });
|
|
16
17
|
var spawn_1 = require("./multithreading/spawn/spawn");
|
|
17
18
|
Object.defineProperty(exports, "spawn", { enumerable: true, get: function () { return spawn_1.spawn; } });
|
|
18
|
-
var extraction_adapter_1 = require("./multithreading/adapters/extraction-adapter");
|
|
19
|
-
Object.defineProperty(exports, "ExtractionAdapter", { enumerable: true, get: function () { return extraction_adapter_1.ExtractionAdapter; } });
|
|
20
|
-
var loading_adapter_1 = require("./multithreading/adapters/loading-adapter");
|
|
21
|
-
Object.defineProperty(exports, "LoadingAdapter", { enumerable: true, get: function () { return loading_adapter_1.LoadingAdapter; } });
|
|
22
19
|
// ── Constants & enums ──
|
|
23
20
|
var constants_1 = require("./common/constants");
|
|
24
21
|
Object.defineProperty(exports, "AirSyncDefaultItemTypes", { enumerable: true, get: function () { return constants_1.AirSyncDefaultItemTypes; } });
|
|
@@ -30,17 +27,14 @@ Object.defineProperty(exports, "installInitialDomainMapping", { enumerable: true
|
|
|
30
27
|
// ── Error formatting ──
|
|
31
28
|
var logger_1 = require("./logger/logger");
|
|
32
29
|
Object.defineProperty(exports, "serializeError", { enumerable: true, get: function () { return logger_1.serializeError; } });
|
|
33
|
-
// ── Common types ──
|
|
34
30
|
var common_1 = require("./types/common");
|
|
35
31
|
Object.defineProperty(exports, "SyncMode", { enumerable: true, get: function () { return common_1.SyncMode; } });
|
|
36
|
-
// ── Extraction types ──
|
|
37
32
|
var extraction_1 = require("./types/extraction");
|
|
38
33
|
Object.defineProperty(exports, "EventType", { enumerable: true, get: function () { return extraction_1.EventType; } });
|
|
39
34
|
Object.defineProperty(exports, "ExtractorEventType", { enumerable: true, get: function () { return extraction_1.ExtractorEventType; } });
|
|
40
35
|
Object.defineProperty(exports, "InitialSyncScope", { enumerable: true, get: function () { return extraction_1.InitialSyncScope; } });
|
|
41
36
|
Object.defineProperty(exports, "TimeUnit", { enumerable: true, get: function () { return extraction_1.TimeUnit; } });
|
|
42
37
|
Object.defineProperty(exports, "TimeValueType", { enumerable: true, get: function () { return extraction_1.TimeValueType; } });
|
|
43
|
-
// ── Loading types ──
|
|
44
38
|
var loading_1 = require("./types/loading");
|
|
45
39
|
Object.defineProperty(exports, "LoaderEventType", { enumerable: true, get: function () { return loading_1.LoaderEventType; } });
|
|
46
40
|
// ── Mappers ──
|
|
@@ -49,9 +43,8 @@ Object.defineProperty(exports, "Mappers", { enumerable: true, get: function () {
|
|
|
49
43
|
var mappers_interfaces_1 = require("./mappers/mappers.interfaces");
|
|
50
44
|
Object.defineProperty(exports, "SyncMapperRecordStatus", { enumerable: true, get: function () { return mappers_interfaces_1.SyncMapperRecordStatus; } });
|
|
51
45
|
Object.defineProperty(exports, "SyncMapperRecordTargetType", { enumerable: true, get: function () { return mappers_interfaces_1.SyncMapperRecordTargetType; } });
|
|
52
|
-
// ── Testing utilities (public test-support surface) ──
|
|
53
|
-
var mock_server_1 = require("./testing/mock-server");
|
|
54
|
-
Object.defineProperty(exports, "MockServer", { enumerable: true, get: function () { return mock_server_1.MockServer; } });
|
|
55
|
-
Object.defineProperty(exports, "MOCK_SERVER_DEFAULT_URL", { enumerable: true, get: function () { return mock_server_1.MOCK_SERVER_DEFAULT_URL; } });
|
|
56
46
|
var mock_event_1 = require("./testing/mock-event");
|
|
57
47
|
Object.defineProperty(exports, "createMockEvent", { enumerable: true, get: function () { return mock_event_1.createMockEvent; } });
|
|
48
|
+
var mock_server_1 = require("./testing/mock-server");
|
|
49
|
+
Object.defineProperty(exports, "MOCK_SERVER_DEFAULT_URL", { enumerable: true, get: function () { return mock_server_1.MOCK_SERVER_DEFAULT_URL; } });
|
|
50
|
+
Object.defineProperty(exports, "MockServer", { enumerable: true, get: function () { return mock_server_1.MockServer; } });
|
package/dist/logger/logger.d.ts
CHANGED
|
@@ -1,87 +1,32 @@
|
|
|
1
|
-
import { AxiosError } from 'axios';
|
|
2
1
|
import { Console } from 'node:console';
|
|
3
2
|
import { InspectOptions } from 'node:util';
|
|
3
|
+
import { AxiosError } from 'axios';
|
|
4
4
|
import { AxiosErrorResponse, LoggerFactoryInterface, LogLevel, PrintableState } from './logger.interfaces';
|
|
5
5
|
export declare const MAX_LOG_STRING_LENGTH = 10000;
|
|
6
6
|
export declare const INSPECT_OPTIONS: InspectOptions;
|
|
7
|
-
/**
|
|
8
|
-
* Truncates a message that exceeds the maximum length, appending a suffix
|
|
9
|
-
* noting how many characters were omitted.
|
|
10
|
-
*/
|
|
11
7
|
export declare function truncateMessage(message: string): string;
|
|
12
|
-
/** Runs `fn` with logs marked as user-originated (`is_sdk_log: false`). */
|
|
13
|
-
export declare function runWithUserLogContext<T>(fn: () => T): T;
|
|
14
|
-
/** Runs `fn` with logs marked as SDK-originated (`is_sdk_log: true`). */
|
|
15
|
-
export declare function runWithSdkLogContext<T>(fn: () => T): T;
|
|
16
|
-
/** Returns the current context flag, or `defaultValue` if none is set. */
|
|
17
|
-
export declare function getSdkLogContextValue(defaultValue: boolean): boolean;
|
|
18
8
|
/**
|
|
19
|
-
*
|
|
20
|
-
*
|
|
9
|
+
* Console replacement that tags every log line with the event context. Worker
|
|
10
|
+
* threads forward log lines to the main thread, because the snap-in platform
|
|
11
|
+
* only captures logs written through the main thread's console.
|
|
21
12
|
*/
|
|
22
13
|
export declare class Logger extends Console {
|
|
23
14
|
private originalConsole;
|
|
24
15
|
private options?;
|
|
25
16
|
private tags;
|
|
26
17
|
constructor({ event, options }: LoggerFactoryInterface);
|
|
27
|
-
/**
|
|
28
|
-
* Converts any value to a string using `util.inspect()` for complex types.
|
|
29
|
-
*
|
|
30
|
-
* @param value - The value to convert
|
|
31
|
-
* @returns String representation of the value
|
|
32
|
-
*/
|
|
33
18
|
private valueToString;
|
|
34
|
-
/**
|
|
35
|
-
|
|
36
|
-
* In production mode, wraps the message with JSON formatting and event context tags.
|
|
37
|
-
* In local development mode, logs the message directly without JSON wrapping.
|
|
38
|
-
* This is useful when you need to log already-stringified content.
|
|
39
|
-
*
|
|
40
|
-
* @param message - The pre-formatted message string to log
|
|
41
|
-
* @param level - Log level (info, warn, error)
|
|
42
|
-
* @param isSdkLog - Flag indicating if the log originated from the SDK
|
|
43
|
-
*/
|
|
44
|
-
logFn(message: string, level: LogLevel, isSdkLog?: boolean): void;
|
|
45
|
-
/**
|
|
46
|
-
* Stringifies and logs arguments to the appropriate destination.
|
|
47
|
-
* On main thread, converts arguments to strings and calls logFn.
|
|
48
|
-
* In worker threads, forwards stringified arguments to the main thread for processing.
|
|
49
|
-
* All arguments are converted to strings using util.inspect and joined with spaces.
|
|
50
|
-
*
|
|
51
|
-
* @param args - Values to log (will be stringified and truncated if needed)
|
|
52
|
-
* @param level - Log level (info, warn, error)
|
|
53
|
-
*/
|
|
19
|
+
/** In production wraps the message in JSON with event context tags; in local development logs as-is. */
|
|
20
|
+
logFn(message: string, level: LogLevel): void;
|
|
54
21
|
private stringifyAndLog;
|
|
55
22
|
log(...args: unknown[]): void;
|
|
56
23
|
info(...args: unknown[]): void;
|
|
57
24
|
warn(...args: unknown[]): void;
|
|
58
25
|
error(...args: unknown[]): void;
|
|
59
26
|
}
|
|
60
|
-
/**
|
|
61
|
-
* Converts a state object into a printable format where arrays are summarized.
|
|
62
|
-
* Arrays show their length, first item, and last item instead of all elements.
|
|
63
|
-
* Objects are recursively processed and primitives are returned as-is.
|
|
64
|
-
*
|
|
65
|
-
* @param state - State object to convert
|
|
66
|
-
* @returns Printable representation with summarized arrays
|
|
67
|
-
*/
|
|
27
|
+
/** Summarizes arrays as `{ length, firstItem, lastItem }` instead of listing all elements. */
|
|
68
28
|
export declare function getPrintableState(state: Record<string, any>): PrintableState;
|
|
69
|
-
/**
|
|
70
|
-
* Serializes an error into a structured format.
|
|
71
|
-
* Automatically detects and formats Axios errors with HTTP details.
|
|
72
|
-
* Returns other error types as-is.
|
|
73
|
-
*
|
|
74
|
-
* @param error - Error to serialize
|
|
75
|
-
* @returns Serialized error or original if not an Axios error
|
|
76
|
-
*/
|
|
29
|
+
/** Serializes any error into a loggable string; Axios errors get HTTP details. */
|
|
77
30
|
export declare function serializeError(error: unknown): string;
|
|
78
|
-
/**
|
|
79
|
-
* Serializes an Axios error into a structured format with HTTP request/response details.
|
|
80
|
-
* Extracts method, URL, parameters, status code, headers, and data.
|
|
81
|
-
* Includes CORS/network failure indicator when no response is available.
|
|
82
|
-
*
|
|
83
|
-
* @param error - Axios error to serialize
|
|
84
|
-
* @returns Structured object with error details
|
|
85
|
-
*/
|
|
86
31
|
export declare function serializeAxiosError(error: AxiosError): AxiosErrorResponse;
|
|
87
32
|
//# sourceMappingURL=logger.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../../src/logger/logger.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../../src/logger/logger.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAW,cAAc,EAAE,MAAM,WAAW,CAAC;AAGpD,OAAO,EAAE,UAAU,EAAyC,MAAM,OAAO,CAAC;AAK1E,OAAO,EACL,kBAAkB,EAClB,sBAAsB,EAEtB,QAAQ,EAER,cAAc,EACf,MAAM,qBAAqB,CAAC;AAI7B,eAAO,MAAM,qBAAqB,QAAQ,CAAC;AAI3C,eAAO,MAAM,eAAe,EAAE,cAM7B,CAAC;AAEF,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAOvD;AAED;;;;GAIG;AACH,qBAAa,MAAO,SAAQ,OAAO;IACjC,OAAO,CAAC,eAAe,CAAU;IACjC,OAAO,CAAC,OAAO,CAAC,CAAuB;IACvC,OAAO,CAAC,IAAI,CAAa;gBAEb,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,sBAAsB;IAUtD,OAAO,CAAC,aAAa;IAOrB,wGAAwG;IACxG,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,GAAG,IAAI;IAa7C,OAAO,CAAC,eAAe;IAcd,GAAG,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI;IAI7B,IAAI,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI;IAI9B,IAAI,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI;IAI9B,KAAK,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI;CAGzC;AAED,8FAA8F;AAE9F,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,cAAc,CAqB5E;AAED,kFAAkF;AAClF,wBAAgB,cAAc,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAmCrD;AAED,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,UAAU,GAAG,kBAAkB,CAwBzE"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"logger.interfaces.d.ts","sourceRoot":"","sources":["../../src/logger/logger.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"logger.interfaces.d.ts","sourceRoot":"","sources":["../../src/logger/logger.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,OAAO,CAAC;AAErD,OAAO,KAAK,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACtE,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAE7D,MAAM,WAAW,sBAAsB;IACrC,KAAK,EAAE,YAAY,CAAC;IACpB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED,oBAAY,QAAQ;IAClB,IAAI,SAAS;IACb,IAAI,SAAS;IACb,KAAK,UAAU;CAChB;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,OAAO,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IAEf,SAAS,CAAC,EAAE,GAAG,CAAC;IAEhB,QAAQ,CAAC,EAAE,GAAG,CAAC;CAChB;AAED,MAAM,WAAW,cAAc;IAE7B,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,GAAG,cAAc,GAAG,cAAc,CAAC;CACtD;AAED,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE;QACN,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;QAE3B,MAAM,EAAE,GAAG,CAAC;QACZ,GAAG,EAAE,MAAM,GAAG,SAAS,CAAC;KACzB,CAAC;IACF,YAAY,EAAE,OAAO,CAAC;IACtB,sBAAsB,EAAE,OAAO,CAAC;IAChC,QAAQ,CAAC,EAAE;QACT,IAAI,EAAE,OAAO,CAAC;QACd,OAAO,EAAE,uBAAuB,CAAC;QACjC,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,UAAW,SAAQ,YAAY;IAC9C,WAAW,EAAE,MAAM,CAAC;CACrB"}
|