@devrev/airsync-sdk 2.0.0-beta.3 → 2.0.0-beta.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/MIGRATION.md +1028 -0
- package/README.md +5 -11
- package/dist/attachments-streaming/attachments-streaming-pool.d.ts +7 -9
- package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map +1 -1
- package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts +2 -7
- package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts.map +1 -1
- package/dist/attachments-streaming/attachments-streaming-pool.js +54 -56
- package/dist/common/constants.d.ts +4 -5
- package/dist/common/constants.d.ts.map +1 -1
- package/dist/common/constants.js +6 -7
- package/dist/common/helpers.d.ts +3 -35
- package/dist/common/helpers.d.ts.map +1 -1
- package/dist/common/helpers.js +5 -50
- package/dist/http/{axios-client-internal.d.ts → client.d.ts} +1 -1
- package/dist/http/client.d.ts.map +1 -0
- package/dist/http/{axios-client-internal.js → client.js} +4 -8
- package/dist/index.d.ts +24 -13
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +43 -32
- package/dist/logger/logger.d.ts +12 -53
- package/dist/logger/logger.d.ts.map +1 -1
- package/dist/logger/logger.interfaces.d.ts +0 -1
- package/dist/logger/logger.interfaces.d.ts.map +1 -1
- package/dist/logger/logger.js +36 -68
- package/dist/mappers/mappers.d.ts +7 -46
- package/dist/mappers/mappers.d.ts.map +1 -1
- package/dist/mappers/mappers.interfaces.d.ts +175 -0
- package/dist/mappers/mappers.interfaces.d.ts.map +1 -0
- package/dist/mappers/{mappers.interface.js → mappers.interfaces.js} +4 -13
- package/dist/mappers/mappers.js +11 -45
- package/dist/multithreading/adapters/base-adapter.d.ts +30 -32
- package/dist/multithreading/adapters/base-adapter.d.ts.map +1 -1
- package/dist/multithreading/adapters/base-adapter.js +97 -62
- package/dist/multithreading/adapters/extraction-adapter.d.ts +16 -29
- package/dist/multithreading/adapters/extraction-adapter.d.ts.map +1 -1
- package/dist/multithreading/adapters/extraction-adapter.js +203 -206
- package/dist/multithreading/adapters/loading-adapter.d.ts +8 -13
- package/dist/multithreading/adapters/loading-adapter.d.ts.map +1 -1
- package/dist/multithreading/adapters/loading-adapter.helpers.d.ts +2 -12
- package/dist/multithreading/adapters/loading-adapter.helpers.d.ts.map +1 -1
- package/dist/multithreading/adapters/loading-adapter.helpers.js +2 -12
- package/dist/multithreading/adapters/loading-adapter.js +294 -346
- package/dist/multithreading/emit.d.ts +11 -0
- package/dist/multithreading/emit.d.ts.map +1 -0
- package/dist/{common/control-protocol.js → multithreading/emit.js} +5 -7
- package/dist/multithreading/process-task.d.ts +2 -12
- package/dist/multithreading/process-task.d.ts.map +1 -1
- package/dist/multithreading/process-task.js +46 -43
- package/dist/multithreading/spawn/spawn.d.ts +3 -9
- package/dist/multithreading/spawn/spawn.d.ts.map +1 -1
- package/dist/multithreading/spawn/spawn.helpers.d.ts +12 -11
- package/dist/multithreading/spawn/spawn.helpers.d.ts.map +1 -1
- package/dist/multithreading/spawn/spawn.helpers.js +128 -20
- package/dist/multithreading/spawn/spawn.js +24 -39
- package/dist/multithreading/worker.js +1 -4
- package/dist/repo/repo.d.ts +10 -0
- package/dist/repo/repo.d.ts.map +1 -1
- package/dist/repo/repo.helpers.d.ts +6 -0
- package/dist/repo/repo.helpers.d.ts.map +1 -0
- package/dist/repo/repo.helpers.js +16 -0
- package/dist/repo/repo.interfaces.d.ts +4 -16
- package/dist/repo/repo.interfaces.d.ts.map +1 -1
- package/dist/repo/repo.js +23 -8
- package/dist/state/base-state.d.ts +16 -38
- package/dist/state/base-state.d.ts.map +1 -1
- package/dist/state/base-state.js +19 -41
- package/dist/state/extraction-state.d.ts +6 -21
- package/dist/state/extraction-state.d.ts.map +1 -1
- package/dist/state/extraction-state.js +8 -25
- package/dist/{common → state}/install-initial-domain-mapping.d.ts +1 -1
- package/dist/state/install-initial-domain-mapping.d.ts.map +1 -0
- package/dist/{common → state}/install-initial-domain-mapping.js +5 -7
- package/dist/state/loading-state.d.ts +3 -14
- package/dist/state/loading-state.d.ts.map +1 -1
- package/dist/state/loading-state.js +3 -14
- package/dist/state/state.d.ts +4 -0
- package/dist/state/state.d.ts.map +1 -0
- package/dist/state/state.interfaces.d.ts +33 -37
- package/dist/state/state.interfaces.d.ts.map +1 -1
- package/dist/state/state.interfaces.js +16 -11
- package/dist/state/state.js +11 -0
- package/dist/state/time-value-resolver.d.ts +17 -0
- package/dist/state/time-value-resolver.d.ts.map +1 -0
- package/dist/{common → state}/time-value-resolver.js +9 -41
- package/dist/testing/mock-event.d.ts +10 -0
- package/dist/testing/mock-event.d.ts.map +1 -0
- package/dist/{common/test-utils.js → testing/mock-event.js} +17 -14
- package/dist/testing/mock-server.d.ts +29 -0
- package/dist/testing/mock-server.d.ts.map +1 -0
- package/dist/testing/mock-server.interfaces.d.ts +48 -0
- package/dist/testing/mock-server.interfaces.d.ts.map +1 -0
- package/dist/{mock-server → testing}/mock-server.js +21 -33
- package/dist/types/common.d.ts +1 -12
- package/dist/types/common.d.ts.map +1 -1
- package/dist/types/common.js +1 -6
- package/dist/types/errors.d.ts +9 -0
- package/dist/types/errors.d.ts.map +1 -0
- package/dist/types/external-domain-metadata.d.ts +30 -171
- package/dist/types/external-domain-metadata.d.ts.map +1 -1
- package/dist/types/extraction.d.ts +64 -94
- package/dist/types/extraction.d.ts.map +1 -1
- package/dist/types/extraction.js +6 -34
- package/dist/types/loading.d.ts +1 -2
- package/dist/types/loading.d.ts.map +1 -1
- package/dist/types/loading.js +0 -1
- package/dist/types/workers.d.ts +39 -94
- package/dist/types/workers.d.ts.map +1 -1
- package/dist/types/workers.js +0 -6
- package/dist/uploader/uploader.d.ts +8 -59
- package/dist/uploader/uploader.d.ts.map +1 -1
- package/dist/uploader/uploader.helpers.d.ts +4 -27
- package/dist/uploader/uploader.helpers.d.ts.map +1 -1
- package/dist/uploader/uploader.helpers.js +64 -30
- package/dist/uploader/uploader.interfaces.d.ts +15 -25
- package/dist/uploader/uploader.interfaces.d.ts.map +1 -1
- package/dist/uploader/uploader.interfaces.js +8 -0
- package/dist/uploader/uploader.js +23 -89
- package/package.json +8 -6
- package/dist/airsync-sdk.d.ts +0 -2059
- package/dist/attachments-streaming/attachments-streaming-pool.test.d.ts +0 -2
- package/dist/attachments-streaming/attachments-streaming-pool.test.d.ts.map +0 -1
- package/dist/attachments-streaming/attachments-streaming-pool.test.js +0 -540
- package/dist/common/control-protocol.d.ts +0 -10
- package/dist/common/control-protocol.d.ts.map +0 -1
- package/dist/common/errors.d.ts +0 -6
- package/dist/common/errors.d.ts.map +0 -1
- package/dist/common/install-initial-domain-mapping.d.ts.map +0 -1
- package/dist/common/install-initial-domain-mapping.test.d.ts +0 -2
- package/dist/common/install-initial-domain-mapping.test.d.ts.map +0 -1
- package/dist/common/install-initial-domain-mapping.test.js +0 -211
- package/dist/common/test-utils.d.ts +0 -19
- package/dist/common/test-utils.d.ts.map +0 -1
- package/dist/common/time-value-resolver.d.ts +0 -47
- package/dist/common/time-value-resolver.d.ts.map +0 -1
- package/dist/common/time-value-resolver.test.d.ts +0 -2
- package/dist/common/time-value-resolver.test.d.ts.map +0 -1
- package/dist/common/time-value-resolver.test.js +0 -268
- package/dist/http/axios-client-internal.d.ts.map +0 -1
- package/dist/http/axios-client-internal.test.d.ts +0 -2
- package/dist/http/axios-client-internal.test.d.ts.map +0 -1
- package/dist/http/axios-client-internal.test.js +0 -175
- package/dist/http/axios-client.d.ts +0 -27
- package/dist/http/axios-client.d.ts.map +0 -1
- package/dist/http/axios-client.js +0 -60
- package/dist/http/constants.d.ts +0 -4
- package/dist/http/constants.d.ts.map +0 -1
- package/dist/http/constants.js +0 -6
- package/dist/http/index.d.ts +0 -2
- package/dist/http/index.d.ts.map +0 -1
- package/dist/http/index.js +0 -17
- package/dist/logger/logger.constants.d.ts +0 -6
- package/dist/logger/logger.constants.d.ts.map +0 -1
- package/dist/logger/logger.constants.js +0 -13
- package/dist/logger/logger.context.d.ts +0 -58
- package/dist/logger/logger.context.d.ts.map +0 -1
- package/dist/logger/logger.context.js +0 -86
- package/dist/logger/logger.test.d.ts +0 -2
- package/dist/logger/logger.test.d.ts.map +0 -1
- package/dist/logger/logger.test.js +0 -537
- package/dist/mappers/mappers.interface.d.ts +0 -294
- package/dist/mappers/mappers.interface.d.ts.map +0 -1
- package/dist/mappers/mappers.test.d.ts +0 -2
- package/dist/mappers/mappers.test.d.ts.map +0 -1
- package/dist/mappers/mappers.test.js +0 -105
- package/dist/mock-server/mock-server.d.ts +0 -47
- package/dist/mock-server/mock-server.d.ts.map +0 -1
- package/dist/mock-server/mock-server.interfaces.d.ts +0 -73
- package/dist/mock-server/mock-server.interfaces.d.ts.map +0 -1
- package/dist/multithreading/adapters/base-adapter.emit.test.d.ts +0 -2
- package/dist/multithreading/adapters/base-adapter.emit.test.d.ts.map +0 -1
- package/dist/multithreading/adapters/base-adapter.emit.test.js +0 -403
- package/dist/multithreading/adapters/extraction-adapter.artifacts.test.d.ts +0 -2
- package/dist/multithreading/adapters/extraction-adapter.artifacts.test.d.ts.map +0 -1
- package/dist/multithreading/adapters/extraction-adapter.artifacts.test.js +0 -175
- package/dist/multithreading/adapters/extraction-adapter.test.d.ts +0 -2
- package/dist/multithreading/adapters/extraction-adapter.test.d.ts.map +0 -1
- package/dist/multithreading/adapters/extraction-adapter.test.js +0 -789
- package/dist/multithreading/adapters/loading-adapter.helpers.test.d.ts +0 -2
- package/dist/multithreading/adapters/loading-adapter.helpers.test.d.ts.map +0 -1
- package/dist/multithreading/adapters/loading-adapter.helpers.test.js +0 -514
- package/dist/multithreading/adapters/loading-adapter.test.d.ts +0 -2
- package/dist/multithreading/adapters/loading-adapter.test.d.ts.map +0 -1
- package/dist/multithreading/adapters/loading-adapter.test.js +0 -589
- package/dist/multithreading/adapters/serialization.test.d.ts +0 -2
- package/dist/multithreading/adapters/serialization.test.d.ts.map +0 -1
- package/dist/multithreading/adapters/serialization.test.js +0 -71
- package/dist/multithreading/create-worker.test.d.ts +0 -2
- package/dist/multithreading/create-worker.test.d.ts.map +0 -1
- package/dist/multithreading/create-worker.test.js +0 -108
- package/dist/multithreading/process-task.test.d.ts +0 -2
- package/dist/multithreading/process-task.test.d.ts.map +0 -1
- package/dist/multithreading/process-task.test.js +0 -146
- package/dist/multithreading/spawn/spawn.helpers.test.d.ts +0 -2
- package/dist/multithreading/spawn/spawn.helpers.test.d.ts.map +0 -1
- package/dist/multithreading/spawn/spawn.helpers.test.js +0 -293
- package/dist/multithreading/spawn/spawn.test.d.ts +0 -2
- package/dist/multithreading/spawn/spawn.test.d.ts.map +0 -1
- package/dist/multithreading/spawn/spawn.test.js +0 -232
- package/dist/repo/repo.test.d.ts +0 -2
- package/dist/repo/repo.test.d.ts.map +0 -1
- package/dist/repo/repo.test.js +0 -184
- package/dist/state/state.extract-window.test.d.ts +0 -2
- package/dist/state/state.extract-window.test.d.ts.map +0 -1
- package/dist/state/state.extract-window.test.js +0 -163
- package/dist/state/state.pending-boundaries.test.d.ts +0 -2
- package/dist/state/state.pending-boundaries.test.d.ts.map +0 -1
- package/dist/state/state.pending-boundaries.test.js +0 -189
- package/dist/state/state.post-state.test.d.ts +0 -2
- package/dist/state/state.post-state.test.d.ts.map +0 -1
- package/dist/state/state.post-state.test.js +0 -78
- package/dist/state/state.test.d.ts +0 -2
- package/dist/state/state.test.d.ts.map +0 -1
- package/dist/state/state.test.js +0 -271
- package/dist/state/state.time-value-resolution.test.d.ts +0 -2
- package/dist/state/state.time-value-resolution.test.d.ts.map +0 -1
- package/dist/state/state.time-value-resolution.test.js +0 -175
- package/dist/tsdoc-metadata.json +0 -11
- package/dist/types/extraction.test.d.ts +0 -2
- package/dist/types/extraction.test.d.ts.map +0 -1
- package/dist/types/extraction.test.js +0 -96
- package/dist/types/index.d.ts +0 -10
- package/dist/types/index.d.ts.map +0 -1
- package/dist/types/index.js +0 -21
- package/dist/uploader/uploader.helpers.test.d.ts +0 -2
- package/dist/uploader/uploader.helpers.test.d.ts.map +0 -1
- package/dist/uploader/uploader.helpers.test.js +0 -256
- package/dist/uploader/uploader.test.d.ts +0 -2
- package/dist/uploader/uploader.test.d.ts.map +0 -1
- package/dist/uploader/uploader.test.js +0 -601
- /package/dist/{mock-server → testing}/mock-server.interfaces.js +0 -0
- /package/dist/{common → types}/errors.js +0 -0
package/README.md
CHANGED
|
@@ -1,16 +1,14 @@
|
|
|
1
|
-
#
|
|
1
|
+
# Airdrop SDK
|
|
2
2
|
|
|
3
|
-
[. The package has been renamed from `@devrev/ts-adaas` to `@devrev/airsync-sdk`.
|
|
3
|
+
[](https://coveralls.io/github/devrev/adaas-sdk?branch=main)
|
|
6
4
|
|
|
7
5
|
## Overview
|
|
8
6
|
|
|
9
|
-
The
|
|
7
|
+
The Airdrop SDK for TypeScript helps developers build snap-ins that integrate with DevRev’s Airdrop platform. This SDK simplifies the workflow for handling data extraction and loading, event-driven actions, state management, and artifact handling.
|
|
10
8
|
|
|
11
9
|
It provides features such as:
|
|
12
10
|
|
|
13
|
-
- Type Definitions: Structured types for
|
|
11
|
+
- Type Definitions: Structured types for Airdrop control protocol
|
|
14
12
|
- Event Management: Easily emit events for different extraction or loading phases
|
|
15
13
|
- State Handling: Update and access state in real-time within tasks
|
|
16
14
|
- Artifact Management: Supports batched storage of artifacts
|
|
@@ -19,9 +17,5 @@ It provides features such as:
|
|
|
19
17
|
## Installation
|
|
20
18
|
|
|
21
19
|
```bash
|
|
22
|
-
npm install @devrev/airsync-sdk
|
|
20
|
+
npm install @devrev/airsync-sdk
|
|
23
21
|
```
|
|
24
|
-
|
|
25
|
-
## Changelog
|
|
26
|
-
|
|
27
|
-
See [CHANGELOG.md](./CHANGELOG.md) for release notes and migration guides.
|
|
@@ -1,23 +1,21 @@
|
|
|
1
|
-
import { ProcessAttachmentReturnType } from '../types';
|
|
1
|
+
import { ProcessAttachmentReturnType } from '../types/extraction';
|
|
2
2
|
import { AttachmentsStreamingPoolParams } from './attachments-streaming-pool.interfaces';
|
|
3
3
|
export declare class AttachmentsStreamingPool<ConnectorState> {
|
|
4
4
|
private adapter;
|
|
5
5
|
private attachments;
|
|
6
|
-
private attachmentsMetadata;
|
|
7
6
|
private batchSize;
|
|
8
7
|
private delay;
|
|
9
8
|
private stream;
|
|
10
9
|
private totalProcessedCount;
|
|
11
10
|
private readonly PROGRESS_REPORT_INTERVAL;
|
|
12
|
-
constructor({ adapter, attachments,
|
|
13
|
-
private
|
|
11
|
+
constructor({ adapter, attachments, batchSize, stream, }: AttachmentsStreamingPoolParams<ConnectorState>);
|
|
12
|
+
private recordProcessedAttachment;
|
|
14
13
|
/**
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
* @param attachments - The attachments list to migrate (either string[] or ProcessedAttachment[])
|
|
18
|
-
* @returns Migrated array of ProcessedAttachment objects, or empty array if input is invalid
|
|
14
|
+
* Backfills `status` on entries recorded before the field existed (those
|
|
15
|
+
* only ever held successes); state from >= v1.15.2 needs no shape migration.
|
|
19
16
|
*/
|
|
20
|
-
private
|
|
17
|
+
private backfillProcessedAttachmentStatus;
|
|
18
|
+
private updateProgress;
|
|
21
19
|
streamAll(): Promise<ProcessAttachmentReturnType>;
|
|
22
20
|
startPoolStreaming(): Promise<void>;
|
|
23
21
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"attachments-streaming-pool.d.ts","sourceRoot":"","sources":["../../src/attachments-streaming/attachments-streaming-pool.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"attachments-streaming-pool.d.ts","sourceRoot":"","sources":["../../src/attachments-streaming/attachments-streaming-pool.ts"],"names":[],"mappings":"AAOA,OAAO,EAEL,2BAA2B,EAC5B,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,8BAA8B,EAAE,MAAM,yCAAyC,CAAC;AAEzF,qBAAa,wBAAwB,CAAC,cAAc;IAClD,OAAO,CAAC,OAAO,CAAoC;IACnD,OAAO,CAAC,WAAW,CAAyB;IAC5C,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,KAAK,CAAqB;IAClC,OAAO,CAAC,MAAM,CAA4C;IAE1D,OAAO,CAAC,mBAAmB,CAAa;IACxC,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAM;gBAEnC,EACV,OAAO,EACP,WAAW,EACX,SAAc,EACd,MAAM,GACP,EAAE,8BAA8B,CAAC,cAAc,CAAC;IAQjD,OAAO,CAAC,yBAAyB;IAyBjC;;;OAGG;IACH,OAAO,CAAC,iCAAiC;YAS3B,cAAc;IAUtB,SAAS,IAAI,OAAO,CAAC,2BAA2B,CAAC;IA6CjD,kBAAkB;CAyFzB"}
|
|
@@ -1,14 +1,9 @@
|
|
|
1
|
-
import { ExternalSystemAttachmentStreamingFunction, NormalizedAttachment } from '../types';
|
|
2
1
|
import { ExtractionAdapter } from '../multithreading/adapters/extraction-adapter';
|
|
3
|
-
import {
|
|
2
|
+
import { NormalizedAttachment } from '../repo/repo.interfaces';
|
|
3
|
+
import { ExternalSystemAttachmentStreamingFunction } from '../types/extraction';
|
|
4
4
|
export interface AttachmentsStreamingPoolParams<ConnectorState> {
|
|
5
5
|
adapter: ExtractionAdapter<ConnectorState>;
|
|
6
6
|
attachments: NormalizedAttachment[];
|
|
7
|
-
/**
|
|
8
|
-
* SDK-owned attachments bookkeeping. Passed in directly so the pool does not
|
|
9
|
-
* reach into the (now-encapsulated) adapter state.
|
|
10
|
-
*/
|
|
11
|
-
attachmentsMetadata: ToDevRev['attachmentsMetadata'] | undefined;
|
|
12
7
|
batchSize?: number;
|
|
13
8
|
stream: ExternalSystemAttachmentStreamingFunction;
|
|
14
9
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"attachments-streaming-pool.interfaces.d.ts","sourceRoot":"","sources":["../../src/attachments-streaming/attachments-streaming-pool.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"attachments-streaming-pool.interfaces.d.ts","sourceRoot":"","sources":["../../src/attachments-streaming/attachments-streaming-pool.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,+CAA+C,CAAC;AAClF,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,yCAAyC,EAAE,MAAM,qBAAqB,CAAC;AAEhF,MAAM,WAAW,8BAA8B,CAAC,cAAc;IAC5D,OAAO,EAAE,iBAAiB,CAAC,cAAc,CAAC,CAAC;IAC3C,WAAW,EAAE,oBAAoB,EAAE,CAAC;IACpC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,yCAAyC,CAAC;CACnD"}
|
|
@@ -2,104 +2,105 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.AttachmentsStreamingPool = void 0;
|
|
4
4
|
const helpers_1 = require("../common/helpers");
|
|
5
|
+
const state_interfaces_1 = require("../state/state.interfaces");
|
|
5
6
|
class AttachmentsStreamingPool {
|
|
6
|
-
constructor({ adapter, attachments,
|
|
7
|
+
constructor({ adapter, attachments, batchSize = 10, stream, }) {
|
|
7
8
|
this.totalProcessedCount = 0;
|
|
8
9
|
this.PROGRESS_REPORT_INTERVAL = 50;
|
|
9
10
|
this.adapter = adapter;
|
|
10
11
|
this.attachments = [...attachments]; // Create a copy we can mutate
|
|
11
|
-
this.attachmentsMetadata = attachmentsMetadata;
|
|
12
12
|
this.batchSize = batchSize;
|
|
13
13
|
this.delay = undefined;
|
|
14
14
|
this.stream = stream;
|
|
15
15
|
}
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
16
|
+
recordProcessedAttachment(attachment, status) {
|
|
17
|
+
var _a;
|
|
18
|
+
const attachmentsMetadata = (_a = this.adapter.sdkState.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata;
|
|
19
|
+
if (!(attachmentsMetadata === null || attachmentsMetadata === void 0 ? void 0 : attachmentsMetadata.lastProcessedAttachmentsIdsList)) {
|
|
20
|
+
return;
|
|
21
|
+
}
|
|
22
|
+
const processedAttachmentsIdsList = attachmentsMetadata.lastProcessedAttachmentsIdsList;
|
|
23
|
+
const alreadyRecorded = processedAttachmentsIdsList.some((it) => it.id === attachment.id && it.parent_id === attachment.parent_id);
|
|
24
|
+
if (!alreadyRecorded) {
|
|
25
|
+
processedAttachmentsIdsList.push({
|
|
26
|
+
id: attachment.id,
|
|
27
|
+
parent_id: attachment.parent_id,
|
|
28
|
+
status,
|
|
29
|
+
});
|
|
22
30
|
}
|
|
23
31
|
}
|
|
24
32
|
/**
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
* @param attachments - The attachments list to migrate (either string[] or ProcessedAttachment[])
|
|
28
|
-
* @returns Migrated array of ProcessedAttachment objects, or empty array if input is invalid
|
|
33
|
+
* Backfills `status` on entries recorded before the field existed (those
|
|
34
|
+
* only ever held successes); state from >= v1.15.2 needs no shape migration.
|
|
29
35
|
*/
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
id: it,
|
|
44
|
-
parent_id: '',
|
|
45
|
-
}));
|
|
36
|
+
backfillProcessedAttachmentStatus(attachments) {
|
|
37
|
+
return attachments.map((it) => {
|
|
38
|
+
var _a;
|
|
39
|
+
return (Object.assign(Object.assign({}, it), { status: (_a = it.status) !== null && _a !== void 0 ? _a : state_interfaces_1.ProcessedAttachmentStatus.Success }));
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
async updateProgress() {
|
|
43
|
+
this.totalProcessedCount++;
|
|
44
|
+
if (this.totalProcessedCount % this.PROGRESS_REPORT_INTERVAL === 0) {
|
|
45
|
+
console.info(`Processed ${this.totalProcessedCount} attachments so far.`);
|
|
46
|
+
// Let a pending soft-timeout message (WorkerMessageExit) be delivered so
|
|
47
|
+
// adapter.isTimeout can flip before the next batch is processed.
|
|
48
|
+
await (0, helpers_1.yieldToEventLoop)();
|
|
46
49
|
}
|
|
47
|
-
return [];
|
|
48
50
|
}
|
|
49
51
|
async streamAll() {
|
|
50
52
|
console.log(`Starting download of ${this.attachments.length} attachments, streaming ${this.batchSize} at once.`);
|
|
51
|
-
if (!this.
|
|
53
|
+
if (!this.adapter.sdkState.toDevRev) {
|
|
52
54
|
const error = new Error('toDevRev state is not initialized');
|
|
53
55
|
console.error(error);
|
|
54
56
|
return { error };
|
|
55
57
|
}
|
|
56
|
-
//
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
this.attachmentsMetadata.lastProcessedAttachmentsIdsList =
|
|
58
|
+
// Attachments processed (successfully or not) by a previous, possibly incomplete, run.
|
|
59
|
+
if (!this.adapter.sdkState.toDevRev.attachmentsMetadata
|
|
60
|
+
.lastProcessedAttachmentsIdsList) {
|
|
61
|
+
this.adapter.sdkState.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList =
|
|
62
|
+
[];
|
|
60
63
|
}
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
// Start initial batch of promises up to batchSize limit
|
|
64
|
+
this.adapter.sdkState.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList =
|
|
65
|
+
this.backfillProcessedAttachmentStatus(this.adapter.sdkState.toDevRev.attachmentsMetadata
|
|
66
|
+
.lastProcessedAttachmentsIdsList);
|
|
65
67
|
const initialBatchSize = Math.min(this.batchSize, this.attachments.length);
|
|
66
68
|
const initialPromises = [];
|
|
67
69
|
for (let i = 0; i < initialBatchSize; i++) {
|
|
68
70
|
initialPromises.push(this.startPoolStreaming());
|
|
69
71
|
}
|
|
70
|
-
|
|
71
|
-
|
|
72
|
+
await Promise.race([
|
|
73
|
+
Promise.all(initialPromises),
|
|
74
|
+
this.adapter.timeoutSignal,
|
|
75
|
+
]);
|
|
72
76
|
if (this.delay) {
|
|
73
77
|
return { delay: this.delay };
|
|
74
78
|
}
|
|
75
79
|
return {};
|
|
76
80
|
}
|
|
77
81
|
async startPoolStreaming() {
|
|
78
|
-
var _a
|
|
79
|
-
// Process attachments until the attachments array is empty
|
|
82
|
+
var _a;
|
|
80
83
|
while (this.attachments.length > 0) {
|
|
81
|
-
// If delay is set, stop streaming
|
|
82
84
|
if (this.delay) {
|
|
83
85
|
break;
|
|
84
86
|
}
|
|
85
|
-
// If timeout is set, stop streaming
|
|
86
87
|
if (this.adapter.isTimeout) {
|
|
87
88
|
console.log('Timeout detected while streaming attachments. Stopping streaming.');
|
|
88
89
|
break;
|
|
89
90
|
}
|
|
90
|
-
// Check if we can process next attachment
|
|
91
91
|
const attachment = this.attachments.shift();
|
|
92
92
|
if (!attachment) {
|
|
93
|
-
break;
|
|
93
|
+
break;
|
|
94
94
|
}
|
|
95
|
-
if (
|
|
96
|
-
|
|
95
|
+
if (this.adapter.sdkState.toDevRev &&
|
|
96
|
+
((_a = this.adapter.sdkState.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList) === null || _a === void 0 ? void 0 : _a.some((it) => it.id == attachment.id && it.parent_id == attachment.parent_id))) {
|
|
97
|
+
continue; // Already processed in a previous run
|
|
97
98
|
}
|
|
98
99
|
try {
|
|
99
100
|
const response = await this.adapter.processAttachment(attachment, this.stream);
|
|
100
|
-
//
|
|
101
|
+
// Rate limit hit
|
|
101
102
|
if (response === null || response === void 0 ? void 0 : response.delay) {
|
|
102
|
-
this.delay = response.delay;
|
|
103
|
+
this.delay = response.delay;
|
|
103
104
|
return;
|
|
104
105
|
}
|
|
105
106
|
if (response === null || response === void 0 ? void 0 : response.error) {
|
|
@@ -111,15 +112,12 @@ class AttachmentsStreamingPool {
|
|
|
111
112
|
? `and content_type ${attachment.content_type} `
|
|
112
113
|
: '';
|
|
113
114
|
console.warn(`Skipping attachment with ID ${attachment.id} with extension ${fileExtension} ${fileSizeInfo}${contentTypeInfo}due to error returned by the stream function`, response.error.message);
|
|
115
|
+
this.recordProcessedAttachment(attachment, state_interfaces_1.ProcessedAttachmentStatus.Failed);
|
|
114
116
|
await this.updateProgress();
|
|
115
117
|
continue;
|
|
116
118
|
}
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
this.attachmentsMetadata.lastProcessedAttachmentsIdsList.push({
|
|
120
|
-
id: attachment.id,
|
|
121
|
-
parent_id: attachment.parent_id,
|
|
122
|
-
});
|
|
119
|
+
if (!this.adapter.isTimeout) {
|
|
120
|
+
this.recordProcessedAttachment(attachment, state_interfaces_1.ProcessedAttachmentStatus.Success);
|
|
123
121
|
}
|
|
124
122
|
await this.updateProgress();
|
|
125
123
|
}
|
|
@@ -15,6 +15,7 @@ export declare const MAX_DEVREV_FILENAME_EXTENSION_LENGTH = 20;
|
|
|
15
15
|
export declare const MAX_EVENT_SIZE_BYTES = 200000;
|
|
16
16
|
export declare const EVENT_SIZE_THRESHOLD_BYTES: number;
|
|
17
17
|
export declare const SSOR_ATTACHMENT: string;
|
|
18
|
+
export declare const UNKNOWN_EVENT_TYPE = "UNKNOWN_EVENT_TYPE";
|
|
18
19
|
export declare enum AirSyncDefaultItemTypes {
|
|
19
20
|
EXTERNAL_DOMAIN_METADATA = "external_domain_metadata",
|
|
20
21
|
ATTACHMENTS = "attachments",
|
|
@@ -26,11 +27,9 @@ export declare const HARD_TIMEOUT_MULTIPLIER = 1.3;
|
|
|
26
27
|
export declare const MEMORY_LOG_INTERVAL: number;
|
|
27
28
|
export declare const DEFAULT_SLEEP_DELAY_MS: number;
|
|
28
29
|
/**
|
|
29
|
-
* Sentinel
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
-
* no real extraction timestamp can be earlier, preventing accidental overwrites
|
|
33
|
-
* of the boundary by subsequent syncs (e.g. reconciliation with absolute dates).
|
|
30
|
+
* Sentinel for an unbounded extraction timestamp (TimeValueType.UNBOUNDED),
|
|
31
|
+
* stored as workersOldest. The Unix epoch guarantees no real timestamp can be
|
|
32
|
+
* earlier, so subsequent syncs cannot accidentally overwrite the boundary.
|
|
34
33
|
*/
|
|
35
34
|
export declare const UNBOUNDED_DATE_TIME_VALUE = "1970-01-01T00:00:00.000Z";
|
|
36
35
|
//# sourceMappingURL=constants.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/common/constants.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/common/constants.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAIhD,eAAO,MAAM,8BAA8B,aAS1C,CAAC;AAEF,eAAO,MAAM,2BAA2B,aAKvC,CAAC;AAEF,eAAO,MAAM,mBAAmB,aAG/B,CAAC;AAEF,eAAO,MAAM,gCAAgC,aAI5C,CAAC;AAEF,eAAO,MAAM,6BAA6B,aAGzC,CAAC;AAEF,eAAO,MAAM,qBAAqB,aAGjC,CAAC;AAEF,eAAO,MAAM,+BAA+B,aAGzC,CAAC;AAEJ,eAAO,MAAM,4BAA4B,aAExC,CAAC;AAEF,eAAO,MAAM,oBAAoB,aAGhC,CAAC;AAEF,eAAO,MAAM,mBAAmB,OAAO,CAAC;AACxC,eAAO,MAAM,wBAAwB,QAAyB,CAAC;AAC/D,eAAO,MAAM,0BAA0B,MAAM,CAAC;AAC9C,eAAO,MAAM,oCAAoC,KAAK,CAAC;AAGvD,eAAO,MAAM,oBAAoB,SAAU,CAAC;AAE5C,eAAO,MAAM,0BAA0B,QAEtC,CAAC;AAEF,eAAO,MAAM,eAAe,EAAE,MAA0B,CAAC;AACzD,eAAO,MAAM,kBAAkB,uBAAuB,CAAC;AAEvD,oBAAY,uBAAuB;IACjC,wBAAwB,6BAA6B;IACrD,WAAW,gBAAgB;IAC3B,mBAAmB,wBAAwB;CAC5C;AAED,eAAO,MAAM,eAAe,KAAsB,CAAC;AAEnD,eAAO,MAAM,sBAAsB,QAAiB,CAAC;AACrD,eAAO,MAAM,uBAAuB,MAAM,CAAC;AAC3C,eAAO,MAAM,mBAAmB,QAAY,CAAC;AAE7C,eAAO,MAAM,sBAAsB,QAAgB,CAAC;AAEpD;;;;GAIG;AACH,eAAO,MAAM,yBAAyB,6BAA6B,CAAC"}
|
package/dist/common/constants.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.UNBOUNDED_DATE_TIME_VALUE = exports.DEFAULT_SLEEP_DELAY_MS = exports.MEMORY_LOG_INTERVAL = exports.HARD_TIMEOUT_MULTIPLIER = exports.DEFAULT_LAMBDA_TIMEOUT = exports.LIBRARY_VERSION = exports.AirSyncDefaultItemTypes = exports.SSOR_ATTACHMENT = exports.EVENT_SIZE_THRESHOLD_BYTES = exports.MAX_EVENT_SIZE_BYTES = exports.MAX_DEVREV_FILENAME_EXTENSION_LENGTH = exports.MAX_DEVREV_FILENAME_LENGTH = exports.MAX_DEVREV_ARTIFACT_SIZE = exports.ARTIFACT_BATCH_SIZE = exports.STATEFUL_EVENT_TYPES = exports.STATEFUL_LOADING_EVENT_TYPES = exports.STATEFUL_EXTRACTION_EVENT_TYPES = exports.STATELESS_EVENT_TYPES = exports.STATELESS_LOADING_EVENT_TYPES = exports.STATELESS_EXTRACTION_EVENT_TYPES = exports.ALLOWED_EVENT_TYPES = exports.ALLOWED_LOADING_EVENT_TYPES = exports.ALLOWED_EXTRACTION_EVENT_TYPES = void 0;
|
|
3
|
+
exports.UNBOUNDED_DATE_TIME_VALUE = exports.DEFAULT_SLEEP_DELAY_MS = exports.MEMORY_LOG_INTERVAL = exports.HARD_TIMEOUT_MULTIPLIER = exports.DEFAULT_LAMBDA_TIMEOUT = exports.LIBRARY_VERSION = exports.AirSyncDefaultItemTypes = exports.UNKNOWN_EVENT_TYPE = exports.SSOR_ATTACHMENT = exports.EVENT_SIZE_THRESHOLD_BYTES = exports.MAX_EVENT_SIZE_BYTES = exports.MAX_DEVREV_FILENAME_EXTENSION_LENGTH = exports.MAX_DEVREV_FILENAME_LENGTH = exports.MAX_DEVREV_ARTIFACT_SIZE = exports.ARTIFACT_BATCH_SIZE = exports.STATEFUL_EVENT_TYPES = exports.STATEFUL_LOADING_EVENT_TYPES = exports.STATEFUL_EXTRACTION_EVENT_TYPES = exports.STATELESS_EVENT_TYPES = exports.STATELESS_LOADING_EVENT_TYPES = exports.STATELESS_EXTRACTION_EVENT_TYPES = exports.ALLOWED_EVENT_TYPES = exports.ALLOWED_LOADING_EVENT_TYPES = exports.ALLOWED_EXTRACTION_EVENT_TYPES = void 0;
|
|
4
4
|
const extraction_1 = require("../types/extraction");
|
|
5
5
|
const helpers_1 = require("./helpers");
|
|
6
6
|
exports.ALLOWED_EXTRACTION_EVENT_TYPES = [
|
|
@@ -45,12 +45,13 @@ exports.STATEFUL_EVENT_TYPES = [
|
|
|
45
45
|
exports.ARTIFACT_BATCH_SIZE = 2000;
|
|
46
46
|
exports.MAX_DEVREV_ARTIFACT_SIZE = 2 * 1024 * 1024 * 1024; // 2GB
|
|
47
47
|
exports.MAX_DEVREV_FILENAME_LENGTH = 256;
|
|
48
|
-
exports.MAX_DEVREV_FILENAME_EXTENSION_LENGTH = 20;
|
|
48
|
+
exports.MAX_DEVREV_FILENAME_EXTENSION_LENGTH = 20;
|
|
49
49
|
// Max SQS message size is 250KB, we want to leave some room for the other data in the message
|
|
50
50
|
exports.MAX_EVENT_SIZE_BYTES = 200000;
|
|
51
51
|
// We want to leave some room for the other data in the message and process the rest of queued messages
|
|
52
52
|
exports.EVENT_SIZE_THRESHOLD_BYTES = Math.floor(exports.MAX_EVENT_SIZE_BYTES * 0.8);
|
|
53
53
|
exports.SSOR_ATTACHMENT = 'ssor_attachment';
|
|
54
|
+
exports.UNKNOWN_EVENT_TYPE = 'UNKNOWN_EVENT_TYPE';
|
|
54
55
|
var AirSyncDefaultItemTypes;
|
|
55
56
|
(function (AirSyncDefaultItemTypes) {
|
|
56
57
|
AirSyncDefaultItemTypes["EXTERNAL_DOMAIN_METADATA"] = "external_domain_metadata";
|
|
@@ -63,10 +64,8 @@ exports.HARD_TIMEOUT_MULTIPLIER = 1.3;
|
|
|
63
64
|
exports.MEMORY_LOG_INTERVAL = 30 * 1000; // 30 seconds
|
|
64
65
|
exports.DEFAULT_SLEEP_DELAY_MS = 3 * 60 * 1000; // 3 minutes
|
|
65
66
|
/**
|
|
66
|
-
* Sentinel
|
|
67
|
-
*
|
|
68
|
-
*
|
|
69
|
-
* no real extraction timestamp can be earlier, preventing accidental overwrites
|
|
70
|
-
* of the boundary by subsequent syncs (e.g. reconciliation with absolute dates).
|
|
67
|
+
* Sentinel for an unbounded extraction timestamp (TimeValueType.UNBOUNDED),
|
|
68
|
+
* stored as workersOldest. The Unix epoch guarantees no real timestamp can be
|
|
69
|
+
* earlier, so subsequent syncs cannot accidentally overwrite the boundary.
|
|
71
70
|
*/
|
|
72
71
|
exports.UNBOUNDED_DATE_TIME_VALUE = '1970-01-01T00:00:00.000Z';
|
package/dist/common/helpers.d.ts
CHANGED
|
@@ -1,30 +1,10 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Gets the library version from the package.json file.
|
|
3
|
-
* @returns {string} The library version
|
|
4
|
-
*/
|
|
5
1
|
export declare function getLibraryVersion(): any;
|
|
6
|
-
/**
|
|
7
|
-
* Sleeps for a given number of milliseconds.
|
|
8
|
-
* @param {number} ms - The number of milliseconds to sleep
|
|
9
|
-
* @returns {Promise<void>} A promise that resolves after the given number of milliseconds
|
|
10
|
-
*/
|
|
11
2
|
export declare function sleep(ms: number): Promise<unknown>;
|
|
12
3
|
/**
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
* @returns {string} The truncated filename
|
|
16
|
-
*/
|
|
17
|
-
export declare function truncateFilename(filename: string): string;
|
|
18
|
-
/**
|
|
19
|
-
* MemoryInfo is an interface that represents the memory usage information.
|
|
20
|
-
* @interface MemoryInfo
|
|
21
|
-
* @property {string} rssUsedMB - The RSS used in MB
|
|
22
|
-
* @property {string} rssUsedPercent - The RSS used percentage
|
|
23
|
-
* @property {string} heapUsedPercent - The heap used percentage
|
|
24
|
-
* @property {string} externalMB - The external memory used in MB
|
|
25
|
-
* @property {string} arrayBuffersMB - The array buffers memory used in MB
|
|
26
|
-
* @property {string} formattedMessage - The formatted message
|
|
4
|
+
* Yields once to the event loop so pending events (e.g. worker messages) can
|
|
5
|
+
* be delivered before continuing.
|
|
27
6
|
*/
|
|
7
|
+
export declare function yieldToEventLoop(): Promise<void>;
|
|
28
8
|
export interface MemoryInfo {
|
|
29
9
|
rssUsedMB: string;
|
|
30
10
|
rssUsedPercent: string;
|
|
@@ -33,17 +13,5 @@ export interface MemoryInfo {
|
|
|
33
13
|
arrayBuffersMB: string;
|
|
34
14
|
formattedMessage: string;
|
|
35
15
|
}
|
|
36
|
-
/**
|
|
37
|
-
* Gets the memory usage information.
|
|
38
|
-
* @returns {MemoryInfo} The memory usage information
|
|
39
|
-
*/
|
|
40
16
|
export declare function getMemoryUsage(): MemoryInfo;
|
|
41
|
-
/**
|
|
42
|
-
* Truncates a message if it exceeds the maximum allowed length.
|
|
43
|
-
* Adds a suffix indicating how many characters were omitted.
|
|
44
|
-
*
|
|
45
|
-
* @param message - The message to truncate
|
|
46
|
-
* @returns Truncated message or original if within limits
|
|
47
|
-
*/
|
|
48
|
-
export declare function truncateMessage(message: string): string;
|
|
49
17
|
//# sourceMappingURL=helpers.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"helpers.d.ts","sourceRoot":"","sources":["../../src/common/helpers.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"helpers.d.ts","sourceRoot":"","sources":["../../src/common/helpers.ts"],"names":[],"mappings":"AAIA,wBAAgB,iBAAiB,QAiBhC;AAED,wBAAsB,KAAK,CAAC,EAAE,EAAE,MAAM,oBAErC;AAED;;;GAGG;AACH,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC,CAEtD;AAED,MAAM,WAAW,UAAU;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC;IACvB,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,wBAAgB,cAAc,IAAI,UAAU,CA+C3C"}
|
package/dist/common/helpers.js
CHANGED
|
@@ -35,18 +35,11 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
36
|
exports.getLibraryVersion = getLibraryVersion;
|
|
37
37
|
exports.sleep = sleep;
|
|
38
|
-
exports.
|
|
38
|
+
exports.yieldToEventLoop = yieldToEventLoop;
|
|
39
39
|
exports.getMemoryUsage = getMemoryUsage;
|
|
40
|
-
exports.truncateMessage = truncateMessage;
|
|
41
40
|
const fs_1 = require("fs");
|
|
42
41
|
const path = __importStar(require("path"));
|
|
43
42
|
const v8 = __importStar(require("v8"));
|
|
44
|
-
const constants_1 = require("./constants");
|
|
45
|
-
const logger_constants_1 = require("../logger/logger.constants");
|
|
46
|
-
/**
|
|
47
|
-
* Gets the library version from the package.json file.
|
|
48
|
-
* @returns {string} The library version
|
|
49
|
-
*/
|
|
50
43
|
function getLibraryVersion() {
|
|
51
44
|
var _a;
|
|
52
45
|
try {
|
|
@@ -61,37 +54,16 @@ function getLibraryVersion() {
|
|
|
61
54
|
return '';
|
|
62
55
|
}
|
|
63
56
|
}
|
|
64
|
-
/**
|
|
65
|
-
* Sleeps for a given number of milliseconds.
|
|
66
|
-
* @param {number} ms - The number of milliseconds to sleep
|
|
67
|
-
* @returns {Promise<void>} A promise that resolves after the given number of milliseconds
|
|
68
|
-
*/
|
|
69
57
|
async function sleep(ms) {
|
|
70
|
-
console.log(`Sleeping for ${ms}ms.`);
|
|
71
58
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
72
59
|
}
|
|
73
60
|
/**
|
|
74
|
-
*
|
|
75
|
-
*
|
|
76
|
-
* @returns {string} The truncated filename
|
|
61
|
+
* Yields once to the event loop so pending events (e.g. worker messages) can
|
|
62
|
+
* be delivered before continuing.
|
|
77
63
|
*/
|
|
78
|
-
function
|
|
79
|
-
|
|
80
|
-
if (filename.length <= constants_1.MAX_DEVREV_FILENAME_LENGTH) {
|
|
81
|
-
return filename;
|
|
82
|
-
}
|
|
83
|
-
console.warn(`Filename length exceeds the maximum limit of ${constants_1.MAX_DEVREV_FILENAME_LENGTH} characters. Truncating filename.`);
|
|
84
|
-
const extension = filename.slice(-constants_1.MAX_DEVREV_FILENAME_EXTENSION_LENGTH);
|
|
85
|
-
// Calculate how many characters are available for the name part after accounting for the extension and "..."
|
|
86
|
-
const availableNameLength = constants_1.MAX_DEVREV_FILENAME_LENGTH - constants_1.MAX_DEVREV_FILENAME_EXTENSION_LENGTH - 3; // -3 for "..."
|
|
87
|
-
// Truncate the name part and add an ellipsis
|
|
88
|
-
const truncatedFilename = filename.slice(0, availableNameLength);
|
|
89
|
-
return `${truncatedFilename}...${extension}`;
|
|
64
|
+
async function yieldToEventLoop() {
|
|
65
|
+
return new Promise((resolve) => setImmediate(resolve));
|
|
90
66
|
}
|
|
91
|
-
/**
|
|
92
|
-
* Gets the memory usage information.
|
|
93
|
-
* @returns {MemoryInfo} The memory usage information
|
|
94
|
-
*/
|
|
95
67
|
function getMemoryUsage() {
|
|
96
68
|
try {
|
|
97
69
|
const memUsage = process.memoryUsage();
|
|
@@ -99,16 +71,12 @@ function getMemoryUsage() {
|
|
|
99
71
|
const rssUsedMB = memUsage.rss / 1024 / 1024;
|
|
100
72
|
const heapLimitMB = heapStats.heap_size_limit / 1024 / 1024;
|
|
101
73
|
const effectiveMemoryLimitMB = heapLimitMB;
|
|
102
|
-
// Calculate heap values for consistent format
|
|
103
74
|
const heapUsedMB = heapStats.used_heap_size / 1024 / 1024;
|
|
104
75
|
const heapTotalMB = heapStats.heap_size_limit / 1024 / 1024;
|
|
105
|
-
// Calculate external and buffer values (critical for detecting stream leaks)
|
|
106
76
|
const externalMB = memUsage.external / 1024 / 1024;
|
|
107
77
|
const arrayBuffersMB = memUsage.arrayBuffers / 1024 / 1024;
|
|
108
|
-
// Critical percentages for OOM detection
|
|
109
78
|
const rssUsedPercent = ((rssUsedMB / effectiveMemoryLimitMB) * 100).toFixed(2) + '%';
|
|
110
79
|
const heapUsedPercent = ((heapStats.used_heap_size / heapStats.heap_size_limit) * 100).toFixed(2) + '%';
|
|
111
|
-
// Detailed message showing RSS breakdown for leak detection
|
|
112
80
|
const formattedMessage = `Memory: RSS ${rssUsedMB.toFixed(2)}/${effectiveMemoryLimitMB.toFixed(2)}MB (${rssUsedPercent}) [Heap ${heapUsedMB.toFixed(2)}/${heapTotalMB.toFixed(2)}MB (${heapUsedPercent}) + External ${externalMB.toFixed(2)}MB + Buffers ${arrayBuffersMB.toFixed(2)}MB].`;
|
|
113
81
|
return {
|
|
114
82
|
rssUsedMB: rssUsedMB.toFixed(2),
|
|
@@ -124,16 +92,3 @@ function getMemoryUsage() {
|
|
|
124
92
|
throw err;
|
|
125
93
|
}
|
|
126
94
|
}
|
|
127
|
-
/**
|
|
128
|
-
* Truncates a message if it exceeds the maximum allowed length.
|
|
129
|
-
* Adds a suffix indicating how many characters were omitted.
|
|
130
|
-
*
|
|
131
|
-
* @param message - The message to truncate
|
|
132
|
-
* @returns Truncated message or original if within limits
|
|
133
|
-
*/
|
|
134
|
-
function truncateMessage(message) {
|
|
135
|
-
if (message.length > logger_constants_1.MAX_LOG_STRING_LENGTH) {
|
|
136
|
-
return `${message.substring(0, logger_constants_1.MAX_LOG_STRING_LENGTH)}... ${message.length - logger_constants_1.MAX_LOG_STRING_LENGTH} more characters`;
|
|
137
|
-
}
|
|
138
|
-
return message;
|
|
139
|
-
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../src/http/client.ts"],"names":[],"mappings":"AAGA,QAAA,MAAM,WAAW,+BAEf,CAAC;AAmFH,OAAO,EAAE,WAAW,EAAE,CAAC"}
|
|
@@ -34,11 +34,10 @@ exports.axiosClient = axiosClient;
|
|
|
34
34
|
var _a, _b, _c, _d, _e, _f, _g;
|
|
35
35
|
const retryAfter = ((_b = (_a = error.response) === null || _a === void 0 ? void 0 : _a.headers) === null || _b === void 0 ? void 0 : _b['retry-after']) ||
|
|
36
36
|
((_d = (_c = error.response) === null || _c === void 0 ? void 0 : _c.headers) === null || _d === void 0 ? void 0 : _d['Retry-After']);
|
|
37
|
-
// 5xx errors
|
|
38
37
|
if (((_e = error.response) === null || _e === void 0 ? void 0 : _e.status) && error.response.status >= 500) {
|
|
39
38
|
return true;
|
|
40
39
|
}
|
|
41
|
-
// 429
|
|
40
|
+
// 429 only when a valid non-negative Retry-After header is present
|
|
42
41
|
else if (((_f = error.response) === null || _f === void 0 ? void 0 : _f.status) &&
|
|
43
42
|
error.response.status === 429 &&
|
|
44
43
|
retryAfter &&
|
|
@@ -46,19 +45,16 @@ exports.axiosClient = axiosClient;
|
|
|
46
45
|
Number(retryAfter) >= 0) {
|
|
47
46
|
return true;
|
|
48
47
|
}
|
|
49
|
-
// Network errors for idempotent requests
|
|
48
|
+
// Network errors for idempotent requests; 429 already handled above
|
|
50
49
|
else if (axios_retry_1.default.isNetworkOrIdempotentRequestError(error) &&
|
|
51
50
|
((_g = error.response) === null || _g === void 0 ? void 0 : _g.status) !== 429) {
|
|
52
51
|
return true;
|
|
53
52
|
}
|
|
54
|
-
//
|
|
55
|
-
//
|
|
56
|
-
// Axios only produces ECONNABORTED on client-side timeouts and browser
|
|
57
|
-
// cancellations, never on server responses, so no response guard is needed.
|
|
53
|
+
// axios-retry excludes ECONNABORTED (client-side timeout only, never a
|
|
54
|
+
// server response) from isNetworkError, so handle it here separately.
|
|
58
55
|
else if (error.code === 'ECONNABORTED') {
|
|
59
56
|
return true;
|
|
60
57
|
}
|
|
61
|
-
// all other errors
|
|
62
58
|
else {
|
|
63
59
|
return false;
|
|
64
60
|
}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,16 +1,27 @@
|
|
|
1
|
-
export { AirSyncDefaultItemTypes } from './common/constants';
|
|
2
|
-
export { ExtractionCommonError } from './common/errors';
|
|
3
|
-
export * from './common/install-initial-domain-mapping';
|
|
4
|
-
export * from './http';
|
|
5
|
-
export { serializeAxiosError } from './logger/logger';
|
|
6
|
-
export { MockServer } from './mock-server/mock-server';
|
|
7
|
-
export type { RequestInfo, RetryConfig, RouteConfig, } from './mock-server/mock-server.interfaces';
|
|
8
|
-
export { processExtractionTask, processLoadingTask, } from './multithreading/process-task';
|
|
9
|
-
export { spawn } from './multithreading/spawn/spawn';
|
|
10
1
|
export { ExtractionAdapter } from './multithreading/adapters/extraction-adapter';
|
|
11
2
|
export { LoadingAdapter } from './multithreading/adapters/loading-adapter';
|
|
12
|
-
export {
|
|
13
|
-
export
|
|
14
|
-
export
|
|
15
|
-
export
|
|
3
|
+
export { processExtractionTask, processLoadingTask, } from './multithreading/process-task';
|
|
4
|
+
export { spawn } from './multithreading/spawn/spawn';
|
|
5
|
+
export type { ExtractionScope, ProcessTaskInterface, SpawnFactoryInterface, SpawnInterface, TaskAdapterInterface, TaskResult, TaskStatus, WorkerAdapterInterface, WorkerAdapterOptions, WorkerPathOverrides, } from './types/workers';
|
|
6
|
+
export { AirSyncDefaultItemTypes } from './common/constants';
|
|
7
|
+
export { UNBOUNDED_DATE_TIME_VALUE } from './common/constants';
|
|
8
|
+
export { ExtractionCommonError } from './types/errors';
|
|
9
|
+
export { installInitialDomainMapping } from './state/install-initial-domain-mapping';
|
|
10
|
+
export { serializeError } from './logger/logger';
|
|
11
|
+
export type { ErrorRecord, InitialDomainMapping } from './types/common';
|
|
12
|
+
export { SyncMode } from './types/common';
|
|
13
|
+
export type { AirSyncEvent, AirSyncMessage, ConnectionData, EventContext, EventData, ExternalProcessAttachmentFunction, ExternalSyncUnit, ExternalSystemAttachmentIteratorFunction, ExternalSystemAttachmentReducerFunction, ExternalSystemAttachmentStreamingFunction, ExternalSystemAttachmentStreamingParams, ExternalSystemAttachmentStreamingResponse, ExtractorEvent, HttpStreamResponse, ProcessAttachmentReturnType, TimeValue, } from './types/extraction';
|
|
14
|
+
export { EventType, ExtractorEventType, InitialSyncScope, TimeUnit, TimeValueType, } from './types/extraction';
|
|
15
|
+
export type { ExternalSystemAttachment, ExternalSystemItem, ExternalSystemItemLoadingParams, ExternalSystemItemLoadingResponse, ItemTypeToLoad, } from './types/loading';
|
|
16
|
+
export { LoaderEventType } from './types/loading';
|
|
17
|
+
export type { Item, NormalizedAttachment, NormalizedItem, RepoInterface, } from './repo/repo.interfaces';
|
|
18
|
+
export { Mappers } from './mappers/mappers';
|
|
19
|
+
export type { MappersCreateParams, MappersGetByExternalIdParams, MappersGetByTargetIdParams, MappersUpdateParams, } from './mappers/mappers.interfaces';
|
|
20
|
+
export { SyncMapperRecordStatus, SyncMapperRecordTargetType, } from './mappers/mappers.interfaces';
|
|
21
|
+
export type { Artifact, ArtifactsPrepareResponse, SsorAttachment, StreamAttachmentsResponse, StreamResponse, UploadResponse, } from './uploader/uploader.interfaces';
|
|
22
|
+
export type { CollectionData, ConditionalPrivilegeData, CustomLinkData, CustomLinkNames, CustomStage, CustomState, EnumData, EnumValue, EnumValueKey, ExternalDomainMetadata, Field, FieldCondition, FieldConditionComparator, FieldConditionEffect, FieldConditions, FieldKey, FieldPrivilegeData, FieldReferenceData, FieldType, FloatData, IntData, PermissionData, RecordType, RecordTypeCategory, RecordTypeCategoryKey, RecordTypeKey, RecordTypePrivilegeData, RecordTypeScope, ReferenceData, ReferenceDetail, ReferenceType, SchemaVersion, StageDiagram, StageKey, StateKey, StructData, StructType, StructTypeKey, TargetTypeKeyData, TextData, TypedReferenceData, } from './types/external-domain-metadata';
|
|
23
|
+
export type { DeepPartial } from './testing/mock-event';
|
|
24
|
+
export { createMockEvent } from './testing/mock-event';
|
|
25
|
+
export { MOCK_SERVER_DEFAULT_URL, MockServer } from './testing/mock-server';
|
|
26
|
+
export type { RequestInfo, RetryConfig, RouteConfig, } from './testing/mock-server.interfaces';
|
|
16
27
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,iBAAiB,EAAE,MAAM,8CAA8C,CAAC;AACjF,OAAO,EAAE,cAAc,EAAE,MAAM,2CAA2C,CAAC;AAC3E,OAAO,EACL,qBAAqB,EACrB,kBAAkB,GACnB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,KAAK,EAAE,MAAM,8BAA8B,CAAC;AAGrD,YAAY,EACV,eAAe,EACf,oBAAoB,EACpB,qBAAqB,EACrB,cAAc,EACd,oBAAoB,EACpB,UAAU,EACV,UAAU,EACV,sBAAsB,EACtB,oBAAoB,EACpB,mBAAmB,GACpB,MAAM,iBAAiB,CAAC;AAGzB,OAAO,EAAE,uBAAuB,EAAE,MAAM,oBAAoB,CAAC;AAC7D,OAAO,EAAE,yBAAyB,EAAE,MAAM,oBAAoB,CAAC;AAC/D,OAAO,EAAE,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;AAGvD,OAAO,EAAE,2BAA2B,EAAE,MAAM,wCAAwC,CAAC;AAGrF,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAGjD,YAAY,EAAE,WAAW,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AACxE,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAG1C,YAAY,EACV,YAAY,EACZ,cAAc,EACd,cAAc,EACd,YAAY,EACZ,SAAS,EACT,iCAAiC,EACjC,gBAAgB,EAChB,wCAAwC,EACxC,uCAAuC,EACvC,yCAAyC,EACzC,uCAAuC,EACvC,yCAAyC,EACzC,cAAc,EACd,kBAAkB,EAClB,2BAA2B,EAC3B,SAAS,GACV,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,SAAS,EACT,kBAAkB,EAClB,gBAAgB,EAChB,QAAQ,EACR,aAAa,GACd,MAAM,oBAAoB,CAAC;AAG5B,YAAY,EACV,wBAAwB,EACxB,kBAAkB,EAClB,+BAA+B,EAC/B,iCAAiC,EACjC,cAAc,GACf,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAGlD,YAAY,EACV,IAAI,EACJ,oBAAoB,EACpB,cAAc,EACd,aAAa,GACd,MAAM,wBAAwB,CAAC;AAGhC,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAC5C,YAAY,EACV,mBAAmB,EACnB,4BAA4B,EAC5B,0BAA0B,EAC1B,mBAAmB,GACpB,MAAM,8BAA8B,CAAC;AACtC,OAAO,EACL,sBAAsB,EACtB,0BAA0B,GAC3B,MAAM,8BAA8B,CAAC;AAGtC,YAAY,EACV,QAAQ,EACR,wBAAwB,EACxB,cAAc,EACd,yBAAyB,EACzB,cAAc,EACd,cAAc,GACf,MAAM,gCAAgC,CAAC;AAGxC,YAAY,EACV,cAAc,EACd,wBAAwB,EACxB,cAAc,EACd,eAAe,EACf,WAAW,EACX,WAAW,EACX,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,sBAAsB,EACtB,KAAK,EACL,cAAc,EACd,wBAAwB,EACxB,oBAAoB,EACpB,eAAe,EACf,QAAQ,EACR,kBAAkB,EAClB,kBAAkB,EAClB,SAAS,EACT,SAAS,EACT,OAAO,EACP,cAAc,EACd,UAAU,EACV,kBAAkB,EAClB,qBAAqB,EACrB,aAAa,EACb,uBAAuB,EACvB,eAAe,EACf,aAAa,EACb,eAAe,EACf,aAAa,EACb,aAAa,EACb,YAAY,EACZ,QAAQ,EACR,QAAQ,EACR,UAAU,EACV,UAAU,EACV,aAAa,EACb,iBAAiB,EACjB,QAAQ,EACR,kBAAkB,GACnB,MAAM,kCAAkC,CAAC;AAG1C,YAAY,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,EAAE,uBAAuB,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAC5E,YAAY,EACV,WAAW,EACX,WAAW,EACX,WAAW,GACZ,MAAM,kCAAkC,CAAC"}
|