@devrev/airsync-sdk 2.0.0-beta.2 → 2.0.0-beta.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -11
- package/dist/attachments-streaming/attachments-streaming-pool.d.ts +1 -8
- package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map +1 -1
- package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts +4 -3
- package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts.map +1 -1
- package/dist/attachments-streaming/attachments-streaming-pool.js +11 -39
- package/dist/common/constants.d.ts +1 -0
- package/dist/common/constants.d.ts.map +1 -1
- package/dist/common/constants.js +2 -1
- package/dist/common/helpers.d.ts +0 -14
- package/dist/common/helpers.d.ts.map +1 -1
- package/dist/common/helpers.js +0 -36
- package/dist/http/{axios-client-internal.d.ts → client.d.ts} +1 -1
- package/dist/http/client.d.ts.map +1 -0
- package/dist/index.d.ts +25 -13
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +51 -30
- package/dist/logger/logger.d.ts +14 -0
- package/dist/logger/logger.d.ts.map +1 -1
- package/dist/logger/logger.js +55 -8
- package/dist/mappers/mappers.d.ts +9 -10
- package/dist/mappers/mappers.d.ts.map +1 -1
- package/dist/mappers/{mappers.interface.d.ts → mappers.interfaces.d.ts} +5 -5
- package/dist/mappers/mappers.interfaces.d.ts.map +1 -0
- package/dist/mappers/{mappers.interface.js → mappers.interfaces.js} +3 -3
- package/dist/mappers/mappers.js +13 -9
- package/dist/multithreading/adapters/base-adapter.d.ts +71 -0
- package/dist/multithreading/adapters/base-adapter.d.ts.map +1 -0
- package/dist/multithreading/adapters/base-adapter.js +138 -0
- package/dist/multithreading/adapters/extraction-adapter.d.ts +57 -0
- package/dist/multithreading/adapters/extraction-adapter.d.ts.map +1 -0
- package/dist/multithreading/adapters/extraction-adapter.js +323 -0
- package/dist/multithreading/adapters/loading-adapter.d.ts +44 -0
- package/dist/multithreading/adapters/loading-adapter.d.ts.map +1 -0
- package/dist/multithreading/{worker-adapter/worker-adapter.helpers.d.ts → adapters/loading-adapter.helpers.d.ts} +1 -1
- package/dist/multithreading/adapters/loading-adapter.helpers.d.ts.map +1 -0
- package/dist/multithreading/adapters/loading-adapter.js +431 -0
- package/dist/{common/control-protocol.d.ts → multithreading/emit.d.ts} +1 -1
- package/dist/multithreading/emit.d.ts.map +1 -0
- package/dist/{common/control-protocol.js → multithreading/emit.js} +3 -3
- package/dist/multithreading/process-task.d.ts +16 -1
- package/dist/multithreading/process-task.d.ts.map +1 -1
- package/dist/multithreading/process-task.js +115 -50
- package/dist/multithreading/spawn/spawn.helpers.d.ts +23 -3
- package/dist/multithreading/spawn/spawn.helpers.d.ts.map +1 -1
- package/dist/multithreading/spawn/spawn.helpers.js +152 -2
- package/dist/multithreading/spawn/spawn.js +7 -7
- package/dist/multithreading/worker.js +1 -2
- package/dist/repo/repo.js +2 -2
- package/dist/state/base-state.d.ts +75 -0
- package/dist/state/base-state.d.ts.map +1 -0
- package/dist/state/base-state.js +261 -0
- package/dist/state/extraction-state.d.ts +29 -0
- package/dist/state/extraction-state.d.ts.map +1 -0
- package/dist/state/extraction-state.js +122 -0
- package/dist/state/install-initial-domain-mapping.d.ts.map +1 -0
- package/dist/{common → state}/install-initial-domain-mapping.js +4 -4
- package/dist/state/loading-state.d.ts +18 -0
- package/dist/state/loading-state.d.ts.map +1 -0
- package/dist/state/loading-state.js +39 -0
- package/dist/state/state.d.ts +3 -35
- package/dist/state/state.d.ts.map +1 -1
- package/dist/state/state.interfaces.d.ts +20 -3
- package/dist/state/state.interfaces.d.ts.map +1 -1
- package/dist/state/state.interfaces.js +19 -1
- package/dist/state/state.js +9 -263
- package/dist/{common → state}/time-value-resolver.d.ts +4 -5
- package/dist/state/time-value-resolver.d.ts.map +1 -0
- package/dist/{common → state}/time-value-resolver.js +9 -8
- package/dist/{common/test-utils.d.ts → testing/mock-event.d.ts} +1 -2
- package/dist/testing/mock-event.d.ts.map +1 -0
- package/dist/{common/test-utils.js → testing/mock-event.js} +14 -3
- package/dist/{mock-server → testing}/mock-server.d.ts +5 -0
- package/dist/testing/mock-server.d.ts.map +1 -0
- package/dist/testing/mock-server.interfaces.d.ts.map +1 -0
- package/dist/{mock-server → testing}/mock-server.js +6 -1
- package/dist/types/errors.d.ts +9 -0
- package/dist/types/errors.d.ts.map +1 -0
- package/dist/types/extraction.d.ts +73 -18
- package/dist/types/extraction.d.ts.map +1 -1
- package/dist/types/extraction.js +4 -6
- package/dist/types/loading.d.ts +1 -2
- package/dist/types/loading.d.ts.map +1 -1
- package/dist/types/loading.js +0 -1
- package/dist/types/workers.d.ts +64 -20
- package/dist/types/workers.d.ts.map +1 -1
- package/dist/uploader/uploader.d.ts +5 -4
- package/dist/uploader/uploader.d.ts.map +1 -1
- package/dist/uploader/uploader.interfaces.d.ts +1 -1
- package/dist/uploader/uploader.js +10 -10
- package/package.json +3 -3
- package/dist/airsync-sdk.d.ts +0 -1802
- package/dist/attachments-streaming/attachments-streaming-pool.test.d.ts +0 -2
- package/dist/attachments-streaming/attachments-streaming-pool.test.d.ts.map +0 -1
- package/dist/attachments-streaming/attachments-streaming-pool.test.js +0 -516
- package/dist/common/control-protocol.d.ts.map +0 -1
- package/dist/common/errors.d.ts +0 -6
- package/dist/common/errors.d.ts.map +0 -1
- package/dist/common/install-initial-domain-mapping.d.ts.map +0 -1
- package/dist/common/install-initial-domain-mapping.test.d.ts +0 -2
- package/dist/common/install-initial-domain-mapping.test.d.ts.map +0 -1
- package/dist/common/install-initial-domain-mapping.test.js +0 -211
- package/dist/common/test-utils.d.ts.map +0 -1
- package/dist/common/time-value-resolver.d.ts.map +0 -1
- package/dist/common/time-value-resolver.test.d.ts +0 -2
- package/dist/common/time-value-resolver.test.d.ts.map +0 -1
- package/dist/common/time-value-resolver.test.js +0 -268
- package/dist/http/axios-client-internal.d.ts.map +0 -1
- package/dist/http/axios-client-internal.test.d.ts +0 -2
- package/dist/http/axios-client-internal.test.d.ts.map +0 -1
- package/dist/http/axios-client-internal.test.js +0 -175
- package/dist/http/axios-client.d.ts +0 -27
- package/dist/http/axios-client.d.ts.map +0 -1
- package/dist/http/axios-client.js +0 -60
- package/dist/http/constants.d.ts +0 -4
- package/dist/http/constants.d.ts.map +0 -1
- package/dist/http/constants.js +0 -6
- package/dist/http/index.d.ts +0 -2
- package/dist/http/index.d.ts.map +0 -1
- package/dist/http/index.js +0 -17
- package/dist/logger/logger.constants.d.ts +0 -6
- package/dist/logger/logger.constants.d.ts.map +0 -1
- package/dist/logger/logger.constants.js +0 -13
- package/dist/logger/logger.context.d.ts +0 -58
- package/dist/logger/logger.context.d.ts.map +0 -1
- package/dist/logger/logger.context.js +0 -86
- package/dist/logger/logger.test.d.ts +0 -2
- package/dist/logger/logger.test.d.ts.map +0 -1
- package/dist/logger/logger.test.js +0 -537
- package/dist/mappers/mappers.interface.d.ts.map +0 -1
- package/dist/mappers/mappers.test.d.ts +0 -2
- package/dist/mappers/mappers.test.d.ts.map +0 -1
- package/dist/mappers/mappers.test.js +0 -105
- package/dist/mock-server/mock-server.d.ts.map +0 -1
- package/dist/mock-server/mock-server.interfaces.d.ts.map +0 -1
- package/dist/multithreading/create-worker.test.d.ts +0 -2
- package/dist/multithreading/create-worker.test.d.ts.map +0 -1
- package/dist/multithreading/create-worker.test.js +0 -108
- package/dist/multithreading/process-task.test.d.ts +0 -2
- package/dist/multithreading/process-task.test.d.ts.map +0 -1
- package/dist/multithreading/process-task.test.js +0 -146
- package/dist/multithreading/spawn/spawn.helpers.test.d.ts +0 -2
- package/dist/multithreading/spawn/spawn.helpers.test.d.ts.map +0 -1
- package/dist/multithreading/spawn/spawn.helpers.test.js +0 -293
- package/dist/multithreading/spawn/spawn.test.d.ts +0 -2
- package/dist/multithreading/spawn/spawn.test.d.ts.map +0 -1
- package/dist/multithreading/spawn/spawn.test.js +0 -232
- package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.d.ts +0 -2
- package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.d.ts.map +0 -1
- package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.js +0 -175
- package/dist/multithreading/worker-adapter/worker-adapter.d.ts +0 -97
- package/dist/multithreading/worker-adapter/worker-adapter.d.ts.map +0 -1
- package/dist/multithreading/worker-adapter/worker-adapter.emit.test.d.ts +0 -2
- package/dist/multithreading/worker-adapter/worker-adapter.emit.test.d.ts.map +0 -1
- package/dist/multithreading/worker-adapter/worker-adapter.emit.test.js +0 -390
- package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.d.ts +0 -2
- package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.d.ts.map +0 -1
- package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.js +0 -799
- package/dist/multithreading/worker-adapter/worker-adapter.helpers.d.ts.map +0 -1
- package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.d.ts +0 -2
- package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.d.ts.map +0 -1
- package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.js +0 -514
- package/dist/multithreading/worker-adapter/worker-adapter.js +0 -846
- package/dist/multithreading/worker-adapter/worker-adapter.loading.test.d.ts +0 -2
- package/dist/multithreading/worker-adapter/worker-adapter.loading.test.d.ts.map +0 -1
- package/dist/multithreading/worker-adapter/worker-adapter.loading.test.js +0 -596
- package/dist/multithreading/worker-adapter/worker-adapter.serialization.test.d.ts +0 -2
- package/dist/multithreading/worker-adapter/worker-adapter.serialization.test.d.ts.map +0 -1
- package/dist/multithreading/worker-adapter/worker-adapter.serialization.test.js +0 -71
- package/dist/repo/repo.test.d.ts +0 -2
- package/dist/repo/repo.test.d.ts.map +0 -1
- package/dist/repo/repo.test.js +0 -184
- package/dist/state/state.extract-window.test.d.ts +0 -2
- package/dist/state/state.extract-window.test.d.ts.map +0 -1
- package/dist/state/state.extract-window.test.js +0 -163
- package/dist/state/state.pending-boundaries.test.d.ts +0 -2
- package/dist/state/state.pending-boundaries.test.d.ts.map +0 -1
- package/dist/state/state.pending-boundaries.test.js +0 -189
- package/dist/state/state.post-state.test.d.ts +0 -2
- package/dist/state/state.post-state.test.d.ts.map +0 -1
- package/dist/state/state.post-state.test.js +0 -77
- package/dist/state/state.test.d.ts +0 -2
- package/dist/state/state.test.d.ts.map +0 -1
- package/dist/state/state.test.js +0 -268
- package/dist/state/state.time-value-resolution.test.d.ts +0 -2
- package/dist/state/state.time-value-resolution.test.d.ts.map +0 -1
- package/dist/state/state.time-value-resolution.test.js +0 -175
- package/dist/tsdoc-metadata.json +0 -11
- package/dist/types/extraction.test.d.ts +0 -2
- package/dist/types/extraction.test.d.ts.map +0 -1
- package/dist/types/extraction.test.js +0 -96
- package/dist/types/index.d.ts +0 -11
- package/dist/types/index.d.ts.map +0 -1
- package/dist/types/index.js +0 -21
- package/dist/uploader/uploader.helpers.test.d.ts +0 -2
- package/dist/uploader/uploader.helpers.test.d.ts.map +0 -1
- package/dist/uploader/uploader.helpers.test.js +0 -256
- package/dist/uploader/uploader.test.d.ts +0 -2
- package/dist/uploader/uploader.test.d.ts.map +0 -1
- package/dist/uploader/uploader.test.js +0 -601
- /package/dist/http/{axios-client-internal.js → client.js} +0 -0
- /package/dist/multithreading/{worker-adapter/worker-adapter.helpers.js → adapters/loading-adapter.helpers.js} +0 -0
- /package/dist/{common → state}/install-initial-domain-mapping.d.ts +0 -0
- /package/dist/{mock-server → testing}/mock-server.interfaces.d.ts +0 -0
- /package/dist/{mock-server → testing}/mock-server.interfaces.js +0 -0
- /package/dist/{common → types}/errors.js +0 -0
package/README.md
CHANGED
|
@@ -1,16 +1,14 @@
|
|
|
1
|
-
#
|
|
1
|
+
# Airdrop SDK
|
|
2
2
|
|
|
3
|
-
[. The package has been renamed from `@devrev/ts-adaas` to `@devrev/airsync-sdk`.
|
|
3
|
+
[](https://coveralls.io/github/devrev/adaas-sdk?branch=main)
|
|
6
4
|
|
|
7
5
|
## Overview
|
|
8
6
|
|
|
9
|
-
The
|
|
7
|
+
The Airdrop SDK for TypeScript helps developers build snap-ins that integrate with DevRev’s Airdrop platform. This SDK simplifies the workflow for handling data extraction and loading, event-driven actions, state management, and artifact handling.
|
|
10
8
|
|
|
11
9
|
It provides features such as:
|
|
12
10
|
|
|
13
|
-
- Type Definitions: Structured types for
|
|
11
|
+
- Type Definitions: Structured types for Airdrop control protocol
|
|
14
12
|
- Event Management: Easily emit events for different extraction or loading phases
|
|
15
13
|
- State Handling: Update and access state in real-time within tasks
|
|
16
14
|
- Artifact Management: Supports batched storage of artifacts
|
|
@@ -19,9 +17,5 @@ It provides features such as:
|
|
|
19
17
|
## Installation
|
|
20
18
|
|
|
21
19
|
```bash
|
|
22
|
-
npm install @devrev/airsync-sdk
|
|
20
|
+
npm install @devrev/airsync-sdk
|
|
23
21
|
```
|
|
24
|
-
|
|
25
|
-
## Changelog
|
|
26
|
-
|
|
27
|
-
See [CHANGELOG.md](./CHANGELOG.md) for release notes and migration guides.
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { ProcessAttachmentReturnType } from '../types';
|
|
1
|
+
import { ProcessAttachmentReturnType } from '../types/extraction';
|
|
2
2
|
import { AttachmentsStreamingPoolParams } from './attachments-streaming-pool.interfaces';
|
|
3
3
|
export declare class AttachmentsStreamingPool<ConnectorState> {
|
|
4
4
|
private adapter;
|
|
@@ -10,13 +10,6 @@ export declare class AttachmentsStreamingPool<ConnectorState> {
|
|
|
10
10
|
private readonly PROGRESS_REPORT_INTERVAL;
|
|
11
11
|
constructor({ adapter, attachments, batchSize, stream, }: AttachmentsStreamingPoolParams<ConnectorState>);
|
|
12
12
|
private updateProgress;
|
|
13
|
-
/**
|
|
14
|
-
* Migrates processed attachments from the legacy string[] format to the new ProcessedAttachment[] format.
|
|
15
|
-
*
|
|
16
|
-
* @param attachments - The attachments list to migrate (either string[] or ProcessedAttachment[])
|
|
17
|
-
* @returns Migrated array of ProcessedAttachment objects, or empty array if input is invalid
|
|
18
|
-
*/
|
|
19
|
-
private migrateProcessedAttachments;
|
|
20
13
|
streamAll(): Promise<ProcessAttachmentReturnType>;
|
|
21
14
|
startPoolStreaming(): Promise<void>;
|
|
22
15
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"attachments-streaming-pool.d.ts","sourceRoot":"","sources":["../../src/attachments-streaming/attachments-streaming-pool.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"attachments-streaming-pool.d.ts","sourceRoot":"","sources":["../../src/attachments-streaming/attachments-streaming-pool.ts"],"names":[],"mappings":"AAEA,OAAO,EAEL,2BAA2B,EAC5B,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,8BAA8B,EAAE,MAAM,yCAAyC,CAAC;AAEzF,qBAAa,wBAAwB,CAAC,cAAc;IAClD,OAAO,CAAC,OAAO,CAAoC;IACnD,OAAO,CAAC,WAAW,CAAyB;IAC5C,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,KAAK,CAAqB;IAClC,OAAO,CAAC,MAAM,CAA4C;IAE1D,OAAO,CAAC,mBAAmB,CAAa;IACxC,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAM;gBAEnC,EACV,OAAO,EACP,WAAW,EACX,SAAc,EACd,MAAM,GACP,EAAE,8BAA8B,CAAC,cAAc,CAAC;YAQnC,cAAc;IAWtB,SAAS,IAAI,OAAO,CAAC,2BAA2B,CAAC;IAuCjD,kBAAkB;CA0FzB"}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import { ExternalSystemAttachmentStreamingFunction
|
|
2
|
-
import {
|
|
1
|
+
import { ExternalSystemAttachmentStreamingFunction } from '../types/extraction';
|
|
2
|
+
import { NormalizedAttachment } from '../repo/repo.interfaces';
|
|
3
|
+
import { ExtractionAdapter } from '../multithreading/adapters/extraction-adapter';
|
|
3
4
|
export interface AttachmentsStreamingPoolParams<ConnectorState> {
|
|
4
|
-
adapter:
|
|
5
|
+
adapter: ExtractionAdapter<ConnectorState>;
|
|
5
6
|
attachments: NormalizedAttachment[];
|
|
6
7
|
batchSize?: number;
|
|
7
8
|
stream: ExternalSystemAttachmentStreamingFunction;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"attachments-streaming-pool.interfaces.d.ts","sourceRoot":"","sources":["../../src/attachments-streaming/attachments-streaming-pool.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"attachments-streaming-pool.interfaces.d.ts","sourceRoot":"","sources":["../../src/attachments-streaming/attachments-streaming-pool.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yCAAyC,EAAE,MAAM,qBAAqB,CAAC;AAChF,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,iBAAiB,EAAE,MAAM,+CAA+C,CAAC;AAElF,MAAM,WAAW,8BAA8B,CAAC,cAAc;IAC5D,OAAO,EAAE,iBAAiB,CAAC,cAAc,CAAC,CAAC;IAC3C,WAAW,EAAE,oBAAoB,EAAE,CAAC;IACpC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,yCAAyC,CAAC;CACnD"}
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.AttachmentsStreamingPool = void 0;
|
|
4
|
-
const helpers_1 = require("../common/helpers");
|
|
5
4
|
class AttachmentsStreamingPool {
|
|
6
5
|
constructor({ adapter, attachments, batchSize = 10, stream, }) {
|
|
7
6
|
this.totalProcessedCount = 0;
|
|
@@ -16,53 +15,26 @@ class AttachmentsStreamingPool {
|
|
|
16
15
|
this.totalProcessedCount++;
|
|
17
16
|
if (this.totalProcessedCount % this.PROGRESS_REPORT_INTERVAL === 0) {
|
|
18
17
|
console.info(`Processed ${this.totalProcessedCount} attachments so far.`);
|
|
19
|
-
//
|
|
20
|
-
|
|
18
|
+
// Yield once to the event loop so a pending soft-timeout message
|
|
19
|
+
// (WorkerMessageExit) can be delivered and adapter.isTimeout can flip
|
|
20
|
+
// before the next batch of attachments is processed.
|
|
21
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
21
22
|
}
|
|
22
23
|
}
|
|
23
|
-
/**
|
|
24
|
-
* Migrates processed attachments from the legacy string[] format to the new ProcessedAttachment[] format.
|
|
25
|
-
*
|
|
26
|
-
* @param attachments - The attachments list to migrate (either string[] or ProcessedAttachment[])
|
|
27
|
-
* @returns Migrated array of ProcessedAttachment objects, or empty array if input is invalid
|
|
28
|
-
*/
|
|
29
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
30
|
-
migrateProcessedAttachments(attachments) {
|
|
31
|
-
// Handle null/undefined
|
|
32
|
-
if (!attachments || !Array.isArray(attachments)) {
|
|
33
|
-
return [];
|
|
34
|
-
}
|
|
35
|
-
// If already migrated (first element is an object), return as-is
|
|
36
|
-
if (attachments.length > 0 && typeof attachments[0] === 'object') {
|
|
37
|
-
return attachments;
|
|
38
|
-
}
|
|
39
|
-
// Migrate old string[] format
|
|
40
|
-
if (attachments.length > 0 && typeof attachments[0] === 'string') {
|
|
41
|
-
return attachments.map((it) => ({
|
|
42
|
-
id: it,
|
|
43
|
-
parent_id: '',
|
|
44
|
-
}));
|
|
45
|
-
}
|
|
46
|
-
return [];
|
|
47
|
-
}
|
|
48
24
|
async streamAll() {
|
|
49
25
|
console.log(`Starting download of ${this.attachments.length} attachments, streaming ${this.batchSize} at once.`);
|
|
50
|
-
if (!this.adapter.
|
|
26
|
+
if (!this.adapter.sdkState.toDevRev) {
|
|
51
27
|
const error = new Error('toDevRev state is not initialized');
|
|
52
28
|
console.error(error);
|
|
53
29
|
return { error };
|
|
54
30
|
}
|
|
55
31
|
// Get the list of successfully processed attachments in previous (possibly incomplete) batch extraction.
|
|
56
32
|
// If no such list exists, create an empty one.
|
|
57
|
-
if (!this.adapter.
|
|
33
|
+
if (!this.adapter.sdkState.toDevRev.attachmentsMetadata
|
|
58
34
|
.lastProcessedAttachmentsIdsList) {
|
|
59
|
-
this.adapter.
|
|
35
|
+
this.adapter.sdkState.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList =
|
|
60
36
|
[];
|
|
61
37
|
}
|
|
62
|
-
// Migrate old processed attachments to the new format.
|
|
63
|
-
this.adapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList =
|
|
64
|
-
this.migrateProcessedAttachments(this.adapter.state.toDevRev.attachmentsMetadata
|
|
65
|
-
.lastProcessedAttachmentsIdsList);
|
|
66
38
|
// Start initial batch of promises up to batchSize limit
|
|
67
39
|
const initialBatchSize = Math.min(this.batchSize, this.attachments.length);
|
|
68
40
|
const initialPromises = [];
|
|
@@ -94,8 +66,8 @@ class AttachmentsStreamingPool {
|
|
|
94
66
|
if (!attachment) {
|
|
95
67
|
break; // Exit if no more attachments
|
|
96
68
|
}
|
|
97
|
-
if (this.adapter.
|
|
98
|
-
((_a = this.adapter.
|
|
69
|
+
if (this.adapter.sdkState.toDevRev &&
|
|
70
|
+
((_a = this.adapter.sdkState.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList) === null || _a === void 0 ? void 0 : _a.some((it) => it.id == attachment.id && it.parent_id == attachment.parent_id))) {
|
|
99
71
|
continue; // Skip if the attachment ID is already processed
|
|
100
72
|
}
|
|
101
73
|
try {
|
|
@@ -118,8 +90,8 @@ class AttachmentsStreamingPool {
|
|
|
118
90
|
continue;
|
|
119
91
|
}
|
|
120
92
|
// No rate limiting, process normally
|
|
121
|
-
if ((_c = (_b = this.adapter.
|
|
122
|
-
(_d = this.adapter.
|
|
93
|
+
if ((_c = (_b = this.adapter.sdkState.toDevRev) === null || _b === void 0 ? void 0 : _b.attachmentsMetadata) === null || _c === void 0 ? void 0 : _c.lastProcessedAttachmentsIdsList) {
|
|
94
|
+
(_d = this.adapter.sdkState.toDevRev) === null || _d === void 0 ? void 0 : _d.attachmentsMetadata.lastProcessedAttachmentsIdsList.push({ id: attachment.id, parent_id: attachment.parent_id });
|
|
123
95
|
}
|
|
124
96
|
await this.updateProgress();
|
|
125
97
|
}
|
|
@@ -15,6 +15,7 @@ export declare const MAX_DEVREV_FILENAME_EXTENSION_LENGTH = 20;
|
|
|
15
15
|
export declare const MAX_EVENT_SIZE_BYTES = 200000;
|
|
16
16
|
export declare const EVENT_SIZE_THRESHOLD_BYTES: number;
|
|
17
17
|
export declare const SSOR_ATTACHMENT: string;
|
|
18
|
+
export declare const UNKNOWN_EVENT_TYPE = "UNKNOWN_EVENT_TYPE";
|
|
18
19
|
export declare enum AirSyncDefaultItemTypes {
|
|
19
20
|
EXTERNAL_DOMAIN_METADATA = "external_domain_metadata",
|
|
20
21
|
ATTACHMENTS = "attachments",
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/common/constants.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAGhD,eAAO,MAAM,8BAA8B,aAS1C,CAAC;AAEF,eAAO,MAAM,2BAA2B,aAKvC,CAAC;AAEF,eAAO,MAAM,mBAAmB,aAG/B,CAAC;AAEF,eAAO,MAAM,gCAAgC,aAI5C,CAAC;AAEF,eAAO,MAAM,6BAA6B,aAGzC,CAAC;AAEF,eAAO,MAAM,qBAAqB,aAGjC,CAAC;AAEF,eAAO,MAAM,+BAA+B,aAGzC,CAAC;AAEJ,eAAO,MAAM,4BAA4B,aAExC,CAAC;AAEF,eAAO,MAAM,oBAAoB,aAGhC,CAAC;AAEF,eAAO,MAAM,mBAAmB,OAAO,CAAC;AACxC,eAAO,MAAM,wBAAwB,QAAyB,CAAC;AAC/D,eAAO,MAAM,0BAA0B,MAAM,CAAC;AAC9C,eAAO,MAAM,oCAAoC,KAAK,CAAC;AAGvD,eAAO,MAAM,oBAAoB,SAAU,CAAC;AAE5C,eAAO,MAAM,0BAA0B,QAEtC,CAAC;AAEF,eAAO,MAAM,eAAe,EAAE,MAA0B,CAAC;
|
|
1
|
+
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/common/constants.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAGhD,eAAO,MAAM,8BAA8B,aAS1C,CAAC;AAEF,eAAO,MAAM,2BAA2B,aAKvC,CAAC;AAEF,eAAO,MAAM,mBAAmB,aAG/B,CAAC;AAEF,eAAO,MAAM,gCAAgC,aAI5C,CAAC;AAEF,eAAO,MAAM,6BAA6B,aAGzC,CAAC;AAEF,eAAO,MAAM,qBAAqB,aAGjC,CAAC;AAEF,eAAO,MAAM,+BAA+B,aAGzC,CAAC;AAEJ,eAAO,MAAM,4BAA4B,aAExC,CAAC;AAEF,eAAO,MAAM,oBAAoB,aAGhC,CAAC;AAEF,eAAO,MAAM,mBAAmB,OAAO,CAAC;AACxC,eAAO,MAAM,wBAAwB,QAAyB,CAAC;AAC/D,eAAO,MAAM,0BAA0B,MAAM,CAAC;AAC9C,eAAO,MAAM,oCAAoC,KAAK,CAAC;AAGvD,eAAO,MAAM,oBAAoB,SAAU,CAAC;AAE5C,eAAO,MAAM,0BAA0B,QAEtC,CAAC;AAEF,eAAO,MAAM,eAAe,EAAE,MAA0B,CAAC;AACzD,eAAO,MAAM,kBAAkB,uBAAuB,CAAC;AAEvD,oBAAY,uBAAuB;IACjC,wBAAwB,6BAA6B;IACrD,WAAW,gBAAgB;IAC3B,mBAAmB,wBAAwB;CAC5C;AAED,eAAO,MAAM,eAAe,KAAsB,CAAC;AAEnD,eAAO,MAAM,sBAAsB,QAAiB,CAAC;AACrD,eAAO,MAAM,uBAAuB,MAAM,CAAC;AAC3C,eAAO,MAAM,mBAAmB,QAAY,CAAC;AAE7C,eAAO,MAAM,sBAAsB,QAAgB,CAAC;AAEpD;;;;;;GAMG;AACH,eAAO,MAAM,yBAAyB,6BAA6B,CAAC"}
|
package/dist/common/constants.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.UNBOUNDED_DATE_TIME_VALUE = exports.DEFAULT_SLEEP_DELAY_MS = exports.MEMORY_LOG_INTERVAL = exports.HARD_TIMEOUT_MULTIPLIER = exports.DEFAULT_LAMBDA_TIMEOUT = exports.LIBRARY_VERSION = exports.AirSyncDefaultItemTypes = exports.SSOR_ATTACHMENT = exports.EVENT_SIZE_THRESHOLD_BYTES = exports.MAX_EVENT_SIZE_BYTES = exports.MAX_DEVREV_FILENAME_EXTENSION_LENGTH = exports.MAX_DEVREV_FILENAME_LENGTH = exports.MAX_DEVREV_ARTIFACT_SIZE = exports.ARTIFACT_BATCH_SIZE = exports.STATEFUL_EVENT_TYPES = exports.STATEFUL_LOADING_EVENT_TYPES = exports.STATEFUL_EXTRACTION_EVENT_TYPES = exports.STATELESS_EVENT_TYPES = exports.STATELESS_LOADING_EVENT_TYPES = exports.STATELESS_EXTRACTION_EVENT_TYPES = exports.ALLOWED_EVENT_TYPES = exports.ALLOWED_LOADING_EVENT_TYPES = exports.ALLOWED_EXTRACTION_EVENT_TYPES = void 0;
|
|
3
|
+
exports.UNBOUNDED_DATE_TIME_VALUE = exports.DEFAULT_SLEEP_DELAY_MS = exports.MEMORY_LOG_INTERVAL = exports.HARD_TIMEOUT_MULTIPLIER = exports.DEFAULT_LAMBDA_TIMEOUT = exports.LIBRARY_VERSION = exports.AirSyncDefaultItemTypes = exports.UNKNOWN_EVENT_TYPE = exports.SSOR_ATTACHMENT = exports.EVENT_SIZE_THRESHOLD_BYTES = exports.MAX_EVENT_SIZE_BYTES = exports.MAX_DEVREV_FILENAME_EXTENSION_LENGTH = exports.MAX_DEVREV_FILENAME_LENGTH = exports.MAX_DEVREV_ARTIFACT_SIZE = exports.ARTIFACT_BATCH_SIZE = exports.STATEFUL_EVENT_TYPES = exports.STATEFUL_LOADING_EVENT_TYPES = exports.STATEFUL_EXTRACTION_EVENT_TYPES = exports.STATELESS_EVENT_TYPES = exports.STATELESS_LOADING_EVENT_TYPES = exports.STATELESS_EXTRACTION_EVENT_TYPES = exports.ALLOWED_EVENT_TYPES = exports.ALLOWED_LOADING_EVENT_TYPES = exports.ALLOWED_EXTRACTION_EVENT_TYPES = void 0;
|
|
4
4
|
const extraction_1 = require("../types/extraction");
|
|
5
5
|
const helpers_1 = require("./helpers");
|
|
6
6
|
exports.ALLOWED_EXTRACTION_EVENT_TYPES = [
|
|
@@ -51,6 +51,7 @@ exports.MAX_EVENT_SIZE_BYTES = 200000;
|
|
|
51
51
|
// We want to leave some room for the other data in the message and process the rest of queued messages
|
|
52
52
|
exports.EVENT_SIZE_THRESHOLD_BYTES = Math.floor(exports.MAX_EVENT_SIZE_BYTES * 0.8);
|
|
53
53
|
exports.SSOR_ATTACHMENT = 'ssor_attachment';
|
|
54
|
+
exports.UNKNOWN_EVENT_TYPE = 'UNKNOWN_EVENT_TYPE';
|
|
54
55
|
var AirSyncDefaultItemTypes;
|
|
55
56
|
(function (AirSyncDefaultItemTypes) {
|
|
56
57
|
AirSyncDefaultItemTypes["EXTERNAL_DOMAIN_METADATA"] = "external_domain_metadata";
|
package/dist/common/helpers.d.ts
CHANGED
|
@@ -9,12 +9,6 @@ export declare function getLibraryVersion(): any;
|
|
|
9
9
|
* @returns {Promise<void>} A promise that resolves after the given number of milliseconds
|
|
10
10
|
*/
|
|
11
11
|
export declare function sleep(ms: number): Promise<unknown>;
|
|
12
|
-
/**
|
|
13
|
-
* Truncates a filename if it exceeds the maximum allowed length.
|
|
14
|
-
* @param {string} filename - The filename to truncate
|
|
15
|
-
* @returns {string} The truncated filename
|
|
16
|
-
*/
|
|
17
|
-
export declare function truncateFilename(filename: string): string;
|
|
18
12
|
/**
|
|
19
13
|
* MemoryInfo is an interface that represents the memory usage information.
|
|
20
14
|
* @interface MemoryInfo
|
|
@@ -38,12 +32,4 @@ export interface MemoryInfo {
|
|
|
38
32
|
* @returns {MemoryInfo} The memory usage information
|
|
39
33
|
*/
|
|
40
34
|
export declare function getMemoryUsage(): MemoryInfo;
|
|
41
|
-
/**
|
|
42
|
-
* Truncates a message if it exceeds the maximum allowed length.
|
|
43
|
-
* Adds a suffix indicating how many characters were omitted.
|
|
44
|
-
*
|
|
45
|
-
* @param message - The message to truncate
|
|
46
|
-
* @returns Truncated message or original if within limits
|
|
47
|
-
*/
|
|
48
|
-
export declare function truncateMessage(message: string): string;
|
|
49
35
|
//# sourceMappingURL=helpers.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"helpers.d.ts","sourceRoot":"","sources":["../../src/common/helpers.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"helpers.d.ts","sourceRoot":"","sources":["../../src/common/helpers.ts"],"names":[],"mappings":"AAIA;;;GAGG;AACH,wBAAgB,iBAAiB,QAiBhC;AAED;;;;GAIG;AACH,wBAAsB,KAAK,CAAC,EAAE,EAAE,MAAM,oBAErC;AAED;;;;;;;;;GASG;AACH,MAAM,WAAW,UAAU;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC;IACvB,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED;;;GAGG;AACH,wBAAgB,cAAc,IAAI,UAAU,CAmD3C"}
|
package/dist/common/helpers.js
CHANGED
|
@@ -35,14 +35,10 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
36
|
exports.getLibraryVersion = getLibraryVersion;
|
|
37
37
|
exports.sleep = sleep;
|
|
38
|
-
exports.truncateFilename = truncateFilename;
|
|
39
38
|
exports.getMemoryUsage = getMemoryUsage;
|
|
40
|
-
exports.truncateMessage = truncateMessage;
|
|
41
39
|
const fs_1 = require("fs");
|
|
42
40
|
const path = __importStar(require("path"));
|
|
43
41
|
const v8 = __importStar(require("v8"));
|
|
44
|
-
const constants_1 = require("./constants");
|
|
45
|
-
const logger_constants_1 = require("../logger/logger.constants");
|
|
46
42
|
/**
|
|
47
43
|
* Gets the library version from the package.json file.
|
|
48
44
|
* @returns {string} The library version
|
|
@@ -67,27 +63,8 @@ function getLibraryVersion() {
|
|
|
67
63
|
* @returns {Promise<void>} A promise that resolves after the given number of milliseconds
|
|
68
64
|
*/
|
|
69
65
|
async function sleep(ms) {
|
|
70
|
-
console.log(`Sleeping for ${ms}ms.`);
|
|
71
66
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
72
67
|
}
|
|
73
|
-
/**
|
|
74
|
-
* Truncates a filename if it exceeds the maximum allowed length.
|
|
75
|
-
* @param {string} filename - The filename to truncate
|
|
76
|
-
* @returns {string} The truncated filename
|
|
77
|
-
*/
|
|
78
|
-
function truncateFilename(filename) {
|
|
79
|
-
// If the filename is already within the limit, return it as is.
|
|
80
|
-
if (filename.length <= constants_1.MAX_DEVREV_FILENAME_LENGTH) {
|
|
81
|
-
return filename;
|
|
82
|
-
}
|
|
83
|
-
console.warn(`Filename length exceeds the maximum limit of ${constants_1.MAX_DEVREV_FILENAME_LENGTH} characters. Truncating filename.`);
|
|
84
|
-
const extension = filename.slice(-constants_1.MAX_DEVREV_FILENAME_EXTENSION_LENGTH);
|
|
85
|
-
// Calculate how many characters are available for the name part after accounting for the extension and "..."
|
|
86
|
-
const availableNameLength = constants_1.MAX_DEVREV_FILENAME_LENGTH - constants_1.MAX_DEVREV_FILENAME_EXTENSION_LENGTH - 3; // -3 for "..."
|
|
87
|
-
// Truncate the name part and add an ellipsis
|
|
88
|
-
const truncatedFilename = filename.slice(0, availableNameLength);
|
|
89
|
-
return `${truncatedFilename}...${extension}`;
|
|
90
|
-
}
|
|
91
68
|
/**
|
|
92
69
|
* Gets the memory usage information.
|
|
93
70
|
* @returns {MemoryInfo} The memory usage information
|
|
@@ -124,16 +101,3 @@ function getMemoryUsage() {
|
|
|
124
101
|
throw err;
|
|
125
102
|
}
|
|
126
103
|
}
|
|
127
|
-
/**
|
|
128
|
-
* Truncates a message if it exceeds the maximum allowed length.
|
|
129
|
-
* Adds a suffix indicating how many characters were omitted.
|
|
130
|
-
*
|
|
131
|
-
* @param message - The message to truncate
|
|
132
|
-
* @returns Truncated message or original if within limits
|
|
133
|
-
*/
|
|
134
|
-
function truncateMessage(message) {
|
|
135
|
-
if (message.length > logger_constants_1.MAX_LOG_STRING_LENGTH) {
|
|
136
|
-
return `${message.substring(0, logger_constants_1.MAX_LOG_STRING_LENGTH)}... ${message.length - logger_constants_1.MAX_LOG_STRING_LENGTH} more characters`;
|
|
137
|
-
}
|
|
138
|
-
return message;
|
|
139
|
-
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../src/http/client.ts"],"names":[],"mappings":"AAGA,QAAA,MAAM,WAAW,+BAEf,CAAC;AAyFH,OAAO,EAAE,WAAW,EAAE,CAAC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,15 +1,27 @@
|
|
|
1
|
-
export {
|
|
2
|
-
export { ExtractionCommonError } from './common/errors';
|
|
3
|
-
export * from './common/install-initial-domain-mapping';
|
|
4
|
-
export * from './http';
|
|
5
|
-
export { serializeAxiosError } from './logger/logger';
|
|
6
|
-
export { MockServer } from './mock-server/mock-server';
|
|
7
|
-
export type { RequestInfo, RetryConfig, RouteConfig, } from './mock-server/mock-server.interfaces';
|
|
8
|
-
export { processTask } from './multithreading/process-task';
|
|
1
|
+
export { processExtractionTask, processLoadingTask, } from './multithreading/process-task';
|
|
9
2
|
export { spawn } from './multithreading/spawn/spawn';
|
|
10
|
-
export {
|
|
11
|
-
export {
|
|
12
|
-
export type {
|
|
13
|
-
export
|
|
14
|
-
export
|
|
3
|
+
export { ExtractionAdapter } from './multithreading/adapters/extraction-adapter';
|
|
4
|
+
export { LoadingAdapter } from './multithreading/adapters/loading-adapter';
|
|
5
|
+
export type { ProcessTaskInterface, TaskAdapterInterface, TaskResult, TaskStatus, WorkerAdapterInterface, WorkerAdapterOptions, SpawnFactoryInterface, SpawnInterface, ExtractionScope, WorkerPathOverrides, } from './types/workers';
|
|
6
|
+
export { AirSyncDefaultItemTypes } from './common/constants';
|
|
7
|
+
export { UNBOUNDED_DATE_TIME_VALUE } from './common/constants';
|
|
8
|
+
export { ExtractionCommonError } from './types/errors';
|
|
9
|
+
export { installInitialDomainMapping } from './state/install-initial-domain-mapping';
|
|
10
|
+
export { serializeError } from './logger/logger';
|
|
11
|
+
export { SyncMode } from './types/common';
|
|
12
|
+
export type { ErrorRecord, InitialDomainMapping } from './types/common';
|
|
13
|
+
export { EventType, ExtractorEventType, InitialSyncScope, TimeUnit, TimeValueType, } from './types/extraction';
|
|
14
|
+
export type { AirSyncEvent, AirSyncMessage, ConnectionData, EventContext, EventData, ExternalProcessAttachmentFunction, ExternalSyncUnit, ExternalSystemAttachmentIteratorFunction, ExternalSystemAttachmentReducerFunction, ExternalSystemAttachmentStreamingFunction, ExternalSystemAttachmentStreamingParams, ExternalSystemAttachmentStreamingResponse, ExtractorEvent, HttpStreamResponse, ProcessAttachmentReturnType, TimeValue, } from './types/extraction';
|
|
15
|
+
export { LoaderEventType } from './types/loading';
|
|
16
|
+
export type { ExternalSystemAttachment, ExternalSystemItem, ExternalSystemItemLoadingParams, ExternalSystemItemLoadingResponse, ItemTypeToLoad, } from './types/loading';
|
|
17
|
+
export type { Item, NormalizedAttachment, NormalizedItem, RepoInterface, } from './repo/repo.interfaces';
|
|
18
|
+
export { Mappers } from './mappers/mappers';
|
|
19
|
+
export { SyncMapperRecordStatus, SyncMapperRecordTargetType, } from './mappers/mappers.interfaces';
|
|
20
|
+
export type { MappersCreateParams, MappersGetByExternalIdParams, MappersGetByTargetIdParams, MappersUpdateParams, } from './mappers/mappers.interfaces';
|
|
21
|
+
export type { Artifact, ArtifactsPrepareResponse, SsorAttachment, StreamAttachmentsResponse, StreamResponse, UploadResponse, } from './uploader/uploader.interfaces';
|
|
22
|
+
export type { CollectionData, ConditionalPrivilegeData, CustomLinkData, CustomLinkNames, CustomStage, CustomState, EnumData, EnumValue, EnumValueKey, ExternalDomainMetadata, Field, FieldCondition, FieldConditionComparator, FieldConditionEffect, FieldConditions, FieldKey, FieldPrivilegeData, FieldReferenceData, FieldType, FloatData, IntData, PermissionData, RecordType, RecordTypeCategory, RecordTypeCategoryKey, RecordTypeKey, RecordTypePrivilegeData, RecordTypeScope, ReferenceData, ReferenceDetail, ReferenceType, SchemaVersion, StageKey, StageDiagram, StateKey, StructData, StructTypeKey, StructType, TargetTypeKeyData, TextData, TypedReferenceData, } from './types/external-domain-metadata';
|
|
23
|
+
export { MockServer, MOCK_SERVER_DEFAULT_URL } from './testing/mock-server';
|
|
24
|
+
export type { RequestInfo, RetryConfig, RouteConfig, } from './testing/mock-server.interfaces';
|
|
25
|
+
export { createMockEvent } from './testing/mock-event';
|
|
26
|
+
export type { DeepPartial } from './testing/mock-event';
|
|
15
27
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAUA,OAAO,EACL,qBAAqB,EACrB,kBAAkB,GACnB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,KAAK,EAAE,MAAM,8BAA8B,CAAC;AACrD,OAAO,EAAE,iBAAiB,EAAE,MAAM,8CAA8C,CAAC;AACjF,OAAO,EAAE,cAAc,EAAE,MAAM,2CAA2C,CAAC;AAG3E,YAAY,EACV,oBAAoB,EACpB,oBAAoB,EACpB,UAAU,EACV,UAAU,EACV,sBAAsB,EACtB,oBAAoB,EACpB,qBAAqB,EACrB,cAAc,EACd,eAAe,EACf,mBAAmB,GACpB,MAAM,iBAAiB,CAAC;AAGzB,OAAO,EAAE,uBAAuB,EAAE,MAAM,oBAAoB,CAAC;AAC7D,OAAO,EAAE,yBAAyB,EAAE,MAAM,oBAAoB,CAAC;AAC/D,OAAO,EAAE,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;AAGvD,OAAO,EAAE,2BAA2B,EAAE,MAAM,wCAAwC,CAAC;AAGrF,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAGjD,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAC1C,YAAY,EAAE,WAAW,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAGxE,OAAO,EACL,SAAS,EACT,kBAAkB,EAClB,gBAAgB,EAChB,QAAQ,EACR,aAAa,GACd,MAAM,oBAAoB,CAAC;AAC5B,YAAY,EACV,YAAY,EACZ,cAAc,EACd,cAAc,EACd,YAAY,EACZ,SAAS,EACT,iCAAiC,EACjC,gBAAgB,EAChB,wCAAwC,EACxC,uCAAuC,EACvC,yCAAyC,EACzC,uCAAuC,EACvC,yCAAyC,EACzC,cAAc,EACd,kBAAkB,EAClB,2BAA2B,EAC3B,SAAS,GACV,MAAM,oBAAoB,CAAC;AAG5B,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAClD,YAAY,EACV,wBAAwB,EACxB,kBAAkB,EAClB,+BAA+B,EAC/B,iCAAiC,EACjC,cAAc,GACf,MAAM,iBAAiB,CAAC;AAGzB,YAAY,EACV,IAAI,EACJ,oBAAoB,EACpB,cAAc,EACd,aAAa,GACd,MAAM,wBAAwB,CAAC;AAGhC,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAC5C,OAAO,EACL,sBAAsB,EACtB,0BAA0B,GAC3B,MAAM,8BAA8B,CAAC;AACtC,YAAY,EACV,mBAAmB,EACnB,4BAA4B,EAC5B,0BAA0B,EAC1B,mBAAmB,GACpB,MAAM,8BAA8B,CAAC;AAGtC,YAAY,EACV,QAAQ,EACR,wBAAwB,EACxB,cAAc,EACd,yBAAyB,EACzB,cAAc,EACd,cAAc,GACf,MAAM,gCAAgC,CAAC;AAGxC,YAAY,EACV,cAAc,EACd,wBAAwB,EACxB,cAAc,EACd,eAAe,EACf,WAAW,EACX,WAAW,EACX,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,sBAAsB,EACtB,KAAK,EACL,cAAc,EACd,wBAAwB,EACxB,oBAAoB,EACpB,eAAe,EACf,QAAQ,EACR,kBAAkB,EAClB,kBAAkB,EAClB,SAAS,EACT,SAAS,EACT,OAAO,EACP,cAAc,EACd,UAAU,EACV,kBAAkB,EAClB,qBAAqB,EACrB,aAAa,EACb,uBAAuB,EACvB,eAAe,EACf,aAAa,EACb,eAAe,EACf,aAAa,EACb,aAAa,EACb,QAAQ,EACR,YAAY,EACZ,QAAQ,EACR,UAAU,EACV,aAAa,EACb,UAAU,EACV,iBAAiB,EACjB,QAAQ,EACR,kBAAkB,GACnB,MAAM,kCAAkC,CAAC;AAG1C,OAAO,EAAE,UAAU,EAAE,uBAAuB,EAAE,MAAM,uBAAuB,CAAC;AAC5E,YAAY,EACV,WAAW,EACX,WAAW,EACX,WAAW,GACZ,MAAM,kCAAkC,CAAC;AAC1C,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AACvD,YAAY,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -1,36 +1,57 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
-
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
-
};
|
|
2
|
+
// ────────────────────────────────────────────────────────────────────────────
|
|
3
|
+
// Public API barrel for @devrev/airsync-sdk.
|
|
4
|
+
//
|
|
5
|
+
// This is the SINGLE source of the public surface — every exported symbol is
|
|
6
|
+
// named explicitly (no `export *`), so any change to the public API shows up as
|
|
7
|
+
// a diff here. Adding a symbol to the package's public contract means adding it
|
|
8
|
+
// to this file.
|
|
9
|
+
// ────────────────────────────────────────────────────────────────────────────
|
|
16
10
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
-
exports.MOCK_SERVER_DEFAULT_URL = exports.
|
|
11
|
+
exports.createMockEvent = exports.MOCK_SERVER_DEFAULT_URL = exports.MockServer = exports.SyncMapperRecordTargetType = exports.SyncMapperRecordStatus = exports.Mappers = exports.LoaderEventType = exports.TimeValueType = exports.TimeUnit = exports.InitialSyncScope = exports.ExtractorEventType = exports.EventType = exports.SyncMode = exports.serializeError = exports.installInitialDomainMapping = exports.UNBOUNDED_DATE_TIME_VALUE = exports.AirSyncDefaultItemTypes = exports.LoadingAdapter = exports.ExtractionAdapter = exports.spawn = exports.processLoadingTask = exports.processExtractionTask = void 0;
|
|
12
|
+
// ── Entry points & adapters ──
|
|
13
|
+
var process_task_1 = require("./multithreading/process-task");
|
|
14
|
+
Object.defineProperty(exports, "processExtractionTask", { enumerable: true, get: function () { return process_task_1.processExtractionTask; } });
|
|
15
|
+
Object.defineProperty(exports, "processLoadingTask", { enumerable: true, get: function () { return process_task_1.processLoadingTask; } });
|
|
16
|
+
var spawn_1 = require("./multithreading/spawn/spawn");
|
|
17
|
+
Object.defineProperty(exports, "spawn", { enumerable: true, get: function () { return spawn_1.spawn; } });
|
|
18
|
+
var extraction_adapter_1 = require("./multithreading/adapters/extraction-adapter");
|
|
19
|
+
Object.defineProperty(exports, "ExtractionAdapter", { enumerable: true, get: function () { return extraction_adapter_1.ExtractionAdapter; } });
|
|
20
|
+
var loading_adapter_1 = require("./multithreading/adapters/loading-adapter");
|
|
21
|
+
Object.defineProperty(exports, "LoadingAdapter", { enumerable: true, get: function () { return loading_adapter_1.LoadingAdapter; } });
|
|
22
|
+
// ── Constants & enums ──
|
|
18
23
|
var constants_1 = require("./common/constants");
|
|
19
24
|
Object.defineProperty(exports, "AirSyncDefaultItemTypes", { enumerable: true, get: function () { return constants_1.AirSyncDefaultItemTypes; } });
|
|
20
|
-
|
|
21
|
-
|
|
25
|
+
var constants_2 = require("./common/constants");
|
|
26
|
+
Object.defineProperty(exports, "UNBOUNDED_DATE_TIME_VALUE", { enumerable: true, get: function () { return constants_2.UNBOUNDED_DATE_TIME_VALUE; } });
|
|
27
|
+
// ── Domain mapping install ──
|
|
28
|
+
var install_initial_domain_mapping_1 = require("./state/install-initial-domain-mapping");
|
|
29
|
+
Object.defineProperty(exports, "installInitialDomainMapping", { enumerable: true, get: function () { return install_initial_domain_mapping_1.installInitialDomainMapping; } });
|
|
30
|
+
// ── Error formatting ──
|
|
22
31
|
var logger_1 = require("./logger/logger");
|
|
23
|
-
Object.defineProperty(exports, "
|
|
24
|
-
|
|
32
|
+
Object.defineProperty(exports, "serializeError", { enumerable: true, get: function () { return logger_1.serializeError; } });
|
|
33
|
+
// ── Common types ──
|
|
34
|
+
var common_1 = require("./types/common");
|
|
35
|
+
Object.defineProperty(exports, "SyncMode", { enumerable: true, get: function () { return common_1.SyncMode; } });
|
|
36
|
+
// ── Extraction types ──
|
|
37
|
+
var extraction_1 = require("./types/extraction");
|
|
38
|
+
Object.defineProperty(exports, "EventType", { enumerable: true, get: function () { return extraction_1.EventType; } });
|
|
39
|
+
Object.defineProperty(exports, "ExtractorEventType", { enumerable: true, get: function () { return extraction_1.ExtractorEventType; } });
|
|
40
|
+
Object.defineProperty(exports, "InitialSyncScope", { enumerable: true, get: function () { return extraction_1.InitialSyncScope; } });
|
|
41
|
+
Object.defineProperty(exports, "TimeUnit", { enumerable: true, get: function () { return extraction_1.TimeUnit; } });
|
|
42
|
+
Object.defineProperty(exports, "TimeValueType", { enumerable: true, get: function () { return extraction_1.TimeValueType; } });
|
|
43
|
+
// ── Loading types ──
|
|
44
|
+
var loading_1 = require("./types/loading");
|
|
45
|
+
Object.defineProperty(exports, "LoaderEventType", { enumerable: true, get: function () { return loading_1.LoaderEventType; } });
|
|
46
|
+
// ── Mappers ──
|
|
47
|
+
var mappers_1 = require("./mappers/mappers");
|
|
48
|
+
Object.defineProperty(exports, "Mappers", { enumerable: true, get: function () { return mappers_1.Mappers; } });
|
|
49
|
+
var mappers_interfaces_1 = require("./mappers/mappers.interfaces");
|
|
50
|
+
Object.defineProperty(exports, "SyncMapperRecordStatus", { enumerable: true, get: function () { return mappers_interfaces_1.SyncMapperRecordStatus; } });
|
|
51
|
+
Object.defineProperty(exports, "SyncMapperRecordTargetType", { enumerable: true, get: function () { return mappers_interfaces_1.SyncMapperRecordTargetType; } });
|
|
52
|
+
// ── Testing utilities (public test-support surface) ──
|
|
53
|
+
var mock_server_1 = require("./testing/mock-server");
|
|
25
54
|
Object.defineProperty(exports, "MockServer", { enumerable: true, get: function () { return mock_server_1.MockServer; } });
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
Object.defineProperty(exports, "spawn", { enumerable: true, get: function () { return spawn_1.spawn; } });
|
|
30
|
-
var worker_adapter_1 = require("./multithreading/worker-adapter/worker-adapter");
|
|
31
|
-
Object.defineProperty(exports, "WorkerAdapter", { enumerable: true, get: function () { return worker_adapter_1.WorkerAdapter; } });
|
|
32
|
-
var test_utils_1 = require("./common/test-utils");
|
|
33
|
-
Object.defineProperty(exports, "createMockEvent", { enumerable: true, get: function () { return test_utils_1.createMockEvent; } });
|
|
34
|
-
Object.defineProperty(exports, "MOCK_SERVER_DEFAULT_URL", { enumerable: true, get: function () { return test_utils_1.MOCK_SERVER_DEFAULT_URL; } });
|
|
35
|
-
__exportStar(require("./types"), exports);
|
|
36
|
-
__exportStar(require("./types/workers"), exports);
|
|
55
|
+
Object.defineProperty(exports, "MOCK_SERVER_DEFAULT_URL", { enumerable: true, get: function () { return mock_server_1.MOCK_SERVER_DEFAULT_URL; } });
|
|
56
|
+
var mock_event_1 = require("./testing/mock-event");
|
|
57
|
+
Object.defineProperty(exports, "createMockEvent", { enumerable: true, get: function () { return mock_event_1.createMockEvent; } });
|
package/dist/logger/logger.d.ts
CHANGED
|
@@ -1,6 +1,20 @@
|
|
|
1
1
|
import { AxiosError } from 'axios';
|
|
2
2
|
import { Console } from 'node:console';
|
|
3
|
+
import { InspectOptions } from 'node:util';
|
|
3
4
|
import { AxiosErrorResponse, LoggerFactoryInterface, LogLevel, PrintableState } from './logger.interfaces';
|
|
5
|
+
export declare const MAX_LOG_STRING_LENGTH = 10000;
|
|
6
|
+
export declare const INSPECT_OPTIONS: InspectOptions;
|
|
7
|
+
/**
|
|
8
|
+
* Truncates a message that exceeds the maximum length, appending a suffix
|
|
9
|
+
* noting how many characters were omitted.
|
|
10
|
+
*/
|
|
11
|
+
export declare function truncateMessage(message: string): string;
|
|
12
|
+
/** Runs `fn` with logs marked as user-originated (`is_sdk_log: false`). */
|
|
13
|
+
export declare function runWithUserLogContext<T>(fn: () => T): T;
|
|
14
|
+
/** Runs `fn` with logs marked as SDK-originated (`is_sdk_log: true`). */
|
|
15
|
+
export declare function runWithSdkLogContext<T>(fn: () => T): T;
|
|
16
|
+
/** Returns the current context flag, or `defaultValue` if none is set. */
|
|
17
|
+
export declare function getSdkLogContextValue(defaultValue: boolean): boolean;
|
|
4
18
|
/**
|
|
5
19
|
* Custom logger that extends Node.js Console with context-aware logging.
|
|
6
20
|
* Handles local development, main thread, and worker thread logging differently.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../../src/logger/logger.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAyC,MAAM,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../../src/logger/logger.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAyC,MAAM,OAAO,CAAC;AAG1E,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAW,cAAc,EAAE,MAAM,WAAW,CAAC;AAMpD,OAAO,EACL,kBAAkB,EAClB,sBAAsB,EAEtB,QAAQ,EAER,cAAc,EACf,MAAM,qBAAqB,CAAC;AAI7B,eAAO,MAAM,qBAAqB,QAAQ,CAAC;AAI3C,eAAO,MAAM,eAAe,EAAE,cAM7B,CAAC;AAEF;;;GAGG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAOvD;AAYD,2EAA2E;AAC3E,wBAAgB,qBAAqB,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,CAEvD;AAED,yEAAyE;AACzE,wBAAgB,oBAAoB,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,CAEtD;AAED,0EAA0E;AAC1E,wBAAgB,qBAAqB,CAAC,YAAY,EAAE,OAAO,GAAG,OAAO,CAMpE;AAED;;;GAGG;AACH,qBAAa,MAAO,SAAQ,OAAO;IACjC,OAAO,CAAC,eAAe,CAAU;IACjC,OAAO,CAAC,OAAO,CAAC,CAAuB;IACvC,OAAO,CAAC,IAAI,CAAa;gBAEb,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,sBAAsB;IAWtD;;;;;OAKG;IACH,OAAO,CAAC,aAAa;IAOrB;;;;;;;;;OASG;IACH,KAAK,CACH,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,QAAQ,EACf,QAAQ,GAAE,OAAqC,GAC9C,IAAI;IAcP;;;;;;;;OAQG;IACH,OAAO,CAAC,eAAe;IAgBd,GAAG,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI;IAI7B,IAAI,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI;IAI9B,IAAI,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI;IAI9B,KAAK,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI;CAGzC;AACD;;;;;;;GAOG;AAEH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,cAAc,CAqB5E;AAED;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAoCrD;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,UAAU,GAAG,kBAAkB,CAwBzE"}
|
package/dist/logger/logger.js
CHANGED
|
@@ -1,19 +1,66 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.Logger = void 0;
|
|
3
|
+
exports.Logger = exports.INSPECT_OPTIONS = exports.MAX_LOG_STRING_LENGTH = void 0;
|
|
4
|
+
exports.truncateMessage = truncateMessage;
|
|
5
|
+
exports.runWithUserLogContext = runWithUserLogContext;
|
|
6
|
+
exports.runWithSdkLogContext = runWithSdkLogContext;
|
|
7
|
+
exports.getSdkLogContextValue = getSdkLogContextValue;
|
|
4
8
|
exports.getPrintableState = getPrintableState;
|
|
5
9
|
exports.serializeError = serializeError;
|
|
6
10
|
exports.serializeAxiosError = serializeAxiosError;
|
|
7
11
|
const axios_1 = require("axios");
|
|
12
|
+
const node_async_hooks_1 = require("node:async_hooks");
|
|
8
13
|
const node_console_1 = require("node:console");
|
|
9
14
|
const node_util_1 = require("node:util");
|
|
10
15
|
const node_worker_threads_1 = require("node:worker_threads");
|
|
11
16
|
const constants_1 = require("../common/constants");
|
|
12
17
|
const workers_1 = require("../types/workers");
|
|
13
|
-
const logger_constants_1 = require("./logger.constants");
|
|
14
|
-
const logger_context_1 = require("./logger.context");
|
|
15
18
|
const logger_interfaces_1 = require("./logger.interfaces");
|
|
16
|
-
|
|
19
|
+
// ── Log formatting ──
|
|
20
|
+
exports.MAX_LOG_STRING_LENGTH = 10000;
|
|
21
|
+
const MAX_LOG_DEPTH = 10;
|
|
22
|
+
const MAX_LOG_ARRAY_LENGTH = 100;
|
|
23
|
+
exports.INSPECT_OPTIONS = {
|
|
24
|
+
compact: false,
|
|
25
|
+
breakLength: Infinity,
|
|
26
|
+
depth: MAX_LOG_DEPTH,
|
|
27
|
+
maxArrayLength: MAX_LOG_ARRAY_LENGTH,
|
|
28
|
+
maxStringLength: exports.MAX_LOG_STRING_LENGTH,
|
|
29
|
+
};
|
|
30
|
+
/**
|
|
31
|
+
* Truncates a message that exceeds the maximum length, appending a suffix
|
|
32
|
+
* noting how many characters were omitted.
|
|
33
|
+
*/
|
|
34
|
+
function truncateMessage(message) {
|
|
35
|
+
if (message.length > exports.MAX_LOG_STRING_LENGTH) {
|
|
36
|
+
return `${message.substring(0, exports.MAX_LOG_STRING_LENGTH)}... ${message.length - exports.MAX_LOG_STRING_LENGTH} more characters`;
|
|
37
|
+
}
|
|
38
|
+
return message;
|
|
39
|
+
}
|
|
40
|
+
// ── SDK log context ──
|
|
41
|
+
/**
|
|
42
|
+
* Async-aware flag tracking whether code is running in SDK internals (`true`)
|
|
43
|
+
* or user-provided handlers (`false`). AsyncLocalStorage propagates the value
|
|
44
|
+
* across await/Promise/timer boundaries, so logs stay correctly attributed
|
|
45
|
+
* without threading the flag through call signatures.
|
|
46
|
+
*/
|
|
47
|
+
const sdkLogContext = new node_async_hooks_1.AsyncLocalStorage();
|
|
48
|
+
/** Runs `fn` with logs marked as user-originated (`is_sdk_log: false`). */
|
|
49
|
+
function runWithUserLogContext(fn) {
|
|
50
|
+
return sdkLogContext.run(false, fn);
|
|
51
|
+
}
|
|
52
|
+
/** Runs `fn` with logs marked as SDK-originated (`is_sdk_log: true`). */
|
|
53
|
+
function runWithSdkLogContext(fn) {
|
|
54
|
+
return sdkLogContext.run(true, fn);
|
|
55
|
+
}
|
|
56
|
+
/** Returns the current context flag, or `defaultValue` if none is set. */
|
|
57
|
+
function getSdkLogContextValue(defaultValue) {
|
|
58
|
+
const storeValue = sdkLogContext.getStore();
|
|
59
|
+
if (typeof storeValue === 'boolean') {
|
|
60
|
+
return storeValue;
|
|
61
|
+
}
|
|
62
|
+
return defaultValue;
|
|
63
|
+
}
|
|
17
64
|
/**
|
|
18
65
|
* Custom logger that extends Node.js Console with context-aware logging.
|
|
19
66
|
* Handles local development, main thread, and worker thread logging differently.
|
|
@@ -35,7 +82,7 @@ class Logger extends node_console_1.Console {
|
|
|
35
82
|
if (typeof value === 'string') {
|
|
36
83
|
return value;
|
|
37
84
|
}
|
|
38
|
-
return (0, node_util_1.inspect)(value,
|
|
85
|
+
return (0, node_util_1.inspect)(value, exports.INSPECT_OPTIONS);
|
|
39
86
|
}
|
|
40
87
|
/**
|
|
41
88
|
* Logs a pre-formatted message string to the console.
|
|
@@ -47,7 +94,7 @@ class Logger extends node_console_1.Console {
|
|
|
47
94
|
* @param level - Log level (info, warn, error)
|
|
48
95
|
* @param isSdkLog - Flag indicating if the log originated from the SDK
|
|
49
96
|
*/
|
|
50
|
-
logFn(message, level, isSdkLog =
|
|
97
|
+
logFn(message, level, isSdkLog = getSdkLogContextValue(true)) {
|
|
51
98
|
var _a;
|
|
52
99
|
if ((_a = this.options) === null || _a === void 0 ? void 0 : _a.isLocalDevelopment) {
|
|
53
100
|
this.originalConsole[level](message);
|
|
@@ -67,8 +114,8 @@ class Logger extends node_console_1.Console {
|
|
|
67
114
|
*/
|
|
68
115
|
stringifyAndLog(args, level) {
|
|
69
116
|
let stringifiedArgs = args.map((arg) => this.valueToString(arg)).join(' ');
|
|
70
|
-
stringifiedArgs =
|
|
71
|
-
const isSdkLog =
|
|
117
|
+
stringifiedArgs = truncateMessage(stringifiedArgs);
|
|
118
|
+
const isSdkLog = getSdkLogContextValue(true);
|
|
72
119
|
if (node_worker_threads_1.isMainThread) {
|
|
73
120
|
this.logFn(stringifiedArgs, level, isSdkLog);
|
|
74
121
|
}
|