@devrev/airsync-sdk 2.0.0-beta.1 → 2.0.0-beta.11

package/README.md

@@ -1,31 +1,17 @@
 # AirSync SDK
 
-[![Coverage Status](https://coveralls.io/repos/github/devrev/adaas-sdk/badge.svg?branch=v2&t=s4Otlm)](https://coveralls.io/github/devrev/adaas-sdk?branch=v2)
+[![Coverage Status](https://coveralls.io/repos/github/devrev/adaas-sdk/badge.svg?branch=main&t=s4Otlm)](https://coveralls.io/github/devrev/adaas-sdk?branch=main)
 
-> **Note:** This is the v2 beta of the AirSync SDK (formerly Airdrop SDK). The package has been renamed from `@devrev/ts-adaas` to `@devrev/airsync-sdk`.
-
-## Overview
-
-The AirSync SDK for TypeScript helps developers build snap-ins that integrate with DevRev's AirSync platform. This SDK simplifies the workflow for handling data extraction and loading, event-driven actions, state management, and artifact handling.
-
-It provides features such as:
-
-- Type Definitions: Structured types for AirSync control protocol
-- Event Management: Easily emit events for different extraction or loading phases
-- State Handling: Update and access state in real-time within tasks
-- Artifact Management: Supports batched storage of artifacts
-- Error & Timeout Support: Error handling and timeout management for long-running tasks
+TypeScript SDK for building AirSync snap-ins (connectors) that sync data between external systems and DevRev. It handles the AirSync control protocol for you: worker threads, timeouts, state, artifact uploads, attachment streaming and loading.
 
 ## Installation
 
 ```bash
-npm install @devrev/airsync-sdk@beta
+npm install @devrev/airsync-sdk
 ```
 
-## Changelog
-
-See [CHANGELOG.md](./CHANGELOG.md) for release notes and migration guides.
-
-## Reference
+## Documentation
 
-Please refer to the [REFERENCE.md](./REFERENCE.md) file for more information on the types, interfaces and functions used in the library.
+- [Architecture](docs/ARCHITECTURE.md): how the SDK works and how it talks to the AirSync platform.
+- [Migration from v1](docs/MIGRATION.md): moving from `@devrev/ts-adaas` to `@devrev/airsync-sdk`.
+- [Contributing](docs/CONTRIBUTING.md): workflow, versioning and testing conventions.

package/dist/attachments-streaming/attachments-streaming-pool.d.ts

@@ -1,4 +1,4 @@
-import { ProcessAttachmentReturnType } from '../types';
+import { ProcessAttachmentReturnType } from '../types/extraction';
 import { AttachmentsStreamingPoolParams } from './attachments-streaming-pool.interfaces';
 export declare class AttachmentsStreamingPool<ConnectorState> {
     private adapter;
@@ -9,14 +9,13 @@ export declare class AttachmentsStreamingPool<ConnectorState> {
     private totalProcessedCount;
     private readonly PROGRESS_REPORT_INTERVAL;
     constructor({ adapter, attachments, batchSize, stream, }: AttachmentsStreamingPoolParams<ConnectorState>);
-    private updateProgress;
+    private recordProcessedAttachment;
     /**
-     * Migrates processed attachments from the legacy string[] format to the new ProcessedAttachment[] format.
-     *
-     * @param attachments - The attachments list to migrate (either string[] or ProcessedAttachment[])
-     * @returns Migrated array of ProcessedAttachment objects, or empty array if input is invalid
+     * Backfills `status` on entries recorded before the field existed (those
+     * only ever held successes); state from >= v1.15.2 needs no shape migration.
      */
-    private migrateProcessedAttachments;
+    private backfillProcessedAttachmentStatus;
+    private updateProgress;
     streamAll(): Promise<ProcessAttachmentReturnType>;
     startPoolStreaming(): Promise<void>;
 }

package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"attachments-streaming-pool.d.ts","sourceRoot":"","sources":["../../src/attachments-streaming/attachments-streaming-pool.ts"],"names":[],"mappings":"AACA,OAAO,EAGL,2BAA2B,EAC5B,MAAM,UAAU,CAAC;AAElB,OAAO,EAAE,8BAA8B,EAAE,MAAM,yCAAyC,CAAC;AAGzF,qBAAa,wBAAwB,CAAC,cAAc;IAClD,OAAO,CAAC,OAAO,CAAgC;IAC/C,OAAO,CAAC,WAAW,CAAyB;IAC5C,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,KAAK,CAAqB;IAClC,OAAO,CAAC,MAAM,CAA4C;IAE1D,OAAO,CAAC,mBAAmB,CAAa;IACxC,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAM;gBAEnC,EACV,OAAO,EACP,WAAW,EACX,SAAc,EACd,MAAM,GACP,EAAE,8BAA8B,CAAC,cAAc,CAAC;YAQnC,cAAc;IAS5B;;;;;OAKG;IAEH,OAAO,CAAC,2BAA2B;IAsB7B,SAAS,IAAI,OAAO,CAAC,2BAA2B,CAAC;IA8CjD,kBAAkB;CA0EzB"}
+{"version":3,"file":"attachments-streaming-pool.d.ts","sourceRoot":"","sources":["../../src/attachments-streaming/attachments-streaming-pool.ts"],"names":[],"mappings":"AAOA,OAAO,EAEL,2BAA2B,EAC5B,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,8BAA8B,EAAE,MAAM,yCAAyC,CAAC;AAEzF,qBAAa,wBAAwB,CAAC,cAAc;IAClD,OAAO,CAAC,OAAO,CAAoC;IACnD,OAAO,CAAC,WAAW,CAAyB;IAC5C,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,KAAK,CAAqB;IAClC,OAAO,CAAC,MAAM,CAA4C;IAE1D,OAAO,CAAC,mBAAmB,CAAa;IACxC,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAM;gBAEnC,EACV,OAAO,EACP,WAAW,EACX,SAAc,EACd,MAAM,GACP,EAAE,8BAA8B,CAAC,cAAc,CAAC;IAQjD,OAAO,CAAC,yBAAyB;IAyBjC;;;OAGG;IACH,OAAO,CAAC,iCAAiC;YAS3B,cAAc;IAUtB,SAAS,IAAI,OAAO,CAAC,2BAA2B,CAAC;IA6CjD,kBAAkB;CAwFzB"}

package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts

@@ -1,7 +1,8 @@
-import { ExternalSystemAttachmentStreamingFunction, NormalizedAttachment } from '../types';
-import { WorkerAdapter } from '../multithreading/worker-adapter/worker-adapter';
+import { ExtractionAdapter } from '../multithreading/adapters/extraction-adapter';
+import { NormalizedAttachment } from '../repo/repo.interfaces';
+import { ExternalSystemAttachmentStreamingFunction } from '../types/extraction';
 export interface AttachmentsStreamingPoolParams<ConnectorState> {
-    adapter: WorkerAdapter<ConnectorState>;
+    adapter: ExtractionAdapter<ConnectorState>;
     attachments: NormalizedAttachment[];
     batchSize?: number;
     stream: ExternalSystemAttachmentStreamingFunction;

package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"attachments-streaming-pool.interfaces.d.ts","sourceRoot":"","sources":["../../src/attachments-streaming/attachments-streaming-pool.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,yCAAyC,EACzC,oBAAoB,EACrB,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,aAAa,EAAE,MAAM,iDAAiD,CAAC;AAEhF,MAAM,WAAW,8BAA8B,CAAC,cAAc;IAC5D,OAAO,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;IACvC,WAAW,EAAE,oBAAoB,EAAE,CAAC;IACpC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,yCAAyC,CAAC;CACnD"}
+{"version":3,"file":"attachments-streaming-pool.interfaces.d.ts","sourceRoot":"","sources":["../../src/attachments-streaming/attachments-streaming-pool.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,+CAA+C,CAAC;AAClF,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,yCAAyC,EAAE,MAAM,qBAAqB,CAAC;AAEhF,MAAM,WAAW,8BAA8B,CAAC,cAAc;IAC5D,OAAO,EAAE,iBAAiB,CAAC,cAAc,CAAC,CAAC;IAC3C,WAAW,EAAE,oBAAoB,EAAE,CAAC;IACpC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,yCAAyC,CAAC;CACnD"}

package/dist/attachments-streaming/attachments-streaming-pool.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.AttachmentsStreamingPool = void 0;
 const helpers_1 = require("../common/helpers");
+const state_interfaces_1 = require("../state/state.interfaces");
 class AttachmentsStreamingPool {
     constructor({ adapter, attachments, batchSize = 10, stream, }) {
         this.totalProcessedCount = 0;
@@ -12,112 +13,119 @@ class AttachmentsStreamingPool {
         this.delay = undefined;
         this.stream = stream;
     }
-    async updateProgress() {
-        this.totalProcessedCount++;
-        if (this.totalProcessedCount % this.PROGRESS_REPORT_INTERVAL === 0) {
-            console.info(`Processed ${this.totalProcessedCount} attachments so far.`);
-            // Sleep for 100ms to avoid blocking the event loop
-            await (0, helpers_1.sleep)(100);
+    recordProcessedAttachment(attachment, status) {
+        var _a;
+        const attachmentsMetadata = (_a = this.adapter.sdkState.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata;
+        if (!(attachmentsMetadata === null || attachmentsMetadata === void 0 ? void 0 : attachmentsMetadata.lastProcessedAttachmentsIdsList)) {
+            return;
+        }
+        const processedAttachmentsIdsList = attachmentsMetadata.lastProcessedAttachmentsIdsList;
+        const alreadyRecorded = processedAttachmentsIdsList.some((it) => it.id === attachment.id && it.parent_id === attachment.parent_id);
+        if (!alreadyRecorded) {
+            processedAttachmentsIdsList.push({
+                id: attachment.id,
+                parent_id: attachment.parent_id,
+                status,
+            });
         }
     }
     /**
-     * Migrates processed attachments from the legacy string[] format to the new ProcessedAttachment[] format.
-     *
-     * @param attachments - The attachments list to migrate (either string[] or ProcessedAttachment[])
-     * @returns Migrated array of ProcessedAttachment objects, or empty array if input is invalid
+     * Backfills `status` on entries recorded before the field existed (those
+     * only ever held successes); state from >= v1.15.2 needs no shape migration.
      */
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    migrateProcessedAttachments(attachments) {
-        // Handle null/undefined
-        if (!attachments || !Array.isArray(attachments)) {
-            return [];
-        }
-        // If already migrated (first element is an object), return as-is
-        if (attachments.length > 0 && typeof attachments[0] === 'object') {
-            return attachments;
-        }
-        // Migrate old string[] format
-        if (attachments.length > 0 && typeof attachments[0] === 'string') {
-            return attachments.map((it) => ({
-                id: it,
-                parent_id: '',
-            }));
+    backfillProcessedAttachmentStatus(attachments) {
+        return attachments.map((it) => {
+            var _a;
+            return (Object.assign(Object.assign({}, it), { status: (_a = it.status) !== null && _a !== void 0 ? _a : state_interfaces_1.ProcessedAttachmentStatus.Success }));
+        });
+    }
+    async updateProgress() {
+        this.totalProcessedCount++;
+        if (this.totalProcessedCount % this.PROGRESS_REPORT_INTERVAL === 0) {
+            console.info(`Processed ${this.totalProcessedCount} attachments so far.`);
+            // Let a pending soft-timeout message (WorkerMessageExit) be delivered so
+            // adapter.isTimeout can flip before the next batch is processed.
+            await (0, helpers_1.yieldToEventLoop)();
         }
-        return [];
     }
     async streamAll() {
         console.log(`Starting download of ${this.attachments.length} attachments, streaming ${this.batchSize} at once.`);
-        if (!this.adapter.state.toDevRev) {
-            const error = new Error('toDevRev state is not initialized');
-            console.error(error);
-            return { error };
+        if (!this.adapter.sdkState.toDevRev) {
+            const message = 'toDevRev state is not initialized';
+            console.error(message);
+            return { status: 'error', error: { message } };
         }
-        // Get the list of successfully processed attachments in previous (possibly incomplete) batch extraction.
-        // If no such list exists, create an empty one.
-        if (!this.adapter.state.toDevRev.attachmentsMetadata
+        // Attachments processed (successfully or not) by a previous, possibly incomplete, run.
+        if (!this.adapter.sdkState.toDevRev.attachmentsMetadata
             .lastProcessedAttachmentsIdsList) {
-            this.adapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList =
+            this.adapter.sdkState.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList =
                 [];
         }
-        // Migrate old processed attachments to the new format.
-        this.adapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList =
-            this.migrateProcessedAttachments(this.adapter.state.toDevRev.attachmentsMetadata
+        this.adapter.sdkState.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList =
+            this.backfillProcessedAttachmentStatus(this.adapter.sdkState.toDevRev.attachmentsMetadata
                 .lastProcessedAttachmentsIdsList);
-        // Start initial batch of promises up to batchSize limit
         const initialBatchSize = Math.min(this.batchSize, this.attachments.length);
         const initialPromises = [];
         for (let i = 0; i < initialBatchSize; i++) {
             initialPromises.push(this.startPoolStreaming());
         }
-        // Wait for all promises to complete
-        await Promise.all(initialPromises);
+        await Promise.race([
+            Promise.all(initialPromises),
+            this.adapter.timeoutSignal,
+        ]);
         if (this.delay) {
-            return { delay: this.delay };
+            return { status: 'delay', delaySeconds: this.delay };
         }
-        return {};
+        return { status: 'success' };
     }
     async startPoolStreaming() {
-        var _a, _b, _c, _d;
-        // Process attachments until the attachments array is empty
+        var _a;
         while (this.attachments.length > 0) {
-            // If delay is set, stop streaming
             if (this.delay) {
                 break;
             }
-            // If timeout is set, stop streaming
             if (this.adapter.isTimeout) {
                 console.log('Timeout detected while streaming attachments. Stopping streaming.');
                 break;
             }
-            // Check if we can process next attachment
             const attachment = this.attachments.shift();
             if (!attachment) {
-                break; // Exit if no more attachments
+                break;
             }
-            if (this.adapter.state.toDevRev &&
-                ((_a = this.adapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList) === null || _a === void 0 ? void 0 : _a.some((it) => it.id == attachment.id && it.parent_id == attachment.parent_id))) {
-                continue; // Skip if the attachment ID is already processed
+            if (this.adapter.sdkState.toDevRev &&
+                ((_a = this.adapter.sdkState.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList) === null || _a === void 0 ? void 0 : _a.some((it) => it.id == attachment.id && it.parent_id == attachment.parent_id))) {
+                continue; // Already processed in a previous run
             }
             try {
                 const response = await this.adapter.processAttachment(attachment, this.stream);
-                // Check if rate limit was hit
-                if (response === null || response === void 0 ? void 0 : response.delay) {
-                    this.delay = response.delay; // Set the delay for rate limiting
+                if (response.status === 'delay') {
+                    this.delay = response.delaySeconds;
                     return;
                 }
-                if (response === null || response === void 0 ? void 0 : response.error) {
-                    console.warn(`Skipping attachment with ID ${attachment.id} due to error returned by the stream function`, response.error);
+                if (response.status === 'error') {
+                    const fileExtension = attachment.file_name.split('.').pop() || '';
+                    const fileSizeInfo = response.error.fileSize
+                        ? `and size ${response.error.fileSize} bytes `
+                        : '';
+                    const contentTypeInfo = attachment.content_type
+                        ? `and content_type ${attachment.content_type} `
+                        : '';
+                    console.warn(`Skipping attachment with ID ${attachment.id} with extension ${fileExtension} ${fileSizeInfo}${contentTypeInfo}due to error returned by the stream function`, response.error.message);
+                    this.recordProcessedAttachment(attachment, state_interfaces_1.ProcessedAttachmentStatus.Failed);
                     await this.updateProgress();
                     continue;
                 }
-                // No rate limiting, process normally
-                if ((_c = (_b = this.adapter.state.toDevRev) === null || _b === void 0 ? void 0 : _b.attachmentsMetadata) === null || _c === void 0 ? void 0 : _c.lastProcessedAttachmentsIdsList) {
-                    (_d = this.adapter.state.toDevRev) === null || _d === void 0 ? void 0 : _d.attachmentsMetadata.lastProcessedAttachmentsIdsList.push({ id: attachment.id, parent_id: attachment.parent_id });
+                if (!this.adapter.isTimeout) {
+                    this.recordProcessedAttachment(attachment, state_interfaces_1.ProcessedAttachmentStatus.Success);
                 }
                 await this.updateProgress();
             }
             catch (error) {
-                console.warn(`Skipping attachment with ID ${attachment.id} due to error in processAttachment function`, error);
+                const fileExtension = attachment.file_name.split('.').pop() || '';
+                const contentTypeInfo = attachment.content_type
+                    ? ` and content_type ${attachment.content_type}`
+                    : '';
+                console.warn(`Skipping attachment with ID ${attachment.id} with extension ${fileExtension}${contentTypeInfo} due to error in processAttachment function`, error);
                 await this.updateProgress();
             }
         }

package/dist/common/constants.d.ts

@@ -1,25 +1,28 @@
 import { EventType } from '../types/extraction';
-export declare const ALLOWED_EXTRACTION_EVENT_TYPES: EventType[];
-export declare const ALLOWED_LOADING_EVENT_TYPES: EventType[];
-export declare const ALLOWED_EVENT_TYPES: EventType[];
 export declare const STATELESS_EXTRACTION_EVENT_TYPES: EventType[];
 export declare const STATELESS_LOADING_EVENT_TYPES: EventType[];
 export declare const STATELESS_EVENT_TYPES: EventType[];
-export declare const STATEFUL_EXTRACTION_EVENT_TYPES: EventType[];
-export declare const STATEFUL_LOADING_EVENT_TYPES: EventType[];
-export declare const STATEFUL_EVENT_TYPES: EventType[];
 export declare const ARTIFACT_BATCH_SIZE = 2000;
 export declare const MAX_DEVREV_ARTIFACT_SIZE: number;
 export declare const MAX_DEVREV_FILENAME_LENGTH = 256;
 export declare const MAX_DEVREV_FILENAME_EXTENSION_LENGTH = 20;
-export declare const AIRDROP_DEFAULT_ITEM_TYPES: {
-    EXTERNAL_DOMAIN_METADATA: string;
-    ATTACHMENTS: string;
-    SSOR_ATTACHMENT: string;
-};
+export declare const MAX_EVENT_SIZE_BYTES = 200000;
+export declare const EVENT_SIZE_THRESHOLD_BYTES: number;
+export declare const SSOR_ATTACHMENT: string;
+export declare const UNKNOWN_EVENT_TYPE = "UNKNOWN_EVENT_TYPE";
+export declare enum AirSyncDefaultItemTypes {
+    EXTERNAL_DOMAIN_METADATA = "external_domain_metadata",
+    ATTACHMENTS = "attachments",
+    EXTERNAL_SYNC_UNITS = "external_sync_units"
+}
 export declare const LIBRARY_VERSION: any;
 export declare const DEFAULT_LAMBDA_TIMEOUT: number;
 export declare const HARD_TIMEOUT_MULTIPLIER = 1.3;
 export declare const MEMORY_LOG_INTERVAL: number;
-export declare const DEFAULT_SLEEP_DELAY_MS: number;
+/**
+ * Sentinel for an unbounded extraction timestamp (TimeValueType.UNBOUNDED),
+ * stored as workersOldest. The Unix epoch guarantees no real timestamp can be
+ * earlier, so subsequent syncs cannot accidentally overwrite the boundary.
+ */
+export declare const UNBOUNDED_DATE_TIME_VALUE = "1970-01-01T00:00:00.000Z";
 //# sourceMappingURL=constants.d.ts.map

package/dist/common/constants.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/common/constants.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAGhD,eAAO,MAAM,8BAA8B,aAS1C,CAAC;AAEF,eAAO,MAAM,2BAA2B,aAKvC,CAAC;AAEF,eAAO,MAAM,mBAAmB,aAG/B,CAAC;AAEF,eAAO,MAAM,gCAAgC,aAI5C,CAAC;AAEF,eAAO,MAAM,6BAA6B,aAGzC,CAAC;AAEF,eAAO,MAAM,qBAAqB,aAGjC,CAAC;AAEF,eAAO,MAAM,+BAA+B,aAGzC,CAAC;AAEJ,eAAO,MAAM,4BAA4B,aAExC,CAAC;AAEF,eAAO,MAAM,oBAAoB,aAGhC,CAAC;AAEF,eAAO,MAAM,mBAAmB,OAAO,CAAC;AACxC,eAAO,MAAM,wBAAwB,QAAyB,CAAC;AAC/D,eAAO,MAAM,0BAA0B,MAAM,CAAC;AAC9C,eAAO,MAAM,oCAAoC,KAAK,CAAC;AAEvD,eAAO,MAAM,0BAA0B;;;;CAItC,CAAC;AAEF,eAAO,MAAM,eAAe,KAAsB,CAAC;AAEnD,eAAO,MAAM,sBAAsB,QAAiB,CAAC;AACrD,eAAO,MAAM,uBAAuB,MAAM,CAAC;AAC3C,eAAO,MAAM,mBAAmB,QAAY,CAAC;AAE7C,eAAO,MAAM,sBAAsB,QAAgB,CAAC"}
+{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/common/constants.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAIhD,eAAO,MAAM,gCAAgC,aAI5C,CAAC;AAEF,eAAO,MAAM,6BAA6B,aAGzC,CAAC;AAEF,eAAO,MAAM,qBAAqB,aAGjC,CAAC;AAEF,eAAO,MAAM,mBAAmB,OAAO,CAAC;AACxC,eAAO,MAAM,wBAAwB,QAAyB,CAAC;AAC/D,eAAO,MAAM,0BAA0B,MAAM,CAAC;AAC9C,eAAO,MAAM,oCAAoC,KAAK,CAAC;AAGvD,eAAO,MAAM,oBAAoB,SAAU,CAAC;AAE5C,eAAO,MAAM,0BAA0B,QAEtC,CAAC;AAEF,eAAO,MAAM,eAAe,EAAE,MAA0B,CAAC;AACzD,eAAO,MAAM,kBAAkB,uBAAuB,CAAC;AAEvD,oBAAY,uBAAuB;IACjC,wBAAwB,6BAA6B;IACrD,WAAW,gBAAgB;IAC3B,mBAAmB,wBAAwB;CAC5C;AAED,eAAO,MAAM,eAAe,KAAsB,CAAC;AAEnD,eAAO,MAAM,sBAAsB,QAAiB,CAAC;AACrD,eAAO,MAAM,uBAAuB,MAAM,CAAC;AAC3C,eAAO,MAAM,mBAAmB,QAAY,CAAC;AAE7C;;;;GAIG;AACH,eAAO,MAAM,yBAAyB,6BAA6B,CAAC"}

package/dist/common/constants.js

@@ -1,28 +1,8 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.DEFAULT_SLEEP_DELAY_MS = exports.MEMORY_LOG_INTERVAL = exports.HARD_TIMEOUT_MULTIPLIER = exports.DEFAULT_LAMBDA_TIMEOUT = exports.LIBRARY_VERSION = exports.AIRDROP_DEFAULT_ITEM_TYPES = exports.MAX_DEVREV_FILENAME_EXTENSION_LENGTH = exports.MAX_DEVREV_FILENAME_LENGTH = exports.MAX_DEVREV_ARTIFACT_SIZE = exports.ARTIFACT_BATCH_SIZE = exports.STATEFUL_EVENT_TYPES = exports.STATEFUL_LOADING_EVENT_TYPES = exports.STATEFUL_EXTRACTION_EVENT_TYPES = exports.STATELESS_EVENT_TYPES = exports.STATELESS_LOADING_EVENT_TYPES = exports.STATELESS_EXTRACTION_EVENT_TYPES = exports.ALLOWED_EVENT_TYPES = exports.ALLOWED_LOADING_EVENT_TYPES = exports.ALLOWED_EXTRACTION_EVENT_TYPES = void 0;
+exports.UNBOUNDED_DATE_TIME_VALUE = exports.MEMORY_LOG_INTERVAL = exports.HARD_TIMEOUT_MULTIPLIER = exports.DEFAULT_LAMBDA_TIMEOUT = exports.LIBRARY_VERSION = exports.AirSyncDefaultItemTypes = exports.UNKNOWN_EVENT_TYPE = exports.SSOR_ATTACHMENT = exports.EVENT_SIZE_THRESHOLD_BYTES = exports.MAX_EVENT_SIZE_BYTES = exports.MAX_DEVREV_FILENAME_EXTENSION_LENGTH = exports.MAX_DEVREV_FILENAME_LENGTH = exports.MAX_DEVREV_ARTIFACT_SIZE = exports.ARTIFACT_BATCH_SIZE = exports.STATELESS_EVENT_TYPES = exports.STATELESS_LOADING_EVENT_TYPES = exports.STATELESS_EXTRACTION_EVENT_TYPES = void 0;
 const extraction_1 = require("../types/extraction");
 const helpers_1 = require("./helpers");
-exports.ALLOWED_EXTRACTION_EVENT_TYPES = [
-    extraction_1.EventType.StartExtractingExternalSyncUnits,
-    extraction_1.EventType.StartExtractingMetadata,
-    extraction_1.EventType.StartExtractingData,
-    extraction_1.EventType.ContinueExtractingData,
-    extraction_1.EventType.StartDeletingExtractorState,
-    extraction_1.EventType.StartExtractingAttachments,
-    extraction_1.EventType.ContinueExtractingAttachments,
-    extraction_1.EventType.StartDeletingExtractorAttachmentsState,
-];
-exports.ALLOWED_LOADING_EVENT_TYPES = [
-    extraction_1.EventType.StartLoadingData,
-    extraction_1.EventType.ContinueLoadingData,
-    extraction_1.EventType.StartDeletingLoaderState,
-    extraction_1.EventType.StartDeletingLoaderAttachmentState,
-];
-exports.ALLOWED_EVENT_TYPES = [
-    ...exports.ALLOWED_EXTRACTION_EVENT_TYPES,
-    ...exports.ALLOWED_LOADING_EVENT_TYPES,
-];
 exports.STATELESS_EXTRACTION_EVENT_TYPES = [
     extraction_1.EventType.StartExtractingExternalSyncUnits,
     extraction_1.EventType.StartDeletingExtractorState,
@@ -36,23 +16,29 @@ exports.STATELESS_EVENT_TYPES = [
     ...exports.STATELESS_EXTRACTION_EVENT_TYPES,
     ...exports.STATELESS_LOADING_EVENT_TYPES,
 ];
-exports.STATEFUL_EXTRACTION_EVENT_TYPES = exports.ALLOWED_EXTRACTION_EVENT_TYPES.filter((eventType) => !exports.STATELESS_EXTRACTION_EVENT_TYPES.includes(eventType));
-exports.STATEFUL_LOADING_EVENT_TYPES = exports.ALLOWED_LOADING_EVENT_TYPES.filter((eventType) => !exports.STATELESS_LOADING_EVENT_TYPES.includes(eventType));
-exports.STATEFUL_EVENT_TYPES = [
-    ...exports.STATEFUL_EXTRACTION_EVENT_TYPES,
-    ...exports.STATEFUL_LOADING_EVENT_TYPES,
-];
 exports.ARTIFACT_BATCH_SIZE = 2000;
 exports.MAX_DEVREV_ARTIFACT_SIZE = 2 * 1024 * 1024 * 1024; // 2GB
 exports.MAX_DEVREV_FILENAME_LENGTH = 256;
-exports.MAX_DEVREV_FILENAME_EXTENSION_LENGTH = 20; // 20 characters for the file extension
-exports.AIRDROP_DEFAULT_ITEM_TYPES = {
-    EXTERNAL_DOMAIN_METADATA: 'external_domain_metadata',
-    ATTACHMENTS: 'attachments',
-    SSOR_ATTACHMENT: 'ssor_attachment',
-};
+exports.MAX_DEVREV_FILENAME_EXTENSION_LENGTH = 20;
+// Max SQS message size is 250KB, we want to leave some room for the other data in the message
+exports.MAX_EVENT_SIZE_BYTES = 200000;
+// We want to leave some room for the other data in the message and process the rest of queued messages
+exports.EVENT_SIZE_THRESHOLD_BYTES = Math.floor(exports.MAX_EVENT_SIZE_BYTES * 0.8);
+exports.SSOR_ATTACHMENT = 'ssor_attachment';
+exports.UNKNOWN_EVENT_TYPE = 'UNKNOWN_EVENT_TYPE';
+var AirSyncDefaultItemTypes;
+(function (AirSyncDefaultItemTypes) {
+    AirSyncDefaultItemTypes["EXTERNAL_DOMAIN_METADATA"] = "external_domain_metadata";
+    AirSyncDefaultItemTypes["ATTACHMENTS"] = "attachments";
+    AirSyncDefaultItemTypes["EXTERNAL_SYNC_UNITS"] = "external_sync_units";
+})(AirSyncDefaultItemTypes || (exports.AirSyncDefaultItemTypes = AirSyncDefaultItemTypes = {}));
 exports.LIBRARY_VERSION = (0, helpers_1.getLibraryVersion)();
 exports.DEFAULT_LAMBDA_TIMEOUT = 10 * 60 * 1000; // 10 minutes
 exports.HARD_TIMEOUT_MULTIPLIER = 1.3;
 exports.MEMORY_LOG_INTERVAL = 30 * 1000; // 30 seconds
-exports.DEFAULT_SLEEP_DELAY_MS = 3 * 60 * 1000; // 3 minutes
+/**
+ * Sentinel for an unbounded extraction timestamp (TimeValueType.UNBOUNDED),
+ * stored as workersOldest. The Unix epoch guarantees no real timestamp can be
+ * earlier, so subsequent syncs cannot accidentally overwrite the boundary.
+ */
+exports.UNBOUNDED_DATE_TIME_VALUE = '1970-01-01T00:00:00.000Z';

package/dist/common/helpers.d.ts

@@ -1,30 +1,10 @@
-/**
- * Gets the library version from the package.json file.
- * @returns {string} The library version
- */
 export declare function getLibraryVersion(): any;
-/**
- * Sleeps for a given number of milliseconds.
- * @param {number} ms - The number of milliseconds to sleep
- * @returns {Promise<void>} A promise that resolves after the given number of milliseconds
- */
 export declare function sleep(ms: number): Promise<unknown>;
 /**
- * Truncates a filename if it exceeds the maximum allowed length.
- * @param {string} filename - The filename to truncate
- * @returns {string} The truncated filename
- */
-export declare function truncateFilename(filename: string): string;
-/**
- * MemoryInfo is an interface that represents the memory usage information.
- * @interface MemoryInfo
- * @property {string} rssUsedMB - The RSS used in MB
- * @property {string} rssUsedPercent - The RSS used percentage
- * @property {string} heapUsedPercent - The heap used percentage
- * @property {string} externalMB - The external memory used in MB
- * @property {string} arrayBuffersMB - The array buffers memory used in MB
- * @property {string} formattedMessage - The formatted message
+ * Yields once to the event loop so pending events (e.g. worker messages) can
+ * be delivered before continuing.
  */
+export declare function yieldToEventLoop(): Promise<void>;
 export interface MemoryInfo {
     rssUsedMB: string;
     rssUsedPercent: string;
@@ -33,9 +13,5 @@ export interface MemoryInfo {
     arrayBuffersMB: string;
     formattedMessage: string;
 }
-/**
- * Gets the memory usage information.
- * @returns {MemoryInfo} The memory usage information
- */
 export declare function getMemoryUsage(): MemoryInfo;
 //# sourceMappingURL=helpers.d.ts.map

package/dist/common/helpers.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"helpers.d.ts","sourceRoot":"","sources":["../../src/common/helpers.ts"],"names":[],"mappings":"AASA;;;GAGG;AACH,wBAAgB,iBAAiB,QAiBhC;AAED;;;;GAIG;AACH,wBAAsB,KAAK,CAAC,EAAE,EAAE,MAAM,oBAGrC;AAED;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAmBzD;AAED;;;;;;;;;GASG;AACH,MAAM,WAAW,UAAU;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC;IACvB,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED;;;GAGG;AACH,wBAAgB,cAAc,IAAI,UAAU,CAmD3C"}
+{"version":3,"file":"helpers.d.ts","sourceRoot":"","sources":["../../src/common/helpers.ts"],"names":[],"mappings":"AAIA,wBAAgB,iBAAiB,QAchC;AAED,wBAAsB,KAAK,CAAC,EAAE,EAAE,MAAM,oBAErC;AAED;;;GAGG;AACH,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC,CAEtD;AAED,MAAM,WAAW,UAAU;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC;IACvB,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,wBAAgB,cAAc,IAAI,UAAU,CA+C3C"}

package/dist/common/helpers.js

@@ -35,16 +35,11 @@ var __importStar = (this && this.__importStar) || (function () {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.getLibraryVersion = getLibraryVersion;
 exports.sleep = sleep;
-exports.truncateFilename = truncateFilename;
+exports.yieldToEventLoop = yieldToEventLoop;
 exports.getMemoryUsage = getMemoryUsage;
 const fs_1 = require("fs");
 const path = __importStar(require("path"));
 const v8 = __importStar(require("v8"));
-const constants_1 = require("./constants");
-/**
- * Gets the library version from the package.json file.
- * @returns {string} The library version
- */
 function getLibraryVersion() {
     var _a;
     try {
@@ -55,41 +50,20 @@ function getLibraryVersion() {
         return '';
     }
     catch (error) {
-        console.error('Error reading adaas library version from package.json', error);
+        console.error('Error reading SDK version from package.json', error);
         return '';
     }
 }
-/**
- * Sleeps for a given number of milliseconds.
- * @param {number} ms - The number of milliseconds to sleep
- * @returns {Promise<void>} A promise that resolves after the given number of milliseconds
- */
 async function sleep(ms) {
-    console.log(`Sleeping for ${ms}ms.`);
     return new Promise((resolve) => setTimeout(resolve, ms));
 }
 /**
- * Truncates a filename if it exceeds the maximum allowed length.
- * @param {string} filename - The filename to truncate
- * @returns {string} The truncated filename
+ * Yields once to the event loop so pending events (e.g. worker messages) can
+ * be delivered before continuing.
  */
-function truncateFilename(filename) {
-    // If the filename is already within the limit, return it as is.
-    if (filename.length <= constants_1.MAX_DEVREV_FILENAME_LENGTH) {
-        return filename;
-    }
-    console.warn(`Filename length exceeds the maximum limit of ${constants_1.MAX_DEVREV_FILENAME_LENGTH} characters. Truncating filename.`);
-    const extension = filename.slice(-constants_1.MAX_DEVREV_FILENAME_EXTENSION_LENGTH);
-    // Calculate how many characters are available for the name part after accounting for the extension and "..."
-    const availableNameLength = constants_1.MAX_DEVREV_FILENAME_LENGTH - constants_1.MAX_DEVREV_FILENAME_EXTENSION_LENGTH - 3; // -3 for "..."
-    // Truncate the name part and add an ellipsis
-    const truncatedFilename = filename.slice(0, availableNameLength);
-    return `${truncatedFilename}...${extension}`;
+async function yieldToEventLoop() {
+    return new Promise((resolve) => setImmediate(resolve));
 }
-/**
- * Gets the memory usage information.
- * @returns {MemoryInfo} The memory usage information
- */
 function getMemoryUsage() {
     try {
         const memUsage = process.memoryUsage();
@@ -97,16 +71,12 @@ function getMemoryUsage() {
         const rssUsedMB = memUsage.rss / 1024 / 1024;
         const heapLimitMB = heapStats.heap_size_limit / 1024 / 1024;
         const effectiveMemoryLimitMB = heapLimitMB;
-        // Calculate heap values for consistent format
         const heapUsedMB = heapStats.used_heap_size / 1024 / 1024;
         const heapTotalMB = heapStats.heap_size_limit / 1024 / 1024;
-        // Calculate external and buffer values (critical for detecting stream leaks)
         const externalMB = memUsage.external / 1024 / 1024;
         const arrayBuffersMB = memUsage.arrayBuffers / 1024 / 1024;
-        // Critical percentages for OOM detection
         const rssUsedPercent = ((rssUsedMB / effectiveMemoryLimitMB) * 100).toFixed(2) + '%';
         const heapUsedPercent = ((heapStats.used_heap_size / heapStats.heap_size_limit) * 100).toFixed(2) + '%';
-        // Detailed message showing RSS breakdown for leak detection
         const formattedMessage = `Memory: RSS ${rssUsedMB.toFixed(2)}/${effectiveMemoryLimitMB.toFixed(2)}MB (${rssUsedPercent}) [Heap ${heapUsedMB.toFixed(2)}/${heapTotalMB.toFixed(2)}MB (${heapUsedPercent}) + External ${externalMB.toFixed(2)}MB + Buffers ${arrayBuffersMB.toFixed(2)}MB].`;
         return {
             rssUsedMB: rssUsedMB.toFixed(2),

package/dist/http/client.d.ts

@@ -0,0 +1,4 @@
+import { AxiosInstance } from 'axios';
+declare const axiosClient: Pick<AxiosInstance, 'get' | 'post'>;
+export { axiosClient };
+//# sourceMappingURL=client.d.ts.map

package/dist/http/client.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../src/http/client.ts"],"names":[],"mappings":"AAAA,OAAc,EAAc,aAAa,EAAE,MAAM,OAAO,CAAC;AAmGzD,QAAA,MAAM,WAAW,EAAE,IAAI,CAAC,aAAa,EAAE,KAAK,GAAG,MAAM,CAGpD,CAAC;AAEF,OAAO,EAAE,WAAW,EAAE,CAAC"}