@devrev/airsync-sdk 2.0.0-beta.1 → 2.0.0-beta.10

package/dist/multithreading/spawn/spawn.helpers.js

@@ -1,93 +1,158 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.getEventTypeForResult = getEventTypeForResult;
 exports.getTimeoutErrorEventType = getTimeoutErrorEventType;
 exports.getNoScriptEventType = getNoScriptEventType;
+const constants_1 = require("../../common/constants");
 const extraction_1 = require("../../types/extraction");
 const loading_1 = require("../../types/loading");
 /**
- * Gets the event type for the timeout error.
- * @param {EventType} eventType - The event type to get the timeout error event type for
- * @returns {ExtractorEventType | LoaderEventType} The event type for the timeout error
+ * Resolves the outgoing event type to emit for an incoming event type and a
+ * {@link TaskResult} status. Resumable phases honor every status
+ * (success/progress/delay/error -> *_DONE/*_PROGRESS/*_DELAYED/*_ERROR);
+ * non-resumable phases only have done/error, so `progress`/`delay` there is
+ * illegal and maps to the phase's error event.
  */
+function getEventTypeForResult(eventType, status) {
+    const phase = EVENT_PHASE_MAP[eventType];
+    if (!phase) {
+        console.error('Event type not recognized in getEventTypeForResult function: ' +
+            eventType);
+        return {
+            eventType: constants_1.UNKNOWN_EVENT_TYPE,
+            illegal: true,
+        };
+    }
+    // Non-resumable phases only define done/error events.
+    if (!phase.resumable) {
+        if (status === 'success') {
+            return { eventType: phase.done, illegal: false };
+        }
+        // progress/delay are illegal here; collapse them (and error) to the error event.
+        return { eventType: phase.error, illegal: status !== 'error' };
+    }
+    switch (status) {
+        case 'success':
+            return { eventType: phase.done, illegal: false };
+        case 'progress':
+            return { eventType: phase.progress, illegal: false };
+        case 'delay':
+            return { eventType: phase.delayed, illegal: false };
+        case 'error':
+            return { eventType: phase.error, illegal: false };
+    }
+}
+/** Per-phase outgoing event types, keyed by the incoming {@link EventType}. */
+const EVENT_PHASE_MAP = {
+    [extraction_1.EventType.StartExtractingExternalSyncUnits]: {
+        resumable: false,
+        done: extraction_1.ExtractorEventType.ExternalSyncUnitExtractionDone,
+        error: extraction_1.ExtractorEventType.ExternalSyncUnitExtractionError,
+    },
+    [extraction_1.EventType.StartExtractingMetadata]: {
+        resumable: true,
+        done: extraction_1.ExtractorEventType.MetadataExtractionDone,
+        error: extraction_1.ExtractorEventType.MetadataExtractionError,
+        progress: extraction_1.ExtractorEventType.MetadataExtractionProgress,
+        delayed: extraction_1.ExtractorEventType.MetadataExtractionDelayed,
+    },
+    [extraction_1.EventType.ContinueExtractingMetadata]: {
+        resumable: true,
+        done: extraction_1.ExtractorEventType.MetadataExtractionDone,
+        error: extraction_1.ExtractorEventType.MetadataExtractionError,
+        progress: extraction_1.ExtractorEventType.MetadataExtractionProgress,
+        delayed: extraction_1.ExtractorEventType.MetadataExtractionDelayed,
+    },
+    [extraction_1.EventType.StartExtractingData]: {
+        resumable: true,
+        done: extraction_1.ExtractorEventType.DataExtractionDone,
+        error: extraction_1.ExtractorEventType.DataExtractionError,
+        progress: extraction_1.ExtractorEventType.DataExtractionProgress,
+        delayed: extraction_1.ExtractorEventType.DataExtractionDelayed,
+    },
+    [extraction_1.EventType.ContinueExtractingData]: {
+        resumable: true,
+        done: extraction_1.ExtractorEventType.DataExtractionDone,
+        error: extraction_1.ExtractorEventType.DataExtractionError,
+        progress: extraction_1.ExtractorEventType.DataExtractionProgress,
+        delayed: extraction_1.ExtractorEventType.DataExtractionDelayed,
+    },
+    [extraction_1.EventType.StartDeletingExtractorState]: {
+        resumable: false,
+        done: extraction_1.ExtractorEventType.ExtractorStateDeletionDone,
+        error: extraction_1.ExtractorEventType.ExtractorStateDeletionError,
+    },
+    [extraction_1.EventType.StartExtractingAttachments]: {
+        resumable: true,
+        done: extraction_1.ExtractorEventType.AttachmentExtractionDone,
+        error: extraction_1.ExtractorEventType.AttachmentExtractionError,
+        progress: extraction_1.ExtractorEventType.AttachmentExtractionProgress,
+        delayed: extraction_1.ExtractorEventType.AttachmentExtractionDelayed,
+    },
+    [extraction_1.EventType.ContinueExtractingAttachments]: {
+        resumable: true,
+        done: extraction_1.ExtractorEventType.AttachmentExtractionDone,
+        error: extraction_1.ExtractorEventType.AttachmentExtractionError,
+        progress: extraction_1.ExtractorEventType.AttachmentExtractionProgress,
+        delayed: extraction_1.ExtractorEventType.AttachmentExtractionDelayed,
+    },
+    [extraction_1.EventType.StartDeletingExtractorAttachmentsState]: {
+        resumable: false,
+        done: extraction_1.ExtractorEventType.ExtractorAttachmentsStateDeletionDone,
+        error: extraction_1.ExtractorEventType.ExtractorAttachmentsStateDeletionError,
+    },
+    [extraction_1.EventType.StartLoadingData]: {
+        resumable: true,
+        done: loading_1.LoaderEventType.DataLoadingDone,
+        error: loading_1.LoaderEventType.DataLoadingError,
+        progress: loading_1.LoaderEventType.DataLoadingProgress,
+        delayed: loading_1.LoaderEventType.DataLoadingDelayed,
+    },
+    [extraction_1.EventType.ContinueLoadingData]: {
+        resumable: true,
+        done: loading_1.LoaderEventType.DataLoadingDone,
+        error: loading_1.LoaderEventType.DataLoadingError,
+        progress: loading_1.LoaderEventType.DataLoadingProgress,
+        delayed: loading_1.LoaderEventType.DataLoadingDelayed,
+    },
+    [extraction_1.EventType.StartLoadingAttachments]: {
+        resumable: true,
+        done: loading_1.LoaderEventType.AttachmentLoadingDone,
+        error: loading_1.LoaderEventType.AttachmentLoadingError,
+        progress: loading_1.LoaderEventType.AttachmentLoadingProgress,
+        delayed: loading_1.LoaderEventType.AttachmentLoadingDelayed,
+    },
+    [extraction_1.EventType.ContinueLoadingAttachments]: {
+        resumable: true,
+        done: loading_1.LoaderEventType.AttachmentLoadingDone,
+        error: loading_1.LoaderEventType.AttachmentLoadingError,
+        progress: loading_1.LoaderEventType.AttachmentLoadingProgress,
+        delayed: loading_1.LoaderEventType.AttachmentLoadingDelayed,
+    },
+    [extraction_1.EventType.StartDeletingLoaderState]: {
+        resumable: false,
+        done: loading_1.LoaderEventType.LoaderStateDeletionDone,
+        error: loading_1.LoaderEventType.LoaderStateDeletionError,
+    },
+    [extraction_1.EventType.StartDeletingLoaderAttachmentState]: {
+        resumable: false,
+        done: loading_1.LoaderEventType.LoaderAttachmentStateDeletionDone,
+        error: loading_1.LoaderEventType.LoaderAttachmentStateDeletionError,
+    },
+};
 function getTimeoutErrorEventType(eventType) {
+    return { eventType: getEventTypeForResult(eventType, 'error').eventType };
+}
+/** Event type to emit when no worker script exists for the incoming event. */
+function getNoScriptEventType(eventType) {
     switch (eventType) {
-        // Metadata extraction (handles both old and new enum members)
+        // A missing metadata worker is a misconfigured connector, so report it as a
+        // phase error rather than an unknown event type.
         case extraction_1.EventType.StartExtractingMetadata:
-        case extraction_1.EventType.ExtractionMetadataStart:
+        case extraction_1.EventType.ContinueExtractingMetadata:
             return {
                 eventType: extraction_1.ExtractorEventType.MetadataExtractionError,
             };
-        // Data extraction (handles both old and new enum members)
-        case extraction_1.EventType.StartExtractingData:
-        case extraction_1.EventType.ContinueExtractingData:
-        case extraction_1.EventType.ExtractionDataStart:
-        case extraction_1.EventType.ExtractionDataContinue:
-            return {
-                eventType: extraction_1.ExtractorEventType.DataExtractionError,
-            };
-        // Data deletion (handles both old and new enum members)
-        case extraction_1.EventType.StartDeletingExtractorState:
-        case extraction_1.EventType.ExtractionDataDelete:
-            return {
-                eventType: extraction_1.ExtractorEventType.ExtractorStateDeletionError,
-            };
-        // Attachments extraction (handles both old and new enum members)
-        case extraction_1.EventType.StartExtractingAttachments:
-        case extraction_1.EventType.ContinueExtractingAttachments:
-        case extraction_1.EventType.ExtractionAttachmentsStart:
-        case extraction_1.EventType.ExtractionAttachmentsContinue:
-            return {
-                eventType: extraction_1.ExtractorEventType.AttachmentExtractionError,
-            };
-        // Attachments deletion (handles both old and new enum members)
-        case extraction_1.EventType.StartDeletingExtractorAttachmentsState:
-        case extraction_1.EventType.ExtractionAttachmentsDelete:
-            return {
-                eventType: extraction_1.ExtractorEventType.ExtractorAttachmentsStateDeletionError,
-            };
-        // External sync units (handles both old and new enum members)
-        case extraction_1.EventType.StartExtractingExternalSyncUnits:
-        case extraction_1.EventType.ExtractionExternalSyncUnitsStart:
-            return {
-                eventType: extraction_1.ExtractorEventType.ExternalSyncUnitExtractionError,
-            };
-        // Loading data
-        case extraction_1.EventType.StartLoadingData:
-        case extraction_1.EventType.ContinueLoadingData:
-            return {
-                eventType: loading_1.LoaderEventType.DataLoadingError,
-            };
-        // Deleting loader state
-        case extraction_1.EventType.StartDeletingLoaderState:
-            return {
-                eventType: loading_1.LoaderEventType.LoaderStateDeletionError,
-            };
-        // Loading attachments
-        case extraction_1.EventType.StartLoadingAttachments:
-        case extraction_1.EventType.ContinueLoadingAttachments:
-            return {
-                eventType: loading_1.LoaderEventType.AttachmentLoadingError,
-            };
-        // Deleting loader attachment state
-        case extraction_1.EventType.StartDeletingLoaderAttachmentState:
-            return {
-                eventType: loading_1.LoaderEventType.LoaderAttachmentStateDeletionError,
-            };
-        default:
-            console.error('Event type not recognized in getTimeoutErrorEventType function: ' +
-                eventType);
-            return {
-                eventType: loading_1.LoaderEventType.UnknownEventType,
-            };
-    }
-}
-/**
- * Gets the event type for the no script error.
- * @param {EventType} eventType - The event type to get the no script error event type for
- * @returns {ExtractorEventType | LoaderEventType} The event type for the no script error
- */
-function getNoScriptEventType(eventType) {
-    switch (eventType) {
         case extraction_1.EventType.StartDeletingExtractorState:
             return {
                 eventType: extraction_1.ExtractorEventType.ExtractorStateDeletionDone,
@@ -108,7 +173,7 @@ function getNoScriptEventType(eventType) {
             console.error('Event type not recognized in getNoScriptEventType function: ' +
                 eventType);
             return {
-                eventType: loading_1.LoaderEventType.UnknownEventType,
+                eventType: constants_1.UNKNOWN_EVENT_TYPE,
             };
     }
 }

package/dist/multithreading/spawn/spawn.js

@@ -1,21 +1,36 @@
 "use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Spawn = void 0;
+exports.isLocalDevelopment = isLocalDevelopment;
 exports.spawn = spawn;
-const yargs_1 = __importDefault(require("yargs"));
-const helpers_1 = require("yargs/helpers");
-const control_protocol_1 = require("../../common/control-protocol");
-const event_type_translation_1 = require("../../common/event-type-translation");
-const helpers_2 = require("../../common/helpers");
+const constants_1 = require("../../common/constants");
+const helpers_1 = require("../../common/helpers");
 const logger_1 = require("../../logger/logger");
 const extraction_1 = require("../../types/extraction");
 const workers_1 = require("../../types/workers");
-const constants_1 = require("../../common/constants");
 const create_worker_1 = require("../create-worker");
+const emit_1 = require("../emit");
 const spawn_helpers_1 = require("./spawn.helpers");
+const LAMBDA_ENV_MARKERS = [
+    'AWS_LAMBDA_FUNCTION_NAME',
+    'AWS_EXECUTION_ENV',
+    'LAMBDA_TASK_ROOT',
+];
+/**
+ * Local mode is opt-in through AIRSYNC_LOCAL_DEVELOPMENT only, and refused on
+ * Lambda so a stray flag can never change production behavior.
+ */
+function isLocalDevelopment(env = process.env) {
+    if (env.AIRSYNC_LOCAL_DEVELOPMENT !== '1') {
+        return false;
+    }
+    const marker = LAMBDA_ENV_MARKERS.find((name) => env[name]);
+    if (marker) {
+        console.warn(`Ignoring AIRSYNC_LOCAL_DEVELOPMENT because ${marker} is set (running on Lambda).`);
+        return false;
+    }
+    return true;
+}
 function getWorkerPath({ event, workerBasePath, }) {
     let path = null;
     switch (event.payload.event_type) {
@@ -23,6 +38,7 @@ function getWorkerPath({ event, workerBasePath, }) {
             path = '/workers/external-sync-units-extraction';
             break;
         case extraction_1.EventType.StartExtractingMetadata:
+        case extraction_1.EventType.ContinueExtractingMetadata:
             path = '/workers/metadata-extraction';
             break;
         case extraction_1.EventType.StartExtractingData:
@@ -45,55 +61,22 @@ function getWorkerPath({ event, workerBasePath, }) {
     return path ? workerBasePath + path : null;
 }
 /**
- * Creates a new instance of Spawn class.
- * Spawn class is responsible for spawning a new worker thread and managing the lifecycle of the worker.
- * The class provides utilities to emit control events to the platform and exit the worker gracefully.
- * In case of lambda timeout, the class emits a lambda timeout event to the platform.
- * @param {SpawnFactoryInterface} options - The options to create a new instance of Spawn class
- * @param {AirdropEvent} options.event - The event object received from the platform
- * @param {object} options.initialState - The initial state of the adapter
- * @param {string} [options.workerPath] Remove getWorkerPath function and use baseWorkerPath: __dirname instead of workerPath
- * @param {string} [options.baseWorkerPath] - The base path for the worker files, usually `__dirname`
- * @returns {Promise<Spawn>} - A new instance of Spawn class
+ * Spawns a worker thread for the event and manages its lifecycle (control
+ * events, graceful exit, lambda timeout handling). Resolves when the worker
+ * run is fully finished.
  */
-async function spawn({ event, initialState, workerPath, initialDomainMapping, options, baseWorkerPath, }) {
-    // Translate incoming event type for backwards compatibility. This allows the
-    // SDK to accept both old and new event type formats. Then update the event with the translated event type.
-    const originalEventType = event.payload.event_type;
-    const translatedEventType = (0, event_type_translation_1.translateIncomingEventType)(event.payload.event_type);
-    event.payload.event_type = translatedEventType;
-    // Read the command line arguments to check if the local flag is passed.
-    const argv = await (0, yargs_1.default)((0, helpers_1.hideBin)(process.argv)).argv;
-    if (argv._.includes('local')) {
-        options = Object.assign(Object.assign({}, (options || {})), { isLocalDevelopment: true });
-    }
+async function spawn({ event, initialState, initialDomainMapping, options: spawnOptions, baseWorkerPath, workerPathOverrides, }) {
+    const options = Object.assign(Object.assign({}, spawnOptions), { isLocalDevelopment: isLocalDevelopment() });
     const originalConsole = console;
     // eslint-disable-next-line no-global-assign
     console = new logger_1.Logger({ event, options });
-    if (translatedEventType !== originalEventType) {
-        console.log(`Event type translated from ${originalEventType} to ${translatedEventType}.`);
-    }
     if (options === null || options === void 0 ? void 0 : options.isLocalDevelopment) {
         console.log('Snap-in is running in local development mode.');
     }
-    let script = null;
-    if (workerPath != null) {
-        script = workerPath;
-    }
-    else if (baseWorkerPath != null &&
-        (options === null || options === void 0 ? void 0 : options.workerPathOverrides) != null &&
-        options.workerPathOverrides[translatedEventType] != null) {
-        script =
-            baseWorkerPath +
-                options.workerPathOverrides[translatedEventType];
-    }
-    else {
-        script = getWorkerPath({
-            event,
-            workerBasePath: baseWorkerPath !== null && baseWorkerPath !== void 0 ? baseWorkerPath : __dirname,
-        });
-    }
-    // If a script is found for the event type, spawn a new worker.
+    const override = workerPathOverrides === null || workerPathOverrides === void 0 ? void 0 : workerPathOverrides[event.payload.event_type];
+    const script = override != null
+        ? baseWorkerPath + override
+        : getWorkerPath({ event, workerBasePath: baseWorkerPath });
     if (script) {
         try {
             const worker = await (0, create_worker_1.createWorker)({
@@ -122,7 +105,7 @@ async function spawn({ event, initialState, workerPath, initialDomainMapping, op
     }
     else {
         const { eventType } = (0, spawn_helpers_1.getNoScriptEventType)(event.payload.event_type);
-        await (0, control_protocol_1.emit)({
+        await (0, emit_1.emit)({
             event,
             eventType,
         });
@@ -143,7 +126,7 @@ class Spawn {
             ? Math.min(options.timeout, this.defaultLambdaTimeout)
             : this.defaultLambdaTimeout;
         this.resolve = resolve;
-        // If soft timeout is reached, send a message to the worker to gracefully exit.
+        // Soft timeout: ask the worker to gracefully exit.
         this.softTimeoutTimer = setTimeout(() => void (async () => {
             console.log('SOFT TIMEOUT: Sending a message to the worker to gracefully exit.');
             this.softTimeoutSent = true;
@@ -157,7 +140,7 @@ class Spawn {
                 await this.exitFromMainThread();
             }
         })(), this.lambdaTimeout);
-        // If hard timeout is reached, that means the worker did not exit in time. Terminate the worker.
+        // Hard timeout: the worker did not exit in time, terminate it.
         this.hardTimeoutTimer = setTimeout(() => void (async () => {
             console.error('HARD TIMEOUT: Worker did not exit in time. Terminating the worker.');
             if (worker) {
@@ -168,13 +151,10 @@ class Spawn {
                 await this.exitFromMainThread();
             }
         })(), this.lambdaTimeout * constants_1.HARD_TIMEOUT_MULTIPLIER);
-        // If worker exits with process.exit(code), clear the timeouts and exit from
-        // main thread. When a soft timeout was sent, we use setImmediate to defer
-        // processing so that any pending WorkerMessage events (e.g.
-        // WorkerMessageEmitted from onTimeout) already queued in the event loop are
-        // handled first, preventing a race condition where exitFromMainThread sees
-        // alreadyEmitted=false and emits an error even though the worker
-        // successfully emitted an event.
+        // After a soft timeout, defer exit handling via setImmediate so pending
+        // WorkerMessage events (e.g. WorkerMessageEmitted from onTimeout) are
+        // handled first; otherwise exitFromMainThread could see
+        // alreadyEmitted=false and emit a spurious error.
         worker.on(workers_1.WorkerEvent.WorkerExit, (code) => {
             const handler = async () => {
                 console.info('Worker exited with exit code: ' + code + '.');
@@ -189,32 +169,32 @@ class Spawn {
             }
         });
         worker.on(workers_1.WorkerEvent.WorkerMessage, (message) => {
-            var _a, _b, _c, _d;
-            // Since logs from the worker thread are handled differently in snap-in
-            // platform,  we need to catch the log messages from worker thread and log
-            // them in main thread.
+            var _a, _b, _c;
+            // The snap-in platform handles worker-thread logs differently, so worker
+            // log messages are re-logged in the main thread.
             if ((message === null || message === void 0 ? void 0 : message.subject) === workers_1.WorkerMessageSubject.WorkerMessageLog) {
                 const stringifiedArgs = (_a = message.payload) === null || _a === void 0 ? void 0 : _a.stringifiedArgs;
                 const level = (_b = message.payload) === null || _b === void 0 ? void 0 : _b.level;
-                const isSdkLog = (_d = (_c = message.payload) === null || _c === void 0 ? void 0 : _c.isSdkLog) !== null && _d !== void 0 ? _d : false;
-                this.logger.logFn(stringifiedArgs, level, isSdkLog);
+                this.logger.logFn(stringifiedArgs, level);
             }
-            // If worker sends a message that it has emitted an event, then set alreadyEmitted to true.
             if ((message === null || message === void 0 ? void 0 : message.subject) === workers_1.WorkerMessageSubject.WorkerMessageEmitted) {
-                console.info('Worker has emitted message to ADaaS.');
+                console.info('Worker has emitted message to AirSync.');
                 this.alreadyEmitted = true;
             }
+            // Capture the worker's failure reason for use in the error event.
+            if ((message === null || message === void 0 ? void 0 : message.subject) === workers_1.WorkerMessageSubject.WorkerMessageFailed) {
+                this.workerFailedMessage = (_c = message.payload) === null || _c === void 0 ? void 0 : _c.message;
+            }
         });
-        // Log memory usage every 30 seconds
         this.memoryMonitoringInterval = setInterval(() => {
             try {
-                const memoryInfo = (0, helpers_2.getMemoryUsage)();
+                const memoryInfo = (0, helpers_1.getMemoryUsage)();
                 if (memoryInfo) {
                     console.info(memoryInfo.formattedMessage);
                 }
             }
             catch (error) {
-                // If memory monitoring fails, log the warning and clear the interval to prevent further issues
+                // Stop monitoring on failure to prevent repeated crashes
                 console.warn('Memory monitoring failed, stopping logging of memory usage interval', error);
                 if (this.memoryMonitoringInterval) {
                     clearInterval(this.memoryMonitoringInterval);
@@ -245,20 +225,24 @@ class Spawn {
         this.alreadyEmitted = true;
         const { eventType } = (0, spawn_helpers_1.getTimeoutErrorEventType)(this.event.payload.event_type);
         try {
-            await (0, control_protocol_1.emit)({
+            await (0, emit_1.emit)({
                 eventType,
                 event: this.event,
                 data: {
                     error: {
-                        message: 'Worker exited the process without emitting an event. Check other logs for more information.',
+                        message: `Worker exited without emitting event. ${this.workerFailedMessage
+                            ? `Error: ${this.workerFailedMessage}`
+                            : 'Check the logs for more information.'}`,
                     },
                 },
             });
-            this.resolve();
         }
         catch (error) {
             console.error('Error while emitting event.', (0, logger_1.serializeError)(error));
         }
+        finally {
+            this.resolve();
+        }
     }
 }
 exports.Spawn = Spawn;

package/dist/multithreading/worker.js

@@ -1,9 +1,20 @@
 "use strict";
 const { workerData } = require('node:worker_threads');
-require('ts-node').register();
+let resolvedWorkerPath;
+try {
+    resolvedWorkerPath = require.resolve(workerData.workerPath);
+}
+catch (_a) {
+    // Unresolvable without a TypeScript loader, so the worker is a .ts file.
+}
+if (!resolvedWorkerPath || resolvedWorkerPath.endsWith('.ts')) {
+    try {
+        require('ts-node').register();
+    }
+    catch (error) {
+        throw new Error(`Worker ${workerData.workerPath} needs ts-node to run TypeScript. Install ts-node or point the worker at compiled JavaScript. ${error.message}`);
+    }
+}
 const { Logger } = require('../logger/logger');
-const { runWithUserLogContext } = require('../logger/logger.context');
 console = new Logger({ event: workerData.event, options: workerData.options });
-runWithUserLogContext(() => {
-    require(workerData.workerPath);
-});
+require(workerData.workerPath);

package/dist/repo/repo.d.ts

@@ -1,7 +1,6 @@
 import { Item } from '../repo/repo.interfaces';
-import { ErrorRecord } from '../types/common';
 import { Artifact } from '../uploader/uploader.interfaces';
-import { NormalizedAttachment, NormalizedItem, RepoFactoryInterface } from './repo.interfaces';
+import { NormalizedAttachment, NormalizedItem } from './repo.interfaces';
 export declare class Repo {
     readonly itemType: string;
     private items;
@@ -10,9 +9,19 @@ export declare class Repo {
     private onUpload;
     private options?;
     uploadedArtifacts: Artifact[];
-    constructor({ event, itemType, normalize, onUpload, options, }: RepoFactoryInterface);
+    dateRanges: {
+        creationDate: {
+            oldest?: number;
+            newest?: number;
+        };
+        modifiedDate: {
+            oldest?: number;
+            newest?: number;
+        };
+    };
     getItems(): (NormalizedItem | NormalizedAttachment | Item)[];
-    upload(batch?: (NormalizedItem | NormalizedAttachment | Item)[]): Promise<void | ErrorRecord>;
-    push(items: Item[]): Promise<boolean>;
+    upload(batch?: (NormalizedItem | NormalizedAttachment | Item)[]): Promise<void>;
+    /** Buffers items and uploads every full batch. Throws if an upload fails. */
+    push(items: Item[]): Promise<void>;
 }
 //# sourceMappingURL=repo.d.ts.map

package/dist/repo/repo.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"repo.d.ts","sourceRoot":"","sources":["../../src/repo/repo.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,IAAI,EAAE,MAAM,yBAAyB,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAG3D,OAAO,EACL,oBAAoB,EACpB,cAAc,EACd,oBAAoB,EACrB,MAAM,mBAAmB,CAAC;AAE3B,qBAAa,IAAI;IACf,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,OAAO,CAAC,KAAK,CAAmD;IAChE,OAAO,CAAC,SAAS,CAAC,CAAwD;IAC1E,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,QAAQ,CAA+B;IAC/C,OAAO,CAAC,OAAO,CAAC,CAAuB;IAChC,iBAAiB,EAAE,QAAQ,EAAE,CAAC;gBAEzB,EACV,KAAK,EACL,QAAQ,EACR,SAAS,EACT,QAAQ,EACR,OAAO,GACR,EAAE,oBAAoB;IAUvB,QAAQ,IAAI,CAAC,cAAc,GAAG,oBAAoB,GAAG,IAAI,CAAC,EAAE;IAItD,MAAM,CACV,KAAK,CAAC,EAAE,CAAC,cAAc,GAAG,oBAAoB,GAAG,IAAI,CAAC,EAAE,GACvD,OAAO,CAAC,IAAI,GAAG,WAAW,CAAC;IAqCxB,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC;CAuC5C"}
+{"version":3,"file":"repo.d.ts","sourceRoot":"","sources":["../../src/repo/repo.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,IAAI,EAAE,MAAM,yBAAyB,CAAC;AAG/C,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAG3D,OAAO,EACL,oBAAoB,EACpB,cAAc,EAEf,MAAM,mBAAmB,CAAC;AAE3B,qBAAa,IAAI;IACf,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,OAAO,CAAC,KAAK,CAAmD;IAChE,OAAO,CAAC,SAAS,CAAC,CAAwD;IAC1E,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,QAAQ,CAA+B;IAC/C,OAAO,CAAC,OAAO,CAAC,CAAiB;IAC1B,iBAAiB,EAAE,QAAQ,EAAE,CAAC;IAC9B,UAAU,EAAE;QACjB,YAAY,EAAE;YAAE,MAAM,CAAC,EAAE,MAAM,CAAC;YAAC,MAAM,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QACnD,YAAY,EAAE;YAAE,MAAM,CAAC,EAAE,MAAM,CAAC;YAAC,MAAM,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;KACpD,CAGC;IAoBF,QAAQ,IAAI,CAAC,cAAc,GAAG,oBAAoB,GAAG,IAAI,CAAC,EAAE;IAItD,MAAM,CACV,KAAK,CAAC,EAAE,CAAC,cAAc,GAAG,oBAAoB,GAAG,IAAI,CAAC,EAAE,GACvD,OAAO,CAAC,IAAI,CAAC;IA2DhB,6EAA6E;IACvE,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;CA6BzC"}

package/dist/repo/repo.helpers.d.ts

@@ -0,0 +1,6 @@
+export declare function updateRange(range: {
+    oldest?: number;
+    newest?: number;
+}, ms: number): void;
+export declare function toValidTimestamp(value: string): number | undefined;
+//# sourceMappingURL=repo.helpers.d.ts.map

package/dist/repo/repo.helpers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"repo.helpers.d.ts","sourceRoot":"","sources":["../../src/repo/repo.helpers.ts"],"names":[],"mappings":"AAAA,wBAAgB,WAAW,CACzB,KAAK,EAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,EAC3C,EAAE,EAAE,MAAM,GACT,IAAI,CAON;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAGlE"}

package/dist/repo/repo.helpers.js

@@ -0,0 +1,16 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.updateRange = updateRange;
+exports.toValidTimestamp = toValidTimestamp;
+function updateRange(range, ms) {
+    if (range.oldest === undefined || ms < range.oldest) {
+        range.oldest = ms;
+    }
+    if (range.newest === undefined || ms > range.newest) {
+        range.newest = ms;
+    }
+}
+function toValidTimestamp(value) {
+    const ms = new Date(value).getTime();
+    return Number.isFinite(ms) ? ms : undefined;
+}

package/dist/repo/repo.interfaces.d.ts

@@ -1,35 +1,26 @@
+import { AirSyncEvent } from '../types/extraction';
+import { RepoOptions, RuntimeOptions } from '../types/workers';
 import { Artifact } from '../uploader/uploader.interfaces';
-import { AirdropEvent } from '../types/extraction';
-import { WorkerAdapterOptions } from '../types/workers';
-/**
- * RepoInterface is an interface that defines the structure of a repo which is used to store and upload extracted data.
- */
+/** Stores and uploads extracted data. */
 export interface RepoInterface {
     itemType: string;
     normalize?: (record: object) => NormalizedItem | NormalizedAttachment;
+    overriddenOptions?: RepoOptions;
 }
-/**
- * RepoFactoryInterface is an interface that defines the structure of a repo factory which is used to create a repo.
- */
 export interface RepoFactoryInterface {
-    event: AirdropEvent;
+    event: AirSyncEvent;
     itemType: string;
     normalize?: (record: object) => NormalizedItem | NormalizedAttachment;
     onUpload: (artifact: Artifact) => void;
-    options?: WorkerAdapterOptions;
+    options?: RuntimeOptions;
+    skipConfirmation?: boolean;
 }
-/**
- * NormalizedItem is an interface of item after normalization.
- */
 export interface NormalizedItem {
     id: string;
     created_date: string;
     modified_date: string;
     data: object;
 }
-/**
- * NormalizedAttachment is an interface of attachment after normalization.
- */
 export interface NormalizedAttachment {
     url: string;
     id: string;
@@ -37,10 +28,10 @@ export interface NormalizedAttachment {
     parent_id: string;
     author_id?: string;
     inline?: boolean;
-    grand_parent_id?: number | string;
+    content_type?: string;
+    created_date?: string;
+    modified_date?: string;
+    grand_parent_id?: string;
 }
-/**
- * Item is an interface that defines the structure of an item.
- */
 export type Item = Record<string, any>;
 //# sourceMappingURL=repo.interfaces.d.ts.map

package/dist/repo/repo.interfaces.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"repo.interfaces.d.ts","sourceRoot":"","sources":["../../src/repo/repo.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,cAAc,GAAG,oBAAoB,CAAC;CACvE;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,YAAY,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,cAAc,GAAG,oBAAoB,CAAC;IACtE,QAAQ,EAAE,CAAC,QAAQ,EAAE,QAAQ,KAAK,IAAI,CAAC;IACvC,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;IAIjB,eAAe,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;CACnC;AAED;;GAEG;AAEH,MAAM,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC"}
+{"version":3,"file":"repo.interfaces.d.ts","sourceRoot":"","sources":["../../src/repo/repo.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAC/D,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,yCAAyC;AACzC,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,cAAc,GAAG,oBAAoB,CAAC;IACtE,iBAAiB,CAAC,EAAE,WAAW,CAAC;CACjC;AAED,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,YAAY,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,cAAc,GAAG,oBAAoB,CAAC;IACtE,QAAQ,EAAE,CAAC,QAAQ,EAAE,QAAQ,KAAK,IAAI,CAAC;IACvC,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,oBAAoB;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAGD,MAAM,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC"}