@devrev/airsync-sdk 2.0.0 → 2.1.0-alpha.0

package/README.md

@@ -1,6 +1,6 @@
 # AirSync SDK
 
-[![Coverage Status](https://coveralls.io/repos/github/devrev/adaas-sdk/badge.svg?branch=main&t=s4Otlm)](https://coveralls.io/github/devrev/adaas-sdk?branch=main)
+[![Coverage Status](https://coveralls.io/repos/github/devrev/airsync-sdk/badge.svg?branch=main&t=s4Otlm)](https://coveralls.io/github/devrev/airsync-sdk?branch=main)
 
 TypeScript SDK for building AirSync snap-ins (connectors) that sync data between external systems and DevRev. It handles the AirSync control protocol for you: worker threads, timeouts, state, artifact uploads, attachment streaming and loading.
 

package/dist/common/constants.d.ts

@@ -4,6 +4,7 @@ export declare const STATELESS_LOADING_EVENT_TYPES: EventType[];
 export declare const STATELESS_EVENT_TYPES: EventType[];
 export declare const ARTIFACT_BATCH_SIZE = 2000;
 export declare const MAX_DEVREV_ARTIFACT_SIZE: number;
+export declare const S3_MULTIPART_CHUNK_SIZE: number;
 export declare const MAX_DEVREV_FILENAME_LENGTH = 256;
 export declare const MAX_DEVREV_FILENAME_EXTENSION_LENGTH = 20;
 export declare const MAX_EVENT_SIZE_BYTES = 200000;

package/dist/common/constants.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/common/constants.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAIhD,eAAO,MAAM,gCAAgC,aAI5C,CAAC;AAEF,eAAO,MAAM,6BAA6B,aAGzC,CAAC;AAEF,eAAO,MAAM,qBAAqB,aAGjC,CAAC;AAEF,eAAO,MAAM,mBAAmB,OAAO,CAAC;AACxC,eAAO,MAAM,wBAAwB,QAAyB,CAAC;AAC/D,eAAO,MAAM,0BAA0B,MAAM,CAAC;AAC9C,eAAO,MAAM,oCAAoC,KAAK,CAAC;AAGvD,eAAO,MAAM,oBAAoB,SAAU,CAAC;AAE5C,eAAO,MAAM,0BAA0B,QAEtC,CAAC;AAEF,eAAO,MAAM,eAAe,EAAE,MAA0B,CAAC;AACzD,eAAO,MAAM,kBAAkB,uBAAuB,CAAC;AAEvD,oBAAY,uBAAuB;IACjC,wBAAwB,6BAA6B;IACrD,WAAW,gBAAgB;IAC3B,mBAAmB,wBAAwB;CAC5C;AAED,eAAO,MAAM,eAAe,KAAsB,CAAC;AAEnD,eAAO,MAAM,sBAAsB,QAAiB,CAAC;AACrD,eAAO,MAAM,uBAAuB,MAAM,CAAC;AAC3C,eAAO,MAAM,mBAAmB,QAAY,CAAC;AAE7C;;;;GAIG;AACH,eAAO,MAAM,yBAAyB,6BAA6B,CAAC"}
+{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/common/constants.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAIhD,eAAO,MAAM,gCAAgC,aAI5C,CAAC;AAEF,eAAO,MAAM,6BAA6B,aAGzC,CAAC;AAEF,eAAO,MAAM,qBAAqB,aAGjC,CAAC;AAEF,eAAO,MAAM,mBAAmB,OAAO,CAAC;AACxC,eAAO,MAAM,wBAAwB,QAAyB,CAAC;AAG/D,eAAO,MAAM,uBAAuB,QAAmB,CAAC;AACxD,eAAO,MAAM,0BAA0B,MAAM,CAAC;AAC9C,eAAO,MAAM,oCAAoC,KAAK,CAAC;AAGvD,eAAO,MAAM,oBAAoB,SAAU,CAAC;AAE5C,eAAO,MAAM,0BAA0B,QAEtC,CAAC;AAEF,eAAO,MAAM,eAAe,EAAE,MAA0B,CAAC;AACzD,eAAO,MAAM,kBAAkB,uBAAuB,CAAC;AAEvD,oBAAY,uBAAuB;IACjC,wBAAwB,6BAA6B;IACrD,WAAW,gBAAgB;IAC3B,mBAAmB,wBAAwB;CAC5C;AAED,eAAO,MAAM,eAAe,KAAsB,CAAC;AAEnD,eAAO,MAAM,sBAAsB,QAAiB,CAAC;AACrD,eAAO,MAAM,uBAAuB,MAAM,CAAC;AAC3C,eAAO,MAAM,mBAAmB,QAAY,CAAC;AAE7C;;;;GAIG;AACH,eAAO,MAAM,yBAAyB,6BAA6B,CAAC"}

package/dist/common/constants.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.UNBOUNDED_DATE_TIME_VALUE = exports.MEMORY_LOG_INTERVAL = exports.HARD_TIMEOUT_MULTIPLIER = exports.DEFAULT_LAMBDA_TIMEOUT = exports.LIBRARY_VERSION = exports.AirSyncDefaultItemTypes = exports.UNKNOWN_EVENT_TYPE = exports.SSOR_ATTACHMENT = exports.EVENT_SIZE_THRESHOLD_BYTES = exports.MAX_EVENT_SIZE_BYTES = exports.MAX_DEVREV_FILENAME_EXTENSION_LENGTH = exports.MAX_DEVREV_FILENAME_LENGTH = exports.MAX_DEVREV_ARTIFACT_SIZE = exports.ARTIFACT_BATCH_SIZE = exports.STATELESS_EVENT_TYPES = exports.STATELESS_LOADING_EVENT_TYPES = exports.STATELESS_EXTRACTION_EVENT_TYPES = void 0;
+exports.UNBOUNDED_DATE_TIME_VALUE = exports.MEMORY_LOG_INTERVAL = exports.HARD_TIMEOUT_MULTIPLIER = exports.DEFAULT_LAMBDA_TIMEOUT = exports.LIBRARY_VERSION = exports.AirSyncDefaultItemTypes = exports.UNKNOWN_EVENT_TYPE = exports.SSOR_ATTACHMENT = exports.EVENT_SIZE_THRESHOLD_BYTES = exports.MAX_EVENT_SIZE_BYTES = exports.MAX_DEVREV_FILENAME_EXTENSION_LENGTH = exports.MAX_DEVREV_FILENAME_LENGTH = exports.S3_MULTIPART_CHUNK_SIZE = exports.MAX_DEVREV_ARTIFACT_SIZE = exports.ARTIFACT_BATCH_SIZE = exports.STATELESS_EVENT_TYPES = exports.STATELESS_LOADING_EVENT_TYPES = exports.STATELESS_EXTRACTION_EVENT_TYPES = void 0;
 const extraction_1 = require("../types/extraction");
 const helpers_1 = require("./helpers");
 exports.STATELESS_EXTRACTION_EVENT_TYPES = [
@@ -18,6 +18,9 @@ exports.STATELESS_EVENT_TYPES = [
 ];
 exports.ARTIFACT_BATCH_SIZE = 2000;
 exports.MAX_DEVREV_ARTIFACT_SIZE = 2 * 1024 * 1024 * 1024; // 2GB
+// S3 multipart uploads require every non-final part to be at least 5 MiB. A
+// 10 MiB part gives us a useful buffer while keeping the memory overhead low.
+exports.S3_MULTIPART_CHUNK_SIZE = 10 * 1024 * 1024;
 exports.MAX_DEVREV_FILENAME_LENGTH = 256;
 exports.MAX_DEVREV_FILENAME_EXTENSION_LENGTH = 20;
 // Max SQS message size is 250KB, we want to leave some room for the other data in the message

package/dist/index.d.ts

@@ -4,7 +4,7 @@ export type { LoadingAdapter } from './multithreading/adapters/loading-adapter';
 export { processExtractionTask, processLoadingTask, } from './multithreading/process-task';
 export { spawn } from './multithreading/spawn/spawn';
 export type { Repo } from './repo/repo';
-export type { ExtractionScope, ProcessTaskInterface, RepoOptions, SpawnFactoryInterface, SpawnOptions, TaskAdapterInterface, TaskResult, TaskStatus, WorkerPathOverrides, } from './types/workers';
+export type { DatasetQueryInfo, ExtractionQueries, ExtractionScope, ProcessTaskInterface, RepoOptions, SpawnFactoryInterface, SpawnOptions, TaskAdapterInterface, TaskResult, TaskStatus, WorkerPathOverrides, } from './types/workers';
 export { AirSyncDefaultItemTypes, UNBOUNDED_DATE_TIME_VALUE, } from './common/constants';
 export { AirSyncErrorCode } from './types/errors';
 export { installInitialDomainMapping } from './state/install-initial-domain-mapping';

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAQA,YAAY,EAAE,WAAW,EAAE,MAAM,wCAAwC,CAAC;AAC1E,YAAY,EAAE,iBAAiB,EAAE,MAAM,8CAA8C,CAAC;AACtF,YAAY,EAAE,cAAc,EAAE,MAAM,2CAA2C,CAAC;AAChF,OAAO,EACL,qBAAqB,EACrB,kBAAkB,GACnB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,KAAK,EAAE,MAAM,8BAA8B,CAAC;AACrD,YAAY,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AAGxC,YAAY,EACV,eAAe,EACf,oBAAoB,EACpB,WAAW,EACX,qBAAqB,EACrB,YAAY,EACZ,oBAAoB,EACpB,UAAU,EACV,UAAU,EACV,mBAAmB,GACpB,MAAM,iBAAiB,CAAC;AAGzB,OAAO,EACL,uBAAuB,EACvB,yBAAyB,GAC1B,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAGlD,OAAO,EAAE,2BAA2B,EAAE,MAAM,wCAAwC,CAAC;AAGrF,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAGjD,YAAY,EAAE,WAAW,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AACxE,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAG1C,YAAY,EACV,YAAY,EACZ,cAAc,EACd,cAAc,EACd,YAAY,EACZ,SAAS,EACT,iCAAiC,EACjC,gBAAgB,EAChB,wCAAwC,EACxC,kCAAkC,EAClC,uCAAuC,EACvC,yCAAyC,EACzC,uCAAuC,EACvC,yCAAyC,EACzC,cAAc,EACd,kBAAkB,EAClB,SAAS,EACT,aAAa,EACb,2BAA2B,EAC3B,SAAS,EACT,cAAc,GACf,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,SAAS,EACT,gBAAgB,EAChB,aAAa,EACb,QAAQ,EACR,aAAa,GACd,MAAM,oBAAoB,CAAC;AAG5B,YAAY,EACV,wBAAwB,EACxB,kBAAkB,EAClB,+BAA+B,EAC/B,iCAAiC,EACjC,6BAA6B,EAC7B,UAAU,EACV,qBAAqB,EACrB,cAAc,EACd,YAAY,GACb,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAG7C,YAAY,EACV,IAAI,EACJ,oBAAoB,EACpB,cAAc,EACd,aAAa,GACd,MAAM,wBAAwB,CAAC;AAGhC,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAC5C,YAAY,EACV,mBAAmB,EACnB,qBAAqB,EACrB,uBAAuB,EACvB,4BAA4B,EAC5B,8BAA8B,EAC9B,0BAA0B,EAC1B,4BAA4B,EAC5B,mBAAmB,EACnB,qBAAqB,EACrB,gBAAgB,EAChB,+BAA+B,EAC/B,4BAA4B,GAC7B,MAAM,8BAA8B,CAAC;AACtC,OAAO,EACL,sBAAsB,EACtB,0BAA0B,GAC3B,MAAM,8BAA8B,CAAC;AAGtC,YAAY,EACV,QAAQ,EACR,cAAc,EACd,cAAc,GACf,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAAE,iBAAiB,EAAE,MAAM,gCAAgC,CAAC;AAGnE,YAAY,EACV,WAAW,EACX,cAAc,EACd,wBAAwB,EACxB,cAAc,EACd,eAAe,EACf,WAAW,EACX,WAAW,EACX,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,sBAAsB,EACtB,KAAK,EACL,cAAc,EACd,wBAAwB,EACxB,oBAAoB,EACpB,eAAe,EACf,QAAQ,EACR,kBAAkB,EAClB,kBAAkB,EAClB,SAAS,EACT,SAAS,EACT,OAAO,EACP,iBAAiB,EACjB,cAAc,EACd,UAAU,EACV,kBAAkB,EAClB,qBAAqB,EACrB,aAAa,EACb,uBAAuB,EACvB,eAAe,EACf,aAAa,EACb,eAAe,EACf,aAAa,EACb,aAAa,EACb,YAAY,EACZ,QAAQ,EACR,QAAQ,EACR,UAAU,EACV,UAAU,EACV,aAAa,EACb,iBAAiB,EACjB,QAAQ,EACR,kBAAkB,GACnB,MAAM,kCAAkC,CAAC;AAG1C,YAAY,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,EAAE,uBAAuB,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAC5E,YAAY,EACV,WAAW,EACX,WAAW,EACX,WAAW,GACZ,MAAM,kCAAkC,CAAC;AAC1C,YAAY,EACV,iBAAiB,EACjB,wBAAwB,GACzB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACL,2BAA2B,EAC3B,wBAAwB,GACzB,MAAM,yBAAyB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAQA,YAAY,EAAE,WAAW,EAAE,MAAM,wCAAwC,CAAC;AAC1E,YAAY,EAAE,iBAAiB,EAAE,MAAM,8CAA8C,CAAC;AACtF,YAAY,EAAE,cAAc,EAAE,MAAM,2CAA2C,CAAC;AAChF,OAAO,EACL,qBAAqB,EACrB,kBAAkB,GACnB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,KAAK,EAAE,MAAM,8BAA8B,CAAC;AACrD,YAAY,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AAGxC,YAAY,EACV,gBAAgB,EAChB,iBAAiB,EACjB,eAAe,EACf,oBAAoB,EACpB,WAAW,EACX,qBAAqB,EACrB,YAAY,EACZ,oBAAoB,EACpB,UAAU,EACV,UAAU,EACV,mBAAmB,GACpB,MAAM,iBAAiB,CAAC;AAGzB,OAAO,EACL,uBAAuB,EACvB,yBAAyB,GAC1B,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAGlD,OAAO,EAAE,2BAA2B,EAAE,MAAM,wCAAwC,CAAC;AAGrF,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAGjD,YAAY,EAAE,WAAW,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AACxE,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAG1C,YAAY,EACV,YAAY,EACZ,cAAc,EACd,cAAc,EACd,YAAY,EACZ,SAAS,EACT,iCAAiC,EACjC,gBAAgB,EAChB,wCAAwC,EACxC,kCAAkC,EAClC,uCAAuC,EACvC,yCAAyC,EACzC,uCAAuC,EACvC,yCAAyC,EACzC,cAAc,EACd,kBAAkB,EAClB,SAAS,EACT,aAAa,EACb,2BAA2B,EAC3B,SAAS,EACT,cAAc,GACf,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,SAAS,EACT,gBAAgB,EAChB,aAAa,EACb,QAAQ,EACR,aAAa,GACd,MAAM,oBAAoB,CAAC;AAG5B,YAAY,EACV,wBAAwB,EACxB,kBAAkB,EAClB,+BAA+B,EAC/B,iCAAiC,EACjC,6BAA6B,EAC7B,UAAU,EACV,qBAAqB,EACrB,cAAc,EACd,YAAY,GACb,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAG7C,YAAY,EACV,IAAI,EACJ,oBAAoB,EACpB,cAAc,EACd,aAAa,GACd,MAAM,wBAAwB,CAAC;AAGhC,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAC5C,YAAY,EACV,mBAAmB,EACnB,qBAAqB,EACrB,uBAAuB,EACvB,4BAA4B,EAC5B,8BAA8B,EAC9B,0BAA0B,EAC1B,4BAA4B,EAC5B,mBAAmB,EACnB,qBAAqB,EACrB,gBAAgB,EAChB,+BAA+B,EAC/B,4BAA4B,GAC7B,MAAM,8BAA8B,CAAC;AACtC,OAAO,EACL,sBAAsB,EACtB,0BAA0B,GAC3B,MAAM,8BAA8B,CAAC;AAGtC,YAAY,EACV,QAAQ,EACR,cAAc,EACd,cAAc,GACf,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAAE,iBAAiB,EAAE,MAAM,gCAAgC,CAAC;AAGnE,YAAY,EACV,WAAW,EACX,cAAc,EACd,wBAAwB,EACxB,cAAc,EACd,eAAe,EACf,WAAW,EACX,WAAW,EACX,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,sBAAsB,EACtB,KAAK,EACL,cAAc,EACd,wBAAwB,EACxB,oBAAoB,EACpB,eAAe,EACf,QAAQ,EACR,kBAAkB,EAClB,kBAAkB,EAClB,SAAS,EACT,SAAS,EACT,OAAO,EACP,iBAAiB,EACjB,cAAc,EACd,UAAU,EACV,kBAAkB,EAClB,qBAAqB,EACrB,aAAa,EACb,uBAAuB,EACvB,eAAe,EACf,aAAa,EACb,eAAe,EACf,aAAa,EACb,aAAa,EACb,YAAY,EACZ,QAAQ,EACR,QAAQ,EACR,UAAU,EACV,UAAU,EACV,aAAa,EACb,iBAAiB,EACjB,QAAQ,EACR,kBAAkB,GACnB,MAAM,kCAAkC,CAAC;AAG1C,YAAY,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,EAAE,uBAAuB,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAC5E,YAAY,EACV,WAAW,EACX,WAAW,EACX,WAAW,GACZ,MAAM,kCAAkC,CAAC;AAC1C,YAAY,EACV,iBAAiB,EACjB,wBAAwB,GACzB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACL,2BAA2B,EAC3B,wBAAwB,GACzB,MAAM,yBAAyB,CAAC"}

package/dist/multithreading/adapters/base-adapter.d.ts

@@ -17,6 +17,7 @@ export declare abstract class BaseAdapter<ConnectorState> {
     /** Connector-owned state exposed to snap-in code. */
     get state(): ConnectorState;
     set state(value: ConnectorState);
+    get extractionQueries(): import("../../types/workers").ExtractionQueries;
     get extractionScope(): import("../../types/workers").ExtractionScope;
     /**
      * Persists connector and SDK state right now. State is saved on every emit

package/dist/multithreading/adapters/base-adapter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"base-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/base-adapter.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAKhD,OAAO,EACL,YAAY,EAIb,MAAM,wBAAwB,CAAC;AAYhC;;;GAGG;AACH,8BAAsB,WAAW,CAAC,cAAc;IAC9C,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IAI7B,OAAO,CAAC,iBAAiB,CAAkB;IAC3C,OAAO,CAAC,UAAU,CAAkB;IACpC,OAAO,CAAC,oBAAoB,CAAc;IAU1C,OAAO,CAAC,QAAQ,CAAU;IAsB1B,sFAAsF;IACtF,IAAI,OAAO,IAAI,OAAO,CAErB;IAOD,2EAA2E;IAC3E,IAAI,SAAS,IAAI,OAAO,CAEvB;IAUD,qDAAqD;IACrD,IAAI,KAAK,IAAI,cAAc,CAE1B;IAED,IAAI,KAAK,CAAC,KAAK,EAAE,cAAc,EAE9B;IAOD,IAAI,eAAe,kDAElB;IAED;;;;OAIG;IACG,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IAIhC;;;;OAIG;IACG,cAAc,CAClB,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,MAAM,EAAE,GACb,OAAO,CAAC;QAAE,EAAE,EAAE,MAAM,CAAA;KAAE,CAAC;IAY1B,kFAAkF;IAC5E,gBAAgB,CAAC,CAAC,GAAG,OAAO,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC;YAmIvD,IAAI;CAmCnB"}
+{"version":3,"file":"base-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/base-adapter.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAKhD,OAAO,EACL,YAAY,EAIb,MAAM,wBAAwB,CAAC;AAYhC;;;GAGG;AACH,8BAAsB,WAAW,CAAC,cAAc;IAC9C,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IAI7B,OAAO,CAAC,iBAAiB,CAAkB;IAC3C,OAAO,CAAC,UAAU,CAAkB;IACpC,OAAO,CAAC,oBAAoB,CAAc;IAU1C,OAAO,CAAC,QAAQ,CAAU;IAsB1B,sFAAsF;IACtF,IAAI,OAAO,IAAI,OAAO,CAErB;IAOD,2EAA2E;IAC3E,IAAI,SAAS,IAAI,OAAO,CAEvB;IAUD,qDAAqD;IACrD,IAAI,KAAK,IAAI,cAAc,CAE1B;IAED,IAAI,KAAK,CAAC,KAAK,EAAE,cAAc,EAE9B;IAOD,IAAI,iBAAiB,oDAEpB;IAED,IAAI,eAAe,kDAElB;IAED;;;;OAIG;IACG,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IAIhC;;;;OAIG;IACG,cAAc,CAClB,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,MAAM,EAAE,GACb,OAAO,CAAC;QAAE,EAAE,EAAE,MAAM,CAAA;KAAE,CAAC;IAY1B,kFAAkF;IAC5E,gBAAgB,CAAC,CAAC,GAAG,OAAO,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC;YAmIvD,IAAI;CAmCnB"}

package/dist/multithreading/adapters/base-adapter.js

@@ -63,6 +63,9 @@ class BaseAdapter {
     get sdkState() {
         return this.adapterState.sdkState;
     }
+    get extractionQueries() {
+        return this.adapterState.extractionQueries;
+    }
     get extractionScope() {
         return this.adapterState.extractionScope;
     }

package/dist/multithreading/adapters/extraction-adapter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"extraction-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/extraction-adapter.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AACvC,OAAO,EACL,oBAAoB,EACpB,aAAa,EACd,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EAIL,kCAAkC,EAClC,yCAAyC,EAEzC,aAAa,EACb,2BAA2B,EAE5B,MAAM,wBAAwB,CAAC;AAEhC,OAAO,EAAkB,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjE,OAAO,EAAE,QAAQ,EAAkB,MAAM,oCAAoC,CAAC;AAE9E,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,2FAA2F;AAC3F,qBAAa,iBAAiB,CAC5B,cAAc,CACd,SAAQ,WAAW,CAAC,cAAc,CAAC;IACnC,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,wBAAwB,CAAC,CAAkB;IACnD,OAAO,CAAC,KAAK,CAAc;IAC3B,OAAO,CAAC,qBAAqB,CAAC,CAAS;IACvC,OAAO,CAAC,sBAAsB,CAAa;IAY3C,6EAA6E;IAC7E,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAOxC,eAAe,CAAC,KAAK,EAAE,aAAa,EAAE;IAatC,OAAO,CAAC,UAAU;IAoDlB;;;OAGG;IACH,OAAO,CAAC,cAAc;IAItB,OAAO,CAAC,kBAAkB;IAQ1B,OAAO,CAAC,mBAAmB;IAS3B,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS;IAW3C,gEAAgE;IAChE,IAAI,SAAS,IAAI,SAAS,QAAQ,EAAE,CAEnC;IAED,OAAO,CAAC,YAAY;IAMpB;;;OAGG;IACH,IAAI,uBAAuB,IAAI,aAAa,EAAE,GAAG,SAAS,CAEzD;IAED,IAAI,uBAAuB,CAAC,MAAM,EAAE,aAAa,EAAE,GAAG,SAAS,EAE9D;IA6HD,OAAO,CAAC,kBAAkB;YAaZ,cAAc;IAmB5B;;;;OAIG;IACG,iBAAiB,CACrB,UAAU,EAAE,oBAAoB,EAChC,MAAM,EAAE,yCAAyC,GAChD,OAAO,CAAC,2BAA2B,CAAC;IAwGjC,iBAAiB,CAAC,QAAQ,EAAE,EAChC,MAAM,EACN,UAAU,EACV,SAAa,GACd,EAAE;QACD,MAAM,EAAE,yCAAyC,CAAC;QAClD,UAAU,CAAC,EAAE,kCAAkC,CAC7C,cAAc,EACd,oBAAoB,EAAE,EACtB,QAAQ,CACT,CAAC;QACF,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,GAAG,OAAO,CAAC,UAAU,CAAC;CAsHxB"}
+{"version":3,"file":"extraction-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/extraction-adapter.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AACvC,OAAO,EACL,oBAAoB,EACpB,aAAa,EACd,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EAIL,kCAAkC,EAClC,yCAAyC,EAEzC,aAAa,EACb,2BAA2B,EAE5B,MAAM,wBAAwB,CAAC;AAEhC,OAAO,EAAkB,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjE,OAAO,EAAE,QAAQ,EAAkB,MAAM,oCAAoC,CAAC;AAE9E,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,2FAA2F;AAC3F,qBAAa,iBAAiB,CAC5B,cAAc,CACd,SAAQ,WAAW,CAAC,cAAc,CAAC;IACnC,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,wBAAwB,CAAC,CAAkB;IACnD,OAAO,CAAC,KAAK,CAAc;IAC3B,OAAO,CAAC,qBAAqB,CAAC,CAAS;IACvC,OAAO,CAAC,sBAAsB,CAAa;IAY3C,6EAA6E;IAC7E,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAOxC,eAAe,CAAC,KAAK,EAAE,aAAa,EAAE;IAatC,OAAO,CAAC,UAAU;IAoDlB;;;OAGG;IACH,OAAO,CAAC,cAAc;IAItB,OAAO,CAAC,kBAAkB;IAQ1B,OAAO,CAAC,mBAAmB;IAS3B,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS;IAW3C,gEAAgE;IAChE,IAAI,SAAS,IAAI,SAAS,QAAQ,EAAE,CAEnC;IAED,OAAO,CAAC,YAAY;IAMpB;;;OAGG;IACH,IAAI,uBAAuB,IAAI,aAAa,EAAE,GAAG,SAAS,CAEzD;IAED,IAAI,uBAAuB,CAAC,MAAM,EAAE,aAAa,EAAE,GAAG,SAAS,EAE9D;IA6HD,OAAO,CAAC,kBAAkB;YAaZ,cAAc;IAmB5B;;;;OAIG;IACG,iBAAiB,CACrB,UAAU,EAAE,oBAAoB,EAChC,MAAM,EAAE,yCAAyC,GAChD,OAAO,CAAC,2BAA2B,CAAC;IA4IjC,iBAAiB,CAAC,QAAQ,EAAE,EAChC,MAAM,EACN,UAAU,EACV,SAAa,GACd,EAAE;QACD,MAAM,EAAE,yCAAyC,CAAC;QAClD,UAAU,CAAC,EAAE,kCAAkC,CAC7C,cAAc,EACd,oBAAoB,EAAE,EACtB,QAAQ,CACT,CAAC;QACF,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,GAAG,OAAO,CAAC,UAAU,CAAC;CAsHxB"}

package/dist/multithreading/adapters/extraction-adapter.js

@@ -230,44 +230,70 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
             ((_a = httpStream.headers['content-type']) === null || _a === void 0 ? void 0 : _a.toString()) ||
             'application/octet-stream';
         const contentLength = (_b = httpStream.headers['content-length']) === null || _b === void 0 ? void 0 : _b.toString();
-        const fileSize = contentLength ? parseInt(contentLength) : undefined;
-        const { error: artifactUrlError, response: artifactUrlResponse } = await this.uploader.getArtifactUploadUrl(attachment.file_name, fileType, fileSize);
-        if (artifactUrlError) {
-            (0, uploader_helpers_1.destroyHttpStream)(httpStream);
-            return {
-                status: 'error',
-                error: {
-                    message: `Error while preparing artifact for attachment ID ${attachment.id}. Skipping attachment. ${(0, logger_1.serializeError)(artifactUrlError)}`,
-                    fileSize,
-                },
-            };
-        }
-        const { error: uploadedArtifactError } = await this.uploader.streamArtifact(artifactUrlResponse, httpStream);
-        if (uploadedArtifactError) {
-            (0, uploader_helpers_1.destroyHttpStream)(httpStream);
-            return {
-                status: 'error',
-                error: {
-                    message: `Error while streaming to artifact for attachment ID ${attachment.id}. Skipping attachment. ` +
-                        (0, logger_1.serializeError)(uploadedArtifactError),
-                    fileSize,
-                },
-            };
+        const parsedFileSize = contentLength
+            ? Number.parseInt(contentLength, 10)
+            : undefined;
+        const fileSize = parsedFileSize !== undefined && Number.isFinite(parsedFileSize)
+            ? parsedFileSize
+            : undefined;
+        let artifactId;
+        if (fileSize === undefined) {
+            // Unknown-length streams cannot be uploaded using a presigned POST with
+            // a guessed Content-Length. Stream them through S3 multipart upload.
+            const { response: multipartUpload, error: multipartUploadError } = await this.uploader.streamArtifactMultipart(attachment.file_name, fileType, httpStream);
+            if (multipartUploadError || !multipartUpload) {
+                return {
+                    status: 'error',
+                    error: {
+                        message: `Error while uploading attachment for attachment ID ${attachment.id}. Skipping attachment. ` +
+                            (0, logger_1.serializeError)(multipartUploadError !== null && multipartUploadError !== void 0 ? multipartUploadError : new Error('Multipart upload returned no response.')),
+                        fileSize,
+                    },
+                };
+            }
+            artifactId = multipartUpload.artifact_id;
         }
-        const { error: confirmArtifactUploadError } = await this.uploader.confirmArtifactUpload(artifactUrlResponse.artifact_id);
-        if (confirmArtifactUploadError) {
-            return {
-                status: 'error',
-                error: {
-                    message: `Error while confirming upload for attachment ID ${attachment.id}. ` +
-                        (0, logger_1.serializeError)(confirmArtifactUploadError),
-                    fileSize,
-                },
-            };
+        else {
+            // Known-size attachments retain the existing single-request upload flow.
+            const { error: artifactUrlError, response: artifactUrlResponse } = await this.uploader.getArtifactUploadUrl(attachment.file_name, fileType, fileSize);
+            if (artifactUrlError || !artifactUrlResponse) {
+                (0, uploader_helpers_1.destroyHttpStream)(httpStream);
+                return {
+                    status: 'error',
+                    error: {
+                        message: `Error while preparing artifact for attachment ID ${attachment.id}. Skipping attachment. ${(0, logger_1.serializeError)(artifactUrlError !== null && artifactUrlError !== void 0 ? artifactUrlError : new Error('No artifact upload response.'))}`,
+                        fileSize,
+                    },
+                };
+            }
+            const { error: uploadedArtifactError } = await this.uploader.streamArtifact(artifactUrlResponse, httpStream);
+            if (uploadedArtifactError) {
+                (0, uploader_helpers_1.destroyHttpStream)(httpStream);
+                return {
+                    status: 'error',
+                    error: {
+                        message: `Error while streaming to artifact for attachment ID ${attachment.id}. Skipping attachment. ` +
+                            (0, logger_1.serializeError)(uploadedArtifactError),
+                        fileSize,
+                    },
+                };
+            }
+            const { error: confirmArtifactUploadError } = await this.uploader.confirmArtifactUpload(artifactUrlResponse.artifact_id);
+            if (confirmArtifactUploadError) {
+                return {
+                    status: 'error',
+                    error: {
+                        message: `Error while confirming upload for attachment ID ${attachment.id}. ` +
+                            (0, logger_1.serializeError)(confirmArtifactUploadError),
+                        fileSize,
+                    },
+                };
+            }
+            artifactId = artifactUrlResponse.artifact_id;
         }
         const ssorAttachment = {
             id: {
-                devrev: artifactUrlResponse.artifact_id,
+                devrev: artifactId,
                 external: attachment.id,
             },
             parent_id: {

package/dist/state/base-state.d.ts

@@ -1,6 +1,6 @@
 import { InitialDomainMapping } from '../types/common';
 import { AirSyncEvent } from '../types/extraction';
-import { ExtractionScope } from '../types/workers';
+import { ExtractionQueries, ExtractionScope } from '../types/workers';
 import { SdkState, StateInterface } from './state.interfaces';
 /**
  * State lifecycle shared by all sync modes: connector/SDK state separation,
@@ -11,6 +11,7 @@ export declare abstract class BaseState<ConnectorState> {
     protected _connectorState: ConnectorState;
     protected _sdkState: SdkState;
     protected _extractionScope: ExtractionScope;
+    protected _extractionQueries: ExtractionQueries;
     protected readonly initialSdkState: SdkState;
     protected readonly event: AirSyncEvent;
     private workerUrl;
@@ -24,6 +25,7 @@ export declare abstract class BaseState<ConnectorState> {
     /** SDK-internal bookkeeping state. Never exposed to connector code. */
     get sdkState(): SdkState;
     set sdkState(value: SdkState);
+    get extractionQueries(): ExtractionQueries;
     get extractionScope(): ExtractionScope;
     /**
      * Installs the initial domain mapping when the snap-in version in state
@@ -53,6 +55,7 @@ export declare abstract class BaseState<ConnectorState> {
     fetchState(): Promise<{
         state: string;
         objects?: string;
+        queries?: string;
     }>;
 }
 //# sourceMappingURL=base-state.d.ts.map

package/dist/state/base-state.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"base-state.d.ts","sourceRoot":"","sources":["../../src/state/base-state.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAEnD,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAGnD,OAAO,EAEL,QAAQ,EAER,cAAc,EAEf,MAAM,oBAAoB,CAAC;AAE5B;;;;GAIG;AACH,8BAAsB,SAAS,CAAC,cAAc;IAC5C,SAAS,CAAC,eAAe,EAAE,cAAc,CAAC;IAC1C,SAAS,CAAC,SAAS,EAAE,QAAQ,CAAC;IAC9B,SAAS,CAAC,gBAAgB,EAAE,eAAe,CAAM;IACjD,SAAS,CAAC,QAAQ,CAAC,eAAe,EAAE,QAAQ,CAAC;IAC7C,SAAS,CAAC,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IACvC,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,SAAS,CAAS;gBAGxB,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,cAAc,CAAC,cAAc,CAAC,EACvD,eAAe,EAAE,QAAQ;IAY3B,2EAA2E;IAC3E,IAAI,KAAK,IAAI,cAAc,CAE1B;IAED,IAAI,KAAK,CAAC,KAAK,EAAE,cAAc,EAE9B;IAED,uEAAuE;IACvE,IAAI,QAAQ,IAAI,QAAQ,CAEvB;IAED,IAAI,QAAQ,CAAC,KAAK,EAAE,QAAQ,EAE3B;IAED,IAAI,eAAe,IAAI,eAAe,CAErC;IAED;;;;OAIG;IACG,mCAAmC,CACvC,oBAAoB,CAAC,EAAE,oBAAoB,GAC1C,OAAO,CAAC,IAAI,CAAC;IAwChB;;;;OAIG;IACG,IAAI,CAAC,YAAY,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IAuDvD;;;;;OAKG;IACH,OAAO,CAAC,qBAAqB;IAyC7B,OAAO,CAAC,YAAY;IAepB;;;OAGG;IACG,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IA0C1B,UAAU,IAAI,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAqBjE"}
+{"version":3,"file":"base-state.d.ts","sourceRoot":"","sources":["../../src/state/base-state.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAEnD,OAAO,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAGtE,OAAO,EAEL,QAAQ,EAER,cAAc,EAEf,MAAM,oBAAoB,CAAC;AAE5B;;;;GAIG;AACH,8BAAsB,SAAS,CAAC,cAAc;IAC5C,SAAS,CAAC,eAAe,EAAE,cAAc,CAAC;IAC1C,SAAS,CAAC,SAAS,EAAE,QAAQ,CAAC;IAC9B,SAAS,CAAC,gBAAgB,EAAE,eAAe,CAAM;IACjD,SAAS,CAAC,kBAAkB,EAAE,iBAAiB,CAAM;IACrD,SAAS,CAAC,QAAQ,CAAC,eAAe,EAAE,QAAQ,CAAC;IAC7C,SAAS,CAAC,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IACvC,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,SAAS,CAAS;gBAGxB,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,cAAc,CAAC,cAAc,CAAC,EACvD,eAAe,EAAE,QAAQ;IAY3B,2EAA2E;IAC3E,IAAI,KAAK,IAAI,cAAc,CAE1B;IAED,IAAI,KAAK,CAAC,KAAK,EAAE,cAAc,EAE9B;IAED,uEAAuE;IACvE,IAAI,QAAQ,IAAI,QAAQ,CAEvB;IAED,IAAI,QAAQ,CAAC,KAAK,EAAE,QAAQ,EAE3B;IAED,IAAI,iBAAiB,IAAI,iBAAiB,CAEzC;IAED,IAAI,eAAe,IAAI,eAAe,CAErC;IAED;;;;OAIG;IACG,mCAAmC,CACvC,oBAAoB,CAAC,EAAE,oBAAoB,GAC1C,OAAO,CAAC,IAAI,CAAC;IAwChB;;;;OAIG;IACG,IAAI,CAAC,YAAY,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IAmFvD;;;;;OAKG;IACH,OAAO,CAAC,qBAAqB;IAyC7B,OAAO,CAAC,YAAY;IAepB;;;OAGG;IACG,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IA0C1B,UAAU,IAAI,OAAO,CAAC;QAC1B,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,CAAC;CAsBH"}

package/dist/state/base-state.js

@@ -19,6 +19,7 @@ const state_interfaces_1 = require("./state.interfaces");
 class BaseState {
     constructor({ event, initialState }, initialSdkState) {
         this._extractionScope = {};
+        this._extractionQueries = {};
         this.event = event;
         this.initialSdkState = initialSdkState;
         this._connectorState = initialState;
@@ -42,6 +43,9 @@ class BaseState {
     set sdkState(value) {
         this._sdkState = value;
     }
+    get extractionQueries() {
+        return this._extractionQueries;
+    }
     get extractionScope() {
         return this._extractionScope;
     }
@@ -87,7 +91,7 @@ class BaseState {
     async init(initialState) {
         var _a;
         try {
-            const { state: stringifiedState, objects } = await this.fetchState();
+            const { state: stringifiedState, objects, queries, } = await this.fetchState();
             if (!stringifiedState) {
                 throw new Error('No state found in response.');
             }
@@ -105,6 +109,25 @@ class BaseState {
                 connectorState: (0, logger_1.getPrintableState)(this.state),
                 sdkState: (0, logger_1.getPrintableState)(this.sdkState),
             });
+            this._extractionQueries = {};
+            if (queries) {
+                try {
+                    const parsedQueries = JSON.parse(queries);
+                    if (parsedQueries === null ||
+                        typeof parsedQueries !== 'object' ||
+                        Array.isArray(parsedQueries) ||
+                        Object.values(parsedQueries).some((query) => query === null ||
+                            typeof query !== 'object' ||
+                            Array.isArray(query) ||
+                            typeof query.sql_query !== 'string')) {
+                        throw new Error('Invalid extraction queries.');
+                    }
+                    this._extractionQueries = parsedQueries;
+                }
+                catch (_b) {
+                    console.warn('Failed to parse extractionQueries.');
+                }
+            }
             if (objects) {
                 try {
                     this._extractionScope = JSON.parse(objects);
@@ -218,7 +241,7 @@ class BaseState {
         });
     }
     async fetchState() {
-        var _a, _b;
+        var _a, _b, _c;
         console.log(`Fetching state with sync unit id ${this.syncUnitId} and request id ${this.requestId}.`);
         const url = this.workerUrl + '.get';
         const response = await client_1.axiosClient.get(url, {
@@ -233,6 +256,7 @@ class BaseState {
         return {
             state: (_a = response.data) === null || _a === void 0 ? void 0 : _a.state,
             objects: (_b = response.data) === null || _b === void 0 ? void 0 : _b.objects,
+            queries: (_c = response.data) === null || _c === void 0 ? void 0 : _c.queries,
         };
     }
 }

package/dist/types/workers.d.ts

@@ -7,6 +7,10 @@ import { LoaderEventType } from './loading';
 export type ExtractionScope = Record<string, {
     extract: boolean;
 }>;
+export interface DatasetQueryInfo {
+    sql_query: string;
+}
+export type ExtractionQueries = Record<string, DatasetQueryInfo>;
 /** Per-repo overrides for `RepoInterface.overriddenOptions`. */
 export interface RepoOptions {
     /** Maximum number of extracted items in a batch. */

package/dist/types/workers.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"workers.d.ts","sourceRoot":"","sources":["../../src/types/workers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AAExC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,6BAA6B,CAAC;AAE5D,OAAO,EAAE,WAAW,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AAC7D,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAC3E,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAE5C,0EAA0E;AAC1E,MAAM,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,EAAE;IAAE,OAAO,EAAE,OAAO,CAAA;CAAE,CAAC,CAAC;AAEnE,gEAAgE;AAChE,MAAM,WAAW,WAAW;IAC1B,oDAAoD;IACpD,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,YAAa,SAAQ,WAAW;IAC/C,gDAAgD;IAChD,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;GAGG;AACH,MAAM,WAAW,cAAe,SAAQ,YAAY;IAClD,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAED,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,YAAY,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,OAAO,EAAE,CAAC,KAAK,EAAE,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC;IACnD,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B;AAED,MAAM,WAAW,qBAAqB,CAAC,cAAc;IACnD,KAAK,EAAE,YAAY,CAAC;IACpB,YAAY,EAAE,cAAc,CAAC;IAC7B,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,uEAAuE;IACvE,cAAc,EAAE,MAAM,CAAC;IACvB,+EAA+E;IAC/E,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;IAC1C,OAAO,CAAC,EAAE,YAAY,CAAC;CACxB;AAED;;;;;;;;;;;GAWG;AACH,MAAM,MAAM,UAAU,GAClB;IAAE,MAAM,EAAE,SAAS,CAAA;CAAE,GACrB;IAAE,MAAM,EAAE,UAAU,CAAA;CAAE,GACtB;IAAE,MAAM,EAAE,OAAO,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,GACzC;IAAE,MAAM,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,WAAW,CAAA;CAAE,CAAC;AAE5C,MAAM,MAAM,UAAU,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;AAE9C,yEAAyE;AACzE,MAAM,WAAW,oBAAoB,CAAC,OAAO;IAC3C,OAAO,EAAE,OAAO,CAAC;CAClB;AAED;;;GAGG;AACH,MAAM,WAAW,oBAAoB,CAAC,OAAO;IAC3C,IAAI,EAAE,CAAC,MAAM,EAAE,oBAAoB,CAAC,OAAO,CAAC,KAAK,OAAO,CAAC,UAAU,CAAC,CAAC;IACrE,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,oBAAoB,CAAC,OAAO,CAAC,KAAK,OAAO,CAAC,UAAU,CAAC,CAAC;CAC5E;AAED,oBAAY,WAAW;IACrB,aAAa,YAAY;IACzB,YAAY,WAAW;IACvB,WAAW,UAAU;IACrB,UAAU,SAAS;CACpB;AAED,oBAAY,oBAAoB;IAC9B,oBAAoB,SAAS;IAC7B,iBAAiB,SAAS;IAC1B,gBAAgB,QAAQ;IACxB,mBAAmB,WAAW;CAC/B;AAED,MAAM,WAAW,oBAAoB;IACnC,OAAO,EAAE,oBAAoB,CAAC,oBAAoB,CAAC;IACnD,OAAO,EAAE;QACP,SAAS,EAAE,kBAAkB,GAAG,eAAe,CAAC;KACjD,CAAC;CACH;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,oBAAoB,CAAC,iBAAiB,CAAC;CACjD;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,oBAAoB,CAAC,gBAAgB,CAAC;IAC/C,OAAO,EAAE;QACP,eAAe,EAAE,MAAM,CAAC;QACxB,KAAK,EAAE,QAAQ,CAAC;KACjB,CAAC;CACH;AAED,wGAAwG;AACxG,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,oBAAoB,CAAC,mBAAmB,CAAC;IAClD,OAAO,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;CAC9B;AAED,MAAM,MAAM,aAAa,GACrB,oBAAoB,GACpB,iBAAiB,GACjB,gBAAgB,GAChB,mBAAmB,CAAC;AAExB,MAAM,WAAW,UAAU,CAAC,cAAc;IACxC,KAAK,EAAE,YAAY,CAAC;IACpB,YAAY,EAAE,cAAc,CAAC;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,OAAO,CAAC,EAAE,cAAc,CAAC;CAC1B;AAED,MAAM,WAAW,sBAAsB;IACrC,KAAK,EAAE,YAAY,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,wEAAwE;AACxE,MAAM,MAAM,mBAAmB,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC"}
+{"version":3,"file":"workers.d.ts","sourceRoot":"","sources":["../../src/types/workers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AAExC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,6BAA6B,CAAC;AAE5D,OAAO,EAAE,WAAW,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AAC7D,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAC3E,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAE5C,0EAA0E;AAC1E,MAAM,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,EAAE;IAAE,OAAO,EAAE,OAAO,CAAA;CAAE,CAAC,CAAC;AAEnE,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,MAAM,iBAAiB,GAAG,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;AAEjE,gEAAgE;AAChE,MAAM,WAAW,WAAW;IAC1B,oDAAoD;IACpD,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,YAAa,SAAQ,WAAW;IAC/C,gDAAgD;IAChD,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;GAGG;AACH,MAAM,WAAW,cAAe,SAAQ,YAAY;IAClD,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAED,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,YAAY,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,OAAO,EAAE,CAAC,KAAK,EAAE,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC;IACnD,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B;AAED,MAAM,WAAW,qBAAqB,CAAC,cAAc;IACnD,KAAK,EAAE,YAAY,CAAC;IACpB,YAAY,EAAE,cAAc,CAAC;IAC7B,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,uEAAuE;IACvE,cAAc,EAAE,MAAM,CAAC;IACvB,+EAA+E;IAC/E,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;IAC1C,OAAO,CAAC,EAAE,YAAY,CAAC;CACxB;AAED;;;;;;;;;;;GAWG;AACH,MAAM,MAAM,UAAU,GAClB;IAAE,MAAM,EAAE,SAAS,CAAA;CAAE,GACrB;IAAE,MAAM,EAAE,UAAU,CAAA;CAAE,GACtB;IAAE,MAAM,EAAE,OAAO,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,GACzC;IAAE,MAAM,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,WAAW,CAAA;CAAE,CAAC;AAE5C,MAAM,MAAM,UAAU,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;AAE9C,yEAAyE;AACzE,MAAM,WAAW,oBAAoB,CAAC,OAAO;IAC3C,OAAO,EAAE,OAAO,CAAC;CAClB;AAED;;;GAGG;AACH,MAAM,WAAW,oBAAoB,CAAC,OAAO;IAC3C,IAAI,EAAE,CAAC,MAAM,EAAE,oBAAoB,CAAC,OAAO,CAAC,KAAK,OAAO,CAAC,UAAU,CAAC,CAAC;IACrE,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,oBAAoB,CAAC,OAAO,CAAC,KAAK,OAAO,CAAC,UAAU,CAAC,CAAC;CAC5E;AAED,oBAAY,WAAW;IACrB,aAAa,YAAY;IACzB,YAAY,WAAW;IACvB,WAAW,UAAU;IACrB,UAAU,SAAS;CACpB;AAED,oBAAY,oBAAoB;IAC9B,oBAAoB,SAAS;IAC7B,iBAAiB,SAAS;IAC1B,gBAAgB,QAAQ;IACxB,mBAAmB,WAAW;CAC/B;AAED,MAAM,WAAW,oBAAoB;IACnC,OAAO,EAAE,oBAAoB,CAAC,oBAAoB,CAAC;IACnD,OAAO,EAAE;QACP,SAAS,EAAE,kBAAkB,GAAG,eAAe,CAAC;KACjD,CAAC;CACH;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,oBAAoB,CAAC,iBAAiB,CAAC;CACjD;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,oBAAoB,CAAC,gBAAgB,CAAC;IAC/C,OAAO,EAAE;QACP,eAAe,EAAE,MAAM,CAAC;QACxB,KAAK,EAAE,QAAQ,CAAC;KACjB,CAAC;CACH;AAED,wGAAwG;AACxG,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,oBAAoB,CAAC,mBAAmB,CAAC;IAClD,OAAO,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;CAC9B;AAED,MAAM,MAAM,aAAa,GACrB,oBAAoB,GACpB,iBAAiB,GACjB,gBAAgB,GAChB,mBAAmB,CAAC;AAExB,MAAM,WAAW,UAAU,CAAC,cAAc;IACxC,KAAK,EAAE,YAAY,CAAC;IACpB,YAAY,EAAE,cAAc,CAAC;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,OAAO,CAAC,EAAE,cAAc,CAAC;CAC1B;AAED,MAAM,WAAW,sBAAsB;IACrC,KAAK,EAAE,YAAY,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,wEAAwE;AACxE,MAAM,MAAM,mBAAmB,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC"}

package/dist/uploader/uploader.d.ts

@@ -1,7 +1,7 @@
 import { AxiosResponse } from 'axios';
 import { NormalizedAttachment } from '../repo/repo.interfaces';
 import { HttpStreamResponse } from '../types/extraction';
-import { ArtifactToUpload, UploaderFactoryInterface, UploaderResult, UploadResponse } from './uploader.interfaces';
+import { ArtifactToUpload, MultipartUpload, UploaderFactoryInterface, UploaderResult, UploadResponse } from './uploader.interfaces';
 export declare class Uploader {
     private isLocalDevelopment?;
     private devrevApiEndpoint;
@@ -17,6 +17,26 @@ export declare class Uploader {
     uploadArtifact(artifact: ArtifactToUpload, file: Buffer): Promise<UploaderResult<AxiosResponse>>;
     /** Streams a file from an HTTP response directly to the artifact's upload URL. */
     streamArtifact(artifact: ArtifactToUpload, fileStream: HttpStreamResponse): Promise<UploaderResult<AxiosResponse>>;
+    /**
+     * Creates an S3 multipart upload for an artifact whose size is not known.
+     * The returned URL is for part 1; subsequent URLs are obtained with
+     * getNextMultipartUploadPart.
+     */
+    createMultipartUpload(filename: string, fileType: string, partSize?: number): Promise<UploaderResult<MultipartUpload>>;
+    /** Gets a presigned URL for the next multipart upload part. */
+    getNextMultipartUploadPart(artifactId: string, cursor: string, partSize?: number): Promise<UploaderResult<MultipartUpload>>;
+    /** Finalizes and publishes an S3 multipart artifact upload. */
+    completeMultipartUpload(artifactId: string, cursor: string): Promise<UploaderResult<AxiosResponse>>;
+    /**
+     * Streams an attachment of unknown size through an S3 multipart upload.
+     * At most one 10 MiB part is buffered at a time. The final part may be
+     * smaller than S3's 5 MiB minimum.
+     */
+    streamArtifactMultipart(filename: string, fileType: string, fileStream: HttpStreamResponse): Promise<UploaderResult<{
+        artifact_id: string;
+    }>>;
+    /** Uploads one raw part to its presigned S3 URL. */
+    private uploadMultipartPart;
     confirmArtifactUpload(artifactId: string): Promise<{
         response?: AxiosResponse;
         error?: unknown;

package/dist/uploader/uploader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAOtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAWzD,OAAO,EAEL,gBAAgB,EAChB,wBAAwB,EACxB,cAAc,EACd,cAAc,EACf,MAAM,uBAAuB,CAAC;AAE/B,qBAAa,QAAQ;IACnB,OAAO,CAAC,kBAAkB,CAAC,CAAU;IACrC,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,cAAc,CAAyB;IAC/C,OAAO,CAAC,gBAAgB,CAAU;gBAEtB,EAAE,KAAK,EAAE,OAAO,EAAE,gBAAgB,EAAE,EAAE,wBAAwB;IAW1E,mFAAmF;IAC7E,MAAM,CACV,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,EAAE,GAAG,MAAM,GAChC,OAAO,CAAC,cAAc,CAAC;IAwEpB,oBAAoB,CACxB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;IA2B5C,mFAAmF;IAC7E,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAmBzC,kFAAkF;IAC5E,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,UAAU,EAAE,kBAAkB,GAC7B,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAqCnC,qBAAqB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC;QACvD,QAAQ,CAAC,EAAE,aAAa,CAAC;QACzB,KAAK,CAAC,EAAE,OAAO,CAAC;KACjB,CAAC;IAgCF,oEAAoE;IAC9D,4BAA4B,CAAC,EACjC,QAAQ,GACT,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;KAClB,GAAG,OAAO,CAAC;QACV,WAAW,CAAC,EAAE,oBAAoB,EAAE,CAAC;QACrC,KAAK,CAAC,EAAE;YAAE,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC;KAC7B,CAAC;YAuDY,sBAAsB;YAsBtB,gBAAgB;IAc9B,yEAAyE;IACnE,yBAAyB,CAAC,EAC9B,UAAU,EACV,SAAiB,GAClB,EAAE;QACD,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,CAAC,EAAE,OAAO,CAAC;KACrB,GAAG,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC;CA6B/C"}
+{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAUtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAWzD,OAAO,EAEL,gBAAgB,EAChB,eAAe,EACf,wBAAwB,EACxB,cAAc,EACd,cAAc,EACf,MAAM,uBAAuB,CAAC;AAE/B,qBAAa,QAAQ;IACnB,OAAO,CAAC,kBAAkB,CAAC,CAAU;IACrC,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,cAAc,CAAyB;IAC/C,OAAO,CAAC,gBAAgB,CAAU;gBAEtB,EAAE,KAAK,EAAE,OAAO,EAAE,gBAAgB,EAAE,EAAE,wBAAwB;IAW1E,mFAAmF;IAC7E,MAAM,CACV,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,EAAE,GAAG,MAAM,GAChC,OAAO,CAAC,cAAc,CAAC;IAwEpB,oBAAoB,CACxB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;IA2B5C,mFAAmF;IAC7E,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAmBzC,kFAAkF;IAC5E,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,UAAU,EAAE,kBAAkB,GAC7B,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAqCzC;;;;OAIG;IACG,qBAAqB,CACzB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,QAAQ,SAA0B,GACjC,OAAO,CAAC,cAAc,CAAC,eAAe,CAAC,CAAC;IAkB3C,+DAA+D;IACzD,0BAA0B,CAC9B,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,QAAQ,SAA0B,GACjC,OAAO,CAAC,cAAc,CAAC,eAAe,CAAC,CAAC;IAkB3C,+DAA+D;IACzD,uBAAuB,CAC3B,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAoBzC;;;;OAIG;IACG,uBAAuB,CAC3B,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,kBAAkB,GAC7B,OAAO,CAAC,cAAc,CAAC;QAAE,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IA6HnD,oDAAoD;YACtC,mBAAmB;IAsD3B,qBAAqB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC;QACvD,QAAQ,CAAC,EAAE,aAAa,CAAC;QACzB,KAAK,CAAC,EAAE,OAAO,CAAC;KACjB,CAAC;IAgCF,oEAAoE;IAC9D,4BAA4B,CAAC,EACjC,QAAQ,GACT,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;KAClB,GAAG,OAAO,CAAC;QACV,WAAW,CAAC,EAAE,oBAAoB,EAAE,CAAC;QACrC,KAAK,CAAC,EAAE;YAAE,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC;KAC7B,CAAC;YAuDY,sBAAsB;YAsBtB,gBAAgB;IAc9B,yEAAyE;IACnE,yBAAyB,CAAC,EAC9B,UAAU,EACV,SAAiB,GAClB,EAAE;QACD,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,CAAC,EAAE,OAAO,CAAC;KACrB,GAAG,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC;CA6B/C"}

package/dist/uploader/uploader.interfaces.d.ts

@@ -40,6 +40,12 @@ export interface ArtifactToUpload {
         value: string;
     }[];
 }
+/** Multipart upload state returned by the artifacts upload endpoints. */
+export interface MultipartUpload {
+    id: string;
+    url: string;
+    cursor: string;
+}
 export interface UploadResponse {
     artifact?: Artifact;
     error?: ErrorRecord;

package/dist/uploader/uploader.interfaces.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"uploader.interfaces.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAElD,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,YAAY,CAAC;IACpB,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,uFAAuF;IACvF,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,MAAM,cAAc,CAAC,CAAC,IACxB;IAAE,QAAQ,EAAE,CAAC,CAAC;IAAC,KAAK,CAAC,EAAE,KAAK,CAAA;CAAE,GAC9B;IAAE,QAAQ,CAAC,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE,CAAC;AAEzC,oBAAY,iBAAiB;IAC3B,iBAAiB,wBAAwB;IACzC,iBAAiB,wBAAwB;IACzC,kBAAkB,yBAAyB;IAC3C,kBAAkB,yBAAyB;CAC5C;AAED,iEAAiE;AACjE,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,CAAC,EAAE,MAAM,CAAC;IAC/C,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,CAAC,EAAE,MAAM,CAAC;IAC/C,CAAC,iBAAiB,CAAC,kBAAkB,CAAC,CAAC,EAAE,MAAM,CAAC;IAChD,CAAC,iBAAiB,CAAC,kBAAkB,CAAC,CAAC,EAAE,MAAM,CAAC;CACjD;AAED,MAAM,MAAM,kBAAkB,GAAG,IAAI,CACnC,QAAQ,EACN,iBAAiB,CAAC,iBAAiB,GACnC,iBAAiB,CAAC,iBAAiB,GACnC,iBAAiB,CAAC,kBAAkB,GACpC,iBAAiB,CAAC,kBAAkB,CACvC,CAAC;AAEF,iDAAiD;AACjD,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE;QACT,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;KACf,EAAE,CAAC;CACL;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE;QACF,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,SAAS,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,QAAQ,CAAC,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB"}
+{"version":3,"file":"uploader.interfaces.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAElD,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,YAAY,CAAC;IACpB,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,uFAAuF;IACvF,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,MAAM,cAAc,CAAC,CAAC,IACxB;IAAE,QAAQ,EAAE,CAAC,CAAC;IAAC,KAAK,CAAC,EAAE,KAAK,CAAA;CAAE,GAC9B;IAAE,QAAQ,CAAC,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE,CAAC;AAEzC,oBAAY,iBAAiB;IAC3B,iBAAiB,wBAAwB;IACzC,iBAAiB,wBAAwB;IACzC,kBAAkB,yBAAyB;IAC3C,kBAAkB,yBAAyB;CAC5C;AAED,iEAAiE;AACjE,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,CAAC,EAAE,MAAM,CAAC;IAC/C,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,CAAC,EAAE,MAAM,CAAC;IAC/C,CAAC,iBAAiB,CAAC,kBAAkB,CAAC,CAAC,EAAE,MAAM,CAAC;IAChD,CAAC,iBAAiB,CAAC,kBAAkB,CAAC,CAAC,EAAE,MAAM,CAAC;CACjD;AAED,MAAM,MAAM,kBAAkB,GAAG,IAAI,CACnC,QAAQ,EACN,iBAAiB,CAAC,iBAAiB,GACnC,iBAAiB,CAAC,iBAAiB,GACnC,iBAAiB,CAAC,kBAAkB,GACpC,iBAAiB,CAAC,kBAAkB,CACvC,CAAC;AAEF,iDAAiD;AACjD,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE;QACT,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;KACf,EAAE,CAAC;CACL;AAED,yEAAyE;AACzE,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE;QACF,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,SAAS,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,QAAQ,CAAC,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB"}

package/dist/uploader/uploader.js

@@ -4,6 +4,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Uploader = void 0;
+const http_1 = __importDefault(require("http"));
+const https_1 = __importDefault(require("https"));
 const form_data_1 = __importDefault(require("form-data"));
 const js_jsonl_1 = require("js-jsonl");
 const constants_1 = require("../common/constants");
@@ -136,6 +138,201 @@ class Uploader {
             return { error };
         }
     }
+    /**
+     * Creates an S3 multipart upload for an artifact whose size is not known.
+     * The returned URL is for part 1; subsequent URLs are obtained with
+     * getNextMultipartUploadPart.
+     */
+    async createMultipartUpload(filename, fileType, partSize = constants_1.S3_MULTIPART_CHUNK_SIZE) {
+        const url = `${this.devrevApiEndpoint}/internal/artifacts.upload.create`;
+        try {
+            const response = await client_1.axiosClient.post(url, {
+                file_name: (0, uploader_helpers_1.truncateFilename)(filename),
+                file_type: fileType,
+                part_size: partSize,
+            }, { headers: Object.assign({}, this.defaultHeaders) });
+            return { response: response.data };
+        }
+        catch (error) {
+            return { error };
+        }
+    }
+    /** Gets a presigned URL for the next multipart upload part. */
+    async getNextMultipartUploadPart(artifactId, cursor, partSize = constants_1.S3_MULTIPART_CHUNK_SIZE) {
+        const url = `${this.devrevApiEndpoint}/internal/artifacts.upload.next`;
+        try {
+            const response = await client_1.axiosClient.post(url, {
+                id: artifactId,
+                cursor,
+                part_size: partSize,
+            }, { headers: Object.assign({}, this.defaultHeaders) });
+            return { response: response.data };
+        }
+        catch (error) {
+            return { error };
+        }
+    }
+    /** Finalizes and publishes an S3 multipart artifact upload. */
+    async completeMultipartUpload(artifactId, cursor) {
+        const url = `${this.devrevApiEndpoint}/internal/artifacts.upload.complete`;
+        try {
+            const response = await client_1.axiosClient.post(url, {
+                id: artifactId,
+                cursor,
+            }, { headers: Object.assign({}, this.defaultHeaders) });
+            if (response.status >= 200 && response.status < 300) {
+                return { response };
+            }
+            return { error: response };
+        }
+        catch (error) {
+            return { error };
+        }
+    }
+    /**
+     * Streams an attachment of unknown size through an S3 multipart upload.
+     * At most one 10 MiB part is buffered at a time. The final part may be
+     * smaller than S3's 5 MiB minimum.
+     */
+    async streamArtifactMultipart(filename, fileType, fileStream) {
+        try {
+            const iterator = fileStream.data[Symbol.asyncIterator]();
+            let pendingChunk;
+            // Read a complete part before requesting its presigned URL. The gateway
+            // signs Content-Length using part_size, so the requested part size must
+            // be the exact byte length that will be sent (including the final part).
+            const readPart = async () => {
+                const chunks = [];
+                let length = 0;
+                while (length < constants_1.S3_MULTIPART_CHUNK_SIZE) {
+                    let chunk = pendingChunk;
+                    pendingChunk = undefined;
+                    if (!chunk) {
+                        const next = await iterator.next();
+                        if (next.done) {
+                            break;
+                        }
+                        chunk = Buffer.isBuffer(next.value)
+                            ? next.value
+                            : Buffer.from(next.value);
+                    }
+                    if (chunk.length === 0) {
+                        continue;
+                    }
+                    const capacity = constants_1.S3_MULTIPART_CHUNK_SIZE - length;
+                    if (chunk.length <= capacity) {
+                        chunks.push(chunk);
+                        length += chunk.length;
+                    }
+                    else {
+                        chunks.push(chunk.subarray(0, capacity));
+                        pendingChunk = chunk.subarray(capacity);
+                        length += capacity;
+                    }
+                }
+                return length > 0 ? Buffer.concat(chunks, length) : undefined;
+            };
+            let part = await readPart();
+            if (!part) {
+                throw new Error('Cannot complete a multipart upload with an empty file.');
+            }
+            const { response: createdUpload, error: createError } = await this.createMultipartUpload(filename, fileType, part.length);
+            if (createError || !createdUpload) {
+                return {
+                    error: createError !== null && createError !== void 0 ? createError : new Error('Multipart upload creation returned no response.'),
+                };
+            }
+            const artifactId = createdUpload.id;
+            let cursor = createdUpload.cursor;
+            let uploadUrl = createdUpload.url;
+            if (!artifactId || !cursor || !uploadUrl) {
+                return {
+                    error: new Error('Multipart upload creation returned incomplete upload information.'),
+                };
+            }
+            while (part) {
+                const { response, error } = await this.uploadMultipartPart(uploadUrl, part, part.length);
+                if (error || !response) {
+                    return {
+                        error: error !== null && error !== void 0 ? error : new Error('Multipart part upload returned no response.'),
+                    };
+                }
+                const nextPart = part.length === constants_1.S3_MULTIPART_CHUNK_SIZE
+                    ? await readPart()
+                    : undefined;
+                if (!nextPart) {
+                    break;
+                }
+                const { response: nextUpload, error: nextError } = await this.getNextMultipartUploadPart(artifactId, cursor, nextPart.length);
+                if (nextError || !(nextUpload === null || nextUpload === void 0 ? void 0 : nextUpload.url) || !nextUpload.cursor) {
+                    return {
+                        error: nextError !== null && nextError !== void 0 ? nextError : new Error('Could not obtain the next multipart upload URL.'),
+                    };
+                }
+                part = nextPart;
+                uploadUrl = nextUpload.url;
+                cursor = nextUpload.cursor;
+            }
+            const { error: completeError } = await this.completeMultipartUpload(artifactId, cursor);
+            if (completeError) {
+                return { error: completeError };
+            }
+            return { response: { artifact_id: artifactId } };
+        }
+        catch (error) {
+            return { error };
+        }
+        finally {
+            (0, uploader_helpers_1.destroyHttpStream)(fileStream);
+        }
+    }
+    /** Uploads one raw part to its presigned S3 URL. */
+    async uploadMultipartPart(uploadUrl, part, contentLength) {
+        try {
+            // Use Node's native request instead of Axios for the presigned request.
+            // Axios may normalize, remove, or recalculate Content-Length; that is
+            // not allowed when the presigned URL includes this header in its
+            // signature.
+            const parsedUrl = new URL(uploadUrl);
+            const transport = parsedUrl.protocol === 'https:' ? https_1.default : http_1.default;
+            const response = await new Promise((resolve, reject) => {
+                const request = transport.request(parsedUrl, {
+                    method: 'PUT',
+                    headers: {
+                        'Content-Length': String(contentLength),
+                    },
+                }, (incomingResponse) => {
+                    const chunks = [];
+                    incomingResponse.on('data', (chunk) => chunks.push(chunk));
+                    incomingResponse.on('end', () => {
+                        var _a, _b;
+                        const data = Buffer.concat(chunks);
+                        const status = (_a = incomingResponse.statusCode) !== null && _a !== void 0 ? _a : 0;
+                        const axiosResponse = {
+                            data,
+                            status,
+                            statusText: (_b = incomingResponse.statusMessage) !== null && _b !== void 0 ? _b : '',
+                            headers: incomingResponse.headers,
+                            config: {},
+                            request,
+                        };
+                        if (status >= 200 && status < 400) {
+                            resolve(axiosResponse);
+                        }
+                        else {
+                            reject(new Error(`Multipart part upload returned HTTP ${status}.`));
+                        }
+                    });
+                });
+                request.once('error', reject);
+                request.end(part);
+            });
+            return { response };
+        }
+        catch (error) {
+            return { error };
+        }
+    }
     async confirmArtifactUpload(artifactId) {
         const url = `${this.devrevApiEndpoint}/internal/airdrop.artifacts.confirm-upload`;
         try {

package/docs/MIGRATION.md

@@ -394,7 +394,7 @@ async function extractList(adapter: ExtractionAdapter<State>) { ... }
 |---------|----|
 | `initializeRepos`, `getRepo`, `streamAttachments`, `processAttachment`, `shouldExtract`, `artifacts`, `preExtractionItemCounts` | `ExtractionAdapter` |
 | `loadItemTypes`, `loadAttachments`, `reports`, `processedFiles` | `LoadingAdapter` |
-| `event`, `state`, `mappers`, `postState`, `uploadArtifact`, `downloadArtifact`, `isTimeout`, `extractionScope` | both |
+| `event`, `state`, `mappers`, `postState`, `uploadArtifact`, `downloadArtifact`, `isTimeout`, `extractionScope`, `extractionQueries` | both |
 
 - The adapters and `Repo` are **type-only**: `new ExtractionAdapter(...)` and
   `class X extends LoadingAdapter<S>` or `extends Repo` (test doubles that

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devrev/airsync-sdk",
-  "version": "2.0.0",
+  "version": "2.1.0-alpha.0",
   "description": "Typescript SDK for building AirSync snap-ins on the DevRev platform.",
   "type": "commonjs",
   "main": "./dist/index.js",