@devrev/airsync-sdk 2.2.0-alpha.3 → 2.2.0-alpha.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/common/constants.d.ts +1 -3
- package/dist/common/constants.d.ts.map +1 -1
- package/dist/common/constants.js +0 -2
- package/dist/multithreading/adapters/extraction-adapter.d.ts +0 -2
- package/dist/multithreading/adapters/extraction-adapter.d.ts.map +1 -1
- package/dist/multithreading/adapters/extraction-adapter.js +11 -18
- package/dist/repo/repo.d.ts +0 -15
- package/dist/repo/repo.d.ts.map +1 -1
- package/dist/repo/repo.interfaces.d.ts +1 -1
- package/dist/repo/repo.interfaces.d.ts.map +1 -1
- package/dist/repo/repo.js +2 -43
- package/dist/state/base-state.d.ts.map +1 -1
- package/dist/state/base-state.js +39 -5
- package/dist/testing/mock-server.d.ts.map +1 -1
- package/dist/testing/mock-server.js +1 -0
- package/dist/types/extraction.d.ts +3 -3
- package/dist/uploader/uploader.d.ts +1 -5
- package/dist/uploader/uploader.d.ts.map +1 -1
- package/dist/uploader/uploader.js +23 -41
- package/package.json +1 -1
|
@@ -14,9 +14,7 @@ export declare const UNKNOWN_EVENT_TYPE = "UNKNOWN_EVENT_TYPE";
|
|
|
14
14
|
export declare enum AirSyncDefaultItemTypes {
|
|
15
15
|
EXTERNAL_DOMAIN_METADATA = "external_domain_metadata",
|
|
16
16
|
ATTACHMENTS = "attachments",
|
|
17
|
-
EXTERNAL_SYNC_UNITS = "external_sync_units"
|
|
18
|
-
INITIAL_DOMAIN_MAPPING = "initial_domain_mapping",
|
|
19
|
-
QUERY_RESULT_METADATA = "query_result_metadata"
|
|
17
|
+
EXTERNAL_SYNC_UNITS = "external_sync_units"
|
|
20
18
|
}
|
|
21
19
|
export declare const LIBRARY_VERSION: any;
|
|
22
20
|
export declare const DEFAULT_LAMBDA_TIMEOUT: number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/common/constants.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAIhD,eAAO,MAAM,gCAAgC,aAI5C,CAAC;AAEF,eAAO,MAAM,6BAA6B,aAGzC,CAAC;AAEF,eAAO,MAAM,qBAAqB,aAGjC,CAAC;AAEF,eAAO,MAAM,mBAAmB,OAAO,CAAC;AACxC,eAAO,MAAM,wBAAwB,QAAyB,CAAC;AAG/D,eAAO,MAAM,uBAAuB,QAAmB,CAAC;AACxD,eAAO,MAAM,0BAA0B,MAAM,CAAC;AAC9C,eAAO,MAAM,oCAAoC,KAAK,CAAC;AAGvD,eAAO,MAAM,oBAAoB,SAAU,CAAC;AAE5C,eAAO,MAAM,0BAA0B,QAEtC,CAAC;AAEF,eAAO,MAAM,eAAe,EAAE,MAA0B,CAAC;AACzD,eAAO,MAAM,kBAAkB,uBAAuB,CAAC;AAEvD,oBAAY,uBAAuB;IACjC,wBAAwB,6BAA6B;IACrD,WAAW,gBAAgB;IAC3B,mBAAmB,wBAAwB;
|
|
1
|
+
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/common/constants.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAIhD,eAAO,MAAM,gCAAgC,aAI5C,CAAC;AAEF,eAAO,MAAM,6BAA6B,aAGzC,CAAC;AAEF,eAAO,MAAM,qBAAqB,aAGjC,CAAC;AAEF,eAAO,MAAM,mBAAmB,OAAO,CAAC;AACxC,eAAO,MAAM,wBAAwB,QAAyB,CAAC;AAG/D,eAAO,MAAM,uBAAuB,QAAmB,CAAC;AACxD,eAAO,MAAM,0BAA0B,MAAM,CAAC;AAC9C,eAAO,MAAM,oCAAoC,KAAK,CAAC;AAGvD,eAAO,MAAM,oBAAoB,SAAU,CAAC;AAE5C,eAAO,MAAM,0BAA0B,QAEtC,CAAC;AAEF,eAAO,MAAM,eAAe,EAAE,MAA0B,CAAC;AACzD,eAAO,MAAM,kBAAkB,uBAAuB,CAAC;AAEvD,oBAAY,uBAAuB;IACjC,wBAAwB,6BAA6B;IACrD,WAAW,gBAAgB;IAC3B,mBAAmB,wBAAwB;CAC5C;AAED,eAAO,MAAM,eAAe,KAAsB,CAAC;AAEnD,eAAO,MAAM,sBAAsB,QAAiB,CAAC;AACrD,eAAO,MAAM,uBAAuB,MAAM,CAAC;AAC3C,eAAO,MAAM,mBAAmB,QAAY,CAAC;AAE7C;;;;GAIG;AACH,eAAO,MAAM,yBAAyB,6BAA6B,CAAC"}
|
package/dist/common/constants.js
CHANGED
|
@@ -34,8 +34,6 @@ var AirSyncDefaultItemTypes;
|
|
|
34
34
|
AirSyncDefaultItemTypes["EXTERNAL_DOMAIN_METADATA"] = "external_domain_metadata";
|
|
35
35
|
AirSyncDefaultItemTypes["ATTACHMENTS"] = "attachments";
|
|
36
36
|
AirSyncDefaultItemTypes["EXTERNAL_SYNC_UNITS"] = "external_sync_units";
|
|
37
|
-
AirSyncDefaultItemTypes["INITIAL_DOMAIN_MAPPING"] = "initial_domain_mapping";
|
|
38
|
-
AirSyncDefaultItemTypes["QUERY_RESULT_METADATA"] = "query_result_metadata";
|
|
39
37
|
})(AirSyncDefaultItemTypes || (exports.AirSyncDefaultItemTypes = AirSyncDefaultItemTypes = {}));
|
|
40
38
|
exports.LIBRARY_VERSION = (0, helpers_1.getLibraryVersion)();
|
|
41
39
|
exports.DEFAULT_LAMBDA_TIMEOUT = 10 * 60 * 1000; // 10 minutes
|
|
@@ -11,7 +11,6 @@ export declare class ExtractionAdapter<ConnectorState> extends BaseAdapter<Conne
|
|
|
11
11
|
private _preExtractionItemCounts?;
|
|
12
12
|
private lastExtractedItemType?;
|
|
13
13
|
private currentEventDataLength;
|
|
14
|
-
/** Defaults to true if the scope is empty or the item type is not listed. */
|
|
15
14
|
shouldExtract(itemType: string): boolean;
|
|
16
15
|
initializeRepos(repos: RepoInterface[]): void;
|
|
17
16
|
private createRepo;
|
|
@@ -25,7 +24,6 @@ export declare class ExtractionAdapter<ConnectorState> extends BaseAdapter<Conne
|
|
|
25
24
|
getRepo(itemType: string): Repo | undefined;
|
|
26
25
|
/** Artifacts uploaded so far that the next emit will report. */
|
|
27
26
|
get artifacts(): readonly Artifact[];
|
|
28
|
-
private trackArtifactSize;
|
|
29
27
|
private addArtifacts;
|
|
30
28
|
/**
|
|
31
29
|
* Per-record-type counts for sync-duration estimation. Set during the
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extraction-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/extraction-adapter.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AACvC,OAAO,EACL,oBAAoB,EACpB,aAAa,EACd,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EAIL,kCAAkC,EAClC,yCAAyC,EAEzC,aAAa,EACb,2BAA2B,EAE5B,MAAM,wBAAwB,CAAC;AAEhC,OAAO,EAAkB,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjE,OAAO,EAAE,QAAQ,EAAkB,MAAM,oCAAoC,CAAC;AAE9E,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,2FAA2F;AAC3F,qBAAa,iBAAiB,CAC5B,cAAc,CACd,SAAQ,WAAW,CAAC,cAAc,CAAC;IACnC,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,KAAK,CAAc;IAC3B,OAAO,CAAC,wBAAwB,CAAC,CAAkB;IACnD,OAAO,CAAC,qBAAqB,CAAC,CAAS;IACvC,OAAO,CAAC,sBAAsB,CAAa;IAY3C,
|
|
1
|
+
{"version":3,"file":"extraction-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/extraction-adapter.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AACvC,OAAO,EACL,oBAAoB,EACpB,aAAa,EACd,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EAIL,kCAAkC,EAClC,yCAAyC,EAEzC,aAAa,EACb,2BAA2B,EAE5B,MAAM,wBAAwB,CAAC;AAEhC,OAAO,EAAkB,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjE,OAAO,EAAE,QAAQ,EAAkB,MAAM,oCAAoC,CAAC;AAE9E,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,2FAA2F;AAC3F,qBAAa,iBAAiB,CAC5B,cAAc,CACd,SAAQ,WAAW,CAAC,cAAc,CAAC;IACnC,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,KAAK,CAAc;IAC3B,OAAO,CAAC,wBAAwB,CAAC,CAAkB;IACnD,OAAO,CAAC,qBAAqB,CAAC,CAAS;IACvC,OAAO,CAAC,sBAAsB,CAAa;IAY3C,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAIxC,eAAe,CAAC,KAAK,EAAE,aAAa,EAAE;IAatC,OAAO,CAAC,UAAU;IAwDlB;;;OAGG;IACH,OAAO,CAAC,cAAc;IAItB,OAAO,CAAC,kBAAkB;IAQ1B,OAAO,CAAC,mBAAmB;IAS3B,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS;IAW3C,gEAAgE;IAChE,IAAI,SAAS,IAAI,SAAS,QAAQ,EAAE,CAEnC;IAED,OAAO,CAAC,YAAY;IAMpB;;;OAGG;IACH,IAAI,uBAAuB,IAAI,aAAa,EAAE,GAAG,SAAS,CAEzD;IAED,IAAI,uBAAuB,CAAC,MAAM,EAAE,aAAa,EAAE,GAAG,SAAS,EAE9D;IA6HD,OAAO,CAAC,kBAAkB;YAaZ,cAAc;IAmB5B;;;;OAIG;IACG,iBAAiB,CACrB,UAAU,EAAE,oBAAoB,EAChC,MAAM,EAAE,yCAAyC,GAChD,OAAO,CAAC,2BAA2B,CAAC;IA4IjC,iBAAiB,CAAC,QAAQ,EAAE,EAChC,MAAM,EACN,UAAU,EACV,SAAa,GACd,EAAE;QACD,MAAM,EAAE,yCAAyC,CAAC;QAClD,UAAU,CAAC,EAAE,kCAAkC,CAC7C,cAAc,EACd,oBAAoB,EAAE,EACtB,QAAQ,CACT,CAAC;QACF,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,GAAG,OAAO,CAAC,UAAU,CAAC;CAsHxB"}
|
|
@@ -17,14 +17,9 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
|
|
|
17
17
|
this.currentEventDataLength = 0;
|
|
18
18
|
this._artifacts = [];
|
|
19
19
|
}
|
|
20
|
-
/** Defaults to true if the scope is empty or the item type is not listed. */
|
|
21
20
|
shouldExtract(itemType) {
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
return true;
|
|
25
|
-
if (!(itemType in scope))
|
|
26
|
-
return true;
|
|
27
|
-
return scope[itemType].extract;
|
|
21
|
+
var _a;
|
|
22
|
+
return ((_a = this.extractionScope[itemType]) === null || _a === void 0 ? void 0 : _a.extract) === true;
|
|
28
23
|
}
|
|
29
24
|
initializeRepos(repos) {
|
|
30
25
|
const existingSsorRepo = this.repos.find((repo) => repo.itemType === constants_1.SSOR_ATTACHMENT);
|
|
@@ -40,7 +35,7 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
|
|
|
40
35
|
repo.itemType !== constants_1.SSOR_ATTACHMENT;
|
|
41
36
|
const isDeferred = this.isDeferredRepo(repo.itemType);
|
|
42
37
|
const skipTimestampTracking = (_a = repo.skipTimestampTracking) !== null && _a !== void 0 ? _a : false;
|
|
43
|
-
return new repo_1.Repo(Object.assign(Object.assign({ event: this.event, itemType: repo.itemType, skipTimestampTracking }, (shouldNormalize && { normalize: repo.normalize })), { skipConfirmation: repo.itemType === constants_1.AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS, onUpload: (artifact
|
|
38
|
+
return new repo_1.Repo(Object.assign(Object.assign({ event: this.event, itemType: repo.itemType, skipTimestampTracking }, (shouldNormalize && { normalize: repo.normalize })), { skipConfirmation: repo.itemType === constants_1.AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS, onUpload: (artifact) => {
|
|
44
39
|
var _a, _b;
|
|
45
40
|
if (isDeferred) {
|
|
46
41
|
const toDevRev = this.adapterState.sdkState.toDevRev;
|
|
@@ -50,14 +45,20 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
|
|
|
50
45
|
}
|
|
51
46
|
return;
|
|
52
47
|
}
|
|
53
|
-
if (!skipTimestampTracking
|
|
48
|
+
if (!skipTimestampTracking) {
|
|
54
49
|
this.lastExtractedItemType = repo.itemType;
|
|
55
50
|
}
|
|
56
51
|
// Artifact ids are kept in state for later attachment streaming
|
|
57
52
|
if (repo.itemType === constants_1.AirSyncDefaultItemTypes.ATTACHMENTS) {
|
|
58
53
|
(_b = this.adapterState.sdkState.toDevRev) === null || _b === void 0 ? void 0 : _b.attachmentsMetadata.artifactIds.push(artifact.id);
|
|
59
54
|
}
|
|
60
|
-
|
|
55
|
+
// Track size of artifact objects that go in the SQS message; flip
|
|
56
|
+
// isTimeout once the threshold is exceeded to force an early emit.
|
|
57
|
+
this.currentEventDataLength += Buffer.byteLength(JSON.stringify(artifact), 'utf8');
|
|
58
|
+
if (this.currentEventDataLength > constants_1.EVENT_SIZE_THRESHOLD_BYTES &&
|
|
59
|
+
!this.isTimeout) {
|
|
60
|
+
this.isTimeout = true;
|
|
61
|
+
}
|
|
61
62
|
}, options: Object.assign(Object.assign({}, this.options), repo.overriddenOptions) }));
|
|
62
63
|
}
|
|
63
64
|
/**
|
|
@@ -92,14 +93,6 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
|
|
|
92
93
|
get artifacts() {
|
|
93
94
|
return this._artifacts;
|
|
94
95
|
}
|
|
95
|
-
trackArtifactSize(artifact) {
|
|
96
|
-
// Count each upload once, not when repos are collected again before emit.
|
|
97
|
-
this.currentEventDataLength += Buffer.byteLength(JSON.stringify(artifact), 'utf8');
|
|
98
|
-
if (this.currentEventDataLength > constants_1.EVENT_SIZE_THRESHOLD_BYTES &&
|
|
99
|
-
!this.isTimeout) {
|
|
100
|
-
this.isTimeout = true;
|
|
101
|
-
}
|
|
102
|
-
}
|
|
103
96
|
addArtifacts(artifacts) {
|
|
104
97
|
this._artifacts = this._artifacts
|
|
105
98
|
.concat(artifacts)
|
package/dist/repo/repo.d.ts
CHANGED
|
@@ -9,8 +9,6 @@ export declare class Repo {
|
|
|
9
9
|
private onUpload;
|
|
10
10
|
private options?;
|
|
11
11
|
private trackTimestamps;
|
|
12
|
-
private canUploadParquet;
|
|
13
|
-
private format?;
|
|
14
12
|
uploadedArtifacts: Artifact[];
|
|
15
13
|
dateRanges: {
|
|
16
14
|
creationDate: {
|
|
@@ -24,20 +22,7 @@ export declare class Repo {
|
|
|
24
22
|
};
|
|
25
23
|
getItems(): (NormalizedItem | NormalizedAttachment | Item)[];
|
|
26
24
|
upload(batch?: (NormalizedItem | NormalizedAttachment | Item)[]): Promise<void>;
|
|
27
|
-
/**
|
|
28
|
-
* Uploads and confirms one ready-made Parquet file immediately during data
|
|
29
|
-
* extraction. The connector supplies valid bytes and the actual positive
|
|
30
|
-
* safe-integer row count. Bypasses normalization, batching and timestamp
|
|
31
|
-
* tracking. Confirmed files are reported with the repo's next callback,
|
|
32
|
-
* including progress and errors. Throws on failure; files are not queued.
|
|
33
|
-
* A repo cannot mix row and file submissions.
|
|
34
|
-
*/
|
|
35
|
-
push(file: Buffer, options: {
|
|
36
|
-
format: 'parquet';
|
|
37
|
-
itemCount: number;
|
|
38
|
-
}): Promise<void>;
|
|
39
25
|
/** Buffers items and uploads every full batch. Throws if an upload fails. */
|
|
40
26
|
push(items: Item[]): Promise<void>;
|
|
41
|
-
private selectFormat;
|
|
42
27
|
}
|
|
43
28
|
//# sourceMappingURL=repo.d.ts.map
|
package/dist/repo/repo.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"repo.d.ts","sourceRoot":"","sources":["../../src/repo/repo.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"repo.d.ts","sourceRoot":"","sources":["../../src/repo/repo.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,IAAI,EAAE,MAAM,yBAAyB,CAAC;AAG/C,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAG3D,OAAO,EACL,oBAAoB,EACpB,cAAc,EAEf,MAAM,mBAAmB,CAAC;AAE3B,qBAAa,IAAI;IACf,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,OAAO,CAAC,KAAK,CAAmD;IAChE,OAAO,CAAC,SAAS,CAAC,CAAwD;IAC1E,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,QAAQ,CAA+B;IAC/C,OAAO,CAAC,OAAO,CAAC,CAAiB;IACjC,OAAO,CAAC,eAAe,CAAU;IAC1B,iBAAiB,EAAE,QAAQ,EAAE,CAAC;IAC9B,UAAU,EAAE;QACjB,YAAY,EAAE;YAAE,MAAM,CAAC,EAAE,MAAM,CAAC;YAAC,MAAM,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QACnD,YAAY,EAAE;YAAE,MAAM,CAAC,EAAE,MAAM,CAAC;YAAC,MAAM,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;KACpD,CAGC;IAsBF,QAAQ,IAAI,CAAC,cAAc,GAAG,oBAAoB,GAAG,IAAI,CAAC,EAAE;IAItD,MAAM,CACV,KAAK,CAAC,EAAE,CAAC,cAAc,GAAG,oBAAoB,GAAG,IAAI,CAAC,EAAE,GACvD,OAAO,CAAC,IAAI,CAAC;IA6DhB,6EAA6E;IACvE,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;CA6BzC"}
|
|
@@ -19,7 +19,7 @@ export interface RepoFactoryInterface {
|
|
|
19
19
|
itemType: string;
|
|
20
20
|
normalize?: (record: object) => NormalizedItem | NormalizedAttachment;
|
|
21
21
|
skipTimestampTracking?: boolean;
|
|
22
|
-
onUpload: (artifact: Artifact
|
|
22
|
+
onUpload: (artifact: Artifact) => void;
|
|
23
23
|
options?: RuntimeOptions;
|
|
24
24
|
skipConfirmation?: boolean;
|
|
25
25
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"repo.interfaces.d.ts","sourceRoot":"","sources":["../../src/repo/repo.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAC/D,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,yCAAyC;AACzC,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,cAAc,GAAG,oBAAoB,CAAC;IACtE;;;;;OAKG;IACH,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,iBAAiB,CAAC,EAAE,WAAW,CAAC;CACjC;AAED,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,YAAY,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,cAAc,GAAG,oBAAoB,CAAC;IACtE,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,QAAQ,EAAE,CAAC,QAAQ,EAAE,QAAQ,
|
|
1
|
+
{"version":3,"file":"repo.interfaces.d.ts","sourceRoot":"","sources":["../../src/repo/repo.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAC/D,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,yCAAyC;AACzC,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,cAAc,GAAG,oBAAoB,CAAC;IACtE;;;;;OAKG;IACH,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,iBAAiB,CAAC,EAAE,WAAW,CAAC;CACjC;AAED,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,YAAY,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,cAAc,GAAG,oBAAoB,CAAC;IACtE,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,QAAQ,EAAE,CAAC,QAAQ,EAAE,QAAQ,KAAK,IAAI,CAAC;IACvC,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,oBAAoB;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAGD,MAAM,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC"}
|
package/dist/repo/repo.js
CHANGED
|
@@ -2,8 +2,6 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.Repo = void 0;
|
|
4
4
|
const constants_1 = require("../common/constants");
|
|
5
|
-
const errors_1 = require("../http/errors");
|
|
6
|
-
const extraction_1 = require("../types/extraction");
|
|
7
5
|
const uploader_1 = require("../uploader/uploader");
|
|
8
6
|
const repo_helpers_1 = require("./repo.helpers");
|
|
9
7
|
class Repo {
|
|
@@ -21,9 +19,6 @@ class Repo {
|
|
|
21
19
|
this.uploader = new uploader_1.Uploader({ event, options, skipConfirmation });
|
|
22
20
|
this.options = options;
|
|
23
21
|
this.uploadedArtifacts = [];
|
|
24
|
-
this.canUploadParquet =
|
|
25
|
-
event.payload.event_type === extraction_1.EventType.StartExtractingData ||
|
|
26
|
-
event.payload.event_type === extraction_1.EventType.ContinueExtractingData;
|
|
27
22
|
}
|
|
28
23
|
getItems() {
|
|
29
24
|
return this.items;
|
|
@@ -32,7 +27,6 @@ class Repo {
|
|
|
32
27
|
var _a;
|
|
33
28
|
const itemsToUpload = batch || this.items;
|
|
34
29
|
if (itemsToUpload.length > 0) {
|
|
35
|
-
this.selectFormat('jsonl');
|
|
36
30
|
if (this.trackTimestamps) {
|
|
37
31
|
for (const item of itemsToUpload) {
|
|
38
32
|
const createdDate = item === null || item === void 0 ? void 0 : item.created_date;
|
|
@@ -68,43 +62,14 @@ class Repo {
|
|
|
68
62
|
console.log(`No items to upload for type ${this.itemType}. Skipping upload.`);
|
|
69
63
|
}
|
|
70
64
|
}
|
|
71
|
-
|
|
65
|
+
/** Buffers items and uploads every full batch. Throws if an upload fails. */
|
|
66
|
+
async push(items) {
|
|
72
67
|
var _a;
|
|
73
|
-
if (options !== undefined || Buffer.isBuffer(items)) {
|
|
74
|
-
if (!this.canUploadParquet) {
|
|
75
|
-
throw new Error('Extraction files can only be uploaded during data extraction.');
|
|
76
|
-
}
|
|
77
|
-
if (!this.itemType.trim() ||
|
|
78
|
-
Object.values(constants_1.AirSyncDefaultItemTypes).some((type) => type === this.itemType) ||
|
|
79
|
-
this.itemType === constants_1.SSOR_ATTACHMENT) {
|
|
80
|
-
throw new Error('itemType must be a dataset/query key, not a metadata or attachment item type.');
|
|
81
|
-
}
|
|
82
|
-
if (!Buffer.isBuffer(items) || items.length === 0) {
|
|
83
|
-
throw new Error('file must be a nonempty Buffer containing Parquet bytes.');
|
|
84
|
-
}
|
|
85
|
-
if ((options === null || options === void 0 ? void 0 : options.format) !== 'parquet') {
|
|
86
|
-
throw new Error('format must be parquet for file uploads.');
|
|
87
|
-
}
|
|
88
|
-
if (!Number.isSafeInteger(options.itemCount) || options.itemCount <= 0) {
|
|
89
|
-
throw new Error('itemCount must be a positive safe integer.');
|
|
90
|
-
}
|
|
91
|
-
this.selectFormat('parquet');
|
|
92
|
-
const { response: artifact, error } = await this.uploader.uploadParquet(this.itemType, items, options.itemCount);
|
|
93
|
-
if (error !== undefined || !artifact) {
|
|
94
|
-
throw error === undefined
|
|
95
|
-
? new errors_1.AirSyncHttpError('No extraction artifact returned.')
|
|
96
|
-
: (0, errors_1.toAirSyncHttpError)(error);
|
|
97
|
-
}
|
|
98
|
-
this.uploadedArtifacts.push(artifact);
|
|
99
|
-
this.onUpload(artifact, 'parquet');
|
|
100
|
-
return;
|
|
101
|
-
}
|
|
102
68
|
let recordsToPush;
|
|
103
69
|
if (!items || items.length === 0) {
|
|
104
70
|
console.log(`No items to push for type ${this.itemType}. Skipping push.`);
|
|
105
71
|
return;
|
|
106
72
|
}
|
|
107
|
-
this.selectFormat('jsonl');
|
|
108
73
|
if (this.normalize &&
|
|
109
74
|
this.itemType != constants_1.AirSyncDefaultItemTypes.EXTERNAL_DOMAIN_METADATA &&
|
|
110
75
|
this.itemType != constants_1.SSOR_ATTACHMENT) {
|
|
@@ -123,11 +88,5 @@ class Repo {
|
|
|
123
88
|
this.items.splice(0, batchSize);
|
|
124
89
|
}
|
|
125
90
|
}
|
|
126
|
-
selectFormat(format) {
|
|
127
|
-
if (this.format && this.format !== format) {
|
|
128
|
-
throw new Error('A repo cannot mix JSONL rows and Parquet files.');
|
|
129
|
-
}
|
|
130
|
-
this.format = format;
|
|
131
|
-
}
|
|
132
91
|
}
|
|
133
92
|
exports.Repo = Repo;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"base-state.d.ts","sourceRoot":"","sources":["../../src/state/base-state.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,YAAY,
|
|
1
|
+
{"version":3,"file":"base-state.d.ts","sourceRoot":"","sources":["../../src/state/base-state.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,YAAY,EAAa,MAAM,qBAAqB,CAAC;AAE9D,OAAO,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAGtE,OAAO,EAEL,QAAQ,EAER,cAAc,EAEf,MAAM,oBAAoB,CAAC;AAiC5B;;;;GAIG;AACH,8BAAsB,SAAS,CAAC,cAAc;IAC5C,SAAS,CAAC,eAAe,EAAE,cAAc,CAAC;IAC1C,SAAS,CAAC,SAAS,EAAE,QAAQ,CAAC;IAC9B,SAAS,CAAC,gBAAgB,EAAE,eAAe,CAAM;IACjD,SAAS,CAAC,kBAAkB,EAAE,iBAAiB,CAAM;IACrD,SAAS,CAAC,QAAQ,CAAC,eAAe,EAAE,QAAQ,CAAC;IAC7C,SAAS,CAAC,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IACvC,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,SAAS,CAAS;gBAGxB,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,cAAc,CAAC,cAAc,CAAC,EACvD,eAAe,EAAE,QAAQ;IAY3B,2EAA2E;IAC3E,IAAI,KAAK,IAAI,cAAc,CAE1B;IAED,IAAI,KAAK,CAAC,KAAK,EAAE,cAAc,EAE9B;IAED,uEAAuE;IACvE,IAAI,QAAQ,IAAI,QAAQ,CAEvB;IAED,IAAI,QAAQ,CAAC,KAAK,EAAE,QAAQ,EAE3B;IAED,IAAI,iBAAiB,IAAI,iBAAiB,CAEzC;IAED,IAAI,eAAe,IAAI,eAAe,CAErC;IAED;;;;OAIG;IACG,mCAAmC,CACvC,oBAAoB,CAAC,EAAE,oBAAoB,GAC1C,OAAO,CAAC,IAAI,CAAC;IAwChB;;;;OAIG;IACG,IAAI,CAAC,YAAY,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IA8FvD;;;;;OAKG;IACH,OAAO,CAAC,qBAAqB;IA4C7B,OAAO,CAAC,YAAY;IAkBpB;;;OAGG;IACG,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IA0C1B,UAAU,IAAI,OAAO,CAAC;QAC1B,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,CAAC;CAsBH"}
|
package/dist/state/base-state.js
CHANGED
|
@@ -8,9 +8,35 @@ const node_worker_threads_1 = require("node:worker_threads");
|
|
|
8
8
|
const axios_1 = __importDefault(require("axios"));
|
|
9
9
|
const client_1 = require("../http/client");
|
|
10
10
|
const logger_1 = require("../logger/logger");
|
|
11
|
+
const extraction_1 = require("../types/extraction");
|
|
11
12
|
const workers_1 = require("../types/workers");
|
|
12
13
|
const install_initial_domain_mapping_1 = require("./install-initial-domain-mapping");
|
|
13
14
|
const state_interfaces_1 = require("./state.interfaces");
|
|
15
|
+
function parseExtractionScope(objects) {
|
|
16
|
+
let scope;
|
|
17
|
+
try {
|
|
18
|
+
scope = JSON.parse(objects);
|
|
19
|
+
}
|
|
20
|
+
catch (error) {
|
|
21
|
+
throw new Error(`Failed to parse extraction scope. ${error}`);
|
|
22
|
+
}
|
|
23
|
+
if (scope === null ||
|
|
24
|
+
typeof scope !== 'object' ||
|
|
25
|
+
Array.isArray(scope) ||
|
|
26
|
+
Object.values(scope).some((value) => value === null ||
|
|
27
|
+
typeof value !== 'object' ||
|
|
28
|
+
Array.isArray(value) ||
|
|
29
|
+
(value.extract !== undefined && typeof value.extract !== 'boolean'))) {
|
|
30
|
+
throw new Error('Invalid extraction scope in worker response.');
|
|
31
|
+
}
|
|
32
|
+
return Object.fromEntries(Object.entries(scope).map(([itemType, requirement]) => {
|
|
33
|
+
var _a;
|
|
34
|
+
return [
|
|
35
|
+
itemType,
|
|
36
|
+
Object.assign(Object.assign({}, requirement), { extract: (_a = requirement.extract) !== null && _a !== void 0 ? _a : false }),
|
|
37
|
+
];
|
|
38
|
+
}));
|
|
39
|
+
}
|
|
14
40
|
/**
|
|
15
41
|
* State lifecycle shared by all sync modes: connector/SDK state separation,
|
|
16
42
|
* fetch/init/post, the v1 to v2 migration shim, and the snap-in-version-gated
|
|
@@ -128,12 +154,20 @@ class BaseState {
|
|
|
128
154
|
console.warn('Failed to parse extractionQueries.');
|
|
129
155
|
}
|
|
130
156
|
}
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
157
|
+
const requiresScope = [
|
|
158
|
+
extraction_1.EventType.StartExtractingData,
|
|
159
|
+
extraction_1.EventType.ContinueExtractingData,
|
|
160
|
+
].includes(this.event.payload.event_type);
|
|
161
|
+
if (objects === undefined || objects === null) {
|
|
162
|
+
if (requiresScope) {
|
|
163
|
+
throw new Error('Missing extraction scope in worker response.');
|
|
134
164
|
}
|
|
135
|
-
|
|
136
|
-
|
|
165
|
+
}
|
|
166
|
+
else {
|
|
167
|
+
this._extractionScope = parseExtractionScope(objects);
|
|
168
|
+
if (requiresScope) {
|
|
169
|
+
console.log('Extraction requirements from worker response:', objects);
|
|
170
|
+
console.log('Effective extraction requirements:', JSON.stringify(this._extractionScope));
|
|
137
171
|
}
|
|
138
172
|
}
|
|
139
173
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mock-server.d.ts","sourceRoot":"","sources":["../../src/testing/mock-server.ts"],"names":[],"mappings":"AAEA,OAAO,EAKL,WAAW,EACX,WAAW,EAEZ,MAAM,0BAA0B,CAAC;AAIlC,eAAO,MAAM,uBAAuB,uBAAuB,CAAC;AAsE5D;;;GAGG;AACH,qBAAa,UAAU;IACrB,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,YAAY,CAAS;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,aAAa,CAA4B;IACjD,OAAO,CAAC,QAAQ,CAAqB;IACrC,OAAO,CAAC,aAAa,CAA4B;gBAErC,IAAI,GAAE,MAAiC;IAMtC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAsBtB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;YAkBpB,aAAa;IA+B3B,qEAAqE;IACrE,OAAO,CAAC,mBAAmB;
|
|
1
|
+
{"version":3,"file":"mock-server.d.ts","sourceRoot":"","sources":["../../src/testing/mock-server.ts"],"names":[],"mappings":"AAEA,OAAO,EAKL,WAAW,EACX,WAAW,EAEZ,MAAM,0BAA0B,CAAC;AAIlC,eAAO,MAAM,uBAAuB,uBAAuB,CAAC;AAsE5D;;;GAGG;AACH,qBAAa,UAAU;IACrB,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,YAAY,CAAS;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,aAAa,CAA4B;IACjD,OAAO,CAAC,QAAQ,CAAqB;IACrC,OAAO,CAAC,aAAa,CAA4B;gBAErC,IAAI,GAAE,MAAiC;IAMtC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAsBtB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;YAkBpB,aAAa;IA+B3B,qEAAqE;IACrE,OAAO,CAAC,mBAAmB;IA8B3B,OAAO,CAAC,WAAW;IAIZ,QAAQ,CAAC,MAAM,EAAE,WAAW,GAAG,IAAI;IA+G1C,kEAAkE;IAC3D,WAAW,IAAI,IAAI;IAMnB,cAAc,IAAI,WAAW,GAAG,SAAS;IAOhD,uEAAuE;IAChE,mBAAmB,IAAI,WAAW,EAAE;IAOpC,eAAe,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM;IAIrD,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,WAAW,EAAE;CAQhE"}
|
|
@@ -139,6 +139,7 @@ class MockServer {
|
|
|
139
139
|
if (req.method === 'GET' && req.path === '/worker_data_url.get') {
|
|
140
140
|
res.status(200).json({
|
|
141
141
|
state: JSON.stringify({}),
|
|
142
|
+
objects: '{}',
|
|
142
143
|
});
|
|
143
144
|
}
|
|
144
145
|
else if (req.method === 'GET' && req.path === '/internal/snap-ins.get') {
|
|
@@ -155,9 +155,9 @@ export interface EventData {
|
|
|
155
155
|
error?: ErrorRecord;
|
|
156
156
|
delay?: number;
|
|
157
157
|
/**
|
|
158
|
-
* Artifacts produced by the worker's repos, attached to the emitted event
|
|
159
|
-
*
|
|
160
|
-
*
|
|
158
|
+
* Artifacts produced by the worker's repos, attached to the emitted event by
|
|
159
|
+
* the SDK. Includes external sync units, which are pushed to the
|
|
160
|
+
* AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS repo and uploaded as artifacts.
|
|
161
161
|
*/
|
|
162
162
|
artifacts?: Artifact[];
|
|
163
163
|
reports?: LoaderReport[];
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { AxiosResponse } from 'axios';
|
|
2
2
|
import { NormalizedAttachment } from '../repo/repo.interfaces';
|
|
3
3
|
import { HttpStreamResponse } from '../types/extraction';
|
|
4
|
-
import {
|
|
4
|
+
import { ArtifactToUpload, MultipartUpload, UploaderFactoryInterface, UploaderResult, UploadResponse } from './uploader.interfaces';
|
|
5
5
|
export declare class Uploader {
|
|
6
6
|
private isLocalDevelopment?;
|
|
7
7
|
private devrevApiEndpoint;
|
|
@@ -12,10 +12,6 @@ export declare class Uploader {
|
|
|
12
12
|
constructor({ event, options, skipConfirmation }: UploaderFactoryInterface);
|
|
13
13
|
/** Compresses fetched objects to gzipped JSONL and uploads them as an artifact. */
|
|
14
14
|
upload(itemType: string, fetchedObjects: object[] | object): Promise<UploadResponse>;
|
|
15
|
-
/** Uploads ready-made Parquet bytes without serializing or compressing them. */
|
|
16
|
-
uploadParquet(itemType: string, file: Buffer, itemCount: number): Promise<UploaderResult<Artifact>>;
|
|
17
|
-
/** Keeps raw errors for callers that need HTTP diagnostics. */
|
|
18
|
-
private uploadFile;
|
|
19
15
|
getArtifactUploadUrl(filename: string, fileType: string, fileSize?: number): Promise<UploaderResult<ArtifactToUpload>>;
|
|
20
16
|
/** Uploads the file buffer to the artifact's upload URL as multipart form data. */
|
|
21
17
|
uploadArtifact(artifact: ArtifactToUpload, file: Buffer): Promise<UploaderResult<AxiosResponse>>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAUtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAWzD,OAAO,
|
|
1
|
+
{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAUtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAWzD,OAAO,EAEL,gBAAgB,EAChB,eAAe,EACf,wBAAwB,EACxB,cAAc,EACd,cAAc,EACf,MAAM,uBAAuB,CAAC;AAE/B,qBAAa,QAAQ;IACnB,OAAO,CAAC,kBAAkB,CAAC,CAAU;IACrC,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,cAAc,CAAyB;IAC/C,OAAO,CAAC,gBAAgB,CAAU;gBAEtB,EAAE,KAAK,EAAE,OAAO,EAAE,gBAAgB,EAAE,EAAE,wBAAwB;IAW1E,mFAAmF;IAC7E,MAAM,CACV,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,EAAE,GAAG,MAAM,GAChC,OAAO,CAAC,cAAc,CAAC;IAwEpB,oBAAoB,CACxB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;IA2B5C,mFAAmF;IAC7E,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAmBzC,kFAAkF;IAC5E,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,UAAU,EAAE,kBAAkB,GAC7B,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAqCzC;;;;OAIG;IACG,qBAAqB,CACzB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,QAAQ,SAA0B,GACjC,OAAO,CAAC,cAAc,CAAC,eAAe,CAAC,CAAC;IAkB3C,+DAA+D;IACzD,0BAA0B,CAC9B,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,QAAQ,SAA0B,GACjC,OAAO,CAAC,cAAc,CAAC,eAAe,CAAC,CAAC;IAkB3C,+DAA+D;IACzD,uBAAuB,CAC3B,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAoBzC;;;;OAIG;IACG,uBAAuB,CAC3B,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,kBAAkB,GAC7B,OAAO,CAAC,cAAc,CAAC;QAAE,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IA6HnD,oDAAoD;YACtC,mBAAmB;IAsD3B,qBAAqB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC;QACvD,QAAQ,CAAC,EAAE,aAAa,CAAC;QACzB,KAAK,CAAC,EAAE,OAAO,CAAC;KACjB,CAAC;IAgCF,oEAAoE;IAC9D,4BAA4B,CAAC,EACjC,QAAQ,GACT,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;KAClB,GAAG,OAAO,CAAC;QACV,WAAW,CAAC,EAAE,oBAAoB,EAAE,CAAC;QACrC,KAAK,CAAC,EAAE;YAAE,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC;KAC7B,CAAC;YAuDY,sBAAsB;YAsBtB,gBAAgB;IAc9B,yEAAyE;IACnE,yBAAyB,CAAC,EAC9B,UAAU,EACV,SAAiB,GAClB,EAAE;QACD,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,CAAC,EAAE,OAAO,CAAC;KACrB,GAAG,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC;CA6B/C"}
|
|
@@ -37,58 +37,40 @@ class Uploader {
|
|
|
37
37
|
},
|
|
38
38
|
};
|
|
39
39
|
}
|
|
40
|
-
const
|
|
41
|
-
|
|
40
|
+
const filename = itemType + '.jsonl.gz';
|
|
41
|
+
const fileType = 'application/x-gzip';
|
|
42
|
+
const { error: preparedArtifactError, response: preparedArtifact } = await this.getArtifactUploadUrl(filename, fileType);
|
|
43
|
+
if (preparedArtifactError) {
|
|
42
44
|
return {
|
|
43
45
|
error: {
|
|
44
|
-
message:
|
|
46
|
+
message: 'Error while getting artifact upload URL: ' +
|
|
47
|
+
(0, logger_1.serializeError)(preparedArtifactError),
|
|
45
48
|
},
|
|
46
49
|
};
|
|
47
50
|
}
|
|
48
|
-
const
|
|
49
|
-
|
|
50
|
-
return { artifact };
|
|
51
|
-
}
|
|
52
|
-
/** Uploads ready-made Parquet bytes without serializing or compressing them. */
|
|
53
|
-
async uploadParquet(itemType, file, itemCount) {
|
|
54
|
-
const { response: artifactId, error } = await this.uploadFile(`${itemType}.parquet`, 'application/vnd.apache.parquet', file, file.length);
|
|
55
|
-
if (error !== undefined) {
|
|
56
|
-
return { error };
|
|
57
|
-
}
|
|
58
|
-
return {
|
|
59
|
-
response: {
|
|
60
|
-
id: artifactId,
|
|
61
|
-
item_type: itemType,
|
|
62
|
-
item_count: itemCount,
|
|
63
|
-
},
|
|
64
|
-
};
|
|
65
|
-
}
|
|
66
|
-
/** Keeps raw errors for callers that need HTTP diagnostics. */
|
|
67
|
-
async uploadFile(filename, fileType, file, fileSize, skipConfirmation = false) {
|
|
68
|
-
const { response: preparedArtifact, error: prepareError } = await this.getArtifactUploadUrl(filename, fileType, fileSize);
|
|
69
|
-
if (prepareError !== undefined || !preparedArtifact) {
|
|
51
|
+
const { error: uploadItemError } = await this.uploadArtifact(preparedArtifact, file);
|
|
52
|
+
if (uploadItemError) {
|
|
70
53
|
return {
|
|
71
|
-
error:
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
const { error: uploadError } = await this.uploadArtifact(preparedArtifact, file);
|
|
76
|
-
if (uploadError !== undefined) {
|
|
77
|
-
return {
|
|
78
|
-
error: uploadError,
|
|
79
|
-
errorMessage: 'Error while uploading artifact: ',
|
|
54
|
+
error: {
|
|
55
|
+
message: 'Error while uploading artifact: ' +
|
|
56
|
+
(0, logger_1.serializeError)(uploadItemError),
|
|
57
|
+
},
|
|
80
58
|
};
|
|
81
59
|
}
|
|
82
|
-
if (!skipConfirmation) {
|
|
83
|
-
const { error:
|
|
84
|
-
if (
|
|
60
|
+
if (!this.skipConfirmation) {
|
|
61
|
+
const { error: confirmArtifactUploadError } = await this.confirmArtifactUpload(preparedArtifact.artifact_id);
|
|
62
|
+
if (confirmArtifactUploadError) {
|
|
85
63
|
return {
|
|
86
|
-
error:
|
|
87
|
-
|
|
64
|
+
error: {
|
|
65
|
+
message: 'Error while confirming artifact upload. ' +
|
|
66
|
+
JSON.stringify(confirmArtifactUploadError),
|
|
67
|
+
},
|
|
88
68
|
};
|
|
89
69
|
}
|
|
90
70
|
}
|
|
91
|
-
|
|
71
|
+
const artifactDateRanges = (0, uploader_helpers_1.computeArtifactDateRanges)(fetchedObjects);
|
|
72
|
+
const artifact = Object.assign({ id: preparedArtifact.artifact_id, item_type: itemType, item_count: Array.isArray(fetchedObjects) ? fetchedObjects.length : 1 }, artifactDateRanges);
|
|
73
|
+
return { artifact };
|
|
92
74
|
}
|
|
93
75
|
async getArtifactUploadUrl(filename, fileType, fileSize) {
|
|
94
76
|
const url = `${this.devrevApiEndpoint}/internal/airdrop.artifacts.upload-url`;
|
|
@@ -373,7 +355,7 @@ class Uploader {
|
|
|
373
355
|
}
|
|
374
356
|
}
|
|
375
357
|
catch (error) {
|
|
376
|
-
return { error };
|
|
358
|
+
return { error: { message: (0, logger_1.serializeError)(error) } };
|
|
377
359
|
}
|
|
378
360
|
}
|
|
379
361
|
/** Downloads an artifact and parses it into attachment metadata. */
|