@devrev/airsync-sdk 2.2.0-alpha.6 → 2.2.0-alpha.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/common/constants.d.ts +3 -1
- package/dist/common/constants.d.ts.map +1 -1
- package/dist/common/constants.js +2 -0
- package/dist/multithreading/adapters/extraction-adapter.d.ts +2 -0
- package/dist/multithreading/adapters/extraction-adapter.d.ts.map +1 -1
- package/dist/multithreading/adapters/extraction-adapter.js +18 -11
- package/dist/repo/repo.d.ts +15 -0
- package/dist/repo/repo.d.ts.map +1 -1
- package/dist/repo/repo.interfaces.d.ts +1 -1
- package/dist/repo/repo.interfaces.d.ts.map +1 -1
- package/dist/repo/repo.js +43 -2
- package/dist/state/base-state.d.ts.map +1 -1
- package/dist/state/base-state.js +5 -41
- package/dist/testing/mock-server.d.ts.map +1 -1
- package/dist/testing/mock-server.js +0 -1
- package/dist/types/extraction.d.ts +3 -3
- package/dist/uploader/uploader.d.ts +5 -1
- package/dist/uploader/uploader.d.ts.map +1 -1
- package/dist/uploader/uploader.js +41 -23
- package/package.json +1 -1
|
@@ -14,7 +14,9 @@ export declare const UNKNOWN_EVENT_TYPE = "UNKNOWN_EVENT_TYPE";
|
|
|
14
14
|
export declare enum AirSyncDefaultItemTypes {
|
|
15
15
|
EXTERNAL_DOMAIN_METADATA = "external_domain_metadata",
|
|
16
16
|
ATTACHMENTS = "attachments",
|
|
17
|
-
EXTERNAL_SYNC_UNITS = "external_sync_units"
|
|
17
|
+
EXTERNAL_SYNC_UNITS = "external_sync_units",
|
|
18
|
+
INITIAL_DOMAIN_MAPPING = "initial_domain_mapping",
|
|
19
|
+
QUERY_RESULT_METADATA = "query_result_metadata"
|
|
18
20
|
}
|
|
19
21
|
export declare const LIBRARY_VERSION: any;
|
|
20
22
|
export declare const DEFAULT_LAMBDA_TIMEOUT: number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/common/constants.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAIhD,eAAO,MAAM,gCAAgC,aAI5C,CAAC;AAEF,eAAO,MAAM,6BAA6B,aAGzC,CAAC;AAEF,eAAO,MAAM,qBAAqB,aAGjC,CAAC;AAEF,eAAO,MAAM,mBAAmB,OAAO,CAAC;AACxC,eAAO,MAAM,wBAAwB,QAAyB,CAAC;AAG/D,eAAO,MAAM,uBAAuB,QAAmB,CAAC;AACxD,eAAO,MAAM,0BAA0B,MAAM,CAAC;AAC9C,eAAO,MAAM,oCAAoC,KAAK,CAAC;AAGvD,eAAO,MAAM,oBAAoB,SAAU,CAAC;AAE5C,eAAO,MAAM,0BAA0B,QAEtC,CAAC;AAEF,eAAO,MAAM,eAAe,EAAE,MAA0B,CAAC;AACzD,eAAO,MAAM,kBAAkB,uBAAuB,CAAC;AAEvD,oBAAY,uBAAuB;IACjC,wBAAwB,6BAA6B;IACrD,WAAW,gBAAgB;IAC3B,mBAAmB,wBAAwB;
|
|
1
|
+
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/common/constants.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAIhD,eAAO,MAAM,gCAAgC,aAI5C,CAAC;AAEF,eAAO,MAAM,6BAA6B,aAGzC,CAAC;AAEF,eAAO,MAAM,qBAAqB,aAGjC,CAAC;AAEF,eAAO,MAAM,mBAAmB,OAAO,CAAC;AACxC,eAAO,MAAM,wBAAwB,QAAyB,CAAC;AAG/D,eAAO,MAAM,uBAAuB,QAAmB,CAAC;AACxD,eAAO,MAAM,0BAA0B,MAAM,CAAC;AAC9C,eAAO,MAAM,oCAAoC,KAAK,CAAC;AAGvD,eAAO,MAAM,oBAAoB,SAAU,CAAC;AAE5C,eAAO,MAAM,0BAA0B,QAEtC,CAAC;AAEF,eAAO,MAAM,eAAe,EAAE,MAA0B,CAAC;AACzD,eAAO,MAAM,kBAAkB,uBAAuB,CAAC;AAEvD,oBAAY,uBAAuB;IACjC,wBAAwB,6BAA6B;IACrD,WAAW,gBAAgB;IAC3B,mBAAmB,wBAAwB;IAC3C,sBAAsB,2BAA2B;IACjD,qBAAqB,0BAA0B;CAChD;AAED,eAAO,MAAM,eAAe,KAAsB,CAAC;AAEnD,eAAO,MAAM,sBAAsB,QAAiB,CAAC;AACrD,eAAO,MAAM,uBAAuB,MAAM,CAAC;AAC3C,eAAO,MAAM,mBAAmB,QAAY,CAAC;AAE7C;;;;GAIG;AACH,eAAO,MAAM,yBAAyB,6BAA6B,CAAC"}
|
package/dist/common/constants.js
CHANGED
|
@@ -34,6 +34,8 @@ var AirSyncDefaultItemTypes;
|
|
|
34
34
|
AirSyncDefaultItemTypes["EXTERNAL_DOMAIN_METADATA"] = "external_domain_metadata";
|
|
35
35
|
AirSyncDefaultItemTypes["ATTACHMENTS"] = "attachments";
|
|
36
36
|
AirSyncDefaultItemTypes["EXTERNAL_SYNC_UNITS"] = "external_sync_units";
|
|
37
|
+
AirSyncDefaultItemTypes["INITIAL_DOMAIN_MAPPING"] = "initial_domain_mapping";
|
|
38
|
+
AirSyncDefaultItemTypes["QUERY_RESULT_METADATA"] = "query_result_metadata";
|
|
37
39
|
})(AirSyncDefaultItemTypes || (exports.AirSyncDefaultItemTypes = AirSyncDefaultItemTypes = {}));
|
|
38
40
|
exports.LIBRARY_VERSION = (0, helpers_1.getLibraryVersion)();
|
|
39
41
|
exports.DEFAULT_LAMBDA_TIMEOUT = 10 * 60 * 1000; // 10 minutes
|
|
@@ -11,6 +11,7 @@ export declare class ExtractionAdapter<ConnectorState> extends BaseAdapter<Conne
|
|
|
11
11
|
private _preExtractionItemCounts?;
|
|
12
12
|
private lastExtractedItemType?;
|
|
13
13
|
private currentEventDataLength;
|
|
14
|
+
/** Defaults to true if the scope is empty or the item type is not listed. */
|
|
14
15
|
shouldExtract(itemType: string): boolean;
|
|
15
16
|
initializeRepos(repos: RepoInterface[]): void;
|
|
16
17
|
private createRepo;
|
|
@@ -24,6 +25,7 @@ export declare class ExtractionAdapter<ConnectorState> extends BaseAdapter<Conne
|
|
|
24
25
|
getRepo(itemType: string): Repo | undefined;
|
|
25
26
|
/** Artifacts uploaded so far that the next emit will report. */
|
|
26
27
|
get artifacts(): readonly Artifact[];
|
|
28
|
+
private trackArtifactSize;
|
|
27
29
|
private addArtifacts;
|
|
28
30
|
/**
|
|
29
31
|
* Per-record-type counts for sync-duration estimation. Set during the
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extraction-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/extraction-adapter.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AACvC,OAAO,EACL,oBAAoB,EACpB,aAAa,EACd,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EAIL,kCAAkC,EAClC,yCAAyC,EAEzC,aAAa,EACb,2BAA2B,EAE5B,MAAM,wBAAwB,CAAC;AAEhC,OAAO,EAAkB,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjE,OAAO,EAAE,QAAQ,EAAkB,MAAM,oCAAoC,CAAC;AAE9E,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,2FAA2F;AAC3F,qBAAa,iBAAiB,CAC5B,cAAc,CACd,SAAQ,WAAW,CAAC,cAAc,CAAC;IACnC,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,KAAK,CAAc;IAC3B,OAAO,CAAC,wBAAwB,CAAC,CAAkB;IACnD,OAAO,CAAC,qBAAqB,CAAC,CAAS;IACvC,OAAO,CAAC,sBAAsB,CAAa;IAY3C,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;
|
|
1
|
+
{"version":3,"file":"extraction-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/extraction-adapter.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AACvC,OAAO,EACL,oBAAoB,EACpB,aAAa,EACd,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EAIL,kCAAkC,EAClC,yCAAyC,EAEzC,aAAa,EACb,2BAA2B,EAE5B,MAAM,wBAAwB,CAAC;AAEhC,OAAO,EAAkB,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjE,OAAO,EAAE,QAAQ,EAAkB,MAAM,oCAAoC,CAAC;AAE9E,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,2FAA2F;AAC3F,qBAAa,iBAAiB,CAC5B,cAAc,CACd,SAAQ,WAAW,CAAC,cAAc,CAAC;IACnC,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,KAAK,CAAc;IAC3B,OAAO,CAAC,wBAAwB,CAAC,CAAkB;IACnD,OAAO,CAAC,qBAAqB,CAAC,CAAS;IACvC,OAAO,CAAC,sBAAsB,CAAa;IAY3C,6EAA6E;IAC7E,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAOxC,eAAe,CAAC,KAAK,EAAE,aAAa,EAAE;IAatC,OAAO,CAAC,UAAU;IA4ClB;;;OAGG;IACH,OAAO,CAAC,cAAc;IAItB,OAAO,CAAC,kBAAkB;IAQ1B,OAAO,CAAC,mBAAmB;IAS3B,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS;IAW3C,gEAAgE;IAChE,IAAI,SAAS,IAAI,SAAS,QAAQ,EAAE,CAEnC;IAED,OAAO,CAAC,iBAAiB;IAczB,OAAO,CAAC,YAAY;IAMpB;;;OAGG;IACH,IAAI,uBAAuB,IAAI,aAAa,EAAE,GAAG,SAAS,CAEzD;IAED,IAAI,uBAAuB,CAAC,MAAM,EAAE,aAAa,EAAE,GAAG,SAAS,EAE9D;IA6HD,OAAO,CAAC,kBAAkB;YAaZ,cAAc;IAmB5B;;;;OAIG;IACG,iBAAiB,CACrB,UAAU,EAAE,oBAAoB,EAChC,MAAM,EAAE,yCAAyC,GAChD,OAAO,CAAC,2BAA2B,CAAC;IA4IjC,iBAAiB,CAAC,QAAQ,EAAE,EAChC,MAAM,EACN,UAAU,EACV,SAAa,GACd,EAAE;QACD,MAAM,EAAE,yCAAyC,CAAC;QAClD,UAAU,CAAC,EAAE,kCAAkC,CAC7C,cAAc,EACd,oBAAoB,EAAE,EACtB,QAAQ,CACT,CAAC;QACF,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,GAAG,OAAO,CAAC,UAAU,CAAC;CAsHxB"}
|
|
@@ -17,9 +17,14 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
|
|
|
17
17
|
this.currentEventDataLength = 0;
|
|
18
18
|
this._artifacts = [];
|
|
19
19
|
}
|
|
20
|
+
/** Defaults to true if the scope is empty or the item type is not listed. */
|
|
20
21
|
shouldExtract(itemType) {
|
|
21
|
-
|
|
22
|
-
|
|
22
|
+
const scope = this.extractionScope;
|
|
23
|
+
if (Object.keys(scope).length === 0)
|
|
24
|
+
return true;
|
|
25
|
+
if (!(itemType in scope))
|
|
26
|
+
return true;
|
|
27
|
+
return scope[itemType].extract;
|
|
23
28
|
}
|
|
24
29
|
initializeRepos(repos) {
|
|
25
30
|
const existingSsorRepo = this.repos.find((repo) => repo.itemType === constants_1.SSOR_ATTACHMENT);
|
|
@@ -35,7 +40,7 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
|
|
|
35
40
|
repo.itemType !== constants_1.SSOR_ATTACHMENT;
|
|
36
41
|
const isDeferred = this.isDeferredRepo(repo.itemType);
|
|
37
42
|
const skipTimestampTracking = (_a = repo.skipTimestampTracking) !== null && _a !== void 0 ? _a : false;
|
|
38
|
-
return new repo_1.Repo(Object.assign(Object.assign({ event: this.event, itemType: repo.itemType, skipTimestampTracking }, (shouldNormalize && { normalize: repo.normalize })), { skipConfirmation: repo.itemType === constants_1.AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS, onUpload: (artifact) => {
|
|
43
|
+
return new repo_1.Repo(Object.assign(Object.assign({ event: this.event, itemType: repo.itemType, skipTimestampTracking }, (shouldNormalize && { normalize: repo.normalize })), { skipConfirmation: repo.itemType === constants_1.AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS, onUpload: (artifact, format) => {
|
|
39
44
|
var _a, _b;
|
|
40
45
|
if (isDeferred) {
|
|
41
46
|
const toDevRev = this.adapterState.sdkState.toDevRev;
|
|
@@ -45,20 +50,14 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
|
|
|
45
50
|
}
|
|
46
51
|
return;
|
|
47
52
|
}
|
|
48
|
-
if (!skipTimestampTracking) {
|
|
53
|
+
if (!skipTimestampTracking && format !== 'parquet') {
|
|
49
54
|
this.lastExtractedItemType = repo.itemType;
|
|
50
55
|
}
|
|
51
56
|
// Artifact ids are kept in state for later attachment streaming
|
|
52
57
|
if (repo.itemType === constants_1.AirSyncDefaultItemTypes.ATTACHMENTS) {
|
|
53
58
|
(_b = this.adapterState.sdkState.toDevRev) === null || _b === void 0 ? void 0 : _b.attachmentsMetadata.artifactIds.push(artifact.id);
|
|
54
59
|
}
|
|
55
|
-
|
|
56
|
-
// isTimeout once the threshold is exceeded to force an early emit.
|
|
57
|
-
this.currentEventDataLength += Buffer.byteLength(JSON.stringify(artifact), 'utf8');
|
|
58
|
-
if (this.currentEventDataLength > constants_1.EVENT_SIZE_THRESHOLD_BYTES &&
|
|
59
|
-
!this.isTimeout) {
|
|
60
|
-
this.isTimeout = true;
|
|
61
|
-
}
|
|
60
|
+
this.trackArtifactSize(artifact);
|
|
62
61
|
}, options: Object.assign(Object.assign({}, this.options), repo.overriddenOptions) }));
|
|
63
62
|
}
|
|
64
63
|
/**
|
|
@@ -93,6 +92,14 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
|
|
|
93
92
|
get artifacts() {
|
|
94
93
|
return this._artifacts;
|
|
95
94
|
}
|
|
95
|
+
trackArtifactSize(artifact) {
|
|
96
|
+
// Count each upload once, not when repos are collected again before emit.
|
|
97
|
+
this.currentEventDataLength += Buffer.byteLength(JSON.stringify(artifact), 'utf8');
|
|
98
|
+
if (this.currentEventDataLength > constants_1.EVENT_SIZE_THRESHOLD_BYTES &&
|
|
99
|
+
!this.isTimeout) {
|
|
100
|
+
this.isTimeout = true;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
96
103
|
addArtifacts(artifacts) {
|
|
97
104
|
this._artifacts = this._artifacts
|
|
98
105
|
.concat(artifacts)
|
package/dist/repo/repo.d.ts
CHANGED
|
@@ -9,6 +9,8 @@ export declare class Repo {
|
|
|
9
9
|
private onUpload;
|
|
10
10
|
private options?;
|
|
11
11
|
private trackTimestamps;
|
|
12
|
+
private canUploadParquet;
|
|
13
|
+
private format?;
|
|
12
14
|
uploadedArtifacts: Artifact[];
|
|
13
15
|
dateRanges: {
|
|
14
16
|
creationDate: {
|
|
@@ -22,7 +24,20 @@ export declare class Repo {
|
|
|
22
24
|
};
|
|
23
25
|
getItems(): (NormalizedItem | NormalizedAttachment | Item)[];
|
|
24
26
|
upload(batch?: (NormalizedItem | NormalizedAttachment | Item)[]): Promise<void>;
|
|
27
|
+
/**
|
|
28
|
+
* Uploads and confirms one ready-made Parquet file immediately during data
|
|
29
|
+
* extraction. The connector supplies valid bytes and the actual positive
|
|
30
|
+
* safe-integer row count. Bypasses normalization, batching and timestamp
|
|
31
|
+
* tracking. Confirmed files are reported with the repo's next callback,
|
|
32
|
+
* including progress and errors. Throws on failure; files are not queued.
|
|
33
|
+
* A repo cannot mix row and file submissions.
|
|
34
|
+
*/
|
|
35
|
+
push(file: Buffer, options: {
|
|
36
|
+
format: 'parquet';
|
|
37
|
+
itemCount: number;
|
|
38
|
+
}): Promise<void>;
|
|
25
39
|
/** Buffers items and uploads every full batch. Throws if an upload fails. */
|
|
26
40
|
push(items: Item[]): Promise<void>;
|
|
41
|
+
private selectFormat;
|
|
27
42
|
}
|
|
28
43
|
//# sourceMappingURL=repo.d.ts.map
|
package/dist/repo/repo.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"repo.d.ts","sourceRoot":"","sources":["../../src/repo/repo.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"repo.d.ts","sourceRoot":"","sources":["../../src/repo/repo.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,IAAI,EAAE,MAAM,yBAAyB,CAAC;AAI/C,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAG3D,OAAO,EACL,oBAAoB,EACpB,cAAc,EAEf,MAAM,mBAAmB,CAAC;AAE3B,qBAAa,IAAI;IACf,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,OAAO,CAAC,KAAK,CAAmD;IAChE,OAAO,CAAC,SAAS,CAAC,CAAwD;IAC1E,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,QAAQ,CAAmC;IACnD,OAAO,CAAC,OAAO,CAAC,CAAiB;IACjC,OAAO,CAAC,eAAe,CAAU;IACjC,OAAO,CAAC,gBAAgB,CAAU;IAClC,OAAO,CAAC,MAAM,CAAC,CAAsB;IAC9B,iBAAiB,EAAE,QAAQ,EAAE,CAAC;IAC9B,UAAU,EAAE;QACjB,YAAY,EAAE;YAAE,MAAM,CAAC,EAAE,MAAM,CAAC;YAAC,MAAM,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QACnD,YAAY,EAAE;YAAE,MAAM,CAAC,EAAE,MAAM,CAAC;YAAC,MAAM,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;KACpD,CAGC;IAyBF,QAAQ,IAAI,CAAC,cAAc,GAAG,oBAAoB,GAAG,IAAI,CAAC,EAAE;IAItD,MAAM,CACV,KAAK,CAAC,EAAE,CAAC,cAAc,GAAG,oBAAoB,GAAG,IAAI,CAAC,EAAE,GACvD,OAAO,CAAC,IAAI,CAAC;IA8DhB;;;;;;;OAOG;IACG,IAAI,CACR,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE;QAAE,MAAM,EAAE,SAAS,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,GAChD,OAAO,CAAC,IAAI,CAAC;IAChB,6EAA6E;IACvE,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAgFxC,OAAO,CAAC,YAAY;CAMrB"}
|
|
@@ -19,7 +19,7 @@ export interface RepoFactoryInterface {
|
|
|
19
19
|
itemType: string;
|
|
20
20
|
normalize?: (record: object) => NormalizedItem | NormalizedAttachment;
|
|
21
21
|
skipTimestampTracking?: boolean;
|
|
22
|
-
onUpload: (artifact: Artifact) => void;
|
|
22
|
+
onUpload: (artifact: Artifact, format?: 'parquet') => void;
|
|
23
23
|
options?: RuntimeOptions;
|
|
24
24
|
skipConfirmation?: boolean;
|
|
25
25
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"repo.interfaces.d.ts","sourceRoot":"","sources":["../../src/repo/repo.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAC/D,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,yCAAyC;AACzC,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,cAAc,GAAG,oBAAoB,CAAC;IACtE;;;;;OAKG;IACH,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,iBAAiB,CAAC,EAAE,WAAW,CAAC;CACjC;AAED,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,YAAY,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,cAAc,GAAG,oBAAoB,CAAC;IACtE,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,QAAQ,EAAE,CAAC,QAAQ,EAAE,QAAQ,KAAK,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"repo.interfaces.d.ts","sourceRoot":"","sources":["../../src/repo/repo.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAC/D,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,yCAAyC;AACzC,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,cAAc,GAAG,oBAAoB,CAAC;IACtE;;;;;OAKG;IACH,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,iBAAiB,CAAC,EAAE,WAAW,CAAC;CACjC;AAED,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,YAAY,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,cAAc,GAAG,oBAAoB,CAAC;IACtE,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,QAAQ,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,CAAC,EAAE,SAAS,KAAK,IAAI,CAAC;IAC3D,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,oBAAoB;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAGD,MAAM,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC"}
|
package/dist/repo/repo.js
CHANGED
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.Repo = void 0;
|
|
4
4
|
const constants_1 = require("../common/constants");
|
|
5
|
+
const errors_1 = require("../http/errors");
|
|
6
|
+
const extraction_1 = require("../types/extraction");
|
|
5
7
|
const uploader_1 = require("../uploader/uploader");
|
|
6
8
|
const repo_helpers_1 = require("./repo.helpers");
|
|
7
9
|
class Repo {
|
|
@@ -19,6 +21,9 @@ class Repo {
|
|
|
19
21
|
this.uploader = new uploader_1.Uploader({ event, options, skipConfirmation });
|
|
20
22
|
this.options = options;
|
|
21
23
|
this.uploadedArtifacts = [];
|
|
24
|
+
this.canUploadParquet =
|
|
25
|
+
event.payload.event_type === extraction_1.EventType.StartExtractingData ||
|
|
26
|
+
event.payload.event_type === extraction_1.EventType.ContinueExtractingData;
|
|
22
27
|
}
|
|
23
28
|
getItems() {
|
|
24
29
|
return this.items;
|
|
@@ -27,6 +32,7 @@ class Repo {
|
|
|
27
32
|
var _a;
|
|
28
33
|
const itemsToUpload = batch || this.items;
|
|
29
34
|
if (itemsToUpload.length > 0) {
|
|
35
|
+
this.selectFormat('jsonl');
|
|
30
36
|
if (this.trackTimestamps) {
|
|
31
37
|
for (const item of itemsToUpload) {
|
|
32
38
|
const createdDate = item === null || item === void 0 ? void 0 : item.created_date;
|
|
@@ -62,14 +68,43 @@ class Repo {
|
|
|
62
68
|
console.log(`No items to upload for type ${this.itemType}. Skipping upload.`);
|
|
63
69
|
}
|
|
64
70
|
}
|
|
65
|
-
|
|
66
|
-
async push(items) {
|
|
71
|
+
async push(items, options) {
|
|
67
72
|
var _a;
|
|
73
|
+
if (options !== undefined || Buffer.isBuffer(items)) {
|
|
74
|
+
if (!this.canUploadParquet) {
|
|
75
|
+
throw new Error('Extraction files can only be uploaded during data extraction.');
|
|
76
|
+
}
|
|
77
|
+
if (!this.itemType.trim() ||
|
|
78
|
+
Object.values(constants_1.AirSyncDefaultItemTypes).some((type) => type === this.itemType) ||
|
|
79
|
+
this.itemType === constants_1.SSOR_ATTACHMENT) {
|
|
80
|
+
throw new Error('itemType must be a dataset/query key, not a metadata or attachment item type.');
|
|
81
|
+
}
|
|
82
|
+
if (!Buffer.isBuffer(items) || items.length === 0) {
|
|
83
|
+
throw new Error('file must be a nonempty Buffer containing Parquet bytes.');
|
|
84
|
+
}
|
|
85
|
+
if ((options === null || options === void 0 ? void 0 : options.format) !== 'parquet') {
|
|
86
|
+
throw new Error('format must be parquet for file uploads.');
|
|
87
|
+
}
|
|
88
|
+
if (!Number.isSafeInteger(options.itemCount) || options.itemCount <= 0) {
|
|
89
|
+
throw new Error('itemCount must be a positive safe integer.');
|
|
90
|
+
}
|
|
91
|
+
this.selectFormat('parquet');
|
|
92
|
+
const { response: artifact, error } = await this.uploader.uploadParquet(this.itemType, items, options.itemCount);
|
|
93
|
+
if (error !== undefined || !artifact) {
|
|
94
|
+
throw error === undefined
|
|
95
|
+
? new errors_1.AirSyncHttpError('No extraction artifact returned.')
|
|
96
|
+
: (0, errors_1.toAirSyncHttpError)(error);
|
|
97
|
+
}
|
|
98
|
+
this.uploadedArtifacts.push(artifact);
|
|
99
|
+
this.onUpload(artifact, 'parquet');
|
|
100
|
+
return;
|
|
101
|
+
}
|
|
68
102
|
let recordsToPush;
|
|
69
103
|
if (!items || items.length === 0) {
|
|
70
104
|
console.log(`No items to push for type ${this.itemType}. Skipping push.`);
|
|
71
105
|
return;
|
|
72
106
|
}
|
|
107
|
+
this.selectFormat('jsonl');
|
|
73
108
|
if (this.normalize &&
|
|
74
109
|
this.itemType != constants_1.AirSyncDefaultItemTypes.EXTERNAL_DOMAIN_METADATA &&
|
|
75
110
|
this.itemType != constants_1.SSOR_ATTACHMENT) {
|
|
@@ -88,5 +123,11 @@ class Repo {
|
|
|
88
123
|
this.items.splice(0, batchSize);
|
|
89
124
|
}
|
|
90
125
|
}
|
|
126
|
+
selectFormat(format) {
|
|
127
|
+
if (this.format && this.format !== format) {
|
|
128
|
+
throw new Error('A repo cannot mix JSONL rows and Parquet files.');
|
|
129
|
+
}
|
|
130
|
+
this.format = format;
|
|
131
|
+
}
|
|
91
132
|
}
|
|
92
133
|
exports.Repo = Repo;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"base-state.d.ts","sourceRoot":"","sources":["../../src/state/base-state.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,YAAY,
|
|
1
|
+
{"version":3,"file":"base-state.d.ts","sourceRoot":"","sources":["../../src/state/base-state.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAEnD,OAAO,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAGtE,OAAO,EAEL,QAAQ,EAER,cAAc,EAEf,MAAM,oBAAoB,CAAC;AAE5B;;;;GAIG;AACH,8BAAsB,SAAS,CAAC,cAAc;IAC5C,SAAS,CAAC,eAAe,EAAE,cAAc,CAAC;IAC1C,SAAS,CAAC,SAAS,EAAE,QAAQ,CAAC;IAC9B,SAAS,CAAC,gBAAgB,EAAE,eAAe,CAAM;IACjD,SAAS,CAAC,kBAAkB,EAAE,iBAAiB,CAAM;IACrD,SAAS,CAAC,QAAQ,CAAC,eAAe,EAAE,QAAQ,CAAC;IAC7C,SAAS,CAAC,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IACvC,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,SAAS,CAAS;gBAGxB,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,cAAc,CAAC,cAAc,CAAC,EACvD,eAAe,EAAE,QAAQ;IAY3B,2EAA2E;IAC3E,IAAI,KAAK,IAAI,cAAc,CAE1B;IAED,IAAI,KAAK,CAAC,KAAK,EAAE,cAAc,EAE9B;IAED,uEAAuE;IACvE,IAAI,QAAQ,IAAI,QAAQ,CAEvB;IAED,IAAI,QAAQ,CAAC,KAAK,EAAE,QAAQ,EAE3B;IAED,IAAI,iBAAiB,IAAI,iBAAiB,CAEzC;IAED,IAAI,eAAe,IAAI,eAAe,CAErC;IAED;;;;OAIG;IACG,mCAAmC,CACvC,oBAAoB,CAAC,EAAE,oBAAoB,GAC1C,OAAO,CAAC,IAAI,CAAC;IAwChB;;;;OAIG;IACG,IAAI,CAAC,YAAY,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IAmFvD;;;;;OAKG;IACH,OAAO,CAAC,qBAAqB;IA4C7B,OAAO,CAAC,YAAY;IAkBpB;;;OAGG;IACG,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IA0C1B,UAAU,IAAI,OAAO,CAAC;QAC1B,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,CAAC;CAsBH"}
|
package/dist/state/base-state.js
CHANGED
|
@@ -8,35 +8,9 @@ const node_worker_threads_1 = require("node:worker_threads");
|
|
|
8
8
|
const axios_1 = __importDefault(require("axios"));
|
|
9
9
|
const client_1 = require("../http/client");
|
|
10
10
|
const logger_1 = require("../logger/logger");
|
|
11
|
-
const extraction_1 = require("../types/extraction");
|
|
12
11
|
const workers_1 = require("../types/workers");
|
|
13
12
|
const install_initial_domain_mapping_1 = require("./install-initial-domain-mapping");
|
|
14
13
|
const state_interfaces_1 = require("./state.interfaces");
|
|
15
|
-
function parseExtractionScope(objects) {
|
|
16
|
-
let scope;
|
|
17
|
-
try {
|
|
18
|
-
scope = JSON.parse(objects);
|
|
19
|
-
}
|
|
20
|
-
catch (error) {
|
|
21
|
-
throw new Error(`Failed to parse extraction scope. ${error}`);
|
|
22
|
-
}
|
|
23
|
-
if (scope === null ||
|
|
24
|
-
typeof scope !== 'object' ||
|
|
25
|
-
Array.isArray(scope) ||
|
|
26
|
-
Object.values(scope).some((value) => value === null ||
|
|
27
|
-
typeof value !== 'object' ||
|
|
28
|
-
Array.isArray(value) ||
|
|
29
|
-
(value.extract !== undefined && typeof value.extract !== 'boolean'))) {
|
|
30
|
-
throw new Error('Invalid extraction scope in worker response.');
|
|
31
|
-
}
|
|
32
|
-
return Object.fromEntries(Object.entries(scope).map(([itemType, requirement]) => {
|
|
33
|
-
var _a;
|
|
34
|
-
return [
|
|
35
|
-
itemType,
|
|
36
|
-
Object.assign(Object.assign({}, requirement), { extract: (_a = requirement.extract) !== null && _a !== void 0 ? _a : false }),
|
|
37
|
-
];
|
|
38
|
-
}));
|
|
39
|
-
}
|
|
40
14
|
/**
|
|
41
15
|
* State lifecycle shared by all sync modes: connector/SDK state separation,
|
|
42
16
|
* fetch/init/post, the v1 to v2 migration shim, and the snap-in-version-gated
|
|
@@ -154,22 +128,12 @@ class BaseState {
|
|
|
154
128
|
console.warn('Failed to parse extractionQueries.');
|
|
155
129
|
}
|
|
156
130
|
}
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
].includes(this.event.payload.event_type);
|
|
161
|
-
if (objects === undefined ||
|
|
162
|
-
objects === null ||
|
|
163
|
-
(objects === '' && !requiresScope)) {
|
|
164
|
-
if (requiresScope) {
|
|
165
|
-
throw new Error('Missing extraction scope in worker response.');
|
|
131
|
+
if (objects) {
|
|
132
|
+
try {
|
|
133
|
+
this._extractionScope = JSON.parse(objects);
|
|
166
134
|
}
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
this._extractionScope = parseExtractionScope(objects);
|
|
170
|
-
if (requiresScope) {
|
|
171
|
-
console.log('Extraction requirements from worker response:', objects);
|
|
172
|
-
console.log('Effective extraction requirements:', JSON.stringify(this._extractionScope));
|
|
135
|
+
catch (error) {
|
|
136
|
+
console.warn(`Failed to parse extractionScope. ${error}`);
|
|
173
137
|
}
|
|
174
138
|
}
|
|
175
139
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mock-server.d.ts","sourceRoot":"","sources":["../../src/testing/mock-server.ts"],"names":[],"mappings":"AAEA,OAAO,EAKL,WAAW,EACX,WAAW,EAEZ,MAAM,0BAA0B,CAAC;AAIlC,eAAO,MAAM,uBAAuB,uBAAuB,CAAC;AAsE5D;;;GAGG;AACH,qBAAa,UAAU;IACrB,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,YAAY,CAAS;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,aAAa,CAA4B;IACjD,OAAO,CAAC,QAAQ,CAAqB;IACrC,OAAO,CAAC,aAAa,CAA4B;gBAErC,IAAI,GAAE,MAAiC;IAMtC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAsBtB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;YAkBpB,aAAa;IA+B3B,qEAAqE;IACrE,OAAO,CAAC,mBAAmB;
|
|
1
|
+
{"version":3,"file":"mock-server.d.ts","sourceRoot":"","sources":["../../src/testing/mock-server.ts"],"names":[],"mappings":"AAEA,OAAO,EAKL,WAAW,EACX,WAAW,EAEZ,MAAM,0BAA0B,CAAC;AAIlC,eAAO,MAAM,uBAAuB,uBAAuB,CAAC;AAsE5D;;;GAGG;AACH,qBAAa,UAAU;IACrB,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,YAAY,CAAS;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,aAAa,CAA4B;IACjD,OAAO,CAAC,QAAQ,CAAqB;IACrC,OAAO,CAAC,aAAa,CAA4B;gBAErC,IAAI,GAAE,MAAiC;IAMtC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAsBtB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;YAkBpB,aAAa;IA+B3B,qEAAqE;IACrE,OAAO,CAAC,mBAAmB;IA6B3B,OAAO,CAAC,WAAW;IAIZ,QAAQ,CAAC,MAAM,EAAE,WAAW,GAAG,IAAI;IA+G1C,kEAAkE;IAC3D,WAAW,IAAI,IAAI;IAMnB,cAAc,IAAI,WAAW,GAAG,SAAS;IAOhD,uEAAuE;IAChE,mBAAmB,IAAI,WAAW,EAAE;IAOpC,eAAe,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM;IAIrD,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,WAAW,EAAE;CAQhE"}
|
|
@@ -139,7 +139,6 @@ class MockServer {
|
|
|
139
139
|
if (req.method === 'GET' && req.path === '/worker_data_url.get') {
|
|
140
140
|
res.status(200).json({
|
|
141
141
|
state: JSON.stringify({}),
|
|
142
|
-
objects: '{}',
|
|
143
142
|
});
|
|
144
143
|
}
|
|
145
144
|
else if (req.method === 'GET' && req.path === '/internal/snap-ins.get') {
|
|
@@ -155,9 +155,9 @@ export interface EventData {
|
|
|
155
155
|
error?: ErrorRecord;
|
|
156
156
|
delay?: number;
|
|
157
157
|
/**
|
|
158
|
-
* Artifacts produced by the worker's repos, attached to the emitted event
|
|
159
|
-
* the SDK. Includes
|
|
160
|
-
*
|
|
158
|
+
* Artifacts produced by the worker's repos, attached to the emitted event
|
|
159
|
+
* by the SDK. Includes row batches, ready-made Parquet files and external
|
|
160
|
+
* sync units pushed to the AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS repo.
|
|
161
161
|
*/
|
|
162
162
|
artifacts?: Artifact[];
|
|
163
163
|
reports?: LoaderReport[];
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { AxiosResponse } from 'axios';
|
|
2
2
|
import { NormalizedAttachment } from '../repo/repo.interfaces';
|
|
3
3
|
import { HttpStreamResponse } from '../types/extraction';
|
|
4
|
-
import { ArtifactToUpload, MultipartUpload, UploaderFactoryInterface, UploaderResult, UploadResponse } from './uploader.interfaces';
|
|
4
|
+
import { Artifact, ArtifactToUpload, MultipartUpload, UploaderFactoryInterface, UploaderResult, UploadResponse } from './uploader.interfaces';
|
|
5
5
|
export declare class Uploader {
|
|
6
6
|
private isLocalDevelopment?;
|
|
7
7
|
private devrevApiEndpoint;
|
|
@@ -12,6 +12,10 @@ export declare class Uploader {
|
|
|
12
12
|
constructor({ event, options, skipConfirmation }: UploaderFactoryInterface);
|
|
13
13
|
/** Compresses fetched objects to gzipped JSONL and uploads them as an artifact. */
|
|
14
14
|
upload(itemType: string, fetchedObjects: object[] | object): Promise<UploadResponse>;
|
|
15
|
+
/** Uploads ready-made Parquet bytes without serializing or compressing them. */
|
|
16
|
+
uploadParquet(itemType: string, file: Buffer, itemCount: number): Promise<UploaderResult<Artifact>>;
|
|
17
|
+
/** Keeps raw errors for callers that need HTTP diagnostics. */
|
|
18
|
+
private uploadFile;
|
|
15
19
|
getArtifactUploadUrl(filename: string, fileType: string, fileSize?: number): Promise<UploaderResult<ArtifactToUpload>>;
|
|
16
20
|
/** Uploads the file buffer to the artifact's upload URL as multipart form data. */
|
|
17
21
|
uploadArtifact(artifact: ArtifactToUpload, file: Buffer): Promise<UploaderResult<AxiosResponse>>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAUtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAWzD,OAAO,
|
|
1
|
+
{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAUtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAWzD,OAAO,EACL,QAAQ,EACR,gBAAgB,EAChB,eAAe,EACf,wBAAwB,EACxB,cAAc,EACd,cAAc,EACf,MAAM,uBAAuB,CAAC;AAE/B,qBAAa,QAAQ;IACnB,OAAO,CAAC,kBAAkB,CAAC,CAAU;IACrC,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,cAAc,CAAyB;IAC/C,OAAO,CAAC,gBAAgB,CAAU;gBAEtB,EAAE,KAAK,EAAE,OAAO,EAAE,gBAAgB,EAAE,EAAE,wBAAwB;IAW1E,mFAAmF;IAC7E,MAAM,CACV,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,EAAE,GAAG,MAAM,GAChC,OAAO,CAAC,cAAc,CAAC;IAgD1B,gFAAgF;IAC1E,aAAa,CACjB,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;IAoBpC,+DAA+D;YACjD,UAAU;IA0ClB,oBAAoB,CACxB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;IA2B5C,mFAAmF;IAC7E,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAmBzC,kFAAkF;IAC5E,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,UAAU,EAAE,kBAAkB,GAC7B,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAqCzC;;;;OAIG;IACG,qBAAqB,CACzB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,QAAQ,SAA0B,GACjC,OAAO,CAAC,cAAc,CAAC,eAAe,CAAC,CAAC;IAkB3C,+DAA+D;IACzD,0BAA0B,CAC9B,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,QAAQ,SAA0B,GACjC,OAAO,CAAC,cAAc,CAAC,eAAe,CAAC,CAAC;IAkB3C,+DAA+D;IACzD,uBAAuB,CAC3B,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAoBzC;;;;OAIG;IACG,uBAAuB,CAC3B,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,kBAAkB,GAC7B,OAAO,CAAC,cAAc,CAAC;QAAE,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IA6HnD,oDAAoD;YACtC,mBAAmB;IAsD3B,qBAAqB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC;QACvD,QAAQ,CAAC,EAAE,aAAa,CAAC;QACzB,KAAK,CAAC,EAAE,OAAO,CAAC;KACjB,CAAC;IAgCF,oEAAoE;IAC9D,4BAA4B,CAAC,EACjC,QAAQ,GACT,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;KAClB,GAAG,OAAO,CAAC;QACV,WAAW,CAAC,EAAE,oBAAoB,EAAE,CAAC;QACrC,KAAK,CAAC,EAAE;YAAE,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC;KAC7B,CAAC;YAuDY,sBAAsB;YAsBtB,gBAAgB;IAc9B,yEAAyE;IACnE,yBAAyB,CAAC,EAC9B,UAAU,EACV,SAAiB,GAClB,EAAE;QACD,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,CAAC,EAAE,OAAO,CAAC;KACrB,GAAG,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC;CA6B/C"}
|
|
@@ -37,40 +37,58 @@ class Uploader {
|
|
|
37
37
|
},
|
|
38
38
|
};
|
|
39
39
|
}
|
|
40
|
-
const
|
|
41
|
-
|
|
42
|
-
const { error: preparedArtifactError, response: preparedArtifact } = await this.getArtifactUploadUrl(filename, fileType);
|
|
43
|
-
if (preparedArtifactError) {
|
|
40
|
+
const { response: artifactId, error, errorMessage, } = await this.uploadFile(itemType + '.jsonl.gz', 'application/x-gzip', file, undefined, this.skipConfirmation);
|
|
41
|
+
if (error !== undefined) {
|
|
44
42
|
return {
|
|
45
43
|
error: {
|
|
46
|
-
message:
|
|
47
|
-
(0, logger_1.serializeError)(preparedArtifactError),
|
|
44
|
+
message: errorMessage + (0, logger_1.serializeError)(error),
|
|
48
45
|
},
|
|
49
46
|
};
|
|
50
47
|
}
|
|
51
|
-
const
|
|
52
|
-
|
|
48
|
+
const artifactDateRanges = (0, uploader_helpers_1.computeArtifactDateRanges)(fetchedObjects);
|
|
49
|
+
const artifact = Object.assign({ id: artifactId, item_type: itemType, item_count: Array.isArray(fetchedObjects) ? fetchedObjects.length : 1 }, artifactDateRanges);
|
|
50
|
+
return { artifact };
|
|
51
|
+
}
|
|
52
|
+
/** Uploads ready-made Parquet bytes without serializing or compressing them. */
|
|
53
|
+
async uploadParquet(itemType, file, itemCount) {
|
|
54
|
+
const { response: artifactId, error } = await this.uploadFile(`${itemType}.parquet`, 'application/vnd.apache.parquet', file, file.length);
|
|
55
|
+
if (error !== undefined) {
|
|
56
|
+
return { error };
|
|
57
|
+
}
|
|
58
|
+
return {
|
|
59
|
+
response: {
|
|
60
|
+
id: artifactId,
|
|
61
|
+
item_type: itemType,
|
|
62
|
+
item_count: itemCount,
|
|
63
|
+
},
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
/** Keeps raw errors for callers that need HTTP diagnostics. */
|
|
67
|
+
async uploadFile(filename, fileType, file, fileSize, skipConfirmation = false) {
|
|
68
|
+
const { response: preparedArtifact, error: prepareError } = await this.getArtifactUploadUrl(filename, fileType, fileSize);
|
|
69
|
+
if (prepareError !== undefined || !preparedArtifact) {
|
|
53
70
|
return {
|
|
54
|
-
error:
|
|
55
|
-
|
|
56
|
-
(0, logger_1.serializeError)(uploadItemError),
|
|
57
|
-
},
|
|
71
|
+
error: prepareError !== null && prepareError !== void 0 ? prepareError : new Error('No artifact upload URL returned.'),
|
|
72
|
+
errorMessage: 'Error while getting artifact upload URL: ',
|
|
58
73
|
};
|
|
59
74
|
}
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
75
|
+
const { error: uploadError } = await this.uploadArtifact(preparedArtifact, file);
|
|
76
|
+
if (uploadError !== undefined) {
|
|
77
|
+
return {
|
|
78
|
+
error: uploadError,
|
|
79
|
+
errorMessage: 'Error while uploading artifact: ',
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
if (!skipConfirmation) {
|
|
83
|
+
const { error: confirmError } = await this.confirmArtifactUpload(preparedArtifact.artifact_id);
|
|
84
|
+
if (confirmError !== undefined) {
|
|
63
85
|
return {
|
|
64
|
-
error:
|
|
65
|
-
|
|
66
|
-
JSON.stringify(confirmArtifactUploadError),
|
|
67
|
-
},
|
|
86
|
+
error: confirmError,
|
|
87
|
+
errorMessage: 'Error while confirming artifact upload. ',
|
|
68
88
|
};
|
|
69
89
|
}
|
|
70
90
|
}
|
|
71
|
-
|
|
72
|
-
const artifact = Object.assign({ id: preparedArtifact.artifact_id, item_type: itemType, item_count: Array.isArray(fetchedObjects) ? fetchedObjects.length : 1 }, artifactDateRanges);
|
|
73
|
-
return { artifact };
|
|
91
|
+
return { response: preparedArtifact.artifact_id };
|
|
74
92
|
}
|
|
75
93
|
async getArtifactUploadUrl(filename, fileType, fileSize) {
|
|
76
94
|
const url = `${this.devrevApiEndpoint}/internal/airdrop.artifacts.upload-url`;
|
|
@@ -355,7 +373,7 @@ class Uploader {
|
|
|
355
373
|
}
|
|
356
374
|
}
|
|
357
375
|
catch (error) {
|
|
358
|
-
return { error
|
|
376
|
+
return { error };
|
|
359
377
|
}
|
|
360
378
|
}
|
|
361
379
|
/** Downloads an artifact and parses it into attachment metadata. */
|