@devrev/airsync-sdk 2.0.0-beta.9 → 2.1.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/attachments-streaming/attachments-streaming-pool.js +2 -2
- package/dist/common/constants.d.ts +1 -0
- package/dist/common/constants.d.ts.map +1 -1
- package/dist/common/constants.js +4 -1
- package/dist/http/client.d.ts +2 -1
- package/dist/http/client.d.ts.map +1 -1
- package/dist/http/client.js +71 -60
- package/dist/multithreading/adapters/extraction-adapter.d.ts.map +1 -1
- package/dist/multithreading/adapters/extraction-adapter.js +60 -34
- package/dist/multithreading/adapters/loading-adapter.d.ts +1 -0
- package/dist/multithreading/adapters/loading-adapter.d.ts.map +1 -1
- package/dist/multithreading/adapters/loading-adapter.js +31 -9
- package/dist/state/base-state.d.ts +2 -2
- package/dist/state/base-state.d.ts.map +1 -1
- package/dist/state/base-state.js +5 -8
- package/dist/state/extraction-state.d.ts.map +1 -1
- package/dist/state/extraction-state.js +10 -4
- package/dist/types/errors.d.ts +3 -3
- package/dist/types/errors.js +3 -3
- package/dist/types/extraction.d.ts +0 -4
- package/dist/types/extraction.d.ts.map +1 -1
- package/dist/uploader/uploader.d.ts +21 -1
- package/dist/uploader/uploader.d.ts.map +1 -1
- package/dist/uploader/uploader.interfaces.d.ts +6 -0
- package/dist/uploader/uploader.interfaces.d.ts.map +1 -1
- package/dist/uploader/uploader.js +197 -0
- package/docs/MIGRATION.md +152 -38
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# AirSync SDK
|
|
2
2
|
|
|
3
|
-
[](https://coveralls.io/github/devrev/airsync-sdk?branch=main)
|
|
4
4
|
|
|
5
5
|
TypeScript SDK for building AirSync snap-ins (connectors) that sync data between external systems and DevRev. It handles the AirSync control protocol for you: worker threads, timeouts, state, artifact uploads, attachment streaming and loading.
|
|
6
6
|
|
|
@@ -73,7 +73,7 @@ class AttachmentsStreamingPool {
|
|
|
73
73
|
Promise.all(initialPromises),
|
|
74
74
|
this.adapter.timeoutSignal,
|
|
75
75
|
]);
|
|
76
|
-
if (this.delay) {
|
|
76
|
+
if (this.delay !== undefined) {
|
|
77
77
|
return { status: 'delay', delaySeconds: this.delay };
|
|
78
78
|
}
|
|
79
79
|
return { status: 'success' };
|
|
@@ -81,7 +81,7 @@ class AttachmentsStreamingPool {
|
|
|
81
81
|
async startPoolStreaming() {
|
|
82
82
|
var _a;
|
|
83
83
|
while (this.attachments.length > 0) {
|
|
84
|
-
if (this.delay) {
|
|
84
|
+
if (this.delay !== undefined) {
|
|
85
85
|
break;
|
|
86
86
|
}
|
|
87
87
|
if (this.adapter.isTimeout) {
|
|
@@ -4,6 +4,7 @@ export declare const STATELESS_LOADING_EVENT_TYPES: EventType[];
|
|
|
4
4
|
export declare const STATELESS_EVENT_TYPES: EventType[];
|
|
5
5
|
export declare const ARTIFACT_BATCH_SIZE = 2000;
|
|
6
6
|
export declare const MAX_DEVREV_ARTIFACT_SIZE: number;
|
|
7
|
+
export declare const S3_MULTIPART_CHUNK_SIZE: number;
|
|
7
8
|
export declare const MAX_DEVREV_FILENAME_LENGTH = 256;
|
|
8
9
|
export declare const MAX_DEVREV_FILENAME_EXTENSION_LENGTH = 20;
|
|
9
10
|
export declare const MAX_EVENT_SIZE_BYTES = 200000;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/common/constants.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAIhD,eAAO,MAAM,gCAAgC,aAI5C,CAAC;AAEF,eAAO,MAAM,6BAA6B,aAGzC,CAAC;AAEF,eAAO,MAAM,qBAAqB,aAGjC,CAAC;AAEF,eAAO,MAAM,mBAAmB,OAAO,CAAC;AACxC,eAAO,MAAM,wBAAwB,QAAyB,CAAC;
|
|
1
|
+
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/common/constants.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAIhD,eAAO,MAAM,gCAAgC,aAI5C,CAAC;AAEF,eAAO,MAAM,6BAA6B,aAGzC,CAAC;AAEF,eAAO,MAAM,qBAAqB,aAGjC,CAAC;AAEF,eAAO,MAAM,mBAAmB,OAAO,CAAC;AACxC,eAAO,MAAM,wBAAwB,QAAyB,CAAC;AAG/D,eAAO,MAAM,uBAAuB,QAAmB,CAAC;AACxD,eAAO,MAAM,0BAA0B,MAAM,CAAC;AAC9C,eAAO,MAAM,oCAAoC,KAAK,CAAC;AAGvD,eAAO,MAAM,oBAAoB,SAAU,CAAC;AAE5C,eAAO,MAAM,0BAA0B,QAEtC,CAAC;AAEF,eAAO,MAAM,eAAe,EAAE,MAA0B,CAAC;AACzD,eAAO,MAAM,kBAAkB,uBAAuB,CAAC;AAEvD,oBAAY,uBAAuB;IACjC,wBAAwB,6BAA6B;IACrD,WAAW,gBAAgB;IAC3B,mBAAmB,wBAAwB;CAC5C;AAED,eAAO,MAAM,eAAe,KAAsB,CAAC;AAEnD,eAAO,MAAM,sBAAsB,QAAiB,CAAC;AACrD,eAAO,MAAM,uBAAuB,MAAM,CAAC;AAC3C,eAAO,MAAM,mBAAmB,QAAY,CAAC;AAE7C;;;;GAIG;AACH,eAAO,MAAM,yBAAyB,6BAA6B,CAAC"}
|
package/dist/common/constants.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.UNBOUNDED_DATE_TIME_VALUE = exports.MEMORY_LOG_INTERVAL = exports.HARD_TIMEOUT_MULTIPLIER = exports.DEFAULT_LAMBDA_TIMEOUT = exports.LIBRARY_VERSION = exports.AirSyncDefaultItemTypes = exports.UNKNOWN_EVENT_TYPE = exports.SSOR_ATTACHMENT = exports.EVENT_SIZE_THRESHOLD_BYTES = exports.MAX_EVENT_SIZE_BYTES = exports.MAX_DEVREV_FILENAME_EXTENSION_LENGTH = exports.MAX_DEVREV_FILENAME_LENGTH = exports.MAX_DEVREV_ARTIFACT_SIZE = exports.ARTIFACT_BATCH_SIZE = exports.STATELESS_EVENT_TYPES = exports.STATELESS_LOADING_EVENT_TYPES = exports.STATELESS_EXTRACTION_EVENT_TYPES = void 0;
|
|
3
|
+
exports.UNBOUNDED_DATE_TIME_VALUE = exports.MEMORY_LOG_INTERVAL = exports.HARD_TIMEOUT_MULTIPLIER = exports.DEFAULT_LAMBDA_TIMEOUT = exports.LIBRARY_VERSION = exports.AirSyncDefaultItemTypes = exports.UNKNOWN_EVENT_TYPE = exports.SSOR_ATTACHMENT = exports.EVENT_SIZE_THRESHOLD_BYTES = exports.MAX_EVENT_SIZE_BYTES = exports.MAX_DEVREV_FILENAME_EXTENSION_LENGTH = exports.MAX_DEVREV_FILENAME_LENGTH = exports.S3_MULTIPART_CHUNK_SIZE = exports.MAX_DEVREV_ARTIFACT_SIZE = exports.ARTIFACT_BATCH_SIZE = exports.STATELESS_EVENT_TYPES = exports.STATELESS_LOADING_EVENT_TYPES = exports.STATELESS_EXTRACTION_EVENT_TYPES = void 0;
|
|
4
4
|
const extraction_1 = require("../types/extraction");
|
|
5
5
|
const helpers_1 = require("./helpers");
|
|
6
6
|
exports.STATELESS_EXTRACTION_EVENT_TYPES = [
|
|
@@ -18,6 +18,9 @@ exports.STATELESS_EVENT_TYPES = [
|
|
|
18
18
|
];
|
|
19
19
|
exports.ARTIFACT_BATCH_SIZE = 2000;
|
|
20
20
|
exports.MAX_DEVREV_ARTIFACT_SIZE = 2 * 1024 * 1024 * 1024; // 2GB
|
|
21
|
+
// S3 multipart uploads require every non-final part to be at least 5 MiB. A
|
|
22
|
+
// 10 MiB part gives us a useful buffer while keeping the memory overhead low.
|
|
23
|
+
exports.S3_MULTIPART_CHUNK_SIZE = 10 * 1024 * 1024;
|
|
21
24
|
exports.MAX_DEVREV_FILENAME_LENGTH = 256;
|
|
22
25
|
exports.MAX_DEVREV_FILENAME_EXTENSION_LENGTH = 20;
|
|
23
26
|
// Max SQS message size is 250KB, we want to leave some room for the other data in the message
|
package/dist/http/client.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../src/http/client.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../src/http/client.ts"],"names":[],"mappings":"AAAA,OAAc,EAAc,aAAa,EAAE,MAAM,OAAO,CAAC;AAmGzD,QAAA,MAAM,WAAW,EAAE,IAAI,CAAC,aAAa,EAAE,KAAK,GAAG,MAAM,CAGpD,CAAC;AAEF,OAAO,EAAE,WAAW,EAAE,CAAC"}
|
package/dist/http/client.js
CHANGED
|
@@ -6,64 +6,75 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
6
6
|
exports.axiosClient = void 0;
|
|
7
7
|
const axios_1 = __importDefault(require("axios"));
|
|
8
8
|
const axios_retry_1 = __importDefault(require("axios-retry"));
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
9
|
+
function createAxiosClient() {
|
|
10
|
+
const client = axios_1.default.create({
|
|
11
|
+
timeout: 30 * 1000,
|
|
12
|
+
});
|
|
13
|
+
(0, axios_retry_1.default)(client, {
|
|
14
|
+
retries: 5,
|
|
15
|
+
shouldResetTimeout: true,
|
|
16
|
+
retryDelay: (retryCount, error) => {
|
|
17
|
+
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o;
|
|
18
|
+
let delay;
|
|
19
|
+
if (((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) === 429) {
|
|
20
|
+
const retryAfter = ((_c = (_b = error.response) === null || _b === void 0 ? void 0 : _b.headers) === null || _c === void 0 ? void 0 : _c['retry-after']) ||
|
|
21
|
+
((_e = (_d = error.response) === null || _d === void 0 ? void 0 : _d.headers) === null || _e === void 0 ? void 0 : _e['Retry-After']);
|
|
22
|
+
delay = parseInt(retryAfter, 10) * 1000;
|
|
23
|
+
}
|
|
24
|
+
else {
|
|
25
|
+
delay = axios_retry_1.default.exponentialDelay(retryCount, error, 1000);
|
|
26
|
+
}
|
|
27
|
+
const requestId = ((_g = (_f = error.config) === null || _f === void 0 ? void 0 : _f.headers) === null || _g === void 0 ? void 0 : _g['x-request-id']) ||
|
|
28
|
+
((_j = (_h = error.config) === null || _h === void 0 ? void 0 : _h.headers) === null || _j === void 0 ? void 0 : _j['X-Request-ID']);
|
|
29
|
+
const delayInSeconds = Math.round(delay / 1000);
|
|
30
|
+
console.warn(`Retrying request to ${(_k = error.config) === null || _k === void 0 ? void 0 : _k.url} in ${delayInSeconds}s due to ${(_m = (_l = error.response) === null || _l === void 0 ? void 0 : _l.status) !== null && _m !== void 0 ? _m : 'unknown'} error.`, Object.assign(Object.assign({ method: (_o = error.config) === null || _o === void 0 ? void 0 : _o.method }, (retryCount && { retryCount })), (requestId && { requestId })));
|
|
31
|
+
return delay;
|
|
32
|
+
},
|
|
33
|
+
retryCondition: (error) => {
|
|
34
|
+
var _a, _b, _c, _d, _e, _f, _g;
|
|
35
|
+
const retryAfter = ((_b = (_a = error.response) === null || _a === void 0 ? void 0 : _a.headers) === null || _b === void 0 ? void 0 : _b['retry-after']) ||
|
|
36
|
+
((_d = (_c = error.response) === null || _c === void 0 ? void 0 : _c.headers) === null || _d === void 0 ? void 0 : _d['Retry-After']);
|
|
37
|
+
if (((_e = error.response) === null || _e === void 0 ? void 0 : _e.status) && error.response.status >= 500) {
|
|
38
|
+
return true;
|
|
39
|
+
}
|
|
40
|
+
// 429 only when a valid non-negative Retry-After header is present
|
|
41
|
+
else if (((_f = error.response) === null || _f === void 0 ? void 0 : _f.status) &&
|
|
42
|
+
error.response.status === 429 &&
|
|
43
|
+
retryAfter &&
|
|
44
|
+
!isNaN(Number(retryAfter)) &&
|
|
45
|
+
Number(retryAfter) >= 0) {
|
|
46
|
+
return true;
|
|
47
|
+
}
|
|
48
|
+
// Network errors for idempotent requests; 429 already handled above
|
|
49
|
+
else if (axios_retry_1.default.isNetworkOrIdempotentRequestError(error) &&
|
|
50
|
+
((_g = error.response) === null || _g === void 0 ? void 0 : _g.status) !== 429) {
|
|
51
|
+
return true;
|
|
52
|
+
}
|
|
53
|
+
// axios-retry excludes ECONNABORTED (client-side timeout only, never a
|
|
54
|
+
// server response) from isNetworkError, so handle it here separately.
|
|
55
|
+
else if (error.code === 'ECONNABORTED') {
|
|
56
|
+
return true;
|
|
57
|
+
}
|
|
58
|
+
else {
|
|
59
|
+
return false;
|
|
60
|
+
}
|
|
61
|
+
},
|
|
62
|
+
onMaxRetryTimesExceeded(error) {
|
|
63
|
+
var _a, _b, _c, _d, _e, _f;
|
|
64
|
+
(_b = (_a = error.config) === null || _a === void 0 ? void 0 : _a.headers) === null || _b === void 0 ? true : delete _b.authorization;
|
|
65
|
+
(_d = (_c = error.config) === null || _c === void 0 ? void 0 : _c.headers) === null || _d === void 0 ? true : delete _d.Authorization;
|
|
66
|
+
(_e = error.request) === null || _e === void 0 ? true : delete _e._header;
|
|
67
|
+
console.error(`Request to ${(_f = error.config) === null || _f === void 0 ? void 0 : _f.url} failed after max retries. Error`, error);
|
|
68
|
+
},
|
|
69
|
+
});
|
|
70
|
+
return client;
|
|
71
|
+
}
|
|
72
|
+
let instance;
|
|
73
|
+
const getInstance = () => (instance !== null && instance !== void 0 ? instance : (instance = createAxiosClient()));
|
|
74
|
+
// Created on first use so that loading the SDK never calls axios.create(),
|
|
75
|
+
// which breaks connector tests that mock axios.
|
|
76
|
+
const axiosClient = {
|
|
77
|
+
get: async (...args) => getInstance().get(...args),
|
|
78
|
+
post: async (...args) => getInstance().post(...args),
|
|
79
|
+
};
|
|
12
80
|
exports.axiosClient = axiosClient;
|
|
13
|
-
(0, axios_retry_1.default)(axiosClient, {
|
|
14
|
-
retries: 5,
|
|
15
|
-
shouldResetTimeout: true,
|
|
16
|
-
retryDelay: (retryCount, error) => {
|
|
17
|
-
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o;
|
|
18
|
-
let delay;
|
|
19
|
-
if (((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) === 429) {
|
|
20
|
-
const retryAfter = ((_c = (_b = error.response) === null || _b === void 0 ? void 0 : _b.headers) === null || _c === void 0 ? void 0 : _c['retry-after']) ||
|
|
21
|
-
((_e = (_d = error.response) === null || _d === void 0 ? void 0 : _d.headers) === null || _e === void 0 ? void 0 : _e['Retry-After']);
|
|
22
|
-
delay = parseInt(retryAfter, 10) * 1000;
|
|
23
|
-
}
|
|
24
|
-
else {
|
|
25
|
-
delay = axios_retry_1.default.exponentialDelay(retryCount, error, 1000);
|
|
26
|
-
}
|
|
27
|
-
const requestId = ((_g = (_f = error.config) === null || _f === void 0 ? void 0 : _f.headers) === null || _g === void 0 ? void 0 : _g['x-request-id']) ||
|
|
28
|
-
((_j = (_h = error.config) === null || _h === void 0 ? void 0 : _h.headers) === null || _j === void 0 ? void 0 : _j['X-Request-ID']);
|
|
29
|
-
const delayInSeconds = Math.round(delay / 1000);
|
|
30
|
-
console.warn(`Retrying request to ${(_k = error.config) === null || _k === void 0 ? void 0 : _k.url} in ${delayInSeconds}s due to ${(_m = (_l = error.response) === null || _l === void 0 ? void 0 : _l.status) !== null && _m !== void 0 ? _m : 'unknown'} error.`, Object.assign(Object.assign({ method: (_o = error.config) === null || _o === void 0 ? void 0 : _o.method }, (retryCount && { retryCount })), (requestId && { requestId })));
|
|
31
|
-
return delay;
|
|
32
|
-
},
|
|
33
|
-
retryCondition: (error) => {
|
|
34
|
-
var _a, _b, _c, _d, _e, _f, _g;
|
|
35
|
-
const retryAfter = ((_b = (_a = error.response) === null || _a === void 0 ? void 0 : _a.headers) === null || _b === void 0 ? void 0 : _b['retry-after']) ||
|
|
36
|
-
((_d = (_c = error.response) === null || _c === void 0 ? void 0 : _c.headers) === null || _d === void 0 ? void 0 : _d['Retry-After']);
|
|
37
|
-
if (((_e = error.response) === null || _e === void 0 ? void 0 : _e.status) && error.response.status >= 500) {
|
|
38
|
-
return true;
|
|
39
|
-
}
|
|
40
|
-
// 429 only when a valid non-negative Retry-After header is present
|
|
41
|
-
else if (((_f = error.response) === null || _f === void 0 ? void 0 : _f.status) &&
|
|
42
|
-
error.response.status === 429 &&
|
|
43
|
-
retryAfter &&
|
|
44
|
-
!isNaN(Number(retryAfter)) &&
|
|
45
|
-
Number(retryAfter) >= 0) {
|
|
46
|
-
return true;
|
|
47
|
-
}
|
|
48
|
-
// Network errors for idempotent requests; 429 already handled above
|
|
49
|
-
else if (axios_retry_1.default.isNetworkOrIdempotentRequestError(error) &&
|
|
50
|
-
((_g = error.response) === null || _g === void 0 ? void 0 : _g.status) !== 429) {
|
|
51
|
-
return true;
|
|
52
|
-
}
|
|
53
|
-
// axios-retry excludes ECONNABORTED (client-side timeout only, never a
|
|
54
|
-
// server response) from isNetworkError, so handle it here separately.
|
|
55
|
-
else if (error.code === 'ECONNABORTED') {
|
|
56
|
-
return true;
|
|
57
|
-
}
|
|
58
|
-
else {
|
|
59
|
-
return false;
|
|
60
|
-
}
|
|
61
|
-
},
|
|
62
|
-
onMaxRetryTimesExceeded(error) {
|
|
63
|
-
var _a, _b, _c, _d, _e, _f;
|
|
64
|
-
(_b = (_a = error.config) === null || _a === void 0 ? void 0 : _a.headers) === null || _b === void 0 ? true : delete _b.authorization;
|
|
65
|
-
(_d = (_c = error.config) === null || _c === void 0 ? void 0 : _c.headers) === null || _d === void 0 ? true : delete _d.Authorization;
|
|
66
|
-
(_e = error.request) === null || _e === void 0 ? true : delete _e._header;
|
|
67
|
-
console.error(`Request to ${(_f = error.config) === null || _f === void 0 ? void 0 : _f.url} failed after max retries. Error`, error);
|
|
68
|
-
},
|
|
69
|
-
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extraction-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/extraction-adapter.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AACvC,OAAO,EACL,oBAAoB,EACpB,aAAa,EACd,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EAIL,kCAAkC,EAClC,yCAAyC,EAEzC,aAAa,EACb,2BAA2B,EAE5B,MAAM,wBAAwB,CAAC;AAEhC,OAAO,EAAkB,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjE,OAAO,EAAE,QAAQ,EAAkB,MAAM,oCAAoC,CAAC;AAE9E,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,2FAA2F;AAC3F,qBAAa,iBAAiB,CAC5B,cAAc,CACd,SAAQ,WAAW,CAAC,cAAc,CAAC;IACnC,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,wBAAwB,CAAC,CAAkB;IACnD,OAAO,CAAC,KAAK,CAAc;IAC3B,OAAO,CAAC,qBAAqB,CAAC,CAAS;IACvC,OAAO,CAAC,sBAAsB,CAAa;IAY3C,6EAA6E;IAC7E,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAOxC,eAAe,CAAC,KAAK,EAAE,aAAa,EAAE;IAatC,OAAO,CAAC,UAAU;IAoDlB;;;OAGG;IACH,OAAO,CAAC,cAAc;IAItB,OAAO,CAAC,kBAAkB;IAQ1B,OAAO,CAAC,mBAAmB;IAS3B,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS;IAW3C,gEAAgE;IAChE,IAAI,SAAS,IAAI,SAAS,QAAQ,EAAE,CAEnC;IAED,OAAO,CAAC,YAAY;IAMpB;;;OAGG;IACH,IAAI,uBAAuB,IAAI,aAAa,EAAE,GAAG,SAAS,CAEzD;IAED,IAAI,uBAAuB,CAAC,MAAM,EAAE,aAAa,EAAE,GAAG,SAAS,EAE9D;IA6HD,OAAO,CAAC,kBAAkB;YAaZ,cAAc;IAmB5B;;;;OAIG;IACG,iBAAiB,CACrB,UAAU,EAAE,oBAAoB,EAChC,MAAM,EAAE,yCAAyC,GAChD,OAAO,CAAC,2BAA2B,CAAC;
|
|
1
|
+
{"version":3,"file":"extraction-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/extraction-adapter.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AACvC,OAAO,EACL,oBAAoB,EACpB,aAAa,EACd,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EAIL,kCAAkC,EAClC,yCAAyC,EAEzC,aAAa,EACb,2BAA2B,EAE5B,MAAM,wBAAwB,CAAC;AAEhC,OAAO,EAAkB,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjE,OAAO,EAAE,QAAQ,EAAkB,MAAM,oCAAoC,CAAC;AAE9E,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,2FAA2F;AAC3F,qBAAa,iBAAiB,CAC5B,cAAc,CACd,SAAQ,WAAW,CAAC,cAAc,CAAC;IACnC,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,wBAAwB,CAAC,CAAkB;IACnD,OAAO,CAAC,KAAK,CAAc;IAC3B,OAAO,CAAC,qBAAqB,CAAC,CAAS;IACvC,OAAO,CAAC,sBAAsB,CAAa;IAY3C,6EAA6E;IAC7E,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAOxC,eAAe,CAAC,KAAK,EAAE,aAAa,EAAE;IAatC,OAAO,CAAC,UAAU;IAoDlB;;;OAGG;IACH,OAAO,CAAC,cAAc;IAItB,OAAO,CAAC,kBAAkB;IAQ1B,OAAO,CAAC,mBAAmB;IAS3B,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS;IAW3C,gEAAgE;IAChE,IAAI,SAAS,IAAI,SAAS,QAAQ,EAAE,CAEnC;IAED,OAAO,CAAC,YAAY;IAMpB;;;OAGG;IACH,IAAI,uBAAuB,IAAI,aAAa,EAAE,GAAG,SAAS,CAEzD;IAED,IAAI,uBAAuB,CAAC,MAAM,EAAE,aAAa,EAAE,GAAG,SAAS,EAE9D;IA6HD,OAAO,CAAC,kBAAkB;YAaZ,cAAc;IAmB5B;;;;OAIG;IACG,iBAAiB,CACrB,UAAU,EAAE,oBAAoB,EAChC,MAAM,EAAE,yCAAyC,GAChD,OAAO,CAAC,2BAA2B,CAAC;IA4IjC,iBAAiB,CAAC,QAAQ,EAAE,EAChC,MAAM,EACN,UAAU,EACV,SAAa,GACd,EAAE;QACD,MAAM,EAAE,yCAAyC,CAAC;QAClD,UAAU,CAAC,EAAE,kCAAkC,CAC7C,cAAc,EACd,oBAAoB,EAAE,EACtB,QAAQ,CACT,CAAC;QACF,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,GAAG,OAAO,CAAC,UAAU,CAAC;CAsHxB"}
|
|
@@ -230,44 +230,70 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
|
|
|
230
230
|
((_a = httpStream.headers['content-type']) === null || _a === void 0 ? void 0 : _a.toString()) ||
|
|
231
231
|
'application/octet-stream';
|
|
232
232
|
const contentLength = (_b = httpStream.headers['content-length']) === null || _b === void 0 ? void 0 : _b.toString();
|
|
233
|
-
const
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
};
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
};
|
|
233
|
+
const parsedFileSize = contentLength
|
|
234
|
+
? Number.parseInt(contentLength, 10)
|
|
235
|
+
: undefined;
|
|
236
|
+
const fileSize = parsedFileSize !== undefined && Number.isFinite(parsedFileSize)
|
|
237
|
+
? parsedFileSize
|
|
238
|
+
: undefined;
|
|
239
|
+
let artifactId;
|
|
240
|
+
if (fileSize === undefined) {
|
|
241
|
+
// Unknown-length streams cannot be uploaded using a presigned POST with
|
|
242
|
+
// a guessed Content-Length. Stream them through S3 multipart upload.
|
|
243
|
+
const { response: multipartUpload, error: multipartUploadError } = await this.uploader.streamArtifactMultipart(attachment.file_name, fileType, httpStream);
|
|
244
|
+
if (multipartUploadError || !multipartUpload) {
|
|
245
|
+
return {
|
|
246
|
+
status: 'error',
|
|
247
|
+
error: {
|
|
248
|
+
message: `Error while uploading attachment for attachment ID ${attachment.id}. Skipping attachment. ` +
|
|
249
|
+
(0, logger_1.serializeError)(multipartUploadError !== null && multipartUploadError !== void 0 ? multipartUploadError : new Error('Multipart upload returned no response.')),
|
|
250
|
+
fileSize,
|
|
251
|
+
},
|
|
252
|
+
};
|
|
253
|
+
}
|
|
254
|
+
artifactId = multipartUpload.artifact_id;
|
|
256
255
|
}
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
256
|
+
else {
|
|
257
|
+
// Known-size attachments retain the existing single-request upload flow.
|
|
258
|
+
const { error: artifactUrlError, response: artifactUrlResponse } = await this.uploader.getArtifactUploadUrl(attachment.file_name, fileType, fileSize);
|
|
259
|
+
if (artifactUrlError || !artifactUrlResponse) {
|
|
260
|
+
(0, uploader_helpers_1.destroyHttpStream)(httpStream);
|
|
261
|
+
return {
|
|
262
|
+
status: 'error',
|
|
263
|
+
error: {
|
|
264
|
+
message: `Error while preparing artifact for attachment ID ${attachment.id}. Skipping attachment. ${(0, logger_1.serializeError)(artifactUrlError !== null && artifactUrlError !== void 0 ? artifactUrlError : new Error('No artifact upload response.'))}`,
|
|
265
|
+
fileSize,
|
|
266
|
+
},
|
|
267
|
+
};
|
|
268
|
+
}
|
|
269
|
+
const { error: uploadedArtifactError } = await this.uploader.streamArtifact(artifactUrlResponse, httpStream);
|
|
270
|
+
if (uploadedArtifactError) {
|
|
271
|
+
(0, uploader_helpers_1.destroyHttpStream)(httpStream);
|
|
272
|
+
return {
|
|
273
|
+
status: 'error',
|
|
274
|
+
error: {
|
|
275
|
+
message: `Error while streaming to artifact for attachment ID ${attachment.id}. Skipping attachment. ` +
|
|
276
|
+
(0, logger_1.serializeError)(uploadedArtifactError),
|
|
277
|
+
fileSize,
|
|
278
|
+
},
|
|
279
|
+
};
|
|
280
|
+
}
|
|
281
|
+
const { error: confirmArtifactUploadError } = await this.uploader.confirmArtifactUpload(artifactUrlResponse.artifact_id);
|
|
282
|
+
if (confirmArtifactUploadError) {
|
|
283
|
+
return {
|
|
284
|
+
status: 'error',
|
|
285
|
+
error: {
|
|
286
|
+
message: `Error while confirming upload for attachment ID ${attachment.id}. ` +
|
|
287
|
+
(0, logger_1.serializeError)(confirmArtifactUploadError),
|
|
288
|
+
fileSize,
|
|
289
|
+
},
|
|
290
|
+
};
|
|
291
|
+
}
|
|
292
|
+
artifactId = artifactUrlResponse.artifact_id;
|
|
267
293
|
}
|
|
268
294
|
const ssorAttachment = {
|
|
269
295
|
id: {
|
|
270
|
-
devrev:
|
|
296
|
+
devrev: artifactId,
|
|
271
297
|
external: attachment.id,
|
|
272
298
|
},
|
|
273
299
|
parent_id: {
|
|
@@ -17,6 +17,7 @@ export declare class LoadingAdapter<ConnectorState> extends BaseAdapter<Connecto
|
|
|
17
17
|
* an item type the connector can't load, which fails the phase.
|
|
18
18
|
*/
|
|
19
19
|
private loadFiles;
|
|
20
|
+
private initializeFilesToLoad;
|
|
20
21
|
private getLoaderBatches;
|
|
21
22
|
private loadItem;
|
|
22
23
|
/** Maps a non-success connector response to a delay or a failed report. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"loading-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/loading-adapter.ts"],"names":[],"mappings":"AASA,OAAO,EAEL,wBAAwB,EAGxB,6BAA6B,EAE7B,qBAAqB,EAGrB,YAAY,EAGb,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAkB,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjE,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAM7C,yFAAyF;AACzF,qBAAa,cAAc,CACzB,cAAc,CACd,SAAQ,WAAW,CAAC,cAAc,CAAC;IACnC,OAAO,CAAC,aAAa,CAAiB;IACtC,OAAO,CAAC,eAAe,CAAW;IAalC,IAAI,OAAO,IAAI,YAAY,EAAE,CAE5B;IAED,IAAI,cAAc,IAAI,MAAM,EAAE,CAE7B;IA2BK,aAAa,CAAC,EAClB,eAAe,GAChB,EAAE,qBAAqB,GAAG,OAAO,CAAC,UAAU,CAAC;
|
|
1
|
+
{"version":3,"file":"loading-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/loading-adapter.ts"],"names":[],"mappings":"AASA,OAAO,EAEL,wBAAwB,EAGxB,6BAA6B,EAE7B,qBAAqB,EAGrB,YAAY,EAGb,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAkB,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjE,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAM7C,yFAAyF;AACzF,qBAAa,cAAc,CACzB,cAAc,CACd,SAAQ,WAAW,CAAC,cAAc,CAAC;IACnC,OAAO,CAAC,aAAa,CAAiB;IACtC,OAAO,CAAC,eAAe,CAAW;IAalC,IAAI,OAAO,IAAI,YAAY,EAAE,CAE5B;IAED,IAAI,cAAc,IAAI,MAAM,EAAE,CAE7B;IA2BK,aAAa,CAAC,EAClB,eAAe,GAChB,EAAE,qBAAqB,GAAG,OAAO,CAAC,UAAU,CAAC;IA6BxC,eAAe,CAAC,EACpB,MAAM,GACP,EAAE;QACD,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,UAAU,CAAC;IAYvB;;;;OAIG;YACW,SAAS;YAqFT,qBAAqB;YAarB,gBAAgB;YA+BhB,QAAQ;IA+HtB,2EAA2E;IAC3E,OAAO,CAAC,YAAY;YAwBN,cAAc;CAiD7B"}
|
|
@@ -46,9 +46,9 @@ class LoadingAdapter extends base_adapter_1.BaseAdapter {
|
|
|
46
46
|
console.warn('No item types to load, returning.');
|
|
47
47
|
return { status: 'success' };
|
|
48
48
|
}
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
49
|
+
const failure = await this.initializeFilesToLoad(itemTypes);
|
|
50
|
+
if (failure)
|
|
51
|
+
return failure;
|
|
52
52
|
}
|
|
53
53
|
return this.loadFiles('data', (fileToLoad) => {
|
|
54
54
|
const itemTypeToLoad = itemTypesToLoad.find((itemTypeToLoad) => itemTypeToLoad.itemType === fileToLoad.itemType);
|
|
@@ -60,9 +60,9 @@ class LoadingAdapter extends base_adapter_1.BaseAdapter {
|
|
|
60
60
|
}
|
|
61
61
|
async loadAttachments({ create, }) {
|
|
62
62
|
if (this.event.payload.event_type === extraction_1.EventType.StartLoadingAttachments) {
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
63
|
+
const failure = await this.initializeFilesToLoad(['attachment']);
|
|
64
|
+
if (failure)
|
|
65
|
+
return failure;
|
|
66
66
|
}
|
|
67
67
|
return this.loadFiles('attachment', () => async (item) => this.loadAttachment({ item, create }));
|
|
68
68
|
}
|
|
@@ -105,8 +105,11 @@ class LoadingAdapter extends base_adapter_1.BaseAdapter {
|
|
|
105
105
|
console.log(`Timeout detected during ${kind} loading. Returning progress to allow continuation.`);
|
|
106
106
|
return { status: 'progress' };
|
|
107
107
|
}
|
|
108
|
-
const { report, rateLimit } = await load(transformerFile[i]);
|
|
109
|
-
if (
|
|
108
|
+
const { report, rateLimit, error } = await load(transformerFile[i]);
|
|
109
|
+
if (error) {
|
|
110
|
+
return { status: 'error', error };
|
|
111
|
+
}
|
|
112
|
+
if (rateLimit) {
|
|
110
113
|
return { status: 'delay', delaySeconds: rateLimit.delay };
|
|
111
114
|
}
|
|
112
115
|
if (report) {
|
|
@@ -132,6 +135,17 @@ class LoadingAdapter extends base_adapter_1.BaseAdapter {
|
|
|
132
135
|
}
|
|
133
136
|
return { status: 'success' };
|
|
134
137
|
}
|
|
138
|
+
async initializeFilesToLoad(supportedItemTypes) {
|
|
139
|
+
try {
|
|
140
|
+
this.adapterState.sdkState.fromDevRev = {
|
|
141
|
+
filesToLoad: await this.getLoaderBatches(supportedItemTypes),
|
|
142
|
+
};
|
|
143
|
+
return undefined;
|
|
144
|
+
}
|
|
145
|
+
catch (error) {
|
|
146
|
+
return { status: 'error', error: { message: (0, logger_1.serializeError)(error) } };
|
|
147
|
+
}
|
|
148
|
+
}
|
|
135
149
|
async getLoaderBatches(supportedItemTypes) {
|
|
136
150
|
var _a;
|
|
137
151
|
const statsFileArtifactId = (_a = this.event.payload.event_data) === null || _a === void 0 ? void 0 : _a.stats_file;
|
|
@@ -142,7 +156,10 @@ class LoadingAdapter extends base_adapter_1.BaseAdapter {
|
|
|
142
156
|
artifactId: statsFileArtifactId,
|
|
143
157
|
});
|
|
144
158
|
const statsFile = response;
|
|
145
|
-
if (statsFileError
|
|
159
|
+
if (statsFileError) {
|
|
160
|
+
throw new Error(`Failed to download loader stats file ${statsFileArtifactId}. ${(0, logger_1.serializeError)(statsFileError)}`);
|
|
161
|
+
}
|
|
162
|
+
if (statsFile.length === 0) {
|
|
146
163
|
return [];
|
|
147
164
|
}
|
|
148
165
|
return (0, loading_adapter_helpers_1.getFilesToLoad)({ supportedItemTypes, statsFile });
|
|
@@ -267,6 +284,11 @@ class LoadingAdapter extends base_adapter_1.BaseAdapter {
|
|
|
267
284
|
}
|
|
268
285
|
catch (error) {
|
|
269
286
|
console.warn('Failed to create sync mapper record.', (0, logger_1.serializeError)(error));
|
|
287
|
+
return {
|
|
288
|
+
error: {
|
|
289
|
+
message: 'Failed to create sync mapper record. ' + (0, logger_1.serializeError)(error),
|
|
290
|
+
},
|
|
291
|
+
};
|
|
270
292
|
}
|
|
271
293
|
return {
|
|
272
294
|
report: {
|
|
@@ -39,8 +39,8 @@ export declare abstract class BaseState<ConnectorState> {
|
|
|
39
39
|
init(initialState: ConnectorState): Promise<void>;
|
|
40
40
|
/**
|
|
41
41
|
* Normalizes fetched state into the `{ connectorState, sdkState }` envelope.
|
|
42
|
-
*
|
|
43
|
-
*
|
|
42
|
+
* Envelopes are recognized by their keys and validate `stateVersion` when
|
|
43
|
+
* present; anything else is a flat v1 blob split by
|
|
44
44
|
* `V1_SDK_STATE_KEYS`.
|
|
45
45
|
*/
|
|
46
46
|
private normalizeFetchedState;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"base-state.d.ts","sourceRoot":"","sources":["../../src/state/base-state.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAEnD,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAGnD,OAAO,EAEL,QAAQ,EAER,cAAc,EAEf,MAAM,oBAAoB,CAAC;AAE5B;;;;GAIG;AACH,8BAAsB,SAAS,CAAC,cAAc;IAC5C,SAAS,CAAC,eAAe,EAAE,cAAc,CAAC;IAC1C,SAAS,CAAC,SAAS,EAAE,QAAQ,CAAC;IAC9B,SAAS,CAAC,gBAAgB,EAAE,eAAe,CAAM;IACjD,SAAS,CAAC,QAAQ,CAAC,eAAe,EAAE,QAAQ,CAAC;IAC7C,SAAS,CAAC,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IACvC,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,SAAS,CAAS;gBAGxB,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,cAAc,CAAC,cAAc,CAAC,EACvD,eAAe,EAAE,QAAQ;IAY3B,2EAA2E;IAC3E,IAAI,KAAK,IAAI,cAAc,CAE1B;IAED,IAAI,KAAK,CAAC,KAAK,EAAE,cAAc,EAE9B;IAED,uEAAuE;IACvE,IAAI,QAAQ,IAAI,QAAQ,CAEvB;IAED,IAAI,QAAQ,CAAC,KAAK,EAAE,QAAQ,EAE3B;IAED,IAAI,eAAe,IAAI,eAAe,CAErC;IAED;;;;OAIG;IACG,mCAAmC,CACvC,oBAAoB,CAAC,EAAE,oBAAoB,GAC1C,OAAO,CAAC,IAAI,CAAC;IAwChB;;;;OAIG;IACG,IAAI,CAAC,YAAY,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IAuDvD;;;;;OAKG;IACH,OAAO,CAAC,qBAAqB;
|
|
1
|
+
{"version":3,"file":"base-state.d.ts","sourceRoot":"","sources":["../../src/state/base-state.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAEnD,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAGnD,OAAO,EAEL,QAAQ,EAER,cAAc,EAEf,MAAM,oBAAoB,CAAC;AAE5B;;;;GAIG;AACH,8BAAsB,SAAS,CAAC,cAAc;IAC5C,SAAS,CAAC,eAAe,EAAE,cAAc,CAAC;IAC1C,SAAS,CAAC,SAAS,EAAE,QAAQ,CAAC;IAC9B,SAAS,CAAC,gBAAgB,EAAE,eAAe,CAAM;IACjD,SAAS,CAAC,QAAQ,CAAC,eAAe,EAAE,QAAQ,CAAC;IAC7C,SAAS,CAAC,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IACvC,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,SAAS,CAAS;gBAGxB,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,cAAc,CAAC,cAAc,CAAC,EACvD,eAAe,EAAE,QAAQ;IAY3B,2EAA2E;IAC3E,IAAI,KAAK,IAAI,cAAc,CAE1B;IAED,IAAI,KAAK,CAAC,KAAK,EAAE,cAAc,EAE9B;IAED,uEAAuE;IACvE,IAAI,QAAQ,IAAI,QAAQ,CAEvB;IAED,IAAI,QAAQ,CAAC,KAAK,EAAE,QAAQ,EAE3B;IAED,IAAI,eAAe,IAAI,eAAe,CAErC;IAED;;;;OAIG;IACG,mCAAmC,CACvC,oBAAoB,CAAC,EAAE,oBAAoB,GAC1C,OAAO,CAAC,IAAI,CAAC;IAwChB;;;;OAIG;IACG,IAAI,CAAC,YAAY,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IAuDvD;;;;;OAKG;IACH,OAAO,CAAC,qBAAqB;IAyC7B,OAAO,CAAC,YAAY;IAepB;;;OAGG;IACG,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IA0C1B,UAAU,IAAI,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAqBjE"}
|
package/dist/state/base-state.js
CHANGED
|
@@ -137,8 +137,8 @@ class BaseState {
|
|
|
137
137
|
}
|
|
138
138
|
/**
|
|
139
139
|
* Normalizes fetched state into the `{ connectorState, sdkState }` envelope.
|
|
140
|
-
*
|
|
141
|
-
*
|
|
140
|
+
* Envelopes are recognized by their keys and validate `stateVersion` when
|
|
141
|
+
* present; anything else is a flat v1 blob split by
|
|
142
142
|
* `V1_SDK_STATE_KEYS`.
|
|
143
143
|
*/
|
|
144
144
|
normalizeFetchedState(parsed) {
|
|
@@ -146,16 +146,13 @@ class BaseState {
|
|
|
146
146
|
throw new Error('Fetched state is not a JSON object.');
|
|
147
147
|
}
|
|
148
148
|
const record = parsed;
|
|
149
|
-
if (record
|
|
150
|
-
if (record.stateVersion !==
|
|
149
|
+
if ('connectorState' in record || 'sdkState' in record) {
|
|
150
|
+
if (record.stateVersion !== undefined &&
|
|
151
|
+
record.stateVersion !== state_interfaces_1.STATE_VERSION) {
|
|
151
152
|
throw new Error(`Unsupported state version ${String(record.stateVersion)}; this SDK reads version ${state_interfaces_1.STATE_VERSION}.`);
|
|
152
153
|
}
|
|
153
154
|
return this.readEnvelope(record);
|
|
154
155
|
}
|
|
155
|
-
// Envelopes written by 2.0.0 betas carry no version marker.
|
|
156
|
-
if ('connectorState' in record || 'sdkState' in record) {
|
|
157
|
-
return this.readEnvelope(record);
|
|
158
|
-
}
|
|
159
156
|
// Legacy flat v1 blob: split known SDK keys out of the connector state.
|
|
160
157
|
const connectorState = {};
|
|
161
158
|
const sdkState = {};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extraction-state.d.ts","sourceRoot":"","sources":["../../src/state/extraction-state.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAsB,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAGxE,sGAAsG;AACtG,qBAAa,eAAe,CAAC,cAAc,CAAE,SAAQ,SAAS,CAAC,cAAc,CAAC;gBAChE,MAAM,EAAE,cAAc,CAAC,cAAc,CAAC;IAIlD;;;;OAIG;IACH,uBAAuB,IAAI,IAAI;
|
|
1
|
+
{"version":3,"file":"extraction-state.d.ts","sourceRoot":"","sources":["../../src/state/extraction-state.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAsB,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAGxE,sGAAsG;AACtG,qBAAa,eAAe,CAAC,cAAc,CAAE,SAAQ,SAAS,CAAC,cAAc,CAAC;gBAChE,MAAM,EAAE,cAAc,CAAC,cAAc,CAAC;IAIlD;;;;OAIG;IACH,uBAAuB,IAAI,IAAI;CA4HhC;AAED,wBAAsB,qBAAqB,CAAC,cAAc,EAAE,EAC1D,KAAK,EACL,YAAY,EACZ,oBAAoB,EACpB,OAAO,GACR,EAAE,cAAc,CAAC,cAAc,CAAC,GAAG,OAAO,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC,CAkB3E"}
|
|
@@ -21,6 +21,7 @@ class ExtractionState extends base_state_1.BaseState {
|
|
|
21
21
|
* all other events reuse those cached boundaries. Validates extract_from < extract_to.
|
|
22
22
|
*/
|
|
23
23
|
resolveExtractionWindow() {
|
|
24
|
+
var _a, _b;
|
|
24
25
|
const sdkState = this.sdkState;
|
|
25
26
|
const eventContext = this.event.payload.event_context;
|
|
26
27
|
const overrideExtractFrom = (value) => {
|
|
@@ -29,10 +30,15 @@ class ExtractionState extends base_state_1.BaseState {
|
|
|
29
30
|
}
|
|
30
31
|
eventContext.extract_from = value;
|
|
31
32
|
};
|
|
32
|
-
if (this.event.payload.event_type === extraction_1.EventType.StartExtractingData
|
|
33
|
-
!sdkState.lastSyncStarted) {
|
|
34
|
-
|
|
35
|
-
|
|
33
|
+
if (this.event.payload.event_type === extraction_1.EventType.StartExtractingData) {
|
|
34
|
+
if (!sdkState.lastSyncStarted) {
|
|
35
|
+
sdkState.lastSyncStarted = new Date().toISOString();
|
|
36
|
+
console.log(`Setting lastSyncStarted to ${sdkState.lastSyncStarted}.`);
|
|
37
|
+
}
|
|
38
|
+
// A sync that failed before attachments must not leak its SSOR records into this one.
|
|
39
|
+
if ((_b = (_a = sdkState.toDevRev) === null || _a === void 0 ? void 0 : _a.deferredSsorArtifacts) === null || _b === void 0 ? void 0 : _b.length) {
|
|
40
|
+
sdkState.toDevRev.deferredSsorArtifacts = [];
|
|
41
|
+
}
|
|
36
42
|
}
|
|
37
43
|
if (this.event.payload.event_type === extraction_1.EventType.StartExtractingMetadata) {
|
|
38
44
|
const timeFields = [
|
package/dist/types/errors.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
3
|
-
*
|
|
4
|
-
* sends
|
|
2
|
+
* Platform-recognized error codes (CommonError), mapped by the snap-in manager
|
|
3
|
+
* in `internal/common/errors_snapin.go`. Set them on `ErrorRecord.code`; the SDK
|
|
4
|
+
* sends exactly `ERROR_CODE=<code>` as the wire message.
|
|
5
5
|
*/
|
|
6
6
|
export declare enum AirSyncErrorCode {
|
|
7
7
|
EXTERNAL_SYNC_UNIT_DELETED = "EXTERNAL_SYNC_UNIT_DELETED",
|
package/dist/types/errors.js
CHANGED
|
@@ -5,9 +5,9 @@ exports.formatErrorCode = formatErrorCode;
|
|
|
5
5
|
const ERROR_PREFIX = 'ERROR_CODE';
|
|
6
6
|
const ERROR_DELIMITER = '=';
|
|
7
7
|
/**
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
* sends
|
|
8
|
+
* Platform-recognized error codes (CommonError), mapped by the snap-in manager
|
|
9
|
+
* in `internal/common/errors_snapin.go`. Set them on `ErrorRecord.code`; the SDK
|
|
10
|
+
* sends exactly `ERROR_CODE=<code>` as the wire message.
|
|
11
11
|
*/
|
|
12
12
|
var AirSyncErrorCode;
|
|
13
13
|
(function (AirSyncErrorCode) {
|
|
@@ -109,10 +109,6 @@ export interface EventContext {
|
|
|
109
109
|
mode: string;
|
|
110
110
|
request_id: string;
|
|
111
111
|
request_id_adaas: string;
|
|
112
|
-
/** @deprecated Never read by the SDK; kept because connectors still branch on it. */
|
|
113
|
-
reset_extraction?: boolean;
|
|
114
|
-
/** @deprecated Use extraction_start_time/extraction_end_time (resolved into extract_from/extract_to). */
|
|
115
|
-
reset_extract_from?: boolean;
|
|
116
112
|
run_id: string;
|
|
117
113
|
sequence_version: string;
|
|
118
114
|
snap_in_slug: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extraction.d.ts","sourceRoot":"","sources":["../../src/types/extraction.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,+CAA+C,CAAC;AAClF,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAEhD,iDAAiD;AACjD,oBAAY,SAAS;IAEnB,gCAAgC,yCAAyC;IACzE,uBAAuB,8BAA8B;IACrD,0BAA0B,iCAAiC;IAC3D,mBAAmB,0BAA0B;IAC7C,sBAAsB,6BAA6B;IACnD,2BAA2B,mCAAmC;IAC9D,0BAA0B,iCAAiC;IAC3D,6BAA6B,oCAAoC;IACjE,sCAAsC,+CAA+C;IAGrF,gBAAgB,uBAAuB;IACvC,mBAAmB,0BAA0B;IAC7C,uBAAuB,8BAA8B;IACrD,0BAA0B,iCAAiC;IAC3D,wBAAwB,gCAAgC;IACxD,kCAAkC,2CAA2C;CAC9E;AAED,iDAAiD;AACjD,oBAAY,kBAAkB;IAE5B,8BAA8B,uCAAuC;IACrE,+BAA+B,wCAAwC;IACvE,0BAA0B,iCAAiC;IAC3D,yBAAyB,gCAAgC;IACzD,sBAAsB,6BAA6B;IACnD,uBAAuB,8BAA8B;IACrD,sBAAsB,6BAA6B;IACnD,qBAAqB,4BAA4B;IACjD,kBAAkB,yBAAyB;IAC3C,mBAAmB,0BAA0B;IAC7C,0BAA0B,kCAAkC;IAC5D,2BAA2B,mCAAmC;IAC9D,4BAA4B,mCAAmC;IAC/D,2BAA2B,kCAAkC;IAC7D,wBAAwB,+BAA+B;IACvD,yBAAyB,gCAAgC;IACzD,qCAAqC,8CAA8C;IACnF,sCAAsC,+CAA+C;CACtF;AAED,uEAAuE;AACvE,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,oBAAY,gBAAgB;IAC1B,YAAY,iBAAiB;IAC7B,WAAW,gBAAgB;CAC5B;AAED,8EAA8E;AAC9E,oBAAY,QAAQ;IAClB,WAAW,OAAO;IAClB,YAAY,OAAO;IACnB,eAAe,YAAO;IACtB,YAAY,OAAO;IACnB,OAAO,MAAM;IACb,OAAO,MAAM;IACb,KAAK,MAAM;CACZ;AAED;;;GAGG;AACH,oBAAY,aAAa;IACvB,cAAc,mBAAmB;IACjC,2BAA2B,gCAAgC;IAC3D,cAAc,mBAAmB;IACjC,0BAA0B,+BAA+B;IACzD,YAAY,iBAAiB;IAC7B,aAAa,kBAAkB;IAC/B,SAAS,cAAc;CACxB;AAED;;;GAGG;AACH,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,aAAa,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,YAAY;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,uBAAuB,EAAE,MAAM,CAAC;IAChC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;IAC7B;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,0EAA0E;IAC1E,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,WAAW,EAAE,MAAM,CAAC;IACpB,kBAAkB,CAAC,EAAE,gBAAgB,CAAC;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,gBAAgB,EAAE,MAAM,CAAC;IACzB,
|
|
1
|
+
{"version":3,"file":"extraction.d.ts","sourceRoot":"","sources":["../../src/types/extraction.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,+CAA+C,CAAC;AAClF,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAEhD,iDAAiD;AACjD,oBAAY,SAAS;IAEnB,gCAAgC,yCAAyC;IACzE,uBAAuB,8BAA8B;IACrD,0BAA0B,iCAAiC;IAC3D,mBAAmB,0BAA0B;IAC7C,sBAAsB,6BAA6B;IACnD,2BAA2B,mCAAmC;IAC9D,0BAA0B,iCAAiC;IAC3D,6BAA6B,oCAAoC;IACjE,sCAAsC,+CAA+C;IAGrF,gBAAgB,uBAAuB;IACvC,mBAAmB,0BAA0B;IAC7C,uBAAuB,8BAA8B;IACrD,0BAA0B,iCAAiC;IAC3D,wBAAwB,gCAAgC;IACxD,kCAAkC,2CAA2C;CAC9E;AAED,iDAAiD;AACjD,oBAAY,kBAAkB;IAE5B,8BAA8B,uCAAuC;IACrE,+BAA+B,wCAAwC;IACvE,0BAA0B,iCAAiC;IAC3D,yBAAyB,gCAAgC;IACzD,sBAAsB,6BAA6B;IACnD,uBAAuB,8BAA8B;IACrD,sBAAsB,6BAA6B;IACnD,qBAAqB,4BAA4B;IACjD,kBAAkB,yBAAyB;IAC3C,mBAAmB,0BAA0B;IAC7C,0BAA0B,kCAAkC;IAC5D,2BAA2B,mCAAmC;IAC9D,4BAA4B,mCAAmC;IAC/D,2BAA2B,kCAAkC;IAC7D,wBAAwB,+BAA+B;IACvD,yBAAyB,gCAAgC;IACzD,qCAAqC,8CAA8C;IACnF,sCAAsC,+CAA+C;CACtF;AAED,uEAAuE;AACvE,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,oBAAY,gBAAgB;IAC1B,YAAY,iBAAiB;IAC7B,WAAW,gBAAgB;CAC5B;AAED,8EAA8E;AAC9E,oBAAY,QAAQ;IAClB,WAAW,OAAO;IAClB,YAAY,OAAO;IACnB,eAAe,YAAO;IACtB,YAAY,OAAO;IACnB,OAAO,MAAM;IACb,OAAO,MAAM;IACb,KAAK,MAAM;CACZ;AAED;;;GAGG;AACH,oBAAY,aAAa;IACvB,cAAc,mBAAmB;IACjC,2BAA2B,gCAAgC;IAC3D,cAAc,mBAAmB;IACjC,0BAA0B,+BAA+B;IACzD,YAAY,iBAAiB;IAC7B,aAAa,kBAAkB;IAC/B,SAAS,cAAc;CACxB;AAED;;;GAGG;AACH,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,aAAa,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,YAAY;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,uBAAuB,EAAE,MAAM,CAAC;IAChC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;IAC7B;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,0EAA0E;IAC1E,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,WAAW,EAAE,MAAM,CAAC;IACpB,kBAAkB,CAAC,EAAE,gBAAgB,CAAC;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,gBAAgB,EAAE,MAAM,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,KAAK,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,uEAAuE;IACvE,qBAAqB,CAAC,EAAE,SAAS,CAAC;IAClC,mEAAmE;IACnE,mBAAmB,CAAC,EAAE,SAAS,CAAC;IAChC,mGAAmG;IACnG,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,kFAAkF;AAClF,oBAAY,aAAa;IACvB,IAAI,SAAS;IACb,KAAK,UAAU;CAChB;AAED,mGAAmG;AACnG,MAAM,WAAW,aAAa;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,gBAAgB,EAAE,aAAa,CAAC;CACjC;AAED,MAAM,WAAW,SAAS;IACxB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;OAIG;IACH,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IAEvB,OAAO,CAAC,EAAE,YAAY,EAAE,CAAC;IACzB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,0BAA0B,CAAC,EAAE,aAAa,EAAE,CAAC;CAC9C;AAED,MAAM,WAAW,cAAc;IAC7B,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAG/B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAG9B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,gDAAgD;AAChD,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE;QACP,OAAO,EAAE;YACP,qBAAqB,EAAE,MAAM,CAAC;SAC/B,CAAC;QACF,kBAAkB,EAAE,MAAM,CAAC;QAC3B,UAAU,EAAE,MAAM,CAAC;QACnB,0DAA0D;QAC1D,OAAO,EAAE,MAAM,CAAC;QAChB,iDAAiD;QACjD,OAAO,EAAE,MAAM,CAAC;QAChB,gEAAgE;QAChE,SAAS,EAAE,MAAM,CAAC;QAClB,yDAAyD;QACzD,kBAAkB,EAAE,MAAM,CAAC;KAC5B,CAAC;IACF,OAAO,EAAE,cAAc,CAAC;IACxB,kBAAkB,EAAE;QAClB,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;IACF,UAAU,EAAE,SAAS,CAAC;CACvB;AAED,8CAA8C;AAC9C,MAAM,WAAW,SAAS;IACxB,iDAAiD;IACjD,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtC,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtC,2EAA2E;IAE3E,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CACjC;AAED,MAAM,WAAW,cAAc;IAC7B,eAAe,EAAE,cAAc,CAAC;IAChC,aAAa,EAAE,YAAY,CAAC;IAC5B,UAAU,EAAE,SAAS,CAAC;IACtB,UAAU,CAAC,EAAE,SAAS,CAAC;CACxB;AAED,gDAAgD;AAChD,MAAM,WAAW,cAAc;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,YAAY,CAAC;IAC5B,UAAU,CAAC,EAAE,SAAS,CAAC;IACvB,eAAe,CAAC,EAAE,cAAc,CAAC;CAClC;AAED,MAAM,MAAM,yCAAyC,GAAG,CAAC,EACvD,IAAI,EACJ,KAAK,GACN,EAAE,uCAAuC,KAAK,OAAO,CAAC,yCAAyC,CAAC,CAAC;AAElG,MAAM,WAAW,uCAAuC;IACtD,IAAI,EAAE,oBAAoB,CAAC;IAC3B,KAAK,EAAE,YAAY,CAAC;CACrB;AAED,MAAM,WAAW,kBAAkB;IAEjC,IAAI,EAAE,GAAG,CAAC;IAEV,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAC9B;AAED,gEAAgE;AAChE,MAAM,MAAM,yCAAyC,GACjD;IAAE,MAAM,EAAE,SAAS,CAAC;IAAC,UAAU,EAAE,kBAAkB,CAAA;CAAE,GACrD;IAAE,MAAM,EAAE,OAAO,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,GACzC;IAAE,MAAM,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,WAAW,CAAA;CAAE,CAAC;AAE5C;;;;GAIG;AACH,MAAM,MAAM,2BAA2B,GACnC;IAAE,MAAM,EAAE,SAAS,CAAA;CAAE,GACrB;IAAE,MAAM,EAAE,OAAO,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,GACzC;IAAE,MAAM,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,WAAW,GAAG;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAA;CAAE,CAAC;AAEpE,MAAM,MAAM,uCAAuC,CACjD,KAAK,EACL,QAAQ,EACR,cAAc,IACZ,CAAC,EACH,WAAW,EACX,OAAO,EACP,SAAS,GACV,EAAE;IACD,WAAW,EAAE,KAAK,CAAC;IACnB,OAAO,EAAE,iBAAiB,CAAC,cAAc,CAAC,CAAC;IAC3C,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,KAAK,QAAQ,CAAC;AAEf,MAAM,MAAM,iCAAiC,GAAG,CAAC,EAC/C,UAAU,EACV,MAAM,GACP,EAAE;IACD,UAAU,EAAE,oBAAoB,CAAC;IACjC,MAAM,EAAE,yCAAyC,CAAC;CACnD,KAAK,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAE3C,MAAM,MAAM,wCAAwC,CAAC,QAAQ,EAAE,cAAc,IAC3E,CAAC,EACC,kBAAkB,EAClB,OAAO,EACP,MAAM,GACP,EAAE;IACD,kBAAkB,EAAE,QAAQ,CAAC;IAC7B,OAAO,EAAE,iBAAiB,CAAC,cAAc,CAAC,CAAC;IAC3C,MAAM,EAAE,yCAAyC,CAAC;CACnD,KAAK,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAE7C,MAAM,WAAW,kCAAkC,CACjD,cAAc,EACd,KAAK,EACL,QAAQ;IAER,OAAO,EAAE,uCAAuC,CAC9C,KAAK,EACL,QAAQ,EACR,cAAc,CACf,CAAC;IACF,QAAQ,EAAE,wCAAwC,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;CAC9E"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { AxiosResponse } from 'axios';
|
|
2
2
|
import { NormalizedAttachment } from '../repo/repo.interfaces';
|
|
3
3
|
import { HttpStreamResponse } from '../types/extraction';
|
|
4
|
-
import { ArtifactToUpload, UploaderFactoryInterface, UploaderResult, UploadResponse } from './uploader.interfaces';
|
|
4
|
+
import { ArtifactToUpload, MultipartUpload, UploaderFactoryInterface, UploaderResult, UploadResponse } from './uploader.interfaces';
|
|
5
5
|
export declare class Uploader {
|
|
6
6
|
private isLocalDevelopment?;
|
|
7
7
|
private devrevApiEndpoint;
|
|
@@ -17,6 +17,26 @@ export declare class Uploader {
|
|
|
17
17
|
uploadArtifact(artifact: ArtifactToUpload, file: Buffer): Promise<UploaderResult<AxiosResponse>>;
|
|
18
18
|
/** Streams a file from an HTTP response directly to the artifact's upload URL. */
|
|
19
19
|
streamArtifact(artifact: ArtifactToUpload, fileStream: HttpStreamResponse): Promise<UploaderResult<AxiosResponse>>;
|
|
20
|
+
/**
|
|
21
|
+
* Creates an S3 multipart upload for an artifact whose size is not known.
|
|
22
|
+
* The returned URL is for part 1; subsequent URLs are obtained with
|
|
23
|
+
* getNextMultipartUploadPart.
|
|
24
|
+
*/
|
|
25
|
+
createMultipartUpload(filename: string, fileType: string, partSize?: number): Promise<UploaderResult<MultipartUpload>>;
|
|
26
|
+
/** Gets a presigned URL for the next multipart upload part. */
|
|
27
|
+
getNextMultipartUploadPart(artifactId: string, cursor: string, partSize?: number): Promise<UploaderResult<MultipartUpload>>;
|
|
28
|
+
/** Finalizes and publishes an S3 multipart artifact upload. */
|
|
29
|
+
completeMultipartUpload(artifactId: string, cursor: string): Promise<UploaderResult<AxiosResponse>>;
|
|
30
|
+
/**
|
|
31
|
+
* Streams an attachment of unknown size through an S3 multipart upload.
|
|
32
|
+
* At most one 10 MiB part is buffered at a time. The final part may be
|
|
33
|
+
* smaller than S3's 5 MiB minimum.
|
|
34
|
+
*/
|
|
35
|
+
streamArtifactMultipart(filename: string, fileType: string, fileStream: HttpStreamResponse): Promise<UploaderResult<{
|
|
36
|
+
artifact_id: string;
|
|
37
|
+
}>>;
|
|
38
|
+
/** Uploads one raw part to its presigned S3 URL. */
|
|
39
|
+
private uploadMultipartPart;
|
|
20
40
|
confirmArtifactUpload(artifactId: string): Promise<{
|
|
21
41
|
response?: AxiosResponse;
|
|
22
42
|
error?: unknown;
|