@devrev/airsync-sdk 2.0.0-beta.8 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/attachments-streaming/attachments-streaming-pool.js +2 -2
- package/dist/http/client.d.ts +2 -1
- package/dist/http/client.d.ts.map +1 -1
- package/dist/http/client.js +71 -60
- package/dist/logger/logger.d.ts.map +1 -1
- package/dist/logger/logger.interfaces.d.ts +0 -2
- package/dist/logger/logger.interfaces.d.ts.map +1 -1
- package/dist/logger/logger.js +0 -1
- package/dist/multithreading/adapters/base-adapter.d.ts +4 -0
- package/dist/multithreading/adapters/base-adapter.d.ts.map +1 -1
- package/dist/multithreading/adapters/base-adapter.js +6 -0
- package/dist/multithreading/adapters/loading-adapter.d.ts +2 -4
- package/dist/multithreading/adapters/loading-adapter.d.ts.map +1 -1
- package/dist/multithreading/adapters/loading-adapter.js +38 -21
- package/dist/multithreading/process-task.d.ts.map +1 -1
- package/dist/multithreading/process-task.js +18 -16
- package/dist/state/base-state.d.ts +2 -2
- package/dist/state/base-state.d.ts.map +1 -1
- package/dist/state/base-state.js +5 -8
- package/dist/state/extraction-state.d.ts.map +1 -1
- package/dist/state/extraction-state.js +10 -4
- package/dist/types/errors.d.ts +3 -3
- package/dist/types/errors.js +3 -3
- package/dist/types/extraction.d.ts +0 -4
- package/dist/types/extraction.d.ts.map +1 -1
- package/docs/MIGRATION.md +168 -52
- package/package.json +2 -2
|
@@ -73,7 +73,7 @@ class AttachmentsStreamingPool {
|
|
|
73
73
|
Promise.all(initialPromises),
|
|
74
74
|
this.adapter.timeoutSignal,
|
|
75
75
|
]);
|
|
76
|
-
if (this.delay) {
|
|
76
|
+
if (this.delay !== undefined) {
|
|
77
77
|
return { status: 'delay', delaySeconds: this.delay };
|
|
78
78
|
}
|
|
79
79
|
return { status: 'success' };
|
|
@@ -81,7 +81,7 @@ class AttachmentsStreamingPool {
|
|
|
81
81
|
async startPoolStreaming() {
|
|
82
82
|
var _a;
|
|
83
83
|
while (this.attachments.length > 0) {
|
|
84
|
-
if (this.delay) {
|
|
84
|
+
if (this.delay !== undefined) {
|
|
85
85
|
break;
|
|
86
86
|
}
|
|
87
87
|
if (this.adapter.isTimeout) {
|
package/dist/http/client.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../src/http/client.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../src/http/client.ts"],"names":[],"mappings":"AAAA,OAAc,EAAc,aAAa,EAAE,MAAM,OAAO,CAAC;AAmGzD,QAAA,MAAM,WAAW,EAAE,IAAI,CAAC,aAAa,EAAE,KAAK,GAAG,MAAM,CAGpD,CAAC;AAEF,OAAO,EAAE,WAAW,EAAE,CAAC"}
|
package/dist/http/client.js
CHANGED
|
@@ -6,64 +6,75 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
6
6
|
exports.axiosClient = void 0;
|
|
7
7
|
const axios_1 = __importDefault(require("axios"));
|
|
8
8
|
const axios_retry_1 = __importDefault(require("axios-retry"));
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
9
|
+
function createAxiosClient() {
|
|
10
|
+
const client = axios_1.default.create({
|
|
11
|
+
timeout: 30 * 1000,
|
|
12
|
+
});
|
|
13
|
+
(0, axios_retry_1.default)(client, {
|
|
14
|
+
retries: 5,
|
|
15
|
+
shouldResetTimeout: true,
|
|
16
|
+
retryDelay: (retryCount, error) => {
|
|
17
|
+
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o;
|
|
18
|
+
let delay;
|
|
19
|
+
if (((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) === 429) {
|
|
20
|
+
const retryAfter = ((_c = (_b = error.response) === null || _b === void 0 ? void 0 : _b.headers) === null || _c === void 0 ? void 0 : _c['retry-after']) ||
|
|
21
|
+
((_e = (_d = error.response) === null || _d === void 0 ? void 0 : _d.headers) === null || _e === void 0 ? void 0 : _e['Retry-After']);
|
|
22
|
+
delay = parseInt(retryAfter, 10) * 1000;
|
|
23
|
+
}
|
|
24
|
+
else {
|
|
25
|
+
delay = axios_retry_1.default.exponentialDelay(retryCount, error, 1000);
|
|
26
|
+
}
|
|
27
|
+
const requestId = ((_g = (_f = error.config) === null || _f === void 0 ? void 0 : _f.headers) === null || _g === void 0 ? void 0 : _g['x-request-id']) ||
|
|
28
|
+
((_j = (_h = error.config) === null || _h === void 0 ? void 0 : _h.headers) === null || _j === void 0 ? void 0 : _j['X-Request-ID']);
|
|
29
|
+
const delayInSeconds = Math.round(delay / 1000);
|
|
30
|
+
console.warn(`Retrying request to ${(_k = error.config) === null || _k === void 0 ? void 0 : _k.url} in ${delayInSeconds}s due to ${(_m = (_l = error.response) === null || _l === void 0 ? void 0 : _l.status) !== null && _m !== void 0 ? _m : 'unknown'} error.`, Object.assign(Object.assign({ method: (_o = error.config) === null || _o === void 0 ? void 0 : _o.method }, (retryCount && { retryCount })), (requestId && { requestId })));
|
|
31
|
+
return delay;
|
|
32
|
+
},
|
|
33
|
+
retryCondition: (error) => {
|
|
34
|
+
var _a, _b, _c, _d, _e, _f, _g;
|
|
35
|
+
const retryAfter = ((_b = (_a = error.response) === null || _a === void 0 ? void 0 : _a.headers) === null || _b === void 0 ? void 0 : _b['retry-after']) ||
|
|
36
|
+
((_d = (_c = error.response) === null || _c === void 0 ? void 0 : _c.headers) === null || _d === void 0 ? void 0 : _d['Retry-After']);
|
|
37
|
+
if (((_e = error.response) === null || _e === void 0 ? void 0 : _e.status) && error.response.status >= 500) {
|
|
38
|
+
return true;
|
|
39
|
+
}
|
|
40
|
+
// 429 only when a valid non-negative Retry-After header is present
|
|
41
|
+
else if (((_f = error.response) === null || _f === void 0 ? void 0 : _f.status) &&
|
|
42
|
+
error.response.status === 429 &&
|
|
43
|
+
retryAfter &&
|
|
44
|
+
!isNaN(Number(retryAfter)) &&
|
|
45
|
+
Number(retryAfter) >= 0) {
|
|
46
|
+
return true;
|
|
47
|
+
}
|
|
48
|
+
// Network errors for idempotent requests; 429 already handled above
|
|
49
|
+
else if (axios_retry_1.default.isNetworkOrIdempotentRequestError(error) &&
|
|
50
|
+
((_g = error.response) === null || _g === void 0 ? void 0 : _g.status) !== 429) {
|
|
51
|
+
return true;
|
|
52
|
+
}
|
|
53
|
+
// axios-retry excludes ECONNABORTED (client-side timeout only, never a
|
|
54
|
+
// server response) from isNetworkError, so handle it here separately.
|
|
55
|
+
else if (error.code === 'ECONNABORTED') {
|
|
56
|
+
return true;
|
|
57
|
+
}
|
|
58
|
+
else {
|
|
59
|
+
return false;
|
|
60
|
+
}
|
|
61
|
+
},
|
|
62
|
+
onMaxRetryTimesExceeded(error) {
|
|
63
|
+
var _a, _b, _c, _d, _e, _f;
|
|
64
|
+
(_b = (_a = error.config) === null || _a === void 0 ? void 0 : _a.headers) === null || _b === void 0 ? true : delete _b.authorization;
|
|
65
|
+
(_d = (_c = error.config) === null || _c === void 0 ? void 0 : _c.headers) === null || _d === void 0 ? true : delete _d.Authorization;
|
|
66
|
+
(_e = error.request) === null || _e === void 0 ? true : delete _e._header;
|
|
67
|
+
console.error(`Request to ${(_f = error.config) === null || _f === void 0 ? void 0 : _f.url} failed after max retries. Error`, error);
|
|
68
|
+
},
|
|
69
|
+
});
|
|
70
|
+
return client;
|
|
71
|
+
}
|
|
72
|
+
let instance;
|
|
73
|
+
const getInstance = () => (instance !== null && instance !== void 0 ? instance : (instance = createAxiosClient()));
|
|
74
|
+
// Created on first use so that loading the SDK never calls axios.create(),
|
|
75
|
+
// which breaks connector tests that mock axios.
|
|
76
|
+
const axiosClient = {
|
|
77
|
+
get: async (...args) => getInstance().get(...args),
|
|
78
|
+
post: async (...args) => getInstance().post(...args),
|
|
79
|
+
};
|
|
12
80
|
exports.axiosClient = axiosClient;
|
|
13
|
-
(0, axios_retry_1.default)(axiosClient, {
|
|
14
|
-
retries: 5,
|
|
15
|
-
shouldResetTimeout: true,
|
|
16
|
-
retryDelay: (retryCount, error) => {
|
|
17
|
-
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o;
|
|
18
|
-
let delay;
|
|
19
|
-
if (((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) === 429) {
|
|
20
|
-
const retryAfter = ((_c = (_b = error.response) === null || _b === void 0 ? void 0 : _b.headers) === null || _c === void 0 ? void 0 : _c['retry-after']) ||
|
|
21
|
-
((_e = (_d = error.response) === null || _d === void 0 ? void 0 : _d.headers) === null || _e === void 0 ? void 0 : _e['Retry-After']);
|
|
22
|
-
delay = parseInt(retryAfter, 10) * 1000;
|
|
23
|
-
}
|
|
24
|
-
else {
|
|
25
|
-
delay = axios_retry_1.default.exponentialDelay(retryCount, error, 1000);
|
|
26
|
-
}
|
|
27
|
-
const requestId = ((_g = (_f = error.config) === null || _f === void 0 ? void 0 : _f.headers) === null || _g === void 0 ? void 0 : _g['x-request-id']) ||
|
|
28
|
-
((_j = (_h = error.config) === null || _h === void 0 ? void 0 : _h.headers) === null || _j === void 0 ? void 0 : _j['X-Request-ID']);
|
|
29
|
-
const delayInSeconds = Math.round(delay / 1000);
|
|
30
|
-
console.warn(`Retrying request to ${(_k = error.config) === null || _k === void 0 ? void 0 : _k.url} in ${delayInSeconds}s due to ${(_m = (_l = error.response) === null || _l === void 0 ? void 0 : _l.status) !== null && _m !== void 0 ? _m : 'unknown'} error.`, Object.assign(Object.assign({ method: (_o = error.config) === null || _o === void 0 ? void 0 : _o.method }, (retryCount && { retryCount })), (requestId && { requestId })));
|
|
31
|
-
return delay;
|
|
32
|
-
},
|
|
33
|
-
retryCondition: (error) => {
|
|
34
|
-
var _a, _b, _c, _d, _e, _f, _g;
|
|
35
|
-
const retryAfter = ((_b = (_a = error.response) === null || _a === void 0 ? void 0 : _a.headers) === null || _b === void 0 ? void 0 : _b['retry-after']) ||
|
|
36
|
-
((_d = (_c = error.response) === null || _c === void 0 ? void 0 : _c.headers) === null || _d === void 0 ? void 0 : _d['Retry-After']);
|
|
37
|
-
if (((_e = error.response) === null || _e === void 0 ? void 0 : _e.status) && error.response.status >= 500) {
|
|
38
|
-
return true;
|
|
39
|
-
}
|
|
40
|
-
// 429 only when a valid non-negative Retry-After header is present
|
|
41
|
-
else if (((_f = error.response) === null || _f === void 0 ? void 0 : _f.status) &&
|
|
42
|
-
error.response.status === 429 &&
|
|
43
|
-
retryAfter &&
|
|
44
|
-
!isNaN(Number(retryAfter)) &&
|
|
45
|
-
Number(retryAfter) >= 0) {
|
|
46
|
-
return true;
|
|
47
|
-
}
|
|
48
|
-
// Network errors for idempotent requests; 429 already handled above
|
|
49
|
-
else if (axios_retry_1.default.isNetworkOrIdempotentRequestError(error) &&
|
|
50
|
-
((_g = error.response) === null || _g === void 0 ? void 0 : _g.status) !== 429) {
|
|
51
|
-
return true;
|
|
52
|
-
}
|
|
53
|
-
// axios-retry excludes ECONNABORTED (client-side timeout only, never a
|
|
54
|
-
// server response) from isNetworkError, so handle it here separately.
|
|
55
|
-
else if (error.code === 'ECONNABORTED') {
|
|
56
|
-
return true;
|
|
57
|
-
}
|
|
58
|
-
else {
|
|
59
|
-
return false;
|
|
60
|
-
}
|
|
61
|
-
},
|
|
62
|
-
onMaxRetryTimesExceeded(error) {
|
|
63
|
-
var _a, _b, _c, _d, _e, _f;
|
|
64
|
-
(_b = (_a = error.config) === null || _a === void 0 ? void 0 : _a.headers) === null || _b === void 0 ? true : delete _b.authorization;
|
|
65
|
-
(_d = (_c = error.config) === null || _c === void 0 ? void 0 : _c.headers) === null || _d === void 0 ? true : delete _d.Authorization;
|
|
66
|
-
(_e = error.request) === null || _e === void 0 ? true : delete _e._header;
|
|
67
|
-
console.error(`Request to ${(_f = error.config) === null || _f === void 0 ? void 0 : _f.url} failed after max retries. Error`, error);
|
|
68
|
-
},
|
|
69
|
-
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../../src/logger/logger.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAW,cAAc,EAAE,MAAM,WAAW,CAAC;AAGpD,OAAO,EAAE,UAAU,
|
|
1
|
+
{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../../src/logger/logger.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAW,cAAc,EAAE,MAAM,WAAW,CAAC;AAGpD,OAAO,EAAE,UAAU,EAAgB,MAAM,OAAO,CAAC;AAKjD,OAAO,EACL,kBAAkB,EAClB,sBAAsB,EAEtB,QAAQ,EAER,cAAc,EACf,MAAM,qBAAqB,CAAC;AAI7B,eAAO,MAAM,qBAAqB,QAAQ,CAAC;AAI3C,eAAO,MAAM,eAAe,EAAE,cAM7B,CAAC;AAEF,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAOvD;AAED;;;;GAIG;AACH,qBAAa,MAAO,SAAQ,OAAO;IACjC,OAAO,CAAC,eAAe,CAAU;IACjC,OAAO,CAAC,OAAO,CAAC,CAAiB;IACjC,OAAO,CAAC,IAAI,CAAa;gBAEb,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,sBAAsB;IAUtD,OAAO,CAAC,aAAa;IAOrB,wGAAwG;IACxG,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,GAAG,IAAI;IAa7C,OAAO,CAAC,eAAe;IAcd,GAAG,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI;IAI7B,IAAI,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI;IAI9B,IAAI,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI;IAI9B,KAAK,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI;CAGzC;AAED,8FAA8F;AAE9F,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,cAAc,CAqB5E;AAED,kFAAkF;AAClF,wBAAgB,cAAc,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAmCrD;AAED,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,UAAU,GAAG,kBAAkB,CAuBzE"}
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import type { RawAxiosResponseHeaders } from 'axios';
|
|
2
1
|
import type { AirSyncEvent, EventContext } from '../types/extraction';
|
|
3
2
|
import type { RuntimeOptions } from '../types/workers';
|
|
4
3
|
export interface LoggerFactoryInterface {
|
|
@@ -29,7 +28,6 @@ export interface AxiosErrorResponse {
|
|
|
29
28
|
isCorsOrNoNetworkError: boolean;
|
|
30
29
|
response?: {
|
|
31
30
|
data: unknown;
|
|
32
|
-
headers: RawAxiosResponseHeaders;
|
|
33
31
|
status: number;
|
|
34
32
|
statusText: string;
|
|
35
33
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"logger.interfaces.d.ts","sourceRoot":"","sources":["../../src/logger/logger.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,
|
|
1
|
+
{"version":3,"file":"logger.interfaces.d.ts","sourceRoot":"","sources":["../../src/logger/logger.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACtE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAEvD,MAAM,WAAW,sBAAsB;IACrC,KAAK,EAAE,YAAY,CAAC;IACpB,OAAO,CAAC,EAAE,cAAc,CAAC;CAC1B;AAED,oBAAY,QAAQ;IAClB,IAAI,SAAS;IACb,IAAI,SAAS;IACb,KAAK,UAAU;CAChB;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,OAAO,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IAEf,SAAS,CAAC,EAAE,GAAG,CAAC;IAEhB,QAAQ,CAAC,EAAE,GAAG,CAAC;CAChB;AAED,MAAM,WAAW,cAAc;IAE7B,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,GAAG,cAAc,GAAG,cAAc,CAAC;CACtD;AAED,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE;QACN,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;QAE3B,MAAM,EAAE,GAAG,CAAC;QACZ,GAAG,EAAE,MAAM,GAAG,SAAS,CAAC;KACzB,CAAC;IACF,YAAY,EAAE,OAAO,CAAC;IACtB,sBAAsB,EAAE,OAAO,CAAC;IAChC,QAAQ,CAAC,EAAE;QACT,IAAI,EAAE,OAAO,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,UAAW,SAAQ,YAAY;IAC9C,WAAW,EAAE,MAAM,CAAC;CACrB"}
|
package/dist/logger/logger.js
CHANGED
|
@@ -159,7 +159,6 @@ function serializeAxiosError(error) {
|
|
|
159
159
|
if (error.response) {
|
|
160
160
|
serializedAxiosError.response = {
|
|
161
161
|
data: error.response.data,
|
|
162
|
-
headers: error.response.headers,
|
|
163
162
|
status: error.response.status,
|
|
164
163
|
statusText: error.response.statusText,
|
|
165
164
|
};
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { Mappers } from '../../mappers/mappers';
|
|
1
2
|
import { AirSyncEvent } from '../../types/extraction';
|
|
2
3
|
/**
|
|
3
4
|
* Shared state/behavior for both sync modes; owns the `emit` control-protocol
|
|
@@ -8,6 +9,9 @@ export declare abstract class BaseAdapter<ConnectorState> {
|
|
|
8
9
|
private _hasWorkerEmitted;
|
|
9
10
|
private _isTimeout;
|
|
10
11
|
private resolveTimeoutSignal;
|
|
12
|
+
private _mappers;
|
|
13
|
+
/** Sync mapper records for this event's sync unit, in both extraction and loading. */
|
|
14
|
+
get mappers(): Mappers;
|
|
11
15
|
/** Set once the soft timeout fires; the task should wrap up and return. */
|
|
12
16
|
get isTimeout(): boolean;
|
|
13
17
|
/** Connector-owned state exposed to snap-in code. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"base-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/base-adapter.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"base-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/base-adapter.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAKhD,OAAO,EACL,YAAY,EAIb,MAAM,wBAAwB,CAAC;AAYhC;;;GAGG;AACH,8BAAsB,WAAW,CAAC,cAAc;IAC9C,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IAI7B,OAAO,CAAC,iBAAiB,CAAkB;IAC3C,OAAO,CAAC,UAAU,CAAkB;IACpC,OAAO,CAAC,oBAAoB,CAAc;IAU1C,OAAO,CAAC,QAAQ,CAAU;IAsB1B,sFAAsF;IACtF,IAAI,OAAO,IAAI,OAAO,CAErB;IAOD,2EAA2E;IAC3E,IAAI,SAAS,IAAI,OAAO,CAEvB;IAUD,qDAAqD;IACrD,IAAI,KAAK,IAAI,cAAc,CAE1B;IAED,IAAI,KAAK,CAAC,KAAK,EAAE,cAAc,EAE9B;IAOD,IAAI,eAAe,kDAElB;IAED;;;;OAIG;IACG,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IAIhC;;;;OAIG;IACG,cAAc,CAClB,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,MAAM,EAAE,GACb,OAAO,CAAC;QAAE,EAAE,EAAE,MAAM,CAAA;KAAE,CAAC;IAY1B,kFAAkF;IAC5E,gBAAgB,CAAC,CAAC,GAAG,OAAO,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC;YAmIvD,IAAI;CAmCnB"}
|
|
@@ -5,6 +5,7 @@ const node_worker_threads_1 = require("node:worker_threads");
|
|
|
5
5
|
const constants_1 = require("../../common/constants");
|
|
6
6
|
const errors_1 = require("../../http/errors");
|
|
7
7
|
const logger_1 = require("../../logger/logger");
|
|
8
|
+
const mappers_1 = require("../../mappers/mappers");
|
|
8
9
|
const errors_2 = require("../../types/errors");
|
|
9
10
|
const workers_1 = require("../../types/workers");
|
|
10
11
|
const uploader_1 = require("../../uploader/uploader");
|
|
@@ -30,6 +31,11 @@ class BaseAdapter {
|
|
|
30
31
|
event,
|
|
31
32
|
options,
|
|
32
33
|
});
|
|
34
|
+
this._mappers = new mappers_1.Mappers({ event });
|
|
35
|
+
}
|
|
36
|
+
/** Sync mapper records for this event's sync unit, in both extraction and loading. */
|
|
37
|
+
get mappers() {
|
|
38
|
+
return this._mappers;
|
|
33
39
|
}
|
|
34
40
|
/** @internal */
|
|
35
41
|
get hasWorkerEmitted() {
|
|
@@ -1,15 +1,12 @@
|
|
|
1
|
-
import { Mappers } from '../../mappers/mappers';
|
|
2
1
|
import { ExternalSystemAttachment, ExternalSystemLoadingFunction, ItemTypesToLoadParams, LoaderReport } from '../../types/loading';
|
|
3
2
|
import { TaskResult } from '../../types/workers';
|
|
4
3
|
import { BaseAdapter } from './base-adapter';
|
|
5
|
-
/** Adapter passed to loading tasks (item/attachment loading,
|
|
4
|
+
/** Adapter passed to loading tasks (item/attachment loading, loader reports). @public */
|
|
6
5
|
export declare class LoadingAdapter<ConnectorState> extends BaseAdapter<ConnectorState> {
|
|
7
6
|
private loaderReports;
|
|
8
7
|
private _processedFiles;
|
|
9
|
-
private _mappers;
|
|
10
8
|
get reports(): LoaderReport[];
|
|
11
9
|
get processedFiles(): string[];
|
|
12
|
-
get mappers(): Mappers;
|
|
13
10
|
loadItemTypes({ itemTypesToLoad, }: ItemTypesToLoadParams): Promise<TaskResult>;
|
|
14
11
|
loadAttachments({ create, }: {
|
|
15
12
|
create: ExternalSystemLoadingFunction<ExternalSystemAttachment>;
|
|
@@ -20,6 +17,7 @@ export declare class LoadingAdapter<ConnectorState> extends BaseAdapter<Connecto
|
|
|
20
17
|
* an item type the connector can't load, which fails the phase.
|
|
21
18
|
*/
|
|
22
19
|
private loadFiles;
|
|
20
|
+
private initializeFilesToLoad;
|
|
23
21
|
private getLoaderBatches;
|
|
24
22
|
private loadItem;
|
|
25
23
|
/** Maps a non-success connector response to a delay or a failed report. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"loading-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/loading-adapter.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"loading-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/loading-adapter.ts"],"names":[],"mappings":"AASA,OAAO,EAEL,wBAAwB,EAGxB,6BAA6B,EAE7B,qBAAqB,EAGrB,YAAY,EAGb,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAkB,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjE,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAM7C,yFAAyF;AACzF,qBAAa,cAAc,CACzB,cAAc,CACd,SAAQ,WAAW,CAAC,cAAc,CAAC;IACnC,OAAO,CAAC,aAAa,CAAiB;IACtC,OAAO,CAAC,eAAe,CAAW;IAalC,IAAI,OAAO,IAAI,YAAY,EAAE,CAE5B;IAED,IAAI,cAAc,IAAI,MAAM,EAAE,CAE7B;IA2BK,aAAa,CAAC,EAClB,eAAe,GAChB,EAAE,qBAAqB,GAAG,OAAO,CAAC,UAAU,CAAC;IA6BxC,eAAe,CAAC,EACpB,MAAM,GACP,EAAE;QACD,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,UAAU,CAAC;IAYvB;;;;OAIG;YACW,SAAS;YAqFT,qBAAqB;YAarB,gBAAgB;YA+BhB,QAAQ;IA+HtB,2EAA2E;IAC3E,OAAO,CAAC,YAAY;YAwBN,cAAc;CAiD7B"}
|
|
@@ -2,20 +2,18 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.LoadingAdapter = void 0;
|
|
4
4
|
const logger_1 = require("../../logger/logger");
|
|
5
|
-
const mappers_1 = require("../../mappers/mappers");
|
|
6
5
|
const mappers_interfaces_1 = require("../../mappers/mappers.interfaces");
|
|
7
6
|
const extraction_1 = require("../../types/extraction");
|
|
8
7
|
const loading_1 = require("../../types/loading");
|
|
9
8
|
const base_adapter_1 = require("./base-adapter");
|
|
10
9
|
const loading_adapter_helpers_1 = require("./loading-adapter.helpers");
|
|
11
|
-
/** Adapter passed to loading tasks (item/attachment loading,
|
|
10
|
+
/** Adapter passed to loading tasks (item/attachment loading, loader reports). @public */
|
|
12
11
|
class LoadingAdapter extends base_adapter_1.BaseAdapter {
|
|
13
12
|
/** @internal */
|
|
14
13
|
constructor(params) {
|
|
15
14
|
super(params);
|
|
16
15
|
this.loaderReports = [];
|
|
17
16
|
this._processedFiles = [];
|
|
18
|
-
this._mappers = new mappers_1.Mappers({ event: params.event });
|
|
19
17
|
}
|
|
20
18
|
get reports() {
|
|
21
19
|
return this.loaderReports;
|
|
@@ -23,9 +21,6 @@ class LoadingAdapter extends base_adapter_1.BaseAdapter {
|
|
|
23
21
|
get processedFiles() {
|
|
24
22
|
return this._processedFiles;
|
|
25
23
|
}
|
|
26
|
-
get mappers() {
|
|
27
|
-
return this._mappers;
|
|
28
|
-
}
|
|
29
24
|
/** @internal */
|
|
30
25
|
async beforeEmit() {
|
|
31
26
|
// Loading has no pre-emit work.
|
|
@@ -51,9 +46,9 @@ class LoadingAdapter extends base_adapter_1.BaseAdapter {
|
|
|
51
46
|
console.warn('No item types to load, returning.');
|
|
52
47
|
return { status: 'success' };
|
|
53
48
|
}
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
49
|
+
const failure = await this.initializeFilesToLoad(itemTypes);
|
|
50
|
+
if (failure)
|
|
51
|
+
return failure;
|
|
57
52
|
}
|
|
58
53
|
return this.loadFiles('data', (fileToLoad) => {
|
|
59
54
|
const itemTypeToLoad = itemTypesToLoad.find((itemTypeToLoad) => itemTypeToLoad.itemType === fileToLoad.itemType);
|
|
@@ -65,9 +60,9 @@ class LoadingAdapter extends base_adapter_1.BaseAdapter {
|
|
|
65
60
|
}
|
|
66
61
|
async loadAttachments({ create, }) {
|
|
67
62
|
if (this.event.payload.event_type === extraction_1.EventType.StartLoadingAttachments) {
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
63
|
+
const failure = await this.initializeFilesToLoad(['attachment']);
|
|
64
|
+
if (failure)
|
|
65
|
+
return failure;
|
|
71
66
|
}
|
|
72
67
|
return this.loadFiles('attachment', () => async (item) => this.loadAttachment({ item, create }));
|
|
73
68
|
}
|
|
@@ -110,8 +105,11 @@ class LoadingAdapter extends base_adapter_1.BaseAdapter {
|
|
|
110
105
|
console.log(`Timeout detected during ${kind} loading. Returning progress to allow continuation.`);
|
|
111
106
|
return { status: 'progress' };
|
|
112
107
|
}
|
|
113
|
-
const { report, rateLimit } = await load(transformerFile[i]);
|
|
114
|
-
if (
|
|
108
|
+
const { report, rateLimit, error } = await load(transformerFile[i]);
|
|
109
|
+
if (error) {
|
|
110
|
+
return { status: 'error', error };
|
|
111
|
+
}
|
|
112
|
+
if (rateLimit) {
|
|
115
113
|
return { status: 'delay', delaySeconds: rateLimit.delay };
|
|
116
114
|
}
|
|
117
115
|
if (report) {
|
|
@@ -137,6 +135,17 @@ class LoadingAdapter extends base_adapter_1.BaseAdapter {
|
|
|
137
135
|
}
|
|
138
136
|
return { status: 'success' };
|
|
139
137
|
}
|
|
138
|
+
async initializeFilesToLoad(supportedItemTypes) {
|
|
139
|
+
try {
|
|
140
|
+
this.adapterState.sdkState.fromDevRev = {
|
|
141
|
+
filesToLoad: await this.getLoaderBatches(supportedItemTypes),
|
|
142
|
+
};
|
|
143
|
+
return undefined;
|
|
144
|
+
}
|
|
145
|
+
catch (error) {
|
|
146
|
+
return { status: 'error', error: { message: (0, logger_1.serializeError)(error) } };
|
|
147
|
+
}
|
|
148
|
+
}
|
|
140
149
|
async getLoaderBatches(supportedItemTypes) {
|
|
141
150
|
var _a;
|
|
142
151
|
const statsFileArtifactId = (_a = this.event.payload.event_data) === null || _a === void 0 ? void 0 : _a.stats_file;
|
|
@@ -147,7 +156,10 @@ class LoadingAdapter extends base_adapter_1.BaseAdapter {
|
|
|
147
156
|
artifactId: statsFileArtifactId,
|
|
148
157
|
});
|
|
149
158
|
const statsFile = response;
|
|
150
|
-
if (statsFileError
|
|
159
|
+
if (statsFileError) {
|
|
160
|
+
throw new Error(`Failed to download loader stats file ${statsFileArtifactId}. ${(0, logger_1.serializeError)(statsFileError)}`);
|
|
161
|
+
}
|
|
162
|
+
if (statsFile.length === 0) {
|
|
151
163
|
return [];
|
|
152
164
|
}
|
|
153
165
|
return (0, loading_adapter_helpers_1.getFilesToLoad)({ supportedItemTypes, statsFile });
|
|
@@ -157,7 +169,7 @@ class LoadingAdapter extends base_adapter_1.BaseAdapter {
|
|
|
157
169
|
const syncUnit = this.event.payload.event_context.sync_unit;
|
|
158
170
|
let syncMapperRecord;
|
|
159
171
|
try {
|
|
160
|
-
syncMapperRecord = await this.
|
|
172
|
+
syncMapperRecord = await this.mappers.getByTargetId({
|
|
161
173
|
sync_unit: syncUnit,
|
|
162
174
|
target: devrevId,
|
|
163
175
|
});
|
|
@@ -170,14 +182,14 @@ class LoadingAdapter extends base_adapter_1.BaseAdapter {
|
|
|
170
182
|
},
|
|
171
183
|
};
|
|
172
184
|
}
|
|
173
|
-
const loadParams = { item, mappers: this.
|
|
185
|
+
const loadParams = { item, mappers: this.mappers, event: this.event };
|
|
174
186
|
if (syncMapperRecord) {
|
|
175
187
|
const response = await itemTypeToLoad.update(loadParams);
|
|
176
188
|
if (response.status !== 'success') {
|
|
177
189
|
return this.toFailedLoad(itemTypeToLoad.itemType, 'update', response);
|
|
178
190
|
}
|
|
179
191
|
try {
|
|
180
|
-
const syncMapperRecordUpdate = await this.
|
|
192
|
+
const syncMapperRecordUpdate = await this.mappers.update(Object.assign(Object.assign({ id: syncMapperRecord.sync_mapper_record.id, sync_unit: syncUnit, status: mappers_interfaces_1.SyncMapperRecordStatus.OPERATIONAL }, (response.modifiedDate && {
|
|
181
193
|
external_versions: {
|
|
182
194
|
add: [
|
|
183
195
|
{
|
|
@@ -213,7 +225,7 @@ class LoadingAdapter extends base_adapter_1.BaseAdapter {
|
|
|
213
225
|
return this.toFailedLoad(itemTypeToLoad.itemType, 'create', response);
|
|
214
226
|
}
|
|
215
227
|
try {
|
|
216
|
-
const syncMapperRecordCreate = await this.
|
|
228
|
+
const syncMapperRecordCreate = await this.mappers.create(Object.assign({ sync_unit: syncUnit, status: mappers_interfaces_1.SyncMapperRecordStatus.OPERATIONAL, external_ids: [response.id], targets: [devrevId] }, (response.modifiedDate && {
|
|
217
229
|
external_versions: [
|
|
218
230
|
{
|
|
219
231
|
modified_date: response.modifiedDate,
|
|
@@ -255,14 +267,14 @@ class LoadingAdapter extends base_adapter_1.BaseAdapter {
|
|
|
255
267
|
async loadAttachment({ item, create, }) {
|
|
256
268
|
const response = await create({
|
|
257
269
|
item,
|
|
258
|
-
mappers: this.
|
|
270
|
+
mappers: this.mappers,
|
|
259
271
|
event: this.event,
|
|
260
272
|
});
|
|
261
273
|
if (response.status !== 'success') {
|
|
262
274
|
return this.toFailedLoad('attachments', 'create', response);
|
|
263
275
|
}
|
|
264
276
|
try {
|
|
265
|
-
const syncMapperRecordCreate = await this.
|
|
277
|
+
const syncMapperRecordCreate = await this.mappers.create({
|
|
266
278
|
sync_unit: this.event.payload.event_context.sync_unit,
|
|
267
279
|
external_ids: [response.id],
|
|
268
280
|
targets: [item.reference_id],
|
|
@@ -272,6 +284,11 @@ class LoadingAdapter extends base_adapter_1.BaseAdapter {
|
|
|
272
284
|
}
|
|
273
285
|
catch (error) {
|
|
274
286
|
console.warn('Failed to create sync mapper record.', (0, logger_1.serializeError)(error));
|
|
287
|
+
return {
|
|
288
|
+
error: {
|
|
289
|
+
message: 'Failed to create sync mapper record. ' + (0, logger_1.serializeError)(error),
|
|
290
|
+
},
|
|
291
|
+
};
|
|
275
292
|
}
|
|
276
293
|
return {
|
|
277
294
|
report: {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"process-task.d.ts","sourceRoot":"","sources":["../../src/multithreading/process-task.ts"],"names":[],"mappings":"AAKA,OAAO,EACL,oBAAoB,EAIrB,MAAM,kBAAkB,CAAC;AAG1B,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;
|
|
1
|
+
{"version":3,"file":"process-task.d.ts","sourceRoot":"","sources":["../../src/multithreading/process-task.ts"],"names":[],"mappings":"AAKA,OAAO,EACL,oBAAoB,EAIrB,MAAM,kBAAkB,CAAC;AAG1B,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAyF5D,oDAAoD;AACpD,wBAAgB,qBAAqB,CAAC,cAAc,EAAE,EACpD,IAAI,EACJ,SAAS,GACV,EAAE,oBAAoB,CAAC,iBAAiB,CAAC,cAAc,CAAC,CAAC,QA6BzD;AAED,gDAAgD;AAChD,wBAAgB,kBAAkB,CAAC,cAAc,EAAE,EACjD,IAAI,EACJ,SAAS,GACV,EAAE,oBAAoB,CAAC,cAAc,CAAC,cAAc,CAAC,CAAC,QA6BtD"}
|
|
@@ -13,9 +13,10 @@ const spawn_helpers_1 = require("./spawn/spawn.helpers");
|
|
|
13
13
|
/**
|
|
14
14
|
* Shared worker-thread driver: builds the adapter, runs the task, maps the
|
|
15
15
|
* returned {@link TaskResult} to a platform event and emits it exactly once.
|
|
16
|
-
* On soft timeout a returned `
|
|
17
|
-
*
|
|
18
|
-
* `delay`
|
|
16
|
+
* On soft timeout a returned `progress` is replaced by the `onTimeout` result
|
|
17
|
+
* (or a phase-appropriate default), and so is a `success` in resumable phases.
|
|
18
|
+
* A returned `error` or `delay`, or a `success` in a non-resumable phase, is
|
|
19
|
+
* emitted as is.
|
|
19
20
|
*/
|
|
20
21
|
async function runWorkerTask(buildAdapter, { task, onTimeout }) {
|
|
21
22
|
let adapter;
|
|
@@ -31,25 +32,26 @@ async function runWorkerTask(buildAdapter, { task, onTimeout }) {
|
|
|
31
32
|
});
|
|
32
33
|
console.log('Event passed from SDK to the connector:', adapter.event);
|
|
33
34
|
let result = await task({ adapter });
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
35
|
+
if (adapter.isTimeout) {
|
|
36
|
+
// A returned error or delay is a real outcome and must reach the
|
|
37
|
+
// platform. A success is replaced only where the phase can hand off
|
|
38
|
+
// with progress; a non-resumable phase that finished late is still done.
|
|
39
|
+
const eventType = adapter.event.payload.event_type;
|
|
40
|
+
const isResumable = !(0, spawn_helpers_1.getEventTypeForResult)(eventType, 'progress').illegal;
|
|
41
|
+
const canBeReplaced = result.status === 'progress' ||
|
|
42
|
+
(result.status === 'success' && isResumable);
|
|
43
|
+
if (canBeReplaced && onTimeout) {
|
|
39
44
|
result = await onTimeout({ adapter });
|
|
40
45
|
}
|
|
41
|
-
else {
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
result = illegal
|
|
46
|
-
? {
|
|
46
|
+
else if (canBeReplaced) {
|
|
47
|
+
result = isResumable
|
|
48
|
+
? { status: 'progress' }
|
|
49
|
+
: {
|
|
47
50
|
status: 'error',
|
|
48
51
|
error: {
|
|
49
52
|
message: `Worker timed out during a non-resumable phase (${eventType}), which cannot be continued.`,
|
|
50
53
|
},
|
|
51
|
-
}
|
|
52
|
-
: { status: 'progress' };
|
|
54
|
+
};
|
|
53
55
|
}
|
|
54
56
|
}
|
|
55
57
|
await adapter.emitFromResult(result);
|
|
@@ -39,8 +39,8 @@ export declare abstract class BaseState<ConnectorState> {
|
|
|
39
39
|
init(initialState: ConnectorState): Promise<void>;
|
|
40
40
|
/**
|
|
41
41
|
* Normalizes fetched state into the `{ connectorState, sdkState }` envelope.
|
|
42
|
-
*
|
|
43
|
-
*
|
|
42
|
+
* Envelopes are recognized by their keys and validate `stateVersion` when
|
|
43
|
+
* present; anything else is a flat v1 blob split by
|
|
44
44
|
* `V1_SDK_STATE_KEYS`.
|
|
45
45
|
*/
|
|
46
46
|
private normalizeFetchedState;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"base-state.d.ts","sourceRoot":"","sources":["../../src/state/base-state.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAEnD,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAGnD,OAAO,EAEL,QAAQ,EAER,cAAc,EAEf,MAAM,oBAAoB,CAAC;AAE5B;;;;GAIG;AACH,8BAAsB,SAAS,CAAC,cAAc;IAC5C,SAAS,CAAC,eAAe,EAAE,cAAc,CAAC;IAC1C,SAAS,CAAC,SAAS,EAAE,QAAQ,CAAC;IAC9B,SAAS,CAAC,gBAAgB,EAAE,eAAe,CAAM;IACjD,SAAS,CAAC,QAAQ,CAAC,eAAe,EAAE,QAAQ,CAAC;IAC7C,SAAS,CAAC,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IACvC,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,SAAS,CAAS;gBAGxB,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,cAAc,CAAC,cAAc,CAAC,EACvD,eAAe,EAAE,QAAQ;IAY3B,2EAA2E;IAC3E,IAAI,KAAK,IAAI,cAAc,CAE1B;IAED,IAAI,KAAK,CAAC,KAAK,EAAE,cAAc,EAE9B;IAED,uEAAuE;IACvE,IAAI,QAAQ,IAAI,QAAQ,CAEvB;IAED,IAAI,QAAQ,CAAC,KAAK,EAAE,QAAQ,EAE3B;IAED,IAAI,eAAe,IAAI,eAAe,CAErC;IAED;;;;OAIG;IACG,mCAAmC,CACvC,oBAAoB,CAAC,EAAE,oBAAoB,GAC1C,OAAO,CAAC,IAAI,CAAC;IAwChB;;;;OAIG;IACG,IAAI,CAAC,YAAY,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IAuDvD;;;;;OAKG;IACH,OAAO,CAAC,qBAAqB;
|
|
1
|
+
{"version":3,"file":"base-state.d.ts","sourceRoot":"","sources":["../../src/state/base-state.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAEnD,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAGnD,OAAO,EAEL,QAAQ,EAER,cAAc,EAEf,MAAM,oBAAoB,CAAC;AAE5B;;;;GAIG;AACH,8BAAsB,SAAS,CAAC,cAAc;IAC5C,SAAS,CAAC,eAAe,EAAE,cAAc,CAAC;IAC1C,SAAS,CAAC,SAAS,EAAE,QAAQ,CAAC;IAC9B,SAAS,CAAC,gBAAgB,EAAE,eAAe,CAAM;IACjD,SAAS,CAAC,QAAQ,CAAC,eAAe,EAAE,QAAQ,CAAC;IAC7C,SAAS,CAAC,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IACvC,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,SAAS,CAAS;gBAGxB,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,cAAc,CAAC,cAAc,CAAC,EACvD,eAAe,EAAE,QAAQ;IAY3B,2EAA2E;IAC3E,IAAI,KAAK,IAAI,cAAc,CAE1B;IAED,IAAI,KAAK,CAAC,KAAK,EAAE,cAAc,EAE9B;IAED,uEAAuE;IACvE,IAAI,QAAQ,IAAI,QAAQ,CAEvB;IAED,IAAI,QAAQ,CAAC,KAAK,EAAE,QAAQ,EAE3B;IAED,IAAI,eAAe,IAAI,eAAe,CAErC;IAED;;;;OAIG;IACG,mCAAmC,CACvC,oBAAoB,CAAC,EAAE,oBAAoB,GAC1C,OAAO,CAAC,IAAI,CAAC;IAwChB;;;;OAIG;IACG,IAAI,CAAC,YAAY,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IAuDvD;;;;;OAKG;IACH,OAAO,CAAC,qBAAqB;IAyC7B,OAAO,CAAC,YAAY;IAepB;;;OAGG;IACG,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IA0C1B,UAAU,IAAI,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAqBjE"}
|
package/dist/state/base-state.js
CHANGED
|
@@ -137,8 +137,8 @@ class BaseState {
|
|
|
137
137
|
}
|
|
138
138
|
/**
|
|
139
139
|
* Normalizes fetched state into the `{ connectorState, sdkState }` envelope.
|
|
140
|
-
*
|
|
141
|
-
*
|
|
140
|
+
* Envelopes are recognized by their keys and validate `stateVersion` when
|
|
141
|
+
* present; anything else is a flat v1 blob split by
|
|
142
142
|
* `V1_SDK_STATE_KEYS`.
|
|
143
143
|
*/
|
|
144
144
|
normalizeFetchedState(parsed) {
|
|
@@ -146,16 +146,13 @@ class BaseState {
|
|
|
146
146
|
throw new Error('Fetched state is not a JSON object.');
|
|
147
147
|
}
|
|
148
148
|
const record = parsed;
|
|
149
|
-
if (record
|
|
150
|
-
if (record.stateVersion !==
|
|
149
|
+
if ('connectorState' in record || 'sdkState' in record) {
|
|
150
|
+
if (record.stateVersion !== undefined &&
|
|
151
|
+
record.stateVersion !== state_interfaces_1.STATE_VERSION) {
|
|
151
152
|
throw new Error(`Unsupported state version ${String(record.stateVersion)}; this SDK reads version ${state_interfaces_1.STATE_VERSION}.`);
|
|
152
153
|
}
|
|
153
154
|
return this.readEnvelope(record);
|
|
154
155
|
}
|
|
155
|
-
// Envelopes written by 2.0.0 betas carry no version marker.
|
|
156
|
-
if ('connectorState' in record || 'sdkState' in record) {
|
|
157
|
-
return this.readEnvelope(record);
|
|
158
|
-
}
|
|
159
156
|
// Legacy flat v1 blob: split known SDK keys out of the connector state.
|
|
160
157
|
const connectorState = {};
|
|
161
158
|
const sdkState = {};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extraction-state.d.ts","sourceRoot":"","sources":["../../src/state/extraction-state.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAsB,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAGxE,sGAAsG;AACtG,qBAAa,eAAe,CAAC,cAAc,CAAE,SAAQ,SAAS,CAAC,cAAc,CAAC;gBAChE,MAAM,EAAE,cAAc,CAAC,cAAc,CAAC;IAIlD;;;;OAIG;IACH,uBAAuB,IAAI,IAAI;
|
|
1
|
+
{"version":3,"file":"extraction-state.d.ts","sourceRoot":"","sources":["../../src/state/extraction-state.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAsB,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAGxE,sGAAsG;AACtG,qBAAa,eAAe,CAAC,cAAc,CAAE,SAAQ,SAAS,CAAC,cAAc,CAAC;gBAChE,MAAM,EAAE,cAAc,CAAC,cAAc,CAAC;IAIlD;;;;OAIG;IACH,uBAAuB,IAAI,IAAI;CA4HhC;AAED,wBAAsB,qBAAqB,CAAC,cAAc,EAAE,EAC1D,KAAK,EACL,YAAY,EACZ,oBAAoB,EACpB,OAAO,GACR,EAAE,cAAc,CAAC,cAAc,CAAC,GAAG,OAAO,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC,CAkB3E"}
|
|
@@ -21,6 +21,7 @@ class ExtractionState extends base_state_1.BaseState {
|
|
|
21
21
|
* all other events reuse those cached boundaries. Validates extract_from < extract_to.
|
|
22
22
|
*/
|
|
23
23
|
resolveExtractionWindow() {
|
|
24
|
+
var _a, _b;
|
|
24
25
|
const sdkState = this.sdkState;
|
|
25
26
|
const eventContext = this.event.payload.event_context;
|
|
26
27
|
const overrideExtractFrom = (value) => {
|
|
@@ -29,10 +30,15 @@ class ExtractionState extends base_state_1.BaseState {
|
|
|
29
30
|
}
|
|
30
31
|
eventContext.extract_from = value;
|
|
31
32
|
};
|
|
32
|
-
if (this.event.payload.event_type === extraction_1.EventType.StartExtractingData
|
|
33
|
-
!sdkState.lastSyncStarted) {
|
|
34
|
-
|
|
35
|
-
|
|
33
|
+
if (this.event.payload.event_type === extraction_1.EventType.StartExtractingData) {
|
|
34
|
+
if (!sdkState.lastSyncStarted) {
|
|
35
|
+
sdkState.lastSyncStarted = new Date().toISOString();
|
|
36
|
+
console.log(`Setting lastSyncStarted to ${sdkState.lastSyncStarted}.`);
|
|
37
|
+
}
|
|
38
|
+
// A sync that failed before attachments must not leak its SSOR records into this one.
|
|
39
|
+
if ((_b = (_a = sdkState.toDevRev) === null || _a === void 0 ? void 0 : _a.deferredSsorArtifacts) === null || _b === void 0 ? void 0 : _b.length) {
|
|
40
|
+
sdkState.toDevRev.deferredSsorArtifacts = [];
|
|
41
|
+
}
|
|
36
42
|
}
|
|
37
43
|
if (this.event.payload.event_type === extraction_1.EventType.StartExtractingMetadata) {
|
|
38
44
|
const timeFields = [
|
package/dist/types/errors.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
3
|
-
*
|
|
4
|
-
* sends
|
|
2
|
+
* Platform-recognized error codes (CommonError), mapped by the snap-in manager
|
|
3
|
+
* in `internal/common/errors_snapin.go`. Set them on `ErrorRecord.code`; the SDK
|
|
4
|
+
* sends exactly `ERROR_CODE=<code>` as the wire message.
|
|
5
5
|
*/
|
|
6
6
|
export declare enum AirSyncErrorCode {
|
|
7
7
|
EXTERNAL_SYNC_UNIT_DELETED = "EXTERNAL_SYNC_UNIT_DELETED",
|
package/dist/types/errors.js
CHANGED
|
@@ -5,9 +5,9 @@ exports.formatErrorCode = formatErrorCode;
|
|
|
5
5
|
const ERROR_PREFIX = 'ERROR_CODE';
|
|
6
6
|
const ERROR_DELIMITER = '=';
|
|
7
7
|
/**
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
* sends
|
|
8
|
+
* Platform-recognized error codes (CommonError), mapped by the snap-in manager
|
|
9
|
+
* in `internal/common/errors_snapin.go`. Set them on `ErrorRecord.code`; the SDK
|
|
10
|
+
* sends exactly `ERROR_CODE=<code>` as the wire message.
|
|
11
11
|
*/
|
|
12
12
|
var AirSyncErrorCode;
|
|
13
13
|
(function (AirSyncErrorCode) {
|
|
@@ -109,10 +109,6 @@ export interface EventContext {
|
|
|
109
109
|
mode: string;
|
|
110
110
|
request_id: string;
|
|
111
111
|
request_id_adaas: string;
|
|
112
|
-
/** @deprecated Never read by the SDK; kept because connectors still branch on it. */
|
|
113
|
-
reset_extraction?: boolean;
|
|
114
|
-
/** @deprecated Use extraction_start_time/extraction_end_time (resolved into extract_from/extract_to). */
|
|
115
|
-
reset_extract_from?: boolean;
|
|
116
112
|
run_id: string;
|
|
117
113
|
sequence_version: string;
|
|
118
114
|
snap_in_slug: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extraction.d.ts","sourceRoot":"","sources":["../../src/types/extraction.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,+CAA+C,CAAC;AAClF,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAEhD,iDAAiD;AACjD,oBAAY,SAAS;IAEnB,gCAAgC,yCAAyC;IACzE,uBAAuB,8BAA8B;IACrD,0BAA0B,iCAAiC;IAC3D,mBAAmB,0BAA0B;IAC7C,sBAAsB,6BAA6B;IACnD,2BAA2B,mCAAmC;IAC9D,0BAA0B,iCAAiC;IAC3D,6BAA6B,oCAAoC;IACjE,sCAAsC,+CAA+C;IAGrF,gBAAgB,uBAAuB;IACvC,mBAAmB,0BAA0B;IAC7C,uBAAuB,8BAA8B;IACrD,0BAA0B,iCAAiC;IAC3D,wBAAwB,gCAAgC;IACxD,kCAAkC,2CAA2C;CAC9E;AAED,iDAAiD;AACjD,oBAAY,kBAAkB;IAE5B,8BAA8B,uCAAuC;IACrE,+BAA+B,wCAAwC;IACvE,0BAA0B,iCAAiC;IAC3D,yBAAyB,gCAAgC;IACzD,sBAAsB,6BAA6B;IACnD,uBAAuB,8BAA8B;IACrD,sBAAsB,6BAA6B;IACnD,qBAAqB,4BAA4B;IACjD,kBAAkB,yBAAyB;IAC3C,mBAAmB,0BAA0B;IAC7C,0BAA0B,kCAAkC;IAC5D,2BAA2B,mCAAmC;IAC9D,4BAA4B,mCAAmC;IAC/D,2BAA2B,kCAAkC;IAC7D,wBAAwB,+BAA+B;IACvD,yBAAyB,gCAAgC;IACzD,qCAAqC,8CAA8C;IACnF,sCAAsC,+CAA+C;CACtF;AAED,uEAAuE;AACvE,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,oBAAY,gBAAgB;IAC1B,YAAY,iBAAiB;IAC7B,WAAW,gBAAgB;CAC5B;AAED,8EAA8E;AAC9E,oBAAY,QAAQ;IAClB,WAAW,OAAO;IAClB,YAAY,OAAO;IACnB,eAAe,YAAO;IACtB,YAAY,OAAO;IACnB,OAAO,MAAM;IACb,OAAO,MAAM;IACb,KAAK,MAAM;CACZ;AAED;;;GAGG;AACH,oBAAY,aAAa;IACvB,cAAc,mBAAmB;IACjC,2BAA2B,gCAAgC;IAC3D,cAAc,mBAAmB;IACjC,0BAA0B,+BAA+B;IACzD,YAAY,iBAAiB;IAC7B,aAAa,kBAAkB;IAC/B,SAAS,cAAc;CACxB;AAED;;;GAGG;AACH,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,aAAa,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,YAAY;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,uBAAuB,EAAE,MAAM,CAAC;IAChC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;IAC7B;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,0EAA0E;IAC1E,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,WAAW,EAAE,MAAM,CAAC;IACpB,kBAAkB,CAAC,EAAE,gBAAgB,CAAC;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,gBAAgB,EAAE,MAAM,CAAC;IACzB,
|
|
1
|
+
{"version":3,"file":"extraction.d.ts","sourceRoot":"","sources":["../../src/types/extraction.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,+CAA+C,CAAC;AAClF,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAEhD,iDAAiD;AACjD,oBAAY,SAAS;IAEnB,gCAAgC,yCAAyC;IACzE,uBAAuB,8BAA8B;IACrD,0BAA0B,iCAAiC;IAC3D,mBAAmB,0BAA0B;IAC7C,sBAAsB,6BAA6B;IACnD,2BAA2B,mCAAmC;IAC9D,0BAA0B,iCAAiC;IAC3D,6BAA6B,oCAAoC;IACjE,sCAAsC,+CAA+C;IAGrF,gBAAgB,uBAAuB;IACvC,mBAAmB,0BAA0B;IAC7C,uBAAuB,8BAA8B;IACrD,0BAA0B,iCAAiC;IAC3D,wBAAwB,gCAAgC;IACxD,kCAAkC,2CAA2C;CAC9E;AAED,iDAAiD;AACjD,oBAAY,kBAAkB;IAE5B,8BAA8B,uCAAuC;IACrE,+BAA+B,wCAAwC;IACvE,0BAA0B,iCAAiC;IAC3D,yBAAyB,gCAAgC;IACzD,sBAAsB,6BAA6B;IACnD,uBAAuB,8BAA8B;IACrD,sBAAsB,6BAA6B;IACnD,qBAAqB,4BAA4B;IACjD,kBAAkB,yBAAyB;IAC3C,mBAAmB,0BAA0B;IAC7C,0BAA0B,kCAAkC;IAC5D,2BAA2B,mCAAmC;IAC9D,4BAA4B,mCAAmC;IAC/D,2BAA2B,kCAAkC;IAC7D,wBAAwB,+BAA+B;IACvD,yBAAyB,gCAAgC;IACzD,qCAAqC,8CAA8C;IACnF,sCAAsC,+CAA+C;CACtF;AAED,uEAAuE;AACvE,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,oBAAY,gBAAgB;IAC1B,YAAY,iBAAiB;IAC7B,WAAW,gBAAgB;CAC5B;AAED,8EAA8E;AAC9E,oBAAY,QAAQ;IAClB,WAAW,OAAO;IAClB,YAAY,OAAO;IACnB,eAAe,YAAO;IACtB,YAAY,OAAO;IACnB,OAAO,MAAM;IACb,OAAO,MAAM;IACb,KAAK,MAAM;CACZ;AAED;;;GAGG;AACH,oBAAY,aAAa;IACvB,cAAc,mBAAmB;IACjC,2BAA2B,gCAAgC;IAC3D,cAAc,mBAAmB;IACjC,0BAA0B,+BAA+B;IACzD,YAAY,iBAAiB;IAC7B,aAAa,kBAAkB;IAC/B,SAAS,cAAc;CACxB;AAED;;;GAGG;AACH,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,aAAa,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,YAAY;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,uBAAuB,EAAE,MAAM,CAAC;IAChC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;IAC7B;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,0EAA0E;IAC1E,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,WAAW,EAAE,MAAM,CAAC;IACpB,kBAAkB,CAAC,EAAE,gBAAgB,CAAC;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,gBAAgB,EAAE,MAAM,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,KAAK,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,uEAAuE;IACvE,qBAAqB,CAAC,EAAE,SAAS,CAAC;IAClC,mEAAmE;IACnE,mBAAmB,CAAC,EAAE,SAAS,CAAC;IAChC,mGAAmG;IACnG,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,kFAAkF;AAClF,oBAAY,aAAa;IACvB,IAAI,SAAS;IACb,KAAK,UAAU;CAChB;AAED,mGAAmG;AACnG,MAAM,WAAW,aAAa;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,gBAAgB,EAAE,aAAa,CAAC;CACjC;AAED,MAAM,WAAW,SAAS;IACxB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;OAIG;IACH,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IAEvB,OAAO,CAAC,EAAE,YAAY,EAAE,CAAC;IACzB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,0BAA0B,CAAC,EAAE,aAAa,EAAE,CAAC;CAC9C;AAED,MAAM,WAAW,cAAc;IAC7B,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAG/B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAG9B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,gDAAgD;AAChD,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE;QACP,OAAO,EAAE;YACP,qBAAqB,EAAE,MAAM,CAAC;SAC/B,CAAC;QACF,kBAAkB,EAAE,MAAM,CAAC;QAC3B,UAAU,EAAE,MAAM,CAAC;QACnB,0DAA0D;QAC1D,OAAO,EAAE,MAAM,CAAC;QAChB,iDAAiD;QACjD,OAAO,EAAE,MAAM,CAAC;QAChB,gEAAgE;QAChE,SAAS,EAAE,MAAM,CAAC;QAClB,yDAAyD;QACzD,kBAAkB,EAAE,MAAM,CAAC;KAC5B,CAAC;IACF,OAAO,EAAE,cAAc,CAAC;IACxB,kBAAkB,EAAE;QAClB,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;IACF,UAAU,EAAE,SAAS,CAAC;CACvB;AAED,8CAA8C;AAC9C,MAAM,WAAW,SAAS;IACxB,iDAAiD;IACjD,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtC,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtC,2EAA2E;IAE3E,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CACjC;AAED,MAAM,WAAW,cAAc;IAC7B,eAAe,EAAE,cAAc,CAAC;IAChC,aAAa,EAAE,YAAY,CAAC;IAC5B,UAAU,EAAE,SAAS,CAAC;IACtB,UAAU,CAAC,EAAE,SAAS,CAAC;CACxB;AAED,gDAAgD;AAChD,MAAM,WAAW,cAAc;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,YAAY,CAAC;IAC5B,UAAU,CAAC,EAAE,SAAS,CAAC;IACvB,eAAe,CAAC,EAAE,cAAc,CAAC;CAClC;AAED,MAAM,MAAM,yCAAyC,GAAG,CAAC,EACvD,IAAI,EACJ,KAAK,GACN,EAAE,uCAAuC,KAAK,OAAO,CAAC,yCAAyC,CAAC,CAAC;AAElG,MAAM,WAAW,uCAAuC;IACtD,IAAI,EAAE,oBAAoB,CAAC;IAC3B,KAAK,EAAE,YAAY,CAAC;CACrB;AAED,MAAM,WAAW,kBAAkB;IAEjC,IAAI,EAAE,GAAG,CAAC;IAEV,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAC9B;AAED,gEAAgE;AAChE,MAAM,MAAM,yCAAyC,GACjD;IAAE,MAAM,EAAE,SAAS,CAAC;IAAC,UAAU,EAAE,kBAAkB,CAAA;CAAE,GACrD;IAAE,MAAM,EAAE,OAAO,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,GACzC;IAAE,MAAM,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,WAAW,CAAA;CAAE,CAAC;AAE5C;;;;GAIG;AACH,MAAM,MAAM,2BAA2B,GACnC;IAAE,MAAM,EAAE,SAAS,CAAA;CAAE,GACrB;IAAE,MAAM,EAAE,OAAO,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,GACzC;IAAE,MAAM,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,WAAW,GAAG;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAA;CAAE,CAAC;AAEpE,MAAM,MAAM,uCAAuC,CACjD,KAAK,EACL,QAAQ,EACR,cAAc,IACZ,CAAC,EACH,WAAW,EACX,OAAO,EACP,SAAS,GACV,EAAE;IACD,WAAW,EAAE,KAAK,CAAC;IACnB,OAAO,EAAE,iBAAiB,CAAC,cAAc,CAAC,CAAC;IAC3C,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,KAAK,QAAQ,CAAC;AAEf,MAAM,MAAM,iCAAiC,GAAG,CAAC,EAC/C,UAAU,EACV,MAAM,GACP,EAAE;IACD,UAAU,EAAE,oBAAoB,CAAC;IACjC,MAAM,EAAE,yCAAyC,CAAC;CACnD,KAAK,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAE3C,MAAM,MAAM,wCAAwC,CAAC,QAAQ,EAAE,cAAc,IAC3E,CAAC,EACC,kBAAkB,EAClB,OAAO,EACP,MAAM,GACP,EAAE;IACD,kBAAkB,EAAE,QAAQ,CAAC;IAC7B,OAAO,EAAE,iBAAiB,CAAC,cAAc,CAAC,CAAC;IAC3C,MAAM,EAAE,yCAAyC,CAAC;CACnD,KAAK,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAE7C,MAAM,WAAW,kCAAkC,CACjD,cAAc,EACd,KAAK,EACL,QAAQ;IAER,OAAO,EAAE,uCAAuC,CAC9C,KAAK,EACL,QAAQ,EACR,cAAc,CACf,CAAC;IACF,QAAQ,EAAE,wCAAwC,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;CAC9E"}
|
package/docs/MIGRATION.md
CHANGED
|
@@ -9,11 +9,14 @@ own:
|
|
|
9
9
|
It ships in the package at `node_modules/@devrev/airsync-sdk/docs/MIGRATION.md`
|
|
10
10
|
(read it from disk; the `exports` map only blocks `import`/`require`). Before
|
|
11
11
|
v2 is installed, use
|
|
12
|
-
`https://raw.githubusercontent.com/devrev/adaas-sdk/v2/docs/MIGRATION.md
|
|
12
|
+
`https://raw.githubusercontent.com/devrev/adaas-sdk/v2/docs/MIGRATION.md`
|
|
13
|
+
(use `main` after GA). The installed guide always takes precedence.
|
|
13
14
|
|
|
14
|
-
**The wire protocol is unchanged.** Event
|
|
15
|
-
artifact
|
|
16
|
-
|
|
15
|
+
**The modern wire protocol is unchanged.** Event names, routes, headers,
|
|
16
|
+
artifact formats and `ERROR_CODE=…` strings remain compatible with platforms
|
|
17
|
+
that send modern events. Legacy `EXTRACTION_*` event names are intentionally
|
|
18
|
+
unsupported. The public TypeScript API changes all at once, without deprecated
|
|
19
|
+
aliases; runtime differences are described below.
|
|
17
20
|
|
|
18
21
|
---
|
|
19
22
|
|
|
@@ -21,7 +24,8 @@ v1. Only the TypeScript API changed, all at once, with no compatibility shims.
|
|
|
21
24
|
|
|
22
25
|
**Rules.** Work autonomously; when a rewrite is ambiguous, pick the option
|
|
23
26
|
that preserves runtime behavior and note it in `MIGRATION_REVIEW.md`. Migrate
|
|
24
|
-
source **and** tests.
|
|
27
|
+
source **and** tests. Preserve runtime string values except the documented
|
|
28
|
+
legacy event-string replacements in §3 and §11. Don't add comments
|
|
25
29
|
about the migration; the `// v1` / `// v2` labels here are reading aids. Leave
|
|
26
30
|
no compile errors or orphaned imports. A section with no hits is a no-op.
|
|
27
31
|
Existing comments and commented-out code that describe v1 behavior (`emit`,
|
|
@@ -30,40 +34,58 @@ v2; templates are copied into new connectors, so stale comments spread.
|
|
|
30
34
|
|
|
31
35
|
1. **Locate the package.** Find the `package.json` that depends on
|
|
32
36
|
`@devrev/ts-adaas` (generated connectors keep it in `code/`). Run everything
|
|
33
|
-
from there. If it already depends on `@devrev/airsync-sdk
|
|
34
|
-
|
|
37
|
+
from there. If it already depends on `@devrev/airsync-sdk`, inspect its
|
|
38
|
+
installed version and migration review. Early betas still need the
|
|
39
|
+
changes under **Rare cases** below; upgrade to the selected v2 version
|
|
40
|
+
and verify source, tests and local runners against this guide. Stop only
|
|
41
|
+
when the current-contract migration and verification are complete.
|
|
35
42
|
2. **Start from green latest v1.** If behind the latest stable 1.x, upgrade and
|
|
36
43
|
apply the release notes of every release in between that mentions a
|
|
37
44
|
migration (the last ones were 1.16.0–1.18.0), then get tsc/lint/build/test
|
|
38
|
-
green on v1.
|
|
45
|
+
green on v1. If the upgrade alone breaks tsc with axios type mismatches,
|
|
46
|
+
bump the connector's `axios` to the version that SDK release depends on.
|
|
47
|
+
Record failures that remain as **Pre-existing**.
|
|
39
48
|
3. **Install v2.**
|
|
40
49
|
```bash
|
|
41
50
|
npm uninstall @devrev/ts-adaas
|
|
42
51
|
npm install @devrev/airsync-sdk@beta # @latest after GA
|
|
43
52
|
```
|
|
44
|
-
|
|
53
|
+
Rename or remove `overrides` / `resolutions` entries keyed on
|
|
54
|
+
`@devrev/ts-adaas`; `npm uninstall` leaves them behind. If the installed
|
|
55
|
+
`docs/MIGRATION.md` differs from this copy, switch to it.
|
|
45
56
|
4. **Apply §1–§14 in order** across source and tests. After §1, search by
|
|
46
57
|
symbol (`\.emit\(`, `WorkerAdapter`, `sync_mapper_record`, …), not by the old
|
|
47
58
|
package name; test files that import nothing from the SDK still need
|
|
48
59
|
changes.
|
|
49
60
|
5. **Verify**: `npm install && npx tsc --noEmit && npm run lint && npm run build && npm test`.
|
|
50
|
-
If tsconfig excludes
|
|
61
|
+
If tsconfig excludes folders the connector still runs (`test/`, a local
|
|
62
|
+
runner), type-check them too: use the repo's `tsconfig.test.json` if it has
|
|
63
|
+
one, otherwise a temporary tsconfig that extends `tsconfig.json` and
|
|
64
|
+
overrides `"include": ["src", "test"]`, `"exclude": ["node_modules", "dist"]`,
|
|
65
|
+
`"rootDir": "."` and `"noEmit": true` (the base `exclude` and `rootDir` still
|
|
66
|
+
apply otherwise), then `npx tsc -p` it. Fix and rerun,
|
|
51
67
|
at most 4 attempts; the migration must add no new failures compared to
|
|
52
68
|
step 2.
|
|
53
69
|
6. **Sweep.** This must return nothing except legitimate hits (an
|
|
54
|
-
`EventEmitter.emit`, the connector's own `axiosClient` from §9
|
|
70
|
+
`EventEmitter.emit`, the connector's own `axiosClient` from §9,
|
|
71
|
+
`@devrev/typescript-sdk` in `package.json` from §1, `git config --local`,
|
|
72
|
+
legacy event names in fields the SDK doesn't read, such as
|
|
73
|
+
`event_type_adaas`):
|
|
55
74
|
```bash
|
|
56
|
-
grep -rnE "@devrev/ts-adaas|airsync-sdk/dist|\bprocessTask\b
|
|
57
|
-
--include='*.ts' --include='*.js' --include='*.json' --include='*.yml' --include='*.yaml' --include='*.md' --include='*.mdc' . \
|
|
75
|
+
grep -rnE "@devrev/ts-adaas|airsync-sdk/dist|\bprocessTask\b|\bWorkerAdapter\b|\.emit\(|hasWorkerEmitted|\bAirdrop(Event|Message)\b|axiosClient|(format|serialize)AxiosError|workerPath:|overridenOptions|skipConfirmation|isLocalDevelopment|--local|-- local\b|@devrev/typescript-sdk|(Extractor|Loader)EventType|ExtractionCommonError|ERROR_CODE=|reset_(extraction|extract_from)|sdkState|onUpload|uploadedArtifacts|currentEventDataLength|EXTRACTION_(DATA|METADATA|ATTACHMENTS|EXTERNAL_SYNC_UNITS)_" \
|
|
76
|
+
--include='*.ts' --include='*.js' --include='*.json' --include='*.yml' --include='*.yaml' --include='*.md' --include='*.mdc' --include='*.sh' . \
|
|
58
77
|
| grep -vE 'node_modules|/dist/|/coverage/|package-lock.json|MIGRATION_REVIEW.md'
|
|
59
78
|
```
|
|
60
79
|
The sweep covers `test/` and other folders that tsc, lint and jest often
|
|
61
80
|
exclude. A hit there is real: `@devrev/typescript-sdk` must be in
|
|
62
81
|
`package.json` (§1), and a `local` argv must become the env var (§3).
|
|
82
|
+
Also check `package.json` scripts and shell scripts for a bare `local`
|
|
83
|
+
argument (`npm run test:server -- local`, `main.ts local`).
|
|
63
84
|
Also grep `getByExternalId|getByTargetId` and check each call against §10:
|
|
64
85
|
existence checks there compile but silently invert.
|
|
65
86
|
Then rewrite any connector docs (`README`, `AGENTS.md`, `CLAUDE.md`, `docs/`)
|
|
66
|
-
that describe v1 behavior
|
|
87
|
+
that describe v1 behavior, including text that paraphrases DevRev docs with
|
|
88
|
+
legacy event names.
|
|
67
89
|
7. **Write `MIGRATION_REVIEW.md`** at the repo root: one line per section that
|
|
68
90
|
changed code with `high` / `medium` / `low confidence`, a sentence on what
|
|
69
91
|
to verify for medium/low, and the final check results with an
|
|
@@ -71,7 +93,7 @@ v2; templates are copied into new connectors, so stale comments spread.
|
|
|
71
93
|
helpers (§4), paths that emitted nothing (§4), `onTimeout` bodies kept (§4),
|
|
72
94
|
delay values that look like milliseconds or headers (§4), the cursor decision
|
|
73
95
|
(§8), ESU repo rewrites (§7), mapper error handling (§10), and a v1 baseline
|
|
74
|
-
upgrade.
|
|
96
|
+
upgrade. Record the exact installed versions.
|
|
75
97
|
```markdown
|
|
76
98
|
# v2 Migration Review
|
|
77
99
|
Migrated `@devrev/ts-adaas` 1.20.2 → `@devrev/airsync-sdk` 2.0.0.
|
|
@@ -99,7 +121,7 @@ v2; templates are copied into new connectors, so stale comments spread.
|
|
|
99
121
|
root equivalent are listed in §11. Never import the SDK's internal HTTP client
|
|
100
122
|
(`dist/http/axios-client-internal`): use the connector's own client (§9).
|
|
101
123
|
- **Node.js ≥ 20.** Bump the connector's `engines`, CI image and `.nvmrc` if
|
|
102
|
-
they pin older. Dependabot / CI configs that name `@devrev/ts-adaas` switch to
|
|
124
|
+
they pin older. `@types/node` can stay as it is. Dependabot / CI configs that name `@devrev/ts-adaas` switch to
|
|
103
125
|
`@devrev/airsync-sdk`.
|
|
104
126
|
- `ts-node` and `@devrev/typescript-sdk` are no longer SDK dependencies. A
|
|
105
127
|
connector that runs `.ts` workers directly needs `ts-node` in its own
|
|
@@ -107,7 +129,7 @@ v2; templates are copied into new connectors, so stale comments spread.
|
|
|
107
129
|
it. The template's local test server (`code/test/runner.ts`, and the copy
|
|
108
130
|
`airsync start-worker` writes to `code/.snapin-test/`) imports
|
|
109
131
|
`@devrev/typescript-sdk/dist/snap-ins`, so template connectors add it to
|
|
110
|
-
`devDependencies
|
|
132
|
+
`devDependencies` (`1.1.78`, the version the template pins). Most connectors exclude `test/` from tsc, lint and jest,
|
|
111
133
|
so the gates pass and the local worker then fails with
|
|
112
134
|
`Cannot find module '@devrev/typescript-sdk/dist/snap-ins'`.
|
|
113
135
|
`InputData` (the type of `event.input_data`) is now exported from the
|
|
@@ -125,7 +147,10 @@ The shapes are unchanged. `AirSyncEvent.context` now also declares `user_id`,
|
|
|
125
147
|
`payload.event_context`). Delete a hand-rolled context type that added these
|
|
126
148
|
(e.g. `interface CustomAirdropEvent extends AirdropEvent { context: CustomContext }`):
|
|
127
149
|
a narrower `context` no longer satisfies `AirSyncEvent`. Keep only the fields
|
|
128
|
-
the SDK type lacks.
|
|
150
|
+
the SDK type lacks. These fields are required, so typed mock events must set
|
|
151
|
+
them. `execution_metadata` is typed `{ devrev_endpoint }`; a custom type that
|
|
152
|
+
widened it with `@devrev/typescript-sdk` fields goes too, along with mock values
|
|
153
|
+
for fields nothing reads.
|
|
129
154
|
|
|
130
155
|
Deprecated `EventContext` aliases are removed. Use the current names:
|
|
131
156
|
|
|
@@ -138,8 +163,18 @@ Deprecated `EventContext` aliases are removed. Use the current names:
|
|
|
138
163
|
| `sync_run`, `sync_run_id` | `run_id` |
|
|
139
164
|
| `uuid` | `request_id_adaas` |
|
|
140
165
|
|
|
141
|
-
|
|
142
|
-
|
|
166
|
+
Rename these keys in fixture and test `event_context` objects too. When an
|
|
167
|
+
object already has the new key (or has two old keys for one new one, like
|
|
168
|
+
`dev_user` and `dev_user_id`), keep the existing new-name value, otherwise the
|
|
169
|
+
first old one, and drop the rest. Untyped
|
|
170
|
+
JSON still compiles, but the SDK reads only the current names (e.g. state
|
|
171
|
+
requests carry `request_id_adaas`).
|
|
172
|
+
|
|
173
|
+
`reset_extraction` and `reset_extract_from` are removed. Delete these keys
|
|
174
|
+
from fixtures and remove connector branches that read them. The SDK never
|
|
175
|
+
interpreted either flag. Configure backfills through the platform's
|
|
176
|
+
`extraction_start_time` / `extraction_end_time`; connectors read the
|
|
177
|
+
SDK-resolved `extract_from` / `extract_to` for the sync window (§8).
|
|
143
178
|
|
|
144
179
|
## 3. `spawn`
|
|
145
180
|
|
|
@@ -204,7 +239,7 @@ export type TaskResult =
|
|
|
204
239
|
|
|
205
240
|
Replace `processTask` with `processExtractionTask` or `processLoadingTask`,
|
|
206
241
|
depending on the worker (extraction workers use repos or `streamAttachments`;
|
|
207
|
-
loading workers use `loadItemTypes
|
|
242
|
+
loading workers use `loadItemTypes` or `loadAttachments`). Their
|
|
208
243
|
generic stays the connector state: `processTask<State>` →
|
|
209
244
|
`processExtractionTask<State>`.
|
|
210
245
|
|
|
@@ -233,14 +268,14 @@ processExtractionTask({
|
|
|
233
268
|
|----|----|
|
|
234
269
|
| `emit(XxxDone)` | `return { status: 'success' }` |
|
|
235
270
|
| `emit(XxxProgress, { progress })` | `return { status: 'progress' }` |
|
|
236
|
-
| `emit(XxxDelayed, { delay })` | `return { status: 'delay', delaySeconds: delay }` — keep the same number; the platform always read it as seconds. If it is clearly milliseconds
|
|
271
|
+
| `emit(XxxDelayed, { delay })` | `return { status: 'delay', delaySeconds: delay }` — keep the same number; the platform always read it as seconds. A raw `Retry-After` header stays `Number(header)`. If it is clearly milliseconds (a `*_MS` constant, a value in the thousands, "ms" in the message), keep the number, flag it and list it under **Unresolved**: v1 waited that many seconds too, and fixing it changes behavior |
|
|
237
272
|
| `emit(XxxDelayed)` with no `delay` | `delaySeconds` is required; use the connector's existing default retry constant, else `0` (what v1 effectively sent), and flag it |
|
|
238
273
|
| `emit(XxxError, { error })` | `return { status: 'error', error }` — `error` is required; fall back to `{ message: '<phase> failed' }` |
|
|
239
274
|
| `emit(XxxDone, { error })` (finished with warnings) | `return { status: 'success' }` and log the error. Returning `error` would fail the phase |
|
|
240
275
|
| `emit(ExternalSyncUnitExtractionDone, { external_sync_units })` | push to a repo, then `success` (§7) |
|
|
241
276
|
| `emit(MetadataExtractionDone, { pre_extraction_item_counts })` | `adapter.preExtractionItemCounts = counts`, then `success` |
|
|
242
277
|
| a path that **emitted nothing** (a `catch` that only logs, a branch that falls through) | `{ status: 'error', error: { message } }`, or rethrow. In v1 the SDK reported these as a phase error ("Worker exited without emitting"); never turn them into `success` or `progress` |
|
|
243
|
-
| any `emit(XxxDone)` after `loadItemTypes` / `loadAttachments` / `streamAttachments`, guarded (`!adapter.hasWorkerEmitted && !adapter.isTimeout`) or not | `return adapter.loadItemTypes(...)` (or the other call). Never discard their result and return `success`: v1 emitted their `delay` / `error` itself. Drop the guard and any module-level "already emitted" flags |
|
|
278
|
+
| any `emit(XxxDone)` after `loadItemTypes` / `loadAttachments` / `streamAttachments`, guarded (`!adapter.hasWorkerEmitted && !adapter.isTimeout`) or not | `return adapter.loadItemTypes(...)` (or the other call). Never discard their result and return `success`: v1 emitted their `delay` / `error` itself. Drop the guard and any module-level "already emitted" flags. If v1 did more work after the call and then emitted `Done` with an `error`, return the call's result and log that error |
|
|
244
279
|
|
|
245
280
|
In non-resumable phases (external sync units, state deletion), `progress` and
|
|
246
281
|
`delay` become `*Error`.
|
|
@@ -265,6 +300,21 @@ If a shared `handleError` emitted from many `catch` blocks that rethrow, v1
|
|
|
265
300
|
kept only the first emit. Record the first result (`firstError ??= …`) and
|
|
266
301
|
return it from the task's outermost `catch`.
|
|
267
302
|
|
|
303
|
+
If a helper **emitted and then kept going** (no throw), the caller often
|
|
304
|
+
emitted again later, e.g. `DataExtractionDone` after the loop, and v1 dropped
|
|
305
|
+
every emit after the first. Record the first result the same way, stop the
|
|
306
|
+
work that followed it (return out of the loop or extractor), and return it
|
|
307
|
+
from the task. Unawaited emits inside loops count too. When the emitting helper
|
|
308
|
+
is deep in the call stack, throw an error class that carries the `TaskResult`
|
|
309
|
+
instead (`class TaskResultError extends Error { constructor(readonly result: TaskResult) { super(); } }`),
|
|
310
|
+
stop the extractor, and return `e.result` from the task's outermost `catch`.
|
|
311
|
+
A free-function handler that must keep only the first result stores it per
|
|
312
|
+
adapter (`WeakMap<adapter, TaskResult>`) and the task reads it back.
|
|
313
|
+
|
|
314
|
+
A helper that returned `{ delay?, error? }` and emitted only when one was set
|
|
315
|
+
needs an explicit result when neither is set: `success` if the caller carried
|
|
316
|
+
on, otherwise `error`.
|
|
317
|
+
|
|
268
318
|
If a helper (often the HTTP client's 429 handler) **emitted and then threw a
|
|
269
319
|
sentinel error** that call sites rethrow, keep the throw, drop the emit, and map
|
|
270
320
|
it in the task's outermost `catch`:
|
|
@@ -284,12 +334,15 @@ helpers, assert on the return value.
|
|
|
284
334
|
`catch` that handles a timeout (`if (adapter.isTimeout || e instanceof
|
|
285
335
|
MyTimeout) return;`). Early `return`s inside `Promise<void>` helpers stay as
|
|
286
336
|
they are, as long as the task checks `isTimeout` after calling them.
|
|
287
|
-
- After the soft timeout, a returned `
|
|
288
|
-
|
|
337
|
+
- After the soft timeout, a returned `progress` is replaced by the
|
|
338
|
+
`onTimeout` result, and so is a returned `success` in resumable phases
|
|
339
|
+
(metadata, data, attachments, loading). A returned `error` or `delay`, and
|
|
340
|
+
a `success` in external sync units or state deletion, are emitted as is.
|
|
289
341
|
- `onTimeout` is optional and defaults to `progress` (or a timeout error in
|
|
290
|
-
external sync units and state deletion
|
|
291
|
-
|
|
292
|
-
|
|
342
|
+
external sync units and state deletion). v1 ran `onTimeout` only when the
|
|
343
|
+
task finished **without emitting**; v2 also runs it when a resumable task
|
|
344
|
+
returns `success` after the timeout, which costs one extra `PROGRESS` →
|
|
345
|
+
`CONTINUE` round instead of `DONE`. So:
|
|
293
346
|
- **Delete** it when it only emitted the usual event or computed a
|
|
294
347
|
`progress` number, when the v1 task emitted on every path, or when the task
|
|
295
348
|
only returns `loadItemTypes` / `loadAttachments` / `streamAttachments`
|
|
@@ -298,7 +351,12 @@ helpers, assert on the return value.
|
|
|
298
351
|
v1 (the task had a path that didn't emit) **and** it did real work or set a
|
|
299
352
|
custom error. Make its body safe to run after a completed task. A custom
|
|
300
353
|
"Lambda timeout" `onTimeout` on a task that emits on every path is
|
|
301
|
-
deleted.
|
|
354
|
+
deleted; v1 never ran it.
|
|
355
|
+
- `onTimeout` runs only after the task has returned, in v1 and v2, so
|
|
356
|
+
cleanup there (cancelling a rate limiter, aborting requests) never
|
|
357
|
+
interrupted a stuck task. Check the public `adapter.isTimeout` between
|
|
358
|
+
batches and perform any needed cancellation inside the task.
|
|
359
|
+
`timeoutSignal` is internal and unavailable to connector code.
|
|
302
360
|
- Don't call `process.exit()` or `postState()` in `onTimeout`; the SDK saves
|
|
303
361
|
state and exits.
|
|
304
362
|
|
|
@@ -309,6 +367,16 @@ now wrong**, and `Boolean(await repo.push(x))` or `if (!(await …push(x)))`
|
|
|
309
367
|
compiles but is always falsy. Make wrappers `await repo.push(x)` and return
|
|
310
368
|
`true` (or `void`). Keep a top-level `try/catch` that maps an error to `delay`
|
|
311
369
|
(e.g. a thrown 429); otherwise it's only needed for a custom message.
|
|
370
|
+
Find every unawaited `repo.push()` / `repo.upload()`, including calls in
|
|
371
|
+
helpers and loops, and await it before advancing the cursor or returning.
|
|
372
|
+
An unhandled upload rejection can bypass the task's error handling.
|
|
373
|
+
`loadItemTypes` and `loadAttachments` don't throw: a `create` / `update` that
|
|
374
|
+
throws becomes their `error` result, so a 429 must be returned as
|
|
375
|
+
`{ status: 'delay', delaySeconds }` from the callback (§6). A `catch` around
|
|
376
|
+
them that checks `isAxiosError(e) && status === 429` is dead; delete it.
|
|
377
|
+
Failures fetching loader stats or reading/writing sync mapper records fail
|
|
378
|
+
the phase and retain the current cursor. A connector callback returning
|
|
379
|
+
`status: 'error'` still counts that item as failed and continues.
|
|
312
380
|
|
|
313
381
|
`ProcessTaskInterface<T>` and `TaskAdapterInterface<T>` now take the
|
|
314
382
|
**adapter** type: `ProcessTaskInterface<ExtractionAdapter<State>>`, not
|
|
@@ -325,17 +393,17 @@ async function extractList(adapter: ExtractionAdapter<State>) { ... }
|
|
|
325
393
|
| Members | On |
|
|
326
394
|
|---------|----|
|
|
327
395
|
| `initializeRepos`, `getRepo`, `streamAttachments`, `processAttachment`, `shouldExtract`, `artifacts`, `preExtractionItemCounts` | `ExtractionAdapter` |
|
|
328
|
-
| `loadItemTypes`, `loadAttachments`, `
|
|
329
|
-
| `event`, `state`, `postState`, `uploadArtifact`, `downloadArtifact`, `isTimeout`, `extractionScope` | both |
|
|
396
|
+
| `loadItemTypes`, `loadAttachments`, `reports`, `processedFiles` | `LoadingAdapter` |
|
|
397
|
+
| `event`, `state`, `mappers`, `postState`, `uploadArtifact`, `downloadArtifact`, `isTimeout`, `extractionScope` | both |
|
|
330
398
|
|
|
331
399
|
- The adapters and `Repo` are **type-only**: `new ExtractionAdapter(...)` and
|
|
332
|
-
`class X extends LoadingAdapter<S>`
|
|
400
|
+
`class X extends LoadingAdapter<S>` or `extends Repo` (test doubles that
|
|
401
|
+
subclassed `Repo` through `dist/`) don't compile; use a plain object with
|
|
402
|
+
the methods the code calls. `interface X extends …`
|
|
333
403
|
does. SDK internals (`emit`, `sdkState`, `options`, constructors, …) aren't
|
|
334
404
|
public. `isTimeout` and `artifacts` are read-only.
|
|
335
|
-
-
|
|
336
|
-
|
|
337
|
-
constructor) and passes it down to helpers as a `mappers` argument typed
|
|
338
|
-
`Pick<Mappers, 'getByExternalId'>`; tests then pass a plain double.
|
|
405
|
+
- `adapter.mappers` works in extraction as in v1; keep those calls and only
|
|
406
|
+
apply the §10 return-shape changes.
|
|
339
407
|
- **Integration tests** that built an adapter by hand use the root factories:
|
|
340
408
|
|
|
341
409
|
```ts
|
|
@@ -399,7 +467,9 @@ return { status: 'success' }; // was a bare `return;`
|
|
|
399
467
|
as `skip` are dropped; every `success` creates or updates a mapper record.
|
|
400
468
|
- Shared helpers typed as these responses change too. Callers that read
|
|
401
469
|
`result.delay` or interpolate `${result.error}` (now an object) switch on
|
|
402
|
-
`result.status`.
|
|
470
|
+
`result.status`. A helper that only builds delays and errors returns the
|
|
471
|
+
narrowed type (`Extract<…, { status: 'delay' | 'error' }>`), so callers and
|
|
472
|
+
tests can read `delaySeconds` and `error` without a check.
|
|
403
473
|
- An error `code` returned from the stream callback is only logged; the pool
|
|
404
474
|
skips that attachment.
|
|
405
475
|
- `httpStream` is typed `HttpStreamResponse` (`{ data, headers }`); an axios
|
|
@@ -415,6 +485,14 @@ its state fields and tests. Workarounds typically override the
|
|
|
415
485
|
`ssor_attachment` repo's `onUpload`, splice `uploadedArtifacts`, adjust
|
|
416
486
|
`currentEventDataLength`, or stash artifact refs in state.
|
|
417
487
|
|
|
488
|
+
- A worker whose only job was re-injecting those refs returns
|
|
489
|
+
`{ status: 'success' }` (or its remaining work); keep its phase wired.
|
|
490
|
+
- A connector's own `{ itemType: 'ssor_attachment' }` repo entry can stay.
|
|
491
|
+
- Tests that mocked the deep `Uploader` use adapter doubles instead:
|
|
492
|
+
`{ uploadArtifact: jest.fn(), downloadArtifact: jest.fn() }`.
|
|
493
|
+
- If the workaround stashed refs in connector state, syncs between data done and
|
|
494
|
+
attachments done lose them after the upgrade; drain them first.
|
|
495
|
+
|
|
418
496
|
## 7. External sync units go through a repo
|
|
419
497
|
|
|
420
498
|
`EventData.external_sync_units` is removed. Push them to a repo:
|
|
@@ -442,9 +520,14 @@ the emit.
|
|
|
442
520
|
- **Persisted state** becomes `{ stateVersion: 2, connectorState, sdkState }`.
|
|
443
521
|
A v1 blob is migrated automatically on first read: known SDK keys move to
|
|
444
522
|
`sdkState` and everything else becomes connector state. **Rolling back to v1
|
|
445
|
-
after a v2 worker has
|
|
523
|
+
after a v2 worker has persisted state is not supported**. A drained sync
|
|
524
|
+
still has a v2 envelope: recovery needs a verified saved v1 state together
|
|
525
|
+
with the v1 connector, or an explicit reset/reinitialization and reimport.
|
|
526
|
+
Upgrade between runs, after v1 workers have finished. A
|
|
446
527
|
v1 state with a top-level key named `connectorState` or `sdkState` is
|
|
447
|
-
misread
|
|
528
|
+
misread. Rename it in a staged v1 deployment and migrate its persisted
|
|
529
|
+
value before upgrading, or explicitly reset/reimport. A top-level
|
|
530
|
+
connector-owned `stateVersion` in a flat v1 blob is preserved.
|
|
448
531
|
- **Sync window**: read
|
|
449
532
|
`adapter.event.payload.event_context.extract_from` / `extract_to`. The SDK
|
|
450
533
|
resolves them from the platform's time values, or from the last successful
|
|
@@ -455,17 +538,34 @@ the emit.
|
|
|
455
538
|
**`lastSyncStarted` / `lastSuccessfulSyncStarted`: rename or repoint.**
|
|
456
539
|
Getting this wrong compiles but silently breaks incremental sync.
|
|
457
540
|
|
|
458
|
-
|
|
459
|
-
|
|
541
|
+
These rules cover **top-level** state keys only; the v1 migration moves only
|
|
542
|
+
those. A cursor nested in the connector's own object (`state.extractor.lastSuccessfulSyncStarted`)
|
|
543
|
+
is connector-owned: leave it alone.
|
|
544
|
+
|
|
545
|
+
- **Not in the connector's `State` interface**, or declared but never written
|
|
546
|
+
by the connector (it came from the SDK): delete writes, and repoint reads to
|
|
547
|
+
`extract_from`.
|
|
460
548
|
- **Declared and used as the connector's own cursor** (derived from data, or
|
|
461
549
|
read alongside `extract_from`): **rename** it (e.g. →
|
|
462
|
-
`lastSuccessfulWindowStart`) everywhere, including fixtures.
|
|
463
|
-
|
|
550
|
+
`lastSuccessfulWindowStart`) everywhere, including fixtures. Before
|
|
551
|
+
deploying v2, use a staged v1 deployment to copy the persisted value into
|
|
552
|
+
the new connector key, retaining the legacy key for v1 SDK bookkeeping.
|
|
553
|
+
Alternatively, explicitly reset/reimport. A source-only rename does not
|
|
554
|
+
migrate persisted data: the v1 auto-migration moves the old key into SDK
|
|
555
|
+
state and leaves the new connector key unset.
|
|
464
556
|
- **Declared but only mirrors the SDK**: its only writes are "now" when data
|
|
465
557
|
extraction starts and a copy on finish. Treat it as not declared, **even if
|
|
466
558
|
it is read next to `extract_from`**. Renaming would leave it empty after the
|
|
467
559
|
upgrade, because the v1 value moves into SDK state.
|
|
468
560
|
|
|
561
|
+
After repointing, branches that chose between the two
|
|
562
|
+
(`reset_extract_from ? extract_from : lastSuccessfulSyncStarted`) read the same
|
|
563
|
+
value; collapse them and drop the `reset_extract_from` read. If the connector
|
|
564
|
+
already used `extract_from` for something else (e.g. a skip filter) and the
|
|
565
|
+
cursor for a second window (e.g. "re-emit and bump the floor"), repointing
|
|
566
|
+
merges the two, so code below the window may become dead. Flag that low
|
|
567
|
+
confidence.
|
|
568
|
+
|
|
469
569
|
`extract_from` equals the last successful sync start only when the platform
|
|
470
570
|
sends no time window. If a repointed read feeds logic beyond "skip items older
|
|
471
571
|
than this" (e.g. distinguishing first syncs), flag it as low confidence; there
|
|
@@ -490,7 +590,7 @@ import axios from 'axios';
|
|
|
490
590
|
import axiosRetry from 'axios-retry';
|
|
491
591
|
|
|
492
592
|
export const axiosClient = axios.create();
|
|
493
|
-
axiosRetry(axiosClient, {
|
|
593
|
+
axiosRetry(axiosClient as Parameters<typeof axiosRetry>[0], {
|
|
494
594
|
retries: 5,
|
|
495
595
|
retryDelay: (retryCount, error) => axiosRetry.exponentialDelay(retryCount, error, 1000),
|
|
496
596
|
retryCondition: (error) =>
|
|
@@ -504,6 +604,9 @@ axiosRetry(axiosClient, {
|
|
|
504
604
|
});
|
|
505
605
|
```
|
|
506
606
|
|
|
607
|
+
The cast is needed under ts-jest, which can resolve axios's `.d.ts` and
|
|
608
|
+
`.d.cts` to different types.
|
|
609
|
+
|
|
507
610
|
A deep import of the SDK's internal client (`dist/http/axios-client-internal`)
|
|
508
611
|
relied on more: a 30 s timeout, retries of 429s that send `Retry-After` (waiting
|
|
509
612
|
that long) and of `ECONNABORTED`. Reproduce what the connector depends on with
|
|
@@ -511,6 +614,9 @@ that long) and of `ECONNABORTED`. Reproduce what the connector depends on with
|
|
|
511
614
|
`retryCondition` / `retryDelay` that honor `Retry-After`.
|
|
512
615
|
|
|
513
616
|
For logging, use the root `serializeError(error)`, which returns a string.
|
|
617
|
+
For an axios error it is JSON of the request method, URL and params plus the
|
|
618
|
+
response status and body, without headers. Interpolating `serializeAxiosError`
|
|
619
|
+
printed `[object Object]` in v1, so such messages now carry these details.
|
|
514
620
|
Where `serializeAxiosError(e)` was used as an object (spread, or `.response`),
|
|
515
621
|
build the fields you need from the axios error yourself, leaving out headers.
|
|
516
622
|
|
|
@@ -546,6 +652,10 @@ return record?.sync_mapper_record.external_ids[0];
|
|
|
546
652
|
- Hand-written structural types (`{ getByTargetId(): Promise<{ data?: … }> }`)
|
|
547
653
|
keep `.data` compiling. Replace them with `Pick<Mappers, 'getByTargetId'>`.
|
|
548
654
|
- Replace 404 `catch` branches with a `null` check.
|
|
655
|
+
- A **generic `catch`** around a lookup (no 404 test) also handled "not
|
|
656
|
+
found": it reported an item error, logged a warning or tried the next ID.
|
|
657
|
+
With `null` that path no longer runs, so add a `null` branch that does what
|
|
658
|
+
the `catch` did, and keep the `catch` for other failures.
|
|
549
659
|
- Replace `isAxiosError` checks on mapper calls with
|
|
550
660
|
`e instanceof AirSyncHttpError` / `e.status`. Checks on the connector's own
|
|
551
661
|
HTTP calls stay.
|
|
@@ -592,8 +702,10 @@ in minor versions, so give `switch` statements over it a `default`.
|
|
|
592
702
|
|
|
593
703
|
## 12. Error codes
|
|
594
704
|
|
|
595
|
-
`ExtractionCommonError` is now `AirSyncErrorCode`.
|
|
596
|
-
|
|
705
|
+
`ExtractionCommonError` is now `AirSyncErrorCode`. These remain the platform's
|
|
706
|
+
known common error codes (`CommonError`), mapped by the snap-in manager in
|
|
707
|
+
`internal/common/errors_snapin.go`. Their values are bare codes, and you set
|
|
708
|
+
them on `error.code`:
|
|
597
709
|
|
|
598
710
|
```ts
|
|
599
711
|
// v1
|
|
@@ -605,9 +717,11 @@ return {
|
|
|
605
717
|
};
|
|
606
718
|
```
|
|
607
719
|
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
720
|
+
`error.code` is an SDK input field. Before sending the callback, the SDK
|
|
721
|
+
converts it into `event_data.error.message = 'ERROR_CODE=<code>'` and omits
|
|
722
|
+
the `code` property. The backend continues reading the common error from
|
|
723
|
+
`error.message`; your human-readable `message` goes to the logs. Write a
|
|
724
|
+
useful `message`. Raw `'ERROR_CODE=…'` strings and connector helpers that
|
|
611
725
|
return them (`HISTORICAL_DONE`, `authErrorCode(e)`) still work at runtime but
|
|
612
726
|
are migrated too: make the helper return `AirSyncErrorCode | undefined`, set it
|
|
613
727
|
as `code`, and update tests that pin the literal. The wire string doesn't
|
|
@@ -640,10 +754,12 @@ change.
|
|
|
640
754
|
- Update doubles and fixtures to the v2 shapes: `delaySeconds`, callback
|
|
641
755
|
`status` unions (§6), mapper `null` / unwrapped bodies (§10), `code` on
|
|
642
756
|
errors (§12), renamed state fields and the removed SDK fields (§8).
|
|
643
|
-
- Source that now calls `new Mappers(...)` needs a `Mappers` mock in the SDK
|
|
644
|
-
module factory.
|
|
645
757
|
- For a deleted `onTimeout`, drop the tests of its body and assert it's
|
|
646
|
-
`undefined`.
|
|
758
|
+
`undefined`. A factory like `processExtractionTask: (_config) => jest.fn()`
|
|
759
|
+
discards the config; use `processExtractionTask: jest.fn()` and read
|
|
760
|
+
`mocked.mock.calls[0][0].onTimeout`. Capture it in a module-level `const`
|
|
761
|
+
right after importing the worker: `jest.clearAllMocks()` in a `beforeEach`
|
|
762
|
+
wipes the calls. Tests of a deleted SSOR workaround go with it.
|
|
647
763
|
|
|
648
764
|
## 14. Smaller changes
|
|
649
765
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@devrev/airsync-sdk",
|
|
3
|
-
"version": "2.0.0
|
|
3
|
+
"version": "2.0.0",
|
|
4
4
|
"description": "Typescript SDK for building AirSync snap-ins on the DevRev platform.",
|
|
5
5
|
"type": "commonjs",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -26,7 +26,7 @@
|
|
|
26
26
|
},
|
|
27
27
|
"repository": {
|
|
28
28
|
"type": "git",
|
|
29
|
-
"url": "git+https://github.com/devrev/
|
|
29
|
+
"url": "git+https://github.com/devrev/airsync-sdk.git"
|
|
30
30
|
},
|
|
31
31
|
"publishConfig": {
|
|
32
32
|
"registry": "https://registry.npmjs.org"
|