@devrev/airsync-sdk 2.0.0-beta.10 → 2.0.0-beta.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/http/client.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../src/http/client.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../src/http/client.ts"],"names":[],"mappings":"AAAA,OAAc,EAAc,aAAa,EAAE,MAAM,OAAO,CAAC;AAmGzD,QAAA,MAAM,WAAW,EAAE,IAAI,CAAC,aAAa,EAAE,KAAK,GAAG,MAAM,CAGpD,CAAC;AAEF,OAAO,EAAE,WAAW,EAAE,CAAC"}
|
package/dist/http/client.js
CHANGED
|
@@ -6,64 +6,75 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
6
6
|
exports.axiosClient = void 0;
|
|
7
7
|
const axios_1 = __importDefault(require("axios"));
|
|
8
8
|
const axios_retry_1 = __importDefault(require("axios-retry"));
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
9
|
+
function createAxiosClient() {
|
|
10
|
+
const client = axios_1.default.create({
|
|
11
|
+
timeout: 30 * 1000,
|
|
12
|
+
});
|
|
13
|
+
(0, axios_retry_1.default)(client, {
|
|
14
|
+
retries: 5,
|
|
15
|
+
shouldResetTimeout: true,
|
|
16
|
+
retryDelay: (retryCount, error) => {
|
|
17
|
+
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o;
|
|
18
|
+
let delay;
|
|
19
|
+
if (((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) === 429) {
|
|
20
|
+
const retryAfter = ((_c = (_b = error.response) === null || _b === void 0 ? void 0 : _b.headers) === null || _c === void 0 ? void 0 : _c['retry-after']) ||
|
|
21
|
+
((_e = (_d = error.response) === null || _d === void 0 ? void 0 : _d.headers) === null || _e === void 0 ? void 0 : _e['Retry-After']);
|
|
22
|
+
delay = parseInt(retryAfter, 10) * 1000;
|
|
23
|
+
}
|
|
24
|
+
else {
|
|
25
|
+
delay = axios_retry_1.default.exponentialDelay(retryCount, error, 1000);
|
|
26
|
+
}
|
|
27
|
+
const requestId = ((_g = (_f = error.config) === null || _f === void 0 ? void 0 : _f.headers) === null || _g === void 0 ? void 0 : _g['x-request-id']) ||
|
|
28
|
+
((_j = (_h = error.config) === null || _h === void 0 ? void 0 : _h.headers) === null || _j === void 0 ? void 0 : _j['X-Request-ID']);
|
|
29
|
+
const delayInSeconds = Math.round(delay / 1000);
|
|
30
|
+
console.warn(`Retrying request to ${(_k = error.config) === null || _k === void 0 ? void 0 : _k.url} in ${delayInSeconds}s due to ${(_m = (_l = error.response) === null || _l === void 0 ? void 0 : _l.status) !== null && _m !== void 0 ? _m : 'unknown'} error.`, Object.assign(Object.assign({ method: (_o = error.config) === null || _o === void 0 ? void 0 : _o.method }, (retryCount && { retryCount })), (requestId && { requestId })));
|
|
31
|
+
return delay;
|
|
32
|
+
},
|
|
33
|
+
retryCondition: (error) => {
|
|
34
|
+
var _a, _b, _c, _d, _e, _f, _g;
|
|
35
|
+
const retryAfter = ((_b = (_a = error.response) === null || _a === void 0 ? void 0 : _a.headers) === null || _b === void 0 ? void 0 : _b['retry-after']) ||
|
|
36
|
+
((_d = (_c = error.response) === null || _c === void 0 ? void 0 : _c.headers) === null || _d === void 0 ? void 0 : _d['Retry-After']);
|
|
37
|
+
if (((_e = error.response) === null || _e === void 0 ? void 0 : _e.status) && error.response.status >= 500) {
|
|
38
|
+
return true;
|
|
39
|
+
}
|
|
40
|
+
// 429 only when a valid non-negative Retry-After header is present
|
|
41
|
+
else if (((_f = error.response) === null || _f === void 0 ? void 0 : _f.status) &&
|
|
42
|
+
error.response.status === 429 &&
|
|
43
|
+
retryAfter &&
|
|
44
|
+
!isNaN(Number(retryAfter)) &&
|
|
45
|
+
Number(retryAfter) >= 0) {
|
|
46
|
+
return true;
|
|
47
|
+
}
|
|
48
|
+
// Network errors for idempotent requests; 429 already handled above
|
|
49
|
+
else if (axios_retry_1.default.isNetworkOrIdempotentRequestError(error) &&
|
|
50
|
+
((_g = error.response) === null || _g === void 0 ? void 0 : _g.status) !== 429) {
|
|
51
|
+
return true;
|
|
52
|
+
}
|
|
53
|
+
// axios-retry excludes ECONNABORTED (client-side timeout only, never a
|
|
54
|
+
// server response) from isNetworkError, so handle it here separately.
|
|
55
|
+
else if (error.code === 'ECONNABORTED') {
|
|
56
|
+
return true;
|
|
57
|
+
}
|
|
58
|
+
else {
|
|
59
|
+
return false;
|
|
60
|
+
}
|
|
61
|
+
},
|
|
62
|
+
onMaxRetryTimesExceeded(error) {
|
|
63
|
+
var _a, _b, _c, _d, _e, _f;
|
|
64
|
+
(_b = (_a = error.config) === null || _a === void 0 ? void 0 : _a.headers) === null || _b === void 0 ? true : delete _b.authorization;
|
|
65
|
+
(_d = (_c = error.config) === null || _c === void 0 ? void 0 : _c.headers) === null || _d === void 0 ? true : delete _d.Authorization;
|
|
66
|
+
(_e = error.request) === null || _e === void 0 ? true : delete _e._header;
|
|
67
|
+
console.error(`Request to ${(_f = error.config) === null || _f === void 0 ? void 0 : _f.url} failed after max retries. Error`, error);
|
|
68
|
+
},
|
|
69
|
+
});
|
|
70
|
+
return client;
|
|
71
|
+
}
|
|
72
|
+
let instance;
|
|
73
|
+
const getInstance = () => (instance !== null && instance !== void 0 ? instance : (instance = createAxiosClient()));
|
|
74
|
+
// Created on first use so that loading the SDK never calls axios.create(),
|
|
75
|
+
// which breaks connector tests that mock axios.
|
|
76
|
+
const axiosClient = {
|
|
77
|
+
get: async (...args) => getInstance().get(...args),
|
|
78
|
+
post: async (...args) => getInstance().post(...args),
|
|
79
|
+
};
|
|
12
80
|
exports.axiosClient = axiosClient;
|
|
13
|
-
(0, axios_retry_1.default)(axiosClient, {
|
|
14
|
-
retries: 5,
|
|
15
|
-
shouldResetTimeout: true,
|
|
16
|
-
retryDelay: (retryCount, error) => {
|
|
17
|
-
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o;
|
|
18
|
-
let delay;
|
|
19
|
-
if (((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) === 429) {
|
|
20
|
-
const retryAfter = ((_c = (_b = error.response) === null || _b === void 0 ? void 0 : _b.headers) === null || _c === void 0 ? void 0 : _c['retry-after']) ||
|
|
21
|
-
((_e = (_d = error.response) === null || _d === void 0 ? void 0 : _d.headers) === null || _e === void 0 ? void 0 : _e['Retry-After']);
|
|
22
|
-
delay = parseInt(retryAfter, 10) * 1000;
|
|
23
|
-
}
|
|
24
|
-
else {
|
|
25
|
-
delay = axios_retry_1.default.exponentialDelay(retryCount, error, 1000);
|
|
26
|
-
}
|
|
27
|
-
const requestId = ((_g = (_f = error.config) === null || _f === void 0 ? void 0 : _f.headers) === null || _g === void 0 ? void 0 : _g['x-request-id']) ||
|
|
28
|
-
((_j = (_h = error.config) === null || _h === void 0 ? void 0 : _h.headers) === null || _j === void 0 ? void 0 : _j['X-Request-ID']);
|
|
29
|
-
const delayInSeconds = Math.round(delay / 1000);
|
|
30
|
-
console.warn(`Retrying request to ${(_k = error.config) === null || _k === void 0 ? void 0 : _k.url} in ${delayInSeconds}s due to ${(_m = (_l = error.response) === null || _l === void 0 ? void 0 : _l.status) !== null && _m !== void 0 ? _m : 'unknown'} error.`, Object.assign(Object.assign({ method: (_o = error.config) === null || _o === void 0 ? void 0 : _o.method }, (retryCount && { retryCount })), (requestId && { requestId })));
|
|
31
|
-
return delay;
|
|
32
|
-
},
|
|
33
|
-
retryCondition: (error) => {
|
|
34
|
-
var _a, _b, _c, _d, _e, _f, _g;
|
|
35
|
-
const retryAfter = ((_b = (_a = error.response) === null || _a === void 0 ? void 0 : _a.headers) === null || _b === void 0 ? void 0 : _b['retry-after']) ||
|
|
36
|
-
((_d = (_c = error.response) === null || _c === void 0 ? void 0 : _c.headers) === null || _d === void 0 ? void 0 : _d['Retry-After']);
|
|
37
|
-
if (((_e = error.response) === null || _e === void 0 ? void 0 : _e.status) && error.response.status >= 500) {
|
|
38
|
-
return true;
|
|
39
|
-
}
|
|
40
|
-
// 429 only when a valid non-negative Retry-After header is present
|
|
41
|
-
else if (((_f = error.response) === null || _f === void 0 ? void 0 : _f.status) &&
|
|
42
|
-
error.response.status === 429 &&
|
|
43
|
-
retryAfter &&
|
|
44
|
-
!isNaN(Number(retryAfter)) &&
|
|
45
|
-
Number(retryAfter) >= 0) {
|
|
46
|
-
return true;
|
|
47
|
-
}
|
|
48
|
-
// Network errors for idempotent requests; 429 already handled above
|
|
49
|
-
else if (axios_retry_1.default.isNetworkOrIdempotentRequestError(error) &&
|
|
50
|
-
((_g = error.response) === null || _g === void 0 ? void 0 : _g.status) !== 429) {
|
|
51
|
-
return true;
|
|
52
|
-
}
|
|
53
|
-
// axios-retry excludes ECONNABORTED (client-side timeout only, never a
|
|
54
|
-
// server response) from isNetworkError, so handle it here separately.
|
|
55
|
-
else if (error.code === 'ECONNABORTED') {
|
|
56
|
-
return true;
|
|
57
|
-
}
|
|
58
|
-
else {
|
|
59
|
-
return false;
|
|
60
|
-
}
|
|
61
|
-
},
|
|
62
|
-
onMaxRetryTimesExceeded(error) {
|
|
63
|
-
var _a, _b, _c, _d, _e, _f;
|
|
64
|
-
(_b = (_a = error.config) === null || _a === void 0 ? void 0 : _a.headers) === null || _b === void 0 ? true : delete _b.authorization;
|
|
65
|
-
(_d = (_c = error.config) === null || _c === void 0 ? void 0 : _c.headers) === null || _d === void 0 ? true : delete _d.Authorization;
|
|
66
|
-
(_e = error.request) === null || _e === void 0 ? true : delete _e._header;
|
|
67
|
-
console.error(`Request to ${(_f = error.config) === null || _f === void 0 ? void 0 : _f.url} failed after max retries. Error`, error);
|
|
68
|
-
},
|
|
69
|
-
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extraction-state.d.ts","sourceRoot":"","sources":["../../src/state/extraction-state.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAsB,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAGxE,sGAAsG;AACtG,qBAAa,eAAe,CAAC,cAAc,CAAE,SAAQ,SAAS,CAAC,cAAc,CAAC;gBAChE,MAAM,EAAE,cAAc,CAAC,cAAc,CAAC;IAIlD;;;;OAIG;IACH,uBAAuB,IAAI,IAAI;
|
|
1
|
+
{"version":3,"file":"extraction-state.d.ts","sourceRoot":"","sources":["../../src/state/extraction-state.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAsB,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAGxE,sGAAsG;AACtG,qBAAa,eAAe,CAAC,cAAc,CAAE,SAAQ,SAAS,CAAC,cAAc,CAAC;gBAChE,MAAM,EAAE,cAAc,CAAC,cAAc,CAAC;IAIlD;;;;OAIG;IACH,uBAAuB,IAAI,IAAI;CA4HhC;AAED,wBAAsB,qBAAqB,CAAC,cAAc,EAAE,EAC1D,KAAK,EACL,YAAY,EACZ,oBAAoB,EACpB,OAAO,GACR,EAAE,cAAc,CAAC,cAAc,CAAC,GAAG,OAAO,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC,CAkB3E"}
|
|
@@ -21,6 +21,7 @@ class ExtractionState extends base_state_1.BaseState {
|
|
|
21
21
|
* all other events reuse those cached boundaries. Validates extract_from < extract_to.
|
|
22
22
|
*/
|
|
23
23
|
resolveExtractionWindow() {
|
|
24
|
+
var _a, _b;
|
|
24
25
|
const sdkState = this.sdkState;
|
|
25
26
|
const eventContext = this.event.payload.event_context;
|
|
26
27
|
const overrideExtractFrom = (value) => {
|
|
@@ -29,10 +30,15 @@ class ExtractionState extends base_state_1.BaseState {
|
|
|
29
30
|
}
|
|
30
31
|
eventContext.extract_from = value;
|
|
31
32
|
};
|
|
32
|
-
if (this.event.payload.event_type === extraction_1.EventType.StartExtractingData
|
|
33
|
-
!sdkState.lastSyncStarted) {
|
|
34
|
-
|
|
35
|
-
|
|
33
|
+
if (this.event.payload.event_type === extraction_1.EventType.StartExtractingData) {
|
|
34
|
+
if (!sdkState.lastSyncStarted) {
|
|
35
|
+
sdkState.lastSyncStarted = new Date().toISOString();
|
|
36
|
+
console.log(`Setting lastSyncStarted to ${sdkState.lastSyncStarted}.`);
|
|
37
|
+
}
|
|
38
|
+
// A sync that failed before attachments must not leak its SSOR records into this one.
|
|
39
|
+
if ((_b = (_a = sdkState.toDevRev) === null || _a === void 0 ? void 0 : _a.deferredSsorArtifacts) === null || _b === void 0 ? void 0 : _b.length) {
|
|
40
|
+
sdkState.toDevRev.deferredSsorArtifacts = [];
|
|
41
|
+
}
|
|
36
42
|
}
|
|
37
43
|
if (this.event.payload.event_type === extraction_1.EventType.StartExtractingMetadata) {
|
|
38
44
|
const timeFields = [
|
package/docs/MIGRATION.md
CHANGED
|
@@ -35,7 +35,9 @@ v2; templates are copied into new connectors, so stale comments spread.
|
|
|
35
35
|
2. **Start from green latest v1.** If behind the latest stable 1.x, upgrade and
|
|
36
36
|
apply the release notes of every release in between that mentions a
|
|
37
37
|
migration (the last ones were 1.16.0–1.18.0), then get tsc/lint/build/test
|
|
38
|
-
green on v1.
|
|
38
|
+
green on v1. If the upgrade alone breaks tsc with axios type mismatches,
|
|
39
|
+
bump the connector's `axios` to the version that SDK release depends on.
|
|
40
|
+
Record failures that remain as **Pre-existing**.
|
|
39
41
|
3. **Install v2.**
|
|
40
42
|
```bash
|
|
41
43
|
npm uninstall @devrev/ts-adaas
|
|
@@ -51,15 +53,19 @@ v2; templates are copied into new connectors, so stale comments spread.
|
|
|
51
53
|
5. **Verify**: `npm install && npx tsc --noEmit && npm run lint && npm run build && npm test`.
|
|
52
54
|
If tsconfig excludes folders the connector still runs (`test/`, a local
|
|
53
55
|
runner), type-check them too: use the repo's `tsconfig.test.json` if it has
|
|
54
|
-
one, otherwise a temporary tsconfig that extends `tsconfig.json`
|
|
55
|
-
`"include": ["src", "test"]
|
|
56
|
+
one, otherwise a temporary tsconfig that extends `tsconfig.json` and
|
|
57
|
+
overrides `"include": ["src", "test"]`, `"exclude": ["node_modules", "dist"]`,
|
|
58
|
+
`"rootDir": "."` and `"noEmit": true` (the base `exclude` and `rootDir` still
|
|
59
|
+
apply otherwise), then `npx tsc -p` it. Fix and rerun,
|
|
56
60
|
at most 4 attempts; the migration must add no new failures compared to
|
|
57
61
|
step 2.
|
|
58
62
|
6. **Sweep.** This must return nothing except legitimate hits (an
|
|
59
63
|
`EventEmitter.emit`, the connector's own `axiosClient` from §9,
|
|
60
|
-
`@devrev/typescript-sdk` in `package.json` from §1, `git config --local
|
|
64
|
+
`@devrev/typescript-sdk` in `package.json` from §1, `git config --local`,
|
|
65
|
+
legacy event names in fields the SDK doesn't read, such as
|
|
66
|
+
`event_type_adaas`):
|
|
61
67
|
```bash
|
|
62
|
-
grep -rnE "@devrev/ts-adaas|airsync-sdk/dist|\bprocessTask\b
|
|
68
|
+
grep -rnE "@devrev/ts-adaas|airsync-sdk/dist|\bprocessTask\b|\bWorkerAdapter\b|\.emit\(|hasWorkerEmitted|\bAirdrop(Event|Message)\b|axiosClient|(format|serialize)AxiosError|workerPath:|overridenOptions|skipConfirmation|isLocalDevelopment|--local|-- local\b|@devrev/typescript-sdk|(Extractor|Loader)EventType|ExtractionCommonError|ERROR_CODE=|sdkState|onUpload|uploadedArtifacts|currentEventDataLength|EXTRACTION_(DATA|METADATA|ATTACHMENTS|EXTERNAL_SYNC_UNITS)_" \
|
|
63
69
|
--include='*.ts' --include='*.js' --include='*.json' --include='*.yml' --include='*.yaml' --include='*.md' --include='*.mdc' --include='*.sh' . \
|
|
64
70
|
| grep -vE 'node_modules|/dist/|/coverage/|package-lock.json|MIGRATION_REVIEW.md'
|
|
65
71
|
```
|
|
@@ -80,7 +86,7 @@ v2; templates are copied into new connectors, so stale comments spread.
|
|
|
80
86
|
helpers (§4), paths that emitted nothing (§4), `onTimeout` bodies kept (§4),
|
|
81
87
|
delay values that look like milliseconds or headers (§4), the cursor decision
|
|
82
88
|
(§8), ESU repo rewrites (§7), mapper error handling (§10), and a v1 baseline
|
|
83
|
-
upgrade.
|
|
89
|
+
upgrade. Record the exact installed versions.
|
|
84
90
|
```markdown
|
|
85
91
|
# v2 Migration Review
|
|
86
92
|
Migrated `@devrev/ts-adaas` 1.20.2 → `@devrev/airsync-sdk` 2.0.0.
|
|
@@ -134,7 +140,10 @@ The shapes are unchanged. `AirSyncEvent.context` now also declares `user_id`,
|
|
|
134
140
|
`payload.event_context`). Delete a hand-rolled context type that added these
|
|
135
141
|
(e.g. `interface CustomAirdropEvent extends AirdropEvent { context: CustomContext }`):
|
|
136
142
|
a narrower `context` no longer satisfies `AirSyncEvent`. Keep only the fields
|
|
137
|
-
the SDK type lacks.
|
|
143
|
+
the SDK type lacks. These fields are required, so typed mock events must set
|
|
144
|
+
them. `execution_metadata` is typed `{ devrev_endpoint }`; a custom type that
|
|
145
|
+
widened it with `@devrev/typescript-sdk` fields goes too, along with mock values
|
|
146
|
+
for fields nothing reads.
|
|
138
147
|
|
|
139
148
|
Deprecated `EventContext` aliases are removed. Use the current names:
|
|
140
149
|
|
|
@@ -147,7 +156,10 @@ Deprecated `EventContext` aliases are removed. Use the current names:
|
|
|
147
156
|
| `sync_run`, `sync_run_id` | `run_id` |
|
|
148
157
|
| `uuid` | `request_id_adaas` |
|
|
149
158
|
|
|
150
|
-
Rename these keys in fixture and test `event_context` objects too.
|
|
159
|
+
Rename these keys in fixture and test `event_context` objects too. When an
|
|
160
|
+
object already has the new key (or has two old keys for one new one, like
|
|
161
|
+
`dev_user` and `dev_user_id`), keep the existing new-name value, otherwise the
|
|
162
|
+
first old one, and drop the rest. Untyped
|
|
151
163
|
JSON still compiles, but the SDK reads only the current names (e.g. state
|
|
152
164
|
requests carry `request_id_adaas`).
|
|
153
165
|
|
|
@@ -246,14 +258,14 @@ processExtractionTask({
|
|
|
246
258
|
|----|----|
|
|
247
259
|
| `emit(XxxDone)` | `return { status: 'success' }` |
|
|
248
260
|
| `emit(XxxProgress, { progress })` | `return { status: 'progress' }` |
|
|
249
|
-
| `emit(XxxDelayed, { delay })` | `return { status: 'delay', delaySeconds: delay }` — keep the same number; the platform always read it as seconds. If it is clearly milliseconds
|
|
261
|
+
| `emit(XxxDelayed, { delay })` | `return { status: 'delay', delaySeconds: delay }` — keep the same number; the platform always read it as seconds. A raw `Retry-After` header stays `Number(header)`. If it is clearly milliseconds (a `*_MS` constant, a value in the thousands, "ms" in the message), keep the number, flag it and list it under **Unresolved**: v1 waited that many seconds too, and fixing it changes behavior |
|
|
250
262
|
| `emit(XxxDelayed)` with no `delay` | `delaySeconds` is required; use the connector's existing default retry constant, else `0` (what v1 effectively sent), and flag it |
|
|
251
263
|
| `emit(XxxError, { error })` | `return { status: 'error', error }` — `error` is required; fall back to `{ message: '<phase> failed' }` |
|
|
252
264
|
| `emit(XxxDone, { error })` (finished with warnings) | `return { status: 'success' }` and log the error. Returning `error` would fail the phase |
|
|
253
265
|
| `emit(ExternalSyncUnitExtractionDone, { external_sync_units })` | push to a repo, then `success` (§7) |
|
|
254
266
|
| `emit(MetadataExtractionDone, { pre_extraction_item_counts })` | `adapter.preExtractionItemCounts = counts`, then `success` |
|
|
255
267
|
| a path that **emitted nothing** (a `catch` that only logs, a branch that falls through) | `{ status: 'error', error: { message } }`, or rethrow. In v1 the SDK reported these as a phase error ("Worker exited without emitting"); never turn them into `success` or `progress` |
|
|
256
|
-
| any `emit(XxxDone)` after `loadItemTypes` / `loadAttachments` / `streamAttachments`, guarded (`!adapter.hasWorkerEmitted && !adapter.isTimeout`) or not | `return adapter.loadItemTypes(...)` (or the other call). Never discard their result and return `success`: v1 emitted their `delay` / `error` itself. Drop the guard and any module-level "already emitted" flags |
|
|
268
|
+
| any `emit(XxxDone)` after `loadItemTypes` / `loadAttachments` / `streamAttachments`, guarded (`!adapter.hasWorkerEmitted && !adapter.isTimeout`) or not | `return adapter.loadItemTypes(...)` (or the other call). Never discard their result and return `success`: v1 emitted their `delay` / `error` itself. Drop the guard and any module-level "already emitted" flags. If v1 did more work after the call and then emitted `Done` with an `error`, return the call's result and log that error |
|
|
257
269
|
|
|
258
270
|
In non-resumable phases (external sync units, state deletion), `progress` and
|
|
259
271
|
`delay` become `*Error`.
|
|
@@ -282,7 +294,16 @@ If a helper **emitted and then kept going** (no throw), the caller often
|
|
|
282
294
|
emitted again later, e.g. `DataExtractionDone` after the loop, and v1 dropped
|
|
283
295
|
every emit after the first. Record the first result the same way, stop the
|
|
284
296
|
work that followed it (return out of the loop or extractor), and return it
|
|
285
|
-
from the task. Unawaited emits inside loops count too.
|
|
297
|
+
from the task. Unawaited emits inside loops count too. When the emitting helper
|
|
298
|
+
is deep in the call stack, throw an error class that carries the `TaskResult`
|
|
299
|
+
instead (`class TaskResultError extends Error { constructor(readonly result: TaskResult) { super(); } }`),
|
|
300
|
+
stop the extractor, and return `e.result` from the task's outermost `catch`.
|
|
301
|
+
A free-function handler that must keep only the first result stores it per
|
|
302
|
+
adapter (`WeakMap<adapter, TaskResult>`) and the task reads it back.
|
|
303
|
+
|
|
304
|
+
A helper that returned `{ delay?, error? }` and emitted only when one was set
|
|
305
|
+
needs an explicit result when neither is set: `success` if the caller carried
|
|
306
|
+
on, otherwise `error`.
|
|
286
307
|
|
|
287
308
|
If a helper (often the HTTP client's 429 handler) **emitted and then threw a
|
|
288
309
|
sentinel error** that call sites rethrow, keep the throw, drop the emit, and map
|
|
@@ -321,6 +342,10 @@ helpers, assert on the return value.
|
|
|
321
342
|
custom error. Make its body safe to run after a completed task. A custom
|
|
322
343
|
"Lambda timeout" `onTimeout` on a task that emits on every path is
|
|
323
344
|
deleted; v1 never ran it.
|
|
345
|
+
- `onTimeout` runs only after the task has returned, in v1 and v2, so
|
|
346
|
+
cleanup there (cancelling a rate limiter, aborting requests) never
|
|
347
|
+
interrupted a stuck task. Move it into the task's `isTimeout` /
|
|
348
|
+
`timeoutSignal` handling if it matters, otherwise delete it.
|
|
324
349
|
- Don't call `process.exit()` or `postState()` in `onTimeout`; the SDK saves
|
|
325
350
|
state and exits.
|
|
326
351
|
|
|
@@ -355,7 +380,9 @@ async function extractList(adapter: ExtractionAdapter<State>) { ... }
|
|
|
355
380
|
| `event`, `state`, `mappers`, `postState`, `uploadArtifact`, `downloadArtifact`, `isTimeout`, `extractionScope` | both |
|
|
356
381
|
|
|
357
382
|
- The adapters and `Repo` are **type-only**: `new ExtractionAdapter(...)` and
|
|
358
|
-
`class X extends LoadingAdapter<S>`
|
|
383
|
+
`class X extends LoadingAdapter<S>` or `extends Repo` (test doubles that
|
|
384
|
+
subclassed `Repo` through `dist/`) don't compile; use a plain object with
|
|
385
|
+
the methods the code calls. `interface X extends …`
|
|
359
386
|
does. SDK internals (`emit`, `sdkState`, `options`, constructors, …) aren't
|
|
360
387
|
public. `isTimeout` and `artifacts` are read-only.
|
|
361
388
|
- `adapter.mappers` works in extraction as in v1; keep those calls and only
|
|
@@ -423,7 +450,9 @@ return { status: 'success' }; // was a bare `return;`
|
|
|
423
450
|
as `skip` are dropped; every `success` creates or updates a mapper record.
|
|
424
451
|
- Shared helpers typed as these responses change too. Callers that read
|
|
425
452
|
`result.delay` or interpolate `${result.error}` (now an object) switch on
|
|
426
|
-
`result.status`.
|
|
453
|
+
`result.status`. A helper that only builds delays and errors returns the
|
|
454
|
+
narrowed type (`Extract<…, { status: 'delay' | 'error' }>`), so callers and
|
|
455
|
+
tests can read `delaySeconds` and `error` without a check.
|
|
427
456
|
- An error `code` returned from the stream callback is only logged; the pool
|
|
428
457
|
skips that attachment.
|
|
429
458
|
- `httpStream` is typed `HttpStreamResponse` (`{ data, headers }`); an axios
|
|
@@ -439,6 +468,14 @@ its state fields and tests. Workarounds typically override the
|
|
|
439
468
|
`ssor_attachment` repo's `onUpload`, splice `uploadedArtifacts`, adjust
|
|
440
469
|
`currentEventDataLength`, or stash artifact refs in state.
|
|
441
470
|
|
|
471
|
+
- A worker whose only job was re-injecting those refs returns
|
|
472
|
+
`{ status: 'success' }` (or its remaining work); keep its phase wired.
|
|
473
|
+
- A connector's own `{ itemType: 'ssor_attachment' }` repo entry can stay.
|
|
474
|
+
- Tests that mocked the deep `Uploader` use adapter doubles instead:
|
|
475
|
+
`{ uploadArtifact: jest.fn(), downloadArtifact: jest.fn() }`.
|
|
476
|
+
- If the workaround stashed refs in connector state, syncs between data done and
|
|
477
|
+
attachments done lose them after the upgrade; drain them first.
|
|
478
|
+
|
|
442
479
|
## 7. External sync units go through a repo
|
|
443
480
|
|
|
444
481
|
`EventData.external_sync_units` is removed. Push them to a repo:
|
|
@@ -479,8 +516,13 @@ the emit.
|
|
|
479
516
|
**`lastSyncStarted` / `lastSuccessfulSyncStarted`: rename or repoint.**
|
|
480
517
|
Getting this wrong compiles but silently breaks incremental sync.
|
|
481
518
|
|
|
482
|
-
|
|
483
|
-
|
|
519
|
+
These rules cover **top-level** state keys only; the v1 migration moves only
|
|
520
|
+
those. A cursor nested in the connector's own object (`state.extractor.lastSuccessfulSyncStarted`)
|
|
521
|
+
is connector-owned: leave it alone.
|
|
522
|
+
|
|
523
|
+
- **Not in the connector's `State` interface**, or declared but never written
|
|
524
|
+
by the connector (it came from the SDK): delete writes, and repoint reads to
|
|
525
|
+
`extract_from`.
|
|
484
526
|
- **Declared and used as the connector's own cursor** (derived from data, or
|
|
485
527
|
read alongside `extract_from`): **rename** it (e.g. →
|
|
486
528
|
`lastSuccessfulWindowStart`) everywhere, including fixtures. Otherwise the v1
|
|
@@ -490,6 +532,14 @@ Getting this wrong compiles but silently breaks incremental sync.
|
|
|
490
532
|
it is read next to `extract_from`**. Renaming would leave it empty after the
|
|
491
533
|
upgrade, because the v1 value moves into SDK state.
|
|
492
534
|
|
|
535
|
+
After repointing, branches that chose between the two
|
|
536
|
+
(`reset_extract_from ? extract_from : lastSuccessfulSyncStarted`) read the same
|
|
537
|
+
value; collapse them and drop the `reset_extract_from` read. If the connector
|
|
538
|
+
already used `extract_from` for something else (e.g. a skip filter) and the
|
|
539
|
+
cursor for a second window (e.g. "re-emit and bump the floor"), repointing
|
|
540
|
+
merges the two, so code below the window may become dead. Flag that low
|
|
541
|
+
confidence.
|
|
542
|
+
|
|
493
543
|
`extract_from` equals the last successful sync start only when the platform
|
|
494
544
|
sends no time window. If a repointed read feeds logic beyond "skip items older
|
|
495
545
|
than this" (e.g. distinguishing first syncs), flag it as low confidence; there
|
|
@@ -514,7 +564,7 @@ import axios from 'axios';
|
|
|
514
564
|
import axiosRetry from 'axios-retry';
|
|
515
565
|
|
|
516
566
|
export const axiosClient = axios.create();
|
|
517
|
-
axiosRetry(axiosClient, {
|
|
567
|
+
axiosRetry(axiosClient as Parameters<typeof axiosRetry>[0], {
|
|
518
568
|
retries: 5,
|
|
519
569
|
retryDelay: (retryCount, error) => axiosRetry.exponentialDelay(retryCount, error, 1000),
|
|
520
570
|
retryCondition: (error) =>
|
|
@@ -528,6 +578,9 @@ axiosRetry(axiosClient, {
|
|
|
528
578
|
});
|
|
529
579
|
```
|
|
530
580
|
|
|
581
|
+
The cast is needed under ts-jest, which can resolve axios's `.d.ts` and
|
|
582
|
+
`.d.cts` to different types.
|
|
583
|
+
|
|
531
584
|
A deep import of the SDK's internal client (`dist/http/axios-client-internal`)
|
|
532
585
|
relied on more: a 30 s timeout, retries of 429s that send `Retry-After` (waiting
|
|
533
586
|
that long) and of `ECONNABORTED`. Reproduce what the connector depends on with
|
|
@@ -674,7 +727,9 @@ change.
|
|
|
674
727
|
- For a deleted `onTimeout`, drop the tests of its body and assert it's
|
|
675
728
|
`undefined`. A factory like `processExtractionTask: (_config) => jest.fn()`
|
|
676
729
|
discards the config; use `processExtractionTask: jest.fn()` and read
|
|
677
|
-
`mocked.mock.calls[0][0].onTimeout`.
|
|
730
|
+
`mocked.mock.calls[0][0].onTimeout`. Capture it in a module-level `const`
|
|
731
|
+
right after importing the worker: `jest.clearAllMocks()` in a `beforeEach`
|
|
732
|
+
wipes the calls. Tests of a deleted SSOR workaround go with it.
|
|
678
733
|
|
|
679
734
|
## 14. Smaller changes
|
|
680
735
|
|