@devrev/airsync-sdk 2.0.0-beta.7 → 2.0.0-beta.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/logger/logger.d.ts.map +1 -1
- package/dist/logger/logger.interfaces.d.ts +0 -2
- package/dist/logger/logger.interfaces.d.ts.map +1 -1
- package/dist/logger/logger.js +0 -1
- package/dist/multithreading/adapters/base-adapter.d.ts +4 -0
- package/dist/multithreading/adapters/base-adapter.d.ts.map +1 -1
- package/dist/multithreading/adapters/base-adapter.js +6 -0
- package/dist/multithreading/adapters/loading-adapter.d.ts +1 -4
- package/dist/multithreading/adapters/loading-adapter.d.ts.map +1 -1
- package/dist/multithreading/adapters/loading-adapter.js +7 -12
- package/dist/multithreading/process-task.d.ts.map +1 -1
- package/dist/multithreading/process-task.js +18 -16
- package/docs/MIGRATION.md +36 -20
- package/package.json +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../../src/logger/logger.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAW,cAAc,EAAE,MAAM,WAAW,CAAC;AAGpD,OAAO,EAAE,UAAU,
|
|
1
|
+
{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../../src/logger/logger.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAW,cAAc,EAAE,MAAM,WAAW,CAAC;AAGpD,OAAO,EAAE,UAAU,EAAgB,MAAM,OAAO,CAAC;AAKjD,OAAO,EACL,kBAAkB,EAClB,sBAAsB,EAEtB,QAAQ,EAER,cAAc,EACf,MAAM,qBAAqB,CAAC;AAI7B,eAAO,MAAM,qBAAqB,QAAQ,CAAC;AAI3C,eAAO,MAAM,eAAe,EAAE,cAM7B,CAAC;AAEF,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAOvD;AAED;;;;GAIG;AACH,qBAAa,MAAO,SAAQ,OAAO;IACjC,OAAO,CAAC,eAAe,CAAU;IACjC,OAAO,CAAC,OAAO,CAAC,CAAiB;IACjC,OAAO,CAAC,IAAI,CAAa;gBAEb,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,sBAAsB;IAUtD,OAAO,CAAC,aAAa;IAOrB,wGAAwG;IACxG,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,GAAG,IAAI;IAa7C,OAAO,CAAC,eAAe;IAcd,GAAG,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI;IAI7B,IAAI,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI;IAI9B,IAAI,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI;IAI9B,KAAK,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI;CAGzC;AAED,8FAA8F;AAE9F,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,cAAc,CAqB5E;AAED,kFAAkF;AAClF,wBAAgB,cAAc,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAmCrD;AAED,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,UAAU,GAAG,kBAAkB,CAuBzE"}
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import type { RawAxiosResponseHeaders } from 'axios';
|
|
2
1
|
import type { AirSyncEvent, EventContext } from '../types/extraction';
|
|
3
2
|
import type { RuntimeOptions } from '../types/workers';
|
|
4
3
|
export interface LoggerFactoryInterface {
|
|
@@ -29,7 +28,6 @@ export interface AxiosErrorResponse {
|
|
|
29
28
|
isCorsOrNoNetworkError: boolean;
|
|
30
29
|
response?: {
|
|
31
30
|
data: unknown;
|
|
32
|
-
headers: RawAxiosResponseHeaders;
|
|
33
31
|
status: number;
|
|
34
32
|
statusText: string;
|
|
35
33
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"logger.interfaces.d.ts","sourceRoot":"","sources":["../../src/logger/logger.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,
|
|
1
|
+
{"version":3,"file":"logger.interfaces.d.ts","sourceRoot":"","sources":["../../src/logger/logger.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACtE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAEvD,MAAM,WAAW,sBAAsB;IACrC,KAAK,EAAE,YAAY,CAAC;IACpB,OAAO,CAAC,EAAE,cAAc,CAAC;CAC1B;AAED,oBAAY,QAAQ;IAClB,IAAI,SAAS;IACb,IAAI,SAAS;IACb,KAAK,UAAU;CAChB;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,OAAO,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IAEf,SAAS,CAAC,EAAE,GAAG,CAAC;IAEhB,QAAQ,CAAC,EAAE,GAAG,CAAC;CAChB;AAED,MAAM,WAAW,cAAc;IAE7B,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,GAAG,cAAc,GAAG,cAAc,CAAC;CACtD;AAED,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE;QACN,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;QAE3B,MAAM,EAAE,GAAG,CAAC;QACZ,GAAG,EAAE,MAAM,GAAG,SAAS,CAAC;KACzB,CAAC;IACF,YAAY,EAAE,OAAO,CAAC;IACtB,sBAAsB,EAAE,OAAO,CAAC;IAChC,QAAQ,CAAC,EAAE;QACT,IAAI,EAAE,OAAO,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,UAAW,SAAQ,YAAY;IAC9C,WAAW,EAAE,MAAM,CAAC;CACrB"}
|
package/dist/logger/logger.js
CHANGED
|
@@ -159,7 +159,6 @@ function serializeAxiosError(error) {
|
|
|
159
159
|
if (error.response) {
|
|
160
160
|
serializedAxiosError.response = {
|
|
161
161
|
data: error.response.data,
|
|
162
|
-
headers: error.response.headers,
|
|
163
162
|
status: error.response.status,
|
|
164
163
|
statusText: error.response.statusText,
|
|
165
164
|
};
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { Mappers } from '../../mappers/mappers';
|
|
1
2
|
import { AirSyncEvent } from '../../types/extraction';
|
|
2
3
|
/**
|
|
3
4
|
* Shared state/behavior for both sync modes; owns the `emit` control-protocol
|
|
@@ -8,6 +9,9 @@ export declare abstract class BaseAdapter<ConnectorState> {
|
|
|
8
9
|
private _hasWorkerEmitted;
|
|
9
10
|
private _isTimeout;
|
|
10
11
|
private resolveTimeoutSignal;
|
|
12
|
+
private _mappers;
|
|
13
|
+
/** Sync mapper records for this event's sync unit, in both extraction and loading. */
|
|
14
|
+
get mappers(): Mappers;
|
|
11
15
|
/** Set once the soft timeout fires; the task should wrap up and return. */
|
|
12
16
|
get isTimeout(): boolean;
|
|
13
17
|
/** Connector-owned state exposed to snap-in code. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"base-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/base-adapter.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"base-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/base-adapter.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAKhD,OAAO,EACL,YAAY,EAIb,MAAM,wBAAwB,CAAC;AAYhC;;;GAGG;AACH,8BAAsB,WAAW,CAAC,cAAc;IAC9C,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IAI7B,OAAO,CAAC,iBAAiB,CAAkB;IAC3C,OAAO,CAAC,UAAU,CAAkB;IACpC,OAAO,CAAC,oBAAoB,CAAc;IAU1C,OAAO,CAAC,QAAQ,CAAU;IAsB1B,sFAAsF;IACtF,IAAI,OAAO,IAAI,OAAO,CAErB;IAOD,2EAA2E;IAC3E,IAAI,SAAS,IAAI,OAAO,CAEvB;IAUD,qDAAqD;IACrD,IAAI,KAAK,IAAI,cAAc,CAE1B;IAED,IAAI,KAAK,CAAC,KAAK,EAAE,cAAc,EAE9B;IAOD,IAAI,eAAe,kDAElB;IAED;;;;OAIG;IACG,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IAIhC;;;;OAIG;IACG,cAAc,CAClB,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,MAAM,EAAE,GACb,OAAO,CAAC;QAAE,EAAE,EAAE,MAAM,CAAA;KAAE,CAAC;IAY1B,kFAAkF;IAC5E,gBAAgB,CAAC,CAAC,GAAG,OAAO,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC;YAmIvD,IAAI;CAmCnB"}
|
|
@@ -5,6 +5,7 @@ const node_worker_threads_1 = require("node:worker_threads");
|
|
|
5
5
|
const constants_1 = require("../../common/constants");
|
|
6
6
|
const errors_1 = require("../../http/errors");
|
|
7
7
|
const logger_1 = require("../../logger/logger");
|
|
8
|
+
const mappers_1 = require("../../mappers/mappers");
|
|
8
9
|
const errors_2 = require("../../types/errors");
|
|
9
10
|
const workers_1 = require("../../types/workers");
|
|
10
11
|
const uploader_1 = require("../../uploader/uploader");
|
|
@@ -30,6 +31,11 @@ class BaseAdapter {
|
|
|
30
31
|
event,
|
|
31
32
|
options,
|
|
32
33
|
});
|
|
34
|
+
this._mappers = new mappers_1.Mappers({ event });
|
|
35
|
+
}
|
|
36
|
+
/** Sync mapper records for this event's sync unit, in both extraction and loading. */
|
|
37
|
+
get mappers() {
|
|
38
|
+
return this._mappers;
|
|
33
39
|
}
|
|
34
40
|
/** @internal */
|
|
35
41
|
get hasWorkerEmitted() {
|
|
@@ -1,15 +1,12 @@
|
|
|
1
|
-
import { Mappers } from '../../mappers/mappers';
|
|
2
1
|
import { ExternalSystemAttachment, ExternalSystemLoadingFunction, ItemTypesToLoadParams, LoaderReport } from '../../types/loading';
|
|
3
2
|
import { TaskResult } from '../../types/workers';
|
|
4
3
|
import { BaseAdapter } from './base-adapter';
|
|
5
|
-
/** Adapter passed to loading tasks (item/attachment loading,
|
|
4
|
+
/** Adapter passed to loading tasks (item/attachment loading, loader reports). @public */
|
|
6
5
|
export declare class LoadingAdapter<ConnectorState> extends BaseAdapter<ConnectorState> {
|
|
7
6
|
private loaderReports;
|
|
8
7
|
private _processedFiles;
|
|
9
|
-
private _mappers;
|
|
10
8
|
get reports(): LoaderReport[];
|
|
11
9
|
get processedFiles(): string[];
|
|
12
|
-
get mappers(): Mappers;
|
|
13
10
|
loadItemTypes({ itemTypesToLoad, }: ItemTypesToLoadParams): Promise<TaskResult>;
|
|
14
11
|
loadAttachments({ create, }: {
|
|
15
12
|
create: ExternalSystemLoadingFunction<ExternalSystemAttachment>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"loading-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/loading-adapter.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"loading-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/loading-adapter.ts"],"names":[],"mappings":"AASA,OAAO,EAEL,wBAAwB,EAGxB,6BAA6B,EAE7B,qBAAqB,EAGrB,YAAY,EAGb,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAkB,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjE,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAM7C,yFAAyF;AACzF,qBAAa,cAAc,CACzB,cAAc,CACd,SAAQ,WAAW,CAAC,cAAc,CAAC;IACnC,OAAO,CAAC,aAAa,CAAiB;IACtC,OAAO,CAAC,eAAe,CAAW;IAalC,IAAI,OAAO,IAAI,YAAY,EAAE,CAE5B;IAED,IAAI,cAAc,IAAI,MAAM,EAAE,CAE7B;IA2BK,aAAa,CAAC,EAClB,eAAe,GAChB,EAAE,qBAAqB,GAAG,OAAO,CAAC,UAAU,CAAC;IA8BxC,eAAe,CAAC,EACpB,MAAM,GACP,EAAE;QACD,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,UAAU,CAAC;IAavB;;;;OAIG;YACW,SAAS;YAiFT,gBAAgB;YAuBhB,QAAQ;IA+HtB,2EAA2E;IAC3E,OAAO,CAAC,YAAY;YAwBN,cAAc;CA2C7B"}
|
|
@@ -2,20 +2,18 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.LoadingAdapter = void 0;
|
|
4
4
|
const logger_1 = require("../../logger/logger");
|
|
5
|
-
const mappers_1 = require("../../mappers/mappers");
|
|
6
5
|
const mappers_interfaces_1 = require("../../mappers/mappers.interfaces");
|
|
7
6
|
const extraction_1 = require("../../types/extraction");
|
|
8
7
|
const loading_1 = require("../../types/loading");
|
|
9
8
|
const base_adapter_1 = require("./base-adapter");
|
|
10
9
|
const loading_adapter_helpers_1 = require("./loading-adapter.helpers");
|
|
11
|
-
/** Adapter passed to loading tasks (item/attachment loading,
|
|
10
|
+
/** Adapter passed to loading tasks (item/attachment loading, loader reports). @public */
|
|
12
11
|
class LoadingAdapter extends base_adapter_1.BaseAdapter {
|
|
13
12
|
/** @internal */
|
|
14
13
|
constructor(params) {
|
|
15
14
|
super(params);
|
|
16
15
|
this.loaderReports = [];
|
|
17
16
|
this._processedFiles = [];
|
|
18
|
-
this._mappers = new mappers_1.Mappers({ event: params.event });
|
|
19
17
|
}
|
|
20
18
|
get reports() {
|
|
21
19
|
return this.loaderReports;
|
|
@@ -23,9 +21,6 @@ class LoadingAdapter extends base_adapter_1.BaseAdapter {
|
|
|
23
21
|
get processedFiles() {
|
|
24
22
|
return this._processedFiles;
|
|
25
23
|
}
|
|
26
|
-
get mappers() {
|
|
27
|
-
return this._mappers;
|
|
28
|
-
}
|
|
29
24
|
/** @internal */
|
|
30
25
|
async beforeEmit() {
|
|
31
26
|
// Loading has no pre-emit work.
|
|
@@ -157,7 +152,7 @@ class LoadingAdapter extends base_adapter_1.BaseAdapter {
|
|
|
157
152
|
const syncUnit = this.event.payload.event_context.sync_unit;
|
|
158
153
|
let syncMapperRecord;
|
|
159
154
|
try {
|
|
160
|
-
syncMapperRecord = await this.
|
|
155
|
+
syncMapperRecord = await this.mappers.getByTargetId({
|
|
161
156
|
sync_unit: syncUnit,
|
|
162
157
|
target: devrevId,
|
|
163
158
|
});
|
|
@@ -170,14 +165,14 @@ class LoadingAdapter extends base_adapter_1.BaseAdapter {
|
|
|
170
165
|
},
|
|
171
166
|
};
|
|
172
167
|
}
|
|
173
|
-
const loadParams = { item, mappers: this.
|
|
168
|
+
const loadParams = { item, mappers: this.mappers, event: this.event };
|
|
174
169
|
if (syncMapperRecord) {
|
|
175
170
|
const response = await itemTypeToLoad.update(loadParams);
|
|
176
171
|
if (response.status !== 'success') {
|
|
177
172
|
return this.toFailedLoad(itemTypeToLoad.itemType, 'update', response);
|
|
178
173
|
}
|
|
179
174
|
try {
|
|
180
|
-
const syncMapperRecordUpdate = await this.
|
|
175
|
+
const syncMapperRecordUpdate = await this.mappers.update(Object.assign(Object.assign({ id: syncMapperRecord.sync_mapper_record.id, sync_unit: syncUnit, status: mappers_interfaces_1.SyncMapperRecordStatus.OPERATIONAL }, (response.modifiedDate && {
|
|
181
176
|
external_versions: {
|
|
182
177
|
add: [
|
|
183
178
|
{
|
|
@@ -213,7 +208,7 @@ class LoadingAdapter extends base_adapter_1.BaseAdapter {
|
|
|
213
208
|
return this.toFailedLoad(itemTypeToLoad.itemType, 'create', response);
|
|
214
209
|
}
|
|
215
210
|
try {
|
|
216
|
-
const syncMapperRecordCreate = await this.
|
|
211
|
+
const syncMapperRecordCreate = await this.mappers.create(Object.assign({ sync_unit: syncUnit, status: mappers_interfaces_1.SyncMapperRecordStatus.OPERATIONAL, external_ids: [response.id], targets: [devrevId] }, (response.modifiedDate && {
|
|
217
212
|
external_versions: [
|
|
218
213
|
{
|
|
219
214
|
modified_date: response.modifiedDate,
|
|
@@ -255,14 +250,14 @@ class LoadingAdapter extends base_adapter_1.BaseAdapter {
|
|
|
255
250
|
async loadAttachment({ item, create, }) {
|
|
256
251
|
const response = await create({
|
|
257
252
|
item,
|
|
258
|
-
mappers: this.
|
|
253
|
+
mappers: this.mappers,
|
|
259
254
|
event: this.event,
|
|
260
255
|
});
|
|
261
256
|
if (response.status !== 'success') {
|
|
262
257
|
return this.toFailedLoad('attachments', 'create', response);
|
|
263
258
|
}
|
|
264
259
|
try {
|
|
265
|
-
const syncMapperRecordCreate = await this.
|
|
260
|
+
const syncMapperRecordCreate = await this.mappers.create({
|
|
266
261
|
sync_unit: this.event.payload.event_context.sync_unit,
|
|
267
262
|
external_ids: [response.id],
|
|
268
263
|
targets: [item.reference_id],
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"process-task.d.ts","sourceRoot":"","sources":["../../src/multithreading/process-task.ts"],"names":[],"mappings":"AAKA,OAAO,EACL,oBAAoB,EAIrB,MAAM,kBAAkB,CAAC;AAG1B,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;
|
|
1
|
+
{"version":3,"file":"process-task.d.ts","sourceRoot":"","sources":["../../src/multithreading/process-task.ts"],"names":[],"mappings":"AAKA,OAAO,EACL,oBAAoB,EAIrB,MAAM,kBAAkB,CAAC;AAG1B,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAyF5D,oDAAoD;AACpD,wBAAgB,qBAAqB,CAAC,cAAc,EAAE,EACpD,IAAI,EACJ,SAAS,GACV,EAAE,oBAAoB,CAAC,iBAAiB,CAAC,cAAc,CAAC,CAAC,QA6BzD;AAED,gDAAgD;AAChD,wBAAgB,kBAAkB,CAAC,cAAc,EAAE,EACjD,IAAI,EACJ,SAAS,GACV,EAAE,oBAAoB,CAAC,cAAc,CAAC,cAAc,CAAC,CAAC,QA6BtD"}
|
|
@@ -13,9 +13,10 @@ const spawn_helpers_1 = require("./spawn/spawn.helpers");
|
|
|
13
13
|
/**
|
|
14
14
|
* Shared worker-thread driver: builds the adapter, runs the task, maps the
|
|
15
15
|
* returned {@link TaskResult} to a platform event and emits it exactly once.
|
|
16
|
-
* On soft timeout a returned `
|
|
17
|
-
*
|
|
18
|
-
* `delay`
|
|
16
|
+
* On soft timeout a returned `progress` is replaced by the `onTimeout` result
|
|
17
|
+
* (or a phase-appropriate default), and so is a `success` in resumable phases.
|
|
18
|
+
* A returned `error` or `delay`, or a `success` in a non-resumable phase, is
|
|
19
|
+
* emitted as is.
|
|
19
20
|
*/
|
|
20
21
|
async function runWorkerTask(buildAdapter, { task, onTimeout }) {
|
|
21
22
|
let adapter;
|
|
@@ -31,25 +32,26 @@ async function runWorkerTask(buildAdapter, { task, onTimeout }) {
|
|
|
31
32
|
});
|
|
32
33
|
console.log('Event passed from SDK to the connector:', adapter.event);
|
|
33
34
|
let result = await task({ adapter });
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
35
|
+
if (adapter.isTimeout) {
|
|
36
|
+
// A returned error or delay is a real outcome and must reach the
|
|
37
|
+
// platform. A success is replaced only where the phase can hand off
|
|
38
|
+
// with progress; a non-resumable phase that finished late is still done.
|
|
39
|
+
const eventType = adapter.event.payload.event_type;
|
|
40
|
+
const isResumable = !(0, spawn_helpers_1.getEventTypeForResult)(eventType, 'progress').illegal;
|
|
41
|
+
const canBeReplaced = result.status === 'progress' ||
|
|
42
|
+
(result.status === 'success' && isResumable);
|
|
43
|
+
if (canBeReplaced && onTimeout) {
|
|
39
44
|
result = await onTimeout({ adapter });
|
|
40
45
|
}
|
|
41
|
-
else {
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
result = illegal
|
|
46
|
-
? {
|
|
46
|
+
else if (canBeReplaced) {
|
|
47
|
+
result = isResumable
|
|
48
|
+
? { status: 'progress' }
|
|
49
|
+
: {
|
|
47
50
|
status: 'error',
|
|
48
51
|
error: {
|
|
49
52
|
message: `Worker timed out during a non-resumable phase (${eventType}), which cannot be continued.`,
|
|
50
53
|
},
|
|
51
|
-
}
|
|
52
|
-
: { status: 'progress' };
|
|
54
|
+
};
|
|
53
55
|
}
|
|
54
56
|
}
|
|
55
57
|
await adapter.emitFromResult(result);
|
package/docs/MIGRATION.md
CHANGED
|
@@ -53,10 +53,13 @@ v2; templates are copied into new connectors, so stale comments spread.
|
|
|
53
53
|
6. **Sweep.** This must return nothing except legitimate hits (an
|
|
54
54
|
`EventEmitter.emit`, the connector's own `axiosClient` from §9):
|
|
55
55
|
```bash
|
|
56
|
-
grep -rnE "@devrev/ts-adaas|airsync-sdk/dist|\bprocessTask\b|WorkerAdapter|\.emit\(|hasWorkerEmitted|Airdrop(Event|Message)|axiosClient|(format|serialize)AxiosError|workerPath:|overridenOptions|skipConfirmation|isLocalDevelopment|--local|(Extractor|Loader)EventType|ExtractionCommonError|ERROR_CODE=|sdkState|onUpload|uploadedArtifacts|currentEventDataLength|EXTRACTION_(DATA|METADATA|ATTACHMENTS|EXTERNAL_SYNC_UNITS)_" \
|
|
56
|
+
grep -rnE "@devrev/ts-adaas|airsync-sdk/dist|\bprocessTask\b|WorkerAdapter|\.emit\(|hasWorkerEmitted|Airdrop(Event|Message)|axiosClient|(format|serialize)AxiosError|workerPath:|overridenOptions|skipConfirmation|isLocalDevelopment|--local|@devrev/typescript-sdk|(Extractor|Loader)EventType|ExtractionCommonError|ERROR_CODE=|sdkState|onUpload|uploadedArtifacts|currentEventDataLength|EXTRACTION_(DATA|METADATA|ATTACHMENTS|EXTERNAL_SYNC_UNITS)_" \
|
|
57
57
|
--include='*.ts' --include='*.js' --include='*.json' --include='*.yml' --include='*.yaml' --include='*.md' --include='*.mdc' . \
|
|
58
58
|
| grep -vE 'node_modules|/dist/|/coverage/|package-lock.json|MIGRATION_REVIEW.md'
|
|
59
59
|
```
|
|
60
|
+
The sweep covers `test/` and other folders that tsc, lint and jest often
|
|
61
|
+
exclude. A hit there is real: `@devrev/typescript-sdk` must be in
|
|
62
|
+
`package.json` (§1), and a `local` argv must become the env var (§3).
|
|
60
63
|
Also grep `getByExternalId|getByTargetId` and check each call against §10:
|
|
61
64
|
existence checks there compile but silently invert.
|
|
62
65
|
Then rewrite any connector docs (`README`, `AGENTS.md`, `CLAUDE.md`, `docs/`)
|
|
@@ -101,7 +104,13 @@ v2; templates are copied into new connectors, so stale comments spread.
|
|
|
101
104
|
- `ts-node` and `@devrev/typescript-sdk` are no longer SDK dependencies. A
|
|
102
105
|
connector that runs `.ts` workers directly needs `ts-node` in its own
|
|
103
106
|
`devDependencies`. A connector that uses `@devrev/typescript-sdk` must list
|
|
104
|
-
it.
|
|
107
|
+
it. The template's local test server (`code/test/runner.ts`, and the copy
|
|
108
|
+
`airsync start-worker` writes to `code/.snapin-test/`) imports
|
|
109
|
+
`@devrev/typescript-sdk/dist/snap-ins`, so template connectors add it to
|
|
110
|
+
`devDependencies`. Most connectors exclude `test/` from tsc, lint and jest,
|
|
111
|
+
so the gates pass and the local worker then fails with
|
|
112
|
+
`Cannot find module '@devrev/typescript-sdk/dist/snap-ins'`.
|
|
113
|
+
`InputData` (the type of `event.input_data`) is now exported from the
|
|
105
114
|
root.
|
|
106
115
|
|
|
107
116
|
## 2. Event types
|
|
@@ -140,7 +149,7 @@ leave connector logic that uses them alone.
|
|
|
140
149
|
| `baseWorkerPath?` | **required**, the directory containing `workers/` (usually `__dirname`) |
|
|
141
150
|
| `options.workerPathOverrides` | top-level `workerPathOverrides` |
|
|
142
151
|
| `options: WorkerAdapterOptions` | `options?: SpawnOptions` (`{ timeout?, batchSize? }`) |
|
|
143
|
-
| `options.isLocalDevelopment`, `--local` argv | removed; set `AIRSYNC_LOCAL_DEVELOPMENT=1` |
|
|
152
|
+
| `options.isLocalDevelopment`, `local` / `--local` argv | removed; set `AIRSYNC_LOCAL_DEVELOPMENT=1` |
|
|
144
153
|
|
|
145
154
|
```ts
|
|
146
155
|
// v1
|
|
@@ -171,9 +180,14 @@ spawn({
|
|
|
171
180
|
`START_DELETING_EXTRACTOR_STATE` and `START_EXTRACTING_ATTACHMENTS` /
|
|
172
181
|
`CONTINUE_EXTRACTING_ATTACHMENTS` / `START_DELETING_EXTRACTOR_ATTACHMENTS_STATE`.
|
|
173
182
|
- **Local mode** comes only from the `AIRSYNC_LOCAL_DEVELOPMENT=1` environment
|
|
174
|
-
variable, and is ignored on Lambda.
|
|
175
|
-
`
|
|
176
|
-
`
|
|
183
|
+
variable, and is ignored on Lambda. v1 also enabled it from a bare `local`
|
|
184
|
+
argument, which the template test server receives (`main.ts local --port …`,
|
|
185
|
+
also used by `airsync start-worker`). v2 ignores both `local` and `--local`
|
|
186
|
+
without an error: logs come out as production JSON and no copies land in
|
|
187
|
+
`extracted_files/`. A local runner that did `process.argv.push('--local')`
|
|
188
|
+
or relied on the `local` argument sets
|
|
189
|
+
`process.env.AIRSYNC_LOCAL_DEVELOPMENT = '1'` before calling `spawn` instead,
|
|
190
|
+
or documents running the worker with `AIRSYNC_LOCAL_DEVELOPMENT=1`.
|
|
177
191
|
|
|
178
192
|
## 4. Workers return a `TaskResult` instead of emitting
|
|
179
193
|
|
|
@@ -190,7 +204,7 @@ export type TaskResult =
|
|
|
190
204
|
|
|
191
205
|
Replace `processTask` with `processExtractionTask` or `processLoadingTask`,
|
|
192
206
|
depending on the worker (extraction workers use repos or `streamAttachments`;
|
|
193
|
-
loading workers use `loadItemTypes
|
|
207
|
+
loading workers use `loadItemTypes` or `loadAttachments`). Their
|
|
194
208
|
generic stays the connector state: `processTask<State>` →
|
|
195
209
|
`processExtractionTask<State>`.
|
|
196
210
|
|
|
@@ -270,12 +284,15 @@ helpers, assert on the return value.
|
|
|
270
284
|
`catch` that handles a timeout (`if (adapter.isTimeout || e instanceof
|
|
271
285
|
MyTimeout) return;`). Early `return`s inside `Promise<void>` helpers stay as
|
|
272
286
|
they are, as long as the task checks `isTimeout` after calling them.
|
|
273
|
-
- After the soft timeout, a returned `
|
|
274
|
-
|
|
287
|
+
- After the soft timeout, a returned `progress` is replaced by the
|
|
288
|
+
`onTimeout` result, and so is a returned `success` in resumable phases
|
|
289
|
+
(metadata, data, attachments, loading). A returned `error` or `delay`, and
|
|
290
|
+
a `success` in external sync units or state deletion, are emitted as is.
|
|
275
291
|
- `onTimeout` is optional and defaults to `progress` (or a timeout error in
|
|
276
|
-
external sync units and state deletion
|
|
277
|
-
|
|
278
|
-
|
|
292
|
+
external sync units and state deletion). v1 ran `onTimeout` only when the
|
|
293
|
+
task finished **without emitting**; v2 also runs it when a resumable task
|
|
294
|
+
returns `success` after the timeout, which costs one extra `PROGRESS` →
|
|
295
|
+
`CONTINUE` round instead of `DONE`. So:
|
|
279
296
|
- **Delete** it when it only emitted the usual event or computed a
|
|
280
297
|
`progress` number, when the v1 task emitted on every path, or when the task
|
|
281
298
|
only returns `loadItemTypes` / `loadAttachments` / `streamAttachments`
|
|
@@ -311,17 +328,15 @@ async function extractList(adapter: ExtractionAdapter<State>) { ... }
|
|
|
311
328
|
| Members | On |
|
|
312
329
|
|---------|----|
|
|
313
330
|
| `initializeRepos`, `getRepo`, `streamAttachments`, `processAttachment`, `shouldExtract`, `artifacts`, `preExtractionItemCounts` | `ExtractionAdapter` |
|
|
314
|
-
| `loadItemTypes`, `loadAttachments`, `
|
|
315
|
-
| `event`, `state`, `postState`, `uploadArtifact`, `downloadArtifact`, `isTimeout`, `extractionScope` | both |
|
|
331
|
+
| `loadItemTypes`, `loadAttachments`, `reports`, `processedFiles` | `LoadingAdapter` |
|
|
332
|
+
| `event`, `state`, `mappers`, `postState`, `uploadArtifact`, `downloadArtifact`, `isTimeout`, `extractionScope` | both |
|
|
316
333
|
|
|
317
334
|
- The adapters and `Repo` are **type-only**: `new ExtractionAdapter(...)` and
|
|
318
335
|
`class X extends LoadingAdapter<S>` don't compile; `interface X extends …`
|
|
319
336
|
does. SDK internals (`emit`, `sdkState`, `options`, constructors, …) aren't
|
|
320
337
|
public. `isTimeout` and `artifacts` are read-only.
|
|
321
|
-
-
|
|
322
|
-
|
|
323
|
-
constructor) and passes it down to helpers as a `mappers` argument typed
|
|
324
|
-
`Pick<Mappers, 'getByExternalId'>`; tests then pass a plain double.
|
|
338
|
+
- `adapter.mappers` works in extraction as in v1; keep those calls and only
|
|
339
|
+
apply the §10 return-shape changes.
|
|
325
340
|
- **Integration tests** that built an adapter by hand use the root factories:
|
|
326
341
|
|
|
327
342
|
```ts
|
|
@@ -497,6 +512,9 @@ that long) and of `ECONNABORTED`. Reproduce what the connector depends on with
|
|
|
497
512
|
`retryCondition` / `retryDelay` that honor `Retry-After`.
|
|
498
513
|
|
|
499
514
|
For logging, use the root `serializeError(error)`, which returns a string.
|
|
515
|
+
For an axios error it is JSON of the request method, URL and params plus the
|
|
516
|
+
response status and body, without headers. Interpolating `serializeAxiosError`
|
|
517
|
+
printed `[object Object]` in v1, so such messages now carry these details.
|
|
500
518
|
Where `serializeAxiosError(e)` was used as an object (spread, or `.response`),
|
|
501
519
|
build the fields you need from the axios error yourself, leaving out headers.
|
|
502
520
|
|
|
@@ -626,8 +644,6 @@ change.
|
|
|
626
644
|
- Update doubles and fixtures to the v2 shapes: `delaySeconds`, callback
|
|
627
645
|
`status` unions (§6), mapper `null` / unwrapped bodies (§10), `code` on
|
|
628
646
|
errors (§12), renamed state fields and the removed SDK fields (§8).
|
|
629
|
-
- Source that now calls `new Mappers(...)` needs a `Mappers` mock in the SDK
|
|
630
|
-
module factory.
|
|
631
647
|
- For a deleted `onTimeout`, drop the tests of its body and assert it's
|
|
632
648
|
`undefined`. Tests of a deleted SSOR workaround go with it.
|
|
633
649
|
|