@devrev/airsync-sdk 2.0.0-beta.5 → 2.0.0-beta.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -12
- package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map +1 -1
- package/dist/attachments-streaming/attachments-streaming-pool.js +8 -9
- package/dist/common/constants.d.ts +0 -7
- package/dist/common/constants.d.ts.map +1 -1
- package/dist/common/constants.js +1 -28
- package/dist/common/helpers.d.ts.map +1 -1
- package/dist/common/helpers.js +1 -1
- package/dist/http/errors.d.ts +8 -0
- package/dist/http/errors.d.ts.map +1 -0
- package/dist/http/errors.js +31 -0
- package/dist/index.d.ts +18 -13
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +14 -11
- package/dist/logger/logger.d.ts.map +1 -1
- package/dist/logger/logger.interfaces.d.ts +2 -2
- package/dist/logger/logger.interfaces.d.ts.map +1 -1
- package/dist/mappers/mappers.d.ts +9 -3
- package/dist/mappers/mappers.d.ts.map +1 -1
- package/dist/mappers/mappers.interfaces.d.ts +0 -2
- package/dist/mappers/mappers.interfaces.d.ts.map +1 -1
- package/dist/mappers/mappers.js +33 -13
- package/dist/multithreading/adapters/base-adapter.d.ts +16 -35
- package/dist/multithreading/adapters/base-adapter.d.ts.map +1 -1
- package/dist/multithreading/adapters/base-adapter.js +89 -23
- package/dist/multithreading/adapters/extraction-adapter.d.ts +24 -20
- package/dist/multithreading/adapters/extraction-adapter.d.ts.map +1 -1
- package/dist/multithreading/adapters/extraction-adapter.js +175 -122
- package/dist/multithreading/adapters/loading-adapter.d.ts +13 -23
- package/dist/multithreading/adapters/loading-adapter.d.ts.map +1 -1
- package/dist/multithreading/adapters/loading-adapter.js +176 -301
- package/dist/multithreading/emit.d.ts.map +1 -1
- package/dist/multithreading/process-task.d.ts.map +1 -1
- package/dist/multithreading/process-task.js +24 -6
- package/dist/multithreading/spawn/spawn.d.ts +6 -1
- package/dist/multithreading/spawn/spawn.d.ts.map +1 -1
- package/dist/multithreading/spawn/spawn.helpers.d.ts.map +1 -1
- package/dist/multithreading/spawn/spawn.helpers.js +17 -52
- package/dist/multithreading/spawn/spawn.js +30 -26
- package/dist/multithreading/worker.js +15 -1
- package/dist/repo/repo.d.ts +4 -5
- package/dist/repo/repo.d.ts.map +1 -1
- package/dist/repo/repo.interfaces.d.ts +5 -4
- package/dist/repo/repo.interfaces.d.ts.map +1 -1
- package/dist/repo/repo.js +13 -15
- package/dist/state/base-state.d.ts +10 -5
- package/dist/state/base-state.d.ts.map +1 -1
- package/dist/state/base-state.js +39 -36
- package/dist/state/extraction-state.d.ts.map +1 -1
- package/dist/state/extraction-state.js +39 -4
- package/dist/state/state.interfaces.d.ts +13 -7
- package/dist/state/state.interfaces.d.ts.map +1 -1
- package/dist/state/state.interfaces.js +6 -8
- package/dist/state/time-value-resolver.d.ts.map +1 -1
- package/dist/state/time-value-resolver.js +3 -2
- package/dist/testing/mock-event.d.ts.map +1 -1
- package/dist/testing/mock-event.js +0 -8
- package/dist/testing/mock-server.d.ts +2 -0
- package/dist/testing/mock-server.d.ts.map +1 -1
- package/dist/testing/mock-server.interfaces.d.ts +6 -0
- package/dist/testing/mock-server.interfaces.d.ts.map +1 -1
- package/dist/testing/mock-server.js +37 -6
- package/dist/testing/test-adapters.d.ts +22 -0
- package/dist/testing/test-adapters.d.ts.map +1 -0
- package/dist/testing/test-adapters.js +32 -0
- package/dist/types/common.d.ts +4 -1
- package/dist/types/common.d.ts.map +1 -1
- package/dist/types/common.js +1 -1
- package/dist/types/errors.d.ts +13 -7
- package/dist/types/errors.d.ts.map +1 -1
- package/dist/types/errors.js +24 -0
- package/dist/types/extraction.d.ts +60 -44
- package/dist/types/extraction.d.ts.map +1 -1
- package/dist/types/extraction.js +10 -1
- package/dist/types/loading.d.ts +11 -30
- package/dist/types/loading.d.ts.map +1 -1
- package/dist/types/workers.d.ts +27 -23
- package/dist/types/workers.d.ts.map +1 -1
- package/dist/uploader/uploader.d.ts +1 -2
- package/dist/uploader/uploader.d.ts.map +1 -1
- package/dist/uploader/uploader.helpers.d.ts +3 -0
- package/dist/uploader/uploader.helpers.d.ts.map +1 -1
- package/dist/uploader/uploader.helpers.js +16 -0
- package/dist/uploader/uploader.interfaces.d.ts +4 -29
- package/dist/uploader/uploader.interfaces.d.ts.map +1 -1
- package/dist/uploader/uploader.js +4 -23
- package/docs/MIGRATION.md +651 -0
- package/package.json +24 -10
- package/MIGRATION.md +0 -1028
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extraction.d.ts","sourceRoot":"","sources":["../../src/types/extraction.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"extraction.d.ts","sourceRoot":"","sources":["../../src/types/extraction.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,+CAA+C,CAAC;AAClF,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAEhD,iDAAiD;AACjD,oBAAY,SAAS;IAEnB,gCAAgC,yCAAyC;IACzE,uBAAuB,8BAA8B;IACrD,0BAA0B,iCAAiC;IAC3D,mBAAmB,0BAA0B;IAC7C,sBAAsB,6BAA6B;IACnD,2BAA2B,mCAAmC;IAC9D,0BAA0B,iCAAiC;IAC3D,6BAA6B,oCAAoC;IACjE,sCAAsC,+CAA+C;IAGrF,gBAAgB,uBAAuB;IACvC,mBAAmB,0BAA0B;IAC7C,uBAAuB,8BAA8B;IACrD,0BAA0B,iCAAiC;IAC3D,wBAAwB,gCAAgC;IACxD,kCAAkC,2CAA2C;CAC9E;AAED,iDAAiD;AACjD,oBAAY,kBAAkB;IAE5B,8BAA8B,uCAAuC;IACrE,+BAA+B,wCAAwC;IACvE,0BAA0B,iCAAiC;IAC3D,yBAAyB,gCAAgC;IACzD,sBAAsB,6BAA6B;IACnD,uBAAuB,8BAA8B;IACrD,sBAAsB,6BAA6B;IACnD,qBAAqB,4BAA4B;IACjD,kBAAkB,yBAAyB;IAC3C,mBAAmB,0BAA0B;IAC7C,0BAA0B,kCAAkC;IAC5D,2BAA2B,mCAAmC;IAC9D,4BAA4B,mCAAmC;IAC/D,2BAA2B,kCAAkC;IAC7D,wBAAwB,+BAA+B;IACvD,yBAAyB,gCAAgC;IACzD,qCAAqC,8CAA8C;IACnF,sCAAsC,+CAA+C;CACtF;AAED,uEAAuE;AACvE,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,oBAAY,gBAAgB;IAC1B,YAAY,iBAAiB;IAC7B,WAAW,gBAAgB;CAC5B;AAED,8EAA8E;AAC9E,oBAAY,QAAQ;IAClB,WAAW,OAAO;IAClB,YAAY,OAAO;IACnB,eAAe,YAAO;IACtB,YAAY,OAAO;IACnB,OAAO,MAAM;IACb,OAAO,MAAM;IACb,KAAK,MAAM;CACZ;AAED;;;GAGG;AACH,oBAAY,aAAa;IACvB,cAAc,mBAAmB;IACjC,2BAA2B,gCAAgC;IAC3D,cAAc,mBAAmB;IACjC,0BAA0B,+BAA+B;IACzD,YAAY,iBAAiB;IAC7B,aAAa,kBAAkB;IAC/B,SAAS,cAAc;CACxB;AAED;;;GAGG;AACH,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,aAAa,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,YAAY;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,uBAAuB,EAAE,MAAM,CAAC;IAChC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;IAC7B;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,0EAA0E;IAC1E,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,WAAW,EAAE,MAAM,CAAC;IACpB,kBAAkB,CAAC,EAAE,gBAAgB,CAAC;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,gBAAgB,EAAE,MAAM,CAAC;IACzB,qFAAqF;IACrF,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,yGAAyG;IACzG,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,KAAK,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,uEAAuE;IACvE,qBAAqB,CAAC,EAAE,SAAS,CAAC;IAClC,mEAAmE;IACnE,mBAAmB,CAAC,EAAE,SAAS,CAAC;IAChC,mGAAmG;IACnG,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,kFAAkF;AAClF,oBAAY,aAAa;IACvB,IAAI,SAAS;IACb,KAAK,UAAU;CAChB;AAED,mGAAmG;AACnG,MAAM,WAAW,aAAa;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,gBAAgB,EAAE,aAAa,CAAC;CACjC;AAED,MAAM,WAAW,SAAS;IACxB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;OAIG;IACH,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IAEvB,OAAO,CAAC,EAAE,YAAY,EAAE,CAAC;IACzB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,0BAA0B,CAAC,EAAE,aAAa,EAAE,CAAC;CAC9C;AAED,MAAM,WAAW,cAAc;IAC7B,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAG/B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAG9B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,gDAAgD;AAChD,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE;QACP,OAAO,EAAE;YACP,qBAAqB,EAAE,MAAM,CAAC;SAC/B,CAAC;QACF,kBAAkB,EAAE,MAAM,CAAC;QAC3B,UAAU,EAAE,MAAM,CAAC;QACnB,0DAA0D;QAC1D,OAAO,EAAE,MAAM,CAAC;QAChB,iDAAiD;QACjD,OAAO,EAAE,MAAM,CAAC;QAChB,gEAAgE;QAChE,SAAS,EAAE,MAAM,CAAC;QAClB,yDAAyD;QACzD,kBAAkB,EAAE,MAAM,CAAC;KAC5B,CAAC;IACF,OAAO,EAAE,cAAc,CAAC;IACxB,kBAAkB,EAAE;QAClB,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;IACF,UAAU,EAAE,SAAS,CAAC;CACvB;AAED,8CAA8C;AAC9C,MAAM,WAAW,SAAS;IACxB,iDAAiD;IACjD,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtC,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtC,2EAA2E;IAE3E,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CACjC;AAED,MAAM,WAAW,cAAc;IAC7B,eAAe,EAAE,cAAc,CAAC;IAChC,aAAa,EAAE,YAAY,CAAC;IAC5B,UAAU,EAAE,SAAS,CAAC;IACtB,UAAU,CAAC,EAAE,SAAS,CAAC;CACxB;AAED,gDAAgD;AAChD,MAAM,WAAW,cAAc;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,YAAY,CAAC;IAC5B,UAAU,CAAC,EAAE,SAAS,CAAC;IACvB,eAAe,CAAC,EAAE,cAAc,CAAC;CAClC;AAED,MAAM,MAAM,yCAAyC,GAAG,CAAC,EACvD,IAAI,EACJ,KAAK,GACN,EAAE,uCAAuC,KAAK,OAAO,CAAC,yCAAyC,CAAC,CAAC;AAElG,MAAM,WAAW,uCAAuC;IACtD,IAAI,EAAE,oBAAoB,CAAC;IAC3B,KAAK,EAAE,YAAY,CAAC;CACrB;AAED,MAAM,WAAW,kBAAkB;IAEjC,IAAI,EAAE,GAAG,CAAC;IAEV,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAC9B;AAED,gEAAgE;AAChE,MAAM,MAAM,yCAAyC,GACjD;IAAE,MAAM,EAAE,SAAS,CAAC;IAAC,UAAU,EAAE,kBAAkB,CAAA;CAAE,GACrD;IAAE,MAAM,EAAE,OAAO,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,GACzC;IAAE,MAAM,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,WAAW,CAAA;CAAE,CAAC;AAE5C;;;;GAIG;AACH,MAAM,MAAM,2BAA2B,GACnC;IAAE,MAAM,EAAE,SAAS,CAAA;CAAE,GACrB;IAAE,MAAM,EAAE,OAAO,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,GACzC;IAAE,MAAM,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,WAAW,GAAG;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAA;CAAE,CAAC;AAEpE,MAAM,MAAM,uCAAuC,CACjD,KAAK,EACL,QAAQ,EACR,cAAc,IACZ,CAAC,EACH,WAAW,EACX,OAAO,EACP,SAAS,GACV,EAAE;IACD,WAAW,EAAE,KAAK,CAAC;IACnB,OAAO,EAAE,iBAAiB,CAAC,cAAc,CAAC,CAAC;IAC3C,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,KAAK,QAAQ,CAAC;AAEf,MAAM,MAAM,iCAAiC,GAAG,CAAC,EAC/C,UAAU,EACV,MAAM,GACP,EAAE;IACD,UAAU,EAAE,oBAAoB,CAAC;IACjC,MAAM,EAAE,yCAAyC,CAAC;CACnD,KAAK,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAE3C,MAAM,MAAM,wCAAwC,CAAC,QAAQ,EAAE,cAAc,IAC3E,CAAC,EACC,kBAAkB,EAClB,OAAO,EACP,MAAM,GACP,EAAE;IACD,kBAAkB,EAAE,QAAQ,CAAC;IAC7B,OAAO,EAAE,iBAAiB,CAAC,cAAc,CAAC,CAAC;IAC3C,MAAM,EAAE,yCAAyC,CAAC;CACnD,KAAK,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAE7C,MAAM,WAAW,kCAAkC,CACjD,cAAc,EACd,KAAK,EACL,QAAQ;IAER,OAAO,EAAE,uCAAuC,CAC9C,KAAK,EACL,QAAQ,EACR,cAAc,CACf,CAAC;IACF,QAAQ,EAAE,wCAAwC,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;CAC9E"}
|
package/dist/types/extraction.js
CHANGED
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.TimeValueType = exports.TimeUnit = exports.InitialSyncScope = exports.ExtractorEventType = exports.EventType = void 0;
|
|
3
|
+
exports.ItemInputType = exports.TimeValueType = exports.TimeUnit = exports.InitialSyncScope = exports.ExtractorEventType = exports.EventType = void 0;
|
|
4
4
|
/** Events sent from AirSync to the connector. */
|
|
5
5
|
var EventType;
|
|
6
6
|
(function (EventType) {
|
|
7
7
|
// Extraction
|
|
8
8
|
EventType["StartExtractingExternalSyncUnits"] = "START_EXTRACTING_EXTERNAL_SYNC_UNITS";
|
|
9
9
|
EventType["StartExtractingMetadata"] = "START_EXTRACTING_METADATA";
|
|
10
|
+
EventType["ContinueExtractingMetadata"] = "CONTINUE_EXTRACTING_METADATA";
|
|
10
11
|
EventType["StartExtractingData"] = "START_EXTRACTING_DATA";
|
|
11
12
|
EventType["ContinueExtractingData"] = "CONTINUE_EXTRACTING_DATA";
|
|
12
13
|
EventType["StartDeletingExtractorState"] = "START_DELETING_EXTRACTOR_STATE";
|
|
@@ -27,6 +28,8 @@ var ExtractorEventType;
|
|
|
27
28
|
// Extraction
|
|
28
29
|
ExtractorEventType["ExternalSyncUnitExtractionDone"] = "EXTERNAL_SYNC_UNIT_EXTRACTION_DONE";
|
|
29
30
|
ExtractorEventType["ExternalSyncUnitExtractionError"] = "EXTERNAL_SYNC_UNIT_EXTRACTION_ERROR";
|
|
31
|
+
ExtractorEventType["MetadataExtractionProgress"] = "METADATA_EXTRACTION_PROGRESS";
|
|
32
|
+
ExtractorEventType["MetadataExtractionDelayed"] = "METADATA_EXTRACTION_DELAYED";
|
|
30
33
|
ExtractorEventType["MetadataExtractionDone"] = "METADATA_EXTRACTION_DONE";
|
|
31
34
|
ExtractorEventType["MetadataExtractionError"] = "METADATA_EXTRACTION_ERROR";
|
|
32
35
|
ExtractorEventType["DataExtractionProgress"] = "DATA_EXTRACTION_PROGRESS";
|
|
@@ -72,3 +75,9 @@ var TimeValueType;
|
|
|
72
75
|
TimeValueType["ABSOLUTE_TIME"] = "absolute_time";
|
|
73
76
|
TimeValueType["UNBOUNDED"] = "unbounded";
|
|
74
77
|
})(TimeValueType || (exports.TimeValueType = TimeValueType = {}));
|
|
78
|
+
/** Sync-duration-estimation model input that a counted record type feeds into. */
|
|
79
|
+
var ItemInputType;
|
|
80
|
+
(function (ItemInputType) {
|
|
81
|
+
ItemInputType["MAIN"] = "main";
|
|
82
|
+
ItemInputType["USERS"] = "users";
|
|
83
|
+
})(ItemInputType || (exports.ItemInputType = ItemInputType = {}));
|
package/dist/types/loading.d.ts
CHANGED
|
@@ -40,31 +40,23 @@ export interface ExternalSystemItem {
|
|
|
40
40
|
modified_date: string;
|
|
41
41
|
data: any;
|
|
42
42
|
}
|
|
43
|
-
export interface ExternalSystemItem {
|
|
44
|
-
id: {
|
|
45
|
-
devrev: DonV2;
|
|
46
|
-
external?: string;
|
|
47
|
-
};
|
|
48
|
-
created_date: string;
|
|
49
|
-
modified_date: string;
|
|
50
|
-
data: any;
|
|
51
|
-
}
|
|
52
43
|
export interface ExternalSystemItemLoadingParams<Type> {
|
|
53
44
|
item: Type;
|
|
54
45
|
mappers: Mappers;
|
|
55
46
|
event: AirSyncEvent;
|
|
56
47
|
}
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
delay?: number;
|
|
62
|
-
}
|
|
63
|
-
export interface ExternalSystemItemLoadedItem {
|
|
64
|
-
id?: string;
|
|
65
|
-
error?: string;
|
|
48
|
+
/** Returned by the connector's loading `create`/`update` functions. */
|
|
49
|
+
export type ExternalSystemItemLoadingResponse = {
|
|
50
|
+
status: 'success';
|
|
51
|
+
id: string;
|
|
66
52
|
modifiedDate?: string;
|
|
67
|
-
}
|
|
53
|
+
} | {
|
|
54
|
+
status: 'delay';
|
|
55
|
+
delaySeconds: number;
|
|
56
|
+
} | {
|
|
57
|
+
status: 'error';
|
|
58
|
+
error: ErrorRecord;
|
|
59
|
+
};
|
|
68
60
|
export type ExternalSystemLoadingFunction<Item> = ({ item, mappers, event, }: ExternalSystemItemLoadingParams<Item>) => Promise<ExternalSystemItemLoadingResponse>;
|
|
69
61
|
export interface ItemTypeToLoad {
|
|
70
62
|
itemType: string;
|
|
@@ -90,10 +82,6 @@ export interface LoadItemResponse {
|
|
|
90
82
|
report?: LoaderReport;
|
|
91
83
|
rateLimit?: RateLimited;
|
|
92
84
|
}
|
|
93
|
-
export interface LoadItemTypesResponse {
|
|
94
|
-
reports: LoaderReport[];
|
|
95
|
-
processed_files: string[];
|
|
96
|
-
}
|
|
97
85
|
export declare enum ActionType {
|
|
98
86
|
CREATED = "created",
|
|
99
87
|
UPDATED = "updated",
|
|
@@ -102,13 +90,6 @@ export declare enum ActionType {
|
|
|
102
90
|
FAILED = "failed"
|
|
103
91
|
}
|
|
104
92
|
export type DonV2 = string;
|
|
105
|
-
export type SyncMapperRecord = {
|
|
106
|
-
external_ids: string[];
|
|
107
|
-
secondary_ids: string[];
|
|
108
|
-
devrev_ids: string[];
|
|
109
|
-
status: string[];
|
|
110
|
-
input_file?: string;
|
|
111
|
-
};
|
|
112
93
|
export declare enum LoaderEventType {
|
|
113
94
|
DataLoadingProgress = "DATA_LOADING_PROGRESS",
|
|
114
95
|
DataLoadingDelayed = "DATA_LOADING_DELAYED",
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"loading.d.ts","sourceRoot":"","sources":["../../src/types/loading.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAE7C,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE5C,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,WAAW,wBAAwB;IACvC,YAAY,EAAE,KAAK,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,mBAAmB,EAAE,KAAK,CAAC;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,kBAAkB;IACjC,EAAE,EAAE;QACF,MAAM,EAAE,KAAK,CAAC;QACd,QAAQ,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IAEtB,IAAI,EAAE,GAAG,CAAC;CACX;AAED,MAAM,WAAW
|
|
1
|
+
{"version":3,"file":"loading.d.ts","sourceRoot":"","sources":["../../src/types/loading.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAE7C,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE5C,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,WAAW,wBAAwB;IACvC,YAAY,EAAE,KAAK,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,mBAAmB,EAAE,KAAK,CAAC;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,kBAAkB;IACjC,EAAE,EAAE;QACF,MAAM,EAAE,KAAK,CAAC;QACd,QAAQ,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IAEtB,IAAI,EAAE,GAAG,CAAC;CACX;AAED,MAAM,WAAW,+BAA+B,CAAC,IAAI;IACnD,IAAI,EAAE,IAAI,CAAC;IACX,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,YAAY,CAAC;CACrB;AAED,uEAAuE;AACvE,MAAM,MAAM,iCAAiC,GACzC;IAAE,MAAM,EAAE,SAAS,CAAC;IAAC,EAAE,EAAE,MAAM,CAAC;IAAC,YAAY,CAAC,EAAE,MAAM,CAAA;CAAE,GACxD;IAAE,MAAM,EAAE,OAAO,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,GACzC;IAAE,MAAM,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,WAAW,CAAA;CAAE,CAAC;AAE5C,MAAM,MAAM,6BAA6B,CAAC,IAAI,IAAI,CAAC,EACjD,IAAI,EACJ,OAAO,EACP,KAAK,GACN,EAAE,+BAA+B,CAAC,IAAI,CAAC,KAAK,OAAO,CAAC,iCAAiC,CAAC,CAAC;AAExF,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,6BAA6B,CAAC,kBAAkB,CAAC,CAAC;IAC1D,MAAM,EAAE,6BAA6B,CAAC,kBAAkB,CAAC,CAAC;CAC3D;AAED,MAAM,WAAW,qBAAqB;IACpC,eAAe,EAAE,cAAc,EAAE,CAAC;CACnC;AAED,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,CAAC;IAC9B,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,CAAC;IAC9B,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,CAAC;IAC9B,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,CAAC;IAC9B,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,SAAS,CAAC,EAAE,WAAW,CAAC;CACzB;AAED,oBAAY,UAAU;IACpB,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,MAAM,WAAW;CAClB;AAED,MAAM,MAAM,KAAK,GAAG,MAAM,CAAC;AAE3B,oBAAY,eAAe;IACzB,mBAAmB,0BAA0B;IAC7C,kBAAkB,yBAAyB;IAC3C,eAAe,sBAAsB;IACrC,gBAAgB,uBAAuB;IAEvC,yBAAyB,gCAAgC;IACzD,wBAAwB,+BAA+B;IACvD,qBAAqB,4BAA4B;IACjD,sBAAsB,6BAA6B;IAEnD,uBAAuB,+BAA+B;IACtD,wBAAwB,gCAAgC;IAExD,iCAAiC,0CAA0C;IAC3E,kCAAkC,2CAA2C;CAC9E"}
|
package/dist/types/workers.d.ts
CHANGED
|
@@ -1,41 +1,44 @@
|
|
|
1
1
|
import { Worker } from 'worker_threads';
|
|
2
2
|
import type { LogLevel } from '../logger/logger.interfaces';
|
|
3
|
-
import { BaseState } from '../state/state';
|
|
4
3
|
import { ErrorRecord, InitialDomainMapping } from './common';
|
|
5
4
|
import { AirSyncEvent, EventType, ExtractorEventType } from './extraction';
|
|
6
5
|
import { LoaderEventType } from './loading';
|
|
7
|
-
export interface WorkerAdapterInterface<ConnectorState> {
|
|
8
|
-
event: AirSyncEvent;
|
|
9
|
-
adapterState: BaseState<ConnectorState>;
|
|
10
|
-
options?: WorkerAdapterOptions;
|
|
11
|
-
}
|
|
12
6
|
/** Parsed extraction scope from the platform, keyed by item type name. */
|
|
13
7
|
export type ExtractionScope = Record<string, {
|
|
14
8
|
extract: boolean;
|
|
15
9
|
}>;
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
/** Worker thread timeout. */
|
|
19
|
-
timeout?: number;
|
|
10
|
+
/** Per-repo overrides for `RepoInterface.overriddenOptions`. */
|
|
11
|
+
export interface RepoOptions {
|
|
20
12
|
/** Maximum number of extracted items in a batch. */
|
|
21
13
|
batchSize?: number;
|
|
22
|
-
|
|
23
|
-
|
|
14
|
+
}
|
|
15
|
+
export interface SpawnOptions extends RepoOptions {
|
|
16
|
+
/** Soft timeout in ms, capped at 10 minutes. */
|
|
17
|
+
timeout?: number;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Options the SDK threads from `spawn` into the worker. `isLocalDevelopment`
|
|
21
|
+
* is never set by connectors; it comes from AIRSYNC_LOCAL_DEVELOPMENT.
|
|
22
|
+
*/
|
|
23
|
+
export interface RuntimeOptions extends SpawnOptions {
|
|
24
|
+
isLocalDevelopment?: boolean;
|
|
24
25
|
}
|
|
25
26
|
export interface SpawnInterface {
|
|
26
27
|
event: AirSyncEvent;
|
|
27
28
|
worker: Worker;
|
|
28
|
-
options?:
|
|
29
|
+
options?: RuntimeOptions;
|
|
29
30
|
resolve: (value: void | PromiseLike<void>) => void;
|
|
30
31
|
originalConsole?: Console;
|
|
31
32
|
}
|
|
32
33
|
export interface SpawnFactoryInterface<ConnectorState> {
|
|
33
34
|
event: AirSyncEvent;
|
|
34
35
|
initialState: ConnectorState;
|
|
35
|
-
options?: WorkerAdapterOptions;
|
|
36
36
|
initialDomainMapping?: InitialDomainMapping;
|
|
37
|
-
/**
|
|
38
|
-
baseWorkerPath
|
|
37
|
+
/** Directory containing the `workers/` folder, usually `__dirname`. */
|
|
38
|
+
baseWorkerPath: string;
|
|
39
|
+
/** Worker paths relative to `baseWorkerPath`, keyed by incoming event type. */
|
|
40
|
+
workerPathOverrides?: WorkerPathOverrides;
|
|
41
|
+
options?: SpawnOptions;
|
|
39
42
|
}
|
|
40
43
|
/**
|
|
41
44
|
* Returned by a worker's `task`/`onTimeout` callback; the SDK (never the connector)
|
|
@@ -43,10 +46,11 @@ export interface SpawnFactoryInterface<ConnectorState> {
|
|
|
43
46
|
* One invocation = one worker = one emitted event; continuation happens in a
|
|
44
47
|
* fresh platform-driven invocation.
|
|
45
48
|
*
|
|
46
|
-
* Status
|
|
47
|
-
* 'progress'
|
|
48
|
-
* (data/attachment extraction and loading), but *_ERROR in
|
|
49
|
-
* phases (external sync units,
|
|
49
|
+
* Status to emitted event: 'success' → *_DONE; 'error' → *_ERROR;
|
|
50
|
+
* 'progress' → *_PROGRESS and 'delay' → *_DELAYED in resumable phases
|
|
51
|
+
* (metadata/data/attachment extraction and loading), but *_ERROR in
|
|
52
|
+
* non-resumable phases (external sync units, state deletion), where they are
|
|
53
|
+
* illegal.
|
|
50
54
|
*/
|
|
51
55
|
export type TaskResult = {
|
|
52
56
|
status: 'success';
|
|
@@ -66,7 +70,7 @@ export interface TaskAdapterInterface<Adapter> {
|
|
|
66
70
|
}
|
|
67
71
|
/**
|
|
68
72
|
* If `onTimeout` is omitted, the SDK emits a phase-appropriate default on
|
|
69
|
-
* timeout: progress for resumable phases, error for
|
|
73
|
+
* timeout: progress for resumable phases, error for non-resumable ones.
|
|
70
74
|
*/
|
|
71
75
|
export interface ProcessTaskInterface<Adapter> {
|
|
72
76
|
task: (params: TaskAdapterInterface<Adapter>) => Promise<TaskResult>;
|
|
@@ -113,11 +117,11 @@ export interface WorkerData<ConnectorState> {
|
|
|
113
117
|
initialState: ConnectorState;
|
|
114
118
|
workerPath: string;
|
|
115
119
|
initialDomainMapping?: InitialDomainMapping;
|
|
116
|
-
options?:
|
|
120
|
+
options?: RuntimeOptions;
|
|
117
121
|
}
|
|
118
122
|
export interface GetWorkerPathInterface {
|
|
119
123
|
event: AirSyncEvent;
|
|
120
|
-
workerBasePath
|
|
124
|
+
workerBasePath: string;
|
|
121
125
|
}
|
|
122
126
|
/** Maps event types to custom worker paths, overriding the defaults. */
|
|
123
127
|
export type WorkerPathOverrides = Partial<Record<EventType, string>>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"workers.d.ts","sourceRoot":"","sources":["../../src/types/workers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AAExC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,6BAA6B,CAAC;
|
|
1
|
+
{"version":3,"file":"workers.d.ts","sourceRoot":"","sources":["../../src/types/workers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AAExC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,6BAA6B,CAAC;AAE5D,OAAO,EAAE,WAAW,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AAC7D,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAC3E,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAE5C,0EAA0E;AAC1E,MAAM,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,EAAE;IAAE,OAAO,EAAE,OAAO,CAAA;CAAE,CAAC,CAAC;AAEnE,gEAAgE;AAChE,MAAM,WAAW,WAAW;IAC1B,oDAAoD;IACpD,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,YAAa,SAAQ,WAAW;IAC/C,gDAAgD;IAChD,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;GAGG;AACH,MAAM,WAAW,cAAe,SAAQ,YAAY;IAClD,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAED,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,YAAY,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,OAAO,EAAE,CAAC,KAAK,EAAE,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC;IACnD,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B;AAED,MAAM,WAAW,qBAAqB,CAAC,cAAc;IACnD,KAAK,EAAE,YAAY,CAAC;IACpB,YAAY,EAAE,cAAc,CAAC;IAC7B,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,uEAAuE;IACvE,cAAc,EAAE,MAAM,CAAC;IACvB,+EAA+E;IAC/E,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;IAC1C,OAAO,CAAC,EAAE,YAAY,CAAC;CACxB;AAED;;;;;;;;;;;GAWG;AACH,MAAM,MAAM,UAAU,GAClB;IAAE,MAAM,EAAE,SAAS,CAAA;CAAE,GACrB;IAAE,MAAM,EAAE,UAAU,CAAA;CAAE,GACtB;IAAE,MAAM,EAAE,OAAO,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,GACzC;IAAE,MAAM,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,WAAW,CAAA;CAAE,CAAC;AAE5C,MAAM,MAAM,UAAU,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;AAE9C,yEAAyE;AACzE,MAAM,WAAW,oBAAoB,CAAC,OAAO;IAC3C,OAAO,EAAE,OAAO,CAAC;CAClB;AAED;;;GAGG;AACH,MAAM,WAAW,oBAAoB,CAAC,OAAO;IAC3C,IAAI,EAAE,CAAC,MAAM,EAAE,oBAAoB,CAAC,OAAO,CAAC,KAAK,OAAO,CAAC,UAAU,CAAC,CAAC;IACrE,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,oBAAoB,CAAC,OAAO,CAAC,KAAK,OAAO,CAAC,UAAU,CAAC,CAAC;CAC5E;AAED,oBAAY,WAAW;IACrB,aAAa,YAAY;IACzB,YAAY,WAAW;IACvB,WAAW,UAAU;IACrB,UAAU,SAAS;CACpB;AAED,oBAAY,oBAAoB;IAC9B,oBAAoB,SAAS;IAC7B,iBAAiB,SAAS;IAC1B,gBAAgB,QAAQ;IACxB,mBAAmB,WAAW;CAC/B;AAED,MAAM,WAAW,oBAAoB;IACnC,OAAO,EAAE,oBAAoB,CAAC,oBAAoB,CAAC;IACnD,OAAO,EAAE;QACP,SAAS,EAAE,kBAAkB,GAAG,eAAe,CAAC;KACjD,CAAC;CACH;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,oBAAoB,CAAC,iBAAiB,CAAC;CACjD;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,oBAAoB,CAAC,gBAAgB,CAAC;IAC/C,OAAO,EAAE;QACP,eAAe,EAAE,MAAM,CAAC;QACxB,KAAK,EAAE,QAAQ,CAAC;KACjB,CAAC;CACH;AAED,wGAAwG;AACxG,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,oBAAoB,CAAC,mBAAmB,CAAC;IAClD,OAAO,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;CAC9B;AAED,MAAM,MAAM,aAAa,GACrB,oBAAoB,GACpB,iBAAiB,GACjB,gBAAgB,GAChB,mBAAmB,CAAC;AAExB,MAAM,WAAW,UAAU,CAAC,cAAc;IACxC,KAAK,EAAE,YAAY,CAAC;IACpB,YAAY,EAAE,cAAc,CAAC;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,OAAO,CAAC,EAAE,cAAc,CAAC;CAC1B;AAED,MAAM,WAAW,sBAAsB;IACrC,KAAK,EAAE,YAAY,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,wEAAwE;AACxE,MAAM,MAAM,mBAAmB,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC"}
|
|
@@ -9,7 +9,7 @@ export declare class Uploader {
|
|
|
9
9
|
private requestId;
|
|
10
10
|
private defaultHeaders;
|
|
11
11
|
private skipConfirmation;
|
|
12
|
-
constructor({ event, options }: UploaderFactoryInterface);
|
|
12
|
+
constructor({ event, options, skipConfirmation }: UploaderFactoryInterface);
|
|
13
13
|
/** Compresses fetched objects to gzipped JSONL and uploads them as an artifact. */
|
|
14
14
|
upload(itemType: string, fetchedObjects: object[] | object): Promise<UploadResponse>;
|
|
15
15
|
getArtifactUploadUrl(filename: string, fileType: string, fileSize?: number): Promise<UploaderResult<ArtifactToUpload>>;
|
|
@@ -21,7 +21,6 @@ export declare class Uploader {
|
|
|
21
21
|
response?: AxiosResponse;
|
|
22
22
|
error?: unknown;
|
|
23
23
|
}>;
|
|
24
|
-
private destroyStream;
|
|
25
24
|
/** Downloads an artifact and parses it into attachment metadata. */
|
|
26
25
|
getAttachmentsFromArtifactId({ artifact, }: {
|
|
27
26
|
artifact: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAOtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAOtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAWzD,OAAO,EAEL,gBAAgB,EAChB,wBAAwB,EACxB,cAAc,EACd,cAAc,EACf,MAAM,uBAAuB,CAAC;AAE/B,qBAAa,QAAQ;IACnB,OAAO,CAAC,kBAAkB,CAAC,CAAU;IACrC,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,cAAc,CAAyB;IAC/C,OAAO,CAAC,gBAAgB,CAAU;gBAEtB,EAAE,KAAK,EAAE,OAAO,EAAE,gBAAgB,EAAE,EAAE,wBAAwB;IAW1E,mFAAmF;IAC7E,MAAM,CACV,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,EAAE,GAAG,MAAM,GAChC,OAAO,CAAC,cAAc,CAAC;IAwEpB,oBAAoB,CACxB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;IA2B5C,mFAAmF;IAC7E,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAmBzC,kFAAkF;IAC5E,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,UAAU,EAAE,kBAAkB,GAC7B,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAqCnC,qBAAqB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC;QACvD,QAAQ,CAAC,EAAE,aAAa,CAAC;QACzB,KAAK,CAAC,EAAE,OAAO,CAAC;KACjB,CAAC;IAgCF,oEAAoE;IAC9D,4BAA4B,CAAC,EACjC,QAAQ,GACT,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;KAClB,GAAG,OAAO,CAAC;QACV,WAAW,CAAC,EAAE,oBAAoB,EAAE,CAAC;QACrC,KAAK,CAAC,EAAE;YAAE,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC;KAC7B,CAAC;YAuDY,sBAAsB;YAsBtB,gBAAgB;IAc9B,yEAAyE;IACnE,yBAAyB,CAAC,EAC9B,UAAU,EACV,SAAiB,GAClB,EAAE;QACD,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,CAAC,EAAE,OAAO,CAAC;KACrB,GAAG,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC;CA6B/C"}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { HttpStreamResponse } from '../types/extraction';
|
|
1
2
|
import { ArtifactDateRanges, UploaderResult } from './uploader.interfaces';
|
|
2
3
|
/** Computes oldest/newest created and modified timestamps (RFC3339) across uploaded items. */
|
|
3
4
|
export declare function computeArtifactDateRanges(fetchedObjects: object[] | object): ArtifactDateRanges;
|
|
@@ -7,4 +8,6 @@ export declare function parseJsonl(jsonlObject: string): UploaderResult<object[]
|
|
|
7
8
|
/** Writes fetched objects to the local file system (local development only). */
|
|
8
9
|
export declare function downloadToLocal(itemType: string, fetchedObjects: object | object[]): Promise<void>;
|
|
9
10
|
export declare function truncateFilename(filename: string): string;
|
|
11
|
+
/** Destroys a stream to prevent memory leaks. */
|
|
12
|
+
export declare function destroyHttpStream(httpStream: HttpStreamResponse): void;
|
|
10
13
|
//# sourceMappingURL=uploader.helpers.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"uploader.helpers.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.helpers.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"uploader.helpers.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.helpers.ts"],"names":[],"mappings":"AAUA,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAEzD,OAAO,EAEL,kBAAkB,EAClB,cAAc,EACf,MAAM,uBAAuB,CAAC;AAE/B,8FAA8F;AAC9F,wBAAgB,yBAAyB,CACvC,cAAc,EAAE,MAAM,EAAE,GAAG,MAAM,GAChC,kBAAkB,CAwEpB;AAED,wBAAgB,YAAY,CAAC,WAAW,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,CAMxE;AAED,wBAAgB,cAAc,CAC5B,kBAAkB,EAAE,MAAM,GACzB,cAAc,CAAC,MAAM,CAAC,CAOxB;AAED,wBAAgB,UAAU,CAAC,WAAW,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,CAMxE;AAED,gFAAgF;AAChF,wBAAsB,eAAe,CACnC,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,GAAG,MAAM,EAAE,GAChC,OAAO,CAAC,IAAI,CAAC,CA4Bf;AAED,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAgBzD;AAED,iDAAiD;AACjD,wBAAgB,iBAAiB,CAAC,UAAU,EAAE,kBAAkB,GAAG,IAAI,CAUtE"}
|
|
@@ -42,6 +42,7 @@ exports.decompressGzip = decompressGzip;
|
|
|
42
42
|
exports.parseJsonl = parseJsonl;
|
|
43
43
|
exports.downloadToLocal = downloadToLocal;
|
|
44
44
|
exports.truncateFilename = truncateFilename;
|
|
45
|
+
exports.destroyHttpStream = destroyHttpStream;
|
|
45
46
|
const fs_1 = __importStar(require("fs"));
|
|
46
47
|
const zlib_1 = __importDefault(require("zlib"));
|
|
47
48
|
const js_jsonl_1 = require("js-jsonl");
|
|
@@ -171,3 +172,18 @@ function truncateFilename(filename) {
|
|
|
171
172
|
const truncatedFilename = filename.slice(0, availableNameLength);
|
|
172
173
|
return `${truncatedFilename}...${extension}`;
|
|
173
174
|
}
|
|
175
|
+
/** Destroys a stream to prevent memory leaks. */
|
|
176
|
+
function destroyHttpStream(httpStream) {
|
|
177
|
+
var _a, _b;
|
|
178
|
+
try {
|
|
179
|
+
if (typeof ((_a = httpStream === null || httpStream === void 0 ? void 0 : httpStream.data) === null || _a === void 0 ? void 0 : _a.destroy) === 'function') {
|
|
180
|
+
httpStream.data.destroy();
|
|
181
|
+
}
|
|
182
|
+
else if (typeof ((_b = httpStream === null || httpStream === void 0 ? void 0 : httpStream.data) === null || _b === void 0 ? void 0 : _b.close) === 'function') {
|
|
183
|
+
httpStream.data.close();
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
catch (error) {
|
|
187
|
+
console.warn('Error while destroying stream:', error);
|
|
188
|
+
}
|
|
189
|
+
}
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import { ErrorRecord } from '../types/common';
|
|
2
2
|
import { AirSyncEvent } from '../types/extraction';
|
|
3
|
-
import {
|
|
4
|
-
import { WorkerAdapterOptions } from '../types/workers';
|
|
3
|
+
import { RuntimeOptions } from '../types/workers';
|
|
5
4
|
export interface UploaderFactoryInterface {
|
|
6
5
|
event: AirSyncEvent;
|
|
7
|
-
options?:
|
|
6
|
+
options?: RuntimeOptions;
|
|
7
|
+
/** Skips confirm-upload; ESU uploads happen before a sync exists to attach them to. */
|
|
8
|
+
skipConfirmation?: boolean;
|
|
8
9
|
}
|
|
9
10
|
export type UploaderResult<T> = {
|
|
10
11
|
response: T;
|
|
@@ -30,15 +31,6 @@ export interface Artifact {
|
|
|
30
31
|
[ArtifactDateField.NewestModifiedDate]?: string;
|
|
31
32
|
}
|
|
32
33
|
export type ArtifactDateRanges = Pick<Artifact, ArtifactDateField.OldestCreatedDate | ArtifactDateField.NewestCreatedDate | ArtifactDateField.OldestModifiedDate | ArtifactDateField.NewestModifiedDate>;
|
|
33
|
-
/** Response from the prepare artifacts endpoint. */
|
|
34
|
-
export interface ArtifactsPrepareResponse {
|
|
35
|
-
url: string;
|
|
36
|
-
id: string;
|
|
37
|
-
form_data: {
|
|
38
|
-
key: string;
|
|
39
|
-
value: string;
|
|
40
|
-
}[];
|
|
41
|
-
}
|
|
42
34
|
/** Response from the get upload url endpoint. */
|
|
43
35
|
export interface ArtifactToUpload {
|
|
44
36
|
upload_url: string;
|
|
@@ -52,15 +44,6 @@ export interface UploadResponse {
|
|
|
52
44
|
artifact?: Artifact;
|
|
53
45
|
error?: ErrorRecord;
|
|
54
46
|
}
|
|
55
|
-
export interface StreamAttachmentsResponse {
|
|
56
|
-
ssorAttachments?: SsorAttachment[];
|
|
57
|
-
error?: ErrorRecord;
|
|
58
|
-
}
|
|
59
|
-
/** Result of streaming a single attachment through Uploader. */
|
|
60
|
-
export interface StreamResponse {
|
|
61
|
-
ssorAttachment?: SsorAttachment;
|
|
62
|
-
error?: ErrorRecord;
|
|
63
|
-
}
|
|
64
47
|
export interface SsorAttachment {
|
|
65
48
|
id: {
|
|
66
49
|
devrev: string;
|
|
@@ -74,12 +57,4 @@ export interface SsorAttachment {
|
|
|
74
57
|
};
|
|
75
58
|
inline?: boolean;
|
|
76
59
|
}
|
|
77
|
-
export interface StatsFileResponse {
|
|
78
|
-
error?: ErrorRecord;
|
|
79
|
-
statsFile?: StatsFileObject[];
|
|
80
|
-
}
|
|
81
|
-
export interface TransformerFileResponse {
|
|
82
|
-
error?: ErrorRecord;
|
|
83
|
-
transformerFile?: ExternalSystemItem[];
|
|
84
|
-
}
|
|
85
60
|
//# sourceMappingURL=uploader.interfaces.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"uploader.interfaces.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"uploader.interfaces.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAElD,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,YAAY,CAAC;IACpB,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,uFAAuF;IACvF,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,MAAM,cAAc,CAAC,CAAC,IACxB;IAAE,QAAQ,EAAE,CAAC,CAAC;IAAC,KAAK,CAAC,EAAE,KAAK,CAAA;CAAE,GAC9B;IAAE,QAAQ,CAAC,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE,CAAC;AAEzC,oBAAY,iBAAiB;IAC3B,iBAAiB,wBAAwB;IACzC,iBAAiB,wBAAwB;IACzC,kBAAkB,yBAAyB;IAC3C,kBAAkB,yBAAyB;CAC5C;AAED,iEAAiE;AACjE,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,CAAC,EAAE,MAAM,CAAC;IAC/C,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,CAAC,EAAE,MAAM,CAAC;IAC/C,CAAC,iBAAiB,CAAC,kBAAkB,CAAC,CAAC,EAAE,MAAM,CAAC;IAChD,CAAC,iBAAiB,CAAC,kBAAkB,CAAC,CAAC,EAAE,MAAM,CAAC;CACjD;AAED,MAAM,MAAM,kBAAkB,GAAG,IAAI,CACnC,QAAQ,EACN,iBAAiB,CAAC,iBAAiB,GACnC,iBAAiB,CAAC,iBAAiB,GACnC,iBAAiB,CAAC,kBAAkB,GACpC,iBAAiB,CAAC,kBAAkB,CACvC,CAAC;AAEF,iDAAiD;AACjD,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE;QACT,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;KACf,EAAE,CAAC;CACL;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE;QACF,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,SAAS,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,QAAQ,CAAC,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB"}
|
|
@@ -11,13 +11,12 @@ const client_1 = require("../http/client");
|
|
|
11
11
|
const logger_1 = require("../logger/logger");
|
|
12
12
|
const uploader_helpers_1 = require("./uploader.helpers");
|
|
13
13
|
class Uploader {
|
|
14
|
-
constructor({ event, options }) {
|
|
15
|
-
var _a;
|
|
14
|
+
constructor({ event, options, skipConfirmation }) {
|
|
16
15
|
this.devrevApiEndpoint = event.execution_metadata.devrev_endpoint;
|
|
17
16
|
this.devrevApiToken = event.context.secrets.service_account_token;
|
|
18
17
|
this.requestId = event.payload.event_context.request_id;
|
|
19
18
|
this.isLocalDevelopment = options === null || options === void 0 ? void 0 : options.isLocalDevelopment;
|
|
20
|
-
this.skipConfirmation =
|
|
19
|
+
this.skipConfirmation = skipConfirmation !== null && skipConfirmation !== void 0 ? skipConfirmation : false;
|
|
21
20
|
this.defaultHeaders = {
|
|
22
21
|
Authorization: `Bearer ${this.devrevApiToken}`,
|
|
23
22
|
};
|
|
@@ -56,8 +55,6 @@ class Uploader {
|
|
|
56
55
|
},
|
|
57
56
|
};
|
|
58
57
|
}
|
|
59
|
-
// Skip confirmation for External Sync Units: confirmation attaches the upload
|
|
60
|
-
// to the sync, which doesn't exist yet when extracting External Sync Units.
|
|
61
58
|
if (!this.skipConfirmation) {
|
|
62
59
|
const { error: confirmArtifactUploadError } = await this.confirmArtifactUpload(preparedArtifact.artifact_id);
|
|
63
60
|
if (confirmArtifactUploadError) {
|
|
@@ -131,11 +128,11 @@ class Uploader {
|
|
|
131
128
|
maxRedirects: 0,
|
|
132
129
|
// 2xx and 3xx are success; 4xx/5xx throw into the catch block
|
|
133
130
|
validateStatus: (status) => status >= 200 && status < 400 }, (!hasContentLength ? { 'axios-retry': { retries: 0 } } : {})));
|
|
134
|
-
|
|
131
|
+
(0, uploader_helpers_1.destroyHttpStream)(fileStream);
|
|
135
132
|
return { response };
|
|
136
133
|
}
|
|
137
134
|
catch (error) {
|
|
138
|
-
|
|
135
|
+
(0, uploader_helpers_1.destroyHttpStream)(fileStream);
|
|
139
136
|
return { error };
|
|
140
137
|
}
|
|
141
138
|
}
|
|
@@ -164,22 +161,6 @@ class Uploader {
|
|
|
164
161
|
return { error: { message: (0, logger_1.serializeError)(error) } };
|
|
165
162
|
}
|
|
166
163
|
}
|
|
167
|
-
destroyStream(fileStream) {
|
|
168
|
-
try {
|
|
169
|
-
if (fileStream && fileStream.data) {
|
|
170
|
-
// For axios response streams, `data` holds the actual stream
|
|
171
|
-
if (typeof fileStream.data.destroy === 'function') {
|
|
172
|
-
fileStream.data.destroy();
|
|
173
|
-
}
|
|
174
|
-
else if (typeof fileStream.data.close === 'function') {
|
|
175
|
-
fileStream.data.close();
|
|
176
|
-
}
|
|
177
|
-
}
|
|
178
|
-
}
|
|
179
|
-
catch (error) {
|
|
180
|
-
console.warn('Error while destroying stream:', (0, logger_1.serializeError)(error));
|
|
181
|
-
}
|
|
182
|
-
}
|
|
183
164
|
/** Downloads an artifact and parses it into attachment metadata. */
|
|
184
165
|
async getAttachmentsFromArtifactId({ artifact, }) {
|
|
185
166
|
const { response: artifactUrl, error: artifactUrlError } = await this.getArtifactDownloadUrl(artifact);
|