@devrev/airsync-sdk 2.1.0-beta.3 → 2.2.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/multithreading/adapters/extraction-adapter.d.ts +9 -1
- package/dist/multithreading/adapters/extraction-adapter.d.ts.map +1 -1
- package/dist/multithreading/adapters/extraction-adapter.js +50 -7
- package/dist/types/extraction.d.ts +4 -3
- package/dist/types/extraction.d.ts.map +1 -1
- package/dist/uploader/uploader.d.ts +3 -1
- package/dist/uploader/uploader.d.ts.map +1 -1
- package/dist/uploader/uploader.interfaces.d.ts +13 -0
- package/dist/uploader/uploader.interfaces.d.ts.map +1 -1
- package/dist/uploader/uploader.js +26 -2
- package/docs/MIGRATION.md +37 -9
- package/package.json +1 -1
package/dist/index.d.ts
CHANGED
|
@@ -20,7 +20,7 @@ export type { Item, NormalizedAttachment, NormalizedItem, RepoInterface, } from
|
|
|
20
20
|
export { Mappers } from './mappers/mappers';
|
|
21
21
|
export type { MappersCreateParams, MappersCreateResponse, MappersFactoryInterface, MappersGetByExternalIdParams, MappersGetByExternalIdResponse, MappersGetByTargetIdParams, MappersGetByTargetIdResponse, MappersUpdateParams, MappersUpdateResponse, SyncMapperRecord, SyncMapperRecordExternalVersion, UpdateSyncMapperRecordParams, } from './mappers/mappers.interfaces';
|
|
22
22
|
export { SyncMapperRecordStatus, SyncMapperRecordTargetType, } from './mappers/mappers.interfaces';
|
|
23
|
-
export type { Artifact, SsorAttachment, UploadResponse, } from './uploader/uploader.interfaces';
|
|
23
|
+
export type { Artifact, ExtractionFileFormat, SsorAttachment, UploadExtractionFileParams, UploadResponse, } from './uploader/uploader.interfaces';
|
|
24
24
|
export { ArtifactDateField } from './uploader/uploader.interfaces';
|
|
25
25
|
export type { Attachments, CollectionData, ConditionalPrivilegeData, CustomLinkData, CustomLinkNames, CustomStage, CustomState, EnumData, EnumValue, EnumValueKey, ExternalDomainMetadata, Field, FieldCondition, FieldConditionComparator, FieldConditionEffect, FieldConditions, FieldKey, FieldPrivilegeData, FieldReferenceData, FieldType, FloatData, IntData, ParticipationData, PermissionData, RecordType, RecordTypeCategory, RecordTypeCategoryKey, RecordTypeKey, RecordTypePrivilegeData, RecordTypeScope, ReferenceData, ReferenceDetail, ReferenceType, SchemaVersion, StageDiagram, StageKey, StateKey, StructData, StructType, StructTypeKey, TargetTypeKeyData, TextData, TypedReferenceData, } from './types/external-domain-metadata';
|
|
26
26
|
export type { DeepPartial } from './testing/mock-event';
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAQA,YAAY,EAAE,WAAW,EAAE,MAAM,wCAAwC,CAAC;AAC1E,YAAY,EAAE,iBAAiB,EAAE,MAAM,8CAA8C,CAAC;AACtF,YAAY,EAAE,cAAc,EAAE,MAAM,2CAA2C,CAAC;AAChF,OAAO,EACL,qBAAqB,EACrB,kBAAkB,GACnB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,KAAK,EAAE,MAAM,8BAA8B,CAAC;AACrD,YAAY,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AAGxC,YAAY,EACV,gBAAgB,EAChB,iBAAiB,EACjB,eAAe,EACf,oBAAoB,EACpB,WAAW,EACX,qBAAqB,EACrB,YAAY,EACZ,oBAAoB,EACpB,UAAU,EACV,UAAU,EACV,mBAAmB,GACpB,MAAM,iBAAiB,CAAC;AAGzB,OAAO,EACL,uBAAuB,EACvB,yBAAyB,GAC1B,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAGlD,OAAO,EAAE,2BAA2B,EAAE,MAAM,wCAAwC,CAAC;AAGrF,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAGjD,YAAY,EAAE,WAAW,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AACxE,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAG1C,YAAY,EACV,YAAY,EACZ,cAAc,EACd,cAAc,EACd,YAAY,EACZ,SAAS,EACT,iCAAiC,EACjC,gBAAgB,EAChB,wCAAwC,EACxC,kCAAkC,EAClC,uCAAuC,EACvC,yCAAyC,EACzC,uCAAuC,EACvC,yCAAyC,EACzC,cAAc,EACd,kBAAkB,EAClB,SAAS,EACT,aAAa,EACb,2BAA2B,EAC3B,SAAS,EACT,cAAc,GACf,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,SAAS,EACT,gBAAgB,EAChB,aAAa,EACb,YAAY,EACZ,QAAQ,EACR,aAAa,GACd,MAAM,oBAAoB,CAAC;AAG5B,YAAY,EACV,wBAAwB,EACxB,kBAAkB,EAClB,+BAA+B,EAC/B,iCAAiC,EACjC,6BAA6B,EAC7B,UAAU,EACV,qBAAqB,EACrB,cAAc,EACd,YAAY,GACb,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAG7C,YAAY,EACV,IAAI,EACJ,oBAAoB,EACpB,cAAc,EACd,aAAa,GACd,MAAM,wBAAwB,CAAC;AAGhC,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAC5C,YAAY,EACV,mBAAmB,EACnB,qBAAqB,EACrB,uBAAuB,EACvB,4BAA4B,EAC5B,8BAA8B,EAC9B,0BAA0B,EAC1B,4BAA4B,EAC5B,mBAAmB,EACnB,qBAAqB,EACrB,gBAAgB,EAChB,+BAA+B,EAC/B,4BAA4B,GAC7B,MAAM,8BAA8B,CAAC;AACtC,OAAO,EACL,sBAAsB,EACtB,0BAA0B,GAC3B,MAAM,8BAA8B,CAAC;AAGtC,YAAY,EACV,QAAQ,EACR,cAAc,EACd,cAAc,GACf,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAAE,iBAAiB,EAAE,MAAM,gCAAgC,CAAC;AAGnE,YAAY,EACV,WAAW,EACX,cAAc,EACd,wBAAwB,EACxB,cAAc,EACd,eAAe,EACf,WAAW,EACX,WAAW,EACX,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,sBAAsB,EACtB,KAAK,EACL,cAAc,EACd,wBAAwB,EACxB,oBAAoB,EACpB,eAAe,EACf,QAAQ,EACR,kBAAkB,EAClB,kBAAkB,EAClB,SAAS,EACT,SAAS,EACT,OAAO,EACP,iBAAiB,EACjB,cAAc,EACd,UAAU,EACV,kBAAkB,EAClB,qBAAqB,EACrB,aAAa,EACb,uBAAuB,EACvB,eAAe,EACf,aAAa,EACb,eAAe,EACf,aAAa,EACb,aAAa,EACb,YAAY,EACZ,QAAQ,EACR,QAAQ,EACR,UAAU,EACV,UAAU,EACV,aAAa,EACb,iBAAiB,EACjB,QAAQ,EACR,kBAAkB,GACnB,MAAM,kCAAkC,CAAC;AAG1C,YAAY,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,EAAE,uBAAuB,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAC5E,YAAY,EACV,WAAW,EACX,WAAW,EACX,WAAW,GACZ,MAAM,kCAAkC,CAAC;AAC1C,YAAY,EACV,iBAAiB,EACjB,wBAAwB,GACzB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACL,2BAA2B,EAC3B,wBAAwB,GACzB,MAAM,yBAAyB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAQA,YAAY,EAAE,WAAW,EAAE,MAAM,wCAAwC,CAAC;AAC1E,YAAY,EAAE,iBAAiB,EAAE,MAAM,8CAA8C,CAAC;AACtF,YAAY,EAAE,cAAc,EAAE,MAAM,2CAA2C,CAAC;AAChF,OAAO,EACL,qBAAqB,EACrB,kBAAkB,GACnB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,KAAK,EAAE,MAAM,8BAA8B,CAAC;AACrD,YAAY,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AAGxC,YAAY,EACV,gBAAgB,EAChB,iBAAiB,EACjB,eAAe,EACf,oBAAoB,EACpB,WAAW,EACX,qBAAqB,EACrB,YAAY,EACZ,oBAAoB,EACpB,UAAU,EACV,UAAU,EACV,mBAAmB,GACpB,MAAM,iBAAiB,CAAC;AAGzB,OAAO,EACL,uBAAuB,EACvB,yBAAyB,GAC1B,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAGlD,OAAO,EAAE,2BAA2B,EAAE,MAAM,wCAAwC,CAAC;AAGrF,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAGjD,YAAY,EAAE,WAAW,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AACxE,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAG1C,YAAY,EACV,YAAY,EACZ,cAAc,EACd,cAAc,EACd,YAAY,EACZ,SAAS,EACT,iCAAiC,EACjC,gBAAgB,EAChB,wCAAwC,EACxC,kCAAkC,EAClC,uCAAuC,EACvC,yCAAyC,EACzC,uCAAuC,EACvC,yCAAyC,EACzC,cAAc,EACd,kBAAkB,EAClB,SAAS,EACT,aAAa,EACb,2BAA2B,EAC3B,SAAS,EACT,cAAc,GACf,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,SAAS,EACT,gBAAgB,EAChB,aAAa,EACb,YAAY,EACZ,QAAQ,EACR,aAAa,GACd,MAAM,oBAAoB,CAAC;AAG5B,YAAY,EACV,wBAAwB,EACxB,kBAAkB,EAClB,+BAA+B,EAC/B,iCAAiC,EACjC,6BAA6B,EAC7B,UAAU,EACV,qBAAqB,EACrB,cAAc,EACd,YAAY,GACb,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAG7C,YAAY,EACV,IAAI,EACJ,oBAAoB,EACpB,cAAc,EACd,aAAa,GACd,MAAM,wBAAwB,CAAC;AAGhC,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAC5C,YAAY,EACV,mBAAmB,EACnB,qBAAqB,EACrB,uBAAuB,EACvB,4BAA4B,EAC5B,8BAA8B,EAC9B,0BAA0B,EAC1B,4BAA4B,EAC5B,mBAAmB,EACnB,qBAAqB,EACrB,gBAAgB,EAChB,+BAA+B,EAC/B,4BAA4B,GAC7B,MAAM,8BAA8B,CAAC;AACtC,OAAO,EACL,sBAAsB,EACtB,0BAA0B,GAC3B,MAAM,8BAA8B,CAAC;AAGtC,YAAY,EACV,QAAQ,EACR,oBAAoB,EACpB,cAAc,EACd,0BAA0B,EAC1B,cAAc,GACf,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAAE,iBAAiB,EAAE,MAAM,gCAAgC,CAAC;AAGnE,YAAY,EACV,WAAW,EACX,cAAc,EACd,wBAAwB,EACxB,cAAc,EACd,eAAe,EACf,WAAW,EACX,WAAW,EACX,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,sBAAsB,EACtB,KAAK,EACL,cAAc,EACd,wBAAwB,EACxB,oBAAoB,EACpB,eAAe,EACf,QAAQ,EACR,kBAAkB,EAClB,kBAAkB,EAClB,SAAS,EACT,SAAS,EACT,OAAO,EACP,iBAAiB,EACjB,cAAc,EACd,UAAU,EACV,kBAAkB,EAClB,qBAAqB,EACrB,aAAa,EACb,uBAAuB,EACvB,eAAe,EACf,aAAa,EACb,eAAe,EACf,aAAa,EACb,aAAa,EACb,YAAY,EACZ,QAAQ,EACR,QAAQ,EACR,UAAU,EACV,UAAU,EACV,aAAa,EACb,iBAAiB,EACjB,QAAQ,EACR,kBAAkB,GACnB,MAAM,kCAAkC,CAAC;AAG1C,YAAY,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,EAAE,uBAAuB,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAC5E,YAAY,EACV,WAAW,EACX,WAAW,EACX,WAAW,GACZ,MAAM,kCAAkC,CAAC;AAC1C,YAAY,EACV,iBAAiB,EACjB,wBAAwB,GACzB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACL,2BAA2B,EAC3B,wBAAwB,GACzB,MAAM,yBAAyB,CAAC"}
|
|
@@ -2,7 +2,7 @@ import { Repo } from '../../repo/repo';
|
|
|
2
2
|
import { NormalizedAttachment, RepoInterface } from '../../repo/repo.interfaces';
|
|
3
3
|
import { ExternalSystemAttachmentProcessors, ExternalSystemAttachmentStreamingFunction, ItemTypeCount, ProcessAttachmentReturnType } from '../../types/extraction';
|
|
4
4
|
import { TaskResult } from '../../types/workers';
|
|
5
|
-
import { Artifact } from '../../uploader/uploader.interfaces';
|
|
5
|
+
import { Artifact, UploadExtractionFileParams } from '../../uploader/uploader.interfaces';
|
|
6
6
|
import { BaseAdapter } from './base-adapter';
|
|
7
7
|
/** Adapter passed to extraction tasks (repos, artifacts, attachment streaming). @public */
|
|
8
8
|
export declare class ExtractionAdapter<ConnectorState> extends BaseAdapter<ConnectorState> {
|
|
@@ -25,6 +25,14 @@ export declare class ExtractionAdapter<ConnectorState> extends BaseAdapter<Conne
|
|
|
25
25
|
getRepo(itemType: string): Repo | undefined;
|
|
26
26
|
/** Artifacts uploaded so far that the next emit will report. */
|
|
27
27
|
get artifacts(): readonly Artifact[];
|
|
28
|
+
/**
|
|
29
|
+
* Uploads and confirms a ready-made Parquet dataset file during data extraction,
|
|
30
|
+
* then registers it for the next callback (including progress and errors).
|
|
31
|
+
* Bypasses repo normalization, batching and timestamp tracking. The connector
|
|
32
|
+
* supplies valid Parquet and its actual row count. Throws on failure.
|
|
33
|
+
*/
|
|
34
|
+
uploadParquetExtractionFile(params: UploadExtractionFileParams<'parquet'>): Promise<Artifact>;
|
|
35
|
+
private trackArtifactSize;
|
|
28
36
|
private addArtifacts;
|
|
29
37
|
/**
|
|
30
38
|
* Per-record-type counts for sync-duration estimation. Set during the
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extraction-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/extraction-adapter.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"extraction-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/extraction-adapter.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AACvC,OAAO,EACL,oBAAoB,EACpB,aAAa,EACd,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EAIL,kCAAkC,EAClC,yCAAyC,EAEzC,aAAa,EACb,2BAA2B,EAE5B,MAAM,wBAAwB,CAAC;AAEhC,OAAO,EAAkB,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjE,OAAO,EACL,QAAQ,EAER,0BAA0B,EAC3B,MAAM,oCAAoC,CAAC;AAE5C,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,2FAA2F;AAC3F,qBAAa,iBAAiB,CAC5B,cAAc,CACd,SAAQ,WAAW,CAAC,cAAc,CAAC;IACnC,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,KAAK,CAAc;IAC3B,OAAO,CAAC,wBAAwB,CAAC,CAAkB;IACnD,OAAO,CAAC,qBAAqB,CAAC,CAAS;IACvC,OAAO,CAAC,sBAAsB,CAAa;IAY3C,6EAA6E;IAC7E,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAOxC,eAAe,CAAC,KAAK,EAAE,aAAa,EAAE;IAatC,OAAO,CAAC,UAAU;IA4ClB;;;OAGG;IACH,OAAO,CAAC,cAAc;IAItB,OAAO,CAAC,kBAAkB;IAQ1B,OAAO,CAAC,mBAAmB;IAS3B,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS;IAW3C,gEAAgE;IAChE,IAAI,SAAS,IAAI,SAAS,QAAQ,EAAE,CAEnC;IAED;;;;;OAKG;IACG,2BAA2B,CAC/B,MAAM,EAAE,0BAA0B,CAAC,SAAS,CAAC,GAC5C,OAAO,CAAC,QAAQ,CAAC;IAiDpB,OAAO,CAAC,iBAAiB;IAczB,OAAO,CAAC,YAAY;IAMpB;;;OAGG;IACH,IAAI,uBAAuB,IAAI,aAAa,EAAE,GAAG,SAAS,CAEzD;IAED,IAAI,uBAAuB,CAAC,MAAM,EAAE,aAAa,EAAE,GAAG,SAAS,EAE9D;IA6HD,OAAO,CAAC,kBAAkB;YAaZ,cAAc;IAmB5B;;;;OAIG;IACG,iBAAiB,CACrB,UAAU,EAAE,oBAAoB,EAChC,MAAM,EAAE,yCAAyC,GAChD,OAAO,CAAC,2BAA2B,CAAC;IA4IjC,iBAAiB,CAAC,QAAQ,EAAE,EAChC,MAAM,EACN,UAAU,EACV,SAAa,GACd,EAAE;QACD,MAAM,EAAE,yCAAyC,CAAC;QAClD,UAAU,CAAC,EAAE,kCAAkC,CAC7C,cAAc,EACd,oBAAoB,EAAE,EACtB,QAAQ,CACT,CAAC;QACF,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,GAAG,OAAO,CAAC,UAAU,CAAC;CAsHxB"}
|
|
@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.ExtractionAdapter = void 0;
|
|
4
4
|
const attachments_streaming_pool_1 = require("../../attachments-streaming/attachments-streaming-pool");
|
|
5
5
|
const constants_1 = require("../../common/constants");
|
|
6
|
+
const errors_1 = require("../../http/errors");
|
|
6
7
|
const logger_1 = require("../../logger/logger");
|
|
7
8
|
const repo_1 = require("../../repo/repo");
|
|
8
9
|
const extraction_1 = require("../../types/extraction");
|
|
@@ -57,13 +58,7 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
|
|
|
57
58
|
if (repo.itemType === constants_1.AirSyncDefaultItemTypes.ATTACHMENTS) {
|
|
58
59
|
(_b = this.adapterState.sdkState.toDevRev) === null || _b === void 0 ? void 0 : _b.attachmentsMetadata.artifactIds.push(artifact.id);
|
|
59
60
|
}
|
|
60
|
-
|
|
61
|
-
// isTimeout once the threshold is exceeded to force an early emit.
|
|
62
|
-
this.currentEventDataLength += Buffer.byteLength(JSON.stringify(artifact), 'utf8');
|
|
63
|
-
if (this.currentEventDataLength > constants_1.EVENT_SIZE_THRESHOLD_BYTES &&
|
|
64
|
-
!this.isTimeout) {
|
|
65
|
-
this.isTimeout = true;
|
|
66
|
-
}
|
|
61
|
+
this.trackArtifactSize(artifact);
|
|
67
62
|
}, options: Object.assign(Object.assign({}, this.options), repo.overriddenOptions) }));
|
|
68
63
|
}
|
|
69
64
|
/**
|
|
@@ -98,6 +93,54 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
|
|
|
98
93
|
get artifacts() {
|
|
99
94
|
return this._artifacts;
|
|
100
95
|
}
|
|
96
|
+
/**
|
|
97
|
+
* Uploads and confirms a ready-made Parquet dataset file during data extraction,
|
|
98
|
+
* then registers it for the next callback (including progress and errors).
|
|
99
|
+
* Bypasses repo normalization, batching and timestamp tracking. The connector
|
|
100
|
+
* supplies valid Parquet and its actual row count. Throws on failure.
|
|
101
|
+
*/
|
|
102
|
+
async uploadParquetExtractionFile(params) {
|
|
103
|
+
const eventType = this.event.payload.event_type;
|
|
104
|
+
if (eventType !== extraction_1.EventType.StartExtractingData &&
|
|
105
|
+
eventType !== extraction_1.EventType.ContinueExtractingData) {
|
|
106
|
+
throw new Error('Extraction files can only be uploaded during data extraction.');
|
|
107
|
+
}
|
|
108
|
+
if (!params.itemType.trim() ||
|
|
109
|
+
Object.values(constants_1.AirSyncDefaultItemTypes).some((type) => type === params.itemType) ||
|
|
110
|
+
[
|
|
111
|
+
'initial_domain_mapping',
|
|
112
|
+
'query_result_metadata',
|
|
113
|
+
constants_1.SSOR_ATTACHMENT,
|
|
114
|
+
].includes(params.itemType)) {
|
|
115
|
+
throw new Error('itemType must be a dataset/query key, not a metadata or attachment item type.');
|
|
116
|
+
}
|
|
117
|
+
if (params.format !== 'parquet') {
|
|
118
|
+
throw new Error('Only parquet extraction files are supported.');
|
|
119
|
+
}
|
|
120
|
+
if (!Buffer.isBuffer(params.file) || params.file.length === 0) {
|
|
121
|
+
throw new Error('file must be a nonempty Buffer containing Parquet bytes.');
|
|
122
|
+
}
|
|
123
|
+
if (!Number.isSafeInteger(params.itemCount) || params.itemCount <= 0) {
|
|
124
|
+
throw new Error('itemCount must be a positive safe integer.');
|
|
125
|
+
}
|
|
126
|
+
const { response: artifact, error } = await this.uploader.uploadParquetExtractionFile(params);
|
|
127
|
+
if (error !== undefined || !artifact) {
|
|
128
|
+
throw error === undefined
|
|
129
|
+
? new errors_1.AirSyncHttpError('No extraction artifact returned.')
|
|
130
|
+
: (0, errors_1.toAirSyncHttpError)(error);
|
|
131
|
+
}
|
|
132
|
+
this.addArtifacts([artifact]);
|
|
133
|
+
this.trackArtifactSize(artifact);
|
|
134
|
+
return artifact;
|
|
135
|
+
}
|
|
136
|
+
trackArtifactSize(artifact) {
|
|
137
|
+
// Count each upload once, not when repos are collected again before emit.
|
|
138
|
+
this.currentEventDataLength += Buffer.byteLength(JSON.stringify(artifact), 'utf8');
|
|
139
|
+
if (this.currentEventDataLength > constants_1.EVENT_SIZE_THRESHOLD_BYTES &&
|
|
140
|
+
!this.isTimeout) {
|
|
141
|
+
this.isTimeout = true;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
101
144
|
addArtifacts(artifacts) {
|
|
102
145
|
this._artifacts = this._artifacts
|
|
103
146
|
.concat(artifacts)
|
|
@@ -155,9 +155,10 @@ export interface EventData {
|
|
|
155
155
|
error?: ErrorRecord;
|
|
156
156
|
delay?: number;
|
|
157
157
|
/**
|
|
158
|
-
* Artifacts produced by the worker's repos
|
|
159
|
-
* the SDK. Includes external sync units,
|
|
160
|
-
* AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS repo and uploaded
|
|
158
|
+
* Artifacts produced by the worker's repos or uploadParquetExtractionFile,
|
|
159
|
+
* attached to the emitted event by the SDK. Includes external sync units,
|
|
160
|
+
* pushed to the AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS repo and uploaded
|
|
161
|
+
* as artifacts.
|
|
161
162
|
*/
|
|
162
163
|
artifacts?: Artifact[];
|
|
163
164
|
reports?: LoaderReport[];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extraction.d.ts","sourceRoot":"","sources":["../../src/types/extraction.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,+CAA+C,CAAC;AAClF,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAEhD,iDAAiD;AACjD,oBAAY,SAAS;IAEnB,gCAAgC,yCAAyC;IACzE,uBAAuB,8BAA8B;IACrD,0BAA0B,iCAAiC;IAC3D,mBAAmB,0BAA0B;IAC7C,sBAAsB,6BAA6B;IACnD,2BAA2B,mCAAmC;IAC9D,0BAA0B,iCAAiC;IAC3D,6BAA6B,oCAAoC;IACjE,sCAAsC,+CAA+C;IAGrF,gBAAgB,uBAAuB;IACvC,mBAAmB,0BAA0B;IAC7C,uBAAuB,8BAA8B;IACrD,0BAA0B,iCAAiC;IAC3D,wBAAwB,gCAAgC;IACxD,kCAAkC,2CAA2C;CAC9E;AAED,iDAAiD;AACjD,oBAAY,kBAAkB;IAE5B,8BAA8B,uCAAuC;IACrE,+BAA+B,wCAAwC;IACvE,0BAA0B,iCAAiC;IAC3D,yBAAyB,gCAAgC;IACzD,sBAAsB,6BAA6B;IACnD,uBAAuB,8BAA8B;IACrD,sBAAsB,6BAA6B;IACnD,qBAAqB,4BAA4B;IACjD,kBAAkB,yBAAyB;IAC3C,mBAAmB,0BAA0B;IAC7C,0BAA0B,kCAAkC;IAC5D,2BAA2B,mCAAmC;IAC9D,4BAA4B,mCAAmC;IAC/D,2BAA2B,kCAAkC;IAC7D,wBAAwB,+BAA+B;IACvD,yBAAyB,gCAAgC;IACzD,qCAAqC,8CAA8C;IACnF,sCAAsC,+CAA+C;CACtF;AAED,uEAAuE;AACvE,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,oBAAY,gBAAgB;IAC1B,YAAY,iBAAiB;IAC7B,WAAW,gBAAgB;CAC5B;AAED,8EAA8E;AAC9E,oBAAY,QAAQ;IAClB,WAAW,OAAO;IAClB,YAAY,OAAO;IACnB,eAAe,YAAO;IACtB,YAAY,OAAO;IACnB,OAAO,MAAM;IACb,OAAO,MAAM;IACb,KAAK,MAAM;CACZ;AAED;;;GAGG;AACH,oBAAY,aAAa;IACvB,cAAc,mBAAmB;IACjC,2BAA2B,gCAAgC;IAC3D,cAAc,mBAAmB;IACjC,0BAA0B,+BAA+B;IACzD,YAAY,iBAAiB;IAC7B,aAAa,kBAAkB;IAC/B,SAAS,cAAc;CACxB;AAED;;;GAGG;AACH,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,aAAa,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,oBAAY,YAAY;IACtB,QAAQ,sCAAsC;IAC9C,UAAU,wCAAwC;IAClD,OAAO,qCAAqC;CAC7C;AAED,MAAM,WAAW,YAAY;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,uBAAuB,EAAE,MAAM,CAAC;IAChC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;IAC7B;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,0EAA0E;IAC1E,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,WAAW,EAAE,MAAM,CAAC;IACpB,kBAAkB,CAAC,EAAE,gBAAgB,CAAC;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,gBAAgB,EAAE,MAAM,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,KAAK,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,CAAC,EAAE,YAAY,CAAC;IAC9B,+EAA+E;IAC/E,yBAAyB,CAAC,EAAE,MAAM,CAAC;IACnC,2EAA2E;IAC3E,gCAAgC,CAAC,EAAE,OAAO,CAAC;IAC3C,eAAe,EAAE,MAAM,CAAC;IACxB,uEAAuE;IACvE,qBAAqB,CAAC,EAAE,SAAS,CAAC;IAClC,mEAAmE;IACnE,mBAAmB,CAAC,EAAE,SAAS,CAAC;IAChC,mGAAmG;IACnG,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,kFAAkF;AAClF,oBAAY,aAAa;IACvB,IAAI,SAAS;IACb,KAAK,UAAU;CAChB;AAED,mGAAmG;AACnG,MAAM,WAAW,aAAa;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,gBAAgB,EAAE,aAAa,CAAC;CACjC;AAED,MAAM,WAAW,SAAS;IACxB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf
|
|
1
|
+
{"version":3,"file":"extraction.d.ts","sourceRoot":"","sources":["../../src/types/extraction.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,+CAA+C,CAAC;AAClF,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAEhD,iDAAiD;AACjD,oBAAY,SAAS;IAEnB,gCAAgC,yCAAyC;IACzE,uBAAuB,8BAA8B;IACrD,0BAA0B,iCAAiC;IAC3D,mBAAmB,0BAA0B;IAC7C,sBAAsB,6BAA6B;IACnD,2BAA2B,mCAAmC;IAC9D,0BAA0B,iCAAiC;IAC3D,6BAA6B,oCAAoC;IACjE,sCAAsC,+CAA+C;IAGrF,gBAAgB,uBAAuB;IACvC,mBAAmB,0BAA0B;IAC7C,uBAAuB,8BAA8B;IACrD,0BAA0B,iCAAiC;IAC3D,wBAAwB,gCAAgC;IACxD,kCAAkC,2CAA2C;CAC9E;AAED,iDAAiD;AACjD,oBAAY,kBAAkB;IAE5B,8BAA8B,uCAAuC;IACrE,+BAA+B,wCAAwC;IACvE,0BAA0B,iCAAiC;IAC3D,yBAAyB,gCAAgC;IACzD,sBAAsB,6BAA6B;IACnD,uBAAuB,8BAA8B;IACrD,sBAAsB,6BAA6B;IACnD,qBAAqB,4BAA4B;IACjD,kBAAkB,yBAAyB;IAC3C,mBAAmB,0BAA0B;IAC7C,0BAA0B,kCAAkC;IAC5D,2BAA2B,mCAAmC;IAC9D,4BAA4B,mCAAmC;IAC/D,2BAA2B,kCAAkC;IAC7D,wBAAwB,+BAA+B;IACvD,yBAAyB,gCAAgC;IACzD,qCAAqC,8CAA8C;IACnF,sCAAsC,+CAA+C;CACtF;AAED,uEAAuE;AACvE,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,oBAAY,gBAAgB;IAC1B,YAAY,iBAAiB;IAC7B,WAAW,gBAAgB;CAC5B;AAED,8EAA8E;AAC9E,oBAAY,QAAQ;IAClB,WAAW,OAAO;IAClB,YAAY,OAAO;IACnB,eAAe,YAAO;IACtB,YAAY,OAAO;IACnB,OAAO,MAAM;IACb,OAAO,MAAM;IACb,KAAK,MAAM;CACZ;AAED;;;GAGG;AACH,oBAAY,aAAa;IACvB,cAAc,mBAAmB;IACjC,2BAA2B,gCAAgC;IAC3D,cAAc,mBAAmB;IACjC,0BAA0B,+BAA+B;IACzD,YAAY,iBAAiB;IAC7B,aAAa,kBAAkB;IAC/B,SAAS,cAAc;CACxB;AAED;;;GAGG;AACH,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,aAAa,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,oBAAY,YAAY;IACtB,QAAQ,sCAAsC;IAC9C,UAAU,wCAAwC;IAClD,OAAO,qCAAqC;CAC7C;AAED,MAAM,WAAW,YAAY;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,uBAAuB,EAAE,MAAM,CAAC;IAChC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;IAC7B;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,0EAA0E;IAC1E,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,WAAW,EAAE,MAAM,CAAC;IACpB,kBAAkB,CAAC,EAAE,gBAAgB,CAAC;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,gBAAgB,EAAE,MAAM,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,KAAK,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,CAAC,EAAE,YAAY,CAAC;IAC9B,+EAA+E;IAC/E,yBAAyB,CAAC,EAAE,MAAM,CAAC;IACnC,2EAA2E;IAC3E,gCAAgC,CAAC,EAAE,OAAO,CAAC;IAC3C,eAAe,EAAE,MAAM,CAAC;IACxB,uEAAuE;IACvE,qBAAqB,CAAC,EAAE,SAAS,CAAC;IAClC,mEAAmE;IACnE,mBAAmB,CAAC,EAAE,SAAS,CAAC;IAChC,mGAAmG;IACnG,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,kFAAkF;AAClF,oBAAY,aAAa;IACvB,IAAI,SAAS;IACb,KAAK,UAAU;CAChB;AAED,mGAAmG;AACnG,MAAM,WAAW,aAAa;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,gBAAgB,EAAE,aAAa,CAAC;CACjC;AAED,MAAM,WAAW,SAAS;IACxB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;OAKG;IACH,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IAEvB,OAAO,CAAC,EAAE,YAAY,EAAE,CAAC;IACzB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,0BAA0B,CAAC,EAAE,aAAa,EAAE,CAAC;CAC9C;AAED,MAAM,WAAW,cAAc;IAC7B,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAG/B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAG9B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,gDAAgD;AAChD,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE;QACP,OAAO,EAAE;YACP,qBAAqB,EAAE,MAAM,CAAC;SAC/B,CAAC;QACF,kBAAkB,EAAE,MAAM,CAAC;QAC3B,UAAU,EAAE,MAAM,CAAC;QACnB,0DAA0D;QAC1D,OAAO,EAAE,MAAM,CAAC;QAChB,iDAAiD;QACjD,OAAO,EAAE,MAAM,CAAC;QAChB,gEAAgE;QAChE,SAAS,EAAE,MAAM,CAAC;QAClB,yDAAyD;QACzD,kBAAkB,EAAE,MAAM,CAAC;KAC5B,CAAC;IACF,OAAO,EAAE,cAAc,CAAC;IACxB,kBAAkB,EAAE;QAClB,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;IACF,UAAU,EAAE,SAAS,CAAC;CACvB;AAED,8CAA8C;AAC9C,MAAM,WAAW,SAAS;IACxB,iDAAiD;IACjD,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtC,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtC,2EAA2E;IAE3E,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CACjC;AAED,MAAM,WAAW,cAAc;IAC7B,eAAe,EAAE,cAAc,CAAC;IAChC,aAAa,EAAE,YAAY,CAAC;IAC5B,UAAU,EAAE,SAAS,CAAC;IACtB,UAAU,CAAC,EAAE,SAAS,CAAC;CACxB;AAED,gDAAgD;AAChD,MAAM,WAAW,cAAc;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,YAAY,CAAC;IAC5B,UAAU,CAAC,EAAE,SAAS,CAAC;IACvB,eAAe,CAAC,EAAE,cAAc,CAAC;CAClC;AAED,MAAM,MAAM,yCAAyC,GAAG,CAAC,EACvD,IAAI,EACJ,KAAK,GACN,EAAE,uCAAuC,KAAK,OAAO,CAAC,yCAAyC,CAAC,CAAC;AAElG,MAAM,WAAW,uCAAuC;IACtD,IAAI,EAAE,oBAAoB,CAAC;IAC3B,KAAK,EAAE,YAAY,CAAC;CACrB;AAED,MAAM,WAAW,kBAAkB;IAEjC,IAAI,EAAE,GAAG,CAAC;IAEV,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAC9B;AAED,gEAAgE;AAChE,MAAM,MAAM,yCAAyC,GACjD;IAAE,MAAM,EAAE,SAAS,CAAC;IAAC,UAAU,EAAE,kBAAkB,CAAA;CAAE,GACrD;IAAE,MAAM,EAAE,OAAO,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,GACzC;IAAE,MAAM,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,WAAW,CAAA;CAAE,CAAC;AAE5C;;;;GAIG;AACH,MAAM,MAAM,2BAA2B,GACnC;IAAE,MAAM,EAAE,SAAS,CAAA;CAAE,GACrB;IAAE,MAAM,EAAE,OAAO,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,GACzC;IAAE,MAAM,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,WAAW,GAAG;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAA;CAAE,CAAC;AAEpE,MAAM,MAAM,uCAAuC,CACjD,KAAK,EACL,QAAQ,EACR,cAAc,IACZ,CAAC,EACH,WAAW,EACX,OAAO,EACP,SAAS,GACV,EAAE;IACD,WAAW,EAAE,KAAK,CAAC;IACnB,OAAO,EAAE,iBAAiB,CAAC,cAAc,CAAC,CAAC;IAC3C,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,KAAK,QAAQ,CAAC;AAEf,MAAM,MAAM,iCAAiC,GAAG,CAAC,EAC/C,UAAU,EACV,MAAM,GACP,EAAE;IACD,UAAU,EAAE,oBAAoB,CAAC;IACjC,MAAM,EAAE,yCAAyC,CAAC;CACnD,KAAK,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAE3C,MAAM,MAAM,wCAAwC,CAAC,QAAQ,EAAE,cAAc,IAC3E,CAAC,EACC,kBAAkB,EAClB,OAAO,EACP,MAAM,GACP,EAAE;IACD,kBAAkB,EAAE,QAAQ,CAAC;IAC7B,OAAO,EAAE,iBAAiB,CAAC,cAAc,CAAC,CAAC;IAC3C,MAAM,EAAE,yCAAyC,CAAC;CACnD,KAAK,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAE7C,MAAM,WAAW,kCAAkC,CACjD,cAAc,EACd,KAAK,EACL,QAAQ;IAER,OAAO,EAAE,uCAAuC,CAC9C,KAAK,EACL,QAAQ,EACR,cAAc,CACf,CAAC;IACF,QAAQ,EAAE,wCAAwC,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;CAC9E"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { AxiosResponse } from 'axios';
|
|
2
2
|
import { NormalizedAttachment } from '../repo/repo.interfaces';
|
|
3
3
|
import { HttpStreamResponse } from '../types/extraction';
|
|
4
|
-
import { ArtifactToUpload, MultipartUpload, UploaderFactoryInterface, UploaderResult, UploadResponse } from './uploader.interfaces';
|
|
4
|
+
import { Artifact, ArtifactToUpload, MultipartUpload, UploaderFactoryInterface, UploaderResult, UploadExtractionFileParams, UploadResponse } from './uploader.interfaces';
|
|
5
5
|
export declare class Uploader {
|
|
6
6
|
private isLocalDevelopment?;
|
|
7
7
|
private devrevApiEndpoint;
|
|
@@ -12,6 +12,8 @@ export declare class Uploader {
|
|
|
12
12
|
constructor({ event, options, skipConfirmation }: UploaderFactoryInterface);
|
|
13
13
|
/** Compresses fetched objects to gzipped JSONL and uploads them as an artifact. */
|
|
14
14
|
upload(itemType: string, fetchedObjects: object[] | object): Promise<UploadResponse>;
|
|
15
|
+
/** Uploads ready-made Parquet bytes without serializing or compressing them. */
|
|
16
|
+
uploadParquetExtractionFile({ itemType, file, itemCount, }: UploadExtractionFileParams<'parquet'>): Promise<UploaderResult<Artifact>>;
|
|
15
17
|
getArtifactUploadUrl(filename: string, fileType: string, fileSize?: number): Promise<UploaderResult<ArtifactToUpload>>;
|
|
16
18
|
/** Uploads the file buffer to the artifact's upload URL as multipart form data. */
|
|
17
19
|
uploadArtifact(artifact: ArtifactToUpload, file: Buffer): Promise<UploaderResult<AxiosResponse>>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAUtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAWzD,OAAO,
|
|
1
|
+
{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAUtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAWzD,OAAO,EACL,QAAQ,EACR,gBAAgB,EAChB,eAAe,EACf,wBAAwB,EACxB,cAAc,EACd,0BAA0B,EAC1B,cAAc,EACf,MAAM,uBAAuB,CAAC;AAE/B,qBAAa,QAAQ;IACnB,OAAO,CAAC,kBAAkB,CAAC,CAAU;IACrC,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,cAAc,CAAyB;IAC/C,OAAO,CAAC,gBAAgB,CAAU;gBAEtB,EAAE,KAAK,EAAE,OAAO,EAAE,gBAAgB,EAAE,EAAE,wBAAwB;IAW1E,mFAAmF;IAC7E,MAAM,CACV,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,EAAE,GAAG,MAAM,GAChC,OAAO,CAAC,cAAc,CAAC;IAwE1B,gFAAgF;IAC1E,2BAA2B,CAAC,EAChC,QAAQ,EACR,IAAI,EACJ,SAAS,GACV,EAAE,0BAA0B,CAAC,SAAS,CAAC,GAAG,OAAO,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;IAqCtE,oBAAoB,CACxB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;IA2B5C,mFAAmF;IAC7E,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAmBzC,kFAAkF;IAC5E,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,UAAU,EAAE,kBAAkB,GAC7B,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAqCzC;;;;OAIG;IACG,qBAAqB,CACzB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,QAAQ,SAA0B,GACjC,OAAO,CAAC,cAAc,CAAC,eAAe,CAAC,CAAC;IAkB3C,+DAA+D;IACzD,0BAA0B,CAC9B,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,QAAQ,SAA0B,GACjC,OAAO,CAAC,cAAc,CAAC,eAAe,CAAC,CAAC;IAkB3C,+DAA+D;IACzD,uBAAuB,CAC3B,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAoBzC;;;;OAIG;IACG,uBAAuB,CAC3B,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,kBAAkB,GAC7B,OAAO,CAAC,cAAc,CAAC;QAAE,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IA6HnD,oDAAoD;YACtC,mBAAmB;IAsD3B,qBAAqB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC;QACvD,QAAQ,CAAC,EAAE,aAAa,CAAC;QACzB,KAAK,CAAC,EAAE,OAAO,CAAC;KACjB,CAAC;IAgCF,oEAAoE;IAC9D,4BAA4B,CAAC,EACjC,QAAQ,GACT,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;KAClB,GAAG,OAAO,CAAC;QACV,WAAW,CAAC,EAAE,oBAAoB,EAAE,CAAC;QACrC,KAAK,CAAC,EAAE;YAAE,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC;KAC7B,CAAC;YAuDY,sBAAsB;YAsBtB,gBAAgB;IAc9B,yEAAyE;IACnE,yBAAyB,CAAC,EAC9B,UAAU,EACV,SAAiB,GAClB,EAAE;QACD,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,CAAC,EAAE,OAAO,CAAC;KACrB,GAAG,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC;CA6B/C"}
|
|
@@ -30,6 +30,19 @@ export interface Artifact {
|
|
|
30
30
|
[ArtifactDateField.OldestModifiedDate]?: string;
|
|
31
31
|
[ArtifactDateField.NewestModifiedDate]?: string;
|
|
32
32
|
}
|
|
33
|
+
/** Formats supported by extraction file upload methods. */
|
|
34
|
+
export type ExtractionFileFormat = 'parquet';
|
|
35
|
+
/** A ready-made dataset file to upload and report in an extraction event. */
|
|
36
|
+
export interface UploadExtractionFileParams<Format extends ExtractionFileFormat> {
|
|
37
|
+
/** Dataset/query key, not a built-in metadata or attachment item type. */
|
|
38
|
+
itemType: string;
|
|
39
|
+
/** Valid file bytes in the specified format. */
|
|
40
|
+
file: Buffer;
|
|
41
|
+
/** Format accepted by the upload method. */
|
|
42
|
+
format: Format;
|
|
43
|
+
/** Actual row count; must be a positive safe integer. Not inferred from the file. */
|
|
44
|
+
itemCount: number;
|
|
45
|
+
}
|
|
33
46
|
export type ArtifactDateRanges = Pick<Artifact, ArtifactDateField.OldestCreatedDate | ArtifactDateField.NewestCreatedDate | ArtifactDateField.OldestModifiedDate | ArtifactDateField.NewestModifiedDate>;
|
|
34
47
|
/** Response from the get upload url endpoint. */
|
|
35
48
|
export interface ArtifactToUpload {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"uploader.interfaces.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAElD,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,YAAY,CAAC;IACpB,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,uFAAuF;IACvF,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,MAAM,cAAc,CAAC,CAAC,IACxB;IAAE,QAAQ,EAAE,CAAC,CAAC;IAAC,KAAK,CAAC,EAAE,KAAK,CAAA;CAAE,GAC9B;IAAE,QAAQ,CAAC,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE,CAAC;AAEzC,oBAAY,iBAAiB;IAC3B,iBAAiB,wBAAwB;IACzC,iBAAiB,wBAAwB;IACzC,kBAAkB,yBAAyB;IAC3C,kBAAkB,yBAAyB;CAC5C;AAED,iEAAiE;AACjE,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,CAAC,EAAE,MAAM,CAAC;IAC/C,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,CAAC,EAAE,MAAM,CAAC;IAC/C,CAAC,iBAAiB,CAAC,kBAAkB,CAAC,CAAC,EAAE,MAAM,CAAC;IAChD,CAAC,iBAAiB,CAAC,kBAAkB,CAAC,CAAC,EAAE,MAAM,CAAC;CACjD;AAED,MAAM,MAAM,kBAAkB,GAAG,IAAI,CACnC,QAAQ,EACN,iBAAiB,CAAC,iBAAiB,GACnC,iBAAiB,CAAC,iBAAiB,GACnC,iBAAiB,CAAC,kBAAkB,GACpC,iBAAiB,CAAC,kBAAkB,CACvC,CAAC;AAEF,iDAAiD;AACjD,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE;QACT,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;KACf,EAAE,CAAC;CACL;AAED,yEAAyE;AACzE,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE;QACF,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,SAAS,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,QAAQ,CAAC,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB"}
|
|
1
|
+
{"version":3,"file":"uploader.interfaces.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAElD,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,YAAY,CAAC;IACpB,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,uFAAuF;IACvF,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,MAAM,cAAc,CAAC,CAAC,IACxB;IAAE,QAAQ,EAAE,CAAC,CAAC;IAAC,KAAK,CAAC,EAAE,KAAK,CAAA;CAAE,GAC9B;IAAE,QAAQ,CAAC,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE,CAAC;AAEzC,oBAAY,iBAAiB;IAC3B,iBAAiB,wBAAwB;IACzC,iBAAiB,wBAAwB;IACzC,kBAAkB,yBAAyB;IAC3C,kBAAkB,yBAAyB;CAC5C;AAED,iEAAiE;AACjE,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,CAAC,EAAE,MAAM,CAAC;IAC/C,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,CAAC,EAAE,MAAM,CAAC;IAC/C,CAAC,iBAAiB,CAAC,kBAAkB,CAAC,CAAC,EAAE,MAAM,CAAC;IAChD,CAAC,iBAAiB,CAAC,kBAAkB,CAAC,CAAC,EAAE,MAAM,CAAC;CACjD;AAED,2DAA2D;AAC3D,MAAM,MAAM,oBAAoB,GAAG,SAAS,CAAC;AAE7C,6EAA6E;AAC7E,MAAM,WAAW,0BAA0B,CACzC,MAAM,SAAS,oBAAoB;IAEnC,0EAA0E;IAC1E,QAAQ,EAAE,MAAM,CAAC;IACjB,gDAAgD;IAChD,IAAI,EAAE,MAAM,CAAC;IACb,4CAA4C;IAC5C,MAAM,EAAE,MAAM,CAAC;IACf,qFAAqF;IACrF,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,MAAM,kBAAkB,GAAG,IAAI,CACnC,QAAQ,EACN,iBAAiB,CAAC,iBAAiB,GACnC,iBAAiB,CAAC,iBAAiB,GACnC,iBAAiB,CAAC,kBAAkB,GACpC,iBAAiB,CAAC,kBAAkB,CACvC,CAAC;AAEF,iDAAiD;AACjD,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE;QACT,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;KACf,EAAE,CAAC;CACL;AAED,yEAAyE;AACzE,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE;QACF,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,SAAS,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,QAAQ,CAAC,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB"}
|
|
@@ -63,7 +63,7 @@ class Uploader {
|
|
|
63
63
|
return {
|
|
64
64
|
error: {
|
|
65
65
|
message: 'Error while confirming artifact upload. ' +
|
|
66
|
-
|
|
66
|
+
(0, logger_1.serializeError)(confirmArtifactUploadError),
|
|
67
67
|
},
|
|
68
68
|
};
|
|
69
69
|
}
|
|
@@ -72,6 +72,30 @@ class Uploader {
|
|
|
72
72
|
const artifact = Object.assign({ id: preparedArtifact.artifact_id, item_type: itemType, item_count: Array.isArray(fetchedObjects) ? fetchedObjects.length : 1 }, artifactDateRanges);
|
|
73
73
|
return { artifact };
|
|
74
74
|
}
|
|
75
|
+
/** Uploads ready-made Parquet bytes without serializing or compressing them. */
|
|
76
|
+
async uploadParquetExtractionFile({ itemType, file, itemCount, }) {
|
|
77
|
+
const { response: preparedArtifact, error: prepareError } = await this.getArtifactUploadUrl(`${itemType}.parquet`, 'application/vnd.apache.parquet', file.length);
|
|
78
|
+
if (prepareError !== undefined || !preparedArtifact) {
|
|
79
|
+
return {
|
|
80
|
+
error: prepareError !== null && prepareError !== void 0 ? prepareError : new Error('No artifact upload URL returned.'),
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
const { error: uploadError } = await this.uploadArtifact(preparedArtifact, file);
|
|
84
|
+
if (uploadError !== undefined) {
|
|
85
|
+
return { error: uploadError };
|
|
86
|
+
}
|
|
87
|
+
const { error: confirmError } = await this.confirmArtifactUpload(preparedArtifact.artifact_id);
|
|
88
|
+
if (confirmError !== undefined) {
|
|
89
|
+
return { error: confirmError };
|
|
90
|
+
}
|
|
91
|
+
return {
|
|
92
|
+
response: {
|
|
93
|
+
id: preparedArtifact.artifact_id,
|
|
94
|
+
item_type: itemType,
|
|
95
|
+
item_count: itemCount,
|
|
96
|
+
},
|
|
97
|
+
};
|
|
98
|
+
}
|
|
75
99
|
async getArtifactUploadUrl(filename, fileType, fileSize) {
|
|
76
100
|
const url = `${this.devrevApiEndpoint}/internal/airdrop.artifacts.upload-url`;
|
|
77
101
|
if (fileSize !== undefined && fileSize <= 0) {
|
|
@@ -355,7 +379,7 @@ class Uploader {
|
|
|
355
379
|
}
|
|
356
380
|
}
|
|
357
381
|
catch (error) {
|
|
358
|
-
return { error
|
|
382
|
+
return { error };
|
|
359
383
|
}
|
|
360
384
|
}
|
|
361
385
|
/** Downloads an artifact and parses it into attachment metadata. */
|
package/docs/MIGRATION.md
CHANGED
|
@@ -9,8 +9,8 @@ own:
|
|
|
9
9
|
It ships in the package at `node_modules/@devrev/airsync-sdk/docs/MIGRATION.md`
|
|
10
10
|
(read it from disk; the `exports` map only blocks `import`/`require`). Before
|
|
11
11
|
v2 is installed, use
|
|
12
|
-
`https://raw.githubusercontent.com/devrev/
|
|
13
|
-
|
|
12
|
+
`https://raw.githubusercontent.com/devrev/airsync-sdk/stable/docs/MIGRATION.md`.
|
|
13
|
+
The installed guide always takes precedence.
|
|
14
14
|
|
|
15
15
|
**The modern wire protocol is unchanged.** Event names, routes, headers,
|
|
16
16
|
artifact formats and `ERROR_CODE=…` strings remain compatible with platforms
|
|
@@ -35,8 +35,8 @@ v2; templates are copied into new connectors, so stale comments spread.
|
|
|
35
35
|
1. **Locate the package.** Find the `package.json` that depends on
|
|
36
36
|
`@devrev/ts-adaas` (generated connectors keep it in `code/`). Run everything
|
|
37
37
|
from there. If it already depends on `@devrev/airsync-sdk`, inspect its
|
|
38
|
-
installed version and migration review.
|
|
39
|
-
changes under **Rare cases** below; upgrade to the
|
|
38
|
+
installed version and migration review. Earlier versions may still need
|
|
39
|
+
the changes under **Rare cases** below; upgrade to the latest v2 version
|
|
40
40
|
and verify source, tests and local runners against this guide. Stop only
|
|
41
41
|
when the current-contract migration and verification are complete.
|
|
42
42
|
2. **Start from green latest v1.** If behind the latest stable 1.x, upgrade and
|
|
@@ -48,7 +48,7 @@ v2; templates are copied into new connectors, so stale comments spread.
|
|
|
48
48
|
3. **Install v2.**
|
|
49
49
|
```bash
|
|
50
50
|
npm uninstall @devrev/ts-adaas
|
|
51
|
-
npm install @devrev/airsync-sdk@
|
|
51
|
+
npm install @devrev/airsync-sdk@latest
|
|
52
52
|
```
|
|
53
53
|
Rename or remove `overrides` / `resolutions` entries keyed on
|
|
54
54
|
`@devrev/ts-adaas`; `npm uninstall` leaves them behind. If the installed
|
|
@@ -395,7 +395,7 @@ async function extractList(adapter: ExtractionAdapter<State>) { ... }
|
|
|
395
395
|
|
|
396
396
|
| Members | On |
|
|
397
397
|
|---------|----|
|
|
398
|
-
| `initializeRepos`, `getRepo`, `streamAttachments`, `processAttachment`, `shouldExtract`, `artifacts`, `preExtractionItemCounts` | `ExtractionAdapter` |
|
|
398
|
+
| `initializeRepos`, `getRepo`, `uploadParquetExtractionFile`, `streamAttachments`, `processAttachment`, `shouldExtract`, `artifacts`, `preExtractionItemCounts` | `ExtractionAdapter` |
|
|
399
399
|
| `loadItemTypes`, `loadAttachments`, `reports`, `processedFiles` | `LoadingAdapter` |
|
|
400
400
|
| `event`, `state`, `mappers`, `postState`, `uploadArtifact`, `downloadArtifact`, `isTimeout`, `extractionScope`, `extractionQueries` | both |
|
|
401
401
|
|
|
@@ -407,6 +407,34 @@ async function extractList(adapter: ExtractionAdapter<State>) { ... }
|
|
|
407
407
|
public. `isTimeout` and `artifacts` are read-only.
|
|
408
408
|
- `adapter.mappers` works in extraction as in v1; keep those calls and only
|
|
409
409
|
apply the §10 return-shape changes.
|
|
410
|
+
- For ready-made DLH Parquet files, use the extraction-only API during a
|
|
411
|
+
data-extraction task:
|
|
412
|
+
|
|
413
|
+
```ts
|
|
414
|
+
await adapter.uploadParquetExtractionFile({
|
|
415
|
+
itemType: queryKey,
|
|
416
|
+
file: parquetBuffer,
|
|
417
|
+
format: 'parquet',
|
|
418
|
+
itemCount: rowCount,
|
|
419
|
+
});
|
|
420
|
+
```
|
|
421
|
+
|
|
422
|
+
It uploads and confirms the `Buffer` unchanged, then registers the returned
|
|
423
|
+
`Artifact` for the next callback, including progress/errors. It throws on
|
|
424
|
+
failure. The request type is `UploadExtractionFileParams<'parquet'>`: the
|
|
425
|
+
generic is constrained by `ExtractionFileFormat`, which currently allows
|
|
426
|
+
only `'parquet'`, not arbitrary strings. Only the Parquet upload method is provided.
|
|
427
|
+
Supply valid Parquet and the actual positive safe-integer row count;
|
|
428
|
+
normalization, batching and timestamp tracking are bypassed. Built-in
|
|
429
|
+
metadata and attachment item types are not allowed. Continue checking
|
|
430
|
+
`adapter.isTimeout` between files, and split large results into files smaller
|
|
431
|
+
than 2 GiB. This API is buffer-only; it does not write a local JSONL copy.
|
|
432
|
+
Keep `query_result_metadata` in a separate repo, pushing one complete JSON
|
|
433
|
+
object with `format: 'parquet'` and `datasets: { [queryKey]: { schema: ... } }`.
|
|
434
|
+
All data files covered by that metadata must be Parquet. Skip zero-row
|
|
435
|
+
uploads; omitting files does not clear an existing dataset.
|
|
436
|
+
`uploadArtifact` / `downloadArtifact` remain JSONL storage for connector-owned
|
|
437
|
+
data, not extraction output, and `repo.push()` keeps its JSONL behavior.
|
|
410
438
|
- **Integration tests** that built an adapter by hand use the root factories:
|
|
411
439
|
|
|
412
440
|
```ts
|
|
@@ -786,6 +814,6 @@ change.
|
|
|
786
814
|
that matters.
|
|
787
815
|
- **`is_sdk_log`** is no longer on log lines; update dashboards that filter on
|
|
788
816
|
it.
|
|
789
|
-
- **Connectors migrated
|
|
790
|
-
§3–§14 against
|
|
791
|
-
State they wrote is still read.
|
|
817
|
+
- **Connectors migrated before the v2 GA release** must re-apply
|
|
818
|
+
§3–§14 against the latest release, because most of the API above changed
|
|
819
|
+
before GA. State they wrote is still read.
|