@devrev/airsync-sdk 2.2.0-alpha.1 → 2.2.0-alpha.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/common/constants.d.ts +3 -1
- package/dist/common/constants.d.ts.map +1 -1
- package/dist/common/constants.js +2 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/multithreading/adapters/extraction-adapter.d.ts +1 -8
- package/dist/multithreading/adapters/extraction-adapter.d.ts.map +1 -1
- package/dist/multithreading/adapters/extraction-adapter.js +2 -40
- package/dist/repo/repo.d.ts +15 -0
- package/dist/repo/repo.d.ts.map +1 -1
- package/dist/repo/repo.interfaces.d.ts +1 -1
- package/dist/repo/repo.interfaces.d.ts.map +1 -1
- package/dist/repo/repo.js +43 -2
- package/dist/types/extraction.d.ts +3 -4
- package/dist/types/extraction.d.ts.map +1 -1
- package/dist/uploader/uploader.d.ts +4 -2
- package/dist/uploader/uploader.d.ts.map +1 -1
- package/dist/uploader/uploader.interfaces.d.ts +0 -9
- package/dist/uploader/uploader.interfaces.d.ts.map +1 -1
- package/dist/uploader/uploader.js +34 -40
- package/docs/MIGRATION.md +1 -27
- package/package.json +1 -1
|
@@ -14,7 +14,9 @@ export declare const UNKNOWN_EVENT_TYPE = "UNKNOWN_EVENT_TYPE";
|
|
|
14
14
|
export declare enum AirSyncDefaultItemTypes {
|
|
15
15
|
EXTERNAL_DOMAIN_METADATA = "external_domain_metadata",
|
|
16
16
|
ATTACHMENTS = "attachments",
|
|
17
|
-
EXTERNAL_SYNC_UNITS = "external_sync_units"
|
|
17
|
+
EXTERNAL_SYNC_UNITS = "external_sync_units",
|
|
18
|
+
INITIAL_DOMAIN_MAPPING = "initial_domain_mapping",
|
|
19
|
+
QUERY_RESULT_METADATA = "query_result_metadata"
|
|
18
20
|
}
|
|
19
21
|
export declare const LIBRARY_VERSION: any;
|
|
20
22
|
export declare const DEFAULT_LAMBDA_TIMEOUT: number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/common/constants.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAIhD,eAAO,MAAM,gCAAgC,aAI5C,CAAC;AAEF,eAAO,MAAM,6BAA6B,aAGzC,CAAC;AAEF,eAAO,MAAM,qBAAqB,aAGjC,CAAC;AAEF,eAAO,MAAM,mBAAmB,OAAO,CAAC;AACxC,eAAO,MAAM,wBAAwB,QAAyB,CAAC;AAG/D,eAAO,MAAM,uBAAuB,QAAmB,CAAC;AACxD,eAAO,MAAM,0BAA0B,MAAM,CAAC;AAC9C,eAAO,MAAM,oCAAoC,KAAK,CAAC;AAGvD,eAAO,MAAM,oBAAoB,SAAU,CAAC;AAE5C,eAAO,MAAM,0BAA0B,QAEtC,CAAC;AAEF,eAAO,MAAM,eAAe,EAAE,MAA0B,CAAC;AACzD,eAAO,MAAM,kBAAkB,uBAAuB,CAAC;AAEvD,oBAAY,uBAAuB;IACjC,wBAAwB,6BAA6B;IACrD,WAAW,gBAAgB;IAC3B,mBAAmB,wBAAwB;
|
|
1
|
+
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/common/constants.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAIhD,eAAO,MAAM,gCAAgC,aAI5C,CAAC;AAEF,eAAO,MAAM,6BAA6B,aAGzC,CAAC;AAEF,eAAO,MAAM,qBAAqB,aAGjC,CAAC;AAEF,eAAO,MAAM,mBAAmB,OAAO,CAAC;AACxC,eAAO,MAAM,wBAAwB,QAAyB,CAAC;AAG/D,eAAO,MAAM,uBAAuB,QAAmB,CAAC;AACxD,eAAO,MAAM,0BAA0B,MAAM,CAAC;AAC9C,eAAO,MAAM,oCAAoC,KAAK,CAAC;AAGvD,eAAO,MAAM,oBAAoB,SAAU,CAAC;AAE5C,eAAO,MAAM,0BAA0B,QAEtC,CAAC;AAEF,eAAO,MAAM,eAAe,EAAE,MAA0B,CAAC;AACzD,eAAO,MAAM,kBAAkB,uBAAuB,CAAC;AAEvD,oBAAY,uBAAuB;IACjC,wBAAwB,6BAA6B;IACrD,WAAW,gBAAgB;IAC3B,mBAAmB,wBAAwB;IAC3C,sBAAsB,2BAA2B;IACjD,qBAAqB,0BAA0B;CAChD;AAED,eAAO,MAAM,eAAe,KAAsB,CAAC;AAEnD,eAAO,MAAM,sBAAsB,QAAiB,CAAC;AACrD,eAAO,MAAM,uBAAuB,MAAM,CAAC;AAC3C,eAAO,MAAM,mBAAmB,QAAY,CAAC;AAE7C;;;;GAIG;AACH,eAAO,MAAM,yBAAyB,6BAA6B,CAAC"}
|
package/dist/common/constants.js
CHANGED
|
@@ -34,6 +34,8 @@ var AirSyncDefaultItemTypes;
|
|
|
34
34
|
AirSyncDefaultItemTypes["EXTERNAL_DOMAIN_METADATA"] = "external_domain_metadata";
|
|
35
35
|
AirSyncDefaultItemTypes["ATTACHMENTS"] = "attachments";
|
|
36
36
|
AirSyncDefaultItemTypes["EXTERNAL_SYNC_UNITS"] = "external_sync_units";
|
|
37
|
+
AirSyncDefaultItemTypes["INITIAL_DOMAIN_MAPPING"] = "initial_domain_mapping";
|
|
38
|
+
AirSyncDefaultItemTypes["QUERY_RESULT_METADATA"] = "query_result_metadata";
|
|
37
39
|
})(AirSyncDefaultItemTypes || (exports.AirSyncDefaultItemTypes = AirSyncDefaultItemTypes = {}));
|
|
38
40
|
exports.LIBRARY_VERSION = (0, helpers_1.getLibraryVersion)();
|
|
39
41
|
exports.DEFAULT_LAMBDA_TIMEOUT = 10 * 60 * 1000; // 10 minutes
|
package/dist/index.d.ts
CHANGED
|
@@ -20,7 +20,7 @@ export type { Item, NormalizedAttachment, NormalizedItem, RepoInterface, } from
|
|
|
20
20
|
export { Mappers } from './mappers/mappers';
|
|
21
21
|
export type { MappersCreateParams, MappersCreateResponse, MappersFactoryInterface, MappersGetByExternalIdParams, MappersGetByExternalIdResponse, MappersGetByTargetIdParams, MappersGetByTargetIdResponse, MappersUpdateParams, MappersUpdateResponse, SyncMapperRecord, SyncMapperRecordExternalVersion, UpdateSyncMapperRecordParams, } from './mappers/mappers.interfaces';
|
|
22
22
|
export { SyncMapperRecordStatus, SyncMapperRecordTargetType, } from './mappers/mappers.interfaces';
|
|
23
|
-
export type { Artifact, SsorAttachment,
|
|
23
|
+
export type { Artifact, SsorAttachment, UploadResponse, } from './uploader/uploader.interfaces';
|
|
24
24
|
export { ArtifactDateField } from './uploader/uploader.interfaces';
|
|
25
25
|
export type { Attachments, CollectionData, ConditionalPrivilegeData, CustomLinkData, CustomLinkNames, CustomStage, CustomState, EnumData, EnumValue, EnumValueKey, ExternalDomainMetadata, Field, FieldCondition, FieldConditionComparator, FieldConditionEffect, FieldConditions, FieldKey, FieldPrivilegeData, FieldReferenceData, FieldType, FloatData, IntData, ParticipationData, PermissionData, RecordType, RecordTypeCategory, RecordTypeCategoryKey, RecordTypeKey, RecordTypePrivilegeData, RecordTypeScope, ReferenceData, ReferenceDetail, ReferenceType, SchemaVersion, StageDiagram, StageKey, StateKey, StructData, StructType, StructTypeKey, TargetTypeKeyData, TextData, TypedReferenceData, } from './types/external-domain-metadata';
|
|
26
26
|
export type { DeepPartial } from './testing/mock-event';
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAQA,YAAY,EAAE,WAAW,EAAE,MAAM,wCAAwC,CAAC;AAC1E,YAAY,EAAE,iBAAiB,EAAE,MAAM,8CAA8C,CAAC;AACtF,YAAY,EAAE,cAAc,EAAE,MAAM,2CAA2C,CAAC;AAChF,OAAO,EACL,qBAAqB,EACrB,kBAAkB,GACnB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,KAAK,EAAE,MAAM,8BAA8B,CAAC;AACrD,YAAY,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AAGxC,YAAY,EACV,gBAAgB,EAChB,iBAAiB,EACjB,eAAe,EACf,oBAAoB,EACpB,WAAW,EACX,qBAAqB,EACrB,YAAY,EACZ,oBAAoB,EACpB,UAAU,EACV,UAAU,EACV,mBAAmB,GACpB,MAAM,iBAAiB,CAAC;AAGzB,OAAO,EACL,uBAAuB,EACvB,yBAAyB,GAC1B,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAGlD,OAAO,EAAE,2BAA2B,EAAE,MAAM,wCAAwC,CAAC;AAGrF,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAGjD,YAAY,EAAE,WAAW,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AACxE,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAG1C,YAAY,EACV,YAAY,EACZ,cAAc,EACd,cAAc,EACd,YAAY,EACZ,SAAS,EACT,iCAAiC,EACjC,gBAAgB,EAChB,wCAAwC,EACxC,kCAAkC,EAClC,uCAAuC,EACvC,yCAAyC,EACzC,uCAAuC,EACvC,yCAAyC,EACzC,cAAc,EACd,kBAAkB,EAClB,SAAS,EACT,aAAa,EACb,2BAA2B,EAC3B,SAAS,EACT,cAAc,GACf,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,SAAS,EACT,gBAAgB,EAChB,aAAa,EACb,YAAY,EACZ,QAAQ,EACR,aAAa,GACd,MAAM,oBAAoB,CAAC;AAG5B,YAAY,EACV,wBAAwB,EACxB,kBAAkB,EAClB,+BAA+B,EAC/B,iCAAiC,EACjC,6BAA6B,EAC7B,UAAU,EACV,qBAAqB,EACrB,cAAc,EACd,YAAY,GACb,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAG7C,YAAY,EACV,IAAI,EACJ,oBAAoB,EACpB,cAAc,EACd,aAAa,GACd,MAAM,wBAAwB,CAAC;AAGhC,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAC5C,YAAY,EACV,mBAAmB,EACnB,qBAAqB,EACrB,uBAAuB,EACvB,4BAA4B,EAC5B,8BAA8B,EAC9B,0BAA0B,EAC1B,4BAA4B,EAC5B,mBAAmB,EACnB,qBAAqB,EACrB,gBAAgB,EAChB,+BAA+B,EAC/B,4BAA4B,GAC7B,MAAM,8BAA8B,CAAC;AACtC,OAAO,EACL,sBAAsB,EACtB,0BAA0B,GAC3B,MAAM,8BAA8B,CAAC;AAGtC,YAAY,EACV,QAAQ,EACR,cAAc,EACd,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAQA,YAAY,EAAE,WAAW,EAAE,MAAM,wCAAwC,CAAC;AAC1E,YAAY,EAAE,iBAAiB,EAAE,MAAM,8CAA8C,CAAC;AACtF,YAAY,EAAE,cAAc,EAAE,MAAM,2CAA2C,CAAC;AAChF,OAAO,EACL,qBAAqB,EACrB,kBAAkB,GACnB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,KAAK,EAAE,MAAM,8BAA8B,CAAC;AACrD,YAAY,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AAGxC,YAAY,EACV,gBAAgB,EAChB,iBAAiB,EACjB,eAAe,EACf,oBAAoB,EACpB,WAAW,EACX,qBAAqB,EACrB,YAAY,EACZ,oBAAoB,EACpB,UAAU,EACV,UAAU,EACV,mBAAmB,GACpB,MAAM,iBAAiB,CAAC;AAGzB,OAAO,EACL,uBAAuB,EACvB,yBAAyB,GAC1B,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAGlD,OAAO,EAAE,2BAA2B,EAAE,MAAM,wCAAwC,CAAC;AAGrF,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAGjD,YAAY,EAAE,WAAW,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AACxE,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAG1C,YAAY,EACV,YAAY,EACZ,cAAc,EACd,cAAc,EACd,YAAY,EACZ,SAAS,EACT,iCAAiC,EACjC,gBAAgB,EAChB,wCAAwC,EACxC,kCAAkC,EAClC,uCAAuC,EACvC,yCAAyC,EACzC,uCAAuC,EACvC,yCAAyC,EACzC,cAAc,EACd,kBAAkB,EAClB,SAAS,EACT,aAAa,EACb,2BAA2B,EAC3B,SAAS,EACT,cAAc,GACf,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,SAAS,EACT,gBAAgB,EAChB,aAAa,EACb,YAAY,EACZ,QAAQ,EACR,aAAa,GACd,MAAM,oBAAoB,CAAC;AAG5B,YAAY,EACV,wBAAwB,EACxB,kBAAkB,EAClB,+BAA+B,EAC/B,iCAAiC,EACjC,6BAA6B,EAC7B,UAAU,EACV,qBAAqB,EACrB,cAAc,EACd,YAAY,GACb,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAG7C,YAAY,EACV,IAAI,EACJ,oBAAoB,EACpB,cAAc,EACd,aAAa,GACd,MAAM,wBAAwB,CAAC;AAGhC,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAC5C,YAAY,EACV,mBAAmB,EACnB,qBAAqB,EACrB,uBAAuB,EACvB,4BAA4B,EAC5B,8BAA8B,EAC9B,0BAA0B,EAC1B,4BAA4B,EAC5B,mBAAmB,EACnB,qBAAqB,EACrB,gBAAgB,EAChB,+BAA+B,EAC/B,4BAA4B,GAC7B,MAAM,8BAA8B,CAAC;AACtC,OAAO,EACL,sBAAsB,EACtB,0BAA0B,GAC3B,MAAM,8BAA8B,CAAC;AAGtC,YAAY,EACV,QAAQ,EACR,cAAc,EACd,cAAc,GACf,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAAE,iBAAiB,EAAE,MAAM,gCAAgC,CAAC;AAGnE,YAAY,EACV,WAAW,EACX,cAAc,EACd,wBAAwB,EACxB,cAAc,EACd,eAAe,EACf,WAAW,EACX,WAAW,EACX,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,sBAAsB,EACtB,KAAK,EACL,cAAc,EACd,wBAAwB,EACxB,oBAAoB,EACpB,eAAe,EACf,QAAQ,EACR,kBAAkB,EAClB,kBAAkB,EAClB,SAAS,EACT,SAAS,EACT,OAAO,EACP,iBAAiB,EACjB,cAAc,EACd,UAAU,EACV,kBAAkB,EAClB,qBAAqB,EACrB,aAAa,EACb,uBAAuB,EACvB,eAAe,EACf,aAAa,EACb,eAAe,EACf,aAAa,EACb,aAAa,EACb,YAAY,EACZ,QAAQ,EACR,QAAQ,EACR,UAAU,EACV,UAAU,EACV,aAAa,EACb,iBAAiB,EACjB,QAAQ,EACR,kBAAkB,GACnB,MAAM,kCAAkC,CAAC;AAG1C,YAAY,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,EAAE,uBAAuB,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAC5E,YAAY,EACV,WAAW,EACX,WAAW,EACX,WAAW,GACZ,MAAM,kCAAkC,CAAC;AAC1C,YAAY,EACV,iBAAiB,EACjB,wBAAwB,GACzB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACL,2BAA2B,EAC3B,wBAAwB,GACzB,MAAM,yBAAyB,CAAC"}
|
|
@@ -2,7 +2,7 @@ import { Repo } from '../../repo/repo';
|
|
|
2
2
|
import { NormalizedAttachment, RepoInterface } from '../../repo/repo.interfaces';
|
|
3
3
|
import { ExternalSystemAttachmentProcessors, ExternalSystemAttachmentStreamingFunction, ItemTypeCount, ProcessAttachmentReturnType } from '../../types/extraction';
|
|
4
4
|
import { TaskResult } from '../../types/workers';
|
|
5
|
-
import { Artifact
|
|
5
|
+
import { Artifact } from '../../uploader/uploader.interfaces';
|
|
6
6
|
import { BaseAdapter } from './base-adapter';
|
|
7
7
|
/** Adapter passed to extraction tasks (repos, artifacts, attachment streaming). @public */
|
|
8
8
|
export declare class ExtractionAdapter<ConnectorState> extends BaseAdapter<ConnectorState> {
|
|
@@ -25,13 +25,6 @@ export declare class ExtractionAdapter<ConnectorState> extends BaseAdapter<Conne
|
|
|
25
25
|
getRepo(itemType: string): Repo | undefined;
|
|
26
26
|
/** Artifacts uploaded so far that the next emit will report. */
|
|
27
27
|
get artifacts(): readonly Artifact[];
|
|
28
|
-
/**
|
|
29
|
-
* Uploads and confirms a ready-made Parquet dataset file during data extraction,
|
|
30
|
-
* then registers it for the next callback (including progress and errors).
|
|
31
|
-
* Bypasses repo normalization, batching and timestamp tracking. The connector
|
|
32
|
-
* supplies valid Parquet and its actual row count. Throws on failure.
|
|
33
|
-
*/
|
|
34
|
-
uploadParquetExtractionFile(params: UploadExtractionFileParams): Promise<Artifact>;
|
|
35
28
|
private trackArtifactSize;
|
|
36
29
|
private addArtifacts;
|
|
37
30
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extraction-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/extraction-adapter.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"extraction-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/extraction-adapter.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AACvC,OAAO,EACL,oBAAoB,EACpB,aAAa,EACd,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EAIL,kCAAkC,EAClC,yCAAyC,EAEzC,aAAa,EACb,2BAA2B,EAE5B,MAAM,wBAAwB,CAAC;AAEhC,OAAO,EAAkB,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjE,OAAO,EAAE,QAAQ,EAAkB,MAAM,oCAAoC,CAAC;AAE9E,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,2FAA2F;AAC3F,qBAAa,iBAAiB,CAC5B,cAAc,CACd,SAAQ,WAAW,CAAC,cAAc,CAAC;IACnC,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,KAAK,CAAc;IAC3B,OAAO,CAAC,wBAAwB,CAAC,CAAkB;IACnD,OAAO,CAAC,qBAAqB,CAAC,CAAS;IACvC,OAAO,CAAC,sBAAsB,CAAa;IAY3C,6EAA6E;IAC7E,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAOxC,eAAe,CAAC,KAAK,EAAE,aAAa,EAAE;IAatC,OAAO,CAAC,UAAU;IA4ClB;;;OAGG;IACH,OAAO,CAAC,cAAc;IAItB,OAAO,CAAC,kBAAkB;IAQ1B,OAAO,CAAC,mBAAmB;IAS3B,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS;IAW3C,gEAAgE;IAChE,IAAI,SAAS,IAAI,SAAS,QAAQ,EAAE,CAEnC;IAED,OAAO,CAAC,iBAAiB;IAczB,OAAO,CAAC,YAAY;IAMpB;;;OAGG;IACH,IAAI,uBAAuB,IAAI,aAAa,EAAE,GAAG,SAAS,CAEzD;IAED,IAAI,uBAAuB,CAAC,MAAM,EAAE,aAAa,EAAE,GAAG,SAAS,EAE9D;IA6HD,OAAO,CAAC,kBAAkB;YAaZ,cAAc;IAmB5B;;;;OAIG;IACG,iBAAiB,CACrB,UAAU,EAAE,oBAAoB,EAChC,MAAM,EAAE,yCAAyC,GAChD,OAAO,CAAC,2BAA2B,CAAC;IA4IjC,iBAAiB,CAAC,QAAQ,EAAE,EAChC,MAAM,EACN,UAAU,EACV,SAAa,GACd,EAAE;QACD,MAAM,EAAE,yCAAyC,CAAC;QAClD,UAAU,CAAC,EAAE,kCAAkC,CAC7C,cAAc,EACd,oBAAoB,EAAE,EACtB,QAAQ,CACT,CAAC;QACF,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,GAAG,OAAO,CAAC,UAAU,CAAC;CAsHxB"}
|
|
@@ -3,7 +3,6 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.ExtractionAdapter = void 0;
|
|
4
4
|
const attachments_streaming_pool_1 = require("../../attachments-streaming/attachments-streaming-pool");
|
|
5
5
|
const constants_1 = require("../../common/constants");
|
|
6
|
-
const errors_1 = require("../../http/errors");
|
|
7
6
|
const logger_1 = require("../../logger/logger");
|
|
8
7
|
const repo_1 = require("../../repo/repo");
|
|
9
8
|
const extraction_1 = require("../../types/extraction");
|
|
@@ -41,7 +40,7 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
|
|
|
41
40
|
repo.itemType !== constants_1.SSOR_ATTACHMENT;
|
|
42
41
|
const isDeferred = this.isDeferredRepo(repo.itemType);
|
|
43
42
|
const skipTimestampTracking = (_a = repo.skipTimestampTracking) !== null && _a !== void 0 ? _a : false;
|
|
44
|
-
return new repo_1.Repo(Object.assign(Object.assign({ event: this.event, itemType: repo.itemType, skipTimestampTracking }, (shouldNormalize && { normalize: repo.normalize })), { skipConfirmation: repo.itemType === constants_1.AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS, onUpload: (artifact) => {
|
|
43
|
+
return new repo_1.Repo(Object.assign(Object.assign({ event: this.event, itemType: repo.itemType, skipTimestampTracking }, (shouldNormalize && { normalize: repo.normalize })), { skipConfirmation: repo.itemType === constants_1.AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS, onUpload: (artifact, format) => {
|
|
45
44
|
var _a, _b;
|
|
46
45
|
if (isDeferred) {
|
|
47
46
|
const toDevRev = this.adapterState.sdkState.toDevRev;
|
|
@@ -51,7 +50,7 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
|
|
|
51
50
|
}
|
|
52
51
|
return;
|
|
53
52
|
}
|
|
54
|
-
if (!skipTimestampTracking) {
|
|
53
|
+
if (!skipTimestampTracking && format !== 'parquet') {
|
|
55
54
|
this.lastExtractedItemType = repo.itemType;
|
|
56
55
|
}
|
|
57
56
|
// Artifact ids are kept in state for later attachment streaming
|
|
@@ -93,43 +92,6 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
|
|
|
93
92
|
get artifacts() {
|
|
94
93
|
return this._artifacts;
|
|
95
94
|
}
|
|
96
|
-
/**
|
|
97
|
-
* Uploads and confirms a ready-made Parquet dataset file during data extraction,
|
|
98
|
-
* then registers it for the next callback (including progress and errors).
|
|
99
|
-
* Bypasses repo normalization, batching and timestamp tracking. The connector
|
|
100
|
-
* supplies valid Parquet and its actual row count. Throws on failure.
|
|
101
|
-
*/
|
|
102
|
-
async uploadParquetExtractionFile(params) {
|
|
103
|
-
const eventType = this.event.payload.event_type;
|
|
104
|
-
if (eventType !== extraction_1.EventType.StartExtractingData &&
|
|
105
|
-
eventType !== extraction_1.EventType.ContinueExtractingData) {
|
|
106
|
-
throw new Error('Extraction files can only be uploaded during data extraction.');
|
|
107
|
-
}
|
|
108
|
-
if (!params.itemType.trim() ||
|
|
109
|
-
Object.values(constants_1.AirSyncDefaultItemTypes).some((type) => type === params.itemType) ||
|
|
110
|
-
[
|
|
111
|
-
'initial_domain_mapping',
|
|
112
|
-
'query_result_metadata',
|
|
113
|
-
constants_1.SSOR_ATTACHMENT,
|
|
114
|
-
].includes(params.itemType)) {
|
|
115
|
-
throw new Error('itemType must be a dataset/query key, not a metadata or attachment item type.');
|
|
116
|
-
}
|
|
117
|
-
if (!Buffer.isBuffer(params.file) || params.file.length === 0) {
|
|
118
|
-
throw new Error('file must be a nonempty Buffer containing Parquet bytes.');
|
|
119
|
-
}
|
|
120
|
-
if (!Number.isSafeInteger(params.itemCount) || params.itemCount <= 0) {
|
|
121
|
-
throw new Error('itemCount must be a positive safe integer.');
|
|
122
|
-
}
|
|
123
|
-
const { response: artifact, error } = await this.uploader.uploadParquetExtractionFile(params);
|
|
124
|
-
if (error !== undefined || !artifact) {
|
|
125
|
-
throw error === undefined
|
|
126
|
-
? new errors_1.AirSyncHttpError('No extraction artifact returned.')
|
|
127
|
-
: (0, errors_1.toAirSyncHttpError)(error);
|
|
128
|
-
}
|
|
129
|
-
this.addArtifacts([artifact]);
|
|
130
|
-
this.trackArtifactSize(artifact);
|
|
131
|
-
return artifact;
|
|
132
|
-
}
|
|
133
95
|
trackArtifactSize(artifact) {
|
|
134
96
|
// Count each upload once, not when repos are collected again before emit.
|
|
135
97
|
this.currentEventDataLength += Buffer.byteLength(JSON.stringify(artifact), 'utf8');
|
package/dist/repo/repo.d.ts
CHANGED
|
@@ -9,6 +9,8 @@ export declare class Repo {
|
|
|
9
9
|
private onUpload;
|
|
10
10
|
private options?;
|
|
11
11
|
private trackTimestamps;
|
|
12
|
+
private canUploadParquet;
|
|
13
|
+
private format?;
|
|
12
14
|
uploadedArtifacts: Artifact[];
|
|
13
15
|
dateRanges: {
|
|
14
16
|
creationDate: {
|
|
@@ -22,7 +24,20 @@ export declare class Repo {
|
|
|
22
24
|
};
|
|
23
25
|
getItems(): (NormalizedItem | NormalizedAttachment | Item)[];
|
|
24
26
|
upload(batch?: (NormalizedItem | NormalizedAttachment | Item)[]): Promise<void>;
|
|
27
|
+
/**
|
|
28
|
+
* Uploads and confirms one ready-made Parquet file immediately during data
|
|
29
|
+
* extraction. The connector supplies valid bytes and the actual positive
|
|
30
|
+
* safe-integer row count. Bypasses normalization, batching and timestamp
|
|
31
|
+
* tracking. Confirmed files are reported with the repo's next callback,
|
|
32
|
+
* including progress and errors. Throws on failure; files are not queued.
|
|
33
|
+
* A repo cannot mix row and file submissions.
|
|
34
|
+
*/
|
|
35
|
+
push(file: Buffer, options: {
|
|
36
|
+
format: 'parquet';
|
|
37
|
+
itemCount: number;
|
|
38
|
+
}): Promise<void>;
|
|
25
39
|
/** Buffers items and uploads every full batch. Throws if an upload fails. */
|
|
26
40
|
push(items: Item[]): Promise<void>;
|
|
41
|
+
private selectFormat;
|
|
27
42
|
}
|
|
28
43
|
//# sourceMappingURL=repo.d.ts.map
|
package/dist/repo/repo.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"repo.d.ts","sourceRoot":"","sources":["../../src/repo/repo.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"repo.d.ts","sourceRoot":"","sources":["../../src/repo/repo.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,IAAI,EAAE,MAAM,yBAAyB,CAAC;AAI/C,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAG3D,OAAO,EACL,oBAAoB,EACpB,cAAc,EAEf,MAAM,mBAAmB,CAAC;AAE3B,qBAAa,IAAI;IACf,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,OAAO,CAAC,KAAK,CAAmD;IAChE,OAAO,CAAC,SAAS,CAAC,CAAwD;IAC1E,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,QAAQ,CAAmC;IACnD,OAAO,CAAC,OAAO,CAAC,CAAiB;IACjC,OAAO,CAAC,eAAe,CAAU;IACjC,OAAO,CAAC,gBAAgB,CAAU;IAClC,OAAO,CAAC,MAAM,CAAC,CAAsB;IAC9B,iBAAiB,EAAE,QAAQ,EAAE,CAAC;IAC9B,UAAU,EAAE;QACjB,YAAY,EAAE;YAAE,MAAM,CAAC,EAAE,MAAM,CAAC;YAAC,MAAM,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QACnD,YAAY,EAAE;YAAE,MAAM,CAAC,EAAE,MAAM,CAAC;YAAC,MAAM,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;KACpD,CAGC;IAyBF,QAAQ,IAAI,CAAC,cAAc,GAAG,oBAAoB,GAAG,IAAI,CAAC,EAAE;IAItD,MAAM,CACV,KAAK,CAAC,EAAE,CAAC,cAAc,GAAG,oBAAoB,GAAG,IAAI,CAAC,EAAE,GACvD,OAAO,CAAC,IAAI,CAAC;IA8DhB;;;;;;;OAOG;IACG,IAAI,CACR,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE;QAAE,MAAM,EAAE,SAAS,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,GAChD,OAAO,CAAC,IAAI,CAAC;IAChB,6EAA6E;IACvE,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAgFxC,OAAO,CAAC,YAAY;CAMrB"}
|
|
@@ -19,7 +19,7 @@ export interface RepoFactoryInterface {
|
|
|
19
19
|
itemType: string;
|
|
20
20
|
normalize?: (record: object) => NormalizedItem | NormalizedAttachment;
|
|
21
21
|
skipTimestampTracking?: boolean;
|
|
22
|
-
onUpload: (artifact: Artifact) => void;
|
|
22
|
+
onUpload: (artifact: Artifact, format?: 'parquet') => void;
|
|
23
23
|
options?: RuntimeOptions;
|
|
24
24
|
skipConfirmation?: boolean;
|
|
25
25
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"repo.interfaces.d.ts","sourceRoot":"","sources":["../../src/repo/repo.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAC/D,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,yCAAyC;AACzC,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,cAAc,GAAG,oBAAoB,CAAC;IACtE;;;;;OAKG;IACH,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,iBAAiB,CAAC,EAAE,WAAW,CAAC;CACjC;AAED,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,YAAY,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,cAAc,GAAG,oBAAoB,CAAC;IACtE,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,QAAQ,EAAE,CAAC,QAAQ,EAAE,QAAQ,KAAK,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"repo.interfaces.d.ts","sourceRoot":"","sources":["../../src/repo/repo.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAC/D,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,yCAAyC;AACzC,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,cAAc,GAAG,oBAAoB,CAAC;IACtE;;;;;OAKG;IACH,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,iBAAiB,CAAC,EAAE,WAAW,CAAC;CACjC;AAED,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,YAAY,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,cAAc,GAAG,oBAAoB,CAAC;IACtE,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,QAAQ,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,CAAC,EAAE,SAAS,KAAK,IAAI,CAAC;IAC3D,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,oBAAoB;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAGD,MAAM,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC"}
|
package/dist/repo/repo.js
CHANGED
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.Repo = void 0;
|
|
4
4
|
const constants_1 = require("../common/constants");
|
|
5
|
+
const errors_1 = require("../http/errors");
|
|
6
|
+
const extraction_1 = require("../types/extraction");
|
|
5
7
|
const uploader_1 = require("../uploader/uploader");
|
|
6
8
|
const repo_helpers_1 = require("./repo.helpers");
|
|
7
9
|
class Repo {
|
|
@@ -19,6 +21,9 @@ class Repo {
|
|
|
19
21
|
this.uploader = new uploader_1.Uploader({ event, options, skipConfirmation });
|
|
20
22
|
this.options = options;
|
|
21
23
|
this.uploadedArtifacts = [];
|
|
24
|
+
this.canUploadParquet =
|
|
25
|
+
event.payload.event_type === extraction_1.EventType.StartExtractingData ||
|
|
26
|
+
event.payload.event_type === extraction_1.EventType.ContinueExtractingData;
|
|
22
27
|
}
|
|
23
28
|
getItems() {
|
|
24
29
|
return this.items;
|
|
@@ -27,6 +32,7 @@ class Repo {
|
|
|
27
32
|
var _a;
|
|
28
33
|
const itemsToUpload = batch || this.items;
|
|
29
34
|
if (itemsToUpload.length > 0) {
|
|
35
|
+
this.selectFormat('jsonl');
|
|
30
36
|
if (this.trackTimestamps) {
|
|
31
37
|
for (const item of itemsToUpload) {
|
|
32
38
|
const createdDate = item === null || item === void 0 ? void 0 : item.created_date;
|
|
@@ -62,14 +68,43 @@ class Repo {
|
|
|
62
68
|
console.log(`No items to upload for type ${this.itemType}. Skipping upload.`);
|
|
63
69
|
}
|
|
64
70
|
}
|
|
65
|
-
|
|
66
|
-
async push(items) {
|
|
71
|
+
async push(items, options) {
|
|
67
72
|
var _a;
|
|
73
|
+
if (options !== undefined || Buffer.isBuffer(items)) {
|
|
74
|
+
if (!this.canUploadParquet) {
|
|
75
|
+
throw new Error('Extraction files can only be uploaded during data extraction.');
|
|
76
|
+
}
|
|
77
|
+
if (!this.itemType.trim() ||
|
|
78
|
+
Object.values(constants_1.AirSyncDefaultItemTypes).some((type) => type === this.itemType) ||
|
|
79
|
+
this.itemType === constants_1.SSOR_ATTACHMENT) {
|
|
80
|
+
throw new Error('itemType must be a dataset/query key, not a metadata or attachment item type.');
|
|
81
|
+
}
|
|
82
|
+
if (!Buffer.isBuffer(items) || items.length === 0) {
|
|
83
|
+
throw new Error('file must be a nonempty Buffer containing Parquet bytes.');
|
|
84
|
+
}
|
|
85
|
+
if ((options === null || options === void 0 ? void 0 : options.format) !== 'parquet') {
|
|
86
|
+
throw new Error('format must be parquet for file uploads.');
|
|
87
|
+
}
|
|
88
|
+
if (!Number.isSafeInteger(options.itemCount) || options.itemCount <= 0) {
|
|
89
|
+
throw new Error('itemCount must be a positive safe integer.');
|
|
90
|
+
}
|
|
91
|
+
this.selectFormat('parquet');
|
|
92
|
+
const { response: artifact, error } = await this.uploader.uploadParquet(this.itemType, items, options.itemCount);
|
|
93
|
+
if (error !== undefined || !artifact) {
|
|
94
|
+
throw error === undefined
|
|
95
|
+
? new errors_1.AirSyncHttpError('No extraction artifact returned.')
|
|
96
|
+
: (0, errors_1.toAirSyncHttpError)(error);
|
|
97
|
+
}
|
|
98
|
+
this.uploadedArtifacts.push(artifact);
|
|
99
|
+
this.onUpload(artifact, 'parquet');
|
|
100
|
+
return;
|
|
101
|
+
}
|
|
68
102
|
let recordsToPush;
|
|
69
103
|
if (!items || items.length === 0) {
|
|
70
104
|
console.log(`No items to push for type ${this.itemType}. Skipping push.`);
|
|
71
105
|
return;
|
|
72
106
|
}
|
|
107
|
+
this.selectFormat('jsonl');
|
|
73
108
|
if (this.normalize &&
|
|
74
109
|
this.itemType != constants_1.AirSyncDefaultItemTypes.EXTERNAL_DOMAIN_METADATA &&
|
|
75
110
|
this.itemType != constants_1.SSOR_ATTACHMENT) {
|
|
@@ -88,5 +123,11 @@ class Repo {
|
|
|
88
123
|
this.items.splice(0, batchSize);
|
|
89
124
|
}
|
|
90
125
|
}
|
|
126
|
+
selectFormat(format) {
|
|
127
|
+
if (this.format && this.format !== format) {
|
|
128
|
+
throw new Error('A repo cannot mix JSONL rows and Parquet files.');
|
|
129
|
+
}
|
|
130
|
+
this.format = format;
|
|
131
|
+
}
|
|
91
132
|
}
|
|
92
133
|
exports.Repo = Repo;
|
|
@@ -155,10 +155,9 @@ export interface EventData {
|
|
|
155
155
|
error?: ErrorRecord;
|
|
156
156
|
delay?: number;
|
|
157
157
|
/**
|
|
158
|
-
* Artifacts produced by the worker's repos
|
|
159
|
-
*
|
|
160
|
-
* pushed to the AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS repo
|
|
161
|
-
* as artifacts.
|
|
158
|
+
* Artifacts produced by the worker's repos, attached to the emitted event
|
|
159
|
+
* by the SDK. Includes row batches, ready-made Parquet files and external
|
|
160
|
+
* sync units pushed to the AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS repo.
|
|
162
161
|
*/
|
|
163
162
|
artifacts?: Artifact[];
|
|
164
163
|
reports?: LoaderReport[];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extraction.d.ts","sourceRoot":"","sources":["../../src/types/extraction.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,+CAA+C,CAAC;AAClF,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAEhD,iDAAiD;AACjD,oBAAY,SAAS;IAEnB,gCAAgC,yCAAyC;IACzE,uBAAuB,8BAA8B;IACrD,0BAA0B,iCAAiC;IAC3D,mBAAmB,0BAA0B;IAC7C,sBAAsB,6BAA6B;IACnD,2BAA2B,mCAAmC;IAC9D,0BAA0B,iCAAiC;IAC3D,6BAA6B,oCAAoC;IACjE,sCAAsC,+CAA+C;IAGrF,gBAAgB,uBAAuB;IACvC,mBAAmB,0BAA0B;IAC7C,uBAAuB,8BAA8B;IACrD,0BAA0B,iCAAiC;IAC3D,wBAAwB,gCAAgC;IACxD,kCAAkC,2CAA2C;CAC9E;AAED,iDAAiD;AACjD,oBAAY,kBAAkB;IAE5B,8BAA8B,uCAAuC;IACrE,+BAA+B,wCAAwC;IACvE,0BAA0B,iCAAiC;IAC3D,yBAAyB,gCAAgC;IACzD,sBAAsB,6BAA6B;IACnD,uBAAuB,8BAA8B;IACrD,sBAAsB,6BAA6B;IACnD,qBAAqB,4BAA4B;IACjD,kBAAkB,yBAAyB;IAC3C,mBAAmB,0BAA0B;IAC7C,0BAA0B,kCAAkC;IAC5D,2BAA2B,mCAAmC;IAC9D,4BAA4B,mCAAmC;IAC/D,2BAA2B,kCAAkC;IAC7D,wBAAwB,+BAA+B;IACvD,yBAAyB,gCAAgC;IACzD,qCAAqC,8CAA8C;IACnF,sCAAsC,+CAA+C;CACtF;AAED,uEAAuE;AACvE,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,oBAAY,gBAAgB;IAC1B,YAAY,iBAAiB;IAC7B,WAAW,gBAAgB;CAC5B;AAED,8EAA8E;AAC9E,oBAAY,QAAQ;IAClB,WAAW,OAAO;IAClB,YAAY,OAAO;IACnB,eAAe,YAAO;IACtB,YAAY,OAAO;IACnB,OAAO,MAAM;IACb,OAAO,MAAM;IACb,KAAK,MAAM;CACZ;AAED;;;GAGG;AACH,oBAAY,aAAa;IACvB,cAAc,mBAAmB;IACjC,2BAA2B,gCAAgC;IAC3D,cAAc,mBAAmB;IACjC,0BAA0B,+BAA+B;IACzD,YAAY,iBAAiB;IAC7B,aAAa,kBAAkB;IAC/B,SAAS,cAAc;CACxB;AAED;;;GAGG;AACH,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,aAAa,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,oBAAY,YAAY;IACtB,QAAQ,sCAAsC;IAC9C,UAAU,wCAAwC;IAClD,OAAO,qCAAqC;CAC7C;AAED,MAAM,WAAW,YAAY;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,uBAAuB,EAAE,MAAM,CAAC;IAChC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;IAC7B;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,0EAA0E;IAC1E,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,WAAW,EAAE,MAAM,CAAC;IACpB,kBAAkB,CAAC,EAAE,gBAAgB,CAAC;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,gBAAgB,EAAE,MAAM,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,KAAK,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,CAAC,EAAE,YAAY,CAAC;IAC9B,+EAA+E;IAC/E,yBAAyB,CAAC,EAAE,MAAM,CAAC;IACnC,2EAA2E;IAC3E,gCAAgC,CAAC,EAAE,OAAO,CAAC;IAC3C,eAAe,EAAE,MAAM,CAAC;IACxB,uEAAuE;IACvE,qBAAqB,CAAC,EAAE,SAAS,CAAC;IAClC,mEAAmE;IACnE,mBAAmB,CAAC,EAAE,SAAS,CAAC;IAChC,mGAAmG;IACnG,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,kFAAkF;AAClF,oBAAY,aAAa;IACvB,IAAI,SAAS;IACb,KAAK,UAAU;CAChB;AAED,mGAAmG;AACnG,MAAM,WAAW,aAAa;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,gBAAgB,EAAE,aAAa,CAAC;CACjC;AAED,MAAM,WAAW,SAAS;IACxB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf
|
|
1
|
+
{"version":3,"file":"extraction.d.ts","sourceRoot":"","sources":["../../src/types/extraction.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,+CAA+C,CAAC;AAClF,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAEhD,iDAAiD;AACjD,oBAAY,SAAS;IAEnB,gCAAgC,yCAAyC;IACzE,uBAAuB,8BAA8B;IACrD,0BAA0B,iCAAiC;IAC3D,mBAAmB,0BAA0B;IAC7C,sBAAsB,6BAA6B;IACnD,2BAA2B,mCAAmC;IAC9D,0BAA0B,iCAAiC;IAC3D,6BAA6B,oCAAoC;IACjE,sCAAsC,+CAA+C;IAGrF,gBAAgB,uBAAuB;IACvC,mBAAmB,0BAA0B;IAC7C,uBAAuB,8BAA8B;IACrD,0BAA0B,iCAAiC;IAC3D,wBAAwB,gCAAgC;IACxD,kCAAkC,2CAA2C;CAC9E;AAED,iDAAiD;AACjD,oBAAY,kBAAkB;IAE5B,8BAA8B,uCAAuC;IACrE,+BAA+B,wCAAwC;IACvE,0BAA0B,iCAAiC;IAC3D,yBAAyB,gCAAgC;IACzD,sBAAsB,6BAA6B;IACnD,uBAAuB,8BAA8B;IACrD,sBAAsB,6BAA6B;IACnD,qBAAqB,4BAA4B;IACjD,kBAAkB,yBAAyB;IAC3C,mBAAmB,0BAA0B;IAC7C,0BAA0B,kCAAkC;IAC5D,2BAA2B,mCAAmC;IAC9D,4BAA4B,mCAAmC;IAC/D,2BAA2B,kCAAkC;IAC7D,wBAAwB,+BAA+B;IACvD,yBAAyB,gCAAgC;IACzD,qCAAqC,8CAA8C;IACnF,sCAAsC,+CAA+C;CACtF;AAED,uEAAuE;AACvE,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,oBAAY,gBAAgB;IAC1B,YAAY,iBAAiB;IAC7B,WAAW,gBAAgB;CAC5B;AAED,8EAA8E;AAC9E,oBAAY,QAAQ;IAClB,WAAW,OAAO;IAClB,YAAY,OAAO;IACnB,eAAe,YAAO;IACtB,YAAY,OAAO;IACnB,OAAO,MAAM;IACb,OAAO,MAAM;IACb,KAAK,MAAM;CACZ;AAED;;;GAGG;AACH,oBAAY,aAAa;IACvB,cAAc,mBAAmB;IACjC,2BAA2B,gCAAgC;IAC3D,cAAc,mBAAmB;IACjC,0BAA0B,+BAA+B;IACzD,YAAY,iBAAiB;IAC7B,aAAa,kBAAkB;IAC/B,SAAS,cAAc;CACxB;AAED;;;GAGG;AACH,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,aAAa,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,oBAAY,YAAY;IACtB,QAAQ,sCAAsC;IAC9C,UAAU,wCAAwC;IAClD,OAAO,qCAAqC;CAC7C;AAED,MAAM,WAAW,YAAY;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,uBAAuB,EAAE,MAAM,CAAC;IAChC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;IAC7B;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,0EAA0E;IAC1E,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,WAAW,EAAE,MAAM,CAAC;IACpB,kBAAkB,CAAC,EAAE,gBAAgB,CAAC;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,gBAAgB,EAAE,MAAM,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,KAAK,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,CAAC,EAAE,YAAY,CAAC;IAC9B,+EAA+E;IAC/E,yBAAyB,CAAC,EAAE,MAAM,CAAC;IACnC,2EAA2E;IAC3E,gCAAgC,CAAC,EAAE,OAAO,CAAC;IAC3C,eAAe,EAAE,MAAM,CAAC;IACxB,uEAAuE;IACvE,qBAAqB,CAAC,EAAE,SAAS,CAAC;IAClC,mEAAmE;IACnE,mBAAmB,CAAC,EAAE,SAAS,CAAC;IAChC,mGAAmG;IACnG,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,kFAAkF;AAClF,oBAAY,aAAa;IACvB,IAAI,SAAS;IACb,KAAK,UAAU;CAChB;AAED,mGAAmG;AACnG,MAAM,WAAW,aAAa;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,gBAAgB,EAAE,aAAa,CAAC;CACjC;AAED,MAAM,WAAW,SAAS;IACxB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;OAIG;IACH,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IAEvB,OAAO,CAAC,EAAE,YAAY,EAAE,CAAC;IACzB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,0BAA0B,CAAC,EAAE,aAAa,EAAE,CAAC;CAC9C;AAED,MAAM,WAAW,cAAc;IAC7B,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAG/B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAG9B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,gDAAgD;AAChD,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE;QACP,OAAO,EAAE;YACP,qBAAqB,EAAE,MAAM,CAAC;SAC/B,CAAC;QACF,kBAAkB,EAAE,MAAM,CAAC;QAC3B,UAAU,EAAE,MAAM,CAAC;QACnB,0DAA0D;QAC1D,OAAO,EAAE,MAAM,CAAC;QAChB,iDAAiD;QACjD,OAAO,EAAE,MAAM,CAAC;QAChB,gEAAgE;QAChE,SAAS,EAAE,MAAM,CAAC;QAClB,yDAAyD;QACzD,kBAAkB,EAAE,MAAM,CAAC;KAC5B,CAAC;IACF,OAAO,EAAE,cAAc,CAAC;IACxB,kBAAkB,EAAE;QAClB,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;IACF,UAAU,EAAE,SAAS,CAAC;CACvB;AAED,8CAA8C;AAC9C,MAAM,WAAW,SAAS;IACxB,iDAAiD;IACjD,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtC,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtC,2EAA2E;IAE3E,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CACjC;AAED,MAAM,WAAW,cAAc;IAC7B,eAAe,EAAE,cAAc,CAAC;IAChC,aAAa,EAAE,YAAY,CAAC;IAC5B,UAAU,EAAE,SAAS,CAAC;IACtB,UAAU,CAAC,EAAE,SAAS,CAAC;CACxB;AAED,gDAAgD;AAChD,MAAM,WAAW,cAAc;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,YAAY,CAAC;IAC5B,UAAU,CAAC,EAAE,SAAS,CAAC;IACvB,eAAe,CAAC,EAAE,cAAc,CAAC;CAClC;AAED,MAAM,MAAM,yCAAyC,GAAG,CAAC,EACvD,IAAI,EACJ,KAAK,GACN,EAAE,uCAAuC,KAAK,OAAO,CAAC,yCAAyC,CAAC,CAAC;AAElG,MAAM,WAAW,uCAAuC;IACtD,IAAI,EAAE,oBAAoB,CAAC;IAC3B,KAAK,EAAE,YAAY,CAAC;CACrB;AAED,MAAM,WAAW,kBAAkB;IAEjC,IAAI,EAAE,GAAG,CAAC;IAEV,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAC9B;AAED,gEAAgE;AAChE,MAAM,MAAM,yCAAyC,GACjD;IAAE,MAAM,EAAE,SAAS,CAAC;IAAC,UAAU,EAAE,kBAAkB,CAAA;CAAE,GACrD;IAAE,MAAM,EAAE,OAAO,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,GACzC;IAAE,MAAM,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,WAAW,CAAA;CAAE,CAAC;AAE5C;;;;GAIG;AACH,MAAM,MAAM,2BAA2B,GACnC;IAAE,MAAM,EAAE,SAAS,CAAA;CAAE,GACrB;IAAE,MAAM,EAAE,OAAO,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,GACzC;IAAE,MAAM,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,WAAW,GAAG;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAA;CAAE,CAAC;AAEpE,MAAM,MAAM,uCAAuC,CACjD,KAAK,EACL,QAAQ,EACR,cAAc,IACZ,CAAC,EACH,WAAW,EACX,OAAO,EACP,SAAS,GACV,EAAE;IACD,WAAW,EAAE,KAAK,CAAC;IACnB,OAAO,EAAE,iBAAiB,CAAC,cAAc,CAAC,CAAC;IAC3C,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,KAAK,QAAQ,CAAC;AAEf,MAAM,MAAM,iCAAiC,GAAG,CAAC,EAC/C,UAAU,EACV,MAAM,GACP,EAAE;IACD,UAAU,EAAE,oBAAoB,CAAC;IACjC,MAAM,EAAE,yCAAyC,CAAC;CACnD,KAAK,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAE3C,MAAM,MAAM,wCAAwC,CAAC,QAAQ,EAAE,cAAc,IAC3E,CAAC,EACC,kBAAkB,EAClB,OAAO,EACP,MAAM,GACP,EAAE;IACD,kBAAkB,EAAE,QAAQ,CAAC;IAC7B,OAAO,EAAE,iBAAiB,CAAC,cAAc,CAAC,CAAC;IAC3C,MAAM,EAAE,yCAAyC,CAAC;CACnD,KAAK,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAE7C,MAAM,WAAW,kCAAkC,CACjD,cAAc,EACd,KAAK,EACL,QAAQ;IAER,OAAO,EAAE,uCAAuC,CAC9C,KAAK,EACL,QAAQ,EACR,cAAc,CACf,CAAC;IACF,QAAQ,EAAE,wCAAwC,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;CAC9E"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { AxiosResponse } from 'axios';
|
|
2
2
|
import { NormalizedAttachment } from '../repo/repo.interfaces';
|
|
3
3
|
import { HttpStreamResponse } from '../types/extraction';
|
|
4
|
-
import { Artifact, ArtifactToUpload, MultipartUpload, UploaderFactoryInterface, UploaderResult,
|
|
4
|
+
import { Artifact, ArtifactToUpload, MultipartUpload, UploaderFactoryInterface, UploaderResult, UploadResponse } from './uploader.interfaces';
|
|
5
5
|
export declare class Uploader {
|
|
6
6
|
private isLocalDevelopment?;
|
|
7
7
|
private devrevApiEndpoint;
|
|
@@ -13,7 +13,9 @@ export declare class Uploader {
|
|
|
13
13
|
/** Compresses fetched objects to gzipped JSONL and uploads them as an artifact. */
|
|
14
14
|
upload(itemType: string, fetchedObjects: object[] | object): Promise<UploadResponse>;
|
|
15
15
|
/** Uploads ready-made Parquet bytes without serializing or compressing them. */
|
|
16
|
-
|
|
16
|
+
uploadParquet(itemType: string, file: Buffer, itemCount: number): Promise<UploaderResult<Artifact>>;
|
|
17
|
+
/** Keeps raw errors for callers that need HTTP diagnostics. */
|
|
18
|
+
private uploadFile;
|
|
17
19
|
getArtifactUploadUrl(filename: string, fileType: string, fileSize?: number): Promise<UploaderResult<ArtifactToUpload>>;
|
|
18
20
|
/** Uploads the file buffer to the artifact's upload URL as multipart form data. */
|
|
19
21
|
uploadArtifact(artifact: ArtifactToUpload, file: Buffer): Promise<UploaderResult<AxiosResponse>>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAUtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAWzD,OAAO,EACL,QAAQ,EACR,gBAAgB,EAChB,eAAe,EACf,wBAAwB,EACxB,cAAc,EACd,
|
|
1
|
+
{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAUtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAWzD,OAAO,EACL,QAAQ,EACR,gBAAgB,EAChB,eAAe,EACf,wBAAwB,EACxB,cAAc,EACd,cAAc,EACf,MAAM,uBAAuB,CAAC;AAE/B,qBAAa,QAAQ;IACnB,OAAO,CAAC,kBAAkB,CAAC,CAAU;IACrC,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,cAAc,CAAyB;IAC/C,OAAO,CAAC,gBAAgB,CAAU;gBAEtB,EAAE,KAAK,EAAE,OAAO,EAAE,gBAAgB,EAAE,EAAE,wBAAwB;IAW1E,mFAAmF;IAC7E,MAAM,CACV,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,EAAE,GAAG,MAAM,GAChC,OAAO,CAAC,cAAc,CAAC;IAgD1B,gFAAgF;IAC1E,aAAa,CACjB,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;IAoBpC,+DAA+D;YACjD,UAAU;IA0ClB,oBAAoB,CACxB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;IA2B5C,mFAAmF;IAC7E,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAmBzC,kFAAkF;IAC5E,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,UAAU,EAAE,kBAAkB,GAC7B,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAqCzC;;;;OAIG;IACG,qBAAqB,CACzB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,QAAQ,SAA0B,GACjC,OAAO,CAAC,cAAc,CAAC,eAAe,CAAC,CAAC;IAkB3C,+DAA+D;IACzD,0BAA0B,CAC9B,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,QAAQ,SAA0B,GACjC,OAAO,CAAC,cAAc,CAAC,eAAe,CAAC,CAAC;IAkB3C,+DAA+D;IACzD,uBAAuB,CAC3B,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAoBzC;;;;OAIG;IACG,uBAAuB,CAC3B,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,kBAAkB,GAC7B,OAAO,CAAC,cAAc,CAAC;QAAE,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IA6HnD,oDAAoD;YACtC,mBAAmB;IAsD3B,qBAAqB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC;QACvD,QAAQ,CAAC,EAAE,aAAa,CAAC;QACzB,KAAK,CAAC,EAAE,OAAO,CAAC;KACjB,CAAC;IAgCF,oEAAoE;IAC9D,4BAA4B,CAAC,EACjC,QAAQ,GACT,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;KAClB,GAAG,OAAO,CAAC;QACV,WAAW,CAAC,EAAE,oBAAoB,EAAE,CAAC;QACrC,KAAK,CAAC,EAAE;YAAE,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC;KAC7B,CAAC;YAuDY,sBAAsB;YAsBtB,gBAAgB;IAc9B,yEAAyE;IACnE,yBAAyB,CAAC,EAC9B,UAAU,EACV,SAAiB,GAClB,EAAE;QACD,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,CAAC,EAAE,OAAO,CAAC;KACrB,GAAG,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC;CA6B/C"}
|
|
@@ -30,15 +30,6 @@ export interface Artifact {
|
|
|
30
30
|
[ArtifactDateField.OldestModifiedDate]?: string;
|
|
31
31
|
[ArtifactDateField.NewestModifiedDate]?: string;
|
|
32
32
|
}
|
|
33
|
-
/** A ready-made dataset file to upload and report in an extraction event. */
|
|
34
|
-
export interface UploadExtractionFileParams {
|
|
35
|
-
/** Dataset/query key, not a built-in metadata or attachment item type. */
|
|
36
|
-
itemType: string;
|
|
37
|
-
/** Valid file bytes in the format accepted by the upload method. */
|
|
38
|
-
file: Buffer;
|
|
39
|
-
/** Actual row count; must be a positive safe integer. Not inferred from the file. */
|
|
40
|
-
itemCount: number;
|
|
41
|
-
}
|
|
42
33
|
export type ArtifactDateRanges = Pick<Artifact, ArtifactDateField.OldestCreatedDate | ArtifactDateField.NewestCreatedDate | ArtifactDateField.OldestModifiedDate | ArtifactDateField.NewestModifiedDate>;
|
|
43
34
|
/** Response from the get upload url endpoint. */
|
|
44
35
|
export interface ArtifactToUpload {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"uploader.interfaces.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAElD,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,YAAY,CAAC;IACpB,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,uFAAuF;IACvF,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,MAAM,cAAc,CAAC,CAAC,IACxB;IAAE,QAAQ,EAAE,CAAC,CAAC;IAAC,KAAK,CAAC,EAAE,KAAK,CAAA;CAAE,GAC9B;IAAE,QAAQ,CAAC,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE,CAAC;AAEzC,oBAAY,iBAAiB;IAC3B,iBAAiB,wBAAwB;IACzC,iBAAiB,wBAAwB;IACzC,kBAAkB,yBAAyB;IAC3C,kBAAkB,yBAAyB;CAC5C;AAED,iEAAiE;AACjE,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,CAAC,EAAE,MAAM,CAAC;IAC/C,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,CAAC,EAAE,MAAM,CAAC;IAC/C,CAAC,iBAAiB,CAAC,kBAAkB,CAAC,CAAC,EAAE,MAAM,CAAC;IAChD,CAAC,iBAAiB,CAAC,kBAAkB,CAAC,CAAC,EAAE,MAAM,CAAC;CACjD;AAED,
|
|
1
|
+
{"version":3,"file":"uploader.interfaces.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAElD,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,YAAY,CAAC;IACpB,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,uFAAuF;IACvF,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,MAAM,cAAc,CAAC,CAAC,IACxB;IAAE,QAAQ,EAAE,CAAC,CAAC;IAAC,KAAK,CAAC,EAAE,KAAK,CAAA;CAAE,GAC9B;IAAE,QAAQ,CAAC,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE,CAAC;AAEzC,oBAAY,iBAAiB;IAC3B,iBAAiB,wBAAwB;IACzC,iBAAiB,wBAAwB;IACzC,kBAAkB,yBAAyB;IAC3C,kBAAkB,yBAAyB;CAC5C;AAED,iEAAiE;AACjE,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,CAAC,EAAE,MAAM,CAAC;IAC/C,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,CAAC,EAAE,MAAM,CAAC;IAC/C,CAAC,iBAAiB,CAAC,kBAAkB,CAAC,CAAC,EAAE,MAAM,CAAC;IAChD,CAAC,iBAAiB,CAAC,kBAAkB,CAAC,CAAC,EAAE,MAAM,CAAC;CACjD;AAED,MAAM,MAAM,kBAAkB,GAAG,IAAI,CACnC,QAAQ,EACN,iBAAiB,CAAC,iBAAiB,GACnC,iBAAiB,CAAC,iBAAiB,GACnC,iBAAiB,CAAC,kBAAkB,GACpC,iBAAiB,CAAC,kBAAkB,CACvC,CAAC;AAEF,iDAAiD;AACjD,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE;QACT,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;KACf,EAAE,CAAC;CACL;AAED,yEAAyE;AACzE,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE;QACF,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,SAAS,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,QAAQ,CAAC,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB"}
|
|
@@ -37,64 +37,58 @@ class Uploader {
|
|
|
37
37
|
},
|
|
38
38
|
};
|
|
39
39
|
}
|
|
40
|
-
const
|
|
41
|
-
|
|
42
|
-
const { error: preparedArtifactError, response: preparedArtifact } = await this.getArtifactUploadUrl(filename, fileType);
|
|
43
|
-
if (preparedArtifactError) {
|
|
40
|
+
const { response: artifactId, error, errorMessage, } = await this.uploadFile(itemType + '.jsonl.gz', 'application/x-gzip', file, undefined, this.skipConfirmation);
|
|
41
|
+
if (error !== undefined) {
|
|
44
42
|
return {
|
|
45
43
|
error: {
|
|
46
|
-
message:
|
|
47
|
-
(0, logger_1.serializeError)(preparedArtifactError),
|
|
44
|
+
message: errorMessage + (0, logger_1.serializeError)(error),
|
|
48
45
|
},
|
|
49
46
|
};
|
|
50
47
|
}
|
|
51
|
-
const { error: uploadItemError } = await this.uploadArtifact(preparedArtifact, file);
|
|
52
|
-
if (uploadItemError) {
|
|
53
|
-
return {
|
|
54
|
-
error: {
|
|
55
|
-
message: 'Error while uploading artifact: ' +
|
|
56
|
-
(0, logger_1.serializeError)(uploadItemError),
|
|
57
|
-
},
|
|
58
|
-
};
|
|
59
|
-
}
|
|
60
|
-
if (!this.skipConfirmation) {
|
|
61
|
-
const { error: confirmArtifactUploadError } = await this.confirmArtifactUpload(preparedArtifact.artifact_id);
|
|
62
|
-
if (confirmArtifactUploadError) {
|
|
63
|
-
return {
|
|
64
|
-
error: {
|
|
65
|
-
message: 'Error while confirming artifact upload. ' +
|
|
66
|
-
(0, logger_1.serializeError)(confirmArtifactUploadError),
|
|
67
|
-
},
|
|
68
|
-
};
|
|
69
|
-
}
|
|
70
|
-
}
|
|
71
48
|
const artifactDateRanges = (0, uploader_helpers_1.computeArtifactDateRanges)(fetchedObjects);
|
|
72
|
-
const artifact = Object.assign({ id:
|
|
49
|
+
const artifact = Object.assign({ id: artifactId, item_type: itemType, item_count: Array.isArray(fetchedObjects) ? fetchedObjects.length : 1 }, artifactDateRanges);
|
|
73
50
|
return { artifact };
|
|
74
51
|
}
|
|
75
52
|
/** Uploads ready-made Parquet bytes without serializing or compressing them. */
|
|
76
|
-
async
|
|
77
|
-
const { response:
|
|
53
|
+
async uploadParquet(itemType, file, itemCount) {
|
|
54
|
+
const { response: artifactId, error } = await this.uploadFile(`${itemType}.parquet`, 'application/vnd.apache.parquet', file, file.length);
|
|
55
|
+
if (error !== undefined) {
|
|
56
|
+
return { error };
|
|
57
|
+
}
|
|
58
|
+
return {
|
|
59
|
+
response: {
|
|
60
|
+
id: artifactId,
|
|
61
|
+
item_type: itemType,
|
|
62
|
+
item_count: itemCount,
|
|
63
|
+
},
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
/** Keeps raw errors for callers that need HTTP diagnostics. */
|
|
67
|
+
async uploadFile(filename, fileType, file, fileSize, skipConfirmation = false) {
|
|
68
|
+
const { response: preparedArtifact, error: prepareError } = await this.getArtifactUploadUrl(filename, fileType, fileSize);
|
|
78
69
|
if (prepareError !== undefined || !preparedArtifact) {
|
|
79
70
|
return {
|
|
80
71
|
error: prepareError !== null && prepareError !== void 0 ? prepareError : new Error('No artifact upload URL returned.'),
|
|
72
|
+
errorMessage: 'Error while getting artifact upload URL: ',
|
|
81
73
|
};
|
|
82
74
|
}
|
|
83
75
|
const { error: uploadError } = await this.uploadArtifact(preparedArtifact, file);
|
|
84
76
|
if (uploadError !== undefined) {
|
|
85
|
-
return {
|
|
77
|
+
return {
|
|
78
|
+
error: uploadError,
|
|
79
|
+
errorMessage: 'Error while uploading artifact: ',
|
|
80
|
+
};
|
|
86
81
|
}
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
82
|
+
if (!skipConfirmation) {
|
|
83
|
+
const { error: confirmError } = await this.confirmArtifactUpload(preparedArtifact.artifact_id);
|
|
84
|
+
if (confirmError !== undefined) {
|
|
85
|
+
return {
|
|
86
|
+
error: confirmError,
|
|
87
|
+
errorMessage: 'Error while confirming artifact upload. ',
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
90
|
}
|
|
91
|
-
return {
|
|
92
|
-
response: {
|
|
93
|
-
id: preparedArtifact.artifact_id,
|
|
94
|
-
item_type: itemType,
|
|
95
|
-
item_count: itemCount,
|
|
96
|
-
},
|
|
97
|
-
};
|
|
91
|
+
return { response: preparedArtifact.artifact_id };
|
|
98
92
|
}
|
|
99
93
|
async getArtifactUploadUrl(filename, fileType, fileSize) {
|
|
100
94
|
const url = `${this.devrevApiEndpoint}/internal/airdrop.artifacts.upload-url`;
|
package/docs/MIGRATION.md
CHANGED
|
@@ -395,7 +395,7 @@ async function extractList(adapter: ExtractionAdapter<State>) { ... }
|
|
|
395
395
|
|
|
396
396
|
| Members | On |
|
|
397
397
|
|---------|----|
|
|
398
|
-
| `initializeRepos`, `getRepo`, `
|
|
398
|
+
| `initializeRepos`, `getRepo`, `streamAttachments`, `processAttachment`, `shouldExtract`, `artifacts`, `preExtractionItemCounts` | `ExtractionAdapter` |
|
|
399
399
|
| `loadItemTypes`, `loadAttachments`, `reports`, `processedFiles` | `LoadingAdapter` |
|
|
400
400
|
| `event`, `state`, `mappers`, `postState`, `uploadArtifact`, `downloadArtifact`, `isTimeout`, `extractionScope`, `extractionQueries` | both |
|
|
401
401
|
|
|
@@ -407,32 +407,6 @@ async function extractList(adapter: ExtractionAdapter<State>) { ... }
|
|
|
407
407
|
public. `isTimeout` and `artifacts` are read-only.
|
|
408
408
|
- `adapter.mappers` works in extraction as in v1; keep those calls and only
|
|
409
409
|
apply the §10 return-shape changes.
|
|
410
|
-
- For ready-made DLH Parquet files, use the extraction-only API during a
|
|
411
|
-
data-extraction task:
|
|
412
|
-
|
|
413
|
-
```ts
|
|
414
|
-
await adapter.uploadParquetExtractionFile({
|
|
415
|
-
itemType: queryKey,
|
|
416
|
-
file: parquetBuffer,
|
|
417
|
-
itemCount: rowCount,
|
|
418
|
-
});
|
|
419
|
-
```
|
|
420
|
-
|
|
421
|
-
It uploads and confirms the `Buffer` unchanged, then registers the returned
|
|
422
|
-
`Artifact` for the next callback, including progress/errors. It throws on
|
|
423
|
-
failure. The request type is `UploadExtractionFileParams`; the method selects
|
|
424
|
-
the Parquet MIME type and filename extension.
|
|
425
|
-
Supply valid Parquet and the actual positive safe-integer row count;
|
|
426
|
-
normalization, batching and timestamp tracking are bypassed. Built-in
|
|
427
|
-
metadata and attachment item types are not allowed. Continue checking
|
|
428
|
-
`adapter.isTimeout` between files, and split large results into files smaller
|
|
429
|
-
than 2 GiB. This API is buffer-only; it does not write a local JSONL copy.
|
|
430
|
-
Keep `query_result_metadata` in a separate repo, pushing one complete JSON
|
|
431
|
-
object with `format: 'parquet'` and `datasets: { [queryKey]: { schema: ... } }`.
|
|
432
|
-
All data files covered by that metadata must be Parquet. Skip zero-row
|
|
433
|
-
uploads; omitting files does not clear an existing dataset.
|
|
434
|
-
`uploadArtifact` / `downloadArtifact` remain JSONL storage for connector-owned
|
|
435
|
-
data, not extraction output, and `repo.push()` keeps its JSONL behavior.
|
|
436
410
|
- **Integration tests** that built an adapter by hand use the root factories:
|
|
437
411
|
|
|
438
412
|
```ts
|