@devrev/airsync-sdk 2.0.0-beta.3 → 2.0.0-beta.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/MIGRATION.md +1028 -0
- package/README.md +5 -11
- package/dist/attachments-streaming/attachments-streaming-pool.d.ts +7 -9
- package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map +1 -1
- package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts +2 -7
- package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts.map +1 -1
- package/dist/attachments-streaming/attachments-streaming-pool.js +54 -56
- package/dist/common/constants.d.ts +4 -5
- package/dist/common/constants.d.ts.map +1 -1
- package/dist/common/constants.js +6 -7
- package/dist/common/helpers.d.ts +3 -35
- package/dist/common/helpers.d.ts.map +1 -1
- package/dist/common/helpers.js +5 -50
- package/dist/http/{axios-client-internal.d.ts → client.d.ts} +1 -1
- package/dist/http/client.d.ts.map +1 -0
- package/dist/http/{axios-client-internal.js → client.js} +4 -8
- package/dist/index.d.ts +24 -13
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +43 -32
- package/dist/logger/logger.d.ts +12 -53
- package/dist/logger/logger.d.ts.map +1 -1
- package/dist/logger/logger.interfaces.d.ts +0 -1
- package/dist/logger/logger.interfaces.d.ts.map +1 -1
- package/dist/logger/logger.js +36 -68
- package/dist/mappers/mappers.d.ts +7 -46
- package/dist/mappers/mappers.d.ts.map +1 -1
- package/dist/mappers/mappers.interfaces.d.ts +175 -0
- package/dist/mappers/mappers.interfaces.d.ts.map +1 -0
- package/dist/mappers/{mappers.interface.js → mappers.interfaces.js} +4 -13
- package/dist/mappers/mappers.js +11 -45
- package/dist/multithreading/adapters/base-adapter.d.ts +30 -32
- package/dist/multithreading/adapters/base-adapter.d.ts.map +1 -1
- package/dist/multithreading/adapters/base-adapter.js +97 -62
- package/dist/multithreading/adapters/extraction-adapter.d.ts +16 -29
- package/dist/multithreading/adapters/extraction-adapter.d.ts.map +1 -1
- package/dist/multithreading/adapters/extraction-adapter.js +203 -206
- package/dist/multithreading/adapters/loading-adapter.d.ts +8 -13
- package/dist/multithreading/adapters/loading-adapter.d.ts.map +1 -1
- package/dist/multithreading/adapters/loading-adapter.helpers.d.ts +2 -12
- package/dist/multithreading/adapters/loading-adapter.helpers.d.ts.map +1 -1
- package/dist/multithreading/adapters/loading-adapter.helpers.js +2 -12
- package/dist/multithreading/adapters/loading-adapter.js +294 -346
- package/dist/multithreading/emit.d.ts +11 -0
- package/dist/multithreading/emit.d.ts.map +1 -0
- package/dist/{common/control-protocol.js → multithreading/emit.js} +5 -7
- package/dist/multithreading/process-task.d.ts +2 -12
- package/dist/multithreading/process-task.d.ts.map +1 -1
- package/dist/multithreading/process-task.js +46 -43
- package/dist/multithreading/spawn/spawn.d.ts +3 -9
- package/dist/multithreading/spawn/spawn.d.ts.map +1 -1
- package/dist/multithreading/spawn/spawn.helpers.d.ts +12 -11
- package/dist/multithreading/spawn/spawn.helpers.d.ts.map +1 -1
- package/dist/multithreading/spawn/spawn.helpers.js +128 -20
- package/dist/multithreading/spawn/spawn.js +24 -39
- package/dist/multithreading/worker.js +1 -4
- package/dist/repo/repo.d.ts +10 -0
- package/dist/repo/repo.d.ts.map +1 -1
- package/dist/repo/repo.helpers.d.ts +6 -0
- package/dist/repo/repo.helpers.d.ts.map +1 -0
- package/dist/repo/repo.helpers.js +16 -0
- package/dist/repo/repo.interfaces.d.ts +4 -16
- package/dist/repo/repo.interfaces.d.ts.map +1 -1
- package/dist/repo/repo.js +23 -8
- package/dist/state/base-state.d.ts +16 -38
- package/dist/state/base-state.d.ts.map +1 -1
- package/dist/state/base-state.js +19 -41
- package/dist/state/extraction-state.d.ts +6 -21
- package/dist/state/extraction-state.d.ts.map +1 -1
- package/dist/state/extraction-state.js +8 -25
- package/dist/{common → state}/install-initial-domain-mapping.d.ts +1 -1
- package/dist/state/install-initial-domain-mapping.d.ts.map +1 -0
- package/dist/{common → state}/install-initial-domain-mapping.js +5 -7
- package/dist/state/loading-state.d.ts +3 -14
- package/dist/state/loading-state.d.ts.map +1 -1
- package/dist/state/loading-state.js +3 -14
- package/dist/state/state.d.ts +4 -0
- package/dist/state/state.d.ts.map +1 -0
- package/dist/state/state.interfaces.d.ts +33 -37
- package/dist/state/state.interfaces.d.ts.map +1 -1
- package/dist/state/state.interfaces.js +16 -11
- package/dist/state/state.js +11 -0
- package/dist/state/time-value-resolver.d.ts +17 -0
- package/dist/state/time-value-resolver.d.ts.map +1 -0
- package/dist/{common → state}/time-value-resolver.js +9 -41
- package/dist/testing/mock-event.d.ts +10 -0
- package/dist/testing/mock-event.d.ts.map +1 -0
- package/dist/{common/test-utils.js → testing/mock-event.js} +17 -14
- package/dist/testing/mock-server.d.ts +29 -0
- package/dist/testing/mock-server.d.ts.map +1 -0
- package/dist/testing/mock-server.interfaces.d.ts +48 -0
- package/dist/testing/mock-server.interfaces.d.ts.map +1 -0
- package/dist/{mock-server → testing}/mock-server.js +21 -33
- package/dist/types/common.d.ts +1 -12
- package/dist/types/common.d.ts.map +1 -1
- package/dist/types/common.js +1 -6
- package/dist/types/errors.d.ts +9 -0
- package/dist/types/errors.d.ts.map +1 -0
- package/dist/types/external-domain-metadata.d.ts +30 -171
- package/dist/types/external-domain-metadata.d.ts.map +1 -1
- package/dist/types/extraction.d.ts +64 -94
- package/dist/types/extraction.d.ts.map +1 -1
- package/dist/types/extraction.js +6 -34
- package/dist/types/loading.d.ts +1 -2
- package/dist/types/loading.d.ts.map +1 -1
- package/dist/types/loading.js +0 -1
- package/dist/types/workers.d.ts +39 -94
- package/dist/types/workers.d.ts.map +1 -1
- package/dist/types/workers.js +0 -6
- package/dist/uploader/uploader.d.ts +8 -59
- package/dist/uploader/uploader.d.ts.map +1 -1
- package/dist/uploader/uploader.helpers.d.ts +4 -27
- package/dist/uploader/uploader.helpers.d.ts.map +1 -1
- package/dist/uploader/uploader.helpers.js +64 -30
- package/dist/uploader/uploader.interfaces.d.ts +15 -25
- package/dist/uploader/uploader.interfaces.d.ts.map +1 -1
- package/dist/uploader/uploader.interfaces.js +8 -0
- package/dist/uploader/uploader.js +23 -89
- package/package.json +8 -6
- package/dist/airsync-sdk.d.ts +0 -2059
- package/dist/attachments-streaming/attachments-streaming-pool.test.d.ts +0 -2
- package/dist/attachments-streaming/attachments-streaming-pool.test.d.ts.map +0 -1
- package/dist/attachments-streaming/attachments-streaming-pool.test.js +0 -540
- package/dist/common/control-protocol.d.ts +0 -10
- package/dist/common/control-protocol.d.ts.map +0 -1
- package/dist/common/errors.d.ts +0 -6
- package/dist/common/errors.d.ts.map +0 -1
- package/dist/common/install-initial-domain-mapping.d.ts.map +0 -1
- package/dist/common/install-initial-domain-mapping.test.d.ts +0 -2
- package/dist/common/install-initial-domain-mapping.test.d.ts.map +0 -1
- package/dist/common/install-initial-domain-mapping.test.js +0 -211
- package/dist/common/test-utils.d.ts +0 -19
- package/dist/common/test-utils.d.ts.map +0 -1
- package/dist/common/time-value-resolver.d.ts +0 -47
- package/dist/common/time-value-resolver.d.ts.map +0 -1
- package/dist/common/time-value-resolver.test.d.ts +0 -2
- package/dist/common/time-value-resolver.test.d.ts.map +0 -1
- package/dist/common/time-value-resolver.test.js +0 -268
- package/dist/http/axios-client-internal.d.ts.map +0 -1
- package/dist/http/axios-client-internal.test.d.ts +0 -2
- package/dist/http/axios-client-internal.test.d.ts.map +0 -1
- package/dist/http/axios-client-internal.test.js +0 -175
- package/dist/http/axios-client.d.ts +0 -27
- package/dist/http/axios-client.d.ts.map +0 -1
- package/dist/http/axios-client.js +0 -60
- package/dist/http/constants.d.ts +0 -4
- package/dist/http/constants.d.ts.map +0 -1
- package/dist/http/constants.js +0 -6
- package/dist/http/index.d.ts +0 -2
- package/dist/http/index.d.ts.map +0 -1
- package/dist/http/index.js +0 -17
- package/dist/logger/logger.constants.d.ts +0 -6
- package/dist/logger/logger.constants.d.ts.map +0 -1
- package/dist/logger/logger.constants.js +0 -13
- package/dist/logger/logger.context.d.ts +0 -58
- package/dist/logger/logger.context.d.ts.map +0 -1
- package/dist/logger/logger.context.js +0 -86
- package/dist/logger/logger.test.d.ts +0 -2
- package/dist/logger/logger.test.d.ts.map +0 -1
- package/dist/logger/logger.test.js +0 -537
- package/dist/mappers/mappers.interface.d.ts +0 -294
- package/dist/mappers/mappers.interface.d.ts.map +0 -1
- package/dist/mappers/mappers.test.d.ts +0 -2
- package/dist/mappers/mappers.test.d.ts.map +0 -1
- package/dist/mappers/mappers.test.js +0 -105
- package/dist/mock-server/mock-server.d.ts +0 -47
- package/dist/mock-server/mock-server.d.ts.map +0 -1
- package/dist/mock-server/mock-server.interfaces.d.ts +0 -73
- package/dist/mock-server/mock-server.interfaces.d.ts.map +0 -1
- package/dist/multithreading/adapters/base-adapter.emit.test.d.ts +0 -2
- package/dist/multithreading/adapters/base-adapter.emit.test.d.ts.map +0 -1
- package/dist/multithreading/adapters/base-adapter.emit.test.js +0 -403
- package/dist/multithreading/adapters/extraction-adapter.artifacts.test.d.ts +0 -2
- package/dist/multithreading/adapters/extraction-adapter.artifacts.test.d.ts.map +0 -1
- package/dist/multithreading/adapters/extraction-adapter.artifacts.test.js +0 -175
- package/dist/multithreading/adapters/extraction-adapter.test.d.ts +0 -2
- package/dist/multithreading/adapters/extraction-adapter.test.d.ts.map +0 -1
- package/dist/multithreading/adapters/extraction-adapter.test.js +0 -789
- package/dist/multithreading/adapters/loading-adapter.helpers.test.d.ts +0 -2
- package/dist/multithreading/adapters/loading-adapter.helpers.test.d.ts.map +0 -1
- package/dist/multithreading/adapters/loading-adapter.helpers.test.js +0 -514
- package/dist/multithreading/adapters/loading-adapter.test.d.ts +0 -2
- package/dist/multithreading/adapters/loading-adapter.test.d.ts.map +0 -1
- package/dist/multithreading/adapters/loading-adapter.test.js +0 -589
- package/dist/multithreading/adapters/serialization.test.d.ts +0 -2
- package/dist/multithreading/adapters/serialization.test.d.ts.map +0 -1
- package/dist/multithreading/adapters/serialization.test.js +0 -71
- package/dist/multithreading/create-worker.test.d.ts +0 -2
- package/dist/multithreading/create-worker.test.d.ts.map +0 -1
- package/dist/multithreading/create-worker.test.js +0 -108
- package/dist/multithreading/process-task.test.d.ts +0 -2
- package/dist/multithreading/process-task.test.d.ts.map +0 -1
- package/dist/multithreading/process-task.test.js +0 -146
- package/dist/multithreading/spawn/spawn.helpers.test.d.ts +0 -2
- package/dist/multithreading/spawn/spawn.helpers.test.d.ts.map +0 -1
- package/dist/multithreading/spawn/spawn.helpers.test.js +0 -293
- package/dist/multithreading/spawn/spawn.test.d.ts +0 -2
- package/dist/multithreading/spawn/spawn.test.d.ts.map +0 -1
- package/dist/multithreading/spawn/spawn.test.js +0 -232
- package/dist/repo/repo.test.d.ts +0 -2
- package/dist/repo/repo.test.d.ts.map +0 -1
- package/dist/repo/repo.test.js +0 -184
- package/dist/state/state.extract-window.test.d.ts +0 -2
- package/dist/state/state.extract-window.test.d.ts.map +0 -1
- package/dist/state/state.extract-window.test.js +0 -163
- package/dist/state/state.pending-boundaries.test.d.ts +0 -2
- package/dist/state/state.pending-boundaries.test.d.ts.map +0 -1
- package/dist/state/state.pending-boundaries.test.js +0 -189
- package/dist/state/state.post-state.test.d.ts +0 -2
- package/dist/state/state.post-state.test.d.ts.map +0 -1
- package/dist/state/state.post-state.test.js +0 -78
- package/dist/state/state.test.d.ts +0 -2
- package/dist/state/state.test.d.ts.map +0 -1
- package/dist/state/state.test.js +0 -271
- package/dist/state/state.time-value-resolution.test.d.ts +0 -2
- package/dist/state/state.time-value-resolution.test.d.ts.map +0 -1
- package/dist/state/state.time-value-resolution.test.js +0 -175
- package/dist/tsdoc-metadata.json +0 -11
- package/dist/types/extraction.test.d.ts +0 -2
- package/dist/types/extraction.test.d.ts.map +0 -1
- package/dist/types/extraction.test.js +0 -96
- package/dist/types/index.d.ts +0 -10
- package/dist/types/index.d.ts.map +0 -1
- package/dist/types/index.js +0 -21
- package/dist/uploader/uploader.helpers.test.d.ts +0 -2
- package/dist/uploader/uploader.helpers.test.d.ts.map +0 -1
- package/dist/uploader/uploader.helpers.test.js +0 -256
- package/dist/uploader/uploader.test.d.ts +0 -2
- package/dist/uploader/uploader.test.d.ts.map +0 -1
- package/dist/uploader/uploader.test.js +0 -601
- /package/dist/{mock-server → testing}/mock-server.interfaces.js +0 -0
- /package/dist/{common → types}/errors.js +0 -0
package/dist/types/workers.d.ts
CHANGED
|
@@ -1,54 +1,27 @@
|
|
|
1
1
|
import { Worker } from 'worker_threads';
|
|
2
2
|
import type { LogLevel } from '../logger/logger.interfaces';
|
|
3
|
-
import { BaseState } from '../state/
|
|
4
|
-
import {
|
|
5
|
-
import { BaseAdapter } from '../multithreading/adapters/base-adapter';
|
|
3
|
+
import { BaseState } from '../state/state';
|
|
4
|
+
import { ErrorRecord, InitialDomainMapping } from './common';
|
|
6
5
|
import { AirSyncEvent, EventType, ExtractorEventType } from './extraction';
|
|
7
6
|
import { LoaderEventType } from './loading';
|
|
8
|
-
|
|
9
|
-
/**
|
|
10
|
-
* WorkerAdapterInterface is the constructor parameter shape for an adapter.
|
|
11
|
-
* @interface WorkerAdapterInterface
|
|
12
|
-
* @constructor
|
|
13
|
-
* @param {AirSyncEvent} event - The event object received from the platform
|
|
14
|
-
* @param {BaseState} adapterState - The state instance for the adapter
|
|
15
|
-
* @param {WorkerAdapterOptions} options - The options to create a new adapter
|
|
16
|
-
*/
|
|
17
|
-
export interface WorkerAdapterInterface<ConnectorState, StateClass extends BaseState<ConnectorState, BaseSdkState>> {
|
|
7
|
+
export interface WorkerAdapterInterface<ConnectorState> {
|
|
18
8
|
event: AirSyncEvent;
|
|
19
|
-
adapterState:
|
|
9
|
+
adapterState: BaseState<ConnectorState>;
|
|
20
10
|
options?: WorkerAdapterOptions;
|
|
21
11
|
}
|
|
22
|
-
/**
|
|
23
|
-
* ExtractionScope represents the parsed extraction scope from the platform.
|
|
24
|
-
* Each key is an item type name, and the value indicates whether it should be extracted.
|
|
25
|
-
*/
|
|
12
|
+
/** Parsed extraction scope from the platform, keyed by item type name. */
|
|
26
13
|
export type ExtractionScope = Record<string, {
|
|
27
14
|
extract: boolean;
|
|
28
15
|
}>;
|
|
29
|
-
/**
|
|
30
|
-
* WorkerAdapterOptions represents the options for WorkerAdapter class.
|
|
31
|
-
* @interface WorkerAdapterOptions
|
|
32
|
-
* @constructor
|
|
33
|
-
* @param {boolean=} isLocalDevelopment - A flag to indicate if the adapter is being used in local development
|
|
34
|
-
* @param {number=} timeout - The timeout for the worker thread
|
|
35
|
-
* @param {number=} batchSize - Maximum number of extracted items in a batch
|
|
36
|
-
* @param {Record<EventType, string>=} workerPathOverrides - A map of event types to custom worker paths to override default worker paths
|
|
37
|
-
*/
|
|
38
16
|
export interface WorkerAdapterOptions {
|
|
39
17
|
isLocalDevelopment?: boolean;
|
|
18
|
+
/** Worker thread timeout. */
|
|
40
19
|
timeout?: number;
|
|
20
|
+
/** Maximum number of extracted items in a batch. */
|
|
41
21
|
batchSize?: number;
|
|
42
22
|
workerPathOverrides?: WorkerPathOverrides;
|
|
43
23
|
skipConfirmation?: boolean;
|
|
44
24
|
}
|
|
45
|
-
/**
|
|
46
|
-
* SpawnInterface is an interface for Spawn class.
|
|
47
|
-
* @interface SpawnInterface
|
|
48
|
-
* @constructor
|
|
49
|
-
* @param {AirSyncEvent} event - The event object received from the platform
|
|
50
|
-
* @param {Worker} worker - The worker thread
|
|
51
|
-
*/
|
|
52
25
|
export interface SpawnInterface {
|
|
53
26
|
event: AirSyncEvent;
|
|
54
27
|
worker: Worker;
|
|
@@ -56,108 +29,85 @@ export interface SpawnInterface {
|
|
|
56
29
|
resolve: (value: void | PromiseLike<void>) => void;
|
|
57
30
|
originalConsole?: Console;
|
|
58
31
|
}
|
|
59
|
-
/**
|
|
60
|
-
* SpawnFactoryInterface is an interface for Spawn class factory.
|
|
61
|
-
* Spawn class is responsible for spawning a new worker thread and managing the lifecycle of the worker.
|
|
62
|
-
* The class provides utilities to emit control events to the platform and exit the worker gracefully.
|
|
63
|
-
* In case of lambda timeout, the class emits a lambda timeout event to the platform.
|
|
64
|
-
* @interface SpawnFactoryInterface
|
|
65
|
-
* @constructor
|
|
66
|
-
* @param {AirSyncEvent} event - The event object received from the platform
|
|
67
|
-
* @param {object=} initialState - The initial state of the adapter
|
|
68
|
-
* @param {string} initialDomainMapping - The initial domain mapping
|
|
69
|
-
* @param {WorkerAdapterOptions} options - The options to create a new instance of Spawn class
|
|
70
|
-
* @param {string=} baseWorkerPath - The base path for the worker files, usually `__dirname`
|
|
71
|
-
*/
|
|
72
32
|
export interface SpawnFactoryInterface<ConnectorState> {
|
|
73
33
|
event: AirSyncEvent;
|
|
74
34
|
initialState: ConnectorState;
|
|
75
35
|
options?: WorkerAdapterOptions;
|
|
76
36
|
initialDomainMapping?: InitialDomainMapping;
|
|
37
|
+
/** Base path for the worker files, usually `__dirname`. */
|
|
77
38
|
baseWorkerPath?: string;
|
|
78
39
|
}
|
|
79
40
|
/**
|
|
80
|
-
*
|
|
81
|
-
*
|
|
82
|
-
*
|
|
83
|
-
*
|
|
84
|
-
|
|
85
|
-
|
|
41
|
+
* Returned by a worker's `task`/`onTimeout` callback; the SDK (never the connector)
|
|
42
|
+
* maps it to the phase-appropriate platform event and emits it exactly once.
|
|
43
|
+
* One invocation = one worker = one emitted event; continuation happens in a
|
|
44
|
+
* fresh platform-driven invocation.
|
|
45
|
+
*
|
|
46
|
+
* Status -> emitted event: 'success' -> *_DONE; 'error' -> *_ERROR;
|
|
47
|
+
* 'progress' -> *_PROGRESS and 'delay' -> *_DELAYED in resumable phases
|
|
48
|
+
* (data/attachment extraction and loading), but *_ERROR in non-resumable
|
|
49
|
+
* phases (external sync units, metadata), where they are illegal.
|
|
50
|
+
*/
|
|
51
|
+
export type TaskResult = {
|
|
52
|
+
status: 'success';
|
|
53
|
+
} | {
|
|
54
|
+
status: 'progress';
|
|
55
|
+
} | {
|
|
56
|
+
status: 'delay';
|
|
57
|
+
delaySeconds: number;
|
|
58
|
+
} | {
|
|
59
|
+
status: 'error';
|
|
60
|
+
error: ErrorRecord;
|
|
61
|
+
};
|
|
62
|
+
export type TaskStatus = TaskResult['status'];
|
|
63
|
+
/** Parameter shape passed to a worker's task and onTimeout callbacks. */
|
|
64
|
+
export interface TaskAdapterInterface<Adapter> {
|
|
86
65
|
adapter: Adapter;
|
|
87
66
|
}
|
|
88
67
|
/**
|
|
89
|
-
*
|
|
90
|
-
*
|
|
91
|
-
* @constructor
|
|
92
|
-
* @param {function} task - The task to be executed, returns exit code
|
|
93
|
-
* @param {function} onTimeout - The task to be executed on timeout, returns exit code
|
|
68
|
+
* If `onTimeout` is omitted, the SDK emits a phase-appropriate default on
|
|
69
|
+
* timeout: progress for resumable phases, error for ESU/metadata.
|
|
94
70
|
*/
|
|
95
|
-
export interface ProcessTaskInterface<Adapter
|
|
96
|
-
task: (params: TaskAdapterInterface<Adapter>) => Promise<
|
|
97
|
-
onTimeout
|
|
71
|
+
export interface ProcessTaskInterface<Adapter> {
|
|
72
|
+
task: (params: TaskAdapterInterface<Adapter>) => Promise<TaskResult>;
|
|
73
|
+
onTimeout?: (params: TaskAdapterInterface<Adapter>) => Promise<TaskResult>;
|
|
98
74
|
}
|
|
99
|
-
/**
|
|
100
|
-
* WorkerEvent represents the standard worker events.
|
|
101
|
-
*/
|
|
102
75
|
export declare enum WorkerEvent {
|
|
103
76
|
WorkerMessage = "message",
|
|
104
77
|
WorkerOnline = "online",
|
|
105
78
|
WorkerError = "error",
|
|
106
79
|
WorkerExit = "exit"
|
|
107
80
|
}
|
|
108
|
-
/**
|
|
109
|
-
* WorkerMessageSubject represents the handled worker message subjects.
|
|
110
|
-
*/
|
|
111
81
|
export declare enum WorkerMessageSubject {
|
|
112
82
|
WorkerMessageEmitted = "emit",
|
|
113
83
|
WorkerMessageExit = "exit",
|
|
114
84
|
WorkerMessageLog = "log",
|
|
115
85
|
WorkerMessageFailed = "failed"
|
|
116
86
|
}
|
|
117
|
-
/**
|
|
118
|
-
* WorkerMessageEmitted interface represents the structure of the emitted worker message.
|
|
119
|
-
*/
|
|
120
87
|
export interface WorkerMessageEmitted {
|
|
121
88
|
subject: WorkerMessageSubject.WorkerMessageEmitted;
|
|
122
89
|
payload: {
|
|
123
90
|
eventType: ExtractorEventType | LoaderEventType;
|
|
124
91
|
};
|
|
125
92
|
}
|
|
126
|
-
/**
|
|
127
|
-
* WorkerMessageExit interface represents the structure of the exit worker message.
|
|
128
|
-
*/
|
|
129
93
|
export interface WorkerMessageExit {
|
|
130
94
|
subject: WorkerMessageSubject.WorkerMessageExit;
|
|
131
95
|
}
|
|
132
|
-
/**
|
|
133
|
-
* WorkerMessageLog interface represents the structure of the worker log message.
|
|
134
|
-
*/
|
|
135
96
|
export interface WorkerMessageLog {
|
|
136
97
|
subject: WorkerMessageSubject.WorkerMessageLog;
|
|
137
98
|
payload: {
|
|
138
99
|
stringifiedArgs: string;
|
|
139
100
|
level: LogLevel;
|
|
140
|
-
isSdkLog?: boolean;
|
|
141
101
|
};
|
|
142
102
|
}
|
|
143
|
-
/**
|
|
144
|
-
* WorkerMessageFailed interface represents the structure of the worker failed message.
|
|
145
|
-
* Sent from the worker thread before calling process.exit(1) to convey the specific
|
|
146
|
-
* error reason to the main thread.
|
|
147
|
-
*/
|
|
103
|
+
/** Sent from the worker thread before process.exit(1) to convey the error reason to the main thread. */
|
|
148
104
|
export interface WorkerMessageFailed {
|
|
149
105
|
subject: WorkerMessageSubject.WorkerMessageFailed;
|
|
150
106
|
payload: {
|
|
151
107
|
message: string;
|
|
152
108
|
};
|
|
153
109
|
}
|
|
154
|
-
/**
|
|
155
|
-
* WorkerMessage represents the structure of the worker message.
|
|
156
|
-
*/
|
|
157
110
|
export type WorkerMessage = WorkerMessageEmitted | WorkerMessageExit | WorkerMessageLog | WorkerMessageFailed;
|
|
158
|
-
/**
|
|
159
|
-
* WorkerData represents the structure of the worker data object.
|
|
160
|
-
*/
|
|
161
111
|
export interface WorkerData<ConnectorState> {
|
|
162
112
|
event: AirSyncEvent;
|
|
163
113
|
initialState: ConnectorState;
|
|
@@ -165,15 +115,10 @@ export interface WorkerData<ConnectorState> {
|
|
|
165
115
|
initialDomainMapping?: InitialDomainMapping;
|
|
166
116
|
options?: WorkerAdapterOptions;
|
|
167
117
|
}
|
|
168
|
-
/**
|
|
169
|
-
* GetWorkerPathInterface is an interface for getting the worker path.
|
|
170
|
-
*/
|
|
171
118
|
export interface GetWorkerPathInterface {
|
|
172
119
|
event: AirSyncEvent;
|
|
173
120
|
workerBasePath?: string | null;
|
|
174
121
|
}
|
|
175
|
-
/**
|
|
176
|
-
* WorkerPathOverrides represents a mapping of event types to custom worker paths.
|
|
177
|
-
*/
|
|
122
|
+
/** Maps event types to custom worker paths, overriding the defaults. */
|
|
178
123
|
export type WorkerPathOverrides = Partial<Record<EventType, string>>;
|
|
179
124
|
//# sourceMappingURL=workers.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"workers.d.ts","sourceRoot":"","sources":["../../src/types/workers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AAExC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,6BAA6B,CAAC;AAC5D,OAAO,EAAE,SAAS,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"workers.d.ts","sourceRoot":"","sources":["../../src/types/workers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AAExC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,6BAA6B,CAAC;AAC5D,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAE3C,OAAO,EAAE,WAAW,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AAC7D,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAC3E,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAE5C,MAAM,WAAW,sBAAsB,CAAC,cAAc;IACpD,KAAK,EAAE,YAAY,CAAC;IACpB,YAAY,EAAE,SAAS,CAAC,cAAc,CAAC,CAAC;IACxC,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED,0EAA0E;AAC1E,MAAM,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,EAAE;IAAE,OAAO,EAAE,OAAO,CAAA;CAAE,CAAC,CAAC;AAEnE,MAAM,WAAW,oBAAoB;IACnC,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,6BAA6B;IAC7B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,oDAAoD;IACpD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;IAC1C,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,YAAY,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,OAAO,EAAE,CAAC,KAAK,EAAE,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC;IACnD,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B;AAED,MAAM,WAAW,qBAAqB,CAAC,cAAc;IACnD,KAAK,EAAE,YAAY,CAAC;IACpB,YAAY,EAAE,cAAc,CAAC;IAC7B,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,2DAA2D;IAC3D,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;;;;;;;;;GAUG;AACH,MAAM,MAAM,UAAU,GAClB;IAAE,MAAM,EAAE,SAAS,CAAA;CAAE,GACrB;IAAE,MAAM,EAAE,UAAU,CAAA;CAAE,GACtB;IAAE,MAAM,EAAE,OAAO,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,GACzC;IAAE,MAAM,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,WAAW,CAAA;CAAE,CAAC;AAE5C,MAAM,MAAM,UAAU,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;AAE9C,yEAAyE;AACzE,MAAM,WAAW,oBAAoB,CAAC,OAAO;IAC3C,OAAO,EAAE,OAAO,CAAC;CAClB;AAED;;;GAGG;AACH,MAAM,WAAW,oBAAoB,CAAC,OAAO;IAC3C,IAAI,EAAE,CAAC,MAAM,EAAE,oBAAoB,CAAC,OAAO,CAAC,KAAK,OAAO,CAAC,UAAU,CAAC,CAAC;IACrE,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,oBAAoB,CAAC,OAAO,CAAC,KAAK,OAAO,CAAC,UAAU,CAAC,CAAC;CAC5E;AAED,oBAAY,WAAW;IACrB,aAAa,YAAY;IACzB,YAAY,WAAW;IACvB,WAAW,UAAU;IACrB,UAAU,SAAS;CACpB;AAED,oBAAY,oBAAoB;IAC9B,oBAAoB,SAAS;IAC7B,iBAAiB,SAAS;IAC1B,gBAAgB,QAAQ;IACxB,mBAAmB,WAAW;CAC/B;AAED,MAAM,WAAW,oBAAoB;IACnC,OAAO,EAAE,oBAAoB,CAAC,oBAAoB,CAAC;IACnD,OAAO,EAAE;QACP,SAAS,EAAE,kBAAkB,GAAG,eAAe,CAAC;KACjD,CAAC;CACH;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,oBAAoB,CAAC,iBAAiB,CAAC;CACjD;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,oBAAoB,CAAC,gBAAgB,CAAC;IAC/C,OAAO,EAAE;QACP,eAAe,EAAE,MAAM,CAAC;QACxB,KAAK,EAAE,QAAQ,CAAC;KACjB,CAAC;CACH;AAED,wGAAwG;AACxG,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,oBAAoB,CAAC,mBAAmB,CAAC;IAClD,OAAO,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;CAC9B;AAED,MAAM,MAAM,aAAa,GACrB,oBAAoB,GACpB,iBAAiB,GACjB,gBAAgB,GAChB,mBAAmB,CAAC;AAExB,MAAM,WAAW,UAAU,CAAC,cAAc;IACxC,KAAK,EAAE,YAAY,CAAC;IACpB,YAAY,EAAE,cAAc,CAAC;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED,MAAM,WAAW,sBAAsB;IACrC,KAAK,EAAE,YAAY,CAAC;IACpB,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAChC;AAED,wEAAwE;AACxE,MAAM,MAAM,mBAAmB,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC"}
|
package/dist/types/workers.js
CHANGED
|
@@ -1,9 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.WorkerMessageSubject = exports.WorkerEvent = void 0;
|
|
4
|
-
/**
|
|
5
|
-
* WorkerEvent represents the standard worker events.
|
|
6
|
-
*/
|
|
7
4
|
var WorkerEvent;
|
|
8
5
|
(function (WorkerEvent) {
|
|
9
6
|
WorkerEvent["WorkerMessage"] = "message";
|
|
@@ -11,9 +8,6 @@ var WorkerEvent;
|
|
|
11
8
|
WorkerEvent["WorkerError"] = "error";
|
|
12
9
|
WorkerEvent["WorkerExit"] = "exit";
|
|
13
10
|
})(WorkerEvent || (exports.WorkerEvent = WorkerEvent = {}));
|
|
14
|
-
/**
|
|
15
|
-
* WorkerMessageSubject represents the handled worker message subjects.
|
|
16
|
-
*/
|
|
17
11
|
var WorkerMessageSubject;
|
|
18
12
|
(function (WorkerMessageSubject) {
|
|
19
13
|
WorkerMessageSubject["WorkerMessageEmitted"] = "emit";
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { AxiosResponse } from 'axios';
|
|
2
2
|
import { NormalizedAttachment } from '../repo/repo.interfaces';
|
|
3
|
-
import {
|
|
3
|
+
import { HttpStreamResponse } from '../types/extraction';
|
|
4
|
+
import { ArtifactToUpload, UploaderFactoryInterface, UploaderResult, UploadResponse } from './uploader.interfaces';
|
|
4
5
|
export declare class Uploader {
|
|
5
6
|
private isLocalDevelopment?;
|
|
6
7
|
private devrevApiEndpoint;
|
|
@@ -9,55 +10,19 @@ export declare class Uploader {
|
|
|
9
10
|
private defaultHeaders;
|
|
10
11
|
private skipConfirmation;
|
|
11
12
|
constructor({ event, options }: UploaderFactoryInterface);
|
|
12
|
-
/**
|
|
13
|
-
* Uploads the fetched objects to the DevRev platform. Fetched objects are compressed to a gzipped jsonl object and uploaded to the platform.
|
|
14
|
-
* @param {string} itemType - The type of the item to be uploaded
|
|
15
|
-
* @param {object[] | object} fetchedObjects - The objects to be uploaded
|
|
16
|
-
* @returns {Promise<UploadResponse>} - The response object containing the artifact information or error information if there was an error
|
|
17
|
-
*/
|
|
13
|
+
/** Compresses fetched objects to gzipped JSONL and uploads them as an artifact. */
|
|
18
14
|
upload(itemType: string, fetchedObjects: object[] | object): Promise<UploadResponse>;
|
|
19
|
-
/**
|
|
20
|
-
* Gets the upload URL for an artifact from the DevRev API.
|
|
21
|
-
* @param {string} filename - The name of the file to upload
|
|
22
|
-
* @param {string} fileType - The MIME type of the file
|
|
23
|
-
* @param {number} [fileSize] - Optional file size in bytes
|
|
24
|
-
* @returns {Promise<ArtifactToUpload | void>} The artifact upload information or undefined on error
|
|
25
|
-
*/
|
|
26
15
|
getArtifactUploadUrl(filename: string, fileType: string, fileSize?: number): Promise<UploaderResult<ArtifactToUpload>>;
|
|
27
|
-
/**
|
|
28
|
-
* Uploads an artifact file to the provided upload URL using multipart form data.
|
|
29
|
-
* @param {ArtifactToUpload} artifact - The artifact upload information containing upload URL and form data
|
|
30
|
-
* @param {Buffer} file - The file buffer to upload
|
|
31
|
-
* @returns {Promise<AxiosResponse | void>} The axios response or undefined on error
|
|
32
|
-
*/
|
|
16
|
+
/** Uploads the file buffer to the artifact's upload URL as multipart form data. */
|
|
33
17
|
uploadArtifact(artifact: ArtifactToUpload, file: Buffer): Promise<UploaderResult<AxiosResponse>>;
|
|
34
|
-
/**
|
|
35
|
-
|
|
36
|
-
* @param {ArtifactToUpload} artifact - The artifact upload information containing upload URL and form data
|
|
37
|
-
* @param {AxiosResponse} fileStream - The axios response stream containing the file data
|
|
38
|
-
* @returns {Promise<AxiosResponse | void>} The axios response or undefined on error
|
|
39
|
-
*/
|
|
40
|
-
streamArtifact(artifact: ArtifactToUpload, fileStream: AxiosResponse): Promise<UploaderResult<AxiosResponse>>;
|
|
41
|
-
/**
|
|
42
|
-
* Confirms that an artifact upload has been completed successfully.
|
|
43
|
-
* @param {string} artifactId - The ID of the artifact to confirm
|
|
44
|
-
* @returns {Promise<AxiosResponse | void>} The axios response or undefined on error
|
|
45
|
-
*/
|
|
18
|
+
/** Streams a file from an HTTP response directly to the artifact's upload URL. */
|
|
19
|
+
streamArtifact(artifact: ArtifactToUpload, fileStream: HttpStreamResponse): Promise<UploaderResult<AxiosResponse>>;
|
|
46
20
|
confirmArtifactUpload(artifactId: string): Promise<{
|
|
47
21
|
response?: AxiosResponse;
|
|
48
22
|
error?: unknown;
|
|
49
23
|
}>;
|
|
50
|
-
/**
|
|
51
|
-
* Destroys a stream to prevent resource leaks.
|
|
52
|
-
* @param {any} fileStream - The axios response stream to destroy
|
|
53
|
-
*/
|
|
54
24
|
private destroyStream;
|
|
55
|
-
/**
|
|
56
|
-
* Retrieves attachment metadata from an artifact by downloading and parsing it.
|
|
57
|
-
* @param {object} param0 - Configuration object
|
|
58
|
-
* @param {string} param0.artifact - The artifact ID to download attachments from
|
|
59
|
-
* @returns {Promise<{attachments?: NormalizedAttachment[], error?: {message: string}}>} The attachments array or error object
|
|
60
|
-
*/
|
|
25
|
+
/** Downloads an artifact and parses it into attachment metadata. */
|
|
61
26
|
getAttachmentsFromArtifactId({ artifact, }: {
|
|
62
27
|
artifact: string;
|
|
63
28
|
}): Promise<{
|
|
@@ -66,25 +31,9 @@ export declare class Uploader {
|
|
|
66
31
|
message: string;
|
|
67
32
|
};
|
|
68
33
|
}>;
|
|
69
|
-
/**
|
|
70
|
-
* Gets the download URL for an artifact from the DevRev API.
|
|
71
|
-
* @param {string} artifactId - The ID of the artifact to download
|
|
72
|
-
* @returns {Promise<string | void>} The download URL or undefined on error
|
|
73
|
-
*/
|
|
74
34
|
private getArtifactDownloadUrl;
|
|
75
|
-
/**
|
|
76
|
-
* Downloads an artifact file from the given URL.
|
|
77
|
-
* @param {string} artifactUrl - The URL to download the artifact from
|
|
78
|
-
* @returns {Promise<Buffer | void>} The artifact file buffer or undefined on error
|
|
79
|
-
*/
|
|
80
35
|
private downloadArtifact;
|
|
81
|
-
/**
|
|
82
|
-
* Retrieves and parses JSON objects from an artifact by artifact ID.
|
|
83
|
-
* @param {object} param0 - Configuration object
|
|
84
|
-
* @param {string} param0.artifactId - The artifact ID to download and parse
|
|
85
|
-
* @param {boolean} [param0.isGzipped=false] - Whether the artifact is gzipped
|
|
86
|
-
* @returns {Promise<object[] | object | void>} The parsed JSON objects or undefined on error
|
|
87
|
-
*/
|
|
36
|
+
/** Downloads an artifact (optionally gzipped) and parses it as JSONL. */
|
|
88
37
|
getJsonObjectByArtifactId({ artifactId, isGzipped, }: {
|
|
89
38
|
artifactId: string;
|
|
90
39
|
isGzipped?: boolean;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAOtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAUzD,OAAO,EAEL,gBAAgB,EAChB,wBAAwB,EACxB,cAAc,EACd,cAAc,EACf,MAAM,uBAAuB,CAAC;AAE/B,qBAAa,QAAQ;IACnB,OAAO,CAAC,kBAAkB,CAAC,CAAU;IACrC,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,cAAc,CAAyB;IAC/C,OAAO,CAAC,gBAAgB,CAAU;gBAEtB,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,wBAAwB;IAWxD,mFAAmF;IAC7E,MAAM,CACV,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,EAAE,GAAG,MAAM,GAChC,OAAO,CAAC,cAAc,CAAC;IA0EpB,oBAAoB,CACxB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;IA2B5C,mFAAmF;IAC7E,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAmBzC,kFAAkF;IAC5E,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,UAAU,EAAE,kBAAkB,GAC7B,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAqCnC,qBAAqB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC;QACvD,QAAQ,CAAC,EAAE,aAAa,CAAC;QACzB,KAAK,CAAC,EAAE,OAAO,CAAC;KACjB,CAAC;IAgCF,OAAO,CAAC,aAAa;IAerB,oEAAoE;IAC9D,4BAA4B,CAAC,EACjC,QAAQ,GACT,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;KAClB,GAAG,OAAO,CAAC;QACV,WAAW,CAAC,EAAE,oBAAoB,EAAE,CAAC;QACrC,KAAK,CAAC,EAAE;YAAE,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC;KAC7B,CAAC;YAuDY,sBAAsB;YAsBtB,gBAAgB;IAc9B,yEAAyE;IACnE,yBAAyB,CAAC,EAC9B,UAAU,EACV,SAAiB,GAClB,EAAE;QACD,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,CAAC,EAAE,OAAO,CAAC;KACrB,GAAG,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC;CA6B/C"}
|
|
@@ -1,33 +1,10 @@
|
|
|
1
|
-
import { UploaderResult } from './uploader.interfaces';
|
|
2
|
-
/**
|
|
3
|
-
|
|
4
|
-
* @param {string} jsonlObject - The JSONL string to compress
|
|
5
|
-
* @returns {Buffer | void} The compressed buffer or undefined on error
|
|
6
|
-
*/
|
|
1
|
+
import { ArtifactDateRanges, UploaderResult } from './uploader.interfaces';
|
|
2
|
+
/** Computes oldest/newest created and modified timestamps (RFC3339) across uploaded items. */
|
|
3
|
+
export declare function computeArtifactDateRanges(fetchedObjects: object[] | object): ArtifactDateRanges;
|
|
7
4
|
export declare function compressGzip(jsonlObject: string): UploaderResult<Buffer>;
|
|
8
|
-
/**
|
|
9
|
-
* Decompresses a gzipped buffer to a JSONL string.
|
|
10
|
-
* @param {Buffer} gzippedJsonlObject - The gzipped buffer to decompress
|
|
11
|
-
* @returns {string | void} The decompressed JSONL string or undefined on error
|
|
12
|
-
*/
|
|
13
5
|
export declare function decompressGzip(gzippedJsonlObject: Buffer): UploaderResult<string>;
|
|
14
|
-
/**
|
|
15
|
-
* Parses a JSONL string into an array of objects.
|
|
16
|
-
* @param {string} jsonlObject - The JSONL string to parse
|
|
17
|
-
* @returns {object[] | null} The parsed array of objects or null on error
|
|
18
|
-
*/
|
|
19
6
|
export declare function parseJsonl(jsonlObject: string): UploaderResult<object[]>;
|
|
20
|
-
/**
|
|
21
|
-
* Downloads fetched objects to the local file system (for local development).
|
|
22
|
-
* @param {string} itemType - The type of items being downloaded
|
|
23
|
-
* @param {object | object[]} fetchedObjects - The objects to write to file
|
|
24
|
-
* @returns {Promise<void>} Resolves when the file is written or rejects on error
|
|
25
|
-
*/
|
|
7
|
+
/** Writes fetched objects to the local file system (local development only). */
|
|
26
8
|
export declare function downloadToLocal(itemType: string, fetchedObjects: object | object[]): Promise<void>;
|
|
27
|
-
/**
|
|
28
|
-
* Truncates a filename if it exceeds the maximum allowed length.
|
|
29
|
-
* @param {string} filename - The filename to truncate
|
|
30
|
-
* @returns {string} The truncated filename
|
|
31
|
-
*/
|
|
32
9
|
export declare function truncateFilename(filename: string): string;
|
|
33
10
|
//# sourceMappingURL=uploader.helpers.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"uploader.helpers.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.helpers.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"uploader.helpers.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.helpers.ts"],"names":[],"mappings":"AAWA,OAAO,EAEL,kBAAkB,EAClB,cAAc,EACf,MAAM,uBAAuB,CAAC;AAE/B,8FAA8F;AAC9F,wBAAgB,yBAAyB,CACvC,cAAc,EAAE,MAAM,EAAE,GAAG,MAAM,GAChC,kBAAkB,CAwEpB;AAED,wBAAgB,YAAY,CAAC,WAAW,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,CAMxE;AAED,wBAAgB,cAAc,CAC5B,kBAAkB,EAAE,MAAM,GACzB,cAAc,CAAC,MAAM,CAAC,CAOxB;AAED,wBAAgB,UAAU,CAAC,WAAW,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,CAMxE;AAED,gFAAgF;AAChF,wBAAsB,eAAe,CACnC,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,GAAG,MAAM,EAAE,GAChC,OAAO,CAAC,IAAI,CAAC,CA4Bf;AAED,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAgBzD"}
|
|
@@ -36,20 +36,77 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
36
36
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
37
37
|
};
|
|
38
38
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
|
+
exports.computeArtifactDateRanges = computeArtifactDateRanges;
|
|
39
40
|
exports.compressGzip = compressGzip;
|
|
40
41
|
exports.decompressGzip = decompressGzip;
|
|
41
42
|
exports.parseJsonl = parseJsonl;
|
|
42
43
|
exports.downloadToLocal = downloadToLocal;
|
|
43
44
|
exports.truncateFilename = truncateFilename;
|
|
44
45
|
const fs_1 = __importStar(require("fs"));
|
|
45
|
-
const js_jsonl_1 = require("js-jsonl");
|
|
46
46
|
const zlib_1 = __importDefault(require("zlib"));
|
|
47
|
+
const js_jsonl_1 = require("js-jsonl");
|
|
47
48
|
const constants_1 = require("../common/constants");
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
49
|
+
const uploader_interfaces_1 = require("./uploader.interfaces");
|
|
50
|
+
/** Computes oldest/newest created and modified timestamps (RFC3339) across uploaded items. */
|
|
51
|
+
function computeArtifactDateRanges(fetchedObjects) {
|
|
52
|
+
const items = Array.isArray(fetchedObjects)
|
|
53
|
+
? fetchedObjects
|
|
54
|
+
: [fetchedObjects];
|
|
55
|
+
const created = {
|
|
56
|
+
min: Number.POSITIVE_INFINITY,
|
|
57
|
+
max: Number.NEGATIVE_INFINITY,
|
|
58
|
+
};
|
|
59
|
+
const modified = {
|
|
60
|
+
min: Number.POSITIVE_INFINITY,
|
|
61
|
+
max: Number.NEGATIVE_INFINITY,
|
|
62
|
+
};
|
|
63
|
+
let hasCreated = false;
|
|
64
|
+
let hasModified = false;
|
|
65
|
+
const parseTimestamp = (value) => {
|
|
66
|
+
const ts = new Date(value).getTime();
|
|
67
|
+
return Number.isNaN(ts) ? undefined : ts;
|
|
68
|
+
};
|
|
69
|
+
for (const obj of items) {
|
|
70
|
+
if (!obj || typeof obj !== 'object') {
|
|
71
|
+
continue;
|
|
72
|
+
}
|
|
73
|
+
const item = obj;
|
|
74
|
+
if (item.created_date != undefined) {
|
|
75
|
+
const ts = parseTimestamp(item.created_date);
|
|
76
|
+
if (ts != undefined) {
|
|
77
|
+
if (ts < created.min) {
|
|
78
|
+
created.min = ts;
|
|
79
|
+
}
|
|
80
|
+
if (ts > created.max) {
|
|
81
|
+
created.max = ts;
|
|
82
|
+
}
|
|
83
|
+
hasCreated = true;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
if (item.modified_date != undefined) {
|
|
87
|
+
const ts = parseTimestamp(item.modified_date);
|
|
88
|
+
if (ts != undefined) {
|
|
89
|
+
if (ts < modified.min) {
|
|
90
|
+
modified.min = ts;
|
|
91
|
+
}
|
|
92
|
+
if (ts > modified.max) {
|
|
93
|
+
modified.max = ts;
|
|
94
|
+
}
|
|
95
|
+
hasModified = true;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
const result = {};
|
|
100
|
+
if (hasCreated) {
|
|
101
|
+
result[uploader_interfaces_1.ArtifactDateField.OldestCreatedDate] = new Date(created.min).toISOString();
|
|
102
|
+
result[uploader_interfaces_1.ArtifactDateField.NewestCreatedDate] = new Date(created.max).toISOString();
|
|
103
|
+
}
|
|
104
|
+
if (hasModified) {
|
|
105
|
+
result[uploader_interfaces_1.ArtifactDateField.OldestModifiedDate] = new Date(modified.min).toISOString();
|
|
106
|
+
result[uploader_interfaces_1.ArtifactDateField.NewestModifiedDate] = new Date(modified.max).toISOString();
|
|
107
|
+
}
|
|
108
|
+
return result;
|
|
109
|
+
}
|
|
53
110
|
function compressGzip(jsonlObject) {
|
|
54
111
|
try {
|
|
55
112
|
return { response: zlib_1.default.gzipSync(jsonlObject) };
|
|
@@ -58,11 +115,6 @@ function compressGzip(jsonlObject) {
|
|
|
58
115
|
return { error };
|
|
59
116
|
}
|
|
60
117
|
}
|
|
61
|
-
/**
|
|
62
|
-
* Decompresses a gzipped buffer to a JSONL string.
|
|
63
|
-
* @param {Buffer} gzippedJsonlObject - The gzipped buffer to decompress
|
|
64
|
-
* @returns {string | void} The decompressed JSONL string or undefined on error
|
|
65
|
-
*/
|
|
66
118
|
function decompressGzip(gzippedJsonlObject) {
|
|
67
119
|
try {
|
|
68
120
|
const jsonlObject = zlib_1.default.gunzipSync(gzippedJsonlObject);
|
|
@@ -72,11 +124,6 @@ function decompressGzip(gzippedJsonlObject) {
|
|
|
72
124
|
return { error };
|
|
73
125
|
}
|
|
74
126
|
}
|
|
75
|
-
/**
|
|
76
|
-
* Parses a JSONL string into an array of objects.
|
|
77
|
-
* @param {string} jsonlObject - The JSONL string to parse
|
|
78
|
-
* @returns {object[] | null} The parsed array of objects or null on error
|
|
79
|
-
*/
|
|
80
127
|
function parseJsonl(jsonlObject) {
|
|
81
128
|
try {
|
|
82
129
|
return { response: js_jsonl_1.jsonl.parse(jsonlObject) };
|
|
@@ -85,12 +132,7 @@ function parseJsonl(jsonlObject) {
|
|
|
85
132
|
return { error };
|
|
86
133
|
}
|
|
87
134
|
}
|
|
88
|
-
/**
|
|
89
|
-
* Downloads fetched objects to the local file system (for local development).
|
|
90
|
-
* @param {string} itemType - The type of items being downloaded
|
|
91
|
-
* @param {object | object[]} fetchedObjects - The objects to write to file
|
|
92
|
-
* @returns {Promise<void>} Resolves when the file is written or rejects on error
|
|
93
|
-
*/
|
|
135
|
+
/** Writes fetched objects to the local file system (local development only). */
|
|
94
136
|
async function downloadToLocal(itemType, fetchedObjects) {
|
|
95
137
|
console.log(`Downloading ${itemType} to local file system.`);
|
|
96
138
|
try {
|
|
@@ -119,21 +161,13 @@ async function downloadToLocal(itemType, fetchedObjects) {
|
|
|
119
161
|
return Promise.reject(error);
|
|
120
162
|
}
|
|
121
163
|
}
|
|
122
|
-
/**
|
|
123
|
-
* Truncates a filename if it exceeds the maximum allowed length.
|
|
124
|
-
* @param {string} filename - The filename to truncate
|
|
125
|
-
* @returns {string} The truncated filename
|
|
126
|
-
*/
|
|
127
164
|
function truncateFilename(filename) {
|
|
128
|
-
// If the filename is already within the limit, return it as is.
|
|
129
165
|
if (filename.length <= constants_1.MAX_DEVREV_FILENAME_LENGTH) {
|
|
130
166
|
return filename;
|
|
131
167
|
}
|
|
132
168
|
console.warn(`Filename length exceeds the maximum limit of ${constants_1.MAX_DEVREV_FILENAME_LENGTH} characters. Truncating filename.`);
|
|
133
169
|
const extension = filename.slice(-constants_1.MAX_DEVREV_FILENAME_EXTENSION_LENGTH);
|
|
134
|
-
// Calculate how many characters are available for the name part after accounting for the extension and "..."
|
|
135
170
|
const availableNameLength = constants_1.MAX_DEVREV_FILENAME_LENGTH - constants_1.MAX_DEVREV_FILENAME_EXTENSION_LENGTH - 3; // -3 for "..."
|
|
136
|
-
// Truncate the name part and add an ellipsis
|
|
137
171
|
const truncatedFilename = filename.slice(0, availableNameLength);
|
|
138
172
|
return `${truncatedFilename}...${extension}`;
|
|
139
173
|
}
|
|
@@ -6,10 +6,6 @@ export interface UploaderFactoryInterface {
|
|
|
6
6
|
event: AirSyncEvent;
|
|
7
7
|
options?: WorkerAdapterOptions;
|
|
8
8
|
}
|
|
9
|
-
/**
|
|
10
|
-
* Generic result type for uploader operations that can either succeed with a response or fail with an error.
|
|
11
|
-
* @template T The type of the successful response data
|
|
12
|
-
*/
|
|
13
9
|
export type UploaderResult<T> = {
|
|
14
10
|
response: T;
|
|
15
11
|
error?: never;
|
|
@@ -17,17 +13,24 @@ export type UploaderResult<T> = {
|
|
|
17
13
|
response?: never;
|
|
18
14
|
error: unknown;
|
|
19
15
|
};
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
16
|
+
export declare enum ArtifactDateField {
|
|
17
|
+
OldestCreatedDate = "oldest_created_date",
|
|
18
|
+
NewestCreatedDate = "newest_created_date",
|
|
19
|
+
OldestModifiedDate = "oldest_modified_date",
|
|
20
|
+
NewestModifiedDate = "newest_modified_date"
|
|
21
|
+
}
|
|
22
|
+
/** A file generated by the extractor and uploaded to AirSync. */
|
|
23
23
|
export interface Artifact {
|
|
24
24
|
id: string;
|
|
25
25
|
item_type: string;
|
|
26
26
|
item_count: number;
|
|
27
|
+
[ArtifactDateField.OldestCreatedDate]?: string;
|
|
28
|
+
[ArtifactDateField.NewestCreatedDate]?: string;
|
|
29
|
+
[ArtifactDateField.OldestModifiedDate]?: string;
|
|
30
|
+
[ArtifactDateField.NewestModifiedDate]?: string;
|
|
27
31
|
}
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
*/
|
|
32
|
+
export type ArtifactDateRanges = Pick<Artifact, ArtifactDateField.OldestCreatedDate | ArtifactDateField.NewestCreatedDate | ArtifactDateField.OldestModifiedDate | ArtifactDateField.NewestModifiedDate>;
|
|
33
|
+
/** Response from the prepare artifacts endpoint. */
|
|
31
34
|
export interface ArtifactsPrepareResponse {
|
|
32
35
|
url: string;
|
|
33
36
|
id: string;
|
|
@@ -36,9 +39,7 @@ export interface ArtifactsPrepareResponse {
|
|
|
36
39
|
value: string;
|
|
37
40
|
}[];
|
|
38
41
|
}
|
|
39
|
-
/**
|
|
40
|
-
* ArtifactToUpload is an interface that defines the structure of the response from the get upload url endpoint.
|
|
41
|
-
*/
|
|
42
|
+
/** Response from the get upload url endpoint. */
|
|
42
43
|
export interface ArtifactToUpload {
|
|
43
44
|
upload_url: string;
|
|
44
45
|
artifact_id: string;
|
|
@@ -47,30 +48,19 @@ export interface ArtifactToUpload {
|
|
|
47
48
|
value: string;
|
|
48
49
|
}[];
|
|
49
50
|
}
|
|
50
|
-
/**
|
|
51
|
-
* UploadResponse is an interface that defines the structure of the response from upload through Uploader.
|
|
52
|
-
*/
|
|
53
51
|
export interface UploadResponse {
|
|
54
52
|
artifact?: Artifact;
|
|
55
53
|
error?: ErrorRecord;
|
|
56
54
|
}
|
|
57
|
-
/**
|
|
58
|
-
* StreamAttachmentsResponse is an interface that defines the structure of the response from the stream attachments through Uploader.
|
|
59
|
-
*/
|
|
60
55
|
export interface StreamAttachmentsResponse {
|
|
61
56
|
ssorAttachments?: SsorAttachment[];
|
|
62
57
|
error?: ErrorRecord;
|
|
63
58
|
}
|
|
64
|
-
/**
|
|
65
|
-
* StreamResponse is an interface that defines the structure of the response from the stream of single attachment through Uploader.
|
|
66
|
-
*/
|
|
59
|
+
/** Result of streaming a single attachment through Uploader. */
|
|
67
60
|
export interface StreamResponse {
|
|
68
61
|
ssorAttachment?: SsorAttachment;
|
|
69
62
|
error?: ErrorRecord;
|
|
70
63
|
}
|
|
71
|
-
/**
|
|
72
|
-
* SsorAttachment is an interface that defines the structure of the SSOR attachment.
|
|
73
|
-
*/
|
|
74
64
|
export interface SsorAttachment {
|
|
75
65
|
id: {
|
|
76
66
|
devrev: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"uploader.interfaces.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACvE,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,YAAY,CAAC;IACpB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED
|
|
1
|
+
{"version":3,"file":"uploader.interfaces.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACvE,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,YAAY,CAAC;IACpB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED,MAAM,MAAM,cAAc,CAAC,CAAC,IACxB;IAAE,QAAQ,EAAE,CAAC,CAAC;IAAC,KAAK,CAAC,EAAE,KAAK,CAAA;CAAE,GAC9B;IAAE,QAAQ,CAAC,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE,CAAC;AAEzC,oBAAY,iBAAiB;IAC3B,iBAAiB,wBAAwB;IACzC,iBAAiB,wBAAwB;IACzC,kBAAkB,yBAAyB;IAC3C,kBAAkB,yBAAyB;CAC5C;AAED,iEAAiE;AACjE,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,CAAC,EAAE,MAAM,CAAC;IAC/C,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,CAAC,EAAE,MAAM,CAAC;IAC/C,CAAC,iBAAiB,CAAC,kBAAkB,CAAC,CAAC,EAAE,MAAM,CAAC;IAChD,CAAC,iBAAiB,CAAC,kBAAkB,CAAC,CAAC,EAAE,MAAM,CAAC;CACjD;AAED,MAAM,MAAM,kBAAkB,GAAG,IAAI,CACnC,QAAQ,EACN,iBAAiB,CAAC,iBAAiB,GACnC,iBAAiB,CAAC,iBAAiB,GACnC,iBAAiB,CAAC,kBAAkB,GACpC,iBAAiB,CAAC,kBAAkB,CACvC,CAAC;AAEF,oDAAoD;AACpD,MAAM,WAAW,wBAAwB;IACvC,GAAG,EAAE,MAAM,CAAC;IACZ,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE;QACT,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;KACf,EAAE,CAAC;CACL;AAED,iDAAiD;AACjD,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE;QACT,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;KACf,EAAE,CAAC;CACL;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED,MAAM,WAAW,yBAAyB;IACxC,eAAe,CAAC,EAAE,cAAc,EAAE,CAAC;IACnC,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED,gEAAgE;AAChE,MAAM,WAAW,cAAc;IAC7B,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE;QACF,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,SAAS,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,QAAQ,CAAC,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,SAAS,CAAC,EAAE,eAAe,EAAE,CAAC;CAC/B;AAED,MAAM,WAAW,uBAAuB;IACtC,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,eAAe,CAAC,EAAE,kBAAkB,EAAE,CAAC;CACxC"}
|