@devrev/airsync-sdk 2.0.0-beta.2 → 2.0.0-beta.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. package/dist/airsync-sdk.d.ts +414 -157
  2. package/dist/attachments-streaming/attachments-streaming-pool.d.ts +2 -1
  3. package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map +1 -1
  4. package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts +8 -2
  5. package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts.map +1 -1
  6. package/dist/attachments-streaming/attachments-streaming-pool.js +14 -14
  7. package/dist/attachments-streaming/attachments-streaming-pool.test.js +47 -23
  8. package/dist/common/time-value-resolver.d.ts +2 -2
  9. package/dist/common/time-value-resolver.d.ts.map +1 -1
  10. package/dist/index.d.ts +3 -2
  11. package/dist/index.d.ts.map +1 -1
  12. package/dist/index.js +7 -4
  13. package/dist/multithreading/adapters/base-adapter.d.ts +56 -0
  14. package/dist/multithreading/adapters/base-adapter.d.ts.map +1 -0
  15. package/dist/multithreading/adapters/base-adapter.emit.test.d.ts +2 -0
  16. package/dist/multithreading/adapters/base-adapter.emit.test.d.ts.map +1 -0
  17. package/dist/multithreading/{worker-adapter/worker-adapter.emit.test.js → adapters/base-adapter.emit.test.js} +66 -53
  18. package/dist/multithreading/adapters/base-adapter.js +105 -0
  19. package/dist/multithreading/adapters/extraction-adapter.artifacts.test.d.ts +2 -0
  20. package/dist/multithreading/adapters/extraction-adapter.artifacts.test.d.ts.map +1 -0
  21. package/dist/multithreading/{worker-adapter/worker-adapter.artifacts.test.js → adapters/extraction-adapter.artifacts.test.js} +6 -6
  22. package/dist/multithreading/adapters/extraction-adapter.d.ts +58 -0
  23. package/dist/multithreading/adapters/extraction-adapter.d.ts.map +1 -0
  24. package/dist/multithreading/adapters/extraction-adapter.js +325 -0
  25. package/dist/multithreading/adapters/extraction-adapter.test.d.ts +2 -0
  26. package/dist/multithreading/adapters/extraction-adapter.test.d.ts.map +1 -0
  27. package/dist/multithreading/{worker-adapter/worker-adapter.extraction.test.js → adapters/extraction-adapter.test.js} +41 -51
  28. package/dist/multithreading/adapters/loading-adapter.d.ts +44 -0
  29. package/dist/multithreading/adapters/loading-adapter.d.ts.map +1 -0
  30. package/dist/multithreading/{worker-adapter/worker-adapter.helpers.d.ts → adapters/loading-adapter.helpers.d.ts} +1 -1
  31. package/dist/multithreading/adapters/loading-adapter.helpers.d.ts.map +1 -0
  32. package/dist/multithreading/adapters/loading-adapter.helpers.test.d.ts +2 -0
  33. package/dist/multithreading/adapters/loading-adapter.helpers.test.d.ts.map +1 -0
  34. package/dist/multithreading/{worker-adapter/worker-adapter.helpers.test.js → adapters/loading-adapter.helpers.test.js} +26 -26
  35. package/dist/multithreading/adapters/loading-adapter.js +461 -0
  36. package/dist/multithreading/adapters/loading-adapter.test.d.ts +2 -0
  37. package/dist/multithreading/adapters/loading-adapter.test.d.ts.map +1 -0
  38. package/dist/multithreading/{worker-adapter/worker-adapter.loading.test.js → adapters/loading-adapter.test.js} +19 -26
  39. package/dist/multithreading/adapters/serialization.test.d.ts +2 -0
  40. package/dist/multithreading/adapters/serialization.test.d.ts.map +1 -0
  41. package/dist/multithreading/process-task.d.ts +16 -1
  42. package/dist/multithreading/process-task.d.ts.map +1 -1
  43. package/dist/multithreading/process-task.js +97 -49
  44. package/dist/multithreading/process-task.test.js +15 -15
  45. package/dist/state/base-state.d.ts +75 -0
  46. package/dist/state/base-state.d.ts.map +1 -0
  47. package/dist/state/base-state.js +261 -0
  48. package/dist/state/extraction-state.d.ts +29 -0
  49. package/dist/state/extraction-state.d.ts.map +1 -0
  50. package/dist/state/extraction-state.js +122 -0
  51. package/dist/state/loading-state.d.ts +19 -0
  52. package/dist/state/loading-state.d.ts.map +1 -0
  53. package/dist/state/loading-state.js +40 -0
  54. package/dist/state/state.extract-window.test.js +7 -7
  55. package/dist/state/state.interfaces.d.ts +32 -25
  56. package/dist/state/state.interfaces.d.ts.map +1 -1
  57. package/dist/state/state.interfaces.js +15 -4
  58. package/dist/state/state.pending-boundaries.test.js +15 -15
  59. package/dist/state/state.post-state.test.js +9 -8
  60. package/dist/state/state.test.js +22 -19
  61. package/dist/state/state.time-value-resolution.test.js +8 -8
  62. package/dist/types/extraction.d.ts +3 -3
  63. package/dist/types/extraction.d.ts.map +1 -1
  64. package/dist/types/index.d.ts +0 -1
  65. package/dist/types/index.d.ts.map +1 -1
  66. package/dist/types/workers.d.ts +16 -15
  67. package/dist/types/workers.d.ts.map +1 -1
  68. package/package.json +1 -1
  69. package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.d.ts +0 -2
  70. package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.d.ts.map +0 -1
  71. package/dist/multithreading/worker-adapter/worker-adapter.d.ts +0 -97
  72. package/dist/multithreading/worker-adapter/worker-adapter.d.ts.map +0 -1
  73. package/dist/multithreading/worker-adapter/worker-adapter.emit.test.d.ts +0 -2
  74. package/dist/multithreading/worker-adapter/worker-adapter.emit.test.d.ts.map +0 -1
  75. package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.d.ts +0 -2
  76. package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.d.ts.map +0 -1
  77. package/dist/multithreading/worker-adapter/worker-adapter.helpers.d.ts.map +0 -1
  78. package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.d.ts +0 -2
  79. package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.d.ts.map +0 -1
  80. package/dist/multithreading/worker-adapter/worker-adapter.js +0 -846
  81. package/dist/multithreading/worker-adapter/worker-adapter.loading.test.d.ts +0 -2
  82. package/dist/multithreading/worker-adapter/worker-adapter.loading.test.d.ts.map +0 -1
  83. package/dist/multithreading/worker-adapter/worker-adapter.serialization.test.d.ts +0 -2
  84. package/dist/multithreading/worker-adapter/worker-adapter.serialization.test.d.ts.map +0 -1
  85. package/dist/state/state.d.ts +0 -36
  86. package/dist/state/state.d.ts.map +0 -1
  87. package/dist/state/state.js +0 -265
  88. /package/dist/multithreading/{worker-adapter/worker-adapter.helpers.js → adapters/loading-adapter.helpers.js} +0 -0
  89. /package/dist/multithreading/{worker-adapter/worker-adapter.serialization.test.js → adapters/serialization.test.js} +0 -0
@@ -14,12 +14,6 @@ declare enum ActionType {
14
14
  FAILED = "failed"
15
15
  }
16
16
 
17
- /**
18
- * AdapterState is an interface that defines the structure of the adapter state that is used by the external extractor.
19
- * It extends the connector state with additional fields including snapInVersionId and attachmentsMetadata.
20
- */
21
- export declare type AdapterState<ConnectorState> = ConnectorState & SdkState;
22
-
23
17
  export declare enum AirSyncDefaultItemTypes {
24
18
  EXTERNAL_DOMAIN_METADATA = "external_domain_metadata",
25
19
  ATTACHMENTS = "attachments",
@@ -76,6 +70,18 @@ export declare interface ArtifactsPrepareResponse {
76
70
  }[];
77
71
  }
78
72
 
73
+ /**
74
+ * ArtifactToUpload is an interface that defines the structure of the response from the get upload url endpoint.
75
+ */
76
+ declare interface ArtifactToUpload {
77
+ upload_url: string;
78
+ artifact_id: string;
79
+ form_data: {
80
+ key: string;
81
+ value: string;
82
+ }[];
83
+ }
84
+
79
85
  /**
80
86
  * Attachments configuration for a record type.
81
87
  */
@@ -108,6 +114,134 @@ declare interface AxiosErrorResponse {
108
114
  message?: string;
109
115
  }
110
116
 
117
+ /**
118
+ * BaseAdapter holds the state and behavior shared by both sync modes and owns
119
+ * the `emit` control-protocol flow as a template method. Mode-specific adapters
120
+ * (`ExtractionAdapter`, `LoadingAdapter`) implement the abstract hooks to inject
121
+ * their own pre-emit work and event payload shaping.
122
+ *
123
+ * @typeParam ConnectorState - the connector-owned state shape
124
+ * @typeParam StateClass - the mode's `BaseState` subclass
125
+ */
126
+ declare abstract class BaseAdapter<ConnectorState, StateClass extends BaseState<ConnectorState, BaseSdkState>> {
127
+ readonly event: AirSyncEvent;
128
+ readonly options?: WorkerAdapterOptions;
129
+ isTimeout: boolean;
130
+ hasWorkerEmitted: boolean;
131
+ protected adapterState: StateClass;
132
+ protected uploader: Uploader;
133
+ constructor({ event, adapterState, options, }: {
134
+ event: AirSyncEvent;
135
+ adapterState: StateClass;
136
+ options?: WorkerAdapterOptions;
137
+ });
138
+ get state(): ConnectorState;
139
+ set state(value: ConnectorState);
140
+ postState(): Promise<void>;
141
+ /**
142
+ * Pre-emit hook run before any state is persisted or the event is sent.
143
+ * Extraction uploads pending repos and updates extraction boundaries here;
144
+ * loading has nothing to do. Throwing aborts the emit (the caller signals
145
+ * worker exit).
146
+ */
147
+ protected abstract beforeEmit(newEventType: ExtractorEventType | LoaderEventType): Promise<void>;
148
+ /**
149
+ * Builds the mode-specific extras merged into the emitted event payload
150
+ * (extraction: artifacts; loading: reports + processed files).
151
+ */
152
+ protected abstract buildEmitPayload(newEventType: ExtractorEventType | LoaderEventType): EventData;
153
+ /**
154
+ * Post-emit hook run after the event has been sent successfully. Extraction
155
+ * clears its accumulated artifacts here; loading has nothing to do.
156
+ */
157
+ protected abstract afterEmit(newEventType: ExtractorEventType | LoaderEventType): void;
158
+ /**
159
+ * Emits an event to the platform.
160
+ *
161
+ * @param newEventType - The event type to be emitted
162
+ * @param data - The data to be sent with the event
163
+ */
164
+ emit(newEventType: ExtractorEventType | LoaderEventType, data?: EventData): Promise<void>;
165
+ }
166
+
167
+ /**
168
+ * BaseSdkState holds SDK bookkeeping common to every sync mode.
169
+ */
170
+ declare interface BaseSdkState {
171
+ snapInVersionId?: string;
172
+ }
173
+
174
+ /**
175
+ * BaseState owns the state lifecycle shared by every sync mode: connector vs.
176
+ * SDK state separation, fetch/init/post against the platform, the v1->v2
177
+ * migration shim, and the snap-in-version-gated initial domain mapping install.
178
+ *
179
+ * Mode-specific subclasses (`ExtractionState`, `LoadingState`) narrow `sdkState`
180
+ * to their own SDK state shape and add mode-specific setup in their factories.
181
+ *
182
+ * @typeParam ConnectorState - the connector-owned state shape
183
+ * @typeParam SdkState - the SDK bookkeeping shape for this mode
184
+ */
185
+ declare abstract class BaseState<ConnectorState, SdkState extends BaseSdkState> {
186
+ protected _connectorState: ConnectorState;
187
+ protected _sdkState: SdkState;
188
+ protected _extractionScope: ExtractionScope;
189
+ protected readonly initialSdkState: SdkState;
190
+ protected readonly event: AirSyncEvent;
191
+ private workerUrl;
192
+ private devrevToken;
193
+ private syncUnitId;
194
+ private requestId;
195
+ constructor({ event, initialState }: StateInterface<ConnectorState>, initialSdkState: SdkState);
196
+ /** Connector-owned state. This is what `adapter.state` exposes to snap-in code. */
197
+ get state(): ConnectorState;
198
+ set state(value: ConnectorState);
199
+ /** SDK-internal bookkeeping state. Never exposed to connector code. */
200
+ get sdkState(): SdkState;
201
+ set sdkState(value: SdkState);
202
+ get extractionScope(): ExtractionScope;
203
+ /**
204
+ * Installs the initial domain mapping when the snap-in version in state does
205
+ * not match the version in the event context. Shared by all modes so that a
206
+ * loading run still installs the mapping if extraction has not done so.
207
+ */
208
+ installInitialDomainMappingIfNeeded(initialDomainMapping?: InitialDomainMapping): Promise<void>;
209
+ /**
210
+ * Initializes the state for this adapter instance by fetching from API
211
+ * or creating an initial state if none exists (404).
212
+ *
213
+ * Reads both the v2 `{ connectorState, sdkState }` envelope and a legacy flat
214
+ * v1 blob (connector keys merged with SDK keys), migrating the latter on read.
215
+ * Always persists the v2 envelope going forward.
216
+ * @param initialState The initial connector state provided by the spawn function
217
+ */
218
+ init(initialState: ConnectorState): Promise<void>;
219
+ /**
220
+ * Normalizes a parsed on-disk state into the `{ connectorState, sdkState }`
221
+ * envelope, migrating a legacy flat v1 blob if needed.
222
+ *
223
+ * - v2 envelope (`{ connectorState, sdkState }`): used as-is.
224
+ * - v1 flat blob: SDK-owned keys (`V1_SDK_STATE_KEYS`) split into `sdkState`,
225
+ * everything else becomes connector state.
226
+ * - Malformed envelope (one side present, the other missing) fails loud.
227
+ */
228
+ private normalizeFetchedState;
229
+ /**
230
+ * Updates the state of the adapter by posting to API.
231
+ * Persists the v2 `{ connectorState, sdkState }` envelope.
232
+ * @param {object} state - The connector state to be updated
233
+ */
234
+ postState(state?: ConnectorState): Promise<void>;
235
+ /**
236
+ * Fetches the state of the adapter from API.
237
+ * @return The raw state data from API
238
+ */
239
+ fetchState(): Promise<{
240
+ state: string;
241
+ objects?: string;
242
+ }>;
243
+ }
244
+
111
245
  /**
112
246
  * Collection constraints for fields that are collections of values.
113
247
  */
@@ -361,7 +495,7 @@ export declare interface ExternalSystemAttachment {
361
495
 
362
496
  export declare type ExternalSystemAttachmentIteratorFunction<NewBatch, ConnectorState> = ({ reducedAttachments, adapter, stream, }: {
363
497
  reducedAttachments: NewBatch;
364
- adapter: WorkerAdapter<ConnectorState>;
498
+ adapter: ExtractionAdapter<ConnectorState>;
365
499
  stream: ExternalSystemAttachmentStreamingFunction;
366
500
  }) => Promise<ProcessAttachmentReturnType>;
367
501
 
@@ -372,7 +506,7 @@ declare interface ExternalSystemAttachmentProcessors<ConnectorState, Batch, NewB
372
506
 
373
507
  export declare type ExternalSystemAttachmentReducerFunction<Batch, NewBatch, ConnectorState> = ({ attachments, adapter, batchSize, }: {
374
508
  attachments: Batch;
375
- adapter: WorkerAdapter<ConnectorState>;
509
+ adapter: ExtractionAdapter<ConnectorState>;
376
510
  batchSize?: number;
377
511
  }) => NewBatch;
378
512
 
@@ -424,6 +558,56 @@ export declare interface ExternalSystemItemLoadingResponse {
424
558
 
425
559
  declare type ExternalSystemLoadingFunction<Item> = ({ item, mappers, event, }: ExternalSystemItemLoadingParams<Item>) => Promise<ExternalSystemItemLoadingResponse>;
426
560
 
561
+ /**
562
+ * ExtractionAdapter is the adapter passed to extraction tasks. It exposes the
563
+ * extraction surface (repos, artifacts, attachment streaming) and updates the
564
+ * extraction boundaries before emitting.
565
+ *
566
+ * @public
567
+ */
568
+ export declare class ExtractionAdapter<ConnectorState> extends BaseAdapter<ConnectorState, ExtractionState<ConnectorState>> {
569
+ private _artifacts;
570
+ private repos;
571
+ private currentEventDataLength;
572
+ constructor(params: {
573
+ event: AirSyncEvent;
574
+ adapterState: ExtractionState<ConnectorState>;
575
+ options?: WorkerAdapterOptions;
576
+ });
577
+ get extractionScope(): ExtractionScope;
578
+ /**
579
+ * Returns whether the given item type should be extracted.
580
+ * Defaults to true if the scope is empty or the item type is not listed.
581
+ */
582
+ shouldExtract(itemType: string): boolean;
583
+ initializeRepos(repos: RepoInterface[]): void;
584
+ getRepo(itemType: string): Repo | undefined;
585
+ get artifacts(): Artifact[];
586
+ set artifacts(artifacts: Artifact[]);
587
+ protected beforeEmit(newEventType: ExtractorEventType | LoaderEventType): Promise<void>;
588
+ protected buildEmitPayload(newEventType: ExtractorEventType | LoaderEventType): EventData;
589
+ protected afterEmit(): void;
590
+ uploadAllRepos(): Promise<void>;
591
+ processAttachment(attachment: NormalizedAttachment, stream: ExternalSystemAttachmentStreamingFunction): Promise<ProcessAttachmentReturnType>;
592
+ /**
593
+ * Destroys a stream to prevent memory leaks.
594
+ * @param httpStream - The axios response stream to destroy
595
+ */
596
+ private destroyHttpStream;
597
+ /**
598
+ * Streams the attachments to the DevRev platform.
599
+ * The attachments are streamed to the platform and the artifact information is returned.
600
+ * @param params - The parameters to stream the attachments
601
+ * @returns The response object containing the ssorAttachment artifact information
602
+ * or error information if there was an error
603
+ */
604
+ streamAttachments<NewBatch>({ stream, processors, batchSize, }: {
605
+ stream: ExternalSystemAttachmentStreamingFunction;
606
+ processors?: ExternalSystemAttachmentProcessors<ConnectorState, NormalizedAttachment[], NewBatch>;
607
+ batchSize?: number;
608
+ }): Promise<StreamAttachmentsReturnType>;
609
+ }
610
+
427
611
  export declare const enum ExtractionCommonError {
428
612
  EXTERNAL_SYNC_UNIT_DELETED = "ERROR_CODE=EXTERNAL_SYNC_UNIT_DELETED",
429
613
  EXTERNAL_SYNC_UNIT_DEACTIVATED = "ERROR_CODE=EXTERNAL_SYNC_UNIT_DEACTIVATED",
@@ -438,6 +622,41 @@ export declare type ExtractionScope = Record<string, {
438
622
  extract: boolean;
439
623
  }>;
440
624
 
625
+ /**
626
+ * ExtractionSdkState holds SDK bookkeeping owned by the extraction modes.
627
+ */
628
+ declare interface ExtractionSdkState extends BaseSdkState {
629
+ /** The pending (not yet committed) oldest extraction boundary (ISO 8601 timestamp).
630
+ * Set on StartExtractingMetadata, reused across subsequent phases, cleared on AttachmentExtractionDone. */
631
+ pendingWorkersOldest?: string;
632
+ /** The pending (not yet committed) newest extraction boundary (ISO 8601 timestamp).
633
+ * Set on StartExtractingMetadata, reused across subsequent phases, cleared on AttachmentExtractionDone. */
634
+ pendingWorkersNewest?: string;
635
+ /** The oldest point of extraction (ISO 8601 timestamp). */
636
+ workersOldest?: string;
637
+ /** The newest point of extraction (ISO 8601 timestamp). */
638
+ workersNewest?: string;
639
+ toDevRev?: ToDevRev;
640
+ }
641
+
642
+ /**
643
+ * ExtractionState is the per-mode state for extraction workers. It narrows
644
+ * `sdkState` to `ExtractionSdkState` (extraction boundaries + attachments
645
+ * bookkeeping) on top of the shared lifecycle provided by `BaseState`.
646
+ */
647
+ declare class ExtractionState<ConnectorState> extends BaseState<ConnectorState, ExtractionSdkState> {
648
+ constructor(params: StateInterface<ConnectorState>);
649
+ /**
650
+ * Resolves the extraction window onto the event context.
651
+ *
652
+ * On StartExtractingMetadata: resolve fresh from the TimeValue objects in the
653
+ * event context and cache them as pending boundaries (always overwrite).
654
+ * On all other events: reuse the pending boundaries cached during
655
+ * StartExtractingMetadata. Finally, validate that extract_from < extract_to.
656
+ */
657
+ resolveExtractionWindow(): void;
658
+ }
659
+
441
660
  /**
442
661
  * ExtractorEvent is an interface that defines the structure of the event that is sent from the external extractor to the AirSync platform.
443
662
  * It contains the event type, event context, extractor state, and event data.
@@ -659,6 +878,60 @@ declare interface LoaderReport {
659
878
  [ActionType.FAILED]?: number;
660
879
  }
661
880
 
881
+ /**
882
+ * LoadingAdapter is the adapter passed to loading tasks. It exposes the loading
883
+ * surface (item/attachment loading, mappers, loader reports).
884
+ *
885
+ * @public
886
+ */
887
+ export declare class LoadingAdapter<ConnectorState> extends BaseAdapter<ConnectorState, LoadingState<ConnectorState>> {
888
+ private loaderReports;
889
+ private _processedFiles;
890
+ private _mappers;
891
+ constructor(params: {
892
+ event: AirSyncEvent;
893
+ adapterState: LoadingState<ConnectorState>;
894
+ options?: WorkerAdapterOptions;
895
+ });
896
+ get reports(): LoaderReport[];
897
+ get processedFiles(): string[];
898
+ get mappers(): Mappers;
899
+ protected beforeEmit(): Promise<void>;
900
+ protected buildEmitPayload(newEventType: ExtractorEventType | LoaderEventType): EventData;
901
+ protected afterEmit(): void;
902
+ loadItemTypes({ itemTypesToLoad, }: ItemTypesToLoadParams): Promise<LoadItemTypesResponse>;
903
+ getLoaderBatches({ supportedItemTypes, }: {
904
+ supportedItemTypes: string[];
905
+ }): Promise<FileToLoad[]>;
906
+ loadAttachments({ create, }: {
907
+ create: ExternalSystemLoadingFunction<ExternalSystemAttachment>;
908
+ }): Promise<LoadItemTypesResponse>;
909
+ loadItem({ item, itemTypeToLoad, }: {
910
+ item: ExternalSystemItem;
911
+ itemTypeToLoad: ItemTypeToLoad;
912
+ }): Promise<LoadItemResponse>;
913
+ loadAttachment({ item, create, }: {
914
+ item: ExternalSystemAttachment;
915
+ create: ExternalSystemLoadingFunction<ExternalSystemAttachment>;
916
+ }): Promise<LoadItemResponse>;
917
+ }
918
+
919
+ /**
920
+ * LoadingSdkState holds SDK bookkeeping owned by the loading modes.
921
+ */
922
+ declare interface LoadingSdkState extends BaseSdkState {
923
+ fromDevRev?: FromDevRev;
924
+ }
925
+
926
+ /**
927
+ * LoadingState is the per-mode state for loading workers. It narrows `sdkState`
928
+ * to `LoadingSdkState` (files-to-load bookkeeping) on top of the shared
929
+ * lifecycle provided by `BaseState`.
930
+ */
931
+ declare class LoadingState<ConnectorState> extends BaseState<ConnectorState, LoadingSdkState> {
932
+ constructor(params: StateInterface<ConnectorState>);
933
+ }
934
+
662
935
  declare interface LoadItemResponse {
663
936
  error?: ErrorRecord;
664
937
  report?: LoaderReport;
@@ -984,18 +1257,32 @@ declare interface ProcessedAttachment {
984
1257
  parent_id: string;
985
1258
  }
986
1259
 
987
- export declare function processTask<ConnectorState>({ task, onTimeout, }: ProcessTaskInterface<ConnectorState>): void;
1260
+ /**
1261
+ * Entry point for an extraction worker. Builds an {@link ExtractionAdapter} and
1262
+ * runs the provided task against it.
1263
+ *
1264
+ * @public
1265
+ */
1266
+ export declare function processExtractionTask<ConnectorState>({ task, onTimeout, }: ProcessTaskInterface<ExtractionAdapter<ConnectorState>>): void;
988
1267
 
989
1268
  /**
990
- * ProcessTaskInterface is an interface for ProcessTask class.
1269
+ * Entry point for a loading worker. Builds a {@link LoadingAdapter} and runs the
1270
+ * provided task against it.
1271
+ *
1272
+ * @public
1273
+ */
1274
+ export declare function processLoadingTask<ConnectorState>({ task, onTimeout, }: ProcessTaskInterface<LoadingAdapter<ConnectorState>>): void;
1275
+
1276
+ /**
1277
+ * ProcessTaskInterface is the parameter shape for the process-task entry points.
991
1278
  * @interface ProcessTaskInterface
992
1279
  * @constructor
993
1280
  * @param {function} task - The task to be executed, returns exit code
994
1281
  * @param {function} onTimeout - The task to be executed on timeout, returns exit code
995
1282
  */
996
- export declare interface ProcessTaskInterface<ConnectorState> {
997
- task: (params: TaskAdapterInterface<ConnectorState>) => Promise<void>;
998
- onTimeout: (params: TaskAdapterInterface<ConnectorState>) => Promise<void>;
1283
+ export declare interface ProcessTaskInterface<Adapter extends BaseAdapter<unknown, BaseState<unknown, BaseSdkState>>> {
1284
+ task: (params: TaskAdapterInterface<Adapter>) => Promise<void>;
1285
+ onTimeout: (params: TaskAdapterInterface<Adapter>) => Promise<void>;
999
1286
  }
1000
1287
 
1001
1288
  declare interface RateLimited {
@@ -1170,22 +1457,6 @@ export declare interface RouteConfig {
1170
1457
  */
1171
1458
  export declare type SchemaVersion = 'v0.2.0';
1172
1459
 
1173
- declare interface SdkState {
1174
- /** The pending (not yet committed) oldest extraction boundary (ISO 8601 timestamp).
1175
- * Set on StartExtractingMetadata, reused across subsequent phases, cleared on AttachmentExtractionDone. */
1176
- pendingWorkersOldest?: string;
1177
- /** The pending (not yet committed) newest extraction boundary (ISO 8601 timestamp).
1178
- * Set on StartExtractingMetadata, reused across subsequent phases, cleared on AttachmentExtractionDone. */
1179
- pendingWorkersNewest?: string;
1180
- /** The oldest point of extraction (ISO 8601 timestamp). */
1181
- workersOldest?: string;
1182
- /** The newest point of extraction (ISO 8601 timestamp). */
1183
- workersNewest?: string;
1184
- toDevRev?: ToDevRev;
1185
- fromDevRev?: FromDevRev;
1186
- snapInVersionId?: string;
1187
- }
1188
-
1189
1460
  /**
1190
1461
  * Serializes an Axios error into a structured format with HTTP request/response details.
1191
1462
  * Extracts method, URL, parameters, status code, headers, and data.
@@ -1281,39 +1552,6 @@ export declare interface StageDiagram {
1281
1552
  /** Key identifying a stage in the stage diagram stages map. */
1282
1553
  export declare type StageKey = string;
1283
1554
 
1284
- declare class State<ConnectorState> {
1285
- private _state;
1286
- private _extractionScope;
1287
- private initialSdkState;
1288
- private workerUrl;
1289
- private devrevToken;
1290
- private syncUnitId;
1291
- private requestId;
1292
- constructor({ event, initialState }: StateInterface<ConnectorState>);
1293
- get state(): AdapterState<ConnectorState>;
1294
- set state(value: AdapterState<ConnectorState>);
1295
- get extractionScope(): ExtractionScope;
1296
- /**
1297
- * Initializes the state for this adapter instance by fetching from API
1298
- * or creating an initial state if none exists (404).
1299
- * @param initialState The initial connector state provided by the spawn function
1300
- */
1301
- init(initialState: ConnectorState): Promise<void>;
1302
- /**
1303
- * Updates the state of the adapter by posting to API.
1304
- * @param {object} state - The state to be updated
1305
- */
1306
- postState(state?: AdapterState<ConnectorState>): Promise<void>;
1307
- /**
1308
- * Fetches the state of the adapter from API.
1309
- * @return The raw state data from API
1310
- */
1311
- fetchState(): Promise<{
1312
- state: string;
1313
- objects?: string;
1314
- }>;
1315
- }
1316
-
1317
1555
  declare interface StateInterface<ConnectorState> {
1318
1556
  event: AirSyncEvent;
1319
1557
  initialState: ConnectorState;
@@ -1485,13 +1723,13 @@ export declare interface TargetTypeKeyData {
1485
1723
  }
1486
1724
 
1487
1725
  /**
1488
- * TaskAdapterInterface is an interface for TaskAdapter class.
1726
+ * TaskAdapterInterface is the parameter shape passed to a task callback.
1489
1727
  * @interface TaskAdapterInterface
1490
1728
  * @constructor
1491
- * @param {WorkerAdapter} adapter - The adapter object
1729
+ * @param {BaseAdapter} adapter - The adapter object
1492
1730
  */
1493
- export declare interface TaskAdapterInterface<ConnectorState> {
1494
- adapter: WorkerAdapter<ConnectorState>;
1731
+ export declare interface TaskAdapterInterface<Adapter extends BaseAdapter<unknown, BaseState<unknown, BaseSdkState>>> {
1732
+ adapter: Adapter;
1495
1733
  }
1496
1734
 
1497
1735
  /**
@@ -1583,113 +1821,132 @@ export declare interface TypedReferenceData {
1583
1821
  */
1584
1822
  export declare const UNBOUNDED_DATE_TIME_VALUE = "1970-01-01T00:00:00.000Z";
1585
1823
 
1586
- /**
1587
- * UploadResponse is an interface that defines the structure of the response from upload through Uploader.
1588
- */
1589
- export declare interface UploadResponse {
1590
- artifact?: Artifact;
1591
- error?: ErrorRecord;
1592
- }
1593
-
1594
- /**
1595
- * WorkerAdapter class is used to interact with AirSync platform. It is passed to the snap-in
1596
- * as parameter in processTask and onTimeout functions. The class provides
1597
- * utilities to emit control events to the platform, update the state of the connector,
1598
- * and upload artifacts to the platform.
1599
- * @class WorkerAdapter
1600
- * @constructor
1601
- * @param options - The options to create a new instance of WorkerAdapter class
1602
- * @param event - The event object received from the platform
1603
- * @param initialState - The initial state of the adapter
1604
- * @param isLocalDevelopment - A flag to indicate if the adapter is being used in local development
1605
- * @param workerPath - The path to the worker file
1606
- *
1607
- * @public
1608
- */
1609
- export declare class WorkerAdapter<ConnectorState> {
1610
- readonly event: AirSyncEvent;
1611
- readonly options?: WorkerAdapterOptions;
1612
- isTimeout: boolean;
1613
- hasWorkerEmitted: boolean;
1614
- private adapterState;
1615
- private _artifacts;
1616
- private repos;
1617
- private currentEventDataLength;
1618
- private loaderReports;
1619
- private _processedFiles;
1620
- private _mappers;
1621
- private uploader;
1622
- constructor({ event, adapterState, options, }: WorkerAdapterInterface<ConnectorState>);
1623
- get state(): AdapterState<ConnectorState>;
1624
- set state(value: AdapterState<ConnectorState>);
1625
- get reports(): LoaderReport[];
1626
- get processedFiles(): string[];
1627
- get mappers(): Mappers;
1628
- get extractionScope(): ExtractionScope;
1824
+ declare class Uploader {
1825
+ private isLocalDevelopment?;
1826
+ private devrevApiEndpoint;
1827
+ private devrevApiToken;
1828
+ private requestId;
1829
+ private defaultHeaders;
1830
+ private skipConfirmation;
1831
+ constructor({ event, options }: UploaderFactoryInterface);
1629
1832
  /**
1630
- * Returns whether the given item type should be extracted.
1631
- * Defaults to true if the scope is empty or the item type is not listed.
1833
+ * Uploads the fetched objects to the DevRev platform. Fetched objects are compressed to a gzipped jsonl object and uploaded to the platform.
1834
+ * @param {string} itemType - The type of the item to be uploaded
1835
+ * @param {object[] | object} fetchedObjects - The objects to be uploaded
1836
+ * @returns {Promise<UploadResponse>} - The response object containing the artifact information or error information if there was an error
1632
1837
  */
1633
- shouldExtract(itemType: string): boolean;
1634
- initializeRepos(repos: RepoInterface[]): void;
1635
- getRepo(itemType: string): Repo | undefined;
1636
- postState(): Promise<void>;
1637
- get artifacts(): Artifact[];
1638
- set artifacts(artifacts: Artifact[]);
1838
+ upload(itemType: string, fetchedObjects: object[] | object): Promise<UploadResponse>;
1639
1839
  /**
1640
- * Emits an event to the platform.
1641
- *
1642
- * @param newEventType - The event type to be emitted
1643
- * @param data - The data to be sent with the event
1840
+ * Gets the upload URL for an artifact from the DevRev API.
1841
+ * @param {string} filename - The name of the file to upload
1842
+ * @param {string} fileType - The MIME type of the file
1843
+ * @param {number} [fileSize] - Optional file size in bytes
1844
+ * @returns {Promise<ArtifactToUpload | void>} The artifact upload information or undefined on error
1644
1845
  */
1645
- emit(newEventType: ExtractorEventType | LoaderEventType, data?: EventData): Promise<void>;
1646
- uploadAllRepos(): Promise<void>;
1647
- loadItemTypes({ itemTypesToLoad, }: ItemTypesToLoadParams): Promise<LoadItemTypesResponse>;
1648
- getLoaderBatches({ supportedItemTypes, }: {
1649
- supportedItemTypes: string[];
1650
- }): Promise<FileToLoad[]>;
1651
- loadAttachments({ create, }: {
1652
- create: ExternalSystemLoadingFunction<ExternalSystemAttachment>;
1653
- }): Promise<LoadItemTypesResponse>;
1654
- loadItem({ item, itemTypeToLoad, }: {
1655
- item: ExternalSystemItem;
1656
- itemTypeToLoad: ItemTypeToLoad;
1657
- }): Promise<LoadItemResponse>;
1658
- processAttachment(attachment: NormalizedAttachment, stream: ExternalSystemAttachmentStreamingFunction): Promise<ProcessAttachmentReturnType>;
1846
+ getArtifactUploadUrl(filename: string, fileType: string, fileSize?: number): Promise<UploaderResult<ArtifactToUpload>>;
1659
1847
  /**
1660
- * Destroys a stream to prevent memory leaks.
1661
- * @param httpStream - The axios response stream to destroy
1848
+ * Uploads an artifact file to the provided upload URL using multipart form data.
1849
+ * @param {ArtifactToUpload} artifact - The artifact upload information containing upload URL and form data
1850
+ * @param {Buffer} file - The file buffer to upload
1851
+ * @returns {Promise<AxiosResponse | void>} The axios response or undefined on error
1662
1852
  */
1663
- private destroyHttpStream;
1664
- loadAttachment({ item, create, }: {
1665
- item: ExternalSystemAttachment;
1666
- create: ExternalSystemLoadingFunction<ExternalSystemAttachment>;
1667
- }): Promise<LoadItemResponse>;
1853
+ uploadArtifact(artifact: ArtifactToUpload, file: Buffer): Promise<UploaderResult<AxiosResponse>>;
1668
1854
  /**
1669
- * Streams the attachments to the DevRev platform.
1670
- * The attachments are streamed to the platform and the artifact information is returned.
1671
- * @param params - The parameters to stream the attachments
1672
- * @returns The response object containing the ssorAttachment artifact information
1673
- * or error information if there was an error
1855
+ * Streams an artifact file from an axios response to the upload URL.
1856
+ * @param {ArtifactToUpload} artifact - The artifact upload information containing upload URL and form data
1857
+ * @param {AxiosResponse} fileStream - The axios response stream containing the file data
1858
+ * @returns {Promise<AxiosResponse | void>} The axios response or undefined on error
1674
1859
  */
1675
- streamAttachments<NewBatch>({ stream, processors, batchSize, }: {
1676
- stream: ExternalSystemAttachmentStreamingFunction;
1677
- processors?: ExternalSystemAttachmentProcessors<ConnectorState, NormalizedAttachment[], NewBatch>;
1678
- batchSize?: number;
1679
- }): Promise<StreamAttachmentsReturnType>;
1860
+ streamArtifact(artifact: ArtifactToUpload, fileStream: AxiosResponse): Promise<UploaderResult<AxiosResponse>>;
1861
+ /**
1862
+ * Confirms that an artifact upload has been completed successfully.
1863
+ * @param {string} artifactId - The ID of the artifact to confirm
1864
+ * @returns {Promise<AxiosResponse | void>} The axios response or undefined on error
1865
+ */
1866
+ confirmArtifactUpload(artifactId: string): Promise<{
1867
+ response?: AxiosResponse;
1868
+ error?: unknown;
1869
+ }>;
1870
+ /**
1871
+ * Destroys a stream to prevent resource leaks.
1872
+ * @param {any} fileStream - The axios response stream to destroy
1873
+ */
1874
+ private destroyStream;
1875
+ /**
1876
+ * Retrieves attachment metadata from an artifact by downloading and parsing it.
1877
+ * @param {object} param0 - Configuration object
1878
+ * @param {string} param0.artifact - The artifact ID to download attachments from
1879
+ * @returns {Promise<{attachments?: NormalizedAttachment[], error?: {message: string}}>} The attachments array or error object
1880
+ */
1881
+ getAttachmentsFromArtifactId({ artifact, }: {
1882
+ artifact: string;
1883
+ }): Promise<{
1884
+ attachments?: NormalizedAttachment[];
1885
+ error?: {
1886
+ message: string;
1887
+ };
1888
+ }>;
1889
+ /**
1890
+ * Gets the download URL for an artifact from the DevRev API.
1891
+ * @param {string} artifactId - The ID of the artifact to download
1892
+ * @returns {Promise<string | void>} The download URL or undefined on error
1893
+ */
1894
+ private getArtifactDownloadUrl;
1895
+ /**
1896
+ * Downloads an artifact file from the given URL.
1897
+ * @param {string} artifactUrl - The URL to download the artifact from
1898
+ * @returns {Promise<Buffer | void>} The artifact file buffer or undefined on error
1899
+ */
1900
+ private downloadArtifact;
1901
+ /**
1902
+ * Retrieves and parses JSON objects from an artifact by artifact ID.
1903
+ * @param {object} param0 - Configuration object
1904
+ * @param {string} param0.artifactId - The artifact ID to download and parse
1905
+ * @param {boolean} [param0.isGzipped=false] - Whether the artifact is gzipped
1906
+ * @returns {Promise<object[] | object | void>} The parsed JSON objects or undefined on error
1907
+ */
1908
+ getJsonObjectByArtifactId({ artifactId, isGzipped, }: {
1909
+ artifactId: string;
1910
+ isGzipped?: boolean;
1911
+ }): Promise<UploaderResult<object[] | object>>;
1912
+ }
1913
+
1914
+ declare interface UploaderFactoryInterface {
1915
+ event: AirSyncEvent;
1916
+ options?: WorkerAdapterOptions;
1917
+ }
1918
+
1919
+ /**
1920
+ * Generic result type for uploader operations that can either succeed with a response or fail with an error.
1921
+ * @template T The type of the successful response data
1922
+ */
1923
+ declare type UploaderResult<T> = {
1924
+ response: T;
1925
+ error?: never;
1926
+ } | {
1927
+ response?: never;
1928
+ error: unknown;
1929
+ };
1930
+
1931
+ /**
1932
+ * UploadResponse is an interface that defines the structure of the response from upload through Uploader.
1933
+ */
1934
+ export declare interface UploadResponse {
1935
+ artifact?: Artifact;
1936
+ error?: ErrorRecord;
1680
1937
  }
1681
1938
 
1682
1939
  /**
1683
- * WorkerAdapterInterface is an interface for WorkerAdapter class.
1940
+ * WorkerAdapterInterface is the constructor parameter shape for an adapter.
1684
1941
  * @interface WorkerAdapterInterface
1685
1942
  * @constructor
1686
1943
  * @param {AirSyncEvent} event - The event object received from the platform
1687
- * @param {object=} initialState - The initial state of the adapter
1688
- * @param {WorkerAdapterInterface} options - The options to create a new instance of WorkerAdapter class
1944
+ * @param {BaseState} adapterState - The state instance for the adapter
1945
+ * @param {WorkerAdapterOptions} options - The options to create a new adapter
1689
1946
  */
1690
- export declare interface WorkerAdapterInterface<ConnectorState> {
1947
+ export declare interface WorkerAdapterInterface<ConnectorState, StateClass extends BaseState<ConnectorState, BaseSdkState>> {
1691
1948
  event: AirSyncEvent;
1692
- adapterState: State<ConnectorState>;
1949
+ adapterState: StateClass;
1693
1950
  options?: WorkerAdapterOptions;
1694
1951
  }
1695
1952