@devrev/airsync-sdk 2.0.0-beta.6 → 2.0.0-beta.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. package/README.md +8 -12
  2. package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map +1 -1
  3. package/dist/attachments-streaming/attachments-streaming-pool.js +8 -9
  4. package/dist/common/constants.d.ts +0 -7
  5. package/dist/common/constants.d.ts.map +1 -1
  6. package/dist/common/constants.js +1 -28
  7. package/dist/common/helpers.d.ts.map +1 -1
  8. package/dist/common/helpers.js +1 -1
  9. package/dist/http/errors.d.ts +8 -0
  10. package/dist/http/errors.d.ts.map +1 -0
  11. package/dist/http/errors.js +31 -0
  12. package/dist/index.d.ts +18 -13
  13. package/dist/index.d.ts.map +1 -1
  14. package/dist/index.js +14 -11
  15. package/dist/logger/logger.d.ts.map +1 -1
  16. package/dist/logger/logger.interfaces.d.ts +2 -2
  17. package/dist/logger/logger.interfaces.d.ts.map +1 -1
  18. package/dist/mappers/mappers.d.ts +9 -3
  19. package/dist/mappers/mappers.d.ts.map +1 -1
  20. package/dist/mappers/mappers.interfaces.d.ts +0 -2
  21. package/dist/mappers/mappers.interfaces.d.ts.map +1 -1
  22. package/dist/mappers/mappers.js +33 -13
  23. package/dist/multithreading/adapters/base-adapter.d.ts +16 -35
  24. package/dist/multithreading/adapters/base-adapter.d.ts.map +1 -1
  25. package/dist/multithreading/adapters/base-adapter.js +89 -23
  26. package/dist/multithreading/adapters/extraction-adapter.d.ts +24 -20
  27. package/dist/multithreading/adapters/extraction-adapter.d.ts.map +1 -1
  28. package/dist/multithreading/adapters/extraction-adapter.js +175 -122
  29. package/dist/multithreading/adapters/loading-adapter.d.ts +13 -23
  30. package/dist/multithreading/adapters/loading-adapter.d.ts.map +1 -1
  31. package/dist/multithreading/adapters/loading-adapter.js +176 -301
  32. package/dist/multithreading/emit.d.ts.map +1 -1
  33. package/dist/multithreading/process-task.d.ts.map +1 -1
  34. package/dist/multithreading/process-task.js +24 -6
  35. package/dist/multithreading/spawn/spawn.d.ts +6 -1
  36. package/dist/multithreading/spawn/spawn.d.ts.map +1 -1
  37. package/dist/multithreading/spawn/spawn.helpers.d.ts.map +1 -1
  38. package/dist/multithreading/spawn/spawn.helpers.js +17 -52
  39. package/dist/multithreading/spawn/spawn.js +30 -26
  40. package/dist/multithreading/worker.js +15 -1
  41. package/dist/repo/repo.d.ts +4 -5
  42. package/dist/repo/repo.d.ts.map +1 -1
  43. package/dist/repo/repo.interfaces.d.ts +5 -4
  44. package/dist/repo/repo.interfaces.d.ts.map +1 -1
  45. package/dist/repo/repo.js +13 -15
  46. package/dist/state/base-state.d.ts +10 -5
  47. package/dist/state/base-state.d.ts.map +1 -1
  48. package/dist/state/base-state.js +39 -36
  49. package/dist/state/extraction-state.d.ts.map +1 -1
  50. package/dist/state/extraction-state.js +39 -4
  51. package/dist/state/state.interfaces.d.ts +13 -7
  52. package/dist/state/state.interfaces.d.ts.map +1 -1
  53. package/dist/state/state.interfaces.js +6 -8
  54. package/dist/state/time-value-resolver.d.ts.map +1 -1
  55. package/dist/state/time-value-resolver.js +3 -2
  56. package/dist/testing/mock-event.d.ts.map +1 -1
  57. package/dist/testing/mock-event.js +0 -8
  58. package/dist/testing/mock-server.d.ts +2 -0
  59. package/dist/testing/mock-server.d.ts.map +1 -1
  60. package/dist/testing/mock-server.interfaces.d.ts +6 -0
  61. package/dist/testing/mock-server.interfaces.d.ts.map +1 -1
  62. package/dist/testing/mock-server.js +37 -6
  63. package/dist/testing/test-adapters.d.ts +22 -0
  64. package/dist/testing/test-adapters.d.ts.map +1 -0
  65. package/dist/testing/test-adapters.js +32 -0
  66. package/dist/types/common.d.ts +4 -1
  67. package/dist/types/common.d.ts.map +1 -1
  68. package/dist/types/common.js +1 -1
  69. package/dist/types/errors.d.ts +13 -7
  70. package/dist/types/errors.d.ts.map +1 -1
  71. package/dist/types/errors.js +24 -0
  72. package/dist/types/extraction.d.ts +60 -44
  73. package/dist/types/extraction.d.ts.map +1 -1
  74. package/dist/types/extraction.js +10 -1
  75. package/dist/types/loading.d.ts +11 -30
  76. package/dist/types/loading.d.ts.map +1 -1
  77. package/dist/types/workers.d.ts +27 -23
  78. package/dist/types/workers.d.ts.map +1 -1
  79. package/dist/uploader/uploader.d.ts +1 -2
  80. package/dist/uploader/uploader.d.ts.map +1 -1
  81. package/dist/uploader/uploader.helpers.d.ts +3 -0
  82. package/dist/uploader/uploader.helpers.d.ts.map +1 -1
  83. package/dist/uploader/uploader.helpers.js +16 -0
  84. package/dist/uploader/uploader.interfaces.d.ts +4 -29
  85. package/dist/uploader/uploader.interfaces.d.ts.map +1 -1
  86. package/dist/uploader/uploader.js +4 -23
  87. package/docs/MIGRATION.md +651 -0
  88. package/package.json +24 -10
  89. package/MIGRATION.md +0 -1028
@@ -6,9 +6,11 @@ const constants_1 = require("../../common/constants");
6
6
  const logger_1 = require("../../logger/logger");
7
7
  const repo_1 = require("../../repo/repo");
8
8
  const extraction_1 = require("../../types/extraction");
9
+ const uploader_helpers_1 = require("../../uploader/uploader.helpers");
9
10
  const base_adapter_1 = require("./base-adapter");
10
11
  /** Adapter passed to extraction tasks (repos, artifacts, attachment streaming). @public */
11
12
  class ExtractionAdapter extends base_adapter_1.BaseAdapter {
13
+ /** @internal */
12
14
  constructor(params) {
13
15
  super(params);
14
16
  this.repos = [];
@@ -25,25 +27,60 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
25
27
  return scope[itemType].extract;
26
28
  }
27
29
  initializeRepos(repos) {
28
- this.repos = repos.map((repo) => {
29
- const shouldNormalize = repo.itemType !== constants_1.AirSyncDefaultItemTypes.EXTERNAL_DOMAIN_METADATA &&
30
- repo.itemType !== constants_1.SSOR_ATTACHMENT;
31
- return new repo_1.Repo(Object.assign(Object.assign({ event: this.event, itemType: repo.itemType }, (shouldNormalize && { normalize: repo.normalize })), { onUpload: (artifact) => {
32
- var _a;
33
- this.lastExtractedItemType = repo.itemType;
34
- // Artifact ids are kept in state for later attachment streaming
35
- if (repo.itemType === constants_1.AirSyncDefaultItemTypes.ATTACHMENTS) {
36
- (_a = this.sdkState.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata.artifactIds.push(artifact.id);
37
- }
38
- // Track size of artifact objects that go in the SQS message; flip
39
- // isTimeout once the threshold is exceeded to force an early emit.
40
- this.currentEventDataLength += Buffer.byteLength(JSON.stringify(artifact), 'utf8');
41
- if (this.currentEventDataLength > constants_1.EVENT_SIZE_THRESHOLD_BYTES &&
42
- !this.isTimeout) {
43
- this.isTimeout = true;
30
+ const existingSsorRepo = this.repos.find((repo) => repo.itemType === constants_1.SSOR_ATTACHMENT);
31
+ this.repos = repos.map((repo) => this.createRepo(repo));
32
+ if (existingSsorRepo &&
33
+ !this.repos.some((repo) => repo.itemType === constants_1.SSOR_ATTACHMENT)) {
34
+ this.repos.push(existingSsorRepo);
35
+ }
36
+ }
37
+ createRepo(repo) {
38
+ const shouldNormalize = repo.itemType !== constants_1.AirSyncDefaultItemTypes.EXTERNAL_DOMAIN_METADATA &&
39
+ repo.itemType !== constants_1.SSOR_ATTACHMENT;
40
+ const isDeferred = this.isDeferredRepo(repo.itemType);
41
+ return new repo_1.Repo(Object.assign(Object.assign({ event: this.event, itemType: repo.itemType }, (shouldNormalize && { normalize: repo.normalize })), { skipConfirmation: repo.itemType === constants_1.AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS, onUpload: (artifact) => {
42
+ var _a, _b;
43
+ if (isDeferred) {
44
+ const toDevRev = this.adapterState.sdkState.toDevRev;
45
+ if (toDevRev) {
46
+ (_a = toDevRev.deferredSsorArtifacts) !== null && _a !== void 0 ? _a : (toDevRev.deferredSsorArtifacts = []);
47
+ toDevRev.deferredSsorArtifacts.push(artifact);
44
48
  }
45
- }, options: Object.assign(Object.assign({}, this.options), repo.overridenOptions) }));
46
- });
49
+ return;
50
+ }
51
+ this.lastExtractedItemType = repo.itemType;
52
+ // Artifact ids are kept in state for later attachment streaming
53
+ if (repo.itemType === constants_1.AirSyncDefaultItemTypes.ATTACHMENTS) {
54
+ (_b = this.adapterState.sdkState.toDevRev) === null || _b === void 0 ? void 0 : _b.attachmentsMetadata.artifactIds.push(artifact.id);
55
+ }
56
+ // Track size of artifact objects that go in the SQS message; flip
57
+ // isTimeout once the threshold is exceeded to force an early emit.
58
+ this.currentEventDataLength += Buffer.byteLength(JSON.stringify(artifact), 'utf8');
59
+ if (this.currentEventDataLength > constants_1.EVENT_SIZE_THRESHOLD_BYTES &&
60
+ !this.isTimeout) {
61
+ this.isTimeout = true;
62
+ }
63
+ }, options: Object.assign(Object.assign({}, this.options), repo.overriddenOptions) }));
64
+ }
65
+ /**
66
+ * SSOR attachments created outside the attachments phase are held in state
67
+ * and reported with ATTACHMENT_EXTRACTION_DONE instead of the current event.
68
+ */
69
+ isDeferredRepo(itemType) {
70
+ return itemType === constants_1.SSOR_ATTACHMENT && !this.isAttachmentsPhase();
71
+ }
72
+ isAttachmentsPhase() {
73
+ const eventType = this.event.payload.event_type;
74
+ return (eventType === extraction_1.EventType.StartExtractingAttachments ||
75
+ eventType === extraction_1.EventType.ContinueExtractingAttachments);
76
+ }
77
+ getOrCreateSsorRepo() {
78
+ let repo = this.repos.find((repo) => repo.itemType === constants_1.SSOR_ATTACHMENT);
79
+ if (!repo) {
80
+ repo = this.createRepo({ itemType: constants_1.SSOR_ATTACHMENT });
81
+ this.repos.push(repo);
82
+ }
83
+ return repo;
47
84
  }
48
85
  getRepo(itemType) {
49
86
  const repo = this.repos.find((repo) => repo.itemType === itemType);
@@ -53,21 +90,43 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
53
90
  }
54
91
  return repo;
55
92
  }
93
+ /** Artifacts uploaded so far that the next emit will report. */
56
94
  get artifacts() {
57
95
  return this._artifacts;
58
96
  }
59
- set artifacts(artifacts) {
97
+ addArtifacts(artifacts) {
60
98
  this._artifacts = this._artifacts
61
99
  .concat(artifacts)
62
100
  .filter((value, index, self) => self.indexOf(value) === index);
63
101
  }
102
+ /**
103
+ * Per-record-type counts for sync-duration estimation. Set during the
104
+ * metadata phase; the SDK attaches them to the metadata-done event.
105
+ */
106
+ get preExtractionItemCounts() {
107
+ return this._preExtractionItemCounts;
108
+ }
109
+ set preExtractionItemCounts(counts) {
110
+ this._preExtractionItemCounts = counts;
111
+ }
112
+ /** @internal */
64
113
  async beforeEmit(newEventType) {
114
+ var _a;
65
115
  console.log(`Uploading all repos before emitting event with event type: ${newEventType}.`);
66
116
  await this.uploadAllRepos();
67
117
  // When the full extraction cycle completes, commit the extraction window
68
118
  // boundaries so subsequent incremental syncs can resume from them.
69
119
  if (newEventType === extraction_1.ExtractorEventType.AttachmentExtractionDone) {
70
- const sdkState = this.sdkState;
120
+ const sdkState = this.adapterState.sdkState;
121
+ // Released before the state save so they're never reported twice.
122
+ const toDevRev = sdkState.toDevRev;
123
+ if ((_a = toDevRev === null || toDevRev === void 0 ? void 0 : toDevRev.deferredSsorArtifacts) === null || _a === void 0 ? void 0 : _a.length) {
124
+ this.addArtifacts(toDevRev.deferredSsorArtifacts);
125
+ toDevRev.deferredSsorArtifacts = [];
126
+ }
127
+ console.log(`Overwriting lastSuccessfulSyncStarted with lastSyncStarted (${sdkState.lastSyncStarted}).`);
128
+ sdkState.lastSuccessfulSyncStarted = sdkState.lastSyncStarted;
129
+ sdkState.lastSyncStarted = '';
71
130
  sdkState.pendingWorkersOldest = '';
72
131
  sdkState.pendingWorkersNewest = '';
73
132
  // Expand boundaries: workersOldest keeps the earliest timestamp seen,
@@ -86,11 +145,21 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
86
145
  }
87
146
  }
88
147
  }
148
+ /** @internal */
89
149
  buildEmitPayload(newEventType) {
90
150
  const isExtractionEvent = Object.values(extraction_1.ExtractorEventType).includes(newEventType);
91
- return isExtractionEvent ? { artifacts: this.artifacts } : {};
151
+ if (!isExtractionEvent) {
152
+ return {};
153
+ }
154
+ const payload = { artifacts: [...this._artifacts] };
155
+ if (newEventType === extraction_1.ExtractorEventType.MetadataExtractionDone &&
156
+ this._preExtractionItemCounts) {
157
+ payload.pre_extraction_item_counts = this._preExtractionItemCounts;
158
+ }
159
+ return payload;
92
160
  }
93
161
  /**
162
+ * @internal
94
163
  * Attaches the last-extracted item type and its created/modified date ranges
95
164
  * to data/attachment done/progress events, mirroring the metadata v1 sent.
96
165
  */
@@ -121,116 +190,108 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
121
190
  }
122
191
  return new Date(ms).toISOString();
123
192
  }
193
+ /** @internal */
124
194
  afterEmit() {
125
- this.artifacts = [];
195
+ this._artifacts = [];
126
196
  }
127
197
  async uploadAllRepos() {
128
198
  for (const repo of this.repos) {
129
- const error = await repo.upload();
130
- this.artifacts.push(...repo.uploadedArtifacts);
131
- if (error) {
132
- throw error;
199
+ await repo.upload();
200
+ if (!this.isDeferredRepo(repo.itemType)) {
201
+ this.addArtifacts(repo.uploadedArtifacts);
133
202
  }
134
203
  }
135
204
  }
205
+ /** @internal */
206
+ beforeErrorEmit() {
207
+ // Report artifacts that uploaded before the failure so they aren't orphaned.
208
+ for (const repo of this.repos) {
209
+ if (!this.isDeferredRepo(repo.itemType)) {
210
+ this.addArtifacts(repo.uploadedArtifacts);
211
+ }
212
+ }
213
+ }
214
+ /**
215
+ * Streams one attachment into DevRev. Usable in any extraction phase; SSOR
216
+ * records created outside the attachments phase are reported with
217
+ * ATTACHMENT_EXTRACTION_DONE.
218
+ */
136
219
  async processAttachment(attachment, stream) {
137
- var _a, _b, _c;
138
- const { httpStream, delay, error } = await stream({
220
+ var _a, _b;
221
+ const response = await stream({
139
222
  item: attachment,
140
223
  event: this.event,
141
224
  });
142
- if (error) {
143
- return { error };
225
+ if (response.status !== 'success') {
226
+ return response;
144
227
  }
145
- else if (delay) {
146
- return { delay };
228
+ const { httpStream } = response;
229
+ const fileType = attachment.content_type ||
230
+ ((_a = httpStream.headers['content-type']) === null || _a === void 0 ? void 0 : _a.toString()) ||
231
+ 'application/octet-stream';
232
+ const contentLength = (_b = httpStream.headers['content-length']) === null || _b === void 0 ? void 0 : _b.toString();
233
+ const fileSize = contentLength ? parseInt(contentLength) : undefined;
234
+ const { error: artifactUrlError, response: artifactUrlResponse } = await this.uploader.getArtifactUploadUrl(attachment.file_name, fileType, fileSize);
235
+ if (artifactUrlError) {
236
+ (0, uploader_helpers_1.destroyHttpStream)(httpStream);
237
+ return {
238
+ status: 'error',
239
+ error: {
240
+ message: `Error while preparing artifact for attachment ID ${attachment.id}. Skipping attachment. ${(0, logger_1.serializeError)(artifactUrlError)}`,
241
+ fileSize,
242
+ },
243
+ };
147
244
  }
148
- if (httpStream) {
149
- const fileType = attachment.content_type ||
150
- ((_a = httpStream.headers['content-type']) === null || _a === void 0 ? void 0 : _a.toString()) ||
151
- 'application/octet-stream';
152
- const contentLength = (_b = httpStream.headers['content-length']) === null || _b === void 0 ? void 0 : _b.toString();
153
- const fileSize = contentLength ? parseInt(contentLength) : undefined;
154
- const { error: artifactUrlError, response: artifactUrlResponse } = await this.uploader.getArtifactUploadUrl(attachment.file_name, fileType, fileSize);
155
- if (artifactUrlError) {
156
- this.destroyHttpStream(httpStream);
157
- return {
158
- error: {
159
- message: `Error while preparing artifact for attachment ID ${attachment.id}. Skipping attachment. ${(0, logger_1.serializeError)(artifactUrlError)}`,
160
- fileSize: fileSize,
161
- },
162
- };
163
- }
164
- const { error: uploadedArtifactError } = await this.uploader.streamArtifact(artifactUrlResponse, httpStream);
165
- if (uploadedArtifactError) {
166
- this.destroyHttpStream(httpStream);
167
- return {
168
- error: {
169
- message: `Error while streaming to artifact for attachment ID ${attachment.id}. Skipping attachment. ` +
170
- (0, logger_1.serializeError)(uploadedArtifactError),
171
- fileSize: fileSize,
172
- },
173
- };
174
- }
175
- const { error: confirmArtifactUploadError } = await this.uploader.confirmArtifactUpload(artifactUrlResponse.artifact_id);
176
- if (confirmArtifactUploadError) {
177
- return {
178
- error: {
179
- message: `Error while confirming upload for attachment ID ${attachment.id}. ` +
180
- (0, logger_1.serializeError)(confirmArtifactUploadError),
181
- fileSize: fileSize,
182
- },
183
- };
184
- }
185
- const ssorAttachment = {
186
- id: {
187
- devrev: artifactUrlResponse.artifact_id,
188
- external: attachment.id,
245
+ const { error: uploadedArtifactError } = await this.uploader.streamArtifact(artifactUrlResponse, httpStream);
246
+ if (uploadedArtifactError) {
247
+ (0, uploader_helpers_1.destroyHttpStream)(httpStream);
248
+ return {
249
+ status: 'error',
250
+ error: {
251
+ message: `Error while streaming to artifact for attachment ID ${attachment.id}. Skipping attachment. ` +
252
+ (0, logger_1.serializeError)(uploadedArtifactError),
253
+ fileSize,
189
254
  },
190
- parent_id: {
191
- external: attachment.parent_id,
255
+ };
256
+ }
257
+ const { error: confirmArtifactUploadError } = await this.uploader.confirmArtifactUpload(artifactUrlResponse.artifact_id);
258
+ if (confirmArtifactUploadError) {
259
+ return {
260
+ status: 'error',
261
+ error: {
262
+ message: `Error while confirming upload for attachment ID ${attachment.id}. ` +
263
+ (0, logger_1.serializeError)(confirmArtifactUploadError),
264
+ fileSize,
192
265
  },
193
266
  };
194
- if (attachment.author_id) {
195
- ssorAttachment.actor_id = {
196
- external: attachment.author_id,
197
- };
198
- }
199
- // Set inline flag only if it is explicitly set on the attachment.
200
- if (attachment.inline === true) {
201
- ssorAttachment.inline = true;
202
- }
203
- else if (attachment.inline === false) {
204
- ssorAttachment.inline = false;
205
- }
206
- if (this.isTimeout) {
207
- this.destroyHttpStream(httpStream);
208
- return;
209
- }
210
- await ((_c = this.getRepo('ssor_attachment')) === null || _c === void 0 ? void 0 : _c.push([ssorAttachment]));
211
- return;
212
267
  }
213
- return {
214
- error: {
215
- message: `Error while opening attachment stream. Skipping attachment.`,
268
+ const ssorAttachment = {
269
+ id: {
270
+ devrev: artifactUrlResponse.artifact_id,
271
+ external: attachment.id,
272
+ },
273
+ parent_id: {
274
+ external: attachment.parent_id,
216
275
  },
217
276
  };
218
- }
219
- /** Destroys a stream to prevent memory leaks. */
220
- destroyHttpStream(httpStream) {
221
- try {
222
- if (httpStream && httpStream.data) {
223
- if (typeof httpStream.data.destroy === 'function') {
224
- httpStream.data.destroy();
225
- }
226
- else if (typeof httpStream.data.close === 'function') {
227
- httpStream.data.close();
228
- }
229
- }
277
+ if (attachment.author_id) {
278
+ ssorAttachment.actor_id = {
279
+ external: attachment.author_id,
280
+ };
281
+ }
282
+ // Set inline flag only if it is explicitly set on the attachment.
283
+ if (attachment.inline === true) {
284
+ ssorAttachment.inline = true;
230
285
  }
231
- catch (error) {
232
- console.warn('Error while destroying HTTP stream:', error);
286
+ else if (attachment.inline === false) {
287
+ ssorAttachment.inline = false;
288
+ }
289
+ if (this.isTimeout) {
290
+ (0, uploader_helpers_1.destroyHttpStream)(httpStream);
291
+ return { status: 'success' };
233
292
  }
293
+ await this.getOrCreateSsorRepo().push([ssorAttachment]);
294
+ return { status: 'success' };
234
295
  }
235
296
  async streamAttachments({ stream, processors, batchSize = 1, }) {
236
297
  var _a, _b;
@@ -242,13 +303,8 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
242
303
  console.warn(`The specified batch size (${batchSize}) is too large. Using 50 instead.`);
243
304
  batchSize = 50;
244
305
  }
245
- const repos = [
246
- {
247
- itemType: 'ssor_attachment',
248
- },
249
- ];
250
- this.initializeRepos(repos);
251
- const attachmentsMetadata = (_a = this.sdkState.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata;
306
+ this.initializeRepos([{ itemType: constants_1.SSOR_ATTACHMENT }]);
307
+ const attachmentsMetadata = (_a = this.adapterState.sdkState.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata;
252
308
  if (!((_b = attachmentsMetadata === null || attachmentsMetadata === void 0 ? void 0 : attachmentsMetadata.artifactIds) === null || _b === void 0 ? void 0 : _b.length)) {
253
309
  console.log(`No attachments metadata artifact IDs found in state.`);
254
310
  return { status: 'success' };
@@ -299,11 +355,8 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
299
355
  });
300
356
  response = await attachmentsPool.streamAll();
301
357
  }
302
- if (response === null || response === void 0 ? void 0 : response.error) {
303
- return { status: 'error', error: response.error };
304
- }
305
- if (response === null || response === void 0 ? void 0 : response.delay) {
306
- return { status: 'delay', delaySeconds: response.delay };
358
+ if (response.status !== 'success') {
359
+ return response;
307
360
  }
308
361
  if (this.isTimeout) {
309
362
  console.log(`Timeout detected after processing attachments for artifact ID: ${attachmentsMetadataArtifactId}. Returning progress to allow continuation.`);
@@ -1,39 +1,29 @@
1
1
  import { Mappers } from '../../mappers/mappers';
2
- import { BaseState } from '../../state/state';
3
- import { AirSyncEvent, EventData, ExtractorEventType } from '../../types/extraction';
4
- import { ExternalSystemAttachment, ExternalSystemItem, ExternalSystemLoadingFunction, FileToLoad, ItemTypesToLoadParams, ItemTypeToLoad, LoaderEventType, LoaderReport, LoadItemResponse } from '../../types/loading';
5
- import { TaskResult, WorkerAdapterOptions } from '../../types/workers';
2
+ import { ExternalSystemAttachment, ExternalSystemLoadingFunction, ItemTypesToLoadParams, LoaderReport } from '../../types/loading';
3
+ import { TaskResult } from '../../types/workers';
6
4
  import { BaseAdapter } from './base-adapter';
7
5
  /** Adapter passed to loading tasks (item/attachment loading, mappers, loader reports). @public */
8
6
  export declare class LoadingAdapter<ConnectorState> extends BaseAdapter<ConnectorState> {
9
7
  private loaderReports;
10
8
  private _processedFiles;
11
9
  private _mappers;
12
- constructor(params: {
13
- event: AirSyncEvent;
14
- adapterState: BaseState<ConnectorState>;
15
- options?: WorkerAdapterOptions;
16
- });
17
10
  get reports(): LoaderReport[];
18
11
  get processedFiles(): string[];
19
12
  get mappers(): Mappers;
20
- protected beforeEmit(): Promise<void>;
21
- protected buildEmitPayload(newEventType: ExtractorEventType | LoaderEventType): EventData;
22
- protected afterEmit(): void;
23
13
  loadItemTypes({ itemTypesToLoad, }: ItemTypesToLoadParams): Promise<TaskResult>;
24
- getLoaderBatches({ supportedItemTypes, }: {
25
- supportedItemTypes: string[];
26
- }): Promise<FileToLoad[]>;
27
14
  loadAttachments({ create, }: {
28
15
  create: ExternalSystemLoadingFunction<ExternalSystemAttachment>;
29
16
  }): Promise<TaskResult>;
30
- loadItem({ item, itemTypeToLoad, }: {
31
- item: ExternalSystemItem;
32
- itemTypeToLoad: ItemTypeToLoad;
33
- }): Promise<LoadItemResponse>;
34
- loadAttachment({ item, create, }: {
35
- item: ExternalSystemAttachment;
36
- create: ExternalSystemLoadingFunction<ExternalSystemAttachment>;
37
- }): Promise<LoadItemResponse>;
17
+ /**
18
+ * Walks the transformer files in `fromDevRev.filesToLoad`, resuming from
19
+ * each file's `lineToProcess`. `getLoader` returns undefined when a file has
20
+ * an item type the connector can't load, which fails the phase.
21
+ */
22
+ private loadFiles;
23
+ private getLoaderBatches;
24
+ private loadItem;
25
+ /** Maps a non-success connector response to a delay or a failed report. */
26
+ private toFailedLoad;
27
+ private loadAttachment;
38
28
  }
39
29
  //# sourceMappingURL=loading-adapter.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"loading-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/loading-adapter.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAEhD,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EACL,YAAY,EACZ,SAAS,EAET,kBAAkB,EACnB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAEL,wBAAwB,EACxB,kBAAkB,EAClB,6BAA6B,EAC7B,UAAU,EACV,qBAAqB,EACrB,cAAc,EACd,eAAe,EACf,YAAY,EACZ,gBAAgB,EAEjB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,UAAU,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAEvE,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAM7C,kGAAkG;AAClG,qBAAa,cAAc,CACzB,cAAc,CACd,SAAQ,WAAW,CAAC,cAAc,CAAC;IACnC,OAAO,CAAC,aAAa,CAAiB;IACtC,OAAO,CAAC,eAAe,CAAW;IAClC,OAAO,CAAC,QAAQ,CAAU;gBAEd,MAAM,EAAE;QAClB,KAAK,EAAE,YAAY,CAAC;QACpB,YAAY,EAAE,SAAS,CAAC,cAAc,CAAC,CAAC;QACxC,OAAO,CAAC,EAAE,oBAAoB,CAAC;KAChC;IAUD,IAAI,OAAO,IAAI,YAAY,EAAE,CAE5B;IAED,IAAI,cAAc,IAAI,MAAM,EAAE,CAE7B;IAED,IAAI,OAAO,IAAI,OAAO,CAErB;cAEe,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAI3C,SAAS,CAAC,gBAAgB,CACxB,YAAY,EAAE,kBAAkB,GAAG,eAAe,GACjD,SAAS;IAYZ,SAAS,CAAC,SAAS,IAAI,IAAI;IAIrB,aAAa,CAAC,EAClB,eAAe,GAChB,EAAE,qBAAqB,GAAG,OAAO,CAAC,UAAU,CAAC;IAoHxC,gBAAgB,CAAC,EACrB,kBAAkB,GACnB,EAAE;QACD,kBAAkB,EAAE,MAAM,EAAE,CAAC;KAC9B;IA0BK,eAAe,CAAC,EACpB,MAAM,GACP,EAAE;QACD,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,UAAU,CAAC;IAqFjB,QAAQ,CAAC,EACb,IAAI,EACJ,cAAc,GACf,EAAE;QACD,IAAI,EAAE,kBAAkB,CAAC;QACzB,cAAc,EAAE,cAAc,CAAC;KAChC,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAwLvB,cAAc,CAAC,EACnB,IAAI,EACJ,MAAM,GACP,EAAE;QACD,IAAI,EAAE,wBAAwB,CAAC;QAC/B,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAiD9B"}
1
+ {"version":3,"file":"loading-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/loading-adapter.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAShD,OAAO,EAEL,wBAAwB,EAGxB,6BAA6B,EAE7B,qBAAqB,EAGrB,YAAY,EAGb,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAkB,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjE,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAM7C,kGAAkG;AAClG,qBAAa,cAAc,CACzB,cAAc,CACd,SAAQ,WAAW,CAAC,cAAc,CAAC;IACnC,OAAO,CAAC,aAAa,CAAiB;IACtC,OAAO,CAAC,eAAe,CAAW;IAClC,OAAO,CAAC,QAAQ,CAAU;IAc1B,IAAI,OAAO,IAAI,YAAY,EAAE,CAE5B;IAED,IAAI,cAAc,IAAI,MAAM,EAAE,CAE7B;IAED,IAAI,OAAO,IAAI,OAAO,CAErB;IA2BK,aAAa,CAAC,EAClB,eAAe,GAChB,EAAE,qBAAqB,GAAG,OAAO,CAAC,UAAU,CAAC;IA8BxC,eAAe,CAAC,EACpB,MAAM,GACP,EAAE;QACD,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,UAAU,CAAC;IAavB;;;;OAIG;YACW,SAAS;YAiFT,gBAAgB;YAuBhB,QAAQ;IA+HtB,2EAA2E;IAC3E,OAAO,CAAC,YAAY;YAwBN,cAAc;CA2C7B"}