@devrev/airsync-sdk 2.0.0-beta.4 → 2.0.0-beta.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (112) hide show
  1. package/MIGRATION.md +1028 -0
  2. package/dist/attachments-streaming/attachments-streaming-pool.d.ts +6 -0
  3. package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map +1 -1
  4. package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts +2 -2
  5. package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts.map +1 -1
  6. package/dist/attachments-streaming/attachments-streaming-pool.js +47 -21
  7. package/dist/common/constants.d.ts +3 -5
  8. package/dist/common/constants.d.ts.map +1 -1
  9. package/dist/common/constants.js +4 -6
  10. package/dist/common/helpers.d.ts +3 -21
  11. package/dist/common/helpers.d.ts.map +1 -1
  12. package/dist/common/helpers.js +6 -15
  13. package/dist/http/client.d.ts.map +1 -1
  14. package/dist/http/client.js +4 -8
  15. package/dist/index.d.ts +11 -11
  16. package/dist/index.d.ts.map +1 -1
  17. package/dist/index.js +11 -18
  18. package/dist/logger/logger.d.ts +8 -63
  19. package/dist/logger/logger.d.ts.map +1 -1
  20. package/dist/logger/logger.interfaces.d.ts +0 -1
  21. package/dist/logger/logger.interfaces.d.ts.map +1 -1
  22. package/dist/logger/logger.js +14 -93
  23. package/dist/mappers/mappers.d.ts +2 -40
  24. package/dist/mappers/mappers.d.ts.map +1 -1
  25. package/dist/mappers/mappers.interfaces.d.ts +35 -154
  26. package/dist/mappers/mappers.interfaces.d.ts.map +1 -1
  27. package/dist/mappers/mappers.interfaces.js +1 -10
  28. package/dist/mappers/mappers.js +2 -40
  29. package/dist/multithreading/adapters/base-adapter.d.ts +19 -36
  30. package/dist/multithreading/adapters/base-adapter.d.ts.map +1 -1
  31. package/dist/multithreading/adapters/base-adapter.js +73 -71
  32. package/dist/multithreading/adapters/extraction-adapter.d.ts +12 -24
  33. package/dist/multithreading/adapters/extraction-adapter.d.ts.map +1 -1
  34. package/dist/multithreading/adapters/extraction-adapter.js +199 -200
  35. package/dist/multithreading/adapters/loading-adapter.d.ts +1 -6
  36. package/dist/multithreading/adapters/loading-adapter.d.ts.map +1 -1
  37. package/dist/multithreading/adapters/loading-adapter.helpers.d.ts +2 -12
  38. package/dist/multithreading/adapters/loading-adapter.helpers.d.ts.map +1 -1
  39. package/dist/multithreading/adapters/loading-adapter.helpers.js +2 -12
  40. package/dist/multithreading/adapters/loading-adapter.js +292 -314
  41. package/dist/multithreading/emit.d.ts +3 -2
  42. package/dist/multithreading/emit.d.ts.map +1 -1
  43. package/dist/multithreading/emit.js +3 -5
  44. package/dist/multithreading/process-task.d.ts +2 -12
  45. package/dist/multithreading/process-task.d.ts.map +1 -1
  46. package/dist/multithreading/process-task.js +45 -59
  47. package/dist/multithreading/spawn/spawn.d.ts +3 -9
  48. package/dist/multithreading/spawn/spawn.d.ts.map +1 -1
  49. package/dist/multithreading/spawn/spawn.helpers.d.ts +6 -25
  50. package/dist/multithreading/spawn/spawn.helpers.d.ts.map +1 -1
  51. package/dist/multithreading/spawn/spawn.helpers.js +7 -49
  52. package/dist/multithreading/spawn/spawn.js +18 -33
  53. package/dist/multithreading/worker.js +2 -4
  54. package/dist/repo/repo.d.ts +10 -0
  55. package/dist/repo/repo.d.ts.map +1 -1
  56. package/dist/repo/repo.helpers.d.ts +6 -0
  57. package/dist/repo/repo.helpers.d.ts.map +1 -0
  58. package/dist/repo/repo.helpers.js +16 -0
  59. package/dist/repo/repo.interfaces.d.ts +4 -16
  60. package/dist/repo/repo.interfaces.d.ts.map +1 -1
  61. package/dist/repo/repo.js +23 -8
  62. package/dist/state/base-state.d.ts +14 -36
  63. package/dist/state/base-state.d.ts.map +1 -1
  64. package/dist/state/base-state.js +16 -38
  65. package/dist/state/extraction-state.d.ts +4 -19
  66. package/dist/state/extraction-state.d.ts.map +1 -1
  67. package/dist/state/extraction-state.js +6 -23
  68. package/dist/state/install-initial-domain-mapping.d.ts +1 -1
  69. package/dist/state/install-initial-domain-mapping.d.ts.map +1 -1
  70. package/dist/state/install-initial-domain-mapping.js +1 -3
  71. package/dist/state/loading-state.d.ts +1 -11
  72. package/dist/state/loading-state.d.ts.map +1 -1
  73. package/dist/state/loading-state.js +2 -12
  74. package/dist/state/state.d.ts +2 -2
  75. package/dist/state/state.d.ts.map +1 -1
  76. package/dist/state/state.interfaces.d.ts +10 -24
  77. package/dist/state/state.interfaces.d.ts.map +1 -1
  78. package/dist/state/state.interfaces.js +9 -11
  79. package/dist/state/state.js +3 -3
  80. package/dist/state/time-value-resolver.d.ts +7 -36
  81. package/dist/state/time-value-resolver.d.ts.map +1 -1
  82. package/dist/state/time-value-resolver.js +7 -40
  83. package/dist/testing/mock-event.d.ts +2 -10
  84. package/dist/testing/mock-event.d.ts.map +1 -1
  85. package/dist/testing/mock-event.js +3 -11
  86. package/dist/testing/mock-server.d.ts +4 -27
  87. package/dist/testing/mock-server.d.ts.map +1 -1
  88. package/dist/testing/mock-server.interfaces.d.ts +10 -35
  89. package/dist/testing/mock-server.interfaces.d.ts.map +1 -1
  90. package/dist/testing/mock-server.js +19 -36
  91. package/dist/types/common.d.ts +1 -12
  92. package/dist/types/common.d.ts.map +1 -1
  93. package/dist/types/common.js +1 -6
  94. package/dist/types/external-domain-metadata.d.ts +30 -171
  95. package/dist/types/external-domain-metadata.d.ts.map +1 -1
  96. package/dist/types/extraction.d.ts +35 -120
  97. package/dist/types/extraction.d.ts.map +1 -1
  98. package/dist/types/extraction.js +5 -31
  99. package/dist/types/loading.d.ts.map +1 -1
  100. package/dist/types/workers.d.ts +18 -116
  101. package/dist/types/workers.d.ts.map +1 -1
  102. package/dist/types/workers.js +0 -6
  103. package/dist/uploader/uploader.d.ts +6 -58
  104. package/dist/uploader/uploader.d.ts.map +1 -1
  105. package/dist/uploader/uploader.helpers.d.ts +4 -27
  106. package/dist/uploader/uploader.helpers.d.ts.map +1 -1
  107. package/dist/uploader/uploader.helpers.js +64 -30
  108. package/dist/uploader/uploader.interfaces.d.ts +15 -25
  109. package/dist/uploader/uploader.interfaces.d.ts.map +1 -1
  110. package/dist/uploader/uploader.interfaces.js +8 -0
  111. package/dist/uploader/uploader.js +18 -84
  112. package/package.json +6 -4
@@ -7,13 +7,7 @@ const logger_1 = require("../../logger/logger");
7
7
  const repo_1 = require("../../repo/repo");
8
8
  const extraction_1 = require("../../types/extraction");
9
9
  const base_adapter_1 = require("./base-adapter");
10
- /**
11
- * ExtractionAdapter is the adapter passed to extraction tasks. It exposes the
12
- * extraction surface (repos, artifacts, attachment streaming) and uploads
13
- * pending repos and updates the extraction boundaries before emitting.
14
- *
15
- * @public
16
- */
10
+ /** Adapter passed to extraction tasks (repos, artifacts, attachment streaming). @public */
17
11
  class ExtractionAdapter extends base_adapter_1.BaseAdapter {
18
12
  constructor(params) {
19
13
  super(params);
@@ -21,10 +15,7 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
21
15
  this.currentEventDataLength = 0;
22
16
  this._artifacts = [];
23
17
  }
24
- /**
25
- * Returns whether the given item type should be extracted.
26
- * Defaults to true if the scope is empty or the item type is not listed.
27
- */
18
+ /** Defaults to true if the scope is empty or the item type is not listed. */
28
19
  shouldExtract(itemType) {
29
20
  const scope = this.extractionScope;
30
21
  if (Object.keys(scope).length === 0)
@@ -39,11 +30,13 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
39
30
  repo.itemType !== constants_1.SSOR_ATTACHMENT;
40
31
  return new repo_1.Repo(Object.assign(Object.assign({ event: this.event, itemType: repo.itemType }, (shouldNormalize && { normalize: repo.normalize })), { onUpload: (artifact) => {
41
32
  var _a;
42
- // We need to store artifacts ids in state for later use when streaming attachments
33
+ this.lastExtractedItemType = repo.itemType;
34
+ // Artifact ids are kept in state for later attachment streaming
43
35
  if (repo.itemType === constants_1.AirSyncDefaultItemTypes.ATTACHMENTS) {
44
36
  (_a = this.sdkState.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata.artifactIds.push(artifact.id);
45
37
  }
46
- // Calculate size of the entire artifact object that goes in the SQS message
38
+ // Track size of artifact objects that go in the SQS message; flip
39
+ // isTimeout once the threshold is exceeded to force an early emit.
47
40
  this.currentEventDataLength += Buffer.byteLength(JSON.stringify(artifact), 'utf8');
48
41
  if (this.currentEventDataLength > constants_1.EVENT_SIZE_THRESHOLD_BYTES &&
49
42
  !this.isTimeout) {
@@ -53,14 +46,12 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
53
46
  });
54
47
  }
55
48
  getRepo(itemType) {
56
- return (0, logger_1.runWithSdkLogContext)(() => {
57
- const repo = this.repos.find((repo) => repo.itemType === itemType);
58
- if (!repo) {
59
- console.error(`Repo for item type ${itemType} not found.`);
60
- return;
61
- }
62
- return repo;
63
- });
49
+ const repo = this.repos.find((repo) => repo.itemType === itemType);
50
+ if (!repo) {
51
+ console.error(`Repo for item type ${itemType} not found.`);
52
+ return;
53
+ }
54
+ return repo;
64
55
  }
65
56
  get artifacts() {
66
57
  return this._artifacts;
@@ -71,18 +62,16 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
71
62
  .filter((value, index, self) => self.indexOf(value) === index);
72
63
  }
73
64
  async beforeEmit(newEventType) {
74
- // Upload all repos before emitting the event
75
65
  console.log(`Uploading all repos before emitting event with event type: ${newEventType}.`);
76
66
  await this.uploadAllRepos();
77
67
  // When the full extraction cycle completes, commit the extraction window
78
68
  // boundaries so subsequent incremental syncs can resume from them.
79
69
  if (newEventType === extraction_1.ExtractorEventType.AttachmentExtractionDone) {
80
70
  const sdkState = this.sdkState;
81
- // Clear pending extraction boundaries now that the cycle is complete
82
71
  sdkState.pendingWorkersOldest = '';
83
72
  sdkState.pendingWorkersNewest = '';
84
- // Update workersOldest and workersNewest boundaries from resolved extraction timestamps.
85
- // Expand boundaries: workersOldest gets the earliest timestamp, workersNewest gets the latest.
73
+ // Expand boundaries: workersOldest keeps the earliest timestamp seen,
74
+ // workersNewest the latest.
86
75
  const extractionStart = this.event.payload.event_context.extract_from;
87
76
  const extractionEnd = this.event.payload.event_context.extract_to;
88
77
  if (extractionStart &&
@@ -101,6 +90,37 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
101
90
  const isExtractionEvent = Object.values(extraction_1.ExtractorEventType).includes(newEventType);
102
91
  return isExtractionEvent ? { artifacts: this.artifacts } : {};
103
92
  }
93
+ /**
94
+ * Attaches the last-extracted item type and its created/modified date ranges
95
+ * to data/attachment done/progress events, mirroring the metadata v1 sent.
96
+ */
97
+ buildWorkerMetadata(newEventType) {
98
+ if (newEventType !== extraction_1.ExtractorEventType.DataExtractionDone &&
99
+ newEventType !== extraction_1.ExtractorEventType.DataExtractionProgress &&
100
+ newEventType !== extraction_1.ExtractorEventType.AttachmentExtractionDone &&
101
+ newEventType !== extraction_1.ExtractorEventType.AttachmentExtractionProgress) {
102
+ return undefined;
103
+ }
104
+ const repo = this.lastExtractedItemType
105
+ ? this.repos.find((r) => r.itemType === this.lastExtractedItemType)
106
+ : undefined;
107
+ if (!repo) {
108
+ return undefined;
109
+ }
110
+ return {
111
+ item_type: repo.itemType,
112
+ newest_created_date: this.toRfc3339Timestamp(repo.dateRanges.creationDate.newest),
113
+ oldest_created_date: this.toRfc3339Timestamp(repo.dateRanges.creationDate.oldest),
114
+ newest_modified_date: this.toRfc3339Timestamp(repo.dateRanges.modifiedDate.newest),
115
+ oldest_modified_date: this.toRfc3339Timestamp(repo.dateRanges.modifiedDate.oldest),
116
+ };
117
+ }
118
+ toRfc3339Timestamp(ms) {
119
+ if (ms === undefined || !Number.isFinite(ms)) {
120
+ return undefined;
121
+ }
122
+ return new Date(ms).toISOString();
123
+ }
104
124
  afterEmit() {
105
125
  this.artifacts = [];
106
126
  }
@@ -114,94 +134,89 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
114
134
  }
115
135
  }
116
136
  async processAttachment(attachment, stream) {
117
- return (0, logger_1.runWithSdkLogContext)(async () => {
118
- var _a, _b, _c;
119
- const { httpStream, delay, error } = await (0, logger_1.runWithUserLogContext)(async () => stream({
120
- item: attachment,
121
- event: this.event,
122
- }));
123
- if (error) {
124
- return { error };
125
- }
126
- else if (delay) {
127
- return { delay };
137
+ var _a, _b, _c;
138
+ const { httpStream, delay, error } = await stream({
139
+ item: attachment,
140
+ event: this.event,
141
+ });
142
+ if (error) {
143
+ return { error };
144
+ }
145
+ else if (delay) {
146
+ return { delay };
147
+ }
148
+ if (httpStream) {
149
+ const fileType = attachment.content_type ||
150
+ ((_a = httpStream.headers['content-type']) === null || _a === void 0 ? void 0 : _a.toString()) ||
151
+ 'application/octet-stream';
152
+ const contentLength = (_b = httpStream.headers['content-length']) === null || _b === void 0 ? void 0 : _b.toString();
153
+ const fileSize = contentLength ? parseInt(contentLength) : undefined;
154
+ const { error: artifactUrlError, response: artifactUrlResponse } = await this.uploader.getArtifactUploadUrl(attachment.file_name, fileType, fileSize);
155
+ if (artifactUrlError) {
156
+ this.destroyHttpStream(httpStream);
157
+ return {
158
+ error: {
159
+ message: `Error while preparing artifact for attachment ID ${attachment.id}. Skipping attachment. ${(0, logger_1.serializeError)(artifactUrlError)}`,
160
+ fileSize: fileSize,
161
+ },
162
+ };
128
163
  }
129
- if (httpStream) {
130
- const fileType = attachment.content_type ||
131
- ((_a = httpStream.headers['content-type']) === null || _a === void 0 ? void 0 : _a.toString()) ||
132
- 'application/octet-stream';
133
- const contentLength = (_b = httpStream.headers['content-length']) === null || _b === void 0 ? void 0 : _b.toString();
134
- const fileSize = contentLength ? parseInt(contentLength) : undefined;
135
- // Get upload URL
136
- const { error: artifactUrlError, response: artifactUrlResponse } = await this.uploader.getArtifactUploadUrl(attachment.file_name, fileType, fileSize);
137
- if (artifactUrlError) {
138
- this.destroyHttpStream(httpStream);
139
- return {
140
- error: {
141
- message: `Error while preparing artifact for attachment ID ${attachment.id}. Skipping attachment. ${(0, logger_1.serializeError)(artifactUrlError)}`,
142
- fileSize: fileSize,
143
- },
144
- };
145
- }
146
- // Stream attachment
147
- const { error: uploadedArtifactError } = await this.uploader.streamArtifact(artifactUrlResponse, httpStream);
148
- if (uploadedArtifactError) {
149
- this.destroyHttpStream(httpStream);
150
- return {
151
- error: {
152
- message: `Error while streaming to artifact for attachment ID ${attachment.id}. Skipping attachment. ` +
153
- (0, logger_1.serializeError)(uploadedArtifactError),
154
- fileSize: fileSize,
155
- },
156
- };
157
- }
158
- // Confirm attachment upload
159
- const { error: confirmArtifactUploadError } = await this.uploader.confirmArtifactUpload(artifactUrlResponse.artifact_id);
160
- if (confirmArtifactUploadError) {
161
- return {
162
- error: {
163
- message: `Error while confirming upload for attachment ID ${attachment.id}. ` +
164
- (0, logger_1.serializeError)(confirmArtifactUploadError),
165
- fileSize: fileSize,
166
- },
167
- };
168
- }
169
- const ssorAttachment = {
170
- id: {
171
- devrev: artifactUrlResponse.artifact_id,
172
- external: attachment.id,
164
+ const { error: uploadedArtifactError } = await this.uploader.streamArtifact(artifactUrlResponse, httpStream);
165
+ if (uploadedArtifactError) {
166
+ this.destroyHttpStream(httpStream);
167
+ return {
168
+ error: {
169
+ message: `Error while streaming to artifact for attachment ID ${attachment.id}. Skipping attachment. ` +
170
+ (0, logger_1.serializeError)(uploadedArtifactError),
171
+ fileSize: fileSize,
173
172
  },
174
- parent_id: {
175
- external: attachment.parent_id,
173
+ };
174
+ }
175
+ const { error: confirmArtifactUploadError } = await this.uploader.confirmArtifactUpload(artifactUrlResponse.artifact_id);
176
+ if (confirmArtifactUploadError) {
177
+ return {
178
+ error: {
179
+ message: `Error while confirming upload for attachment ID ${attachment.id}. ` +
180
+ (0, logger_1.serializeError)(confirmArtifactUploadError),
181
+ fileSize: fileSize,
176
182
  },
177
183
  };
178
- if (attachment.author_id) {
179
- ssorAttachment.actor_id = {
180
- external: attachment.author_id,
181
- };
182
- }
183
- // This will set inline flag in ssor_attachment only if it is explicity
184
- // set in the attachment object.
185
- if (attachment.inline === true) {
186
- ssorAttachment.inline = true;
187
- }
188
- else if (attachment.inline === false) {
189
- ssorAttachment.inline = false;
190
- }
191
- await ((_c = this.getRepo('ssor_attachment')) === null || _c === void 0 ? void 0 : _c.push([ssorAttachment]));
192
- return;
193
184
  }
194
- return {
195
- error: {
196
- message: `Error while opening attachment stream. Skipping attachment.`,
185
+ const ssorAttachment = {
186
+ id: {
187
+ devrev: artifactUrlResponse.artifact_id,
188
+ external: attachment.id,
189
+ },
190
+ parent_id: {
191
+ external: attachment.parent_id,
197
192
  },
198
193
  };
199
- });
194
+ if (attachment.author_id) {
195
+ ssorAttachment.actor_id = {
196
+ external: attachment.author_id,
197
+ };
198
+ }
199
+ // Set inline flag only if it is explicitly set on the attachment.
200
+ if (attachment.inline === true) {
201
+ ssorAttachment.inline = true;
202
+ }
203
+ else if (attachment.inline === false) {
204
+ ssorAttachment.inline = false;
205
+ }
206
+ if (this.isTimeout) {
207
+ this.destroyHttpStream(httpStream);
208
+ return;
209
+ }
210
+ await ((_c = this.getRepo('ssor_attachment')) === null || _c === void 0 ? void 0 : _c.push([ssorAttachment]));
211
+ return;
212
+ }
213
+ return {
214
+ error: {
215
+ message: `Error while opening attachment stream. Skipping attachment.`,
216
+ },
217
+ };
200
218
  }
201
- /**
202
- * Destroys a stream to prevent memory leaks.
203
- * @param httpStream - The axios response stream to destroy
204
- */
219
+ /** Destroys a stream to prevent memory leaks. */
205
220
  destroyHttpStream(httpStream) {
206
221
  try {
207
222
  if (httpStream && httpStream.data) {
@@ -217,107 +232,91 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
217
232
  console.warn('Error while destroying HTTP stream:', error);
218
233
  }
219
234
  }
220
- /**
221
- * Streams the attachments to the DevRev platform.
222
- * The attachments are streamed to the platform and the artifact information is returned.
223
- * @param params - The parameters to stream the attachments
224
- * @returns The response object containing the ssorAttachment artifact information
225
- * or error information if there was an error
226
- */
227
- async streamAttachments({ stream, processors, batchSize = 1, // By default, we want to stream one attachment at a time
228
- }) {
229
- return (0, logger_1.runWithSdkLogContext)(async () => {
230
- var _a, _b;
231
- if (batchSize <= 0) {
232
- console.warn(`The specified batch size (${batchSize}) is invalid. Using 1 instead.`);
233
- batchSize = 1;
235
+ async streamAttachments({ stream, processors, batchSize = 1, }) {
236
+ var _a, _b;
237
+ if (batchSize <= 0) {
238
+ console.warn(`The specified batch size (${batchSize}) is invalid. Using 1 instead.`);
239
+ batchSize = 1;
240
+ }
241
+ if (batchSize > 50) {
242
+ console.warn(`The specified batch size (${batchSize}) is too large. Using 50 instead.`);
243
+ batchSize = 50;
244
+ }
245
+ const repos = [
246
+ {
247
+ itemType: 'ssor_attachment',
248
+ },
249
+ ];
250
+ this.initializeRepos(repos);
251
+ const attachmentsMetadata = (_a = this.sdkState.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata;
252
+ if (!((_b = attachmentsMetadata === null || attachmentsMetadata === void 0 ? void 0 : attachmentsMetadata.artifactIds) === null || _b === void 0 ? void 0 : _b.length)) {
253
+ console.log(`No attachments metadata artifact IDs found in state.`);
254
+ return { status: 'success' };
255
+ }
256
+ else {
257
+ console.log(`Found ${attachmentsMetadata.artifactIds.length} attachments metadata artifact IDs in state.`);
258
+ }
259
+ while (attachmentsMetadata.artifactIds.length > 0) {
260
+ const attachmentsMetadataArtifactId = attachmentsMetadata.artifactIds[0];
261
+ console.log(`Started processing attachments for attachments metadata artifact ID: ${attachmentsMetadataArtifactId}.`);
262
+ const { attachments, error } = await this.uploader.getAttachmentsFromArtifactId({
263
+ artifact: attachmentsMetadataArtifactId,
264
+ });
265
+ if (error) {
266
+ console.error(`Failed to get attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
267
+ return { status: 'error', error };
234
268
  }
235
- if (batchSize > 50) {
236
- console.warn(`The specified batch size (${batchSize}) is too large. Using 50 instead.`);
237
- batchSize = 50;
269
+ if (!attachments || attachments.length === 0) {
270
+ console.warn(`No attachments found for artifact ID: ${attachmentsMetadataArtifactId}.`);
271
+ attachmentsMetadata.artifactIds.shift();
272
+ attachmentsMetadata.lastProcessed = 0;
273
+ continue;
238
274
  }
239
- const repos = [
240
- {
241
- itemType: 'ssor_attachment',
242
- },
243
- ];
244
- this.initializeRepos(repos);
245
- const attachmentsMetadata = (_a = this.sdkState.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata;
246
- // If there are no attachments metadata artifact IDs in state, finish here
247
- if (!((_b = attachmentsMetadata === null || attachmentsMetadata === void 0 ? void 0 : attachmentsMetadata.artifactIds) === null || _b === void 0 ? void 0 : _b.length)) {
248
- console.log(`No attachments metadata artifact IDs found in state.`);
249
- return { status: 'success' };
275
+ console.log(`Found ${attachments.length} attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
276
+ let response;
277
+ if (processors) {
278
+ console.log(`Using custom processors for attachments.`);
279
+ const reducer = processors.reducer;
280
+ const iterator = processors.iterator;
281
+ const reducedAttachments = reducer({
282
+ attachments,
283
+ adapter: this,
284
+ batchSize,
285
+ });
286
+ response = await iterator({
287
+ reducedAttachments,
288
+ adapter: this,
289
+ stream,
290
+ });
250
291
  }
251
292
  else {
252
- console.log(`Found ${attachmentsMetadata.artifactIds.length} attachments metadata artifact IDs in state.`);
253
- }
254
- // Loop through the attachments metadata artifact IDs
255
- while (attachmentsMetadata.artifactIds.length > 0) {
256
- const attachmentsMetadataArtifactId = attachmentsMetadata.artifactIds[0];
257
- console.log(`Started processing attachments for attachments metadata artifact ID: ${attachmentsMetadataArtifactId}.`);
258
- const { attachments, error } = await this.uploader.getAttachmentsFromArtifactId({
259
- artifact: attachmentsMetadataArtifactId,
293
+ console.log(`Using attachments streaming pool for attachments streaming.`);
294
+ const attachmentsPool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
295
+ adapter: this,
296
+ attachments,
297
+ batchSize,
298
+ stream,
260
299
  });
261
- if (error) {
262
- console.error(`Failed to get attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
263
- return { status: 'error', error };
264
- }
265
- if (!attachments || attachments.length === 0) {
266
- console.warn(`No attachments found for artifact ID: ${attachmentsMetadataArtifactId}.`);
267
- // Remove empty artifact and reset lastProcessed
268
- attachmentsMetadata.artifactIds.shift();
269
- attachmentsMetadata.lastProcessed = 0;
270
- continue;
271
- }
272
- console.log(`Found ${attachments.length} attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
273
- let response;
274
- if (processors) {
275
- console.log(`Using custom processors for attachments.`);
276
- const reducer = processors.reducer;
277
- const iterator = processors.iterator;
278
- const reducedAttachments = (0, logger_1.runWithUserLogContext)(() => reducer({
279
- attachments,
280
- adapter: this,
281
- batchSize,
282
- }));
283
- response = await (0, logger_1.runWithUserLogContext)(async () => {
284
- return await iterator({
285
- reducedAttachments,
286
- adapter: this,
287
- stream,
288
- });
289
- });
290
- }
291
- else {
292
- console.log(`Using attachments streaming pool for attachments streaming.`);
293
- const attachmentsPool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
294
- adapter: this,
295
- attachments,
296
- batchSize,
297
- stream,
298
- });
299
- response = await attachmentsPool.streamAll();
300
- }
301
- if (response === null || response === void 0 ? void 0 : response.error) {
302
- return { status: 'error', error: response.error };
303
- }
304
- if (response === null || response === void 0 ? void 0 : response.delay) {
305
- return { status: 'delay', delaySeconds: response.delay };
306
- }
307
- // On timeout, return progress to allow continuation in a fresh invocation.
308
- if (this.isTimeout) {
309
- console.log(`Timeout detected after processing attachments for artifact ID: ${attachmentsMetadataArtifactId}. Returning progress to allow continuation.`);
310
- return { status: 'progress' };
311
- }
312
- console.log(`Finished processing all attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
313
- attachmentsMetadata.artifactIds.shift();
314
- attachmentsMetadata.lastProcessed = 0;
315
- if (attachmentsMetadata.lastProcessedAttachmentsIdsList) {
316
- attachmentsMetadata.lastProcessedAttachmentsIdsList.length = 0;
317
- }
300
+ response = await attachmentsPool.streamAll();
318
301
  }
319
- return { status: 'success' };
320
- });
302
+ if (response === null || response === void 0 ? void 0 : response.error) {
303
+ return { status: 'error', error: response.error };
304
+ }
305
+ if (response === null || response === void 0 ? void 0 : response.delay) {
306
+ return { status: 'delay', delaySeconds: response.delay };
307
+ }
308
+ if (this.isTimeout) {
309
+ console.log(`Timeout detected after processing attachments for artifact ID: ${attachmentsMetadataArtifactId}. Returning progress to allow continuation.`);
310
+ return { status: 'progress' };
311
+ }
312
+ console.log(`Finished processing all attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
313
+ attachmentsMetadata.artifactIds.shift();
314
+ attachmentsMetadata.lastProcessed = 0;
315
+ if (attachmentsMetadata.lastProcessedAttachmentsIdsList) {
316
+ attachmentsMetadata.lastProcessedAttachmentsIdsList.length = 0;
317
+ }
318
+ }
319
+ return { status: 'success' };
321
320
  }
322
321
  }
323
322
  exports.ExtractionAdapter = ExtractionAdapter;
@@ -4,12 +4,7 @@ import { AirSyncEvent, EventData, ExtractorEventType } from '../../types/extract
4
4
  import { ExternalSystemAttachment, ExternalSystemItem, ExternalSystemLoadingFunction, FileToLoad, ItemTypesToLoadParams, ItemTypeToLoad, LoaderEventType, LoaderReport, LoadItemResponse } from '../../types/loading';
5
5
  import { TaskResult, WorkerAdapterOptions } from '../../types/workers';
6
6
  import { BaseAdapter } from './base-adapter';
7
- /**
8
- * LoadingAdapter is the adapter passed to loading tasks. It exposes the loading
9
- * surface (item/attachment loading, mappers, loader reports).
10
- *
11
- * @public
12
- */
7
+ /** Adapter passed to loading tasks (item/attachment loading, mappers, loader reports). @public */
13
8
  export declare class LoadingAdapter<ConnectorState> extends BaseAdapter<ConnectorState> {
14
9
  private loaderReports;
15
10
  private _processedFiles;
@@ -1 +1 @@
1
- {"version":3,"file":"loading-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/loading-adapter.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAEhD,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EACL,YAAY,EACZ,SAAS,EAET,kBAAkB,EACnB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAEL,wBAAwB,EACxB,kBAAkB,EAClB,6BAA6B,EAC7B,UAAU,EACV,qBAAqB,EACrB,cAAc,EACd,eAAe,EACf,YAAY,EACZ,gBAAgB,EAEjB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,UAAU,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAEvE,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAM7C;;;;;GAKG;AACH,qBAAa,cAAc,CACzB,cAAc,CACd,SAAQ,WAAW,CAAC,cAAc,CAAC;IACnC,OAAO,CAAC,aAAa,CAAiB;IACtC,OAAO,CAAC,eAAe,CAAW;IAClC,OAAO,CAAC,QAAQ,CAAU;gBAEd,MAAM,EAAE;QAClB,KAAK,EAAE,YAAY,CAAC;QACpB,YAAY,EAAE,SAAS,CAAC,cAAc,CAAC,CAAC;QACxC,OAAO,CAAC,EAAE,oBAAoB,CAAC;KAChC;IAUD,IAAI,OAAO,IAAI,YAAY,EAAE,CAE5B;IAED,IAAI,cAAc,IAAI,MAAM,EAAE,CAE7B;IAED,IAAI,OAAO,IAAI,OAAO,CAErB;cAEe,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAI3C,SAAS,CAAC,gBAAgB,CACxB,YAAY,EAAE,kBAAkB,GAAG,eAAe,GACjD,SAAS;IAYZ,SAAS,CAAC,SAAS,IAAI,IAAI;IAIrB,aAAa,CAAC,EAClB,eAAe,GAChB,EAAE,qBAAqB,GAAG,OAAO,CAAC,UAAU,CAAC;IAsHxC,gBAAgB,CAAC,EACrB,kBAAkB,GACnB,EAAE;QACD,kBAAkB,EAAE,MAAM,EAAE,CAAC;KAC9B;IA4BK,eAAe,CAAC,EACpB,MAAM,GACP,EAAE;QACD,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,UAAU,CAAC;IA4FjB,QAAQ,CAAC,EACb,IAAI,EACJ,cAAc,GACf,EAAE;QACD,IAAI,EAAE,kBAAkB,CAAC;QACzB,cAAc,EAAE,cAAc,CAAC;KAChC,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAsMvB,cAAc,CAAC,EACnB,IAAI,EACJ,MAAM,GACP,EAAE;QACD,IAAI,EAAE,wBAAwB,CAAC;QAC/B,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAsD9B"}
1
+ {"version":3,"file":"loading-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/loading-adapter.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAEhD,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EACL,YAAY,EACZ,SAAS,EAET,kBAAkB,EACnB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAEL,wBAAwB,EACxB,kBAAkB,EAClB,6BAA6B,EAC7B,UAAU,EACV,qBAAqB,EACrB,cAAc,EACd,eAAe,EACf,YAAY,EACZ,gBAAgB,EAEjB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,UAAU,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAEvE,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAM7C,kGAAkG;AAClG,qBAAa,cAAc,CACzB,cAAc,CACd,SAAQ,WAAW,CAAC,cAAc,CAAC;IACnC,OAAO,CAAC,aAAa,CAAiB;IACtC,OAAO,CAAC,eAAe,CAAW;IAClC,OAAO,CAAC,QAAQ,CAAU;gBAEd,MAAM,EAAE;QAClB,KAAK,EAAE,YAAY,CAAC;QACpB,YAAY,EAAE,SAAS,CAAC,cAAc,CAAC,CAAC;QACxC,OAAO,CAAC,EAAE,oBAAoB,CAAC;KAChC;IAUD,IAAI,OAAO,IAAI,YAAY,EAAE,CAE5B;IAED,IAAI,cAAc,IAAI,MAAM,EAAE,CAE7B;IAED,IAAI,OAAO,IAAI,OAAO,CAErB;cAEe,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAI3C,SAAS,CAAC,gBAAgB,CACxB,YAAY,EAAE,kBAAkB,GAAG,eAAe,GACjD,SAAS;IAYZ,SAAS,CAAC,SAAS,IAAI,IAAI;IAIrB,aAAa,CAAC,EAClB,eAAe,GAChB,EAAE,qBAAqB,GAAG,OAAO,CAAC,UAAU,CAAC;IAoHxC,gBAAgB,CAAC,EACrB,kBAAkB,GACnB,EAAE;QACD,kBAAkB,EAAE,MAAM,EAAE,CAAC;KAC9B;IA0BK,eAAe,CAAC,EACpB,MAAM,GACP,EAAE;QACD,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,UAAU,CAAC;IAqFjB,QAAQ,CAAC,EACb,IAAI,EACJ,cAAc,GACf,EAAE;QACD,IAAI,EAAE,kBAAkB,CAAC;QACzB,cAAc,EAAE,cAAc,CAAC;KAChC,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAwLvB,cAAc,CAAC,EACnB,IAAI,EACJ,MAAM,GACP,EAAE;QACD,IAAI,EAAE,wBAAwB,CAAC;QAC/B,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAiD9B"}
@@ -1,20 +1,10 @@
1
1
  import { FileToLoad, LoaderReport, StatsFileObject } from '../../types/loading';
2
- /**
3
- * Gets the files to load for the loader.
4
- * @param {string[]} supportedItemTypes - The supported item types
5
- * @param {StatsFileObject[]} statsFile - The stats file
6
- * @returns {FileToLoad[]} The files to load
7
- */
2
+ /** Filters the stats file to supported item types, ordered by their position in supportedItemTypes. */
8
3
  export declare function getFilesToLoad({ supportedItemTypes, statsFile, }: {
9
4
  supportedItemTypes: string[];
10
5
  statsFile: StatsFileObject[];
11
6
  }): FileToLoad[];
12
- /**
13
- * Adds a report to the loader report.
14
- * @param {LoaderReport[]} loaderReports - The loader reports
15
- * @param {LoaderReport} report - The report to add
16
- * @returns {LoaderReport[]} The updated loader reports
17
- */
7
+ /** Merges a report into the loader reports, summing counts per item type. */
18
8
  export declare function addReportToLoaderReport({ loaderReports, report, }: {
19
9
  loaderReports: LoaderReport[];
20
10
  report: LoaderReport;
@@ -1 +1 @@
1
- {"version":3,"file":"loading-adapter.helpers.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/loading-adapter.helpers.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,UAAU,EACV,YAAY,EACZ,eAAe,EAChB,MAAM,qBAAqB,CAAC;AAE7B;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,EAC7B,kBAAkB,EAClB,SAAS,GACV,EAAE;IACD,kBAAkB,EAAE,MAAM,EAAE,CAAC;IAC7B,SAAS,EAAE,eAAe,EAAE,CAAC;CAC9B,GAAG,UAAU,EAAE,CA8Bf;AAED;;;;;GAKG;AACH,wBAAgB,uBAAuB,CAAC,EACtC,aAAa,EACb,MAAM,GACP,EAAE;IACD,aAAa,EAAE,YAAY,EAAE,CAAC;IAC9B,MAAM,EAAE,YAAY,CAAC;CACtB,GAAG,YAAY,EAAE,CA4BjB"}
1
+ {"version":3,"file":"loading-adapter.helpers.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/loading-adapter.helpers.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,UAAU,EACV,YAAY,EACZ,eAAe,EAChB,MAAM,qBAAqB,CAAC;AAE7B,uGAAuG;AACvG,wBAAgB,cAAc,CAAC,EAC7B,kBAAkB,EAClB,SAAS,GACV,EAAE;IACD,kBAAkB,EAAE,MAAM,EAAE,CAAC;IAC7B,SAAS,EAAE,eAAe,EAAE,CAAC;CAC9B,GAAG,UAAU,EAAE,CA8Bf;AAED,6EAA6E;AAC7E,wBAAgB,uBAAuB,CAAC,EACtC,aAAa,EACb,MAAM,GACP,EAAE;IACD,aAAa,EAAE,YAAY,EAAE,CAAC;IAC9B,MAAM,EAAE,YAAY,CAAC;CACtB,GAAG,YAAY,EAAE,CA4BjB"}
@@ -3,12 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.getFilesToLoad = getFilesToLoad;
4
4
  exports.addReportToLoaderReport = addReportToLoaderReport;
5
5
  const loading_1 = require("../../types/loading");
6
- /**
7
- * Gets the files to load for the loader.
8
- * @param {string[]} supportedItemTypes - The supported item types
9
- * @param {StatsFileObject[]} statsFile - The stats file
10
- * @returns {FileToLoad[]} The files to load
11
- */
6
+ /** Filters the stats file to supported item types, ordered by their position in supportedItemTypes. */
12
7
  function getFilesToLoad({ supportedItemTypes, statsFile, }) {
13
8
  const filesToLoad = [];
14
9
  if (supportedItemTypes.length === 0 || statsFile.length === 0) {
@@ -32,12 +27,7 @@ function getFilesToLoad({ supportedItemTypes, statsFile, }) {
32
27
  }
33
28
  return filesToLoad;
34
29
  }
35
- /**
36
- * Adds a report to the loader report.
37
- * @param {LoaderReport[]} loaderReports - The loader reports
38
- * @param {LoaderReport} report - The report to add
39
- * @returns {LoaderReport[]} The updated loader reports
40
- */
30
+ /** Merges a report into the loader reports, summing counts per item type. */
41
31
  function addReportToLoaderReport({ loaderReports, report, }) {
42
32
  const existingReport = loaderReports.find((loaderReport) => loaderReport.item_type === report.item_type);
43
33
  if (existingReport) {