@devrev/airsync-sdk 2.0.0-beta.4 → 2.0.0-beta.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/MIGRATION.md +1028 -0
- package/dist/attachments-streaming/attachments-streaming-pool.d.ts +6 -0
- package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map +1 -1
- package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts +2 -2
- package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts.map +1 -1
- package/dist/attachments-streaming/attachments-streaming-pool.js +47 -21
- package/dist/common/constants.d.ts +3 -5
- package/dist/common/constants.d.ts.map +1 -1
- package/dist/common/constants.js +4 -6
- package/dist/common/helpers.d.ts +3 -21
- package/dist/common/helpers.d.ts.map +1 -1
- package/dist/common/helpers.js +6 -15
- package/dist/http/client.d.ts.map +1 -1
- package/dist/http/client.js +4 -8
- package/dist/index.d.ts +11 -11
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +11 -18
- package/dist/logger/logger.d.ts +8 -63
- package/dist/logger/logger.d.ts.map +1 -1
- package/dist/logger/logger.interfaces.d.ts +0 -1
- package/dist/logger/logger.interfaces.d.ts.map +1 -1
- package/dist/logger/logger.js +14 -93
- package/dist/mappers/mappers.d.ts +2 -40
- package/dist/mappers/mappers.d.ts.map +1 -1
- package/dist/mappers/mappers.interfaces.d.ts +35 -154
- package/dist/mappers/mappers.interfaces.d.ts.map +1 -1
- package/dist/mappers/mappers.interfaces.js +1 -10
- package/dist/mappers/mappers.js +2 -40
- package/dist/multithreading/adapters/base-adapter.d.ts +19 -36
- package/dist/multithreading/adapters/base-adapter.d.ts.map +1 -1
- package/dist/multithreading/adapters/base-adapter.js +73 -71
- package/dist/multithreading/adapters/extraction-adapter.d.ts +12 -24
- package/dist/multithreading/adapters/extraction-adapter.d.ts.map +1 -1
- package/dist/multithreading/adapters/extraction-adapter.js +199 -200
- package/dist/multithreading/adapters/loading-adapter.d.ts +1 -6
- package/dist/multithreading/adapters/loading-adapter.d.ts.map +1 -1
- package/dist/multithreading/adapters/loading-adapter.helpers.d.ts +2 -12
- package/dist/multithreading/adapters/loading-adapter.helpers.d.ts.map +1 -1
- package/dist/multithreading/adapters/loading-adapter.helpers.js +2 -12
- package/dist/multithreading/adapters/loading-adapter.js +292 -314
- package/dist/multithreading/emit.d.ts +3 -2
- package/dist/multithreading/emit.d.ts.map +1 -1
- package/dist/multithreading/emit.js +3 -5
- package/dist/multithreading/process-task.d.ts +2 -12
- package/dist/multithreading/process-task.d.ts.map +1 -1
- package/dist/multithreading/process-task.js +45 -59
- package/dist/multithreading/spawn/spawn.d.ts +3 -9
- package/dist/multithreading/spawn/spawn.d.ts.map +1 -1
- package/dist/multithreading/spawn/spawn.helpers.d.ts +6 -25
- package/dist/multithreading/spawn/spawn.helpers.d.ts.map +1 -1
- package/dist/multithreading/spawn/spawn.helpers.js +7 -49
- package/dist/multithreading/spawn/spawn.js +18 -33
- package/dist/multithreading/worker.js +2 -4
- package/dist/repo/repo.d.ts +10 -0
- package/dist/repo/repo.d.ts.map +1 -1
- package/dist/repo/repo.helpers.d.ts +6 -0
- package/dist/repo/repo.helpers.d.ts.map +1 -0
- package/dist/repo/repo.helpers.js +16 -0
- package/dist/repo/repo.interfaces.d.ts +4 -16
- package/dist/repo/repo.interfaces.d.ts.map +1 -1
- package/dist/repo/repo.js +23 -8
- package/dist/state/base-state.d.ts +14 -36
- package/dist/state/base-state.d.ts.map +1 -1
- package/dist/state/base-state.js +16 -38
- package/dist/state/extraction-state.d.ts +4 -19
- package/dist/state/extraction-state.d.ts.map +1 -1
- package/dist/state/extraction-state.js +6 -23
- package/dist/state/install-initial-domain-mapping.d.ts +1 -1
- package/dist/state/install-initial-domain-mapping.d.ts.map +1 -1
- package/dist/state/install-initial-domain-mapping.js +1 -3
- package/dist/state/loading-state.d.ts +1 -11
- package/dist/state/loading-state.d.ts.map +1 -1
- package/dist/state/loading-state.js +2 -12
- package/dist/state/state.d.ts +2 -2
- package/dist/state/state.d.ts.map +1 -1
- package/dist/state/state.interfaces.d.ts +10 -24
- package/dist/state/state.interfaces.d.ts.map +1 -1
- package/dist/state/state.interfaces.js +9 -11
- package/dist/state/state.js +3 -3
- package/dist/state/time-value-resolver.d.ts +7 -36
- package/dist/state/time-value-resolver.d.ts.map +1 -1
- package/dist/state/time-value-resolver.js +7 -40
- package/dist/testing/mock-event.d.ts +2 -10
- package/dist/testing/mock-event.d.ts.map +1 -1
- package/dist/testing/mock-event.js +3 -11
- package/dist/testing/mock-server.d.ts +4 -27
- package/dist/testing/mock-server.d.ts.map +1 -1
- package/dist/testing/mock-server.interfaces.d.ts +10 -35
- package/dist/testing/mock-server.interfaces.d.ts.map +1 -1
- package/dist/testing/mock-server.js +19 -36
- package/dist/types/common.d.ts +1 -12
- package/dist/types/common.d.ts.map +1 -1
- package/dist/types/common.js +1 -6
- package/dist/types/external-domain-metadata.d.ts +30 -171
- package/dist/types/external-domain-metadata.d.ts.map +1 -1
- package/dist/types/extraction.d.ts +35 -120
- package/dist/types/extraction.d.ts.map +1 -1
- package/dist/types/extraction.js +5 -31
- package/dist/types/loading.d.ts.map +1 -1
- package/dist/types/workers.d.ts +18 -116
- package/dist/types/workers.d.ts.map +1 -1
- package/dist/types/workers.js +0 -6
- package/dist/uploader/uploader.d.ts +6 -58
- package/dist/uploader/uploader.d.ts.map +1 -1
- package/dist/uploader/uploader.helpers.d.ts +4 -27
- package/dist/uploader/uploader.helpers.d.ts.map +1 -1
- package/dist/uploader/uploader.helpers.js +64 -30
- package/dist/uploader/uploader.interfaces.d.ts +15 -25
- package/dist/uploader/uploader.interfaces.d.ts.map +1 -1
- package/dist/uploader/uploader.interfaces.js +8 -0
- package/dist/uploader/uploader.js +18 -84
- package/package.json +6 -4
|
@@ -7,13 +7,7 @@ const logger_1 = require("../../logger/logger");
|
|
|
7
7
|
const repo_1 = require("../../repo/repo");
|
|
8
8
|
const extraction_1 = require("../../types/extraction");
|
|
9
9
|
const base_adapter_1 = require("./base-adapter");
|
|
10
|
-
/**
|
|
11
|
-
* ExtractionAdapter is the adapter passed to extraction tasks. It exposes the
|
|
12
|
-
* extraction surface (repos, artifacts, attachment streaming) and uploads
|
|
13
|
-
* pending repos and updates the extraction boundaries before emitting.
|
|
14
|
-
*
|
|
15
|
-
* @public
|
|
16
|
-
*/
|
|
10
|
+
/** Adapter passed to extraction tasks (repos, artifacts, attachment streaming). @public */
|
|
17
11
|
class ExtractionAdapter extends base_adapter_1.BaseAdapter {
|
|
18
12
|
constructor(params) {
|
|
19
13
|
super(params);
|
|
@@ -21,10 +15,7 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
|
|
|
21
15
|
this.currentEventDataLength = 0;
|
|
22
16
|
this._artifacts = [];
|
|
23
17
|
}
|
|
24
|
-
/**
|
|
25
|
-
* Returns whether the given item type should be extracted.
|
|
26
|
-
* Defaults to true if the scope is empty or the item type is not listed.
|
|
27
|
-
*/
|
|
18
|
+
/** Defaults to true if the scope is empty or the item type is not listed. */
|
|
28
19
|
shouldExtract(itemType) {
|
|
29
20
|
const scope = this.extractionScope;
|
|
30
21
|
if (Object.keys(scope).length === 0)
|
|
@@ -39,11 +30,13 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
|
|
|
39
30
|
repo.itemType !== constants_1.SSOR_ATTACHMENT;
|
|
40
31
|
return new repo_1.Repo(Object.assign(Object.assign({ event: this.event, itemType: repo.itemType }, (shouldNormalize && { normalize: repo.normalize })), { onUpload: (artifact) => {
|
|
41
32
|
var _a;
|
|
42
|
-
|
|
33
|
+
this.lastExtractedItemType = repo.itemType;
|
|
34
|
+
// Artifact ids are kept in state for later attachment streaming
|
|
43
35
|
if (repo.itemType === constants_1.AirSyncDefaultItemTypes.ATTACHMENTS) {
|
|
44
36
|
(_a = this.sdkState.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata.artifactIds.push(artifact.id);
|
|
45
37
|
}
|
|
46
|
-
//
|
|
38
|
+
// Track size of artifact objects that go in the SQS message; flip
|
|
39
|
+
// isTimeout once the threshold is exceeded to force an early emit.
|
|
47
40
|
this.currentEventDataLength += Buffer.byteLength(JSON.stringify(artifact), 'utf8');
|
|
48
41
|
if (this.currentEventDataLength > constants_1.EVENT_SIZE_THRESHOLD_BYTES &&
|
|
49
42
|
!this.isTimeout) {
|
|
@@ -53,14 +46,12 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
|
|
|
53
46
|
});
|
|
54
47
|
}
|
|
55
48
|
getRepo(itemType) {
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
return repo;
|
|
63
|
-
});
|
|
49
|
+
const repo = this.repos.find((repo) => repo.itemType === itemType);
|
|
50
|
+
if (!repo) {
|
|
51
|
+
console.error(`Repo for item type ${itemType} not found.`);
|
|
52
|
+
return;
|
|
53
|
+
}
|
|
54
|
+
return repo;
|
|
64
55
|
}
|
|
65
56
|
get artifacts() {
|
|
66
57
|
return this._artifacts;
|
|
@@ -71,18 +62,16 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
|
|
|
71
62
|
.filter((value, index, self) => self.indexOf(value) === index);
|
|
72
63
|
}
|
|
73
64
|
async beforeEmit(newEventType) {
|
|
74
|
-
// Upload all repos before emitting the event
|
|
75
65
|
console.log(`Uploading all repos before emitting event with event type: ${newEventType}.`);
|
|
76
66
|
await this.uploadAllRepos();
|
|
77
67
|
// When the full extraction cycle completes, commit the extraction window
|
|
78
68
|
// boundaries so subsequent incremental syncs can resume from them.
|
|
79
69
|
if (newEventType === extraction_1.ExtractorEventType.AttachmentExtractionDone) {
|
|
80
70
|
const sdkState = this.sdkState;
|
|
81
|
-
// Clear pending extraction boundaries now that the cycle is complete
|
|
82
71
|
sdkState.pendingWorkersOldest = '';
|
|
83
72
|
sdkState.pendingWorkersNewest = '';
|
|
84
|
-
//
|
|
85
|
-
//
|
|
73
|
+
// Expand boundaries: workersOldest keeps the earliest timestamp seen,
|
|
74
|
+
// workersNewest the latest.
|
|
86
75
|
const extractionStart = this.event.payload.event_context.extract_from;
|
|
87
76
|
const extractionEnd = this.event.payload.event_context.extract_to;
|
|
88
77
|
if (extractionStart &&
|
|
@@ -101,6 +90,37 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
|
|
|
101
90
|
const isExtractionEvent = Object.values(extraction_1.ExtractorEventType).includes(newEventType);
|
|
102
91
|
return isExtractionEvent ? { artifacts: this.artifacts } : {};
|
|
103
92
|
}
|
|
93
|
+
/**
|
|
94
|
+
* Attaches the last-extracted item type and its created/modified date ranges
|
|
95
|
+
* to data/attachment done/progress events, mirroring the metadata v1 sent.
|
|
96
|
+
*/
|
|
97
|
+
buildWorkerMetadata(newEventType) {
|
|
98
|
+
if (newEventType !== extraction_1.ExtractorEventType.DataExtractionDone &&
|
|
99
|
+
newEventType !== extraction_1.ExtractorEventType.DataExtractionProgress &&
|
|
100
|
+
newEventType !== extraction_1.ExtractorEventType.AttachmentExtractionDone &&
|
|
101
|
+
newEventType !== extraction_1.ExtractorEventType.AttachmentExtractionProgress) {
|
|
102
|
+
return undefined;
|
|
103
|
+
}
|
|
104
|
+
const repo = this.lastExtractedItemType
|
|
105
|
+
? this.repos.find((r) => r.itemType === this.lastExtractedItemType)
|
|
106
|
+
: undefined;
|
|
107
|
+
if (!repo) {
|
|
108
|
+
return undefined;
|
|
109
|
+
}
|
|
110
|
+
return {
|
|
111
|
+
item_type: repo.itemType,
|
|
112
|
+
newest_created_date: this.toRfc3339Timestamp(repo.dateRanges.creationDate.newest),
|
|
113
|
+
oldest_created_date: this.toRfc3339Timestamp(repo.dateRanges.creationDate.oldest),
|
|
114
|
+
newest_modified_date: this.toRfc3339Timestamp(repo.dateRanges.modifiedDate.newest),
|
|
115
|
+
oldest_modified_date: this.toRfc3339Timestamp(repo.dateRanges.modifiedDate.oldest),
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
toRfc3339Timestamp(ms) {
|
|
119
|
+
if (ms === undefined || !Number.isFinite(ms)) {
|
|
120
|
+
return undefined;
|
|
121
|
+
}
|
|
122
|
+
return new Date(ms).toISOString();
|
|
123
|
+
}
|
|
104
124
|
afterEmit() {
|
|
105
125
|
this.artifacts = [];
|
|
106
126
|
}
|
|
@@ -114,94 +134,89 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
|
|
|
114
134
|
}
|
|
115
135
|
}
|
|
116
136
|
async processAttachment(attachment, stream) {
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
137
|
+
var _a, _b, _c;
|
|
138
|
+
const { httpStream, delay, error } = await stream({
|
|
139
|
+
item: attachment,
|
|
140
|
+
event: this.event,
|
|
141
|
+
});
|
|
142
|
+
if (error) {
|
|
143
|
+
return { error };
|
|
144
|
+
}
|
|
145
|
+
else if (delay) {
|
|
146
|
+
return { delay };
|
|
147
|
+
}
|
|
148
|
+
if (httpStream) {
|
|
149
|
+
const fileType = attachment.content_type ||
|
|
150
|
+
((_a = httpStream.headers['content-type']) === null || _a === void 0 ? void 0 : _a.toString()) ||
|
|
151
|
+
'application/octet-stream';
|
|
152
|
+
const contentLength = (_b = httpStream.headers['content-length']) === null || _b === void 0 ? void 0 : _b.toString();
|
|
153
|
+
const fileSize = contentLength ? parseInt(contentLength) : undefined;
|
|
154
|
+
const { error: artifactUrlError, response: artifactUrlResponse } = await this.uploader.getArtifactUploadUrl(attachment.file_name, fileType, fileSize);
|
|
155
|
+
if (artifactUrlError) {
|
|
156
|
+
this.destroyHttpStream(httpStream);
|
|
157
|
+
return {
|
|
158
|
+
error: {
|
|
159
|
+
message: `Error while preparing artifact for attachment ID ${attachment.id}. Skipping attachment. ${(0, logger_1.serializeError)(artifactUrlError)}`,
|
|
160
|
+
fileSize: fileSize,
|
|
161
|
+
},
|
|
162
|
+
};
|
|
128
163
|
}
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
if (artifactUrlError) {
|
|
138
|
-
this.destroyHttpStream(httpStream);
|
|
139
|
-
return {
|
|
140
|
-
error: {
|
|
141
|
-
message: `Error while preparing artifact for attachment ID ${attachment.id}. Skipping attachment. ${(0, logger_1.serializeError)(artifactUrlError)}`,
|
|
142
|
-
fileSize: fileSize,
|
|
143
|
-
},
|
|
144
|
-
};
|
|
145
|
-
}
|
|
146
|
-
// Stream attachment
|
|
147
|
-
const { error: uploadedArtifactError } = await this.uploader.streamArtifact(artifactUrlResponse, httpStream);
|
|
148
|
-
if (uploadedArtifactError) {
|
|
149
|
-
this.destroyHttpStream(httpStream);
|
|
150
|
-
return {
|
|
151
|
-
error: {
|
|
152
|
-
message: `Error while streaming to artifact for attachment ID ${attachment.id}. Skipping attachment. ` +
|
|
153
|
-
(0, logger_1.serializeError)(uploadedArtifactError),
|
|
154
|
-
fileSize: fileSize,
|
|
155
|
-
},
|
|
156
|
-
};
|
|
157
|
-
}
|
|
158
|
-
// Confirm attachment upload
|
|
159
|
-
const { error: confirmArtifactUploadError } = await this.uploader.confirmArtifactUpload(artifactUrlResponse.artifact_id);
|
|
160
|
-
if (confirmArtifactUploadError) {
|
|
161
|
-
return {
|
|
162
|
-
error: {
|
|
163
|
-
message: `Error while confirming upload for attachment ID ${attachment.id}. ` +
|
|
164
|
-
(0, logger_1.serializeError)(confirmArtifactUploadError),
|
|
165
|
-
fileSize: fileSize,
|
|
166
|
-
},
|
|
167
|
-
};
|
|
168
|
-
}
|
|
169
|
-
const ssorAttachment = {
|
|
170
|
-
id: {
|
|
171
|
-
devrev: artifactUrlResponse.artifact_id,
|
|
172
|
-
external: attachment.id,
|
|
164
|
+
const { error: uploadedArtifactError } = await this.uploader.streamArtifact(artifactUrlResponse, httpStream);
|
|
165
|
+
if (uploadedArtifactError) {
|
|
166
|
+
this.destroyHttpStream(httpStream);
|
|
167
|
+
return {
|
|
168
|
+
error: {
|
|
169
|
+
message: `Error while streaming to artifact for attachment ID ${attachment.id}. Skipping attachment. ` +
|
|
170
|
+
(0, logger_1.serializeError)(uploadedArtifactError),
|
|
171
|
+
fileSize: fileSize,
|
|
173
172
|
},
|
|
174
|
-
|
|
175
|
-
|
|
173
|
+
};
|
|
174
|
+
}
|
|
175
|
+
const { error: confirmArtifactUploadError } = await this.uploader.confirmArtifactUpload(artifactUrlResponse.artifact_id);
|
|
176
|
+
if (confirmArtifactUploadError) {
|
|
177
|
+
return {
|
|
178
|
+
error: {
|
|
179
|
+
message: `Error while confirming upload for attachment ID ${attachment.id}. ` +
|
|
180
|
+
(0, logger_1.serializeError)(confirmArtifactUploadError),
|
|
181
|
+
fileSize: fileSize,
|
|
176
182
|
},
|
|
177
183
|
};
|
|
178
|
-
if (attachment.author_id) {
|
|
179
|
-
ssorAttachment.actor_id = {
|
|
180
|
-
external: attachment.author_id,
|
|
181
|
-
};
|
|
182
|
-
}
|
|
183
|
-
// This will set inline flag in ssor_attachment only if it is explicity
|
|
184
|
-
// set in the attachment object.
|
|
185
|
-
if (attachment.inline === true) {
|
|
186
|
-
ssorAttachment.inline = true;
|
|
187
|
-
}
|
|
188
|
-
else if (attachment.inline === false) {
|
|
189
|
-
ssorAttachment.inline = false;
|
|
190
|
-
}
|
|
191
|
-
await ((_c = this.getRepo('ssor_attachment')) === null || _c === void 0 ? void 0 : _c.push([ssorAttachment]));
|
|
192
|
-
return;
|
|
193
184
|
}
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
185
|
+
const ssorAttachment = {
|
|
186
|
+
id: {
|
|
187
|
+
devrev: artifactUrlResponse.artifact_id,
|
|
188
|
+
external: attachment.id,
|
|
189
|
+
},
|
|
190
|
+
parent_id: {
|
|
191
|
+
external: attachment.parent_id,
|
|
197
192
|
},
|
|
198
193
|
};
|
|
199
|
-
|
|
194
|
+
if (attachment.author_id) {
|
|
195
|
+
ssorAttachment.actor_id = {
|
|
196
|
+
external: attachment.author_id,
|
|
197
|
+
};
|
|
198
|
+
}
|
|
199
|
+
// Set inline flag only if it is explicitly set on the attachment.
|
|
200
|
+
if (attachment.inline === true) {
|
|
201
|
+
ssorAttachment.inline = true;
|
|
202
|
+
}
|
|
203
|
+
else if (attachment.inline === false) {
|
|
204
|
+
ssorAttachment.inline = false;
|
|
205
|
+
}
|
|
206
|
+
if (this.isTimeout) {
|
|
207
|
+
this.destroyHttpStream(httpStream);
|
|
208
|
+
return;
|
|
209
|
+
}
|
|
210
|
+
await ((_c = this.getRepo('ssor_attachment')) === null || _c === void 0 ? void 0 : _c.push([ssorAttachment]));
|
|
211
|
+
return;
|
|
212
|
+
}
|
|
213
|
+
return {
|
|
214
|
+
error: {
|
|
215
|
+
message: `Error while opening attachment stream. Skipping attachment.`,
|
|
216
|
+
},
|
|
217
|
+
};
|
|
200
218
|
}
|
|
201
|
-
/**
|
|
202
|
-
* Destroys a stream to prevent memory leaks.
|
|
203
|
-
* @param httpStream - The axios response stream to destroy
|
|
204
|
-
*/
|
|
219
|
+
/** Destroys a stream to prevent memory leaks. */
|
|
205
220
|
destroyHttpStream(httpStream) {
|
|
206
221
|
try {
|
|
207
222
|
if (httpStream && httpStream.data) {
|
|
@@ -217,107 +232,91 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
|
|
|
217
232
|
console.warn('Error while destroying HTTP stream:', error);
|
|
218
233
|
}
|
|
219
234
|
}
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
235
|
+
async streamAttachments({ stream, processors, batchSize = 1, }) {
|
|
236
|
+
var _a, _b;
|
|
237
|
+
if (batchSize <= 0) {
|
|
238
|
+
console.warn(`The specified batch size (${batchSize}) is invalid. Using 1 instead.`);
|
|
239
|
+
batchSize = 1;
|
|
240
|
+
}
|
|
241
|
+
if (batchSize > 50) {
|
|
242
|
+
console.warn(`The specified batch size (${batchSize}) is too large. Using 50 instead.`);
|
|
243
|
+
batchSize = 50;
|
|
244
|
+
}
|
|
245
|
+
const repos = [
|
|
246
|
+
{
|
|
247
|
+
itemType: 'ssor_attachment',
|
|
248
|
+
},
|
|
249
|
+
];
|
|
250
|
+
this.initializeRepos(repos);
|
|
251
|
+
const attachmentsMetadata = (_a = this.sdkState.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata;
|
|
252
|
+
if (!((_b = attachmentsMetadata === null || attachmentsMetadata === void 0 ? void 0 : attachmentsMetadata.artifactIds) === null || _b === void 0 ? void 0 : _b.length)) {
|
|
253
|
+
console.log(`No attachments metadata artifact IDs found in state.`);
|
|
254
|
+
return { status: 'success' };
|
|
255
|
+
}
|
|
256
|
+
else {
|
|
257
|
+
console.log(`Found ${attachmentsMetadata.artifactIds.length} attachments metadata artifact IDs in state.`);
|
|
258
|
+
}
|
|
259
|
+
while (attachmentsMetadata.artifactIds.length > 0) {
|
|
260
|
+
const attachmentsMetadataArtifactId = attachmentsMetadata.artifactIds[0];
|
|
261
|
+
console.log(`Started processing attachments for attachments metadata artifact ID: ${attachmentsMetadataArtifactId}.`);
|
|
262
|
+
const { attachments, error } = await this.uploader.getAttachmentsFromArtifactId({
|
|
263
|
+
artifact: attachmentsMetadataArtifactId,
|
|
264
|
+
});
|
|
265
|
+
if (error) {
|
|
266
|
+
console.error(`Failed to get attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
|
|
267
|
+
return { status: 'error', error };
|
|
234
268
|
}
|
|
235
|
-
if (
|
|
236
|
-
console.warn(`
|
|
237
|
-
|
|
269
|
+
if (!attachments || attachments.length === 0) {
|
|
270
|
+
console.warn(`No attachments found for artifact ID: ${attachmentsMetadataArtifactId}.`);
|
|
271
|
+
attachmentsMetadata.artifactIds.shift();
|
|
272
|
+
attachmentsMetadata.lastProcessed = 0;
|
|
273
|
+
continue;
|
|
238
274
|
}
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
275
|
+
console.log(`Found ${attachments.length} attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
|
|
276
|
+
let response;
|
|
277
|
+
if (processors) {
|
|
278
|
+
console.log(`Using custom processors for attachments.`);
|
|
279
|
+
const reducer = processors.reducer;
|
|
280
|
+
const iterator = processors.iterator;
|
|
281
|
+
const reducedAttachments = reducer({
|
|
282
|
+
attachments,
|
|
283
|
+
adapter: this,
|
|
284
|
+
batchSize,
|
|
285
|
+
});
|
|
286
|
+
response = await iterator({
|
|
287
|
+
reducedAttachments,
|
|
288
|
+
adapter: this,
|
|
289
|
+
stream,
|
|
290
|
+
});
|
|
250
291
|
}
|
|
251
292
|
else {
|
|
252
|
-
console.log(`
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
const { attachments, error } = await this.uploader.getAttachmentsFromArtifactId({
|
|
259
|
-
artifact: attachmentsMetadataArtifactId,
|
|
293
|
+
console.log(`Using attachments streaming pool for attachments streaming.`);
|
|
294
|
+
const attachmentsPool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
|
|
295
|
+
adapter: this,
|
|
296
|
+
attachments,
|
|
297
|
+
batchSize,
|
|
298
|
+
stream,
|
|
260
299
|
});
|
|
261
|
-
|
|
262
|
-
console.error(`Failed to get attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
|
|
263
|
-
return { status: 'error', error };
|
|
264
|
-
}
|
|
265
|
-
if (!attachments || attachments.length === 0) {
|
|
266
|
-
console.warn(`No attachments found for artifact ID: ${attachmentsMetadataArtifactId}.`);
|
|
267
|
-
// Remove empty artifact and reset lastProcessed
|
|
268
|
-
attachmentsMetadata.artifactIds.shift();
|
|
269
|
-
attachmentsMetadata.lastProcessed = 0;
|
|
270
|
-
continue;
|
|
271
|
-
}
|
|
272
|
-
console.log(`Found ${attachments.length} attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
|
|
273
|
-
let response;
|
|
274
|
-
if (processors) {
|
|
275
|
-
console.log(`Using custom processors for attachments.`);
|
|
276
|
-
const reducer = processors.reducer;
|
|
277
|
-
const iterator = processors.iterator;
|
|
278
|
-
const reducedAttachments = (0, logger_1.runWithUserLogContext)(() => reducer({
|
|
279
|
-
attachments,
|
|
280
|
-
adapter: this,
|
|
281
|
-
batchSize,
|
|
282
|
-
}));
|
|
283
|
-
response = await (0, logger_1.runWithUserLogContext)(async () => {
|
|
284
|
-
return await iterator({
|
|
285
|
-
reducedAttachments,
|
|
286
|
-
adapter: this,
|
|
287
|
-
stream,
|
|
288
|
-
});
|
|
289
|
-
});
|
|
290
|
-
}
|
|
291
|
-
else {
|
|
292
|
-
console.log(`Using attachments streaming pool for attachments streaming.`);
|
|
293
|
-
const attachmentsPool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
|
|
294
|
-
adapter: this,
|
|
295
|
-
attachments,
|
|
296
|
-
batchSize,
|
|
297
|
-
stream,
|
|
298
|
-
});
|
|
299
|
-
response = await attachmentsPool.streamAll();
|
|
300
|
-
}
|
|
301
|
-
if (response === null || response === void 0 ? void 0 : response.error) {
|
|
302
|
-
return { status: 'error', error: response.error };
|
|
303
|
-
}
|
|
304
|
-
if (response === null || response === void 0 ? void 0 : response.delay) {
|
|
305
|
-
return { status: 'delay', delaySeconds: response.delay };
|
|
306
|
-
}
|
|
307
|
-
// On timeout, return progress to allow continuation in a fresh invocation.
|
|
308
|
-
if (this.isTimeout) {
|
|
309
|
-
console.log(`Timeout detected after processing attachments for artifact ID: ${attachmentsMetadataArtifactId}. Returning progress to allow continuation.`);
|
|
310
|
-
return { status: 'progress' };
|
|
311
|
-
}
|
|
312
|
-
console.log(`Finished processing all attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
|
|
313
|
-
attachmentsMetadata.artifactIds.shift();
|
|
314
|
-
attachmentsMetadata.lastProcessed = 0;
|
|
315
|
-
if (attachmentsMetadata.lastProcessedAttachmentsIdsList) {
|
|
316
|
-
attachmentsMetadata.lastProcessedAttachmentsIdsList.length = 0;
|
|
317
|
-
}
|
|
300
|
+
response = await attachmentsPool.streamAll();
|
|
318
301
|
}
|
|
319
|
-
|
|
320
|
-
|
|
302
|
+
if (response === null || response === void 0 ? void 0 : response.error) {
|
|
303
|
+
return { status: 'error', error: response.error };
|
|
304
|
+
}
|
|
305
|
+
if (response === null || response === void 0 ? void 0 : response.delay) {
|
|
306
|
+
return { status: 'delay', delaySeconds: response.delay };
|
|
307
|
+
}
|
|
308
|
+
if (this.isTimeout) {
|
|
309
|
+
console.log(`Timeout detected after processing attachments for artifact ID: ${attachmentsMetadataArtifactId}. Returning progress to allow continuation.`);
|
|
310
|
+
return { status: 'progress' };
|
|
311
|
+
}
|
|
312
|
+
console.log(`Finished processing all attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
|
|
313
|
+
attachmentsMetadata.artifactIds.shift();
|
|
314
|
+
attachmentsMetadata.lastProcessed = 0;
|
|
315
|
+
if (attachmentsMetadata.lastProcessedAttachmentsIdsList) {
|
|
316
|
+
attachmentsMetadata.lastProcessedAttachmentsIdsList.length = 0;
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
return { status: 'success' };
|
|
321
320
|
}
|
|
322
321
|
}
|
|
323
322
|
exports.ExtractionAdapter = ExtractionAdapter;
|
|
@@ -4,12 +4,7 @@ import { AirSyncEvent, EventData, ExtractorEventType } from '../../types/extract
|
|
|
4
4
|
import { ExternalSystemAttachment, ExternalSystemItem, ExternalSystemLoadingFunction, FileToLoad, ItemTypesToLoadParams, ItemTypeToLoad, LoaderEventType, LoaderReport, LoadItemResponse } from '../../types/loading';
|
|
5
5
|
import { TaskResult, WorkerAdapterOptions } from '../../types/workers';
|
|
6
6
|
import { BaseAdapter } from './base-adapter';
|
|
7
|
-
/**
|
|
8
|
-
* LoadingAdapter is the adapter passed to loading tasks. It exposes the loading
|
|
9
|
-
* surface (item/attachment loading, mappers, loader reports).
|
|
10
|
-
*
|
|
11
|
-
* @public
|
|
12
|
-
*/
|
|
7
|
+
/** Adapter passed to loading tasks (item/attachment loading, mappers, loader reports). @public */
|
|
13
8
|
export declare class LoadingAdapter<ConnectorState> extends BaseAdapter<ConnectorState> {
|
|
14
9
|
private loaderReports;
|
|
15
10
|
private _processedFiles;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"loading-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/loading-adapter.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"loading-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/loading-adapter.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAEhD,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EACL,YAAY,EACZ,SAAS,EAET,kBAAkB,EACnB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAEL,wBAAwB,EACxB,kBAAkB,EAClB,6BAA6B,EAC7B,UAAU,EACV,qBAAqB,EACrB,cAAc,EACd,eAAe,EACf,YAAY,EACZ,gBAAgB,EAEjB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,UAAU,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAEvE,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAM7C,kGAAkG;AAClG,qBAAa,cAAc,CACzB,cAAc,CACd,SAAQ,WAAW,CAAC,cAAc,CAAC;IACnC,OAAO,CAAC,aAAa,CAAiB;IACtC,OAAO,CAAC,eAAe,CAAW;IAClC,OAAO,CAAC,QAAQ,CAAU;gBAEd,MAAM,EAAE;QAClB,KAAK,EAAE,YAAY,CAAC;QACpB,YAAY,EAAE,SAAS,CAAC,cAAc,CAAC,CAAC;QACxC,OAAO,CAAC,EAAE,oBAAoB,CAAC;KAChC;IAUD,IAAI,OAAO,IAAI,YAAY,EAAE,CAE5B;IAED,IAAI,cAAc,IAAI,MAAM,EAAE,CAE7B;IAED,IAAI,OAAO,IAAI,OAAO,CAErB;cAEe,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAI3C,SAAS,CAAC,gBAAgB,CACxB,YAAY,EAAE,kBAAkB,GAAG,eAAe,GACjD,SAAS;IAYZ,SAAS,CAAC,SAAS,IAAI,IAAI;IAIrB,aAAa,CAAC,EAClB,eAAe,GAChB,EAAE,qBAAqB,GAAG,OAAO,CAAC,UAAU,CAAC;IAoHxC,gBAAgB,CAAC,EACrB,kBAAkB,GACnB,EAAE;QACD,kBAAkB,EAAE,MAAM,EAAE,CAAC;KAC9B;IA0BK,eAAe,CAAC,EACpB,MAAM,GACP,EAAE;QACD,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,UAAU,CAAC;IAqFjB,QAAQ,CAAC,EACb,IAAI,EACJ,cAAc,GACf,EAAE;QACD,IAAI,EAAE,kBAAkB,CAAC;QACzB,cAAc,EAAE,cAAc,CAAC;KAChC,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAwLvB,cAAc,CAAC,EACnB,IAAI,EACJ,MAAM,GACP,EAAE;QACD,IAAI,EAAE,wBAAwB,CAAC;QAC/B,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAiD9B"}
|
|
@@ -1,20 +1,10 @@
|
|
|
1
1
|
import { FileToLoad, LoaderReport, StatsFileObject } from '../../types/loading';
|
|
2
|
-
/**
|
|
3
|
-
* Gets the files to load for the loader.
|
|
4
|
-
* @param {string[]} supportedItemTypes - The supported item types
|
|
5
|
-
* @param {StatsFileObject[]} statsFile - The stats file
|
|
6
|
-
* @returns {FileToLoad[]} The files to load
|
|
7
|
-
*/
|
|
2
|
+
/** Filters the stats file to supported item types, ordered by their position in supportedItemTypes. */
|
|
8
3
|
export declare function getFilesToLoad({ supportedItemTypes, statsFile, }: {
|
|
9
4
|
supportedItemTypes: string[];
|
|
10
5
|
statsFile: StatsFileObject[];
|
|
11
6
|
}): FileToLoad[];
|
|
12
|
-
/**
|
|
13
|
-
* Adds a report to the loader report.
|
|
14
|
-
* @param {LoaderReport[]} loaderReports - The loader reports
|
|
15
|
-
* @param {LoaderReport} report - The report to add
|
|
16
|
-
* @returns {LoaderReport[]} The updated loader reports
|
|
17
|
-
*/
|
|
7
|
+
/** Merges a report into the loader reports, summing counts per item type. */
|
|
18
8
|
export declare function addReportToLoaderReport({ loaderReports, report, }: {
|
|
19
9
|
loaderReports: LoaderReport[];
|
|
20
10
|
report: LoaderReport;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"loading-adapter.helpers.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/loading-adapter.helpers.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,UAAU,EACV,YAAY,EACZ,eAAe,EAChB,MAAM,qBAAqB,CAAC;AAE7B
|
|
1
|
+
{"version":3,"file":"loading-adapter.helpers.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/loading-adapter.helpers.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,UAAU,EACV,YAAY,EACZ,eAAe,EAChB,MAAM,qBAAqB,CAAC;AAE7B,uGAAuG;AACvG,wBAAgB,cAAc,CAAC,EAC7B,kBAAkB,EAClB,SAAS,GACV,EAAE;IACD,kBAAkB,EAAE,MAAM,EAAE,CAAC;IAC7B,SAAS,EAAE,eAAe,EAAE,CAAC;CAC9B,GAAG,UAAU,EAAE,CA8Bf;AAED,6EAA6E;AAC7E,wBAAgB,uBAAuB,CAAC,EACtC,aAAa,EACb,MAAM,GACP,EAAE;IACD,aAAa,EAAE,YAAY,EAAE,CAAC;IAC9B,MAAM,EAAE,YAAY,CAAC;CACtB,GAAG,YAAY,EAAE,CA4BjB"}
|
|
@@ -3,12 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.getFilesToLoad = getFilesToLoad;
|
|
4
4
|
exports.addReportToLoaderReport = addReportToLoaderReport;
|
|
5
5
|
const loading_1 = require("../../types/loading");
|
|
6
|
-
/**
|
|
7
|
-
* Gets the files to load for the loader.
|
|
8
|
-
* @param {string[]} supportedItemTypes - The supported item types
|
|
9
|
-
* @param {StatsFileObject[]} statsFile - The stats file
|
|
10
|
-
* @returns {FileToLoad[]} The files to load
|
|
11
|
-
*/
|
|
6
|
+
/** Filters the stats file to supported item types, ordered by their position in supportedItemTypes. */
|
|
12
7
|
function getFilesToLoad({ supportedItemTypes, statsFile, }) {
|
|
13
8
|
const filesToLoad = [];
|
|
14
9
|
if (supportedItemTypes.length === 0 || statsFile.length === 0) {
|
|
@@ -32,12 +27,7 @@ function getFilesToLoad({ supportedItemTypes, statsFile, }) {
|
|
|
32
27
|
}
|
|
33
28
|
return filesToLoad;
|
|
34
29
|
}
|
|
35
|
-
/**
|
|
36
|
-
* Adds a report to the loader report.
|
|
37
|
-
* @param {LoaderReport[]} loaderReports - The loader reports
|
|
38
|
-
* @param {LoaderReport} report - The report to add
|
|
39
|
-
* @returns {LoaderReport[]} The updated loader reports
|
|
40
|
-
*/
|
|
30
|
+
/** Merges a report into the loader reports, summing counts per item type. */
|
|
41
31
|
function addReportToLoaderReport({ loaderReports, report, }) {
|
|
42
32
|
const existingReport = loaderReports.find((loaderReport) => loaderReport.item_type === report.item_type);
|
|
43
33
|
if (existingReport) {
|