@devrev/airsync-sdk 2.0.0-beta.5 → 2.0.0-beta.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -12
- package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map +1 -1
- package/dist/attachments-streaming/attachments-streaming-pool.js +8 -9
- package/dist/common/constants.d.ts +0 -7
- package/dist/common/constants.d.ts.map +1 -1
- package/dist/common/constants.js +1 -28
- package/dist/common/helpers.d.ts.map +1 -1
- package/dist/common/helpers.js +1 -1
- package/dist/http/errors.d.ts +8 -0
- package/dist/http/errors.d.ts.map +1 -0
- package/dist/http/errors.js +31 -0
- package/dist/index.d.ts +18 -13
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +14 -11
- package/dist/logger/logger.d.ts.map +1 -1
- package/dist/logger/logger.interfaces.d.ts +2 -2
- package/dist/logger/logger.interfaces.d.ts.map +1 -1
- package/dist/mappers/mappers.d.ts +9 -3
- package/dist/mappers/mappers.d.ts.map +1 -1
- package/dist/mappers/mappers.interfaces.d.ts +0 -2
- package/dist/mappers/mappers.interfaces.d.ts.map +1 -1
- package/dist/mappers/mappers.js +33 -13
- package/dist/multithreading/adapters/base-adapter.d.ts +16 -35
- package/dist/multithreading/adapters/base-adapter.d.ts.map +1 -1
- package/dist/multithreading/adapters/base-adapter.js +89 -23
- package/dist/multithreading/adapters/extraction-adapter.d.ts +24 -20
- package/dist/multithreading/adapters/extraction-adapter.d.ts.map +1 -1
- package/dist/multithreading/adapters/extraction-adapter.js +175 -122
- package/dist/multithreading/adapters/loading-adapter.d.ts +13 -23
- package/dist/multithreading/adapters/loading-adapter.d.ts.map +1 -1
- package/dist/multithreading/adapters/loading-adapter.js +176 -301
- package/dist/multithreading/emit.d.ts.map +1 -1
- package/dist/multithreading/process-task.d.ts.map +1 -1
- package/dist/multithreading/process-task.js +24 -6
- package/dist/multithreading/spawn/spawn.d.ts +6 -1
- package/dist/multithreading/spawn/spawn.d.ts.map +1 -1
- package/dist/multithreading/spawn/spawn.helpers.d.ts.map +1 -1
- package/dist/multithreading/spawn/spawn.helpers.js +17 -52
- package/dist/multithreading/spawn/spawn.js +30 -26
- package/dist/multithreading/worker.js +15 -1
- package/dist/repo/repo.d.ts +4 -5
- package/dist/repo/repo.d.ts.map +1 -1
- package/dist/repo/repo.interfaces.d.ts +5 -4
- package/dist/repo/repo.interfaces.d.ts.map +1 -1
- package/dist/repo/repo.js +13 -15
- package/dist/state/base-state.d.ts +10 -5
- package/dist/state/base-state.d.ts.map +1 -1
- package/dist/state/base-state.js +39 -36
- package/dist/state/extraction-state.d.ts.map +1 -1
- package/dist/state/extraction-state.js +39 -4
- package/dist/state/state.interfaces.d.ts +13 -7
- package/dist/state/state.interfaces.d.ts.map +1 -1
- package/dist/state/state.interfaces.js +6 -8
- package/dist/state/time-value-resolver.d.ts.map +1 -1
- package/dist/state/time-value-resolver.js +3 -2
- package/dist/testing/mock-event.d.ts.map +1 -1
- package/dist/testing/mock-event.js +0 -8
- package/dist/testing/mock-server.d.ts +2 -0
- package/dist/testing/mock-server.d.ts.map +1 -1
- package/dist/testing/mock-server.interfaces.d.ts +6 -0
- package/dist/testing/mock-server.interfaces.d.ts.map +1 -1
- package/dist/testing/mock-server.js +37 -6
- package/dist/testing/test-adapters.d.ts +22 -0
- package/dist/testing/test-adapters.d.ts.map +1 -0
- package/dist/testing/test-adapters.js +32 -0
- package/dist/types/common.d.ts +4 -1
- package/dist/types/common.d.ts.map +1 -1
- package/dist/types/common.js +1 -1
- package/dist/types/errors.d.ts +13 -7
- package/dist/types/errors.d.ts.map +1 -1
- package/dist/types/errors.js +24 -0
- package/dist/types/extraction.d.ts +60 -44
- package/dist/types/extraction.d.ts.map +1 -1
- package/dist/types/extraction.js +10 -1
- package/dist/types/loading.d.ts +11 -30
- package/dist/types/loading.d.ts.map +1 -1
- package/dist/types/workers.d.ts +27 -23
- package/dist/types/workers.d.ts.map +1 -1
- package/dist/uploader/uploader.d.ts +1 -2
- package/dist/uploader/uploader.d.ts.map +1 -1
- package/dist/uploader/uploader.helpers.d.ts +3 -0
- package/dist/uploader/uploader.helpers.d.ts.map +1 -1
- package/dist/uploader/uploader.helpers.js +16 -0
- package/dist/uploader/uploader.interfaces.d.ts +4 -29
- package/dist/uploader/uploader.interfaces.d.ts.map +1 -1
- package/dist/uploader/uploader.js +4 -23
- package/docs/MIGRATION.md +651 -0
- package/package.json +24 -10
- package/MIGRATION.md +0 -1028
|
@@ -6,9 +6,11 @@ const constants_1 = require("../../common/constants");
|
|
|
6
6
|
const logger_1 = require("../../logger/logger");
|
|
7
7
|
const repo_1 = require("../../repo/repo");
|
|
8
8
|
const extraction_1 = require("../../types/extraction");
|
|
9
|
+
const uploader_helpers_1 = require("../../uploader/uploader.helpers");
|
|
9
10
|
const base_adapter_1 = require("./base-adapter");
|
|
10
11
|
/** Adapter passed to extraction tasks (repos, artifacts, attachment streaming). @public */
|
|
11
12
|
class ExtractionAdapter extends base_adapter_1.BaseAdapter {
|
|
13
|
+
/** @internal */
|
|
12
14
|
constructor(params) {
|
|
13
15
|
super(params);
|
|
14
16
|
this.repos = [];
|
|
@@ -25,25 +27,60 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
|
|
|
25
27
|
return scope[itemType].extract;
|
|
26
28
|
}
|
|
27
29
|
initializeRepos(repos) {
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
30
|
+
const existingSsorRepo = this.repos.find((repo) => repo.itemType === constants_1.SSOR_ATTACHMENT);
|
|
31
|
+
this.repos = repos.map((repo) => this.createRepo(repo));
|
|
32
|
+
if (existingSsorRepo &&
|
|
33
|
+
!this.repos.some((repo) => repo.itemType === constants_1.SSOR_ATTACHMENT)) {
|
|
34
|
+
this.repos.push(existingSsorRepo);
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
createRepo(repo) {
|
|
38
|
+
const shouldNormalize = repo.itemType !== constants_1.AirSyncDefaultItemTypes.EXTERNAL_DOMAIN_METADATA &&
|
|
39
|
+
repo.itemType !== constants_1.SSOR_ATTACHMENT;
|
|
40
|
+
const isDeferred = this.isDeferredRepo(repo.itemType);
|
|
41
|
+
return new repo_1.Repo(Object.assign(Object.assign({ event: this.event, itemType: repo.itemType }, (shouldNormalize && { normalize: repo.normalize })), { skipConfirmation: repo.itemType === constants_1.AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS, onUpload: (artifact) => {
|
|
42
|
+
var _a, _b;
|
|
43
|
+
if (isDeferred) {
|
|
44
|
+
const toDevRev = this.adapterState.sdkState.toDevRev;
|
|
45
|
+
if (toDevRev) {
|
|
46
|
+
(_a = toDevRev.deferredSsorArtifacts) !== null && _a !== void 0 ? _a : (toDevRev.deferredSsorArtifacts = []);
|
|
47
|
+
toDevRev.deferredSsorArtifacts.push(artifact);
|
|
44
48
|
}
|
|
45
|
-
|
|
46
|
-
|
|
49
|
+
return;
|
|
50
|
+
}
|
|
51
|
+
this.lastExtractedItemType = repo.itemType;
|
|
52
|
+
// Artifact ids are kept in state for later attachment streaming
|
|
53
|
+
if (repo.itemType === constants_1.AirSyncDefaultItemTypes.ATTACHMENTS) {
|
|
54
|
+
(_b = this.adapterState.sdkState.toDevRev) === null || _b === void 0 ? void 0 : _b.attachmentsMetadata.artifactIds.push(artifact.id);
|
|
55
|
+
}
|
|
56
|
+
// Track size of artifact objects that go in the SQS message; flip
|
|
57
|
+
// isTimeout once the threshold is exceeded to force an early emit.
|
|
58
|
+
this.currentEventDataLength += Buffer.byteLength(JSON.stringify(artifact), 'utf8');
|
|
59
|
+
if (this.currentEventDataLength > constants_1.EVENT_SIZE_THRESHOLD_BYTES &&
|
|
60
|
+
!this.isTimeout) {
|
|
61
|
+
this.isTimeout = true;
|
|
62
|
+
}
|
|
63
|
+
}, options: Object.assign(Object.assign({}, this.options), repo.overriddenOptions) }));
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* SSOR attachments created outside the attachments phase are held in state
|
|
67
|
+
* and reported with ATTACHMENT_EXTRACTION_DONE instead of the current event.
|
|
68
|
+
*/
|
|
69
|
+
isDeferredRepo(itemType) {
|
|
70
|
+
return itemType === constants_1.SSOR_ATTACHMENT && !this.isAttachmentsPhase();
|
|
71
|
+
}
|
|
72
|
+
isAttachmentsPhase() {
|
|
73
|
+
const eventType = this.event.payload.event_type;
|
|
74
|
+
return (eventType === extraction_1.EventType.StartExtractingAttachments ||
|
|
75
|
+
eventType === extraction_1.EventType.ContinueExtractingAttachments);
|
|
76
|
+
}
|
|
77
|
+
getOrCreateSsorRepo() {
|
|
78
|
+
let repo = this.repos.find((repo) => repo.itemType === constants_1.SSOR_ATTACHMENT);
|
|
79
|
+
if (!repo) {
|
|
80
|
+
repo = this.createRepo({ itemType: constants_1.SSOR_ATTACHMENT });
|
|
81
|
+
this.repos.push(repo);
|
|
82
|
+
}
|
|
83
|
+
return repo;
|
|
47
84
|
}
|
|
48
85
|
getRepo(itemType) {
|
|
49
86
|
const repo = this.repos.find((repo) => repo.itemType === itemType);
|
|
@@ -53,21 +90,43 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
|
|
|
53
90
|
}
|
|
54
91
|
return repo;
|
|
55
92
|
}
|
|
93
|
+
/** Artifacts uploaded so far that the next emit will report. */
|
|
56
94
|
get artifacts() {
|
|
57
95
|
return this._artifacts;
|
|
58
96
|
}
|
|
59
|
-
|
|
97
|
+
addArtifacts(artifacts) {
|
|
60
98
|
this._artifacts = this._artifacts
|
|
61
99
|
.concat(artifacts)
|
|
62
100
|
.filter((value, index, self) => self.indexOf(value) === index);
|
|
63
101
|
}
|
|
102
|
+
/**
|
|
103
|
+
* Per-record-type counts for sync-duration estimation. Set during the
|
|
104
|
+
* metadata phase; the SDK attaches them to the metadata-done event.
|
|
105
|
+
*/
|
|
106
|
+
get preExtractionItemCounts() {
|
|
107
|
+
return this._preExtractionItemCounts;
|
|
108
|
+
}
|
|
109
|
+
set preExtractionItemCounts(counts) {
|
|
110
|
+
this._preExtractionItemCounts = counts;
|
|
111
|
+
}
|
|
112
|
+
/** @internal */
|
|
64
113
|
async beforeEmit(newEventType) {
|
|
114
|
+
var _a;
|
|
65
115
|
console.log(`Uploading all repos before emitting event with event type: ${newEventType}.`);
|
|
66
116
|
await this.uploadAllRepos();
|
|
67
117
|
// When the full extraction cycle completes, commit the extraction window
|
|
68
118
|
// boundaries so subsequent incremental syncs can resume from them.
|
|
69
119
|
if (newEventType === extraction_1.ExtractorEventType.AttachmentExtractionDone) {
|
|
70
|
-
const sdkState = this.sdkState;
|
|
120
|
+
const sdkState = this.adapterState.sdkState;
|
|
121
|
+
// Released before the state save so they're never reported twice.
|
|
122
|
+
const toDevRev = sdkState.toDevRev;
|
|
123
|
+
if ((_a = toDevRev === null || toDevRev === void 0 ? void 0 : toDevRev.deferredSsorArtifacts) === null || _a === void 0 ? void 0 : _a.length) {
|
|
124
|
+
this.addArtifacts(toDevRev.deferredSsorArtifacts);
|
|
125
|
+
toDevRev.deferredSsorArtifacts = [];
|
|
126
|
+
}
|
|
127
|
+
console.log(`Overwriting lastSuccessfulSyncStarted with lastSyncStarted (${sdkState.lastSyncStarted}).`);
|
|
128
|
+
sdkState.lastSuccessfulSyncStarted = sdkState.lastSyncStarted;
|
|
129
|
+
sdkState.lastSyncStarted = '';
|
|
71
130
|
sdkState.pendingWorkersOldest = '';
|
|
72
131
|
sdkState.pendingWorkersNewest = '';
|
|
73
132
|
// Expand boundaries: workersOldest keeps the earliest timestamp seen,
|
|
@@ -86,11 +145,21 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
|
|
|
86
145
|
}
|
|
87
146
|
}
|
|
88
147
|
}
|
|
148
|
+
/** @internal */
|
|
89
149
|
buildEmitPayload(newEventType) {
|
|
90
150
|
const isExtractionEvent = Object.values(extraction_1.ExtractorEventType).includes(newEventType);
|
|
91
|
-
|
|
151
|
+
if (!isExtractionEvent) {
|
|
152
|
+
return {};
|
|
153
|
+
}
|
|
154
|
+
const payload = { artifacts: [...this._artifacts] };
|
|
155
|
+
if (newEventType === extraction_1.ExtractorEventType.MetadataExtractionDone &&
|
|
156
|
+
this._preExtractionItemCounts) {
|
|
157
|
+
payload.pre_extraction_item_counts = this._preExtractionItemCounts;
|
|
158
|
+
}
|
|
159
|
+
return payload;
|
|
92
160
|
}
|
|
93
161
|
/**
|
|
162
|
+
* @internal
|
|
94
163
|
* Attaches the last-extracted item type and its created/modified date ranges
|
|
95
164
|
* to data/attachment done/progress events, mirroring the metadata v1 sent.
|
|
96
165
|
*/
|
|
@@ -121,116 +190,108 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
|
|
|
121
190
|
}
|
|
122
191
|
return new Date(ms).toISOString();
|
|
123
192
|
}
|
|
193
|
+
/** @internal */
|
|
124
194
|
afterEmit() {
|
|
125
|
-
this.
|
|
195
|
+
this._artifacts = [];
|
|
126
196
|
}
|
|
127
197
|
async uploadAllRepos() {
|
|
128
198
|
for (const repo of this.repos) {
|
|
129
|
-
|
|
130
|
-
this.
|
|
131
|
-
|
|
132
|
-
throw error;
|
|
199
|
+
await repo.upload();
|
|
200
|
+
if (!this.isDeferredRepo(repo.itemType)) {
|
|
201
|
+
this.addArtifacts(repo.uploadedArtifacts);
|
|
133
202
|
}
|
|
134
203
|
}
|
|
135
204
|
}
|
|
205
|
+
/** @internal */
|
|
206
|
+
beforeErrorEmit() {
|
|
207
|
+
// Report artifacts that uploaded before the failure so they aren't orphaned.
|
|
208
|
+
for (const repo of this.repos) {
|
|
209
|
+
if (!this.isDeferredRepo(repo.itemType)) {
|
|
210
|
+
this.addArtifacts(repo.uploadedArtifacts);
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
/**
|
|
215
|
+
* Streams one attachment into DevRev. Usable in any extraction phase; SSOR
|
|
216
|
+
* records created outside the attachments phase are reported with
|
|
217
|
+
* ATTACHMENT_EXTRACTION_DONE.
|
|
218
|
+
*/
|
|
136
219
|
async processAttachment(attachment, stream) {
|
|
137
|
-
var _a, _b
|
|
138
|
-
const
|
|
220
|
+
var _a, _b;
|
|
221
|
+
const response = await stream({
|
|
139
222
|
item: attachment,
|
|
140
223
|
event: this.event,
|
|
141
224
|
});
|
|
142
|
-
if (
|
|
143
|
-
return
|
|
225
|
+
if (response.status !== 'success') {
|
|
226
|
+
return response;
|
|
144
227
|
}
|
|
145
|
-
|
|
146
|
-
|
|
228
|
+
const { httpStream } = response;
|
|
229
|
+
const fileType = attachment.content_type ||
|
|
230
|
+
((_a = httpStream.headers['content-type']) === null || _a === void 0 ? void 0 : _a.toString()) ||
|
|
231
|
+
'application/octet-stream';
|
|
232
|
+
const contentLength = (_b = httpStream.headers['content-length']) === null || _b === void 0 ? void 0 : _b.toString();
|
|
233
|
+
const fileSize = contentLength ? parseInt(contentLength) : undefined;
|
|
234
|
+
const { error: artifactUrlError, response: artifactUrlResponse } = await this.uploader.getArtifactUploadUrl(attachment.file_name, fileType, fileSize);
|
|
235
|
+
if (artifactUrlError) {
|
|
236
|
+
(0, uploader_helpers_1.destroyHttpStream)(httpStream);
|
|
237
|
+
return {
|
|
238
|
+
status: 'error',
|
|
239
|
+
error: {
|
|
240
|
+
message: `Error while preparing artifact for attachment ID ${attachment.id}. Skipping attachment. ${(0, logger_1.serializeError)(artifactUrlError)}`,
|
|
241
|
+
fileSize,
|
|
242
|
+
},
|
|
243
|
+
};
|
|
147
244
|
}
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
return {
|
|
158
|
-
error: {
|
|
159
|
-
message: `Error while preparing artifact for attachment ID ${attachment.id}. Skipping attachment. ${(0, logger_1.serializeError)(artifactUrlError)}`,
|
|
160
|
-
fileSize: fileSize,
|
|
161
|
-
},
|
|
162
|
-
};
|
|
163
|
-
}
|
|
164
|
-
const { error: uploadedArtifactError } = await this.uploader.streamArtifact(artifactUrlResponse, httpStream);
|
|
165
|
-
if (uploadedArtifactError) {
|
|
166
|
-
this.destroyHttpStream(httpStream);
|
|
167
|
-
return {
|
|
168
|
-
error: {
|
|
169
|
-
message: `Error while streaming to artifact for attachment ID ${attachment.id}. Skipping attachment. ` +
|
|
170
|
-
(0, logger_1.serializeError)(uploadedArtifactError),
|
|
171
|
-
fileSize: fileSize,
|
|
172
|
-
},
|
|
173
|
-
};
|
|
174
|
-
}
|
|
175
|
-
const { error: confirmArtifactUploadError } = await this.uploader.confirmArtifactUpload(artifactUrlResponse.artifact_id);
|
|
176
|
-
if (confirmArtifactUploadError) {
|
|
177
|
-
return {
|
|
178
|
-
error: {
|
|
179
|
-
message: `Error while confirming upload for attachment ID ${attachment.id}. ` +
|
|
180
|
-
(0, logger_1.serializeError)(confirmArtifactUploadError),
|
|
181
|
-
fileSize: fileSize,
|
|
182
|
-
},
|
|
183
|
-
};
|
|
184
|
-
}
|
|
185
|
-
const ssorAttachment = {
|
|
186
|
-
id: {
|
|
187
|
-
devrev: artifactUrlResponse.artifact_id,
|
|
188
|
-
external: attachment.id,
|
|
245
|
+
const { error: uploadedArtifactError } = await this.uploader.streamArtifact(artifactUrlResponse, httpStream);
|
|
246
|
+
if (uploadedArtifactError) {
|
|
247
|
+
(0, uploader_helpers_1.destroyHttpStream)(httpStream);
|
|
248
|
+
return {
|
|
249
|
+
status: 'error',
|
|
250
|
+
error: {
|
|
251
|
+
message: `Error while streaming to artifact for attachment ID ${attachment.id}. Skipping attachment. ` +
|
|
252
|
+
(0, logger_1.serializeError)(uploadedArtifactError),
|
|
253
|
+
fileSize,
|
|
189
254
|
},
|
|
190
|
-
|
|
191
|
-
|
|
255
|
+
};
|
|
256
|
+
}
|
|
257
|
+
const { error: confirmArtifactUploadError } = await this.uploader.confirmArtifactUpload(artifactUrlResponse.artifact_id);
|
|
258
|
+
if (confirmArtifactUploadError) {
|
|
259
|
+
return {
|
|
260
|
+
status: 'error',
|
|
261
|
+
error: {
|
|
262
|
+
message: `Error while confirming upload for attachment ID ${attachment.id}. ` +
|
|
263
|
+
(0, logger_1.serializeError)(confirmArtifactUploadError),
|
|
264
|
+
fileSize,
|
|
192
265
|
},
|
|
193
266
|
};
|
|
194
|
-
if (attachment.author_id) {
|
|
195
|
-
ssorAttachment.actor_id = {
|
|
196
|
-
external: attachment.author_id,
|
|
197
|
-
};
|
|
198
|
-
}
|
|
199
|
-
// Set inline flag only if it is explicitly set on the attachment.
|
|
200
|
-
if (attachment.inline === true) {
|
|
201
|
-
ssorAttachment.inline = true;
|
|
202
|
-
}
|
|
203
|
-
else if (attachment.inline === false) {
|
|
204
|
-
ssorAttachment.inline = false;
|
|
205
|
-
}
|
|
206
|
-
if (this.isTimeout) {
|
|
207
|
-
this.destroyHttpStream(httpStream);
|
|
208
|
-
return;
|
|
209
|
-
}
|
|
210
|
-
await ((_c = this.getRepo('ssor_attachment')) === null || _c === void 0 ? void 0 : _c.push([ssorAttachment]));
|
|
211
|
-
return;
|
|
212
267
|
}
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
268
|
+
const ssorAttachment = {
|
|
269
|
+
id: {
|
|
270
|
+
devrev: artifactUrlResponse.artifact_id,
|
|
271
|
+
external: attachment.id,
|
|
272
|
+
},
|
|
273
|
+
parent_id: {
|
|
274
|
+
external: attachment.parent_id,
|
|
216
275
|
},
|
|
217
276
|
};
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
else if (typeof httpStream.data.close === 'function') {
|
|
227
|
-
httpStream.data.close();
|
|
228
|
-
}
|
|
229
|
-
}
|
|
277
|
+
if (attachment.author_id) {
|
|
278
|
+
ssorAttachment.actor_id = {
|
|
279
|
+
external: attachment.author_id,
|
|
280
|
+
};
|
|
281
|
+
}
|
|
282
|
+
// Set inline flag only if it is explicitly set on the attachment.
|
|
283
|
+
if (attachment.inline === true) {
|
|
284
|
+
ssorAttachment.inline = true;
|
|
230
285
|
}
|
|
231
|
-
|
|
232
|
-
|
|
286
|
+
else if (attachment.inline === false) {
|
|
287
|
+
ssorAttachment.inline = false;
|
|
288
|
+
}
|
|
289
|
+
if (this.isTimeout) {
|
|
290
|
+
(0, uploader_helpers_1.destroyHttpStream)(httpStream);
|
|
291
|
+
return { status: 'success' };
|
|
233
292
|
}
|
|
293
|
+
await this.getOrCreateSsorRepo().push([ssorAttachment]);
|
|
294
|
+
return { status: 'success' };
|
|
234
295
|
}
|
|
235
296
|
async streamAttachments({ stream, processors, batchSize = 1, }) {
|
|
236
297
|
var _a, _b;
|
|
@@ -242,13 +303,8 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
|
|
|
242
303
|
console.warn(`The specified batch size (${batchSize}) is too large. Using 50 instead.`);
|
|
243
304
|
batchSize = 50;
|
|
244
305
|
}
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
itemType: 'ssor_attachment',
|
|
248
|
-
},
|
|
249
|
-
];
|
|
250
|
-
this.initializeRepos(repos);
|
|
251
|
-
const attachmentsMetadata = (_a = this.sdkState.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata;
|
|
306
|
+
this.initializeRepos([{ itemType: constants_1.SSOR_ATTACHMENT }]);
|
|
307
|
+
const attachmentsMetadata = (_a = this.adapterState.sdkState.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata;
|
|
252
308
|
if (!((_b = attachmentsMetadata === null || attachmentsMetadata === void 0 ? void 0 : attachmentsMetadata.artifactIds) === null || _b === void 0 ? void 0 : _b.length)) {
|
|
253
309
|
console.log(`No attachments metadata artifact IDs found in state.`);
|
|
254
310
|
return { status: 'success' };
|
|
@@ -299,11 +355,8 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
|
|
|
299
355
|
});
|
|
300
356
|
response = await attachmentsPool.streamAll();
|
|
301
357
|
}
|
|
302
|
-
if (response
|
|
303
|
-
return
|
|
304
|
-
}
|
|
305
|
-
if (response === null || response === void 0 ? void 0 : response.delay) {
|
|
306
|
-
return { status: 'delay', delaySeconds: response.delay };
|
|
358
|
+
if (response.status !== 'success') {
|
|
359
|
+
return response;
|
|
307
360
|
}
|
|
308
361
|
if (this.isTimeout) {
|
|
309
362
|
console.log(`Timeout detected after processing attachments for artifact ID: ${attachmentsMetadataArtifactId}. Returning progress to allow continuation.`);
|
|
@@ -1,39 +1,29 @@
|
|
|
1
1
|
import { Mappers } from '../../mappers/mappers';
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import { ExternalSystemAttachment, ExternalSystemItem, ExternalSystemLoadingFunction, FileToLoad, ItemTypesToLoadParams, ItemTypeToLoad, LoaderEventType, LoaderReport, LoadItemResponse } from '../../types/loading';
|
|
5
|
-
import { TaskResult, WorkerAdapterOptions } from '../../types/workers';
|
|
2
|
+
import { ExternalSystemAttachment, ExternalSystemLoadingFunction, ItemTypesToLoadParams, LoaderReport } from '../../types/loading';
|
|
3
|
+
import { TaskResult } from '../../types/workers';
|
|
6
4
|
import { BaseAdapter } from './base-adapter';
|
|
7
5
|
/** Adapter passed to loading tasks (item/attachment loading, mappers, loader reports). @public */
|
|
8
6
|
export declare class LoadingAdapter<ConnectorState> extends BaseAdapter<ConnectorState> {
|
|
9
7
|
private loaderReports;
|
|
10
8
|
private _processedFiles;
|
|
11
9
|
private _mappers;
|
|
12
|
-
constructor(params: {
|
|
13
|
-
event: AirSyncEvent;
|
|
14
|
-
adapterState: BaseState<ConnectorState>;
|
|
15
|
-
options?: WorkerAdapterOptions;
|
|
16
|
-
});
|
|
17
10
|
get reports(): LoaderReport[];
|
|
18
11
|
get processedFiles(): string[];
|
|
19
12
|
get mappers(): Mappers;
|
|
20
|
-
protected beforeEmit(): Promise<void>;
|
|
21
|
-
protected buildEmitPayload(newEventType: ExtractorEventType | LoaderEventType): EventData;
|
|
22
|
-
protected afterEmit(): void;
|
|
23
13
|
loadItemTypes({ itemTypesToLoad, }: ItemTypesToLoadParams): Promise<TaskResult>;
|
|
24
|
-
getLoaderBatches({ supportedItemTypes, }: {
|
|
25
|
-
supportedItemTypes: string[];
|
|
26
|
-
}): Promise<FileToLoad[]>;
|
|
27
14
|
loadAttachments({ create, }: {
|
|
28
15
|
create: ExternalSystemLoadingFunction<ExternalSystemAttachment>;
|
|
29
16
|
}): Promise<TaskResult>;
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
17
|
+
/**
|
|
18
|
+
* Walks the transformer files in `fromDevRev.filesToLoad`, resuming from
|
|
19
|
+
* each file's `lineToProcess`. `getLoader` returns undefined when a file has
|
|
20
|
+
* an item type the connector can't load, which fails the phase.
|
|
21
|
+
*/
|
|
22
|
+
private loadFiles;
|
|
23
|
+
private getLoaderBatches;
|
|
24
|
+
private loadItem;
|
|
25
|
+
/** Maps a non-success connector response to a delay or a failed report. */
|
|
26
|
+
private toFailedLoad;
|
|
27
|
+
private loadAttachment;
|
|
38
28
|
}
|
|
39
29
|
//# sourceMappingURL=loading-adapter.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"loading-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/loading-adapter.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"loading-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/loading-adapter.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAShD,OAAO,EAEL,wBAAwB,EAGxB,6BAA6B,EAE7B,qBAAqB,EAGrB,YAAY,EAGb,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAkB,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjE,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAM7C,kGAAkG;AAClG,qBAAa,cAAc,CACzB,cAAc,CACd,SAAQ,WAAW,CAAC,cAAc,CAAC;IACnC,OAAO,CAAC,aAAa,CAAiB;IACtC,OAAO,CAAC,eAAe,CAAW;IAClC,OAAO,CAAC,QAAQ,CAAU;IAc1B,IAAI,OAAO,IAAI,YAAY,EAAE,CAE5B;IAED,IAAI,cAAc,IAAI,MAAM,EAAE,CAE7B;IAED,IAAI,OAAO,IAAI,OAAO,CAErB;IA2BK,aAAa,CAAC,EAClB,eAAe,GAChB,EAAE,qBAAqB,GAAG,OAAO,CAAC,UAAU,CAAC;IA8BxC,eAAe,CAAC,EACpB,MAAM,GACP,EAAE;QACD,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,UAAU,CAAC;IAavB;;;;OAIG;YACW,SAAS;YAiFT,gBAAgB;YAuBhB,QAAQ;IA+HtB,2EAA2E;IAC3E,OAAO,CAAC,YAAY;YAwBN,cAAc;CA2C7B"}
|