@devrev/ts-adaas 1.19.11-beta.2 → 1.19.11-beta.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/attachments-streaming/attachments-streaming-pool.d.ts +8 -0
- package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map +1 -1
- package/dist/attachments-streaming/attachments-streaming-pool.js +50 -5
- package/dist/attachments-streaming/attachments-streaming-pool.test.js +102 -1
- package/dist/multithreading/worker-adapter/worker-adapter.d.ts.map +1 -1
- package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.js +65 -2
- package/dist/multithreading/worker-adapter/worker-adapter.helpers.d.ts +6 -0
- package/dist/multithreading/worker-adapter/worker-adapter.helpers.d.ts.map +1 -1
- package/dist/multithreading/worker-adapter/worker-adapter.helpers.js +29 -0
- package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.js +46 -0
- package/dist/multithreading/worker-adapter/worker-adapter.js +23 -7
- package/dist/multithreading/worker-adapter/worker-adapter.loading.test.js +3 -4
- package/dist/state/state.interfaces.d.ts +11 -0
- package/dist/state/state.interfaces.d.ts.map +1 -1
- package/dist/state/state.interfaces.js +1 -0
- package/dist/types/extraction.d.ts +20 -1
- package/dist/types/extraction.d.ts.map +1 -1
- package/dist/uploader/uploader.d.ts.map +1 -1
- package/dist/uploader/uploader.helpers.d.ts +8 -0
- package/dist/uploader/uploader.helpers.d.ts.map +1 -1
- package/dist/uploader/uploader.helpers.js +16 -0
- package/dist/uploader/uploader.helpers.test.js +39 -0
- package/dist/uploader/uploader.js +3 -8
- package/package.json +1 -1
|
@@ -9,6 +9,14 @@ export declare class AttachmentsStreamingPool<ConnectorState> {
|
|
|
9
9
|
private totalProcessedCount;
|
|
10
10
|
private readonly PROGRESS_REPORT_INTERVAL;
|
|
11
11
|
constructor({ adapter, attachments, batchSize, stream, }: AttachmentsStreamingPoolParams<ConnectorState>);
|
|
12
|
+
/**
|
|
13
|
+
* Marks an attachment as permanently failed after a transient error (ECONNABORTED,
|
|
14
|
+
* 5xx), so subsequent invocations skip it instead of retrying a deterministically-
|
|
15
|
+
* failing request forever. The stream function's own HTTP client (axiosClient) already
|
|
16
|
+
* retries transient errors with exponential backoff before returning one here, so a
|
|
17
|
+
* second retry loop at this level would just double up on an already-exhausted budget.
|
|
18
|
+
*/
|
|
19
|
+
private markPermanentlyFailed;
|
|
12
20
|
private updateProgress;
|
|
13
21
|
/**
|
|
14
22
|
* Migrates processed attachments from the legacy string[] format to the new ProcessedAttachment[] format.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"attachments-streaming-pool.d.ts","sourceRoot":"","sources":["../../src/attachments-streaming/attachments-streaming-pool.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"attachments-streaming-pool.d.ts","sourceRoot":"","sources":["../../src/attachments-streaming/attachments-streaming-pool.ts"],"names":[],"mappings":"AAMA,OAAO,EAGL,2BAA2B,EAC5B,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,8BAA8B,EAAE,MAAM,yCAAyC,CAAC;AAEzF,qBAAa,wBAAwB,CAAC,cAAc;IAClD,OAAO,CAAC,OAAO,CAAgC;IAC/C,OAAO,CAAC,WAAW,CAAyB;IAC5C,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,KAAK,CAAqB;IAClC,OAAO,CAAC,MAAM,CAA4C;IAE1D,OAAO,CAAC,mBAAmB,CAAa;IACxC,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAM;gBAEnC,EACV,OAAO,EACP,WAAW,EACX,SAAc,EACd,MAAM,GACP,EAAE,8BAA8B,CAAC,cAAc,CAAC;IAQjD;;;;;;OAMG;IACH,OAAO,CAAC,qBAAqB;YAsBf,cAAc;IAS5B;;;;;OAKG;IAEH,OAAO,CAAC,2BAA2B;IAsB7B,SAAS,IAAI,OAAO,CAAC,2BAA2B,CAAC;IAyDjD,kBAAkB;CAuHzB"}
|
|
@@ -12,6 +12,29 @@ class AttachmentsStreamingPool {
|
|
|
12
12
|
this.delay = undefined;
|
|
13
13
|
this.stream = stream;
|
|
14
14
|
}
|
|
15
|
+
/**
|
|
16
|
+
* Marks an attachment as permanently failed after a transient error (ECONNABORTED,
|
|
17
|
+
* 5xx), so subsequent invocations skip it instead of retrying a deterministically-
|
|
18
|
+
* failing request forever. The stream function's own HTTP client (axiosClient) already
|
|
19
|
+
* retries transient errors with exponential backoff before returning one here, so a
|
|
20
|
+
* second retry loop at this level would just double up on an already-exhausted budget.
|
|
21
|
+
*/
|
|
22
|
+
markPermanentlyFailed(attachment) {
|
|
23
|
+
var _a, _b;
|
|
24
|
+
const attachmentsMetadata = (_a = this.adapter.state.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata;
|
|
25
|
+
if (!attachmentsMetadata) {
|
|
26
|
+
return;
|
|
27
|
+
}
|
|
28
|
+
const failedAttachmentsIdsList = (_b = attachmentsMetadata.failedAttachmentsIdsList) !== null && _b !== void 0 ? _b : [];
|
|
29
|
+
attachmentsMetadata.failedAttachmentsIdsList = failedAttachmentsIdsList;
|
|
30
|
+
const alreadyMarked = failedAttachmentsIdsList.some((it) => it.id === attachment.id && it.parent_id === attachment.parent_id);
|
|
31
|
+
if (!alreadyMarked) {
|
|
32
|
+
failedAttachmentsIdsList.push({
|
|
33
|
+
id: attachment.id,
|
|
34
|
+
parent_id: attachment.parent_id,
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
}
|
|
15
38
|
async updateProgress() {
|
|
16
39
|
this.totalProcessedCount++;
|
|
17
40
|
if (this.totalProcessedCount % this.PROGRESS_REPORT_INTERVAL === 0) {
|
|
@@ -63,6 +86,12 @@ class AttachmentsStreamingPool {
|
|
|
63
86
|
this.adapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList =
|
|
64
87
|
this.migrateProcessedAttachments(this.adapter.state.toDevRev.attachmentsMetadata
|
|
65
88
|
.lastProcessedAttachmentsIdsList);
|
|
89
|
+
// Get the list of attachments that have repeatedly failed with a transient error in
|
|
90
|
+
// previous invocations. If no such list exists, create an empty one.
|
|
91
|
+
if (!this.adapter.state.toDevRev.attachmentsMetadata.failedAttachmentsIdsList) {
|
|
92
|
+
this.adapter.state.toDevRev.attachmentsMetadata.failedAttachmentsIdsList =
|
|
93
|
+
[];
|
|
94
|
+
}
|
|
66
95
|
// Start initial batch of promises up to batchSize limit
|
|
67
96
|
const initialBatchSize = Math.min(this.batchSize, this.attachments.length);
|
|
68
97
|
const initialPromises = [];
|
|
@@ -79,7 +108,7 @@ class AttachmentsStreamingPool {
|
|
|
79
108
|
return {};
|
|
80
109
|
}
|
|
81
110
|
async startPoolStreaming() {
|
|
82
|
-
var _a, _b, _c, _d;
|
|
111
|
+
var _a, _b, _c, _d, _e;
|
|
83
112
|
// Process attachments until the attachments array is empty
|
|
84
113
|
while (this.attachments.length > 0) {
|
|
85
114
|
// If delay is set, stop streaming
|
|
@@ -100,6 +129,11 @@ class AttachmentsStreamingPool {
|
|
|
100
129
|
((_a = this.adapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList) === null || _a === void 0 ? void 0 : _a.some((it) => it.id == attachment.id && it.parent_id == attachment.parent_id))) {
|
|
101
130
|
continue; // Skip if the attachment ID is already processed
|
|
102
131
|
}
|
|
132
|
+
if (this.adapter.state.toDevRev &&
|
|
133
|
+
((_b = this.adapter.state.toDevRev.attachmentsMetadata.failedAttachmentsIdsList) === null || _b === void 0 ? void 0 : _b.some((it) => it.id == attachment.id && it.parent_id == attachment.parent_id))) {
|
|
134
|
+
console.warn(`Skipping attachment with ID ${attachment.id} fetched from URL ${attachment.url}: previously failed with a transient error.`);
|
|
135
|
+
continue; // Skip if the attachment was already marked as permanently failed
|
|
136
|
+
}
|
|
103
137
|
try {
|
|
104
138
|
const response = await this.adapter.processAttachment(attachment, this.stream);
|
|
105
139
|
// Check if rate limit was hit
|
|
@@ -115,13 +149,24 @@ class AttachmentsStreamingPool {
|
|
|
115
149
|
const contentTypeInfo = attachment.content_type
|
|
116
150
|
? `and content_type ${attachment.content_type} `
|
|
117
151
|
: '';
|
|
118
|
-
|
|
152
|
+
const fetchHeadersInfo = response.error.fetchHeaders &&
|
|
153
|
+
Object.keys(response.error.fetchHeaders).length > 0
|
|
154
|
+
? `and fetch response headers ${JSON.stringify(response.error.fetchHeaders)} `
|
|
155
|
+
: '';
|
|
156
|
+
console.warn(`Skipping attachment with ID ${attachment.id} with extension ${fileExtension} ${fileSizeInfo}${contentTypeInfo}${fetchHeadersInfo}fetched from URL ${attachment.url} due to error returned by the stream function`, response.error.message);
|
|
157
|
+
// The stream function's HTTP client already retried this request with
|
|
158
|
+
// exponential backoff before returning an error, so a transient failure here
|
|
159
|
+
// means that budget is exhausted — mark it permanently failed rather than
|
|
160
|
+
// retrying it again from scratch next invocation.
|
|
161
|
+
if (response.error.isTransient) {
|
|
162
|
+
this.markPermanentlyFailed(attachment);
|
|
163
|
+
}
|
|
119
164
|
await this.updateProgress();
|
|
120
165
|
continue;
|
|
121
166
|
}
|
|
122
167
|
if (!this.adapter.isTimeout &&
|
|
123
|
-
((
|
|
124
|
-
(
|
|
168
|
+
((_d = (_c = this.adapter.state.toDevRev) === null || _c === void 0 ? void 0 : _c.attachmentsMetadata) === null || _d === void 0 ? void 0 : _d.lastProcessedAttachmentsIdsList)) {
|
|
169
|
+
(_e = this.adapter.state.toDevRev) === null || _e === void 0 ? void 0 : _e.attachmentsMetadata.lastProcessedAttachmentsIdsList.push({ id: attachment.id, parent_id: attachment.parent_id });
|
|
125
170
|
}
|
|
126
171
|
await this.updateProgress();
|
|
127
172
|
}
|
|
@@ -130,7 +175,7 @@ class AttachmentsStreamingPool {
|
|
|
130
175
|
const contentTypeInfo = attachment.content_type
|
|
131
176
|
? ` and content_type ${attachment.content_type}`
|
|
132
177
|
: '';
|
|
133
|
-
console.warn(`Skipping attachment with ID ${attachment.id} with extension ${fileExtension}${contentTypeInfo} due to error in processAttachment function`, error);
|
|
178
|
+
console.warn(`Skipping attachment with ID ${attachment.id} with extension ${fileExtension}${contentTypeInfo} fetched from URL ${attachment.url} due to error in processAttachment function`, error);
|
|
134
179
|
await this.updateProgress();
|
|
135
180
|
}
|
|
136
181
|
}
|
|
@@ -155,6 +155,107 @@ describe(attachments_streaming_pool_1.AttachmentsStreamingPool.name, () => {
|
|
|
155
155
|
]);
|
|
156
156
|
expect(result).toEqual({});
|
|
157
157
|
});
|
|
158
|
+
it('should skip attachments already marked as permanently failed', async () => {
|
|
159
|
+
mockAdapter.state.toDevRev.attachmentsMetadata.failedAttachmentsIdsList =
|
|
160
|
+
[{ id: 'attachment-1', parent_id: 'parent-1' }];
|
|
161
|
+
mockAdapter.processAttachment.mockResolvedValue({});
|
|
162
|
+
const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
|
|
163
|
+
adapter: mockAdapter,
|
|
164
|
+
attachments: mockAttachments,
|
|
165
|
+
stream: mockStream,
|
|
166
|
+
});
|
|
167
|
+
await pool.streamAll();
|
|
168
|
+
expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(2); // Only 2 out of 3
|
|
169
|
+
});
|
|
170
|
+
it('should include the attachment URL when logging a previously-failed skip', async () => {
|
|
171
|
+
const warnSpy = jest.spyOn(console, 'warn').mockImplementation();
|
|
172
|
+
mockAdapter.state.toDevRev.attachmentsMetadata.failedAttachmentsIdsList =
|
|
173
|
+
[{ id: 'attachment-1', parent_id: 'parent-1' }];
|
|
174
|
+
mockAdapter.processAttachment.mockResolvedValue({});
|
|
175
|
+
const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
|
|
176
|
+
adapter: mockAdapter,
|
|
177
|
+
attachments: [mockAttachments[0]],
|
|
178
|
+
stream: mockStream,
|
|
179
|
+
batchSize: 1,
|
|
180
|
+
});
|
|
181
|
+
await pool.streamAll();
|
|
182
|
+
expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('https://example.com/file1.pdf'));
|
|
183
|
+
});
|
|
184
|
+
it('should mark an attachment as permanently failed on a transient error', async () => {
|
|
185
|
+
mockAdapter.processAttachment.mockResolvedValueOnce({
|
|
186
|
+
error: { message: 'timeout', isTransient: true },
|
|
187
|
+
});
|
|
188
|
+
const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
|
|
189
|
+
adapter: mockAdapter,
|
|
190
|
+
attachments: [mockAttachments[0]],
|
|
191
|
+
stream: mockStream,
|
|
192
|
+
batchSize: 1,
|
|
193
|
+
});
|
|
194
|
+
await pool.streamAll();
|
|
195
|
+
expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(1);
|
|
196
|
+
expect(mockAdapter.state.toDevRev.attachmentsMetadata.failedAttachmentsIdsList).toEqual([{ id: 'attachment-1', parent_id: 'parent-1' }]);
|
|
197
|
+
// A subsequent invocation should now skip it entirely instead of calling processAttachment again.
|
|
198
|
+
mockAdapter.processAttachment.mockClear();
|
|
199
|
+
const secondPool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
|
|
200
|
+
adapter: mockAdapter,
|
|
201
|
+
attachments: [mockAttachments[0]],
|
|
202
|
+
stream: mockStream,
|
|
203
|
+
batchSize: 1,
|
|
204
|
+
});
|
|
205
|
+
await secondPool.streamAll();
|
|
206
|
+
expect(mockAdapter.processAttachment).not.toHaveBeenCalled();
|
|
207
|
+
});
|
|
208
|
+
it('should not mark a non-transient error as permanently failed', async () => {
|
|
209
|
+
mockAdapter.processAttachment.mockResolvedValueOnce({
|
|
210
|
+
error: { message: 'File size is 0 or less.' },
|
|
211
|
+
});
|
|
212
|
+
const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
|
|
213
|
+
adapter: mockAdapter,
|
|
214
|
+
attachments: [mockAttachments[0]],
|
|
215
|
+
stream: mockStream,
|
|
216
|
+
batchSize: 1,
|
|
217
|
+
});
|
|
218
|
+
await pool.streamAll();
|
|
219
|
+
expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(1);
|
|
220
|
+
expect(mockAdapter.state.toDevRev.attachmentsMetadata.failedAttachmentsIdsList).toEqual([]);
|
|
221
|
+
});
|
|
222
|
+
it('should include the attachment URL and fetch response headers in the skip log', async () => {
|
|
223
|
+
const warnSpy = jest.spyOn(console, 'warn').mockImplementation();
|
|
224
|
+
mockAdapter.processAttachment.mockResolvedValueOnce({
|
|
225
|
+
error: {
|
|
226
|
+
message: 'timeout',
|
|
227
|
+
isTransient: true,
|
|
228
|
+
fileSize: 2048,
|
|
229
|
+
fetchHeaders: {
|
|
230
|
+
'content-length': '2048',
|
|
231
|
+
'content-type': 'application/pdf',
|
|
232
|
+
},
|
|
233
|
+
},
|
|
234
|
+
});
|
|
235
|
+
const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
|
|
236
|
+
adapter: mockAdapter,
|
|
237
|
+
attachments: [mockAttachments[0]],
|
|
238
|
+
stream: mockStream,
|
|
239
|
+
batchSize: 1,
|
|
240
|
+
});
|
|
241
|
+
await pool.streamAll();
|
|
242
|
+
expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('https://example.com/file1.pdf'), 'timeout');
|
|
243
|
+
expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('{"content-length":"2048","content-type":"application/pdf"}'), 'timeout');
|
|
244
|
+
});
|
|
245
|
+
it('should include the attachment URL in the skip log when the error has no fetch headers', async () => {
|
|
246
|
+
const warnSpy = jest.spyOn(console, 'warn').mockImplementation();
|
|
247
|
+
mockAdapter.processAttachment.mockResolvedValueOnce({
|
|
248
|
+
error: { message: 'File size is 0 or less.' },
|
|
249
|
+
});
|
|
250
|
+
const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
|
|
251
|
+
adapter: mockAdapter,
|
|
252
|
+
attachments: [mockAttachments[0]],
|
|
253
|
+
stream: mockStream,
|
|
254
|
+
batchSize: 1,
|
|
255
|
+
});
|
|
256
|
+
await pool.streamAll();
|
|
257
|
+
expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('https://example.com/file1.pdf'), 'File size is 0 or less.');
|
|
258
|
+
});
|
|
158
259
|
it('should handle all attachments failing with different file types and sizes', async () => {
|
|
159
260
|
const largeAttachments = [
|
|
160
261
|
{
|
|
@@ -238,7 +339,7 @@ describe(attachments_streaming_pool_1.AttachmentsStreamingPool.name, () => {
|
|
|
238
339
|
stream: mockStream,
|
|
239
340
|
});
|
|
240
341
|
await pool.streamAll();
|
|
241
|
-
expect(warnSpy).toHaveBeenCalledWith('Skipping attachment with ID attachment-2 with extension jpg due to error in processAttachment function', error);
|
|
342
|
+
expect(warnSpy).toHaveBeenCalledWith('Skipping attachment with ID attachment-2 with extension jpg fetched from URL https://example.com/file2.jpg due to error in processAttachment function', error);
|
|
242
343
|
expect(mockAdapter.state.toDevRev.attachmentsMetadata
|
|
243
344
|
.lastProcessedAttachmentsIdsList).toEqual([
|
|
244
345
|
{
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"worker-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"worker-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.ts"],"names":[],"mappings":"AAsBA,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAEhD,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AACvC,OAAO,EACL,oBAAoB,EACpB,aAAa,EACd,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EAAE,YAAY,EAAE,MAAM,8BAA8B,CAAC;AAC5D,OAAO,EACL,YAAY,EACZ,SAAS,EAET,kCAAkC,EAClC,yCAAyC,EACzC,kBAAkB,EAClB,2BAA2B,EAC3B,2BAA2B,EAC5B,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAEL,wBAAwB,EACxB,kBAAkB,EAClB,6BAA6B,EAC7B,UAAU,EACV,qBAAqB,EACrB,cAAc,EACd,eAAe,EACf,YAAY,EACZ,gBAAgB,EAChB,qBAAqB,EAEtB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,sBAAsB,EACtB,oBAAoB,EAGrB,MAAM,qBAAqB,CAAC;AAG7B,OAAO,EAAE,QAAQ,EAAkB,MAAM,oCAAoC,CAAC;AAI9E,wBAAgB,mBAAmB,CAAC,cAAc,EAAE,EAClD,KAAK,EACL,YAAY,EACZ,OAAO,GACR,EAAE,sBAAsB,CAAC,cAAc,CAAC,GAAG,aAAa,CAAC,cAAc,CAAC,CAMxE;AAED;;;;;;;;;;;;;;GAcG;AACH,qBAAa,aAAa,CAAC,cAAc;IACvC,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IAC7B,QAAQ,CAAC,OAAO,CAAC,EAAE,oBAAoB,CAAC;IACxC,gBAAgB,EAAE,OAAO,CAAC;IAE1B,OAAO,CAAC,UAAU,CAAkB;IACpC,OAAO,CAAC,oBAAoB,CAAc;IAC1C,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC,IAAI,CAAC,CAElC;IAEH,OAAO,CAAC,YAAY,CAAwB;IAC5C,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,KAAK,CAAc;IAC3B,OAAO,CAAC,qBAAqB,CAAC,CAAS;IACvC,OAAO,CAAC,sBAAsB,CAAa;IAG3C,OAAO,CAAC,aAAa,CAAiB;IACtC,OAAO,CAAC,eAAe,CAAW;IAClC,OAAO,CAAC,QAAQ,CAAU;IAC1B,OAAO,CAAC,QAAQ,CAAW;gBAEf,EACV,KAAK,EACL,YAAY,EACZ,OAAO,GACR,EAAE,sBAAsB,CAAC,cAAc,CAAC;IAoBzC,IAAI,SAAS,IAAI,OAAO,CAEvB;IAED,IAAI,SAAS,CAAC,KAAK,EAAE,OAAO,EAK3B;IAED,IAAI,KAAK,IAAI,YAAY,CAAC,cAAc,CAAC,CAExC;IAED,IAAI,KAAK,CAAC,KAAK,EAAE,YAAY,CAAC,cAAc,CAAC,EAE5C;IAED,IAAI,OAAO,IAAI,YAAY,EAAE,CAE5B;IAED,IAAI,cAAc,IAAI,MAAM,EAAE,CAE7B;IAED,IAAI,OAAO,IAAI,OAAO,CAErB;IAED,IAAI,eAAe,kDAElB;IAED;;;OAGG;IACH,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAOxC,eAAe,CAAC,KAAK,EAAE,aAAa,EAAE;IAyCtC,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS;IAarC,SAAS;IAMf,IAAI,SAAS,IAAI,QAAQ,EAAE,CAE1B;IAED,IAAI,SAAS,CAAC,SAAS,EAAE,QAAQ,EAAE,EAIlC;IAED;;;;;OAKG;IACG,IAAI,CACR,YAAY,EAAE,kBAAkB,GAAG,eAAe,EAClD,IAAI,CAAC,EAAE,SAAS,GACf,OAAO,CAAC,IAAI,CAAC;IAwLV,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;IAU/B,aAAa,CAAC,EAClB,eAAe,GAChB,EAAE,qBAAqB,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAwInD,gBAAgB,CAAC,EACrB,kBAAkB,GACnB,EAAE;QACD,kBAAkB,EAAE,MAAM,EAAE,CAAC;KAC9B;IA4BK,eAAe,CAAC,EACpB,MAAM,GACP,EAAE;QACD,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAyG5B,QAAQ,CAAC,EACb,IAAI,EACJ,cAAc,GACf,EAAE;QACD,IAAI,EAAE,kBAAkB,CAAC;QACzB,cAAc,EAAE,cAAc,CAAC;KAChC,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAwMvB,iBAAiB,CACrB,UAAU,EAAE,oBAAoB,EAChC,MAAM,EAAE,yCAAyC,GAChD,OAAO,CAAC,2BAA2B,CAAC;IAgIvC;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAcnB,cAAc,CAAC,EACnB,IAAI,EACJ,MAAM,GACP,EAAE;QACD,IAAI,EAAE,wBAAwB,CAAC;QAC/B,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAuD7B;;;;;;OAMG;IACG,iBAAiB,CAAC,QAAQ,EAAE,EAChC,MAAM,EACN,UAAU,EACV,SAAa,GACd,EAAE;QACD,MAAM,EAAE,yCAAyC,CAAC;QAClD,UAAU,CAAC,EAAE,kCAAkC,CAC7C,cAAc,EACd,oBAAoB,EAAE,EACtB,QAAQ,CACT,CAAC;QACF,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,GAAG,OAAO,CAAC,2BAA2B,CAAC;CA0IzC"}
|
|
@@ -561,7 +561,8 @@ describe(`${worker_adapter_1.WorkerAdapter.name}.processAttachment`, () => {
|
|
|
561
561
|
expect(adapter['uploader'].getArtifactUploadUrl).toHaveBeenCalledWith('file.bin', 'application/octet-stream', undefined);
|
|
562
562
|
});
|
|
563
563
|
// ---- error paths ----
|
|
564
|
-
it('should return the stream error
|
|
564
|
+
it('should return the stream error message when the stream function returns an error', async () => {
|
|
565
|
+
var _a, _b;
|
|
565
566
|
// Arrange
|
|
566
567
|
const stream = jest
|
|
567
568
|
.fn()
|
|
@@ -569,7 +570,41 @@ describe(`${worker_adapter_1.WorkerAdapter.name}.processAttachment`, () => {
|
|
|
569
570
|
// Act
|
|
570
571
|
const result = await adapter.processAttachment(makeAttachment(), stream);
|
|
571
572
|
// Assert
|
|
572
|
-
expect(result === null || result === void 0 ? void 0 : result.error).
|
|
573
|
+
expect((_a = result === null || result === void 0 ? void 0 : result.error) === null || _a === void 0 ? void 0 : _a.message).toBe('stream failed');
|
|
574
|
+
expect((_b = result === null || result === void 0 ? void 0 : result.error) === null || _b === void 0 ? void 0 : _b.isTransient).toBe(false);
|
|
575
|
+
});
|
|
576
|
+
it('should classify a stream error with a 5xx statusCode as transient', async () => {
|
|
577
|
+
var _a;
|
|
578
|
+
// Arrange
|
|
579
|
+
const stream = jest.fn().mockResolvedValue({
|
|
580
|
+
error: { message: 'export failed', statusCode: 500 },
|
|
581
|
+
});
|
|
582
|
+
// Act
|
|
583
|
+
const result = await adapter.processAttachment(makeAttachment(), stream);
|
|
584
|
+
// Assert
|
|
585
|
+
expect((_a = result === null || result === void 0 ? void 0 : result.error) === null || _a === void 0 ? void 0 : _a.isTransient).toBe(true);
|
|
586
|
+
});
|
|
587
|
+
it('should not classify a stream error with a 4xx statusCode as transient', async () => {
|
|
588
|
+
var _a;
|
|
589
|
+
// Arrange
|
|
590
|
+
const stream = jest.fn().mockResolvedValue({
|
|
591
|
+
error: { message: 'not found', statusCode: 404 },
|
|
592
|
+
});
|
|
593
|
+
// Act
|
|
594
|
+
const result = await adapter.processAttachment(makeAttachment(), stream);
|
|
595
|
+
// Assert
|
|
596
|
+
expect((_a = result === null || result === void 0 ? void 0 : result.error) === null || _a === void 0 ? void 0 : _a.isTransient).toBe(false);
|
|
597
|
+
});
|
|
598
|
+
it('should not classify a stream error without a statusCode as transient', async () => {
|
|
599
|
+
var _a;
|
|
600
|
+
// Arrange
|
|
601
|
+
const stream = jest.fn().mockResolvedValue({
|
|
602
|
+
error: { message: 'unknown failure' },
|
|
603
|
+
});
|
|
604
|
+
// Act
|
|
605
|
+
const result = await adapter.processAttachment(makeAttachment(), stream);
|
|
606
|
+
// Assert
|
|
607
|
+
expect((_a = result === null || result === void 0 ? void 0 : result.error) === null || _a === void 0 ? void 0 : _a.isTransient).toBe(false);
|
|
573
608
|
});
|
|
574
609
|
it('should propagate a rate-limit delay from the stream function', async () => {
|
|
575
610
|
// Arrange
|
|
@@ -615,6 +650,34 @@ describe(`${worker_adapter_1.WorkerAdapter.name}.processAttachment`, () => {
|
|
|
615
650
|
// Assert
|
|
616
651
|
expect((_a = result === null || result === void 0 ? void 0 : result.error) === null || _a === void 0 ? void 0 : _a.message).toContain('streaming to artifact');
|
|
617
652
|
});
|
|
653
|
+
it('should include allowlisted fetch response headers in the error when streamArtifact fails', async () => {
|
|
654
|
+
var _a;
|
|
655
|
+
// Arrange
|
|
656
|
+
const stream = jest.fn().mockResolvedValue({
|
|
657
|
+
httpStream: createMockHttpStream({
|
|
658
|
+
'content-length': '2048',
|
|
659
|
+
'content-type': 'application/pdf',
|
|
660
|
+
authorization: 'Bearer secret-token',
|
|
661
|
+
}),
|
|
662
|
+
});
|
|
663
|
+
adapter['uploader'].getArtifactUploadUrl = jest.fn().mockResolvedValue({
|
|
664
|
+
response: {
|
|
665
|
+
artifact_id: 'art-1',
|
|
666
|
+
upload_url: 'https://upload',
|
|
667
|
+
form_data: [],
|
|
668
|
+
},
|
|
669
|
+
});
|
|
670
|
+
adapter['uploader'].streamArtifact = jest
|
|
671
|
+
.fn()
|
|
672
|
+
.mockResolvedValue({ error: new Error('stream failed') });
|
|
673
|
+
// Act
|
|
674
|
+
const result = await adapter.processAttachment(makeAttachment(), stream);
|
|
675
|
+
// Assert
|
|
676
|
+
expect((_a = result === null || result === void 0 ? void 0 : result.error) === null || _a === void 0 ? void 0 : _a.fetchHeaders).toEqual({
|
|
677
|
+
'content-length': '2048',
|
|
678
|
+
'content-type': 'application/pdf',
|
|
679
|
+
});
|
|
680
|
+
});
|
|
618
681
|
it('should return an error when confirmArtifactUpload fails', async () => {
|
|
619
682
|
var _a;
|
|
620
683
|
// Arrange
|
|
@@ -1,4 +1,10 @@
|
|
|
1
1
|
import { FileToLoad, LoaderReport, StatsFileObject } from '../../types/loading';
|
|
2
|
+
/**
|
|
3
|
+
* Extracts a fixed allowlist of structural response headers (size, type, cache metadata)
|
|
4
|
+
* from an attachment fetch response, for inclusion in "skipping attachment" logs. Headers
|
|
5
|
+
* outside the allowlist (e.g. anything auth-adjacent) are never included.
|
|
6
|
+
*/
|
|
7
|
+
export declare function getSafeResponseHeaders(headers: Record<string, unknown>): Record<string, string>;
|
|
2
8
|
/**
|
|
3
9
|
* Gets the files to load for the loader.
|
|
4
10
|
* @param {string[]} supportedItemTypes - The supported item types
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"worker-adapter.helpers.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.helpers.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,UAAU,EACV,YAAY,EACZ,eAAe,EAChB,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"worker-adapter.helpers.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.helpers.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,UAAU,EACV,YAAY,EACZ,eAAe,EAChB,MAAM,qBAAqB,CAAC;AAgB7B;;;;GAIG;AACH,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC/B,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CASxB;AAED;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,EAC7B,kBAAkB,EAClB,SAAS,GACV,EAAE;IACD,kBAAkB,EAAE,MAAM,EAAE,CAAC;IAC7B,SAAS,EAAE,eAAe,EAAE,CAAC;CAC9B,GAAG,UAAU,EAAE,CA8Bf;AAED;;;;;GAKG;AACH,wBAAgB,uBAAuB,CAAC,EACtC,aAAa,EACb,MAAM,GACP,EAAE;IACD,aAAa,EAAE,YAAY,EAAE,CAAC;IAC9B,MAAM,EAAE,YAAY,CAAC;CACtB,GAAG,YAAY,EAAE,CA4BjB;AAED,wBAAgB,kBAAkB,CAAC,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAMlE"}
|
|
@@ -1,9 +1,38 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.getSafeResponseHeaders = getSafeResponseHeaders;
|
|
3
4
|
exports.getFilesToLoad = getFilesToLoad;
|
|
4
5
|
exports.addReportToLoaderReport = addReportToLoaderReport;
|
|
5
6
|
exports.toRfc3339Timestamp = toRfc3339Timestamp;
|
|
6
7
|
const loading_1 = require("../../types/loading");
|
|
8
|
+
// Response headers safe to include in logs: structural metadata about the fetched file,
|
|
9
|
+
// not the request itself. Everything else (auth-adjacent headers, cookies, etc.) is
|
|
10
|
+
// deliberately excluded — an allowlist rather than a denylist, so a header we haven't
|
|
11
|
+
// thought about doesn't leak through.
|
|
12
|
+
const SAFE_ATTACHMENT_RESPONSE_HEADERS = [
|
|
13
|
+
'content-length',
|
|
14
|
+
'content-type',
|
|
15
|
+
'content-disposition',
|
|
16
|
+
'content-encoding',
|
|
17
|
+
'last-modified',
|
|
18
|
+
'etag',
|
|
19
|
+
'cache-control',
|
|
20
|
+
];
|
|
21
|
+
/**
|
|
22
|
+
* Extracts a fixed allowlist of structural response headers (size, type, cache metadata)
|
|
23
|
+
* from an attachment fetch response, for inclusion in "skipping attachment" logs. Headers
|
|
24
|
+
* outside the allowlist (e.g. anything auth-adjacent) are never included.
|
|
25
|
+
*/
|
|
26
|
+
function getSafeResponseHeaders(headers) {
|
|
27
|
+
const safeHeaders = {};
|
|
28
|
+
for (const key of SAFE_ATTACHMENT_RESPONSE_HEADERS) {
|
|
29
|
+
const value = headers[key];
|
|
30
|
+
if (value !== undefined) {
|
|
31
|
+
safeHeaders[key] = String(value);
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
return safeHeaders;
|
|
35
|
+
}
|
|
7
36
|
/**
|
|
8
37
|
* Gets the files to load for the loader.
|
|
9
38
|
* @param {string[]} supportedItemTypes - The supported item types
|
|
@@ -540,3 +540,49 @@ describe(worker_adapter_helpers_1.toRfc3339Timestamp.name, () => {
|
|
|
540
540
|
expect(result).toBe('1969-12-31T00:00:00.000Z');
|
|
541
541
|
});
|
|
542
542
|
});
|
|
543
|
+
describe(worker_adapter_helpers_1.getSafeResponseHeaders.name, () => {
|
|
544
|
+
it('should extract allowlisted headers', () => {
|
|
545
|
+
const headers = {
|
|
546
|
+
'content-length': '1024',
|
|
547
|
+
'content-type': 'application/pdf',
|
|
548
|
+
'content-disposition': 'attachment; filename="file.pdf"',
|
|
549
|
+
'content-encoding': 'gzip',
|
|
550
|
+
'last-modified': 'Wed, 21 Oct 2015 07:28:00 GMT',
|
|
551
|
+
etag: '"abc123"',
|
|
552
|
+
'cache-control': 'no-cache',
|
|
553
|
+
};
|
|
554
|
+
const result = (0, worker_adapter_helpers_1.getSafeResponseHeaders)(headers);
|
|
555
|
+
expect(result).toEqual({
|
|
556
|
+
'content-length': '1024',
|
|
557
|
+
'content-type': 'application/pdf',
|
|
558
|
+
'content-disposition': 'attachment; filename="file.pdf"',
|
|
559
|
+
'content-encoding': 'gzip',
|
|
560
|
+
'last-modified': 'Wed, 21 Oct 2015 07:28:00 GMT',
|
|
561
|
+
etag: '"abc123"',
|
|
562
|
+
'cache-control': 'no-cache',
|
|
563
|
+
});
|
|
564
|
+
});
|
|
565
|
+
it('should exclude headers not on the allowlist', () => {
|
|
566
|
+
const headers = {
|
|
567
|
+
'content-length': '1024',
|
|
568
|
+
authorization: 'Bearer secret-token',
|
|
569
|
+
cookie: 'session=abc123',
|
|
570
|
+
'set-cookie': 'session=abc123',
|
|
571
|
+
'x-api-key': 'secret-key',
|
|
572
|
+
};
|
|
573
|
+
const result = (0, worker_adapter_helpers_1.getSafeResponseHeaders)(headers);
|
|
574
|
+
expect(result).toEqual({ 'content-length': '1024' });
|
|
575
|
+
});
|
|
576
|
+
it('[edge] should return an empty object when no headers are present', () => {
|
|
577
|
+
const result = (0, worker_adapter_helpers_1.getSafeResponseHeaders)({});
|
|
578
|
+
expect(result).toEqual({});
|
|
579
|
+
});
|
|
580
|
+
it('[edge] should return an empty object when headers only contain non-allowlisted keys', () => {
|
|
581
|
+
const result = (0, worker_adapter_helpers_1.getSafeResponseHeaders)({ authorization: 'Bearer token' });
|
|
582
|
+
expect(result).toEqual({});
|
|
583
|
+
});
|
|
584
|
+
it('[edge] should stringify non-string header values', () => {
|
|
585
|
+
const result = (0, worker_adapter_helpers_1.getSafeResponseHeaders)({ 'content-length': 1024 });
|
|
586
|
+
expect(result).toEqual({ 'content-length': '1024' });
|
|
587
|
+
});
|
|
588
|
+
});
|
|
@@ -20,6 +20,7 @@ const extraction_1 = require("../../types/extraction");
|
|
|
20
20
|
const loading_1 = require("../../types/loading");
|
|
21
21
|
const workers_1 = require("../../types/workers");
|
|
22
22
|
const uploader_1 = require("../../uploader/uploader");
|
|
23
|
+
const uploader_helpers_1 = require("../../uploader/uploader.helpers");
|
|
23
24
|
const event_type_translation_1 = require("../../common/event-type-translation");
|
|
24
25
|
const helpers_1 = require("../../common/helpers");
|
|
25
26
|
function createWorkerAdapter({ event, adapterState, options, }) {
|
|
@@ -649,22 +650,28 @@ class WorkerAdapter {
|
|
|
649
650
|
}
|
|
650
651
|
async processAttachment(attachment, stream) {
|
|
651
652
|
return (0, logger_context_1.runWithSdkLogContext)(async () => {
|
|
652
|
-
var _a, _b, _c;
|
|
653
|
+
var _a, _b, _c, _d;
|
|
653
654
|
const { httpStream, delay, error } = await (0, logger_context_1.runWithUserLogContext)(async () => stream({
|
|
654
655
|
item: attachment,
|
|
655
656
|
event: this.event,
|
|
656
657
|
}));
|
|
657
658
|
if (error) {
|
|
658
|
-
return {
|
|
659
|
+
return {
|
|
660
|
+
error: {
|
|
661
|
+
message: error.message,
|
|
662
|
+
isTransient: ((_a = error.statusCode) !== null && _a !== void 0 ? _a : 0) >= 500,
|
|
663
|
+
},
|
|
664
|
+
};
|
|
659
665
|
}
|
|
660
666
|
else if (delay) {
|
|
661
667
|
return { delay };
|
|
662
668
|
}
|
|
663
669
|
if (httpStream) {
|
|
670
|
+
const fetchHeaders = (0, worker_adapter_helpers_1.getSafeResponseHeaders)(httpStream.headers);
|
|
664
671
|
const fileType = attachment.content_type ||
|
|
665
|
-
((
|
|
672
|
+
((_b = httpStream.headers['content-type']) === null || _b === void 0 ? void 0 : _b.toString()) ||
|
|
666
673
|
'application/octet-stream';
|
|
667
|
-
const contentLength = (
|
|
674
|
+
const contentLength = (_c = httpStream.headers['content-length']) === null || _c === void 0 ? void 0 : _c.toString();
|
|
668
675
|
const fileSize = contentLength ? parseInt(contentLength) : undefined;
|
|
669
676
|
// Get upload URL
|
|
670
677
|
const { error: artifactUrlError, response: artifactUrlResponse } = await this.uploader.getArtifactUploadUrl(attachment.file_name, fileType, fileSize);
|
|
@@ -674,6 +681,8 @@ class WorkerAdapter {
|
|
|
674
681
|
error: {
|
|
675
682
|
message: `Error while preparing artifact for attachment ID ${attachment.id}. Skipping attachment. ${(0, logger_1.serializeError)(artifactUrlError)}`,
|
|
676
683
|
fileSize: fileSize,
|
|
684
|
+
isTransient: (0, uploader_helpers_1.isTransientUploadError)(artifactUrlError),
|
|
685
|
+
fetchHeaders,
|
|
677
686
|
},
|
|
678
687
|
};
|
|
679
688
|
}
|
|
@@ -686,6 +695,8 @@ class WorkerAdapter {
|
|
|
686
695
|
message: `Error while streaming to artifact for attachment ID ${attachment.id}. Skipping attachment. ` +
|
|
687
696
|
(0, logger_1.serializeError)(uploadedArtifactError),
|
|
688
697
|
fileSize: fileSize,
|
|
698
|
+
isTransient: (0, uploader_helpers_1.isTransientUploadError)(uploadedArtifactError),
|
|
699
|
+
fetchHeaders,
|
|
689
700
|
},
|
|
690
701
|
};
|
|
691
702
|
}
|
|
@@ -697,6 +708,8 @@ class WorkerAdapter {
|
|
|
697
708
|
message: `Error while confirming upload for attachment ID ${attachment.id}. ` +
|
|
698
709
|
(0, logger_1.serializeError)(confirmArtifactUploadError),
|
|
699
710
|
fileSize: fileSize,
|
|
711
|
+
isTransient: (0, uploader_helpers_1.isTransientUploadError)(confirmArtifactUploadError),
|
|
712
|
+
fetchHeaders,
|
|
700
713
|
},
|
|
701
714
|
};
|
|
702
715
|
}
|
|
@@ -726,7 +739,7 @@ class WorkerAdapter {
|
|
|
726
739
|
this.destroyHttpStream(httpStream);
|
|
727
740
|
return;
|
|
728
741
|
}
|
|
729
|
-
await ((
|
|
742
|
+
await ((_d = this.getRepo('ssor_attachment')) === null || _d === void 0 ? void 0 : _d.push([ssorAttachment]));
|
|
730
743
|
return;
|
|
731
744
|
}
|
|
732
745
|
return {
|
|
@@ -785,7 +798,7 @@ class WorkerAdapter {
|
|
|
785
798
|
}
|
|
786
799
|
return {
|
|
787
800
|
report: {
|
|
788
|
-
item_type: '
|
|
801
|
+
item_type: 'attachment',
|
|
789
802
|
[loading_1.ActionType.CREATED]: 1,
|
|
790
803
|
},
|
|
791
804
|
};
|
|
@@ -794,7 +807,7 @@ class WorkerAdapter {
|
|
|
794
807
|
console.warn('Failed to create attachment in external system', error);
|
|
795
808
|
return {
|
|
796
809
|
report: {
|
|
797
|
-
item_type: '
|
|
810
|
+
item_type: 'attachment',
|
|
798
811
|
[loading_1.ActionType.FAILED]: 1,
|
|
799
812
|
},
|
|
800
813
|
};
|
|
@@ -898,6 +911,9 @@ class WorkerAdapter {
|
|
|
898
911
|
if (attachmentsMetadata.lastProcessedAttachmentsIdsList) {
|
|
899
912
|
attachmentsMetadata.lastProcessedAttachmentsIdsList.length = 0;
|
|
900
913
|
}
|
|
914
|
+
if (attachmentsMetadata.failedAttachmentsIdsList) {
|
|
915
|
+
attachmentsMetadata.failedAttachmentsIdsList.length = 0;
|
|
916
|
+
}
|
|
901
917
|
}
|
|
902
918
|
return;
|
|
903
919
|
});
|
|
@@ -553,7 +553,7 @@ describe(`${worker_adapter_1.WorkerAdapter.name}.loadAttachment`, () => {
|
|
|
553
553
|
create,
|
|
554
554
|
});
|
|
555
555
|
// Assert
|
|
556
|
-
expect((_a = result.report) === null || _a === void 0 ? void 0 : _a.item_type).toBe('
|
|
556
|
+
expect((_a = result.report) === null || _a === void 0 ? void 0 : _a.item_type).toBe('attachment');
|
|
557
557
|
expect((_b = result.report) === null || _b === void 0 ? void 0 : _b[loading_1.ActionType.CREATED]).toBe(1);
|
|
558
558
|
});
|
|
559
559
|
it('should still return CREATED even when mapper create fails — attachment loading is resilient', async () => {
|
|
@@ -584,7 +584,7 @@ describe(`${worker_adapter_1.WorkerAdapter.name}.loadAttachment`, () => {
|
|
|
584
584
|
expect((_a = result.rateLimit) === null || _a === void 0 ? void 0 : _a.delay).toBe(30);
|
|
585
585
|
});
|
|
586
586
|
it('should return a FAILED report when create returns neither id nor delay', async () => {
|
|
587
|
-
var _a
|
|
587
|
+
var _a;
|
|
588
588
|
// Arrange
|
|
589
589
|
const create = jest.fn().mockResolvedValue({ id: null, delay: null });
|
|
590
590
|
// Act
|
|
@@ -593,7 +593,6 @@ describe(`${worker_adapter_1.WorkerAdapter.name}.loadAttachment`, () => {
|
|
|
593
593
|
create,
|
|
594
594
|
});
|
|
595
595
|
// Assert
|
|
596
|
-
expect((_a = result.report) === null || _a === void 0 ? void 0 : _a.
|
|
597
|
-
expect((_b = result.report) === null || _b === void 0 ? void 0 : _b[loading_1.ActionType.FAILED]).toBe(1);
|
|
596
|
+
expect((_a = result.report) === null || _a === void 0 ? void 0 : _a[loading_1.ActionType.FAILED]).toBe(1);
|
|
598
597
|
});
|
|
599
598
|
});
|
|
@@ -37,6 +37,7 @@ export interface ToDevRev {
|
|
|
37
37
|
artifactIds: string[];
|
|
38
38
|
lastProcessed: number;
|
|
39
39
|
lastProcessedAttachmentsIdsList?: ProcessedAttachment[];
|
|
40
|
+
failedAttachmentsIdsList?: FailedAttachment[];
|
|
40
41
|
};
|
|
41
42
|
}
|
|
42
43
|
/**
|
|
@@ -46,6 +47,15 @@ export interface ProcessedAttachment {
|
|
|
46
47
|
id: string;
|
|
47
48
|
parent_id: string;
|
|
48
49
|
}
|
|
50
|
+
/**
|
|
51
|
+
* Attachment that exhausted its transient-error retry budget (ECONNABORTED, 5xx) within a
|
|
52
|
+
* single invocation. Persisted across invocations so it's permanently skipped instead of
|
|
53
|
+
* being retried forever on a deterministically-failing request.
|
|
54
|
+
*/
|
|
55
|
+
export interface FailedAttachment {
|
|
56
|
+
id: string;
|
|
57
|
+
parent_id: string;
|
|
58
|
+
}
|
|
49
59
|
export interface FromDevRev {
|
|
50
60
|
filesToLoad: FileToLoad[];
|
|
51
61
|
}
|
|
@@ -68,6 +78,7 @@ export declare const extractionSdkState: {
|
|
|
68
78
|
artifactIds: never[];
|
|
69
79
|
lastProcessed: number;
|
|
70
80
|
lastProcessedAttachmentsIdsList: never[];
|
|
81
|
+
failedAttachmentsIdsList: never[];
|
|
71
82
|
};
|
|
72
83
|
};
|
|
73
84
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"state.interfaces.d.ts","sourceRoot":"","sources":["../../src/state/state.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD,MAAM,WAAW,QAAQ;IACvB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,yBAAyB,CAAC,EAAE,MAAM,CAAC;IACnC;gHAC4G;IAC5G,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;gHAC4G;IAC5G,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,2DAA2D;IAC3D,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,2DAA2D;IAC3D,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED;;;GAGG;AACH,MAAM,MAAM,YAAY,CAAC,cAAc,IAAI,cAAc,GAAG,QAAQ,CAAC;AAErE,MAAM,WAAW,QAAQ;IACvB,mBAAmB,EAAE;QACnB,WAAW,EAAE,MAAM,EAAE,CAAC;QACtB,aAAa,EAAE,MAAM,CAAC;QACtB,+BAA+B,CAAC,EAAE,mBAAmB,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"state.interfaces.d.ts","sourceRoot":"","sources":["../../src/state/state.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD,MAAM,WAAW,QAAQ;IACvB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,yBAAyB,CAAC,EAAE,MAAM,CAAC;IACnC;gHAC4G;IAC5G,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;gHAC4G;IAC5G,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,2DAA2D;IAC3D,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,2DAA2D;IAC3D,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED;;;GAGG;AACH,MAAM,MAAM,YAAY,CAAC,cAAc,IAAI,cAAc,GAAG,QAAQ,CAAC;AAErE,MAAM,WAAW,QAAQ;IACvB,mBAAmB,EAAE;QACnB,WAAW,EAAE,MAAM,EAAE,CAAC;QACtB,aAAa,EAAE,MAAM,CAAC;QACtB,+BAA+B,CAAC,EAAE,mBAAmB,EAAE,CAAC;QACxD,wBAAwB,CAAC,EAAE,gBAAgB,EAAE,CAAC;KAC/C,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;;;GAIG;AACH,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,UAAU;IACzB,WAAW,EAAE,UAAU,EAAE,CAAC;CAC3B;AAED,MAAM,WAAW,cAAc,CAAC,cAAc;IAC5C,KAAK,EAAE,YAAY,CAAC;IACpB,YAAY,EAAE,cAAc,CAAC;IAC7B,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;;;;CAgB9B,CAAC;AAEF,eAAO,MAAM,eAAe;;;;;CAK3B,CAAC"}
|
|
@@ -465,7 +465,14 @@ export interface ExternalSystemAttachmentStreamingParams {
|
|
|
465
465
|
}
|
|
466
466
|
export interface ExternalSystemAttachmentStreamingResponse {
|
|
467
467
|
httpStream?: AxiosResponse;
|
|
468
|
-
error?: ErrorRecord
|
|
468
|
+
error?: ErrorRecord & {
|
|
469
|
+
/**
|
|
470
|
+
* The HTTP status code of the failed request, if the error originated from one
|
|
471
|
+
* (e.g. a non-2xx response from the external system). Used to classify the error
|
|
472
|
+
* as transient (e.g. 5xx) so it counts toward the attachment's failure limit.
|
|
473
|
+
*/
|
|
474
|
+
statusCode?: number;
|
|
475
|
+
};
|
|
469
476
|
delay?: number;
|
|
470
477
|
}
|
|
471
478
|
export interface StreamAttachmentsResponse {
|
|
@@ -478,6 +485,18 @@ export type ProcessAttachmentReturnType = {
|
|
|
478
485
|
error?: {
|
|
479
486
|
message: string;
|
|
480
487
|
fileSize?: number;
|
|
488
|
+
/**
|
|
489
|
+
* Whether the error is a transient upload failure (ECONNABORTED, 5xx) expected
|
|
490
|
+
* to keep recurring for this attachment on retry. Used to count repeated
|
|
491
|
+
* failures per attachment so it can eventually be skipped.
|
|
492
|
+
*/
|
|
493
|
+
isTransient?: boolean;
|
|
494
|
+
/**
|
|
495
|
+
* A fixed allowlist of structural response headers (content-length, content-type,
|
|
496
|
+
* etc.) from the attachment fetch, for diagnostics. Deliberately excludes any
|
|
497
|
+
* header not on the allowlist so auth-adjacent headers never end up here.
|
|
498
|
+
*/
|
|
499
|
+
fetchHeaders?: Record<string, string>;
|
|
481
500
|
};
|
|
482
501
|
} | undefined;
|
|
483
502
|
export type StreamAttachmentsReturnType = {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extraction.d.ts","sourceRoot":"","sources":["../../src/types/extraction.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,sCAAsC,CAAC;AAEjE,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AACtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,iDAAiD,CAAC;AAChF,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAE7D;;;GAGG;AACH,oBAAY,SAAS;IAEnB;;OAEG;IACH,gCAAgC,yCAAyC;IACzE;;OAEG;IACH,uBAAuB,8BAA8B;IACrD;;OAEG;IACH,mBAAmB,0BAA0B;IAC7C;;OAEG;IACH,sBAAsB,6BAA6B;IACnD;;OAEG;IACH,oBAAoB,2BAA2B;IAC/C;;OAEG;IACH,0BAA0B,iCAAiC;IAC3D;;OAEG;IACH,6BAA6B,oCAAoC;IACjE;;OAEG;IACH,2BAA2B,kCAAkC;IAG7D,gBAAgB,uBAAuB;IACvC,mBAAmB,0BAA0B;IAC7C,uBAAuB,8BAA8B;IACrD,0BAA0B,iCAAiC;IAC3D,wBAAwB,gCAAgC;IACxD,kCAAkC,2CAA2C;IAG7E,gBAAgB,uBAAuB;IAGvC,gCAAgC,yCAAyC;IACzE,uBAAuB,8BAA8B;IACrD,mBAAmB,0BAA0B;IAC7C,sBAAsB,6BAA6B;IACnD,2BAA2B,mCAAmC;IAC9D,0BAA0B,iCAAiC;IAC3D,6BAA6B,oCAAoC;IACjE,sCAAsC,+CAA+C;CACtF;AAED;;;GAGG;AACH,oBAAY,kBAAkB;IAE5B;;OAEG;IACH,+BAA+B,wCAAwC;IACvE;;OAEG;IACH,gCAAgC,yCAAyC;IACzE;;OAEG;IACH,sBAAsB,6BAA6B;IACnD;;OAEG;IACH,uBAAuB,8BAA8B;IACrD;;OAEG;IACH,sBAAsB,6BAA6B;IACnD;;OAEG;IACH,mBAAmB,0BAA0B;IAC7C;;OAEG;IACH,kBAAkB,yBAAyB;IAC3C;;OAEG;IACH,mBAAmB,0BAA0B;IAC7C;;OAEG;IACH,wBAAwB,gCAAgC;IACxD;;OAEG;IACH,yBAAyB,iCAAiC;IAC1D;;OAEG;IACH,6BAA6B,oCAAoC;IACjE;;OAEG;IACH,0BAA0B,iCAAiC;IAC3D;;OAEG;IACH,yBAAyB,gCAAgC;IACzD;;OAEG;IACH,0BAA0B,iCAAiC;IAC3D;;OAEG;IACH,+BAA+B,uCAAuC;IACtE;;OAEG;IACH,gCAAgC,wCAAwC;IAGxE,gBAAgB,uBAAuB;IAGvC,8BAA8B,uCAAuC;IACrE,+BAA+B,wCAAwC;IACvE,sBAAsB,6BAA6B;IACnD,uBAAuB,8BAA8B;IACrD,sBAAsB,6BAA6B;IACnD,qBAAqB,4BAA4B;IACjD,kBAAkB,yBAAyB;IAC3C,mBAAmB,0BAA0B;IAC7C,0BAA0B,kCAAkC;IAC5D,2BAA2B,mCAAmC;IAC9D,4BAA4B,mCAAmC;IAC/D,2BAA2B,kCAAkC;IAC7D,wBAAwB,+BAA+B;IACvD,yBAAyB,gCAAgC;IACzD,qCAAqC,8CAA8C;IACnF,sCAAsC,+CAA+C;CACtF;AAED;;;;GAIG;AACH,oBAAY,cAAc;IACxB,OAAO,YAAY;IACnB,WAAW,gBAAgB;CAC5B;AAED;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,oBAAY,gBAAgB;IAC1B,YAAY,iBAAiB;IAC7B,WAAW,gBAAgB;CAC5B;AAED;;;GAGG;AACH,oBAAY,QAAQ;IAClB,kBAAkB;IAClB,WAAW,OAAO;IAClB,iCAAiC;IACjC,YAAY,OAAO;IACnB,mCAAmC;IACnC,eAAe,YAAO;IACtB,mBAAmB;IACnB,YAAY,OAAO;IACnB,cAAc;IACd,OAAO,MAAM;IACb,cAAc;IACd,OAAO,MAAM;IACb,YAAY;IACZ,KAAK,MAAM;CACZ;AAED;;;GAGG;AACH,oBAAY,aAAa;IACvB,yCAAyC;IACzC,cAAc,mBAAmB;IACjC,iEAAiE;IACjE,2BAA2B,gCAAgC;IAC3D,yCAAyC;IACzC,cAAc,mBAAmB;IACjC,gEAAgE;IAChE,0BAA0B,+BAA+B;IACzD,mBAAmB;IACnB,YAAY,iBAAiB;IAC7B,wCAAwC;IACxC,aAAa,kBAAkB;IAC/B,4CAA4C;IAC5C,SAAS,cAAc;CACxB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,aAAa,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,uBAAuB,EAAE,MAAM,CAAC;IAChC,eAAe,EAAE,MAAM,CAAC;IACxB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,KAAK,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,CAAC;CACzB;AAED;;;GAGG;AACH,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,gBAAgB,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,kBAAkB,EAAE,MAAM,CAAC;IAC3B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,uBAAuB,EAAE,MAAM,CAAC;IAChC;;OAEG;IACH,eAAe,EAAE,MAAM,CAAC;IACxB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;IAC7B;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,kBAAkB,CAAC,EAAE,gBAAgB,CAAC;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,gBAAgB,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B;;;OAGG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,kBAAkB,EAAE,MAAM,CAAC;IAC3B;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,KAAK,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,SAAS,CAAC;IAClC;;;OAGG;IACH,mBAAmB,CAAC,EAAE,SAAS,CAAC;IAChC;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,gBAAgB,EAAE,CAAC;IACzC;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IAGvB,OAAO,CAAC,EAAE,YAAY,EAAE,CAAC;IACzB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAG/B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAI9B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED;;;;GAIG;AACH,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE;QACN,KAAK,EAAE,MAAM,EAAE,CAAC;KACjB,CAAC;IACF,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;GAGG;AACH,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE;QACP,OAAO,EAAE;YACP,qBAAqB,EAAE,MAAM,CAAC;SAC/B,CAAC;QACF,kBAAkB,EAAE,MAAM,CAAC;QAC3B,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,OAAO,EAAE,cAAc,CAAC;IACxB,kBAAkB,EAAE;QAClB,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;IACF,UAAU,EAAE,SAAS,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,eAAe,EAAE,cAAc,CAAC;IAChC,aAAa,EAAE,YAAY,CAAC;IAC5B,UAAU,EAAE,SAAS,CAAC;IACtB,UAAU,CAAC,EAAE,SAAS,CAAC;CACxB;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,YAAY,CAAC;IAC5B,UAAU,CAAC,EAAE,SAAS,CAAC;IACvB,eAAe,CAAC,EAAE,cAAc,CAAC;CAClC;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,YAAY,CAAC;IAC5B,UAAU,CAAC,EAAE,SAAS,CAAC;IACvB,eAAe,CAAC,EAAE,cAAc,CAAC;CAClC;AAED,MAAM,MAAM,yCAAyC,GAAG,CAAC,EACvD,IAAI,EACJ,KAAK,GACN,EAAE,uCAAuC,KAAK,OAAO,CAAC,yCAAyC,CAAC,CAAC;AAElG,MAAM,WAAW,uCAAuC;IACtD,IAAI,EAAE,oBAAoB,CAAC;IAC3B,KAAK,EAAE,YAAY,CAAC;CACrB;AAED,MAAM,WAAW,yCAAyC;IACxD,UAAU,CAAC,EAAE,aAAa,CAAC;IAC3B,KAAK,CAAC,EAAE,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"extraction.d.ts","sourceRoot":"","sources":["../../src/types/extraction.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,sCAAsC,CAAC;AAEjE,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AACtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,iDAAiD,CAAC;AAChF,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAE7D;;;GAGG;AACH,oBAAY,SAAS;IAEnB;;OAEG;IACH,gCAAgC,yCAAyC;IACzE;;OAEG;IACH,uBAAuB,8BAA8B;IACrD;;OAEG;IACH,mBAAmB,0BAA0B;IAC7C;;OAEG;IACH,sBAAsB,6BAA6B;IACnD;;OAEG;IACH,oBAAoB,2BAA2B;IAC/C;;OAEG;IACH,0BAA0B,iCAAiC;IAC3D;;OAEG;IACH,6BAA6B,oCAAoC;IACjE;;OAEG;IACH,2BAA2B,kCAAkC;IAG7D,gBAAgB,uBAAuB;IACvC,mBAAmB,0BAA0B;IAC7C,uBAAuB,8BAA8B;IACrD,0BAA0B,iCAAiC;IAC3D,wBAAwB,gCAAgC;IACxD,kCAAkC,2CAA2C;IAG7E,gBAAgB,uBAAuB;IAGvC,gCAAgC,yCAAyC;IACzE,uBAAuB,8BAA8B;IACrD,mBAAmB,0BAA0B;IAC7C,sBAAsB,6BAA6B;IACnD,2BAA2B,mCAAmC;IAC9D,0BAA0B,iCAAiC;IAC3D,6BAA6B,oCAAoC;IACjE,sCAAsC,+CAA+C;CACtF;AAED;;;GAGG;AACH,oBAAY,kBAAkB;IAE5B;;OAEG;IACH,+BAA+B,wCAAwC;IACvE;;OAEG;IACH,gCAAgC,yCAAyC;IACzE;;OAEG;IACH,sBAAsB,6BAA6B;IACnD;;OAEG;IACH,uBAAuB,8BAA8B;IACrD;;OAEG;IACH,sBAAsB,6BAA6B;IACnD;;OAEG;IACH,mBAAmB,0BAA0B;IAC7C;;OAEG;IACH,kBAAkB,yBAAyB;IAC3C;;OAEG;IACH,mBAAmB,0BAA0B;IAC7C;;OAEG;IACH,wBAAwB,gCAAgC;IACxD;;OAEG;IACH,yBAAyB,iCAAiC;IAC1D;;OAEG;IACH,6BAA6B,oCAAoC;IACjE;;OAEG;IACH,0BAA0B,iCAAiC;IAC3D;;OAEG;IACH,yBAAyB,gCAAgC;IACzD;;OAEG;IACH,0BAA0B,iCAAiC;IAC3D;;OAEG;IACH,+BAA+B,uCAAuC;IACtE;;OAEG;IACH,gCAAgC,wCAAwC;IAGxE,gBAAgB,uBAAuB;IAGvC,8BAA8B,uCAAuC;IACrE,+BAA+B,wCAAwC;IACvE,sBAAsB,6BAA6B;IACnD,uBAAuB,8BAA8B;IACrD,sBAAsB,6BAA6B;IACnD,qBAAqB,4BAA4B;IACjD,kBAAkB,yBAAyB;IAC3C,mBAAmB,0BAA0B;IAC7C,0BAA0B,kCAAkC;IAC5D,2BAA2B,mCAAmC;IAC9D,4BAA4B,mCAAmC;IAC/D,2BAA2B,kCAAkC;IAC7D,wBAAwB,+BAA+B;IACvD,yBAAyB,gCAAgC;IACzD,qCAAqC,8CAA8C;IACnF,sCAAsC,+CAA+C;CACtF;AAED;;;;GAIG;AACH,oBAAY,cAAc;IACxB,OAAO,YAAY;IACnB,WAAW,gBAAgB;CAC5B;AAED;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,oBAAY,gBAAgB;IAC1B,YAAY,iBAAiB;IAC7B,WAAW,gBAAgB;CAC5B;AAED;;;GAGG;AACH,oBAAY,QAAQ;IAClB,kBAAkB;IAClB,WAAW,OAAO;IAClB,iCAAiC;IACjC,YAAY,OAAO;IACnB,mCAAmC;IACnC,eAAe,YAAO;IACtB,mBAAmB;IACnB,YAAY,OAAO;IACnB,cAAc;IACd,OAAO,MAAM;IACb,cAAc;IACd,OAAO,MAAM;IACb,YAAY;IACZ,KAAK,MAAM;CACZ;AAED;;;GAGG;AACH,oBAAY,aAAa;IACvB,yCAAyC;IACzC,cAAc,mBAAmB;IACjC,iEAAiE;IACjE,2BAA2B,gCAAgC;IAC3D,yCAAyC;IACzC,cAAc,mBAAmB;IACjC,gEAAgE;IAChE,0BAA0B,+BAA+B;IACzD,mBAAmB;IACnB,YAAY,iBAAiB;IAC7B,wCAAwC;IACxC,aAAa,kBAAkB;IAC/B,4CAA4C;IAC5C,SAAS,cAAc;CACxB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,aAAa,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,uBAAuB,EAAE,MAAM,CAAC;IAChC,eAAe,EAAE,MAAM,CAAC;IACxB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,KAAK,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,CAAC;CACzB;AAED;;;GAGG;AACH,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,gBAAgB,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,kBAAkB,EAAE,MAAM,CAAC;IAC3B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,uBAAuB,EAAE,MAAM,CAAC;IAChC;;OAEG;IACH,eAAe,EAAE,MAAM,CAAC;IACxB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;IAC7B;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,kBAAkB,CAAC,EAAE,gBAAgB,CAAC;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,gBAAgB,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B;;;OAGG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,kBAAkB,EAAE,MAAM,CAAC;IAC3B;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,KAAK,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,SAAS,CAAC;IAClC;;;OAGG;IACH,mBAAmB,CAAC,EAAE,SAAS,CAAC;IAChC;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,gBAAgB,EAAE,CAAC;IACzC;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IAGvB,OAAO,CAAC,EAAE,YAAY,EAAE,CAAC;IACzB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAG/B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAI9B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED;;;;GAIG;AACH,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE;QACN,KAAK,EAAE,MAAM,EAAE,CAAC;KACjB,CAAC;IACF,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;GAGG;AACH,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE;QACP,OAAO,EAAE;YACP,qBAAqB,EAAE,MAAM,CAAC;SAC/B,CAAC;QACF,kBAAkB,EAAE,MAAM,CAAC;QAC3B,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,OAAO,EAAE,cAAc,CAAC;IACxB,kBAAkB,EAAE;QAClB,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;IACF,UAAU,EAAE,SAAS,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,eAAe,EAAE,cAAc,CAAC;IAChC,aAAa,EAAE,YAAY,CAAC;IAC5B,UAAU,EAAE,SAAS,CAAC;IACtB,UAAU,CAAC,EAAE,SAAS,CAAC;CACxB;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,YAAY,CAAC;IAC5B,UAAU,CAAC,EAAE,SAAS,CAAC;IACvB,eAAe,CAAC,EAAE,cAAc,CAAC;CAClC;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,YAAY,CAAC;IAC5B,UAAU,CAAC,EAAE,SAAS,CAAC;IACvB,eAAe,CAAC,EAAE,cAAc,CAAC;CAClC;AAED,MAAM,MAAM,yCAAyC,GAAG,CAAC,EACvD,IAAI,EACJ,KAAK,GACN,EAAE,uCAAuC,KAAK,OAAO,CAAC,yCAAyC,CAAC,CAAC;AAElG,MAAM,WAAW,uCAAuC;IACtD,IAAI,EAAE,oBAAoB,CAAC;IAC3B,KAAK,EAAE,YAAY,CAAC;CACrB;AAED,MAAM,WAAW,yCAAyC;IACxD,UAAU,CAAC,EAAE,aAAa,CAAC;IAC3B,KAAK,CAAC,EAAE,WAAW,GAAG;QACpB;;;;WAIG;QACH,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,CAAC;IACF,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,yBAAyB;IACxC,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,SAAS,CAAC,EAAE,WAAW,CAAC;CACzB;AAED,MAAM,MAAM,2BAA2B,GACnC;IACE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE;QACN,OAAO,EAAE,MAAM,CAAC;QAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB;;;;WAIG;QACH,WAAW,CAAC,EAAE,OAAO,CAAC;QACtB;;;;WAIG;QACH,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KACvC,CAAC;CACH,GACD,SAAS,CAAC;AAEd,MAAM,MAAM,2BAA2B,GACnC;IACE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GACD,SAAS,CAAC;AAEd,MAAM,MAAM,uCAAuC,CACjD,KAAK,EACL,QAAQ,EACR,cAAc,IACZ,CAAC,EACH,WAAW,EACX,OAAO,EACP,SAAS,GACV,EAAE;IACD,WAAW,EAAE,KAAK,CAAC;IACnB,OAAO,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;IACvC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,KAAK,QAAQ,CAAC;AAEf,MAAM,MAAM,iCAAiC,GAAG,CAAC,EAC/C,UAAU,EACV,MAAM,GACP,EAAE;IACD,UAAU,EAAE,oBAAoB,CAAC;IACjC,MAAM,EAAE,yCAAyC,CAAC;CACnD,KAAK,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAE3C,MAAM,MAAM,wCAAwC,CAAC,QAAQ,EAAE,cAAc,IAC3E,CAAC,EACC,kBAAkB,EAClB,OAAO,EACP,MAAM,GACP,EAAE;IACD,kBAAkB,EAAE,QAAQ,CAAC;IAC7B,OAAO,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;IACvC,MAAM,EAAE,yCAAyC,CAAC;CACnD,KAAK,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAE7C,MAAM,WAAW,kCAAkC,CACjD,cAAc,EACd,KAAK,EACL,QAAQ;IAER,OAAO,EAAE,uCAAuC,CAC9C,KAAK,EACL,QAAQ,EACR,cAAc,CACf,CAAC;IACF,QAAQ,EAAE,wCAAwC,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;CAC9E"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAMtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAW/D,OAAO,EAEL,gBAAgB,EAChB,cAAc,EACd,wBAAwB,EACxB,cAAc,EACf,MAAM,uBAAuB,CAAC;AAE/B,qBAAa,QAAQ;IACnB,OAAO,CAAC,kBAAkB,CAAC,CAAU;IACrC,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,cAAc,CAAyB;IAC/C,OAAO,CAAC,gBAAgB,CAAU;gBAEtB,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,wBAAwB;IAWxD;;;;;OAKG;IACG,MAAM,CACV,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,EAAE,GAAG,MAAM,GAChC,OAAO,CAAC,cAAc,CAAC;IA+E1B;;;;;;OAMG;IACG,oBAAoB,CACxB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;IA2B5C;;;;;OAKG;IACG,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAmBzC;;;;;OAKG;IACG,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,UAAU,EAAE,aAAa,GACxB,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAsCzC;;;;OAIG;IACG,qBAAqB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC;QACvD,QAAQ,CAAC,EAAE,aAAa,CAAC;QACzB,KAAK,CAAC,EAAE,OAAO,CAAC;KACjB,CAAC;
|
|
1
|
+
{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAMtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAW/D,OAAO,EAEL,gBAAgB,EAChB,cAAc,EACd,wBAAwB,EACxB,cAAc,EACf,MAAM,uBAAuB,CAAC;AAE/B,qBAAa,QAAQ;IACnB,OAAO,CAAC,kBAAkB,CAAC,CAAU;IACrC,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,cAAc,CAAyB;IAC/C,OAAO,CAAC,gBAAgB,CAAU;gBAEtB,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,wBAAwB;IAWxD;;;;;OAKG;IACG,MAAM,CACV,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,EAAE,GAAG,MAAM,GAChC,OAAO,CAAC,cAAc,CAAC;IA+E1B;;;;;;OAMG;IACG,oBAAoB,CACxB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;IA2B5C;;;;;OAKG;IACG,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAmBzC;;;;;OAKG;IACG,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,UAAU,EAAE,aAAa,GACxB,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAsCzC;;;;OAIG;IACG,qBAAqB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC;QACvD,QAAQ,CAAC,EAAE,aAAa,CAAC;QACzB,KAAK,CAAC,EAAE,OAAO,CAAC;KACjB,CAAC;IA2BF;;;OAGG;IACH,OAAO,CAAC,aAAa;IAerB;;;;;OAKG;IACG,4BAA4B,CAAC,EACjC,QAAQ,GACT,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;KAClB,GAAG,OAAO,CAAC;QACV,WAAW,CAAC,EAAE,oBAAoB,EAAE,CAAC;QACrC,KAAK,CAAC,EAAE;YAAE,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC;KAC7B,CAAC;IA2DF;;;;OAIG;YACW,sBAAsB;IAsBpC;;;;OAIG;YACW,gBAAgB;IAc9B;;;;;;OAMG;IACG,yBAAyB,CAAC,EAC9B,UAAU,EACV,SAAiB,GAClB,EAAE;QACD,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,CAAC,EAAE,OAAO,CAAC;KACrB,GAAG,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC;CA6B/C"}
|
|
@@ -1,4 +1,12 @@
|
|
|
1
1
|
import { ArtifactDateRanges, UploaderResult } from './uploader.interfaces';
|
|
2
|
+
/**
|
|
3
|
+
* Determines whether an error from the SDK-owned upload path (get upload URL, stream
|
|
4
|
+
* artifact, confirm upload) is a transient failure that's expected to keep recurring on
|
|
5
|
+
* retry for the same attachment: a client-side timeout (ECONNABORTED) or a 5xx response.
|
|
6
|
+
* Used to count repeated failures per attachment so it can eventually be skipped instead
|
|
7
|
+
* of retried forever.
|
|
8
|
+
*/
|
|
9
|
+
export declare function isTransientUploadError(error: unknown): boolean;
|
|
2
10
|
/**
|
|
3
11
|
* Computes oldest/newest created and modified timestamps (RFC3339) across uploaded items.
|
|
4
12
|
* @param fetchedObjects - Single object or array of objects (e.g. NormalizedItem[])
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"uploader.helpers.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.helpers.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"uploader.helpers.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.helpers.ts"],"names":[],"mappings":"AAUA,OAAO,EAEL,kBAAkB,EAClB,cAAc,EACf,MAAM,uBAAuB,CAAC;AAE/B;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAK9D;AAED;;;GAGG;AACH,wBAAgB,yBAAyB,CACvC,cAAc,EAAE,MAAM,EAAE,GAAG,MAAM,GAChC,kBAAkB,CAwEpB;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,WAAW,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,CAMxE;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAC5B,kBAAkB,EAAE,MAAM,GACzB,cAAc,CAAC,MAAM,CAAC,CAOxB;AAED;;;;GAIG;AACH,wBAAgB,UAAU,CAAC,WAAW,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,CAMxE;AAED;;;;;GAKG;AACH,wBAAsB,eAAe,CACnC,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,GAAG,MAAM,EAAE,GAChC,OAAO,CAAC,IAAI,CAAC,CA4Bf;AAED;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAmBzD"}
|
|
@@ -36,17 +36,33 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
36
36
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
37
37
|
};
|
|
38
38
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
|
+
exports.isTransientUploadError = isTransientUploadError;
|
|
39
40
|
exports.computeArtifactDateRanges = computeArtifactDateRanges;
|
|
40
41
|
exports.compressGzip = compressGzip;
|
|
41
42
|
exports.decompressGzip = decompressGzip;
|
|
42
43
|
exports.parseJsonl = parseJsonl;
|
|
43
44
|
exports.downloadToLocal = downloadToLocal;
|
|
44
45
|
exports.truncateFilename = truncateFilename;
|
|
46
|
+
const axios_1 = __importDefault(require("axios"));
|
|
45
47
|
const fs_1 = __importStar(require("fs"));
|
|
46
48
|
const js_jsonl_1 = require("js-jsonl");
|
|
47
49
|
const zlib_1 = __importDefault(require("zlib"));
|
|
48
50
|
const constants_1 = require("../common/constants");
|
|
49
51
|
const uploader_interfaces_1 = require("./uploader.interfaces");
|
|
52
|
+
/**
|
|
53
|
+
* Determines whether an error from the SDK-owned upload path (get upload URL, stream
|
|
54
|
+
* artifact, confirm upload) is a transient failure that's expected to keep recurring on
|
|
55
|
+
* retry for the same attachment: a client-side timeout (ECONNABORTED) or a 5xx response.
|
|
56
|
+
* Used to count repeated failures per attachment so it can eventually be skipped instead
|
|
57
|
+
* of retried forever.
|
|
58
|
+
*/
|
|
59
|
+
function isTransientUploadError(error) {
|
|
60
|
+
var _a, _b;
|
|
61
|
+
if (!axios_1.default.isAxiosError(error)) {
|
|
62
|
+
return false;
|
|
63
|
+
}
|
|
64
|
+
return error.code === 'ECONNABORTED' || ((_b = (_a = error.response) === null || _a === void 0 ? void 0 : _a.status) !== null && _b !== void 0 ? _b : 0) >= 500;
|
|
65
|
+
}
|
|
50
66
|
/**
|
|
51
67
|
* Computes oldest/newest created and modified timestamps (RFC3339) across uploaded items.
|
|
52
68
|
* @param fetchedObjects - Single object or array of objects (e.g. NormalizedItem[])
|
|
@@ -394,4 +394,43 @@ describe('uploader.helpers', () => {
|
|
|
394
394
|
expect(result.length).toBe(constants_1.MAX_DEVREV_FILENAME_LENGTH);
|
|
395
395
|
});
|
|
396
396
|
});
|
|
397
|
+
describe(uploader_helpers_1.isTransientUploadError.name, () => {
|
|
398
|
+
it('should return true for ECONNABORTED errors', () => {
|
|
399
|
+
// Arrange
|
|
400
|
+
const error = {
|
|
401
|
+
isAxiosError: true,
|
|
402
|
+
code: 'ECONNABORTED',
|
|
403
|
+
};
|
|
404
|
+
// Act & Assert
|
|
405
|
+
expect((0, uploader_helpers_1.isTransientUploadError)(error)).toBe(true);
|
|
406
|
+
});
|
|
407
|
+
it('should return true for 5xx response errors', () => {
|
|
408
|
+
// Arrange
|
|
409
|
+
const error = {
|
|
410
|
+
isAxiosError: true,
|
|
411
|
+
response: { status: 503 },
|
|
412
|
+
};
|
|
413
|
+
// Act & Assert
|
|
414
|
+
expect((0, uploader_helpers_1.isTransientUploadError)(error)).toBe(true);
|
|
415
|
+
});
|
|
416
|
+
it('should return false for 4xx response errors', () => {
|
|
417
|
+
// Arrange
|
|
418
|
+
const error = {
|
|
419
|
+
isAxiosError: true,
|
|
420
|
+
response: { status: 404 },
|
|
421
|
+
};
|
|
422
|
+
// Act & Assert
|
|
423
|
+
expect((0, uploader_helpers_1.isTransientUploadError)(error)).toBe(false);
|
|
424
|
+
});
|
|
425
|
+
it('should return false for non-axios errors', () => {
|
|
426
|
+
// Arrange
|
|
427
|
+
const error = new Error('some other error');
|
|
428
|
+
// Act & Assert
|
|
429
|
+
expect((0, uploader_helpers_1.isTransientUploadError)(error)).toBe(false);
|
|
430
|
+
});
|
|
431
|
+
it('[edge] should return false when error is undefined', () => {
|
|
432
|
+
// Act & Assert
|
|
433
|
+
expect((0, uploader_helpers_1.isTransientUploadError)(undefined)).toBe(false);
|
|
434
|
+
});
|
|
435
|
+
});
|
|
397
436
|
});
|
|
@@ -73,7 +73,7 @@ class Uploader {
|
|
|
73
73
|
return {
|
|
74
74
|
error: {
|
|
75
75
|
message: 'Error while confirming artifact upload. ' +
|
|
76
|
-
|
|
76
|
+
(0, logger_1.serializeError)(confirmArtifactUploadError),
|
|
77
77
|
},
|
|
78
78
|
};
|
|
79
79
|
}
|
|
@@ -185,16 +185,11 @@ class Uploader {
|
|
|
185
185
|
return { response };
|
|
186
186
|
}
|
|
187
187
|
else {
|
|
188
|
-
return {
|
|
189
|
-
error: {
|
|
190
|
-
message: 'Error while confirming artifact upload. ' +
|
|
191
|
-
(0, logger_1.serializeError)(response),
|
|
192
|
-
},
|
|
193
|
-
};
|
|
188
|
+
return { error: response };
|
|
194
189
|
}
|
|
195
190
|
}
|
|
196
191
|
catch (error) {
|
|
197
|
-
return { error
|
|
192
|
+
return { error };
|
|
198
193
|
}
|
|
199
194
|
}
|
|
200
195
|
/**
|
package/package.json
CHANGED