@devrev/ts-adaas 1.19.11-beta.3 → 1.19.11-beta.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/attachments-streaming/attachments-streaming-pool.d.ts +5 -10
- package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map +1 -1
- package/dist/attachments-streaming/attachments-streaming-pool.js +31 -54
- package/dist/attachments-streaming/attachments-streaming-pool.test.js +78 -83
- package/dist/multithreading/worker-adapter/worker-adapter.d.ts.map +1 -1
- package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.js +1 -63
- package/dist/multithreading/worker-adapter/worker-adapter.helpers.d.ts +0 -6
- package/dist/multithreading/worker-adapter/worker-adapter.helpers.d.ts.map +1 -1
- package/dist/multithreading/worker-adapter/worker-adapter.helpers.js +0 -29
- package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.js +0 -46
- package/dist/multithreading/worker-adapter/worker-adapter.js +7 -23
- package/dist/multithreading/worker-adapter/worker-adapter.loading.test.js +4 -3
- package/dist/state/state.interfaces.d.ts +8 -12
- package/dist/state/state.interfaces.d.ts.map +1 -1
- package/dist/state/state.interfaces.js +7 -2
- package/dist/types/extraction.d.ts +1 -17
- package/dist/types/extraction.d.ts.map +1 -1
- package/dist/uploader/uploader.helpers.d.ts +0 -8
- package/dist/uploader/uploader.helpers.d.ts.map +1 -1
- package/dist/uploader/uploader.helpers.js +0 -16
- package/dist/uploader/uploader.helpers.test.js +0 -39
- package/package.json +1 -1
|
@@ -9,19 +9,14 @@ export declare class AttachmentsStreamingPool<ConnectorState> {
|
|
|
9
9
|
private totalProcessedCount;
|
|
10
10
|
private readonly PROGRESS_REPORT_INTERVAL;
|
|
11
11
|
constructor({ adapter, attachments, batchSize, stream, }: AttachmentsStreamingPoolParams<ConnectorState>);
|
|
12
|
-
|
|
13
|
-
* Marks an attachment as permanently failed after a transient error (ECONNABORTED,
|
|
14
|
-
* 5xx), so subsequent invocations skip it instead of retrying a deterministically-
|
|
15
|
-
* failing request forever. The stream function's own HTTP client (axiosClient) already
|
|
16
|
-
* retries transient errors with exponential backoff before returning one here, so a
|
|
17
|
-
* second retry loop at this level would just double up on an already-exhausted budget.
|
|
18
|
-
*/
|
|
19
|
-
private markPermanentlyFailed;
|
|
12
|
+
private recordProcessedAttachment;
|
|
20
13
|
private updateProgress;
|
|
21
14
|
/**
|
|
22
|
-
* Migrates processed attachments from
|
|
15
|
+
* Migrates processed attachments from older state shapes to the current
|
|
16
|
+
* ProcessedAttachment[] format: legacy string[] IDs, and entries recorded before the
|
|
17
|
+
* `status` field existed (which only ever held successes).
|
|
23
18
|
*
|
|
24
|
-
* @param attachments - The attachments list to migrate (
|
|
19
|
+
* @param attachments - The attachments list to migrate (string[] or partial ProcessedAttachment[])
|
|
25
20
|
* @returns Migrated array of ProcessedAttachment objects, or empty array if input is invalid
|
|
26
21
|
*/
|
|
27
22
|
private migrateProcessedAttachments;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"attachments-streaming-pool.d.ts","sourceRoot":"","sources":["../../src/attachments-streaming/attachments-streaming-pool.ts"],"names":[],"mappings":"AAMA,OAAO,EAGL,2BAA2B,EAC5B,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,8BAA8B,EAAE,MAAM,yCAAyC,CAAC;AAEzF,qBAAa,wBAAwB,CAAC,cAAc;IAClD,OAAO,CAAC,OAAO,CAAgC;IAC/C,OAAO,CAAC,WAAW,CAAyB;IAC5C,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,KAAK,CAAqB;IAClC,OAAO,CAAC,MAAM,CAA4C;IAE1D,OAAO,CAAC,mBAAmB,CAAa;IACxC,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAM;gBAEnC,EACV,OAAO,EACP,WAAW,EACX,SAAc,EACd,MAAM,GACP,EAAE,8BAA8B,CAAC,cAAc,CAAC;IAQjD
|
|
1
|
+
{"version":3,"file":"attachments-streaming-pool.d.ts","sourceRoot":"","sources":["../../src/attachments-streaming/attachments-streaming-pool.ts"],"names":[],"mappings":"AAMA,OAAO,EAGL,2BAA2B,EAC5B,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,8BAA8B,EAAE,MAAM,yCAAyC,CAAC;AAEzF,qBAAa,wBAAwB,CAAC,cAAc;IAClD,OAAO,CAAC,OAAO,CAAgC;IAC/C,OAAO,CAAC,WAAW,CAAyB;IAC5C,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,KAAK,CAAqB;IAClC,OAAO,CAAC,MAAM,CAA4C;IAE1D,OAAO,CAAC,mBAAmB,CAAa;IACxC,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAM;gBAEnC,EACV,OAAO,EACP,WAAW,EACX,SAAc,EACd,MAAM,GACP,EAAE,8BAA8B,CAAC,cAAc,CAAC;IAQjD,OAAO,CAAC,yBAAyB;YAyBnB,cAAc;IAS5B;;;;;;;OAOG;IAEH,OAAO,CAAC,2BAA2B;IAyB7B,SAAS,IAAI,OAAO,CAAC,2BAA2B,CAAC;IAgDjD,kBAAkB;CA6FzB"}
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.AttachmentsStreamingPool = void 0;
|
|
4
4
|
const helpers_1 = require("../common/helpers");
|
|
5
|
+
const state_interfaces_1 = require("../state/state.interfaces");
|
|
5
6
|
class AttachmentsStreamingPool {
|
|
6
7
|
constructor({ adapter, attachments, batchSize = 10, stream, }) {
|
|
7
8
|
this.totalProcessedCount = 0;
|
|
@@ -12,26 +13,19 @@ class AttachmentsStreamingPool {
|
|
|
12
13
|
this.delay = undefined;
|
|
13
14
|
this.stream = stream;
|
|
14
15
|
}
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
* 5xx), so subsequent invocations skip it instead of retrying a deterministically-
|
|
18
|
-
* failing request forever. The stream function's own HTTP client (axiosClient) already
|
|
19
|
-
* retries transient errors with exponential backoff before returning one here, so a
|
|
20
|
-
* second retry loop at this level would just double up on an already-exhausted budget.
|
|
21
|
-
*/
|
|
22
|
-
markPermanentlyFailed(attachment) {
|
|
23
|
-
var _a, _b;
|
|
16
|
+
recordProcessedAttachment(attachment, status) {
|
|
17
|
+
var _a;
|
|
24
18
|
const attachmentsMetadata = (_a = this.adapter.state.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata;
|
|
25
|
-
if (!attachmentsMetadata) {
|
|
19
|
+
if (!(attachmentsMetadata === null || attachmentsMetadata === void 0 ? void 0 : attachmentsMetadata.lastProcessedAttachmentsIdsList)) {
|
|
26
20
|
return;
|
|
27
21
|
}
|
|
28
|
-
const
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
failedAttachmentsIdsList.push({
|
|
22
|
+
const processedAttachmentsIdsList = attachmentsMetadata.lastProcessedAttachmentsIdsList;
|
|
23
|
+
const alreadyRecorded = processedAttachmentsIdsList.some((it) => it.id === attachment.id && it.parent_id === attachment.parent_id);
|
|
24
|
+
if (!alreadyRecorded) {
|
|
25
|
+
processedAttachmentsIdsList.push({
|
|
33
26
|
id: attachment.id,
|
|
34
27
|
parent_id: attachment.parent_id,
|
|
28
|
+
status,
|
|
35
29
|
});
|
|
36
30
|
}
|
|
37
31
|
}
|
|
@@ -44,28 +38,33 @@ class AttachmentsStreamingPool {
|
|
|
44
38
|
}
|
|
45
39
|
}
|
|
46
40
|
/**
|
|
47
|
-
* Migrates processed attachments from
|
|
41
|
+
* Migrates processed attachments from older state shapes to the current
|
|
42
|
+
* ProcessedAttachment[] format: legacy string[] IDs, and entries recorded before the
|
|
43
|
+
* `status` field existed (which only ever held successes).
|
|
48
44
|
*
|
|
49
|
-
* @param attachments - The attachments list to migrate (
|
|
45
|
+
* @param attachments - The attachments list to migrate (string[] or partial ProcessedAttachment[])
|
|
50
46
|
* @returns Migrated array of ProcessedAttachment objects, or empty array if input is invalid
|
|
51
47
|
*/
|
|
52
48
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
53
49
|
migrateProcessedAttachments(attachments) {
|
|
54
|
-
// Handle null/undefined
|
|
55
50
|
if (!attachments || !Array.isArray(attachments)) {
|
|
56
51
|
return [];
|
|
57
52
|
}
|
|
58
|
-
// If already migrated (first element is an object), return as-is
|
|
59
|
-
if (attachments.length > 0 && typeof attachments[0] === 'object') {
|
|
60
|
-
return attachments;
|
|
61
|
-
}
|
|
62
53
|
// Migrate old string[] format
|
|
63
54
|
if (attachments.length > 0 && typeof attachments[0] === 'string') {
|
|
64
55
|
return attachments.map((it) => ({
|
|
65
56
|
id: it,
|
|
66
57
|
parent_id: '',
|
|
58
|
+
status: state_interfaces_1.ProcessedAttachmentStatus.Success,
|
|
67
59
|
}));
|
|
68
60
|
}
|
|
61
|
+
// Backfill status on entries recorded before it existed
|
|
62
|
+
if (attachments.length > 0 && typeof attachments[0] === 'object') {
|
|
63
|
+
return attachments.map((it) => {
|
|
64
|
+
var _a;
|
|
65
|
+
return (Object.assign(Object.assign({}, it), { status: (_a = it.status) !== null && _a !== void 0 ? _a : state_interfaces_1.ProcessedAttachmentStatus.Success }));
|
|
66
|
+
});
|
|
67
|
+
}
|
|
69
68
|
return [];
|
|
70
69
|
}
|
|
71
70
|
async streamAll() {
|
|
@@ -75,23 +74,17 @@ class AttachmentsStreamingPool {
|
|
|
75
74
|
console.error(error);
|
|
76
75
|
return { error };
|
|
77
76
|
}
|
|
78
|
-
// Get the list of
|
|
79
|
-
// If no such list exists, create an empty one.
|
|
77
|
+
// Get the list of attachments processed (successfully or not) in a previous
|
|
78
|
+
// (possibly incomplete) batch extraction. If no such list exists, create an empty one.
|
|
80
79
|
if (!this.adapter.state.toDevRev.attachmentsMetadata
|
|
81
80
|
.lastProcessedAttachmentsIdsList) {
|
|
82
81
|
this.adapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList =
|
|
83
82
|
[];
|
|
84
83
|
}
|
|
85
|
-
// Migrate old processed attachments to the
|
|
84
|
+
// Migrate old processed attachments to the current format.
|
|
86
85
|
this.adapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList =
|
|
87
86
|
this.migrateProcessedAttachments(this.adapter.state.toDevRev.attachmentsMetadata
|
|
88
87
|
.lastProcessedAttachmentsIdsList);
|
|
89
|
-
// Get the list of attachments that have repeatedly failed with a transient error in
|
|
90
|
-
// previous invocations. If no such list exists, create an empty one.
|
|
91
|
-
if (!this.adapter.state.toDevRev.attachmentsMetadata.failedAttachmentsIdsList) {
|
|
92
|
-
this.adapter.state.toDevRev.attachmentsMetadata.failedAttachmentsIdsList =
|
|
93
|
-
[];
|
|
94
|
-
}
|
|
95
88
|
// Start initial batch of promises up to batchSize limit
|
|
96
89
|
const initialBatchSize = Math.min(this.batchSize, this.attachments.length);
|
|
97
90
|
const initialPromises = [];
|
|
@@ -108,7 +101,7 @@ class AttachmentsStreamingPool {
|
|
|
108
101
|
return {};
|
|
109
102
|
}
|
|
110
103
|
async startPoolStreaming() {
|
|
111
|
-
var _a
|
|
104
|
+
var _a;
|
|
112
105
|
// Process attachments until the attachments array is empty
|
|
113
106
|
while (this.attachments.length > 0) {
|
|
114
107
|
// If delay is set, stop streaming
|
|
@@ -127,12 +120,7 @@ class AttachmentsStreamingPool {
|
|
|
127
120
|
}
|
|
128
121
|
if (this.adapter.state.toDevRev &&
|
|
129
122
|
((_a = this.adapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList) === null || _a === void 0 ? void 0 : _a.some((it) => it.id == attachment.id && it.parent_id == attachment.parent_id))) {
|
|
130
|
-
continue; // Skip if the attachment
|
|
131
|
-
}
|
|
132
|
-
if (this.adapter.state.toDevRev &&
|
|
133
|
-
((_b = this.adapter.state.toDevRev.attachmentsMetadata.failedAttachmentsIdsList) === null || _b === void 0 ? void 0 : _b.some((it) => it.id == attachment.id && it.parent_id == attachment.parent_id))) {
|
|
134
|
-
console.warn(`Skipping attachment with ID ${attachment.id} fetched from URL ${attachment.url}: previously failed with a transient error.`);
|
|
135
|
-
continue; // Skip if the attachment was already marked as permanently failed
|
|
123
|
+
continue; // Skip if the attachment was already processed (succeeded or failed)
|
|
136
124
|
}
|
|
137
125
|
try {
|
|
138
126
|
const response = await this.adapter.processAttachment(attachment, this.stream);
|
|
@@ -149,24 +137,13 @@ class AttachmentsStreamingPool {
|
|
|
149
137
|
const contentTypeInfo = attachment.content_type
|
|
150
138
|
? `and content_type ${attachment.content_type} `
|
|
151
139
|
: '';
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
? `and fetch response headers ${JSON.stringify(response.error.fetchHeaders)} `
|
|
155
|
-
: '';
|
|
156
|
-
console.warn(`Skipping attachment with ID ${attachment.id} with extension ${fileExtension} ${fileSizeInfo}${contentTypeInfo}${fetchHeadersInfo}fetched from URL ${attachment.url} due to error returned by the stream function`, response.error.message);
|
|
157
|
-
// The stream function's HTTP client already retried this request with
|
|
158
|
-
// exponential backoff before returning an error, so a transient failure here
|
|
159
|
-
// means that budget is exhausted — mark it permanently failed rather than
|
|
160
|
-
// retrying it again from scratch next invocation.
|
|
161
|
-
if (response.error.isTransient) {
|
|
162
|
-
this.markPermanentlyFailed(attachment);
|
|
163
|
-
}
|
|
140
|
+
console.warn(`Skipping attachment with ID ${attachment.id} with extension ${fileExtension} ${fileSizeInfo}${contentTypeInfo}due to error returned by the stream function`, response.error.message);
|
|
141
|
+
this.recordProcessedAttachment(attachment, state_interfaces_1.ProcessedAttachmentStatus.Failed);
|
|
164
142
|
await this.updateProgress();
|
|
165
143
|
continue;
|
|
166
144
|
}
|
|
167
|
-
if (!this.adapter.isTimeout
|
|
168
|
-
|
|
169
|
-
(_e = this.adapter.state.toDevRev) === null || _e === void 0 ? void 0 : _e.attachmentsMetadata.lastProcessedAttachmentsIdsList.push({ id: attachment.id, parent_id: attachment.parent_id });
|
|
145
|
+
if (!this.adapter.isTimeout) {
|
|
146
|
+
this.recordProcessedAttachment(attachment, state_interfaces_1.ProcessedAttachmentStatus.Success);
|
|
170
147
|
}
|
|
171
148
|
await this.updateProgress();
|
|
172
149
|
}
|
|
@@ -175,7 +152,7 @@ class AttachmentsStreamingPool {
|
|
|
175
152
|
const contentTypeInfo = attachment.content_type
|
|
176
153
|
? ` and content_type ${attachment.content_type}`
|
|
177
154
|
: '';
|
|
178
|
-
console.warn(`Skipping attachment with ID ${attachment.id} with extension ${fileExtension}${contentTypeInfo}
|
|
155
|
+
console.warn(`Skipping attachment with ID ${attachment.id} with extension ${fileExtension}${contentTypeInfo} due to error in processAttachment function`, error);
|
|
179
156
|
await this.updateProgress();
|
|
180
157
|
}
|
|
181
158
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const state_interfaces_1 = require("../state/state.interfaces");
|
|
3
4
|
const attachments_streaming_pool_1 = require("./attachments-streaming-pool");
|
|
4
5
|
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
5
6
|
describe(attachments_streaming_pool_1.AttachmentsStreamingPool.name, () => {
|
|
@@ -147,17 +148,35 @@ describe(attachments_streaming_pool_1.AttachmentsStreamingPool.name, () => {
|
|
|
147
148
|
});
|
|
148
149
|
const result = await pool.streamAll();
|
|
149
150
|
expect((_a = mockAdapter.state.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata.lastProcessedAttachmentsIdsList).toEqual([
|
|
150
|
-
{ id: 'attachment-1', parent_id: '' },
|
|
151
|
-
{ id: 'attachment-2', parent_id: '' },
|
|
152
|
-
{
|
|
153
|
-
|
|
154
|
-
|
|
151
|
+
{ id: 'attachment-1', parent_id: '', status: state_interfaces_1.ProcessedAttachmentStatus.Success },
|
|
152
|
+
{ id: 'attachment-2', parent_id: '', status: state_interfaces_1.ProcessedAttachmentStatus.Success },
|
|
153
|
+
{
|
|
154
|
+
id: 'attachment-1',
|
|
155
|
+
parent_id: 'parent-1',
|
|
156
|
+
status: state_interfaces_1.ProcessedAttachmentStatus.Success,
|
|
157
|
+
},
|
|
158
|
+
{
|
|
159
|
+
id: 'attachment-2',
|
|
160
|
+
parent_id: 'parent-2',
|
|
161
|
+
status: state_interfaces_1.ProcessedAttachmentStatus.Success,
|
|
162
|
+
},
|
|
163
|
+
{
|
|
164
|
+
id: 'attachment-3',
|
|
165
|
+
parent_id: 'parent-3',
|
|
166
|
+
status: state_interfaces_1.ProcessedAttachmentStatus.Success,
|
|
167
|
+
},
|
|
155
168
|
]);
|
|
156
169
|
expect(result).toEqual({});
|
|
157
170
|
});
|
|
158
171
|
it('should skip attachments already marked as permanently failed', async () => {
|
|
159
|
-
mockAdapter.state.toDevRev.attachmentsMetadata.
|
|
160
|
-
[
|
|
172
|
+
mockAdapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList =
|
|
173
|
+
[
|
|
174
|
+
{
|
|
175
|
+
id: 'attachment-1',
|
|
176
|
+
parent_id: 'parent-1',
|
|
177
|
+
status: state_interfaces_1.ProcessedAttachmentStatus.Failed,
|
|
178
|
+
},
|
|
179
|
+
];
|
|
161
180
|
mockAdapter.processAttachment.mockResolvedValue({});
|
|
162
181
|
const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
|
|
163
182
|
adapter: mockAdapter,
|
|
@@ -167,23 +186,9 @@ describe(attachments_streaming_pool_1.AttachmentsStreamingPool.name, () => {
|
|
|
167
186
|
await pool.streamAll();
|
|
168
187
|
expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(2); // Only 2 out of 3
|
|
169
188
|
});
|
|
170
|
-
it('should
|
|
171
|
-
const warnSpy = jest.spyOn(console, 'warn').mockImplementation();
|
|
172
|
-
mockAdapter.state.toDevRev.attachmentsMetadata.failedAttachmentsIdsList =
|
|
173
|
-
[{ id: 'attachment-1', parent_id: 'parent-1' }];
|
|
174
|
-
mockAdapter.processAttachment.mockResolvedValue({});
|
|
175
|
-
const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
|
|
176
|
-
adapter: mockAdapter,
|
|
177
|
-
attachments: [mockAttachments[0]],
|
|
178
|
-
stream: mockStream,
|
|
179
|
-
batchSize: 1,
|
|
180
|
-
});
|
|
181
|
-
await pool.streamAll();
|
|
182
|
-
expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('https://example.com/file1.pdf'));
|
|
183
|
-
});
|
|
184
|
-
it('should mark an attachment as permanently failed on a transient error', async () => {
|
|
189
|
+
it('should mark an attachment as permanently failed on any error', async () => {
|
|
185
190
|
mockAdapter.processAttachment.mockResolvedValueOnce({
|
|
186
|
-
error: { message: 'timeout'
|
|
191
|
+
error: { message: 'timeout' },
|
|
187
192
|
});
|
|
188
193
|
const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
|
|
189
194
|
adapter: mockAdapter,
|
|
@@ -193,7 +198,14 @@ describe(attachments_streaming_pool_1.AttachmentsStreamingPool.name, () => {
|
|
|
193
198
|
});
|
|
194
199
|
await pool.streamAll();
|
|
195
200
|
expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(1);
|
|
196
|
-
expect(mockAdapter.state.toDevRev.attachmentsMetadata
|
|
201
|
+
expect(mockAdapter.state.toDevRev.attachmentsMetadata
|
|
202
|
+
.lastProcessedAttachmentsIdsList).toEqual([
|
|
203
|
+
{
|
|
204
|
+
id: 'attachment-1',
|
|
205
|
+
parent_id: 'parent-1',
|
|
206
|
+
status: state_interfaces_1.ProcessedAttachmentStatus.Failed,
|
|
207
|
+
},
|
|
208
|
+
]);
|
|
197
209
|
// A subsequent invocation should now skip it entirely instead of calling processAttachment again.
|
|
198
210
|
mockAdapter.processAttachment.mockClear();
|
|
199
211
|
const secondPool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
|
|
@@ -205,57 +217,6 @@ describe(attachments_streaming_pool_1.AttachmentsStreamingPool.name, () => {
|
|
|
205
217
|
await secondPool.streamAll();
|
|
206
218
|
expect(mockAdapter.processAttachment).not.toHaveBeenCalled();
|
|
207
219
|
});
|
|
208
|
-
it('should not mark a non-transient error as permanently failed', async () => {
|
|
209
|
-
mockAdapter.processAttachment.mockResolvedValueOnce({
|
|
210
|
-
error: { message: 'File size is 0 or less.' },
|
|
211
|
-
});
|
|
212
|
-
const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
|
|
213
|
-
adapter: mockAdapter,
|
|
214
|
-
attachments: [mockAttachments[0]],
|
|
215
|
-
stream: mockStream,
|
|
216
|
-
batchSize: 1,
|
|
217
|
-
});
|
|
218
|
-
await pool.streamAll();
|
|
219
|
-
expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(1);
|
|
220
|
-
expect(mockAdapter.state.toDevRev.attachmentsMetadata.failedAttachmentsIdsList).toEqual([]);
|
|
221
|
-
});
|
|
222
|
-
it('should include the attachment URL and fetch response headers in the skip log', async () => {
|
|
223
|
-
const warnSpy = jest.spyOn(console, 'warn').mockImplementation();
|
|
224
|
-
mockAdapter.processAttachment.mockResolvedValueOnce({
|
|
225
|
-
error: {
|
|
226
|
-
message: 'timeout',
|
|
227
|
-
isTransient: true,
|
|
228
|
-
fileSize: 2048,
|
|
229
|
-
fetchHeaders: {
|
|
230
|
-
'content-length': '2048',
|
|
231
|
-
'content-type': 'application/pdf',
|
|
232
|
-
},
|
|
233
|
-
},
|
|
234
|
-
});
|
|
235
|
-
const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
|
|
236
|
-
adapter: mockAdapter,
|
|
237
|
-
attachments: [mockAttachments[0]],
|
|
238
|
-
stream: mockStream,
|
|
239
|
-
batchSize: 1,
|
|
240
|
-
});
|
|
241
|
-
await pool.streamAll();
|
|
242
|
-
expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('https://example.com/file1.pdf'), 'timeout');
|
|
243
|
-
expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('{"content-length":"2048","content-type":"application/pdf"}'), 'timeout');
|
|
244
|
-
});
|
|
245
|
-
it('should include the attachment URL in the skip log when the error has no fetch headers', async () => {
|
|
246
|
-
const warnSpy = jest.spyOn(console, 'warn').mockImplementation();
|
|
247
|
-
mockAdapter.processAttachment.mockResolvedValueOnce({
|
|
248
|
-
error: { message: 'File size is 0 or less.' },
|
|
249
|
-
});
|
|
250
|
-
const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
|
|
251
|
-
adapter: mockAdapter,
|
|
252
|
-
attachments: [mockAttachments[0]],
|
|
253
|
-
stream: mockStream,
|
|
254
|
-
batchSize: 1,
|
|
255
|
-
});
|
|
256
|
-
await pool.streamAll();
|
|
257
|
-
expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('https://example.com/file1.pdf'), 'File size is 0 or less.');
|
|
258
|
-
});
|
|
259
220
|
it('should handle all attachments failing with different file types and sizes', async () => {
|
|
260
221
|
const largeAttachments = [
|
|
261
222
|
{
|
|
@@ -301,7 +262,13 @@ describe(attachments_streaming_pool_1.AttachmentsStreamingPool.name, () => {
|
|
|
301
262
|
describe(attachments_streaming_pool_1.AttachmentsStreamingPool.prototype.startPoolStreaming.name, () => {
|
|
302
263
|
it('should skip already processed attachments', async () => {
|
|
303
264
|
mockAdapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList =
|
|
304
|
-
[
|
|
265
|
+
[
|
|
266
|
+
{
|
|
267
|
+
id: 'attachment-1',
|
|
268
|
+
parent_id: 'parent-1',
|
|
269
|
+
status: state_interfaces_1.ProcessedAttachmentStatus.Success,
|
|
270
|
+
},
|
|
271
|
+
];
|
|
305
272
|
mockAdapter.processAttachment.mockResolvedValue({});
|
|
306
273
|
const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
|
|
307
274
|
adapter: mockAdapter,
|
|
@@ -321,9 +288,21 @@ describe(attachments_streaming_pool_1.AttachmentsStreamingPool.name, () => {
|
|
|
321
288
|
await pool.streamAll();
|
|
322
289
|
expect(mockAdapter.state.toDevRev.attachmentsMetadata
|
|
323
290
|
.lastProcessedAttachmentsIdsList).toEqual([
|
|
324
|
-
{
|
|
325
|
-
|
|
326
|
-
|
|
291
|
+
{
|
|
292
|
+
id: 'attachment-1',
|
|
293
|
+
parent_id: 'parent-1',
|
|
294
|
+
status: state_interfaces_1.ProcessedAttachmentStatus.Success,
|
|
295
|
+
},
|
|
296
|
+
{
|
|
297
|
+
id: 'attachment-2',
|
|
298
|
+
parent_id: 'parent-2',
|
|
299
|
+
status: state_interfaces_1.ProcessedAttachmentStatus.Success,
|
|
300
|
+
},
|
|
301
|
+
{
|
|
302
|
+
id: 'attachment-3',
|
|
303
|
+
parent_id: 'parent-3',
|
|
304
|
+
status: state_interfaces_1.ProcessedAttachmentStatus.Success,
|
|
305
|
+
},
|
|
327
306
|
]);
|
|
328
307
|
});
|
|
329
308
|
it('should handle processing errors gracefully', async () => {
|
|
@@ -339,16 +318,18 @@ describe(attachments_streaming_pool_1.AttachmentsStreamingPool.name, () => {
|
|
|
339
318
|
stream: mockStream,
|
|
340
319
|
});
|
|
341
320
|
await pool.streamAll();
|
|
342
|
-
expect(warnSpy).toHaveBeenCalledWith('Skipping attachment with ID attachment-2 with extension jpg
|
|
321
|
+
expect(warnSpy).toHaveBeenCalledWith('Skipping attachment with ID attachment-2 with extension jpg due to error in processAttachment function', error);
|
|
343
322
|
expect(mockAdapter.state.toDevRev.attachmentsMetadata
|
|
344
323
|
.lastProcessedAttachmentsIdsList).toEqual([
|
|
345
324
|
{
|
|
346
325
|
id: 'attachment-1',
|
|
347
326
|
parent_id: 'parent-1',
|
|
327
|
+
status: state_interfaces_1.ProcessedAttachmentStatus.Success,
|
|
348
328
|
},
|
|
349
329
|
{
|
|
350
330
|
id: 'attachment-3',
|
|
351
331
|
parent_id: 'parent-3',
|
|
332
|
+
status: state_interfaces_1.ProcessedAttachmentStatus.Success,
|
|
352
333
|
},
|
|
353
334
|
]);
|
|
354
335
|
});
|
|
@@ -366,8 +347,16 @@ describe(attachments_streaming_pool_1.AttachmentsStreamingPool.name, () => {
|
|
|
366
347
|
expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(3);
|
|
367
348
|
expect(mockAdapter.state.toDevRev.attachmentsMetadata
|
|
368
349
|
.lastProcessedAttachmentsIdsList).toEqual([
|
|
369
|
-
{
|
|
370
|
-
|
|
350
|
+
{
|
|
351
|
+
id: 'attachment-1',
|
|
352
|
+
parent_id: 'parent-1',
|
|
353
|
+
status: state_interfaces_1.ProcessedAttachmentStatus.Success,
|
|
354
|
+
},
|
|
355
|
+
{
|
|
356
|
+
id: 'attachment-3',
|
|
357
|
+
parent_id: 'parent-3',
|
|
358
|
+
status: state_interfaces_1.ProcessedAttachmentStatus.Success,
|
|
359
|
+
},
|
|
371
360
|
]);
|
|
372
361
|
});
|
|
373
362
|
it('should pass correct parameters to processAttachment', async () => {
|
|
@@ -674,7 +663,13 @@ describe(attachments_streaming_pool_1.AttachmentsStreamingPool.name, () => {
|
|
|
674
663
|
});
|
|
675
664
|
await pool.streamAll();
|
|
676
665
|
expect(mockAdapter.state.toDevRev.attachmentsMetadata
|
|
677
|
-
.lastProcessedAttachmentsIdsList).toEqual([
|
|
666
|
+
.lastProcessedAttachmentsIdsList).toEqual([
|
|
667
|
+
{
|
|
668
|
+
id: 'attachment-1',
|
|
669
|
+
parent_id: 'parent-1',
|
|
670
|
+
status: state_interfaces_1.ProcessedAttachmentStatus.Success,
|
|
671
|
+
},
|
|
672
|
+
]);
|
|
678
673
|
// 2nd was started then abandoned; 3rd never started.
|
|
679
674
|
expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(2);
|
|
680
675
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"worker-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"worker-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.ts"],"names":[],"mappings":"AAqBA,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAEhD,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AACvC,OAAO,EACL,oBAAoB,EACpB,aAAa,EACd,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EAAE,YAAY,EAAE,MAAM,8BAA8B,CAAC;AAC5D,OAAO,EACL,YAAY,EACZ,SAAS,EAET,kCAAkC,EAClC,yCAAyC,EACzC,kBAAkB,EAClB,2BAA2B,EAC3B,2BAA2B,EAC5B,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAEL,wBAAwB,EACxB,kBAAkB,EAClB,6BAA6B,EAC7B,UAAU,EACV,qBAAqB,EACrB,cAAc,EACd,eAAe,EACf,YAAY,EACZ,gBAAgB,EAChB,qBAAqB,EAEtB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,sBAAsB,EACtB,oBAAoB,EAGrB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,QAAQ,EAAkB,MAAM,oCAAoC,CAAC;AAI9E,wBAAgB,mBAAmB,CAAC,cAAc,EAAE,EAClD,KAAK,EACL,YAAY,EACZ,OAAO,GACR,EAAE,sBAAsB,CAAC,cAAc,CAAC,GAAG,aAAa,CAAC,cAAc,CAAC,CAMxE;AAED;;;;;;;;;;;;;;GAcG;AACH,qBAAa,aAAa,CAAC,cAAc;IACvC,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IAC7B,QAAQ,CAAC,OAAO,CAAC,EAAE,oBAAoB,CAAC;IACxC,gBAAgB,EAAE,OAAO,CAAC;IAE1B,OAAO,CAAC,UAAU,CAAkB;IACpC,OAAO,CAAC,oBAAoB,CAAc;IAC1C,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC,IAAI,CAAC,CAElC;IAEH,OAAO,CAAC,YAAY,CAAwB;IAC5C,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,KAAK,CAAc;IAC3B,OAAO,CAAC,qBAAqB,CAAC,CAAS;IACvC,OAAO,CAAC,sBAAsB,CAAa;IAG3C,OAAO,CAAC,aAAa,CAAiB;IACtC,OAAO,CAAC,eAAe,CAAW;IAClC,OAAO,CAAC,QAAQ,CAAU;IAC1B,OAAO,CAAC,QAAQ,CAAW;gBAEf,EACV,KAAK,EACL,YAAY,EACZ,OAAO,GACR,EAAE,sBAAsB,CAAC,cAAc,CAAC;IAoBzC,IAAI,SAAS,IAAI,OAAO,CAEvB;IAED,IAAI,SAAS,CAAC,KAAK,EAAE,OAAO,EAK3B;IAED,IAAI,KAAK,IAAI,YAAY,CAAC,cAAc,CAAC,CAExC;IAED,IAAI,KAAK,CAAC,KAAK,EAAE,YAAY,CAAC,cAAc,CAAC,EAE5C;IAED,IAAI,OAAO,IAAI,YAAY,EAAE,CAE5B;IAED,IAAI,cAAc,IAAI,MAAM,EAAE,CAE7B;IAED,IAAI,OAAO,IAAI,OAAO,CAErB;IAED,IAAI,eAAe,kDAElB;IAED;;;OAGG;IACH,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAOxC,eAAe,CAAC,KAAK,EAAE,aAAa,EAAE;IAyCtC,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS;IAarC,SAAS;IAMf,IAAI,SAAS,IAAI,QAAQ,EAAE,CAE1B;IAED,IAAI,SAAS,CAAC,SAAS,EAAE,QAAQ,EAAE,EAIlC;IAED;;;;;OAKG;IACG,IAAI,CACR,YAAY,EAAE,kBAAkB,GAAG,eAAe,EAClD,IAAI,CAAC,EAAE,SAAS,GACf,OAAO,CAAC,IAAI,CAAC;IAwLV,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;IAU/B,aAAa,CAAC,EAClB,eAAe,GAChB,EAAE,qBAAqB,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAwInD,gBAAgB,CAAC,EACrB,kBAAkB,GACnB,EAAE;QACD,kBAAkB,EAAE,MAAM,EAAE,CAAC;KAC9B;IA4BK,eAAe,CAAC,EACpB,MAAM,GACP,EAAE;QACD,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAyG5B,QAAQ,CAAC,EACb,IAAI,EACJ,cAAc,GACf,EAAE;QACD,IAAI,EAAE,kBAAkB,CAAC;QACzB,cAAc,EAAE,cAAc,CAAC;KAChC,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAwMvB,iBAAiB,CACrB,UAAU,EAAE,oBAAoB,EAChC,MAAM,EAAE,yCAAyC,GAChD,OAAO,CAAC,2BAA2B,CAAC;IAoHvC;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAcnB,cAAc,CAAC,EACnB,IAAI,EACJ,MAAM,GACP,EAAE;QACD,IAAI,EAAE,wBAAwB,CAAC;QAC/B,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAuD7B;;;;;;OAMG;IACG,iBAAiB,CAAC,QAAQ,EAAE,EAChC,MAAM,EACN,UAAU,EACV,SAAa,GACd,EAAE;QACD,MAAM,EAAE,yCAAyC,CAAC;QAClD,UAAU,CAAC,EAAE,kCAAkC,CAC7C,cAAc,EACd,oBAAoB,EAAE,EACtB,QAAQ,CACT,CAAC;QACF,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,GAAG,OAAO,CAAC,2BAA2B,CAAC;CAuIzC"}
|
|
@@ -562,7 +562,7 @@ describe(`${worker_adapter_1.WorkerAdapter.name}.processAttachment`, () => {
|
|
|
562
562
|
});
|
|
563
563
|
// ---- error paths ----
|
|
564
564
|
it('should return the stream error message when the stream function returns an error', async () => {
|
|
565
|
-
var _a
|
|
565
|
+
var _a;
|
|
566
566
|
// Arrange
|
|
567
567
|
const stream = jest
|
|
568
568
|
.fn()
|
|
@@ -571,40 +571,6 @@ describe(`${worker_adapter_1.WorkerAdapter.name}.processAttachment`, () => {
|
|
|
571
571
|
const result = await adapter.processAttachment(makeAttachment(), stream);
|
|
572
572
|
// Assert
|
|
573
573
|
expect((_a = result === null || result === void 0 ? void 0 : result.error) === null || _a === void 0 ? void 0 : _a.message).toBe('stream failed');
|
|
574
|
-
expect((_b = result === null || result === void 0 ? void 0 : result.error) === null || _b === void 0 ? void 0 : _b.isTransient).toBe(false);
|
|
575
|
-
});
|
|
576
|
-
it('should classify a stream error with a 5xx statusCode as transient', async () => {
|
|
577
|
-
var _a;
|
|
578
|
-
// Arrange
|
|
579
|
-
const stream = jest.fn().mockResolvedValue({
|
|
580
|
-
error: { message: 'export failed', statusCode: 500 },
|
|
581
|
-
});
|
|
582
|
-
// Act
|
|
583
|
-
const result = await adapter.processAttachment(makeAttachment(), stream);
|
|
584
|
-
// Assert
|
|
585
|
-
expect((_a = result === null || result === void 0 ? void 0 : result.error) === null || _a === void 0 ? void 0 : _a.isTransient).toBe(true);
|
|
586
|
-
});
|
|
587
|
-
it('should not classify a stream error with a 4xx statusCode as transient', async () => {
|
|
588
|
-
var _a;
|
|
589
|
-
// Arrange
|
|
590
|
-
const stream = jest.fn().mockResolvedValue({
|
|
591
|
-
error: { message: 'not found', statusCode: 404 },
|
|
592
|
-
});
|
|
593
|
-
// Act
|
|
594
|
-
const result = await adapter.processAttachment(makeAttachment(), stream);
|
|
595
|
-
// Assert
|
|
596
|
-
expect((_a = result === null || result === void 0 ? void 0 : result.error) === null || _a === void 0 ? void 0 : _a.isTransient).toBe(false);
|
|
597
|
-
});
|
|
598
|
-
it('should not classify a stream error without a statusCode as transient', async () => {
|
|
599
|
-
var _a;
|
|
600
|
-
// Arrange
|
|
601
|
-
const stream = jest.fn().mockResolvedValue({
|
|
602
|
-
error: { message: 'unknown failure' },
|
|
603
|
-
});
|
|
604
|
-
// Act
|
|
605
|
-
const result = await adapter.processAttachment(makeAttachment(), stream);
|
|
606
|
-
// Assert
|
|
607
|
-
expect((_a = result === null || result === void 0 ? void 0 : result.error) === null || _a === void 0 ? void 0 : _a.isTransient).toBe(false);
|
|
608
574
|
});
|
|
609
575
|
it('should propagate a rate-limit delay from the stream function', async () => {
|
|
610
576
|
// Arrange
|
|
@@ -650,34 +616,6 @@ describe(`${worker_adapter_1.WorkerAdapter.name}.processAttachment`, () => {
|
|
|
650
616
|
// Assert
|
|
651
617
|
expect((_a = result === null || result === void 0 ? void 0 : result.error) === null || _a === void 0 ? void 0 : _a.message).toContain('streaming to artifact');
|
|
652
618
|
});
|
|
653
|
-
it('should include allowlisted fetch response headers in the error when streamArtifact fails', async () => {
|
|
654
|
-
var _a;
|
|
655
|
-
// Arrange
|
|
656
|
-
const stream = jest.fn().mockResolvedValue({
|
|
657
|
-
httpStream: createMockHttpStream({
|
|
658
|
-
'content-length': '2048',
|
|
659
|
-
'content-type': 'application/pdf',
|
|
660
|
-
authorization: 'Bearer secret-token',
|
|
661
|
-
}),
|
|
662
|
-
});
|
|
663
|
-
adapter['uploader'].getArtifactUploadUrl = jest.fn().mockResolvedValue({
|
|
664
|
-
response: {
|
|
665
|
-
artifact_id: 'art-1',
|
|
666
|
-
upload_url: 'https://upload',
|
|
667
|
-
form_data: [],
|
|
668
|
-
},
|
|
669
|
-
});
|
|
670
|
-
adapter['uploader'].streamArtifact = jest
|
|
671
|
-
.fn()
|
|
672
|
-
.mockResolvedValue({ error: new Error('stream failed') });
|
|
673
|
-
// Act
|
|
674
|
-
const result = await adapter.processAttachment(makeAttachment(), stream);
|
|
675
|
-
// Assert
|
|
676
|
-
expect((_a = result === null || result === void 0 ? void 0 : result.error) === null || _a === void 0 ? void 0 : _a.fetchHeaders).toEqual({
|
|
677
|
-
'content-length': '2048',
|
|
678
|
-
'content-type': 'application/pdf',
|
|
679
|
-
});
|
|
680
|
-
});
|
|
681
619
|
it('should return an error when confirmArtifactUpload fails', async () => {
|
|
682
620
|
var _a;
|
|
683
621
|
// Arrange
|
|
@@ -1,10 +1,4 @@
|
|
|
1
1
|
import { FileToLoad, LoaderReport, StatsFileObject } from '../../types/loading';
|
|
2
|
-
/**
|
|
3
|
-
* Extracts a fixed allowlist of structural response headers (size, type, cache metadata)
|
|
4
|
-
* from an attachment fetch response, for inclusion in "skipping attachment" logs. Headers
|
|
5
|
-
* outside the allowlist (e.g. anything auth-adjacent) are never included.
|
|
6
|
-
*/
|
|
7
|
-
export declare function getSafeResponseHeaders(headers: Record<string, unknown>): Record<string, string>;
|
|
8
2
|
/**
|
|
9
3
|
* Gets the files to load for the loader.
|
|
10
4
|
* @param {string[]} supportedItemTypes - The supported item types
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"worker-adapter.helpers.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.helpers.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,UAAU,EACV,YAAY,EACZ,eAAe,EAChB,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"worker-adapter.helpers.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.helpers.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,UAAU,EACV,YAAY,EACZ,eAAe,EAChB,MAAM,qBAAqB,CAAC;AAE7B;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,EAC7B,kBAAkB,EAClB,SAAS,GACV,EAAE;IACD,kBAAkB,EAAE,MAAM,EAAE,CAAC;IAC7B,SAAS,EAAE,eAAe,EAAE,CAAC;CAC9B,GAAG,UAAU,EAAE,CA8Bf;AAED;;;;;GAKG;AACH,wBAAgB,uBAAuB,CAAC,EACtC,aAAa,EACb,MAAM,GACP,EAAE;IACD,aAAa,EAAE,YAAY,EAAE,CAAC;IAC9B,MAAM,EAAE,YAAY,CAAC;CACtB,GAAG,YAAY,EAAE,CA4BjB;AAED,wBAAgB,kBAAkB,CAAC,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAMlE"}
|
|
@@ -1,38 +1,9 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.getSafeResponseHeaders = getSafeResponseHeaders;
|
|
4
3
|
exports.getFilesToLoad = getFilesToLoad;
|
|
5
4
|
exports.addReportToLoaderReport = addReportToLoaderReport;
|
|
6
5
|
exports.toRfc3339Timestamp = toRfc3339Timestamp;
|
|
7
6
|
const loading_1 = require("../../types/loading");
|
|
8
|
-
// Response headers safe to include in logs: structural metadata about the fetched file,
|
|
9
|
-
// not the request itself. Everything else (auth-adjacent headers, cookies, etc.) is
|
|
10
|
-
// deliberately excluded — an allowlist rather than a denylist, so a header we haven't
|
|
11
|
-
// thought about doesn't leak through.
|
|
12
|
-
const SAFE_ATTACHMENT_RESPONSE_HEADERS = [
|
|
13
|
-
'content-length',
|
|
14
|
-
'content-type',
|
|
15
|
-
'content-disposition',
|
|
16
|
-
'content-encoding',
|
|
17
|
-
'last-modified',
|
|
18
|
-
'etag',
|
|
19
|
-
'cache-control',
|
|
20
|
-
];
|
|
21
|
-
/**
|
|
22
|
-
* Extracts a fixed allowlist of structural response headers (size, type, cache metadata)
|
|
23
|
-
* from an attachment fetch response, for inclusion in "skipping attachment" logs. Headers
|
|
24
|
-
* outside the allowlist (e.g. anything auth-adjacent) are never included.
|
|
25
|
-
*/
|
|
26
|
-
function getSafeResponseHeaders(headers) {
|
|
27
|
-
const safeHeaders = {};
|
|
28
|
-
for (const key of SAFE_ATTACHMENT_RESPONSE_HEADERS) {
|
|
29
|
-
const value = headers[key];
|
|
30
|
-
if (value !== undefined) {
|
|
31
|
-
safeHeaders[key] = String(value);
|
|
32
|
-
}
|
|
33
|
-
}
|
|
34
|
-
return safeHeaders;
|
|
35
|
-
}
|
|
36
7
|
/**
|
|
37
8
|
* Gets the files to load for the loader.
|
|
38
9
|
* @param {string[]} supportedItemTypes - The supported item types
|
|
@@ -540,49 +540,3 @@ describe(worker_adapter_helpers_1.toRfc3339Timestamp.name, () => {
|
|
|
540
540
|
expect(result).toBe('1969-12-31T00:00:00.000Z');
|
|
541
541
|
});
|
|
542
542
|
});
|
|
543
|
-
describe(worker_adapter_helpers_1.getSafeResponseHeaders.name, () => {
|
|
544
|
-
it('should extract allowlisted headers', () => {
|
|
545
|
-
const headers = {
|
|
546
|
-
'content-length': '1024',
|
|
547
|
-
'content-type': 'application/pdf',
|
|
548
|
-
'content-disposition': 'attachment; filename="file.pdf"',
|
|
549
|
-
'content-encoding': 'gzip',
|
|
550
|
-
'last-modified': 'Wed, 21 Oct 2015 07:28:00 GMT',
|
|
551
|
-
etag: '"abc123"',
|
|
552
|
-
'cache-control': 'no-cache',
|
|
553
|
-
};
|
|
554
|
-
const result = (0, worker_adapter_helpers_1.getSafeResponseHeaders)(headers);
|
|
555
|
-
expect(result).toEqual({
|
|
556
|
-
'content-length': '1024',
|
|
557
|
-
'content-type': 'application/pdf',
|
|
558
|
-
'content-disposition': 'attachment; filename="file.pdf"',
|
|
559
|
-
'content-encoding': 'gzip',
|
|
560
|
-
'last-modified': 'Wed, 21 Oct 2015 07:28:00 GMT',
|
|
561
|
-
etag: '"abc123"',
|
|
562
|
-
'cache-control': 'no-cache',
|
|
563
|
-
});
|
|
564
|
-
});
|
|
565
|
-
it('should exclude headers not on the allowlist', () => {
|
|
566
|
-
const headers = {
|
|
567
|
-
'content-length': '1024',
|
|
568
|
-
authorization: 'Bearer secret-token',
|
|
569
|
-
cookie: 'session=abc123',
|
|
570
|
-
'set-cookie': 'session=abc123',
|
|
571
|
-
'x-api-key': 'secret-key',
|
|
572
|
-
};
|
|
573
|
-
const result = (0, worker_adapter_helpers_1.getSafeResponseHeaders)(headers);
|
|
574
|
-
expect(result).toEqual({ 'content-length': '1024' });
|
|
575
|
-
});
|
|
576
|
-
it('[edge] should return an empty object when no headers are present', () => {
|
|
577
|
-
const result = (0, worker_adapter_helpers_1.getSafeResponseHeaders)({});
|
|
578
|
-
expect(result).toEqual({});
|
|
579
|
-
});
|
|
580
|
-
it('[edge] should return an empty object when headers only contain non-allowlisted keys', () => {
|
|
581
|
-
const result = (0, worker_adapter_helpers_1.getSafeResponseHeaders)({ authorization: 'Bearer token' });
|
|
582
|
-
expect(result).toEqual({});
|
|
583
|
-
});
|
|
584
|
-
it('[edge] should stringify non-string header values', () => {
|
|
585
|
-
const result = (0, worker_adapter_helpers_1.getSafeResponseHeaders)({ 'content-length': 1024 });
|
|
586
|
-
expect(result).toEqual({ 'content-length': '1024' });
|
|
587
|
-
});
|
|
588
|
-
});
|
|
@@ -20,7 +20,6 @@ const extraction_1 = require("../../types/extraction");
|
|
|
20
20
|
const loading_1 = require("../../types/loading");
|
|
21
21
|
const workers_1 = require("../../types/workers");
|
|
22
22
|
const uploader_1 = require("../../uploader/uploader");
|
|
23
|
-
const uploader_helpers_1 = require("../../uploader/uploader.helpers");
|
|
24
23
|
const event_type_translation_1 = require("../../common/event-type-translation");
|
|
25
24
|
const helpers_1 = require("../../common/helpers");
|
|
26
25
|
function createWorkerAdapter({ event, adapterState, options, }) {
|
|
@@ -650,28 +649,22 @@ class WorkerAdapter {
|
|
|
650
649
|
}
|
|
651
650
|
async processAttachment(attachment, stream) {
|
|
652
651
|
return (0, logger_context_1.runWithSdkLogContext)(async () => {
|
|
653
|
-
var _a, _b, _c
|
|
652
|
+
var _a, _b, _c;
|
|
654
653
|
const { httpStream, delay, error } = await (0, logger_context_1.runWithUserLogContext)(async () => stream({
|
|
655
654
|
item: attachment,
|
|
656
655
|
event: this.event,
|
|
657
656
|
}));
|
|
658
657
|
if (error) {
|
|
659
|
-
return {
|
|
660
|
-
error: {
|
|
661
|
-
message: error.message,
|
|
662
|
-
isTransient: ((_a = error.statusCode) !== null && _a !== void 0 ? _a : 0) >= 500,
|
|
663
|
-
},
|
|
664
|
-
};
|
|
658
|
+
return { error };
|
|
665
659
|
}
|
|
666
660
|
else if (delay) {
|
|
667
661
|
return { delay };
|
|
668
662
|
}
|
|
669
663
|
if (httpStream) {
|
|
670
|
-
const fetchHeaders = (0, worker_adapter_helpers_1.getSafeResponseHeaders)(httpStream.headers);
|
|
671
664
|
const fileType = attachment.content_type ||
|
|
672
|
-
((
|
|
665
|
+
((_a = httpStream.headers['content-type']) === null || _a === void 0 ? void 0 : _a.toString()) ||
|
|
673
666
|
'application/octet-stream';
|
|
674
|
-
const contentLength = (
|
|
667
|
+
const contentLength = (_b = httpStream.headers['content-length']) === null || _b === void 0 ? void 0 : _b.toString();
|
|
675
668
|
const fileSize = contentLength ? parseInt(contentLength) : undefined;
|
|
676
669
|
// Get upload URL
|
|
677
670
|
const { error: artifactUrlError, response: artifactUrlResponse } = await this.uploader.getArtifactUploadUrl(attachment.file_name, fileType, fileSize);
|
|
@@ -681,8 +674,6 @@ class WorkerAdapter {
|
|
|
681
674
|
error: {
|
|
682
675
|
message: `Error while preparing artifact for attachment ID ${attachment.id}. Skipping attachment. ${(0, logger_1.serializeError)(artifactUrlError)}`,
|
|
683
676
|
fileSize: fileSize,
|
|
684
|
-
isTransient: (0, uploader_helpers_1.isTransientUploadError)(artifactUrlError),
|
|
685
|
-
fetchHeaders,
|
|
686
677
|
},
|
|
687
678
|
};
|
|
688
679
|
}
|
|
@@ -695,8 +686,6 @@ class WorkerAdapter {
|
|
|
695
686
|
message: `Error while streaming to artifact for attachment ID ${attachment.id}. Skipping attachment. ` +
|
|
696
687
|
(0, logger_1.serializeError)(uploadedArtifactError),
|
|
697
688
|
fileSize: fileSize,
|
|
698
|
-
isTransient: (0, uploader_helpers_1.isTransientUploadError)(uploadedArtifactError),
|
|
699
|
-
fetchHeaders,
|
|
700
689
|
},
|
|
701
690
|
};
|
|
702
691
|
}
|
|
@@ -708,8 +697,6 @@ class WorkerAdapter {
|
|
|
708
697
|
message: `Error while confirming upload for attachment ID ${attachment.id}. ` +
|
|
709
698
|
(0, logger_1.serializeError)(confirmArtifactUploadError),
|
|
710
699
|
fileSize: fileSize,
|
|
711
|
-
isTransient: (0, uploader_helpers_1.isTransientUploadError)(confirmArtifactUploadError),
|
|
712
|
-
fetchHeaders,
|
|
713
700
|
},
|
|
714
701
|
};
|
|
715
702
|
}
|
|
@@ -739,7 +726,7 @@ class WorkerAdapter {
|
|
|
739
726
|
this.destroyHttpStream(httpStream);
|
|
740
727
|
return;
|
|
741
728
|
}
|
|
742
|
-
await ((
|
|
729
|
+
await ((_c = this.getRepo('ssor_attachment')) === null || _c === void 0 ? void 0 : _c.push([ssorAttachment]));
|
|
743
730
|
return;
|
|
744
731
|
}
|
|
745
732
|
return {
|
|
@@ -798,7 +785,7 @@ class WorkerAdapter {
|
|
|
798
785
|
}
|
|
799
786
|
return {
|
|
800
787
|
report: {
|
|
801
|
-
item_type: '
|
|
788
|
+
item_type: 'attachments',
|
|
802
789
|
[loading_1.ActionType.CREATED]: 1,
|
|
803
790
|
},
|
|
804
791
|
};
|
|
@@ -807,7 +794,7 @@ class WorkerAdapter {
|
|
|
807
794
|
console.warn('Failed to create attachment in external system', error);
|
|
808
795
|
return {
|
|
809
796
|
report: {
|
|
810
|
-
item_type: '
|
|
797
|
+
item_type: 'attachments',
|
|
811
798
|
[loading_1.ActionType.FAILED]: 1,
|
|
812
799
|
},
|
|
813
800
|
};
|
|
@@ -911,9 +898,6 @@ class WorkerAdapter {
|
|
|
911
898
|
if (attachmentsMetadata.lastProcessedAttachmentsIdsList) {
|
|
912
899
|
attachmentsMetadata.lastProcessedAttachmentsIdsList.length = 0;
|
|
913
900
|
}
|
|
914
|
-
if (attachmentsMetadata.failedAttachmentsIdsList) {
|
|
915
|
-
attachmentsMetadata.failedAttachmentsIdsList.length = 0;
|
|
916
|
-
}
|
|
917
901
|
}
|
|
918
902
|
return;
|
|
919
903
|
});
|
|
@@ -553,7 +553,7 @@ describe(`${worker_adapter_1.WorkerAdapter.name}.loadAttachment`, () => {
|
|
|
553
553
|
create,
|
|
554
554
|
});
|
|
555
555
|
// Assert
|
|
556
|
-
expect((_a = result.report) === null || _a === void 0 ? void 0 : _a.item_type).toBe('
|
|
556
|
+
expect((_a = result.report) === null || _a === void 0 ? void 0 : _a.item_type).toBe('attachments');
|
|
557
557
|
expect((_b = result.report) === null || _b === void 0 ? void 0 : _b[loading_1.ActionType.CREATED]).toBe(1);
|
|
558
558
|
});
|
|
559
559
|
it('should still return CREATED even when mapper create fails — attachment loading is resilient', async () => {
|
|
@@ -584,7 +584,7 @@ describe(`${worker_adapter_1.WorkerAdapter.name}.loadAttachment`, () => {
|
|
|
584
584
|
expect((_a = result.rateLimit) === null || _a === void 0 ? void 0 : _a.delay).toBe(30);
|
|
585
585
|
});
|
|
586
586
|
it('should return a FAILED report when create returns neither id nor delay', async () => {
|
|
587
|
-
var _a;
|
|
587
|
+
var _a, _b;
|
|
588
588
|
// Arrange
|
|
589
589
|
const create = jest.fn().mockResolvedValue({ id: null, delay: null });
|
|
590
590
|
// Act
|
|
@@ -593,6 +593,7 @@ describe(`${worker_adapter_1.WorkerAdapter.name}.loadAttachment`, () => {
|
|
|
593
593
|
create,
|
|
594
594
|
});
|
|
595
595
|
// Assert
|
|
596
|
-
expect((_a = result.report) === null || _a === void 0 ? void 0 : _a
|
|
596
|
+
expect((_a = result.report) === null || _a === void 0 ? void 0 : _a.item_type).toBe('attachments');
|
|
597
|
+
expect((_b = result.report) === null || _b === void 0 ? void 0 : _b[loading_1.ActionType.FAILED]).toBe(1);
|
|
597
598
|
});
|
|
598
599
|
});
|
|
@@ -37,24 +37,21 @@ export interface ToDevRev {
|
|
|
37
37
|
artifactIds: string[];
|
|
38
38
|
lastProcessed: number;
|
|
39
39
|
lastProcessedAttachmentsIdsList?: ProcessedAttachment[];
|
|
40
|
-
failedAttachmentsIdsList?: FailedAttachment[];
|
|
41
40
|
};
|
|
42
41
|
}
|
|
43
|
-
/**
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
id: string;
|
|
48
|
-
parent_id: string;
|
|
42
|
+
/** Outcome of sending an attachment for processing. */
|
|
43
|
+
export declare enum ProcessedAttachmentStatus {
|
|
44
|
+
Success = "success",
|
|
45
|
+
Failed = "failed"
|
|
49
46
|
}
|
|
50
47
|
/**
|
|
51
|
-
* Attachment that
|
|
52
|
-
*
|
|
53
|
-
* being retried forever on a deterministically-failing request.
|
|
48
|
+
* Attachment structure, that stores both attachment id and its parent_id for deduplication
|
|
49
|
+
* on the SDK side, along with whether it succeeded or failed.
|
|
54
50
|
*/
|
|
55
|
-
export interface
|
|
51
|
+
export interface ProcessedAttachment {
|
|
56
52
|
id: string;
|
|
57
53
|
parent_id: string;
|
|
54
|
+
status: ProcessedAttachmentStatus;
|
|
58
55
|
}
|
|
59
56
|
export interface FromDevRev {
|
|
60
57
|
filesToLoad: FileToLoad[];
|
|
@@ -78,7 +75,6 @@ export declare const extractionSdkState: {
|
|
|
78
75
|
artifactIds: never[];
|
|
79
76
|
lastProcessed: number;
|
|
80
77
|
lastProcessedAttachmentsIdsList: never[];
|
|
81
|
-
failedAttachmentsIdsList: never[];
|
|
82
78
|
};
|
|
83
79
|
};
|
|
84
80
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"state.interfaces.d.ts","sourceRoot":"","sources":["../../src/state/state.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD,MAAM,WAAW,QAAQ;IACvB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,yBAAyB,CAAC,EAAE,MAAM,CAAC;IACnC;gHAC4G;IAC5G,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;gHAC4G;IAC5G,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,2DAA2D;IAC3D,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,2DAA2D;IAC3D,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED;;;GAGG;AACH,MAAM,MAAM,YAAY,CAAC,cAAc,IAAI,cAAc,GAAG,QAAQ,CAAC;AAErE,MAAM,WAAW,QAAQ;IACvB,mBAAmB,EAAE;QACnB,WAAW,EAAE,MAAM,EAAE,CAAC;QACtB,aAAa,EAAE,MAAM,CAAC;QACtB,+BAA+B,CAAC,EAAE,mBAAmB,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"state.interfaces.d.ts","sourceRoot":"","sources":["../../src/state/state.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD,MAAM,WAAW,QAAQ;IACvB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,yBAAyB,CAAC,EAAE,MAAM,CAAC;IACnC;gHAC4G;IAC5G,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;gHAC4G;IAC5G,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,2DAA2D;IAC3D,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,2DAA2D;IAC3D,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED;;;GAGG;AACH,MAAM,MAAM,YAAY,CAAC,cAAc,IAAI,cAAc,GAAG,QAAQ,CAAC;AAErE,MAAM,WAAW,QAAQ;IACvB,mBAAmB,EAAE;QACnB,WAAW,EAAE,MAAM,EAAE,CAAC;QACtB,aAAa,EAAE,MAAM,CAAC;QACtB,+BAA+B,CAAC,EAAE,mBAAmB,EAAE,CAAC;KACzD,CAAC;CACH;AAED,uDAAuD;AACvD,oBAAY,yBAAyB;IACnC,OAAO,YAAY;IACnB,MAAM,WAAW;CAClB;AAED;;;GAGG;AACH,MAAM,WAAW,mBAAmB;IAClC,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,yBAAyB,CAAC;CACnC;AAED,MAAM,WAAW,UAAU;IACzB,WAAW,EAAE,UAAU,EAAE,CAAC;CAC3B;AAED,MAAM,WAAW,cAAc,CAAC,cAAc;IAC5C,KAAK,EAAE,YAAY,CAAC;IACpB,YAAY,EAAE,cAAc,CAAC;IAC7B,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;;;CAe9B,CAAC;AAEF,eAAO,MAAM,eAAe;;;;;CAK3B,CAAC"}
|
|
@@ -1,6 +1,12 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.loadingSdkState = exports.extractionSdkState = void 0;
|
|
3
|
+
exports.loadingSdkState = exports.extractionSdkState = exports.ProcessedAttachmentStatus = void 0;
|
|
4
|
+
/** Outcome of sending an attachment for processing. */
|
|
5
|
+
var ProcessedAttachmentStatus;
|
|
6
|
+
(function (ProcessedAttachmentStatus) {
|
|
7
|
+
ProcessedAttachmentStatus["Success"] = "success";
|
|
8
|
+
ProcessedAttachmentStatus["Failed"] = "failed";
|
|
9
|
+
})(ProcessedAttachmentStatus || (exports.ProcessedAttachmentStatus = ProcessedAttachmentStatus = {}));
|
|
4
10
|
exports.extractionSdkState = {
|
|
5
11
|
lastSyncStarted: '',
|
|
6
12
|
lastSuccessfulSyncStarted: '',
|
|
@@ -14,7 +20,6 @@ exports.extractionSdkState = {
|
|
|
14
20
|
artifactIds: [],
|
|
15
21
|
lastProcessed: 0,
|
|
16
22
|
lastProcessedAttachmentsIdsList: [],
|
|
17
|
-
failedAttachmentsIdsList: [],
|
|
18
23
|
},
|
|
19
24
|
},
|
|
20
25
|
};
|
|
@@ -466,11 +466,7 @@ export interface ExternalSystemAttachmentStreamingParams {
|
|
|
466
466
|
export interface ExternalSystemAttachmentStreamingResponse {
|
|
467
467
|
httpStream?: AxiosResponse;
|
|
468
468
|
error?: ErrorRecord & {
|
|
469
|
-
/**
|
|
470
|
-
* The HTTP status code of the failed request, if the error originated from one
|
|
471
|
-
* (e.g. a non-2xx response from the external system). Used to classify the error
|
|
472
|
-
* as transient (e.g. 5xx) so it counts toward the attachment's failure limit.
|
|
473
|
-
*/
|
|
469
|
+
/** The HTTP status code of the failed request, if the error originated from one. */
|
|
474
470
|
statusCode?: number;
|
|
475
471
|
};
|
|
476
472
|
delay?: number;
|
|
@@ -485,18 +481,6 @@ export type ProcessAttachmentReturnType = {
|
|
|
485
481
|
error?: {
|
|
486
482
|
message: string;
|
|
487
483
|
fileSize?: number;
|
|
488
|
-
/**
|
|
489
|
-
* Whether the error is a transient upload failure (ECONNABORTED, 5xx) expected
|
|
490
|
-
* to keep recurring for this attachment on retry. Used to count repeated
|
|
491
|
-
* failures per attachment so it can eventually be skipped.
|
|
492
|
-
*/
|
|
493
|
-
isTransient?: boolean;
|
|
494
|
-
/**
|
|
495
|
-
* A fixed allowlist of structural response headers (content-length, content-type,
|
|
496
|
-
* etc.) from the attachment fetch, for diagnostics. Deliberately excludes any
|
|
497
|
-
* header not on the allowlist so auth-adjacent headers never end up here.
|
|
498
|
-
*/
|
|
499
|
-
fetchHeaders?: Record<string, string>;
|
|
500
484
|
};
|
|
501
485
|
} | undefined;
|
|
502
486
|
export type StreamAttachmentsReturnType = {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extraction.d.ts","sourceRoot":"","sources":["../../src/types/extraction.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,sCAAsC,CAAC;AAEjE,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AACtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,iDAAiD,CAAC;AAChF,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAE7D;;;GAGG;AACH,oBAAY,SAAS;IAEnB;;OAEG;IACH,gCAAgC,yCAAyC;IACzE;;OAEG;IACH,uBAAuB,8BAA8B;IACrD;;OAEG;IACH,mBAAmB,0BAA0B;IAC7C;;OAEG;IACH,sBAAsB,6BAA6B;IACnD;;OAEG;IACH,oBAAoB,2BAA2B;IAC/C;;OAEG;IACH,0BAA0B,iCAAiC;IAC3D;;OAEG;IACH,6BAA6B,oCAAoC;IACjE;;OAEG;IACH,2BAA2B,kCAAkC;IAG7D,gBAAgB,uBAAuB;IACvC,mBAAmB,0BAA0B;IAC7C,uBAAuB,8BAA8B;IACrD,0BAA0B,iCAAiC;IAC3D,wBAAwB,gCAAgC;IACxD,kCAAkC,2CAA2C;IAG7E,gBAAgB,uBAAuB;IAGvC,gCAAgC,yCAAyC;IACzE,uBAAuB,8BAA8B;IACrD,mBAAmB,0BAA0B;IAC7C,sBAAsB,6BAA6B;IACnD,2BAA2B,mCAAmC;IAC9D,0BAA0B,iCAAiC;IAC3D,6BAA6B,oCAAoC;IACjE,sCAAsC,+CAA+C;CACtF;AAED;;;GAGG;AACH,oBAAY,kBAAkB;IAE5B;;OAEG;IACH,+BAA+B,wCAAwC;IACvE;;OAEG;IACH,gCAAgC,yCAAyC;IACzE;;OAEG;IACH,sBAAsB,6BAA6B;IACnD;;OAEG;IACH,uBAAuB,8BAA8B;IACrD;;OAEG;IACH,sBAAsB,6BAA6B;IACnD;;OAEG;IACH,mBAAmB,0BAA0B;IAC7C;;OAEG;IACH,kBAAkB,yBAAyB;IAC3C;;OAEG;IACH,mBAAmB,0BAA0B;IAC7C;;OAEG;IACH,wBAAwB,gCAAgC;IACxD;;OAEG;IACH,yBAAyB,iCAAiC;IAC1D;;OAEG;IACH,6BAA6B,oCAAoC;IACjE;;OAEG;IACH,0BAA0B,iCAAiC;IAC3D;;OAEG;IACH,yBAAyB,gCAAgC;IACzD;;OAEG;IACH,0BAA0B,iCAAiC;IAC3D;;OAEG;IACH,+BAA+B,uCAAuC;IACtE;;OAEG;IACH,gCAAgC,wCAAwC;IAGxE,gBAAgB,uBAAuB;IAGvC,8BAA8B,uCAAuC;IACrE,+BAA+B,wCAAwC;IACvE,sBAAsB,6BAA6B;IACnD,uBAAuB,8BAA8B;IACrD,sBAAsB,6BAA6B;IACnD,qBAAqB,4BAA4B;IACjD,kBAAkB,yBAAyB;IAC3C,mBAAmB,0BAA0B;IAC7C,0BAA0B,kCAAkC;IAC5D,2BAA2B,mCAAmC;IAC9D,4BAA4B,mCAAmC;IAC/D,2BAA2B,kCAAkC;IAC7D,wBAAwB,+BAA+B;IACvD,yBAAyB,gCAAgC;IACzD,qCAAqC,8CAA8C;IACnF,sCAAsC,+CAA+C;CACtF;AAED;;;;GAIG;AACH,oBAAY,cAAc;IACxB,OAAO,YAAY;IACnB,WAAW,gBAAgB;CAC5B;AAED;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,oBAAY,gBAAgB;IAC1B,YAAY,iBAAiB;IAC7B,WAAW,gBAAgB;CAC5B;AAED;;;GAGG;AACH,oBAAY,QAAQ;IAClB,kBAAkB;IAClB,WAAW,OAAO;IAClB,iCAAiC;IACjC,YAAY,OAAO;IACnB,mCAAmC;IACnC,eAAe,YAAO;IACtB,mBAAmB;IACnB,YAAY,OAAO;IACnB,cAAc;IACd,OAAO,MAAM;IACb,cAAc;IACd,OAAO,MAAM;IACb,YAAY;IACZ,KAAK,MAAM;CACZ;AAED;;;GAGG;AACH,oBAAY,aAAa;IACvB,yCAAyC;IACzC,cAAc,mBAAmB;IACjC,iEAAiE;IACjE,2BAA2B,gCAAgC;IAC3D,yCAAyC;IACzC,cAAc,mBAAmB;IACjC,gEAAgE;IAChE,0BAA0B,+BAA+B;IACzD,mBAAmB;IACnB,YAAY,iBAAiB;IAC7B,wCAAwC;IACxC,aAAa,kBAAkB;IAC/B,4CAA4C;IAC5C,SAAS,cAAc;CACxB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,aAAa,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,uBAAuB,EAAE,MAAM,CAAC;IAChC,eAAe,EAAE,MAAM,CAAC;IACxB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,KAAK,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,CAAC;CACzB;AAED;;;GAGG;AACH,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,gBAAgB,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,kBAAkB,EAAE,MAAM,CAAC;IAC3B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,uBAAuB,EAAE,MAAM,CAAC;IAChC;;OAEG;IACH,eAAe,EAAE,MAAM,CAAC;IACxB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;IAC7B;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,kBAAkB,CAAC,EAAE,gBAAgB,CAAC;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,gBAAgB,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B;;;OAGG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,kBAAkB,EAAE,MAAM,CAAC;IAC3B;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,KAAK,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,SAAS,CAAC;IAClC;;;OAGG;IACH,mBAAmB,CAAC,EAAE,SAAS,CAAC;IAChC;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,gBAAgB,EAAE,CAAC;IACzC;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IAGvB,OAAO,CAAC,EAAE,YAAY,EAAE,CAAC;IACzB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAG/B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAI9B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED;;;;GAIG;AACH,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE;QACN,KAAK,EAAE,MAAM,EAAE,CAAC;KACjB,CAAC;IACF,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;GAGG;AACH,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE;QACP,OAAO,EAAE;YACP,qBAAqB,EAAE,MAAM,CAAC;SAC/B,CAAC;QACF,kBAAkB,EAAE,MAAM,CAAC;QAC3B,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,OAAO,EAAE,cAAc,CAAC;IACxB,kBAAkB,EAAE;QAClB,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;IACF,UAAU,EAAE,SAAS,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,eAAe,EAAE,cAAc,CAAC;IAChC,aAAa,EAAE,YAAY,CAAC;IAC5B,UAAU,EAAE,SAAS,CAAC;IACtB,UAAU,CAAC,EAAE,SAAS,CAAC;CACxB;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,YAAY,CAAC;IAC5B,UAAU,CAAC,EAAE,SAAS,CAAC;IACvB,eAAe,CAAC,EAAE,cAAc,CAAC;CAClC;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,YAAY,CAAC;IAC5B,UAAU,CAAC,EAAE,SAAS,CAAC;IACvB,eAAe,CAAC,EAAE,cAAc,CAAC;CAClC;AAED,MAAM,MAAM,yCAAyC,GAAG,CAAC,EACvD,IAAI,EACJ,KAAK,GACN,EAAE,uCAAuC,KAAK,OAAO,CAAC,yCAAyC,CAAC,CAAC;AAElG,MAAM,WAAW,uCAAuC;IACtD,IAAI,EAAE,oBAAoB,CAAC;IAC3B,KAAK,EAAE,YAAY,CAAC;CACrB;AAED,MAAM,WAAW,yCAAyC;IACxD,UAAU,CAAC,EAAE,aAAa,CAAC;IAC3B,KAAK,CAAC,EAAE,WAAW,GAAG;QACpB
|
|
1
|
+
{"version":3,"file":"extraction.d.ts","sourceRoot":"","sources":["../../src/types/extraction.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,sCAAsC,CAAC;AAEjE,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AACtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,iDAAiD,CAAC;AAChF,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAE7D;;;GAGG;AACH,oBAAY,SAAS;IAEnB;;OAEG;IACH,gCAAgC,yCAAyC;IACzE;;OAEG;IACH,uBAAuB,8BAA8B;IACrD;;OAEG;IACH,mBAAmB,0BAA0B;IAC7C;;OAEG;IACH,sBAAsB,6BAA6B;IACnD;;OAEG;IACH,oBAAoB,2BAA2B;IAC/C;;OAEG;IACH,0BAA0B,iCAAiC;IAC3D;;OAEG;IACH,6BAA6B,oCAAoC;IACjE;;OAEG;IACH,2BAA2B,kCAAkC;IAG7D,gBAAgB,uBAAuB;IACvC,mBAAmB,0BAA0B;IAC7C,uBAAuB,8BAA8B;IACrD,0BAA0B,iCAAiC;IAC3D,wBAAwB,gCAAgC;IACxD,kCAAkC,2CAA2C;IAG7E,gBAAgB,uBAAuB;IAGvC,gCAAgC,yCAAyC;IACzE,uBAAuB,8BAA8B;IACrD,mBAAmB,0BAA0B;IAC7C,sBAAsB,6BAA6B;IACnD,2BAA2B,mCAAmC;IAC9D,0BAA0B,iCAAiC;IAC3D,6BAA6B,oCAAoC;IACjE,sCAAsC,+CAA+C;CACtF;AAED;;;GAGG;AACH,oBAAY,kBAAkB;IAE5B;;OAEG;IACH,+BAA+B,wCAAwC;IACvE;;OAEG;IACH,gCAAgC,yCAAyC;IACzE;;OAEG;IACH,sBAAsB,6BAA6B;IACnD;;OAEG;IACH,uBAAuB,8BAA8B;IACrD;;OAEG;IACH,sBAAsB,6BAA6B;IACnD;;OAEG;IACH,mBAAmB,0BAA0B;IAC7C;;OAEG;IACH,kBAAkB,yBAAyB;IAC3C;;OAEG;IACH,mBAAmB,0BAA0B;IAC7C;;OAEG;IACH,wBAAwB,gCAAgC;IACxD;;OAEG;IACH,yBAAyB,iCAAiC;IAC1D;;OAEG;IACH,6BAA6B,oCAAoC;IACjE;;OAEG;IACH,0BAA0B,iCAAiC;IAC3D;;OAEG;IACH,yBAAyB,gCAAgC;IACzD;;OAEG;IACH,0BAA0B,iCAAiC;IAC3D;;OAEG;IACH,+BAA+B,uCAAuC;IACtE;;OAEG;IACH,gCAAgC,wCAAwC;IAGxE,gBAAgB,uBAAuB;IAGvC,8BAA8B,uCAAuC;IACrE,+BAA+B,wCAAwC;IACvE,sBAAsB,6BAA6B;IACnD,uBAAuB,8BAA8B;IACrD,sBAAsB,6BAA6B;IACnD,qBAAqB,4BAA4B;IACjD,kBAAkB,yBAAyB;IAC3C,mBAAmB,0BAA0B;IAC7C,0BAA0B,kCAAkC;IAC5D,2BAA2B,mCAAmC;IAC9D,4BAA4B,mCAAmC;IAC/D,2BAA2B,kCAAkC;IAC7D,wBAAwB,+BAA+B;IACvD,yBAAyB,gCAAgC;IACzD,qCAAqC,8CAA8C;IACnF,sCAAsC,+CAA+C;CACtF;AAED;;;;GAIG;AACH,oBAAY,cAAc;IACxB,OAAO,YAAY;IACnB,WAAW,gBAAgB;CAC5B;AAED;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,oBAAY,gBAAgB;IAC1B,YAAY,iBAAiB;IAC7B,WAAW,gBAAgB;CAC5B;AAED;;;GAGG;AACH,oBAAY,QAAQ;IAClB,kBAAkB;IAClB,WAAW,OAAO;IAClB,iCAAiC;IACjC,YAAY,OAAO;IACnB,mCAAmC;IACnC,eAAe,YAAO;IACtB,mBAAmB;IACnB,YAAY,OAAO;IACnB,cAAc;IACd,OAAO,MAAM;IACb,cAAc;IACd,OAAO,MAAM;IACb,YAAY;IACZ,KAAK,MAAM;CACZ;AAED;;;GAGG;AACH,oBAAY,aAAa;IACvB,yCAAyC;IACzC,cAAc,mBAAmB;IACjC,iEAAiE;IACjE,2BAA2B,gCAAgC;IAC3D,yCAAyC;IACzC,cAAc,mBAAmB;IACjC,gEAAgE;IAChE,0BAA0B,+BAA+B;IACzD,mBAAmB;IACnB,YAAY,iBAAiB;IAC7B,wCAAwC;IACxC,aAAa,kBAAkB;IAC/B,4CAA4C;IAC5C,SAAS,cAAc;CACxB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,aAAa,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,uBAAuB,EAAE,MAAM,CAAC;IAChC,eAAe,EAAE,MAAM,CAAC;IACxB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,KAAK,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,CAAC;CACzB;AAED;;;GAGG;AACH,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,gBAAgB,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,kBAAkB,EAAE,MAAM,CAAC;IAC3B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,uBAAuB,EAAE,MAAM,CAAC;IAChC;;OAEG;IACH,eAAe,EAAE,MAAM,CAAC;IACxB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;IAC7B;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,kBAAkB,CAAC,EAAE,gBAAgB,CAAC;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,gBAAgB,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B;;;OAGG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,kBAAkB,EAAE,MAAM,CAAC;IAC3B;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,KAAK,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,SAAS,CAAC;IAClC;;;OAGG;IACH,mBAAmB,CAAC,EAAE,SAAS,CAAC;IAChC;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,gBAAgB,EAAE,CAAC;IACzC;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IAGvB,OAAO,CAAC,EAAE,YAAY,EAAE,CAAC;IACzB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAG/B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAI9B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED;;;;GAIG;AACH,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE;QACN,KAAK,EAAE,MAAM,EAAE,CAAC;KACjB,CAAC;IACF,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;GAGG;AACH,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE;QACP,OAAO,EAAE;YACP,qBAAqB,EAAE,MAAM,CAAC;SAC/B,CAAC;QACF,kBAAkB,EAAE,MAAM,CAAC;QAC3B,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,OAAO,EAAE,cAAc,CAAC;IACxB,kBAAkB,EAAE;QAClB,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;IACF,UAAU,EAAE,SAAS,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,eAAe,EAAE,cAAc,CAAC;IAChC,aAAa,EAAE,YAAY,CAAC;IAC5B,UAAU,EAAE,SAAS,CAAC;IACtB,UAAU,CAAC,EAAE,SAAS,CAAC;CACxB;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,YAAY,CAAC;IAC5B,UAAU,CAAC,EAAE,SAAS,CAAC;IACvB,eAAe,CAAC,EAAE,cAAc,CAAC;CAClC;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,YAAY,CAAC;IAC5B,UAAU,CAAC,EAAE,SAAS,CAAC;IACvB,eAAe,CAAC,EAAE,cAAc,CAAC;CAClC;AAED,MAAM,MAAM,yCAAyC,GAAG,CAAC,EACvD,IAAI,EACJ,KAAK,GACN,EAAE,uCAAuC,KAAK,OAAO,CAAC,yCAAyC,CAAC,CAAC;AAElG,MAAM,WAAW,uCAAuC;IACtD,IAAI,EAAE,oBAAoB,CAAC;IAC3B,KAAK,EAAE,YAAY,CAAC;CACrB;AAED,MAAM,WAAW,yCAAyC;IACxD,UAAU,CAAC,EAAE,aAAa,CAAC;IAC3B,KAAK,CAAC,EAAE,WAAW,GAAG;QACpB,oFAAoF;QACpF,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,CAAC;IACF,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,yBAAyB;IACxC,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,SAAS,CAAC,EAAE,WAAW,CAAC;CACzB;AAED,MAAM,MAAM,2BAA2B,GACnC;IACE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE;QACN,OAAO,EAAE,MAAM,CAAC;QAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;CACH,GACD,SAAS,CAAC;AAEd,MAAM,MAAM,2BAA2B,GACnC;IACE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GACD,SAAS,CAAC;AAEd,MAAM,MAAM,uCAAuC,CACjD,KAAK,EACL,QAAQ,EACR,cAAc,IACZ,CAAC,EACH,WAAW,EACX,OAAO,EACP,SAAS,GACV,EAAE;IACD,WAAW,EAAE,KAAK,CAAC;IACnB,OAAO,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;IACvC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,KAAK,QAAQ,CAAC;AAEf,MAAM,MAAM,iCAAiC,GAAG,CAAC,EAC/C,UAAU,EACV,MAAM,GACP,EAAE;IACD,UAAU,EAAE,oBAAoB,CAAC;IACjC,MAAM,EAAE,yCAAyC,CAAC;CACnD,KAAK,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAE3C,MAAM,MAAM,wCAAwC,CAAC,QAAQ,EAAE,cAAc,IAC3E,CAAC,EACC,kBAAkB,EAClB,OAAO,EACP,MAAM,GACP,EAAE;IACD,kBAAkB,EAAE,QAAQ,CAAC;IAC7B,OAAO,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;IACvC,MAAM,EAAE,yCAAyC,CAAC;CACnD,KAAK,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAE7C,MAAM,WAAW,kCAAkC,CACjD,cAAc,EACd,KAAK,EACL,QAAQ;IAER,OAAO,EAAE,uCAAuC,CAC9C,KAAK,EACL,QAAQ,EACR,cAAc,CACf,CAAC;IACF,QAAQ,EAAE,wCAAwC,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;CAC9E"}
|
|
@@ -1,12 +1,4 @@
|
|
|
1
1
|
import { ArtifactDateRanges, UploaderResult } from './uploader.interfaces';
|
|
2
|
-
/**
|
|
3
|
-
* Determines whether an error from the SDK-owned upload path (get upload URL, stream
|
|
4
|
-
* artifact, confirm upload) is a transient failure that's expected to keep recurring on
|
|
5
|
-
* retry for the same attachment: a client-side timeout (ECONNABORTED) or a 5xx response.
|
|
6
|
-
* Used to count repeated failures per attachment so it can eventually be skipped instead
|
|
7
|
-
* of retried forever.
|
|
8
|
-
*/
|
|
9
|
-
export declare function isTransientUploadError(error: unknown): boolean;
|
|
10
2
|
/**
|
|
11
3
|
* Computes oldest/newest created and modified timestamps (RFC3339) across uploaded items.
|
|
12
4
|
* @param fetchedObjects - Single object or array of objects (e.g. NormalizedItem[])
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"uploader.helpers.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.helpers.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"uploader.helpers.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.helpers.ts"],"names":[],"mappings":"AASA,OAAO,EAEL,kBAAkB,EAClB,cAAc,EACf,MAAM,uBAAuB,CAAC;AAE/B;;;GAGG;AACH,wBAAgB,yBAAyB,CACvC,cAAc,EAAE,MAAM,EAAE,GAAG,MAAM,GAChC,kBAAkB,CAwEpB;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,WAAW,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,CAMxE;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAC5B,kBAAkB,EAAE,MAAM,GACzB,cAAc,CAAC,MAAM,CAAC,CAOxB;AAED;;;;GAIG;AACH,wBAAgB,UAAU,CAAC,WAAW,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,CAMxE;AAED;;;;;GAKG;AACH,wBAAsB,eAAe,CACnC,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,GAAG,MAAM,EAAE,GAChC,OAAO,CAAC,IAAI,CAAC,CA4Bf;AAED;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAmBzD"}
|
|
@@ -36,33 +36,17 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
36
36
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
37
37
|
};
|
|
38
38
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
|
-
exports.isTransientUploadError = isTransientUploadError;
|
|
40
39
|
exports.computeArtifactDateRanges = computeArtifactDateRanges;
|
|
41
40
|
exports.compressGzip = compressGzip;
|
|
42
41
|
exports.decompressGzip = decompressGzip;
|
|
43
42
|
exports.parseJsonl = parseJsonl;
|
|
44
43
|
exports.downloadToLocal = downloadToLocal;
|
|
45
44
|
exports.truncateFilename = truncateFilename;
|
|
46
|
-
const axios_1 = __importDefault(require("axios"));
|
|
47
45
|
const fs_1 = __importStar(require("fs"));
|
|
48
46
|
const js_jsonl_1 = require("js-jsonl");
|
|
49
47
|
const zlib_1 = __importDefault(require("zlib"));
|
|
50
48
|
const constants_1 = require("../common/constants");
|
|
51
49
|
const uploader_interfaces_1 = require("./uploader.interfaces");
|
|
52
|
-
/**
|
|
53
|
-
* Determines whether an error from the SDK-owned upload path (get upload URL, stream
|
|
54
|
-
* artifact, confirm upload) is a transient failure that's expected to keep recurring on
|
|
55
|
-
* retry for the same attachment: a client-side timeout (ECONNABORTED) or a 5xx response.
|
|
56
|
-
* Used to count repeated failures per attachment so it can eventually be skipped instead
|
|
57
|
-
* of retried forever.
|
|
58
|
-
*/
|
|
59
|
-
function isTransientUploadError(error) {
|
|
60
|
-
var _a, _b;
|
|
61
|
-
if (!axios_1.default.isAxiosError(error)) {
|
|
62
|
-
return false;
|
|
63
|
-
}
|
|
64
|
-
return error.code === 'ECONNABORTED' || ((_b = (_a = error.response) === null || _a === void 0 ? void 0 : _a.status) !== null && _b !== void 0 ? _b : 0) >= 500;
|
|
65
|
-
}
|
|
66
50
|
/**
|
|
67
51
|
* Computes oldest/newest created and modified timestamps (RFC3339) across uploaded items.
|
|
68
52
|
* @param fetchedObjects - Single object or array of objects (e.g. NormalizedItem[])
|
|
@@ -394,43 +394,4 @@ describe('uploader.helpers', () => {
|
|
|
394
394
|
expect(result.length).toBe(constants_1.MAX_DEVREV_FILENAME_LENGTH);
|
|
395
395
|
});
|
|
396
396
|
});
|
|
397
|
-
describe(uploader_helpers_1.isTransientUploadError.name, () => {
|
|
398
|
-
it('should return true for ECONNABORTED errors', () => {
|
|
399
|
-
// Arrange
|
|
400
|
-
const error = {
|
|
401
|
-
isAxiosError: true,
|
|
402
|
-
code: 'ECONNABORTED',
|
|
403
|
-
};
|
|
404
|
-
// Act & Assert
|
|
405
|
-
expect((0, uploader_helpers_1.isTransientUploadError)(error)).toBe(true);
|
|
406
|
-
});
|
|
407
|
-
it('should return true for 5xx response errors', () => {
|
|
408
|
-
// Arrange
|
|
409
|
-
const error = {
|
|
410
|
-
isAxiosError: true,
|
|
411
|
-
response: { status: 503 },
|
|
412
|
-
};
|
|
413
|
-
// Act & Assert
|
|
414
|
-
expect((0, uploader_helpers_1.isTransientUploadError)(error)).toBe(true);
|
|
415
|
-
});
|
|
416
|
-
it('should return false for 4xx response errors', () => {
|
|
417
|
-
// Arrange
|
|
418
|
-
const error = {
|
|
419
|
-
isAxiosError: true,
|
|
420
|
-
response: { status: 404 },
|
|
421
|
-
};
|
|
422
|
-
// Act & Assert
|
|
423
|
-
expect((0, uploader_helpers_1.isTransientUploadError)(error)).toBe(false);
|
|
424
|
-
});
|
|
425
|
-
it('should return false for non-axios errors', () => {
|
|
426
|
-
// Arrange
|
|
427
|
-
const error = new Error('some other error');
|
|
428
|
-
// Act & Assert
|
|
429
|
-
expect((0, uploader_helpers_1.isTransientUploadError)(error)).toBe(false);
|
|
430
|
-
});
|
|
431
|
-
it('[edge] should return false when error is undefined', () => {
|
|
432
|
-
// Act & Assert
|
|
433
|
-
expect((0, uploader_helpers_1.isTransientUploadError)(undefined)).toBe(false);
|
|
434
|
-
});
|
|
435
|
-
});
|
|
436
397
|
});
|
package/package.json
CHANGED