@devrev/ts-adaas 1.5.1 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/common/constants.d.ts +5 -1
- package/dist/common/constants.js +6 -2
- package/dist/common/control-protocol.js +1 -1
- package/dist/common/helpers.d.ts +2 -1
- package/dist/common/helpers.js +15 -0
- package/dist/{tests/from_devrev/loading.test.js → common/helpers.test.js} +1 -1
- package/dist/common/install-initial-domain-mapping.js +37 -56
- package/dist/common/install-initial-domain-mapping.test.js +215 -0
- package/dist/repo/repo.interfaces.d.ts +1 -0
- package/dist/state/state.js +21 -12
- package/dist/tests/mock-server.d.ts +20 -0
- package/dist/tests/mock-server.js +278 -0
- package/dist/tests/test-helpers.d.ts +1 -1
- package/dist/tests/test-helpers.interfaces.d.ts +3 -1
- package/dist/tests/test-helpers.js +27 -28
- package/dist/tests/timeout-handling/extraction.d.ts +3 -0
- package/dist/tests/timeout-handling/extraction.js +21 -0
- package/dist/tests/timeout-handling/timeout-1.d.ts +1 -0
- package/dist/tests/timeout-handling/timeout-1.js +14 -0
- package/dist/tests/timeout-handling/timeout-1.test.d.ts +1 -0
- package/dist/tests/timeout-handling/timeout-1.test.js +45 -0
- package/dist/tests/timeout-handling/timeout-2.d.ts +1 -0
- package/dist/tests/timeout-handling/timeout-2.js +42 -0
- package/dist/tests/timeout-handling/timeout-2.test.d.ts +1 -0
- package/dist/tests/timeout-handling/timeout-2.test.js +46 -0
- package/dist/tests/timeout-handling/timeout-3a.d.ts +1 -0
- package/dist/tests/timeout-handling/timeout-3a.js +37 -0
- package/dist/tests/timeout-handling/timeout-3a.test.d.ts +1 -0
- package/dist/tests/timeout-handling/timeout-3a.test.js +46 -0
- package/dist/tests/timeout-handling/timeout-3b.d.ts +1 -0
- package/dist/tests/timeout-handling/timeout-3b.js +38 -0
- package/dist/tests/timeout-handling/timeout-3b.test.d.ts +1 -0
- package/dist/tests/timeout-handling/timeout-3b.test.js +46 -0
- package/dist/types/workers.d.ts +1 -8
- package/dist/types/workers.js +0 -1
- package/dist/uploader/uploader.js +2 -1
- package/dist/uploader/uploader.test.js +3 -0
- package/dist/workers/process-task.js +0 -4
- package/dist/workers/spawn.d.ts +3 -1
- package/dist/workers/spawn.js +47 -39
- package/package.json +3 -2
- package/dist/tests/test-worker.js +0 -14
- /package/dist/{tests/from_devrev/loading.test.d.ts → common/helpers.test.d.ts} +0 -0
- /package/dist/{tests/test-worker.d.ts → common/install-initial-domain-mapping.test.d.ts} +0 -0
|
@@ -0,0 +1,278 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.MockServer = void 0;
|
|
37
|
+
const http = __importStar(require("http"));
|
|
38
|
+
const url = __importStar(require("url"));
|
|
39
|
+
class MockServer {
|
|
40
|
+
constructor(port) {
|
|
41
|
+
this.server = null;
|
|
42
|
+
this.consoleLogs = [];
|
|
43
|
+
this.requests = [];
|
|
44
|
+
this.port = port || 3001; // Default to standard test port
|
|
45
|
+
}
|
|
46
|
+
getDefaultWorkerState() {
|
|
47
|
+
return {
|
|
48
|
+
lastSyncStarted: '',
|
|
49
|
+
lastSuccessfulSyncStarted: '',
|
|
50
|
+
snapInVersionId: 'test-snap-in-version-id',
|
|
51
|
+
toDevRev: {
|
|
52
|
+
attachmentsMetadata: {
|
|
53
|
+
artifactIds: [],
|
|
54
|
+
lastProcessed: 0,
|
|
55
|
+
lastProcessedAttachmentsIdsList: [],
|
|
56
|
+
},
|
|
57
|
+
},
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
handleRequest(req, res) {
|
|
61
|
+
const parsedUrl = url.parse(req.url || '', true);
|
|
62
|
+
const pathname = parsedUrl.pathname || '';
|
|
63
|
+
const method = req.method || 'GET';
|
|
64
|
+
// Track this request
|
|
65
|
+
const requestInfo = {
|
|
66
|
+
method,
|
|
67
|
+
url: req.url || '',
|
|
68
|
+
};
|
|
69
|
+
// For POST requests, capture the body
|
|
70
|
+
if (method === 'POST') {
|
|
71
|
+
let body = '';
|
|
72
|
+
req.on('data', (chunk) => (body += chunk.toString()));
|
|
73
|
+
req.on('end', () => {
|
|
74
|
+
try {
|
|
75
|
+
requestInfo.body = JSON.parse(body);
|
|
76
|
+
}
|
|
77
|
+
catch (_a) {
|
|
78
|
+
requestInfo.body = body;
|
|
79
|
+
}
|
|
80
|
+
this.requests.push(requestInfo);
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
else {
|
|
84
|
+
this.requests.push(requestInfo);
|
|
85
|
+
}
|
|
86
|
+
// Set CORS headers for all requests
|
|
87
|
+
res.setHeader('Access-Control-Allow-Origin', '*');
|
|
88
|
+
res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS');
|
|
89
|
+
res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Authorization');
|
|
90
|
+
res.setHeader('Content-Type', 'application/json');
|
|
91
|
+
switch (method) {
|
|
92
|
+
case 'OPTIONS':
|
|
93
|
+
// Handle preflight CORS requests
|
|
94
|
+
res.writeHead(200);
|
|
95
|
+
res.end();
|
|
96
|
+
break;
|
|
97
|
+
case 'GET':
|
|
98
|
+
switch (true) {
|
|
99
|
+
case pathname.includes('airdrop.external-worker'):
|
|
100
|
+
// Worker state GET endpoint
|
|
101
|
+
res.writeHead(200);
|
|
102
|
+
res.end(JSON.stringify({
|
|
103
|
+
state: JSON.stringify(this.getDefaultWorkerState()),
|
|
104
|
+
}));
|
|
105
|
+
break;
|
|
106
|
+
case pathname.includes('snap-ins.get'):
|
|
107
|
+
// Snap-ins endpoint for install-initial-domain-mapping
|
|
108
|
+
res.writeHead(200);
|
|
109
|
+
res.end(JSON.stringify({
|
|
110
|
+
snap_in: {
|
|
111
|
+
imports: [{ name: 'test_import_slug' }],
|
|
112
|
+
snap_in_version: { slug: 'test_snap_in_slug' },
|
|
113
|
+
},
|
|
114
|
+
}));
|
|
115
|
+
break;
|
|
116
|
+
case pathname.includes('/upload') || pathname.includes('/download'):
|
|
117
|
+
// Generic file download endpoints for artifacts
|
|
118
|
+
res.writeHead(200);
|
|
119
|
+
res.setHeader('Content-Type', 'application/octet-stream');
|
|
120
|
+
res.end(Buffer.from('mock file content'));
|
|
121
|
+
break;
|
|
122
|
+
default:
|
|
123
|
+
// Unknown GET endpoint
|
|
124
|
+
res.writeHead(404);
|
|
125
|
+
res.end(JSON.stringify({
|
|
126
|
+
error: 'GET endpoint not found',
|
|
127
|
+
code: 'ENDPOINT_NOT_FOUND',
|
|
128
|
+
}));
|
|
129
|
+
}
|
|
130
|
+
break;
|
|
131
|
+
case 'POST':
|
|
132
|
+
switch (true) {
|
|
133
|
+
case pathname.includes('airdrop.external-worker'):
|
|
134
|
+
// Worker state POST/UPDATE endpoint
|
|
135
|
+
res.writeHead(200);
|
|
136
|
+
res.end(JSON.stringify({
|
|
137
|
+
success: true,
|
|
138
|
+
updated_at: new Date().toISOString(),
|
|
139
|
+
}));
|
|
140
|
+
break;
|
|
141
|
+
case pathname.includes('airdrop.external-extractor.message'):
|
|
142
|
+
// Callback endpoint for event emissions
|
|
143
|
+
res.writeHead(200);
|
|
144
|
+
res.end(JSON.stringify({
|
|
145
|
+
success: true,
|
|
146
|
+
received_at: new Date().toISOString(),
|
|
147
|
+
}));
|
|
148
|
+
break;
|
|
149
|
+
case pathname.includes('recipe.blueprints.create'):
|
|
150
|
+
// Recipe blueprint creation endpoint
|
|
151
|
+
res.writeHead(200);
|
|
152
|
+
res.end(JSON.stringify({
|
|
153
|
+
recipe_blueprint: { id: 'test_recipe_blueprint_id' },
|
|
154
|
+
}));
|
|
155
|
+
break;
|
|
156
|
+
case pathname.includes('initial-domain-mappings.install'):
|
|
157
|
+
// Domain mappings installation endpoint
|
|
158
|
+
res.writeHead(200);
|
|
159
|
+
res.end(JSON.stringify({
|
|
160
|
+
success: true,
|
|
161
|
+
message: 'Initial domain mappings installed successfully',
|
|
162
|
+
}));
|
|
163
|
+
break;
|
|
164
|
+
case pathname.includes('artifacts.prepare'):
|
|
165
|
+
// Artifacts prepare endpoint for uploader
|
|
166
|
+
res.writeHead(200);
|
|
167
|
+
res.end(JSON.stringify({
|
|
168
|
+
id: `test_artifact_${Date.now()}`,
|
|
169
|
+
url: `${this.getBaseUrl()}/upload`,
|
|
170
|
+
form_data: [
|
|
171
|
+
{ key: 'key', value: 'test-key' },
|
|
172
|
+
{ key: 'x-amz-algorithm', value: 'AWS4-HMAC-SHA256' },
|
|
173
|
+
{ key: 'x-amz-credential', value: 'test-credential' },
|
|
174
|
+
{ key: 'x-amz-date', value: new Date().toISOString() },
|
|
175
|
+
{ key: 'policy', value: 'test-policy' },
|
|
176
|
+
{ key: 'x-amz-signature', value: 'test-signature' },
|
|
177
|
+
],
|
|
178
|
+
}));
|
|
179
|
+
break;
|
|
180
|
+
case pathname.includes('artifacts.locate'):
|
|
181
|
+
// Artifacts locate endpoint for uploader
|
|
182
|
+
res.writeHead(200);
|
|
183
|
+
res.end(JSON.stringify({
|
|
184
|
+
url: `${this.getBaseUrl()}/download/artifact.jsonl.gz`,
|
|
185
|
+
}));
|
|
186
|
+
break;
|
|
187
|
+
case pathname.includes('/upload'):
|
|
188
|
+
// File upload endpoint for artifacts
|
|
189
|
+
res.writeHead(200);
|
|
190
|
+
res.end(JSON.stringify({
|
|
191
|
+
success: true,
|
|
192
|
+
message: 'File uploaded successfully',
|
|
193
|
+
ETag: '"test-etag-12345"',
|
|
194
|
+
}));
|
|
195
|
+
break;
|
|
196
|
+
default:
|
|
197
|
+
// Unknown POST endpoint
|
|
198
|
+
res.writeHead(404);
|
|
199
|
+
res.end(JSON.stringify({
|
|
200
|
+
error: 'POST endpoint not found',
|
|
201
|
+
code: 'ENDPOINT_NOT_FOUND',
|
|
202
|
+
}));
|
|
203
|
+
}
|
|
204
|
+
break;
|
|
205
|
+
case 'PUT':
|
|
206
|
+
switch (true) {
|
|
207
|
+
case pathname.includes('/upload'):
|
|
208
|
+
// PUT file upload endpoint for artifacts (alternative upload method)
|
|
209
|
+
res.writeHead(200);
|
|
210
|
+
res.end(JSON.stringify({
|
|
211
|
+
success: true,
|
|
212
|
+
message: 'File uploaded successfully via PUT',
|
|
213
|
+
ETag: '"test-etag-put-12345"',
|
|
214
|
+
}));
|
|
215
|
+
break;
|
|
216
|
+
default:
|
|
217
|
+
// Unknown PUT endpoint
|
|
218
|
+
res.writeHead(404);
|
|
219
|
+
res.end(JSON.stringify({
|
|
220
|
+
error: 'PUT endpoint not found',
|
|
221
|
+
code: 'ENDPOINT_NOT_FOUND',
|
|
222
|
+
}));
|
|
223
|
+
}
|
|
224
|
+
break;
|
|
225
|
+
default:
|
|
226
|
+
// Unsupported HTTP method
|
|
227
|
+
res.writeHead(405);
|
|
228
|
+
res.end(JSON.stringify({
|
|
229
|
+
error: `Method ${method} not allowed`,
|
|
230
|
+
code: 'METHOD_NOT_ALLOWED',
|
|
231
|
+
}));
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
async start() {
|
|
235
|
+
return new Promise((resolve, reject) => {
|
|
236
|
+
this.server = http.createServer((req, res) => {
|
|
237
|
+
this.handleRequest(req, res);
|
|
238
|
+
});
|
|
239
|
+
this.server.listen(this.port, () => {
|
|
240
|
+
console.log(`Mock server running on port ${this.port}`);
|
|
241
|
+
resolve();
|
|
242
|
+
});
|
|
243
|
+
this.server.on('error', (error) => {
|
|
244
|
+
reject(error);
|
|
245
|
+
});
|
|
246
|
+
});
|
|
247
|
+
}
|
|
248
|
+
async stop() {
|
|
249
|
+
return new Promise((resolve) => {
|
|
250
|
+
if (this.server) {
|
|
251
|
+
this.server.close(() => {
|
|
252
|
+
console.log('Mock server stopped');
|
|
253
|
+
this.server = null;
|
|
254
|
+
resolve();
|
|
255
|
+
});
|
|
256
|
+
}
|
|
257
|
+
else {
|
|
258
|
+
resolve();
|
|
259
|
+
}
|
|
260
|
+
});
|
|
261
|
+
}
|
|
262
|
+
getBaseUrl() {
|
|
263
|
+
return `http://localhost:${this.port}`;
|
|
264
|
+
}
|
|
265
|
+
getPort() {
|
|
266
|
+
return this.port;
|
|
267
|
+
}
|
|
268
|
+
getConsoleLogs() {
|
|
269
|
+
return [...this.consoleLogs];
|
|
270
|
+
}
|
|
271
|
+
getRequests() {
|
|
272
|
+
return [...this.requests];
|
|
273
|
+
}
|
|
274
|
+
clearRequests() {
|
|
275
|
+
this.requests = [];
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
exports.MockServer = MockServer;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { Item, NormalizedAttachment, NormalizedItem } from '../repo/repo.interfaces';
|
|
2
2
|
import { AirdropEvent } from '../types/extraction';
|
|
3
3
|
import { CreateEventInterface } from './test-helpers.interfaces';
|
|
4
|
-
export declare function createEvent({ eventType, externalSyncUnits, progress, error, delay, contextOverrides, payloadOverrides, }: CreateEventInterface): AirdropEvent;
|
|
4
|
+
export declare function createEvent({ eventType, externalSyncUnits, progress, error, delay, contextOverrides, payloadOverrides, eventContextOverrides, executionMetadataOverrides, }: CreateEventInterface): AirdropEvent;
|
|
5
5
|
export declare function createItem(id: number): Item;
|
|
6
6
|
export declare function createItems(count: number): Item[];
|
|
7
7
|
export declare function normalizeItem(item: Item): NormalizedItem;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { ErrorRecord } from '../types/common';
|
|
2
|
-
import { AirdropEvent, EventType, ExternalSyncUnit } from '../types/extraction';
|
|
2
|
+
import { AirdropEvent, EventType, ExternalSyncUnit, EventContext } from '../types/extraction';
|
|
3
3
|
export interface CreateEventInterface {
|
|
4
4
|
eventType: EventType;
|
|
5
5
|
externalSyncUnits?: ExternalSyncUnit[];
|
|
@@ -8,4 +8,6 @@ export interface CreateEventInterface {
|
|
|
8
8
|
delay?: number;
|
|
9
9
|
contextOverrides?: Partial<AirdropEvent['context']>;
|
|
10
10
|
payloadOverrides?: Partial<AirdropEvent['payload']>;
|
|
11
|
+
eventContextOverrides?: Partial<EventContext>;
|
|
12
|
+
executionMetadataOverrides?: Partial<AirdropEvent['execution_metadata']>;
|
|
11
13
|
}
|
|
@@ -6,7 +6,31 @@ exports.createItems = createItems;
|
|
|
6
6
|
exports.normalizeItem = normalizeItem;
|
|
7
7
|
exports.createAttachment = createAttachment;
|
|
8
8
|
exports.createAttachments = createAttachments;
|
|
9
|
-
function createEvent({ eventType, externalSyncUnits = [], progress, error, delay, contextOverrides = {}, payloadOverrides = {}, }) {
|
|
9
|
+
function createEvent({ eventType, externalSyncUnits = [], progress, error, delay, contextOverrides = {}, payloadOverrides = {}, eventContextOverrides = {}, executionMetadataOverrides = {}, }) {
|
|
10
|
+
const defaultEventContext = {
|
|
11
|
+
callback_url: 'test_callback_url',
|
|
12
|
+
dev_org: 'test_dev_org',
|
|
13
|
+
dev_org_id: 'test_dev_org_id',
|
|
14
|
+
dev_user: 'test_dev_user',
|
|
15
|
+
dev_user_id: 'test_dev_user_id',
|
|
16
|
+
external_sync_unit: 'test_external_sync_unit',
|
|
17
|
+
external_sync_unit_id: 'test_external_sync_unit_id',
|
|
18
|
+
external_sync_unit_name: 'test_external_sync_unit_name',
|
|
19
|
+
external_system: 'test_external_system',
|
|
20
|
+
external_system_type: 'test_external_system_type',
|
|
21
|
+
import_slug: 'test_import_slug',
|
|
22
|
+
mode: 'test_mode',
|
|
23
|
+
request_id: 'test_request_id',
|
|
24
|
+
snap_in_slug: 'test_snap_in_slug',
|
|
25
|
+
snap_in_version_id: 'test_snap_in_version_id',
|
|
26
|
+
sync_run: 'test_sync_run',
|
|
27
|
+
sync_run_id: 'test_sync_run_id',
|
|
28
|
+
sync_tier: 'test_sync_tier',
|
|
29
|
+
sync_unit: 'test_sync_unit',
|
|
30
|
+
sync_unit_id: 'test_sync_unit_id',
|
|
31
|
+
uuid: 'test_uuid',
|
|
32
|
+
worker_data_url: 'test_worker_data_url',
|
|
33
|
+
};
|
|
10
34
|
return {
|
|
11
35
|
context: Object.assign({ secrets: {
|
|
12
36
|
service_account_token: 'test_token',
|
|
@@ -16,38 +40,13 @@ function createEvent({ eventType, externalSyncUnits = [], progress, error, delay
|
|
|
16
40
|
org_name: 'test_org_name',
|
|
17
41
|
key: 'test_key',
|
|
18
42
|
key_type: 'test_key_type',
|
|
19
|
-
}, event_context: {
|
|
20
|
-
callback_url: 'test_callback_url',
|
|
21
|
-
dev_org: 'test_dev_org',
|
|
22
|
-
dev_org_id: 'test_dev_org_id',
|
|
23
|
-
dev_user: 'test_dev_user',
|
|
24
|
-
dev_user_id: 'test_dev_user_id',
|
|
25
|
-
external_sync_unit: 'test_external_sync_unit',
|
|
26
|
-
external_sync_unit_id: 'test_external_sync_unit_id',
|
|
27
|
-
external_sync_unit_name: 'test_external_sync_unit_name',
|
|
28
|
-
external_system: 'test_external_system',
|
|
29
|
-
external_system_type: 'test_external_system_type',
|
|
30
|
-
import_slug: 'test_import_slug',
|
|
31
|
-
mode: 'test_mode',
|
|
32
|
-
request_id: 'test_request_id',
|
|
33
|
-
snap_in_slug: 'test_snap_in_slug',
|
|
34
|
-
snap_in_version_id: 'test_snap_in_version_id',
|
|
35
|
-
sync_run: 'test_sync_run',
|
|
36
|
-
sync_run_id: 'test_sync_run_id',
|
|
37
|
-
sync_tier: 'test_sync_tier',
|
|
38
|
-
sync_unit: 'test_sync_unit',
|
|
39
|
-
sync_unit_id: 'test_sync_unit_id',
|
|
40
|
-
uuid: 'test_uuid',
|
|
41
|
-
worker_data_url: 'test_worker_data_url',
|
|
42
|
-
}, event_type: eventType, event_data: {
|
|
43
|
+
}, event_context: Object.assign(Object.assign({}, defaultEventContext), eventContextOverrides), event_type: eventType, event_data: {
|
|
43
44
|
external_sync_units: externalSyncUnits,
|
|
44
45
|
progress,
|
|
45
46
|
error,
|
|
46
47
|
delay,
|
|
47
48
|
} }, payloadOverrides),
|
|
48
|
-
execution_metadata: {
|
|
49
|
-
devrev_endpoint: 'test_devrev_endpoint',
|
|
50
|
-
},
|
|
49
|
+
execution_metadata: Object.assign({ devrev_endpoint: 'test_devrev_endpoint' }, executionMetadataOverrides),
|
|
51
50
|
input_data: {
|
|
52
51
|
global_values: {
|
|
53
52
|
test_global_key: 'test_global_value',
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const index_1 = require("../../index");
|
|
4
|
+
const initialState = {};
|
|
5
|
+
const initialDomainMapping = {};
|
|
6
|
+
const run = async (events, workerPath) => {
|
|
7
|
+
for (const event of events) {
|
|
8
|
+
await (0, index_1.spawn)({
|
|
9
|
+
event,
|
|
10
|
+
initialState,
|
|
11
|
+
workerPath,
|
|
12
|
+
initialDomainMapping,
|
|
13
|
+
options: {
|
|
14
|
+
batchSize: 1000,
|
|
15
|
+
timeout: 5 * 1000, // 5 seconds
|
|
16
|
+
isLocalDevelopment: true,
|
|
17
|
+
},
|
|
18
|
+
});
|
|
19
|
+
}
|
|
20
|
+
};
|
|
21
|
+
exports.default = run;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const index_1 = require("../../index");
|
|
4
|
+
(0, index_1.processTask)({
|
|
5
|
+
task: async ({ adapter }) => {
|
|
6
|
+
for (let i = 0; i < 10; i++) {
|
|
7
|
+
console.log('timeout-1 iteration', i);
|
|
8
|
+
}
|
|
9
|
+
await adapter.emit(index_1.ExtractorEventType.ExtractionDataDone);
|
|
10
|
+
},
|
|
11
|
+
onTimeout: async ({ adapter }) => {
|
|
12
|
+
await adapter.emit(index_1.ExtractorEventType.ExtractionDataProgress);
|
|
13
|
+
},
|
|
14
|
+
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
const extraction_1 = require("../../types/extraction");
|
|
7
|
+
const test_helpers_1 = require("../test-helpers");
|
|
8
|
+
const extraction_2 = __importDefault(require("./extraction"));
|
|
9
|
+
const mock_server_1 = require("../mock-server");
|
|
10
|
+
describe('timeout-1', () => {
|
|
11
|
+
let mockServer;
|
|
12
|
+
beforeAll(async () => {
|
|
13
|
+
mockServer = new mock_server_1.MockServer(3001);
|
|
14
|
+
await mockServer.start();
|
|
15
|
+
});
|
|
16
|
+
afterAll(async () => {
|
|
17
|
+
if (mockServer) {
|
|
18
|
+
await mockServer.stop();
|
|
19
|
+
}
|
|
20
|
+
});
|
|
21
|
+
beforeEach(() => {
|
|
22
|
+
jest.clearAllMocks();
|
|
23
|
+
mockServer.clearRequests();
|
|
24
|
+
});
|
|
25
|
+
it('should emit done event since there is no timeout', async () => {
|
|
26
|
+
const baseUrl = mockServer.getBaseUrl();
|
|
27
|
+
const event = (0, test_helpers_1.createEvent)({
|
|
28
|
+
eventType: extraction_1.EventType.ExtractionDataStart,
|
|
29
|
+
eventContextOverrides: {
|
|
30
|
+
callback_url: `${baseUrl}/internal/airdrop.external-extractor.message`,
|
|
31
|
+
worker_data_url: `${baseUrl}/internal/airdrop.external-worker`,
|
|
32
|
+
},
|
|
33
|
+
executionMetadataOverrides: {
|
|
34
|
+
devrev_endpoint: `${baseUrl}`,
|
|
35
|
+
},
|
|
36
|
+
});
|
|
37
|
+
await (0, extraction_2.default)([event], __dirname + '/timeout-1');
|
|
38
|
+
const requests = mockServer.getRequests();
|
|
39
|
+
const lastRequest = requests[requests.length - 1];
|
|
40
|
+
// Expect last request to be emission of done event
|
|
41
|
+
expect(lastRequest.url).toContain('airdrop.external-extractor.message');
|
|
42
|
+
expect(lastRequest.method).toBe('POST');
|
|
43
|
+
expect(lastRequest.body.event_type).toBe('EXTRACTION_DATA_DONE');
|
|
44
|
+
});
|
|
45
|
+
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const index_1 = require("../../index");
|
|
4
|
+
// Simulate a network request with variable delays
|
|
5
|
+
const simulateNetworkRequest = (url, delay) => {
|
|
6
|
+
return new Promise((resolve) => {
|
|
7
|
+
console.log(`Starting network request to ${url}...`);
|
|
8
|
+
setTimeout(() => {
|
|
9
|
+
console.log(`Completed network request to ${url} after ${delay}ms`);
|
|
10
|
+
resolve({ status: 200, data: `Response from ${url}` });
|
|
11
|
+
}, delay);
|
|
12
|
+
});
|
|
13
|
+
};
|
|
14
|
+
(0, index_1.processTask)({
|
|
15
|
+
task: async ({ adapter }) => {
|
|
16
|
+
console.log('Starting network requests simulation...');
|
|
17
|
+
// Define request configurations
|
|
18
|
+
const requestConfigs = [
|
|
19
|
+
{ url: '/api/users', delay: 3000 },
|
|
20
|
+
{ url: '/api/products', delay: 4000 },
|
|
21
|
+
{ url: '/api/orders', delay: 5000 },
|
|
22
|
+
{ url: '/api/analytics', delay: 6000 },
|
|
23
|
+
{ url: '/api/reports', delay: 4000 },
|
|
24
|
+
];
|
|
25
|
+
try {
|
|
26
|
+
// Execute requests sequentially - this will take approximately 22 seconds total
|
|
27
|
+
for (const config of requestConfigs) {
|
|
28
|
+
const response = await simulateNetworkRequest(config.url, config.delay);
|
|
29
|
+
console.log('Network request completed:', response.data);
|
|
30
|
+
}
|
|
31
|
+
console.log('All network requests completed successfully');
|
|
32
|
+
await adapter.emit(index_1.ExtractorEventType.ExtractionDataDone);
|
|
33
|
+
}
|
|
34
|
+
catch (error) {
|
|
35
|
+
console.error('Network request failed:', error);
|
|
36
|
+
await adapter.emit(index_1.ExtractorEventType.ExtractionDataDone);
|
|
37
|
+
}
|
|
38
|
+
},
|
|
39
|
+
onTimeout: async ({ adapter }) => {
|
|
40
|
+
await adapter.emit(index_1.ExtractorEventType.ExtractionDataProgress);
|
|
41
|
+
},
|
|
42
|
+
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
const extraction_1 = require("../../types/extraction");
|
|
7
|
+
const test_helpers_1 = require("../test-helpers");
|
|
8
|
+
const extraction_2 = __importDefault(require("./extraction"));
|
|
9
|
+
const mock_server_1 = require("../mock-server");
|
|
10
|
+
jest.setTimeout(15000);
|
|
11
|
+
describe('timeout-2', () => {
|
|
12
|
+
let mockServer;
|
|
13
|
+
beforeAll(async () => {
|
|
14
|
+
mockServer = new mock_server_1.MockServer(3001);
|
|
15
|
+
await mockServer.start();
|
|
16
|
+
});
|
|
17
|
+
afterAll(async () => {
|
|
18
|
+
if (mockServer) {
|
|
19
|
+
await mockServer.stop();
|
|
20
|
+
}
|
|
21
|
+
});
|
|
22
|
+
beforeEach(() => {
|
|
23
|
+
jest.clearAllMocks();
|
|
24
|
+
mockServer.clearRequests();
|
|
25
|
+
});
|
|
26
|
+
it('should emit progress event when soft timeout is reached', async () => {
|
|
27
|
+
const baseUrl = mockServer.getBaseUrl();
|
|
28
|
+
const event = (0, test_helpers_1.createEvent)({
|
|
29
|
+
eventType: extraction_1.EventType.ExtractionDataStart,
|
|
30
|
+
eventContextOverrides: {
|
|
31
|
+
callback_url: `${baseUrl}/internal/airdrop.external-extractor.message`,
|
|
32
|
+
worker_data_url: `${baseUrl}/internal/airdrop.external-worker`,
|
|
33
|
+
},
|
|
34
|
+
executionMetadataOverrides: {
|
|
35
|
+
devrev_endpoint: `${baseUrl}`,
|
|
36
|
+
},
|
|
37
|
+
});
|
|
38
|
+
await (0, extraction_2.default)([event], __dirname + '/timeout-2');
|
|
39
|
+
const requests = mockServer.getRequests();
|
|
40
|
+
const lastRequest = requests[requests.length - 1];
|
|
41
|
+
// Expect last request to be emission of progress event
|
|
42
|
+
expect(lastRequest.url).toContain('airdrop.external-extractor.message');
|
|
43
|
+
expect(lastRequest.method).toBe('POST');
|
|
44
|
+
expect(lastRequest.body.event_type).toBe('EXTRACTION_DATA_PROGRESS');
|
|
45
|
+
});
|
|
46
|
+
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const index_1 = require("../../index");
|
|
4
|
+
(0, index_1.processTask)({
|
|
5
|
+
task: async ({ adapter }) => {
|
|
6
|
+
console.log('Starting intensive computation...');
|
|
7
|
+
let result = 0;
|
|
8
|
+
// Intensive loop - calibrated to exceed timeout
|
|
9
|
+
for (let i = 1; i <= 30000; i++) {
|
|
10
|
+
for (let num = 2; num <= 2000; num++) {
|
|
11
|
+
let isPrime = true;
|
|
12
|
+
for (let divisor = 2; divisor <= Math.sqrt(num); divisor++) {
|
|
13
|
+
if (num % divisor === 0) {
|
|
14
|
+
isPrime = false;
|
|
15
|
+
break;
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
if (isPrime) {
|
|
19
|
+
for (let j = 0; j < 200; j++) {
|
|
20
|
+
result +=
|
|
21
|
+
Math.sin(num * j) * Math.cos(i * j) * Math.sqrt(num + i + j);
|
|
22
|
+
result = Math.abs(result) % 1000000;
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
// Log every 500 iterations to see progress before timeout
|
|
27
|
+
if (i % 500 === 0) {
|
|
28
|
+
console.log(`timeout-3a iteration ${i}, result: ${result}`);
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
console.log(`Final computation result: ${result}`);
|
|
32
|
+
await adapter.emit(index_1.ExtractorEventType.ExtractionDataDone);
|
|
33
|
+
},
|
|
34
|
+
onTimeout: async ({ adapter }) => {
|
|
35
|
+
await adapter.emit(index_1.ExtractorEventType.ExtractionDataProgress);
|
|
36
|
+
},
|
|
37
|
+
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
const extraction_1 = require("../../types/extraction");
|
|
7
|
+
const test_helpers_1 = require("../test-helpers");
|
|
8
|
+
const extraction_2 = __importDefault(require("./extraction"));
|
|
9
|
+
const mock_server_1 = require("../mock-server");
|
|
10
|
+
jest.setTimeout(15000);
|
|
11
|
+
describe('timeout-3a', () => {
|
|
12
|
+
let mockServer;
|
|
13
|
+
beforeAll(async () => {
|
|
14
|
+
mockServer = new mock_server_1.MockServer(3001);
|
|
15
|
+
await mockServer.start();
|
|
16
|
+
});
|
|
17
|
+
afterAll(async () => {
|
|
18
|
+
if (mockServer) {
|
|
19
|
+
await mockServer.stop();
|
|
20
|
+
}
|
|
21
|
+
});
|
|
22
|
+
beforeEach(() => {
|
|
23
|
+
jest.clearAllMocks();
|
|
24
|
+
mockServer.clearRequests();
|
|
25
|
+
});
|
|
26
|
+
it('should emit error event when hard timeout is reached', async () => {
|
|
27
|
+
const baseUrl = mockServer.getBaseUrl();
|
|
28
|
+
const event = (0, test_helpers_1.createEvent)({
|
|
29
|
+
eventType: extraction_1.EventType.ExtractionDataStart,
|
|
30
|
+
eventContextOverrides: {
|
|
31
|
+
callback_url: `${baseUrl}/internal/airdrop.external-extractor.message`,
|
|
32
|
+
worker_data_url: `${baseUrl}/internal/airdrop.external-worker`,
|
|
33
|
+
},
|
|
34
|
+
executionMetadataOverrides: {
|
|
35
|
+
devrev_endpoint: `${baseUrl}`,
|
|
36
|
+
},
|
|
37
|
+
});
|
|
38
|
+
await (0, extraction_2.default)([event], __dirname + '/timeout-3a');
|
|
39
|
+
const requests = mockServer.getRequests();
|
|
40
|
+
const lastRequest = requests[requests.length - 1];
|
|
41
|
+
// Expect last request to be emission of error event since we force-kill the worker
|
|
42
|
+
expect(lastRequest.url).toContain('airdrop.external-extractor.message');
|
|
43
|
+
expect(lastRequest.method).toBe('POST');
|
|
44
|
+
expect(lastRequest.body.event_type).toBe('EXTRACTION_DATA_ERROR');
|
|
45
|
+
});
|
|
46
|
+
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|