@devrev/airsync-sdk 2.0.0-beta.0 → 2.0.0-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +0 -4
- package/dist/airsync-sdk.d.ts +1802 -0
- package/dist/attachments-streaming/attachments-streaming-pool.d.ts +7 -0
- package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map +1 -1
- package/dist/attachments-streaming/attachments-streaming-pool.js +44 -4
- package/dist/attachments-streaming/attachments-streaming-pool.test.js +259 -10
- package/dist/common/constants.d.ts +16 -5
- package/dist/common/constants.d.ts.map +1 -1
- package/dist/common/constants.js +20 -6
- package/dist/common/control-protocol.d.ts +2 -2
- package/dist/common/control-protocol.d.ts.map +1 -1
- package/dist/common/control-protocol.js +1 -5
- package/dist/common/helpers.d.ts +8 -0
- package/dist/common/helpers.d.ts.map +1 -1
- package/dist/common/helpers.js +15 -0
- package/dist/common/install-initial-domain-mapping.d.ts +2 -2
- package/dist/common/install-initial-domain-mapping.test.js +7 -3
- package/dist/common/test-utils.d.ts +19 -0
- package/dist/common/test-utils.d.ts.map +1 -0
- package/dist/common/test-utils.js +101 -0
- package/dist/common/time-value-resolver.d.ts +47 -0
- package/dist/common/time-value-resolver.d.ts.map +1 -0
- package/dist/common/time-value-resolver.js +141 -0
- package/dist/common/time-value-resolver.test.d.ts +2 -0
- package/dist/common/time-value-resolver.test.d.ts.map +1 -0
- package/dist/common/time-value-resolver.test.js +268 -0
- package/dist/http/axios-client-internal.d.ts.map +1 -1
- package/dist/http/axios-client-internal.js +7 -0
- package/dist/http/axios-client-internal.test.js +27 -6
- package/dist/http/axios-client.d.ts.map +1 -1
- package/dist/http/axios-client.js +6 -3
- package/dist/http/index.d.ts +0 -1
- package/dist/http/index.d.ts.map +1 -1
- package/dist/http/index.js +0 -1
- package/dist/index.d.ts +9 -8
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +12 -10
- package/dist/logger/logger.d.ts +1 -17
- package/dist/logger/logger.d.ts.map +1 -1
- package/dist/logger/logger.interfaces.d.ts +2 -2
- package/dist/logger/logger.js +35 -27
- package/dist/logger/logger.test.js +66 -19
- package/dist/mappers/mappers.interface.d.ts +5 -5
- package/dist/mappers/mappers.interface.js +3 -3
- package/dist/mappers/mappers.test.js +8 -10
- package/dist/mock-server/mock-server.d.ts +47 -0
- package/dist/mock-server/mock-server.d.ts.map +1 -0
- package/dist/mock-server/mock-server.interfaces.d.ts +73 -0
- package/dist/mock-server/mock-server.interfaces.d.ts.map +1 -0
- package/dist/mock-server/mock-server.interfaces.js +4 -0
- package/dist/mock-server/mock-server.js +278 -0
- package/dist/multithreading/create-worker.test.js +36 -17
- package/dist/multithreading/process-task.d.ts.map +1 -1
- package/dist/multithreading/process-task.js +46 -44
- package/dist/multithreading/process-task.test.d.ts +2 -0
- package/dist/multithreading/process-task.test.d.ts.map +1 -0
- package/dist/multithreading/process-task.test.js +146 -0
- package/dist/multithreading/spawn/spawn.d.ts +4 -3
- package/dist/multithreading/spawn/spawn.d.ts.map +1 -1
- package/dist/multithreading/spawn/spawn.helpers.d.ts.map +1 -1
- package/dist/multithreading/spawn/spawn.helpers.js +6 -14
- package/dist/multithreading/spawn/spawn.helpers.test.js +8 -8
- package/dist/multithreading/spawn/spawn.js +41 -31
- package/dist/multithreading/spawn/spawn.test.d.ts +2 -0
- package/dist/multithreading/spawn/spawn.test.d.ts.map +1 -0
- package/dist/multithreading/spawn/spawn.test.js +232 -0
- package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.js +49 -1
- package/dist/multithreading/worker-adapter/worker-adapter.d.ts +11 -5
- package/dist/multithreading/worker-adapter/worker-adapter.d.ts.map +1 -1
- package/dist/multithreading/worker-adapter/worker-adapter.emit.test.d.ts +2 -0
- package/dist/multithreading/worker-adapter/worker-adapter.emit.test.d.ts.map +1 -0
- package/dist/multithreading/worker-adapter/worker-adapter.emit.test.js +390 -0
- package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.d.ts +2 -0
- package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.d.ts.map +1 -0
- package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.js +799 -0
- package/dist/multithreading/worker-adapter/worker-adapter.js +233 -134
- package/dist/multithreading/worker-adapter/worker-adapter.loading.test.d.ts +2 -0
- package/dist/multithreading/worker-adapter/worker-adapter.loading.test.d.ts.map +1 -0
- package/dist/multithreading/worker-adapter/worker-adapter.loading.test.js +596 -0
- package/dist/multithreading/worker-adapter/worker-adapter.serialization.test.d.ts +2 -0
- package/dist/multithreading/worker-adapter/worker-adapter.serialization.test.d.ts.map +1 -0
- package/dist/multithreading/worker-adapter/worker-adapter.serialization.test.js +71 -0
- package/dist/repo/repo.d.ts.map +1 -1
- package/dist/repo/repo.interfaces.d.ts +4 -2
- package/dist/repo/repo.interfaces.d.ts.map +1 -1
- package/dist/repo/repo.js +4 -3
- package/dist/repo/repo.test.js +60 -7
- package/dist/state/state.d.ts +7 -1
- package/dist/state/state.d.ts.map +1 -1
- package/dist/state/state.extract-window.test.d.ts +2 -0
- package/dist/state/state.extract-window.test.d.ts.map +1 -0
- package/dist/state/state.extract-window.test.js +163 -0
- package/dist/state/state.interfaces.d.ts +25 -8
- package/dist/state/state.interfaces.d.ts.map +1 -1
- package/dist/state/state.interfaces.js +4 -2
- package/dist/state/state.js +111 -12
- package/dist/state/state.pending-boundaries.test.d.ts +2 -0
- package/dist/state/state.pending-boundaries.test.d.ts.map +1 -0
- package/dist/state/state.pending-boundaries.test.js +189 -0
- package/dist/state/state.post-state.test.d.ts +2 -0
- package/dist/state/state.post-state.test.d.ts.map +1 -0
- package/dist/state/state.post-state.test.js +77 -0
- package/dist/state/state.test.js +117 -73
- package/dist/state/state.time-value-resolution.test.d.ts +2 -0
- package/dist/state/state.time-value-resolution.test.d.ts.map +1 -0
- package/dist/state/state.time-value-resolution.test.js +175 -0
- package/dist/tsdoc-metadata.json +11 -0
- package/dist/types/common.d.ts +0 -25
- package/dist/types/common.d.ts.map +1 -1
- package/dist/types/common.js +1 -11
- package/dist/types/external-domain-metadata.d.ts +338 -0
- package/dist/types/external-domain-metadata.d.ts.map +1 -0
- package/dist/types/extraction.d.ts +89 -218
- package/dist/types/extraction.d.ts.map +1 -1
- package/dist/types/extraction.js +56 -124
- package/dist/types/extraction.test.js +69 -43
- package/dist/types/index.d.ts +4 -2
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/index.js +6 -3
- package/dist/types/loading.d.ts +4 -24
- package/dist/types/loading.d.ts.map +1 -1
- package/dist/types/loading.js +1 -22
- package/dist/types/workers.d.ts +31 -14
- package/dist/types/workers.d.ts.map +1 -1
- package/dist/types/workers.js +1 -0
- package/dist/uploader/uploader.d.ts +1 -0
- package/dist/uploader/uploader.d.ts.map +1 -1
- package/dist/uploader/uploader.helpers.test.js +0 -11
- package/dist/uploader/uploader.interfaces.d.ts +3 -3
- package/dist/uploader/uploader.js +53 -20
- package/dist/uploader/uploader.test.js +22 -10
- package/package.json +12 -11
- package/dist/common/event-type-translation.d.ts +0 -24
- package/dist/common/event-type-translation.d.ts.map +0 -1
- package/dist/common/event-type-translation.js +0 -117
- package/dist/deprecated/adapter/index.d.ts +0 -62
- package/dist/deprecated/adapter/index.d.ts.map +0 -1
- package/dist/deprecated/adapter/index.js +0 -151
- package/dist/deprecated/common/helpers.d.ts +0 -7
- package/dist/deprecated/common/helpers.d.ts.map +0 -1
- package/dist/deprecated/common/helpers.js +0 -47
- package/dist/deprecated/demo-extractor/external_domain_metadata.json +0 -38
- package/dist/deprecated/demo-extractor/index.d.ts +0 -18
- package/dist/deprecated/demo-extractor/index.d.ts.map +0 -1
- package/dist/deprecated/demo-extractor/index.js +0 -161
- package/dist/deprecated/http/client.d.ts +0 -22
- package/dist/deprecated/http/client.d.ts.map +0 -1
- package/dist/deprecated/http/client.js +0 -161
- package/dist/deprecated/uploader/index.d.ts +0 -35
- package/dist/deprecated/uploader/index.d.ts.map +0 -1
- package/dist/deprecated/uploader/index.js +0 -161
- package/dist/http/types.d.ts +0 -17
- package/dist/http/types.d.ts.map +0 -1
- package/dist/multithreading/worker-adapter/worker-adapter.test.d.ts +0 -2
- package/dist/multithreading/worker-adapter/worker-adapter.test.d.ts.map +0 -1
- package/dist/multithreading/worker-adapter/worker-adapter.test.js +0 -483
- /package/dist/{http/types.js → types/external-domain-metadata.js} +0 -0
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const events_1 = require("events");
|
|
4
|
+
const constants_1 = require("../../common/constants");
|
|
5
|
+
const extraction_1 = require("../../types/extraction");
|
|
6
|
+
const workers_1 = require("../../types/workers");
|
|
7
|
+
const test_utils_1 = require("../../common/test-utils");
|
|
8
|
+
// ---------------------------------------------------------------------------
|
|
9
|
+
// Mocks
|
|
10
|
+
// ---------------------------------------------------------------------------
|
|
11
|
+
jest.mock('../create-worker', () => ({
|
|
12
|
+
createWorker: jest.fn(),
|
|
13
|
+
}));
|
|
14
|
+
jest.mock('../../common/control-protocol', () => ({
|
|
15
|
+
emit: jest.fn().mockResolvedValue({}),
|
|
16
|
+
}));
|
|
17
|
+
jest.mock('../../logger/logger', () => ({
|
|
18
|
+
Logger: jest.fn().mockImplementation(() => ({
|
|
19
|
+
log: jest.fn(),
|
|
20
|
+
warn: jest.fn(),
|
|
21
|
+
error: jest.fn(),
|
|
22
|
+
info: jest.fn(),
|
|
23
|
+
logFn: jest.fn(),
|
|
24
|
+
})),
|
|
25
|
+
serializeError: jest.fn((e) => String(e)),
|
|
26
|
+
}));
|
|
27
|
+
jest.mock('../../common/helpers', () => ({
|
|
28
|
+
getLibraryVersion: jest.fn().mockReturnValue('1.0.0-test'),
|
|
29
|
+
getMemoryUsage: jest.fn().mockReturnValue({
|
|
30
|
+
formattedMessage: 'Memory: RSS 100/512MB (19.53%) [...]',
|
|
31
|
+
rssUsedMB: '100.00',
|
|
32
|
+
rssUsedPercent: '19.53%',
|
|
33
|
+
heapUsedPercent: '30.00%',
|
|
34
|
+
externalMB: '10.00',
|
|
35
|
+
arrayBuffersMB: '5.00',
|
|
36
|
+
}),
|
|
37
|
+
sleep: jest.fn(),
|
|
38
|
+
truncateFilename: jest.fn((f) => f),
|
|
39
|
+
truncateMessage: jest.fn((m) => m),
|
|
40
|
+
}));
|
|
41
|
+
// ---------------------------------------------------------------------------
|
|
42
|
+
// Imports after mocks
|
|
43
|
+
// ---------------------------------------------------------------------------
|
|
44
|
+
const spawn_1 = require("./spawn");
|
|
45
|
+
const create_worker_1 = require("../create-worker");
|
|
46
|
+
const control_protocol_1 = require("../../common/control-protocol");
|
|
47
|
+
const helpers_1 = require("../../common/helpers");
|
|
48
|
+
// ---------------------------------------------------------------------------
|
|
49
|
+
// Factory for a fake worker (EventEmitter with postMessage + terminate)
|
|
50
|
+
// ---------------------------------------------------------------------------
|
|
51
|
+
function makeWorker() {
|
|
52
|
+
const w = new events_1.EventEmitter();
|
|
53
|
+
w.postMessage = jest.fn();
|
|
54
|
+
w.terminate = jest.fn().mockResolvedValue(0);
|
|
55
|
+
return w;
|
|
56
|
+
}
|
|
57
|
+
// ---------------------------------------------------------------------------
|
|
58
|
+
// Helper: instantiate Spawn directly, injecting a mock logger via console swap
|
|
59
|
+
// ---------------------------------------------------------------------------
|
|
60
|
+
function buildSpawn(overrides) {
|
|
61
|
+
var _a;
|
|
62
|
+
const event = (0, test_utils_1.createMockEvent)('http://localhost:0', {
|
|
63
|
+
payload: { event_type: extraction_1.EventType.StartExtractingData },
|
|
64
|
+
});
|
|
65
|
+
const mockLogger = {
|
|
66
|
+
log: jest.fn(),
|
|
67
|
+
warn: jest.fn(),
|
|
68
|
+
error: jest.fn(),
|
|
69
|
+
info: jest.fn(),
|
|
70
|
+
logFn: jest.fn(),
|
|
71
|
+
};
|
|
72
|
+
const originalConsole = console;
|
|
73
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
74
|
+
global.console = mockLogger;
|
|
75
|
+
const s = new spawn_1.Spawn({
|
|
76
|
+
event,
|
|
77
|
+
worker: overrides.worker,
|
|
78
|
+
options: overrides.options,
|
|
79
|
+
resolve: (_a = overrides.resolve) !== null && _a !== void 0 ? _a : jest.fn(),
|
|
80
|
+
originalConsole: originalConsole,
|
|
81
|
+
});
|
|
82
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
83
|
+
global.console = originalConsole;
|
|
84
|
+
return s;
|
|
85
|
+
}
|
|
86
|
+
// ---------------------------------------------------------------------------
|
|
87
|
+
// spawn() factory tests
|
|
88
|
+
// ---------------------------------------------------------------------------
|
|
89
|
+
describe('spawn() factory', () => {
|
|
90
|
+
beforeEach(() => {
|
|
91
|
+
jest.clearAllMocks();
|
|
92
|
+
jest.useFakeTimers({ legacyFakeTimers: true });
|
|
93
|
+
});
|
|
94
|
+
afterEach(() => {
|
|
95
|
+
jest.useRealTimers();
|
|
96
|
+
});
|
|
97
|
+
it('should emit a no-script event and NOT spawn a worker for an unknown event type', async () => {
|
|
98
|
+
// Arrange
|
|
99
|
+
const event = (0, test_utils_1.createMockEvent)('http://localhost:0', {
|
|
100
|
+
payload: { event_type: extraction_1.EventType.UnknownEventType },
|
|
101
|
+
});
|
|
102
|
+
// Act
|
|
103
|
+
await (0, spawn_1.spawn)({ event, initialState: {} });
|
|
104
|
+
// Assert: no worker process should be started, but the platform should
|
|
105
|
+
// still receive a terminal event (so the run doesn't hang).
|
|
106
|
+
expect(create_worker_1.createWorker).not.toHaveBeenCalled();
|
|
107
|
+
expect(control_protocol_1.emit).toHaveBeenCalledWith(expect.objectContaining({ event, eventType: expect.any(String) }));
|
|
108
|
+
});
|
|
109
|
+
it('should reject the returned promise when createWorker throws', async () => {
|
|
110
|
+
// Arrange
|
|
111
|
+
create_worker_1.createWorker.mockRejectedValue(new Error('worker boom'));
|
|
112
|
+
const event = (0, test_utils_1.createMockEvent)('http://localhost:0', {
|
|
113
|
+
payload: { event_type: extraction_1.EventType.StartExtractingData },
|
|
114
|
+
});
|
|
115
|
+
// Act & Assert
|
|
116
|
+
await expect((0, spawn_1.spawn)({
|
|
117
|
+
event,
|
|
118
|
+
initialState: {},
|
|
119
|
+
baseWorkerPath: '/fake',
|
|
120
|
+
options: {
|
|
121
|
+
workerPathOverrides: {
|
|
122
|
+
[extraction_1.EventType.StartExtractingData]: '/path.js',
|
|
123
|
+
},
|
|
124
|
+
},
|
|
125
|
+
})).rejects.toThrow('worker boom');
|
|
126
|
+
});
|
|
127
|
+
});
|
|
128
|
+
// ---------------------------------------------------------------------------
|
|
129
|
+
// Spawn class — lifecycle tests
|
|
130
|
+
// ---------------------------------------------------------------------------
|
|
131
|
+
describe('Spawn class', () => {
|
|
132
|
+
let worker;
|
|
133
|
+
let resolveMock;
|
|
134
|
+
beforeEach(() => {
|
|
135
|
+
jest.clearAllMocks();
|
|
136
|
+
jest.useFakeTimers({ legacyFakeTimers: true });
|
|
137
|
+
worker = makeWorker();
|
|
138
|
+
resolveMock = jest.fn();
|
|
139
|
+
});
|
|
140
|
+
afterEach(() => {
|
|
141
|
+
jest.useRealTimers();
|
|
142
|
+
});
|
|
143
|
+
// -------------------------------------------------------------------------
|
|
144
|
+
// WorkerMessageFailed captured and propagated in error emit
|
|
145
|
+
// -------------------------------------------------------------------------
|
|
146
|
+
it('should include the WorkerMessageFailed reason in the error event emitted to the platform', async () => {
|
|
147
|
+
// Arrange
|
|
148
|
+
buildSpawn({ worker, resolve: resolveMock });
|
|
149
|
+
// Act
|
|
150
|
+
worker.emit(workers_1.WorkerEvent.WorkerMessage, {
|
|
151
|
+
subject: workers_1.WorkerMessageSubject.WorkerMessageFailed,
|
|
152
|
+
payload: { message: 'connector exploded' },
|
|
153
|
+
});
|
|
154
|
+
worker.emit(workers_1.WorkerEvent.WorkerExit, 1);
|
|
155
|
+
await Promise.resolve();
|
|
156
|
+
await Promise.resolve();
|
|
157
|
+
// Assert: the platform receives an error event whose message contains
|
|
158
|
+
// the reason sent by the worker — this is what operators see in the run log.
|
|
159
|
+
expect(control_protocol_1.emit).toHaveBeenCalledWith(expect.objectContaining({
|
|
160
|
+
data: expect.objectContaining({
|
|
161
|
+
error: expect.objectContaining({
|
|
162
|
+
message: expect.stringContaining('connector exploded'),
|
|
163
|
+
}),
|
|
164
|
+
}),
|
|
165
|
+
}));
|
|
166
|
+
expect(resolveMock).toHaveBeenCalled();
|
|
167
|
+
});
|
|
168
|
+
it('should emit a data extraction error when the worker exits without ever emitting', async () => {
|
|
169
|
+
// Arrange
|
|
170
|
+
buildSpawn({ worker, resolve: resolveMock });
|
|
171
|
+
// Act
|
|
172
|
+
worker.emit(workers_1.WorkerEvent.WorkerExit, 1);
|
|
173
|
+
await Promise.resolve();
|
|
174
|
+
await Promise.resolve();
|
|
175
|
+
// Assert: a silent worker exit must surface a DataExtractionError to the
|
|
176
|
+
// platform with a non-empty message. The event type is the upstream
|
|
177
|
+
// contract; the exact message wording is implementation detail.
|
|
178
|
+
expect(control_protocol_1.emit).toHaveBeenCalledTimes(1);
|
|
179
|
+
const [emitted] = control_protocol_1.emit.mock.calls[0];
|
|
180
|
+
expect(emitted.eventType).toBe(extraction_1.ExtractorEventType.DataExtractionError);
|
|
181
|
+
expect(emitted.data.error.message).toEqual(expect.any(String));
|
|
182
|
+
expect(emitted.data.error.message.length).toBeGreaterThan(0);
|
|
183
|
+
expect(resolveMock).toHaveBeenCalled();
|
|
184
|
+
});
|
|
185
|
+
// Soft-timeout and hard-timeout timer behavior is covered end-to-end by
|
|
186
|
+
// src/tests/timeout-handling/ (real workers, real timers). Unit tests with
|
|
187
|
+
// fake timers only re-asserted the mocked setTimeout call and gave no signal.
|
|
188
|
+
// -------------------------------------------------------------------------
|
|
189
|
+
// Memory monitoring — error clears the interval
|
|
190
|
+
// -------------------------------------------------------------------------
|
|
191
|
+
it('should clear the memory monitoring interval when getMemoryUsage throws to prevent repeated crashes', async () => {
|
|
192
|
+
// Arrange
|
|
193
|
+
helpers_1.getMemoryUsage.mockImplementation(() => {
|
|
194
|
+
throw new Error('OOM');
|
|
195
|
+
});
|
|
196
|
+
const clearIntervalSpy = jest.spyOn(global, 'clearInterval');
|
|
197
|
+
buildSpawn({ worker, resolve: resolveMock });
|
|
198
|
+
// Act
|
|
199
|
+
jest.advanceTimersByTime(30001); // MEMORY_LOG_INTERVAL = 30 s
|
|
200
|
+
await Promise.resolve();
|
|
201
|
+
// Assert
|
|
202
|
+
expect(clearIntervalSpy).toHaveBeenCalled();
|
|
203
|
+
});
|
|
204
|
+
// -------------------------------------------------------------------------
|
|
205
|
+
// Soft-timeout race: worker emits AFTER softTimeoutSent — no double-emit
|
|
206
|
+
// -------------------------------------------------------------------------
|
|
207
|
+
it('should NOT emit an error when the worker successfully emits just after receiving the soft-timeout signal', async () => {
|
|
208
|
+
// Arrange
|
|
209
|
+
buildSpawn({ worker, resolve: resolveMock });
|
|
210
|
+
// Act: trigger soft timeout — sends WorkerMessageExit to the worker
|
|
211
|
+
jest.advanceTimersByTime(constants_1.DEFAULT_LAMBDA_TIMEOUT + 1);
|
|
212
|
+
await Promise.resolve();
|
|
213
|
+
// Confirm the soft-timeout path actually fired — guards against a silent
|
|
214
|
+
// pass if the timeout constant changes and the timer never runs.
|
|
215
|
+
expect(worker.postMessage).toHaveBeenCalledWith(expect.objectContaining({
|
|
216
|
+
subject: workers_1.WorkerMessageSubject.WorkerMessageExit,
|
|
217
|
+
}));
|
|
218
|
+
// Worker responds: emits its event successfully, then exits normally
|
|
219
|
+
worker.emit(workers_1.WorkerEvent.WorkerMessage, {
|
|
220
|
+
subject: workers_1.WorkerMessageSubject.WorkerMessageEmitted,
|
|
221
|
+
});
|
|
222
|
+
worker.emit(workers_1.WorkerEvent.WorkerExit, 0);
|
|
223
|
+
// The exit handler defers via setImmediate when softTimeoutSent=true
|
|
224
|
+
jest.runAllImmediates();
|
|
225
|
+
await Promise.resolve();
|
|
226
|
+
await Promise.resolve();
|
|
227
|
+
// Assert: no error should reach the platform — the worker completed its job
|
|
228
|
+
const errorEmits = control_protocol_1.emit.mock.calls.filter((call) => { var _a, _b; return (_b = (_a = call[0]) === null || _a === void 0 ? void 0 : _a.data) === null || _b === void 0 ? void 0 : _b.error; });
|
|
229
|
+
expect(errorEmits).toHaveLength(0);
|
|
230
|
+
expect(resolveMock).toHaveBeenCalled();
|
|
231
|
+
});
|
|
232
|
+
});
|
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const constants_1 = require("../../common/constants");
|
|
3
4
|
const state_1 = require("../../state/state");
|
|
5
|
+
const jest_setup_1 = require("../../tests/jest.setup");
|
|
4
6
|
const test_helpers_1 = require("../../tests/test-helpers");
|
|
7
|
+
const test_utils_1 = require("../../common/test-utils");
|
|
5
8
|
const types_1 = require("../../types");
|
|
6
9
|
const worker_adapter_1 = require("./worker-adapter");
|
|
7
10
|
// 1. Create a mock function for the method you want to override.
|
|
@@ -49,7 +52,9 @@ describe('Artifact ordering when artifacts overflow batch sizes in repositories'
|
|
|
49
52
|
let testAdapter;
|
|
50
53
|
beforeEach(() => {
|
|
51
54
|
// Create a fresh adapter instance for this test to avoid mocking conflicts
|
|
52
|
-
const mockEvent = (0,
|
|
55
|
+
const mockEvent = (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
|
|
56
|
+
payload: { event_type: types_1.EventType.StartExtractingData },
|
|
57
|
+
});
|
|
53
58
|
const mockAdapterState = new state_1.State({
|
|
54
59
|
event: mockEvent,
|
|
55
60
|
initialState: { attachments: { completed: false } },
|
|
@@ -125,3 +130,46 @@ describe('Artifact ordering when artifacts overflow batch sizes in repositories'
|
|
|
125
130
|
expect(checkArtifactOrder(artifacts, repos)).toBe(true);
|
|
126
131
|
});
|
|
127
132
|
});
|
|
133
|
+
describe('External sync units splitting into artifacts', () => {
|
|
134
|
+
let testAdapter;
|
|
135
|
+
beforeEach(() => {
|
|
136
|
+
const mockEvent = (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
|
|
137
|
+
payload: { event_type: types_1.EventType.StartExtractingExternalSyncUnits },
|
|
138
|
+
});
|
|
139
|
+
const mockAdapterState = new state_1.State({
|
|
140
|
+
event: mockEvent,
|
|
141
|
+
initialState: {},
|
|
142
|
+
});
|
|
143
|
+
testAdapter = new worker_adapter_1.WorkerAdapter({
|
|
144
|
+
event: mockEvent,
|
|
145
|
+
adapterState: mockAdapterState,
|
|
146
|
+
});
|
|
147
|
+
});
|
|
148
|
+
it('should split 125k external sync units into 5 artifacts', async () => {
|
|
149
|
+
const BATCH_SIZE = 25000;
|
|
150
|
+
const TOTAL_UNITS = 125000;
|
|
151
|
+
const externalSyncUnits = Array.from({ length: TOTAL_UNITS }, (_, i) => ({
|
|
152
|
+
id: String(i),
|
|
153
|
+
name: `Unit ${i}`,
|
|
154
|
+
description: `Description ${i}`,
|
|
155
|
+
}));
|
|
156
|
+
testAdapter.initializeRepos([
|
|
157
|
+
{
|
|
158
|
+
itemType: constants_1.AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS,
|
|
159
|
+
overridenOptions: {
|
|
160
|
+
batchSize: BATCH_SIZE,
|
|
161
|
+
skipConfirmation: true,
|
|
162
|
+
},
|
|
163
|
+
},
|
|
164
|
+
]);
|
|
165
|
+
const repo = testAdapter.getRepo(constants_1.AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS);
|
|
166
|
+
const chunkSize = 10000;
|
|
167
|
+
for (let i = 0; i < TOTAL_UNITS; i += chunkSize) {
|
|
168
|
+
await (repo === null || repo === void 0 ? void 0 : repo.push(externalSyncUnits.slice(i, i + chunkSize)));
|
|
169
|
+
}
|
|
170
|
+
await testAdapter.uploadAllRepos();
|
|
171
|
+
expect(testAdapter.artifacts.length).toBe(TOTAL_UNITS / BATCH_SIZE // 5
|
|
172
|
+
);
|
|
173
|
+
expect(testAdapter.artifacts.every((a) => a.item_type === constants_1.AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS)).toBe(true);
|
|
174
|
+
});
|
|
175
|
+
});
|
|
@@ -2,13 +2,13 @@ import { Mappers } from '../../mappers/mappers';
|
|
|
2
2
|
import { Repo } from '../../repo/repo';
|
|
3
3
|
import { NormalizedAttachment, RepoInterface } from '../../repo/repo.interfaces';
|
|
4
4
|
import { AdapterState } from '../../state/state.interfaces';
|
|
5
|
-
import {
|
|
5
|
+
import { AirSyncEvent, EventData, ExternalSystemAttachmentProcessors, ExternalSystemAttachmentStreamingFunction, ExtractorEventType, ProcessAttachmentReturnType, StreamAttachmentsReturnType } from '../../types/extraction';
|
|
6
6
|
import { ExternalSystemAttachment, ExternalSystemItem, ExternalSystemLoadingFunction, FileToLoad, ItemTypesToLoadParams, ItemTypeToLoad, LoaderEventType, LoaderReport, LoadItemResponse, LoadItemTypesResponse } from '../../types/loading';
|
|
7
7
|
import { WorkerAdapterInterface, WorkerAdapterOptions } from '../../types/workers';
|
|
8
8
|
import { Artifact } from '../../uploader/uploader.interfaces';
|
|
9
9
|
export declare function createWorkerAdapter<ConnectorState>({ event, adapterState, options, }: WorkerAdapterInterface<ConnectorState>): WorkerAdapter<ConnectorState>;
|
|
10
10
|
/**
|
|
11
|
-
* WorkerAdapter class is used to interact with
|
|
11
|
+
* WorkerAdapter class is used to interact with AirSync platform. It is passed to the snap-in
|
|
12
12
|
* as parameter in processTask and onTimeout functions. The class provides
|
|
13
13
|
* utilities to emit control events to the platform, update the state of the connector,
|
|
14
14
|
* and upload artifacts to the platform.
|
|
@@ -23,13 +23,14 @@ export declare function createWorkerAdapter<ConnectorState>({ event, adapterStat
|
|
|
23
23
|
* @public
|
|
24
24
|
*/
|
|
25
25
|
export declare class WorkerAdapter<ConnectorState> {
|
|
26
|
-
readonly event:
|
|
26
|
+
readonly event: AirSyncEvent;
|
|
27
27
|
readonly options?: WorkerAdapterOptions;
|
|
28
28
|
isTimeout: boolean;
|
|
29
|
+
hasWorkerEmitted: boolean;
|
|
29
30
|
private adapterState;
|
|
30
31
|
private _artifacts;
|
|
31
|
-
private hasWorkerEmitted;
|
|
32
32
|
private repos;
|
|
33
|
+
private currentEventDataLength;
|
|
33
34
|
private loaderReports;
|
|
34
35
|
private _processedFiles;
|
|
35
36
|
private _mappers;
|
|
@@ -40,6 +41,12 @@ export declare class WorkerAdapter<ConnectorState> {
|
|
|
40
41
|
get reports(): LoaderReport[];
|
|
41
42
|
get processedFiles(): string[];
|
|
42
43
|
get mappers(): Mappers;
|
|
44
|
+
get extractionScope(): import("../../types/workers").ExtractionScope;
|
|
45
|
+
/**
|
|
46
|
+
* Returns whether the given item type should be extracted.
|
|
47
|
+
* Defaults to true if the scope is empty or the item type is not listed.
|
|
48
|
+
*/
|
|
49
|
+
shouldExtract(itemType: string): boolean;
|
|
43
50
|
initializeRepos(repos: RepoInterface[]): void;
|
|
44
51
|
getRepo(itemType: string): Repo | undefined;
|
|
45
52
|
postState(): Promise<void>;
|
|
@@ -53,7 +60,6 @@ export declare class WorkerAdapter<ConnectorState> {
|
|
|
53
60
|
*/
|
|
54
61
|
emit(newEventType: ExtractorEventType | LoaderEventType, data?: EventData): Promise<void>;
|
|
55
62
|
uploadAllRepos(): Promise<void>;
|
|
56
|
-
handleTimeout(): void;
|
|
57
63
|
loadItemTypes({ itemTypesToLoad, }: ItemTypesToLoadParams): Promise<LoadItemTypesResponse>;
|
|
58
64
|
getLoaderBatches({ supportedItemTypes, }: {
|
|
59
65
|
supportedItemTypes: string[];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"worker-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"worker-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.ts"],"names":[],"mappings":"AAmBA,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAEhD,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AACvC,OAAO,EACL,oBAAoB,EACpB,aAAa,EACd,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EAAE,YAAY,EAAE,MAAM,8BAA8B,CAAC;AAC5D,OAAO,EACL,YAAY,EACZ,SAAS,EAET,kCAAkC,EAClC,yCAAyC,EACzC,kBAAkB,EAClB,2BAA2B,EAC3B,2BAA2B,EAC5B,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAEL,wBAAwB,EACxB,kBAAkB,EAClB,6BAA6B,EAC7B,UAAU,EACV,qBAAqB,EACrB,cAAc,EACd,eAAe,EACf,YAAY,EACZ,gBAAgB,EAChB,qBAAqB,EAEtB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,sBAAsB,EACtB,oBAAoB,EAGrB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,QAAQ,EAAkB,MAAM,oCAAoC,CAAC;AAG9E,wBAAgB,mBAAmB,CAAC,cAAc,EAAE,EAClD,KAAK,EACL,YAAY,EACZ,OAAO,GACR,EAAE,sBAAsB,CAAC,cAAc,CAAC,GAAG,aAAa,CAAC,cAAc,CAAC,CAMxE;AAED;;;;;;;;;;;;;;GAcG;AACH,qBAAa,aAAa,CAAC,cAAc;IACvC,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IAC7B,QAAQ,CAAC,OAAO,CAAC,EAAE,oBAAoB,CAAC;IACxC,SAAS,EAAE,OAAO,CAAC;IACnB,gBAAgB,EAAE,OAAO,CAAC;IAE1B,OAAO,CAAC,YAAY,CAAwB;IAC5C,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,KAAK,CAAc;IAC3B,OAAO,CAAC,sBAAsB,CAAa;IAG3C,OAAO,CAAC,aAAa,CAAiB;IACtC,OAAO,CAAC,eAAe,CAAW;IAClC,OAAO,CAAC,QAAQ,CAAU;IAC1B,OAAO,CAAC,QAAQ,CAAW;gBAEf,EACV,KAAK,EACL,YAAY,EACZ,OAAO,GACR,EAAE,sBAAsB,CAAC,cAAc,CAAC;IAqBzC,IAAI,KAAK,IAAI,YAAY,CAAC,cAAc,CAAC,CAExC;IAED,IAAI,KAAK,CAAC,KAAK,EAAE,YAAY,CAAC,cAAc,CAAC,EAE5C;IAED,IAAI,OAAO,IAAI,YAAY,EAAE,CAE5B;IAED,IAAI,cAAc,IAAI,MAAM,EAAE,CAE7B;IAED,IAAI,OAAO,IAAI,OAAO,CAErB;IAED,IAAI,eAAe,kDAElB;IAED;;;OAGG;IACH,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAOxC,eAAe,CAAC,KAAK,EAAE,aAAa,EAAE;IAuCtC,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS;IAarC,SAAS;IAMf,IAAI,SAAS,IAAI,QAAQ,EAAE,CAE1B;IAED,IAAI,SAAS,CAAC,SAAS,EAAE,QAAQ,EAAE,EAIlC;IAED;;;;;OAKG;IACG,IAAI,CACR,YAAY,EAAE,kBAAkB,GAAG,eAAe,EAClD,IAAI,CAAC,EAAE,SAAS,GACf,OAAO,CAAC,IAAI,CAAC;IAmHV,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;IAU/B,aAAa,CAAC,EAClB,eAAe,GAChB,EAAE,qBAAqB,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAwInD,gBAAgB,CAAC,EACrB,kBAAkB,GACnB,EAAE;QACD,kBAAkB,EAAE,MAAM,EAAE,CAAC;KAC9B;IA4BK,eAAe,CAAC,EACpB,MAAM,GACP,EAAE;QACD,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAyG5B,QAAQ,CAAC,EACb,IAAI,EACJ,cAAc,GACf,EAAE;QACD,IAAI,EAAE,kBAAkB,CAAC;QACzB,cAAc,EAAE,cAAc,CAAC;KAChC,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAwMvB,iBAAiB,CACrB,UAAU,EAAE,oBAAoB,EAChC,MAAM,EAAE,yCAAyC,GAChD,OAAO,CAAC,2BAA2B,CAAC;IA+GvC;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAcnB,cAAc,CAAC,EACnB,IAAI,EACJ,MAAM,GACP,EAAE;QACD,IAAI,EAAE,wBAAwB,CAAC;QAC/B,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAuD7B;;;;;;OAMG;IACG,iBAAiB,CAAC,QAAQ,EAAE,EAChC,MAAM,EACN,UAAU,EACV,SAAa,GACd,EAAE;QACD,MAAM,EAAE,yCAAyC,CAAC;QAClD,UAAU,CAAC,EAAE,kCAAkC,CAC7C,cAAc,EACd,oBAAoB,EAAE,EACtB,QAAQ,CACT,CAAC;QACF,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,GAAG,OAAO,CAAC,2BAA2B,CAAC;CAuIzC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"worker-adapter.emit.test.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.emit.test.ts"],"names":[],"mappings":""}
|