@devrev/airsync-sdk 2.0.0-beta.2 → 2.0.0-beta.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/airsync-sdk.d.ts +414 -157
- package/dist/attachments-streaming/attachments-streaming-pool.d.ts +2 -1
- package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map +1 -1
- package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts +8 -2
- package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts.map +1 -1
- package/dist/attachments-streaming/attachments-streaming-pool.js +14 -14
- package/dist/attachments-streaming/attachments-streaming-pool.test.js +47 -23
- package/dist/common/time-value-resolver.d.ts +2 -2
- package/dist/common/time-value-resolver.d.ts.map +1 -1
- package/dist/index.d.ts +3 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -4
- package/dist/multithreading/adapters/base-adapter.d.ts +56 -0
- package/dist/multithreading/adapters/base-adapter.d.ts.map +1 -0
- package/dist/multithreading/adapters/base-adapter.emit.test.d.ts +2 -0
- package/dist/multithreading/adapters/base-adapter.emit.test.d.ts.map +1 -0
- package/dist/multithreading/{worker-adapter/worker-adapter.emit.test.js → adapters/base-adapter.emit.test.js} +66 -53
- package/dist/multithreading/adapters/base-adapter.js +105 -0
- package/dist/multithreading/adapters/extraction-adapter.artifacts.test.d.ts +2 -0
- package/dist/multithreading/adapters/extraction-adapter.artifacts.test.d.ts.map +1 -0
- package/dist/multithreading/{worker-adapter/worker-adapter.artifacts.test.js → adapters/extraction-adapter.artifacts.test.js} +6 -6
- package/dist/multithreading/adapters/extraction-adapter.d.ts +58 -0
- package/dist/multithreading/adapters/extraction-adapter.d.ts.map +1 -0
- package/dist/multithreading/adapters/extraction-adapter.js +325 -0
- package/dist/multithreading/adapters/extraction-adapter.test.d.ts +2 -0
- package/dist/multithreading/adapters/extraction-adapter.test.d.ts.map +1 -0
- package/dist/multithreading/{worker-adapter/worker-adapter.extraction.test.js → adapters/extraction-adapter.test.js} +41 -51
- package/dist/multithreading/adapters/loading-adapter.d.ts +44 -0
- package/dist/multithreading/adapters/loading-adapter.d.ts.map +1 -0
- package/dist/multithreading/{worker-adapter/worker-adapter.helpers.d.ts → adapters/loading-adapter.helpers.d.ts} +1 -1
- package/dist/multithreading/adapters/loading-adapter.helpers.d.ts.map +1 -0
- package/dist/multithreading/adapters/loading-adapter.helpers.test.d.ts +2 -0
- package/dist/multithreading/adapters/loading-adapter.helpers.test.d.ts.map +1 -0
- package/dist/multithreading/{worker-adapter/worker-adapter.helpers.test.js → adapters/loading-adapter.helpers.test.js} +26 -26
- package/dist/multithreading/adapters/loading-adapter.js +461 -0
- package/dist/multithreading/adapters/loading-adapter.test.d.ts +2 -0
- package/dist/multithreading/adapters/loading-adapter.test.d.ts.map +1 -0
- package/dist/multithreading/{worker-adapter/worker-adapter.loading.test.js → adapters/loading-adapter.test.js} +19 -26
- package/dist/multithreading/adapters/serialization.test.d.ts +2 -0
- package/dist/multithreading/adapters/serialization.test.d.ts.map +1 -0
- package/dist/multithreading/process-task.d.ts +16 -1
- package/dist/multithreading/process-task.d.ts.map +1 -1
- package/dist/multithreading/process-task.js +97 -49
- package/dist/multithreading/process-task.test.js +15 -15
- package/dist/state/base-state.d.ts +75 -0
- package/dist/state/base-state.d.ts.map +1 -0
- package/dist/state/base-state.js +261 -0
- package/dist/state/extraction-state.d.ts +29 -0
- package/dist/state/extraction-state.d.ts.map +1 -0
- package/dist/state/extraction-state.js +122 -0
- package/dist/state/loading-state.d.ts +19 -0
- package/dist/state/loading-state.d.ts.map +1 -0
- package/dist/state/loading-state.js +40 -0
- package/dist/state/state.extract-window.test.js +7 -7
- package/dist/state/state.interfaces.d.ts +32 -25
- package/dist/state/state.interfaces.d.ts.map +1 -1
- package/dist/state/state.interfaces.js +15 -4
- package/dist/state/state.pending-boundaries.test.js +15 -15
- package/dist/state/state.post-state.test.js +9 -8
- package/dist/state/state.test.js +22 -19
- package/dist/state/state.time-value-resolution.test.js +8 -8
- package/dist/types/extraction.d.ts +3 -3
- package/dist/types/extraction.d.ts.map +1 -1
- package/dist/types/index.d.ts +0 -1
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/workers.d.ts +16 -15
- package/dist/types/workers.d.ts.map +1 -1
- package/package.json +1 -1
- package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.d.ts +0 -2
- package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.d.ts.map +0 -1
- package/dist/multithreading/worker-adapter/worker-adapter.d.ts +0 -97
- package/dist/multithreading/worker-adapter/worker-adapter.d.ts.map +0 -1
- package/dist/multithreading/worker-adapter/worker-adapter.emit.test.d.ts +0 -2
- package/dist/multithreading/worker-adapter/worker-adapter.emit.test.d.ts.map +0 -1
- package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.d.ts +0 -2
- package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.d.ts.map +0 -1
- package/dist/multithreading/worker-adapter/worker-adapter.helpers.d.ts.map +0 -1
- package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.d.ts +0 -2
- package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.d.ts.map +0 -1
- package/dist/multithreading/worker-adapter/worker-adapter.js +0 -846
- package/dist/multithreading/worker-adapter/worker-adapter.loading.test.d.ts +0 -2
- package/dist/multithreading/worker-adapter/worker-adapter.loading.test.d.ts.map +0 -1
- package/dist/multithreading/worker-adapter/worker-adapter.serialization.test.d.ts +0 -2
- package/dist/multithreading/worker-adapter/worker-adapter.serialization.test.d.ts.map +0 -1
- package/dist/state/state.d.ts +0 -36
- package/dist/state/state.d.ts.map +0 -1
- package/dist/state/state.js +0 -265
- /package/dist/multithreading/{worker-adapter/worker-adapter.helpers.js → adapters/loading-adapter.helpers.js} +0 -0
- /package/dist/multithreading/{worker-adapter/worker-adapter.serialization.test.js → adapters/serialization.test.js} +0 -0
|
@@ -1,60 +1,108 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
3
|
+
exports.processExtractionTask = processExtractionTask;
|
|
4
|
+
exports.processLoadingTask = processLoadingTask;
|
|
4
5
|
const node_worker_threads_1 = require("node:worker_threads");
|
|
5
6
|
const logger_1 = require("../logger/logger");
|
|
6
7
|
const logger_context_1 = require("../logger/logger.context");
|
|
7
|
-
const
|
|
8
|
+
const extraction_state_1 = require("../state/extraction-state");
|
|
9
|
+
const loading_state_1 = require("../state/loading-state");
|
|
8
10
|
const workers_1 = require("../types/workers");
|
|
9
|
-
const
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
event,
|
|
25
|
-
initialState,
|
|
26
|
-
initialDomainMapping,
|
|
27
|
-
options,
|
|
28
|
-
});
|
|
29
|
-
const adapter = new worker_adapter_1.WorkerAdapter({
|
|
30
|
-
event,
|
|
31
|
-
adapterState,
|
|
32
|
-
options,
|
|
33
|
-
});
|
|
34
|
-
node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.on(workers_1.WorkerEvent.WorkerMessage, (message) => {
|
|
35
|
-
if (message.subject !== workers_1.WorkerMessageSubject.WorkerMessageExit) {
|
|
36
|
-
return;
|
|
37
|
-
}
|
|
38
|
-
console.log('Timeout received. Waiting for the task to finish.');
|
|
39
|
-
adapter.isTimeout = true;
|
|
40
|
-
});
|
|
41
|
-
await (0, logger_context_1.runWithUserLogContext)(async () => task({ adapter }));
|
|
42
|
-
if (adapter.isTimeout && !adapter.hasWorkerEmitted) {
|
|
43
|
-
await (0, logger_context_1.runWithUserLogContext)(async () => onTimeout({ adapter }));
|
|
11
|
+
const extraction_adapter_1 = require("./adapters/extraction-adapter");
|
|
12
|
+
const loading_adapter_1 = require("./adapters/loading-adapter");
|
|
13
|
+
/**
|
|
14
|
+
* Shared worker-thread driver. Builds the logger context, runs the task and
|
|
15
|
+
* (on timeout) the onTimeout callback against the provided adapter, and wires
|
|
16
|
+
* the error/exit plumbing. The adapter is constructed by the caller so each
|
|
17
|
+
* entry point can build its own typed adapter.
|
|
18
|
+
*/
|
|
19
|
+
async function runWorkerTask(buildAdapter, { task, onTimeout }) {
|
|
20
|
+
await (0, logger_context_1.runWithSdkLogContext)(async () => {
|
|
21
|
+
try {
|
|
22
|
+
const adapter = await buildAdapter();
|
|
23
|
+
node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.on(workers_1.WorkerEvent.WorkerMessage, (message) => {
|
|
24
|
+
if (message.subject !== workers_1.WorkerMessageSubject.WorkerMessageExit) {
|
|
25
|
+
return;
|
|
44
26
|
}
|
|
45
|
-
|
|
27
|
+
console.log('Timeout received. Waiting for the task to finish.');
|
|
28
|
+
adapter.isTimeout = true;
|
|
29
|
+
});
|
|
30
|
+
await (0, logger_context_1.runWithUserLogContext)(async () => task({ adapter }));
|
|
31
|
+
if (adapter.isTimeout && !adapter.hasWorkerEmitted) {
|
|
32
|
+
await (0, logger_context_1.runWithUserLogContext)(async () => onTimeout({ adapter }));
|
|
46
33
|
}
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
34
|
+
process.exit(0);
|
|
35
|
+
}
|
|
36
|
+
catch (error) {
|
|
37
|
+
(0, logger_context_1.runWithUserLogContext)(() => {
|
|
38
|
+
const errorMessage = `Error while processing task. ${(0, logger_1.serializeError)(error)}`;
|
|
39
|
+
console.error(errorMessage);
|
|
40
|
+
node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage({
|
|
41
|
+
subject: workers_1.WorkerMessageSubject.WorkerMessageFailed,
|
|
42
|
+
payload: { message: errorMessage },
|
|
56
43
|
});
|
|
57
|
-
|
|
44
|
+
process.exit(1);
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Entry point for an extraction worker. Builds an {@link ExtractionAdapter} and
|
|
51
|
+
* runs the provided task against it.
|
|
52
|
+
*
|
|
53
|
+
* @public
|
|
54
|
+
*/
|
|
55
|
+
function processExtractionTask({ task, onTimeout, }) {
|
|
56
|
+
if (node_worker_threads_1.isMainThread) {
|
|
57
|
+
return;
|
|
58
|
+
}
|
|
59
|
+
void runWorkerTask(async () => {
|
|
60
|
+
const event = node_worker_threads_1.workerData.event;
|
|
61
|
+
const initialState = node_worker_threads_1.workerData.initialState;
|
|
62
|
+
const initialDomainMapping = node_worker_threads_1.workerData.initialDomainMapping;
|
|
63
|
+
const options = node_worker_threads_1.workerData.options;
|
|
64
|
+
// eslint-disable-next-line no-global-assign
|
|
65
|
+
console = new logger_1.Logger({ event, options });
|
|
66
|
+
const adapterState = await (0, extraction_state_1.createExtractionState)({
|
|
67
|
+
event,
|
|
68
|
+
initialState,
|
|
69
|
+
initialDomainMapping,
|
|
70
|
+
options,
|
|
71
|
+
});
|
|
72
|
+
return new extraction_adapter_1.ExtractionAdapter({
|
|
73
|
+
event,
|
|
74
|
+
adapterState,
|
|
75
|
+
options,
|
|
76
|
+
});
|
|
77
|
+
}, { task, onTimeout });
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Entry point for a loading worker. Builds a {@link LoadingAdapter} and runs the
|
|
81
|
+
* provided task against it.
|
|
82
|
+
*
|
|
83
|
+
* @public
|
|
84
|
+
*/
|
|
85
|
+
function processLoadingTask({ task, onTimeout, }) {
|
|
86
|
+
if (node_worker_threads_1.isMainThread) {
|
|
87
|
+
return;
|
|
88
|
+
}
|
|
89
|
+
void runWorkerTask(async () => {
|
|
90
|
+
const event = node_worker_threads_1.workerData.event;
|
|
91
|
+
const initialState = node_worker_threads_1.workerData.initialState;
|
|
92
|
+
const initialDomainMapping = node_worker_threads_1.workerData.initialDomainMapping;
|
|
93
|
+
const options = node_worker_threads_1.workerData.options;
|
|
94
|
+
// eslint-disable-next-line no-global-assign
|
|
95
|
+
console = new logger_1.Logger({ event, options });
|
|
96
|
+
const adapterState = await (0, loading_state_1.createLoadingState)({
|
|
97
|
+
event,
|
|
98
|
+
initialState,
|
|
99
|
+
initialDomainMapping,
|
|
100
|
+
options,
|
|
101
|
+
});
|
|
102
|
+
return new loading_adapter_1.LoadingAdapter({
|
|
103
|
+
event,
|
|
104
|
+
adapterState,
|
|
105
|
+
options,
|
|
58
106
|
});
|
|
59
|
-
})
|
|
107
|
+
}, { task, onTimeout });
|
|
60
108
|
}
|
|
@@ -46,18 +46,18 @@ jest.mock('../logger/logger.context', () => ({
|
|
|
46
46
|
runWithSdkLogContext: jest.fn((fn) => fn()),
|
|
47
47
|
runWithUserLogContext: jest.fn((fn) => fn()),
|
|
48
48
|
}));
|
|
49
|
-
jest.mock('../state/state', () => ({
|
|
50
|
-
|
|
49
|
+
jest.mock('../state/extraction-state', () => ({
|
|
50
|
+
createExtractionState: jest.fn(),
|
|
51
51
|
}));
|
|
52
|
-
jest.mock('./
|
|
53
|
-
|
|
52
|
+
jest.mock('./adapters/extraction-adapter', () => ({
|
|
53
|
+
ExtractionAdapter: jest.fn().mockImplementation(() => ({
|
|
54
54
|
isTimeout: false,
|
|
55
55
|
hasWorkerEmitted: false,
|
|
56
56
|
})),
|
|
57
57
|
}));
|
|
58
58
|
const process_task_1 = require("./process-task");
|
|
59
|
-
const
|
|
60
|
-
const
|
|
59
|
+
const extraction_state_1 = require("../state/extraction-state");
|
|
60
|
+
const extraction_adapter_1 = require("./adapters/extraction-adapter");
|
|
61
61
|
const test_utils_1 = require("../common/test-utils");
|
|
62
62
|
function setWorkerData(data) {
|
|
63
63
|
global.__workerData__ = data;
|
|
@@ -67,9 +67,9 @@ function makeEvent(eventType = extraction_1.EventType.StartExtractingData) {
|
|
|
67
67
|
payload: { event_type: eventType },
|
|
68
68
|
});
|
|
69
69
|
}
|
|
70
|
-
// Flush the microtask queue enough to let the async IIFE inside
|
|
70
|
+
// Flush the microtask queue enough to let the async IIFE inside processExtractionTask run.
|
|
71
71
|
const flush = async () => new Promise((r) => setTimeout(r, 0));
|
|
72
|
-
describe(process_task_1.
|
|
72
|
+
describe(process_task_1.processExtractionTask.name, () => {
|
|
73
73
|
let processExitSpy;
|
|
74
74
|
beforeEach(() => {
|
|
75
75
|
jest.clearAllMocks();
|
|
@@ -77,7 +77,7 @@ describe(process_task_1.processTask.name, () => {
|
|
|
77
77
|
processExitSpy = jest
|
|
78
78
|
.spyOn(process, 'exit')
|
|
79
79
|
.mockImplementation((() => { }));
|
|
80
|
-
|
|
80
|
+
extraction_state_1.createExtractionState.mockResolvedValue({});
|
|
81
81
|
});
|
|
82
82
|
afterEach(() => {
|
|
83
83
|
processExitSpy.mockRestore();
|
|
@@ -90,11 +90,11 @@ describe(process_task_1.processTask.name, () => {
|
|
|
90
90
|
// onTimeout must be skipped. This is the guard the integration suite cannot
|
|
91
91
|
// target cleanly because it requires a precise race between emit and timeout.
|
|
92
92
|
const mockAdapter = { isTimeout: true, hasWorkerEmitted: true };
|
|
93
|
-
|
|
93
|
+
extraction_adapter_1.ExtractionAdapter.mockImplementation(() => mockAdapter);
|
|
94
94
|
const task = jest.fn().mockResolvedValue(undefined);
|
|
95
95
|
const onTimeout = jest.fn().mockResolvedValue(undefined);
|
|
96
96
|
// Act
|
|
97
|
-
(0, process_task_1.
|
|
97
|
+
(0, process_task_1.processExtractionTask)({ task, onTimeout });
|
|
98
98
|
await flush();
|
|
99
99
|
// Assert
|
|
100
100
|
expect(onTimeout).not.toHaveBeenCalled();
|
|
@@ -105,11 +105,11 @@ describe(process_task_1.processTask.name, () => {
|
|
|
105
105
|
const event = makeEvent();
|
|
106
106
|
setWorkerData({ event, initialState: {}, options: {} });
|
|
107
107
|
const mockAdapter = { isTimeout: false, hasWorkerEmitted: false };
|
|
108
|
-
|
|
108
|
+
extraction_adapter_1.ExtractionAdapter.mockImplementation(() => mockAdapter);
|
|
109
109
|
const task = jest.fn().mockResolvedValue(undefined);
|
|
110
110
|
const onTimeout = jest.fn().mockResolvedValue(undefined);
|
|
111
111
|
// Act
|
|
112
|
-
(0, process_task_1.
|
|
112
|
+
(0, process_task_1.processExtractionTask)({ task, onTimeout });
|
|
113
113
|
await flush();
|
|
114
114
|
// Grab the handler registered for WorkerMessage events and invoke it with
|
|
115
115
|
// subjects that must NOT flip isTimeout (log messages, unknown subjects).
|
|
@@ -126,12 +126,12 @@ describe(process_task_1.processTask.name, () => {
|
|
|
126
126
|
const event = makeEvent();
|
|
127
127
|
setWorkerData({ event, initialState: {}, options: {} });
|
|
128
128
|
const mockAdapter = { isTimeout: false, hasWorkerEmitted: false };
|
|
129
|
-
|
|
129
|
+
extraction_adapter_1.ExtractionAdapter.mockImplementation(() => mockAdapter);
|
|
130
130
|
const taskError = new Error('task boom');
|
|
131
131
|
const task = jest.fn().mockRejectedValue(taskError);
|
|
132
132
|
const onTimeout = jest.fn().mockResolvedValue(undefined);
|
|
133
133
|
// Act
|
|
134
|
-
(0, process_task_1.
|
|
134
|
+
(0, process_task_1.processExtractionTask)({ task, onTimeout });
|
|
135
135
|
await flush();
|
|
136
136
|
// Assert
|
|
137
137
|
expect(mockParentPortPostMessage).toHaveBeenCalledWith(expect.objectContaining({
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import { InitialDomainMapping } from '../types/common';
|
|
2
|
+
import { AirSyncEvent } from '../types/extraction';
|
|
3
|
+
import { ExtractionScope } from '../types/workers';
|
|
4
|
+
import { BaseSdkState, StateInterface } from './state.interfaces';
|
|
5
|
+
/**
|
|
6
|
+
* BaseState owns the state lifecycle shared by every sync mode: connector vs.
|
|
7
|
+
* SDK state separation, fetch/init/post against the platform, the v1->v2
|
|
8
|
+
* migration shim, and the snap-in-version-gated initial domain mapping install.
|
|
9
|
+
*
|
|
10
|
+
* Mode-specific subclasses (`ExtractionState`, `LoadingState`) narrow `sdkState`
|
|
11
|
+
* to their own SDK state shape and add mode-specific setup in their factories.
|
|
12
|
+
*
|
|
13
|
+
* @typeParam ConnectorState - the connector-owned state shape
|
|
14
|
+
* @typeParam SdkState - the SDK bookkeeping shape for this mode
|
|
15
|
+
*/
|
|
16
|
+
export declare abstract class BaseState<ConnectorState, SdkState extends BaseSdkState> {
|
|
17
|
+
protected _connectorState: ConnectorState;
|
|
18
|
+
protected _sdkState: SdkState;
|
|
19
|
+
protected _extractionScope: ExtractionScope;
|
|
20
|
+
protected readonly initialSdkState: SdkState;
|
|
21
|
+
protected readonly event: AirSyncEvent;
|
|
22
|
+
private workerUrl;
|
|
23
|
+
private devrevToken;
|
|
24
|
+
private syncUnitId;
|
|
25
|
+
private requestId;
|
|
26
|
+
constructor({ event, initialState }: StateInterface<ConnectorState>, initialSdkState: SdkState);
|
|
27
|
+
/** Connector-owned state. This is what `adapter.state` exposes to snap-in code. */
|
|
28
|
+
get state(): ConnectorState;
|
|
29
|
+
set state(value: ConnectorState);
|
|
30
|
+
/** SDK-internal bookkeeping state. Never exposed to connector code. */
|
|
31
|
+
get sdkState(): SdkState;
|
|
32
|
+
set sdkState(value: SdkState);
|
|
33
|
+
get extractionScope(): ExtractionScope;
|
|
34
|
+
/**
|
|
35
|
+
* Installs the initial domain mapping when the snap-in version in state does
|
|
36
|
+
* not match the version in the event context. Shared by all modes so that a
|
|
37
|
+
* loading run still installs the mapping if extraction has not done so.
|
|
38
|
+
*/
|
|
39
|
+
installInitialDomainMappingIfNeeded(initialDomainMapping?: InitialDomainMapping): Promise<void>;
|
|
40
|
+
/**
|
|
41
|
+
* Initializes the state for this adapter instance by fetching from API
|
|
42
|
+
* or creating an initial state if none exists (404).
|
|
43
|
+
*
|
|
44
|
+
* Reads both the v2 `{ connectorState, sdkState }` envelope and a legacy flat
|
|
45
|
+
* v1 blob (connector keys merged with SDK keys), migrating the latter on read.
|
|
46
|
+
* Always persists the v2 envelope going forward.
|
|
47
|
+
* @param initialState The initial connector state provided by the spawn function
|
|
48
|
+
*/
|
|
49
|
+
init(initialState: ConnectorState): Promise<void>;
|
|
50
|
+
/**
|
|
51
|
+
* Normalizes a parsed on-disk state into the `{ connectorState, sdkState }`
|
|
52
|
+
* envelope, migrating a legacy flat v1 blob if needed.
|
|
53
|
+
*
|
|
54
|
+
* - v2 envelope (`{ connectorState, sdkState }`): used as-is.
|
|
55
|
+
* - v1 flat blob: SDK-owned keys (`V1_SDK_STATE_KEYS`) split into `sdkState`,
|
|
56
|
+
* everything else becomes connector state.
|
|
57
|
+
* - Malformed envelope (one side present, the other missing) fails loud.
|
|
58
|
+
*/
|
|
59
|
+
private normalizeFetchedState;
|
|
60
|
+
/**
|
|
61
|
+
* Updates the state of the adapter by posting to API.
|
|
62
|
+
* Persists the v2 `{ connectorState, sdkState }` envelope.
|
|
63
|
+
* @param {object} state - The connector state to be updated
|
|
64
|
+
*/
|
|
65
|
+
postState(state?: ConnectorState): Promise<void>;
|
|
66
|
+
/**
|
|
67
|
+
* Fetches the state of the adapter from API.
|
|
68
|
+
* @return The raw state data from API
|
|
69
|
+
*/
|
|
70
|
+
fetchState(): Promise<{
|
|
71
|
+
state: string;
|
|
72
|
+
objects?: string;
|
|
73
|
+
}>;
|
|
74
|
+
}
|
|
75
|
+
//# sourceMappingURL=base-state.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"base-state.d.ts","sourceRoot":"","sources":["../../src/state/base-state.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAEnD,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAEnD,OAAO,EAEL,YAAY,EACZ,cAAc,EAEf,MAAM,oBAAoB,CAAC;AAE5B;;;;;;;;;;GAUG;AACH,8BAAsB,SAAS,CAAC,cAAc,EAAE,QAAQ,SAAS,YAAY;IAC3E,SAAS,CAAC,eAAe,EAAE,cAAc,CAAC;IAC1C,SAAS,CAAC,SAAS,EAAE,QAAQ,CAAC;IAC9B,SAAS,CAAC,gBAAgB,EAAE,eAAe,CAAM;IACjD,SAAS,CAAC,QAAQ,CAAC,eAAe,EAAE,QAAQ,CAAC;IAC7C,SAAS,CAAC,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IACvC,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,SAAS,CAAS;gBAGxB,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,cAAc,CAAC,cAAc,CAAC,EACvD,eAAe,EAAE,QAAQ;IAY3B,mFAAmF;IACnF,IAAI,KAAK,IAAI,cAAc,CAE1B;IAED,IAAI,KAAK,CAAC,KAAK,EAAE,cAAc,EAE9B;IAED,uEAAuE;IACvE,IAAI,QAAQ,IAAI,QAAQ,CAEvB;IAED,IAAI,QAAQ,CAAC,KAAK,EAAE,QAAQ,EAE3B;IAED,IAAI,eAAe,IAAI,eAAe,CAErC;IAED;;;;OAIG;IACG,mCAAmC,CACvC,oBAAoB,CAAC,EAAE,oBAAoB,GAC1C,OAAO,CAAC,IAAI,CAAC;IAwChB;;;;;;;;OAQG;IACG,IAAI,CAAC,YAAY,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IAkDvD;;;;;;;;OAQG;IACH,OAAO,CAAC,qBAAqB;IAyC7B;;;;OAIG;IACG,SAAS,CAAC,KAAK,CAAC,EAAE,cAAc;IA4DtC;;;OAGG;IACG,UAAU,IAAI,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAqBjE"}
|
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.BaseState = void 0;
|
|
7
|
+
const axios_1 = __importDefault(require("axios"));
|
|
8
|
+
const node_worker_threads_1 = require("node:worker_threads");
|
|
9
|
+
const install_initial_domain_mapping_1 = require("../common/install-initial-domain-mapping");
|
|
10
|
+
const axios_client_internal_1 = require("../http/axios-client-internal");
|
|
11
|
+
const logger_1 = require("../logger/logger");
|
|
12
|
+
const workers_1 = require("../types/workers");
|
|
13
|
+
const state_interfaces_1 = require("./state.interfaces");
|
|
14
|
+
/**
|
|
15
|
+
* BaseState owns the state lifecycle shared by every sync mode: connector vs.
|
|
16
|
+
* SDK state separation, fetch/init/post against the platform, the v1->v2
|
|
17
|
+
* migration shim, and the snap-in-version-gated initial domain mapping install.
|
|
18
|
+
*
|
|
19
|
+
* Mode-specific subclasses (`ExtractionState`, `LoadingState`) narrow `sdkState`
|
|
20
|
+
* to their own SDK state shape and add mode-specific setup in their factories.
|
|
21
|
+
*
|
|
22
|
+
* @typeParam ConnectorState - the connector-owned state shape
|
|
23
|
+
* @typeParam SdkState - the SDK bookkeeping shape for this mode
|
|
24
|
+
*/
|
|
25
|
+
class BaseState {
|
|
26
|
+
constructor({ event, initialState }, initialSdkState) {
|
|
27
|
+
this._extractionScope = {};
|
|
28
|
+
this.event = event;
|
|
29
|
+
this.initialSdkState = initialSdkState;
|
|
30
|
+
this._connectorState = initialState;
|
|
31
|
+
this._sdkState = Object.assign({}, initialSdkState);
|
|
32
|
+
this.workerUrl = event.payload.event_context.worker_data_url;
|
|
33
|
+
this.devrevToken = event.context.secrets.service_account_token;
|
|
34
|
+
this.syncUnitId = event.payload.event_context.sync_unit_id;
|
|
35
|
+
this.requestId = event.payload.event_context.request_id_adaas;
|
|
36
|
+
}
|
|
37
|
+
/** Connector-owned state. This is what `adapter.state` exposes to snap-in code. */
|
|
38
|
+
get state() {
|
|
39
|
+
return this._connectorState;
|
|
40
|
+
}
|
|
41
|
+
set state(value) {
|
|
42
|
+
this._connectorState = value;
|
|
43
|
+
}
|
|
44
|
+
/** SDK-internal bookkeeping state. Never exposed to connector code. */
|
|
45
|
+
get sdkState() {
|
|
46
|
+
return this._sdkState;
|
|
47
|
+
}
|
|
48
|
+
set sdkState(value) {
|
|
49
|
+
this._sdkState = value;
|
|
50
|
+
}
|
|
51
|
+
get extractionScope() {
|
|
52
|
+
return this._extractionScope;
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Installs the initial domain mapping when the snap-in version in state does
|
|
56
|
+
* not match the version in the event context. Shared by all modes so that a
|
|
57
|
+
* loading run still installs the mapping if extraction has not done so.
|
|
58
|
+
*/
|
|
59
|
+
async installInitialDomainMappingIfNeeded(initialDomainMapping) {
|
|
60
|
+
const snapInVersionId = this.event.context.snap_in_version_id;
|
|
61
|
+
const hasSnapInVersionInState = 'snapInVersionId' in this.sdkState;
|
|
62
|
+
const shouldUpdateIDM = !hasSnapInVersionInState ||
|
|
63
|
+
this.sdkState.snapInVersionId !== snapInVersionId;
|
|
64
|
+
if (!shouldUpdateIDM) {
|
|
65
|
+
console.log(`Snap-in version in state matches the version in event context "${snapInVersionId}". Skipping initial domain mapping installation.`);
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
try {
|
|
69
|
+
console.log(`Snap-in version in state "${this.sdkState.snapInVersionId}" does not match the version in event context "${snapInVersionId}". Installing initial domain mapping.`);
|
|
70
|
+
if (initialDomainMapping) {
|
|
71
|
+
await (0, install_initial_domain_mapping_1.installInitialDomainMapping)(this.event, initialDomainMapping);
|
|
72
|
+
this.sdkState.snapInVersionId = snapInVersionId;
|
|
73
|
+
}
|
|
74
|
+
else {
|
|
75
|
+
throw new Error('No initial domain mapping was passed to spawn function. Skipping initial domain mapping installation.');
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
catch (error) {
|
|
79
|
+
const errorMessage = `Error while installing initial domain mapping. ${(0, logger_1.serializeError)(error)}`;
|
|
80
|
+
console.error(errorMessage);
|
|
81
|
+
node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage({
|
|
82
|
+
subject: workers_1.WorkerMessageSubject.WorkerMessageFailed,
|
|
83
|
+
payload: { message: errorMessage },
|
|
84
|
+
});
|
|
85
|
+
process.exit(1);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Initializes the state for this adapter instance by fetching from API
|
|
90
|
+
* or creating an initial state if none exists (404).
|
|
91
|
+
*
|
|
92
|
+
* Reads both the v2 `{ connectorState, sdkState }` envelope and a legacy flat
|
|
93
|
+
* v1 blob (connector keys merged with SDK keys), migrating the latter on read.
|
|
94
|
+
* Always persists the v2 envelope going forward.
|
|
95
|
+
* @param initialState The initial connector state provided by the spawn function
|
|
96
|
+
*/
|
|
97
|
+
async init(initialState) {
|
|
98
|
+
var _a;
|
|
99
|
+
try {
|
|
100
|
+
const { state: stringifiedState, objects } = await this.fetchState();
|
|
101
|
+
if (!stringifiedState) {
|
|
102
|
+
throw new Error('No state found in response.');
|
|
103
|
+
}
|
|
104
|
+
let parsed;
|
|
105
|
+
try {
|
|
106
|
+
parsed = JSON.parse(stringifiedState);
|
|
107
|
+
}
|
|
108
|
+
catch (error) {
|
|
109
|
+
throw new Error(`Failed to parse state. ${error}`);
|
|
110
|
+
}
|
|
111
|
+
const { connectorState, sdkState } = this.normalizeFetchedState(parsed);
|
|
112
|
+
this.state = connectorState;
|
|
113
|
+
this.sdkState = sdkState;
|
|
114
|
+
console.log('State fetched successfully. Current state', {
|
|
115
|
+
connectorState: (0, logger_1.getPrintableState)(this.state),
|
|
116
|
+
sdkState: (0, logger_1.getPrintableState)(this.sdkState),
|
|
117
|
+
});
|
|
118
|
+
if (objects) {
|
|
119
|
+
try {
|
|
120
|
+
this._extractionScope = JSON.parse(objects);
|
|
121
|
+
}
|
|
122
|
+
catch (error) {
|
|
123
|
+
console.warn(`Failed to parse extractionScope. ${error}`);
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
catch (error) {
|
|
128
|
+
if (axios_1.default.isAxiosError(error) && ((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) === 404) {
|
|
129
|
+
console.log('State not found. Initializing state with initial state.');
|
|
130
|
+
this.state = initialState;
|
|
131
|
+
this.sdkState = Object.assign({}, this.initialSdkState);
|
|
132
|
+
await this.postState();
|
|
133
|
+
}
|
|
134
|
+
else {
|
|
135
|
+
const errorMessage = `Failed to init state. ${(0, logger_1.serializeError)(error)}`;
|
|
136
|
+
console.error(errorMessage);
|
|
137
|
+
node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage({
|
|
138
|
+
subject: workers_1.WorkerMessageSubject.WorkerMessageFailed,
|
|
139
|
+
payload: { message: errorMessage },
|
|
140
|
+
});
|
|
141
|
+
process.exit(1);
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
/**
|
|
146
|
+
* Normalizes a parsed on-disk state into the `{ connectorState, sdkState }`
|
|
147
|
+
* envelope, migrating a legacy flat v1 blob if needed.
|
|
148
|
+
*
|
|
149
|
+
* - v2 envelope (`{ connectorState, sdkState }`): used as-is.
|
|
150
|
+
* - v1 flat blob: SDK-owned keys (`V1_SDK_STATE_KEYS`) split into `sdkState`,
|
|
151
|
+
* everything else becomes connector state.
|
|
152
|
+
* - Malformed envelope (one side present, the other missing) fails loud.
|
|
153
|
+
*/
|
|
154
|
+
normalizeFetchedState(parsed) {
|
|
155
|
+
if (parsed === null || typeof parsed !== 'object') {
|
|
156
|
+
throw new Error('Fetched state is not a JSON object.');
|
|
157
|
+
}
|
|
158
|
+
const record = parsed;
|
|
159
|
+
const hasConnector = 'connectorState' in record;
|
|
160
|
+
const hasSdk = 'sdkState' in record;
|
|
161
|
+
if (hasConnector || hasSdk) {
|
|
162
|
+
if (!hasConnector || !hasSdk) {
|
|
163
|
+
throw new Error('Malformed state envelope: expected both "connectorState" and "sdkState".');
|
|
164
|
+
}
|
|
165
|
+
return {
|
|
166
|
+
connectorState: record.connectorState,
|
|
167
|
+
sdkState: Object.assign(Object.assign({}, this.initialSdkState), record.sdkState),
|
|
168
|
+
};
|
|
169
|
+
}
|
|
170
|
+
// Legacy flat v1 blob: split known SDK keys out of the connector state.
|
|
171
|
+
const connectorState = {};
|
|
172
|
+
const sdkState = {};
|
|
173
|
+
for (const [key, value] of Object.entries(record)) {
|
|
174
|
+
if (state_interfaces_1.V1_SDK_STATE_KEYS.has(key)) {
|
|
175
|
+
sdkState[key] = value;
|
|
176
|
+
}
|
|
177
|
+
else {
|
|
178
|
+
connectorState[key] = value;
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
return {
|
|
182
|
+
connectorState: connectorState,
|
|
183
|
+
sdkState: Object.assign(Object.assign({}, this.initialSdkState), sdkState),
|
|
184
|
+
};
|
|
185
|
+
}
|
|
186
|
+
/**
|
|
187
|
+
* Updates the state of the adapter by posting to API.
|
|
188
|
+
* Persists the v2 `{ connectorState, sdkState }` envelope.
|
|
189
|
+
* @param {object} state - The connector state to be updated
|
|
190
|
+
*/
|
|
191
|
+
async postState(state) {
|
|
192
|
+
const url = this.workerUrl + '.update';
|
|
193
|
+
this.state = state || this.state;
|
|
194
|
+
const envelope = {
|
|
195
|
+
connectorState: this.state,
|
|
196
|
+
sdkState: this.sdkState,
|
|
197
|
+
};
|
|
198
|
+
let stringifiedState;
|
|
199
|
+
try {
|
|
200
|
+
stringifiedState = JSON.stringify(envelope);
|
|
201
|
+
}
|
|
202
|
+
catch (error) {
|
|
203
|
+
const errorMessage = `Failed to stringify state. ${(0, logger_1.serializeError)(error)}`;
|
|
204
|
+
console.error(errorMessage);
|
|
205
|
+
node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage({
|
|
206
|
+
subject: workers_1.WorkerMessageSubject.WorkerMessageFailed,
|
|
207
|
+
payload: { message: errorMessage },
|
|
208
|
+
});
|
|
209
|
+
process.exit(1);
|
|
210
|
+
}
|
|
211
|
+
try {
|
|
212
|
+
await axios_client_internal_1.axiosClient.post(url, {
|
|
213
|
+
state: stringifiedState,
|
|
214
|
+
}, {
|
|
215
|
+
headers: {
|
|
216
|
+
Authorization: this.devrevToken,
|
|
217
|
+
},
|
|
218
|
+
params: {
|
|
219
|
+
sync_unit: this.syncUnitId,
|
|
220
|
+
request_id: this.requestId,
|
|
221
|
+
},
|
|
222
|
+
});
|
|
223
|
+
console.log('State updated successfully to', {
|
|
224
|
+
connectorState: (0, logger_1.getPrintableState)(this.state),
|
|
225
|
+
sdkState: (0, logger_1.getPrintableState)(this.sdkState),
|
|
226
|
+
});
|
|
227
|
+
}
|
|
228
|
+
catch (error) {
|
|
229
|
+
const errorMessage = `Failed to update the state. ${(0, logger_1.serializeError)(error)}`;
|
|
230
|
+
console.error(errorMessage);
|
|
231
|
+
node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage({
|
|
232
|
+
subject: workers_1.WorkerMessageSubject.WorkerMessageFailed,
|
|
233
|
+
payload: { message: errorMessage },
|
|
234
|
+
});
|
|
235
|
+
process.exit(1);
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
/**
|
|
239
|
+
* Fetches the state of the adapter from API.
|
|
240
|
+
* @return The raw state data from API
|
|
241
|
+
*/
|
|
242
|
+
async fetchState() {
|
|
243
|
+
var _a, _b;
|
|
244
|
+
console.log(`Fetching state with sync unit id ${this.syncUnitId} and request id ${this.requestId}.`);
|
|
245
|
+
const url = this.workerUrl + '.get';
|
|
246
|
+
const response = await axios_client_internal_1.axiosClient.get(url, {
|
|
247
|
+
headers: {
|
|
248
|
+
Authorization: this.devrevToken,
|
|
249
|
+
},
|
|
250
|
+
params: {
|
|
251
|
+
sync_unit: this.syncUnitId,
|
|
252
|
+
request_id: this.requestId,
|
|
253
|
+
},
|
|
254
|
+
});
|
|
255
|
+
return {
|
|
256
|
+
state: (_a = response.data) === null || _a === void 0 ? void 0 : _a.state,
|
|
257
|
+
objects: (_b = response.data) === null || _b === void 0 ? void 0 : _b.objects,
|
|
258
|
+
};
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
exports.BaseState = BaseState;
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { BaseState } from './base-state';
|
|
2
|
+
import { ExtractionSdkState, StateInterface } from './state.interfaces';
|
|
3
|
+
/**
|
|
4
|
+
* ExtractionState is the per-mode state for extraction workers. It narrows
|
|
5
|
+
* `sdkState` to `ExtractionSdkState` (extraction boundaries + attachments
|
|
6
|
+
* bookkeeping) on top of the shared lifecycle provided by `BaseState`.
|
|
7
|
+
*/
|
|
8
|
+
export declare class ExtractionState<ConnectorState> extends BaseState<ConnectorState, ExtractionSdkState> {
|
|
9
|
+
constructor(params: StateInterface<ConnectorState>);
|
|
10
|
+
/**
|
|
11
|
+
* Resolves the extraction window onto the event context.
|
|
12
|
+
*
|
|
13
|
+
* On StartExtractingMetadata: resolve fresh from the TimeValue objects in the
|
|
14
|
+
* event context and cache them as pending boundaries (always overwrite).
|
|
15
|
+
* On all other events: reuse the pending boundaries cached during
|
|
16
|
+
* StartExtractingMetadata. Finally, validate that extract_from < extract_to.
|
|
17
|
+
*/
|
|
18
|
+
resolveExtractionWindow(): void;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Creates and initializes an `ExtractionState` for an extraction worker.
|
|
22
|
+
*
|
|
23
|
+
* For non-stateless events this fetches/migrates persisted state, installs the
|
|
24
|
+
* initial domain mapping if the snap-in version changed, then resolves the
|
|
25
|
+
* extraction window (time-value resolution + pending boundary reuse) and
|
|
26
|
+
* validates it.
|
|
27
|
+
*/
|
|
28
|
+
export declare function createExtractionState<ConnectorState>({ event, initialState, initialDomainMapping, options, }: StateInterface<ConnectorState>): Promise<ExtractionState<ConnectorState>>;
|
|
29
|
+
//# sourceMappingURL=extraction-state.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"extraction-state.d.ts","sourceRoot":"","sources":["../../src/state/extraction-state.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EACL,kBAAkB,EAElB,cAAc,EACf,MAAM,oBAAoB,CAAC;AAE5B;;;;GAIG;AACH,qBAAa,eAAe,CAAC,cAAc,CAAE,SAAQ,SAAS,CAC5D,cAAc,EACd,kBAAkB,CACnB;gBACa,MAAM,EAAE,cAAc,CAAC,cAAc,CAAC;IAIlD;;;;;;;OAOG;IACH,uBAAuB,IAAI,IAAI;CA8EhC;AAED;;;;;;;GAOG;AACH,wBAAsB,qBAAqB,CAAC,cAAc,EAAE,EAC1D,KAAK,EACL,YAAY,EACZ,oBAAoB,EACpB,OAAO,GACR,EAAE,cAAc,CAAC,cAAc,CAAC,GAAG,OAAO,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC,CAkB3E"}
|