@devrev/airsync-sdk 2.0.0-beta.2 → 2.0.0-beta.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/airsync-sdk.d.ts +414 -157
- package/dist/attachments-streaming/attachments-streaming-pool.d.ts +2 -1
- package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map +1 -1
- package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts +8 -2
- package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts.map +1 -1
- package/dist/attachments-streaming/attachments-streaming-pool.js +14 -14
- package/dist/attachments-streaming/attachments-streaming-pool.test.js +47 -23
- package/dist/common/time-value-resolver.d.ts +2 -2
- package/dist/common/time-value-resolver.d.ts.map +1 -1
- package/dist/index.d.ts +3 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -4
- package/dist/multithreading/adapters/base-adapter.d.ts +56 -0
- package/dist/multithreading/adapters/base-adapter.d.ts.map +1 -0
- package/dist/multithreading/adapters/base-adapter.emit.test.d.ts +2 -0
- package/dist/multithreading/adapters/base-adapter.emit.test.d.ts.map +1 -0
- package/dist/multithreading/{worker-adapter/worker-adapter.emit.test.js → adapters/base-adapter.emit.test.js} +66 -53
- package/dist/multithreading/adapters/base-adapter.js +105 -0
- package/dist/multithreading/adapters/extraction-adapter.artifacts.test.d.ts +2 -0
- package/dist/multithreading/adapters/extraction-adapter.artifacts.test.d.ts.map +1 -0
- package/dist/multithreading/{worker-adapter/worker-adapter.artifacts.test.js → adapters/extraction-adapter.artifacts.test.js} +6 -6
- package/dist/multithreading/adapters/extraction-adapter.d.ts +58 -0
- package/dist/multithreading/adapters/extraction-adapter.d.ts.map +1 -0
- package/dist/multithreading/adapters/extraction-adapter.js +325 -0
- package/dist/multithreading/adapters/extraction-adapter.test.d.ts +2 -0
- package/dist/multithreading/adapters/extraction-adapter.test.d.ts.map +1 -0
- package/dist/multithreading/{worker-adapter/worker-adapter.extraction.test.js → adapters/extraction-adapter.test.js} +41 -51
- package/dist/multithreading/adapters/loading-adapter.d.ts +44 -0
- package/dist/multithreading/adapters/loading-adapter.d.ts.map +1 -0
- package/dist/multithreading/{worker-adapter/worker-adapter.helpers.d.ts → adapters/loading-adapter.helpers.d.ts} +1 -1
- package/dist/multithreading/adapters/loading-adapter.helpers.d.ts.map +1 -0
- package/dist/multithreading/adapters/loading-adapter.helpers.test.d.ts +2 -0
- package/dist/multithreading/adapters/loading-adapter.helpers.test.d.ts.map +1 -0
- package/dist/multithreading/{worker-adapter/worker-adapter.helpers.test.js → adapters/loading-adapter.helpers.test.js} +26 -26
- package/dist/multithreading/adapters/loading-adapter.js +461 -0
- package/dist/multithreading/adapters/loading-adapter.test.d.ts +2 -0
- package/dist/multithreading/adapters/loading-adapter.test.d.ts.map +1 -0
- package/dist/multithreading/{worker-adapter/worker-adapter.loading.test.js → adapters/loading-adapter.test.js} +19 -26
- package/dist/multithreading/adapters/serialization.test.d.ts +2 -0
- package/dist/multithreading/adapters/serialization.test.d.ts.map +1 -0
- package/dist/multithreading/process-task.d.ts +16 -1
- package/dist/multithreading/process-task.d.ts.map +1 -1
- package/dist/multithreading/process-task.js +97 -49
- package/dist/multithreading/process-task.test.js +15 -15
- package/dist/state/base-state.d.ts +75 -0
- package/dist/state/base-state.d.ts.map +1 -0
- package/dist/state/base-state.js +261 -0
- package/dist/state/extraction-state.d.ts +29 -0
- package/dist/state/extraction-state.d.ts.map +1 -0
- package/dist/state/extraction-state.js +122 -0
- package/dist/state/loading-state.d.ts +19 -0
- package/dist/state/loading-state.d.ts.map +1 -0
- package/dist/state/loading-state.js +40 -0
- package/dist/state/state.extract-window.test.js +7 -7
- package/dist/state/state.interfaces.d.ts +32 -25
- package/dist/state/state.interfaces.d.ts.map +1 -1
- package/dist/state/state.interfaces.js +15 -4
- package/dist/state/state.pending-boundaries.test.js +15 -15
- package/dist/state/state.post-state.test.js +9 -8
- package/dist/state/state.test.js +22 -19
- package/dist/state/state.time-value-resolution.test.js +8 -8
- package/dist/types/extraction.d.ts +3 -3
- package/dist/types/extraction.d.ts.map +1 -1
- package/dist/types/index.d.ts +0 -1
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/workers.d.ts +16 -15
- package/dist/types/workers.d.ts.map +1 -1
- package/package.json +1 -1
- package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.d.ts +0 -2
- package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.d.ts.map +0 -1
- package/dist/multithreading/worker-adapter/worker-adapter.d.ts +0 -97
- package/dist/multithreading/worker-adapter/worker-adapter.d.ts.map +0 -1
- package/dist/multithreading/worker-adapter/worker-adapter.emit.test.d.ts +0 -2
- package/dist/multithreading/worker-adapter/worker-adapter.emit.test.d.ts.map +0 -1
- package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.d.ts +0 -2
- package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.d.ts.map +0 -1
- package/dist/multithreading/worker-adapter/worker-adapter.helpers.d.ts.map +0 -1
- package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.d.ts +0 -2
- package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.d.ts.map +0 -1
- package/dist/multithreading/worker-adapter/worker-adapter.js +0 -846
- package/dist/multithreading/worker-adapter/worker-adapter.loading.test.d.ts +0 -2
- package/dist/multithreading/worker-adapter/worker-adapter.loading.test.d.ts.map +0 -1
- package/dist/multithreading/worker-adapter/worker-adapter.serialization.test.d.ts +0 -2
- package/dist/multithreading/worker-adapter/worker-adapter.serialization.test.d.ts.map +0 -1
- package/dist/state/state.d.ts +0 -36
- package/dist/state/state.d.ts.map +0 -1
- package/dist/state/state.js +0 -265
- /package/dist/multithreading/{worker-adapter/worker-adapter.helpers.js → adapters/loading-adapter.helpers.js} +0 -0
- /package/dist/multithreading/{worker-adapter/worker-adapter.serialization.test.js → adapters/serialization.test.js} +0 -0
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.ExtractionState = void 0;
|
|
4
|
+
exports.createExtractionState = createExtractionState;
|
|
5
|
+
const node_worker_threads_1 = require("node:worker_threads");
|
|
6
|
+
const constants_1 = require("../common/constants");
|
|
7
|
+
const time_value_resolver_1 = require("../common/time-value-resolver");
|
|
8
|
+
const logger_1 = require("../logger/logger");
|
|
9
|
+
const extraction_1 = require("../types/extraction");
|
|
10
|
+
const workers_1 = require("../types/workers");
|
|
11
|
+
const base_state_1 = require("./base-state");
|
|
12
|
+
const state_interfaces_1 = require("./state.interfaces");
|
|
13
|
+
/**
|
|
14
|
+
* ExtractionState is the per-mode state for extraction workers. It narrows
|
|
15
|
+
* `sdkState` to `ExtractionSdkState` (extraction boundaries + attachments
|
|
16
|
+
* bookkeeping) on top of the shared lifecycle provided by `BaseState`.
|
|
17
|
+
*/
|
|
18
|
+
class ExtractionState extends base_state_1.BaseState {
|
|
19
|
+
constructor(params) {
|
|
20
|
+
super(params, state_interfaces_1.initialExtractionSdkState);
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Resolves the extraction window onto the event context.
|
|
24
|
+
*
|
|
25
|
+
* On StartExtractingMetadata: resolve fresh from the TimeValue objects in the
|
|
26
|
+
* event context and cache them as pending boundaries (always overwrite).
|
|
27
|
+
* On all other events: reuse the pending boundaries cached during
|
|
28
|
+
* StartExtractingMetadata. Finally, validate that extract_from < extract_to.
|
|
29
|
+
*/
|
|
30
|
+
resolveExtractionWindow() {
|
|
31
|
+
const eventContext = this.event.payload.event_context;
|
|
32
|
+
const sdkState = this.sdkState;
|
|
33
|
+
if (this.event.payload.event_type === extraction_1.EventType.StartExtractingMetadata) {
|
|
34
|
+
const timeFields = [
|
|
35
|
+
{
|
|
36
|
+
source: 'extraction_start_time',
|
|
37
|
+
target: 'extract_from',
|
|
38
|
+
pending: 'pendingWorkersOldest',
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
source: 'extraction_end_time',
|
|
42
|
+
target: 'extract_to',
|
|
43
|
+
pending: 'pendingWorkersNewest',
|
|
44
|
+
},
|
|
45
|
+
];
|
|
46
|
+
for (const { source, target, pending } of timeFields) {
|
|
47
|
+
const timeValue = eventContext[source];
|
|
48
|
+
if (timeValue && timeValue.type) {
|
|
49
|
+
try {
|
|
50
|
+
const resolved = (0, time_value_resolver_1.resolveTimeValue)(timeValue, sdkState);
|
|
51
|
+
eventContext[target] = resolved;
|
|
52
|
+
sdkState[pending] = resolved;
|
|
53
|
+
console.log(`Resolved ${target} to ${resolved}. Stored in ${pending}.`);
|
|
54
|
+
}
|
|
55
|
+
catch (error) {
|
|
56
|
+
const errorMessage = `Failed to resolve ${source}: ${(0, logger_1.serializeError)(error)}`;
|
|
57
|
+
console.error(errorMessage);
|
|
58
|
+
node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage({
|
|
59
|
+
subject: workers_1.WorkerMessageSubject.WorkerMessageFailed,
|
|
60
|
+
payload: { message: errorMessage },
|
|
61
|
+
});
|
|
62
|
+
process.exit(1);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
else {
|
|
68
|
+
// Non-StartExtractingMetadata events: reuse pending values from state
|
|
69
|
+
if (sdkState.pendingWorkersOldest) {
|
|
70
|
+
eventContext.extract_from = sdkState.pendingWorkersOldest;
|
|
71
|
+
console.log(`Reusing pendingWorkersOldest as extract_from: ${sdkState.pendingWorkersOldest}.`);
|
|
72
|
+
}
|
|
73
|
+
else {
|
|
74
|
+
console.log('pendingWorkersOldest is not set in state. extract_from will not be populated for this invocation.');
|
|
75
|
+
}
|
|
76
|
+
if (sdkState.pendingWorkersNewest) {
|
|
77
|
+
eventContext.extract_to = sdkState.pendingWorkersNewest;
|
|
78
|
+
console.log(`Reusing pendingWorkersNewest as extract_to: ${sdkState.pendingWorkersNewest}.`);
|
|
79
|
+
}
|
|
80
|
+
else {
|
|
81
|
+
console.log('pendingWorkersNewest is not set in state. extract_to will not be populated for this invocation.');
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
// Validate that extract_from is before extract_to
|
|
85
|
+
if (eventContext.extract_from && eventContext.extract_to) {
|
|
86
|
+
if (eventContext.extract_from >= eventContext.extract_to) {
|
|
87
|
+
const errorMessage = `Invalid extraction window: extract_from (${eventContext.extract_from}) must be older than extract_to (${eventContext.extract_to}). This indicates an error in the platform.`;
|
|
88
|
+
console.error(errorMessage);
|
|
89
|
+
node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage({
|
|
90
|
+
subject: workers_1.WorkerMessageSubject.WorkerMessageFailed,
|
|
91
|
+
payload: { message: errorMessage },
|
|
92
|
+
});
|
|
93
|
+
process.exit(1);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
exports.ExtractionState = ExtractionState;
|
|
99
|
+
/**
|
|
100
|
+
* Creates and initializes an `ExtractionState` for an extraction worker.
|
|
101
|
+
*
|
|
102
|
+
* For non-stateless events this fetches/migrates persisted state, installs the
|
|
103
|
+
* initial domain mapping if the snap-in version changed, then resolves the
|
|
104
|
+
* extraction window (time-value resolution + pending boundary reuse) and
|
|
105
|
+
* validates it.
|
|
106
|
+
*/
|
|
107
|
+
async function createExtractionState({ event, initialState, initialDomainMapping, options, }) {
|
|
108
|
+
// Deep clone the initial state to avoid mutating the original state
|
|
109
|
+
const deepCloneInitialState = structuredClone(initialState);
|
|
110
|
+
const state = new ExtractionState({
|
|
111
|
+
event,
|
|
112
|
+
initialState: deepCloneInitialState,
|
|
113
|
+
initialDomainMapping,
|
|
114
|
+
options,
|
|
115
|
+
});
|
|
116
|
+
if (!constants_1.STATELESS_EVENT_TYPES.includes(event.payload.event_type)) {
|
|
117
|
+
await state.init(deepCloneInitialState);
|
|
118
|
+
await state.installInitialDomainMappingIfNeeded(initialDomainMapping);
|
|
119
|
+
state.resolveExtractionWindow();
|
|
120
|
+
}
|
|
121
|
+
return state;
|
|
122
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { BaseState } from './base-state';
|
|
2
|
+
import { LoadingSdkState, StateInterface } from './state.interfaces';
|
|
3
|
+
/**
|
|
4
|
+
* LoadingState is the per-mode state for loading workers. It narrows `sdkState`
|
|
5
|
+
* to `LoadingSdkState` (files-to-load bookkeeping) on top of the shared
|
|
6
|
+
* lifecycle provided by `BaseState`.
|
|
7
|
+
*/
|
|
8
|
+
export declare class LoadingState<ConnectorState> extends BaseState<ConnectorState, LoadingSdkState> {
|
|
9
|
+
constructor(params: StateInterface<ConnectorState>);
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Creates and initializes a `LoadingState` for a loading worker.
|
|
13
|
+
*
|
|
14
|
+
* For non-stateless events this fetches/migrates persisted state and installs
|
|
15
|
+
* the initial domain mapping if the snap-in version changed. Loading has no
|
|
16
|
+
* extraction-window resolution, so the factory is otherwise minimal.
|
|
17
|
+
*/
|
|
18
|
+
export declare function createLoadingState<ConnectorState>({ event, initialState, initialDomainMapping, options, }: StateInterface<ConnectorState>): Promise<LoadingState<ConnectorState>>;
|
|
19
|
+
//# sourceMappingURL=loading-state.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"loading-state.d.ts","sourceRoot":"","sources":["../../src/state/loading-state.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAEL,eAAe,EACf,cAAc,EACf,MAAM,oBAAoB,CAAC;AAE5B;;;;GAIG;AACH,qBAAa,YAAY,CAAC,cAAc,CAAE,SAAQ,SAAS,CACzD,cAAc,EACd,eAAe,CAChB;gBACa,MAAM,EAAE,cAAc,CAAC,cAAc,CAAC;CAGnD;AAED;;;;;;GAMG;AACH,wBAAsB,kBAAkB,CAAC,cAAc,EAAE,EACvD,KAAK,EACL,YAAY,EACZ,oBAAoB,EACpB,OAAO,GACR,EAAE,cAAc,CAAC,cAAc,CAAC,GAAG,OAAO,CAAC,YAAY,CAAC,cAAc,CAAC,CAAC,CAiBxE"}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.LoadingState = void 0;
|
|
4
|
+
exports.createLoadingState = createLoadingState;
|
|
5
|
+
const constants_1 = require("../common/constants");
|
|
6
|
+
const base_state_1 = require("./base-state");
|
|
7
|
+
const state_interfaces_1 = require("./state.interfaces");
|
|
8
|
+
/**
|
|
9
|
+
* LoadingState is the per-mode state for loading workers. It narrows `sdkState`
|
|
10
|
+
* to `LoadingSdkState` (files-to-load bookkeeping) on top of the shared
|
|
11
|
+
* lifecycle provided by `BaseState`.
|
|
12
|
+
*/
|
|
13
|
+
class LoadingState extends base_state_1.BaseState {
|
|
14
|
+
constructor(params) {
|
|
15
|
+
super(params, state_interfaces_1.initialLoadingSdkState);
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
exports.LoadingState = LoadingState;
|
|
19
|
+
/**
|
|
20
|
+
* Creates and initializes a `LoadingState` for a loading worker.
|
|
21
|
+
*
|
|
22
|
+
* For non-stateless events this fetches/migrates persisted state and installs
|
|
23
|
+
* the initial domain mapping if the snap-in version changed. Loading has no
|
|
24
|
+
* extraction-window resolution, so the factory is otherwise minimal.
|
|
25
|
+
*/
|
|
26
|
+
async function createLoadingState({ event, initialState, initialDomainMapping, options, }) {
|
|
27
|
+
// Deep clone the initial state to avoid mutating the original state
|
|
28
|
+
const deepCloneInitialState = structuredClone(initialState);
|
|
29
|
+
const state = new LoadingState({
|
|
30
|
+
event,
|
|
31
|
+
initialState: deepCloneInitialState,
|
|
32
|
+
initialDomainMapping,
|
|
33
|
+
options,
|
|
34
|
+
});
|
|
35
|
+
if (!constants_1.STATELESS_EVENT_TYPES.includes(event.payload.event_type)) {
|
|
36
|
+
await state.init(deepCloneInitialState);
|
|
37
|
+
await state.installInitialDomainMappingIfNeeded(initialDomainMapping);
|
|
38
|
+
}
|
|
39
|
+
return state;
|
|
40
|
+
}
|
|
@@ -3,14 +3,14 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
const jest_setup_1 = require("../tests/jest.setup");
|
|
4
4
|
const test_utils_1 = require("../common/test-utils");
|
|
5
5
|
const extraction_1 = require("../types/extraction");
|
|
6
|
-
const
|
|
6
|
+
const extraction_state_1 = require("./extraction-state");
|
|
7
7
|
describe('State — extraction window validation', () => {
|
|
8
8
|
let fetchStateSpy;
|
|
9
9
|
let processExitSpy;
|
|
10
10
|
beforeEach(() => {
|
|
11
11
|
jest.clearAllMocks();
|
|
12
12
|
jest.restoreAllMocks();
|
|
13
|
-
fetchStateSpy = jest.spyOn(
|
|
13
|
+
fetchStateSpy = jest.spyOn(extraction_state_1.ExtractionState.prototype, 'fetchState');
|
|
14
14
|
processExitSpy = jest.spyOn(process, 'exit').mockImplementation(() => {
|
|
15
15
|
throw new Error('process.exit called');
|
|
16
16
|
});
|
|
@@ -38,7 +38,7 @@ describe('State — extraction window validation', () => {
|
|
|
38
38
|
});
|
|
39
39
|
fetchStateSpy.mockResolvedValue({ state: stringifiedState });
|
|
40
40
|
// Act & Assert
|
|
41
|
-
await expect((0,
|
|
41
|
+
await expect((0, extraction_state_1.createExtractionState)({
|
|
42
42
|
event,
|
|
43
43
|
initialState: {},
|
|
44
44
|
initialDomainMapping: {},
|
|
@@ -67,7 +67,7 @@ describe('State — extraction window validation', () => {
|
|
|
67
67
|
});
|
|
68
68
|
fetchStateSpy.mockResolvedValue({ state: stringifiedState });
|
|
69
69
|
// Act & Assert
|
|
70
|
-
await expect((0,
|
|
70
|
+
await expect((0, extraction_state_1.createExtractionState)({
|
|
71
71
|
event,
|
|
72
72
|
initialState: {},
|
|
73
73
|
initialDomainMapping: {},
|
|
@@ -96,7 +96,7 @@ describe('State — extraction window validation', () => {
|
|
|
96
96
|
});
|
|
97
97
|
fetchStateSpy.mockResolvedValue({ state: stringifiedState });
|
|
98
98
|
// Act
|
|
99
|
-
await (0,
|
|
99
|
+
await (0, extraction_state_1.createExtractionState)({
|
|
100
100
|
event,
|
|
101
101
|
initialState: {},
|
|
102
102
|
initialDomainMapping: {},
|
|
@@ -122,7 +122,7 @@ describe('State — extraction window validation', () => {
|
|
|
122
122
|
});
|
|
123
123
|
fetchStateSpy.mockResolvedValue({ state: stringifiedState });
|
|
124
124
|
// Act
|
|
125
|
-
await (0,
|
|
125
|
+
await (0, extraction_state_1.createExtractionState)({
|
|
126
126
|
event,
|
|
127
127
|
initialState: {},
|
|
128
128
|
initialDomainMapping: {},
|
|
@@ -151,7 +151,7 @@ describe('State — extraction window validation', () => {
|
|
|
151
151
|
});
|
|
152
152
|
fetchStateSpy.mockResolvedValue({ state: stringifiedState });
|
|
153
153
|
// Act
|
|
154
|
-
await (0,
|
|
154
|
+
await (0, extraction_state_1.createExtractionState)({
|
|
155
155
|
event,
|
|
156
156
|
initialState: {},
|
|
157
157
|
initialDomainMapping: {},
|
|
@@ -2,7 +2,16 @@ import { InitialDomainMapping } from '../types/common';
|
|
|
2
2
|
import { AirSyncEvent } from '../types/extraction';
|
|
3
3
|
import { FileToLoad } from '../types/loading';
|
|
4
4
|
import { WorkerAdapterOptions } from '../types/workers';
|
|
5
|
-
|
|
5
|
+
/**
|
|
6
|
+
* BaseSdkState holds SDK bookkeeping common to every sync mode.
|
|
7
|
+
*/
|
|
8
|
+
export interface BaseSdkState {
|
|
9
|
+
snapInVersionId?: string;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* ExtractionSdkState holds SDK bookkeeping owned by the extraction modes.
|
|
13
|
+
*/
|
|
14
|
+
export interface ExtractionSdkState extends BaseSdkState {
|
|
6
15
|
/** The pending (not yet committed) oldest extraction boundary (ISO 8601 timestamp).
|
|
7
16
|
* Set on StartExtractingMetadata, reused across subsequent phases, cleared on AttachmentExtractionDone. */
|
|
8
17
|
pendingWorkersOldest?: string;
|
|
@@ -14,14 +23,22 @@ export interface SdkState {
|
|
|
14
23
|
/** The newest point of extraction (ISO 8601 timestamp). */
|
|
15
24
|
workersNewest?: string;
|
|
16
25
|
toDevRev?: ToDevRev;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* LoadingSdkState holds SDK bookkeeping owned by the loading modes.
|
|
29
|
+
*/
|
|
30
|
+
export interface LoadingSdkState extends BaseSdkState {
|
|
17
31
|
fromDevRev?: FromDevRev;
|
|
18
|
-
snapInVersionId?: string;
|
|
19
32
|
}
|
|
20
33
|
/**
|
|
21
|
-
*
|
|
22
|
-
*
|
|
34
|
+
* AdapterStateEnvelope is the v2 on-disk state shape: connector state and SDK
|
|
35
|
+
* bookkeeping are stored as disjoint sub-objects so SDK internals stay
|
|
36
|
+
* encapsulated and never collide with connector keys.
|
|
23
37
|
*/
|
|
24
|
-
export
|
|
38
|
+
export interface AdapterStateEnvelope<ConnectorState, SdkState extends BaseSdkState> {
|
|
39
|
+
connectorState: ConnectorState;
|
|
40
|
+
sdkState: SdkState;
|
|
41
|
+
}
|
|
25
42
|
export interface ToDevRev {
|
|
26
43
|
attachmentsMetadata: {
|
|
27
44
|
artifactIds: string[];
|
|
@@ -45,24 +62,14 @@ export interface StateInterface<ConnectorState> {
|
|
|
45
62
|
initialDomainMapping?: InitialDomainMapping;
|
|
46
63
|
options?: WorkerAdapterOptions;
|
|
47
64
|
}
|
|
48
|
-
export declare const
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
lastProcessedAttachmentsIdsList: never[];
|
|
59
|
-
};
|
|
60
|
-
};
|
|
61
|
-
};
|
|
62
|
-
export declare const loadingSdkState: {
|
|
63
|
-
snapInVersionId: string;
|
|
64
|
-
fromDevRev: {
|
|
65
|
-
filesToLoad: never[];
|
|
66
|
-
};
|
|
67
|
-
};
|
|
65
|
+
export declare const initialExtractionSdkState: ExtractionSdkState;
|
|
66
|
+
export declare const initialLoadingSdkState: LoadingSdkState;
|
|
67
|
+
/**
|
|
68
|
+
* The set of top-level state keys owned by the SDK. Derived from the initial
|
|
69
|
+
* SDK state constants so it auto-updates whenever a new SDK field is added.
|
|
70
|
+
* Used by the migration shim to split a flat v1 state blob into the
|
|
71
|
+
* `{ connectorState, sdkState }` envelope: keys in this set go to `sdkState`,
|
|
72
|
+
* everything else is connector state.
|
|
73
|
+
*/
|
|
74
|
+
export declare const V1_SDK_STATE_KEYS: ReadonlySet<string>;
|
|
68
75
|
//# sourceMappingURL=state.interfaces.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"state.interfaces.d.ts","sourceRoot":"","sources":["../../src/state/state.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD,MAAM,WAAW,
|
|
1
|
+
{"version":3,"file":"state.interfaces.d.ts","sourceRoot":"","sources":["../../src/state/state.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED;;GAEG;AACH,MAAM,WAAW,kBAAmB,SAAQ,YAAY;IACtD;gHAC4G;IAC5G,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;gHAC4G;IAC5G,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,2DAA2D;IAC3D,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,2DAA2D;IAC3D,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,EAAE,QAAQ,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,eAAgB,SAAQ,YAAY;IACnD,UAAU,CAAC,EAAE,UAAU,CAAC;CACzB;AAED;;;;GAIG;AACH,MAAM,WAAW,oBAAoB,CACnC,cAAc,EACd,QAAQ,SAAS,YAAY;IAE7B,cAAc,EAAE,cAAc,CAAC;IAC/B,QAAQ,EAAE,QAAQ,CAAC;CACpB;AAED,MAAM,WAAW,QAAQ;IACvB,mBAAmB,EAAE;QACnB,WAAW,EAAE,MAAM,EAAE,CAAC;QACtB,aAAa,EAAE,MAAM,CAAC;QACtB,+BAA+B,CAAC,EAAE,mBAAmB,EAAE,CAAC;KACzD,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,UAAU;IACzB,WAAW,EAAE,UAAU,EAAE,CAAC;CAC3B;AAED,MAAM,WAAW,cAAc,CAAC,cAAc;IAC5C,KAAK,EAAE,YAAY,CAAC;IACpB,YAAY,EAAE,cAAc,CAAC;IAC7B,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED,eAAO,MAAM,yBAAyB,EAAE,kBAavC,CAAC;AAEF,eAAO,MAAM,sBAAsB,EAAE,eAKpC,CAAC;AAEF;;;;;;GAMG;AACH,eAAO,MAAM,iBAAiB,EAAE,WAAW,CAAC,MAAM,CAGhD,CAAC"}
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
4
|
-
exports.
|
|
3
|
+
exports.V1_SDK_STATE_KEYS = exports.initialLoadingSdkState = exports.initialExtractionSdkState = void 0;
|
|
4
|
+
exports.initialExtractionSdkState = {
|
|
5
|
+
snapInVersionId: '',
|
|
5
6
|
pendingWorkersOldest: '',
|
|
6
7
|
pendingWorkersNewest: '',
|
|
7
8
|
workersOldest: '',
|
|
8
9
|
workersNewest: '',
|
|
9
|
-
snapInVersionId: '',
|
|
10
10
|
toDevRev: {
|
|
11
11
|
attachmentsMetadata: {
|
|
12
12
|
artifactIds: [],
|
|
@@ -15,9 +15,20 @@ exports.extractionSdkState = {
|
|
|
15
15
|
},
|
|
16
16
|
},
|
|
17
17
|
};
|
|
18
|
-
exports.
|
|
18
|
+
exports.initialLoadingSdkState = {
|
|
19
19
|
snapInVersionId: '',
|
|
20
20
|
fromDevRev: {
|
|
21
21
|
filesToLoad: [],
|
|
22
22
|
},
|
|
23
23
|
};
|
|
24
|
+
/**
|
|
25
|
+
* The set of top-level state keys owned by the SDK. Derived from the initial
|
|
26
|
+
* SDK state constants so it auto-updates whenever a new SDK field is added.
|
|
27
|
+
* Used by the migration shim to split a flat v1 state blob into the
|
|
28
|
+
* `{ connectorState, sdkState }` envelope: keys in this set go to `sdkState`,
|
|
29
|
+
* everything else is connector state.
|
|
30
|
+
*/
|
|
31
|
+
exports.V1_SDK_STATE_KEYS = new Set([
|
|
32
|
+
...Object.keys(exports.initialExtractionSdkState),
|
|
33
|
+
...Object.keys(exports.initialLoadingSdkState),
|
|
34
|
+
]);
|
|
@@ -3,7 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
const jest_setup_1 = require("../tests/jest.setup");
|
|
4
4
|
const test_utils_1 = require("../common/test-utils");
|
|
5
5
|
const extraction_1 = require("../types/extraction");
|
|
6
|
-
const
|
|
6
|
+
const extraction_state_1 = require("./extraction-state");
|
|
7
7
|
/* eslint-disable @typescript-eslint/no-require-imports */
|
|
8
8
|
const FIXED_NOW = '2026-03-26T10:00:00.000Z';
|
|
9
9
|
describe('State — pending extraction boundaries', () => {
|
|
@@ -13,8 +13,8 @@ describe('State — pending extraction boundaries', () => {
|
|
|
13
13
|
beforeEach(() => {
|
|
14
14
|
jest.clearAllMocks();
|
|
15
15
|
jest.restoreAllMocks();
|
|
16
|
-
postStateSpy = jest.spyOn(
|
|
17
|
-
fetchStateSpy = jest.spyOn(
|
|
16
|
+
postStateSpy = jest.spyOn(extraction_state_1.ExtractionState.prototype, 'postState');
|
|
17
|
+
fetchStateSpy = jest.spyOn(extraction_state_1.ExtractionState.prototype, 'fetchState');
|
|
18
18
|
installInitialDomainMappingSpy = jest.spyOn(require('../common/install-initial-domain-mapping'), 'installInitialDomainMapping');
|
|
19
19
|
jest.spyOn(process, 'exit').mockImplementation(() => {
|
|
20
20
|
throw new Error('process.exit called');
|
|
@@ -50,14 +50,14 @@ describe('State — pending extraction boundaries', () => {
|
|
|
50
50
|
installInitialDomainMappingSpy.mockResolvedValue({ success: true });
|
|
51
51
|
postStateSpy.mockResolvedValue({ success: true });
|
|
52
52
|
// Act
|
|
53
|
-
const state = await (0,
|
|
53
|
+
const state = await (0, extraction_state_1.createExtractionState)({
|
|
54
54
|
event,
|
|
55
55
|
initialState: {},
|
|
56
56
|
initialDomainMapping: {},
|
|
57
57
|
});
|
|
58
58
|
// Assert
|
|
59
|
-
expect(state.
|
|
60
|
-
expect(state.
|
|
59
|
+
expect(state.sdkState.pendingWorkersOldest).toBe('1970-01-01T00:00:00.000Z');
|
|
60
|
+
expect(state.sdkState.pendingWorkersNewest).toBe(FIXED_NOW);
|
|
61
61
|
expect(event.payload.event_context.extract_from).toBe('1970-01-01T00:00:00.000Z');
|
|
62
62
|
expect(event.payload.event_context.extract_to).toBe(FIXED_NOW);
|
|
63
63
|
});
|
|
@@ -88,15 +88,15 @@ describe('State — pending extraction boundaries', () => {
|
|
|
88
88
|
});
|
|
89
89
|
fetchStateSpy.mockResolvedValue({ state: stringifiedState });
|
|
90
90
|
// Act
|
|
91
|
-
const state = await (0,
|
|
91
|
+
const state = await (0, extraction_state_1.createExtractionState)({
|
|
92
92
|
event,
|
|
93
93
|
initialState: {},
|
|
94
94
|
initialDomainMapping: {},
|
|
95
95
|
});
|
|
96
96
|
// Assert: pending values are overwritten with fresh resolution, not stale values
|
|
97
|
-
expect(state.
|
|
98
|
-
expect(state.
|
|
99
|
-
expect(state.
|
|
97
|
+
expect(state.sdkState.pendingWorkersOldest).toBe('1970-01-01T00:00:00.000Z');
|
|
98
|
+
expect(state.sdkState.pendingWorkersNewest).toBe(FIXED_NOW);
|
|
99
|
+
expect(state.sdkState.pendingWorkersNewest).not.toBe(staleNewest);
|
|
100
100
|
});
|
|
101
101
|
it('should reuse pending values from state on ContinueExtractingData instead of re-resolving', async () => {
|
|
102
102
|
// Arrange: state has pending values from a prior StartExtractingMetadata phase
|
|
@@ -126,7 +126,7 @@ describe('State — pending extraction boundaries', () => {
|
|
|
126
126
|
});
|
|
127
127
|
fetchStateSpy.mockResolvedValue({ state: stringifiedState });
|
|
128
128
|
// Act
|
|
129
|
-
const state = await (0,
|
|
129
|
+
const state = await (0, extraction_state_1.createExtractionState)({
|
|
130
130
|
event,
|
|
131
131
|
initialState: {},
|
|
132
132
|
initialDomainMapping: {},
|
|
@@ -135,8 +135,8 @@ describe('State — pending extraction boundaries', () => {
|
|
|
135
135
|
expect(event.payload.event_context.extract_from).toBe(pendingOldest);
|
|
136
136
|
expect(event.payload.event_context.extract_to).toBe(pendingNewest);
|
|
137
137
|
// Pending values in state remain unchanged
|
|
138
|
-
expect(state.
|
|
139
|
-
expect(state.
|
|
138
|
+
expect(state.sdkState.pendingWorkersOldest).toBe(pendingOldest);
|
|
139
|
+
expect(state.sdkState.pendingWorkersNewest).toBe(pendingNewest);
|
|
140
140
|
});
|
|
141
141
|
it('should not set extract_from/extract_to on ContinueExtractingData if no pending values exist', async () => {
|
|
142
142
|
// Arrange: state has no pending values (e.g. old state from before this feature)
|
|
@@ -151,7 +151,7 @@ describe('State — pending extraction boundaries', () => {
|
|
|
151
151
|
});
|
|
152
152
|
fetchStateSpy.mockResolvedValue({ state: stringifiedState });
|
|
153
153
|
// Act
|
|
154
|
-
await (0,
|
|
154
|
+
await (0, extraction_state_1.createExtractionState)({
|
|
155
155
|
event,
|
|
156
156
|
initialState: {},
|
|
157
157
|
initialDomainMapping: {},
|
|
@@ -177,7 +177,7 @@ describe('State — pending extraction boundaries', () => {
|
|
|
177
177
|
});
|
|
178
178
|
fetchStateSpy.mockResolvedValue({ state: stringifiedState });
|
|
179
179
|
// Act
|
|
180
|
-
await (0,
|
|
180
|
+
await (0, extraction_state_1.createExtractionState)({
|
|
181
181
|
event,
|
|
182
182
|
initialState: {},
|
|
183
183
|
initialDomainMapping: {},
|
|
@@ -3,7 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
const jest_setup_1 = require("../tests/jest.setup");
|
|
4
4
|
const test_utils_1 = require("../common/test-utils");
|
|
5
5
|
const extraction_1 = require("../types/extraction");
|
|
6
|
-
const
|
|
6
|
+
const extraction_state_1 = require("./extraction-state");
|
|
7
7
|
/* eslint-disable @typescript-eslint/no-require-imports */
|
|
8
8
|
describe('State.postState', () => {
|
|
9
9
|
let postStateSpy;
|
|
@@ -12,8 +12,8 @@ describe('State.postState', () => {
|
|
|
12
12
|
beforeEach(() => {
|
|
13
13
|
jest.clearAllMocks();
|
|
14
14
|
jest.restoreAllMocks();
|
|
15
|
-
postStateSpy = jest.spyOn(
|
|
16
|
-
fetchStateSpy = jest.spyOn(
|
|
15
|
+
postStateSpy = jest.spyOn(extraction_state_1.ExtractionState.prototype, 'postState');
|
|
16
|
+
fetchStateSpy = jest.spyOn(extraction_state_1.ExtractionState.prototype, 'fetchState');
|
|
17
17
|
processExitSpy = jest.spyOn(process, 'exit').mockImplementation(() => {
|
|
18
18
|
throw new Error('process.exit called');
|
|
19
19
|
});
|
|
@@ -32,7 +32,7 @@ describe('State.postState', () => {
|
|
|
32
32
|
state: JSON.stringify({ snapInVersionId: '1.0.0' }),
|
|
33
33
|
});
|
|
34
34
|
postStateSpy.mockRestore();
|
|
35
|
-
const adapterState = await (0,
|
|
35
|
+
const adapterState = await (0, extraction_state_1.createExtractionState)({
|
|
36
36
|
event,
|
|
37
37
|
initialState: {},
|
|
38
38
|
initialDomainMapping: {},
|
|
@@ -43,11 +43,12 @@ describe('State.postState', () => {
|
|
|
43
43
|
const requests = jest_setup_1.mockServer.getRequests('POST', '/worker_data_url.update');
|
|
44
44
|
expect(requests).toHaveLength(1);
|
|
45
45
|
const body = requests[0].body;
|
|
46
|
-
// Body must contain the stringified
|
|
46
|
+
// Body must contain the stringified v2 envelope: connector state and SDK
|
|
47
|
+
// bookkeeping stored as disjoint sub-objects.
|
|
47
48
|
expect(typeof body.state).toBe('string');
|
|
48
49
|
const parsed = JSON.parse(body.state);
|
|
49
|
-
expect(parsed.foo).toBe('bar');
|
|
50
|
-
expect(parsed.snapInVersionId).toBe('1.0.0');
|
|
50
|
+
expect(parsed.connectorState.foo).toBe('bar');
|
|
51
|
+
expect(parsed.sdkState.snapInVersionId).toBe('1.0.0');
|
|
51
52
|
});
|
|
52
53
|
it('should exit(1) when postState HTTP request fails', async () => {
|
|
53
54
|
// Arrange
|
|
@@ -59,7 +60,7 @@ describe('State.postState', () => {
|
|
|
59
60
|
state: JSON.stringify({ snapInVersionId: '1.0.0' }),
|
|
60
61
|
});
|
|
61
62
|
postStateSpy.mockRestore();
|
|
62
|
-
const adapterState = await (0,
|
|
63
|
+
const adapterState = await (0, extraction_state_1.createExtractionState)({
|
|
63
64
|
event,
|
|
64
65
|
initialState: {},
|
|
65
66
|
initialDomainMapping: {},
|