@x12i/memorix-service 3.4.0 → 3.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/app.d.ts.map +1 -1
- package/dist/app.js +6 -0
- package/dist/app.js.map +1 -1
- package/dist/connector-ops/checkpoint-store.d.ts +7 -0
- package/dist/connector-ops/checkpoint-store.d.ts.map +1 -0
- package/dist/connector-ops/checkpoint-store.js +135 -0
- package/dist/connector-ops/checkpoint-store.js.map +1 -0
- package/dist/connector-ops/checkpoint-types.d.ts +46 -0
- package/dist/connector-ops/checkpoint-types.d.ts.map +1 -0
- package/dist/connector-ops/checkpoint-types.js +14 -0
- package/dist/connector-ops/checkpoint-types.js.map +1 -0
- package/dist/connector-ops/continuous-types.d.ts +141 -0
- package/dist/connector-ops/continuous-types.d.ts.map +1 -0
- package/dist/connector-ops/continuous-types.js +13 -0
- package/dist/connector-ops/continuous-types.js.map +1 -0
- package/dist/connector-ops/coordinator.d.ts +88 -0
- package/dist/connector-ops/coordinator.d.ts.map +1 -0
- package/dist/connector-ops/coordinator.js +174 -0
- package/dist/connector-ops/coordinator.js.map +1 -0
- package/dist/connector-ops/index.d.ts +6 -0
- package/dist/connector-ops/index.d.ts.map +1 -1
- package/dist/connector-ops/index.js +6 -0
- package/dist/connector-ops/index.js.map +1 -1
- package/dist/connector-ops/run-store.d.ts +16 -0
- package/dist/connector-ops/run-store.d.ts.map +1 -0
- package/dist/connector-ops/run-store.js +249 -0
- package/dist/connector-ops/run-store.js.map +1 -0
- package/dist/connector-ops/schedule-lease-store.d.ts +8 -0
- package/dist/connector-ops/schedule-lease-store.d.ts.map +1 -0
- package/dist/connector-ops/schedule-lease-store.js +318 -0
- package/dist/connector-ops/schedule-lease-store.js.map +1 -0
- package/dist/incremental-state/index.d.ts +3 -0
- package/dist/incremental-state/index.d.ts.map +1 -0
- package/dist/incremental-state/index.js +3 -0
- package/dist/incremental-state/index.js.map +1 -0
- package/dist/incremental-state/state-store.d.ts +7 -0
- package/dist/incremental-state/state-store.d.ts.map +1 -0
- package/dist/incremental-state/state-store.js +183 -0
- package/dist/incremental-state/state-store.js.map +1 -0
- package/dist/incremental-state/types.d.ts +104 -0
- package/dist/incremental-state/types.d.ts.map +1 -0
- package/dist/incremental-state/types.js +8 -0
- package/dist/incremental-state/types.js.map +1 -0
- package/dist/platform.d.ts +26 -1
- package/dist/platform.d.ts.map +1 -1
- package/dist/platform.js +111 -1
- package/dist/platform.js.map +1 -1
- package/dist/routes/continuous-acquisition.d.ts +5 -0
- package/dist/routes/continuous-acquisition.d.ts.map +1 -0
- package/dist/routes/continuous-acquisition.js +259 -0
- package/dist/routes/continuous-acquisition.js.map +1 -0
- package/dist/routes/incremental-state.d.ts +5 -0
- package/dist/routes/incremental-state.d.ts.map +1 -0
- package/dist/routes/incremental-state.js +173 -0
- package/dist/routes/incremental-state.js.map +1 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -7,7 +7,7 @@ Remote apps should **call** this service over HTTP — not reinvent wiring by in
|
|
|
7
7
|
## Install / run
|
|
8
8
|
|
|
9
9
|
```bash
|
|
10
|
-
npm install @x12i/memorix-service@3.
|
|
10
|
+
npm install @x12i/memorix-service@3.6.0 # composition library / CLI
|
|
11
11
|
```
|
|
12
12
|
|
|
13
13
|
Prefer the monorepo stack (service + Studio + opx-source):
|
package/dist/app.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"app.d.ts","sourceRoot":"","sources":["../src/app.ts"],"names":[],"mappings":"AAIA,OAAO,OAAO,EAAE,EAAE,KAAK,eAAe,EAAE,MAAM,SAAS,CAAC;AAIxD,OAAO,EAEL,KAAK,eAAe,EACpB,KAAK,YAAY,EAClB,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"app.d.ts","sourceRoot":"","sources":["../src/app.ts"],"names":[],"mappings":"AAIA,OAAO,OAAO,EAAE,EAAE,KAAK,eAAe,EAAE,MAAM,SAAS,CAAC;AAIxD,OAAO,EAEL,KAAK,eAAe,EACpB,KAAK,YAAY,EAClB,MAAM,eAAe,CAAC;AA4BvB,MAAM,MAAM,8BAA8B,GAAG,YAAY,GAAG;IAC1D,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,QAAQ,CAAC,EAAE,eAAe,CAAC;CAC5B,CAAC;AAEF,wBAAsB,uBAAuB,CAC3C,IAAI,EAAE,8BAA8B,GACnC,OAAO,CAAC;IAAE,GAAG,EAAE,eAAe,CAAC;IAAC,QAAQ,EAAE,eAAe,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,CA+F5E;AAED,MAAM,MAAM,0BAA0B,GAAG,8BAA8B,GAAG;IACxE,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,wBAAsB,uBAAuB,CAAC,IAAI,EAAE,0BAA0B;;;;;GAM7E"}
|
package/dist/app.js
CHANGED
|
@@ -15,8 +15,10 @@ import { registerRelationshipRoutes } from "./routes/relationships.js";
|
|
|
15
15
|
import { registerMemoryRoutes } from "./routes/memory.js";
|
|
16
16
|
import { registerConnectorInvocationRoutes } from "./routes/connector-invocations.js";
|
|
17
17
|
import { registerConnectorOpsRoutes } from "./routes/connector-ops.js";
|
|
18
|
+
import { registerContinuousAcquisitionRoutes } from "./routes/continuous-acquisition.js";
|
|
18
19
|
import { registerIntegrationRoutes } from "./routes/integrations.js";
|
|
19
20
|
import { registerRestrictedStoreRoutes } from "./routes/restricted-store.js";
|
|
21
|
+
import { registerIncrementalStateRoutes } from "./routes/incremental-state.js";
|
|
20
22
|
import { registerPipelineRoutes } from "./routes/pipelines.js";
|
|
21
23
|
import { registerJobsRoutes } from "./routes/jobs.js";
|
|
22
24
|
import { registerOperationsRoutes } from "./routes/operations.js";
|
|
@@ -98,8 +100,10 @@ export async function createMemorixServiceApp(opts) {
|
|
|
98
100
|
await registerMemoryRoutes(app, platform);
|
|
99
101
|
await registerConnectorInvocationRoutes(app, platform);
|
|
100
102
|
await registerConnectorOpsRoutes(app, platform);
|
|
103
|
+
await registerContinuousAcquisitionRoutes(app, platform);
|
|
101
104
|
await registerIntegrationRoutes(app, platform);
|
|
102
105
|
await registerRestrictedStoreRoutes(app, platform);
|
|
106
|
+
await registerIncrementalStateRoutes(app, platform);
|
|
103
107
|
await registerPipelineRoutes(app, platform);
|
|
104
108
|
await registerJobsRoutes(app, platform);
|
|
105
109
|
await registerOperationsRoutes(app, platform);
|
|
@@ -112,6 +116,8 @@ export async function createMemorixServiceApp(opts) {
|
|
|
112
116
|
platform.startMemoryScheduleTicker({ intervalMs: 60_000 });
|
|
113
117
|
// Jobs continuous Work evaluation + held-run promotion.
|
|
114
118
|
platform.startJobsWorkTicker({ intervalMs: 15_000 });
|
|
119
|
+
// Continuous acquisition coordinator (schedule → lease → bounded pipeline).
|
|
120
|
+
platform.startContinuousAcquisitionTicker({ intervalMs: 30_000 });
|
|
115
121
|
return { app, platform, port: core.port };
|
|
116
122
|
}
|
|
117
123
|
export async function startMemorixServiceHttp(opts) {
|
package/dist/app.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"app.js","sourceRoot":"","sources":["../src/app.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,OAAiC,MAAM,SAAS,CAAC;AACxD,OAAO,IAAI,MAAM,eAAe,CAAC;AACjC,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAC;AAC/D,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5D,OAAO,EACL,qBAAqB,GAGtB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAC9D,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAC5D,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAE,0BAA0B,EAAE,MAAM,2BAA2B,CAAC;AACvE,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,EAAE,iCAAiC,EAAE,MAAM,mCAAmC,CAAC;AACtF,OAAO,EAAE,0BAA0B,EAAE,MAAM,2BAA2B,CAAC;AACvE,OAAO,EAAE,yBAAyB,EAAE,MAAM,0BAA0B,CAAC;AACrE,OAAO,EAAE,6BAA6B,EAAE,MAAM,8BAA8B,CAAC;AAC7E,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAE,wBAAwB,EAAE,MAAM,wBAAwB,CAAC;AAClE,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAC9D,OAAO,EAAE,0BAA0B,EAAE,MAAM,0BAA0B,CAAC;AACtE,OAAO,EAAE,yBAAyB,EAAE,MAAM,0BAA0B,CAAC;AACrE,OAAO,EAAE,wBAAwB,EAAE,MAAM,wBAAwB,CAAC;AAClE,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,EAAE,wBAAwB,EAAE,MAAM,yBAAyB,CAAC;AACnE,OAAO,EAAE,0BAA0B,EAAE,MAAM,iBAAiB,CAAC;AAC7D,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,EAAE,iCAAiC,EAAE,MAAM,yBAAyB,CAAC;AAE5E,MAAM,SAAS,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAO1D,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,IAAoC;IAEpC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,CAAC,MAAM,qBAAqB,CAAC,IAAI,CAAC,CAAC,CAAC;IACtE,MAAM,UAAU,GAAG,0BAA0B,EAAE,CAAC;IAChD,MAAM,GAAG,GAAG,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,KAAK,EAAE,CAAC,CAAC;IACtD,MAAM,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;IAE3C,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAC,GAAY,EAAE;QACjD,IAAI,EAAE,eAAe;QACrB,QAAQ,EAAE;YACR,EAAE,EAAE,uBAAuB;YAC3B,KAAK,EAAE,iBAAiB;YACxB,IAAI,EAAE,KAAK;YACX,MAAM,EAAE,SAAS;YACjB,OAAO,EAAE,SAAS;YAClB,OAAO,EAAE,sBAAsB;YAC/B,QAAQ,EAAE,mBAAmB;YAC7B,WAAW,EAAE,4BAA4B;SAC1C;QACD,SAAS,EAAE,KAAK,IAAI,EAAE;YACpB,MAAM,GAAG,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC;YAClC,OAAO;gBACL,EAAE,EAAE,IAAI;gBACR,OAAO,EAAE,uBAAuB;gBAChC,IAAI,EAAE,QAAQ,CAAC,IAAI;gBACnB,UAAU,EAAE;oBACV,SAAS,EAAE,GAAG,CAAC,SAAS;oBACxB,MAAM,EAAE,GAAG,CAAC,MAAM;oBAClB,aAAa,EAAE,GAAG,CAAC,aAAa;oBAChC,SAAS,EAAE,GAAG,CAAC,SAAS;iBACzB;aACF,CAAC;QACJ,CAAC;KACF,CAAC,CAAC;IAEH,GAAG,CAAC,eAAe,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE;QACtC,MAAM,aAAa,GAChB,GAAkC,CAAC,aAAa,IAAI,UAAU,EAAE,CAAC;QACpE,MAAM,MAAM,GAAG,cAAc,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;QAClD,MAAM,CAAC,IAAI,CAAC,KAAK,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAA6B,CAAC;QACjF,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;IAEH,GAAG,CAAC,GAAG,CAAC,mBAAmB,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;QACjD,MAAM,IAAI,GAAG,OAAO,CAAC,SAAS,EAAE,yBAAyB,CAAC,CAAC;QAC3D,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACxC,OAAO,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;IAC/D,CAAC,CAAC,CAAC;IAEH,GAAG,CAAC,GAAG,CAAC,wBAAwB,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QAC9C,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;QAChC,MAAM,IAAI,GAAG,QAAQ,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QACzC,OAAO;YACL,KAAK;YACL,QAAQ,EAAE;gBACR,GAAG,IAAI;gBACP,UAAU,EAAE,iCAAiC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBAC1D,aAAa,EAAE,CAAC,CAAC,aAAa;oBAC9B,GAAG,EAAE,CAAC,CAAC,GAAG;oBACV,GAAG,EAAE,CAAC,CAAC,GAAG;oBACV,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;oBAC1B,WAAW,EAAE,CAAC,CAAC,WAAW;iBAC3B,CAAC,CAAC;aACJ;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,wBAAwB,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;IAC1C,MAAM,sBAAsB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAC5C,MAAM,qBAAqB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAC3C,MAAM,kBAAkB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IACxC,MAAM,0BAA0B,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAChD,MAAM,oBAAoB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAC1C,MAAM,iCAAiC,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IACvD,MAAM,0BAA0B,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAChD,MAAM,yBAAyB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAC/C,MAAM,6BAA6B,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IACnD,MAAM,sBAAsB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAC5C,MAAM,kBAAkB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IACxC,MAAM,wBAAwB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAC9C,MAAM,sBAAsB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAC5C,MAAM,0BAA0B,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAChD,MAAM,yBAAyB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAC/C,MAAM,oBAAoB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAC1C,MAAM,wBAAwB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAE9C,iFAAiF;IACjF,QAAQ,CAAC,yBAAyB,CAAC,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,CAAC;IAC3D,wDAAwD;IACxD,QAAQ,CAAC,mBAAmB,CAAC,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"app.js","sourceRoot":"","sources":["../src/app.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,OAAiC,MAAM,SAAS,CAAC;AACxD,OAAO,IAAI,MAAM,eAAe,CAAC;AACjC,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAC;AAC/D,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5D,OAAO,EACL,qBAAqB,GAGtB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAC9D,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAC5D,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAE,0BAA0B,EAAE,MAAM,2BAA2B,CAAC;AACvE,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,EAAE,iCAAiC,EAAE,MAAM,mCAAmC,CAAC;AACtF,OAAO,EAAE,0BAA0B,EAAE,MAAM,2BAA2B,CAAC;AACvE,OAAO,EAAE,mCAAmC,EAAE,MAAM,oCAAoC,CAAC;AACzF,OAAO,EAAE,yBAAyB,EAAE,MAAM,0BAA0B,CAAC;AACrE,OAAO,EAAE,6BAA6B,EAAE,MAAM,8BAA8B,CAAC;AAC7E,OAAO,EAAE,8BAA8B,EAAE,MAAM,+BAA+B,CAAC;AAC/E,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAE,wBAAwB,EAAE,MAAM,wBAAwB,CAAC;AAClE,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAC9D,OAAO,EAAE,0BAA0B,EAAE,MAAM,0BAA0B,CAAC;AACtE,OAAO,EAAE,yBAAyB,EAAE,MAAM,0BAA0B,CAAC;AACrE,OAAO,EAAE,wBAAwB,EAAE,MAAM,wBAAwB,CAAC;AAClE,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,EAAE,wBAAwB,EAAE,MAAM,yBAAyB,CAAC;AACnE,OAAO,EAAE,0BAA0B,EAAE,MAAM,iBAAiB,CAAC;AAC7D,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,EAAE,iCAAiC,EAAE,MAAM,yBAAyB,CAAC;AAE5E,MAAM,SAAS,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAO1D,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,IAAoC;IAEpC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,CAAC,MAAM,qBAAqB,CAAC,IAAI,CAAC,CAAC,CAAC;IACtE,MAAM,UAAU,GAAG,0BAA0B,EAAE,CAAC;IAChD,MAAM,GAAG,GAAG,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,KAAK,EAAE,CAAC,CAAC;IACtD,MAAM,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;IAE3C,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAC,GAAY,EAAE;QACjD,IAAI,EAAE,eAAe;QACrB,QAAQ,EAAE;YACR,EAAE,EAAE,uBAAuB;YAC3B,KAAK,EAAE,iBAAiB;YACxB,IAAI,EAAE,KAAK;YACX,MAAM,EAAE,SAAS;YACjB,OAAO,EAAE,SAAS;YAClB,OAAO,EAAE,sBAAsB;YAC/B,QAAQ,EAAE,mBAAmB;YAC7B,WAAW,EAAE,4BAA4B;SAC1C;QACD,SAAS,EAAE,KAAK,IAAI,EAAE;YACpB,MAAM,GAAG,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC;YAClC,OAAO;gBACL,EAAE,EAAE,IAAI;gBACR,OAAO,EAAE,uBAAuB;gBAChC,IAAI,EAAE,QAAQ,CAAC,IAAI;gBACnB,UAAU,EAAE;oBACV,SAAS,EAAE,GAAG,CAAC,SAAS;oBACxB,MAAM,EAAE,GAAG,CAAC,MAAM;oBAClB,aAAa,EAAE,GAAG,CAAC,aAAa;oBAChC,SAAS,EAAE,GAAG,CAAC,SAAS;iBACzB;aACF,CAAC;QACJ,CAAC;KACF,CAAC,CAAC;IAEH,GAAG,CAAC,eAAe,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE;QACtC,MAAM,aAAa,GAChB,GAAkC,CAAC,aAAa,IAAI,UAAU,EAAE,CAAC;QACpE,MAAM,MAAM,GAAG,cAAc,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;QAClD,MAAM,CAAC,IAAI,CAAC,KAAK,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAA6B,CAAC;QACjF,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;IAEH,GAAG,CAAC,GAAG,CAAC,mBAAmB,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;QACjD,MAAM,IAAI,GAAG,OAAO,CAAC,SAAS,EAAE,yBAAyB,CAAC,CAAC;QAC3D,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACxC,OAAO,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;IAC/D,CAAC,CAAC,CAAC;IAEH,GAAG,CAAC,GAAG,CAAC,wBAAwB,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QAC9C,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;QAChC,MAAM,IAAI,GAAG,QAAQ,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QACzC,OAAO;YACL,KAAK;YACL,QAAQ,EAAE;gBACR,GAAG,IAAI;gBACP,UAAU,EAAE,iCAAiC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBAC1D,aAAa,EAAE,CAAC,CAAC,aAAa;oBAC9B,GAAG,EAAE,CAAC,CAAC,GAAG;oBACV,GAAG,EAAE,CAAC,CAAC,GAAG;oBACV,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;oBAC1B,WAAW,EAAE,CAAC,CAAC,WAAW;iBAC3B,CAAC,CAAC;aACJ;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,wBAAwB,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;IAC1C,MAAM,sBAAsB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAC5C,MAAM,qBAAqB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAC3C,MAAM,kBAAkB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IACxC,MAAM,0BAA0B,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAChD,MAAM,oBAAoB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAC1C,MAAM,iCAAiC,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IACvD,MAAM,0BAA0B,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAChD,MAAM,mCAAmC,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IACzD,MAAM,yBAAyB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAC/C,MAAM,6BAA6B,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IACnD,MAAM,8BAA8B,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IACpD,MAAM,sBAAsB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAC5C,MAAM,kBAAkB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IACxC,MAAM,wBAAwB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAC9C,MAAM,sBAAsB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAC5C,MAAM,0BAA0B,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAChD,MAAM,yBAAyB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAC/C,MAAM,oBAAoB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAC1C,MAAM,wBAAwB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAE9C,iFAAiF;IACjF,QAAQ,CAAC,yBAAyB,CAAC,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,CAAC;IAC3D,wDAAwD;IACxD,QAAQ,CAAC,mBAAmB,CAAC,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,CAAC;IACrD,4EAA4E;IAC5E,QAAQ,CAAC,gCAAgC,CAAC,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,CAAC;IAElE,OAAO,EAAE,GAAG,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC;AAC5C,CAAC;AAOD,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,IAAgC;IAC5E,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,MAAM,uBAAuB,CAAC,IAAI,CAAC,CAAC;IAC9E,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,QAAQ,CAAC;IACnC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,WAAW,CAAC;IACtC,MAAM,GAAG,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IACjC,OAAO,EAAE,GAAG,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;AACvC,CAAC"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { MongoClient } from "mongodb";
|
|
2
|
+
import { type PutStreamCheckpoint, type StreamCheckpointMethod, type StreamCheckpointStore } from "./checkpoint-types.js";
|
|
3
|
+
export declare function assertCheckpointMethod(raw: unknown): StreamCheckpointMethod;
|
|
4
|
+
export declare function validatePutCheckpoint(body: PutStreamCheckpoint): void;
|
|
5
|
+
export declare function createInMemoryStreamCheckpointStore(orgId: string): StreamCheckpointStore;
|
|
6
|
+
export declare function createMongoStreamCheckpointStore(client: MongoClient, orgId: string): StreamCheckpointStore;
|
|
7
|
+
//# sourceMappingURL=checkpoint-store.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"checkpoint-store.d.ts","sourceRoot":"","sources":["../../src/connector-ops/checkpoint-store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAM,WAAW,EAAE,MAAM,SAAS,CAAC;AAG/C,OAAO,EAGL,KAAK,mBAAmB,EAExB,KAAK,sBAAsB,EAC3B,KAAK,qBAAqB,EAC3B,MAAM,uBAAuB,CAAC;AAW/B,wBAAgB,sBAAsB,CACpC,GAAG,EAAE,OAAO,GACX,sBAAsB,CAUxB;AAED,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,mBAAmB,GAAG,IAAI,CAoBrE;AAUD,wBAAgB,mCAAmC,CACjD,KAAK,EAAE,MAAM,GACZ,qBAAqB,CA+CvB;AAED,wBAAgB,gCAAgC,CAC9C,MAAM,EAAE,WAAW,EACnB,KAAK,EAAE,MAAM,GACZ,qBAAqB,CAyEvB"}
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
import { ServiceError } from "../errors.js";
|
|
2
|
+
import { hasSecretPayload } from "./sanitize.js";
|
|
3
|
+
import { MCO_COLLECTIONS_CHECKPOINT, STREAM_CHECKPOINT_METHODS, } from "./checkpoint-types.js";
|
|
4
|
+
function operationalDbName(orgId) {
|
|
5
|
+
return `${orgId}_operational`;
|
|
6
|
+
}
|
|
7
|
+
function stripMongo(doc) {
|
|
8
|
+
const { _id: _drop, ...rest } = doc;
|
|
9
|
+
return rest;
|
|
10
|
+
}
|
|
11
|
+
export function assertCheckpointMethod(raw) {
|
|
12
|
+
if (typeof raw === "string" &&
|
|
13
|
+
STREAM_CHECKPOINT_METHODS.includes(raw)) {
|
|
14
|
+
return raw;
|
|
15
|
+
}
|
|
16
|
+
throw new Error(`method must be one of: ${STREAM_CHECKPOINT_METHODS.join(", ")}`);
|
|
17
|
+
}
|
|
18
|
+
export function validatePutCheckpoint(body) {
|
|
19
|
+
assertCheckpointMethod(body.method);
|
|
20
|
+
if (hasSecretPayload(body)) {
|
|
21
|
+
throw new Error("checkpoint payload must not contain secrets or response bodies");
|
|
22
|
+
}
|
|
23
|
+
const method = body.method;
|
|
24
|
+
if (method === "watermark" && body.watermark == null) {
|
|
25
|
+
throw new Error("watermark required when method is watermark");
|
|
26
|
+
}
|
|
27
|
+
if (method === "revision" && body.revision == null) {
|
|
28
|
+
throw new Error("revision required when method is revision");
|
|
29
|
+
}
|
|
30
|
+
if (method === "fullScanHash" && body.fullScanHash == null) {
|
|
31
|
+
throw new Error("fullScanHash required when method is fullScanHash");
|
|
32
|
+
}
|
|
33
|
+
if (method === "cursor" && (body.cursor == null || body.cursor === "")) {
|
|
34
|
+
throw new Error("cursor required when method is cursor");
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
function streamKey(connectorId, instanceId, streamId) {
|
|
38
|
+
return `${connectorId}\0${instanceId}\0${streamId}`;
|
|
39
|
+
}
|
|
40
|
+
export function createInMemoryStreamCheckpointStore(orgId) {
|
|
41
|
+
const byKey = new Map();
|
|
42
|
+
return {
|
|
43
|
+
async get(connectorId, instanceId, streamId) {
|
|
44
|
+
const doc = byKey.get(streamKey(connectorId, instanceId, streamId));
|
|
45
|
+
return doc ? { ...doc } : null;
|
|
46
|
+
},
|
|
47
|
+
async put(connectorId, instanceId, streamId, body) {
|
|
48
|
+
validatePutCheckpoint(body);
|
|
49
|
+
const k = streamKey(connectorId, instanceId, streamId);
|
|
50
|
+
const existing = byKey.get(k);
|
|
51
|
+
if (body.expectedCheckpointRevision != null &&
|
|
52
|
+
(existing?.checkpointRevision ?? 0) !== body.expectedCheckpointRevision) {
|
|
53
|
+
throw new ServiceError("CHECKPOINT_REVISION_CONFLICT", `checkpoint revision conflict: expected ${body.expectedCheckpointRevision}, got ${existing?.checkpointRevision ?? 0}`, 409);
|
|
54
|
+
}
|
|
55
|
+
const now = new Date().toISOString();
|
|
56
|
+
const doc = {
|
|
57
|
+
orgId,
|
|
58
|
+
connectorId,
|
|
59
|
+
instanceId,
|
|
60
|
+
streamId,
|
|
61
|
+
method: body.method,
|
|
62
|
+
watermark: body.watermark ?? null,
|
|
63
|
+
revision: body.revision ?? null,
|
|
64
|
+
fullScanHash: body.fullScanHash ?? null,
|
|
65
|
+
cursor: body.cursor ?? null,
|
|
66
|
+
contentHash: body.contentHash ?? null,
|
|
67
|
+
producerRunId: body.producerRunId ?? null,
|
|
68
|
+
checkpointRevision: (existing?.checkpointRevision ?? 0) + 1,
|
|
69
|
+
updatedAt: now,
|
|
70
|
+
};
|
|
71
|
+
byKey.set(k, doc);
|
|
72
|
+
return { ...doc };
|
|
73
|
+
},
|
|
74
|
+
async clear(connectorId, instanceId, streamId) {
|
|
75
|
+
return byKey.delete(streamKey(connectorId, instanceId, streamId));
|
|
76
|
+
},
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
export function createMongoStreamCheckpointStore(client, orgId) {
|
|
80
|
+
const db = client.db(operationalDbName(orgId));
|
|
81
|
+
const col = () => db.collection(MCO_COLLECTIONS_CHECKPOINT.checkpoints);
|
|
82
|
+
let indexesReady = null;
|
|
83
|
+
const ensureIndexes = async () => {
|
|
84
|
+
if (!indexesReady) {
|
|
85
|
+
indexesReady = (async () => {
|
|
86
|
+
await col().createIndex({ connectorId: 1, instanceId: 1, streamId: 1 }, { unique: true, name: "mco_stream_checkpoint_unique" });
|
|
87
|
+
})();
|
|
88
|
+
}
|
|
89
|
+
await indexesReady;
|
|
90
|
+
};
|
|
91
|
+
return {
|
|
92
|
+
async get(connectorId, instanceId, streamId) {
|
|
93
|
+
await ensureIndexes();
|
|
94
|
+
const doc = await col().findOne({ connectorId, instanceId, streamId });
|
|
95
|
+
return doc ? stripMongo(doc) : null;
|
|
96
|
+
},
|
|
97
|
+
async put(connectorId, instanceId, streamId, body) {
|
|
98
|
+
await ensureIndexes();
|
|
99
|
+
validatePutCheckpoint(body);
|
|
100
|
+
const existing = await col().findOne({
|
|
101
|
+
connectorId,
|
|
102
|
+
instanceId,
|
|
103
|
+
streamId,
|
|
104
|
+
});
|
|
105
|
+
if (body.expectedCheckpointRevision != null &&
|
|
106
|
+
(existing?.checkpointRevision ?? 0) !== body.expectedCheckpointRevision) {
|
|
107
|
+
throw new ServiceError("CHECKPOINT_REVISION_CONFLICT", `checkpoint revision conflict: expected ${body.expectedCheckpointRevision}, got ${existing?.checkpointRevision ?? 0}`, 409);
|
|
108
|
+
}
|
|
109
|
+
const now = new Date().toISOString();
|
|
110
|
+
const doc = {
|
|
111
|
+
orgId,
|
|
112
|
+
connectorId,
|
|
113
|
+
instanceId,
|
|
114
|
+
streamId,
|
|
115
|
+
method: body.method,
|
|
116
|
+
watermark: body.watermark ?? null,
|
|
117
|
+
revision: body.revision ?? null,
|
|
118
|
+
fullScanHash: body.fullScanHash ?? null,
|
|
119
|
+
cursor: body.cursor ?? null,
|
|
120
|
+
contentHash: body.contentHash ?? null,
|
|
121
|
+
producerRunId: body.producerRunId ?? null,
|
|
122
|
+
checkpointRevision: (existing?.checkpointRevision ?? 0) + 1,
|
|
123
|
+
updatedAt: now,
|
|
124
|
+
};
|
|
125
|
+
await col().replaceOne({ connectorId, instanceId, streamId }, doc, { upsert: true });
|
|
126
|
+
return { ...doc };
|
|
127
|
+
},
|
|
128
|
+
async clear(connectorId, instanceId, streamId) {
|
|
129
|
+
await ensureIndexes();
|
|
130
|
+
const res = await col().deleteOne({ connectorId, instanceId, streamId });
|
|
131
|
+
return res.deletedCount > 0;
|
|
132
|
+
},
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
//# sourceMappingURL=checkpoint-store.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"checkpoint-store.js","sourceRoot":"","sources":["../../src/connector-ops/checkpoint-store.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,EACL,0BAA0B,EAC1B,yBAAyB,GAK1B,MAAM,uBAAuB,CAAC;AAE/B,SAAS,iBAAiB,CAAC,KAAa;IACtC,OAAO,GAAG,KAAK,cAAc,CAAC;AAChC,CAAC;AAED,SAAS,UAAU,CAAmB,GAA0B;IAC9D,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,IAAI,EAAE,GAAG,GAA4B,CAAC;IAC7D,OAAO,IAAS,CAAC;AACnB,CAAC;AAED,MAAM,UAAU,sBAAsB,CACpC,GAAY;IAEZ,IACE,OAAO,GAAG,KAAK,QAAQ;QACtB,yBAA+C,CAAC,QAAQ,CAAC,GAAG,CAAC,EAC9D,CAAC;QACD,OAAO,GAA6B,CAAC;IACvC,CAAC;IACD,MAAM,IAAI,KAAK,CACb,0BAA0B,yBAAyB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACjE,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,IAAyB;IAC7D,sBAAsB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACpC,IAAI,gBAAgB,CAAC,IAAI,CAAC,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CACb,gEAAgE,CACjE,CAAC;IACJ,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;IAC3B,IAAI,MAAM,KAAK,WAAW,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,EAAE,CAAC;QACrD,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;IACjE,CAAC;IACD,IAAI,MAAM,KAAK,UAAU,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,EAAE,CAAC;QACnD,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;IAC/D,CAAC;IACD,IAAI,MAAM,KAAK,cAAc,IAAI,IAAI,CAAC,YAAY,IAAI,IAAI,EAAE,CAAC;QAC3D,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;IACvE,CAAC;IACD,IAAI,MAAM,KAAK,QAAQ,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,EAAE,CAAC,EAAE,CAAC;QACvE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAC3D,CAAC;AACH,CAAC;AAED,SAAS,SAAS,CAChB,WAAmB,EACnB,UAAkB,EAClB,QAAgB;IAEhB,OAAO,GAAG,WAAW,KAAK,UAAU,KAAK,QAAQ,EAAE,CAAC;AACtD,CAAC;AAED,MAAM,UAAU,mCAAmC,CACjD,KAAa;IAEb,MAAM,KAAK,GAAG,IAAI,GAAG,EAA4B,CAAC;IAElD,OAAO;QACL,KAAK,CAAC,GAAG,CAAC,WAAW,EAAE,UAAU,EAAE,QAAQ;YACzC,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,WAAW,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC,CAAC;YACpE,OAAO,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QACjC,CAAC;QAED,KAAK,CAAC,GAAG,CAAC,WAAW,EAAE,UAAU,EAAE,QAAQ,EAAE,IAAI;YAC/C,qBAAqB,CAAC,IAAI,CAAC,CAAC;YAC5B,MAAM,CAAC,GAAG,SAAS,CAAC,WAAW,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;YACvD,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAC9B,IACE,IAAI,CAAC,0BAA0B,IAAI,IAAI;gBACvC,CAAC,QAAQ,EAAE,kBAAkB,IAAI,CAAC,CAAC,KAAK,IAAI,CAAC,0BAA0B,EACvE,CAAC;gBACD,MAAM,IAAI,YAAY,CACpB,8BAA8B,EAC9B,0CAA0C,IAAI,CAAC,0BAA0B,SAAS,QAAQ,EAAE,kBAAkB,IAAI,CAAC,EAAE,EACrH,GAAG,CACJ,CAAC;YACJ,CAAC;YACD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YACrC,MAAM,GAAG,GAAqB;gBAC5B,KAAK;gBACL,WAAW;gBACX,UAAU;gBACV,QAAQ;gBACR,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,SAAS,EAAE,IAAI,CAAC,SAAS,IAAI,IAAI;gBACjC,QAAQ,EAAE,IAAI,CAAC,QAAQ,IAAI,IAAI;gBAC/B,YAAY,EAAE,IAAI,CAAC,YAAY,IAAI,IAAI;gBACvC,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,IAAI;gBAC3B,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,IAAI;gBACrC,aAAa,EAAE,IAAI,CAAC,aAAa,IAAI,IAAI;gBACzC,kBAAkB,EAAE,CAAC,QAAQ,EAAE,kBAAkB,IAAI,CAAC,CAAC,GAAG,CAAC;gBAC3D,SAAS,EAAE,GAAG;aACf,CAAC;YACF,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YAClB,OAAO,EAAE,GAAG,GAAG,EAAE,CAAC;QACpB,CAAC;QAED,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE,UAAU,EAAE,QAAQ;YAC3C,OAAO,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,WAAW,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC,CAAC;QACpE,CAAC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,gCAAgC,CAC9C,MAAmB,EACnB,KAAa;IAEb,MAAM,EAAE,GAAO,MAAM,CAAC,EAAE,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC;IACnD,MAAM,GAAG,GAAG,GAAG,EAAE,CACf,EAAE,CAAC,UAAU,CAAmB,0BAA0B,CAAC,WAAW,CAAC,CAAC;IAC1E,IAAI,YAAY,GAAyB,IAAI,CAAC;IAE9C,MAAM,aAAa,GAAG,KAAK,IAAI,EAAE;QAC/B,IAAI,CAAC,YAAY,EAAE,CAAC;YAClB,YAAY,GAAG,CAAC,KAAK,IAAI,EAAE;gBACzB,MAAM,GAAG,EAAE,CAAC,WAAW,CACrB,EAAE,WAAW,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,EAC9C,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,8BAA8B,EAAE,CACvD,CAAC;YACJ,CAAC,CAAC,EAAE,CAAC;QACP,CAAC;QACD,MAAM,YAAY,CAAC;IACrB,CAAC,CAAC;IAEF,OAAO;QACL,KAAK,CAAC,GAAG,CAAC,WAAW,EAAE,UAAU,EAAE,QAAQ;YACzC,MAAM,aAAa,EAAE,CAAC;YACtB,MAAM,GAAG,GAAG,MAAM,GAAG,EAAE,CAAC,OAAO,CAAC,EAAE,WAAW,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC,CAAC;YACvE,OAAO,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACtC,CAAC;QAED,KAAK,CAAC,GAAG,CAAC,WAAW,EAAE,UAAU,EAAE,QAAQ,EAAE,IAAI;YAC/C,MAAM,aAAa,EAAE,CAAC;YACtB,qBAAqB,CAAC,IAAI,CAAC,CAAC;YAC5B,MAAM,QAAQ,GAAG,MAAM,GAAG,EAAE,CAAC,OAAO,CAAC;gBACnC,WAAW;gBACX,UAAU;gBACV,QAAQ;aACT,CAAC,CAAC;YACH,IACE,IAAI,CAAC,0BAA0B,IAAI,IAAI;gBACvC,CAAC,QAAQ,EAAE,kBAAkB,IAAI,CAAC,CAAC,KAAK,IAAI,CAAC,0BAA0B,EACvE,CAAC;gBACD,MAAM,IAAI,YAAY,CACpB,8BAA8B,EAC9B,0CAA0C,IAAI,CAAC,0BAA0B,SAAS,QAAQ,EAAE,kBAAkB,IAAI,CAAC,EAAE,EACrH,GAAG,CACJ,CAAC;YACJ,CAAC;YACD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YACrC,MAAM,GAAG,GAAqB;gBAC5B,KAAK;gBACL,WAAW;gBACX,UAAU;gBACV,QAAQ;gBACR,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,SAAS,EAAE,IAAI,CAAC,SAAS,IAAI,IAAI;gBACjC,QAAQ,EAAE,IAAI,CAAC,QAAQ,IAAI,IAAI;gBAC/B,YAAY,EAAE,IAAI,CAAC,YAAY,IAAI,IAAI;gBACvC,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,IAAI;gBAC3B,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,IAAI;gBACrC,aAAa,EAAE,IAAI,CAAC,aAAa,IAAI,IAAI;gBACzC,kBAAkB,EAAE,CAAC,QAAQ,EAAE,kBAAkB,IAAI,CAAC,CAAC,GAAG,CAAC;gBAC3D,SAAS,EAAE,GAAG;aACf,CAAC;YACF,MAAM,GAAG,EAAE,CAAC,UAAU,CACpB,EAAE,WAAW,EAAE,UAAU,EAAE,QAAQ,EAAE,EACrC,GAAG,EACH,EAAE,MAAM,EAAE,IAAI,EAAE,CACjB,CAAC;YACF,OAAO,EAAE,GAAG,GAAG,EAAE,CAAC;QACpB,CAAC;QAED,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE,UAAU,EAAE,QAAQ;YAC3C,MAAM,aAAa,EAAE,CAAC;YACtB,MAAM,GAAG,GAAG,MAAM,GAAG,EAAE,CAAC,SAAS,CAAC,EAAE,WAAW,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC,CAAC;YACzE,OAAO,GAAG,CAAC,YAAY,GAAG,CAAC,CAAC;QAC9B,CAAC;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* INC-01 stream checkpoints — offsets/cursors for continuous acquisition.
|
|
3
|
+
* Distinct from Memory OT source checkpoints (sourceId × mode × workRef).
|
|
4
|
+
*/
|
|
5
|
+
export type StreamCheckpointMethod = "watermark" | "revision" | "fullScanHash" | "cursor";
|
|
6
|
+
export type StreamCheckpoint = {
|
|
7
|
+
orgId: string;
|
|
8
|
+
connectorId: string;
|
|
9
|
+
instanceId: string;
|
|
10
|
+
streamId: string;
|
|
11
|
+
method: StreamCheckpointMethod;
|
|
12
|
+
/** Splunk-style _indextime / time watermark. */
|
|
13
|
+
watermark?: string | number | null;
|
|
14
|
+
/** FortiGate revision / sequence. */
|
|
15
|
+
revision?: string | number | null;
|
|
16
|
+
/** Full-scan content hash gate. */
|
|
17
|
+
fullScanHash?: string | null;
|
|
18
|
+
/** Opaque cursor (Entra @odata.deltaLink, Falcon offset, …). */
|
|
19
|
+
cursor?: string | null;
|
|
20
|
+
contentHash?: string | null;
|
|
21
|
+
producerRunId?: string | null;
|
|
22
|
+
/** Checkpoint document revision for optional CAS on put. */
|
|
23
|
+
checkpointRevision: number;
|
|
24
|
+
updatedAt: string;
|
|
25
|
+
};
|
|
26
|
+
export type PutStreamCheckpoint = {
|
|
27
|
+
method: StreamCheckpointMethod;
|
|
28
|
+
watermark?: string | number | null;
|
|
29
|
+
revision?: string | number | null;
|
|
30
|
+
fullScanHash?: string | null;
|
|
31
|
+
cursor?: string | null;
|
|
32
|
+
contentHash?: string | null;
|
|
33
|
+
producerRunId?: string | null;
|
|
34
|
+
/** When set, put fails unless current checkpointRevision matches. */
|
|
35
|
+
expectedCheckpointRevision?: number;
|
|
36
|
+
};
|
|
37
|
+
export type StreamCheckpointStore = {
|
|
38
|
+
get(connectorId: string, instanceId: string, streamId: string): Promise<StreamCheckpoint | null>;
|
|
39
|
+
put(connectorId: string, instanceId: string, streamId: string, body: PutStreamCheckpoint): Promise<StreamCheckpoint>;
|
|
40
|
+
clear(connectorId: string, instanceId: string, streamId: string): Promise<boolean>;
|
|
41
|
+
};
|
|
42
|
+
export declare const STREAM_CHECKPOINT_METHODS: readonly StreamCheckpointMethod[];
|
|
43
|
+
export declare const MCO_COLLECTIONS_CHECKPOINT: {
|
|
44
|
+
readonly checkpoints: "mco_stream_checkpoints";
|
|
45
|
+
};
|
|
46
|
+
//# sourceMappingURL=checkpoint-types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"checkpoint-types.d.ts","sourceRoot":"","sources":["../../src/connector-ops/checkpoint-types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,MAAM,MAAM,sBAAsB,GAC9B,WAAW,GACX,UAAU,GACV,cAAc,GACd,QAAQ,CAAC;AAEb,MAAM,MAAM,gBAAgB,GAAG;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,sBAAsB,CAAC;IAC/B,gDAAgD;IAChD,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;IACnC,qCAAqC;IACrC,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;IAClC,mCAAmC;IACnC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,gEAAgE;IAChE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,4DAA4D;IAC5D,kBAAkB,EAAE,MAAM,CAAC;IAC3B,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAChC,MAAM,EAAE,sBAAsB,CAAC;IAC/B,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;IAClC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,qEAAqE;IACrE,0BAA0B,CAAC,EAAE,MAAM,CAAC;CACrC,CAAC;AAEF,MAAM,MAAM,qBAAqB,GAAG;IAClC,GAAG,CACD,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAAC;IACpC,GAAG,CACD,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,mBAAmB,GACxB,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAC7B,KAAK,CACH,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,OAAO,CAAC,CAAC;CACrB,CAAC;AAEF,eAAO,MAAM,yBAAyB,EAAE,SAAS,sBAAsB,EAK7D,CAAC;AAEX,eAAO,MAAM,0BAA0B;;CAE7B,CAAC"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* INC-01 stream checkpoints — offsets/cursors for continuous acquisition.
|
|
3
|
+
* Distinct from Memory OT source checkpoints (sourceId × mode × workRef).
|
|
4
|
+
*/
|
|
5
|
+
export const STREAM_CHECKPOINT_METHODS = [
|
|
6
|
+
"watermark",
|
|
7
|
+
"revision",
|
|
8
|
+
"fullScanHash",
|
|
9
|
+
"cursor",
|
|
10
|
+
];
|
|
11
|
+
export const MCO_COLLECTIONS_CHECKPOINT = {
|
|
12
|
+
checkpoints: "mco_stream_checkpoints",
|
|
13
|
+
};
|
|
14
|
+
//# sourceMappingURL=checkpoint-types.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"checkpoint-types.js","sourceRoot":"","sources":["../../src/connector-ops/checkpoint-types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AA4DH,MAAM,CAAC,MAAM,yBAAyB,GAAsC;IAC1E,WAAW;IACX,UAAU;IACV,cAAc;IACd,QAAQ;CACA,CAAC;AAEX,MAAM,CAAC,MAAM,0BAA0B,GAAG;IACxC,WAAW,EAAE,wBAAwB;CAC7B,CAAC"}
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* MX-CONT-01 continuous acquisition — schedule, lease, run ledger, freshness.
|
|
3
|
+
* Product-plane under connector-ops; not MGC.
|
|
4
|
+
*/
|
|
5
|
+
import type { StreamCheckpoint } from "./checkpoint-types.js";
|
|
6
|
+
export type AcquisitionCadence = {
|
|
7
|
+
kind: "interval";
|
|
8
|
+
intervalSeconds: number;
|
|
9
|
+
} | {
|
|
10
|
+
kind: "cron";
|
|
11
|
+
expression: string;
|
|
12
|
+
};
|
|
13
|
+
export type StreamSchedule = {
|
|
14
|
+
orgId: string;
|
|
15
|
+
connectorId: string;
|
|
16
|
+
instanceId: string;
|
|
17
|
+
streamId: string;
|
|
18
|
+
enabled: boolean;
|
|
19
|
+
cadence: AcquisitionCadence;
|
|
20
|
+
/** Soft bounds passed to bounded /pipeline invokes. */
|
|
21
|
+
bounds?: {
|
|
22
|
+
maxDurationMs?: number;
|
|
23
|
+
maxPages?: number;
|
|
24
|
+
maxEvents?: number;
|
|
25
|
+
};
|
|
26
|
+
updatedAt: string;
|
|
27
|
+
updatedBy?: string;
|
|
28
|
+
};
|
|
29
|
+
export type PutStreamSchedule = {
|
|
30
|
+
enabled?: boolean;
|
|
31
|
+
cadence: AcquisitionCadence;
|
|
32
|
+
bounds?: StreamSchedule["bounds"];
|
|
33
|
+
updatedBy?: string;
|
|
34
|
+
};
|
|
35
|
+
export type StreamLease = {
|
|
36
|
+
orgId: string;
|
|
37
|
+
connectorId: string;
|
|
38
|
+
instanceId: string;
|
|
39
|
+
streamId: string;
|
|
40
|
+
ownerId: string;
|
|
41
|
+
leaseId: string;
|
|
42
|
+
acquiredAt: string;
|
|
43
|
+
expiresAt: string;
|
|
44
|
+
renewCount: number;
|
|
45
|
+
};
|
|
46
|
+
export type AcquireLeaseInput = {
|
|
47
|
+
ownerId: string;
|
|
48
|
+
/** Lease TTL in ms (default 60_000). */
|
|
49
|
+
ttlMs?: number;
|
|
50
|
+
};
|
|
51
|
+
export type AcquisitionRunOutcome = "ok" | "fail" | "skip" | "canceled" | "lease_lost";
|
|
52
|
+
export type AcquisitionRunRecord = {
|
|
53
|
+
orgId: string;
|
|
54
|
+
connectorId: string;
|
|
55
|
+
instanceId: string;
|
|
56
|
+
streamId: string;
|
|
57
|
+
runId: string;
|
|
58
|
+
startedAt: string;
|
|
59
|
+
endedAt?: string | null;
|
|
60
|
+
outcome?: AcquisitionRunOutcome | null;
|
|
61
|
+
errorClass?: string | null;
|
|
62
|
+
errorMessage?: string | null;
|
|
63
|
+
checkpointBefore?: Partial<StreamCheckpoint> | null;
|
|
64
|
+
checkpointAfter?: Partial<StreamCheckpoint> | null;
|
|
65
|
+
bounds?: StreamSchedule["bounds"] | null;
|
|
66
|
+
ownerId?: string | null;
|
|
67
|
+
cancelRequested?: boolean;
|
|
68
|
+
};
|
|
69
|
+
export type StartAcquisitionRunInput = {
|
|
70
|
+
runId?: string;
|
|
71
|
+
ownerId?: string;
|
|
72
|
+
bounds?: StreamSchedule["bounds"];
|
|
73
|
+
checkpointBefore?: Partial<StreamCheckpoint> | null;
|
|
74
|
+
};
|
|
75
|
+
export type EndAcquisitionRunInput = {
|
|
76
|
+
outcome: AcquisitionRunOutcome;
|
|
77
|
+
errorClass?: string;
|
|
78
|
+
errorMessage?: string;
|
|
79
|
+
checkpointAfter?: Partial<StreamCheckpoint> | null;
|
|
80
|
+
};
|
|
81
|
+
export type StreamFreshness = {
|
|
82
|
+
orgId: string;
|
|
83
|
+
connectorId: string;
|
|
84
|
+
instanceId: string;
|
|
85
|
+
streamId: string;
|
|
86
|
+
scheduleEnabled: boolean;
|
|
87
|
+
lastSuccessAt?: string | null;
|
|
88
|
+
lastFailAt?: string | null;
|
|
89
|
+
lastRunAt?: string | null;
|
|
90
|
+
lastOutcome?: AcquisitionRunOutcome | null;
|
|
91
|
+
/** Seconds since last success; null if never succeeded. */
|
|
92
|
+
lagSeconds?: number | null;
|
|
93
|
+
/** True when lag exceeds 2× cadence interval (or 1h for cron). */
|
|
94
|
+
aging?: boolean;
|
|
95
|
+
/** True when no success within 3× cadence (or 3h for cron). */
|
|
96
|
+
lapsed?: boolean;
|
|
97
|
+
activeLeaseOwnerId?: string | null;
|
|
98
|
+
activeLeaseExpiresAt?: string | null;
|
|
99
|
+
cancelRequested?: boolean;
|
|
100
|
+
};
|
|
101
|
+
export type StreamScheduleStore = {
|
|
102
|
+
get(connectorId: string, instanceId: string, streamId: string): Promise<StreamSchedule | null>;
|
|
103
|
+
put(connectorId: string, instanceId: string, streamId: string, body: PutStreamSchedule): Promise<StreamSchedule>;
|
|
104
|
+
clear(connectorId: string, instanceId: string, streamId: string): Promise<boolean>;
|
|
105
|
+
listEnabled(opts?: {
|
|
106
|
+
connectorId?: string;
|
|
107
|
+
instanceId?: string;
|
|
108
|
+
}): Promise<StreamSchedule[]>;
|
|
109
|
+
};
|
|
110
|
+
export type StreamLeaseStore = {
|
|
111
|
+
get(connectorId: string, instanceId: string, streamId: string): Promise<StreamLease | null>;
|
|
112
|
+
acquire(connectorId: string, instanceId: string, streamId: string, input: AcquireLeaseInput): Promise<StreamLease>;
|
|
113
|
+
renew(connectorId: string, instanceId: string, streamId: string, input: {
|
|
114
|
+
ownerId: string;
|
|
115
|
+
leaseId: string;
|
|
116
|
+
ttlMs?: number;
|
|
117
|
+
}): Promise<StreamLease>;
|
|
118
|
+
release(connectorId: string, instanceId: string, streamId: string, input: {
|
|
119
|
+
ownerId: string;
|
|
120
|
+
leaseId: string;
|
|
121
|
+
}): Promise<boolean>;
|
|
122
|
+
};
|
|
123
|
+
export type AcquisitionRunStore = {
|
|
124
|
+
start(connectorId: string, instanceId: string, streamId: string, input: StartAcquisitionRunInput): Promise<AcquisitionRunRecord>;
|
|
125
|
+
end(connectorId: string, instanceId: string, streamId: string, runId: string, input: EndAcquisitionRunInput): Promise<AcquisitionRunRecord>;
|
|
126
|
+
get(connectorId: string, instanceId: string, streamId: string, runId: string): Promise<AcquisitionRunRecord | null>;
|
|
127
|
+
requestCancel(connectorId: string, instanceId: string, streamId: string, runId: string): Promise<AcquisitionRunRecord | null>;
|
|
128
|
+
list(connectorId: string, instanceId: string, streamId: string, opts?: {
|
|
129
|
+
limit?: number;
|
|
130
|
+
}): Promise<AcquisitionRunRecord[]>;
|
|
131
|
+
latestSuccess(connectorId: string, instanceId: string, streamId: string): Promise<AcquisitionRunRecord | null>;
|
|
132
|
+
};
|
|
133
|
+
export declare const CONT_COLLECTIONS: {
|
|
134
|
+
readonly schedules: "mco_stream_schedules";
|
|
135
|
+
readonly leases: "mco_stream_leases";
|
|
136
|
+
readonly runs: "mco_acquisition_runs";
|
|
137
|
+
};
|
|
138
|
+
export declare const DEFAULT_LEASE_TTL_MS = 60000;
|
|
139
|
+
export declare const ACQUISITION_RUNS_DEFAULT_LIMIT = 50;
|
|
140
|
+
export declare const ACQUISITION_RUNS_MAX_LIMIT = 200;
|
|
141
|
+
//# sourceMappingURL=continuous-types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"continuous-types.d.ts","sourceRoot":"","sources":["../../src/connector-ops/continuous-types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAE9D,MAAM,MAAM,kBAAkB,GAC1B;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,eAAe,EAAE,MAAM,CAAA;CAAE,GAC7C;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,CAAC;AAEzC,MAAM,MAAM,cAAc,GAAG;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,kBAAkB,CAAC;IAC5B,uDAAuD;IACvD,MAAM,CAAC,EAAE;QACP,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,OAAO,EAAE,kBAAkB,CAAC;IAC5B,MAAM,CAAC,EAAE,cAAc,CAAC,QAAQ,CAAC,CAAC;IAClC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,wCAAwC;IACxC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,qBAAqB,GAC7B,IAAI,GACJ,MAAM,GACN,MAAM,GACN,UAAU,GACV,YAAY,CAAC;AAEjB,MAAM,MAAM,oBAAoB,GAAG;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,OAAO,CAAC,EAAE,qBAAqB,GAAG,IAAI,CAAC;IACvC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,gBAAgB,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAAC;IACpD,eAAe,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAAC;IACnD,MAAM,CAAC,EAAE,cAAc,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC;IACzC,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B,CAAC;AAEF,MAAM,MAAM,wBAAwB,GAAG;IACrC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,cAAc,CAAC,QAAQ,CAAC,CAAC;IAClC,gBAAgB,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAAC;CACrD,CAAC;AAEF,MAAM,MAAM,sBAAsB,GAAG;IACnC,OAAO,EAAE,qBAAqB,CAAC;IAC/B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAAC;CACpD,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,eAAe,EAAE,OAAO,CAAC;IACzB,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,WAAW,CAAC,EAAE,qBAAqB,GAAG,IAAI,CAAC;IAC3C,2DAA2D;IAC3D,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,kEAAkE;IAClE,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,+DAA+D;IAC/D,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,kBAAkB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,oBAAoB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAChC,GAAG,CACD,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,CAAC;IAClC,GAAG,CACD,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,iBAAiB,GACtB,OAAO,CAAC,cAAc,CAAC,CAAC;IAC3B,KAAK,CACH,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,OAAO,CAAC,CAAC;IACpB,WAAW,CAAC,IAAI,CAAC,EAAE;QACjB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC;CAC/B,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,GAAG,CACD,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,CAAC;IAC/B,OAAO,CACL,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,iBAAiB,GACvB,OAAO,CAAC,WAAW,CAAC,CAAC;IACxB,KAAK,CACH,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAC1D,OAAO,CAAC,WAAW,CAAC,CAAC;IACxB,OAAO,CACL,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,GAC1C,OAAO,CAAC,OAAO,CAAC,CAAC;CACrB,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAChC,KAAK,CACH,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,wBAAwB,GAC9B,OAAO,CAAC,oBAAoB,CAAC,CAAC;IACjC,GAAG,CACD,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,sBAAsB,GAC5B,OAAO,CAAC,oBAAoB,CAAC,CAAC;IACjC,GAAG,CACD,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,oBAAoB,GAAG,IAAI,CAAC,CAAC;IACxC,aAAa,CACX,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,oBAAoB,GAAG,IAAI,CAAC,CAAC;IACxC,IAAI,CACF,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GACxB,OAAO,CAAC,oBAAoB,EAAE,CAAC,CAAC;IACnC,aAAa,CACX,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,oBAAoB,GAAG,IAAI,CAAC,CAAC;CACzC,CAAC;AAEF,eAAO,MAAM,gBAAgB;;;;CAInB,CAAC;AAEX,eAAO,MAAM,oBAAoB,QAAS,CAAC;AAC3C,eAAO,MAAM,8BAA8B,KAAK,CAAC;AACjD,eAAO,MAAM,0BAA0B,MAAM,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* MX-CONT-01 continuous acquisition — schedule, lease, run ledger, freshness.
|
|
3
|
+
* Product-plane under connector-ops; not MGC.
|
|
4
|
+
*/
|
|
5
|
+
export const CONT_COLLECTIONS = {
|
|
6
|
+
schedules: "mco_stream_schedules",
|
|
7
|
+
leases: "mco_stream_leases",
|
|
8
|
+
runs: "mco_acquisition_runs",
|
|
9
|
+
};
|
|
10
|
+
export const DEFAULT_LEASE_TTL_MS = 60_000;
|
|
11
|
+
export const ACQUISITION_RUNS_DEFAULT_LIMIT = 50;
|
|
12
|
+
export const ACQUISITION_RUNS_MAX_LIMIT = 200;
|
|
13
|
+
//# sourceMappingURL=continuous-types.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"continuous-types.js","sourceRoot":"","sources":["../../src/connector-ops/continuous-types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAsMH,MAAM,CAAC,MAAM,gBAAgB,GAAG;IAC9B,SAAS,EAAE,sBAAsB;IACjC,MAAM,EAAE,mBAAmB;IAC3B,IAAI,EAAE,sBAAsB;CACpB,CAAC;AAEX,MAAM,CAAC,MAAM,oBAAoB,GAAG,MAAM,CAAC;AAC3C,MAAM,CAAC,MAAM,8BAA8B,GAAG,EAAE,CAAC;AACjD,MAAM,CAAC,MAAM,0BAA0B,GAAG,GAAG,CAAC"}
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Jobs-ticked continuous acquisition coordinator (MX-CONT-01 host).
|
|
3
|
+
* Bounded pipeline invokes only — no always-on CF-A5 fleet daemon.
|
|
4
|
+
*/
|
|
5
|
+
import type { IntegrationInstanceStore } from "../integrations/types.js";
|
|
6
|
+
import type { McoOverrideStore } from "./types.js";
|
|
7
|
+
import type { StreamCheckpointStore } from "./checkpoint-types.js";
|
|
8
|
+
import type { AcquisitionRunStore, StreamLeaseStore, StreamSchedule, StreamScheduleStore } from "./continuous-types.js";
|
|
9
|
+
export type PipelineInvokeBounds = {
|
|
10
|
+
maxDurationMs?: number;
|
|
11
|
+
maxPages?: number;
|
|
12
|
+
maxEvents?: number;
|
|
13
|
+
};
|
|
14
|
+
export type BoundedPipelineInvokeResult = {
|
|
15
|
+
ok: boolean;
|
|
16
|
+
applyAck: boolean;
|
|
17
|
+
errorClass?: string;
|
|
18
|
+
errorMessage?: string;
|
|
19
|
+
/** Checkpoint payload to commit only when applyAck is true. */
|
|
20
|
+
nextCheckpoint?: {
|
|
21
|
+
method: "watermark" | "revision" | "fullScanHash" | "cursor";
|
|
22
|
+
watermark?: string | number | null;
|
|
23
|
+
revision?: string | number | null;
|
|
24
|
+
fullScanHash?: string | null;
|
|
25
|
+
cursor?: string | null;
|
|
26
|
+
contentHash?: string | null;
|
|
27
|
+
};
|
|
28
|
+
};
|
|
29
|
+
export type BoundedPipelineInvoker = (input: {
|
|
30
|
+
orgId: string;
|
|
31
|
+
connectorId: string;
|
|
32
|
+
instanceId: string;
|
|
33
|
+
streamId: string;
|
|
34
|
+
schedule: StreamSchedule;
|
|
35
|
+
bounds: PipelineInvokeBounds;
|
|
36
|
+
runId: string;
|
|
37
|
+
checkpoint: Awaited<ReturnType<StreamCheckpointStore["get"]>>;
|
|
38
|
+
abortSignal?: AbortSignal;
|
|
39
|
+
}) => Promise<BoundedPipelineInvokeResult>;
|
|
40
|
+
export type ContinuousAcquisitionTickDeps = {
|
|
41
|
+
orgId: string;
|
|
42
|
+
ownerId: string;
|
|
43
|
+
schedules: StreamScheduleStore;
|
|
44
|
+
leases: StreamLeaseStore;
|
|
45
|
+
runs: AcquisitionRunStore;
|
|
46
|
+
checkpoints: StreamCheckpointStore;
|
|
47
|
+
integrations: IntegrationInstanceStore;
|
|
48
|
+
overrides: McoOverrideStore;
|
|
49
|
+
invokePipeline: BoundedPipelineInvoker;
|
|
50
|
+
/** Optional op id used for MCO kill-switch (default continuous-acquire). */
|
|
51
|
+
mcoOpId?: string;
|
|
52
|
+
leaseTtlMs?: number;
|
|
53
|
+
};
|
|
54
|
+
export type ContinuousAcquisitionTickResult = {
|
|
55
|
+
orgId: string;
|
|
56
|
+
considered: number;
|
|
57
|
+
started: number;
|
|
58
|
+
skipped: number;
|
|
59
|
+
succeeded: number;
|
|
60
|
+
failed: number;
|
|
61
|
+
details: Array<{
|
|
62
|
+
connectorId: string;
|
|
63
|
+
instanceId: string;
|
|
64
|
+
streamId: string;
|
|
65
|
+
outcome: string;
|
|
66
|
+
reason?: string;
|
|
67
|
+
runId?: string;
|
|
68
|
+
}>;
|
|
69
|
+
};
|
|
70
|
+
/**
|
|
71
|
+
* Profile helpers for acceptance proofs (CF-A5 vs Splunk watermark window).
|
|
72
|
+
*/
|
|
73
|
+
export declare const COORDINATOR_PROFILES: {
|
|
74
|
+
/** CrowdStrike CF-A5-style: hard event/partition cap. */
|
|
75
|
+
readonly cfA5: {
|
|
76
|
+
maxEvents: number;
|
|
77
|
+
maxPages: number;
|
|
78
|
+
maxDurationMs: number;
|
|
79
|
+
};
|
|
80
|
+
/** Splunk-style watermark window (bounded lookback pages). */
|
|
81
|
+
readonly splunkWatermark: {
|
|
82
|
+
maxPages: number;
|
|
83
|
+
maxEvents: number;
|
|
84
|
+
maxDurationMs: number;
|
|
85
|
+
};
|
|
86
|
+
};
|
|
87
|
+
export declare function tickContinuousAcquisition(deps: ContinuousAcquisitionTickDeps): Promise<ContinuousAcquisitionTickResult>;
|
|
88
|
+
//# sourceMappingURL=coordinator.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"coordinator.d.ts","sourceRoot":"","sources":["../../src/connector-ops/coordinator.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,0BAA0B,CAAC;AACzE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AACnD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AACnE,OAAO,KAAK,EACV,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,mBAAmB,EACpB,MAAM,uBAAuB,CAAC;AAE/B,MAAM,MAAM,oBAAoB,GAAG;IACjC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,2BAA2B,GAAG;IACxC,EAAE,EAAE,OAAO,CAAC;IACZ,QAAQ,EAAE,OAAO,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,+DAA+D;IAC/D,cAAc,CAAC,EAAE;QACf,MAAM,EAAE,WAAW,GAAG,UAAU,GAAG,cAAc,GAAG,QAAQ,CAAC;QAC7D,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;QACnC,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;QAClC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAC7B,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QACvB,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;KAC7B,CAAC;CACH,CAAC;AAEF,MAAM,MAAM,sBAAsB,GAAG,CAAC,KAAK,EAAE;IAC3C,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,cAAc,CAAC;IACzB,MAAM,EAAE,oBAAoB,CAAC;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,OAAO,CAAC,UAAU,CAAC,qBAAqB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC9D,WAAW,CAAC,EAAE,WAAW,CAAC;CAC3B,KAAK,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAE3C,MAAM,MAAM,6BAA6B,GAAG;IAC1C,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,mBAAmB,CAAC;IAC/B,MAAM,EAAE,gBAAgB,CAAC;IACzB,IAAI,EAAE,mBAAmB,CAAC;IAC1B,WAAW,EAAE,qBAAqB,CAAC;IACnC,YAAY,EAAE,wBAAwB,CAAC;IACvC,SAAS,EAAE,gBAAgB,CAAC;IAC5B,cAAc,EAAE,sBAAsB,CAAC;IACvC,4EAA4E;IAC5E,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,CAAC;AAEF,MAAM,MAAM,+BAA+B,GAAG;IAC5C,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,KAAK,CAAC;QACb,WAAW,EAAE,MAAM,CAAC;QACpB,UAAU,EAAE,MAAM,CAAC;QACnB,QAAQ,EAAE,MAAM,CAAC;QACjB,OAAO,EAAE,MAAM,CAAC;QAChB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;CACJ,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,oBAAoB;IAC/B,yDAAyD;;;;;;IAMzD,8DAA8D;;;;;;CAMtD,CAAC;AAEX,wBAAsB,yBAAyB,CAC7C,IAAI,EAAE,6BAA6B,GAClC,OAAO,CAAC,+BAA+B,CAAC,CA4K1C"}
|