@x12i/memorix-service 3.4.0 → 3.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/app.d.ts.map +1 -1
- package/dist/app.js +26 -14
- package/dist/app.js.map +1 -1
- package/dist/connector-ops/checkpoint-store.d.ts +7 -0
- package/dist/connector-ops/checkpoint-store.d.ts.map +1 -0
- package/dist/connector-ops/checkpoint-store.js +135 -0
- package/dist/connector-ops/checkpoint-store.js.map +1 -0
- package/dist/connector-ops/checkpoint-types.d.ts +46 -0
- package/dist/connector-ops/checkpoint-types.d.ts.map +1 -0
- package/dist/connector-ops/checkpoint-types.js +14 -0
- package/dist/connector-ops/checkpoint-types.js.map +1 -0
- package/dist/connector-ops/continuous-types.d.ts +141 -0
- package/dist/connector-ops/continuous-types.d.ts.map +1 -0
- package/dist/connector-ops/continuous-types.js +13 -0
- package/dist/connector-ops/continuous-types.js.map +1 -0
- package/dist/connector-ops/coordinator.d.ts +88 -0
- package/dist/connector-ops/coordinator.d.ts.map +1 -0
- package/dist/connector-ops/coordinator.js +174 -0
- package/dist/connector-ops/coordinator.js.map +1 -0
- package/dist/connector-ops/index.d.ts +6 -0
- package/dist/connector-ops/index.d.ts.map +1 -1
- package/dist/connector-ops/index.js +6 -0
- package/dist/connector-ops/index.js.map +1 -1
- package/dist/connector-ops/run-store.d.ts +16 -0
- package/dist/connector-ops/run-store.d.ts.map +1 -0
- package/dist/connector-ops/run-store.js +249 -0
- package/dist/connector-ops/run-store.js.map +1 -0
- package/dist/connector-ops/schedule-lease-store.d.ts +8 -0
- package/dist/connector-ops/schedule-lease-store.d.ts.map +1 -0
- package/dist/connector-ops/schedule-lease-store.js +318 -0
- package/dist/connector-ops/schedule-lease-store.js.map +1 -0
- package/dist/incremental-state/index.d.ts +3 -0
- package/dist/incremental-state/index.d.ts.map +1 -0
- package/dist/incremental-state/index.js +3 -0
- package/dist/incremental-state/index.js.map +1 -0
- package/dist/incremental-state/state-store.d.ts +7 -0
- package/dist/incremental-state/state-store.d.ts.map +1 -0
- package/dist/incremental-state/state-store.js +183 -0
- package/dist/incremental-state/state-store.js.map +1 -0
- package/dist/incremental-state/types.d.ts +104 -0
- package/dist/incremental-state/types.d.ts.map +1 -0
- package/dist/incremental-state/types.js +8 -0
- package/dist/incremental-state/types.js.map +1 -0
- package/dist/magit/magit-pack-service.d.ts +18 -0
- package/dist/magit/magit-pack-service.d.ts.map +1 -1
- package/dist/magit/magit-pack-service.js +46 -1
- package/dist/magit/magit-pack-service.js.map +1 -1
- package/dist/openapi/openapi.json +144 -0
- package/dist/pipeline-compile.d.ts +6 -0
- package/dist/pipeline-compile.d.ts.map +1 -1
- package/dist/pipeline-compile.js +18 -0
- package/dist/pipeline-compile.js.map +1 -1
- package/dist/platform.d.ts +26 -1
- package/dist/platform.d.ts.map +1 -1
- package/dist/platform.js +111 -1
- package/dist/platform.js.map +1 -1
- package/dist/routes/continuous-acquisition.d.ts +5 -0
- package/dist/routes/continuous-acquisition.d.ts.map +1 -0
- package/dist/routes/continuous-acquisition.js +259 -0
- package/dist/routes/continuous-acquisition.js.map +1 -0
- package/dist/routes/incremental-state.d.ts +5 -0
- package/dist/routes/incremental-state.d.ts.map +1 -0
- package/dist/routes/incremental-state.js +173 -0
- package/dist/routes/incremental-state.js.map +1 -0
- package/dist/routes/metadata.d.ts.map +1 -1
- package/dist/routes/metadata.js +409 -406
- package/dist/routes/metadata.js.map +1 -1
- package/openapi/openapi.json +144 -0
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -7,7 +7,7 @@ Remote apps should **call** this service over HTTP — not reinvent wiring by in
|
|
|
7
7
|
## Install / run
|
|
8
8
|
|
|
9
9
|
```bash
|
|
10
|
-
npm install @x12i/memorix-service@3.
|
|
10
|
+
npm install @x12i/memorix-service@3.6.0 # composition library / CLI
|
|
11
11
|
```
|
|
12
12
|
|
|
13
13
|
Prefer the monorepo stack (service + Studio + opx-source):
|
package/dist/app.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"app.d.ts","sourceRoot":"","sources":["../src/app.ts"],"names":[],"mappings":"AAIA,OAAO,OAAO,EAAE,EAAE,KAAK,eAAe,EAAE,MAAM,SAAS,CAAC;AAIxD,OAAO,EAEL,KAAK,eAAe,EACpB,KAAK,YAAY,EAClB,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"app.d.ts","sourceRoot":"","sources":["../src/app.ts"],"names":[],"mappings":"AAIA,OAAO,OAAO,EAAE,EAAE,KAAK,eAAe,EAAE,MAAM,SAAS,CAAC;AAIxD,OAAO,EAEL,KAAK,eAAe,EACpB,KAAK,YAAY,EAClB,MAAM,eAAe,CAAC;AA4BvB,MAAM,MAAM,8BAA8B,GAAG,YAAY,GAAG;IAC1D,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,QAAQ,CAAC,EAAE,eAAe,CAAC;CAC5B,CAAC;AAEF,wBAAsB,uBAAuB,CAC3C,IAAI,EAAE,8BAA8B,GACnC,OAAO,CAAC;IAAE,GAAG,EAAE,eAAe,CAAC;IAAC,QAAQ,EAAE,eAAe,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,CAuG5E;AAED,MAAM,MAAM,0BAA0B,GAAG,8BAA8B,GAAG;IACxE,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,wBAAsB,uBAAuB,CAAC,IAAI,EAAE,0BAA0B;;;;;GAM7E"}
|
package/dist/app.js
CHANGED
|
@@ -15,8 +15,10 @@ import { registerRelationshipRoutes } from "./routes/relationships.js";
|
|
|
15
15
|
import { registerMemoryRoutes } from "./routes/memory.js";
|
|
16
16
|
import { registerConnectorInvocationRoutes } from "./routes/connector-invocations.js";
|
|
17
17
|
import { registerConnectorOpsRoutes } from "./routes/connector-ops.js";
|
|
18
|
+
import { registerContinuousAcquisitionRoutes } from "./routes/continuous-acquisition.js";
|
|
18
19
|
import { registerIntegrationRoutes } from "./routes/integrations.js";
|
|
19
20
|
import { registerRestrictedStoreRoutes } from "./routes/restricted-store.js";
|
|
21
|
+
import { registerIncrementalStateRoutes } from "./routes/incremental-state.js";
|
|
20
22
|
import { registerPipelineRoutes } from "./routes/pipelines.js";
|
|
21
23
|
import { registerJobsRoutes } from "./routes/jobs.js";
|
|
22
24
|
import { registerOperationsRoutes } from "./routes/operations.js";
|
|
@@ -35,6 +37,24 @@ export async function createMemorixServiceApp(opts) {
|
|
|
35
37
|
const simulation = createSimulationController();
|
|
36
38
|
const app = Fastify({ logger: opts.logger ?? false });
|
|
37
39
|
await app.register(cors, { origin: true });
|
|
40
|
+
const healthPayload = async () => {
|
|
41
|
+
const sim = simulation.getState();
|
|
42
|
+
return {
|
|
43
|
+
ok: true,
|
|
44
|
+
service: "@x12i/memorix-service",
|
|
45
|
+
mode: platform.mode,
|
|
46
|
+
"write-record": {
|
|
47
|
+
live: true,
|
|
48
|
+
mode: "nexum",
|
|
49
|
+
},
|
|
50
|
+
simulation: {
|
|
51
|
+
supported: sim.supported,
|
|
52
|
+
active: sim.active,
|
|
53
|
+
apsPackageSim: sim.apsPackageSim,
|
|
54
|
+
blockedOn: sim.blockedOn,
|
|
55
|
+
},
|
|
56
|
+
};
|
|
57
|
+
};
|
|
38
58
|
const core = await attachCoreFastify(app, {
|
|
39
59
|
ring: memorixLiveRing,
|
|
40
60
|
identity: {
|
|
@@ -47,20 +67,7 @@ export async function createMemorixServiceApp(opts) {
|
|
|
47
67
|
docsPath: "/api/openapi.json",
|
|
48
68
|
description: "Sole HTTP composition root",
|
|
49
69
|
},
|
|
50
|
-
getHealth:
|
|
51
|
-
const sim = simulation.getState();
|
|
52
|
-
return {
|
|
53
|
-
ok: true,
|
|
54
|
-
service: "@x12i/memorix-service",
|
|
55
|
-
mode: platform.mode,
|
|
56
|
-
simulation: {
|
|
57
|
-
supported: sim.supported,
|
|
58
|
-
active: sim.active,
|
|
59
|
-
apsPackageSim: sim.apsPackageSim,
|
|
60
|
-
blockedOn: sim.blockedOn,
|
|
61
|
-
},
|
|
62
|
-
};
|
|
63
|
-
},
|
|
70
|
+
getHealth: healthPayload,
|
|
64
71
|
});
|
|
65
72
|
app.setErrorHandler((err, req, reply) => {
|
|
66
73
|
const correlationId = req.correlationId ?? randomUUID();
|
|
@@ -68,6 +75,7 @@ export async function createMemorixServiceApp(opts) {
|
|
|
68
75
|
mapped.body.error = redactSecrets(mapped.body.error);
|
|
69
76
|
return reply.code(mapped.status).send(mapped.body);
|
|
70
77
|
});
|
|
78
|
+
app.get("/", async () => healthPayload());
|
|
71
79
|
app.get("/api/openapi.json", async (_req, reply) => {
|
|
72
80
|
const path = resolve(__dirname, "../openapi/openapi.json");
|
|
73
81
|
const body = readFileSync(path, "utf8");
|
|
@@ -98,8 +106,10 @@ export async function createMemorixServiceApp(opts) {
|
|
|
98
106
|
await registerMemoryRoutes(app, platform);
|
|
99
107
|
await registerConnectorInvocationRoutes(app, platform);
|
|
100
108
|
await registerConnectorOpsRoutes(app, platform);
|
|
109
|
+
await registerContinuousAcquisitionRoutes(app, platform);
|
|
101
110
|
await registerIntegrationRoutes(app, platform);
|
|
102
111
|
await registerRestrictedStoreRoutes(app, platform);
|
|
112
|
+
await registerIncrementalStateRoutes(app, platform);
|
|
103
113
|
await registerPipelineRoutes(app, platform);
|
|
104
114
|
await registerJobsRoutes(app, platform);
|
|
105
115
|
await registerOperationsRoutes(app, platform);
|
|
@@ -112,6 +122,8 @@ export async function createMemorixServiceApp(opts) {
|
|
|
112
122
|
platform.startMemoryScheduleTicker({ intervalMs: 60_000 });
|
|
113
123
|
// Jobs continuous Work evaluation + held-run promotion.
|
|
114
124
|
platform.startJobsWorkTicker({ intervalMs: 15_000 });
|
|
125
|
+
// Continuous acquisition coordinator (schedule → lease → bounded pipeline).
|
|
126
|
+
platform.startContinuousAcquisitionTicker({ intervalMs: 30_000 });
|
|
115
127
|
return { app, platform, port: core.port };
|
|
116
128
|
}
|
|
117
129
|
export async function startMemorixServiceHttp(opts) {
|
package/dist/app.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"app.js","sourceRoot":"","sources":["../src/app.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,OAAiC,MAAM,SAAS,CAAC;AACxD,OAAO,IAAI,MAAM,eAAe,CAAC;AACjC,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAC;AAC/D,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5D,OAAO,EACL,qBAAqB,GAGtB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAC9D,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAC5D,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAE,0BAA0B,EAAE,MAAM,2BAA2B,CAAC;AACvE,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,EAAE,iCAAiC,EAAE,MAAM,mCAAmC,CAAC;AACtF,OAAO,EAAE,0BAA0B,EAAE,MAAM,2BAA2B,CAAC;AACvE,OAAO,EAAE,yBAAyB,EAAE,MAAM,0BAA0B,CAAC;AACrE,OAAO,EAAE,6BAA6B,EAAE,MAAM,8BAA8B,CAAC;AAC7E,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAE,wBAAwB,EAAE,MAAM,wBAAwB,CAAC;AAClE,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAC9D,OAAO,EAAE,0BAA0B,EAAE,MAAM,0BAA0B,CAAC;AACtE,OAAO,EAAE,yBAAyB,EAAE,MAAM,0BAA0B,CAAC;AACrE,OAAO,EAAE,wBAAwB,EAAE,MAAM,wBAAwB,CAAC;AAClE,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,EAAE,wBAAwB,EAAE,MAAM,yBAAyB,CAAC;AACnE,OAAO,EAAE,0BAA0B,EAAE,MAAM,iBAAiB,CAAC;AAC7D,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,EAAE,iCAAiC,EAAE,MAAM,yBAAyB,CAAC;AAE5E,MAAM,SAAS,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAO1D,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,IAAoC;IAEpC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,CAAC,MAAM,qBAAqB,CAAC,IAAI,CAAC,CAAC,CAAC;IACtE,MAAM,UAAU,GAAG,0BAA0B,EAAE,CAAC;IAChD,MAAM,GAAG,GAAG,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,KAAK,EAAE,CAAC,CAAC;IACtD,MAAM,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;IAE3C,MAAM,
|
|
1
|
+
{"version":3,"file":"app.js","sourceRoot":"","sources":["../src/app.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,OAAiC,MAAM,SAAS,CAAC;AACxD,OAAO,IAAI,MAAM,eAAe,CAAC;AACjC,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAC;AAC/D,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5D,OAAO,EACL,qBAAqB,GAGtB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAC9D,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAC5D,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAE,0BAA0B,EAAE,MAAM,2BAA2B,CAAC;AACvE,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,EAAE,iCAAiC,EAAE,MAAM,mCAAmC,CAAC;AACtF,OAAO,EAAE,0BAA0B,EAAE,MAAM,2BAA2B,CAAC;AACvE,OAAO,EAAE,mCAAmC,EAAE,MAAM,oCAAoC,CAAC;AACzF,OAAO,EAAE,yBAAyB,EAAE,MAAM,0BAA0B,CAAC;AACrE,OAAO,EAAE,6BAA6B,EAAE,MAAM,8BAA8B,CAAC;AAC7E,OAAO,EAAE,8BAA8B,EAAE,MAAM,+BAA+B,CAAC;AAC/E,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAE,wBAAwB,EAAE,MAAM,wBAAwB,CAAC;AAClE,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAC9D,OAAO,EAAE,0BAA0B,EAAE,MAAM,0BAA0B,CAAC;AACtE,OAAO,EAAE,yBAAyB,EAAE,MAAM,0BAA0B,CAAC;AACrE,OAAO,EAAE,wBAAwB,EAAE,MAAM,wBAAwB,CAAC;AAClE,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,EAAE,wBAAwB,EAAE,MAAM,yBAAyB,CAAC;AACnE,OAAO,EAAE,0BAA0B,EAAE,MAAM,iBAAiB,CAAC;AAC7D,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,EAAE,iCAAiC,EAAE,MAAM,yBAAyB,CAAC;AAE5E,MAAM,SAAS,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAO1D,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,IAAoC;IAEpC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,CAAC,MAAM,qBAAqB,CAAC,IAAI,CAAC,CAAC,CAAC;IACtE,MAAM,UAAU,GAAG,0BAA0B,EAAE,CAAC;IAChD,MAAM,GAAG,GAAG,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,KAAK,EAAE,CAAC,CAAC;IACtD,MAAM,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;IAE3C,MAAM,aAAa,GAAG,KAAK,IAAI,EAAE;QAC/B,MAAM,GAAG,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC;QAClC,OAAO;YACL,EAAE,EAAE,IAAI;YACR,OAAO,EAAE,uBAAuB;YAChC,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,cAAc,EAAE;gBACd,IAAI,EAAE,IAAI;gBACV,IAAI,EAAE,OAAO;aACd;YACD,UAAU,EAAE;gBACV,SAAS,EAAE,GAAG,CAAC,SAAS;gBACxB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,aAAa,EAAE,GAAG,CAAC,aAAa;gBAChC,SAAS,EAAE,GAAG,CAAC,SAAS;aACzB;SACF,CAAC;IACJ,CAAC,CAAC;IAEF,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAC,GAAY,EAAE;QACjD,IAAI,EAAE,eAAe;QACrB,QAAQ,EAAE;YACR,EAAE,EAAE,uBAAuB;YAC3B,KAAK,EAAE,iBAAiB;YACxB,IAAI,EAAE,KAAK;YACX,MAAM,EAAE,SAAS;YACjB,OAAO,EAAE,SAAS;YAClB,OAAO,EAAE,sBAAsB;YAC/B,QAAQ,EAAE,mBAAmB;YAC7B,WAAW,EAAE,4BAA4B;SAC1C;QACD,SAAS,EAAE,aAAa;KACzB,CAAC,CAAC;IAEH,GAAG,CAAC,eAAe,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE;QACtC,MAAM,aAAa,GAChB,GAAkC,CAAC,aAAa,IAAI,UAAU,EAAE,CAAC;QACpE,MAAM,MAAM,GAAG,cAAc,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;QAClD,MAAM,CAAC,IAAI,CAAC,KAAK,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAA6B,CAAC;QACjF,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;IAEH,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,IAAI,EAAE,CAAC,aAAa,EAAE,CAAC,CAAC;IAE1C,GAAG,CAAC,GAAG,CAAC,mBAAmB,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;QACjD,MAAM,IAAI,GAAG,OAAO,CAAC,SAAS,EAAE,yBAAyB,CAAC,CAAC;QAC3D,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACxC,OAAO,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;IAC/D,CAAC,CAAC,CAAC;IAEH,GAAG,CAAC,GAAG,CAAC,wBAAwB,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QAC9C,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;QAChC,MAAM,IAAI,GAAG,QAAQ,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QACzC,OAAO;YACL,KAAK;YACL,QAAQ,EAAE;gBACR,GAAG,IAAI;gBACP,UAAU,EAAE,iCAAiC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBAC1D,aAAa,EAAE,CAAC,CAAC,aAAa;oBAC9B,GAAG,EAAE,CAAC,CAAC,GAAG;oBACV,GAAG,EAAE,CAAC,CAAC,GAAG;oBACV,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;oBAC1B,WAAW,EAAE,CAAC,CAAC,WAAW;iBAC3B,CAAC,CAAC;aACJ;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,wBAAwB,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;IAC1C,MAAM,sBAAsB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAC5C,MAAM,qBAAqB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAC3C,MAAM,kBAAkB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IACxC,MAAM,0BAA0B,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAChD,MAAM,oBAAoB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAC1C,MAAM,iCAAiC,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IACvD,MAAM,0BAA0B,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAChD,MAAM,mCAAmC,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IACzD,MAAM,yBAAyB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAC/C,MAAM,6BAA6B,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IACnD,MAAM,8BAA8B,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IACpD,MAAM,sBAAsB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAC5C,MAAM,kBAAkB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IACxC,MAAM,wBAAwB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAC9C,MAAM,sBAAsB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAC5C,MAAM,0BAA0B,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAChD,MAAM,yBAAyB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAC/C,MAAM,oBAAoB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAC1C,MAAM,wBAAwB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAE9C,iFAAiF;IACjF,QAAQ,CAAC,yBAAyB,CAAC,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,CAAC;IAC3D,wDAAwD;IACxD,QAAQ,CAAC,mBAAmB,CAAC,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,CAAC;IACrD,4EAA4E;IAC5E,QAAQ,CAAC,gCAAgC,CAAC,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,CAAC;IAElE,OAAO,EAAE,GAAG,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC;AAC5C,CAAC;AAOD,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,IAAgC;IAC5E,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,MAAM,uBAAuB,CAAC,IAAI,CAAC,CAAC;IAC9E,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,QAAQ,CAAC;IACnC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,WAAW,CAAC;IACtC,MAAM,GAAG,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IACjC,OAAO,EAAE,GAAG,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;AACvC,CAAC"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { MongoClient } from "mongodb";
|
|
2
|
+
import { type PutStreamCheckpoint, type StreamCheckpointMethod, type StreamCheckpointStore } from "./checkpoint-types.js";
|
|
3
|
+
export declare function assertCheckpointMethod(raw: unknown): StreamCheckpointMethod;
|
|
4
|
+
export declare function validatePutCheckpoint(body: PutStreamCheckpoint): void;
|
|
5
|
+
export declare function createInMemoryStreamCheckpointStore(orgId: string): StreamCheckpointStore;
|
|
6
|
+
export declare function createMongoStreamCheckpointStore(client: MongoClient, orgId: string): StreamCheckpointStore;
|
|
7
|
+
//# sourceMappingURL=checkpoint-store.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"checkpoint-store.d.ts","sourceRoot":"","sources":["../../src/connector-ops/checkpoint-store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAM,WAAW,EAAE,MAAM,SAAS,CAAC;AAG/C,OAAO,EAGL,KAAK,mBAAmB,EAExB,KAAK,sBAAsB,EAC3B,KAAK,qBAAqB,EAC3B,MAAM,uBAAuB,CAAC;AAW/B,wBAAgB,sBAAsB,CACpC,GAAG,EAAE,OAAO,GACX,sBAAsB,CAUxB;AAED,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,mBAAmB,GAAG,IAAI,CAoBrE;AAUD,wBAAgB,mCAAmC,CACjD,KAAK,EAAE,MAAM,GACZ,qBAAqB,CA+CvB;AAED,wBAAgB,gCAAgC,CAC9C,MAAM,EAAE,WAAW,EACnB,KAAK,EAAE,MAAM,GACZ,qBAAqB,CAyEvB"}
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
import { ServiceError } from "../errors.js";
|
|
2
|
+
import { hasSecretPayload } from "./sanitize.js";
|
|
3
|
+
import { MCO_COLLECTIONS_CHECKPOINT, STREAM_CHECKPOINT_METHODS, } from "./checkpoint-types.js";
|
|
4
|
+
function operationalDbName(orgId) {
|
|
5
|
+
return `${orgId}_operational`;
|
|
6
|
+
}
|
|
7
|
+
function stripMongo(doc) {
|
|
8
|
+
const { _id: _drop, ...rest } = doc;
|
|
9
|
+
return rest;
|
|
10
|
+
}
|
|
11
|
+
export function assertCheckpointMethod(raw) {
|
|
12
|
+
if (typeof raw === "string" &&
|
|
13
|
+
STREAM_CHECKPOINT_METHODS.includes(raw)) {
|
|
14
|
+
return raw;
|
|
15
|
+
}
|
|
16
|
+
throw new Error(`method must be one of: ${STREAM_CHECKPOINT_METHODS.join(", ")}`);
|
|
17
|
+
}
|
|
18
|
+
export function validatePutCheckpoint(body) {
|
|
19
|
+
assertCheckpointMethod(body.method);
|
|
20
|
+
if (hasSecretPayload(body)) {
|
|
21
|
+
throw new Error("checkpoint payload must not contain secrets or response bodies");
|
|
22
|
+
}
|
|
23
|
+
const method = body.method;
|
|
24
|
+
if (method === "watermark" && body.watermark == null) {
|
|
25
|
+
throw new Error("watermark required when method is watermark");
|
|
26
|
+
}
|
|
27
|
+
if (method === "revision" && body.revision == null) {
|
|
28
|
+
throw new Error("revision required when method is revision");
|
|
29
|
+
}
|
|
30
|
+
if (method === "fullScanHash" && body.fullScanHash == null) {
|
|
31
|
+
throw new Error("fullScanHash required when method is fullScanHash");
|
|
32
|
+
}
|
|
33
|
+
if (method === "cursor" && (body.cursor == null || body.cursor === "")) {
|
|
34
|
+
throw new Error("cursor required when method is cursor");
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
function streamKey(connectorId, instanceId, streamId) {
|
|
38
|
+
return `${connectorId}\0${instanceId}\0${streamId}`;
|
|
39
|
+
}
|
|
40
|
+
export function createInMemoryStreamCheckpointStore(orgId) {
|
|
41
|
+
const byKey = new Map();
|
|
42
|
+
return {
|
|
43
|
+
async get(connectorId, instanceId, streamId) {
|
|
44
|
+
const doc = byKey.get(streamKey(connectorId, instanceId, streamId));
|
|
45
|
+
return doc ? { ...doc } : null;
|
|
46
|
+
},
|
|
47
|
+
async put(connectorId, instanceId, streamId, body) {
|
|
48
|
+
validatePutCheckpoint(body);
|
|
49
|
+
const k = streamKey(connectorId, instanceId, streamId);
|
|
50
|
+
const existing = byKey.get(k);
|
|
51
|
+
if (body.expectedCheckpointRevision != null &&
|
|
52
|
+
(existing?.checkpointRevision ?? 0) !== body.expectedCheckpointRevision) {
|
|
53
|
+
throw new ServiceError("CHECKPOINT_REVISION_CONFLICT", `checkpoint revision conflict: expected ${body.expectedCheckpointRevision}, got ${existing?.checkpointRevision ?? 0}`, 409);
|
|
54
|
+
}
|
|
55
|
+
const now = new Date().toISOString();
|
|
56
|
+
const doc = {
|
|
57
|
+
orgId,
|
|
58
|
+
connectorId,
|
|
59
|
+
instanceId,
|
|
60
|
+
streamId,
|
|
61
|
+
method: body.method,
|
|
62
|
+
watermark: body.watermark ?? null,
|
|
63
|
+
revision: body.revision ?? null,
|
|
64
|
+
fullScanHash: body.fullScanHash ?? null,
|
|
65
|
+
cursor: body.cursor ?? null,
|
|
66
|
+
contentHash: body.contentHash ?? null,
|
|
67
|
+
producerRunId: body.producerRunId ?? null,
|
|
68
|
+
checkpointRevision: (existing?.checkpointRevision ?? 0) + 1,
|
|
69
|
+
updatedAt: now,
|
|
70
|
+
};
|
|
71
|
+
byKey.set(k, doc);
|
|
72
|
+
return { ...doc };
|
|
73
|
+
},
|
|
74
|
+
async clear(connectorId, instanceId, streamId) {
|
|
75
|
+
return byKey.delete(streamKey(connectorId, instanceId, streamId));
|
|
76
|
+
},
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
export function createMongoStreamCheckpointStore(client, orgId) {
|
|
80
|
+
const db = client.db(operationalDbName(orgId));
|
|
81
|
+
const col = () => db.collection(MCO_COLLECTIONS_CHECKPOINT.checkpoints);
|
|
82
|
+
let indexesReady = null;
|
|
83
|
+
const ensureIndexes = async () => {
|
|
84
|
+
if (!indexesReady) {
|
|
85
|
+
indexesReady = (async () => {
|
|
86
|
+
await col().createIndex({ connectorId: 1, instanceId: 1, streamId: 1 }, { unique: true, name: "mco_stream_checkpoint_unique" });
|
|
87
|
+
})();
|
|
88
|
+
}
|
|
89
|
+
await indexesReady;
|
|
90
|
+
};
|
|
91
|
+
return {
|
|
92
|
+
async get(connectorId, instanceId, streamId) {
|
|
93
|
+
await ensureIndexes();
|
|
94
|
+
const doc = await col().findOne({ connectorId, instanceId, streamId });
|
|
95
|
+
return doc ? stripMongo(doc) : null;
|
|
96
|
+
},
|
|
97
|
+
async put(connectorId, instanceId, streamId, body) {
|
|
98
|
+
await ensureIndexes();
|
|
99
|
+
validatePutCheckpoint(body);
|
|
100
|
+
const existing = await col().findOne({
|
|
101
|
+
connectorId,
|
|
102
|
+
instanceId,
|
|
103
|
+
streamId,
|
|
104
|
+
});
|
|
105
|
+
if (body.expectedCheckpointRevision != null &&
|
|
106
|
+
(existing?.checkpointRevision ?? 0) !== body.expectedCheckpointRevision) {
|
|
107
|
+
throw new ServiceError("CHECKPOINT_REVISION_CONFLICT", `checkpoint revision conflict: expected ${body.expectedCheckpointRevision}, got ${existing?.checkpointRevision ?? 0}`, 409);
|
|
108
|
+
}
|
|
109
|
+
const now = new Date().toISOString();
|
|
110
|
+
const doc = {
|
|
111
|
+
orgId,
|
|
112
|
+
connectorId,
|
|
113
|
+
instanceId,
|
|
114
|
+
streamId,
|
|
115
|
+
method: body.method,
|
|
116
|
+
watermark: body.watermark ?? null,
|
|
117
|
+
revision: body.revision ?? null,
|
|
118
|
+
fullScanHash: body.fullScanHash ?? null,
|
|
119
|
+
cursor: body.cursor ?? null,
|
|
120
|
+
contentHash: body.contentHash ?? null,
|
|
121
|
+
producerRunId: body.producerRunId ?? null,
|
|
122
|
+
checkpointRevision: (existing?.checkpointRevision ?? 0) + 1,
|
|
123
|
+
updatedAt: now,
|
|
124
|
+
};
|
|
125
|
+
await col().replaceOne({ connectorId, instanceId, streamId }, doc, { upsert: true });
|
|
126
|
+
return { ...doc };
|
|
127
|
+
},
|
|
128
|
+
async clear(connectorId, instanceId, streamId) {
|
|
129
|
+
await ensureIndexes();
|
|
130
|
+
const res = await col().deleteOne({ connectorId, instanceId, streamId });
|
|
131
|
+
return res.deletedCount > 0;
|
|
132
|
+
},
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
//# sourceMappingURL=checkpoint-store.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"checkpoint-store.js","sourceRoot":"","sources":["../../src/connector-ops/checkpoint-store.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,EACL,0BAA0B,EAC1B,yBAAyB,GAK1B,MAAM,uBAAuB,CAAC;AAE/B,SAAS,iBAAiB,CAAC,KAAa;IACtC,OAAO,GAAG,KAAK,cAAc,CAAC;AAChC,CAAC;AAED,SAAS,UAAU,CAAmB,GAA0B;IAC9D,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,IAAI,EAAE,GAAG,GAA4B,CAAC;IAC7D,OAAO,IAAS,CAAC;AACnB,CAAC;AAED,MAAM,UAAU,sBAAsB,CACpC,GAAY;IAEZ,IACE,OAAO,GAAG,KAAK,QAAQ;QACtB,yBAA+C,CAAC,QAAQ,CAAC,GAAG,CAAC,EAC9D,CAAC;QACD,OAAO,GAA6B,CAAC;IACvC,CAAC;IACD,MAAM,IAAI,KAAK,CACb,0BAA0B,yBAAyB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACjE,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,IAAyB;IAC7D,sBAAsB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACpC,IAAI,gBAAgB,CAAC,IAAI,CAAC,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CACb,gEAAgE,CACjE,CAAC;IACJ,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;IAC3B,IAAI,MAAM,KAAK,WAAW,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,EAAE,CAAC;QACrD,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;IACjE,CAAC;IACD,IAAI,MAAM,KAAK,UAAU,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,EAAE,CAAC;QACnD,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;IAC/D,CAAC;IACD,IAAI,MAAM,KAAK,cAAc,IAAI,IAAI,CAAC,YAAY,IAAI,IAAI,EAAE,CAAC;QAC3D,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;IACvE,CAAC;IACD,IAAI,MAAM,KAAK,QAAQ,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,EAAE,CAAC,EAAE,CAAC;QACvE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAC3D,CAAC;AACH,CAAC;AAED,SAAS,SAAS,CAChB,WAAmB,EACnB,UAAkB,EAClB,QAAgB;IAEhB,OAAO,GAAG,WAAW,KAAK,UAAU,KAAK,QAAQ,EAAE,CAAC;AACtD,CAAC;AAED,MAAM,UAAU,mCAAmC,CACjD,KAAa;IAEb,MAAM,KAAK,GAAG,IAAI,GAAG,EAA4B,CAAC;IAElD,OAAO;QACL,KAAK,CAAC,GAAG,CAAC,WAAW,EAAE,UAAU,EAAE,QAAQ;YACzC,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,WAAW,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC,CAAC;YACpE,OAAO,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QACjC,CAAC;QAED,KAAK,CAAC,GAAG,CAAC,WAAW,EAAE,UAAU,EAAE,QAAQ,EAAE,IAAI;YAC/C,qBAAqB,CAAC,IAAI,CAAC,CAAC;YAC5B,MAAM,CAAC,GAAG,SAAS,CAAC,WAAW,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;YACvD,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAC9B,IACE,IAAI,CAAC,0BAA0B,IAAI,IAAI;gBACvC,CAAC,QAAQ,EAAE,kBAAkB,IAAI,CAAC,CAAC,KAAK,IAAI,CAAC,0BAA0B,EACvE,CAAC;gBACD,MAAM,IAAI,YAAY,CACpB,8BAA8B,EAC9B,0CAA0C,IAAI,CAAC,0BAA0B,SAAS,QAAQ,EAAE,kBAAkB,IAAI,CAAC,EAAE,EACrH,GAAG,CACJ,CAAC;YACJ,CAAC;YACD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YACrC,MAAM,GAAG,GAAqB;gBAC5B,KAAK;gBACL,WAAW;gBACX,UAAU;gBACV,QAAQ;gBACR,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,SAAS,EAAE,IAAI,CAAC,SAAS,IAAI,IAAI;gBACjC,QAAQ,EAAE,IAAI,CAAC,QAAQ,IAAI,IAAI;gBAC/B,YAAY,EAAE,IAAI,CAAC,YAAY,IAAI,IAAI;gBACvC,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,IAAI;gBAC3B,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,IAAI;gBACrC,aAAa,EAAE,IAAI,CAAC,aAAa,IAAI,IAAI;gBACzC,kBAAkB,EAAE,CAAC,QAAQ,EAAE,kBAAkB,IAAI,CAAC,CAAC,GAAG,CAAC;gBAC3D,SAAS,EAAE,GAAG;aACf,CAAC;YACF,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YAClB,OAAO,EAAE,GAAG,GAAG,EAAE,CAAC;QACpB,CAAC;QAED,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE,UAAU,EAAE,QAAQ;YAC3C,OAAO,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,WAAW,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC,CAAC;QACpE,CAAC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,gCAAgC,CAC9C,MAAmB,EACnB,KAAa;IAEb,MAAM,EAAE,GAAO,MAAM,CAAC,EAAE,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC;IACnD,MAAM,GAAG,GAAG,GAAG,EAAE,CACf,EAAE,CAAC,UAAU,CAAmB,0BAA0B,CAAC,WAAW,CAAC,CAAC;IAC1E,IAAI,YAAY,GAAyB,IAAI,CAAC;IAE9C,MAAM,aAAa,GAAG,KAAK,IAAI,EAAE;QAC/B,IAAI,CAAC,YAAY,EAAE,CAAC;YAClB,YAAY,GAAG,CAAC,KAAK,IAAI,EAAE;gBACzB,MAAM,GAAG,EAAE,CAAC,WAAW,CACrB,EAAE,WAAW,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,EAC9C,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,8BAA8B,EAAE,CACvD,CAAC;YACJ,CAAC,CAAC,EAAE,CAAC;QACP,CAAC;QACD,MAAM,YAAY,CAAC;IACrB,CAAC,CAAC;IAEF,OAAO;QACL,KAAK,CAAC,GAAG,CAAC,WAAW,EAAE,UAAU,EAAE,QAAQ;YACzC,MAAM,aAAa,EAAE,CAAC;YACtB,MAAM,GAAG,GAAG,MAAM,GAAG,EAAE,CAAC,OAAO,CAAC,EAAE,WAAW,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC,CAAC;YACvE,OAAO,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACtC,CAAC;QAED,KAAK,CAAC,GAAG,CAAC,WAAW,EAAE,UAAU,EAAE,QAAQ,EAAE,IAAI;YAC/C,MAAM,aAAa,EAAE,CAAC;YACtB,qBAAqB,CAAC,IAAI,CAAC,CAAC;YAC5B,MAAM,QAAQ,GAAG,MAAM,GAAG,EAAE,CAAC,OAAO,CAAC;gBACnC,WAAW;gBACX,UAAU;gBACV,QAAQ;aACT,CAAC,CAAC;YACH,IACE,IAAI,CAAC,0BAA0B,IAAI,IAAI;gBACvC,CAAC,QAAQ,EAAE,kBAAkB,IAAI,CAAC,CAAC,KAAK,IAAI,CAAC,0BAA0B,EACvE,CAAC;gBACD,MAAM,IAAI,YAAY,CACpB,8BAA8B,EAC9B,0CAA0C,IAAI,CAAC,0BAA0B,SAAS,QAAQ,EAAE,kBAAkB,IAAI,CAAC,EAAE,EACrH,GAAG,CACJ,CAAC;YACJ,CAAC;YACD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YACrC,MAAM,GAAG,GAAqB;gBAC5B,KAAK;gBACL,WAAW;gBACX,UAAU;gBACV,QAAQ;gBACR,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,SAAS,EAAE,IAAI,CAAC,SAAS,IAAI,IAAI;gBACjC,QAAQ,EAAE,IAAI,CAAC,QAAQ,IAAI,IAAI;gBAC/B,YAAY,EAAE,IAAI,CAAC,YAAY,IAAI,IAAI;gBACvC,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,IAAI;gBAC3B,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,IAAI;gBACrC,aAAa,EAAE,IAAI,CAAC,aAAa,IAAI,IAAI;gBACzC,kBAAkB,EAAE,CAAC,QAAQ,EAAE,kBAAkB,IAAI,CAAC,CAAC,GAAG,CAAC;gBAC3D,SAAS,EAAE,GAAG;aACf,CAAC;YACF,MAAM,GAAG,EAAE,CAAC,UAAU,CACpB,EAAE,WAAW,EAAE,UAAU,EAAE,QAAQ,EAAE,EACrC,GAAG,EACH,EAAE,MAAM,EAAE,IAAI,EAAE,CACjB,CAAC;YACF,OAAO,EAAE,GAAG,GAAG,EAAE,CAAC;QACpB,CAAC;QAED,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE,UAAU,EAAE,QAAQ;YAC3C,MAAM,aAAa,EAAE,CAAC;YACtB,MAAM,GAAG,GAAG,MAAM,GAAG,EAAE,CAAC,SAAS,CAAC,EAAE,WAAW,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC,CAAC;YACzE,OAAO,GAAG,CAAC,YAAY,GAAG,CAAC,CAAC;QAC9B,CAAC;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* INC-01 stream checkpoints — offsets/cursors for continuous acquisition.
|
|
3
|
+
* Distinct from Memory OT source checkpoints (sourceId × mode × workRef).
|
|
4
|
+
*/
|
|
5
|
+
export type StreamCheckpointMethod = "watermark" | "revision" | "fullScanHash" | "cursor";
|
|
6
|
+
export type StreamCheckpoint = {
|
|
7
|
+
orgId: string;
|
|
8
|
+
connectorId: string;
|
|
9
|
+
instanceId: string;
|
|
10
|
+
streamId: string;
|
|
11
|
+
method: StreamCheckpointMethod;
|
|
12
|
+
/** Splunk-style _indextime / time watermark. */
|
|
13
|
+
watermark?: string | number | null;
|
|
14
|
+
/** FortiGate revision / sequence. */
|
|
15
|
+
revision?: string | number | null;
|
|
16
|
+
/** Full-scan content hash gate. */
|
|
17
|
+
fullScanHash?: string | null;
|
|
18
|
+
/** Opaque cursor (Entra @odata.deltaLink, Falcon offset, …). */
|
|
19
|
+
cursor?: string | null;
|
|
20
|
+
contentHash?: string | null;
|
|
21
|
+
producerRunId?: string | null;
|
|
22
|
+
/** Checkpoint document revision for optional CAS on put. */
|
|
23
|
+
checkpointRevision: number;
|
|
24
|
+
updatedAt: string;
|
|
25
|
+
};
|
|
26
|
+
export type PutStreamCheckpoint = {
|
|
27
|
+
method: StreamCheckpointMethod;
|
|
28
|
+
watermark?: string | number | null;
|
|
29
|
+
revision?: string | number | null;
|
|
30
|
+
fullScanHash?: string | null;
|
|
31
|
+
cursor?: string | null;
|
|
32
|
+
contentHash?: string | null;
|
|
33
|
+
producerRunId?: string | null;
|
|
34
|
+
/** When set, put fails unless current checkpointRevision matches. */
|
|
35
|
+
expectedCheckpointRevision?: number;
|
|
36
|
+
};
|
|
37
|
+
export type StreamCheckpointStore = {
|
|
38
|
+
get(connectorId: string, instanceId: string, streamId: string): Promise<StreamCheckpoint | null>;
|
|
39
|
+
put(connectorId: string, instanceId: string, streamId: string, body: PutStreamCheckpoint): Promise<StreamCheckpoint>;
|
|
40
|
+
clear(connectorId: string, instanceId: string, streamId: string): Promise<boolean>;
|
|
41
|
+
};
|
|
42
|
+
export declare const STREAM_CHECKPOINT_METHODS: readonly StreamCheckpointMethod[];
|
|
43
|
+
export declare const MCO_COLLECTIONS_CHECKPOINT: {
|
|
44
|
+
readonly checkpoints: "mco_stream_checkpoints";
|
|
45
|
+
};
|
|
46
|
+
//# sourceMappingURL=checkpoint-types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"checkpoint-types.d.ts","sourceRoot":"","sources":["../../src/connector-ops/checkpoint-types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,MAAM,MAAM,sBAAsB,GAC9B,WAAW,GACX,UAAU,GACV,cAAc,GACd,QAAQ,CAAC;AAEb,MAAM,MAAM,gBAAgB,GAAG;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,sBAAsB,CAAC;IAC/B,gDAAgD;IAChD,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;IACnC,qCAAqC;IACrC,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;IAClC,mCAAmC;IACnC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,gEAAgE;IAChE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,4DAA4D;IAC5D,kBAAkB,EAAE,MAAM,CAAC;IAC3B,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAChC,MAAM,EAAE,sBAAsB,CAAC;IAC/B,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;IAClC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,qEAAqE;IACrE,0BAA0B,CAAC,EAAE,MAAM,CAAC;CACrC,CAAC;AAEF,MAAM,MAAM,qBAAqB,GAAG;IAClC,GAAG,CACD,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAAC;IACpC,GAAG,CACD,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,mBAAmB,GACxB,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAC7B,KAAK,CACH,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,OAAO,CAAC,CAAC;CACrB,CAAC;AAEF,eAAO,MAAM,yBAAyB,EAAE,SAAS,sBAAsB,EAK7D,CAAC;AAEX,eAAO,MAAM,0BAA0B;;CAE7B,CAAC"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* INC-01 stream checkpoints — offsets/cursors for continuous acquisition.
|
|
3
|
+
* Distinct from Memory OT source checkpoints (sourceId × mode × workRef).
|
|
4
|
+
*/
|
|
5
|
+
export const STREAM_CHECKPOINT_METHODS = [
|
|
6
|
+
"watermark",
|
|
7
|
+
"revision",
|
|
8
|
+
"fullScanHash",
|
|
9
|
+
"cursor",
|
|
10
|
+
];
|
|
11
|
+
export const MCO_COLLECTIONS_CHECKPOINT = {
|
|
12
|
+
checkpoints: "mco_stream_checkpoints",
|
|
13
|
+
};
|
|
14
|
+
//# sourceMappingURL=checkpoint-types.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"checkpoint-types.js","sourceRoot":"","sources":["../../src/connector-ops/checkpoint-types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AA4DH,MAAM,CAAC,MAAM,yBAAyB,GAAsC;IAC1E,WAAW;IACX,UAAU;IACV,cAAc;IACd,QAAQ;CACA,CAAC;AAEX,MAAM,CAAC,MAAM,0BAA0B,GAAG;IACxC,WAAW,EAAE,wBAAwB;CAC7B,CAAC"}
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* MX-CONT-01 continuous acquisition — schedule, lease, run ledger, freshness.
|
|
3
|
+
* Product-plane under connector-ops; not MGC.
|
|
4
|
+
*/
|
|
5
|
+
import type { StreamCheckpoint } from "./checkpoint-types.js";
|
|
6
|
+
export type AcquisitionCadence = {
|
|
7
|
+
kind: "interval";
|
|
8
|
+
intervalSeconds: number;
|
|
9
|
+
} | {
|
|
10
|
+
kind: "cron";
|
|
11
|
+
expression: string;
|
|
12
|
+
};
|
|
13
|
+
export type StreamSchedule = {
|
|
14
|
+
orgId: string;
|
|
15
|
+
connectorId: string;
|
|
16
|
+
instanceId: string;
|
|
17
|
+
streamId: string;
|
|
18
|
+
enabled: boolean;
|
|
19
|
+
cadence: AcquisitionCadence;
|
|
20
|
+
/** Soft bounds passed to bounded /pipeline invokes. */
|
|
21
|
+
bounds?: {
|
|
22
|
+
maxDurationMs?: number;
|
|
23
|
+
maxPages?: number;
|
|
24
|
+
maxEvents?: number;
|
|
25
|
+
};
|
|
26
|
+
updatedAt: string;
|
|
27
|
+
updatedBy?: string;
|
|
28
|
+
};
|
|
29
|
+
export type PutStreamSchedule = {
|
|
30
|
+
enabled?: boolean;
|
|
31
|
+
cadence: AcquisitionCadence;
|
|
32
|
+
bounds?: StreamSchedule["bounds"];
|
|
33
|
+
updatedBy?: string;
|
|
34
|
+
};
|
|
35
|
+
export type StreamLease = {
|
|
36
|
+
orgId: string;
|
|
37
|
+
connectorId: string;
|
|
38
|
+
instanceId: string;
|
|
39
|
+
streamId: string;
|
|
40
|
+
ownerId: string;
|
|
41
|
+
leaseId: string;
|
|
42
|
+
acquiredAt: string;
|
|
43
|
+
expiresAt: string;
|
|
44
|
+
renewCount: number;
|
|
45
|
+
};
|
|
46
|
+
export type AcquireLeaseInput = {
|
|
47
|
+
ownerId: string;
|
|
48
|
+
/** Lease TTL in ms (default 60_000). */
|
|
49
|
+
ttlMs?: number;
|
|
50
|
+
};
|
|
51
|
+
export type AcquisitionRunOutcome = "ok" | "fail" | "skip" | "canceled" | "lease_lost";
|
|
52
|
+
export type AcquisitionRunRecord = {
|
|
53
|
+
orgId: string;
|
|
54
|
+
connectorId: string;
|
|
55
|
+
instanceId: string;
|
|
56
|
+
streamId: string;
|
|
57
|
+
runId: string;
|
|
58
|
+
startedAt: string;
|
|
59
|
+
endedAt?: string | null;
|
|
60
|
+
outcome?: AcquisitionRunOutcome | null;
|
|
61
|
+
errorClass?: string | null;
|
|
62
|
+
errorMessage?: string | null;
|
|
63
|
+
checkpointBefore?: Partial<StreamCheckpoint> | null;
|
|
64
|
+
checkpointAfter?: Partial<StreamCheckpoint> | null;
|
|
65
|
+
bounds?: StreamSchedule["bounds"] | null;
|
|
66
|
+
ownerId?: string | null;
|
|
67
|
+
cancelRequested?: boolean;
|
|
68
|
+
};
|
|
69
|
+
export type StartAcquisitionRunInput = {
|
|
70
|
+
runId?: string;
|
|
71
|
+
ownerId?: string;
|
|
72
|
+
bounds?: StreamSchedule["bounds"];
|
|
73
|
+
checkpointBefore?: Partial<StreamCheckpoint> | null;
|
|
74
|
+
};
|
|
75
|
+
export type EndAcquisitionRunInput = {
|
|
76
|
+
outcome: AcquisitionRunOutcome;
|
|
77
|
+
errorClass?: string;
|
|
78
|
+
errorMessage?: string;
|
|
79
|
+
checkpointAfter?: Partial<StreamCheckpoint> | null;
|
|
80
|
+
};
|
|
81
|
+
export type StreamFreshness = {
|
|
82
|
+
orgId: string;
|
|
83
|
+
connectorId: string;
|
|
84
|
+
instanceId: string;
|
|
85
|
+
streamId: string;
|
|
86
|
+
scheduleEnabled: boolean;
|
|
87
|
+
lastSuccessAt?: string | null;
|
|
88
|
+
lastFailAt?: string | null;
|
|
89
|
+
lastRunAt?: string | null;
|
|
90
|
+
lastOutcome?: AcquisitionRunOutcome | null;
|
|
91
|
+
/** Seconds since last success; null if never succeeded. */
|
|
92
|
+
lagSeconds?: number | null;
|
|
93
|
+
/** True when lag exceeds 2× cadence interval (or 1h for cron). */
|
|
94
|
+
aging?: boolean;
|
|
95
|
+
/** True when no success within 3× cadence (or 3h for cron). */
|
|
96
|
+
lapsed?: boolean;
|
|
97
|
+
activeLeaseOwnerId?: string | null;
|
|
98
|
+
activeLeaseExpiresAt?: string | null;
|
|
99
|
+
cancelRequested?: boolean;
|
|
100
|
+
};
|
|
101
|
+
export type StreamScheduleStore = {
|
|
102
|
+
get(connectorId: string, instanceId: string, streamId: string): Promise<StreamSchedule | null>;
|
|
103
|
+
put(connectorId: string, instanceId: string, streamId: string, body: PutStreamSchedule): Promise<StreamSchedule>;
|
|
104
|
+
clear(connectorId: string, instanceId: string, streamId: string): Promise<boolean>;
|
|
105
|
+
listEnabled(opts?: {
|
|
106
|
+
connectorId?: string;
|
|
107
|
+
instanceId?: string;
|
|
108
|
+
}): Promise<StreamSchedule[]>;
|
|
109
|
+
};
|
|
110
|
+
export type StreamLeaseStore = {
|
|
111
|
+
get(connectorId: string, instanceId: string, streamId: string): Promise<StreamLease | null>;
|
|
112
|
+
acquire(connectorId: string, instanceId: string, streamId: string, input: AcquireLeaseInput): Promise<StreamLease>;
|
|
113
|
+
renew(connectorId: string, instanceId: string, streamId: string, input: {
|
|
114
|
+
ownerId: string;
|
|
115
|
+
leaseId: string;
|
|
116
|
+
ttlMs?: number;
|
|
117
|
+
}): Promise<StreamLease>;
|
|
118
|
+
release(connectorId: string, instanceId: string, streamId: string, input: {
|
|
119
|
+
ownerId: string;
|
|
120
|
+
leaseId: string;
|
|
121
|
+
}): Promise<boolean>;
|
|
122
|
+
};
|
|
123
|
+
export type AcquisitionRunStore = {
|
|
124
|
+
start(connectorId: string, instanceId: string, streamId: string, input: StartAcquisitionRunInput): Promise<AcquisitionRunRecord>;
|
|
125
|
+
end(connectorId: string, instanceId: string, streamId: string, runId: string, input: EndAcquisitionRunInput): Promise<AcquisitionRunRecord>;
|
|
126
|
+
get(connectorId: string, instanceId: string, streamId: string, runId: string): Promise<AcquisitionRunRecord | null>;
|
|
127
|
+
requestCancel(connectorId: string, instanceId: string, streamId: string, runId: string): Promise<AcquisitionRunRecord | null>;
|
|
128
|
+
list(connectorId: string, instanceId: string, streamId: string, opts?: {
|
|
129
|
+
limit?: number;
|
|
130
|
+
}): Promise<AcquisitionRunRecord[]>;
|
|
131
|
+
latestSuccess(connectorId: string, instanceId: string, streamId: string): Promise<AcquisitionRunRecord | null>;
|
|
132
|
+
};
|
|
133
|
+
export declare const CONT_COLLECTIONS: {
|
|
134
|
+
readonly schedules: "mco_stream_schedules";
|
|
135
|
+
readonly leases: "mco_stream_leases";
|
|
136
|
+
readonly runs: "mco_acquisition_runs";
|
|
137
|
+
};
|
|
138
|
+
export declare const DEFAULT_LEASE_TTL_MS = 60000;
|
|
139
|
+
export declare const ACQUISITION_RUNS_DEFAULT_LIMIT = 50;
|
|
140
|
+
export declare const ACQUISITION_RUNS_MAX_LIMIT = 200;
|
|
141
|
+
//# sourceMappingURL=continuous-types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"continuous-types.d.ts","sourceRoot":"","sources":["../../src/connector-ops/continuous-types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAE9D,MAAM,MAAM,kBAAkB,GAC1B;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,eAAe,EAAE,MAAM,CAAA;CAAE,GAC7C;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,CAAC;AAEzC,MAAM,MAAM,cAAc,GAAG;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,kBAAkB,CAAC;IAC5B,uDAAuD;IACvD,MAAM,CAAC,EAAE;QACP,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,OAAO,EAAE,kBAAkB,CAAC;IAC5B,MAAM,CAAC,EAAE,cAAc,CAAC,QAAQ,CAAC,CAAC;IAClC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,wCAAwC;IACxC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,qBAAqB,GAC7B,IAAI,GACJ,MAAM,GACN,MAAM,GACN,UAAU,GACV,YAAY,CAAC;AAEjB,MAAM,MAAM,oBAAoB,GAAG;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,OAAO,CAAC,EAAE,qBAAqB,GAAG,IAAI,CAAC;IACvC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,gBAAgB,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAAC;IACpD,eAAe,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAAC;IACnD,MAAM,CAAC,EAAE,cAAc,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC;IACzC,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B,CAAC;AAEF,MAAM,MAAM,wBAAwB,GAAG;IACrC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,cAAc,CAAC,QAAQ,CAAC,CAAC;IAClC,gBAAgB,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAAC;CACrD,CAAC;AAEF,MAAM,MAAM,sBAAsB,GAAG;IACnC,OAAO,EAAE,qBAAqB,CAAC;IAC/B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAAC;CACpD,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,eAAe,EAAE,OAAO,CAAC;IACzB,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,WAAW,CAAC,EAAE,qBAAqB,GAAG,IAAI,CAAC;IAC3C,2DAA2D;IAC3D,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,kEAAkE;IAClE,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,+DAA+D;IAC/D,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,kBAAkB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,oBAAoB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAChC,GAAG,CACD,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,CAAC;IAClC,GAAG,CACD,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,iBAAiB,GACtB,OAAO,CAAC,cAAc,CAAC,CAAC;IAC3B,KAAK,CACH,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,OAAO,CAAC,CAAC;IACpB,WAAW,CAAC,IAAI,CAAC,EAAE;QACjB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC;CAC/B,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,GAAG,CACD,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,CAAC;IAC/B,OAAO,CACL,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,iBAAiB,GACvB,OAAO,CAAC,WAAW,CAAC,CAAC;IACxB,KAAK,CACH,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAC1D,OAAO,CAAC,WAAW,CAAC,CAAC;IACxB,OAAO,CACL,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,GAC1C,OAAO,CAAC,OAAO,CAAC,CAAC;CACrB,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAChC,KAAK,CACH,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,wBAAwB,GAC9B,OAAO,CAAC,oBAAoB,CAAC,CAAC;IACjC,GAAG,CACD,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,sBAAsB,GAC5B,OAAO,CAAC,oBAAoB,CAAC,CAAC;IACjC,GAAG,CACD,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,oBAAoB,GAAG,IAAI,CAAC,CAAC;IACxC,aAAa,CACX,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,oBAAoB,GAAG,IAAI,CAAC,CAAC;IACxC,IAAI,CACF,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GACxB,OAAO,CAAC,oBAAoB,EAAE,CAAC,CAAC;IACnC,aAAa,CACX,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,oBAAoB,GAAG,IAAI,CAAC,CAAC;CACzC,CAAC;AAEF,eAAO,MAAM,gBAAgB;;;;CAInB,CAAC;AAEX,eAAO,MAAM,oBAAoB,QAAS,CAAC;AAC3C,eAAO,MAAM,8BAA8B,KAAK,CAAC;AACjD,eAAO,MAAM,0BAA0B,MAAM,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* MX-CONT-01 continuous acquisition — schedule, lease, run ledger, freshness.
|
|
3
|
+
* Product-plane under connector-ops; not MGC.
|
|
4
|
+
*/
|
|
5
|
+
export const CONT_COLLECTIONS = {
|
|
6
|
+
schedules: "mco_stream_schedules",
|
|
7
|
+
leases: "mco_stream_leases",
|
|
8
|
+
runs: "mco_acquisition_runs",
|
|
9
|
+
};
|
|
10
|
+
export const DEFAULT_LEASE_TTL_MS = 60_000;
|
|
11
|
+
export const ACQUISITION_RUNS_DEFAULT_LIMIT = 50;
|
|
12
|
+
export const ACQUISITION_RUNS_MAX_LIMIT = 200;
|
|
13
|
+
//# sourceMappingURL=continuous-types.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"continuous-types.js","sourceRoot":"","sources":["../../src/connector-ops/continuous-types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAsMH,MAAM,CAAC,MAAM,gBAAgB,GAAG;IAC9B,SAAS,EAAE,sBAAsB;IACjC,MAAM,EAAE,mBAAmB;IAC3B,IAAI,EAAE,sBAAsB;CACpB,CAAC;AAEX,MAAM,CAAC,MAAM,oBAAoB,GAAG,MAAM,CAAC;AAC3C,MAAM,CAAC,MAAM,8BAA8B,GAAG,EAAE,CAAC;AACjD,MAAM,CAAC,MAAM,0BAA0B,GAAG,GAAG,CAAC"}
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Jobs-ticked continuous acquisition coordinator (MX-CONT-01 host).
|
|
3
|
+
* Bounded pipeline invokes only — no always-on CF-A5 fleet daemon.
|
|
4
|
+
*/
|
|
5
|
+
import type { IntegrationInstanceStore } from "../integrations/types.js";
|
|
6
|
+
import type { McoOverrideStore } from "./types.js";
|
|
7
|
+
import type { StreamCheckpointStore } from "./checkpoint-types.js";
|
|
8
|
+
import type { AcquisitionRunStore, StreamLeaseStore, StreamSchedule, StreamScheduleStore } from "./continuous-types.js";
|
|
9
|
+
export type PipelineInvokeBounds = {
|
|
10
|
+
maxDurationMs?: number;
|
|
11
|
+
maxPages?: number;
|
|
12
|
+
maxEvents?: number;
|
|
13
|
+
};
|
|
14
|
+
export type BoundedPipelineInvokeResult = {
|
|
15
|
+
ok: boolean;
|
|
16
|
+
applyAck: boolean;
|
|
17
|
+
errorClass?: string;
|
|
18
|
+
errorMessage?: string;
|
|
19
|
+
/** Checkpoint payload to commit only when applyAck is true. */
|
|
20
|
+
nextCheckpoint?: {
|
|
21
|
+
method: "watermark" | "revision" | "fullScanHash" | "cursor";
|
|
22
|
+
watermark?: string | number | null;
|
|
23
|
+
revision?: string | number | null;
|
|
24
|
+
fullScanHash?: string | null;
|
|
25
|
+
cursor?: string | null;
|
|
26
|
+
contentHash?: string | null;
|
|
27
|
+
};
|
|
28
|
+
};
|
|
29
|
+
export type BoundedPipelineInvoker = (input: {
|
|
30
|
+
orgId: string;
|
|
31
|
+
connectorId: string;
|
|
32
|
+
instanceId: string;
|
|
33
|
+
streamId: string;
|
|
34
|
+
schedule: StreamSchedule;
|
|
35
|
+
bounds: PipelineInvokeBounds;
|
|
36
|
+
runId: string;
|
|
37
|
+
checkpoint: Awaited<ReturnType<StreamCheckpointStore["get"]>>;
|
|
38
|
+
abortSignal?: AbortSignal;
|
|
39
|
+
}) => Promise<BoundedPipelineInvokeResult>;
|
|
40
|
+
export type ContinuousAcquisitionTickDeps = {
|
|
41
|
+
orgId: string;
|
|
42
|
+
ownerId: string;
|
|
43
|
+
schedules: StreamScheduleStore;
|
|
44
|
+
leases: StreamLeaseStore;
|
|
45
|
+
runs: AcquisitionRunStore;
|
|
46
|
+
checkpoints: StreamCheckpointStore;
|
|
47
|
+
integrations: IntegrationInstanceStore;
|
|
48
|
+
overrides: McoOverrideStore;
|
|
49
|
+
invokePipeline: BoundedPipelineInvoker;
|
|
50
|
+
/** Optional op id used for MCO kill-switch (default continuous-acquire). */
|
|
51
|
+
mcoOpId?: string;
|
|
52
|
+
leaseTtlMs?: number;
|
|
53
|
+
};
|
|
54
|
+
export type ContinuousAcquisitionTickResult = {
|
|
55
|
+
orgId: string;
|
|
56
|
+
considered: number;
|
|
57
|
+
started: number;
|
|
58
|
+
skipped: number;
|
|
59
|
+
succeeded: number;
|
|
60
|
+
failed: number;
|
|
61
|
+
details: Array<{
|
|
62
|
+
connectorId: string;
|
|
63
|
+
instanceId: string;
|
|
64
|
+
streamId: string;
|
|
65
|
+
outcome: string;
|
|
66
|
+
reason?: string;
|
|
67
|
+
runId?: string;
|
|
68
|
+
}>;
|
|
69
|
+
};
|
|
70
|
+
/**
|
|
71
|
+
* Profile helpers for acceptance proofs (CF-A5 vs Splunk watermark window).
|
|
72
|
+
*/
|
|
73
|
+
export declare const COORDINATOR_PROFILES: {
|
|
74
|
+
/** CrowdStrike CF-A5-style: hard event/partition cap. */
|
|
75
|
+
readonly cfA5: {
|
|
76
|
+
maxEvents: number;
|
|
77
|
+
maxPages: number;
|
|
78
|
+
maxDurationMs: number;
|
|
79
|
+
};
|
|
80
|
+
/** Splunk-style watermark window (bounded lookback pages). */
|
|
81
|
+
readonly splunkWatermark: {
|
|
82
|
+
maxPages: number;
|
|
83
|
+
maxEvents: number;
|
|
84
|
+
maxDurationMs: number;
|
|
85
|
+
};
|
|
86
|
+
};
|
|
87
|
+
export declare function tickContinuousAcquisition(deps: ContinuousAcquisitionTickDeps): Promise<ContinuousAcquisitionTickResult>;
|
|
88
|
+
//# sourceMappingURL=coordinator.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"coordinator.d.ts","sourceRoot":"","sources":["../../src/connector-ops/coordinator.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,0BAA0B,CAAC;AACzE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AACnD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AACnE,OAAO,KAAK,EACV,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,mBAAmB,EACpB,MAAM,uBAAuB,CAAC;AAE/B,MAAM,MAAM,oBAAoB,GAAG;IACjC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,2BAA2B,GAAG;IACxC,EAAE,EAAE,OAAO,CAAC;IACZ,QAAQ,EAAE,OAAO,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,+DAA+D;IAC/D,cAAc,CAAC,EAAE;QACf,MAAM,EAAE,WAAW,GAAG,UAAU,GAAG,cAAc,GAAG,QAAQ,CAAC;QAC7D,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;QACnC,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;QAClC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAC7B,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QACvB,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;KAC7B,CAAC;CACH,CAAC;AAEF,MAAM,MAAM,sBAAsB,GAAG,CAAC,KAAK,EAAE;IAC3C,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,cAAc,CAAC;IACzB,MAAM,EAAE,oBAAoB,CAAC;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,OAAO,CAAC,UAAU,CAAC,qBAAqB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC9D,WAAW,CAAC,EAAE,WAAW,CAAC;CAC3B,KAAK,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAE3C,MAAM,MAAM,6BAA6B,GAAG;IAC1C,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,mBAAmB,CAAC;IAC/B,MAAM,EAAE,gBAAgB,CAAC;IACzB,IAAI,EAAE,mBAAmB,CAAC;IAC1B,WAAW,EAAE,qBAAqB,CAAC;IACnC,YAAY,EAAE,wBAAwB,CAAC;IACvC,SAAS,EAAE,gBAAgB,CAAC;IAC5B,cAAc,EAAE,sBAAsB,CAAC;IACvC,4EAA4E;IAC5E,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,CAAC;AAEF,MAAM,MAAM,+BAA+B,GAAG;IAC5C,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,KAAK,CAAC;QACb,WAAW,EAAE,MAAM,CAAC;QACpB,UAAU,EAAE,MAAM,CAAC;QACnB,QAAQ,EAAE,MAAM,CAAC;QACjB,OAAO,EAAE,MAAM,CAAC;QAChB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;CACJ,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,oBAAoB;IAC/B,yDAAyD;;;;;;IAMzD,8DAA8D;;;;;;CAMtD,CAAC;AAEX,wBAAsB,yBAAyB,CAC7C,IAAI,EAAE,6BAA6B,GAClC,OAAO,CAAC,+BAA+B,CAAC,CA4K1C"}
|