@kontextmind/kxm 0.7.72 → 0.7.73
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/docs/contracts/synchronization.md +6 -3
- package/docs/operations.md +35 -4
- package/package.json +1 -1
- package/plugins/kxm/.claude-plugin/plugin.json +1 -1
- package/plugins/kxm/dist/cli.js +307 -235
- package/plugins/kxm/dist/client.js +67 -36
- package/plugins/kxm/dist/core.js +2 -0
- package/plugins/kxm/dist/extension.js +40 -37
- package/plugins/kxm/dist/mcp-server.js +40 -37
- package/plugins/kxm/dist/runtime-supervisor.js +917 -141
- package/plugins/kxm/dist/runtime.js +977 -237
- package/plugins/kxm/dist/server.js +16206 -8256
- package/plugins/kxm/package.json +1 -1
- package/plugins/kxm/src/client.ts +85 -3
- package/plugins/kxm/src/database.ts +4 -4
- package/plugins/kxm/src/hub.ts +221 -0
- package/plugins/kxm/src/mcp-server.ts +1 -1
- package/plugins/kxm/src/project-config.ts +15 -0
- package/plugins/kxm/src/protocol.ts +27 -0
- package/plugins/kxm/src/runtime-store.ts +109 -1
- package/plugins/kxm/src/runtime-supervisor.ts +150 -0
- package/plugins/kxm/src/store.ts +207 -4
- package/plugins/kxm/src/sync-transform.ts +380 -0
|
@@ -4,6 +4,7 @@ import { dirname, join, resolve } from "node:path";
|
|
|
4
4
|
import { DatabaseSync } from "./sqlite.ts";
|
|
5
5
|
import { KxmConfigError, validateCoordinator, validateDriveReceipt, validateIntakeMessage, validateRunEvent, kxmCanonicalJson, type JsonValue, type KxmConfigIssue, type KxmConfigOptions } from "./project-config.ts";
|
|
6
6
|
import { kxmUserStateRoot } from "./bindings.ts";
|
|
7
|
+
import { deriveKxmSyncEvent, kxmSyncEventBytes, KxmSyncRedactor } from "./sync-transform.ts";
|
|
7
8
|
|
|
8
9
|
/* ------------------------------------------------------------------ *
|
|
9
10
|
* Runtime registry (per-user, platform state root)
|
|
@@ -225,6 +226,16 @@ export class KxmRuntimeRegistry {
|
|
|
225
226
|
}
|
|
226
227
|
|
|
227
228
|
/** Register or revalidate a project's home binding. Home Runtime is immutable. */
|
|
229
|
+
/** All projects registered to this Runtime, for restart recovery: the
|
|
230
|
+
* supervisor needs to reopen their contexts so pending outbox rows resume
|
|
231
|
+
* syncing and presence keeps beating. */
|
|
232
|
+
projectsForRuntime(homeRuntimeId: string): Array<{ projectRoot: string; projectId: string }> {
|
|
233
|
+
const rows = this.database.prepare(
|
|
234
|
+
"SELECT project_root, project_id FROM projects WHERE home_runtime_id = ? ORDER BY registered_at",
|
|
235
|
+
).all(homeRuntimeId) as Array<{ project_root: string; project_id: string }>;
|
|
236
|
+
return rows.map((row) => ({ projectRoot: row.project_root, projectId: row.project_id }));
|
|
237
|
+
}
|
|
238
|
+
|
|
228
239
|
registerProject(registration: { projectId: string; projectRoot: string; homeRuntimeId: string; configRevision?: string; now: string }): KxmProjectRegistration {
|
|
229
240
|
const projectRoot = resolve(registration.projectRoot);
|
|
230
241
|
const projectKey = projectRuntimeKey(projectRoot);
|
|
@@ -550,7 +561,7 @@ export interface KxmCommandRecord {
|
|
|
550
561
|
recordedAt: string;
|
|
551
562
|
}
|
|
552
563
|
|
|
553
|
-
export const KXM_EVENT_STORE_SCHEMA_VERSION =
|
|
564
|
+
export const KXM_EVENT_STORE_SCHEMA_VERSION = 6;
|
|
554
565
|
export const KXM_DRIVE_RECEIPT_SCHEMA = "kxm.drive-receipt.v1";
|
|
555
566
|
export const DRIVE_RECEIPT_MAX_BYTES = 8 * 1024;
|
|
556
567
|
|
|
@@ -586,6 +597,7 @@ const EVENT_STORE_TABLES = {
|
|
|
586
597
|
"schema", "record",
|
|
587
598
|
],
|
|
588
599
|
project_controls: ["project_id", "paused", "reason", "updated_at", "actor", "schema", "record"],
|
|
600
|
+
outbox: ["seq", "run_id", "sequence", "sync_event", "attempted_at", "acked_at"],
|
|
589
601
|
} as const;
|
|
590
602
|
|
|
591
603
|
/**
|
|
@@ -780,8 +792,31 @@ CREATE TABLE project_controls (
|
|
|
780
792
|
schema TEXT NOT NULL,
|
|
781
793
|
record TEXT NOT NULL
|
|
782
794
|
) STRICT;
|
|
795
|
+
CREATE TABLE IF NOT EXISTS outbox (
|
|
796
|
+
seq INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
797
|
+
run_id TEXT NOT NULL,
|
|
798
|
+
sequence INTEGER NOT NULL,
|
|
799
|
+
sync_event TEXT NOT NULL,
|
|
800
|
+
attempted_at TEXT,
|
|
801
|
+
acked_at TEXT,
|
|
802
|
+
UNIQUE (run_id, sequence)
|
|
803
|
+
) STRICT;
|
|
804
|
+
CREATE INDEX outbox_pending ON outbox(seq) WHERE acked_at IS NULL;
|
|
783
805
|
`;
|
|
784
806
|
|
|
807
|
+
/**
|
|
808
|
+
* One outbox row: the already-derived `kxm.sync-event.v1` bytes plus retry
|
|
809
|
+
* transport metadata. The local source payload is never kept here.
|
|
810
|
+
*/
|
|
811
|
+
export interface KxmOutboxRow {
|
|
812
|
+
seq: number;
|
|
813
|
+
runId: string;
|
|
814
|
+
sequence: number;
|
|
815
|
+
syncEvent: string;
|
|
816
|
+
attemptedAt?: string;
|
|
817
|
+
ackedAt?: string;
|
|
818
|
+
}
|
|
819
|
+
|
|
785
820
|
/** One persisted coordinator identity (`kxm.coordinator.v1`). */
|
|
786
821
|
export interface KxmCoordinatorRow {
|
|
787
822
|
coordinatorId: string;
|
|
@@ -818,6 +853,9 @@ export interface KxmProjectControlRow {
|
|
|
818
853
|
|
|
819
854
|
export class KxmRunEventStore {
|
|
820
855
|
readonly path: string;
|
|
856
|
+
/** Secret values registered here are replaced in every sync object this
|
|
857
|
+
* store derives. In memory only: they are never written anywhere. */
|
|
858
|
+
readonly syncRedactor = new KxmSyncRedactor();
|
|
821
859
|
private readonly database: DatabaseSync;
|
|
822
860
|
|
|
823
861
|
constructor(path: string) {
|
|
@@ -933,6 +971,16 @@ export class KxmRunEventStore {
|
|
|
933
971
|
return row ? runFromRow(row) : undefined;
|
|
934
972
|
}
|
|
935
973
|
|
|
974
|
+
/** All projects registered to this Runtime, for restart recovery: the
|
|
975
|
+
* supervisor needs to reopen their contexts so pending outbox rows resume
|
|
976
|
+
* syncing and presence keeps beating. */
|
|
977
|
+
projectsForRuntime(homeRuntimeId: string): Array<{ projectRoot: string; projectId: string }> {
|
|
978
|
+
const rows = this.database.prepare(
|
|
979
|
+
"SELECT project_root, project_id FROM projects WHERE home_runtime_id = ? ORDER BY registered_at",
|
|
980
|
+
).all(homeRuntimeId) as Array<{ project_root: string; project_id: string }>;
|
|
981
|
+
return rows.map((row) => ({ projectRoot: row.project_root, projectId: row.project_id }));
|
|
982
|
+
}
|
|
983
|
+
|
|
936
984
|
runsForProject(projectId: string, limit = 50): KxmRunRecord[] {
|
|
937
985
|
const rows = this.database.prepare(`
|
|
938
986
|
SELECT run_id, project_id, home_runtime_id, workflow_id, prompt_sha256, status, config_revision, memory_revision, executor_policy_revision, tool_policy_revision, created_at, updated_at
|
|
@@ -974,6 +1022,46 @@ export class KxmRunEventStore {
|
|
|
974
1022
|
event.schema,
|
|
975
1023
|
event.homeRuntimeId,
|
|
976
1024
|
);
|
|
1025
|
+
// Allowlist before outbox: the row holds only the derived sync object, and
|
|
1026
|
+
// it commits (or rolls back) with the event in the caller's transaction.
|
|
1027
|
+
const syncEvent = deriveKxmSyncEvent(event, { redactor: this.syncRedactor });
|
|
1028
|
+
this.database.prepare("INSERT INTO outbox (run_id, sequence, sync_event) VALUES (?, ?, ?)")
|
|
1029
|
+
.run(event.runId, event.sequence, kxmSyncEventBytes(syncEvent));
|
|
1030
|
+
}
|
|
1031
|
+
|
|
1032
|
+
/** Unacknowledged outbox rows after `afterSeq`, in outbox order, oldest first. */
|
|
1033
|
+
pendingOutbox(limit = 100, afterSeq = 0): KxmOutboxRow[] {
|
|
1034
|
+
const rows = this.database.prepare(`
|
|
1035
|
+
SELECT seq, run_id, sequence, sync_event, attempted_at, acked_at
|
|
1036
|
+
FROM outbox WHERE acked_at IS NULL AND seq > ? ORDER BY seq ASC LIMIT ?
|
|
1037
|
+
`).all(afterSeq, limit) as OutboxSqlRow[];
|
|
1038
|
+
return rows.map(outboxFromRow);
|
|
1039
|
+
}
|
|
1040
|
+
|
|
1041
|
+
/** Every outbox row of one run, acknowledged or not, in sequence order. */
|
|
1042
|
+
outboxForRun(runId: string): KxmOutboxRow[] {
|
|
1043
|
+
const rows = this.database.prepare(`
|
|
1044
|
+
SELECT seq, run_id, sequence, sync_event, attempted_at, acked_at
|
|
1045
|
+
FROM outbox WHERE run_id = ? ORDER BY sequence ASC
|
|
1046
|
+
`).all(runId) as OutboxSqlRow[];
|
|
1047
|
+
return rows.map(outboxFromRow);
|
|
1048
|
+
}
|
|
1049
|
+
|
|
1050
|
+
markOutboxAttempted(seqs: readonly number[], at: string): void {
|
|
1051
|
+
const statement = this.database.prepare("UPDATE outbox SET attempted_at = ? WHERE seq = ? AND acked_at IS NULL");
|
|
1052
|
+
this.transaction(() => {
|
|
1053
|
+
for (const seq of seqs) statement.run(at, seq);
|
|
1054
|
+
});
|
|
1055
|
+
}
|
|
1056
|
+
|
|
1057
|
+
/** Advance the cursor: the hub holds these rows now. Returns rows newly acked. */
|
|
1058
|
+
ackOutbox(seqs: readonly number[], at: string): number {
|
|
1059
|
+
const statement = this.database.prepare("UPDATE outbox SET acked_at = ? WHERE seq = ? AND acked_at IS NULL");
|
|
1060
|
+
return this.transaction(() => {
|
|
1061
|
+
let changed = 0;
|
|
1062
|
+
for (const seq of seqs) changed += Number(statement.run(at, seq).changes);
|
|
1063
|
+
return changed;
|
|
1064
|
+
});
|
|
977
1065
|
}
|
|
978
1066
|
|
|
979
1067
|
events(runId: string, afterSequence = 0, limit = 200): KxmRunEvent[] {
|
|
@@ -1585,6 +1673,26 @@ function parseDriveReceipt(raw: string): KxmDriveReceipt {
|
|
|
1585
1673
|
return parsed as KxmDriveReceipt;
|
|
1586
1674
|
}
|
|
1587
1675
|
|
|
1676
|
+
type OutboxSqlRow = {
|
|
1677
|
+
seq: number;
|
|
1678
|
+
run_id: string;
|
|
1679
|
+
sequence: number;
|
|
1680
|
+
sync_event: string;
|
|
1681
|
+
attempted_at: string | null;
|
|
1682
|
+
acked_at: string | null;
|
|
1683
|
+
};
|
|
1684
|
+
|
|
1685
|
+
function outboxFromRow(row: OutboxSqlRow): KxmOutboxRow {
|
|
1686
|
+
return {
|
|
1687
|
+
seq: row.seq,
|
|
1688
|
+
runId: row.run_id,
|
|
1689
|
+
sequence: row.sequence,
|
|
1690
|
+
syncEvent: row.sync_event,
|
|
1691
|
+
...(row.attempted_at !== null ? { attemptedAt: row.attempted_at } : {}),
|
|
1692
|
+
...(row.acked_at !== null ? { ackedAt: row.acked_at } : {}),
|
|
1693
|
+
};
|
|
1694
|
+
}
|
|
1695
|
+
|
|
1588
1696
|
function runFromRow(row: {
|
|
1589
1697
|
run_id: string;
|
|
1590
1698
|
project_id: string;
|
|
@@ -11,6 +11,7 @@ import {
|
|
|
11
11
|
runtimeError,
|
|
12
12
|
verifyKxmDriveReceipt,
|
|
13
13
|
kxmRuntimePaths,
|
|
14
|
+
type KxmRunEventStore,
|
|
14
15
|
type KxmRuntimePaths,
|
|
15
16
|
} from "./runtime-store.ts";
|
|
16
17
|
import {
|
|
@@ -26,6 +27,10 @@ import { createKxmOneShotProducer } from "./oneshot-producer.ts";
|
|
|
26
27
|
import { isRouteAdmitted } from "./routes.ts";
|
|
27
28
|
import { KxmRunScheduler, createKxmSimulatedProducer, recordDriveReceipt, recoverKxmRun, kxmDrivePollProjection } from "./engine.ts";
|
|
28
29
|
import { kxmDriveSession, kxmOpenDriveSessions } from "./runtime-owner.ts";
|
|
30
|
+
import { RuntimeHubClient } from "./client.ts";
|
|
31
|
+
import { readHubBinding } from "./hub-binding.ts";
|
|
32
|
+
import { resolveClientHubAuthToken } from "./hub-env.ts";
|
|
33
|
+
import { defaultProjectName } from "./project-name.ts";
|
|
29
34
|
|
|
30
35
|
/* ------------------------------------------------------------------ *
|
|
31
36
|
* Token management
|
|
@@ -351,6 +356,96 @@ async function driveSessionStillPending(settled: Promise<unknown>): Promise<bool
|
|
|
351
356
|
return pending;
|
|
352
357
|
}
|
|
353
358
|
|
|
359
|
+
/* ------------------------------------------------------------------ *
|
|
360
|
+
* Runtime → hub sync (P5): outbound only
|
|
361
|
+
* ------------------------------------------------------------------ */
|
|
362
|
+
|
|
363
|
+
export const DEFAULT_RUNTIME_SYNC_INTERVAL_MS = 10_000;
|
|
364
|
+
const MIN_RUNTIME_SYNC_INTERVAL_MS = 250;
|
|
365
|
+
const MAX_RUNTIME_SYNC_INTERVAL_MS = 60_000;
|
|
366
|
+
const OUTBOX_PUSH_BATCH = 32;
|
|
367
|
+
|
|
368
|
+
/** How often the supervisor heartbeats and pushes its outbox. Keep it well
|
|
369
|
+
* under the hub's presence lease (30 s by default). */
|
|
370
|
+
export function runtimeSyncIntervalMs(env: NodeJS.ProcessEnv = process.env): number {
|
|
371
|
+
const raw = env.KXM_RUNTIME_SYNC_INTERVAL_MS?.trim();
|
|
372
|
+
if (!raw) return DEFAULT_RUNTIME_SYNC_INTERVAL_MS;
|
|
373
|
+
const parsed = Number(raw);
|
|
374
|
+
if (!Number.isInteger(parsed)) return DEFAULT_RUNTIME_SYNC_INTERVAL_MS;
|
|
375
|
+
return Math.min(MAX_RUNTIME_SYNC_INTERVAL_MS, Math.max(MIN_RUNTIME_SYNC_INTERVAL_MS, parsed));
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
export interface KxmOutboxSyncResult {
|
|
379
|
+
pushed: number;
|
|
380
|
+
acked: number;
|
|
381
|
+
conflicts: number;
|
|
382
|
+
rejected: number;
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
/**
|
|
386
|
+
* Push every pending outbox row to the bound hub in outbox order and advance
|
|
387
|
+
* the cursor on each acknowledgement. Accepted and duplicate rows are acked;
|
|
388
|
+
* a conflict or rejection stays pending (the hub has raised the alert) and is
|
|
389
|
+
* skipped for the rest of this pass. A transport failure leaves every row
|
|
390
|
+
* pending for a safe retry.
|
|
391
|
+
*/
|
|
392
|
+
export async function syncKxmOutbox(
|
|
393
|
+
eventStore: KxmRunEventStore,
|
|
394
|
+
client: RuntimeHubClient,
|
|
395
|
+
options: { now?: () => string; batchSize?: number } = {},
|
|
396
|
+
): Promise<KxmOutboxSyncResult> {
|
|
397
|
+
const now = options.now ?? (() => new Date().toISOString());
|
|
398
|
+
const batchSize = options.batchSize ?? OUTBOX_PUSH_BATCH;
|
|
399
|
+
const result: KxmOutboxSyncResult = { pushed: 0, acked: 0, conflicts: 0, rejected: 0 };
|
|
400
|
+
let afterSeq = 0;
|
|
401
|
+
for (;;) {
|
|
402
|
+
const rows = eventStore.pendingOutbox(batchSize, afterSeq);
|
|
403
|
+
if (rows.length === 0) return result;
|
|
404
|
+
// Batch by serialized byte size, not just count: the hub rejects requests
|
|
405
|
+
// over its body ceiling (HTTP 413), and retrying the same oversized batch
|
|
406
|
+
// would permanently block the queue. Trim to the byte budget and leave the
|
|
407
|
+
// rest for the next iteration.
|
|
408
|
+
const MAX_BATCH_BYTES = 200_000; // hub ceiling is 256 KiB; leave headroom
|
|
409
|
+
let byteBudget = MAX_BATCH_BYTES;
|
|
410
|
+
let sendCount = 0;
|
|
411
|
+
for (const row of rows) {
|
|
412
|
+
const rowBytes = Buffer.byteLength(row.syncEvent, "utf8") + 64; // JSON overhead
|
|
413
|
+
if (sendCount > 0 && byteBudget - rowBytes < 0) break;
|
|
414
|
+
byteBudget -= rowBytes;
|
|
415
|
+
sendCount += 1;
|
|
416
|
+
}
|
|
417
|
+
const batch = rows.slice(0, sendCount);
|
|
418
|
+
if (batch.length === 0) batch.push(rows[0]!); // one oversized row: send alone, hub will 413
|
|
419
|
+
afterSeq = batch[batch.length - 1]!.seq;
|
|
420
|
+
eventStore.markOutboxAttempted(batch.map((row) => row.seq), now());
|
|
421
|
+
const response = await client.pushSyncEvents(batch.map((row) => JSON.parse(row.syncEvent) as unknown));
|
|
422
|
+
result.pushed += batch.length;
|
|
423
|
+
const outcomes = new Map(response.results.map((entry) => [`${entry.runId}\u0000${entry.sequence}`, entry.outcome]));
|
|
424
|
+
const acked: number[] = [];
|
|
425
|
+
for (const row of batch) {
|
|
426
|
+
const outcome = outcomes.get(`${row.runId}\u0000${row.sequence}`);
|
|
427
|
+
if (outcome === "accepted" || outcome === "duplicate") acked.push(row.seq);
|
|
428
|
+
else if (outcome === "conflict") result.conflicts += 1;
|
|
429
|
+
else result.rejected += 1;
|
|
430
|
+
}
|
|
431
|
+
result.acked += eventStore.ackOutbox(acked, now());
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
/** Where this Runtime reports one project, or nothing when no hub is bound. */
|
|
436
|
+
function runtimeHubClientFor(context: KxmRuntimeContext, env: NodeJS.ProcessEnv): RuntimeHubClient | undefined {
|
|
437
|
+
const serverUrl = env.KXM_SERVER_URL?.trim() || readHubBinding(env)?.url;
|
|
438
|
+
if (!serverUrl) return undefined;
|
|
439
|
+
const project = defaultProjectName(context.projectRoot, env);
|
|
440
|
+
const authToken = resolveClientHubAuthToken(env, project);
|
|
441
|
+
return new RuntimeHubClient({
|
|
442
|
+
serverUrl,
|
|
443
|
+
project,
|
|
444
|
+
runtimeId: context.homeRuntimeId,
|
|
445
|
+
...(authToken ? { authToken } : {}),
|
|
446
|
+
});
|
|
447
|
+
}
|
|
448
|
+
|
|
354
449
|
export interface KxmRuntimeSupervisor {
|
|
355
450
|
server: Server;
|
|
356
451
|
port: number;
|
|
@@ -395,6 +490,21 @@ async function startKxmRuntimeSupervisorInner(
|
|
|
395
490
|
let activeRuntimeId = runtimeId;
|
|
396
491
|
|
|
397
492
|
const contexts = new Map<string, KxmRuntimeContext>();
|
|
493
|
+
const registerSyncCredentials = (context: KxmRuntimeContext): void => {
|
|
494
|
+
// Register credentials on the store's redactor at context creation —
|
|
495
|
+
// BEFORE any event can be appended — so the very first outbox row is
|
|
496
|
+
// already scrubbed. Registering on the sync tick leaves a window where
|
|
497
|
+
// appended events retain credentials.
|
|
498
|
+
const hubToken = resolveClientHubAuthToken(process.env, defaultProjectName(context.projectRoot, process.env));
|
|
499
|
+
if (hubToken) context.eventStore.syncRedactor.register(hubToken);
|
|
500
|
+
for (const key of Object.keys(process.env)) {
|
|
501
|
+
if ((key.startsWith("KXM_") && (key.endsWith("_TOKEN") || key.endsWith("_KEY"))) || key.endsWith("_API_KEY") || key.endsWith("_SECRET")) {
|
|
502
|
+
const value = process.env[key]?.trim();
|
|
503
|
+
if (value) context.eventStore.syncRedactor.register(value);
|
|
504
|
+
}
|
|
505
|
+
}
|
|
506
|
+
};
|
|
507
|
+
|
|
398
508
|
const contextFor = (projectRoot: string): KxmRuntimeContext => {
|
|
399
509
|
if (!isAbsolute(projectRoot)) {
|
|
400
510
|
throw runtimeError("runtime_request_invalid", "projectRoot", "projectRoot must be an absolute path");
|
|
@@ -403,6 +513,7 @@ async function startKxmRuntimeSupervisorInner(
|
|
|
403
513
|
const existing = contexts.get(key);
|
|
404
514
|
if (existing) return existing;
|
|
405
515
|
const context = openKxmRuntimeContext(projectRoot, { homeRuntimeId: activeRuntimeId, stateRoot: paths.stateRoot });
|
|
516
|
+
registerSyncCredentials(context);
|
|
406
517
|
contexts.set(key, context);
|
|
407
518
|
return context;
|
|
408
519
|
};
|
|
@@ -774,11 +885,50 @@ async function startKxmRuntimeSupervisorInner(
|
|
|
774
885
|
}, 1000);
|
|
775
886
|
heartbeat.unref();
|
|
776
887
|
|
|
888
|
+
// Outbound only: the supervisor pulls nothing and exposes nothing to the hub.
|
|
889
|
+
// A tick that finds no bound hub, no credential or an unreachable hub does
|
|
890
|
+
// nothing; outbox rows stay pending and local execution never waits on it.
|
|
891
|
+
//
|
|
892
|
+
// Restart recovery: contexts are only populated on demand (a project request
|
|
893
|
+
// opens one), so a restarted supervisor would see an empty map and silently
|
|
894
|
+
// stop syncing every registered project's pending outbox rows. Reopen the
|
|
895
|
+
// projects this Runtime owns before the first tick.
|
|
896
|
+
for (const reg of registry.projectsForRuntime(activeRuntimeId)) {
|
|
897
|
+
try {
|
|
898
|
+
contextFor(reg.projectRoot);
|
|
899
|
+
} catch {
|
|
900
|
+
// A project whose checkout has moved or been deleted stays skipped; its
|
|
901
|
+
// outbox rows remain pending and its presence expires, which is visible
|
|
902
|
+
// in the ops snapshot as orphaned.
|
|
903
|
+
}
|
|
904
|
+
}
|
|
905
|
+
|
|
906
|
+
let syncing = false;
|
|
907
|
+
const syncTimer = setInterval(() => {
|
|
908
|
+
if (syncing || stopping) return;
|
|
909
|
+
syncing = true;
|
|
910
|
+
void (async () => {
|
|
911
|
+
for (const context of [...contexts.values()]) {
|
|
912
|
+
|
|
913
|
+
try {
|
|
914
|
+
const client = runtimeHubClientFor(context, process.env);
|
|
915
|
+
if (!client) continue;
|
|
916
|
+
await client.heartbeat();
|
|
917
|
+
await syncKxmOutbox(context.eventStore, client, { now });
|
|
918
|
+
} catch {
|
|
919
|
+
// Retry on the next tick.
|
|
920
|
+
}
|
|
921
|
+
}
|
|
922
|
+
})().finally(() => { syncing = false; });
|
|
923
|
+
}, runtimeSyncIntervalMs());
|
|
924
|
+
syncTimer.unref();
|
|
925
|
+
|
|
777
926
|
let stopping = false;
|
|
778
927
|
const stop = async (): Promise<void> => {
|
|
779
928
|
if (stopping) return;
|
|
780
929
|
stopping = true;
|
|
781
930
|
clearInterval(heartbeat);
|
|
931
|
+
clearInterval(syncTimer);
|
|
782
932
|
try { registry.markStopping(process.pid, now()); } catch { /* best effort */ }
|
|
783
933
|
const openSessions = [...contexts.values()].flatMap((context) => (
|
|
784
934
|
kxmOpenDriveSessions(context.eventStore.path).map((session) => ({ context, session }))
|
package/plugins/kxm/src/store.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { resolve } from "node:path";
|
|
2
2
|
import type { DatabaseSync } from "./sqlite.ts";
|
|
3
|
-
import type { AgentIdentity, LeaseRecord, MessageRecord } from "./protocol.ts";
|
|
3
|
+
import type { AgentIdentity, LeaseRecord, MessageRecord, RuntimePresenceRecord, StoredSyncEvent } from "./protocol.ts";
|
|
4
4
|
import type { ContextItem } from "./context.ts";
|
|
5
5
|
import type { WorkflowJournalEntry, WorkflowRun } from "./workflow.ts";
|
|
6
6
|
import {
|
|
@@ -9,7 +9,7 @@ import {
|
|
|
9
9
|
type DatabaseSchemaSpec,
|
|
10
10
|
} from "./database.ts";
|
|
11
11
|
|
|
12
|
-
export const HUB_STORE_SCHEMA_VERSION =
|
|
12
|
+
export const HUB_STORE_SCHEMA_VERSION = 5;
|
|
13
13
|
|
|
14
14
|
export const HUB_STORE_TABLES: Readonly<Record<string, readonly string[]>> = Object.freeze({
|
|
15
15
|
agents: Object.freeze(["id", "record"]),
|
|
@@ -20,9 +20,13 @@ export const HUB_STORE_TABLES: Readonly<Record<string, readonly string[]>> = Obj
|
|
|
20
20
|
workflow_journal: Object.freeze(["id", "run_id", "category", "area", "record"]),
|
|
21
21
|
context_items: Object.freeze(["id", "project", "kind", "record"]),
|
|
22
22
|
leases: Object.freeze(["resource", "holder_agent_id", "fencing_token", "expires_at", "record"]),
|
|
23
|
+
sync_events: Object.freeze([
|
|
24
|
+
"project_id", "run_id", "sequence", "hub_project", "home_runtime_id", "event_type", "content_hash", "received_at", "record",
|
|
25
|
+
]),
|
|
26
|
+
runtime_presence: Object.freeze(["runtime_id", "hub_project", "host", "heartbeat", "record"]),
|
|
23
27
|
});
|
|
24
28
|
|
|
25
|
-
export const
|
|
29
|
+
export const HUB_STORE_SCHEMA_V5 = `
|
|
26
30
|
CREATE TABLE IF NOT EXISTS agents (
|
|
27
31
|
id TEXT PRIMARY KEY,
|
|
28
32
|
record TEXT NOT NULL
|
|
@@ -78,10 +82,31 @@ export const HUB_STORE_SCHEMA_V4 = `
|
|
|
78
82
|
expires_at TEXT NOT NULL,
|
|
79
83
|
record TEXT NOT NULL
|
|
80
84
|
) STRICT;
|
|
85
|
+
CREATE TABLE IF NOT EXISTS sync_events (
|
|
86
|
+
project_id TEXT NOT NULL,
|
|
87
|
+
run_id TEXT NOT NULL,
|
|
88
|
+
sequence INTEGER NOT NULL,
|
|
89
|
+
hub_project TEXT NOT NULL,
|
|
90
|
+
home_runtime_id TEXT NOT NULL,
|
|
91
|
+
event_type TEXT NOT NULL,
|
|
92
|
+
content_hash TEXT NOT NULL,
|
|
93
|
+
received_at TEXT NOT NULL,
|
|
94
|
+
record TEXT NOT NULL,
|
|
95
|
+
PRIMARY KEY (project_id, run_id, sequence)
|
|
96
|
+
) STRICT;
|
|
97
|
+
CREATE INDEX IF NOT EXISTS sync_events_hub_project ON sync_events(hub_project, run_id, sequence);
|
|
98
|
+
CREATE TABLE IF NOT EXISTS runtime_presence (
|
|
99
|
+
runtime_id TEXT NOT NULL,
|
|
100
|
+
hub_project TEXT NOT NULL,
|
|
101
|
+
host TEXT,
|
|
102
|
+
heartbeat TEXT NOT NULL,
|
|
103
|
+
record TEXT NOT NULL,
|
|
104
|
+
PRIMARY KEY (hub_project, runtime_id)
|
|
105
|
+
) STRICT;
|
|
81
106
|
`;
|
|
82
107
|
|
|
83
108
|
export const HUB_STORE_SCHEMA_SPEC: DatabaseSchemaSpec = Object.freeze({
|
|
84
|
-
schema:
|
|
109
|
+
schema: HUB_STORE_SCHEMA_V5,
|
|
85
110
|
version: HUB_STORE_SCHEMA_VERSION,
|
|
86
111
|
tables: HUB_STORE_TABLES,
|
|
87
112
|
});
|
|
@@ -112,6 +137,17 @@ export interface LeaseAcquireInput {
|
|
|
112
137
|
nowMs: number;
|
|
113
138
|
}
|
|
114
139
|
|
|
140
|
+
/** How one pushed sync event landed. `conflict` is a security alert: the
|
|
141
|
+
* same `{projectId, runId, sequence}` already holds different bytes, or the
|
|
142
|
+
* run already belongs to another project or home Runtime. */
|
|
143
|
+
export type SyncIngestOutcome =
|
|
144
|
+
| { outcome: "accepted" | "duplicate" }
|
|
145
|
+
| { outcome: "conflict"; reason: "sequence_reused" | "project_mismatch" | "home_runtime_mismatch"; existingHash?: string };
|
|
146
|
+
|
|
147
|
+
function syncKey(projectId: string, runId: string, sequence: number): string {
|
|
148
|
+
return `${projectId}\u0000${runId}\u0000${sequence}`;
|
|
149
|
+
}
|
|
150
|
+
|
|
115
151
|
function parseLeaseRecord(record: string): LeaseRecord | undefined {
|
|
116
152
|
try {
|
|
117
153
|
return JSON.parse(record) as LeaseRecord;
|
|
@@ -127,6 +163,32 @@ function leaseHasLapsed(lease: LeaseRecord, nowMs: number): boolean {
|
|
|
127
163
|
return !Number.isFinite(expiresAtMs) || expiresAtMs <= nowMs;
|
|
128
164
|
}
|
|
129
165
|
|
|
166
|
+
type SyncEventSqlRow = {
|
|
167
|
+
project_id: string;
|
|
168
|
+
run_id: string;
|
|
169
|
+
sequence: number;
|
|
170
|
+
hub_project: string;
|
|
171
|
+
home_runtime_id: string;
|
|
172
|
+
event_type: string;
|
|
173
|
+
content_hash: string;
|
|
174
|
+
received_at: string;
|
|
175
|
+
record: string;
|
|
176
|
+
};
|
|
177
|
+
|
|
178
|
+
function syncEventFromRow(row: SyncEventSqlRow): StoredSyncEvent {
|
|
179
|
+
return {
|
|
180
|
+
projectId: row.project_id,
|
|
181
|
+
runId: row.run_id,
|
|
182
|
+
sequence: row.sequence,
|
|
183
|
+
hubProject: row.hub_project,
|
|
184
|
+
homeRuntimeId: row.home_runtime_id,
|
|
185
|
+
eventType: row.event_type,
|
|
186
|
+
contentHash: row.content_hash,
|
|
187
|
+
receivedAt: row.received_at,
|
|
188
|
+
bytes: row.record,
|
|
189
|
+
};
|
|
190
|
+
}
|
|
191
|
+
|
|
130
192
|
class MessageMap extends Map<string, MessageRecord> {
|
|
131
193
|
private readonly store: MeshStore;
|
|
132
194
|
|
|
@@ -167,6 +229,9 @@ export class MeshStore {
|
|
|
167
229
|
readonly journal = new Map<string, WorkflowJournalEntry>();
|
|
168
230
|
readonly contextItems = new Map<string, ContextItem>();
|
|
169
231
|
readonly leases = new Map<string, LeaseRecord>();
|
|
232
|
+
/** Memory-only stores keep sync state here; a database store reads SQLite. */
|
|
233
|
+
private readonly syncEvents = new Map<string, StoredSyncEvent>();
|
|
234
|
+
private readonly runtimePresence = new Map<string, RuntimePresenceRecord>();
|
|
170
235
|
readonly path?: string;
|
|
171
236
|
private readonly database?: DatabaseSync;
|
|
172
237
|
private readonly agentSequences = new Map<string, number>();
|
|
@@ -508,6 +573,144 @@ export class MeshStore {
|
|
|
508
573
|
this.database?.prepare("DELETE FROM leases WHERE resource = ?").run(resource);
|
|
509
574
|
}
|
|
510
575
|
|
|
576
|
+
// ----- Runtime → hub sync (P5) -----
|
|
577
|
+
|
|
578
|
+
/**
|
|
579
|
+
* Accept one sync event exactly once by `{projectId, runId, sequence}`,
|
|
580
|
+
* deciding inside one transaction. Identical bytes are an idempotent
|
|
581
|
+
* duplicate; different bytes under a used sequence are refused, and so is a
|
|
582
|
+
* run that already belongs to another hub project or home Runtime.
|
|
583
|
+
*/
|
|
584
|
+
ingestSyncEvent(event: StoredSyncEvent): SyncIngestOutcome {
|
|
585
|
+
const decide = (): SyncIngestOutcome => {
|
|
586
|
+
const existing = this.readSyncEvent(event.projectId, event.runId, event.sequence);
|
|
587
|
+
if (existing) {
|
|
588
|
+
if (existing.contentHash === event.contentHash && existing.hubProject === event.hubProject) return { outcome: "duplicate" };
|
|
589
|
+
return { outcome: "conflict", reason: "sequence_reused", existingHash: existing.contentHash };
|
|
590
|
+
}
|
|
591
|
+
const owner = this.syncProjectOwner(event.projectId);
|
|
592
|
+
if (owner !== undefined && owner !== event.hubProject) return { outcome: "conflict", reason: "project_mismatch" };
|
|
593
|
+
const home = this.syncRunHome(event.projectId, event.runId);
|
|
594
|
+
if (home !== undefined && home !== event.homeRuntimeId) return { outcome: "conflict", reason: "home_runtime_mismatch" };
|
|
595
|
+
this.writeSyncEvent(event);
|
|
596
|
+
return { outcome: "accepted" };
|
|
597
|
+
};
|
|
598
|
+
return this.database ? withDatabaseTransaction(this.database, decide) : decide();
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
/** Every sync event one hub project holds, by run then sequence. */
|
|
602
|
+
listSyncEvents(hubProject: string): StoredSyncEvent[] {
|
|
603
|
+
if (!this.database) {
|
|
604
|
+
return [...this.syncEvents.values()]
|
|
605
|
+
.filter((event) => event.hubProject === hubProject)
|
|
606
|
+
.sort((left, right) => left.runId.localeCompare(right.runId) || left.sequence - right.sequence);
|
|
607
|
+
}
|
|
608
|
+
const rows = this.database.prepare(`
|
|
609
|
+
SELECT project_id, run_id, sequence, hub_project, home_runtime_id, event_type, content_hash, received_at, record
|
|
610
|
+
FROM sync_events WHERE hub_project = ? ORDER BY run_id ASC, sequence ASC
|
|
611
|
+
`).all(hubProject) as SyncEventSqlRow[];
|
|
612
|
+
return rows.map(syncEventFromRow);
|
|
613
|
+
}
|
|
614
|
+
|
|
615
|
+
/** Highest sequence of one run with no gap below it; 0 when sequence 1 is missing. */
|
|
616
|
+
syncCursor(projectId: string, runId: string): number {
|
|
617
|
+
const sequences = this.database
|
|
618
|
+
? (this.database.prepare("SELECT sequence FROM sync_events WHERE project_id = ? AND run_id = ? ORDER BY sequence ASC")
|
|
619
|
+
.all(projectId, runId) as Array<{ sequence: number }>).map((row) => row.sequence)
|
|
620
|
+
: [...this.syncEvents.values()]
|
|
621
|
+
.filter((event) => event.projectId === projectId && event.runId === runId)
|
|
622
|
+
.map((event) => event.sequence)
|
|
623
|
+
.sort((left, right) => left - right);
|
|
624
|
+
let cursor = 0;
|
|
625
|
+
for (const sequence of sequences) {
|
|
626
|
+
if (sequence !== cursor + 1) break;
|
|
627
|
+
cursor = sequence;
|
|
628
|
+
}
|
|
629
|
+
return cursor;
|
|
630
|
+
}
|
|
631
|
+
|
|
632
|
+
saveRuntimePresence(record: RuntimePresenceRecord): void {
|
|
633
|
+
this.runtimePresence.set(`${record.project}\u0000${record.runtimeId}`, record);
|
|
634
|
+
this.database?.prepare(`
|
|
635
|
+
INSERT INTO runtime_presence (runtime_id, hub_project, host, heartbeat, record) VALUES (?, ?, ?, ?, ?)
|
|
636
|
+
ON CONFLICT(hub_project, runtime_id) DO UPDATE SET
|
|
637
|
+
host = excluded.host,
|
|
638
|
+
heartbeat = excluded.heartbeat,
|
|
639
|
+
record = excluded.record
|
|
640
|
+
`).run(record.runtimeId, record.project, record.host ?? null, record.heartbeatAt, JSON.stringify(record));
|
|
641
|
+
}
|
|
642
|
+
|
|
643
|
+
getRuntimePresence(project: string, runtimeId: string): RuntimePresenceRecord | undefined {
|
|
644
|
+
if (!this.database) return this.runtimePresence.get(`${project}\u0000${runtimeId}`);
|
|
645
|
+
const row = this.database.prepare("SELECT record FROM runtime_presence WHERE hub_project = ? AND runtime_id = ?")
|
|
646
|
+
.get(project, runtimeId) as { record: string } | undefined;
|
|
647
|
+
return row ? JSON.parse(row.record) as RuntimePresenceRecord : undefined;
|
|
648
|
+
}
|
|
649
|
+
|
|
650
|
+
listRuntimePresence(project: string): RuntimePresenceRecord[] {
|
|
651
|
+
if (!this.database) {
|
|
652
|
+
return [...this.runtimePresence.values()]
|
|
653
|
+
.filter((record) => record.project === project)
|
|
654
|
+
.sort((left, right) => left.runtimeId.localeCompare(right.runtimeId));
|
|
655
|
+
}
|
|
656
|
+
const rows = this.database.prepare("SELECT record FROM runtime_presence WHERE hub_project = ? ORDER BY runtime_id ASC")
|
|
657
|
+
.all(project) as Array<{ record: string }>;
|
|
658
|
+
return rows.map((row) => JSON.parse(row.record) as RuntimePresenceRecord);
|
|
659
|
+
}
|
|
660
|
+
|
|
661
|
+
private readSyncEvent(projectId: string, runId: string, sequence: number): StoredSyncEvent | undefined {
|
|
662
|
+
if (!this.database) return this.syncEvents.get(syncKey(projectId, runId, sequence));
|
|
663
|
+
const row = this.database.prepare(`
|
|
664
|
+
SELECT project_id, run_id, sequence, hub_project, home_runtime_id, event_type, content_hash, received_at, record
|
|
665
|
+
FROM sync_events WHERE project_id = ? AND run_id = ? AND sequence = ?
|
|
666
|
+
`).get(projectId, runId, sequence) as SyncEventSqlRow | undefined;
|
|
667
|
+
return row ? syncEventFromRow(row) : undefined;
|
|
668
|
+
}
|
|
669
|
+
|
|
670
|
+
private syncProjectOwner(projectId: string): string | undefined {
|
|
671
|
+
if (!this.database) {
|
|
672
|
+
for (const event of this.syncEvents.values()) if (event.projectId === projectId) return event.hubProject;
|
|
673
|
+
return undefined;
|
|
674
|
+
}
|
|
675
|
+
const row = this.database.prepare("SELECT hub_project FROM sync_events WHERE project_id = ? LIMIT 1").get(projectId) as
|
|
676
|
+
| { hub_project: string }
|
|
677
|
+
| undefined;
|
|
678
|
+
return row?.hub_project;
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
private syncRunHome(projectId: string, runId: string): string | undefined {
|
|
682
|
+
if (!this.database) {
|
|
683
|
+
for (const event of this.syncEvents.values()) {
|
|
684
|
+
if (event.projectId === projectId && event.runId === runId) return event.homeRuntimeId;
|
|
685
|
+
}
|
|
686
|
+
return undefined;
|
|
687
|
+
}
|
|
688
|
+
const row = this.database.prepare("SELECT home_runtime_id FROM sync_events WHERE project_id = ? AND run_id = ? LIMIT 1")
|
|
689
|
+
.get(projectId, runId) as { home_runtime_id: string } | undefined;
|
|
690
|
+
return row?.home_runtime_id;
|
|
691
|
+
}
|
|
692
|
+
|
|
693
|
+
private writeSyncEvent(event: StoredSyncEvent): void {
|
|
694
|
+
if (!this.database) {
|
|
695
|
+
this.syncEvents.set(syncKey(event.projectId, event.runId, event.sequence), event);
|
|
696
|
+
return;
|
|
697
|
+
}
|
|
698
|
+
this.database.prepare(`
|
|
699
|
+
INSERT INTO sync_events (project_id, run_id, sequence, hub_project, home_runtime_id, event_type, content_hash, received_at, record)
|
|
700
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
701
|
+
`).run(
|
|
702
|
+
event.projectId,
|
|
703
|
+
event.runId,
|
|
704
|
+
event.sequence,
|
|
705
|
+
event.hubProject,
|
|
706
|
+
event.homeRuntimeId,
|
|
707
|
+
event.eventType,
|
|
708
|
+
event.contentHash,
|
|
709
|
+
event.receivedAt,
|
|
710
|
+
event.bytes,
|
|
711
|
+
);
|
|
712
|
+
}
|
|
713
|
+
|
|
511
714
|
deleteWorkflowRun(runId: string): void {
|
|
512
715
|
this.workflowRuns.delete(runId);
|
|
513
716
|
this.database?.prepare("DELETE FROM workflow_runs WHERE id = ?").run(runId);
|