@kontextmind/kxm 0.7.71 → 0.7.72
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/CHANGELOG.md +22 -0
- package/docs/operations.md +37 -5
- package/docs/test-matrix.md +1 -0
- package/package.json +1 -1
- package/plugins/kxm/.claude-plugin/plugin.json +1 -1
- package/plugins/kxm/dist/cli.js +42 -2
- package/plugins/kxm/dist/client.js +34 -0
- package/plugins/kxm/dist/core.js +8 -0
- package/plugins/kxm/dist/extension.js +34 -0
- package/plugins/kxm/dist/mcp-server.js +35 -1
- package/plugins/kxm/dist/runtime.js +4 -2
- package/plugins/kxm/dist/server.js +275 -6
- package/plugins/kxm/package.json +1 -1
- package/plugins/kxm/skills/kxm/references/protocol.md +18 -0
- package/plugins/kxm/src/client.ts +39 -1
- package/plugins/kxm/src/database.ts +6 -2
- package/plugins/kxm/src/external-effects.ts +390 -66
- package/plugins/kxm/src/hub.ts +137 -1
- package/plugins/kxm/src/mcp-server.ts +1 -1
- package/plugins/kxm/src/protocol.ts +25 -0
- package/plugins/kxm/src/store.ts +213 -5
package/plugins/kxm/src/hub.ts
CHANGED
|
@@ -4,6 +4,7 @@ import { createServer, type IncomingMessage, type Server, type ServerResponse }
|
|
|
4
4
|
import { isIP } from "node:net";
|
|
5
5
|
import { dirname, join, resolve } from "node:path";
|
|
6
6
|
import {
|
|
7
|
+
DEFAULT_LEASE_TTL_MS,
|
|
7
8
|
DEFAULT_MAX_HOPS,
|
|
8
9
|
DEFAULT_MESSAGE_RETENTION_MS,
|
|
9
10
|
DEFAULT_MESSAGE_TTL_MS,
|
|
@@ -13,7 +14,10 @@ import {
|
|
|
13
14
|
MAX_AGENT_HOST_CHARS,
|
|
14
15
|
MAX_BODY_BYTES,
|
|
15
16
|
MAX_CONTENT_CHARS,
|
|
17
|
+
MAX_LEASE_RESOURCE_CHARS,
|
|
18
|
+
MAX_LEASE_TTL_MS,
|
|
16
19
|
MAX_MESSAGE_TTL_MS,
|
|
20
|
+
MIN_LEASE_TTL_MS,
|
|
17
21
|
MIN_MESSAGE_TTL_MS,
|
|
18
22
|
MIN_MESSAGE_RETENTION_MS,
|
|
19
23
|
ProtocolError,
|
|
@@ -27,6 +31,7 @@ import {
|
|
|
27
31
|
type AgentRecord,
|
|
28
32
|
type DeliveryMode,
|
|
29
33
|
type HubEvent,
|
|
34
|
+
type LeaseRecord,
|
|
30
35
|
type MessageRecord,
|
|
31
36
|
type WorkflowMessageContext,
|
|
32
37
|
} from "./protocol.ts";
|
|
@@ -39,7 +44,7 @@ import { NativeStateProvider } from "./state.ts";
|
|
|
39
44
|
import { SkillLifecycle } from "./skills.ts";
|
|
40
45
|
import { compileKnowledgeWiki, lintKnowledgeWiki, type WikiSourcePool } from "./wiki.ts";
|
|
41
46
|
import { buildRetrospective, writeRetrospective } from "./retrospective.ts";
|
|
42
|
-
import { MeshStore, type StoredAgent } from "./store.ts";
|
|
47
|
+
import { MeshStore, type LeaseRefusal, type StoredAgent } from "./store.ts";
|
|
43
48
|
import {
|
|
44
49
|
canonicalWorkflowEvidenceKey,
|
|
45
50
|
approveWorkflowDegradation,
|
|
@@ -91,6 +96,10 @@ export interface MeshHubOptions {
|
|
|
91
96
|
skillsDir?: string;
|
|
92
97
|
skillLifecycle?: SkillLifecycle;
|
|
93
98
|
repoRoot?: string;
|
|
99
|
+
/** The hub clock every lease decision reads. Injectable so a test can move a
|
|
100
|
+
* lease past its deadline instead of sleeping through a real TTL; production
|
|
101
|
+
* leaves it at `Date.now`. It is never a client-supplied time. */
|
|
102
|
+
now?: () => number;
|
|
94
103
|
}
|
|
95
104
|
|
|
96
105
|
export interface MeshHub {
|
|
@@ -132,6 +141,38 @@ function safeTokenEqual(actual: string | undefined, expected: string): boolean {
|
|
|
132
141
|
return timingSafeStringCompare(actual, expected);
|
|
133
142
|
}
|
|
134
143
|
|
|
144
|
+
/** The wire answer for a lease the hub will not extend or release. Every one of
|
|
145
|
+
* these means the caller's token no longer authorizes a shared write: the
|
|
146
|
+
* holder must stop, not retry. The current lease rides along so the caller can
|
|
147
|
+
* record who holds the resource now. */
|
|
148
|
+
function leaseRefusal(reason: LeaseRefusal, name: string, lease?: LeaseRecord): ProtocolError {
|
|
149
|
+
if (reason === "missing") {
|
|
150
|
+
return new ProtocolError(404, `no lease is held on ${name}`, "lease_not_found");
|
|
151
|
+
}
|
|
152
|
+
if (reason === "expired") {
|
|
153
|
+
return new ProtocolError(
|
|
154
|
+
409,
|
|
155
|
+
`lease on ${name} expired at ${lease?.expiresAt ?? "its deadline"}; re-acquire to take it over`,
|
|
156
|
+
"lease_expired",
|
|
157
|
+
{ lease },
|
|
158
|
+
);
|
|
159
|
+
}
|
|
160
|
+
if (reason === "held") {
|
|
161
|
+
return new ProtocolError(
|
|
162
|
+
409,
|
|
163
|
+
`lease on ${name} is held by ${lease?.holderAgentName ?? "another agent"} until ${lease?.expiresAt ?? "its deadline"}`,
|
|
164
|
+
"lease_held",
|
|
165
|
+
{ lease },
|
|
166
|
+
);
|
|
167
|
+
}
|
|
168
|
+
return new ProtocolError(
|
|
169
|
+
409,
|
|
170
|
+
`fencing token for ${name} was superseded; the hub is now at token ${lease?.fencingToken ?? "a newer value"}`,
|
|
171
|
+
"lease_superseded",
|
|
172
|
+
{ lease },
|
|
173
|
+
);
|
|
174
|
+
}
|
|
175
|
+
|
|
135
176
|
function bearerToken(request: IncomingMessage): string | undefined {
|
|
136
177
|
const header = request.headers.authorization;
|
|
137
178
|
return header?.startsWith("Bearer ") ? header.slice(7) : undefined;
|
|
@@ -409,6 +450,7 @@ export function createMeshHub(options: MeshHubOptions = {}): MeshHub {
|
|
|
409
450
|
};
|
|
410
451
|
const webhookWorkflows = new Map((options.webhookWorkflows ?? []).map((workflow) => [workflow.id, workflow]));
|
|
411
452
|
const logger = options.logger ?? (() => undefined);
|
|
453
|
+
const hubNow = options.now ?? (() => Date.now());
|
|
412
454
|
const assetsDir = options.assetsDir;
|
|
413
455
|
const hubRepoRoot = options.repoRoot ?? (options.dataPath && options.dataPath !== ":memory:" ? resolve(dirname(dirname(options.dataPath))) : process.cwd());
|
|
414
456
|
const store = new MeshStore(options.dataPath);
|
|
@@ -461,6 +503,9 @@ export function createMeshHub(options: MeshHubOptions = {}): MeshHub {
|
|
|
461
503
|
contextRequests: 0,
|
|
462
504
|
attemptLatencySecondsTotal: 0,
|
|
463
505
|
meteredCostUsdTotal: 0,
|
|
506
|
+
leasesGranted: 0,
|
|
507
|
+
leasesRefused: 0,
|
|
508
|
+
leasesReleased: 0,
|
|
464
509
|
};
|
|
465
510
|
let cleanupTimer: NodeJS.Timeout | undefined;
|
|
466
511
|
let closed = false;
|
|
@@ -1065,6 +1110,16 @@ export function createMeshHub(options: MeshHubOptions = {}): MeshHub {
|
|
|
1065
1110
|
for (const runId of swept.purgedRuns) {
|
|
1066
1111
|
logger({ event: "workflow_run_purged", runId });
|
|
1067
1112
|
}
|
|
1113
|
+
for (const lease of swept.purgedLeases) {
|
|
1114
|
+
logger({
|
|
1115
|
+
event: "lease_purged",
|
|
1116
|
+
resource: lease.resource,
|
|
1117
|
+
project: lease.project,
|
|
1118
|
+
agentId: lease.holderAgentId,
|
|
1119
|
+
fencingToken: lease.fencingToken,
|
|
1120
|
+
expiresAt: lease.expiresAt,
|
|
1121
|
+
});
|
|
1122
|
+
}
|
|
1068
1123
|
}
|
|
1069
1124
|
|
|
1070
1125
|
function metricsBody(): string {
|
|
@@ -1112,6 +1167,12 @@ export function createMeshHub(options: MeshHubOptions = {}): MeshHub {
|
|
|
1112
1167
|
`kxm_attempt_latency_seconds_total ${counters.attemptLatencySecondsTotal}`,
|
|
1113
1168
|
"# TYPE kxm_metered_cost_usd_total counter",
|
|
1114
1169
|
`kxm_metered_cost_usd_total ${counters.meteredCostUsdTotal}`,
|
|
1170
|
+
"# TYPE kxm_leases_granted_total counter",
|
|
1171
|
+
`kxm_leases_granted_total ${counters.leasesGranted}`,
|
|
1172
|
+
"# TYPE kxm_leases_refused_total counter",
|
|
1173
|
+
`kxm_leases_refused_total ${counters.leasesRefused}`,
|
|
1174
|
+
"# TYPE kxm_leases_released_total counter",
|
|
1175
|
+
`kxm_leases_released_total ${counters.leasesReleased}`,
|
|
1115
1176
|
"",
|
|
1116
1177
|
].join("\n");
|
|
1117
1178
|
}
|
|
@@ -2198,6 +2259,81 @@ export function createMeshHub(options: MeshHubOptions = {}): MeshHub {
|
|
|
2198
2259
|
return;
|
|
2199
2260
|
}
|
|
2200
2261
|
|
|
2262
|
+
// One writer, one clock. Every branch below decides inside a single store
|
|
2263
|
+
// transaction, so two agents racing for the same resource cannot both read it
|
|
2264
|
+
// free. The fencing token increments only on takeover, which is what lets a
|
|
2265
|
+
// holder that slept past its deadline be refused at commit instead of writing
|
|
2266
|
+
// behind whoever replaced it.
|
|
2267
|
+
const leaseMatch = url.pathname.match(/^\/v1\/leases\/([^/]+)\/(acquire|renew|release)$/);
|
|
2268
|
+
if (method === "POST" && leaseMatch) {
|
|
2269
|
+
const holder = requireAgent(request);
|
|
2270
|
+
requireProjectAuth(request, holder.project);
|
|
2271
|
+
const name = requireString(decodeURIComponent(leaseMatch[1]!), "resource", { max: MAX_LEASE_RESOURCE_CHARS });
|
|
2272
|
+
const action = leaseMatch[2]!;
|
|
2273
|
+
const body = await readJson(request);
|
|
2274
|
+
// The caller never names the prefix, so the same branch in two projects is
|
|
2275
|
+
// two leases and neither project can reach the other's.
|
|
2276
|
+
const resource = `${holder.project}/${name}`;
|
|
2277
|
+
const nowMs = hubNow();
|
|
2278
|
+
const leaseLog = { resource, project: holder.project, agentId: holder.id, agentName: holder.name };
|
|
2279
|
+
|
|
2280
|
+
if (action === "acquire") {
|
|
2281
|
+
const ttlMs = parseBoundedInteger(body.ttlMs, "ttlMs", DEFAULT_LEASE_TTL_MS, MIN_LEASE_TTL_MS, MAX_LEASE_TTL_MS);
|
|
2282
|
+
const outcome = store.acquireLease({
|
|
2283
|
+
resource,
|
|
2284
|
+
project: holder.project,
|
|
2285
|
+
name,
|
|
2286
|
+
holderAgentId: holder.id,
|
|
2287
|
+
holderAgentName: holder.name,
|
|
2288
|
+
ttlMs,
|
|
2289
|
+
nowMs,
|
|
2290
|
+
});
|
|
2291
|
+
if (!outcome.ok) {
|
|
2292
|
+
counters.leasesRefused += 1;
|
|
2293
|
+
logger({ event: "lease_denied", ...leaseLog, reason: outcome.reason, heldBy: outcome.lease?.holderAgentId });
|
|
2294
|
+
throw leaseRefusal(outcome.reason, name, outcome.lease);
|
|
2295
|
+
}
|
|
2296
|
+
counters.leasesGranted += 1;
|
|
2297
|
+
logger({
|
|
2298
|
+
event: outcome.renewed ? "lease_renewed" : "lease_acquired",
|
|
2299
|
+
...leaseLog,
|
|
2300
|
+
fencingToken: outcome.lease.fencingToken,
|
|
2301
|
+
expiresAt: outcome.lease.expiresAt,
|
|
2302
|
+
});
|
|
2303
|
+
json(response, 200, { lease: outcome.lease, renewed: outcome.renewed });
|
|
2304
|
+
return;
|
|
2305
|
+
}
|
|
2306
|
+
|
|
2307
|
+
if (body.fencingToken === undefined) {
|
|
2308
|
+
throw new ProtocolError(400, "fencingToken is required", "lease_token_required");
|
|
2309
|
+
}
|
|
2310
|
+
const fencingToken = parseBoundedInteger(body.fencingToken, "fencingToken", 1, 1, Number.MAX_SAFE_INTEGER);
|
|
2311
|
+
if (action === "renew") {
|
|
2312
|
+
const ttlMs = parseBoundedInteger(body.ttlMs, "ttlMs", DEFAULT_LEASE_TTL_MS, MIN_LEASE_TTL_MS, MAX_LEASE_TTL_MS);
|
|
2313
|
+
const outcome = store.renewLease({ resource, holderAgentId: holder.id, fencingToken, ttlMs, nowMs });
|
|
2314
|
+
if (!outcome.ok) {
|
|
2315
|
+
counters.leasesRefused += 1;
|
|
2316
|
+
logger({ event: "lease_denied", ...leaseLog, reason: outcome.reason, fencingToken });
|
|
2317
|
+
throw leaseRefusal(outcome.reason, name, outcome.lease);
|
|
2318
|
+
}
|
|
2319
|
+
counters.leasesGranted += 1;
|
|
2320
|
+
logger({ event: "lease_renewed", ...leaseLog, fencingToken, expiresAt: outcome.lease.expiresAt });
|
|
2321
|
+
json(response, 200, { lease: outcome.lease, renewed: true });
|
|
2322
|
+
return;
|
|
2323
|
+
}
|
|
2324
|
+
|
|
2325
|
+
const outcome = store.releaseLease({ resource, holderAgentId: holder.id, fencingToken });
|
|
2326
|
+
if (!outcome.ok) {
|
|
2327
|
+
counters.leasesRefused += 1;
|
|
2328
|
+
logger({ event: "lease_denied", ...leaseLog, reason: outcome.reason, fencingToken });
|
|
2329
|
+
throw leaseRefusal(outcome.reason, name, outcome.lease);
|
|
2330
|
+
}
|
|
2331
|
+
counters.leasesReleased += 1;
|
|
2332
|
+
logger({ event: "lease_released", ...leaseLog, fencingToken });
|
|
2333
|
+
json(response, 200, { released: true, lease: outcome.lease });
|
|
2334
|
+
return;
|
|
2335
|
+
}
|
|
2336
|
+
|
|
2201
2337
|
if (method === "GET" && url.pathname === "/v1/events") {
|
|
2202
2338
|
const agentId = requireString(url.searchParams.get("agentId"), "agentId", { max: 80 });
|
|
2203
2339
|
const current = requireAgent(request, agentId);
|
|
@@ -8,7 +8,7 @@ import { AGENT_COMMANDS_MAP, enforceToolPolicy, getMcpTools, reconcileInbox } fr
|
|
|
8
8
|
import { deliverInboxNotification } from "./inbox.ts";
|
|
9
9
|
import type { HubEvent, MessageRecord } from "./protocol.ts";
|
|
10
10
|
|
|
11
|
-
const VERSION = "0.7.
|
|
11
|
+
const VERSION = "0.7.72";
|
|
12
12
|
const inbox = new Map<string, MessageRecord>();
|
|
13
13
|
const notifiedInbox = new Set<string>();
|
|
14
14
|
let meshClient: HubClient | undefined;
|
|
@@ -13,6 +13,10 @@ export const DEFAULT_RATE_LIMIT_WINDOW_MS = 60_000;
|
|
|
13
13
|
export const MAX_BODY_BYTES = 256 * 1024;
|
|
14
14
|
export const MAX_CONTENT_CHARS = 32_000;
|
|
15
15
|
export const MAX_AGENT_HOST_CHARS = 64;
|
|
16
|
+
export const MIN_LEASE_TTL_MS = 5_000;
|
|
17
|
+
export const MAX_LEASE_TTL_MS = 10 * 60_000;
|
|
18
|
+
export const DEFAULT_LEASE_TTL_MS = 5 * 60_000;
|
|
19
|
+
export const MAX_LEASE_RESOURCE_CHARS = 200;
|
|
16
20
|
|
|
17
21
|
export type DeliveryMode = "steer" | "followUp" | "nextTurn";
|
|
18
22
|
export type MessageStatus = "queued" | "delivered" | "replied" | "cancelled" | "expired" | "error";
|
|
@@ -68,6 +72,27 @@ export function toAgentRecord(agent: AgentIdentity, staleAfterMs?: number, now?:
|
|
|
68
72
|
return { ...agent, ...agentPresenceView(agent, staleAfterMs, now) };
|
|
69
73
|
}
|
|
70
74
|
|
|
75
|
+
/** One hub-held fenced lease over a project-scoped resource.
|
|
76
|
+
*
|
|
77
|
+
* `resource` is what the store keys on: the caller's project prefixed onto the
|
|
78
|
+
* name it asked for, so two projects naming the same branch never contend.
|
|
79
|
+
* Expiry is the hub's clock, never the holder's, and `fencingToken` increments
|
|
80
|
+
* only when a new holder takes over an expired lease — a renewal keeps its
|
|
81
|
+
* token, so a writer that comes back after a takeover can be told its token
|
|
82
|
+
* was superseded instead of being allowed to commit. */
|
|
83
|
+
export interface LeaseRecord {
|
|
84
|
+
/** `${project}/${name}`. */
|
|
85
|
+
resource: string;
|
|
86
|
+
project: string;
|
|
87
|
+
/** The resource as the caller named it, without the project prefix. */
|
|
88
|
+
name: string;
|
|
89
|
+
holderAgentId: string;
|
|
90
|
+
holderAgentName: string;
|
|
91
|
+
fencingToken: number;
|
|
92
|
+
acquiredAt: string;
|
|
93
|
+
expiresAt: string;
|
|
94
|
+
}
|
|
95
|
+
|
|
71
96
|
export interface MessageReply {
|
|
72
97
|
content: string;
|
|
73
98
|
createdAt: string;
|
package/plugins/kxm/src/store.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { resolve } from "node:path";
|
|
2
2
|
import type { DatabaseSync } from "./sqlite.ts";
|
|
3
|
-
import type { AgentIdentity, MessageRecord } from "./protocol.ts";
|
|
3
|
+
import type { AgentIdentity, LeaseRecord, MessageRecord } from "./protocol.ts";
|
|
4
4
|
import type { ContextItem } from "./context.ts";
|
|
5
5
|
import type { WorkflowJournalEntry, WorkflowRun } from "./workflow.ts";
|
|
6
6
|
import {
|
|
@@ -9,7 +9,7 @@ import {
|
|
|
9
9
|
type DatabaseSchemaSpec,
|
|
10
10
|
} from "./database.ts";
|
|
11
11
|
|
|
12
|
-
export const HUB_STORE_SCHEMA_VERSION =
|
|
12
|
+
export const HUB_STORE_SCHEMA_VERSION = 4;
|
|
13
13
|
|
|
14
14
|
export const HUB_STORE_TABLES: Readonly<Record<string, readonly string[]>> = Object.freeze({
|
|
15
15
|
agents: Object.freeze(["id", "record"]),
|
|
@@ -19,9 +19,10 @@ export const HUB_STORE_TABLES: Readonly<Record<string, readonly string[]>> = Obj
|
|
|
19
19
|
workflow_runs: Object.freeze(["id", "definition_id", "delivery_id", "record"]),
|
|
20
20
|
workflow_journal: Object.freeze(["id", "run_id", "category", "area", "record"]),
|
|
21
21
|
context_items: Object.freeze(["id", "project", "kind", "record"]),
|
|
22
|
+
leases: Object.freeze(["resource", "holder_agent_id", "fencing_token", "expires_at", "record"]),
|
|
22
23
|
});
|
|
23
24
|
|
|
24
|
-
export const
|
|
25
|
+
export const HUB_STORE_SCHEMA_V4 = `
|
|
25
26
|
CREATE TABLE IF NOT EXISTS agents (
|
|
26
27
|
id TEXT PRIMARY KEY,
|
|
27
28
|
record TEXT NOT NULL
|
|
@@ -70,10 +71,17 @@ export const HUB_STORE_SCHEMA_V3 = `
|
|
|
70
71
|
record TEXT NOT NULL
|
|
71
72
|
) STRICT;
|
|
72
73
|
CREATE INDEX IF NOT EXISTS context_items_project ON context_items(project);
|
|
74
|
+
CREATE TABLE IF NOT EXISTS leases (
|
|
75
|
+
resource TEXT PRIMARY KEY,
|
|
76
|
+
holder_agent_id TEXT NOT NULL,
|
|
77
|
+
fencing_token INTEGER NOT NULL,
|
|
78
|
+
expires_at TEXT NOT NULL,
|
|
79
|
+
record TEXT NOT NULL
|
|
80
|
+
) STRICT;
|
|
73
81
|
`;
|
|
74
82
|
|
|
75
83
|
export const HUB_STORE_SCHEMA_SPEC: DatabaseSchemaSpec = Object.freeze({
|
|
76
|
-
schema:
|
|
84
|
+
schema: HUB_STORE_SCHEMA_V4,
|
|
77
85
|
version: HUB_STORE_SCHEMA_VERSION,
|
|
78
86
|
tables: HUB_STORE_TABLES,
|
|
79
87
|
});
|
|
@@ -85,6 +93,40 @@ export interface StoredAgent extends AgentIdentity {
|
|
|
85
93
|
key: string;
|
|
86
94
|
}
|
|
87
95
|
|
|
96
|
+
/** Why a lease call did not get the resource. `held` is live contention,
|
|
97
|
+
* `superseded` is a token the hub has already moved past, `expired` is a lease
|
|
98
|
+
* whose hub-clocked deadline has passed, and `missing` is no lease at all. */
|
|
99
|
+
export type LeaseRefusal = "held" | "superseded" | "expired" | "missing";
|
|
100
|
+
|
|
101
|
+
export type LeaseOutcome =
|
|
102
|
+
| { ok: true; lease: LeaseRecord; renewed: boolean }
|
|
103
|
+
| { ok: false; reason: LeaseRefusal; lease?: LeaseRecord };
|
|
104
|
+
|
|
105
|
+
export interface LeaseAcquireInput {
|
|
106
|
+
resource: string;
|
|
107
|
+
project: string;
|
|
108
|
+
name: string;
|
|
109
|
+
holderAgentId: string;
|
|
110
|
+
holderAgentName: string;
|
|
111
|
+
ttlMs: number;
|
|
112
|
+
nowMs: number;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
function parseLeaseRecord(record: string): LeaseRecord | undefined {
|
|
116
|
+
try {
|
|
117
|
+
return JSON.parse(record) as LeaseRecord;
|
|
118
|
+
} catch {
|
|
119
|
+
return undefined;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/** A deadline the hub cannot read is an expired deadline. The alternative — a
|
|
124
|
+
* lease nobody can ever take over — is the worse failure. */
|
|
125
|
+
function leaseHasLapsed(lease: LeaseRecord, nowMs: number): boolean {
|
|
126
|
+
const expiresAtMs = Date.parse(lease.expiresAt);
|
|
127
|
+
return !Number.isFinite(expiresAtMs) || expiresAtMs <= nowMs;
|
|
128
|
+
}
|
|
129
|
+
|
|
88
130
|
class MessageMap extends Map<string, MessageRecord> {
|
|
89
131
|
private readonly store: MeshStore;
|
|
90
132
|
|
|
@@ -124,6 +166,7 @@ export class MeshStore {
|
|
|
124
166
|
readonly workflowRuns = new Map<string, WorkflowRun>();
|
|
125
167
|
readonly journal = new Map<string, WorkflowJournalEntry>();
|
|
126
168
|
readonly contextItems = new Map<string, ContextItem>();
|
|
169
|
+
readonly leases = new Map<string, LeaseRecord>();
|
|
127
170
|
readonly path?: string;
|
|
128
171
|
private readonly database?: DatabaseSync;
|
|
129
172
|
private readonly agentSequences = new Map<string, number>();
|
|
@@ -331,6 +374,140 @@ export class MeshStore {
|
|
|
331
374
|
return result;
|
|
332
375
|
}
|
|
333
376
|
|
|
377
|
+
/** Read the lease over one already project-scoped resource. */
|
|
378
|
+
getLease(resource: string): LeaseRecord | undefined {
|
|
379
|
+
if (!this.database) return this.leases.get(resource);
|
|
380
|
+
const row = this.database.prepare("SELECT record FROM leases WHERE resource = ?").get(resource) as { record: string } | undefined;
|
|
381
|
+
if (!row) {
|
|
382
|
+
this.leases.delete(resource);
|
|
383
|
+
return undefined;
|
|
384
|
+
}
|
|
385
|
+
const lease = parseLeaseRecord(row.record);
|
|
386
|
+
if (lease) this.leases.set(resource, lease);
|
|
387
|
+
return lease;
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
/**
|
|
391
|
+
* Take or extend the lease over `resource`, deciding the whole outcome inside
|
|
392
|
+
* one transaction so two writers cannot both read "free" and both insert.
|
|
393
|
+
*
|
|
394
|
+
* The token is the fence: a first acquisition starts at 1, the holder's own
|
|
395
|
+
* re-acquisition keeps its token, and only a takeover of an expired lease
|
|
396
|
+
* increments it. That is what lets a holder that wakes up after its deadline
|
|
397
|
+
* be refused at commit rather than silently writing behind the new holder.
|
|
398
|
+
*/
|
|
399
|
+
acquireLease(input: LeaseAcquireInput): LeaseOutcome {
|
|
400
|
+
return this.inLeaseTransaction(() => {
|
|
401
|
+
const current = this.readLeaseForUpdate(input.resource);
|
|
402
|
+
const expired = current !== undefined && leaseHasLapsed(current, input.nowMs);
|
|
403
|
+
if (current && !expired && current.holderAgentId !== input.holderAgentId) {
|
|
404
|
+
return { ok: false, reason: "held", lease: current };
|
|
405
|
+
}
|
|
406
|
+
const fencingToken = current === undefined
|
|
407
|
+
? 1
|
|
408
|
+
: expired
|
|
409
|
+
? current.fencingToken + 1
|
|
410
|
+
: current.fencingToken;
|
|
411
|
+
const renewed = current !== undefined && !expired;
|
|
412
|
+
const lease: LeaseRecord = {
|
|
413
|
+
resource: input.resource,
|
|
414
|
+
project: input.project,
|
|
415
|
+
name: input.name,
|
|
416
|
+
holderAgentId: input.holderAgentId,
|
|
417
|
+
holderAgentName: input.holderAgentName,
|
|
418
|
+
fencingToken,
|
|
419
|
+
acquiredAt: renewed && current ? current.acquiredAt : new Date(input.nowMs).toISOString(),
|
|
420
|
+
expiresAt: new Date(input.nowMs + input.ttlMs).toISOString(),
|
|
421
|
+
};
|
|
422
|
+
this.writeLease(lease);
|
|
423
|
+
return { ok: true, lease, renewed };
|
|
424
|
+
});
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
/** Extend a lease the caller still holds under the token it was given. The
|
|
428
|
+
* token never changes on renewal — a renewal that would need a new token is
|
|
429
|
+
* a takeover, and takeovers go through `acquireLease`. */
|
|
430
|
+
renewLease(input: {
|
|
431
|
+
resource: string;
|
|
432
|
+
holderAgentId: string;
|
|
433
|
+
fencingToken: number;
|
|
434
|
+
ttlMs: number;
|
|
435
|
+
nowMs: number;
|
|
436
|
+
}): LeaseOutcome {
|
|
437
|
+
return this.inLeaseTransaction(() => {
|
|
438
|
+
const current = this.readLeaseForUpdate(input.resource);
|
|
439
|
+
if (!current) return { ok: false, reason: "missing" };
|
|
440
|
+
if (current.holderAgentId !== input.holderAgentId || current.fencingToken !== input.fencingToken) {
|
|
441
|
+
return { ok: false, reason: "superseded", lease: current };
|
|
442
|
+
}
|
|
443
|
+
if (leaseHasLapsed(current, input.nowMs)) {
|
|
444
|
+
// The deadline passed, so the resource is takeable by anyone. Resurrecting it
|
|
445
|
+
// under the old token would let a second holder appear behind the first.
|
|
446
|
+
return { ok: false, reason: "expired", lease: current };
|
|
447
|
+
}
|
|
448
|
+
const lease: LeaseRecord = { ...current, expiresAt: new Date(input.nowMs + input.ttlMs).toISOString() };
|
|
449
|
+
this.writeLease(lease);
|
|
450
|
+
return { ok: true, lease, renewed: true };
|
|
451
|
+
});
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
/** Drop a lease the caller holds. A clean release ends the fence: there is no
|
|
455
|
+
* stale writer left to keep a token for, so the next acquisition starts over. */
|
|
456
|
+
releaseLease(input: { resource: string; holderAgentId: string; fencingToken: number }): LeaseOutcome {
|
|
457
|
+
return this.inLeaseTransaction(() => {
|
|
458
|
+
const current = this.readLeaseForUpdate(input.resource);
|
|
459
|
+
if (!current) return { ok: false, reason: "missing" };
|
|
460
|
+
if (current.holderAgentId !== input.holderAgentId || current.fencingToken !== input.fencingToken) {
|
|
461
|
+
return { ok: false, reason: "superseded", lease: current };
|
|
462
|
+
}
|
|
463
|
+
this.deleteLease(input.resource);
|
|
464
|
+
return { ok: true, lease: current, renewed: false };
|
|
465
|
+
});
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
/** Every lease of one project, newest deadline last. Reader surface only. */
|
|
469
|
+
listLeases(project: string): LeaseRecord[] {
|
|
470
|
+
if (this.database) {
|
|
471
|
+
const rows = this.database.prepare("SELECT record FROM leases").all() as Array<{ record: string }>;
|
|
472
|
+
this.leases.clear();
|
|
473
|
+
for (const row of rows) {
|
|
474
|
+
const lease = parseLeaseRecord(row.record);
|
|
475
|
+
if (lease) this.leases.set(lease.resource, lease);
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
return [...this.leases.values()]
|
|
479
|
+
.filter((lease) => lease.project === project)
|
|
480
|
+
.sort((left, right) => left.resource.localeCompare(right.resource));
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
private inLeaseTransaction(work: () => LeaseOutcome): LeaseOutcome {
|
|
484
|
+
if (!this.database) return work();
|
|
485
|
+
return withDatabaseTransaction(this.database, work);
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
private readLeaseForUpdate(resource: string): LeaseRecord | undefined {
|
|
489
|
+
if (!this.database) return this.leases.get(resource);
|
|
490
|
+
const row = this.database.prepare("SELECT record FROM leases WHERE resource = ?").get(resource) as { record: string } | undefined;
|
|
491
|
+
return row ? parseLeaseRecord(row.record) : undefined;
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
private writeLease(lease: LeaseRecord): void {
|
|
495
|
+
this.leases.set(lease.resource, lease);
|
|
496
|
+
this.database?.prepare(`
|
|
497
|
+
INSERT INTO leases (resource, holder_agent_id, fencing_token, expires_at, record) VALUES (?, ?, ?, ?, ?)
|
|
498
|
+
ON CONFLICT(resource) DO UPDATE SET
|
|
499
|
+
holder_agent_id = excluded.holder_agent_id,
|
|
500
|
+
fencing_token = excluded.fencing_token,
|
|
501
|
+
expires_at = excluded.expires_at,
|
|
502
|
+
record = excluded.record
|
|
503
|
+
`).run(lease.resource, lease.holderAgentId, lease.fencingToken, lease.expiresAt, JSON.stringify(lease));
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
private deleteLease(resource: string): void {
|
|
507
|
+
this.leases.delete(resource);
|
|
508
|
+
this.database?.prepare("DELETE FROM leases WHERE resource = ?").run(resource);
|
|
509
|
+
}
|
|
510
|
+
|
|
334
511
|
deleteWorkflowRun(runId: string): void {
|
|
335
512
|
this.workflowRuns.delete(runId);
|
|
336
513
|
this.database?.prepare("DELETE FROM workflow_runs WHERE id = ?").run(runId);
|
|
@@ -355,6 +532,7 @@ export class MeshStore {
|
|
|
355
532
|
purgedRuns: string[];
|
|
356
533
|
purgedJournal: string[];
|
|
357
534
|
purgedContextItems: string[];
|
|
535
|
+
purgedLeases: LeaseRecord[];
|
|
358
536
|
} {
|
|
359
537
|
const messageCutoffMs = nowMs - messageRetentionMs;
|
|
360
538
|
const messageCutoffIso = new Date(messageCutoffMs).toISOString();
|
|
@@ -363,6 +541,7 @@ export class MeshStore {
|
|
|
363
541
|
const purgedRuns: string[] = [];
|
|
364
542
|
const purgedJournal: string[] = [];
|
|
365
543
|
const purgedContextItems: string[] = [];
|
|
544
|
+
const purgedLeases: LeaseRecord[] = [];
|
|
366
545
|
|
|
367
546
|
// 1. Messages
|
|
368
547
|
if (!this.database) {
|
|
@@ -441,7 +620,31 @@ export class MeshStore {
|
|
|
441
620
|
purgedContextItems.push(id);
|
|
442
621
|
}
|
|
443
622
|
|
|
444
|
-
|
|
623
|
+
// 5. Leases. An expired lease row is kept on purpose — its token is what a
|
|
624
|
+
// takeover increments, so reaping it the moment it lapses would let the
|
|
625
|
+
// fence restart at 1 while a stale holder was still alive. Only rows whose
|
|
626
|
+
// deadline is older than the retention window, far beyond any live holder's
|
|
627
|
+
// TTL, are dropped.
|
|
628
|
+
const leaseCutoffMs = nowMs - runRetentionMs;
|
|
629
|
+
for (const lease of this.listAllLeases()) {
|
|
630
|
+
if (leaseHasLapsed(lease, leaseCutoffMs)) {
|
|
631
|
+
this.deleteLease(lease.resource);
|
|
632
|
+
purgedLeases.push(lease);
|
|
633
|
+
}
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
return { purgedMessages, purgedRuns, purgedJournal, purgedContextItems, purgedLeases };
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
private listAllLeases(): LeaseRecord[] {
|
|
640
|
+
if (!this.database) return [...this.leases.values()];
|
|
641
|
+
const rows = this.database.prepare("SELECT record FROM leases").all() as Array<{ record: string }>;
|
|
642
|
+
const result: LeaseRecord[] = [];
|
|
643
|
+
for (const row of rows) {
|
|
644
|
+
const lease = parseLeaseRecord(row.record);
|
|
645
|
+
if (lease) result.push(lease);
|
|
646
|
+
}
|
|
647
|
+
return result;
|
|
445
648
|
}
|
|
446
649
|
|
|
447
650
|
saveWorkflowRun(run: WorkflowRun): void {
|
|
@@ -528,6 +731,7 @@ export class MeshStore {
|
|
|
528
731
|
const workflowRows = this.database.prepare("SELECT record FROM workflow_runs").all() as Array<{ record: string }>;
|
|
529
732
|
const journalRows = this.database.prepare("SELECT record FROM workflow_journal").all() as Array<{ record: string }>;
|
|
530
733
|
const contextRows = this.database.prepare("SELECT record FROM context_items").all() as Array<{ record: string }>;
|
|
734
|
+
const leaseRows = this.database.prepare("SELECT record FROM leases").all() as Array<{ record: string }>;
|
|
531
735
|
for (const row of agentRows) {
|
|
532
736
|
const agent = JSON.parse(row.record) as StoredAgent;
|
|
533
737
|
this.agents.set(agent.id, agent);
|
|
@@ -544,5 +748,9 @@ export class MeshStore {
|
|
|
544
748
|
const item = JSON.parse(row.record) as ContextItem;
|
|
545
749
|
this.contextItems.set(item.id, item);
|
|
546
750
|
}
|
|
751
|
+
for (const row of leaseRows) {
|
|
752
|
+
const lease = parseLeaseRecord(row.record);
|
|
753
|
+
if (lease) this.leases.set(lease.resource, lease);
|
|
754
|
+
}
|
|
547
755
|
}
|
|
548
756
|
}
|