@sleep2agi/commhub-server 0.9.0-preview.30 → 0.9.0-preview.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/package-version-consistency.test.ts +25 -0
- package/src/server.ts +17 -1
- package/src/side-thread-command-transport.test.ts +162 -0
- package/src/side-thread-command-transport.ts +11 -2
- package/src/side-thread-production.test.ts +72 -0
- package/src/side-thread-production.ts +118 -0
- package/src/side-thread.test.ts +40 -11
- package/src/side-thread.ts +313 -59
- package/src/tools.ts +10 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sleep2agi/commhub-server",
|
|
3
|
-
"version": "0.9.0-preview.
|
|
3
|
+
"version": "0.9.0-preview.31",
|
|
4
4
|
"description": "CommHub Server — AI Agent communication hub with MCP protocol, multi-network isolation, user auth, and MCP tools (17 collaboration-core + node/provider ops tools; authoritative list: docs-site/docs/api/mcp-tools.md).",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.ts",
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { describe, expect, test } from "bun:test";
|
|
2
|
+
import { readFileSync } from "node:fs";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
|
|
5
|
+
type PackageManifest = { version: string };
|
|
6
|
+
type PackageLock = {
|
|
7
|
+
version: string;
|
|
8
|
+
packages: { "": { version: string } };
|
|
9
|
+
};
|
|
10
|
+
|
|
11
|
+
const readJson = <T>(name: string): T =>
|
|
12
|
+
JSON.parse(readFileSync(join(import.meta.dir, "..", name), "utf8")) as T;
|
|
13
|
+
|
|
14
|
+
describe("server package version consistency", () => {
|
|
15
|
+
const manifest = readJson<PackageManifest>("package.json");
|
|
16
|
+
const lock = readJson<PackageLock>("package-lock.json");
|
|
17
|
+
|
|
18
|
+
test("package-lock top-level version matches package.json", () => {
|
|
19
|
+
expect(lock.version).toBe(manifest.version);
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
test("package-lock root package version matches package.json", () => {
|
|
23
|
+
expect(lock.packages[""].version).toBe(manifest.version);
|
|
24
|
+
});
|
|
25
|
+
});
|
package/src/server.ts
CHANGED
|
@@ -51,6 +51,7 @@ import { handleExternalScheduleEditRequest } from "./external-schedule-edits.js"
|
|
|
51
51
|
import { recordDeliveredStaleEvents } from "./task-lifecycle-watcher.js";
|
|
52
52
|
import { SIDE_THREAD_FEATURE_FLAG, SideThreadCoordinator, SideThreadPortRegistry, SideThreadStore, type SideThreadActor, type SideThreadAttachmentRef, type SideThreadExecutionPort } from "./side-thread.js";
|
|
53
53
|
import { handleSideThreadHttpRequest } from "./side-thread-http.js";
|
|
54
|
+
import { createProductionSideThreadTransport } from "./side-thread-production.js";
|
|
54
55
|
|
|
55
56
|
const PORT = resolvePort(process.env.PORT);
|
|
56
57
|
const HOST = process.env.HOST || "127.0.0.1";
|
|
@@ -77,6 +78,10 @@ const sideThreadCoordinator = new SideThreadCoordinator(new SideThreadStore(db),
|
|
|
77
78
|
authorizeNode: (actor, networkId, nodeId) => authorizeSideThreadNode(actor, networkId, nodeId),
|
|
78
79
|
authorizeAttachment: (actor, networkId, ref) => authorizeSideThreadAttachment(actor, networkId, ref),
|
|
79
80
|
});
|
|
81
|
+
const productionSideThreadTransport = process.env[SIDE_THREAD_FEATURE_FLAG] === "1"
|
|
82
|
+
? createProductionSideThreadTransport(db)
|
|
83
|
+
: null;
|
|
84
|
+
if (productionSideThreadTransport) installSideThreadExecutionPort(productionSideThreadTransport.port);
|
|
80
85
|
|
|
81
86
|
if (AUTH_TOKEN) {
|
|
82
87
|
console.warn("[commhub] COMMHUB_AUTH_TOKEN is deprecated and will be removed in v1.0. See RFC-001.");
|
|
@@ -1519,11 +1524,22 @@ return Bun.serve({
|
|
|
1519
1524
|
return withCors(req, Response.json({ ok: false, error: restScope.denied }, { status: 403 }));
|
|
1520
1525
|
}
|
|
1521
1526
|
|
|
1527
|
+
const sideActor = resolveSideThreadActor(req, resolveRequestAuth(req, { allowQueryToken: false }), isAdmin);
|
|
1528
|
+
const nodeCommandActor = sideActor?.kind === "node" && sideActor.boundNodeId && sideActor.boundNetworkId
|
|
1529
|
+
? { tokenId: sideActor.tokenId, networkId: sideActor.boundNetworkId, nodeId: sideActor.boundNodeId }
|
|
1530
|
+
: null;
|
|
1531
|
+
if (productionSideThreadTransport) {
|
|
1532
|
+
const attachmentResponse = await productionSideThreadTransport.attachment(req, url, nodeCommandActor);
|
|
1533
|
+
if (attachmentResponse) return withCors(req, attachmentResponse);
|
|
1534
|
+
const commandResponse = await productionSideThreadTransport.handle({ req, url, actor: nodeCommandActor });
|
|
1535
|
+
if (commandResponse) return withCors(req, commandResponse);
|
|
1536
|
+
}
|
|
1537
|
+
|
|
1522
1538
|
const sideThreadResponse = await handleSideThreadHttpRequest({
|
|
1523
1539
|
req,
|
|
1524
1540
|
url,
|
|
1525
1541
|
// Durable prompts/results never accept query-string credentials.
|
|
1526
|
-
actor:
|
|
1542
|
+
actor: sideActor,
|
|
1527
1543
|
coordinator: sideThreadCoordinator,
|
|
1528
1544
|
resolveCapabilityTarget: resolveSideThreadCapabilityTarget,
|
|
1529
1545
|
});
|
|
@@ -16,6 +16,168 @@ function fixture() {
|
|
|
16
16
|
}
|
|
17
17
|
|
|
18
18
|
describe("Hub durable SideThread command outbox", () => {
|
|
19
|
+
test("late fork and start ACKs reconcile the coordinator across restart", async () => {
|
|
20
|
+
const f = fixture();
|
|
21
|
+
const owner = { userId: "user-1", username: "owner", tokenId: "utok-1", kind: "user" as const };
|
|
22
|
+
const input = { requestKey: "rk-late-create", networkId: "net-1", nodeId: "node-1", sourceThreadId: "source-1", boundary: { kind: "through" as const, turnId: "source-turn" }, prompt: "late receipt", attachments: [] };
|
|
23
|
+
const sideStore = new SideThreadStore(f.db);
|
|
24
|
+
let coordinator = new SideThreadCoordinator(sideStore, f.port, { enabled: true, authorizeNode: () => true });
|
|
25
|
+
const originalTransaction = f.db.transaction.bind(f.db);
|
|
26
|
+
const failTransaction = (...nth: number[]) => { let calls = 0; (f.db as any).transaction = (fn: () => unknown) => nth.includes(++calls) ? (() => { throw new Error("injected create apply failure"); })() : originalTransaction(fn); };
|
|
27
|
+
failTransaction(2);
|
|
28
|
+
await expect(coordinator.create(owner, input)).rejects.toThrow("injected create apply failure");
|
|
29
|
+
const side = sideStore.getByCreateKey(owner, input.requestKey)!.record;
|
|
30
|
+
expect(side).toMatchObject({ state: "creating", attempts: [{ state: "starting" }], operations: [{ kind: "fork", state: "pending" }] });
|
|
31
|
+
const fork = f.store.claim(actor())!;
|
|
32
|
+
f.store.ack(actor(), { protocol: "side_thread.ack.v1", commandId: fork.commandId, operationId: fork.operationId, state: "accepted", errorCode: null, result: { threadId: "derived-late", turnId: null, destinationTurnId: null } });
|
|
33
|
+
(f.db as any).transaction = originalTransaction;
|
|
34
|
+
coordinator.close();
|
|
35
|
+
coordinator = new SideThreadCoordinator(sideStore, f.port, { enabled: true, authorizeNode: () => true });
|
|
36
|
+
failTransaction(2, 3);
|
|
37
|
+
await expect(coordinator.create(owner, input)).rejects.toThrow("injected create apply failure");
|
|
38
|
+
expect(sideStore.getByCreateKey(owner, input.requestKey)!.record).toMatchObject({ state: "reconciling", attempts: [{ state: "reconciling" }], operations: [{ kind: "fork", state: "pending" }] });
|
|
39
|
+
expect(f.store.claim(actor())).toBeNull();
|
|
40
|
+
(f.db as any).transaction = originalTransaction;
|
|
41
|
+
coordinator.close();
|
|
42
|
+
coordinator = new SideThreadCoordinator(sideStore, f.port, { enabled: true, authorizeNode: () => true });
|
|
43
|
+
await expect(coordinator.create(owner, input)).rejects.toMatchObject({ code: "SIDE_THREAD_AMBIGUOUS" });
|
|
44
|
+
const start = f.store.claim(actor())!;
|
|
45
|
+
expect(start.kind).toBe("start");
|
|
46
|
+
f.store.ack(actor(), { protocol: "side_thread.ack.v1", commandId: start.commandId, operationId: start.operationId, state: "accepted", errorCode: null, result: { threadId: null, turnId: "turn-late", destinationTurnId: null } });
|
|
47
|
+
failTransaction(3);
|
|
48
|
+
await expect(coordinator.create(owner, input)).rejects.toMatchObject({ code: "SIDE_THREAD_AMBIGUOUS" });
|
|
49
|
+
expect(f.store.claim(actor())).toBeNull();
|
|
50
|
+
(f.db as any).transaction = originalTransaction;
|
|
51
|
+
coordinator.close();
|
|
52
|
+
coordinator = new SideThreadCoordinator(sideStore, f.port, { enabled: true, authorizeNode: () => true });
|
|
53
|
+
expect(await coordinator.create(owner, input)).toMatchObject({ state: "running", threadId: "derived-late", attempts: [{ turnId: "turn-late" }] });
|
|
54
|
+
const terminal = { protocol: "side_thread.terminal.v1", sideThreadId: side.sideChatId, attemptId: side.activeAttemptId!, threadId: "derived-late", turnId: "turn-late", status: "completed" as const, text: "late answer", errorCode: null };
|
|
55
|
+
expect((await f.port.acceptTerminal(actor(), terminal)).confirmed).toBe(true);
|
|
56
|
+
|
|
57
|
+
expect(await coordinator.create(owner, input)).toMatchObject({ sideChatId: side.sideChatId, state: "completed", threadId: "derived-late", attempts: [{ turnId: "turn-late", result: "late answer" }] });
|
|
58
|
+
expect(coordinator.get(owner, side.sideChatId).operations.filter((operation) => operation.kind === "fork" || operation.kind === "start")).toHaveLength(2);
|
|
59
|
+
expect(f.store.claim(actor())).toBeNull();
|
|
60
|
+
expect((await f.port.acceptTerminal(actor(), terminal)).idempotent).toBe(true);
|
|
61
|
+
expect(coordinator.get(owner, side.sideChatId)).toMatchObject({ state: "completed", attempts: [{ result: "late answer" }] });
|
|
62
|
+
coordinator.close();
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
for (const receipt of [
|
|
66
|
+
{ state: "unsupported", errorCode: "SIDE_THREAD_UNSUPPORTED", code: "SIDE_THREAD_UNSUPPORTED", status: 501 },
|
|
67
|
+
{ state: "failed", errorCode: "SIDE_THREAD_CONFLICT", code: "SIDE_THREAD_CONFLICT", status: 409 },
|
|
68
|
+
] as const) test(`late ${receipt.state} fork ACK converges to stable failed across coordinators`, async () => {
|
|
69
|
+
const f = fixture();
|
|
70
|
+
const owner = { userId: "user-1", username: "owner", tokenId: "utok-1", kind: "user" as const };
|
|
71
|
+
const input = { requestKey: `rk-late-${receipt.state}`, networkId: "net-1", nodeId: "node-1", sourceThreadId: "source-1", boundary: { kind: "through" as const, turnId: "source-turn" }, prompt: "late failure", attachments: [] };
|
|
72
|
+
const sideStore = new SideThreadStore(f.db);
|
|
73
|
+
let first = new SideThreadCoordinator(sideStore, f.port, { enabled: true, authorizeNode: () => true });
|
|
74
|
+
await expect(first.create(owner, input)).rejects.toMatchObject({ code: "SIDE_THREAD_AMBIGUOUS" });
|
|
75
|
+
const command = f.store.claim(actor())!;
|
|
76
|
+
f.store.ack(actor(), { protocol: "side_thread.ack.v1", commandId: command.commandId, operationId: command.operationId, state: receipt.state, errorCode: receipt.errorCode, result: { threadId: null, turnId: null, destinationTurnId: null } });
|
|
77
|
+
first.close();
|
|
78
|
+
|
|
79
|
+
const second = new SideThreadCoordinator(sideStore, f.port, { enabled: true, authorizeNode: () => true });
|
|
80
|
+
await expect(second.create(owner, input)).rejects.toMatchObject({ code: receipt.code, status: receipt.status });
|
|
81
|
+
const failed = sideStore.getByCreateKey(owner, input.requestKey)!.record;
|
|
82
|
+
expect(failed).toMatchObject({ state: "failed", attempts: [{ state: "failed" }], operations: [{ kind: "fork", state: "failed", errorCode: receipt.code }] });
|
|
83
|
+
expect(failed.activeAttemptId).toBeUndefined();
|
|
84
|
+
expect(f.db.get<{ active: string | null }>("SELECT active_attempt_id active FROM side_chats WHERE side_chat_id=?1", failed.sideChatId)?.active).toBeNull();
|
|
85
|
+
expect(await second.create(owner, input)).toMatchObject({ sideChatId: failed.sideChatId, state: "failed" });
|
|
86
|
+
expect(f.store.claim(actor())).toBeNull();
|
|
87
|
+
second.close();
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
for (const action of ["cancel", "archive", "purge", "bring-back"] as const)
|
|
91
|
+
for (const receipt of [
|
|
92
|
+
{ state: "accepted", errorCode: null, code: null },
|
|
93
|
+
{ state: "unsupported", errorCode: "SIDE_THREAD_UNSUPPORTED", code: "SIDE_THREAD_UNSUPPORTED" },
|
|
94
|
+
{ state: "failed", errorCode: "SIDE_THREAD_CONFLICT", code: "SIDE_THREAD_CONFLICT" },
|
|
95
|
+
] as const) test(`${action} consumes a late ${receipt.state} receipt after restart`, async () => {
|
|
96
|
+
const f = fixture();
|
|
97
|
+
const owner = { userId: "user-1", username: "owner", tokenId: "utok-1", kind: "user" as const };
|
|
98
|
+
const input = { requestKey: `rk-create-${action}-${receipt.state}`, networkId: "net-1", nodeId: "node-1", sourceThreadId: "source-1", boundary: { kind: "through" as const, turnId: "source-turn" }, prompt: "lifecycle", attachments: [] };
|
|
99
|
+
const sideStore = new SideThreadStore(f.db);
|
|
100
|
+
let coordinator = new SideThreadCoordinator(sideStore, f.port, { enabled: true, authorizeNode: () => true });
|
|
101
|
+
await coordinator.create(owner, input).catch(() => {});
|
|
102
|
+
const fork = f.store.claim(actor())!;
|
|
103
|
+
f.store.ack(actor(), { protocol: "side_thread.ack.v1", commandId: fork.commandId, operationId: fork.operationId, state: "accepted", errorCode: null, result: { threadId: "derived-life", turnId: null, destinationTurnId: null } });
|
|
104
|
+
await coordinator.create(owner, input).catch(() => {});
|
|
105
|
+
const start = f.store.claim(actor())!;
|
|
106
|
+
f.store.ack(actor(), { protocol: "side_thread.ack.v1", commandId: start.commandId, operationId: start.operationId, state: "accepted", errorCode: null, result: { threadId: null, turnId: "turn-life", destinationTurnId: null } });
|
|
107
|
+
const running = await coordinator.create(owner, input);
|
|
108
|
+
if (action !== "cancel") await f.port.acceptTerminal(actor(), { protocol: "side_thread.terminal.v1", sideThreadId: running.sideChatId, attemptId: running.activeAttemptId!, threadId: "derived-life", turnId: "turn-life", status: "completed", text: "answer", errorCode: null });
|
|
109
|
+
const requestKey = `rk-${action}-${receipt.state}`;
|
|
110
|
+
const invoke = (c: SideThreadCoordinator) => action === "cancel" ? c.cancel(owner, running.sideChatId, { requestKey })
|
|
111
|
+
: action === "archive" ? c.archive(owner, running.sideChatId, { requestKey })
|
|
112
|
+
: action === "purge" ? c.purge(owner, running.sideChatId, { requestKey })
|
|
113
|
+
: c.bringBack(owner, running.sideChatId, { requestKey, destinationThreadId: "source-1" });
|
|
114
|
+
await invoke(coordinator).catch(() => {});
|
|
115
|
+
const command = f.store.claim(actor())!;
|
|
116
|
+
f.store.ack(actor(), { protocol: "side_thread.ack.v1", commandId: command.commandId, operationId: command.operationId, state: receipt.state, errorCode: receipt.errorCode, result: { threadId: null, turnId: null, destinationTurnId: action === "bring-back" && receipt.state === "accepted" ? "destination-life" : null } });
|
|
117
|
+
coordinator.close();
|
|
118
|
+
coordinator = new SideThreadCoordinator(sideStore, f.port, { enabled: true, authorizeNode: () => true });
|
|
119
|
+
if (receipt.code) {
|
|
120
|
+
await expect(invoke(coordinator)).rejects.toMatchObject({ code: receipt.code });
|
|
121
|
+
await expect(invoke(coordinator)).rejects.toMatchObject({ code: receipt.code });
|
|
122
|
+
} else {
|
|
123
|
+
if (action === "archive" || action === "bring-back") {
|
|
124
|
+
const originalTransaction = f.db.transaction.bind(f.db);
|
|
125
|
+
let failApply = true;
|
|
126
|
+
(f.db as any).transaction = (fn: () => unknown) => {
|
|
127
|
+
if (failApply) { failApply = false; throw new Error("injected local apply failure"); }
|
|
128
|
+
return originalTransaction(fn);
|
|
129
|
+
};
|
|
130
|
+
await expect(invoke(coordinator)).rejects.toMatchObject({ code: "SIDE_THREAD_AMBIGUOUS", status: 202 });
|
|
131
|
+
const unsettled = sideStore.getByCreateKey(owner, input.requestKey)!.record.operations.find((op) => op.kind === action && op.requestKey === requestKey)!;
|
|
132
|
+
expect(["ambiguous", "reconciling"]).toContain(unsettled.state);
|
|
133
|
+
expect(unsettled.state).not.toBe("failed");
|
|
134
|
+
(f.db as any).transaction = originalTransaction;
|
|
135
|
+
coordinator.close();
|
|
136
|
+
coordinator = new SideThreadCoordinator(sideStore, f.port, { enabled: true, authorizeNode: () => true });
|
|
137
|
+
}
|
|
138
|
+
const [one, two] = await Promise.all([invoke(coordinator), invoke(coordinator)]);
|
|
139
|
+
if (action === "bring-back") expect(one).toEqual({ bringBackId: expect.any(String), destinationTurnId: "destination-life" });
|
|
140
|
+
else expect(one).toMatchObject({ state: action === "cancel" ? "running" : action === "archive" ? "archived" : "purged" });
|
|
141
|
+
expect(two).toEqual(one);
|
|
142
|
+
const hydrated = coordinator.get(owner, running.sideChatId);
|
|
143
|
+
const operation = hydrated.operations.find((op) => op.kind === action && op.requestKey === requestKey)!;
|
|
144
|
+
expect(operation).toMatchObject({ state: "completed", threadId: action === "bring-back" ? "source-1" : "derived-life" });
|
|
145
|
+
if (action === "bring-back") expect(operation.turnId).toBe("destination-life");
|
|
146
|
+
if (action === "cancel") expect(operation.turnId).toBe("turn-life");
|
|
147
|
+
const domainType = action === "bring-back" ? "side_chat.brought_back" : action === "archive" ? "side_chat.archived" : action === "purge" ? "side_chat.purged" : null;
|
|
148
|
+
const afterEvents = coordinator.listEvents(owner, running.sideChatId);
|
|
149
|
+
if (domainType) expect(afterEvents.filter((event) => event.type === domainType)).toHaveLength(1);
|
|
150
|
+
else expect(afterEvents.filter((event) => ["side_chat.brought_back", "side_chat.archived", "side_chat.purged"].includes(event.type))).toHaveLength(0);
|
|
151
|
+
}
|
|
152
|
+
expect(f.store.claim(actor())).toBeNull();
|
|
153
|
+
coordinator.close();
|
|
154
|
+
});
|
|
155
|
+
|
|
156
|
+
test("two coordinators race a late archive receipt but persist one domain event", async () => {
|
|
157
|
+
const f = fixture();
|
|
158
|
+
const owner = { userId: "user-1", username: "owner", tokenId: "utok-1", kind: "user" as const };
|
|
159
|
+
const input = { requestKey: "rk-multi-archive-create", networkId: "net-1", nodeId: "node-1", sourceThreadId: "source-1", boundary: { kind: "through" as const, turnId: "source-turn" }, prompt: "multi", attachments: [] };
|
|
160
|
+
const sideStore = new SideThreadStore(f.db);
|
|
161
|
+
const bootstrap = new SideThreadCoordinator(sideStore, f.port, { enabled: true, authorizeNode: () => true });
|
|
162
|
+
await bootstrap.create(owner, input).catch(() => {});
|
|
163
|
+
const fork = f.store.claim(actor())!; f.store.ack(actor(), { protocol: "side_thread.ack.v1", commandId: fork.commandId, operationId: fork.operationId, state: "accepted", errorCode: null, result: { threadId: "derived-multi", turnId: null, destinationTurnId: null } });
|
|
164
|
+
await bootstrap.create(owner, input).catch(() => {});
|
|
165
|
+
const start = f.store.claim(actor())!; f.store.ack(actor(), { protocol: "side_thread.ack.v1", commandId: start.commandId, operationId: start.operationId, state: "accepted", errorCode: null, result: { threadId: null, turnId: "turn-multi", destinationTurnId: null } });
|
|
166
|
+
const running = await bootstrap.create(owner, input);
|
|
167
|
+
await f.port.acceptTerminal(actor(), { protocol: "side_thread.terminal.v1", sideThreadId: running.sideChatId, attemptId: running.activeAttemptId!, threadId: "derived-multi", turnId: "turn-multi", status: "completed", text: "done", errorCode: null });
|
|
168
|
+
await bootstrap.archive(owner, running.sideChatId, { requestKey: "rk-multi-archive" }).catch(() => {});
|
|
169
|
+
const archive = f.store.claim(actor())!; f.store.ack(actor(), { protocol: "side_thread.ack.v1", commandId: archive.commandId, operationId: archive.operationId, state: "accepted", errorCode: null, result: { threadId: null, turnId: null, destinationTurnId: null } });
|
|
170
|
+
bootstrap.close();
|
|
171
|
+
const port = () => new DurableSideThreadCommandPort({ store: f.store, networkForNode: () => "net-1", capabilityForNode: () => ({ supported: true, mode: "native-exact-fork", exactBoundary: { through: true, before: true } }), grantAttachment: (_node, ref) => ({ fileId: ref.fileId }) });
|
|
172
|
+
const a = new SideThreadCoordinator(sideStore, port(), { enabled: true, authorizeNode: () => true });
|
|
173
|
+
const b = new SideThreadCoordinator(sideStore, port(), { enabled: true, authorizeNode: () => true });
|
|
174
|
+
const [one, two] = await Promise.all([a.archive(owner, running.sideChatId, { requestKey: "rk-multi-archive" }), b.archive(owner, running.sideChatId, { requestKey: "rk-multi-archive" })]);
|
|
175
|
+
expect(one.state).toBe("archived"); expect(two.state).toBe("archived");
|
|
176
|
+
expect(sideStore.events(running.sideChatId).filter((event) => event.type === "side_chat.archived")).toHaveLength(1);
|
|
177
|
+
expect(f.store.claim(actor())).toBeNull();
|
|
178
|
+
a.close(); b.close();
|
|
179
|
+
});
|
|
180
|
+
|
|
19
181
|
test("fails closed on the currently non-atomic PostgreSQL adapter", () => {
|
|
20
182
|
expect(() => new SideThreadCommandStore({ dialect: "postgres" } as any)).toThrow(/atomic SQLite/);
|
|
21
183
|
});
|
|
@@ -225,6 +225,7 @@ export class DurableSideThreadCommandPort implements SideThreadExecutionPort {
|
|
|
225
225
|
networkForNode: (nodeId: string) => string | null;
|
|
226
226
|
capabilityForNode: (nodeId: string) => SideThreadCapability;
|
|
227
227
|
grantAttachment: (nodeId: string, ref: SideThreadAttachmentRef) => Record<string, unknown>;
|
|
228
|
+
ackWaitMs?: number;
|
|
228
229
|
}) {}
|
|
229
230
|
capability(nodeId: string) { return this.opts.capabilityForNode(nodeId); }
|
|
230
231
|
fork(input: Parameters<SideThreadExecutionPort["fork"]>[0]) { return this.issue(input, "fork", { sourceThreadId: input.sourceThreadId, boundary: input.boundary }, "threadId") as Promise<{threadId:string}>; }
|
|
@@ -261,10 +262,18 @@ export class DurableSideThreadCommandPort implements SideThreadExecutionPort {
|
|
|
261
262
|
requestKey: input.requestKey ?? input.operationId, nodeId: input.nodeId, sideThreadId: input.sideChatId,
|
|
262
263
|
attemptId: input.attemptId ?? null, kind, payload };
|
|
263
264
|
this.opts.store.enqueue(command, { networkId, nodeId: input.nodeId });
|
|
264
|
-
const
|
|
265
|
+
const deadline = Date.now() + (this.opts.ackWaitMs ?? 0);
|
|
266
|
+
let receipt = this.opts.store.receipt(input.operationId);
|
|
267
|
+
while (!receipt && Date.now() < deadline) {
|
|
268
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
269
|
+
receipt = this.opts.store.receipt(input.operationId);
|
|
270
|
+
}
|
|
265
271
|
if (!receipt) throw new SideThreadError("SIDE_THREAD_AMBIGUOUS", "command durably queued; awaiting node ACK", 202, input.operationId, input.sideChatId, input.attemptId);
|
|
266
272
|
if (receipt.state === "accepted") return resultKey ? { [resultKey]: object(receipt.result)[resultKey] } : undefined;
|
|
267
|
-
|
|
273
|
+
const code = receipt.state === "unsupported" ? "SIDE_THREAD_UNSUPPORTED"
|
|
274
|
+
: receipt.state === "failed" ? "SIDE_THREAD_CONFLICT" : "SIDE_THREAD_AMBIGUOUS";
|
|
275
|
+
const status = code === "SIDE_THREAD_UNSUPPORTED" ? 501 : code === "SIDE_THREAD_CONFLICT" ? 409 : 202;
|
|
276
|
+
throw new SideThreadError(code, "node command returned an authoritative result", status, input.operationId, input.sideChatId, input.attemptId);
|
|
268
277
|
}
|
|
269
278
|
}
|
|
270
279
|
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import { afterEach, describe, expect, test } from "bun:test";
|
|
2
|
+
import { Database } from "bun:sqlite";
|
|
3
|
+
import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
|
|
4
|
+
import { tmpdir } from "node:os";
|
|
5
|
+
import { join } from "node:path";
|
|
6
|
+
import { SQLiteAdapter } from "./db-adapter";
|
|
7
|
+
import { createProductionSideThreadTransport } from "./side-thread-production";
|
|
8
|
+
|
|
9
|
+
const roots: string[] = [];
|
|
10
|
+
const priorUploads = process.env.COMMHUB_UPLOADS_DIR;
|
|
11
|
+
afterEach(() => {
|
|
12
|
+
if (priorUploads === undefined) delete process.env.COMMHUB_UPLOADS_DIR;
|
|
13
|
+
else process.env.COMMHUB_UPLOADS_DIR = priorUploads;
|
|
14
|
+
for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true });
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
describe("production Hub SideThread wiring", () => {
|
|
18
|
+
test("advertises only an exact node snapshot and serves a short-lived node-bound grant", async () => {
|
|
19
|
+
const root = mkdtempSync(join(tmpdir(), "side-hub-prod-")); roots.push(root);
|
|
20
|
+
process.env.COMMHUB_UPLOADS_DIR = root;
|
|
21
|
+
const db = new SQLiteAdapter(new Database(":memory:"));
|
|
22
|
+
db.exec("CREATE TABLE nodes(node_id TEXT PRIMARY KEY, network_id TEXT, config_snapshot TEXT)");
|
|
23
|
+
db.run("INSERT INTO nodes VALUES (?1,?2,?3)", ["node-1", "net-1", JSON.stringify({ side_thread_capability: {
|
|
24
|
+
supported: true, runtime: "codex-app-server", runtimeVersion: "0.148.0", topology: "owned-stdio",
|
|
25
|
+
evidenceRevision: "test1190-wire-v2", mode: "native-exact-fork", exactBoundary: { through: true, before: false },
|
|
26
|
+
} })]);
|
|
27
|
+
let now = 1_000;
|
|
28
|
+
const transport = createProductionSideThreadTransport(db, () => now, 0);
|
|
29
|
+
expect(transport.port.capability("node-1")).toMatchObject({ supported: true, exactBoundary: { before: false } });
|
|
30
|
+
expect(transport.port.capability("missing")).toMatchObject({ supported: false, reason: "runtime" });
|
|
31
|
+
|
|
32
|
+
const fileId = "file12345678"; const bytes = Buffer.from("image bytes"); const bucket = "2026-08-26";
|
|
33
|
+
mkdirSync(join(root, ".index"), { recursive: true }); mkdirSync(join(root, bucket), { recursive: true });
|
|
34
|
+
writeFileSync(join(root, bucket, `${fileId}.png`), bytes);
|
|
35
|
+
writeFileSync(join(root, ".index", `${fileId}.json`), JSON.stringify({
|
|
36
|
+
file_id: fileId, date_bucket: bucket, ext: ".png", name: "x.png", mime: "image/png",
|
|
37
|
+
size: bytes.length, owner_id: "user-1", network_id: "net-1", uploaded_at: new Date().toISOString(),
|
|
38
|
+
}));
|
|
39
|
+
await transport.port.start({
|
|
40
|
+
operationId: "op-start", requestKey: "rk-start", sideChatId: "side-1", attemptId: "attempt-1",
|
|
41
|
+
nodeId: "node-1", threadId: "derived-1", prompt: "read it",
|
|
42
|
+
attachments: [{ fileId, mediaType: "image/png", size: bytes.length }],
|
|
43
|
+
}).catch(() => {});
|
|
44
|
+
const command = transport.store.claim({ tokenId: "token-1", networkId: "net-1", nodeId: "node-1" }) as any;
|
|
45
|
+
const grant = command.payload.attachments[0];
|
|
46
|
+
expect(grant).toMatchObject({ fileId, mediaType: "image/png", size: bytes.length });
|
|
47
|
+
expect(grant.sha256).toMatch(/^[a-f0-9]{64}$/);
|
|
48
|
+
const url = new URL(`http://hub/api/side-thread/attachment-grants/${grant.grantId}`);
|
|
49
|
+
const foreign = await transport.attachment(new Request(url), url, { tokenId: "foreign", networkId: "net-1", nodeId: "node-2" });
|
|
50
|
+
expect(foreign?.status).toBe(404);
|
|
51
|
+
const response = await transport.attachment(new Request(url), url, { tokenId: "token-1", networkId: "net-1", nodeId: "node-1" });
|
|
52
|
+
expect(response?.status).toBe(200); expect(Buffer.from(await response!.arrayBuffer())).toEqual(bytes);
|
|
53
|
+
now += 5 * 60_000 + 1;
|
|
54
|
+
expect((await transport.attachment(new Request(url), url, { tokenId: "token-1", networkId: "net-1", nodeId: "node-1" }))?.status).toBe(404);
|
|
55
|
+
await transport.port.start({
|
|
56
|
+
operationId: "op-start-2", requestKey: "rk-start-2", sideChatId: "side-2", attemptId: "attempt-2",
|
|
57
|
+
nodeId: "node-1", threadId: "derived-2", prompt: "read it again",
|
|
58
|
+
attachments: [{ fileId, mediaType: "image/png", size: bytes.length }],
|
|
59
|
+
}).catch(() => {});
|
|
60
|
+
expect(db.get<{ count: number }>("SELECT COUNT(*) AS count FROM side_thread_attachment_grants")?.count).toBe(1);
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
test("rejects forged capability shapes instead of widening the allowlist", () => {
|
|
64
|
+
const db = new SQLiteAdapter(new Database(":memory:"));
|
|
65
|
+
db.exec("CREATE TABLE nodes(node_id TEXT PRIMARY KEY, network_id TEXT, config_snapshot TEXT)");
|
|
66
|
+
db.run("INSERT INTO nodes VALUES (?1,?2,?3)", ["node-1", "net-1", JSON.stringify({ side_thread_capability: {
|
|
67
|
+
supported: true, runtime: "codex-app-server", runtimeVersion: "latest", topology: "owned-stdio",
|
|
68
|
+
evidenceRevision: "test1190-wire-v2", mode: "native-exact-fork", exactBoundary: { through: true, before: true },
|
|
69
|
+
} })]);
|
|
70
|
+
expect(createProductionSideThreadTransport(db).port.capability("node-1")).toMatchObject({ supported: false });
|
|
71
|
+
});
|
|
72
|
+
});
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
+
import { readFileSync } from "node:fs";
|
|
3
|
+
import type { DbAdapter } from "./db-adapter.js";
|
|
4
|
+
import { indexEntryPath, pathForExistingBlob, validateIndexEntry } from "./uploads.js";
|
|
5
|
+
import type { SideThreadAttachmentRef, SideThreadCapability } from "./side-thread.js";
|
|
6
|
+
import {
|
|
7
|
+
DurableSideThreadCommandPort,
|
|
8
|
+
SideThreadCommandStore,
|
|
9
|
+
handleSideThreadCommandRequest,
|
|
10
|
+
type NodeCommandActor,
|
|
11
|
+
} from "./side-thread-command-transport.js";
|
|
12
|
+
|
|
13
|
+
const GRANT_TTL_MS = 5 * 60_000;
|
|
14
|
+
|
|
15
|
+
type GrantRow = {
|
|
16
|
+
grant_id: string; file_id: string; network_id: string; node_id: string;
|
|
17
|
+
sha256: string; size: number; media_type: string; expires_at: number;
|
|
18
|
+
};
|
|
19
|
+
|
|
20
|
+
/** Production-only SideThread transport. It remains absent unless the public
|
|
21
|
+
* feature flag is enabled and the configured DB can provide atomic claims. */
|
|
22
|
+
export function createProductionSideThreadTransport(db: DbAdapter, now = Date.now, ackWaitMs = 2_500) {
|
|
23
|
+
const store = new SideThreadCommandStore(db, now);
|
|
24
|
+
db.exec(`CREATE TABLE IF NOT EXISTS side_thread_attachment_grants (
|
|
25
|
+
grant_id TEXT PRIMARY KEY, file_id TEXT NOT NULL, network_id TEXT NOT NULL,
|
|
26
|
+
node_id TEXT NOT NULL, sha256 TEXT NOT NULL, size INTEGER NOT NULL,
|
|
27
|
+
media_type TEXT NOT NULL, expires_at INTEGER NOT NULL, created_at INTEGER NOT NULL
|
|
28
|
+
)`);
|
|
29
|
+
const port = new DurableSideThreadCommandPort({
|
|
30
|
+
store,
|
|
31
|
+
networkForNode(nodeId) {
|
|
32
|
+
return db.get<{ network_id: string | null }>("SELECT network_id FROM nodes WHERE node_id=?1", nodeId)?.network_id ?? null;
|
|
33
|
+
},
|
|
34
|
+
capabilityForNode(nodeId) {
|
|
35
|
+
const row = db.get<{ config_snapshot: string | null }>("SELECT config_snapshot FROM nodes WHERE node_id=?1", nodeId);
|
|
36
|
+
if (!row?.config_snapshot) return unsupported("runtime");
|
|
37
|
+
try {
|
|
38
|
+
const cap = JSON.parse(row.config_snapshot)?.side_thread_capability;
|
|
39
|
+
return validateCapability(cap) ? cap : unsupported("runtime");
|
|
40
|
+
} catch { return unsupported("runtime"); }
|
|
41
|
+
},
|
|
42
|
+
grantAttachment(nodeId, ref) { return issueGrant(db, nodeId, ref, now); },
|
|
43
|
+
// The dedicated node consumer polls asynchronously. Give its durable ACK
|
|
44
|
+
// a bounded window so the coordinator can advance fork→start atomically;
|
|
45
|
+
// timeout remains an honest ambiguous result, never a task fallback.
|
|
46
|
+
ackWaitMs,
|
|
47
|
+
});
|
|
48
|
+
return {
|
|
49
|
+
store,
|
|
50
|
+
port,
|
|
51
|
+
handle: (input: { req: Request; url: URL; actor: NodeCommandActor | null }) =>
|
|
52
|
+
handleSideThreadCommandRequest({ ...input, store, port }),
|
|
53
|
+
attachment: (req: Request, url: URL, actor: NodeCommandActor | null) =>
|
|
54
|
+
serveGrant(db, req, url, actor, now),
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function issueGrant(db: DbAdapter, nodeId: string, ref: SideThreadAttachmentRef, now: () => number): Record<string, unknown> {
|
|
59
|
+
// Grants are deliberately short-lived and must not become an unbounded
|
|
60
|
+
// metadata cache. Lazy sweeping keeps the hot download path read-only while
|
|
61
|
+
// ensuring every new issuance bounds accumulated expired rows.
|
|
62
|
+
db.run("DELETE FROM side_thread_attachment_grants WHERE expires_at < ?1", [now()]);
|
|
63
|
+
const node = db.get<{ network_id: string | null }>("SELECT network_id FROM nodes WHERE node_id=?1", nodeId);
|
|
64
|
+
if (!node?.network_id) throw new Error("SideThread attachment node scope unavailable");
|
|
65
|
+
const indexPath = indexEntryPath(ref.fileId);
|
|
66
|
+
if (!indexPath) throw new Error("SideThread attachment index unavailable");
|
|
67
|
+
const entry = JSON.parse(readFileSync(indexPath, "utf8"));
|
|
68
|
+
if (!validateIndexEntry(entry) || entry.file_id !== ref.fileId || entry.network_id !== node.network_id)
|
|
69
|
+
throw new Error("SideThread attachment scope mismatch");
|
|
70
|
+
const path = pathForExistingBlob(entry.date_bucket, entry.file_id, entry.ext).absolutePath;
|
|
71
|
+
const bytes = readFileSync(path);
|
|
72
|
+
if (bytes.byteLength !== entry.size) throw new Error("SideThread attachment size changed");
|
|
73
|
+
const sha256 = createHash("sha256").update(bytes).digest("hex");
|
|
74
|
+
const grantId = `stg_${randomUUID().replace(/-/g, "")}`;
|
|
75
|
+
const mediaType = typeof entry.mime === "string" && entry.mime.length <= 200 ? entry.mime : "application/octet-stream";
|
|
76
|
+
db.run(
|
|
77
|
+
"INSERT INTO side_thread_attachment_grants (grant_id,file_id,network_id,node_id,sha256,size,media_type,expires_at,created_at) VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9)",
|
|
78
|
+
[grantId, ref.fileId, node.network_id, nodeId, sha256, bytes.byteLength, mediaType, now() + GRANT_TTL_MS, now()],
|
|
79
|
+
);
|
|
80
|
+
return { fileId: ref.fileId, grantId, sha256, size: bytes.byteLength, mediaType };
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
async function serveGrant(db: DbAdapter, req: Request, url: URL, actor: NodeCommandActor | null, now: () => number): Promise<Response | null> {
|
|
84
|
+
const match = url.pathname.match(/^\/api\/side-thread\/attachment-grants\/([^/]+)$/);
|
|
85
|
+
if (!match) return null;
|
|
86
|
+
if (req.method !== "GET") return Response.json({ ok: false, error: "method_not_allowed" }, { status: 405 });
|
|
87
|
+
if (!actor) return Response.json({ ok: false, error: "node_token_binding_required" }, { status: 403 });
|
|
88
|
+
let grantId: string;
|
|
89
|
+
try { grantId = decodeURIComponent(match[1]); } catch { return Response.json({ ok: false, error: "invalid_grant" }, { status: 400 }); }
|
|
90
|
+
const row = db.get<GrantRow>("SELECT * FROM side_thread_attachment_grants WHERE grant_id=?1", grantId);
|
|
91
|
+
// 404 deliberately hides existence, ownership and expiry.
|
|
92
|
+
if (!row || row.node_id !== actor.nodeId || row.network_id !== actor.networkId || row.expires_at < now())
|
|
93
|
+
return Response.json({ ok: false, error: "not_found" }, { status: 404 });
|
|
94
|
+
const indexPath = indexEntryPath(row.file_id);
|
|
95
|
+
if (!indexPath) return Response.json({ ok: false, error: "not_found" }, { status: 404 });
|
|
96
|
+
try {
|
|
97
|
+
const entry = JSON.parse(readFileSync(indexPath, "utf8"));
|
|
98
|
+
if (!validateIndexEntry(entry) || entry.network_id !== row.network_id || entry.size !== row.size)
|
|
99
|
+
return Response.json({ ok: false, error: "not_found" }, { status: 404 });
|
|
100
|
+
const bytes = readFileSync(pathForExistingBlob(entry.date_bucket, entry.file_id, entry.ext).absolutePath);
|
|
101
|
+
if (bytes.byteLength !== row.size || createHash("sha256").update(bytes).digest("hex") !== row.sha256)
|
|
102
|
+
return Response.json({ ok: false, error: "not_found" }, { status: 404 });
|
|
103
|
+
return new Response(bytes, { headers: { "Content-Type": row.media_type, "Content-Length": String(row.size), "Cache-Control": "no-store" } });
|
|
104
|
+
} catch { return Response.json({ ok: false, error: "not_found" }, { status: 404 }); }
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function validateCapability(value: unknown): value is SideThreadCapability {
|
|
108
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
|
109
|
+
const v = value as Record<string, unknown>;
|
|
110
|
+
return v.supported === true && v.runtime === "codex-app-server" && v.runtimeVersion === "0.148.0"
|
|
111
|
+
&& v.topology === "owned-stdio" && v.evidenceRevision === "test1190-wire-v2"
|
|
112
|
+
&& v.mode === "native-exact-fork" && !!v.exactBoundary && typeof v.exactBoundary === "object"
|
|
113
|
+
&& (v.exactBoundary as any).through === true && typeof (v.exactBoundary as any).before === "boolean";
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
function unsupported(reason: SideThreadCapability["reason"]): SideThreadCapability {
|
|
117
|
+
return { supported: false, runtime: "unknown", runtimeVersion: "unknown", topology: "unknown", evidenceRevision: "none", reason };
|
|
118
|
+
}
|
package/src/side-thread.test.ts
CHANGED
|
@@ -9,6 +9,7 @@ import {
|
|
|
9
9
|
UnsupportedSideThreadPort,
|
|
10
10
|
type SideThreadActor,
|
|
11
11
|
type SideThreadExecutionPort,
|
|
12
|
+
type SideThreadEventRecord,
|
|
12
13
|
type SideThreadRuntimeEvent,
|
|
13
14
|
} from "./side-thread.js";
|
|
14
15
|
import { handleSideThreadHttpRequest } from "./side-thread-http.js";
|
|
@@ -312,6 +313,28 @@ describe("SideThread Hub contract", () => {
|
|
|
312
313
|
).toHaveLength(3);
|
|
313
314
|
});
|
|
314
315
|
|
|
316
|
+
test("throwing observers cannot alter durable truth or starve cursor replay", async () => {
|
|
317
|
+
const f = fixture();
|
|
318
|
+
const record = await f.coordinator.create(owner, createInput());
|
|
319
|
+
const attempt = record.attempts[0];
|
|
320
|
+
const cursor = f.coordinator.listEvents(owner, record.sideChatId).at(-1)!.eventId;
|
|
321
|
+
const delivered: SideThreadEventRecord[] = [];
|
|
322
|
+
const offThrow = f.coordinator.subscribe(record.sideChatId, () => { throw new Error("observer exploded"); });
|
|
323
|
+
const offGood = f.coordinator.subscribe(record.sideChatId, (event) => delivered.push(event));
|
|
324
|
+
f.port.emit({ sideChatId: record.sideChatId, attemptId: attempt.attemptId, threadId: record.threadId!, turnId: attempt.turnId!, status: "completed", text: "durable answer" });
|
|
325
|
+
await Bun.sleep(0);
|
|
326
|
+
expect(f.coordinator.get(owner, record.sideChatId)).toMatchObject({ state: "completed", attempts: [{ result: "durable answer" }] });
|
|
327
|
+
expect(delivered.filter((event) => event.type === "attempt.terminal")).toHaveLength(1);
|
|
328
|
+
const replay = f.coordinator.listEvents(owner, record.sideChatId, cursor);
|
|
329
|
+
expect(replay.filter((event) => event.type === "attempt.terminal")).toHaveLength(1);
|
|
330
|
+
expect(f.coordinator.listEvents(owner, record.sideChatId, replay.at(-1)!.eventId)).toHaveLength(0);
|
|
331
|
+
offThrow(); offGood();
|
|
332
|
+
f.coordinator.close();
|
|
333
|
+
const reopened = new SideThreadCoordinator(f.store, f.port, { enabled: true, authorizeNode: () => true, authorizeAttachment: () => true });
|
|
334
|
+
closers.push(() => reopened.close());
|
|
335
|
+
expect(reopened.listEvents(owner, record.sideChatId, cursor).filter((event) => event.type === "attempt.terminal")).toHaveLength(1);
|
|
336
|
+
});
|
|
337
|
+
|
|
315
338
|
test("a terminal event arriving before start returns settles once", async () => {
|
|
316
339
|
const f = fixture();
|
|
317
340
|
f.port.onStart = (input, turnId) =>
|
|
@@ -391,10 +414,10 @@ describe("SideThread Hub contract", () => {
|
|
|
391
414
|
closers.push(() => second.close());
|
|
392
415
|
const firstCreate = f.coordinator.create(owner, createInput());
|
|
393
416
|
await entered;
|
|
394
|
-
const replay = await second.create(owner, createInput());
|
|
395
|
-
expect(replay.state).toBe("creating");
|
|
396
417
|
releaseFork();
|
|
397
418
|
const created = await firstCreate;
|
|
419
|
+
const replay = await second.create(owner, createInput());
|
|
420
|
+
expect(replay.state).toBe("running");
|
|
398
421
|
expect(f.port.calls.fork).toBe(1);
|
|
399
422
|
|
|
400
423
|
let cancelEntered!: () => void;
|
|
@@ -420,7 +443,9 @@ describe("SideThread Hub contract", () => {
|
|
|
420
443
|
expect(concurrent.state).toBe("running");
|
|
421
444
|
expect(f.port.calls.cancel).toBe(1);
|
|
422
445
|
releaseCancel();
|
|
423
|
-
expect((await firstCancel).state).toBe("
|
|
446
|
+
expect((await firstCancel).state).toBe("running");
|
|
447
|
+
const cancelledAttempt = created.attempts[0];
|
|
448
|
+
f.port.emit({ sideChatId: created.sideChatId, attemptId: cancelledAttempt.attemptId, threadId: created.threadId!, turnId: cancelledAttempt.turnId!, status: "interrupted" });
|
|
424
449
|
|
|
425
450
|
let archiveEntered!: () => void;
|
|
426
451
|
const archiveStarted = new Promise<void>(
|
|
@@ -627,7 +652,7 @@ describe("SideThread Hub contract", () => {
|
|
|
627
652
|
expect(reopened.list(stranger)).toHaveLength(0);
|
|
628
653
|
});
|
|
629
654
|
|
|
630
|
-
test("response loss
|
|
655
|
+
test("response loss remains ambiguous under the same stable start operation", async () => {
|
|
631
656
|
const port = new FakePort();
|
|
632
657
|
let accepted: Parameters<SideThreadExecutionPort["start"]>[0] | undefined;
|
|
633
658
|
port.start = async (input) => {
|
|
@@ -658,11 +683,11 @@ describe("SideThread Hub contract", () => {
|
|
|
658
683
|
state: "ambiguous",
|
|
659
684
|
errorCode: "SIDE_THREAD_RESPONSE_LOST",
|
|
660
685
|
});
|
|
661
|
-
expect(
|
|
686
|
+
await expect(f.coordinator.create(owner, createInput())).rejects.toMatchObject({
|
|
687
|
+
code: "SIDE_THREAD_AMBIGUOUS",
|
|
662
688
|
sideChatId: ambiguous.sideChatId,
|
|
663
|
-
state: "ambiguous",
|
|
664
689
|
});
|
|
665
|
-
expect(port.calls).toMatchObject({ fork: 1, start:
|
|
690
|
+
expect(port.calls).toMatchObject({ fork: 1, start: 2 });
|
|
666
691
|
|
|
667
692
|
f.coordinator.close();
|
|
668
693
|
const reopened = new SideThreadCoordinator(f.store, port, {
|
|
@@ -689,13 +714,13 @@ describe("SideThread Hub contract", () => {
|
|
|
689
714
|
expect(
|
|
690
715
|
reconciled.operations.find((operation) => operation.kind === "start"),
|
|
691
716
|
).toMatchObject({ state: "completed", turnId: "turn_authoritative" });
|
|
692
|
-
expect(port.calls.start).toBe(
|
|
717
|
+
expect(port.calls.start).toBe(2);
|
|
693
718
|
expect(
|
|
694
719
|
JSON.stringify(reopened.listEvents(owner, ambiguous.sideChatId)),
|
|
695
720
|
).not.toContain("secret transport detail");
|
|
696
721
|
});
|
|
697
722
|
|
|
698
|
-
test("ambiguous cancel/archive/purge
|
|
723
|
+
test("ambiguous cancel/archive/purge replay the same stable operation identity", async () => {
|
|
699
724
|
const lose = () =>
|
|
700
725
|
Object.assign(new Error("accepted but ACK lost"), {
|
|
701
726
|
code: "SIDE_THREAD_RESPONSE_LOST",
|
|
@@ -728,7 +753,7 @@ describe("SideThread Hub contract", () => {
|
|
|
728
753
|
code: "SIDE_THREAD_AMBIGUOUS",
|
|
729
754
|
operationId: cancelOperationId,
|
|
730
755
|
});
|
|
731
|
-
expect(cancelPort.calls.cancel).toBe(
|
|
756
|
+
expect(cancelPort.calls.cancel).toBe(2);
|
|
732
757
|
|
|
733
758
|
for (const action of ["archive", "purge"] as const) {
|
|
734
759
|
const port = new FakePort();
|
|
@@ -767,7 +792,7 @@ describe("SideThread Hub contract", () => {
|
|
|
767
792
|
code: "SIDE_THREAD_AMBIGUOUS",
|
|
768
793
|
operationId,
|
|
769
794
|
});
|
|
770
|
-
expect(port.calls[action]).toBe(
|
|
795
|
+
expect(port.calls[action]).toBe(2);
|
|
771
796
|
}
|
|
772
797
|
});
|
|
773
798
|
});
|
|
@@ -919,6 +944,10 @@ describe("SideThread HTTP contract", () => {
|
|
|
919
944
|
coordinator: f.coordinator,
|
|
920
945
|
});
|
|
921
946
|
expect(response?.status).toBe(200);
|
|
947
|
+
if (action === "cancel") {
|
|
948
|
+
const attempt = record.attempts[0];
|
|
949
|
+
f.port.emit({ sideChatId: record.sideChatId, attemptId: attempt.attemptId, threadId: record.threadId!, turnId: attempt.turnId!, status: "interrupted" });
|
|
950
|
+
}
|
|
922
951
|
}
|
|
923
952
|
const hydrated = f.coordinator.get(owner, record.sideChatId);
|
|
924
953
|
expect(
|
package/src/side-thread.ts
CHANGED
|
@@ -770,6 +770,12 @@ export class SideThreadCoordinator {
|
|
|
770
770
|
if (existing) {
|
|
771
771
|
if (existing.fingerprint !== fingerprint)
|
|
772
772
|
throw conflict("idempotency key reused with different create payload");
|
|
773
|
+
if (this.createNeedsReconciliation(existing.record)) {
|
|
774
|
+
return this.singleFlight(
|
|
775
|
+
`create:${actor.userId}:${input.requestKey}`,
|
|
776
|
+
() => this.reconcileCreate(actor, input, attachments, fingerprint),
|
|
777
|
+
);
|
|
778
|
+
}
|
|
773
779
|
return existing.record;
|
|
774
780
|
}
|
|
775
781
|
return this.singleFlight(
|
|
@@ -781,6 +787,8 @@ export class SideThreadCoordinator {
|
|
|
781
787
|
throw conflict(
|
|
782
788
|
"idempotency key reused with different create payload",
|
|
783
789
|
);
|
|
790
|
+
if (this.createNeedsReconciliation(replay.record))
|
|
791
|
+
return this.reconcileCreate(actor, input, attachments, fingerprint);
|
|
784
792
|
return replay.record;
|
|
785
793
|
}
|
|
786
794
|
const capability = await this.assertCapability(
|
|
@@ -880,7 +888,7 @@ export class SideThreadCoordinator {
|
|
|
880
888
|
throw runtimeProtocol(
|
|
881
889
|
"runtime returned source thread as side thread",
|
|
882
890
|
);
|
|
883
|
-
this.
|
|
891
|
+
this.applyRuntimeReceipt(() => {
|
|
884
892
|
const t = this.now();
|
|
885
893
|
this.store.db.run(
|
|
886
894
|
"UPDATE side_chats SET derived_thread_id=?1,updated_at=?2 WHERE side_chat_id=?3 AND state='creating'",
|
|
@@ -921,16 +929,18 @@ export class SideThreadCoordinator {
|
|
|
921
929
|
attachments,
|
|
922
930
|
});
|
|
923
931
|
requireRuntimeIdentity(started.turnId, "turnId");
|
|
924
|
-
this.store.db.
|
|
932
|
+
const attemptBeforeApply = this.store.db.get<any>(
|
|
933
|
+
"SELECT state,turn_id FROM side_chat_attempts WHERE attempt_id=?1",
|
|
934
|
+
attemptId,
|
|
935
|
+
);
|
|
936
|
+
if (!attemptBeforeApply) throw conflict("attempt disappeared during start");
|
|
937
|
+
if (attemptBeforeApply.turn_id && attemptBeforeApply.turn_id !== started.turnId)
|
|
938
|
+
throw runtimeProtocol("runtime turn identity changed during start");
|
|
939
|
+
this.applyRuntimeReceipt(() => {
|
|
925
940
|
const attempt = this.store.db.get<any>(
|
|
926
941
|
"SELECT state,turn_id FROM side_chat_attempts WHERE attempt_id=?1",
|
|
927
942
|
attemptId,
|
|
928
943
|
);
|
|
929
|
-
if (!attempt) throw conflict("attempt disappeared during start");
|
|
930
|
-
if (attempt.turn_id && attempt.turn_id !== started.turnId)
|
|
931
|
-
throw runtimeProtocol(
|
|
932
|
-
"runtime turn identity changed during start",
|
|
933
|
-
);
|
|
934
944
|
this.settleOperation(startOperationId, "completed", {
|
|
935
945
|
threadId: forked.threadId,
|
|
936
946
|
turnId: started.turnId,
|
|
@@ -964,10 +974,7 @@ export class SideThreadCoordinator {
|
|
|
964
974
|
this.markAmbiguous(sideChatId, attemptId, activeOperationId, error);
|
|
965
975
|
throw ambiguousError(activeOperationId, sideChatId, attemptId);
|
|
966
976
|
}
|
|
967
|
-
this.
|
|
968
|
-
errorCode: auditErrorReason(error),
|
|
969
|
-
});
|
|
970
|
-
this.failStarting(sideChatId, attemptId, error);
|
|
977
|
+
this.failStarting(sideChatId, attemptId, error, activeOperationId);
|
|
971
978
|
throw normalizePortError(error);
|
|
972
979
|
}
|
|
973
980
|
return this.mustOwned(sideChatId, actor);
|
|
@@ -975,6 +982,132 @@ export class SideThreadCoordinator {
|
|
|
975
982
|
);
|
|
976
983
|
}
|
|
977
984
|
|
|
985
|
+
/**
|
|
986
|
+
* Resume the stable fork/start operations after an ACK arrived outside the
|
|
987
|
+
* request window. All identity needed to resume lives in SQLite; calling
|
|
988
|
+
* create again from another process therefore consumes the durable receipt
|
|
989
|
+
* instead of issuing a second native operation.
|
|
990
|
+
*/
|
|
991
|
+
private async reconcileCreate(
|
|
992
|
+
actor: SideThreadActor,
|
|
993
|
+
input: CreateInput,
|
|
994
|
+
attachments: SideThreadAttachmentRef[],
|
|
995
|
+
fingerprint: string,
|
|
996
|
+
): Promise<SideThreadRecord> {
|
|
997
|
+
const found = this.store.getByCreateKey(actor, input.requestKey);
|
|
998
|
+
if (!found) throw conflict("create record disappeared during reconciliation");
|
|
999
|
+
if (found.fingerprint !== fingerprint)
|
|
1000
|
+
throw conflict("idempotency key reused with different create payload");
|
|
1001
|
+
const sideChatId = found.record.sideChatId;
|
|
1002
|
+
const attemptId = found.record.activeAttemptId;
|
|
1003
|
+
if (!attemptId) throw conflict("create attempt disappeared during reconciliation");
|
|
1004
|
+
const forkOperationId = stableOperationId(sideChatId, "fork", input.requestKey);
|
|
1005
|
+
const startOperationId = stableOperationId(sideChatId, "start", input.requestKey);
|
|
1006
|
+
let activeOperationId = forkOperationId;
|
|
1007
|
+
try {
|
|
1008
|
+
this.applyRuntimeReceipt(() => {
|
|
1009
|
+
const at = this.now();
|
|
1010
|
+
this.store.db.run(
|
|
1011
|
+
"UPDATE side_chats SET state='reconciling',updated_at=?1 WHERE side_chat_id=?2 AND state IN ('creating','ambiguous')",
|
|
1012
|
+
[at, sideChatId],
|
|
1013
|
+
);
|
|
1014
|
+
this.store.db.run(
|
|
1015
|
+
"UPDATE side_chat_attempts SET state='reconciling',updated_at=?1 WHERE attempt_id=?2 AND state IN ('starting','ambiguous')",
|
|
1016
|
+
[at, attemptId],
|
|
1017
|
+
);
|
|
1018
|
+
this.store.db.run(
|
|
1019
|
+
"UPDATE side_chat_operations SET state='reconciling',updated_at=?1 WHERE operation_id=?2 AND state='ambiguous'",
|
|
1020
|
+
[at, forkOperationId],
|
|
1021
|
+
);
|
|
1022
|
+
});
|
|
1023
|
+
const forkOp = this.store.db.get<any>(
|
|
1024
|
+
"SELECT state,thread_id FROM side_chat_operations WHERE operation_id=?1",
|
|
1025
|
+
forkOperationId,
|
|
1026
|
+
);
|
|
1027
|
+
let threadId = forkOp?.state === "completed" ? forkOp.thread_id : undefined;
|
|
1028
|
+
if (!threadId) {
|
|
1029
|
+
const forked = await this.port.fork({
|
|
1030
|
+
operationId: forkOperationId,
|
|
1031
|
+
requestKey: input.requestKey,
|
|
1032
|
+
sideChatId,
|
|
1033
|
+
nodeId: input.nodeId,
|
|
1034
|
+
sourceThreadId: input.sourceThreadId,
|
|
1035
|
+
boundary: input.boundary,
|
|
1036
|
+
});
|
|
1037
|
+
threadId = forked.threadId;
|
|
1038
|
+
}
|
|
1039
|
+
requireRuntimeIdentity(threadId, "threadId");
|
|
1040
|
+
if (threadId === input.sourceThreadId)
|
|
1041
|
+
throw runtimeProtocol("runtime returned source thread as side thread");
|
|
1042
|
+
this.applyRuntimeReceipt(() => {
|
|
1043
|
+
const at = this.now();
|
|
1044
|
+
this.settleOperation(forkOperationId, "completed", { threadId });
|
|
1045
|
+
this.store.db.run(
|
|
1046
|
+
"UPDATE side_chats SET derived_thread_id=?1,updated_at=?2 WHERE side_chat_id=?3 AND state IN ('creating','ambiguous','reconciling')",
|
|
1047
|
+
[threadId, at, sideChatId],
|
|
1048
|
+
);
|
|
1049
|
+
this.store.db.run(
|
|
1050
|
+
"UPDATE side_chat_attempts SET thread_id=?1,updated_at=?2 WHERE attempt_id=?3 AND state IN ('starting','ambiguous','reconciling')",
|
|
1051
|
+
[threadId, at, attemptId],
|
|
1052
|
+
);
|
|
1053
|
+
const existingStart = this.store.db.get<any>(
|
|
1054
|
+
"SELECT operation_id FROM side_chat_operations WHERE operation_id=?1",
|
|
1055
|
+
startOperationId,
|
|
1056
|
+
);
|
|
1057
|
+
if (!existingStart)
|
|
1058
|
+
this.insertOperation({ operationId: startOperationId, sideChatId, attemptId, kind: "start", requestKey: input.requestKey, fingerprint, threadId, at });
|
|
1059
|
+
else
|
|
1060
|
+
this.store.db.run(
|
|
1061
|
+
"UPDATE side_chat_operations SET state='reconciling',updated_at=?1 WHERE operation_id=?2 AND state='ambiguous'",
|
|
1062
|
+
[at, startOperationId],
|
|
1063
|
+
);
|
|
1064
|
+
});
|
|
1065
|
+
activeOperationId = startOperationId;
|
|
1066
|
+
const startOp = this.store.db.get<any>(
|
|
1067
|
+
"SELECT state,turn_id FROM side_chat_operations WHERE operation_id=?1",
|
|
1068
|
+
startOperationId,
|
|
1069
|
+
);
|
|
1070
|
+
let turnId = startOp?.state === "completed" ? startOp.turn_id : undefined;
|
|
1071
|
+
if (!turnId) {
|
|
1072
|
+
const started = await this.port.start({ operationId: startOperationId, requestKey: input.requestKey, sideChatId, attemptId, nodeId: input.nodeId, threadId, prompt: input.prompt, attachments });
|
|
1073
|
+
turnId = started.turnId;
|
|
1074
|
+
}
|
|
1075
|
+
requireRuntimeIdentity(turnId, "turnId");
|
|
1076
|
+
this.applyRuntimeReceipt(() => {
|
|
1077
|
+
const at = this.now();
|
|
1078
|
+
this.settleOperation(startOperationId, "completed", { threadId, turnId });
|
|
1079
|
+
this.store.db.run(
|
|
1080
|
+
"UPDATE side_chat_attempts SET state='running',thread_id=?1,turn_id=?2,updated_at=?3 WHERE attempt_id=?4 AND state IN ('starting','ambiguous','reconciling')",
|
|
1081
|
+
[threadId, turnId, at, attemptId],
|
|
1082
|
+
);
|
|
1083
|
+
this.store.db.run(
|
|
1084
|
+
"UPDATE side_chats SET state='running',derived_thread_id=?1,updated_at=?2 WHERE side_chat_id=?3 AND active_attempt_id=?4 AND state IN ('creating','ambiguous','reconciling')",
|
|
1085
|
+
[threadId, at, sideChatId, attemptId],
|
|
1086
|
+
);
|
|
1087
|
+
});
|
|
1088
|
+
return this.mustOwned(sideChatId, actor);
|
|
1089
|
+
} catch (error) {
|
|
1090
|
+
if (isAmbiguousRuntimeError(error)) {
|
|
1091
|
+
if (this.operationState(activeOperationId) === "completed")
|
|
1092
|
+
return this.mustOwned(sideChatId, actor);
|
|
1093
|
+
this.markAmbiguous(sideChatId, attemptId, activeOperationId, error);
|
|
1094
|
+
throw ambiguousError(activeOperationId, sideChatId, attemptId);
|
|
1095
|
+
}
|
|
1096
|
+
this.failStarting(sideChatId, attemptId, error, activeOperationId);
|
|
1097
|
+
throw normalizePortError(error);
|
|
1098
|
+
}
|
|
1099
|
+
}
|
|
1100
|
+
|
|
1101
|
+
private createNeedsReconciliation(record: SideThreadRecord): boolean {
|
|
1102
|
+
if (["creating", "ambiguous", "reconciling"].includes(record.state)) return true;
|
|
1103
|
+
const active = record.attempts.find((attempt) => attempt.attemptId === record.activeAttemptId);
|
|
1104
|
+
if (active && ["starting", "ambiguous", "reconciling"].includes(active.state)) return true;
|
|
1105
|
+
return record.operations.some((operation) =>
|
|
1106
|
+
(operation.kind === "fork" || operation.kind === "start") &&
|
|
1107
|
+
["pending", "ambiguous", "reconciling"].includes(operation.state),
|
|
1108
|
+
);
|
|
1109
|
+
}
|
|
1110
|
+
|
|
978
1111
|
get(actor: SideThreadActor, sideChatId: string): SideThreadRecord {
|
|
979
1112
|
this.assertEnabled();
|
|
980
1113
|
this.assertActor(actor);
|
|
@@ -1042,19 +1175,35 @@ export class SideThreadCoordinator {
|
|
|
1042
1175
|
operation_id: string;
|
|
1043
1176
|
attempt_id: string | null;
|
|
1044
1177
|
state: SideThreadOperationRecord["state"];
|
|
1178
|
+
error_code: string | null;
|
|
1045
1179
|
}>(
|
|
1046
|
-
"SELECT operation_id,attempt_id,state FROM side_chat_operations WHERE side_chat_id=?1 AND operation_kind='cancel' AND request_key=?2",
|
|
1180
|
+
"SELECT operation_id,attempt_id,state,error_code FROM side_chat_operations WHERE side_chat_id=?1 AND operation_kind='cancel' AND request_key=?2",
|
|
1047
1181
|
sideChatId,
|
|
1048
1182
|
input.requestKey,
|
|
1049
1183
|
);
|
|
1050
1184
|
if (replay?.state === "completed" || replay?.state === "pending")
|
|
1051
1185
|
return record;
|
|
1052
|
-
if (replay?.state === "
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1186
|
+
if (replay?.state === "failed") throw persistedOperationFailure(replay.error_code);
|
|
1187
|
+
if (replay?.state === "ambiguous" || replay?.state === "reconciling") {
|
|
1188
|
+
const attempt = record.attempts.find((a) => a.attemptId === replay.attempt_id);
|
|
1189
|
+
if (!attempt?.threadId || !attempt.turnId) throw conflict("cancel ownership missing");
|
|
1190
|
+
await this.reconcileLifecycleOperation({
|
|
1191
|
+
sideChatId, attemptId: attempt.attemptId, operationId: replay.operation_id,
|
|
1192
|
+
invoke: () => this.port.cancel({ operationId: replay.operation_id, sideChatId, attemptId: attempt.attemptId, nodeId: record.nodeId, threadId: attempt.threadId!, turnId: attempt.turnId! }),
|
|
1193
|
+
receipt: () => ({ threadId: attempt.threadId, turnId: attempt.turnId }),
|
|
1194
|
+
accepted: () => {
|
|
1195
|
+
const t = this.now();
|
|
1196
|
+
this.store.db.run("UPDATE side_chat_attempts SET state='running',updated_at=?1 WHERE attempt_id=?2 AND state IN ('ambiguous','reconciling')", [t, attempt.attemptId]);
|
|
1197
|
+
this.store.db.run("UPDATE side_chats SET state='running',updated_at=?1 WHERE side_chat_id=?2 AND active_attempt_id=?3 AND state IN ('ambiguous','reconciling')", [t, sideChatId, attempt.attemptId]);
|
|
1198
|
+
},
|
|
1199
|
+
failed: () => {
|
|
1200
|
+
const t = this.now();
|
|
1201
|
+
this.store.db.run("UPDATE side_chat_attempts SET state='running',cancel_requested_at=NULL,updated_at=?1 WHERE attempt_id=?2 AND state IN ('ambiguous','reconciling')", [t, attempt.attemptId]);
|
|
1202
|
+
this.store.db.run("UPDATE side_chats SET state='running',updated_at=?1 WHERE side_chat_id=?2 AND active_attempt_id=?3 AND state IN ('ambiguous','reconciling')", [t, sideChatId, attempt.attemptId]);
|
|
1203
|
+
},
|
|
1204
|
+
});
|
|
1205
|
+
return this.mustOwned(sideChatId, actor);
|
|
1206
|
+
}
|
|
1058
1207
|
const attempt = record.attempts.find(
|
|
1059
1208
|
(a) => a.attemptId === record.activeAttemptId,
|
|
1060
1209
|
);
|
|
@@ -1141,27 +1290,8 @@ export class SideThreadCoordinator {
|
|
|
1141
1290
|
throw normalizePortError(error);
|
|
1142
1291
|
}
|
|
1143
1292
|
this.store.db.transaction(() => {
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
"UPDATE side_chat_attempts SET state='cancelled',updated_at=?1 WHERE attempt_id=?2 AND state='running' AND thread_id=?3 AND turn_id=?4",
|
|
1147
|
-
[t, attempt.attemptId, attempt.threadId, attempt.turnId],
|
|
1148
|
-
);
|
|
1149
|
-
if (changed.changes > 0) {
|
|
1150
|
-
this.store.db.run(
|
|
1151
|
-
"UPDATE side_chats SET state='cancelled',active_attempt_id=NULL,updated_at=?1 WHERE side_chat_id=?2 AND active_attempt_id=?3",
|
|
1152
|
-
[t, sideChatId, attempt.attemptId],
|
|
1153
|
-
);
|
|
1154
|
-
this.emitStored(
|
|
1155
|
-
sideChatId,
|
|
1156
|
-
attempt.attemptId,
|
|
1157
|
-
attempt.threadId,
|
|
1158
|
-
attempt.turnId,
|
|
1159
|
-
"attempt.cancelled",
|
|
1160
|
-
"cancelled",
|
|
1161
|
-
undefined,
|
|
1162
|
-
t,
|
|
1163
|
-
);
|
|
1164
|
-
}
|
|
1293
|
+
// Command acceptance only proves that interrupt was accepted. The
|
|
1294
|
+
// authoritative terminal event owns the cancelled/completed race.
|
|
1165
1295
|
this.settleOperation(prepared.operationId, "completed", {
|
|
1166
1296
|
threadId: attempt.threadId,
|
|
1167
1297
|
turnId: attempt.turnId,
|
|
@@ -1337,10 +1467,7 @@ export class SideThreadCoordinator {
|
|
|
1337
1467
|
this.markAmbiguous(sideChatId, attemptId, startOperationId, error);
|
|
1338
1468
|
throw ambiguousError(startOperationId, sideChatId, attemptId);
|
|
1339
1469
|
}
|
|
1340
|
-
this.
|
|
1341
|
-
errorCode: auditErrorReason(error),
|
|
1342
|
-
});
|
|
1343
|
-
this.failStarting(sideChatId, attemptId, error);
|
|
1470
|
+
this.failStarting(sideChatId, attemptId, error, startOperationId);
|
|
1344
1471
|
throw normalizePortError(error);
|
|
1345
1472
|
}
|
|
1346
1473
|
return this.mustOwned(sideChatId, actor);
|
|
@@ -1379,6 +1506,11 @@ export class SideThreadCoordinator {
|
|
|
1379
1506
|
if (!r.threadId) throw conflict("side chat has no derived thread");
|
|
1380
1507
|
if (r.state === "purged") return r;
|
|
1381
1508
|
if (action === "archive" && r.state === "archived") return r;
|
|
1509
|
+
const priorOperation = this.store.db.get<{ state: string; error_code: string | null }>(
|
|
1510
|
+
"SELECT state,error_code FROM side_chat_operations WHERE side_chat_id=?1 AND operation_kind=?2 AND request_key=?3",
|
|
1511
|
+
sideChatId, action, input.requestKey,
|
|
1512
|
+
);
|
|
1513
|
+
if (priorOperation?.state === "failed") throw persistedOperationFailure(priorOperation.error_code);
|
|
1382
1514
|
await this.assertCapability(r.nodeId);
|
|
1383
1515
|
const prepared = this.prepareStableOperation({
|
|
1384
1516
|
sideChatId,
|
|
@@ -1388,8 +1520,18 @@ export class SideThreadCoordinator {
|
|
|
1388
1520
|
threadId: r.threadId,
|
|
1389
1521
|
});
|
|
1390
1522
|
if (prepared.state === "completed") return r;
|
|
1391
|
-
if (prepared.state === "ambiguous" || prepared.state === "reconciling")
|
|
1392
|
-
|
|
1523
|
+
if (prepared.state === "ambiguous" || prepared.state === "reconciling") {
|
|
1524
|
+
await this.reconcileLifecycleOperation({
|
|
1525
|
+
sideChatId, operationId: prepared.operationId,
|
|
1526
|
+
invoke: () => action === "archive"
|
|
1527
|
+
? this.port.archive({ operationId: prepared.operationId, sideChatId, nodeId: r.nodeId, threadId: r.threadId! })
|
|
1528
|
+
: this.port.purge({ operationId: prepared.operationId, sideChatId, nodeId: r.nodeId, threadId: r.threadId! }),
|
|
1529
|
+
receipt: () => ({ threadId: r.threadId }),
|
|
1530
|
+
accepted: () => this.applyLifecycleAccepted(sideChatId, action, prepared.operationId, r.threadId!),
|
|
1531
|
+
failed: () => this.store.db.run("UPDATE side_chats SET state='failed',lifecycle_operation=NULL,updated_at=?1 WHERE side_chat_id=?2 AND state IN ('ambiguous','reconciling')", [this.now(), sideChatId]),
|
|
1532
|
+
});
|
|
1533
|
+
return this.mustOwned(sideChatId, actor);
|
|
1534
|
+
}
|
|
1393
1535
|
if (!prepared.callable)
|
|
1394
1536
|
throw conflict("side chat lifecycle operation already in progress");
|
|
1395
1537
|
const claim = this.store.db.run(
|
|
@@ -1536,21 +1678,33 @@ export class SideThreadCoordinator {
|
|
|
1536
1678
|
const operation = this.store.db.get<{
|
|
1537
1679
|
operation_id: string;
|
|
1538
1680
|
state: string;
|
|
1681
|
+
error_code: string | null;
|
|
1539
1682
|
}>(
|
|
1540
|
-
"SELECT operation_id,state FROM side_chat_operations WHERE side_chat_id=?1 AND operation_kind='bring-back' AND request_key=?2",
|
|
1683
|
+
"SELECT operation_id,state,error_code FROM side_chat_operations WHERE side_chat_id=?1 AND operation_kind='bring-back' AND request_key=?2",
|
|
1541
1684
|
sideChatId,
|
|
1542
1685
|
input.requestKey,
|
|
1543
1686
|
);
|
|
1544
|
-
if (
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
operation.
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1687
|
+
if (operation && (operation.state === "ambiguous" || operation.state === "reconciling")) {
|
|
1688
|
+
const destinationTurnId = await this.reconcileLifecycleOperation({
|
|
1689
|
+
sideChatId, attemptId: attempt.attemptId, operationId: operation.operation_id,
|
|
1690
|
+
invoke: async () => (await this.port.bringBack({ operationId: operation.operation_id, sideChatId, attemptId: attempt.attemptId, nodeId: r.nodeId, sourceThreadId: attempt.threadId!, sourceTurnId: attempt.turnId!, destinationThreadId: input.destinationThreadId, requestKey: input.requestKey, text: attempt.result! })).destinationTurnId,
|
|
1691
|
+
receipt: (turn) => ({ threadId: input.destinationThreadId, turnId: turn }),
|
|
1692
|
+
accepted: (turn) => {
|
|
1693
|
+
requireRuntimeIdentity(turn, "destinationTurnId");
|
|
1694
|
+
const now = this.now();
|
|
1695
|
+
this.store.db.run("UPDATE side_chat_bring_backs SET state='completed',destination_turn_id=?1,error_text=NULL,updated_at=?2,completed_at=?2 WHERE bring_back_id=?3 AND state IN ('starting','ambiguous','reconciling')", [turn, now, old.bring_back_id]);
|
|
1696
|
+
this.store.db.run("UPDATE side_chats SET state='completed',updated_at=?1 WHERE side_chat_id=?2 AND state IN ('ambiguous','reconciling')", [now, sideChatId]);
|
|
1697
|
+
this.emitStored(sideChatId, attempt.attemptId, attempt.threadId, attempt.turnId, "side_chat.brought_back", "completed", undefined, now);
|
|
1698
|
+
},
|
|
1699
|
+
failed: (error) => {
|
|
1700
|
+
const now = this.now();
|
|
1701
|
+
this.store.db.run("UPDATE side_chat_bring_backs SET state='failed',error_text=?1,updated_at=?2 WHERE bring_back_id=?3 AND state IN ('starting','ambiguous','reconciling')", [safeReason(error instanceof Error ? error.message : String(error)), now, old.bring_back_id]);
|
|
1702
|
+
this.store.db.run("UPDATE side_chats SET state='completed',updated_at=?1 WHERE side_chat_id=?2 AND state IN ('ambiguous','reconciling')", [now, sideChatId]);
|
|
1703
|
+
},
|
|
1704
|
+
});
|
|
1705
|
+
return { bringBackId: old.bring_back_id, destinationTurnId };
|
|
1706
|
+
}
|
|
1707
|
+
if (operation?.state === "failed") throw persistedOperationFailure(operation.error_code);
|
|
1554
1708
|
throw conflict("bring-back is already in progress or failed");
|
|
1555
1709
|
}
|
|
1556
1710
|
return {
|
|
@@ -1723,6 +1877,84 @@ export class SideThreadCoordinator {
|
|
|
1723
1877
|
);
|
|
1724
1878
|
}
|
|
1725
1879
|
|
|
1880
|
+
private applyRuntimeReceipt(apply: () => void): void {
|
|
1881
|
+
try {
|
|
1882
|
+
this.store.db.transaction(apply);
|
|
1883
|
+
} catch {
|
|
1884
|
+
// The runtime receipt remains authoritative and durable; only the local
|
|
1885
|
+
// projection failed. Callers mark the stable operation ambiguous so a
|
|
1886
|
+
// restart can consume the same receipt without another native command.
|
|
1887
|
+
throw new SideThreadError(
|
|
1888
|
+
"SIDE_THREAD_AMBIGUOUS",
|
|
1889
|
+
"runtime receipt accepted; local apply requires reconciliation",
|
|
1890
|
+
202,
|
|
1891
|
+
);
|
|
1892
|
+
}
|
|
1893
|
+
}
|
|
1894
|
+
|
|
1895
|
+
private async reconcileLifecycleOperation<T>(input: {
|
|
1896
|
+
sideChatId: string;
|
|
1897
|
+
attemptId?: string;
|
|
1898
|
+
operationId: string;
|
|
1899
|
+
invoke: () => Promise<T>;
|
|
1900
|
+
receipt: (value: T) => { threadId?: string; turnId?: string };
|
|
1901
|
+
accepted: (value: T) => void;
|
|
1902
|
+
failed: (error: unknown) => void;
|
|
1903
|
+
}): Promise<T> {
|
|
1904
|
+
this.store.db.run(
|
|
1905
|
+
"UPDATE side_chat_operations SET state='reconciling',updated_at=?1 WHERE operation_id=?2 AND state='ambiguous'",
|
|
1906
|
+
[this.now(), input.operationId],
|
|
1907
|
+
);
|
|
1908
|
+
let value: T;
|
|
1909
|
+
try {
|
|
1910
|
+
value = await input.invoke();
|
|
1911
|
+
} catch (error) {
|
|
1912
|
+
if (isAmbiguousRuntimeError(error)) {
|
|
1913
|
+
this.markAmbiguous(input.sideChatId, input.attemptId, input.operationId, error);
|
|
1914
|
+
throw ambiguousError(input.operationId, input.sideChatId, input.attemptId);
|
|
1915
|
+
}
|
|
1916
|
+
try {
|
|
1917
|
+
this.store.db.transaction(() => {
|
|
1918
|
+
const won = this.store.db.run(
|
|
1919
|
+
"UPDATE side_chat_operations SET state='failed',error_code=?1,updated_at=?2 WHERE operation_id=?3 AND state IN ('ambiguous','reconciling')",
|
|
1920
|
+
[auditErrorReason(error), this.now(), input.operationId],
|
|
1921
|
+
);
|
|
1922
|
+
if (won.changes === 1) input.failed(error);
|
|
1923
|
+
});
|
|
1924
|
+
} catch {
|
|
1925
|
+
this.markAmbiguous(input.sideChatId, input.attemptId, input.operationId, { code: "SIDE_THREAD_LOCAL_APPLY_FAILED" });
|
|
1926
|
+
throw ambiguousError(input.operationId, input.sideChatId, input.attemptId);
|
|
1927
|
+
}
|
|
1928
|
+
throw normalizePortError(error);
|
|
1929
|
+
}
|
|
1930
|
+
try {
|
|
1931
|
+
this.store.db.transaction(() => {
|
|
1932
|
+
const receipt = input.receipt(value);
|
|
1933
|
+
const won = this.store.db.run(
|
|
1934
|
+
"UPDATE side_chat_operations SET state='completed',thread_id=COALESCE(?1,thread_id),turn_id=COALESCE(?2,turn_id),error_code=NULL,updated_at=?3 WHERE operation_id=?4 AND state IN ('ambiguous','reconciling')",
|
|
1935
|
+
[receipt.threadId ?? null, receipt.turnId ?? null, this.now(), input.operationId],
|
|
1936
|
+
);
|
|
1937
|
+
if (won.changes === 1) input.accepted(value);
|
|
1938
|
+
});
|
|
1939
|
+
return value;
|
|
1940
|
+
} catch {
|
|
1941
|
+
this.markAmbiguous(input.sideChatId, input.attemptId, input.operationId, { code: "SIDE_THREAD_LOCAL_APPLY_FAILED" });
|
|
1942
|
+
throw ambiguousError(input.operationId, input.sideChatId, input.attemptId);
|
|
1943
|
+
}
|
|
1944
|
+
}
|
|
1945
|
+
|
|
1946
|
+
private applyLifecycleAccepted(sideChatId: string, action: "archive" | "purge", operationId: string, threadId: string): void {
|
|
1947
|
+
const t = this.now();
|
|
1948
|
+
if (action === "purge") {
|
|
1949
|
+
this.store.db.run("UPDATE side_chats SET state='purged',question_text='',attachments_json='[]',lifecycle_operation=NULL,updated_at=?1,purged_at=?1 WHERE side_chat_id=?2 AND active_attempt_id IS NULL", [t, sideChatId]);
|
|
1950
|
+
this.store.db.run("UPDATE side_chat_attempts SET result_text=NULL,error_text=NULL,attachments_json='[]',updated_at=?1 WHERE side_chat_id=?2", [t, sideChatId]);
|
|
1951
|
+
this.store.db.run("UPDATE side_chat_bring_backs SET error_text=NULL,updated_at=?1 WHERE side_chat_id=?2", [t, sideChatId]);
|
|
1952
|
+
} else {
|
|
1953
|
+
this.store.db.run("UPDATE side_chats SET state='archived',lifecycle_operation=NULL,updated_at=?1,archived_at=?1 WHERE side_chat_id=?2 AND active_attempt_id IS NULL", [t, sideChatId]);
|
|
1954
|
+
}
|
|
1955
|
+
this.emitStored(sideChatId, undefined, threadId, undefined, `side_chat.${action === "archive" ? "archived" : "purged"}`, action === "archive" ? "archived" : "purged", undefined, t);
|
|
1956
|
+
}
|
|
1957
|
+
|
|
1726
1958
|
listEvents(
|
|
1727
1959
|
actor: SideThreadActor,
|
|
1728
1960
|
sideChatId: string,
|
|
@@ -1983,14 +2215,19 @@ export class SideThreadCoordinator {
|
|
|
1983
2215
|
sideChatId: string,
|
|
1984
2216
|
attemptId: string,
|
|
1985
2217
|
error: unknown,
|
|
2218
|
+
operationId?: string,
|
|
1986
2219
|
): void {
|
|
1987
2220
|
const t = this.now(),
|
|
1988
2221
|
reason = safeReason(
|
|
1989
2222
|
error instanceof Error ? error.message : String(error),
|
|
1990
2223
|
);
|
|
1991
2224
|
this.store.db.transaction(() => {
|
|
2225
|
+
if (operationId)
|
|
2226
|
+
this.settleOperation(operationId, "failed", {
|
|
2227
|
+
errorCode: auditErrorReason(error),
|
|
2228
|
+
});
|
|
1992
2229
|
const changed = this.store.db.run(
|
|
1993
|
-
"UPDATE side_chat_attempts SET state='failed',error_text=?1,updated_at=?2 WHERE attempt_id=?3 AND state
|
|
2230
|
+
"UPDATE side_chat_attempts SET state='failed',error_text=?1,updated_at=?2 WHERE attempt_id=?3 AND state IN ('starting','ambiguous','reconciling')",
|
|
1994
2231
|
[reason, t, attemptId],
|
|
1995
2232
|
);
|
|
1996
2233
|
if (changed.changes > 0) {
|
|
@@ -2038,7 +2275,7 @@ export class SideThreadCoordinator {
|
|
|
2038
2275
|
"SELECT event_id FROM side_chat_events WHERE side_chat_id=?1 ORDER BY event_id DESC LIMIT 1",
|
|
2039
2276
|
sideChatId,
|
|
2040
2277
|
);
|
|
2041
|
-
|
|
2278
|
+
const event = {
|
|
2042
2279
|
eventId: row?.event_id ?? 0,
|
|
2043
2280
|
sideChatId,
|
|
2044
2281
|
attemptId,
|
|
@@ -2048,7 +2285,19 @@ export class SideThreadCoordinator {
|
|
|
2048
2285
|
state,
|
|
2049
2286
|
reason: safeReason(reason),
|
|
2050
2287
|
createdAt: at,
|
|
2051
|
-
} satisfies SideThreadEventRecord
|
|
2288
|
+
} satisfies SideThreadEventRecord;
|
|
2289
|
+
// Fanout is deliberately deferred until the current synchronous SQLite
|
|
2290
|
+
// transaction has committed. Observers are never part of durable truth:
|
|
2291
|
+
// one throwing listener cannot roll back state or starve another listener.
|
|
2292
|
+
queueMicrotask(() => {
|
|
2293
|
+
for (const listener of this.emitter.listeners(`side:${sideChatId}`)) {
|
|
2294
|
+
try {
|
|
2295
|
+
listener(event);
|
|
2296
|
+
} catch {
|
|
2297
|
+
// Durable side_chat_events + cursor replay are authoritative.
|
|
2298
|
+
}
|
|
2299
|
+
}
|
|
2300
|
+
});
|
|
2052
2301
|
}
|
|
2053
2302
|
private mustOwned(
|
|
2054
2303
|
sideChatId: string,
|
|
@@ -2239,3 +2488,8 @@ function normalizePortError(error: unknown): SideThreadError {
|
|
|
2239
2488
|
502,
|
|
2240
2489
|
);
|
|
2241
2490
|
}
|
|
2491
|
+
function persistedOperationFailure(code: string | null | undefined): SideThreadError {
|
|
2492
|
+
if (code === "SIDE_THREAD_UNSUPPORTED")
|
|
2493
|
+
return new SideThreadError("SIDE_THREAD_UNSUPPORTED", "runtime operation is unsupported", 501);
|
|
2494
|
+
return new SideThreadError(code === "SIDE_THREAD_CONFLICT" ? "SIDE_THREAD_CONFLICT" : "SIDE_THREAD_RUNTIME_ERROR", "runtime operation failed authoritatively", code === "SIDE_THREAD_CONFLICT" ? 409 : 502);
|
|
2495
|
+
}
|
package/src/tools.ts
CHANGED
|
@@ -526,6 +526,16 @@ export function registerTools(server: McpServer, clientIP?: string, enforceNetwo
|
|
|
526
526
|
config_revision: z.number().int().min(0).optional(),
|
|
527
527
|
config_update_capable: z.boolean().optional(),
|
|
528
528
|
peer_reply_inbox_capable: z.literal(true).optional(),
|
|
529
|
+
side_thread_capability: z.object({
|
|
530
|
+
supported: z.boolean(),
|
|
531
|
+
runtime: z.string().max(64),
|
|
532
|
+
runtimeVersion: z.string().max(64),
|
|
533
|
+
topology: z.string().max(64),
|
|
534
|
+
evidenceRevision: z.string().max(100),
|
|
535
|
+
mode: z.literal("native-exact-fork").optional(),
|
|
536
|
+
exactBoundary: z.object({ through: z.boolean(), before: z.boolean() }).strict().optional(),
|
|
537
|
+
reason: z.enum(["runtime", "version", "topology", "experimental-api", "exact-boundary"]).optional(),
|
|
538
|
+
}).strict().optional(),
|
|
529
539
|
// RFC-026 P2 / #338 — daemon role surfaced for hub /api/nodes
|
|
530
540
|
// discovery (#337 extracts this field). "host_supervisor" =
|
|
531
541
|
// anet daemon. Default-stripping zod would drop this otherwise.
|