@fitchmultz/pi-subagents 0.44.4 → 0.44.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/README.md +4 -0
- package/dist/extension/build-info.js +1 -1
- package/dist/extension/compact-tools.js +1 -1
- package/dist/extension/schemas.js +9 -9
- package/dist/history/index.js +14 -9
- package/dist/history/ingest.js +26 -5
- package/dist/history/queries.js +24 -4
- package/dist/history/worker.js +40 -14
- package/dist/pi-intercom/bridge.js +750 -0
- package/dist/runs/foreground/question-control.js +23 -12
- package/dist/runs/foreground/wait-run.js +5 -3
- package/dist/runs/shared/run-history.js +8 -1
- package/dist/runs/shared/supervisor-questions.js +32 -2
- package/dist/shared/status-format.js +1 -1
- package/docs/intercom-bridge.md +212 -0
- package/docs/intercom.md +3 -1
- package/docs/performance.md +3 -3
- package/package.json +2 -1
- package/scripts/intercom-bridge-credentials.mjs +143 -0
- package/scripts/intercom-remote.mjs +103 -0
- package/scripts/package-smoke.mjs +4 -0
- package/src/extension/compact-tools.ts +1 -1
- package/src/extension/schemas.ts +9 -9
- package/src/history/index.ts +13 -9
- package/src/history/ingest.ts +22 -5
- package/src/history/queries.ts +22 -4
- package/src/history/worker.ts +37 -13
- package/src/pi-intercom/bridge.ts +668 -0
- package/src/runs/foreground/question-control.ts +23 -11
- package/src/runs/foreground/wait-run.ts +5 -2
- package/src/runs/shared/run-history.ts +8 -1
- package/src/runs/shared/supervisor-questions.ts +45 -2
- package/src/shared/status-format.ts +2 -2
- package/src/shared/types.ts +2 -1
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import https from "node:https";
|
|
3
|
+
import { lstatSync, readFileSync } from "node:fs";
|
|
4
|
+
import { parseArgs } from "node:util";
|
|
5
|
+
|
|
6
|
+
function help() {
|
|
7
|
+
console.log(`Usage: node intercom-remote.mjs <register|list|inbox|send|ask|reply|ack> [options] [IDs...]
|
|
8
|
+
|
|
9
|
+
Environment (required):
|
|
10
|
+
INTERCOM_BRIDGE_CA Path to trusted bridge CA certificate
|
|
11
|
+
INTERCOM_BRIDGE_CERT Path to your client certificate
|
|
12
|
+
INTERCOM_BRIDGE_KEY Path to your private client key (owned, mode 0600)
|
|
13
|
+
INTERCOM_BRIDGE_URL HTTPS origin (default: https://127.0.0.1:9443)
|
|
14
|
+
|
|
15
|
+
Options:
|
|
16
|
+
--to PEER Recipient name or ID for send/ask
|
|
17
|
+
--message TEXT Message for send/ask/reply
|
|
18
|
+
--reply-to ID Inbound ask ID from inbox, for reply
|
|
19
|
+
--timeout-ms MS Ask timeout (100..120000, default: 120000)
|
|
20
|
+
-h, --help Show help
|
|
21
|
+
|
|
22
|
+
Examples:
|
|
23
|
+
node intercom-remote.mjs list
|
|
24
|
+
node intercom-remote.mjs ask --to planner --message 'What is the current status?'
|
|
25
|
+
node intercom-remote.mjs inbox
|
|
26
|
+
node intercom-remote.mjs reply --reply-to QUESTION_ID --message 'Proceed.'
|
|
27
|
+
node intercom-remote.mjs ack MESSAGE_ID
|
|
28
|
+
|
|
29
|
+
Outputs JSON. Exit: 0 success, 1 API/TLS/transport failure, 2 invalid arguments.
|
|
30
|
+
No automatic retries: a lost response does not prove a message was unsent.`);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
let options, body, action, origin, timeout;
|
|
34
|
+
try {
|
|
35
|
+
const parsed = parseArgs({ allowPositionals: true, options: {
|
|
36
|
+
help: { type: "boolean", short: "h" }, to: { type: "string" }, message: { type: "string" },
|
|
37
|
+
"reply-to": { type: "string" }, "timeout-ms": { type: "string" },
|
|
38
|
+
} });
|
|
39
|
+
if (parsed.values.help) { help(); process.exit(0); }
|
|
40
|
+
options = parsed.values;
|
|
41
|
+
[action] = parsed.positionals;
|
|
42
|
+
if (!["register", "list", "inbox", "send", "ask", "reply", "ack"].includes(action)) throw new Error("Choose an action; use --help.");
|
|
43
|
+
const allowed = ["send", "ask"].includes(action) ? ["to", "message", ...(action === "ask" ? ["timeout-ms"] : [])] : action === "reply" ? ["reply-to", "message"] : [];
|
|
44
|
+
if (Object.keys(options).some((key) => !allowed.includes(key)) ||
|
|
45
|
+
(action !== "ack" && parsed.positionals.length !== 1)) throw new Error("Unexpected arguments for this action.");
|
|
46
|
+
timeout = options["timeout-ms"] === undefined ? 120000 : Number(options["timeout-ms"]);
|
|
47
|
+
if (!Number.isInteger(timeout) || timeout < 100 || timeout > 120000) throw new Error("--timeout-ms must be 100..120000.");
|
|
48
|
+
body = ["send", "ask"].includes(action) ? { to: options.to, message: options.message, ...(action === "ask" ? { timeoutMs: timeout } : {}) }
|
|
49
|
+
: action === "reply" ? { replyTo: options["reply-to"], message: options.message }
|
|
50
|
+
: action === "ack" ? { ids: parsed.positionals.slice(1) } : {};
|
|
51
|
+
if (["send", "ask", "reply"].includes(action) && (!body.message?.trim() || !(action === "reply" ? body.replyTo : body.to)?.trim())) throw new Error("Recipient/ask ID and message are required.");
|
|
52
|
+
if (action === "ack" && body.ids.length === 0) throw new Error("ack needs at least one message ID.");
|
|
53
|
+
origin = new URL(process.env.INTERCOM_BRIDGE_URL || "https://127.0.0.1:9443");
|
|
54
|
+
if (origin.protocol !== "https:" || origin.username || origin.password || origin.pathname !== "/" || origin.search || origin.hash) throw new Error("INTERCOM_BRIDGE_URL must be an HTTPS origin, without credentials or a path.");
|
|
55
|
+
} catch (error) {
|
|
56
|
+
console.error(JSON.stringify({ ok: false, error: { code: "invalid_arguments", message: error.message } }));
|
|
57
|
+
process.exit(2);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
try {
|
|
61
|
+
const files = {};
|
|
62
|
+
for (const field of ["CA", "CERT", "KEY"]) {
|
|
63
|
+
const path = process.env[`INTERCOM_BRIDGE_${field}`];
|
|
64
|
+
if (!path) throw new Error(`INTERCOM_BRIDGE_${field} is required.`);
|
|
65
|
+
if (field === "KEY") {
|
|
66
|
+
const st = lstatSync(path);
|
|
67
|
+
if (!st.isFile() || st.uid !== process.getuid() || (st.mode & 0o077)) throw new Error("Client key must be an owned private regular file (0600).");
|
|
68
|
+
}
|
|
69
|
+
files[field] = readFileSync(path);
|
|
70
|
+
}
|
|
71
|
+
origin.pathname = `/v1/${action}`;
|
|
72
|
+
const payload = JSON.stringify(body);
|
|
73
|
+
const get = ["list", "inbox"].includes(action);
|
|
74
|
+
const result = await new Promise((resolve, reject) => {
|
|
75
|
+
const req = https.request(origin, {
|
|
76
|
+
method: get ? "GET" : "POST", ca: files.CA, cert: files.CERT, key: files.KEY,
|
|
77
|
+
rejectUnauthorized: true, minVersion: "TLSv1.2", agent: false,
|
|
78
|
+
headers: get ? {} : { "Content-Type": "application/json", "Content-Length": Buffer.byteLength(payload) },
|
|
79
|
+
}, (res) => {
|
|
80
|
+
const chunks = [];
|
|
81
|
+
let bytes = 0;
|
|
82
|
+
res.on("data", (chunk) => {
|
|
83
|
+
bytes += chunk.length;
|
|
84
|
+
if (bytes > 16 * 1024 * 1024) { req.destroy(new Error("Bridge response exceeds 16 MiB.")); return; }
|
|
85
|
+
chunks.push(chunk);
|
|
86
|
+
});
|
|
87
|
+
res.on("error", reject);
|
|
88
|
+
res.on("end", () => {
|
|
89
|
+
try { resolve({ status: res.statusCode, value: JSON.parse(Buffer.concat(chunks).toString("utf8")) }); }
|
|
90
|
+
catch { reject(new Error("Bridge returned invalid JSON.")); }
|
|
91
|
+
});
|
|
92
|
+
});
|
|
93
|
+
const timer = setTimeout(() => req.destroy(new Error("Bridge request deadline exceeded; delivery may be unknown.")), (action === "ask" ? timeout : 0) + 15000);
|
|
94
|
+
req.once("close", () => clearTimeout(timer));
|
|
95
|
+
req.once("error", reject);
|
|
96
|
+
req.end(get ? undefined : payload);
|
|
97
|
+
});
|
|
98
|
+
console.log(JSON.stringify(result.value, null, 2));
|
|
99
|
+
if (result.status < 200 || result.status >= 300 || result.value?.ok !== true) process.exitCode = 1;
|
|
100
|
+
} catch (error) {
|
|
101
|
+
console.error(JSON.stringify({ ok: false, error: { code: "transport_error", message: error.message, delivery: "unknown; do not blindly retry mutations" } }));
|
|
102
|
+
process.exitCode = 1;
|
|
103
|
+
}
|
|
@@ -77,6 +77,10 @@ for (const path of [
|
|
|
77
77
|
"dist/extension/index.js",
|
|
78
78
|
"dist/pi-intercom/index.js",
|
|
79
79
|
"dist/pi-intercom/broker/broker.js",
|
|
80
|
+
"dist/pi-intercom/bridge.js",
|
|
81
|
+
"scripts/intercom-bridge-credentials.mjs",
|
|
82
|
+
"scripts/intercom-remote.mjs",
|
|
83
|
+
"docs/intercom-bridge.md",
|
|
80
84
|
"dist/runs/background/subagent-runner-launcher.js",
|
|
81
85
|
"src/extension/index.ts",
|
|
82
86
|
"src/extension/schemas.ts",
|
|
@@ -67,7 +67,7 @@ export function registerCompactSubagentTools(pi: ExtensionAPI, options: {
|
|
|
67
67
|
name: "agent_runs",
|
|
68
68
|
defaultActive: false,
|
|
69
69
|
label: "Agent Runs",
|
|
70
|
-
description: `List ${childSafe ? "only this child's directly owned" : "your delegated"} runs across working directories (questions/failures, then live work, then unreviewed results; 20 per page). Filter globally by agent/state/text, sort, and page with the returned cursor. history reads 100 bounded native-entry previews; search finds saved visible text using words or quoted phrases, not operators or prefixes. Browse freshness is not completion or delivery proof. Inspect concise results, paths and continuations; full:true includes the full task/configuration. Answer durable questions, nudge, stop, continue, or save parent-only review. Review notes are not sent to children; put actionable instructions in continue/nudge. Inspect/review/nudge never restart finished work. Continue/answer can launch a saved child; async:false waits for its actual result. Saved continuations keep settings unless agent selects a current profile; model overrides win. Live guidance never mutates model or acceptance. profiles lists roles, sources, context and model/thinking/fallback defaults. History survives reload.`,
|
|
70
|
+
description: `List ${childSafe ? "only this child's directly owned" : "your delegated"} runs across working directories (questions/failures, then live work, then unreviewed results; 20 per page). Filter globally by agent/state/text, sort, and page with the returned cursor. history reads 100 bounded native-entry previews; search finds saved visible text using words or quoted phrases, not operators or prefixes. questions pages 20 compact previews (offset/limit); questionId selects one, and questionPath/answerPath retain full records. Browse freshness is not completion or delivery proof. Inspect concise results, paths and continuations; full:true includes the full task/configuration. Answer durable questions, nudge, stop, continue, or save parent-only review. Review notes are not sent to children; put actionable instructions in continue/nudge. Inspect/review/nudge never restart finished work. Continue/answer can launch a saved child; async:false waits for its actual result. Saved continuations keep settings unless agent selects a current profile; model overrides win. Live guidance never mutates model or acceptance. profiles lists roles, sources, context and model/thinking/fallback defaults. History survives reload.`,
|
|
71
71
|
parameters: AgentRunsParams,
|
|
72
72
|
async execute(id, params, signal, onUpdate, ctx) {
|
|
73
73
|
const normalized = normalizeEverydayParams(params, true);
|
package/src/extension/schemas.ts
CHANGED
|
@@ -279,13 +279,13 @@ const HistoryQueryFields = {
|
|
|
279
279
|
export const AgentRunsValidationParams = Type.Object({
|
|
280
280
|
action: Type.Enum(["list", "inspect", "history", "search", "nudge", "stop", "continue", "profiles", "questions", "answer", "review"] as const, { type: "string" }),
|
|
281
281
|
id: Type.Optional(Type.String({ minLength: 1, description: "Run ID or unambiguous prefix." })),
|
|
282
|
-
questionId: Type.Optional(Type.String({ minLength: 1, description: "Durable supervisor question ID for answer." })),
|
|
282
|
+
questionId: Type.Optional(Type.String({ minLength: 1, description: "Durable supervisor question ID for exact questions lookup or answer." })),
|
|
283
283
|
async: Type.Optional(Type.Boolean({ description: "Continue/answer: false waits for the actual continuation result." })),
|
|
284
284
|
index: Type.Optional(Type.Integer({ minimum: 0, description: "Child index for a multi-child run." })),
|
|
285
285
|
message: Type.Optional(Type.String({ minLength: 1, description: "Guidance, follow-up, answer, or optional parent-only review note. Review notes are not sent to the child; put actionable instructions in continue/nudge. Only continue/answer can start a saved child." })),
|
|
286
286
|
decision: Type.Optional(Type.Enum(["accepted", "needs_changes"] as const, { type: "string", description: "Parent-only review outcome; not sent to the child. Separate from execution and runtime acceptance checks." })),
|
|
287
|
-
offset: Type.Optional(Type.Integer({ minimum: 0, description: "List offset; history is retained regardless of page size." })),
|
|
288
|
-
limit: Type.Optional(Type.Integer({ minimum: 1, maximum: 100, description: "Results per list/search/history page (default 20; history 100)." })),
|
|
287
|
+
offset: Type.Optional(Type.Integer({ minimum: 0, description: "List/questions offset; history is retained regardless of page size." })),
|
|
288
|
+
limit: Type.Optional(Type.Integer({ minimum: 1, maximum: 100, description: "Results per list/questions/search/history page (default 20; history 100)." })),
|
|
289
289
|
...HistoryQueryFields,
|
|
290
290
|
full: Type.Optional(Type.Boolean({ description: "Inspect only: include the full task and saved launch configuration. Default is a concise report; stored data is unchanged." })),
|
|
291
291
|
agent: Type.Optional(Type.String({ minLength: 1, description: "List/search: filter by child agent. Continue/answer: adopt this current profile, including model, thinking and fallbacks; a separate model override wins. Otherwise keep saved settings. Required for old runs without a saved profile." })),
|
|
@@ -303,8 +303,8 @@ export const AgentRunsValidationParams = Type.Object({
|
|
|
303
303
|
{ if: { properties: { action: { enum: ["review"] } } }, then: requiredObject("decision") },
|
|
304
304
|
{ if: { anyOf: [requiredObject("async"), requiredObject("model"), requiredObject("cwd"), requiredObject("output"), requiredObject("acceptance")] }, then: { properties: { action: { enum: ["continue", "answer"] } } } },
|
|
305
305
|
{ if: requiredObject("agent"), then: { properties: { action: { enum: ["list", "search", "continue", "answer"] } } } },
|
|
306
|
-
{ if: requiredObject("offset"), then: { properties: { action: { enum: ["list"] } } } },
|
|
307
|
-
{ if: requiredObject("limit"), then: { properties: { action: { enum: ["list", "history", "search"] } } } },
|
|
306
|
+
{ if: requiredObject("offset"), then: { properties: { action: { enum: ["list", "questions"] } } } },
|
|
307
|
+
{ if: requiredObject("limit"), then: { properties: { action: { enum: ["list", "questions", "history", "search"] } } } },
|
|
308
308
|
{ if: requiredObject("cursor"), then: { properties: { action: { enum: ["list", "history", "search"] } } } },
|
|
309
309
|
{ if: requiredObject("sort"), then: { properties: { action: { enum: ["list", "search"] } } } },
|
|
310
310
|
{ not: { anyOf: [requiredObject("cursor", "offset"), requiredObject("cursor", "before")] } },
|
|
@@ -339,9 +339,9 @@ export const SubagentParams = Type.Object({
|
|
|
339
339
|
runId: Type.Optional(Type.String({
|
|
340
340
|
description: "Target run ID; prefer id. Defaults to the most recently active controllable run for interrupt/extend/nudge."
|
|
341
341
|
})),
|
|
342
|
-
questionId:
|
|
342
|
+
questionId: AgentRunsValidationParams.properties.questionId,
|
|
343
343
|
decision: Type.Optional(Type.Enum(["accepted", "needs_changes"] as const, { type: "string", description: "Parent-only review decision; not sent to the child. Put actionable instructions in resume/nudge. Does not launch work or change runtime acceptance." })),
|
|
344
|
-
offset: Type.Optional(Type.Integer({ minimum: 0, description: "Status list offset." })),
|
|
344
|
+
offset: Type.Optional(Type.Integer({ minimum: 0, description: "Status list/questions offset." })),
|
|
345
345
|
limit: AgentRunsValidationParams.properties.limit,
|
|
346
346
|
...HistoryQueryFields,
|
|
347
347
|
full: Type.Optional(Type.Boolean({ description: "Exact status only: include the full task and saved launch configuration; default is concise." })),
|
|
@@ -418,8 +418,8 @@ export const SubagentParams = Type.Object({
|
|
|
418
418
|
{ if: requiredObject("decision"), then: { ...requiredObject("action"), properties: { action: { enum: ["review"] } } } },
|
|
419
419
|
{ if: { ...requiredObject("action"), properties: { action: { enum: ["history"] } } }, then: { anyOf: [requiredObject("id"), requiredObject("runId")] } },
|
|
420
420
|
{ if: { ...requiredObject("action"), properties: { action: { enum: ["search"] } } }, then: requiredObject("query") },
|
|
421
|
-
{ if: requiredObject("offset"), then: { ...requiredObject("action"), properties: { action: { enum: ["status"] } } } },
|
|
422
|
-
{ if: requiredObject("limit"), then: { ...requiredObject("action"), properties: { action: { enum: ["status", "history", "search"] } } } },
|
|
421
|
+
{ if: requiredObject("offset"), then: { ...requiredObject("action"), properties: { action: { enum: ["status", "questions"] } } } },
|
|
422
|
+
{ if: requiredObject("limit"), then: { ...requiredObject("action"), properties: { action: { enum: ["status", "questions", "history", "search"] } } } },
|
|
423
423
|
{ if: requiredObject("cursor"), then: { ...requiredObject("action"), properties: { action: { enum: ["status", "history", "search"] } } } },
|
|
424
424
|
{ if: requiredObject("sort"), then: { ...requiredObject("action"), properties: { action: { enum: ["status", "search"] } } } },
|
|
425
425
|
{ if: { allOf: [{ ...requiredObject("action"), properties: { action: { enum: ["search"] } } }, requiredObject("index")] }, then: { anyOf: [requiredObject("id"), requiredObject("runId")] } },
|
package/src/history/index.ts
CHANGED
|
@@ -87,6 +87,15 @@ export class SubagentHistoryIndex {
|
|
|
87
87
|
}
|
|
88
88
|
await this.ready;
|
|
89
89
|
}
|
|
90
|
+
/** Arms an ordinary deadline only in the continuation that saw the current admission and worker settle, never behind a replacement admission. */
|
|
91
|
+
private async admitted<T>(method: string, input?: unknown, signal?: AbortSignal, timeout?: number): Promise<T> {
|
|
92
|
+
await this.ensure();
|
|
93
|
+
const ready = this.ready, child = this.process;
|
|
94
|
+
await ready;
|
|
95
|
+
// setOwner() or a restart may have replaced readiness during either await; wait for the replacement.
|
|
96
|
+
if (!child || ready !== this.ready || child !== this.process) return this.admitted<T>(method, input, signal, timeout);
|
|
97
|
+
return this.send<T>(method, input, signal, timeout);
|
|
98
|
+
}
|
|
90
99
|
async setOwner(input: HistoryOwner): Promise<void> {
|
|
91
100
|
if (this.closed) throw new HistoryIndexError("CLOSED", "History index is closed.");
|
|
92
101
|
if (!this.process) await Promise.all(this.children.values());
|
|
@@ -104,20 +113,16 @@ export class SubagentHistoryIndex {
|
|
|
104
113
|
await this.ensure();
|
|
105
114
|
if (run.ownerSessionId !== this.owner!.ownerSessionId || foreground && foreground.runId !== run.runId) throw new HistoryIndexError("OWNERSHIP", "Run does not belong to the current owner.");
|
|
106
115
|
const copy = structuredClone(run), fg = foreground && structuredClone(foreground);
|
|
107
|
-
await this.
|
|
116
|
+
await this.admitted<void>("updateRun", { run: copy, foreground: fg });
|
|
108
117
|
this.owner!.runs = [...this.owner!.runs.filter((entry) => entry.runId !== run.runId), copy];
|
|
109
118
|
if (fg) this.owner!.foregroundRuns = [...(this.owner!.foregroundRuns ?? []).filter((entry) => entry.runId !== run.runId), fg];
|
|
110
119
|
}
|
|
111
120
|
private async query<T>(method: string, input: { signal?: AbortSignal }): Promise<T> {
|
|
112
|
-
await this.ensure();
|
|
113
121
|
const { signal, ...wire } = input;
|
|
114
|
-
return this.
|
|
122
|
+
return this.admitted<T>(method, wire, signal);
|
|
115
123
|
}
|
|
116
124
|
/** Tool discovery waits for run metadata only, not transcript backfill. */
|
|
117
|
-
|
|
118
|
-
await this.ensure();
|
|
119
|
-
return this.send("needsControls", undefined, undefined, 15_000);
|
|
120
|
-
}
|
|
125
|
+
needsControls(): Promise<boolean> { return this.admitted("needsControls", undefined, undefined, 15_000); }
|
|
121
126
|
listRuns(options: HistoryRunOptions = {}): Promise<HistoryRunPage> { return this.query("listRuns", options); }
|
|
122
127
|
historyPage(input: HistoryPageInput): Promise<HistoryPage> { return this.query("historyPage", input); }
|
|
123
128
|
search(input: HistorySearchInput): Promise<HistorySearchPage> { return this.query("search", input); }
|
|
@@ -130,8 +135,7 @@ export class SubagentHistoryIndex {
|
|
|
130
135
|
result(input: Pick<HistoryEntryInput, "runId" | "index" | "signal">): Promise<HistoryResult | null> { return this.query("result", input); }
|
|
131
136
|
async refresh(runId?: string, options: { signal?: AbortSignal } = {}): Promise<void> {
|
|
132
137
|
if (!this.closed) this.unavailable = undefined;
|
|
133
|
-
|
|
134
|
-
return this.send("refresh", { runId }, options.signal, 120_000);
|
|
138
|
+
return this.admitted("refresh", { runId }, options.signal, 120_000);
|
|
135
139
|
}
|
|
136
140
|
onChanged(listener: () => void): () => void { this.listeners.add(listener); return () => this.listeners.delete(listener); }
|
|
137
141
|
/** Hard cancellation affects this instance's outstanding work, not native sessions or other indexers. */
|
package/src/history/ingest.ts
CHANGED
|
@@ -175,8 +175,13 @@ export class SourceIngest {
|
|
|
175
175
|
private fd: number;
|
|
176
176
|
private initial: fs.BigIntStats;
|
|
177
177
|
private source: SourceRow;
|
|
178
|
-
private prefix: Hash = createHash("sha256");
|
|
179
178
|
private recordDigest: Hash = createHash("sha256");
|
|
179
|
+
/** prefix_digest is "start:sha256" of the last indexed line; earlier releases hashed the whole prefix. */
|
|
180
|
+
private legacy = false;
|
|
181
|
+
private expected: string | null = null;
|
|
182
|
+
private verified: Hash = createHash("sha256");
|
|
183
|
+
private line: Hash = createHash("sha256");
|
|
184
|
+
private lineStart = 0;
|
|
180
185
|
private verifyAt = 0;
|
|
181
186
|
private position: number;
|
|
182
187
|
private frames: JournalFrames;
|
|
@@ -208,6 +213,10 @@ export class SourceIngest {
|
|
|
208
213
|
this.headerSeen = source.cursor > 0;
|
|
209
214
|
store.clearUnpublished(source.id);
|
|
210
215
|
store.run("DELETE FROM pending_text");
|
|
216
|
+
// Appends re-hash only the last indexed line, so live sources cost their new bytes, not their size.
|
|
217
|
+
const anchor = /^(\d+):([0-9a-f]{64})$/.exec(source.prefix_digest ?? "");
|
|
218
|
+
this.legacy = !anchor; this.expected = anchor ? anchor[2] : source.prefix_digest;
|
|
219
|
+
this.verifyAt = this.lineStart = anchor ? Number(anchor[1]) : 0;
|
|
211
220
|
this.phase = !force && source.stamp === stamp(this.initial) ? "done" : source.cursor ? "verify" : "read";
|
|
212
221
|
if (this.phase !== "done") store.run("UPDATE sources SET state='indexing',error=NULL WHERE id=?", source.id);
|
|
213
222
|
this.frames = new JournalFrames(previewProjection, (record) => {
|
|
@@ -224,7 +233,7 @@ export class SourceIngest {
|
|
|
224
233
|
const current = this.store.source(this.id)!;
|
|
225
234
|
if (!current || current.generation !== this.source.generation || current.cursor !== this.source.cursor) throw new HistoryIndexError("SOURCE_CHANGED", "Another writer advanced this source; reconcile before replaying.");
|
|
226
235
|
work?.();
|
|
227
|
-
this.store.run("UPDATE sources SET cursor=?,prefix_digest=?,identity=? WHERE id=?", end, this.
|
|
236
|
+
this.store.run("UPDATE sources SET cursor=?,prefix_digest=?,identity=? WHERE id=?", end, `${this.source.cursor}:${this.recordDigest.copy().digest("hex")}`, identity(this.initial), this.id);
|
|
228
237
|
this.store.bump();
|
|
229
238
|
});
|
|
230
239
|
this.source.cursor = end;
|
|
@@ -293,11 +302,19 @@ export class SourceIngest {
|
|
|
293
302
|
const count = fs.readSync(this.fd, bytes, 0, bytes.length, this.verifyAt);
|
|
294
303
|
if (count !== bytes.length) throw new Error("Journal truncated during prefix verification.");
|
|
295
304
|
this.store.operations.sourceBytesRead += count;
|
|
296
|
-
this.
|
|
305
|
+
this.verified.update(bytes);
|
|
306
|
+
if (this.legacy) {
|
|
307
|
+
// A whole-prefix check also finds the last indexed line, which anchors every later check.
|
|
308
|
+
const before = this.source.cursor - 2 - this.verifyAt, lf = before < 0 ? -1 : bytes.lastIndexOf(10, before);
|
|
309
|
+
if (lf < 0) this.line.update(bytes);
|
|
310
|
+
else { this.lineStart = this.verifyAt + lf + 1; this.line = createHash("sha256").update(bytes.subarray(lf + 1)); }
|
|
311
|
+
}
|
|
312
|
+
this.verifyAt += count;
|
|
297
313
|
if (this.verifyAt === this.source.cursor) {
|
|
298
|
-
if (this.
|
|
314
|
+
if (this.verified.digest("hex") !== this.expected) {
|
|
299
315
|
this.store.resetSource(this.id); throw new HistoryIndexError("SOURCE_CHANGED", "Journal prefix changed; a new physical generation was scheduled.");
|
|
300
316
|
}
|
|
317
|
+
if (this.legacy) this.store.run("UPDATE sources SET prefix_digest=? WHERE id=?", `${this.lineStart}:${this.line.digest("hex")}`, this.id);
|
|
301
318
|
this.phase = "read";
|
|
302
319
|
}
|
|
303
320
|
return false;
|
|
@@ -313,7 +330,7 @@ export class SourceIngest {
|
|
|
313
330
|
}
|
|
314
331
|
const newline = this.buffer.indexOf(10);
|
|
315
332
|
const bytes = newline < 0 ? this.buffer : this.buffer.subarray(0, newline + 1);
|
|
316
|
-
this.recordDigest.update(bytes); this.
|
|
333
|
+
this.recordDigest.update(bytes); this.frames.write(bytes); this.text.flush(this.store);
|
|
317
334
|
this.buffer = this.buffer.subarray(bytes.length);
|
|
318
335
|
if (newline >= 0) this.finishRecord(this.frames.offset);
|
|
319
336
|
return false;
|
package/src/history/queries.ts
CHANGED
|
@@ -11,7 +11,9 @@ interface Cursor { generation: string; owner: string; version: number; query: st
|
|
|
11
11
|
export class HistoryQueries {
|
|
12
12
|
private store: HistoryStore;
|
|
13
13
|
private info: () => HistoryVersion;
|
|
14
|
+
private configurations = new Map<string, HistoryConfiguration>();
|
|
14
15
|
constructor(store: HistoryStore, info: () => HistoryVersion) { this.store = store; this.info = info; }
|
|
16
|
+
sourcesRemoved(): void { this.configurations.clear(); }
|
|
15
17
|
private cursor(query: string, keys: (string | number)[], offset: number): string {
|
|
16
18
|
return Buffer.from(JSON.stringify({ generation: this.store.generation, owner: this.store.owner, version: this.info().version, query, keys, offset } satisfies Cursor)).toString("base64url");
|
|
17
19
|
}
|
|
@@ -80,6 +82,12 @@ export class HistoryQueries {
|
|
|
80
82
|
if (input.leaf === null) return {};
|
|
81
83
|
let boundary: ReturnType<HistoryQueries["boundary"]>;
|
|
82
84
|
try { boundary = this.boundary(source, input); } catch (error) { if (!strict && error instanceof HistoryIndexError && error.code === "BOUNDARY_UNAVAILABLE") return {}; throw error; }
|
|
85
|
+
// Only committed source changes invalidate ancestry, not unrelated run projections.
|
|
86
|
+
// Validate boundaries before reuse; explicit null leaves returned above remain empty.
|
|
87
|
+
const leafKey = input.leaf === undefined ? null : [typeof input.leaf, String(input.leaf)];
|
|
88
|
+
const key = JSON.stringify([source.id, source.generation, source.cursor, source.stamp, input.terminalEntryId || null, input.terminalEntryId ? null : input.endedAt ?? null, leafKey, strict]);
|
|
89
|
+
const cached = this.configurations.get(key);
|
|
90
|
+
if (cached) return { ...cached };
|
|
83
91
|
const leaf = input.leaf ?? this.store.get(`SELECT id FROM entries WHERE ${boundary.clauses.join(" AND ")} ORDER BY start DESC LIMIT 1`, ...boundary.params)?.id;
|
|
84
92
|
if (!leaf) return {};
|
|
85
93
|
// Fix recursive loop order: look up each child's parent by ID instead of
|
|
@@ -93,7 +101,11 @@ export class HistoryQueries {
|
|
|
93
101
|
(SELECT configuration_thinking FROM branch WHERE configuration_thinking IS NOT NULL ORDER BY start DESC LIMIT 1) AS thinking`, ...boundary.params, leaf);
|
|
94
102
|
if (!facts.count && input.leaf && strict) throw new HistoryIndexError("BOUNDARY_UNAVAILABLE", "Selected configuration leaf is not available in the indexed generation.");
|
|
95
103
|
if (facts.count && !facts.rooted) { if (!strict) return {}; throw new HistoryIndexError("MALFORMED_ANCESTRY", "Selected native configuration has cyclic ancestry."); }
|
|
96
|
-
|
|
104
|
+
const configuration = { ...(facts.model ? { model: facts.model, modelRecordedAt: facts.recorded_at ?? undefined } : {}), ...(facts.thinking ? { thinking: facts.thinking } : {}) };
|
|
105
|
+
// Bound memory across arbitrary sources, boundaries and pages. Eviction only recomputes.
|
|
106
|
+
if (this.configurations.size >= 128) this.configurations.delete(this.configurations.keys().next().value!);
|
|
107
|
+
this.configurations.set(key, configuration);
|
|
108
|
+
return { ...configuration };
|
|
97
109
|
}
|
|
98
110
|
historyPage(input: HistoryPageInput): HistoryPage {
|
|
99
111
|
const { source } = this.child(input.runId, input.index);
|
|
@@ -149,6 +161,12 @@ export class HistoryQueries {
|
|
|
149
161
|
if (input.runId) { clauses.push("c.run_id=?"); params.push(input.runId); }
|
|
150
162
|
if (input.index !== undefined) { clauses.push("c.child_index=?"); params.push(input.index); }
|
|
151
163
|
if (input.agent) { if (input.agent.length > 256) throw new HistoryIndexError("INVALID", "Agent filter is too long."); clauses.push("c.agent=?"); params.push(input.agent); }
|
|
164
|
+
const joins = "JOIN sources s ON s.id=e.source_id JOIN children c ON c.source_id=s.id LEFT JOIN entries terminal ON terminal.source_id=s.id AND terminal.generation=s.generation AND terminal.id=c.terminal_entry_id AND terminal.published=1";
|
|
165
|
+
// Scope records before materializing/grouping hits, while bm25 still sees the
|
|
166
|
+
// complete corpus. Scan each linked source once, not once per continuation.
|
|
167
|
+
// Attempt boundaries, attribution and seek filtering remain in the final join.
|
|
168
|
+
const eligible = input.runId || input.agent ? `eligible AS MATERIALIZED (SELECT e.rowid FROM entries e JOIN sources s ON s.id=e.source_id WHERE e.published=1 AND e.generation=s.generation AND e.source_id IN (SELECT c.source_id FROM children c WHERE ${clauses.slice(4).join(" AND ")})),` : "";
|
|
169
|
+
const eligibleParams = eligible ? params.slice(expression.length) : [];
|
|
152
170
|
// Ordering/filters apply globally before limit. Common terms may still require an FTS
|
|
153
171
|
// scan/sort; the parent enforces a hard deadline by killing this separate process.
|
|
154
172
|
const score = sort === "relevance" ? "matched.score" : "-COALESCE(e.timestamp,0)";
|
|
@@ -158,8 +176,8 @@ export class HistoryQueries {
|
|
|
158
176
|
}
|
|
159
177
|
// Match words across all visible fields/windows of one native record. A quoted
|
|
160
178
|
// phrase remains a single FTS expression, so it cannot span unrelated fields.
|
|
161
|
-
const rows = this.store.all(`WITH hits AS MATERIALIZED (
|
|
162
|
-
${expression.map((_term, index) => `SELECT d.id,d.entry_rowid,bm25(corpus) AS score,${index} AS term FROM corpus JOIN documents d ON d.id=corpus.rowid WHERE corpus MATCH
|
|
179
|
+
const rows = this.store.all(`WITH ${eligible} hits AS MATERIALIZED (
|
|
180
|
+
${expression.map((_term, index) => `SELECT d.id,d.entry_rowid,bm25(corpus) AS score,${index} AS term FROM corpus JOIN documents d ON d.id=corpus.rowid WHERE corpus MATCH ?${eligible ? " AND d.entry_rowid IN (SELECT rowid FROM eligible)" : ""}`).join(" UNION ALL ")}
|
|
163
181
|
), terms AS (
|
|
164
182
|
SELECT entry_rowid,term,MIN(score) AS score FROM hits GROUP BY entry_rowid,term
|
|
165
183
|
), matched AS (
|
|
@@ -167,7 +185,7 @@ export class HistoryQueries {
|
|
|
167
185
|
), excerpts AS (
|
|
168
186
|
SELECT id,entry_rowid,ROW_NUMBER() OVER (PARTITION BY entry_rowid ORDER BY score,id) AS choice FROM hits
|
|
169
187
|
)
|
|
170
|
-
SELECT d.*,${score} AS score,e.*,d.id AS document_id,s.path,s.session_id,c.run_id,c.child_index,c.agent,d.preview AS document_preview FROM matched JOIN excerpts ON excerpts.entry_rowid=matched.entry_rowid AND excerpts.choice=1 JOIN documents d ON d.id=excerpts.id JOIN entries e ON e.rowid=matched.entry_rowid
|
|
188
|
+
SELECT d.*,${score} AS score,e.*,d.id AS document_id,s.path,s.session_id,c.run_id,c.child_index,c.agent,d.preview AS document_preview FROM matched JOIN excerpts ON excerpts.entry_rowid=matched.entry_rowid AND excerpts.choice=1 JOIN documents d ON d.id=excerpts.id JOIN entries e ON e.rowid=matched.entry_rowid ${joins} WHERE ${clauses.join(" AND ")} ORDER BY score,d.id,c.run_id,c.child_index LIMIT ?`, ...eligibleParams, ...params, limit + 1);
|
|
171
189
|
const more = rows.length > limit, page = rows.slice(0, limit);
|
|
172
190
|
return { ...this.info(), matches: page.map((row) => ({ id: row.document_id, runId: row.run_id, index: row.child_index, agent: row.agent, entryId: row.id, nativeId: row.native_id, sessionId: row.session_id, sessionFile: row.path, timestamp: row.timestamp,
|
|
173
191
|
ref: { sourceId: row.source_id, generation: row.generation, start: row.start, end: row.end, digest: row.digest }, preview: row.document_preview, field: row.field, textStart: row.text_start, textEnd: row.text_end, score: row.score })),
|
package/src/history/worker.ts
CHANGED
|
@@ -2,7 +2,7 @@ import * as fs from "node:fs";
|
|
|
2
2
|
import * as path from "node:path";
|
|
3
3
|
import { ownedRunView } from "../runs/shared/run-records.ts";
|
|
4
4
|
import { acceptanceHumanAction } from "../runs/shared/acceptance-evaluation.ts";
|
|
5
|
-
import { getRunMetadataDir, listRunQuestions } from "../runs/shared/supervisor-questions.ts";
|
|
5
|
+
import { getRunMetadataDir, listRunQuestions, summarizeSupervisorQuestion } from "../runs/shared/supervisor-questions.ts";
|
|
6
6
|
import { HistoryIndexError } from "./types.ts";
|
|
7
7
|
import type { ForegroundResumeRun, HistoryOwner, HistoryRunRow, OwnedRun, Request, Response } from "./types.ts";
|
|
8
8
|
import type { SubagentState } from "../shared/types.ts";
|
|
@@ -21,6 +21,8 @@ const sourceWatchers = new Map<string, fs.FSWatcher>();
|
|
|
21
21
|
const watchIdentities = new WeakMap<fs.FSWatcher, string>();
|
|
22
22
|
const watchedSources = new Map<string, Map<string, string>>();
|
|
23
23
|
let job: SourceIngest | undefined, scheduled = false, closed = false;
|
|
24
|
+
let sourceObservation: { state: string; error: string | null } | undefined;
|
|
25
|
+
let notifiedVersion: string | undefined;
|
|
24
26
|
let retryTimer: ReturnType<typeof setTimeout> | undefined;
|
|
25
27
|
let runErrors = new Set<string>();
|
|
26
28
|
const barriers: Array<{ id: number; runId?: string }> = [];
|
|
@@ -28,8 +30,18 @@ const controlQueries: number[] = [];
|
|
|
28
30
|
function replyControls(id: number): void {
|
|
29
31
|
send({ id, value: Boolean(store!.get("SELECT id FROM runs WHERE attention<4 OR state='live' LIMIT 1")) });
|
|
30
32
|
}
|
|
31
|
-
function send(response: Response): void {
|
|
32
|
-
|
|
33
|
+
function send(response: Response): void {
|
|
34
|
+
// The parent can disconnect after process.connected was read; then this and later replies are undeliverable, so stop as on disconnect.
|
|
35
|
+
if (process.connected) process.send!(response, undefined, undefined, (error) => { if (error) shutdown(); });
|
|
36
|
+
}
|
|
37
|
+
function changed(observationChanged = false): void {
|
|
38
|
+
const version = store?.get("SELECT value FROM meta WHERE key='version'").value;
|
|
39
|
+
// Unchanged census projections need only one final freshness notification.
|
|
40
|
+
// Non-versioned errors/recovery and real commits still publish immediately.
|
|
41
|
+
if (observationChanged || version !== notifiedVersion || !hasPending()) {
|
|
42
|
+
notifiedVersion = version; send({ changed: true });
|
|
43
|
+
}
|
|
44
|
+
}
|
|
33
45
|
function errorReply(id: number, error: unknown): void {
|
|
34
46
|
const code = error instanceof HistoryIndexError ? error.code : "UNAVAILABLE";
|
|
35
47
|
// Parser errors can include fragments from the malformed body. IPC/logs must never carry them.
|
|
@@ -43,10 +55,10 @@ function compactView(view: HistoryRunRow): HistoryRunRow {
|
|
|
43
55
|
return {
|
|
44
56
|
...view, task: safeText(view.task, 2048), error: view.error && safeText(view.error), diagnosis: view.diagnosis && safeText(view.diagnosis), recoveryError: view.recoveryError && safeText(view.recoveryError),
|
|
45
57
|
review: view.review && { ...view.review, message: view.review.message && safeText(view.review.message) }, continuations: [],
|
|
46
|
-
questions: view.questions?.map((question) =>
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
58
|
+
questions: view.questions?.map((question) => {
|
|
59
|
+
const summary = summarizeSupervisorQuestion(question);
|
|
60
|
+
return { ...summary, message: safeText(question.message, 2048), answer: summary.answer && { ...summary.answer, message: safeText(question.answer?.message, 2048) } };
|
|
61
|
+
}),
|
|
50
62
|
children: view.children.map((child) => ({
|
|
51
63
|
agent: child.agent, index: child.index, workflowNodeId: child.workflowNodeId, sessionFile: child.sessionFile, task: child.task && safeText(child.task), label: child.label && safeText(child.label, 256),
|
|
52
64
|
state: child.state, configuration: child.configuration, missingSession: child.missingSession, identityUnavailable: child.identityUnavailable, modelSelection: child.modelSelection,
|
|
@@ -161,7 +173,11 @@ function prune(previousSources?: string[]): void {
|
|
|
161
173
|
store!.deleteDocuments(source.id);
|
|
162
174
|
store!.run("DELETE FROM entries WHERE source_id=?", source.id); store!.run("DELETE FROM sources WHERE id=?", source.id); removed = true;
|
|
163
175
|
}
|
|
164
|
-
if (removed)
|
|
176
|
+
if (removed) {
|
|
177
|
+
store!.bump();
|
|
178
|
+
// A later admission of the same path can recreate source generation 1.
|
|
179
|
+
queries?.sourcesRemoved();
|
|
180
|
+
}
|
|
165
181
|
});
|
|
166
182
|
}
|
|
167
183
|
function admit(run: OwnedRun): void {
|
|
@@ -198,23 +214,25 @@ function pump(): void {
|
|
|
198
214
|
try {
|
|
199
215
|
if (dirtyRuns.size) {
|
|
200
216
|
const id = dirtyRuns.values().next().value!; dirtyRuns.delete(id);
|
|
217
|
+
const failed = runErrors.has(id);
|
|
201
218
|
installWatches(id);
|
|
202
219
|
const previousSources = store.all("SELECT source_id FROM children WHERE run_id=? AND source_id IS NOT NULL", id).map((child) => child.source_id);
|
|
203
220
|
try { store.operations.runProjections++; const view = projection(runs.get(id)!); store.putView(compactView(view), view); runErrors.delete(id); }
|
|
204
221
|
catch { const view = unknown(runs.get(id)!, "Canonical owner summary is unavailable. Completion remains unconfirmed."); store.putView(compactView(view), view); runErrors.add(id); }
|
|
205
222
|
prune(previousSources);
|
|
206
223
|
for (const child of store.all("SELECT source_id FROM children WHERE run_id=? AND source_id IS NOT NULL", id)) if (child.source_id !== job?.id) dirtySources.set(child.source_id, dirtySources.get(child.source_id) ?? false);
|
|
207
|
-
installWatches(id); changed();
|
|
224
|
+
installWatches(id); changed(failed !== runErrors.has(id));
|
|
208
225
|
} else if (job) {
|
|
209
226
|
if (job.step()) {
|
|
210
227
|
const source = store.source(job.id);
|
|
211
228
|
if (source) watchSources(path.dirname(source.path));
|
|
212
|
-
job = undefined; changed();
|
|
229
|
+
job = undefined; changed(source?.state !== sourceObservation?.state || source?.error !== sourceObservation?.error);
|
|
230
|
+
sourceObservation = undefined;
|
|
213
231
|
}
|
|
214
232
|
} else if (dirtySources.size) {
|
|
215
233
|
const [id, force] = dirtySources.entries().next().value!; dirtySources.delete(id);
|
|
216
234
|
const source = store.source(id);
|
|
217
|
-
if (source) { activeSource = id; job = new SourceIngest(store, source, force); }
|
|
235
|
+
if (source) { activeSource = id; sourceObservation = source; job = new SourceIngest(store, source, force); }
|
|
218
236
|
}
|
|
219
237
|
} catch (error) {
|
|
220
238
|
const id = job?.id ?? activeSource;
|
|
@@ -227,7 +245,11 @@ function pump(): void {
|
|
|
227
245
|
else if ((error as NodeJS.ErrnoException).code === "ENOENT") store.resetSource(id, "missing", "Linked native conversation is missing.");
|
|
228
246
|
else store.run("UPDATE sources SET state='error',error=?,checked_at=? WHERE id=?", "Linked native conversation could not be indexed.", Date.now(), id);
|
|
229
247
|
}
|
|
230
|
-
if (!(error instanceof HistoryIndexError && error.code === "INDEX_BUSY"))
|
|
248
|
+
if (!(error instanceof HistoryIndexError && error.code === "INDEX_BUSY")) {
|
|
249
|
+
const source = id ? store.source(id) : undefined;
|
|
250
|
+
changed(Boolean(source && (source.state !== sourceObservation?.state || source.error !== sourceObservation?.error)));
|
|
251
|
+
}
|
|
252
|
+
sourceObservation = undefined;
|
|
231
253
|
}
|
|
232
254
|
activeSource = undefined;
|
|
233
255
|
finishBarriers();
|
|
@@ -244,6 +266,7 @@ const timer = setInterval(census, 30_000); timer.unref();
|
|
|
244
266
|
function clear(): void {
|
|
245
267
|
clearTimeout(retryTimer); retryTimer = undefined;
|
|
246
268
|
job?.close(); job = undefined;
|
|
269
|
+
sourceObservation = undefined; notifiedVersion = undefined;
|
|
247
270
|
for (const watcher of watchers.values()) watcher.close(); watchers.clear(); watchedSources.clear();
|
|
248
271
|
for (const watcher of sourceWatchers.values()) watcher.close(); sourceWatchers.clear();
|
|
249
272
|
dirtyRuns.clear(); dirtySources.clear(); runs.clear(); foreground.clear(); runErrors = new Set();
|
|
@@ -314,4 +337,5 @@ async function handle(request: Request): Promise<void> {
|
|
|
314
337
|
} catch (error) { errorReply(request.id, error); }
|
|
315
338
|
}
|
|
316
339
|
process.on("message", (request: Request) => { void handle(request); });
|
|
317
|
-
|
|
340
|
+
function shutdown(): void { closed = true; clearInterval(timer); clear(); process.exit(0); }
|
|
341
|
+
process.on("disconnect", shutdown);
|