@fitchmultz/pi-subagents 0.46.1 → 0.47.0

package/scripts/intercom-bridge-credentials.mjs

@@ -22,13 +22,16 @@ function errorMessage(error) {
 }
 
 function help() {
-  console.log(`Usage: node scripts/intercom-bridge-credentials.mjs <init|issue|revoke> DIR [NAME] [--cwd PATH]
+  console.log(`Usage: node scripts/intercom-bridge-credentials.mjs <init|issue|revoke|renew-server> DIR [NAME] [--cwd PATH] [--port N] [--days N]
 
 init    Create a new private CA, loopback server certificate and grok-bot credential.
-issue   Issue a fresh 7-day client certificate; preserve local launch policy.
+issue   Issue a fresh client certificate; preserve local identity policies.
 revoke  Remove the named client from the allowlist. Does not delete saved credentials.
+renew-server  Renew the loopback server key/certificate using the same CA; preserve clients.
 
 Options: --cwd PATH  Local project identity for init/issue (default: current directory)
+         --port N    Loopback bridge port for init (default: 9443)
+         --days N    Certificate validity (1–3650; client default: 7, renew-server: 365)
          -h, --help  Show help
 Environment: OPENSSL  OpenSSL executable (default: openssl; requires -addext support)
 
@@ -38,6 +41,7 @@ Examples:
   node scripts/intercom-bridge-credentials.mjs revoke ~/.pi/intercom-bridge grok-bot
 
 After issue/revoke, send SIGHUP to the helper PID to activate the allowlist.
+After renew-server, restart only the helper; existing client credentials remain valid.
 Rotation/revocation stops that identity's helper-launched Pi children; journals remain.
 Concurrent credential commands fail rather than overwrite each other's authorization.
 After a crashed command, inspect DIR/.credentials.lock/owner and remove that lock
@@ -80,7 +84,7 @@ function key(path) {
   privatePath(path);
 }
 
-function issueCertificate(dir, stem, name, server = false) {
+function issueCertificate(dir, stem, name, { server = false, days = 7 } = {}) {
   const keyPath = join(dir, `${stem}.key`);
   const certPath = join(dir, `${stem}.crt`);
   const csrPath = join(dir, `${stem}.csr`);
@@ -105,7 +109,7 @@ function issueCertificate(dir, stem, name, server = false) {
       "-set_serial",
       `0x${randomBytes(16).toString("hex")}`,
       "-days",
-      server ? "365" : "7",
+      String(days),
       "-sha256",
       "-extfile",
       extPath,
@@ -145,23 +149,41 @@ function readConfig(path) {
 }
 
 let args;
+let port;
+let certificateDays;
 try {
   args = parseArgs({
     allowPositionals: true,
-    options: { help: { type: "boolean", short: "h" }, cwd: { type: "string" } },
+    options: {
+      help: { type: "boolean", short: "h" },
+      cwd: { type: "string" },
+      port: { type: "string" },
+      days: { type: "string" },
+    },
   });
   if (args.values.help) {
     help();
     process.exit(0);
   }
   const [action, dir, name] = args.positionals;
+  port = args.values.port === undefined ? 9443 : Number(args.values.port);
+  const defaultDays = action === "renew-server" ? 365 : 7;
+  certificateDays = args.values.days === undefined ? defaultDays : Number(args.values.days);
   if (
-    !["init", "issue", "revoke"].includes(action) ||
+    !["init", "issue", "revoke", "renew-server"].includes(action) ||
     !dir ||
     args.positionals.length > 3 ||
-    (action !== "init" && !name) ||
+    (["issue", "revoke"].includes(action) && !name) ||
+    (action === "renew-server" && (name || args.values.cwd)) ||
     (name && !/^[a-zA-Z0-9][a-zA-Z0-9_-]{0,63}$/.test(name)) ||
-    (action === "revoke" && args.values.cwd)
+    (action === "revoke" && (args.values.cwd || args.values.days)) ||
+    (action !== "init" && args.values.port !== undefined) ||
+    !Number.isInteger(port) ||
+    port < 0 ||
+    port > 65535 ||
+    !Number.isInteger(certificateDays) ||
+    certificateDays < 1 ||
+    certificateDays > 3650
   ) {
     throw new Error("Invalid arguments; use --help.");
   }
@@ -217,9 +239,9 @@ try {
       "-out",
       join(dir, "ca.crt"),
     );
-    const server = issueCertificate(dir, "server", "localhost", true);
+    const server = issueCertificate(dir, "server", "localhost", { server: true, days: 365 });
     config = {
-      port: 9443,
+      port,
       ca: join(dir, "ca.crt"),
       cert: server.cert,
       key: server.key,
@@ -239,16 +261,36 @@ try {
     console.log(
       JSON.stringify({ revoked: name, config: configPath, next: "Send SIGHUP to the helper PID." }),
     );
+  } else if (action === "renew-server") {
+    privatePath(join(dir, "ca.key"));
+    const server = issueCertificate(dir, `server-${randomBytes(8).toString("hex")}`, "localhost", {
+      server: true,
+      days: certificateDays,
+    });
+    config.cert = server.cert;
+    config.key = server.key;
+    atomicConfig(configPath, config);
+    console.log(
+      JSON.stringify({
+        renewed: "server",
+        ...server,
+        ca: config.ca,
+        config: configPath,
+        next: "Restart only this helper; client keys and CA are unchanged.",
+      }),
+    );
   } else {
     privatePath(join(dir, "ca.key"));
-    const credential = issueCertificate(dir, `${name}-${randomBytes(8).toString("hex")}`, name);
+    const credential = issueCertificate(dir, `${name}-${randomBytes(8).toString("hex")}`, name, {
+      days: certificateDays,
+    });
     const previous = config.clients.find((client) => client.name === name);
     config.clients = config.clients.filter((client) => client.name !== name);
     config.clients.push({
+      ...previous,
       fingerprint256: credential.fingerprint256,
       name,
       cwd: resolve(args.values.cwd || previous?.cwd || process.cwd()),
-      ...(previous?.launch === undefined ? {} : { launch: previous.launch }),
     });
     atomicConfig(configPath, config);
     console.log(

package/scripts/intercom-remote.mjs

@@ -17,6 +17,8 @@ Options:
   --message TEXT       Message for send/ask/reply
   --reply-to ID        Inbound ask ID from inbox, for reply
   --timeout-ms MS      Ask timeout (100..120000, default: 120000)
+  --after CURSOR       Inbox cursor from nextCursor (default: 0)
+  --limit COUNT        Inbox page size (1..256, default: 256)
   --name NAME          Start/reuse a named idle Pi session under local policy
   --session-id ID      Stop only a Pi session launched by this identity/helper
   -h, --help           Show help
@@ -25,6 +27,7 @@ Examples:
   node intercom-remote.mjs list
   node intercom-remote.mjs ask --to planner --message 'What is the current status?'
   node intercom-remote.mjs inbox
+  node intercom-remote.mjs inbox --after NEXT_CURSOR --limit 64
   node intercom-remote.mjs reply --reply-to QUESTION_ID --message 'Proceed.'
   node intercom-remote.mjs ack MESSAGE_ID
   node intercom-remote.mjs start --name planner
@@ -36,6 +39,9 @@ No automatic retries: a lost response does not prove a message was unsent.`);
 }
 
 function allowedOptions(action) {
+  if (action === "inbox") {
+    return ["after", "limit"];
+  }
   if (action === "start") {
     return ["name"];
   }
@@ -119,7 +125,21 @@ function privateKey(path) {
   }
 }
 
-let body, action, origin, timeout;
+function inboxQuery(options) {
+  const after = options.after ?? "0";
+  const limit = options.limit ?? "256";
+  if (
+    !/^(0|[1-9]\d{0,15})$/.test(after) ||
+    !Number.isSafeInteger(Number(after)) ||
+    !/^[1-9]\d{0,2}$/.test(limit) ||
+    Number(limit) > 256
+  ) {
+    throw new Error("--after must be a nonnegative safe integer cursor; --limit must be 1..256.");
+  }
+  return new URLSearchParams({ after, limit }).toString();
+}
+
+let body, action, origin, timeout, query;
 try {
   const parsed = parseArgs({
     allowPositionals: true,
@@ -129,6 +149,8 @@ try {
       message: { type: "string" },
       "reply-to": { type: "string" },
       "timeout-ms": { type: "string" },
+      after: { type: "string" },
+      limit: { type: "string" },
       name: { type: "string" },
       "session-id": { type: "string" },
     },
@@ -167,6 +189,7 @@ try {
     throw new Error("--timeout-ms must be 100..120000.");
   }
   body = requestBody(action, options, parsed.positionals.slice(1), timeout);
+  query = action === "inbox" ? inboxQuery(options) : "";
   origin = new URL(process.env.INTERCOM_BRIDGE_URL || "https://127.0.0.1:9443");
   if (
     origin.protocol !== "https:" ||
@@ -194,6 +217,7 @@ try {
 try {
   const files = tlsFiles();
   origin.pathname = `/v1/${action}`;
+  origin.search = query;
   const payload = JSON.stringify(body);
   const get = ["list", "inbox", "sessions"].includes(action);
   const result = await new Promise((resolve, reject) => {

package/scripts/package-smoke.mjs

@@ -200,7 +200,7 @@ function installProduction(packed) {
   );
   if (
     !credentialHelp.includes(
-      "Usage: node scripts/intercom-bridge-credentials.mjs <init|issue|revoke>",
+      "Usage: node scripts/intercom-bridge-credentials.mjs <init|issue|revoke|renew-server>",
     )
   ) {
     throw new Error("installed credential CLI did not provide usage help");

package/src/agents/agent-management.ts

@@ -1,4 +1,5 @@
-import { discoverAgents, discoverAgentsAll } from "./agents.ts";
+import { discoverAgentsAll } from "./agents.ts";
+import { mergeAgents } from "./agent-selection.ts";
 import { formatAgentGroups, formatAgentDetail, formatChainDetail } from "./management-format.ts";
 import { handleCreate, handleUpdate, handleDelete } from "./management-persistence.ts";
 import {
@@ -24,9 +25,12 @@ export function handleList(
     return result("agentScope must be 'user', 'project', or 'both'.", true);
   }
   const discovery = discoverAgentsAll(ctx.cwd, discoveryOptions(ctx), scope);
-  const agents = discoverAgents(ctx.cwd, scope, discoveryOptions(ctx)).agents.sort((a, b) =>
-    a.name.localeCompare(b.name),
-  );
+  const agents = mergeAgents(discovery.user, discovery.project, [
+    ...discovery.builtin,
+    ...discovery.package,
+  ])
+    .filter((agent) => agent.disabled !== true)
+    .sort((a, b) => a.name.localeCompare(b.name));
   const chains = discovery.chains
     .filter((chain) => scope === "both" || chain.source === scope)
     .sort((a, b) => a.name.localeCompare(b.name));

package/src/agents/agents.ts

@@ -124,17 +124,26 @@ function loadChainsFromDir(
   }
   return { chains, diagnostics };
 }
-function discoverChains(
-  userDir: string,
-  projectDirs: readonly string[],
-  scope: AgentScope,
-): { chains: ChainConfig[]; chainDiagnostics: ChainDiscoveryDiagnostic[] } {
-  const projectChains = (scope === "user" ? [] : projectDirs).map((dir) =>
+export function discoverChains(
+  cwd: string,
+  options: AgentDiscoveryOptions = {},
+  scope: AgentScope = "both",
+): {
+  chains: ChainConfig[];
+  chainDiagnostics: ChainDiscoveryDiagnostic[];
+  userChainDir: string;
+  projectChainDir: string | null;
+} {
+  const userChainDir = path.join(getAgentDir(), "chains");
+  const project = resolveNearestProjectChainDirs(cwd, options);
+  const projectChains = (scope === "user" ? [] : project.readDirs).map((dir) =>
     loadChainsFromDir(dir, "project"),
   );
   const userChains =
-    scope === "project" ? { chains: [], diagnostics: [] } : loadChainsFromDir(userDir, "user");
+    scope === "project" ? { chains: [], diagnostics: [] } : loadChainsFromDir(userChainDir, "user");
   return {
+    userChainDir,
+    projectChainDir: project.preferredDir,
     chains: [...userChains.chains, ...projectChains.flatMap((entry) => entry.chains)],
     chainDiagnostics: [
       ...userChains.diagnostics,
@@ -186,10 +195,8 @@ export function discoverAgentsAll(
   projectSettingsPath: string | null;
 } {
   const paths = profilePaths(cwd, options);
-  const userChainDir = path.join(getAgentDir(), "chains");
-  const projectChainPaths = resolveNearestProjectChainDirs(cwd, options);
   const profiles = discoverLocalProfiles(paths, scope);
-  const chains = discoverChains(userChainDir, projectChainPaths.readDirs, scope);
+  const chains = discoverChains(cwd, options, scope);
   const packages = loadConfiguredPackageAgents(cwd, scope, options);
   return {
     ...profiles,
@@ -201,8 +208,6 @@ export function discoverAgentsAll(
     ],
     userDir: paths.userDirs.at(0) ?? path.join(getAgentDir(), "agents"),
     projectDir: paths.projectDir,
-    userChainDir,
-    projectChainDir: projectChainPaths.preferredDir,
     userSettingsPath: paths.userSettingsPath,
     projectSettingsPath: paths.projectSettingsPath,
   };

package/src/extension/index.ts

@@ -1,18 +1,25 @@
 import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
-import { SUBAGENT_CHILD_ENV } from "../runs/shared/pi-args.ts";
-import { ensureTempRoot } from "../shared/temp-root.ts";
+// Keep deferred imports on Pi's host-peer mapping path in production-only installs.
+// oxlint-disable-next-line import/no-unassigned-import
+import "@earendil-works/pi-coding-agent";
+import { SUBAGENT_CHILD_ENV } from "../runs/shared/child-route-env.ts";
 import { errorMessage } from "../shared/unknown.ts";
 import { loadConfig } from "./config.ts";
-import { ParentSubagentRuntime } from "./parent-runtime.ts";
-import { cleanupStaleRuntime } from "./runtime-reload.ts";
 
 export { loadConfig } from "./config.ts";
 
-export default function registerSubagentExtension(pi: ExtensionAPI): void {
+export default async function registerSubagentExtension(pi: ExtensionAPI): Promise<void> {
   if (process.env[SUBAGENT_CHILD_ENV] === "1") {
     // Explicit fanout-child owns nested delegation; globally discovered parent code remains inert.
     return;
   }
+  // Children discover this entry too; keep the parent execution/history/UI graph behind the guard.
+  const [{ ensureTempRoot }, { ParentSubagentRuntime }, { cleanupStaleRuntime }] =
+    await Promise.all([
+      import("../shared/temp-root.ts"),
+      import("./parent-runtime.ts"),
+      import("./runtime-reload.ts"),
+    ]);
   try {
     ensureTempRoot();
   } catch (error) {

package/src/extension/result-snapshot.ts

@@ -1,32 +1,6 @@
-import type {
-  AgentProgress,
-  Details,
-  SingleResult,
-  SubagentExecutionResult,
-} from "../shared/types.ts";
+import type { SubagentExecutionResult } from "../shared/types.ts";
 import type { ReadonlyInput } from "../shared/types/inputs.ts";
 
-/** A new result owns its progress buffers; native messages and accounting stay shared snapshots. */
-export function copyProgress(progress: ReadonlyInput<AgentProgress>): AgentProgress {
-  return {
-    ...progress,
-    recentTools: progress.recentTools.map((tool) => ({ ...tool })),
-    recentOutput: [...progress.recentOutput],
-  };
-}
-
-export function copyResult(result: ReadonlyInput<SingleResult>): SingleResult {
-  return { ...result, progress: result.progress ? copyProgress(result.progress) : undefined };
-}
-
-export function copyDetails(details: ReadonlyInput<Details>): Details {
-  return {
-    ...details,
-    results: details.results.map(copyResult),
-    progress: details.progress?.map(copyProgress),
-  };
-}
-
 /** Preserve the native result envelope while giving the receiving tool its own content array. */
 export function copyExecutionResult(
   result: ReadonlyInput<SubagentExecutionResult>,

package/src/history/ingest.ts

@@ -9,7 +9,7 @@ import { AnswerTexts } from "./answer-texts.ts";
 import { object } from "./values.ts";
 import { identity, stamp, openSource, requireSource } from "./source-file.ts";
 export { identity, stamp } from "./source-file.ts";
-import { number, text } from "./rows.ts";
+import { number, text, type Row } from "./rows.ts";
 import { stageEntry, publishTools, type IngestRecord } from "./entry-ingestion.ts";
 export { previewProjection, previewLimits, compactEntry } from "./preview.ts";
 
@@ -173,7 +173,6 @@ export class SourceIngest {
       });
       return;
     }
-    this.stagedEntry = stageEntry(this.store, this.source, record);
     this.textCursor = 0;
     this.phase = "text";
   }
@@ -235,64 +234,73 @@ export class SourceIngest {
   }
   private indexText(): void {
     const record = this.record;
-    const rowid = this.stagedEntry;
-    if (!record || rowid === undefined) {
-      throw new HistoryIndexError("CORRUPT", "Missing staged entry.");
+    if (!record) {
+      throw new HistoryIndexError("CORRUPT", "Missing ingestion record.");
     }
     const chunks = this.store.all(
-      "SELECT rowid,* FROM pending_text WHERE rowid>? ORDER BY rowid LIMIT 32",
+      "SELECT rowid,* FROM pending_text WHERE rowid>? ORDER BY rowid LIMIT 33",
       this.textCursor,
     );
-    if (chunks.length > 0) {
-      this.store.transaction(() => {
-        for (const chunk of chunks) {
-          if (!allowed(record.value, text(chunk, "field"))) {
-            continue;
-          }
-          const body = safeText(text(chunk, "text"), 1_048_576);
-          const id = this.store.run(
-            "INSERT INTO documents(entry_rowid,field,text_start,text_end,preview) VALUES (?,?,?,?,?)",
-            rowid,
-            text(chunk, "field"),
-            number(chunk, "position"),
-            number(chunk, "end_position"),
-            safeText(body, 512),
-          ).lastInsertRowid;
-          this.store.run("INSERT INTO corpus(rowid,text) VALUES (?,?)", id, body);
-        }
-      });
-      const last = chunks.at(-1);
-      if (last) {
-        this.textCursor = number(last, "rowid");
+    const complete = chunks.length <= 32;
+    const batch = chunks.slice(0, 32);
+    let rowid: number | undefined;
+    const write = () => {
+      rowid = this.stagedEntry ?? stageEntry(this.store, this.source, record);
+      this.indexChunks(rowid, record, batch);
+      if (complete) {
+        this.publishEntry(rowid, record);
       }
-      return;
+    };
+    if (complete) {
+      this.published(record.end, write);
+      this.stagedEntry = undefined;
+      this.phase = "read";
+    } else {
+      this.store.transaction(write);
+      this.stagedEntry = rowid;
+      this.textCursor = number(batch[31], "rowid");
     }
+  }
+  private indexChunks(rowid: number, record: IngestRecord, chunks: readonly Row[]): void {
+    for (const chunk of chunks) {
+      if (!allowed(record.value, text(chunk, "field"))) {
+        continue;
+      }
+      const body = safeText(text(chunk, "text"), 1_048_576);
+      const id = this.store.run(
+        "INSERT INTO documents(entry_rowid,field,text_start,text_end,preview) VALUES (?,?,?,?,?)",
+        rowid,
+        text(chunk, "field"),
+        number(chunk, "position"),
+        number(chunk, "end_position"),
+        safeText(body, 512),
+      ).lastInsertRowid;
+      this.store.run("INSERT INTO corpus(rowid,text) VALUES (?,?)", id, body);
+    }
+  }
+  private publishEntry(rowid: number, record: IngestRecord): void {
     const value = record.value;
     const answers = this.answerTexts.fingerprints(
       value,
       typeof value.id === "string" ? value.id : `legacy-${record.start}`,
     );
-    this.published(record.end, () => {
-      if (
-        this.store.run(
-          "UPDATE entries SET published=1 WHERE rowid=? AND source_id=? AND generation=? AND published=0",
-          rowid,
-          this.id,
-          this.source.generation,
-        ).changes !== 1
-      ) {
-        throw new HistoryIndexError(
-          "SOURCE_CHANGED",
-          "Staged entry disappeared before publication; reconcile before advancing the source.",
-        );
-      }
-      for (const [digest, item] of answers) {
-        this.store.run("INSERT OR IGNORE INTO answers VALUES (?,?,?)", rowid, digest, item);
-      }
-      publishTools(this.store, this.source, rowid, value);
-    });
-    this.stagedEntry = undefined;
-    this.phase = "read";
+    if (
+      this.store.run(
+        "UPDATE entries SET published=1 WHERE rowid=? AND source_id=? AND generation=? AND published=0",
+        rowid,
+        this.id,
+        this.source.generation,
+      ).changes !== 1
+    ) {
+      throw new HistoryIndexError(
+        "SOURCE_CHANGED",
+        "Staged entry disappeared before publication; reconcile before advancing the source.",
+      );
+    }
+    for (const [digest, item] of answers) {
+      this.store.run("INSERT OR IGNORE INTO answers VALUES (?,?,?)", rowid, digest, item);
+    }
+    publishTools(this.store, this.source, rowid, value);
   }
   /** Each pump reads <=64 KiB or indexes <=32 chunks, then returns to the IPC event loop. */
   step(): boolean {

package/src/history/prune.ts

@@ -30,7 +30,7 @@ export function pruneHistory(
   return store.transaction(() => {
     let removed = previousSources === undefined && pruneRuns(store, runs);
     for (const source of store.all(
-      `SELECT id,path FROM sources WHERE id NOT IN (SELECT source_id FROM children WHERE source_id IS NOT NULL)${previousSources ? ` AND id IN (${previousSources.map(() => "?").join(",")})` : ""}`,
+      `SELECT id,path FROM sources WHERE NOT EXISTS (SELECT 1 FROM children WHERE children.source_id=sources.id)${previousSources ? ` AND id IN (${previousSources.map(() => "?").join(",")})` : ""}`,
       ...(previousSources ?? []),
     )) {
       const id = text(source, "id");

package/src/history/run-projection.ts

@@ -147,10 +147,8 @@ export function projectRun(
   foreground: Readonly<ReadonlyMap<string, ReadonlyForegroundResumeRun>>,
 ): HistoryRunRow {
   const state: OwnedRunReadState = { ownedRuns: runs, foregroundRuns: foreground };
-  const questions = listRunQuestions(getRunMetadataDir(run.runId)).filter(
-    (question) =>
-      question.ownerSessionId === ownerSessionId &&
-      ["awaiting_input", "answer_pending"].includes(question.state),
+  const questions = listRunQuestions(getRunMetadataDir(run.runId), ownerSessionId).filter(
+    (question) => ["awaiting_input", "answer_pending"].includes(question.state),
   );
   return {
     ...ownedRunView(run, state, {

package/src/history/search-queries.ts

@@ -93,7 +93,8 @@ function plan(
     clauses.push("c.agent=?");
     params.push(input.agent);
   }
-  // Scope physical records once before aggregation; bm25 still uses the whole corpus.
+  // Materialized IDs reject common out-of-scope postings without per-hit entry lookups.
+  // Each source is scoped once; bm25 still uses the whole corpus.
   const eligible =
     hasText(input.runId) || hasText(input.agent)
       ? `eligible AS MATERIALIZED (SELECT e.rowid FROM entries e JOIN sources s ON s.id=e.source_id WHERE e.published=1 AND e.generation=s.generation AND e.source_id IN (SELECT c.source_id FROM children c WHERE ${clauses.slice(4).join(" AND ")})),`
@@ -166,6 +167,12 @@ export class SearchQueries {
     limit: number,
   ): readonly Row[] {
     // Match every token across a record's visible windows; a quoted phrase cannot span fields.
+    // Relevance benefits from hit-driven attribution. Newest keeps the native join order:
+    // forcing hit order regresses newest sorting across many shared attempts.
+    const joined =
+      score === "matched.score"
+        ? "FROM matched CROSS JOIN entries e ON e.rowid=matched.entry_rowid CROSS JOIN sources s ON s.id=e.source_id CROSS JOIN children c ON c.source_id=s.id JOIN excerpts ON excerpts.entry_rowid=matched.entry_rowid AND excerpts.choice=1 JOIN documents d ON d.id=excerpts.id"
+        : "FROM matched JOIN excerpts ON excerpts.entry_rowid=matched.entry_rowid AND excerpts.choice=1 JOIN documents d ON d.id=excerpts.id JOIN entries e ON e.rowid=matched.entry_rowid JOIN sources s ON s.id=e.source_id JOIN children c ON c.source_id=s.id";
     // Ranking/filters precede LIMIT. The parent owns the hard deadline for a costly FTS scan.
     return this.scope.store.all(
       `WITH ${filter.eligible} hits AS MATERIALIZED (
@@ -173,7 +180,7 @@ export class SearchQueries {
     ), terms AS (SELECT entry_rowid,term,MIN(score) AS score FROM hits GROUP BY entry_rowid,term),
     matched AS (SELECT entry_rowid,SUM(score) AS score FROM terms GROUP BY entry_rowid HAVING COUNT(*)=${expression.length}),
     excerpts AS (SELECT id,entry_rowid,ROW_NUMBER() OVER (PARTITION BY entry_rowid ORDER BY score,id) AS choice FROM hits)
-    SELECT d.*,${score} AS score,e.*,d.id AS document_id,s.path,s.session_id,c.run_id,c.child_index,c.agent,d.preview AS document_preview FROM matched JOIN excerpts ON excerpts.entry_rowid=matched.entry_rowid AND excerpts.choice=1 JOIN documents d ON d.id=excerpts.id JOIN entries e ON e.rowid=matched.entry_rowid JOIN sources s ON s.id=e.source_id JOIN children c ON c.source_id=s.id LEFT JOIN entries terminal ON terminal.source_id=s.id AND terminal.generation=s.generation AND terminal.id=c.terminal_entry_id AND terminal.published=1 WHERE ${filter.clauses.join(" AND ")} ORDER BY score,d.id,c.run_id,c.child_index LIMIT ?`,
+    SELECT d.*,${score} AS score,e.*,d.id AS document_id,s.path,s.session_id,c.run_id,c.child_index,c.agent,d.preview AS document_preview ${joined} LEFT JOIN entries terminal ON terminal.source_id=s.id AND terminal.generation=s.generation AND terminal.id=c.terminal_entry_id AND terminal.published=1 WHERE ${filter.clauses.join(" AND ")} ORDER BY score,d.id,c.run_id,c.child_index LIMIT ?`,
       ...filter.eligibleParams,
       ...filter.params,
       limit + 1,

package/src/history/store.ts

@@ -1,7 +1,7 @@
 import * as path from "node:path";
 import { openGeneration } from "./database-generation.ts";
 import { OutputDigests } from "./output-digests.ts";
-import type { DatabaseSync } from "node:sqlite";
+import type { DatabaseSync, StatementSync } from "node:sqlite";
 import { historyTransaction, openHistoryDatabase } from "./database.ts";
 import { hash, safeText, hasText } from "./text.ts";
 export { hash, exactTextDigest, safeText, integer, pageLimit } from "./text.ts";
@@ -45,6 +45,7 @@ export class HistoryStore {
     this.counters[operation] += amount;
   }
   private ingestLock?: DatabaseSync;
+  private readonly statements = new Map<string, StatementSync>();
   private readonly outputDigests = new OutputDigests();
   constructor(agentDir: string, owner: string) {
     this.owner = owner;
@@ -72,12 +73,11 @@ export class HistoryStore {
     return row;
   }
   get(sql: string, ...params: readonly IndexInput[]): Row | undefined {
-    const row = this.db.prepare(sql).get(...params);
+    const row = this.statement(sql).get(...params);
     return row === undefined ? undefined : indexRow(row);
   }
   all(sql: string, ...params: readonly IndexInput[]): readonly Row[] {
-    return this.db
-      .prepare(sql)
+    return this.statement(sql)
       .all(...params)
       .map(indexRow);
   }
@@ -85,7 +85,22 @@ export class HistoryStore {
     sql: string,
     ...params: readonly IndexInput[]
   ): ReturnType<ReturnType<DatabaseSync["prepare"]>["run"]> {
-    return this.db.prepare(sql).run(...params);
+    return this.statement(sql).run(...params);
+  }
+  private statement(sql: string): StatementSync {
+    let statement = this.statements.get(sql);
+    if (!statement) {
+      statement = this.db.prepare(sql);
+      // Dynamic browse/filter shapes cannot grow this connection's cache indefinitely.
+      if (this.statements.size >= 128) {
+        const oldest = this.statements.keys().next().value;
+        if (oldest !== undefined) {
+          this.statements.delete(oldest);
+        }
+      }
+      this.statements.set(sql, statement);
+    }
+    return statement;
   }
   deleteDocuments(sourceId: string, unpublishedOnly = false): void {
     const where = `entry_rowid IN (SELECT rowid FROM entries WHERE source_id=?${unpublishedOnly ? " AND published=0" : ""})`;
@@ -279,6 +294,7 @@ export class HistoryStore {
     }
   }
   close(): void {
+    this.statements.clear();
     this.ingestLock?.close();
     this.db.close();
   }

package/src/history/text-chunks.ts

@@ -171,9 +171,14 @@ export class TextChunks {
       this.fields.clear();
     }
     if (this.staged.length > 0) {
-      const insert = store.db.prepare("INSERT INTO pending_text VALUES (?,?,?,?)");
       for (const item of this.staged) {
-        insert.run(item.field, item.start, item.end, item.text);
+        store.run(
+          "INSERT INTO pending_text VALUES (?,?,?,?)",
+          item.field,
+          item.start,
+          item.end,
+          item.text,
+        );
       }
       this.staged = [];
     }