@fitchmultz/pi-subagents 0.49.1 → 0.50.0

package/src/history/selected-record.ts

@@ -1,140 +0,0 @@
-import * as fs from "node:fs";
-import { createHash } from "node:crypto";
-import { JsonProjection } from "../shared/journal-reader.ts";
-import { HistoryIndexError } from "./types.ts";
-import type { EntryRow, SourceRow } from "./rows.ts";
-import { identity } from "./source-file.ts";
-import { previewLimits, previewProjection } from "./preview.ts";
-import { parseObject } from "./values.ts";
-import { isRecord, hasErrorCode } from "../shared/unknown.ts";
-
-function validateHeader(fd: number, source: SourceRow): void {
-  const hash = createHash("sha256");
-  const bytes = Buffer.allocUnsafe(64 * 1024);
-  let position = 0;
-  let complete = false;
-  while (!complete && position < 1024 * 1024) {
-    const count = fs.readSync(fd, bytes, 0, bytes.length, position);
-    if (count === 0) {
-      break;
-    }
-    const newline = bytes.subarray(0, count).indexOf(10);
-    const length = newline < 0 ? count : newline + 1;
-    hash.update(bytes.subarray(0, length));
-    position += length;
-    complete = newline >= 0;
-  }
-  if (!complete || hash.digest("hex") !== source.header_digest) {
-    throw new HistoryIndexError(
-      "SOURCE_CHANGED",
-      "Selected native header changed or exceeds its validation budget.",
-    );
-  }
-}
-function readBody(
-  fd: number,
-  row: EntryRow,
-  full: boolean,
-): { readonly finish: () => Readonly<Record<string, unknown>>; readonly digest: string } {
-  const digest = createHash("sha256");
-  const parser = new JsonProjection(previewProjection, undefined, undefined, previewLimits);
-  const bytes = Buffer.allocUnsafe(64 * 1024);
-  const body: Buffer[] = [];
-  const decoder = new TextDecoder("utf8", { fatal: true });
-  for (let position = row.start; position < row.end;) {
-    const count = fs.readSync(fd, bytes, 0, Math.min(bytes.length, row.end - position), position);
-    if (count === 0) {
-      throw new HistoryIndexError(
-        "SOURCE_CHANGED",
-        "Selected record was truncated during validation.",
-      );
-    }
-    const chunk = bytes.subarray(0, count);
-    digest.update(chunk);
-    if (full) {
-      body.push(Buffer.from(chunk));
-    } else {
-      parser.write(decoder.decode(chunk, { stream: true }));
-    }
-    position += count;
-  }
-  return {
-    digest: digest.digest("hex"),
-    finish: () => {
-      if (full) {
-        return parseObject(new TextDecoder("utf8", { fatal: true }).decode(Buffer.concat(body)));
-      }
-      parser.write(decoder.decode());
-      const projected = parser.finish();
-      if (!isRecord(projected)) {
-        throw new HistoryIndexError("SOURCE_CHANGED", "Selected record is not an object.");
-      }
-      return projected;
-    },
-  };
-}
-function validateSnapshot(
-  fd: number,
-  source: SourceRow,
-  row: EntryRow,
-  snapshot: { readonly before: fs.BigIntStats; readonly digest: string },
-): void {
-  const { before, digest } = snapshot;
-  const current = fs.statSync(source.path, { bigint: true });
-  const after = fs.fstatSync(fd, { bigint: true });
-  if (
-    identity(current) !== identity(before) ||
-    current.size < BigInt(row.end) ||
-    (after.ctimeNs !== before.ctimeNs && after.size <= before.size) ||
-    digest !== row.digest
-  ) {
-    throw new HistoryIndexError(
-      "SOURCE_CHANGED",
-      "Selected record changed; refresh history before reading details.",
-    );
-  }
-}
-/** Revalidate physical identity, native header and exact selected bytes before exposing detail. */
-export function validateRecord(
-  source: SourceRow,
-  row: EntryRow,
-  full: boolean,
-): Readonly<Record<string, unknown>> {
-  let fd: number | undefined;
-  try {
-    fd = fs.openSync(source.path, "r");
-    const stat = fs.fstatSync(fd, { bigint: true });
-    if (identity(stat) !== source.identity || Number(stat.size) < row.end) {
-      throw new HistoryIndexError(
-        "SOURCE_CHANGED",
-        "Selected conversation was replaced or truncated.",
-      );
-    }
-    validateHeader(fd, source);
-    if (full && row.end - row.start > 16 * 1024 * 1024) {
-      throw new HistoryIndexError(
-        "RECORD_TOO_LARGE",
-        "Selected full record exceeds the 16 MiB detail budget; use its validated preview.",
-      );
-    }
-    const body = readBody(fd, row, full);
-    validateSnapshot(fd, source, row, { before: stat, digest: body.digest });
-    // Physical validation wins over full JSON/UTF-8 decode errors if the file changed during reading.
-    return body.finish();
-  } catch (error) {
-    if (hasErrorCode(error, "ENOENT")) {
-      throw new HistoryIndexError("SOURCE_CHANGED", "Selected conversation is missing.");
-    }
-    if (error instanceof RangeError) {
-      throw new HistoryIndexError(
-        "RECORD_COMPLEXITY",
-        "Selected preview exceeds bounded history structure budgets.",
-      );
-    }
-    throw error;
-  } finally {
-    if (fd !== undefined) {
-      fs.closeSync(fd);
-    }
-  }
-}

package/src/history/source-file.ts

@@ -1,51 +0,0 @@
-import * as fs from "node:fs";
-import type { HistoryStore, SourceRow } from "./store.ts";
-import { HistoryIndexError } from "./types.ts";
-
-export function identity(stat: fs.BigIntStats): string {
-  return `${stat.dev}:${stat.ino}`;
-}
-export function stamp(stat: fs.BigIntStats): string {
-  return `${identity(stat)}:${stat.size}:${stat.mtimeNs}:${stat.ctimeNs}`;
-}
-export function requireSource(store: Readonly<HistoryStore>, id: string): SourceRow {
-  const source = store.source(id);
-  if (!source) {
-    throw new HistoryIndexError("SOURCE_CHANGED", "Source was removed before ingestion.");
-  }
-  return source;
-}
-/** Acquires the writer lock before reading any cursor or clearing shared staging. */
-export function openSource(
-  store: Readonly<HistoryStore>,
-  id: string,
-): { readonly fd: number; readonly initial: fs.BigIntStats; readonly source: SourceRow } {
-  store.beginIngest();
-  let fd: number | undefined;
-  try {
-    let source = requireSource(store, id);
-    store.count("sourceChecks");
-    store.count("sourceOpens");
-    fd = fs.openSync(source.path, "r");
-    const initial = fs.fstatSync(fd, { bigint: true });
-    if (!initial.isFile() || initial.size > BigInt(Number.MAX_SAFE_INTEGER)) {
-      throw new Error("Journal is not a supported regular file.");
-    }
-    if (
-      source.identity !== null &&
-      (source.identity !== identity(initial) || Number(initial.size) < source.cursor)
-    ) {
-      store.resetSource(id);
-      source = requireSource(store, id);
-    }
-    store.clearUnpublished(id);
-    store.run("DELETE FROM pending_text");
-    return { fd, initial, source };
-  } catch (error) {
-    if (fd !== undefined) {
-      fs.closeSync(fd);
-    }
-    store.endIngest();
-    throw error;
-  }
-}

package/src/history/source-watches.ts

@@ -1,156 +0,0 @@
-import * as fs from "node:fs";
-import * as path from "node:path";
-import { identity } from "./source-file.ts";
-import type { SourceRow } from "./rows.ts";
-
-interface WatchEvents {
-  readonly run: (id: string) => void;
-  readonly source: (id: string) => void;
-  readonly schedule: () => void;
-}
-/** Owns native handles, physical-directory identity and co-located run/source subscriptions. */
-export class SourceWatches {
-  private readonly runs = new Set<string>();
-  private readonly watchers = new Map<string, fs.FSWatcher>();
-  private readonly sourceWatchers = new Map<string, fs.FSWatcher>();
-  private readonly identities = new WeakMap<fs.FSWatcher, string>();
-  private readonly sources = new Map<string, Map<string, string>>();
-  private readonly root: string;
-  private readonly events: WatchEvents;
-  constructor(agentDir: string, events: WatchEvents) {
-    this.root = path.join(agentDir, "sessions", "subagent-runs");
-    this.events = events;
-  }
-  private handle(
-    directory: string,
-    recursive: boolean,
-    listener: (name: string | null) => void,
-    source = false,
-  ): void {
-    const handles = source ? this.sourceWatchers : this.watchers;
-    const existing = handles.get(directory);
-    try {
-      const physical = identity(fs.statSync(directory, { bigint: true }));
-      if (existing && this.identities.get(existing) === physical) {
-        return;
-      }
-      const watcher = fs.watch(directory, { recursive }, (event, name) => {
-        const filename = name?.toString() ?? null;
-        const selfRename = event === "rename" && filename === path.basename(directory);
-        if (selfRename && handles.get(directory) === watcher) {
-          this.identities.delete(watcher);
-          this.handle(directory, recursive, listener, source);
-          listener(null);
-        } else {
-          listener(filename);
-        }
-        if (!source) {
-          this.notifySources(directory, selfRename ? null : filename);
-        }
-        this.events.schedule();
-      });
-      this.identities.set(watcher, physical);
-      this.retireOnClose(directory, watcher, source);
-      handles.set(directory, watcher);
-      // Native close fences macOS re-registration; admit the replacement first.
-      existing?.close();
-    } catch {
-      if (existing && handles.get(directory) === existing) {
-        this.identities.delete(existing);
-      }
-      // Known-handle census recovers absent directories and dropped native watches.
-    }
-  }
-  private retireOnClose(directory: string, watcher: Readonly<fs.FSWatcher>, source: boolean): void {
-    const handles = source ? this.sourceWatchers : this.watchers;
-    const retire = () => {
-      if (handles.get(directory) === watcher) {
-        handles.delete(directory);
-      }
-    };
-    watcher.on("error", () => {
-      watcher.close();
-      retire();
-    });
-    watcher.on("close", retire);
-  }
-  private reset(directory: string): void {
-    for (const handles of [this.watchers, this.sourceWatchers]) {
-      const watcher = handles.get(directory);
-      if (watcher) {
-        this.identities.delete(watcher);
-      }
-    }
-    for (const id of this.sources.get(directory)?.keys() ?? []) {
-      this.events.source(id);
-    }
-  }
-  private notifySources(directory: string, name: string | null): void {
-    for (const [id, filename] of this.sources.get(directory) ?? []) {
-      if (name === null || name.length === 0 || filename === name) {
-        this.events.source(id);
-      }
-    }
-  }
-  sourceDirectory(directory: string): void {
-    if (!this.sources.has(directory) || this.watchers.has(directory)) {
-      return;
-    }
-    this.handle(
-      directory,
-      false,
-      (name) => {
-        this.notifySources(directory, name);
-      },
-      true,
-    );
-  }
-  install(runIds: readonly string[], sources: readonly Pick<SourceRow, "id" | "path">[]): void {
-    for (const id of runIds) {
-      this.runs.add(id);
-    }
-    // Observe only admitted trees. Foreign run directory churn never causes a global scan.
-    this.handle(this.root, false, (name) => {
-      let ids: Iterable<string> = this.runs;
-      if (name !== null && name.length > 0) {
-        ids = this.runs.has(name) ? [name] : [];
-      }
-      for (const id of ids) {
-        this.reset(path.join(this.root, id));
-        this.events.run(id);
-      }
-    });
-    for (const id of runIds) {
-      const directory = path.join(this.root, id);
-      this.handle(directory, true, (name) => {
-        if (name === null || name.length === 0) {
-          this.reset(directory);
-        }
-        this.events.run(id);
-      });
-      this.sourceDirectory(directory);
-    }
-    for (const source of sources) {
-      const directory = path.dirname(source.path);
-      let linked = this.sources.get(directory);
-      if (!linked) {
-        linked = new Map();
-        this.sources.set(directory, linked);
-      }
-      linked.set(source.id, path.basename(source.path));
-      this.sourceDirectory(directory);
-    }
-  }
-  forget(source: Pick<SourceRow, "id" | "path">): void {
-    this.sources.get(path.dirname(source.path))?.delete(source.id);
-  }
-  close(): void {
-    for (const watcher of [...this.watchers.values(), ...this.sourceWatchers.values()]) {
-      watcher.close();
-    }
-    this.watchers.clear();
-    this.sourceWatchers.clear();
-    this.sources.clear();
-    this.runs.clear();
-  }
-}

package/src/history/store.ts

@@ -1,332 +0,0 @@
-import * as path from "node:path";
-import { openGeneration } from "./database-generation.ts";
-import { OutputDigests } from "./output-digests.ts";
-import type { DatabaseSync, StatementSync } from "node:sqlite";
-import { historyTransaction, openHistoryDatabase } from "./database.ts";
-import { hash, safeText, hasText } from "./text.ts";
-export { hash, exactTextDigest, safeText, integer, pageLimit } from "./text.ts";
-import {
-  HistoryIndexError,
-  type HistoryEntry,
-  type HistoryFreshness,
-  type HistoryRunRow,
-  type HistoryVersion,
-  type HistoryIndexStatus,
-} from "./types.ts";
-import {
-  sourceRow,
-  indexRow,
-  type Row,
-  type IndexInput,
-  type SourceRow,
-  type EntryRow,
-  number,
-  text,
-} from "./rows.ts";
-import { parseObject, errorMessage } from "./values.ts";
-
-export type { SourceRow, EntryRow } from "./rows.ts";
-export class HistoryStore {
-  readonly db: DatabaseSync;
-  readonly generation: string;
-  readonly file: string;
-  readonly owner: string;
-  private readonly counters = {
-    sourceChecks: 0,
-    sourceOpens: 0,
-    sourceBytesRead: 0,
-    runProjections: 0,
-    queries: 0,
-  };
-  get operations(): Readonly<typeof this.counters> {
-    return { ...this.counters };
-  }
-  count(operation: keyof typeof this.counters, amount = 1): void {
-    this.counters[operation] += amount;
-  }
-  private ingestLock?: DatabaseSync;
-  private readonly statements = new Map<string, StatementSync>();
-  private readonly outputDigests = new OutputDigests();
-  constructor(agentDir: string, owner: string) {
-    this.owner = owner;
-    const { db, file } = openGeneration(agentDir, owner);
-    this.db = db;
-    this.file = file;
-    try {
-      this.generation = text(
-        this.require("SELECT value FROM meta WHERE key='generation'"),
-        "value",
-      );
-      this.db.exec(
-        "CREATE TEMP TABLE pending_text(field TEXT, position INTEGER, end_position INTEGER, text TEXT)",
-      );
-    } catch (error) {
-      this.db.close();
-      throw error;
-    }
-  }
-  require(sql: string, ...params: readonly IndexInput[]): Row {
-    const row = this.get(sql, ...params);
-    if (!row) {
-      throw new HistoryIndexError("CORRUPT", "Missing required index row.");
-    }
-    return row;
-  }
-  get(sql: string, ...params: readonly IndexInput[]): Row | undefined {
-    const row = this.statement(sql).get(...params);
-    return row === undefined ? undefined : indexRow(row);
-  }
-  all(sql: string, ...params: readonly IndexInput[]): readonly Row[] {
-    return this.statement(sql)
-      .all(...params)
-      .map(indexRow);
-  }
-  run(
-    sql: string,
-    ...params: readonly IndexInput[]
-  ): ReturnType<ReturnType<DatabaseSync["prepare"]>["run"]> {
-    return this.statement(sql).run(...params);
-  }
-  private statement(sql: string): StatementSync {
-    let statement = this.statements.get(sql);
-    if (!statement) {
-      statement = this.db.prepare(sql);
-      // Dynamic browse/filter shapes cannot grow this connection's cache indefinitely.
-      if (this.statements.size >= 128) {
-        const oldest = this.statements.keys().next().value;
-        if (oldest !== undefined) {
-          this.statements.delete(oldest);
-        }
-      }
-      this.statements.set(sql, statement);
-    }
-    return statement;
-  }
-  deleteDocuments(sourceId: string, unpublishedOnly = false): void {
-    const where = `entry_rowid IN (SELECT rowid FROM entries WHERE source_id=?${unpublishedOnly ? " AND published=0" : ""})`;
-    this.run(
-      `DELETE FROM corpus WHERE rowid IN (SELECT id FROM documents WHERE ${where})`,
-      sourceId,
-    );
-    this.run(`DELETE FROM documents WHERE ${where}`, sourceId);
-  }
-  transaction<T>(work: () => T): T {
-    return historyTransaction(this.db, work);
-  }
-  bump(): void {
-    this.run("UPDATE meta SET value=CAST(value AS INTEGER)+1 WHERE key='version'");
-    this.run("UPDATE meta SET value=? WHERE key='indexedAt'", String(Date.now()));
-  }
-  source(id: string): SourceRow | undefined {
-    const row = this.get("SELECT * FROM sources WHERE id=?", id);
-    return row ? sourceRow(row) : undefined;
-  }
-  resetSource(id: string, state = "pending", error: string | null = null): void {
-    this.transaction(() => {
-      this.deleteDocuments(id);
-      this.run("DELETE FROM entries WHERE source_id=?", id);
-      this.run(
-        "UPDATE sources SET generation=generation+1,identity=NULL,stamp=NULL,session_id=NULL,header_digest=NULL,format_version=NULL,cursor=0,prefix_digest=NULL,state=?,error=?,malformed=0,complexity=0,checked_at=? WHERE id=?",
-        state,
-        error,
-        Date.now(),
-        id,
-      );
-      this.bump();
-    });
-  }
-  clearUnpublished(id: string): void {
-    this.transaction(() => {
-      this.deleteDocuments(id, true);
-      this.run("DELETE FROM entries WHERE source_id=? AND published=0", id);
-    });
-  }
-  putView(view: HistoryRunRow, full: HistoryRunRow = view): void {
-    const attention = attentionRank(view);
-    const json = JSON.stringify(view),
-      filters = new Map(
-        full.children.map((child) => [
-          child.index,
-          safeText(
-            `${child.agent} ${child.label ?? ""} ${child.task ?? full.task}`,
-            Infinity,
-          ).toLowerCase(),
-        ]),
-      );
-    const filterText = safeText(
-      `${full.task} ${[...filters.values()].join(" ")}`,
-      Infinity,
-    ).toLowerCase();
-    const digests = new Map(
-      full.children.map((child) => [child.index, this.outputDigests.read(child.result)]),
-    );
-    const saved = this.get("SELECT view,filter_text FROM runs WHERE id=?", view.runId);
-    if (
-      saved?.view === json &&
-      saved.filter_text === filterText &&
-      this.all("SELECT child_index,output_digest FROM children WHERE run_id=?", view.runId).every(
-        (child) => child.output_digest === digests.get(number(child, "child_index")),
-      )
-    ) {
-      return;
-    }
-    this.transaction(() => {
-      this.run(
-        "INSERT INTO runs VALUES (?,?,?,?,?,?,?,?,?) ON CONFLICT(id) DO UPDATE SET attention=excluded.attention,updated=excluded.updated,negative_updated=excluded.negative_updated,state=excluded.state,filter_text=excluded.filter_text,view=excluded.view,predecessor_id=excluded.predecessor_id,predecessor_index=excluded.predecessor_index",
-        view.runId,
-        attention,
-        view.updatedAt,
-        -view.updatedAt,
-        view.state,
-        filterText,
-        json,
-        view.predecessorRunId ?? null,
-        view.predecessorIndex ?? 0,
-      );
-      this.putChildren(view, filters, digests);
-      this.bump();
-    });
-  }
-  private putChildren(
-    view: HistoryRunRow,
-    filters: Readonly<ReadonlyMap<number, string>>,
-    digests: Readonly<ReadonlyMap<number, string | null>>,
-  ): void {
-    this.run("DELETE FROM children WHERE run_id=?", view.runId);
-    for (const child of view.children) {
-      let source: string | null = null;
-      if (hasText(child.sessionFile)) {
-        const file = path.resolve(child.sessionFile);
-        source = hash(file);
-        this.run(
-          "INSERT INTO sources(id,path) VALUES (?,?) ON CONFLICT(id) DO NOTHING",
-          source,
-          file,
-        );
-      }
-      const terminal = child.state !== "live" && child.state !== "unknown";
-      const boundary = terminalBoundary(child, view.updatedAt, terminal);
-      this.run(
-        "INSERT INTO children VALUES (?,?,?,?,?,?,?,?)",
-        view.runId,
-        child.index,
-        child.agent,
-        source,
-        filters.get(child.index) ?? "",
-        digests.get(child.index) ?? null,
-        boundary.entry,
-        boundary.endedAt,
-      );
-    }
-  }
-  info(pending: number, projectionErrors = 0): HistoryVersion {
-    const sources = this.require(
-      "SELECT COUNT(*) AS total,SUM(state IN ('pending','indexing')) AS pending,SUM(state IN ('degraded','missing','error')) AS errors FROM sources WHERE id IN (SELECT source_id FROM children)",
-    );
-    const indexedAtValue = Number(
-      text(this.require("SELECT value FROM meta WHERE key='indexedAt'"), "value"),
-    );
-    const indexedAt = indexedAtValue === 0 ? null : indexedAtValue;
-    const errors = Number(sources.errors ?? 0) + projectionErrors;
-    const remaining = pending + Number(sources.pending ?? 0);
-    const freshness: HistoryFreshness = {
-      authoritative: false,
-      state: freshnessState(remaining, errors),
-      pending: remaining,
-      errors,
-      indexedAt,
-    };
-    return {
-      version: Number(text(this.require("SELECT value FROM meta WHERE key='version'"), "value")),
-      indexedAt,
-      freshness,
-    };
-  }
-  status(pending: number): HistoryIndexStatus {
-    return {
-      ...this.info(pending),
-      databaseFile: this.file,
-      physicalSources: number(this.require("SELECT COUNT(*) AS count FROM sources"), "count"),
-      publishedEntries: number(
-        this.require("SELECT COUNT(*) AS count FROM entries WHERE published=1"),
-        "count",
-      ),
-      operations: this.operations,
-    };
-  }
-  entry(row: EntryRow): HistoryEntry {
-    return {
-      id: row.id,
-      nativeId: row.native_id,
-      parentId: row.parent_id,
-      sequence: row.start,
-      timestamp: row.timestamp,
-      type: row.type,
-      ref: {
-        sourceId: row.source_id,
-        generation: row.generation,
-        start: row.start,
-        end: row.end,
-        digest: row.digest,
-      },
-      entry: parseObject(row.preview),
-    };
-  }
-  /** A separate SQLite transaction serializes incremental staging without blocking browse queries.
-   * SQLite releases it on process death; no PID lease, expiry, or stale-file deletion is needed. */
-  beginIngest(): void {
-    if (!this.ingestLock) {
-      this.ingestLock = openHistoryDatabase(`${this.file}.ingest.sqlite`);
-      this.ingestLock.exec("PRAGMA busy_timeout=0");
-    }
-    try {
-      this.ingestLock.exec("BEGIN IMMEDIATE");
-    } catch (error) {
-      if (/locked|busy/i.test(errorMessage(error))) {
-        throw new HistoryIndexError("INDEX_BUSY", "Another worker is indexing this generation.");
-      }
-      throw error;
-    }
-  }
-  endIngest(): void {
-    if (this.ingestLock?.isTransaction === true) {
-      this.ingestLock.exec("ROLLBACK");
-    }
-  }
-  close(): void {
-    this.statements.clear();
-    this.ingestLock?.close();
-    this.db.close();
-  }
-}
-function terminalBoundary(
-  child: HistoryRunRow["children"][number],
-  updatedAt: number,
-  terminal: boolean,
-): { readonly entry: string | null; readonly endedAt: number | null } {
-  if (!terminal) {
-    return { entry: null, endedAt: null };
-  }
-  return {
-    entry: child.result?.terminalEntryId ?? null,
-    endedAt: hasText(child.result?.terminalEntryId) ? null : updatedAt,
-  };
-}
-function attentionRank(view: HistoryRunRow): number {
-  if (view.attention.includes("awaiting_input")) {
-    return 0;
-  }
-  if (view.attention.some((reason) => reason !== "unreviewed")) {
-    return 1;
-  }
-  if (view.state === "live") {
-    return 2;
-  }
-  return view.attention.length > 0 ? 3 : 4;
-}
-function freshnessState(pending: number, errors: number): HistoryFreshness["state"] {
-  if (pending > 0) {
-    return "catching-up";
-  }
-  return errors > 0 ? "degraded" : "current";
-}