@fitchmultz/pi-subagents 0.44.6 → 0.45.0

package/src/history/text-chunks.ts

@@ -0,0 +1,181 @@
+import { HistoryIndexError } from "./types.ts";
+import { candidate } from "./preview.ts";
+import type { HistoryStore } from "./store.ts";
+
+interface TextToken {
+  readonly text: string;
+  readonly start: number;
+  readonly end: number;
+}
+interface TextWindow {
+  readonly field: string;
+  readonly start: number;
+  readonly end: number;
+  readonly text: string;
+}
+/** Streaming ECMA-48 removal, retaining original UTF-16 positions without buffering control payloads. */
+class TerminalText {
+  private state: "text" | "escape" | "intermediate" | "csi" | "string" | "stringEscape" = "text";
+  private escape(char: string): void {
+    if (char === "[") {
+      this.state = "csi";
+    } else if ("]PX^_".includes(char)) {
+      this.state = "string";
+    } else {
+      this.state = char >= " " && char <= "/" ? "intermediate" : "text";
+    }
+  }
+  private control(char: string): void {
+    if (this.state === "escape") {
+      this.escape(char);
+    } else if (this.state === "intermediate") {
+      if (char >= "0" && char <= "~") {
+        this.state = "text";
+      }
+    } else if (this.state === "csi") {
+      if (char >= "@" && char <= "~") {
+        this.state = "text";
+      }
+    } else {
+      this.stringControl(char);
+    }
+  }
+  private stringControl(char: string): void {
+    if (char === "\x07" || char === "\x9c" || (this.state === "stringEscape" && char === "\\")) {
+      this.state = "text";
+    } else {
+      this.state = char === "\x1b" ? "stringEscape" : "string";
+    }
+  }
+  write(text: string, visible: (text: string, start: number) => void): void {
+    let position = 0;
+    while (position < text.length) {
+      if (this.state !== "text") {
+        this.control(text.charAt(position++));
+        continue;
+      }
+      // ECMA-48 introducers intentionally identify terminal controls before lexical indexing.
+      // oxlint-disable-next-line no-control-regex
+      const next = text.slice(position).search(/[\x1b\x90\x98\x9b\x9d-\x9f]/);
+      if (next < 0) {
+        visible(text.slice(position), position);
+        return;
+      }
+      visible(text.slice(position, position + next), position);
+      position += next;
+      const control = text.charAt(position++);
+      if (control === "\x1b") {
+        this.state = "escape";
+      } else {
+        this.state = control === "\x9b" ? "csi" : "string";
+      }
+    }
+  }
+}
+/** Owns one lexical stream and its overlap; no caller can mutate its token buffer. */
+class FieldText {
+  private word = "";
+  private wordStart = 0;
+  private received = 0;
+  private tokens: TextToken[] = [];
+  private length = 0;
+  private readonly terminal = new TerminalText();
+  private readonly field: string;
+  private readonly emit: (window: TextWindow) => void;
+  constructor(field: string, emit: (window: TextWindow) => void) {
+    this.field = field;
+    this.emit = emit;
+  }
+  private window(final = false): void {
+    const first = this.tokens.at(0);
+    const last = this.tokens.at(-1);
+    if (!first || !last) {
+      return;
+    }
+    this.emit({
+      field: this.field,
+      start: first.start,
+      end: last.end,
+      text: this.tokens.map((token) => token.text).join(" "),
+    });
+    this.tokens = final ? [] : this.tokens.slice(-12);
+    this.length = this.tokens.reduce((length, token) => length + token.text.length + 1, 0);
+  }
+  private token(end: number): void {
+    if (this.word.length === 0) {
+      return;
+    }
+    this.tokens.push({ text: this.word, start: this.wordStart, end });
+    this.length += this.word.length + 1;
+    this.word = "";
+    if (this.tokens.length >= 256 || (this.length >= 8192 && this.tokens.length > 12)) {
+      this.window();
+    }
+  }
+  write(text: string): void {
+    this.terminal.write(text, (visible, offset) => {
+      for (const part of visible.matchAll(/[\p{L}\p{N}\p{M}]+|[^\p{L}\p{N}\p{M}]+/gu)) {
+        const start = this.received + offset + part.index;
+        if (!/^[\p{L}\p{N}\p{M}]/u.test(part[0])) {
+          this.token(start);
+          continue;
+        }
+        if (this.word.length === 0) {
+          this.wordStart = start;
+        }
+        this.word += part[0];
+        if (this.word.length > 65_536) {
+          throw new HistoryIndexError(
+            "RECORD_COMPLEXITY",
+            "Record contains a lexical token exceeding the 64 KiB indexing budget.",
+          );
+        }
+      }
+    });
+    this.received += text.length;
+  }
+  finish(): void {
+    this.token(this.received);
+    this.window(true);
+  }
+}
+/** Token windows overlap 12 tokens, including phrases across arbitrarily long whitespace. */
+export class TextChunks {
+  private readonly fields = new Map<string, FieldText>();
+  private staged: TextWindow[] = [];
+  write(keys: readonly (string | number)[], text: string): void {
+    if (!candidate(keys)) {
+      return;
+    }
+    const field = JSON.stringify(keys);
+    let buffer = this.fields.get(field);
+    if (!buffer) {
+      if (this.fields.size >= 4096) {
+        throw new HistoryIndexError(
+          "RECORD_COMPLEXITY",
+          "Record exceeds the 4096 visible text-field indexing budget.",
+        );
+      }
+      buffer = new FieldText(field, (window) => {
+        this.staged.push(window);
+      });
+      this.fields.set(field, buffer);
+    }
+    buffer.write(text);
+  }
+  flush(store: Readonly<HistoryStore>, finish = false): void {
+    if (finish) {
+      for (const buffer of this.fields.values()) {
+        buffer.finish();
+      }
+      this.fields.clear();
+    }
+    if (this.staged.length > 0) {
+      const insert = store.db.prepare("INSERT INTO pending_text VALUES (?,?,?,?)");
+      for (const item of this.staged) {
+        insert.run(item.field, item.start, item.end, item.text);
+      }
+      this.staged = [];
+    }
+  }
+}

package/src/history/text.ts

@@ -0,0 +1,43 @@
+import { createHash } from "node:crypto";
+import { stripTerminalSequences } from "../shared/native-tui.ts";
+import { HistoryIndexError } from "./types.ts";
+
+export const hash = (text: string | Buffer): string =>
+  createHash("sha256").update(text).digest("hex");
+export const exactTextDigest = (text: string): string => hash(stripTerminalSequences(text).trim());
+export function hasText(value: unknown): value is string {
+  return typeof value === "string" && value.length > 0;
+}
+export function safeText(value: unknown, limit = 1024): string {
+  if (typeof value !== "string") {
+    return "";
+  }
+  const controls =
+    // Bounded UI/SQLite previews deliberately remove terminal escapes and C0/C1 controls.
+    // oxlint-disable-next-line no-control-regex
+    /\x1b\][^\x07]*(?:\x07|\x1b\\)|\x1b\[[0-?]*[ -/]*[@-~]|[\u0000-\u0008\u000b-\u001f\u007f-\u009f]/g;
+  return value.replace(controls, "").slice(0, limit);
+}
+export function integer(
+  value: unknown,
+  fallback: number,
+  maximum = Number.MAX_SAFE_INTEGER,
+): number {
+  const result = value === undefined ? fallback : value;
+  if (
+    typeof result !== "number" ||
+    !Number.isSafeInteger(result) ||
+    result < 0 ||
+    result > maximum
+  ) {
+    throw new HistoryIndexError("INVALID", "Expected a bounded non-negative integer.");
+  }
+  return result;
+}
+export function pageLimit(value: unknown, fallback = 100): number {
+  const limit = integer(value, fallback, 100);
+  if (limit === 0) {
+    throw new HistoryIndexError("INVALID", "Page limit must be from 1 to 100.");
+  }
+  return limit;
+}

package/src/history/transcript-queries.ts

@@ -0,0 +1,251 @@
+import { HistoryIndexError, type HistoryPageInput, type HistoryPage } from "./types.ts";
+import { integer, pageLimit } from "./text.ts";
+import { entryRow, number, nullableText, type SourceRow, type EntryRow } from "./rows.ts";
+import type { QueryScope, Boundary } from "./query-scope.ts";
+
+function validate(input: HistoryPageInput): void {
+  if (input.before !== undefined && input.after !== undefined) {
+    throw new HistoryIndexError("INVALID", "Use before or after, not both.");
+  }
+  if (
+    input.messageIds &&
+    (!Array.isArray(input.messageIds) ||
+      input.messageIds.length > 1000 ||
+      input.messageIds.some((id) => typeof id !== "string" || id.length > 512))
+  ) {
+    throw new HistoryIndexError("INVALID", "Invalid scoped message metadata request.");
+  }
+  validateReadThrough(input.readThrough);
+}
+function validateReadThrough(value: unknown): void {
+  if (value !== undefined && value !== null && (typeof value !== "string" || value.length > 1024)) {
+    throw new HistoryIndexError("INVALID", "Invalid scoped read metadata request.");
+  }
+}
+function emptyPage(scope: Readonly<QueryScope>, source: SourceRow | undefined): HistoryPage {
+  return {
+    ...scope.info(),
+    entries: [],
+    count: 0,
+    hasMore: false,
+    sourceId: source?.id ?? null,
+    generation: source?.generation ?? null,
+    sessionId: source?.session_id ?? null,
+    sourceState: source?.state ?? "unlinked",
+    configuration: {},
+    deliveredMessages: [],
+    latestEntryId: null,
+  };
+}
+function queryKey(scope: Readonly<QueryScope>, input: HistoryPageInput, source: SourceRow): string {
+  return scope.key([
+    "history",
+    input.runId,
+    input.index,
+    source.id,
+    source.generation,
+    input.terminalEntryId ?? null,
+    input.endedAt ?? null,
+    input.leaf ?? null,
+  ]);
+}
+export class TranscriptQueries {
+  private readonly scope: Readonly<QueryScope>;
+  constructor(scope: Readonly<QueryScope>) {
+    this.scope = scope;
+  }
+  private delivered(
+    boundary: Boundary,
+    ids: readonly string[],
+  ): readonly (readonly [string, boolean])[] {
+    const delivered: (readonly [string, boolean])[] = [];
+    const where = boundary.clauses.join(" AND ");
+    for (const id of new Set(ids)) {
+      const human = this.scope.store.get(
+        `SELECT start FROM entries WHERE ${where} AND human_id=? ORDER BY start DESC LIMIT 1`,
+        ...boundary.params,
+        id,
+      );
+      if (human) {
+        delivered.push([
+          id,
+          this.scope.store.get(
+            `SELECT rowid FROM entries WHERE ${where} AND start>? AND assistant_text=1 LIMIT 1`,
+            ...boundary.params,
+            number(human, "start"),
+          ) !== undefined,
+        ]);
+      }
+    }
+    return delivered;
+  }
+  private readMetadata(
+    boundary: Boundary,
+    readThrough: string | null | undefined,
+  ): { readonly latestEntryId: string | null; readonly unreadAfter?: boolean } {
+    const where = boundary.clauses.join(" AND ");
+    const latest = this.scope.store.get(
+      `SELECT start,visible_id FROM entries WHERE ${where} AND visible_id IS NOT NULL ORDER BY start DESC LIMIT 1`,
+      ...boundary.params,
+    );
+    const latestEntryId = latest ? nullableText(latest, "visible_id") : null;
+    if (readThrough === undefined) {
+      return { latestEntryId };
+    }
+    const marker =
+      typeof readThrough === "string"
+        ? this.scope.store.get(
+            `SELECT start,visible_id FROM entries WHERE ${where} AND id=?`,
+            ...boundary.params,
+            readThrough.replace(/:(?:\d+|error)$/, ""),
+          )
+        : undefined;
+    let unreadAfter = false;
+    if (latest) {
+      unreadAfter =
+        !marker ||
+        number(latest, "start") > number(marker, "start") ||
+        (number(latest, "start") === number(marker, "start") && latestEntryId !== readThrough);
+    }
+    return { latestEntryId, unreadAfter };
+  }
+  private pageRows(
+    boundary: Boundary,
+    before: number | undefined,
+    input: HistoryPageInput,
+  ): readonly ReturnType<typeof entryRow>[] {
+    const clauses = [...boundary.clauses];
+    const params = [...boundary.params];
+    if (before !== undefined) {
+      clauses.push("start<?");
+      params.push(integer(before, 0));
+    }
+    if (input.after !== undefined) {
+      clauses.push("start>?");
+      params.push(integer(input.after, 0));
+    }
+    return this.scope.store
+      .all(
+        `SELECT * FROM entries WHERE ${clauses.join(" AND ")} ORDER BY start ${input.after !== undefined ? "ASC" : "DESC"} LIMIT ?`,
+        ...params,
+        pageLimit(input.limit) + 1,
+      )
+      .map(entryRow);
+  }
+  private pageMetadata(
+    input: HistoryPageInput,
+    boundary: Boundary,
+    rows: readonly ReturnType<typeof entryRow>[],
+  ): Pick<
+    HistoryPage,
+    "count" | "hasMore" | "previousBefore" | "previousCursor" | "nextAfter" | "finalResultId"
+  > & { readonly previous: boolean } {
+    const first = rows.at(0);
+    const last = rows.at(-1);
+    const where = boundary.clauses.join(" AND ");
+    const count = number(
+      this.scope.store.require(
+        `SELECT COUNT(*) AS count FROM entries WHERE ${where}`,
+        ...boundary.params,
+      ),
+      "count",
+    );
+    const child = this.scope.store.get(
+      "SELECT output_digest FROM children WHERE run_id=? AND child_index=?",
+      input.runId,
+      input.index,
+    );
+    const outputDigest = child ? nullableText(child, "output_digest") : null;
+    const finalResultId =
+      outputDigest !== null && outputDigest.length > 0
+        ? this.scope.finalResultId(
+            input,
+            outputDigest,
+            first && last ? [first.start, last.start] : undefined,
+          )
+        : undefined;
+    const previous =
+      first !== undefined &&
+      this.scope.store.get(
+        `SELECT start FROM entries WHERE ${where} AND start<? LIMIT 1`,
+        ...boundary.params,
+        first.start,
+      ) !== undefined;
+    return {
+      count,
+      hasMore: previous,
+      previous,
+      finalResultId,
+      ...(last ? { nextAfter: last.start } : {}),
+    };
+  }
+  private navigation(
+    query: string,
+    rows: readonly EntryRow[],
+    metadata: ReturnType<TranscriptQueries["pageMetadata"]>,
+    forwardMore: boolean | undefined,
+  ): Pick<
+    HistoryPage,
+    "hasMore" | "finalResultId" | "previousBefore" | "previousCursor" | "nextAfter"
+  > {
+    const first = rows.at(0);
+    return {
+      hasMore: forwardMore ?? metadata.previous,
+      ...(metadata.finalResultId === undefined ? {} : { finalResultId: metadata.finalResultId }),
+      ...(metadata.previous && first
+        ? {
+            previousBefore: first.start,
+            previousCursor: this.scope.cursor(query, [first.start], 0),
+          }
+        : {}),
+      ...(metadata.nextAfter === undefined ? {} : { nextAfter: metadata.nextAfter }),
+    };
+  }
+  page(input: HistoryPageInput): HistoryPage {
+    validate(input);
+    const { source } = this.scope.child(input.runId, input.index);
+    const empty = emptyPage(this.scope, source);
+    if (!source) {
+      return {
+        ...empty,
+        unavailable: "This legacy or unstarted child has no linked native conversation.",
+      };
+    }
+    const query = queryKey(this.scope, input, source);
+    const cursor = this.scope.parseCursor(input.cursor, query);
+    if (cursor && (input.before !== undefined || input.after !== undefined)) {
+      throw new HistoryIndexError("INVALID", "Use cursor or sequence, not both.");
+    }
+    const boundary = this.scope.boundary(source, input);
+    const before = cursor ? integer(cursor.keys[0], 0) : input.before;
+    const fetched = this.pageRows(boundary, before, input);
+    const limit = pageLimit(input.limit);
+    const rows = fetched.slice(0, limit);
+    if (input.after === undefined) {
+      rows.reverse();
+    }
+    const metadata = this.pageMetadata(input, boundary, rows);
+    return {
+      ...empty,
+      configuration: this.scope.configuration(source, input),
+      deliveredMessages: this.delivered(boundary, input.messageIds ?? []),
+      ...this.readMetadata(boundary, input.readThrough),
+      terminalSequence: boundary.terminalSequence,
+      count: metadata.count,
+      ...this.navigation(
+        query,
+        rows,
+        metadata,
+        input.after === undefined ? undefined : fetched.length > limit,
+      ),
+      entries: rows.map((row) => this.scope.store.entry(row)),
+      ...unavailable(source.error, source.state),
+    };
+  }
+}
+function unavailable(error: string | null, state: string): { readonly unavailable?: string } {
+  if (error !== null && error.length > 0) {
+    return { unavailable: error };
+  }
+  return state === "missing" ? { unavailable: "Linked conversation is missing." } : {};
+}

package/src/history/types.ts

@@ -1,166 +1,29 @@
-import type { AcceptanceLedgerStatus, ForegroundResumeRun, ManagementRunState, OwnedRun, OwnedRunView } from "../shared/types.ts";
-import type { SupervisorQuestionView } from "../runs/shared/supervisor-questions.ts";
+export type * from "../shared/types/history.ts";
+export type {
+  OwnedRun,
+  OwnedRunView,
+  ForegroundResumeRun,
+  ReadonlyForegroundResumeRun,
+  ManagementRunState,
+} from "../shared/types/owned-runs.ts";
 
-export type { ForegroundResumeRun, ManagementRunState, OwnedRun, OwnedRunView };
-export interface HistoryOwner {
-	ownerSessionId: string;
-	ownerSessionFile?: string;
-	runs: OwnedRun[];
-	foregroundRuns?: ForegroundResumeRun[];
-}
-export interface HistoryFreshness {
-	/** Browse observations only; never proof of completion, ownership, or receipt absence. */
-	authoritative: false;
-	state: "current" | "catching-up" | "degraded";
-	pending: number;
-	errors: number;
-	indexedAt: number | null;
-}
-export interface HistoryVersion {
-	version: number;
-	indexedAt: number | null;
-	freshness: HistoryFreshness;
-}
-export interface HistoryConfiguration { model?: string; thinking?: string; modelRecordedAt?: number }
-export interface HistoryRunRow extends OwnedRunView {
-	summary?: string;
-	questions?: SupervisorQuestionView[];
-	/** Children satisfying the combined filters, before run pagination. */
-	matchedChildIndexes?: number[];
-	children: Array<OwnedRunView["children"][number] & { savedConfiguration?: HistoryConfiguration; nativeConfiguration?: HistoryConfiguration; humanAction?: string; acceptanceStatus?: AcceptanceLedgerStatus }>;
-}
-export interface HistoryRunOptions {
-	offset?: number;
-	limit?: number;
-	cursor?: string;
-	sort?: "attention" | "newest" | "oldest";
-	agent?: string;
-	state?: ManagementRunState;
-	text?: string;
-	/** Hide only child attempts superseded by an admitted continuation. */
-	latestTasksOnly?: boolean;
-	signal?: AbortSignal;
-}
-export interface HistoryRunPage extends HistoryVersion {
-	rows: HistoryRunRow[];
-	total: number;
-	offset: number;
-	nextOffset?: number;
-	nextCursor?: string;
-}
-export interface HistoryByteRef {
-	sourceId: string;
-	generation: number;
-	start: number;
-	end: number;
-	digest: string;
-}
-export interface HistoryEntry {
-	id: string;
-	/** Missing in older journal formats; id then identifies the physical byte range. */
-	nativeId: string | null;
-	parentId: string | null;
-	sequence: number;
-	timestamp: number | null;
-	type: string;
-	ref: HistoryByteRef;
-	/** Safe bounded text, thinking, and argument previews; images and raw details are not hydrated. */
-	entry: Record<string, any>;
-}
-export interface HistoryPageInput {
-	runId: string;
-	index: number;
-	limit?: number;
-	/** Exclusive physical sequence. Omit to read the latest page. */
-	before?: number;
-	/** Exclusive sequence for forward traversal; mutually exclusive with before. */
-	after?: number;
-	cursor?: string;
-	terminalEntryId?: string;
-	endedAt?: number;
-	/** Configuration ancestry; null is an intentionally empty branch. */
-	leaf?: string | null;
-	/** Only known outgoing IDs are returned, at most 1000. */
-	messageIds?: string[];
-	readThrough?: string | null;
-	signal?: AbortSignal;
-}
-export interface HistoryPage extends HistoryVersion {
-	entries: HistoryEntry[];
-	count: number;
-	hasMore: boolean;
-	previousBefore?: number;
-	previousCursor?: string;
-	nextAfter?: number;
-	sourceId: string | null;
-	generation: number | null;
-	sessionId: string | null;
-	sourceState: string;
-	configuration: HistoryConfiguration;
-	deliveredMessages: Array<[string, boolean]>;
-	latestEntryId: string | null;
-	unreadAfter?: boolean;
-	terminalSequence?: number;
-	/** Exact canonical answer match: prefer this page, otherwise locate the latest in the attempt. Never compare truncated previews. */
-	finalResultId?: string;
-	unavailable?: string;
-}
-export interface HistoryResult { text: string; timestamp: number; finalResultId?: string }
-export interface HistorySearchInput {
-	/** Whitespace-separated tokens (AND), or a quoted phrase, at most 12 tokens. No operators/prefixes. */
-	query: string;
-	runId?: string;
-	index?: number;
-	limit?: number;
-	cursor?: string;
-	sort?: "relevance" | "newest";
-	agent?: string;
-	signal?: AbortSignal;
-}
-export interface HistorySearchMatch {
-	id: number;
-	runId: string;
-	index: number;
-	agent: string;
-	entryId: string;
-	nativeId: string | null;
-	sessionId: string | null;
-	sessionFile: string;
-	timestamp: number | null;
-	ref: HistoryByteRef;
-	/** Indexed excerpt, not an authoritative/validated full record. */
-	preview: string;
-	field: string;
-	textStart: number;
-	textEnd: number;
-	score: number;
-}
-export interface HistorySearchPage extends HistoryVersion {
-	matches: HistorySearchMatch[];
-	nextCursor?: string;
-}
-export interface HistoryEntryInput {
-	runId: string;
-	index: number;
-	entryId?: string;
-	ref?: HistoryByteRef;
-	/** Find the call or result even when its paired entry is on another page. */
-	toolCallId?: string;
-	kind?: "call" | "result";
-	terminalEntryId?: string;
-	endedAt?: number;
-	signal?: AbortSignal;
-}
 export class HistoryIndexError extends Error {
-	readonly code: string;
-	constructor(code: string, message: string) { super(message); this.name = "HistoryIndexError"; this.code = code; }
-}
-export interface HistoryIndexStatus extends HistoryVersion {
-	databaseFile: string;
-	physicalSources: number;
-	publishedEntries: number;
-	/** Process-local operational counters, useful for catch-up/latency diagnostics. */
-	operations: { sourceChecks: number; sourceOpens: number; sourceBytesRead: number; runProjections: number; queries: number };
-}
-export interface Request { id: number; method: string; input?: any }
-export type Response = { id: number; value?: any; error?: { code: string; message: string } } | { changed: true };
+  readonly code: string;
+  constructor(code: string, message: string) {
+    super(message);
+    this.name = "HistoryIndexError";
+    this.code = code;
+  }
+}
+export interface Request {
+  readonly id: number;
+  readonly method: string;
+  readonly input?: unknown;
+}
+export type Response =
+  | {
+      readonly id: number;
+      readonly value?: unknown;
+      readonly error?: { readonly code: string; readonly message: string };
+    }
+  | { readonly changed: true };