@timqi/pier 0.0.29 → 0.1.1

package/dist/agent/listing.js

@@ -1,46 +1,40 @@
-// Which sessions exist on disk, and what to call them — without reading a
-// transcript that has not changed since the last scan.
-//
-// Pi's SessionManager.listAll() parses every line of every session file to
-// answer that: ~250ms for 84 sessions of 30MB here, growing with the total
-// bytes ever written rather than with the number of sessions. Every surface
-// asks (the rail, Activity, a task lookup, every reconnect), so the scan sits
-// on the interactive path and gets slower for the life of the instance. But a
-// transcript is append-only and the facts a listing needs are all derived from
-// bytes already read, so each byte is read once: rows live in pier.db, keyed by
-// path and validated by (size, mtime), and a file that grew is picked up at the
-// byte the last scan stopped on.
-//
-// Reading Pi's on-disk entries is the cost of that; agent/ is where the
-// knowledge of Pi's formats already lives, and Pi's own reader is not
-// incremental.
+// Which sessions exist on disk and what to call them, reading each transcript
+// byte once: Pi's own listing parses every file whole (~250ms for 30MB) and
+// every surface asks. Rows live in pier.db keyed by path and validated by
+// (size, mtime); a file that grew resumes where the last scan stopped. The same
+// pass indexes what was said into `session_fts` for the palette.
 import { createReadStream, promises as fs } from "node:fs";
 import { join } from "node:path";
 import { pierDb, statements, transact } from "../db.js";
-import { SESSION_TITLE_MAX } from "../limits.js";
+import { SESSION_TITLE_MAX } from "../core/types.js";
 import { logger } from "../log.js";
 import { defaultAgentDir } from "./config.js";
-import { textOf } from "./events.js";
+import { hasToolCalls, textOf } from "./events.js";
 const log = logger("agent");
-/** How many of the newest sessions a cross-check reads: enough that a format
- *  change surfaces within a boot or two, few enough to cost nothing. */
+/** Enough that a format change surfaces within a boot or two. */
 const AUDIT_SAMPLE = 5;
-/** The name a session was given, else its first message. Two arguments rather
- *  than a row, because the stored row and the parsed one spell the second one
- *  differently and an object would silently accept either. */
 const titleOf = (name, first) => name || first || undefined;
 const str = (v) => (typeof v === "string" ? v : undefined);
-/**
- * One entry folded into what a listing needs. Three answers, not two:
- * `undefined` is "no header yet" (blank and unparseable lines before it are
- * skipped, as Pi's own reader does), `null` is "not a session file" — a first
- * entry that is not a session header — and anything else is the state so far.
- */
-function fold(acc, line) {
-    // The bulk of a transcript is message entries, each longer than this file.
-    // Once the first user message is in hand only a rename still matters, and a
-    // substring test costs a fraction of parsing one of them.
-    if (acc?.first && !line.includes('"session_info"'))
+/** A reply that calls a tool is a step, not a reply (events.ts), and steps are
+ *  not indexed. `clean` takes off the speaker header (core/identity.ts), which
+ *  as text would make "operator" hit every session. */
+function saidIn(message, entryAt, clean) {
+    if (message?.role !== "user" && message?.role !== "assistant")
+        return undefined;
+    if (message.role === "assistant" && hasToolCalls(message))
+        return undefined;
+    const text = clean(textOf(message.content)).trim();
+    if (!text)
+        return undefined;
+    const at = typeof message.timestamp === "number" ? message.timestamp : Date.parse(entryAt ?? "");
+    return { role: message.role, at: Number.isFinite(at) ? at : 0, text };
+}
+/** `undefined` is "no header yet" (leading junk is skipped, as Pi's reader
+ *  does), `null` is "not a session file". */
+function fold(acc, line, clean) {
+    // The bulk of a transcript is tool results; two substring tests cost a
+    // fraction of parsing one.
+    if (acc && !line.includes('"message"') && !line.includes('"session_info"'))
         return acc;
     let entry;
     try {
@@ -49,6 +43,7 @@ function fold(acc, line) {
             entry = value;
     }
     catch {
+        // The tail of a transcript Pi is still appending to; the next scan reads it whole.
         return acc ?? undefined;
     }
     if (!acc) {
@@ -58,35 +53,57 @@ function fold(acc, line) {
         const timestamp = str(entry.timestamp);
         if (entry.type !== "session" || !id || !timestamp)
             return null;
-        return { id, cwd: str(entry.cwd) ?? "", created: Date.parse(timestamp), at: 0 };
+        return { id, cwd: str(entry.cwd) ?? "", created: Date.parse(timestamp), said: [], at: 0 };
     }
     if (entry?.type === "session_info")
         return { ...acc, name: str(entry.name)?.trim() || undefined };
-    if (entry?.type === "message" && !acc.first) {
+    if (entry?.type === "message") {
         const message = entry.message;
-        if (message?.role !== "user")
+        const said = saidIn(message, str(entry.timestamp), clean);
+        if (!said)
+            return acc;
+        acc.said.push(said);
+        // The title keeps the header the index drops; surfaces read it back
+        // through core/identity.ts.
+        if (acc.first || said.role !== "user")
             return acc;
-        const text = textOf(message.content).trim();
-        return text ? { ...acc, first: text.slice(0, SESSION_TITLE_MAX) } : acc;
+        return { ...acc, first: textOf(message?.content).trim().slice(0, SESSION_TITLE_MAX) };
     }
     return acc;
 }
-/** The listing Pier runs on: Pi's session directory, remembered in pier.db. */
+/** A picker, not a results page. */
+const SEARCH_LIMIT = 8;
+/** A trigram token is one character, so `snippet()` is asked for its ceiling
+ *  of 64 tokens; the default 12 fits one word. */
+const SNIPPET_CHARS = 64;
+/** Delimited like `snippet()` (\u0001 … \u0002, `…` for a cut), so a surface
+ *  draws one shape for both paths. ASCII case folding: LIKE found the row by
+ *  the same rule. */
+function around(text, query) {
+    const hit = text.toLowerCase().indexOf(query.toLowerCase());
+    if (hit < 0)
+        return text.slice(0, SNIPPET_CHARS);
+    const start = Math.max(0, hit - SNIPPET_CHARS / 2);
+    const end = Math.min(text.length, hit + query.length + SNIPPET_CHARS / 2);
+    return `${start ? "…" : ""}${text.slice(start, hit)}\u0001${text.slice(hit, hit + query.length)}\u0002${text.slice(hit + query.length, end)}${end < text.length ? "…" : ""}`;
+}
 export class IndexedListing {
     dir;
+    clean;
     #db;
     #statements;
-    constructor(dir = join(defaultAgentDir(), "sessions"), db) {
+    constructor(dir = join(defaultAgentDir(), "sessions"), db, 
+    /** Handed in rather than imported: the header rule is core's, and agent/
+     *  does not import core at runtime. */
+    clean = (text) => text) {
         this.dir = dir;
+        this.clean = clean;
         this.#db = db;
     }
-    /** Opened on the first scan rather than in the constructor: building an
-     *  agent factory must not open the instance database — tests build bare ones
-     *  and never list. */
+    /** Opened on the first scan: building a factory must not open the database. */
     #store() {
         return (this.#db ??= pierDb());
     }
-    /** The scan's three statements, compiled on the first scan and not again. */
     #sql() {
         return (this.#statements ??= statements(this.#store()));
     }
@@ -109,18 +126,17 @@ export class IndexedListing {
                 });
                 continue;
             }
-            // Only a file that *grew* can be resumed mid-way. Anything else is read
-            // whole — new, truncated, or rewritten. Rewritten includes the same
-            // length with a different mtime, which resuming would answer with the
-            // old derived data stamped with the new mtime: a row that then matches on
-            // every later scan and is never corrected.
-            const parsed = await this.#read(file.path, row && file.size > row.size
+            // Only a file that grew can be resumed. Same length with a new mtime is
+            // rewritten, and resuming would stamp old data with the new mtime forever.
+            const resumed = row !== undefined && file.size > row.size;
+            const parsed = await this.#read(file.path, resumed
                 ? {
                     id: row.id,
                     cwd: row.cwd,
                     created: row.created_at,
                     ...(row.name === null ? {} : { name: row.name }),
                     ...(row.first_message === null ? {} : { first: row.first_message }),
+                    said: [],
                     at: row.parsed_bytes,
                 }
                 : undefined);
@@ -136,6 +152,8 @@ export class IndexedListing {
                 size: file.size,
                 mtime: file.modified,
                 parsed_bytes: parsed.at,
+                said: parsed.said,
+                whole: !resumed,
             });
             const title = titleOf(parsed.name, parsed.first);
             records.push({
@@ -150,14 +168,9 @@ export class IndexedListing {
         this.#save(write, [...known.keys()]);
         return records.sort((a, b) => b.modified - a.modified);
     }
-    /**
-     * The reason this file is allowed to exist: it parses Pi's transcripts
-     * itself, and nothing but a comparison keeps it honest when that format
-     * moves. Reads the newest few both ways; a disagreement is logged with both
-     * answers and the index row is dropped, so the next scan reads that file
-     * whole rather than trusting what this one derived. Returns how many
-     * disagreed — off the interactive path, never blocking a boot.
-     */
+    /** This file parses Pi's transcripts itself, and only a comparison keeps it
+     *  honest when the format moves. A disagreement drops the index row, so the
+     *  next scan reads that file whole. */
     async audit(native) {
         const sample = (await this.scan()).slice(0, AUDIT_SAMPLE);
         if (!sample.length)
@@ -181,8 +194,33 @@ export class IndexedListing {
         this.#save([], stale);
         return stale.length;
     }
-    /** Every session file with its size and mtime — the only syscalls a scan
-     *  where nothing changed makes. */
+    /** Under three code points the trigram tokenizer has nothing to match, and a
+     *  two-character query is what a CJK word often is: substring scan instead. */
+    search(query, limit = SEARCH_LIMIT) {
+        const sql = this.#sql();
+        const rows = [...query].length >= 3
+            // Quoted: the query is a string to find, never FTS syntax.
+            ? sql(`SELECT session_id, role, at, snippet(session_fts, 0, char(1), char(2), '…', ${SNIPPET_CHARS}) AS snippet
+         FROM session_fts WHERE text MATCH ? ORDER BY bm25(session_fts), at DESC`).iterate(`"${query.replaceAll('"', '""')}"`)
+            : sql(`SELECT session_id, role, at, text FROM session_fts WHERE text LIKE ? ESCAPE '\\' ORDER BY at DESC`).iterate(`%${query.replaceAll(/[\\%_]/g, "\\$&")}%`);
+        const hits = [];
+        const seen = new Set();
+        // Walked, not fetched: a chatty session has hundreds of rows for one hit.
+        for (const row of rows) {
+            if (seen.has(row.session_id))
+                continue;
+            seen.add(row.session_id);
+            hits.push({
+                sessionId: row.session_id,
+                role: row.role,
+                at: row.at,
+                snippet: row.snippet ?? around(row.text ?? "", query),
+            });
+            if (hits.length >= limit)
+                break;
+        }
+        return hits;
+    }
     async #files() {
         const entries = await fs.readdir(this.dir, { withFileTypes: true }).catch(() => []);
         const dirs = entries.filter((e) => e.isDirectory() || e.isSymbolicLink());
@@ -199,7 +237,6 @@ export class IndexedListing {
         }));
         return found.flat().filter((f) => !!f);
     }
-    /** From `from.at`, or from the start when there is nothing to resume. */
     async #read(path, from) {
         let acc = from;
         let at = from?.at ?? 0;
@@ -211,7 +248,7 @@ export class IndexedListing {
                     const line = buffer.slice(0, nl);
                     buffer = buffer.slice(nl + 1);
                     at += Buffer.byteLength(line) + 1;
-                    const next = fold(acc, line);
+                    const next = fold(acc, line, this.clean);
                     if (next === null)
                         return null;
                     acc = next;
@@ -219,20 +256,22 @@ export class IndexedListing {
             }
         }
         catch (err) {
-            // A file that cannot be read is left out of this listing and out of the
-            // index, so the next scan tries it again rather than remembering a gap.
+            // Left out of the index too, so the next scan tries it again.
             log.warn(`session file ${path} could not be read`, err);
             return null;
         }
         return acc ? { ...acc, at } : null;
     }
-    /** One transaction, after all reading: a half-written index would hand the
-     *  next scan a size it never parsed to. */
+    /** One transaction after all reading: a half-written index would hand the
+     *  next scan a size it never parsed to. A whole read replaces what the file
+     *  had said; a resumed read only adds. */
     #save(rows, gone) {
         if (!rows.length && !gone.length)
             return;
         const db = this.#store();
         const sql = this.#sql();
+        const forget = sql("DELETE FROM session_fts WHERE path = ?");
+        const say = sql("INSERT INTO session_fts(text, session_id, path, role, at) VALUES (?, ?, ?, ?, ?)");
         const upsert = sql(`INSERT INTO session_index(path, id, cwd, created_at, name, first_message, size, mtime, parsed_bytes)
        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
        ON CONFLICT(path) DO UPDATE SET
@@ -243,10 +282,16 @@ export class IndexedListing {
         const drop = sql("DELETE FROM session_index WHERE path = ?");
         transact(db, () => {
             for (const r of rows) {
+                if (r.whole)
+                    forget.run(r.path);
+                for (const s of r.said)
+                    say.run(s.text, r.id, r.path, s.role, s.at);
                 upsert.run(r.path, r.id, r.cwd, r.created_at, r.name, r.first_message, r.size, r.mtime, r.parsed_bytes);
             }
-            for (const path of gone)
+            for (const path of gone) {
                 drop.run(path);
+                forget.run(path);
+            }
         });
     }
 }