@timqi/pier 0.0.8 → 0.0.15

package/README.md

@@ -19,7 +19,7 @@ versioned from `0.0.1` on — earlier databases are not migrated. Read
 - Node 24 or newer (`node:sqlite` is used unflagged)
 - Linux: Python 3, `make` and a C/C++ compiler for `node-pty`
 - A provider account (Anthropic, OpenAI, …) — configure its API key or OAuth
-  login from Console → Settings → Providers after signing in
+  login from Console → Settings → Models after signing in
 - A user-writable global npm prefix if `pier update` should update a service
 - Optional: the `sqlite3` CLI, for backups and password resets
 
@@ -43,10 +43,10 @@ is behind it — there is no default password and no unclaimed window. Lost it?
 Open `http://localhost:3141`, sign in, then:
 
 - **Console → Settings** — everything the instance is configured with, one
-  tab per topic: Providers (API-key/OAuth logins), Models (the menu of
-  favored models agents are advised with), Channels (Slack Socket-Mode or
-  Telegram bot tokens; chats are discovered when the bot first sees traffic,
-  gated by the mention/bind rules you set), Agent files (Pi configuration),
+  tab per topic: Models (API-key/OAuth logins, and the menu of favored models
+  agents are advised with), Agent (Pi configuration, skills, extensions),
+  Channels (Slack Socket-Mode or Telegram bot tokens; chats are discovered
+  when the bot first sees traffic, gated by the mention/bind rules you set),
   plus the public URL, password and master key
 - **New session** — pick a directory; that is where the agent's shell runs
 
@@ -61,17 +61,34 @@ an existing Pi setup:
 PI_CODING_AGENT_DIR="$HOME/.pi/agent" pier serve
 ```
 
+Pier exports that variable for the SDK, so anything it starts inherits it — a
+shell in the Web Terminal included. A second Pier launched from there with its
+own `PIER_HOME` derives its own agent directory rather than adopting the first
+one's; setting `PI_CODING_AGENT_DIR` again on that command line still wins.
+
 Console → Settings is the normal setup path:
 
-- **Providers** configures built-in or custom endpoints and API-key/OAuth login.
-  Stored credentials are sealed in Pier's SQLite database; they are not written
-  back to `models.json`.
-- **Agent files** edits `SYSTEM.md`, `AGENTS.md`, `settings.json`, and advanced
+- **Models** configures built-in or custom endpoints and API-key/OAuth login,
+  then pins the few models this deployment favors. Stored credentials are
+  sealed in Pier's SQLite database; they are not written back to `models.json`.
+  **Test** sends one real request on a model you pick and shows both halves of
+  it — the body as the provider received it, and what came back — so a wrong
+  base URL, a revoked key, a gateway rewriting the request or a model the
+  endpoint never had says so here instead of in a session.
+- **Agent** edits `SYSTEM.md`, `AGENTS.md`, `settings.json`, and advanced
   `models.json` structure in the Pi agent directory — globally, or per project
   scope, where it also shows that project's `.pi/skills` and `.pi/extensions`
   resources. Changes apply when a session next opens; saving here recycles the
   idle ones for you, and **Settings → Instance → Reload** does it for files
   something else changed — an agent, or an editor on the box.
+  The same tab lists the extensions Pier ships with, under Global — they live
+  inside the package, so there is nothing to install and an update never
+  touches your own `extensions` directory, and if an extension of yours
+  already registers the same tool, Pier's copy stands down. `web` is the one
+  shipped today: the public web through the provider's own hosted web tools —
+  `web_search` on an authenticated Anthropic or OpenAI model, `web_fetch` on an
+  Anthropic one (OpenAI hosts no fetch tool), and no other key or service. The
+  page says which tools a switch adds and what each needs before you flip it.
 
 On first credential access, Pier imports an existing `auth.json` into its sealed
 store and renames the source to `auth.json.imported`. Literal provider keys left

package/dist/agent/events.js

@@ -2,7 +2,7 @@
 // purpose: no @earendil-works/pi-* imports, so it stays unit-testable without Pi
 // and Pi types never leak past the seam. The golden-table test in
 // events.test.ts is the mapping's spec; extend types.ts before adding events.
-import { MAX_STEP_OUTPUT } from "../core/types.js";
+import { isThinkingLevel, MAX_STEP_OUTPUT } from "../core/types.js";
 export function textOf(content) {
     if (typeof content === "string")
         return content;
@@ -19,21 +19,44 @@ function systemOrigin(message) {
     const value = message.details;
     if (!value || typeof value !== "object")
         return null;
-    const origin = value;
+    const { source: raw, ...origin } = value;
     if (typeof origin.taskId !== "string" ||
         typeof origin.runId !== "string" ||
         (origin.sourceSessionId !== null && typeof origin.sourceSessionId !== "string"))
         return null;
+    // Rebuilt around a checked `source` rather than cast through it: a
+    // half-valid one drawn by the card is an `undefined` in a chip, which reads
+    // as a bug in the card rather than as bad metadata.
+    const source = inputSource(raw);
+    const shape = { ...origin, ...(source ? { source } : {}) };
     if (origin.kind === "task-delegation" || origin.kind === "task-callback") {
-        return origin;
+        return shape;
     }
     if (origin.kind === "task-message" &&
         typeof origin.messageId === "string" &&
         (origin.messageKind === "steer" || origin.messageKind === "follow_up" ||
             origin.messageKind === "progress" || origin.messageKind === "decision" || origin.messageKind === "reply"))
-        return origin;
+        return shape;
     return null;
 }
+/** What produced a system input, as read back off disk: the name is the whole
+ *  point of it, the model and the effort are each kept only if they are the
+ *  shape they claim to be (core/types.ts). */
+function inputSource(value) {
+    if (!value || typeof value !== "object")
+        return undefined;
+    const { taskName, model, thinking } = value;
+    if (typeof taskName !== "string")
+        return undefined;
+    const ref = (model ?? {});
+    return {
+        taskName,
+        ...(typeof ref.provider === "string" && typeof ref.id === "string"
+            ? { model: { provider: ref.provider, id: ref.id } }
+            : {}),
+        ...(isThinkingLevel(thinking) ? { thinking } : {}),
+    };
+}
 function lastAssistant(messages) {
     if (!messages)
         return undefined;
@@ -86,12 +109,17 @@ export function toChatTurns(messages) {
     const turns = [];
     let steps = []; // activity seen since the last emitted turn
     const pendingTools = new Map();
-    const flush = (role, text, meta, origin) => {
+    const flush = (role, text, meta, origin, at) => {
         const turn = { role, text };
         if (meta)
             turn.meta = meta;
         if (origin)
             turn.origin = origin;
+        // An assistant turn already says when it finished; these two would have no
+        // time at all after a reload, which is the one place the live stream's own
+        // stamp is gone.
+        if (at !== undefined && role !== "assistant")
+            turn.at = at;
         if (steps.length) {
             turn.steps = steps;
             steps = [];
@@ -116,7 +144,7 @@ export function toChatTurns(messages) {
         if (origin) {
             const text = textOf(m.content);
             if (text)
-                flush("system", text, undefined, origin);
+                flush("system", text, undefined, origin, m.timestamp);
             continue;
         }
         if (m.role !== "user" && m.role !== "assistant")
@@ -143,7 +171,7 @@ export function toChatTurns(messages) {
         // step-only assistant messages keep buffering activity
         if (!text)
             continue;
-        flush(m.role, text, m.role === "assistant" ? turnMetaAt(messages, i) : undefined);
+        flush(m.role, text, m.role === "assistant" ? turnMetaAt(messages, i) : undefined, undefined, m.timestamp);
     }
     // Activity with no answer after it (aborted run) still belongs on the page.
     if (steps.length)
@@ -207,6 +235,24 @@ export function toSessionEvents(e) {
                     followUp: [...(e.followUp ?? [])],
                 },
             ];
+        case "compaction_end": {
+            // The only trace compaction leaves anywhere: Pi replaces the summarized
+            // entries with a `compactionSummary` message, which `toChatTurns` above
+            // renders nothing for — so without this event the button's effect is
+            // invisible and the automatic one is invisible twice over (§5b).
+            const r = e.result;
+            if (r && typeof r.tokensBefore === "number") {
+                return [{
+                        type: "context-compacted",
+                        before: r.tokensBefore,
+                        after: r.estimatedTokensAfter ?? r.tokensBefore,
+                    }];
+            }
+            // No result means the context was *not* shrunk — cancelled, or the
+            // summarization call failed. A manual compact reports through its route
+            // as well; an automatic one has no route, and this is all it has.
+            return [{ type: "error", message: e.errorMessage ?? "compaction cancelled" }];
+        }
         case "tool_execution_end":
             return [
                 {

package/dist/agent/listing.js

@@ -0,0 +1,253 @@
+// Which sessions exist on disk, and what to call them — without reading a
+// transcript that has not changed since the last scan.
+//
+// Pi's SessionManager.listAll() parses every line of every session file to
+// answer that: ~250ms for 84 sessions of 30MB here, growing with the total
+// bytes ever written rather than with the number of sessions. Every surface
+// asks (the rail, Activity, a task lookup, every reconnect), so the scan sits
+// on the interactive path and gets slower for the life of the instance. But a
+// transcript is append-only and the facts a listing needs are all derived from
+// bytes already read, so each byte is read once: rows live in pier.db, keyed by
+// path and validated by (size, mtime), and a file that grew is picked up at the
+// byte the last scan stopped on.
+//
+// Reading Pi's on-disk entries is the cost of that; agent/ is where the
+// knowledge of Pi's formats already lives, and Pi's own reader is not
+// incremental.
+import { createReadStream, promises as fs } from "node:fs";
+import { join } from "node:path";
+import { pierDb } from "../db.js";
+import { SESSION_TITLE_MAX } from "../limits.js";
+import { logger } from "../log.js";
+import { defaultAgentDir } from "./config.js";
+import { textOf } from "./events.js";
+const log = logger("agent");
+/** How many of the newest sessions a cross-check reads: enough that a format
+ *  change surfaces within a boot or two, few enough to cost nothing. */
+const AUDIT_SAMPLE = 5;
+/** The name a session was given, else its first message. Two arguments rather
+ *  than a row, because the stored row and the parsed one spell the second one
+ *  differently and an object would silently accept either. */
+const titleOf = (name, first) => name || first || undefined;
+const str = (v) => (typeof v === "string" ? v : undefined);
+/**
+ * One entry folded into what a listing needs. Three answers, not two:
+ * `undefined` is "no header yet" (blank and unparseable lines before it are
+ * skipped, as Pi's own reader does), `null` is "not a session file" — a first
+ * entry that is not a session header — and anything else is the state so far.
+ */
+function fold(acc, line) {
+    // The bulk of a transcript is message entries, each longer than this file.
+    // Once the first user message is in hand only a rename still matters, and a
+    // substring test costs a fraction of parsing one of them.
+    if (acc?.first && !line.includes('"session_info"'))
+        return acc;
+    let entry;
+    try {
+        const value = JSON.parse(line);
+        if (value && typeof value === "object")
+            entry = value;
+    }
+    catch {
+        return acc ?? undefined;
+    }
+    if (!acc) {
+        if (!entry)
+            return undefined;
+        const id = str(entry.id);
+        const timestamp = str(entry.timestamp);
+        if (entry.type !== "session" || !id || !timestamp)
+            return null;
+        return { id, cwd: str(entry.cwd) ?? "", created: Date.parse(timestamp), at: 0 };
+    }
+    if (entry?.type === "session_info")
+        return { ...acc, name: str(entry.name)?.trim() || undefined };
+    if (entry?.type === "message" && !acc.first) {
+        const message = entry.message;
+        if (message?.role !== "user")
+            return acc;
+        const text = textOf(message.content).trim();
+        return text ? { ...acc, first: text.slice(0, SESSION_TITLE_MAX) } : acc;
+    }
+    return acc;
+}
+/** The listing Pier runs on: Pi's session directory, remembered in pier.db. */
+export class IndexedListing {
+    dir;
+    #db;
+    constructor(dir = join(defaultAgentDir(), "sessions"), db) {
+        this.dir = dir;
+        this.#db = db;
+    }
+    /** Opened on the first scan rather than in the constructor: building an
+     *  agent factory must not open the instance database — tests build bare ones
+     *  and never list. */
+    #store() {
+        return (this.#db ??= pierDb());
+    }
+    async scan() {
+        const db = this.#store();
+        const known = new Map(db.prepare("SELECT * FROM session_index").all().map((row) => [row.path, row]));
+        const records = [];
+        const write = [];
+        for (const file of await this.#files()) {
+            const row = known.get(file.path);
+            known.delete(file.path); // what is left over is no longer on disk
+            if (row && row.size === file.size && row.mtime === file.modified) {
+                const title = titleOf(row.name, row.first_message);
+                records.push({
+                    id: row.id,
+                    path: row.path,
+                    cwd: row.cwd,
+                    created: row.created_at,
+                    modified: row.mtime,
+                    ...(title ? { title } : {}),
+                });
+                continue;
+            }
+            // Only a file that *grew* can be resumed mid-way. Anything else is read
+            // whole — new, truncated, or rewritten. Rewritten includes the same
+            // length with a different mtime, which resuming would answer with the
+            // old derived data stamped with the new mtime: a row that then matches on
+            // every later scan and is never corrected.
+            const parsed = await this.#read(file.path, row && file.size > row.size
+                ? {
+                    id: row.id,
+                    cwd: row.cwd,
+                    created: row.created_at,
+                    ...(row.name === null ? {} : { name: row.name }),
+                    ...(row.first_message === null ? {} : { first: row.first_message }),
+                    at: row.parsed_bytes,
+                }
+                : undefined);
+            if (!parsed)
+                continue; // not a session file, or unreadable
+            write.push({
+                path: file.path,
+                id: parsed.id,
+                cwd: parsed.cwd,
+                created_at: parsed.created,
+                name: parsed.name ?? null,
+                first_message: parsed.first ?? null,
+                size: file.size,
+                mtime: file.modified,
+                parsed_bytes: parsed.at,
+            });
+            const title = titleOf(parsed.name, parsed.first);
+            records.push({
+                id: parsed.id,
+                path: file.path,
+                cwd: parsed.cwd,
+                created: parsed.created,
+                modified: file.modified,
+                ...(title ? { title } : {}),
+            });
+        }
+        this.#save(write, [...known.keys()]);
+        return records.sort((a, b) => b.modified - a.modified);
+    }
+    /**
+     * The reason this file is allowed to exist: it parses Pi's transcripts
+     * itself, and nothing but a comparison keeps it honest when that format
+     * moves. Reads the newest few both ways; a disagreement is logged with both
+     * answers and the index row is dropped, so the next scan reads that file
+     * whole rather than trusting what this one derived. Returns how many
+     * disagreed — off the interactive path, never blocking a boot.
+     */
+    async audit(native) {
+        const sample = (await this.scan()).slice(0, AUDIT_SAMPLE);
+        if (!sample.length)
+            return 0;
+        const theirs = new Map((await native()).map((info) => [info.id, info]));
+        const stale = [];
+        for (const mine of sample) {
+            const info = theirs.get(mine.id);
+            const seen = { cwd: mine.cwd, created: mine.created, title: mine.title };
+            const want = info && {
+                cwd: info.cwd,
+                created: info.created.getTime(),
+                title: info.name ?? info.firstMessage?.slice(0, SESSION_TITLE_MAX),
+            };
+            if (want && want.cwd === seen.cwd && want.created === seen.created &&
+                want.title === seen.title)
+                continue;
+            log.warn(`session index disagrees with Pi about ${mine.id}: index ${JSON.stringify(seen)}, Pi ${JSON.stringify(want ?? "no such session")} — dropping ${mine.path}`);
+            stale.push(mine.path);
+        }
+        this.#save([], stale);
+        return stale.length;
+    }
+    /** Every session file with its size and mtime — the only syscalls a scan
+     *  where nothing changed makes. */
+    async #files() {
+        const entries = await fs.readdir(this.dir, { withFileTypes: true }).catch(() => []);
+        const dirs = entries.filter((e) => e.isDirectory() || e.isSymbolicLink());
+        const found = await Promise.all(dirs.map(async (dir) => {
+            const at = join(this.dir, dir.name);
+            const names = await fs.readdir(at).catch(() => []);
+            return Promise.all(names.filter((n) => n.endsWith(".jsonl")).map(async (name) => {
+                const path = join(at, name);
+                const stat = await fs.stat(path).catch(() => null);
+                return stat?.isFile()
+                    ? { path, size: stat.size, modified: Math.round(stat.mtimeMs) }
+                    : null;
+            }));
+        }));
+        return found.flat().filter((f) => !!f);
+    }
+    /** From `from.at`, or from the start when there is nothing to resume. */
+    async #read(path, from) {
+        let acc = from;
+        let at = from?.at ?? 0;
+        let buffer = "";
+        try {
+            for await (const chunk of createReadStream(path, { encoding: "utf8", start: at })) {
+                buffer += chunk;
+                for (let nl = buffer.indexOf("\n"); nl !== -1; nl = buffer.indexOf("\n")) {
+                    const line = buffer.slice(0, nl);
+                    buffer = buffer.slice(nl + 1);
+                    at += Buffer.byteLength(line) + 1;
+                    const next = fold(acc, line);
+                    if (next === null)
+                        return null;
+                    acc = next;
+                }
+            }
+        }
+        catch (err) {
+            // A file that cannot be read is left out of this listing and out of the
+            // index, so the next scan tries it again rather than remembering a gap.
+            log.warn(`session file ${path} could not be read`, err);
+            return null;
+        }
+        return acc ? { ...acc, at } : null;
+    }
+    /** One transaction, after all reading: a half-written index would hand the
+     *  next scan a size it never parsed to. */
+    #save(rows, gone) {
+        if (!rows.length && !gone.length)
+            return;
+        const db = this.#store();
+        const upsert = db.prepare(`INSERT INTO session_index(path, id, cwd, created_at, name, first_message, size, mtime, parsed_bytes)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+       ON CONFLICT(path) DO UPDATE SET
+         id = excluded.id, cwd = excluded.cwd, created_at = excluded.created_at,
+         name = excluded.name, first_message = excluded.first_message,
+         size = excluded.size, mtime = excluded.mtime,
+         parsed_bytes = excluded.parsed_bytes`);
+        const drop = db.prepare("DELETE FROM session_index WHERE path = ?");
+        db.exec("BEGIN");
+        try {
+            for (const r of rows) {
+                upsert.run(r.path, r.id, r.cwd, r.created_at, r.name, r.first_message, r.size, r.mtime, r.parsed_bytes);
+            }
+            for (const path of gone)
+                drop.run(path);
+            db.exec("COMMIT");
+        }
+        catch (err) {
+            db.exec("ROLLBACK");
+            throw err;
+        }
+    }
+}