@thurstonsand/pi-librarian 0.2.0 → 0.2.1

package/README.md

@@ -2,6 +2,10 @@
 
 A GitHub research subagent for the [pi coding agent](https://github.com/badlogic/pi-mono) inspired by [Amp](https://ampcode.com/): deep-dive questions about specific repos ("how does drizzle-orm implement prepared statements?") and discovery across the ecosystem ("compare the most popular TypeScript SQL ORMs").
 
+![calling the librarian tool](images/librarian-run.png)
+
+![librarian tool results](images/librarian-results.png)
+
 ## How it works
 
 The `librarian` tool spawns a nested research agent with purpose-built tools:
@@ -39,17 +43,19 @@ In pi's global `settings.json`:
     "tools": ["search_web", "fetch_web"],
     "extensions": ["~/.pi/agent/extensions/parallel-web-tools"],
     "cacheDir": "/tmp/pi-librarian",
+    "debug": { "persistRuns": false },
   },
 }
 ```
 
-| Setting         | Recommended                                   | Default                        |
-| --------------- | --------------------------------------------- | ------------------------------ |
-| `model`         | `openai-codex/gpt-5.5`                        | current session model          |
-| `thinkingLevel` | `off`                                         | current session thinking level |
-| `tools`         | names of extra tools to activate, when needed | `[]`                           |
-| `extensions`    | escape hatch paths for tools not loaded in pi | `[]`                           |
-| `cacheDir`      | `/tmp/pi-librarian`                           | `/tmp/pi-librarian`            |
+| Setting             | Recommended                                     | Default                        |
+| ------------------- | ----------------------------------------------- | ------------------------------ |
+| `model`             | `openai-codex/gpt-5.5`                          | current session model          |
+| `thinkingLevel`     | `off`                                           | current session thinking level |
+| `tools`             | names of extra tools to activate, when needed   | `[]`                           |
+| `extensions`        | escape hatch paths for tools not loaded in pi   | `[]`                           |
+| `cacheDir`          | `/tmp/pi-librarian`                             | `/tmp/pi-librarian`            |
+| `debug.persistRuns` | persist nested session file paths for debugging | `false`                        |
 
 `librarian.tools` is the activation gate for extra tools. `librarian.extensions` only adds extension paths to the search space when a named tool is not already loaded in the main pi session; listing an extension path does not activate every tool in that bundle. Librarian runs exclude `write` and `edit`.
 

package/RELEASE.md

@@ -2,6 +2,19 @@
 
 # Release notes
 
+## 0.2.1
+
+Fixes TUI rendering corruption.
+
+### Added
+
+- Added `librarian.debug.persistRuns` to keep nested run session files around for debugging.
+
+### Fixed
+
+- Fixed librarian trace rendering leaving stale "N earlier calls" duplicates in the TUI, caused by unsilenced `@octokit/request` deprecation warnings writing to the console.
+- Fixed `search_code` retries riding out a fixed 30s-per-attempt budget; retries now use an escalating timeout schedule so a stalled request recovers in seconds instead of up to a minute.
+
 ## 0.2.0
 
 Adds continuable librarian runs and opt-in for extra tools.

package/extensions/librarian/github.ts

@@ -45,18 +45,26 @@ export async function resolveGitHubToken(): Promise<string | undefined> {
   }
 }
 
+// Any console output corrupts pi's interactive TUI (its differential renderer
+// cannot recover from writes it did not make), so octokit must never log.
+// The top-level `log` alone is not enough: @octokit/request resolves its logger
+// from the per-request options (`request.log || console`) when emitting API
+// deprecation warnings, so the silent logger has to be passed at both levels.
+const SILENT_OCTOKIT_LOG = {
+  debug: () => {},
+  info: () => {},
+  warn: () => {},
+  error: () => {},
+};
+
 export function createGitHubClient(token: string | undefined): GitHubClient {
   return new GitHubClient(
     new Octokit({
       ...(token ? { auth: token } : {}),
       userAgent: "pi-librarian",
-      log: {
-        debug: () => {},
-        info: () => {},
-        warn: () => {},
-        error: () => {},
-      },
+      log: SILENT_OCTOKIT_LOG,
       request: {
+        log: SILENT_OCTOKIT_LOG,
         headers: {
           "x-github-api-version": "2022-11-28",
         },

package/extensions/librarian/grep-app.ts

@@ -2,7 +2,10 @@ import { Type } from "typebox";
 import { safeParseTypeBoxValue } from "../shared/typebox.ts";
 
 const GREP_MCP_URL = "https://mcp.grep.app";
-const GREP_SEARCH_TIMEOUT_MS = 30_000;
+// Escalating per-attempt budgets: the upstream gateway kills stalled requests at ~15s, so
+// early attempts abort fast (p90 success is ~500ms) instead of waiting out the full timeout.
+const GREP_ATTEMPT_TIMEOUTS_MS = [3_000, 5_000, 7_000];
+const GREP_RETRY_DELAY_MS = 300;
 
 export class GrepError extends Error {
   constructor(message: string) {
@@ -159,7 +162,12 @@ export function parseGrepMcpEvents(body: string): GrepCodeSearchResult {
       continue;
     }
     if (result.isError) {
-      throw new GrepError("Grep MCP returned an error result.");
+      const message = (result.content ?? [])
+        .map((rawContent) => safeParseTypeBoxValue(MCP_TEXT_CONTENT_SCHEMA, rawContent)?.text)
+        .filter((text): text is string => text !== undefined)
+        .join("\n")
+        .trim();
+      throw new GrepError(message || "Grep MCP returned an error result.");
     }
 
     for (const rawContent of result.content ?? []) {
@@ -203,11 +211,45 @@ function buildGrepArguments(params: GrepCodeSearchParams): Record<string, unknow
   return args;
 }
 
-export async function searchCodeGrep(
+function extractHtmlTitle(body: string): string | undefined {
+  const match = /<title[^>]*>(.*?)<\/title>/is.exec(body);
+  return match?.[1]?.replace(/\s+/g, " ").trim();
+}
+
+function formatHttpError(status: number, body: string): string {
+  return `Grep MCP returned ${status}: ${extractHtmlTitle(body) ?? body.slice(0, 200)}`;
+}
+
+function formatTransientError(error: unknown): string {
+  const cause =
+    error instanceof GrepTransientHttpError ? `returned ${error.status}` : "request failed";
+  return `Grep MCP ${cause} after retry (transient backend error). Retry, or simplify the query.`;
+}
+
+function delay(milliseconds: number, signal: AbortSignal | undefined): Promise<void> {
+  if (signal?.aborted) {
+    return Promise.reject(signal.reason);
+  }
+
+  return new Promise((resolve, reject) => {
+    const timeout = setTimeout(resolve, milliseconds);
+    signal?.addEventListener(
+      "abort",
+      () => {
+        clearTimeout(timeout);
+        reject(signal.reason);
+      },
+      { once: true },
+    );
+  });
+}
+
+async function searchCodeGrepAttempt(
   params: GrepCodeSearchParams,
   signal: AbortSignal | undefined,
+  timeoutMs: number,
 ): Promise<GrepCodeSearchResult> {
-  const timeoutSignal = AbortSignal.timeout(GREP_SEARCH_TIMEOUT_MS);
+  const timeoutSignal = AbortSignal.timeout(timeoutMs);
   const requestSignal = signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal;
   const response = await fetch(GREP_MCP_URL, {
     method: "POST",
@@ -230,8 +272,49 @@ export async function searchCodeGrep(
 
   const body = await response.text();
   if (!response.ok) {
-    throw new GrepError(`Grep MCP returned ${response.status}: ${body.slice(0, 200)}`);
+    if (response.status >= 500) {
+      throw new GrepTransientHttpError(response.status, body);
+    }
+    throw new GrepError(formatHttpError(response.status, body));
   }
 
   return parseGrepMcpEvents(body);
 }
+
+// Do not extend GrepError: GrepError is the deterministic no-retry signal.
+class GrepTransientHttpError extends Error {
+  readonly status: number;
+
+  constructor(status: number, body: string) {
+    super(formatHttpError(status, body));
+    this.name = "GrepTransientHttpError";
+    this.status = status;
+  }
+}
+
+export async function searchCodeGrep(
+  params: GrepCodeSearchParams,
+  signal: AbortSignal | undefined,
+): Promise<GrepCodeSearchResult> {
+  if (signal?.aborted) {
+    throw signal.reason;
+  }
+
+  let transientError: unknown;
+  for (const [attempt, timeoutMs] of GREP_ATTEMPT_TIMEOUTS_MS.entries()) {
+    if (attempt > 0) {
+      await delay(GREP_RETRY_DELAY_MS, signal);
+    }
+
+    try {
+      return await searchCodeGrepAttempt(params, signal, timeoutMs);
+    } catch (error) {
+      if (error instanceof GrepError || signal?.aborted) {
+        throw error;
+      }
+      transientError = error;
+    }
+  }
+
+  throw new GrepError(formatTransientError(transientError));
+}

package/extensions/librarian/run.ts

@@ -52,6 +52,7 @@ export interface LibrarianRunDetails {
   checkouts: Record<string, string>;
   error?: string;
   runId?: string;
+  debugSessionPath?: string;
   startedAt: number;
   endedAt?: number;
 }
@@ -108,7 +109,11 @@ export async function runLibrarian(
   };
 
   const finalizeDetails = (status: LibrarianRunStatus, content: string): string => {
-    const resultContent = details.runId ? `${content}\n\nrun: ${details.runId}` : content;
+    const suffixes = [
+      ...(details.runId ? [`run: ${details.runId}`] : []),
+      ...(details.debugSessionPath ? [`debug_session: ${details.debugSessionPath}`] : []),
+    ];
+    const resultContent = suffixes.length > 0 ? `${content}\n\n${suffixes.join("\n")}` : content;
     details.status = status;
     details.endedAt = Date.now();
     if (status !== "done") {
@@ -137,6 +142,10 @@ export async function runLibrarian(
     ? await openContinuedSession(options.continueFrom, sessionsDir, options.settings.cacheDir, fail)
     : SessionManager.create(options.settings.cacheDir, sessionsDir);
   details.runId = sessionManager.getSessionId();
+  const debugSessionPath = sessionManager.getSessionFile();
+  if (options.settings.debug.persistRuns && debugSessionPath) {
+    details.debugSessionPath = debugSessionPath;
+  }
 
   let findings: Findings | undefined;
   const repoTools = [

package/extensions/librarian/settings.ts

@@ -20,6 +20,11 @@ const LIBRARIAN_FILE_SETTINGS_SCHEMA = Type.Object({
   extensions: Type.Optional(Type.Array(Type.String())),
   tools: Type.Optional(Type.Array(Type.String())),
   cacheDir: Type.Optional(Type.String()),
+  debug: Type.Optional(
+    Type.Object({
+      persistRuns: Type.Optional(Type.Boolean()),
+    }),
+  ),
 });
 
 const ROOT_SETTINGS_SCHEMA = Type.Object({
@@ -45,6 +50,9 @@ export interface LibrarianSettings {
   extensions: string[];
   tools: string[];
   cacheDir: string;
+  debug: {
+    persistRuns: boolean;
+  };
 }
 
 export function getDefaultCacheDir(): string {
@@ -117,6 +125,9 @@ export function resolveLibrarianSettings(fileSettings: LibrarianFileSettings): L
     extensions: normalizeExtensionPaths(fileSettings.extensions),
     tools: normalizeToolNames(fileSettings.tools),
     cacheDir: normalizeCacheDir(fileSettings.cacheDir),
+    debug: {
+      persistRuns: fileSettings.debug?.persistRuns ?? false,
+    },
   };
 }
 

package/extensions/librarian/tools/search-code.ts

@@ -14,12 +14,11 @@ export interface SearchCodeDetails {
 const SearchCodeParams = Type.Object({
   pattern: Type.String({
     description:
-      "Public code search pattern. Literal by default; set regex: true to treat it as a regular expression.",
+      "Public code search pattern. Use literal code patterns as they appear in files, not keywords: e.g. `useState(` rather than `react hook state`.",
   }),
   regex: Type.Optional(
     Type.Boolean({
-      description:
-        "Treat pattern as a regular expression. Grep supports multi-line regex with (?s).",
+      description: "Treat `pattern` as a regular expression instead of literals.",
     }),
   ),
   repo: Type.Optional(
@@ -62,6 +61,7 @@ export const searchCodeTool = defineTool<typeof SearchCodeParams, SearchCodeDeta
   description: "Cross-repo code search over public source code.",
   promptSnippet: "Search code across repos",
   promptGuidelines: [
+    "When `regex` is true, `pattern` can also use `(?s)` for multi-line patterns.",
     "Results are CANDIDATES to verify via checkout_repo/read_github_file — never cite them directly.",
     "This searches public GitHub code only. Use search_github_code when private repository access matters.",
   ],

package/extensions/librarian/view.ts

@@ -1,21 +1,18 @@
 import path from "node:path";
 import type { AgentToolResult } from "@earendil-works/pi-agent-core";
 import type { Theme } from "@earendil-works/pi-coding-agent";
-import { getMarkdownTheme } from "@earendil-works/pi-coding-agent";
-import { Container, Markdown, Spacer, Text } from "@earendil-works/pi-tui";
+import { getMarkdownTheme, keyHint } from "@earendil-works/pi-coding-agent";
+import { type Component, Container, Markdown, Spacer, Text } from "@earendil-works/pi-tui";
 import type { LibrarianRunDetails, TraceCall } from "./run.ts";
 import { LIBRARIAN_TOOL_NAMES } from "./tools/names.ts";
 
 const COLLAPSED_TRACE_CALLS = 3;
+// Same frames and cadence as pi's Working... loader (pi-tui loader.ts defaults).
 const SPINNER_FRAMES = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"];
-const SPINNER_INTERVAL_MS = 80;
-
-interface LiveRenderContext {
-  invalidate(): void;
-  state: {
-    librarianSpinnerFrame?: number;
-    librarianSpinnerInterval?: ReturnType<typeof setInterval>;
-  };
+export const SPINNER_INTERVAL_MS = 80;
+
+interface LibrarianRenderContext {
+  lastComponent: Component | undefined;
 }
 
 export function formatDuration(milliseconds: number): string {
@@ -153,18 +150,17 @@ export function formatTraceLine(call: TraceCall, cacheDir: string): TraceLine {
   }
 }
 
-function renderTraceCallText(
-  call: TraceCall,
-  cacheDir: string,
-  theme: Theme,
-  spinnerFrame: string,
-): string {
+function currentSpinnerFrame(): string {
+  return SPINNER_FRAMES[Math.floor(Date.now() / SPINNER_INTERVAL_MS) % SPINNER_FRAMES.length] ?? "";
+}
+
+function renderTraceCallText(call: TraceCall, cacheDir: string, theme: Theme): string {
   const { verb, subject } = formatTraceLine(call, cacheDir);
   const running = call.endedAt === undefined;
   const icon = call.isError
     ? theme.fg("error", "✗")
     : running
-      ? theme.fg("warning", spinnerFrame)
+      ? theme.fg("accent", currentSpinnerFrame())
       : theme.fg("success", "✓");
   const duration = formatDuration((call.endedAt ?? Date.now()) - call.startedAt);
   const summary = call.resultSummary
@@ -189,6 +185,25 @@ function renderFooter(details: LibrarianRunDetails, theme: Theme): string {
   );
 }
 
+function collapsedFindingsHiddenText(details: LibrarianRunDetails): string | undefined {
+  const findings = details.findings;
+  if (!findings) {
+    return undefined;
+  }
+
+  const hiddenParts: string[] = [];
+  if (findings.locations.length > 0) {
+    hiddenParts.push(
+      `${findings.locations.length} location${findings.locations.length === 1 ? "" : "s"}`,
+    );
+  }
+  if (findings.description?.trim()) {
+    hiddenParts.push("details");
+  }
+
+  return hiddenParts.length > 0 ? `${hiddenParts.join(" and ")} hidden` : undefined;
+}
+
 function isLibrarianRunDetails(value: unknown): value is LibrarianRunDetails {
   return (
     value !== null &&
@@ -205,55 +220,33 @@ function renderTrace(
   expanded: boolean,
   cacheDir: string,
   theme: Theme,
-  spinnerFrame: string,
 ): string[] {
   const lines: string[] = [];
   const calls = expanded ? details.trace : details.trace.slice(-COLLAPSED_TRACE_CALLS);
   const hiddenCount = details.trace.length - calls.length;
   if (hiddenCount > 0) {
     lines.push(
-      theme.fg(
-        "muted",
-        `  … ${hiddenCount} earlier call${hiddenCount === 1 ? "" : "s"} (Ctrl+O to expand)`,
-      ),
+      theme.fg("muted", `  … ${hiddenCount} earlier call${hiddenCount === 1 ? "" : "s"} (`) +
+        keyHint("app.tools.expand", "to expand") +
+        theme.fg("muted", ")"),
     );
   }
   for (const call of calls) {
-    lines.push(renderTraceCallText(call, cacheDir, theme, spinnerFrame));
+    lines.push(renderTraceCallText(call, cacheDir, theme));
   }
   return lines;
 }
 
-function updateLiveRender(context: LiveRenderContext | undefined, running: boolean): string {
-  if (!context) {
-    return (
-      SPINNER_FRAMES[Math.floor(Date.now() / SPINNER_INTERVAL_MS) % SPINNER_FRAMES.length] ?? ""
-    );
-  }
-
-  context.state.librarianSpinnerFrame ??= 0;
-  if (running && !context.state.librarianSpinnerInterval) {
-    context.state.librarianSpinnerInterval = setInterval(() => {
-      context.state.librarianSpinnerFrame =
-        ((context.state.librarianSpinnerFrame ?? 0) + 1) % SPINNER_FRAMES.length;
-      context.invalidate();
-    }, SPINNER_INTERVAL_MS);
-  } else if (!running && context.state.librarianSpinnerInterval) {
-    clearInterval(context.state.librarianSpinnerInterval);
-    delete context.state.librarianSpinnerInterval;
-  }
-
-  return SPINNER_FRAMES[context.state.librarianSpinnerFrame] ?? "";
-}
-
 export function renderLibrarianResult(
   result: AgentToolResult<LibrarianRunDetails>,
   options: { expanded: boolean; isPartial: boolean },
   theme: Theme,
   cacheDir: string,
-  context?: LiveRenderContext,
+  context?: LibrarianRenderContext,
 ): Container {
-  const container = new Container();
+  const container =
+    context?.lastComponent instanceof Container ? context.lastComponent : new Container();
+  container.clear();
   const details = result.details;
 
   if (!isLibrarianRunDetails(details)) {
@@ -261,22 +254,34 @@ export function renderLibrarianResult(
     container.addChild(
       new Text(firstText && "text" in firstText ? firstText.text : "(no output)", 0, 0),
     );
+    container.invalidate();
     return container;
   }
 
   const running = options.isPartial || details.status === "running";
-  const spinnerFrame = updateLiveRender(context, running);
   container.addChild(new Text(renderQuestion(details, options.expanded, theme), 0, 0));
   container.addChild(new Spacer(1));
 
   if (running || !details.findings) {
-    for (const line of renderTrace(details, options.expanded, cacheDir, theme, spinnerFrame)) {
+    for (const line of renderTrace(details, options.expanded, cacheDir, theme)) {
       container.addChild(new Text(line, 0, 0));
     }
   }
 
   if (!running) {
     if (details.findings) {
+      const hiddenText = options.expanded ? undefined : collapsedFindingsHiddenText(details);
+      if (hiddenText) {
+        container.addChild(
+          new Text(
+            theme.fg("muted", `(${hiddenText}, `) +
+              keyHint("app.tools.expand", "to expand") +
+              theme.fg("muted", ")"),
+            0,
+            0,
+          ),
+        );
+      }
       const markdown = buildFindingsMarkdown(details, options.expanded);
       container.addChild(new Markdown(markdown, 0, 0, getMarkdownTheme()));
     } else if (details.error) {
@@ -290,6 +295,7 @@ export function renderLibrarianResult(
     container.addChild(new Text(theme.fg("muted", `run ${details.runId}`), 0, 0));
   }
   container.addChild(new Text(renderFooter(details, theme), 0, 0));
+  container.invalidate();
   return container;
 }
 

package/extensions/librarian.ts

@@ -17,6 +17,7 @@ import {
   formatTraceLine,
   renderLibrarianCall,
   renderLibrarianResult,
+  SPINNER_INTERVAL_MS,
   shorten,
 } from "./librarian/view.ts";
 
@@ -130,6 +131,16 @@ export default function librarianExtension(pi: ExtensionAPI): void {
     },
 
     renderResult(result, options, theme, context) {
+      // Live spinner/elapsed ticking, same pattern and cadence as pi's bash
+      // tool and Working... loader: re-render on an interval while streaming.
+      const state = context.state as { interval: NodeJS.Timeout | undefined };
+      if (options.isPartial && !state.interval) {
+        state.interval = setInterval(() => context.invalidate(), SPINNER_INTERVAL_MS);
+      }
+      if ((!options.isPartial || context.isError) && state.interval) {
+        clearInterval(state.interval);
+        state.interval = undefined;
+      }
       return renderLibrarianResult(result, options, theme, settings.cacheDir, context);
     },
   });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@thurstonsand/pi-librarian",
-  "version": "0.2.0",
+  "version": "0.2.1",
   "type": "module",
   "description": "GitHub research subagent for pi: deep-dive specific repos, discover across the ecosystem",
   "license": "MIT",
@@ -25,7 +25,8 @@
     "CONTEXT.md",
     "DEV.md",
     "AGENTS.md",
-    "docs"
+    "docs",
+    "images"
   ],
   "pi": {
     "extensions": [