@thurstonsand/pi-librarian 0.1.0

package/extensions/librarian/grep-app.ts

@@ -0,0 +1,237 @@
+import { Type } from "typebox";
+import { safeParseTypeBoxValue } from "../shared/typebox.ts";
+
+const GREP_MCP_URL = "https://mcp.grep.app";
+const GREP_SEARCH_TIMEOUT_MS = 30_000;
+
+export class GrepError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "GrepError";
+  }
+}
+
+export interface GrepCodeSearchParams {
+  query: string;
+  matchCase?: boolean;
+  matchWholeWords?: boolean;
+  useRegexp?: boolean;
+  repo?: string;
+  path?: string;
+  language?: string[];
+}
+
+export interface GrepSnippet {
+  lineNumber: number;
+  content: string;
+}
+
+export interface GrepCodeMatch {
+  repo: string;
+  path: string;
+  url: string | undefined;
+  license: string | undefined;
+  snippets: GrepSnippet[];
+}
+
+export interface GrepCodeSearchResult {
+  matches: GrepCodeMatch[];
+}
+
+const MCP_TEXT_CONTENT_SCHEMA = Type.Object({
+  type: Type.Literal("text"),
+  text: Type.String(),
+});
+
+const MCP_CALL_RESULT_SCHEMA = Type.Object({
+  content: Type.Optional(Type.Array(Type.Unknown())),
+  isError: Type.Optional(Type.Boolean()),
+});
+
+const MCP_ERROR_SCHEMA = Type.Object({
+  message: Type.String(),
+});
+
+const MCP_RESPONSE_SCHEMA = Type.Object({
+  result: Type.Optional(Type.Unknown()),
+  error: Type.Optional(Type.Unknown()),
+});
+
+function parseHeader(line: string, name: string): string | undefined {
+  const prefix = `${name}: `;
+  return line.startsWith(prefix) ? line.slice(prefix.length).trim() : undefined;
+}
+
+export function parseGrepSearchText(text: string): GrepCodeMatch | undefined {
+  if (text.trim() === "No results found for your query.") {
+    return undefined;
+  }
+
+  let repo: string | undefined;
+  let path: string | undefined;
+  let url: string | undefined;
+  let license: string | undefined;
+  const snippets: GrepSnippet[] = [];
+  let currentSnippet: { lineNumber: number; lines: string[] } | undefined;
+
+  function finishSnippet(): void {
+    if (!currentSnippet) {
+      return;
+    }
+
+    snippets.push({
+      lineNumber: currentSnippet.lineNumber,
+      content: currentSnippet.lines.join("\n").trimEnd(),
+    });
+    currentSnippet = undefined;
+  }
+
+  for (const line of text.split("\n")) {
+    const parsedRepo = parseHeader(line, "Repository");
+    if (parsedRepo) {
+      repo = parsedRepo;
+      continue;
+    }
+
+    const parsedPath = parseHeader(line, "Path");
+    if (parsedPath) {
+      path = parsedPath;
+      continue;
+    }
+
+    const parsedUrl = parseHeader(line, "URL");
+    if (parsedUrl) {
+      url = parsedUrl;
+      continue;
+    }
+
+    const parsedLicense = parseHeader(line, "License");
+    if (parsedLicense) {
+      license = parsedLicense === "Unknown" ? undefined : parsedLicense;
+      continue;
+    }
+
+    const snippetStart = /^--- Snippet \d+ \(Line (\d+)\) ---$/.exec(line);
+    if (snippetStart) {
+      finishSnippet();
+      currentSnippet = { lineNumber: Number(snippetStart[1]), lines: [] };
+      continue;
+    }
+
+    if (currentSnippet) {
+      currentSnippet.lines.push(line.startsWith("> ") ? line.slice(2) : line);
+    }
+  }
+
+  finishSnippet();
+
+  if (!repo || !path) {
+    return undefined;
+  }
+
+  return { repo, path, url, license, snippets };
+}
+
+export function parseGrepMcpEvents(body: string): GrepCodeSearchResult {
+  const matches: GrepCodeMatch[] = [];
+
+  for (const rawLine of body.split("\n")) {
+    const line = rawLine.trimEnd();
+    if (!line.startsWith("data: ")) {
+      continue;
+    }
+
+    const response = safeParseTypeBoxValue(
+      MCP_RESPONSE_SCHEMA,
+      JSON.parse(line.slice("data: ".length)),
+    );
+    if (!response) {
+      continue;
+    }
+
+    const error = safeParseTypeBoxValue(MCP_ERROR_SCHEMA, response.error);
+    if (error) {
+      throw new GrepError(error.message);
+    }
+
+    const result = safeParseTypeBoxValue(MCP_CALL_RESULT_SCHEMA, response.result);
+    if (!result) {
+      continue;
+    }
+    if (result.isError) {
+      throw new GrepError("Grep MCP returned an error result.");
+    }
+
+    for (const rawContent of result.content ?? []) {
+      const content = safeParseTypeBoxValue(MCP_TEXT_CONTENT_SCHEMA, rawContent);
+      if (!content) {
+        continue;
+      }
+
+      const match = parseGrepSearchText(content.text);
+      if (match) {
+        matches.push(match);
+      }
+    }
+  }
+
+  return { matches };
+}
+
+function buildGrepArguments(params: GrepCodeSearchParams): Record<string, unknown> {
+  const args: Record<string, unknown> = { query: params.query };
+
+  if (params.matchCase) {
+    args.matchCase = true;
+  }
+  if (params.matchWholeWords) {
+    args.matchWholeWords = true;
+  }
+  if (params.useRegexp) {
+    args.useRegexp = true;
+  }
+  if (params.repo) {
+    args.repo = params.repo;
+  }
+  if (params.path) {
+    args.path = params.path;
+  }
+  if (params.language && params.language.length > 0) {
+    args.language = params.language;
+  }
+
+  return args;
+}
+
+export async function searchCodeGrep(
+  params: GrepCodeSearchParams,
+  signal: AbortSignal | undefined,
+): Promise<GrepCodeSearchResult> {
+  const timeoutSignal = AbortSignal.timeout(GREP_SEARCH_TIMEOUT_MS);
+  const requestSignal = signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal;
+  const response = await fetch(GREP_MCP_URL, {
+    method: "POST",
+    headers: {
+      accept: "application/json, text/event-stream",
+      "content-type": "application/json",
+      "user-agent": "pi-librarian",
+    },
+    body: JSON.stringify({
+      jsonrpc: "2.0",
+      id: 1,
+      method: "tools/call",
+      params: {
+        name: "searchGitHub",
+        arguments: buildGrepArguments(params),
+      },
+    }),
+    signal: requestSignal,
+  });
+
+  const body = await response.text();
+  if (!response.ok) {
+    throw new GrepError(`Grep MCP returned ${response.status}: ${body.slice(0, 200)}`);
+  }
+
+  return parseGrepMcpEvents(body);
+}

package/extensions/librarian/model.ts

@@ -0,0 +1,59 @@
+import type { ThinkingLevel } from "@earendil-works/pi-agent-core";
+import type { Api, Model } from "@earendil-works/pi-ai";
+import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
+import type { ModelReference } from "./settings.ts";
+
+export type LibrarianModelSource = "configured" | "current";
+
+export interface LibrarianModelResolution {
+  model: Model<Api>;
+  thinkingLevel: ThinkingLevel;
+  source: LibrarianModelSource;
+}
+
+function isAlias(modelId: string): boolean {
+  return modelId.endsWith("-latest") || !/-\d{8}$/.test(modelId);
+}
+
+function bestModelMatch(models: Model<Api>[], pattern: string): Model<Api> | undefined {
+  const normalizedPattern = pattern.toLowerCase();
+  const exactMatches = models.filter((model) => model.id.toLowerCase() === normalizedPattern);
+  if (exactMatches.length === 1) {
+    return exactMatches[0];
+  }
+
+  const partialMatches = models.filter(
+    (model) =>
+      model.id.toLowerCase().includes(normalizedPattern) ||
+      model.name?.toLowerCase().includes(normalizedPattern),
+  );
+  if (partialMatches.length === 0) {
+    return undefined;
+  }
+
+  const aliases = partialMatches.filter((model) => isAlias(model.id));
+  const candidates = aliases.length > 0 ? aliases : partialMatches;
+  return candidates.toSorted((a, b) => b.id.localeCompare(a.id))[0];
+}
+
+export function resolveLibrarianModel(
+  ctx: ExtensionContext,
+  configuredModel: ModelReference | undefined,
+  thinkingLevel: ThinkingLevel,
+): LibrarianModelResolution | undefined {
+  if (configuredModel) {
+    const providerModels = ctx.modelRegistry
+      .getAvailable()
+      .filter((model) => model.provider === configuredModel.provider);
+    const match = bestModelMatch(providerModels, configuredModel.modelId);
+    if (match) {
+      return { model: match, thinkingLevel, source: "configured" };
+    }
+  }
+
+  if (!ctx.model) {
+    return undefined;
+  }
+
+  return { model: ctx.model, thinkingLevel, source: "current" };
+}

package/extensions/librarian/prompt.ts

@@ -0,0 +1,54 @@
+export function buildLibrarianSystemPrompt(cacheDir: string): string {
+  return `You are the Librarian, a codebase-understanding agent. You answer questions about large, multi-repository codebases using the provided tools. You run as an isolated subagent and your answer is relayed verbatim back to the main agent.
+
+## Search Strategy
+
+Pick the cheapest instrument that answers the question:
+
+- Question about a SPECIFIC repo (behavior, implementation, flow): checkout_repo first, then grep/read/find/ls on the local clone. This is your workhorse. Use \`git -C <path> log -S <term> / blame / diff\` for history questions.
+- "What's out there / what's popular": search_repos (stars are the popularity signal), plus web search when available for ecosystem context. Read READMEs/docs via read_github_file.
+- "Who does X across public repos": search_code to gather candidates, then verify the promising ones via checkout_repo or read_github_file.
+- Private/authenticated GitHub code search: search_github_code to gather candidates, then verify with checkout_repo or read_github_file.
+- Single-file peek (a package.json, one README): read_github_file — don't clone for one file.
+
+search_code and search_github_code results are CANDIDATES, not evidence. Never cite a search hit you have not verified by reading the file.
+
+Work with common sense: start with the most informative call, expand only when needed, and stop as soon as you have enough evidence to answer confidently. Prefer parallel tool calls for independent lookups.
+
+## Workspace
+
+Clones live under ${cacheDir}/repos/<owner>/<repo>. They are read-only research material: never modify, commit, or push. Keep bash usage to inspection (git log/blame/diff, wc, jq); file content questions go through grep/read.
+
+## Citations
+
+- Only cite files you actually read in this run.
+- Cite as repo + file path + line range when you have one.
+- If evidence is partial, say what is confirmed and what remains uncertain.
+- If access fails (404/403, private repo), report that constraint plainly.
+
+## Finishing
+
+- You MUST end by calling provide_results — exactly once, after your research is complete:
+- Calling provide_results forcefully ends your turn, so it must be the LAST thing you do, by itself. Do not call it before you are ready to answer.
+- Do not write your findings as a plain message; they only count via provide_results.`;
+}
+
+export function buildLibrarianUserPrompt(query: string, repos: string[], owners: string[]): string {
+  const lines = [`Research query: ${query}`];
+  if (repos.length > 0) {
+    lines.push(`Repository scope: ${repos.join(", ")}`);
+  }
+  if (owners.length > 0) {
+    lines.push(`Owner scope: ${owners.join(", ")}`);
+  }
+  if (repos.length > 0 || owners.length > 0) {
+    lines.push(
+      "Treat the scope as the grounding point for the research, not a hard boundary. Start there, but go beyond it when outside repositories or owners are useful to answer the question accurately.",
+    );
+  }
+  return lines.join("\n");
+}
+
+export function buildProvideResultsReminder(): string {
+  return "You have not called provide_results yet. Call it now with your findings (summary, locations, optional description). If you could not answer the query, say so in the summary and cite whatever partial evidence you have.";
+}

package/extensions/librarian/results.ts

@@ -0,0 +1,63 @@
+import type { Findings } from "./tools/provide-results.ts";
+
+function githubOwnerRepo(repo: string): string | undefined {
+  if (/^[^/]+\/[^/]+$/.test(repo)) {
+    return repo;
+  }
+
+  try {
+    const url = new URL(repo);
+    if (url.hostname !== "github.com" && url.hostname !== "www.github.com") {
+      return undefined;
+    }
+    const [owner, name] = url.pathname.replace(/^\/+/, "").split("/");
+    return owner && name ? `${owner}/${name.replace(/\.git$/, "")}` : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+function locationUrl(
+  repo: string,
+  file: string,
+  lines: string | undefined,
+  sha: string | undefined,
+): string | undefined {
+  const ownerRepo = githubOwnerRepo(repo);
+  if (!ownerRepo) {
+    return undefined;
+  }
+
+  const refSegment = sha ? sha.slice(0, 12) : "HEAD";
+  const anchor = lines ? `#L${lines.replace(/[^0-9-]/g, "").replace("-", "-L")}` : "";
+  return `https://github.com/${ownerRepo}/blob/${refSegment}/${file}${anchor}`;
+}
+
+export function renderFindingsMarkdown(
+  findings: Findings,
+  checkouts: Record<string, string>,
+): string {
+  const sections = [findings.summary];
+
+  if (findings.locations.length > 0) {
+    const bullets = findings.locations.map((location) => {
+      const lineSuffix = location.lines ? `:${location.lines}` : "";
+      const ownerRepo = githubOwnerRepo(location.repo);
+      const url = locationUrl(
+        location.repo,
+        location.file,
+        location.lines,
+        ownerRepo ? checkouts[ownerRepo] : undefined,
+      );
+      const citation = `- \`${location.repo}/${location.file}${lineSuffix}\` — ${location.note}`;
+      return url ? `${citation}\n  ${url}` : citation;
+    });
+    sections.push(`## Locations\n\n${bullets.join("\n")}`);
+  }
+
+  if (findings.description) {
+    sections.push(findings.description);
+  }
+
+  return sections.join("\n\n");
+}

package/extensions/librarian/run.ts

@@ -0,0 +1,248 @@
+import fs from "node:fs/promises";
+import type { AgentToolResult, ThinkingLevel } from "@earendil-works/pi-agent-core";
+import type { Api, Model } from "@earendil-works/pi-ai";
+import type { AgentSession, AgentSessionEvent } from "@earendil-works/pi-coding-agent";
+import {
+  createAgentSession,
+  DefaultResourceLoader,
+  getAgentDir,
+  SessionManager,
+} from "@earendil-works/pi-coding-agent";
+import type { GitHubClientProvider } from "./github.ts";
+import {
+  buildLibrarianSystemPrompt,
+  buildLibrarianUserPrompt,
+  buildProvideResultsReminder,
+} from "./prompt.ts";
+import { renderFindingsMarkdown } from "./results.ts";
+import type { LibrarianSettings } from "./settings.ts";
+import { createCheckoutRepoTool } from "./tools/checkout-repo.ts";
+import type { Findings } from "./tools/provide-results.ts";
+import { createProvideResultsTool } from "./tools/provide-results.ts";
+import { createReadGitHubFileTool } from "./tools/read-github-file.ts";
+import { searchCodeTool } from "./tools/search-code.ts";
+import { createSearchGitHubCodeTool } from "./tools/search-github-code.ts";
+import { createSearchReposTool } from "./tools/search-repos.ts";
+import { asRepoToolDetails, summarizeToolResult } from "./trace.ts";
+
+const MAX_PROVIDE_RESULTS_REMINDERS = 3;
+const HARDCODED_EXCLUDED_TOOLS = ["write", "edit"];
+
+export type LibrarianRunStatus = "running" | "done" | "error" | "aborted";
+
+export interface TraceCall {
+  id: string;
+  name: string;
+  args: unknown;
+  startedAt: number;
+  endedAt?: number;
+  isError?: boolean;
+  resultSummary?: string;
+}
+
+export interface LibrarianRunDetails {
+  status: LibrarianRunStatus;
+  query: string;
+  modelLabel: string;
+  thinkingLevel: ThinkingLevel;
+  trace: TraceCall[];
+  findings?: Findings;
+  checkouts: Record<string, string>;
+  error?: string;
+  startedAt: number;
+  endedAt?: number;
+}
+
+export interface LibrarianRunOptions {
+  query: string;
+  repos: string[];
+  owners: string[];
+  model: Model<Api>;
+  thinkingLevel: ThinkingLevel;
+  settings: LibrarianSettings;
+  githubClient: GitHubClientProvider;
+  signal: AbortSignal | undefined;
+  onUpdate: ((details: LibrarianRunDetails) => void) | undefined;
+}
+
+export async function runLibrarian(
+  options: LibrarianRunOptions,
+): Promise<AgentToolResult<LibrarianRunDetails>> {
+  const details: LibrarianRunDetails = {
+    status: "running",
+    query: options.query,
+    modelLabel: `${options.model.provider}/${options.model.id}`,
+    thinkingLevel: options.thinkingLevel,
+    trace: [],
+    checkouts: {},
+    startedAt: Date.now(),
+  };
+
+  let lastEmit = 0;
+  const emit = (force = false) => {
+    const now = Date.now();
+    if (!force && now - lastEmit < 120) {
+      return;
+    }
+    lastEmit = now;
+    options.onUpdate?.({ ...details, trace: [...details.trace] });
+  };
+
+  const finish = (
+    status: LibrarianRunStatus,
+    content: string,
+    isError: boolean,
+  ): AgentToolResult<LibrarianRunDetails> => {
+    details.status = status;
+    details.endedAt = Date.now();
+    if (isError) {
+      details.error = content.split("\n")[0] ?? content;
+    }
+    emit(true);
+    return {
+      content: [{ type: "text", text: content }],
+      details: { ...details, trace: [...details.trace] },
+      ...(isError ? { isError: true } : {}),
+    };
+  };
+
+  await fs.mkdir(options.settings.cacheDir, { recursive: true });
+
+  let findings: Findings | undefined;
+  const repoTools = [
+    createSearchReposTool(options.githubClient),
+    searchCodeTool,
+    createSearchGitHubCodeTool(options.githubClient),
+    createCheckoutRepoTool(options.settings.cacheDir),
+    createReadGitHubFileTool(options.githubClient),
+    createProvideResultsTool((payload) => {
+      findings = payload;
+    }),
+  ];
+
+  const resourceLoader = new DefaultResourceLoader({
+    cwd: options.settings.cacheDir,
+    agentDir: getAgentDir(),
+    noExtensions: true,
+    ...(options.settings.extensions.length > 0
+      ? { additionalExtensionPaths: options.settings.extensions }
+      : {}),
+    noSkills: true,
+    noPromptTemplates: true,
+    noThemes: true,
+    systemPromptOverride: () => buildLibrarianSystemPrompt(options.settings.cacheDir),
+    skillsOverride: () => ({ skills: [], diagnostics: [] }),
+  });
+  await resourceLoader.reload();
+
+  emit(true);
+
+  let session: AgentSession | undefined;
+  let unsubscribe: (() => void) | undefined;
+  let onAbort: (() => void) | undefined;
+
+  try {
+    const created = await createAgentSession({
+      cwd: options.settings.cacheDir,
+      resourceLoader,
+      sessionManager: SessionManager.inMemory(options.settings.cacheDir),
+      model: options.model,
+      thinkingLevel: options.thinkingLevel,
+      customTools: repoTools,
+      excludeTools: [...HARDCODED_EXCLUDED_TOOLS, ...options.settings.disabledTools],
+    });
+    session = created.session;
+
+    const excluded = new Set([...HARDCODED_EXCLUDED_TOOLS, ...options.settings.disabledTools]);
+    const allToolNames = session
+      .getAllTools()
+      .map((tool) => tool.name)
+      .filter((name) => !excluded.has(name));
+    session.setActiveToolsByName(allToolNames);
+
+    unsubscribe = session.subscribe((event: AgentSessionEvent) => {
+      switch (event.type) {
+        case "tool_execution_start": {
+          details.trace.push({
+            id: event.toolCallId,
+            name: event.toolName,
+            args: event.args,
+            startedAt: Date.now(),
+          });
+          emit(true);
+          break;
+        }
+        case "tool_execution_end": {
+          const call = details.trace.find((traced) => traced.id === event.toolCallId);
+          if (!call) {
+            break;
+          }
+          call.endedAt = Date.now();
+          if (event.isError) {
+            call.isError = true;
+          }
+          const summary = summarizeToolResult(event.result, event.isError === true);
+          if (summary !== undefined) {
+            call.resultSummary = summary;
+          }
+          const repoDetails = asRepoToolDetails(event.result?.details);
+          if (repoDetails?.kind === "checkout_repo" && !event.isError) {
+            details.checkouts[repoDetails.repo] = repoDetails.headSha;
+          }
+          emit(true);
+          break;
+        }
+        default:
+          emit();
+      }
+    });
+
+    if (options.signal?.aborted) {
+      return finish("aborted", "Librarian run aborted before it started.", true);
+    }
+
+    const activeSession = session;
+    onAbort = () => {
+      void activeSession.abort();
+    };
+    options.signal?.addEventListener("abort", onAbort);
+
+    await session.prompt(buildLibrarianUserPrompt(options.query, options.repos, options.owners), {
+      expandPromptTemplates: false,
+    });
+
+    let reminders = 0;
+    while (!findings && !options.signal?.aborted && reminders < MAX_PROVIDE_RESULTS_REMINDERS) {
+      reminders += 1;
+      await session.prompt(buildProvideResultsReminder(), { expandPromptTemplates: false });
+    }
+
+    if (options.signal?.aborted) {
+      return finish("aborted", "Librarian run aborted.", true);
+    }
+
+    if (findings) {
+      details.findings = findings;
+      return finish("done", renderFindingsMarkdown(findings, details.checkouts), false);
+    }
+
+    const lastText = session.getLastAssistantText() ?? "";
+    return finish(
+      "error",
+      `Librarian did not report structured findings after ${MAX_PROVIDE_RESULTS_REMINDERS} reminders.${lastText ? `\n\nRaw final message:\n${lastText}` : ""}`,
+      true,
+    );
+  } catch (error) {
+    if (options.signal?.aborted) {
+      return finish("aborted", "Librarian run aborted.", true);
+    }
+    const message = error instanceof Error ? error.message : String(error);
+    return finish("error", `Librarian run failed: ${message}`, true);
+  } finally {
+    if (onAbort) {
+      options.signal?.removeEventListener("abort", onAbort);
+    }
+    unsubscribe?.();
+    session?.dispose();
+  }
+}