@hienlh/ppm 0.20.9 → 0.21.0

package/src/services/chat-search.service.ts

@@ -10,7 +10,7 @@
  */
 import { getSearchIndexDb } from "./search-index-db.service.ts";
 import { chatService } from "./chat.service.ts";
-import type { ChatMessage } from "../types/chat.ts";
+import type { ChatEvent, ChatMessage } from "../types/chat.ts";
 
 export interface ChatSearchHit {
   sessionId: string;
@@ -42,6 +42,108 @@ export function toFtsQuery(raw: string): string {
   return parts.join(" ");
 }
 
+/**
+ * Bump whenever `messageSearchText` changes what it emits. Stored per session in
+ * `session_meta.indexer_version`; `isStale` compares it, so a session indexed by
+ * an older, thinner indexer is re-read rather than left stamped as fresh.
+ *
+ * 2: `MESSAGE_TEXT_CAP`. Sessions indexed at 1 may hold multi-megabyte rows.
+ *
+ * A bump makes *every* session stale at once, which is what `RECONCILE_BUDGET`
+ * below is for — the whole corpus must not be re-read inside one search.
+ */
+export const INDEXER_VERSION = 2;
+
+/** Per-event cap on indexed tool output. A single `Read`/`grep` result can be
+ *  hundreds of KB; indexing all of it bloats the FTS store far more than it
+ *  helps, since matches that deep are rarely what someone is looking for. */
+const EVENT_TEXT_CAP = 4000;
+
+/**
+ * Whole-message cap, which the per-event one does not imply.
+ *
+ * `EVENT_TEXT_CAP` bounds each event and nothing bounds how many there are. One
+ * agent turn is routinely hundreds of tool calls, each with children that
+ * recurse through the same collector — so a single turn could contribute a
+ * multi-megabyte FTS row, for a feature whose output is a twelve-token snippet.
+ * The cap is generous on purpose: this is about the tail, not about trimming
+ * ordinary messages.
+ */
+const MESSAGE_TEXT_CAP = 64_000;
+
+function stringifyToolInput(input: unknown): string {
+  if (input == null) return "";
+  if (typeof input === "string") return input.slice(0, EVENT_TEXT_CAP);
+  try {
+    return JSON.stringify(input).slice(0, EVENT_TEXT_CAP);
+  } catch {
+    return "";
+  }
+}
+
+/**
+ * Appends to `out` until the budget runs out, and reports what is left.
+ *
+ * The budget is threaded through rather than applied to the joined result so a
+ * turn with a thousand tool calls stops *collecting* — building the array and
+ * then slicing it is the same allocation this is meant to avoid.
+ */
+function collectEventText(events: ChatEvent[] | undefined, out: string[], budget: number): number {
+  if (!events) return budget;
+  for (const ev of events) {
+    if (budget <= 0) return 0;
+    switch (ev.type) {
+      case "text":
+        budget = take(ev.content, out, budget);
+        break;
+      case "tool_use":
+        // The tool name and its arguments are what a search for "the commit
+        // where I opened PR 10232" actually has to match.
+        budget = take(ev.tool, out, budget);
+        budget = take(stringifyToolInput(ev.input), out, budget);
+        budget = collectEventText(ev.children, out, budget);
+        break;
+      case "tool_result":
+        budget = take(ev.output.slice(0, EVENT_TEXT_CAP), out, budget);
+        break;
+      case "error":
+        budget = take(ev.message, out, budget);
+        break;
+      // `thinking` is deliberately skipped: it is the model's scratch work, it
+      // is bulky, and a snippet drawn from it reads as noise in the results.
+      default:
+        break;
+    }
+  }
+  return budget;
+}
+
+/** Push as much of `text` as the budget allows; answer with what remains. */
+function take(text: string | undefined, out: string[], budget: number): number {
+  if (!text || budget <= 0) return budget;
+  out.push(text.length <= budget ? text : text.slice(0, budget));
+  return budget - Math.min(text.length, budget);
+}
+
+/**
+ * All searchable text for one normalized message.
+ *
+ * `content` alone misses most of a transcript: `getMessages` merges tool calls
+ * and their results into `events` and leaves `content` empty for any turn that
+ * only used tools — the commit-and-open-a-PR turns are exactly those, so they
+ * were the least searchable part of the history rather than the most.
+ */
+export function messageSearchText(msg: ChatMessage): string {
+  const parts: string[] = [];
+  let budget = MESSAGE_TEXT_CAP;
+  if (msg.content) budget = take(msg.content, parts, budget);
+  collectEventText(msg.events, parts, budget);
+  // Threading the budget is what stops the *collection*; the final slice is
+  // what makes the cap exact, since the separators joined in below are not
+  // part of any part's length.
+  return parts.filter((p) => p && p.trim()).join("\n").trim().slice(0, MESSAGE_TEXT_CAP);
+}
+
 /** Index a concrete set of normalized messages (core, provider-agnostic). */
 export function indexMessages(
   sessionId: string,
@@ -55,25 +157,26 @@ export function indexMessages(
     "INSERT INTO messages_fts (text, session_id, project_path, message_id, role, ts) VALUES (?, ?, ?, ?, ?, ?)",
   );
   const upsertMeta = db.query(`
-    INSERT INTO session_meta (session_id, project_path, jsonl_mtime, indexed_at, msg_count)
-    VALUES (?, ?, ?, ?, ?)
+    INSERT INTO session_meta (session_id, project_path, jsonl_mtime, indexed_at, msg_count, indexer_version)
+    VALUES (?, ?, ?, ?, ?, ?)
     ON CONFLICT(session_id) DO UPDATE SET
-      project_path = excluded.project_path,
-      jsonl_mtime  = excluded.jsonl_mtime,
-      indexed_at   = excluded.indexed_at,
-      msg_count    = excluded.msg_count
+      project_path    = excluded.project_path,
+      jsonl_mtime     = excluded.jsonl_mtime,
+      indexed_at      = excluded.indexed_at,
+      msg_count       = excluded.msg_count,
+      indexer_version = excluded.indexer_version
   `);
 
   const tx = db.transaction(() => {
     del.run(sessionId);
     let count = 0;
     for (const msg of messages) {
-      const text = (msg.content ?? "").trim();
+      const text = messageSearchText(msg);
       if (!text) continue;
       ins.run(text, sessionId, projectPath, msg.id, msg.role, msg.timestamp ?? "");
       count++;
     }
-    upsertMeta.run(sessionId, projectPath, jsonlMtime, Date.now(), count);
+    upsertMeta.run(sessionId, projectPath, jsonlMtime, Date.now(), count, INDEXER_VERSION);
   });
   tx();
 }
@@ -89,12 +192,18 @@ export async function indexSession(
   indexMessages(sessionId, projectPath, messages, jsonlMtime);
 }
 
-/** True when the stored index for a session is missing or older than the JSONL. */
+/**
+ * True when the stored index for a session is missing, older than the JSONL, or
+ * was written by an earlier indexer. Without the version check, widening what
+ * gets indexed would never reach the sessions already on disk: their
+ * `jsonl_mtime` still matches, so they stay stamped as fresh forever.
+ */
 export function isStale(sessionId: string, jsonlMtime: number): boolean {
   const row = getSearchIndexDb()
-    .query("SELECT jsonl_mtime FROM session_meta WHERE session_id = ?")
-    .get(sessionId) as { jsonl_mtime: number } | null;
+    .query("SELECT jsonl_mtime, indexer_version FROM session_meta WHERE session_id = ?")
+    .get(sessionId) as { jsonl_mtime: number; indexer_version: number } | null;
   if (!row) return true;
+  if (row.indexer_version !== INDEXER_VERSION) return true;
   return row.jsonl_mtime !== jsonlMtime;
 }
 
@@ -127,12 +236,59 @@ export function search(projectPath: string, rawQuery: string, limit = 50): ChatS
   return rows;
 }
 
-/** Count of sessions indexed for a project. */
+/**
+ * Sessions indexed for a project *by this version of the indexer*.
+ *
+ * The version matters because it is exactly when the number is read: an
+ * `INDEXER_VERSION` bump makes every row stale at once, and counting them all
+ * reported "1684/1684" while the corpus was being re-read from scratch — a
+ * progress indicator sitting at 100% for the whole of the work it exists to
+ * show. `isStale` already compares the same column.
+ */
 export function getIndexedCount(projectPath: string): number {
+  const row = getSearchIndexDb()
+    .query("SELECT COUNT(*) AS n FROM session_meta WHERE project_path = ? AND indexer_version = ?")
+    .get(projectPath, INDEXER_VERSION) as { n: number };
+  return row.n;
+}
+
+/**
+ * How many sessions the last enumeration saw, per project.
+ *
+ * Keyed by the index it describes rather than held for the process: a wiped search
+ * database — or a test's fresh one — must not inherit a total counted against rows
+ * that no longer exist.
+ */
+const enumerated = new WeakMap<object, Map<string, number>>();
+function enumeratedTotals(): Map<string, number> {
+  const db = getSearchIndexDb();
+  let totals = enumerated.get(db);
+  if (!totals) enumerated.set(db, (totals = new Map()));
+  return totals;
+}
+
+/**
+ * The indexing chip's denominator when nobody has enumerated.
+ *
+ * `GET /chat/search` knows the real total only because it lists the sessions to
+ * match titles against. An empty query has no titles to match, so it should not
+ * pay for a list that pages the SDK to exhaustion.
+ *
+ * Rows alone are not that number, though, and they are wrong in the direction
+ * that matters. On a fresh index a pass reads `RECONCILE_BUDGET` sessions and
+ * leaves the rest with no row at all, so the rows say 200 of 200 — finished —
+ * while the other 1484 were never read. So this is the larger of what the index
+ * holds and what the last enumeration saw. The error that remains runs the safe
+ * way: a session deleted since that enumeration still counts until the next
+ * query lists them again, which reads as "not quite done" rather than "done". And
+ * a restart forgets the enumeration, so between one and the next real query the
+ * rows are all there is.
+ */
+export function getKnownSessionCount(projectPath: string): number {
   const row = getSearchIndexDb()
     .query("SELECT COUNT(*) AS n FROM session_meta WHERE project_path = ?")
     .get(projectPath) as { n: number };
-  return row.n;
+  return Math.max(row.n, enumeratedTotals().get(projectPath) ?? 0);
 }
 
 // --- Reconcile & backfill --------------------------------------------------
@@ -147,29 +303,54 @@ function staleKey(updatedAt?: string, createdAt?: string): number {
   return Number.isFinite(t) ? t : 0;
 }
 
+/**
+ * Sessions one pass will re-read, at most.
+ *
+ * Steady state is a handful — whatever was touched since the last search. The
+ * case this exists for is an `INDEXER_VERSION` bump, which makes every session
+ * stale at once: the first search after an upgrade would otherwise re-read and
+ * re-parse the entire corpus (hundreds of MB over thousands of transcripts)
+ * before the indexing indicator could ever finish. Bounding the pass turns that
+ * into progress the user watches advance, because `startBackfill` runs again on
+ * the next search and the ones left stale are still stale.
+ */
+const RECONCILE_BUDGET = 200;
+
 /**
  * Re-index any session whose transcript changed since last index. Enumerates
  * all providers for the project via `chatService.listSessions`. Sequential to
  * avoid an FS/parse storm over large transcript corpora.
+ *
+ * `sessions` may be supplied by a caller that has already enumerated them.
+ * `GET /chat/search` has: without this it listed every session in the project
+ * twice per keystroke, once for title matching and once in here, and a
+ * dir-scoped list with no limit pages the SDK until it is exhausted.
  */
 export async function reconcile(
   projectPath: string,
   onProgress?: (done: number, total: number) => void,
-): Promise<{ total: number; indexed: number }> {
-  const sessions = await chatService.listSessions(undefined, projectPath);
+  sessions?: { id: string; providerId: string; updatedAt?: string; createdAt?: string }[],
+): Promise<{ total: number; indexed: number; remaining: number }> {
+  const list = sessions ?? (await chatService.listSessions(undefined, projectPath));
+  enumeratedTotals().set(projectPath, list.length);
   let reindexed = 0;
-  for (let i = 0; i < sessions.length; i++) {
-    const s = sessions[i]!;
+  let remaining = 0;
+  for (let i = 0; i < list.length; i++) {
+    const s = list[i]!;
     const mtime = staleKey(s.updatedAt, s.createdAt);
     if (isStale(s.id, mtime)) {
-      try {
-        await indexSession(s.providerId, s.id, projectPath, mtime);
-        reindexed++;
-      } catch { /* skip unreadable/broken transcript */ }
+      if (reindexed >= RECONCILE_BUDGET) {
+        remaining++;
+      } else {
+        try {
+          await indexSession(s.providerId, s.id, projectPath, mtime);
+          reindexed++;
+        } catch { /* skip unreadable/broken transcript */ }
+      }
     }
-    onProgress?.(i + 1, sessions.length);
+    onProgress?.(i + 1, list.length);
   }
-  return { total: sessions.length, indexed: reindexed };
+  return { total: list.length, indexed: reindexed, remaining };
 }
 
 // Module-level dedup guard — one backfill per project at a time (no lifecycle
@@ -177,10 +358,14 @@ export async function reconcile(
 const backfillRuns = new Map<string, Promise<unknown>>();
 
 /** Trigger a lazy backfill/reconcile for a project (idempotent, fire-and-forget safe). */
-export function startBackfill(projectPath: string): Promise<unknown> {
+export function startBackfill(
+  projectPath: string,
+  sessions?: { id: string; providerId: string; updatedAt?: string; createdAt?: string }[],
+): Promise<unknown> {
   const existing = backfillRuns.get(projectPath);
   if (existing) return existing;
-  const run = reconcile(projectPath).finally(() => backfillRuns.delete(projectPath));
+  const run = reconcile(projectPath, undefined, sessions)
+    .finally(() => backfillRuns.delete(projectPath));
   backfillRuns.set(projectPath, run);
   return run;
 }

package/src/services/fs-ops/fs-real-path.ts

@@ -42,3 +42,26 @@ export function realPathOrSelfSync(target: string): string {
     }
   }
 }
+
+/**
+ * Is `child` the same directory as `parent`, or somewhere inside it?
+ *
+ * Case-folded on Windows, where `C:\Users\PC\ppm` and `c:\users\pc\ppm` are
+ * one directory under two spellings: a case-sensitive prefix test refuses the
+ * second, which reads to the caller as "outside the project" for a path the
+ * server itself handed out. Nowhere else — on Linux two spellings really are
+ * two directories.
+ *
+ * The separator follows the platform rather than `sep`, and `platform` is a
+ * parameter, so win32 semantics can be asserted from a Linux test run the way
+ * `assertSafeFilePaths` takes its `PathApi`. A backslash is a separator only on
+ * Windows: on Linux `/home/x\evil` is a file *beside* `/home/x`, not inside it.
+ */
+export function isInsideDir(child: string, parent: string, platform: string = process.platform): boolean {
+  const separators = platform === "win32" ? ["\\", "/"] : ["/"];
+  const fold = (path: string) => (platform === "win32" ? path.toLowerCase() : path);
+  const from = fold(parent).replace(/[\\/]+$/, "");
+  const to = fold(child);
+  if (to === from) return true;
+  return to.startsWith(from) && separators.includes(to.charAt(from.length));
+}

package/src/services/git-blame/commit-line-diff.ts

@@ -0,0 +1,124 @@
+/**
+ * Reading one `git show` into the pieces the editor's blame hover needs: the
+ * commit's own fields, its full message, and the change it made to one line.
+ *
+ * One command carries all of it. `--format=%H%x00%an%x00...%B%x00` puts the
+ * fields before the diff with NUL separators, which is the only separator that
+ * cannot occur inside a commit message — and a commit message is exactly where
+ * a chosen delimiter would eventually turn up.
+ *
+ * `--unified=0` is what makes the line lookup possible: with no context lines,
+ * every line in a hunk body is a real change, so the hunk containing the blamed
+ * line names that line's own edit rather than an edit up to three lines away.
+ */
+
+/**
+ * The byte git writes for `%x00`, built rather than typed. A literal NUL in a
+ * source file is invisible in every diff and makes the file read as binary to
+ * grep; this has already cost this codebase an afternoon once.
+ */
+const NUL = String.fromCharCode(0);
+
+export interface ShowFields {
+  hash: string;
+  author: string;
+  authorMail: string;
+  /** Unix seconds. */
+  authorTime: number;
+  /** Subject and body, as committed, with trailing blank lines removed. */
+  message: string;
+  /** Everything after the message. Empty for a merge commit, which shows none. */
+  diff: string;
+}
+
+export function parseShow(raw: string): ShowFields | null {
+  // Five fields then the diff; a short split means the format did not run, which
+  // is what a bad revision looks like when git still exits zero.
+  const parts = raw.split(NUL);
+  if (parts.length < 6) return null;
+  const [hash, author, authorMail, authorTime, message] = parts;
+  if (!hash?.trim()) return null;
+  return {
+    hash: hash.trim(),
+    author: author ?? "",
+    authorMail: authorMail ?? "",
+    authorTime: Number(authorTime) || 0,
+    message: (message ?? "").replace(/\s+$/, ""),
+    diff: parts.slice(5).join(NUL).replace(/^\s*\n/, ""),
+  };
+}
+
+export interface LineChange {
+  /** What the commit removed at this line, oldest first. */
+  removed: string[];
+  /** The line as this commit wrote it. */
+  added: string[];
+}
+
+interface Hunk {
+  newStart: number;
+  newCount: number;
+  removed: string[];
+  added: string[];
+}
+
+/**
+ * `@@ -12,3 +14,2 @@` — and `@@ -14,0 +15 @@`, because git omits a count of one.
+ * Reading a missing count as zero would make every single-line hunk unmatchable.
+ */
+const HUNK_HEADER = /^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@/;
+
+function parseHunks(diff: string): Hunk[] {
+  const hunks: Hunk[] = [];
+  let current: Hunk | null = null;
+  for (const line of diff.split("\n")) {
+    const header = HUNK_HEADER.exec(line);
+    if (header) {
+      current = {
+        newStart: Number(header[3]),
+        newCount: header[4] === undefined ? 1 : Number(header[4]),
+        removed: [],
+        added: [],
+      };
+      hunks.push(current);
+      continue;
+    }
+    if (!current) continue;
+    // A new `diff --git` ends the previous file's hunks. Checked before the
+    // `-`/`+` tests because `--- a/x` and `+++ b/x` follow it and would
+    // otherwise be read as a removed and an added line.
+    if (line.startsWith("diff --git")) current = null;
+    else if (line.startsWith("+")) current.added.push(line.slice(1));
+    else if (line.startsWith("-")) current.removed.push(line.slice(1));
+  }
+  return hunks;
+}
+
+/**
+ * The change this commit made at `newLine`, where `newLine` is the line number
+ * in the file *as that commit left it* — which is what `git blame --porcelain`
+ * reports as the original line, and not the line number in the file now.
+ *
+ * Null when the commit did not touch that line: a merge commit shows no diff at
+ * all, and a line can also be attributed to a commit that only moved it.
+ */
+export function lineChangeAt(diff: string, newLine: number, maxRemoved = 3): LineChange | null {
+  for (const hunk of parseHunks(diff)) {
+    // `+n,0` means the commit deleted lines here and added none, so this hunk
+    // contains no line of the new file at all.
+    if (hunk.newCount === 0) continue;
+    if (newLine < hunk.newStart || newLine >= hunk.newStart + hunk.newCount) continue;
+
+    const offset = newLine - hunk.newStart;
+    const line = hunk.added[offset];
+    return {
+      // Capped: a hunk that replaced two hundred lines would otherwise put all
+      // of them into a hover.
+      removed: hunk.removed.slice(0, maxRemoved),
+      // The blamed line itself, not the whole hunk. Falls back to the hunk's
+      // first added line if the header's count and the body ever disagree.
+      added: line === undefined ? hunk.added.slice(0, 1) : [line],
+    };
+  }
+  return null;
+}

package/src/services/git-blame/git-blame.service.ts

@@ -7,7 +7,8 @@
  */
 import simpleGit from "simple-git";
 import { parseBlamePorcelain } from "./blame-porcelain.ts";
-import type { BlameResult } from "../../shared/blame.ts";
+import { lineChangeAt, parseShow } from "./commit-line-diff.ts";
+import type { BlameLineDetail, BlameResult } from "../../shared/blame.ts";
 
 /**
  * Reject a path that is absolute, escapes the repository, or could be read as
@@ -56,6 +57,17 @@ export function assertSafeRev(rev: string): string {
   return rev;
 }
 
+/**
+ * Reject anything that is not a commit hash. Stricter than `assertSafeRev` on
+ * purpose: this value is the one argument to `git show`, and the hover only ever
+ * has a hash from a blame it already ran, so `HEAD~1` and friends are not
+ * requests to honour — they are a sign the caller is not the hover.
+ */
+export function assertCommitHash(hash: string): string {
+  if (!/^[0-9a-f]{7,40}$/.test(hash)) throw new Error(`Invalid commit hash: "${hash}"`);
+  return hash;
+}
+
 class GitBlameService {
   /**
    * Blame every line of `filePath`, in the working tree or as it stood at `rev`.
@@ -85,6 +97,89 @@ class GitBlameService {
       throw e;
     }
   }
+
+  /**
+   * The commit message and the one-line diff behind a blamed line, for the
+   * editor's hover.
+   *
+   * `filePath` must be the path the file had *at that commit* — after a rename
+   * the current path did not exist there, and `git show` would report no diff
+   * at all rather than an error. `git blame --porcelain` supplies it as
+   * `filename`, which is why the caller passes it rather than the open file's
+   * path.
+   *
+   * `origLine` is likewise the line number at that commit, not now.
+   *
+   * Returns null when there is nothing to show — an unknown hash, or a path git
+   * has never heard of. A commit with no diff for this line is *not* null: the
+   * message is still worth showing, so `removed` and `added` come back empty.
+   * That is the ordinary case for a merge commit, which shows no diff.
+   */
+  async lineDetail(
+    projectPath: string,
+    hash: string,
+    filePath: string,
+    origLine: number,
+  ): Promise<BlameLineDetail | null> {
+    assertCommitHash(hash);
+    assertSafeFilePath(filePath);
+    const git = simpleGit(projectPath);
+    // Two commands, not one, and the reason is not obvious: a pathspec puts
+    // `git show` through the diff machinery, which drops the commit entirely
+    // when that path is unchanged — the same filtering that makes `git log --
+    // <path>` skip it. So `show --format=… <hash> -- <path>` prints *nothing*,
+    // not a header with an empty diff, and asking one command for both loses
+    // the fields exactly when the hover still wants them: a line the commit
+    // never touched should still say who wrote the commit and why.
+    const metaArgs = [
+      "show",
+      // NUL-separated fields, no diff. See `commit-line-diff.ts`.
+      "--format=%H%x00%an%x00%ae%x00%at%x00%B%x00",
+      "--no-patch",
+      hash,
+    ];
+    // `--unified=0` so every line in a hunk body is a real change.
+    const diffArgs = ["show", "--format=", "--unified=0", hash, "--", filePath];
+
+    /** True for the family of git errors that all mean "nothing to show here". */
+    const isAbsent = (e: unknown): boolean => {
+      const message = e instanceof Error ? e.message : String(e);
+      // git's wording varies by which lookup failed: a hash it cannot resolve is
+      // "bad revision", one that resolves to nothing is "bad object", and a path
+      // that commit never had is "exists on disk, but not in".
+      return /bad revision|unknown revision|bad object|not a valid object name|no such path|does not exist|but not in/i.test(message);
+    };
+
+    let metaRaw: string;
+    try {
+      metaRaw = await git.raw(metaArgs);
+    } catch (e) {
+      if (isAbsent(e)) return null;
+      throw e;
+    }
+
+    const fields = parseShow(metaRaw);
+    if (!fields) return null;
+
+    let diffRaw = "";
+    try {
+      diffRaw = await git.raw(diffArgs);
+    } catch (e) {
+      // A path this commit never had is not a failed hover, just an absent diff
+      // section. Anything else is still worth reporting.
+      if (!isAbsent(e)) throw e;
+    }
+    const change = lineChangeAt(diffRaw, origLine);
+    return {
+      hash: fields.hash,
+      author: fields.author,
+      authorMail: fields.authorMail,
+      authorTime: fields.authorTime,
+      message: fields.message,
+      removed: change?.removed ?? [],
+      added: change?.added ?? [],
+    };
+  }
 }
 
 export const gitBlameService = new GitBlameService();