@hienlh/ppm 0.23.14 → 0.23.15

package/src/services/session-file-baselines/session-file-blame.ts

@@ -9,7 +9,9 @@
  * was kept, or a state that was not text.
  *
  * Replays are kept, per file and per base, and only the lines a log gained since are read, so
- * asking again after every edit costs that edit's diff and nothing more.
+ * asking again after every edit costs that edit's diff and nothing more. The first replay of a
+ * long log is the expensive one — one diff per observation, each allowed 200 ms — so it hands
+ * the event loop back every few milliseconds rather than holding the server for the whole log.
  */
 import { diffLines } from "diff";
 import { statSync } from "node:fs";
@@ -45,6 +47,25 @@ interface Replay {
 const replays = new Map<string, Replay>();
 const REPLAYS_KEPT = 256;
 
+/** Work done between two hand-backs to the event loop while a log is replayed. */
+const YIELD_AFTER_MS = 20;
+
+/** A 0 ms timer, not `setImmediate`: that one starves HTTP (see `file-list-index.service.ts`). */
+const breathe = () => new Promise<void>((done) => setTimeout(done, 0));
+
+const replaying = new Map<string, Promise<unknown>>();
+
+/** One replay of a key at a time: two would advance the same state from the same position. */
+function oneAtATime<T>(key: string, task: () => Promise<T>): Promise<T> {
+  const next = (replaying.get(key) ?? Promise.resolve()).then(task, task);
+  replaying.set(key, next);
+  const settle = () => {
+    if (replaying.get(key) === next) replaying.delete(key);
+  };
+  next.then(settle, settle);
+  return next;
+}
+
 function labelOf(r: Replay, call: string): number {
   let label = r.callLabels.get(call);
   if (label === undefined) {
@@ -125,28 +146,39 @@ function start(baseText: string, markedAt: number | undefined): Replay {
 }
 
 /** The replay of `path` from `baseText` up to the end of its log, reusing what was read before. */
-function replayed(sessionId: string, path: string, baseText: string, markedAt: number | undefined): Replay | null {
+function replayed(sessionId: string, path: string, baseText: string, markedAt: number | undefined): Promise<Replay | null> {
   const file = historyFile(sessionId, path);
-  if (!file) return null;
-  let size: number;
-  try {
-    size = statSync(file).size;
-  } catch {
-    return null;
-  }
+  if (!file) return Promise.resolve(null);
   const key = `${sessionId}\0${path}\0${stateHash(baseText)}\0${markedAt ?? ""}`;
-  let r = replays.get(key);
-  // A log shorter than what was read is one written again, after its session was deleted.
-  if (!r || size < r.pos.bytes) r = start(baseText, markedAt);
-  if (size > r.pos.bytes) {
-    const { entries, end } = readHistory(sessionId, path, r.pos);
-    for (const e of entries) step(r, e, markedAt);
-    r.pos = end;
-  }
-  replays.delete(key);
-  replays.set(key, r);
-  if (replays.size > REPLAYS_KEPT) replays.delete(replays.keys().next().value!);
-  return r;
+  return oneAtATime(key, async () => {
+    let size: number;
+    try {
+      size = statSync(file).size;
+    } catch {
+      return null;
+    }
+    let r = replays.get(key);
+    // A log shorter than what was read is one written again, after its session was deleted.
+    if (!r || size < r.pos.bytes) r = start(baseText, markedAt);
+    if (size > r.pos.bytes) {
+      const { entries, end } = readHistory(sessionId, path, r.pos);
+      // Stepped in place: one left half way, by a throw, could never be told from a whole one.
+      replays.delete(key);
+      let since = performance.now();
+      for (const e of entries) {
+        step(r, e, markedAt);
+        if (performance.now() - since > YIELD_AFTER_MS) {
+          await breathe();
+          since = performance.now();
+        }
+      }
+      r.pos = end;
+    }
+    replays.delete(key);
+    replays.set(key, r);
+    if (replays.size > REPLAYS_KEPT) replays.delete(replays.keys().next().value!);
+    return r;
+  });
 }
 
 /**
@@ -154,15 +186,15 @@ function replayed(sessionId: string, path: string, baseText: string, markedAt: n
  * they ran; an empty list where the history names none. `markedAt` is set when the base is the
  * state the file was marked reviewed in rather than its "before".
  */
-export function blockCalls(p: {
+export async function blockCalls(p: {
   sessionId: string;
   path: string;
   baseText: string;
   currentText: string;
   blocks: ReviewBlock[];
   markedAt?: number;
-}): string[][] | null {
-  const r = replayed(p.sessionId, p.path, p.baseText, p.markedAt);
+}): Promise<string[][] | null> {
+  const r = await replayed(p.sessionId, p.path, p.baseText, p.markedAt);
   if (!r) return null;
   // What changed since the last observation was changed by nobody the history names.
   let current = r.lines;
@@ -197,4 +229,5 @@ export function blockCalls(p: {
 /** Test seam: forget every kept replay. */
 export function _resetSessionFileBlame(): void {
   replays.clear();
+  replaying.clear();
 }

package/src/services/session-file-baselines/session-file-changes.service.ts

@@ -19,7 +19,7 @@
  * kept. Keeping, reverting and undoing those answers is `session-review-actions.ts`.
  */
 import { createHash } from "node:crypto";
-import { existsSync } from "node:fs";
+import { existsSync, statSync } from "node:fs";
 import { readFile, stat } from "node:fs/promises";
 import { dirname, isAbsolute, relative, resolve, sep } from "node:path";
 import simpleGit from "simple-git";
@@ -27,10 +27,12 @@ import { decodeText, isBinaryContent } from "../binary-content.ts";
 import { assertReadPermitted } from "../fs-ops/fs-ops-read-write.service.ts";
 import { realPathOrSelf } from "../fs-ops/fs-real-path.ts";
 import { getBranchRow } from "../session-branch.service.ts";
-import { BASELINE_MAX_BYTES, listBaselines, readBaseline, type FileBaseline } from "./session-file-baselines.service.ts";
+import { BASELINE_MAX_BYTES, baselinePaths, readBaseline, recordFile, sessionDir, type FileBaseline } from "./session-file-baselines.service.ts";
 import { clearReviewMark, findReviewMark, writeReviewMark, type ReviewMark } from "./session-review-marks.ts";
-import { hashBase, keptBlocks } from "./session-review-blocks.ts";
+import { answerRecordFiles, hashBase, keptBlocks } from "./session-review-blocks.ts";
 import { blockCalls } from "./session-file-blame.ts";
+import { historyFile } from "./session-file-history.ts";
+import { onSessionFileWritten } from "./session-file-writes.ts";
 import { computeBlocks, type BlockDiff } from "../../shared/review-blocks.ts";
 import type {
   SessionChangeStatus,
@@ -218,7 +220,7 @@ export async function compare(chain: string[], projectPath: string, path: string
     const baseHash = hashBase(baseText!);
     change.base = baseHash.slice(0, 16);
     const kept = reviewed ? null : keptBlocks(chain, path, baseHash);
-    const calls = blockCalls({
+    const calls = await blockCalls({
       sessionId: chain[0]!,
       path,
       baseText: baseText!,
@@ -246,36 +248,232 @@ export function markReviewed(sessionId: string, compared: Compared): boolean {
     : writeReviewMark(sessionId, { path, existed: false });
 }
 
-/** Every file the session changed: those it kept a "before" for, then any other `paths`. */
-export async function sessionFileChanges(p: {
+/**
+ * Each session's list, kept between asks: one entry per file, with a stamp of what it was worked
+ * out from. Working a file out reads its "before" and its text now, diffs the two and replays
+ * its history, which on a session that touched a few thousand files took minutes per ask;
+ * stamping it is a few `stat`s. So an ask works out again only the files whose stamp moved,
+ * whoever moved them — the agent, the user's answers, an editor, a checkout — and the agent's
+ * own writes are worked out as they land and pushed to the browser (`onSessionFileChanges`).
+ */
+interface Entry {
+  stamp: string;
+  /** Null when the file is no change of the session's, or one it may not read. */
+  change: SessionFileChange | null;
+}
+
+interface KeptList {
   sessionId: string;
   projectPath: string;
-  paths?: string[];
-}): Promise<SessionFileChange[]> {
-  const chain = lineage(p.sessionId);
+  /** The lineage at the last ask, which is what an entry's stamp was taken along. */
+  chain: string[];
+  entries: Map<string, Entry>;
+  /** Asks and pushes one at a time: two would work the same file out side by side. */
+  queue: Promise<unknown>;
+}
+
+const lists = new Map<string, KeptList>();
+const LISTS_KEPT = 32;
+
+/** Work done between two hand-backs to the event loop: a first ask can take minutes. */
+const YIELD_AFTER_MS = 20;
+
+/** A 0 ms timer, not `setImmediate`: that one starves HTTP (see `file-list-index.service.ts`). */
+const breathe = () => new Promise<void>((done) => setTimeout(done, 0));
+
+function listFor(sessionId: string, projectPath: string): KeptList {
+  // By the session's directory, which names the PPM directory too.
+  const key = `${sessionDir(sessionId)}\0${resolve(projectPath)}`;
+  const list = lists.get(key) ?? { sessionId, projectPath, chain: [sessionId], entries: new Map(), queue: Promise.resolve() };
+  lists.delete(key);
+  lists.set(key, list);
+  if (lists.size > LISTS_KEPT) lists.delete(lists.keys().next().value!);
+  return list;
+}
+
+function queued<T>(list: KeptList, task: () => Promise<T>): Promise<T> {
+  const next = list.queue.then(task, task);
+  list.queue = next.catch(() => {});
+  return next;
+}
+
+function statStamp(file: string | null): string {
+  if (!file) return "";
+  try {
+    const st = statSync(file);
+    return `${st.ino}:${st.size}:${st.mtimeMs}`;
+  } catch {
+    return "-";
+  }
+}
+
+type HeadCache = Map<string, Promise<string | null>>;
+
+/** The commit HEAD names in the repository holding `dir`, once per ask and repository. */
+async function headOf(dir: string, repos: RepoCache, heads: HeadCache): Promise<string | null> {
+  const top = await repoTop(dir, repos);
+  if (!top) return null;
+  let head = heads.get(top);
+  if (!head) {
+    head = simpleGit(top).revparse(["HEAD"]).then((h) => h.trim() || null, () => null);
+    heads.set(top, head);
+  }
+  return head;
+}
+
+/**
+ * What `compare` reads for `path`, by `stat` alone: the file now; each session's "before",
+ * review mark and kept blocks along the lineage; the history the blocks are named from; and for
+ * a file with no "before", the HEAD it falls back to. Equal stamps mean an equal answer. Taken
+ * before the file is worked out, so a write landing in between moves the next stamp, not this one.
+ */
+async function stampOf(chain: string[], projectPath: string, path: string, repos: RepoCache, heads: HeadCache): Promise<string> {
+  const parts = [chain.join(","), statStamp(path)];
+  let kept = false;
+  for (const id of chain) {
+    const dir = sessionDir(id);
+    const before = dir ? statStamp(recordFile(dir, path)) : "-";
+    if (before !== "-") kept = true;
+    const answers = answerRecordFiles(id, path);
+    parts.push(before, statStamp(answers?.[0] ?? null), statStamp(answers?.[1] ?? null));
+  }
+  parts.push(statStamp(historyFile(chain[0]!, path)));
+  if (!kept && isInside(projectPath, path)) parts.push((await headOf(dirname(path), repos, heads)) ?? "");
+  return parts.join("|");
+}
+
+/** The file's entry, worked out again only when its stamp moved. */
+async function entryOf(list: KeptList, path: string, repos: RepoCache, heads: HeadCache): Promise<{ change: SessionFileChange | null; redone: boolean }> {
+  const stamp = await stampOf(list.chain, list.projectPath, path, repos, heads);
+  const held = list.entries.get(path);
+  if (held && held.stamp === stamp) return { change: held.change, redone: false };
+  // A path the guard refuses, or one that vanished mid-read, is simply not listed.
+  const change = (await compare(list.chain, list.projectPath, path, repos).catch(() => null))?.change ?? null;
+  list.entries.set(path, { stamp, change });
+  workedOut++;
+  return { change, redone: true };
+}
+
+let workedOut = 0;
+
+/** Test seam: how many files the kept lists have worked out, from scratch or again. */
+export function _sessionFilesWorkedOut(): number {
+  return workedOut;
+}
+
+/**
+ * The files the list is made of: those the session and its ancestors kept a "before" for, oldest
+ * first, then `extra`. Past the cap the newest are kept — on a session that touched thousands of
+ * files, the ones its latest turns changed are the ones being reviewed.
+ */
+function listedPaths(chain: string[], projectPath: string, extra: string[]): string[] {
   const seen = new Set<string>();
   const ordered: string[] = [];
   const add = (raw: string) => {
-    if (!raw || ordered.length >= MAX_SESSION_CHANGE_PATHS) return;
-    const path = resolve(p.projectPath, raw);
+    if (!raw) return;
+    const path = resolve(projectPath, raw);
     if (seen.has(path)) return;
     seen.add(path);
     ordered.push(path);
   };
-  for (const id of [...chain].reverse()) for (const b of listBaselines(id)) add(b.path);
-  for (const raw of p.paths ?? []) add(raw);
+  for (const id of [...chain].reverse()) for (const path of baselinePaths(id)) add(path);
+  for (const raw of extra) add(raw);
+  return ordered.length > MAX_SESSION_CHANGE_PATHS ? ordered.slice(-MAX_SESSION_CHANGE_PATHS) : ordered;
+}
 
-  const out: SessionFileChange[] = [];
-  const repos: RepoCache = new Map();
-  for (const path of ordered) {
-    try {
-      const compared = await compare(chain, p.projectPath, path, repos);
-      if (compared) out.push(compared.change);
-    } catch {
-      // A path the guard refuses, or one that vanished mid-read, is simply not listed.
+/** Every file the session changed: those it kept a "before" for, then any other `paths`. */
+export function sessionFileChanges(p: {
+  sessionId: string;
+  projectPath: string;
+  paths?: string[];
+}): Promise<SessionFileChange[]> {
+  const list = listFor(p.sessionId, p.projectPath);
+  return queued(list, async () => {
+    list.chain = lineage(list.sessionId);
+    const repos: RepoCache = new Map();
+    const heads: HeadCache = new Map();
+    const out: SessionFileChange[] = [];
+    let since = performance.now();
+    for (const path of listedPaths(list.chain, list.projectPath, p.paths ?? [])) {
+      const { change } = await entryOf(list, path, repos, heads);
+      if (change) out.push(change);
+      if (performance.now() - since > YIELD_AFTER_MS) {
+        await breathe();
+        since = performance.now();
+      }
     }
+    return out;
+  });
+}
+
+/** What a push carries: the rows of the files just worked out again, and the files no longer changed. */
+export interface SessionFileChangesPush {
+  sessionId: string;
+  projectPath: string;
+  files: SessionFileChange[];
+  gone: string[];
+}
+
+const pushListeners = new Set<(push: SessionFileChangesPush) => void>();
+
+/** Subscribe to the rows pushed after the agent changed files; returns the unsubscribe function. */
+export function onSessionFileChanges(cb: (push: SessionFileChangesPush) => void): () => void {
+  pushListeners.add(cb);
+  return () => {
+    pushListeners.delete(cb);
+  };
+}
+
+/** Folds one shell command's files, or a burst of edits, into one push. */
+const PUSH_DELAY_MS = 150;
+const written = new Map<string, Set<string>>();
+let pushTimer: ReturnType<typeof setTimeout> | null = null;
+
+onSessionFileWritten((sessionId, path) => {
+  let paths = written.get(sessionId);
+  if (!paths) written.set(sessionId, (paths = new Set()));
+  paths.add(resolve(path));
+  pushTimer ??= setTimeout(pushWritten, PUSH_DELAY_MS);
+});
+
+/** Work out again the files the agent wrote, in every kept list along whose lineage it wrote them, and push them. */
+function pushWritten(): void {
+  pushTimer = null;
+  const batch = new Map(written);
+  written.clear();
+  for (const list of lists.values()) {
+    const paths = new Set<string>();
+    for (const id of list.chain) for (const path of batch.get(id) ?? []) paths.add(path);
+    if (paths.size === 0) continue;
+    void queued(list, async () => {
+      const repos: RepoCache = new Map();
+      const heads: HeadCache = new Map();
+      const push: SessionFileChangesPush = { sessionId: list.sessionId, projectPath: list.projectPath, files: [], gone: [] };
+      for (const path of paths) {
+        const was = list.entries.get(path)?.change ?? null;
+        const { change, redone } = await entryOf(list, path, repos, heads);
+        if (!redone) continue;
+        if (change) push.files.push(change);
+        else if (was) push.gone.push(path);
+      }
+      if (push.files.length === 0 && push.gone.length === 0) return;
+      for (const cb of pushListeners) {
+        try {
+          cb(push);
+        } catch (e) {
+          console.warn(`[session-baselines] push listener failed: ${(e as Error).message}`);
+        }
+      }
+    }).catch((e) => console.warn(`[session-baselines] push failed: ${(e as Error).message}`));
   }
-  return out;
+}
+
+/** Test seam: forget every kept list and pending push. */
+export function _resetSessionFileChanges(): void {
+  lists.clear();
+  written.clear();
+  if (pushTimer) clearTimeout(pushTimer);
+  pushTimer = null;
 }
 
 /** Both sides of one file, for the diff editor. Null when the session did not change it. */

package/src/services/session-file-baselines/session-file-history.ts

@@ -27,6 +27,7 @@ import { isCredentialPath } from "../fs-credential-path-guard.ts";
 import { realPathOrSelf } from "../fs-ops/fs-real-path.ts";
 import { splitLines } from "../../shared/review-blocks.ts";
 import { BASELINE_MAX_BYTES, sessionDir, type BaselineSource } from "./session-file-baselines.service.ts";
+import { sessionFileWritten } from "./session-file-writes.ts";
 
 /** A log past this keeps only hashes: a file rewritten whole hundreds of times is not worth more. */
 export const HISTORY_MAX_FILE_BYTES = 16 * 1024 * 1024;
@@ -270,7 +271,7 @@ export function observeFile(sessionId: string, filePath: string, call: string, p
   if (isCredentialPath(path)) return Promise.resolve();
   const file = join(dir, logName(path));
   const at = Date.now();
-  return serialized(file, async () => {
+  const done = serialized(file, async () => {
     if (isCredentialPath(await realPathOrSelf(path))) return;
     const state = source === undefined ? await readState(path) : stateOf(source);
     if (!state) return;
@@ -300,6 +301,9 @@ export function observeFile(sessionId: string, filePath: string, call: string, p
       utimesSync(dirname(dir), now, now);
     } catch { /* pruned meanwhile: the next capture makes it again */ }
   });
+  // The call is done with the file: whoever keeps the session's list works this one out again.
+  if (phase === "after") void done.then(() => sessionFileWritten(sessionId, path));
+  return done;
 }
 
 /** Every path the session has a history for. */

package/src/services/session-file-baselines/session-file-writes.ts

@@ -0,0 +1,28 @@
+/**
+ * The moment a session's tool has finished changing a file — a file tool's PostToolUse, a shell
+ * command's `git status` bracket, a codex patch — announced by `observeFile`'s "after", which
+ * every one of those goes through. Its own module so the history can announce it without
+ * reaching the list that listens (`session-file-changes.service.ts`), which reads the history.
+ */
+type FileWrittenListener = (sessionId: string, path: string) => void;
+
+const listeners = new Set<FileWrittenListener>();
+
+/** Subscribe; returns the unsubscribe function. */
+export function onSessionFileWritten(cb: FileWrittenListener): () => void {
+  listeners.add(cb);
+  return () => {
+    listeners.delete(cb);
+  };
+}
+
+export function sessionFileWritten(sessionId: string, path: string): void {
+  for (const cb of listeners) {
+    // A failing listener must not turn a write the agent made into an error.
+    try {
+      cb(sessionId, path);
+    } catch (e) {
+      console.warn(`[session-baselines] file-written listener failed: ${(e as Error).message}`);
+    }
+  }
+}

package/src/services/sliding-window-limiter.ts

@@ -0,0 +1,106 @@
+/**
+ * A keyed sliding-window rate limiter with a bounded number of keys.
+ *
+ * Each key keeps the times of its recent hits (at most `max` of them), so a refusal says
+ * exactly how long until the oldest one ages out. `granularityMs` folds hits that land
+ * within that span of each other into one entry, which keeps a large budget (hundreds a
+ * minute) at a few dozen entries per key; 0 keeps every hit exact.
+ *
+ * Keys are kept in least-recently-used order and the oldest is dropped past `maxKeys`, so a
+ * caller that can mint keys at will (a forged client address) cannot grow the map without
+ * bound. Nothing sweeps the whole map: each key is pruned when it is touched.
+ */
+
+export interface WindowRule {
+  windowMs: number;
+  /** Hits allowed per window. */
+  max: number;
+  /** Least time between two hits of one key; 0 or absent for none. */
+  minIntervalMs?: number;
+  /** Hits this close to the previous entry share it; 0 or absent keeps each hit exact. */
+  granularityMs?: number;
+}
+
+export type LimitReason = "interval" | "window";
+
+export interface LimitRefusal {
+  reason: LimitReason;
+  /** How long until the same request would be allowed. */
+  retryMs: number;
+}
+
+interface Slot { t: number; n: number }
+interface KeyState { slots: Slot[]; count: number; last: number }
+
+export interface SlidingWindowLimiter {
+  /** Why `key` may not hit now, or null when it may. Counts nothing. */
+  check(key: string, now: number): LimitRefusal | null;
+  /** Count one hit for `key`, unconditionally. */
+  record(key: string, now: number): void;
+  /** {@link check}, then {@link record} when allowed. */
+  take(key: string, now: number): LimitRefusal | null;
+  reset(): void;
+  readonly size: number;
+}
+
+export function createSlidingWindowLimiter(rule: WindowRule, opts: { maxKeys: number }): SlidingWindowLimiter {
+  const minInterval = rule.minIntervalMs ?? 0;
+  const granularity = rule.granularityMs ?? 0;
+  const keys = new Map<string, KeyState>();
+
+  /** The key's live state with aged-out hits removed, or undefined when nothing is left. */
+  function live(key: string, now: number): KeyState | undefined {
+    const state = keys.get(key);
+    if (!state) return undefined;
+    let drop = 0;
+    while (drop < state.slots.length && now - state.slots[drop]!.t >= rule.windowMs) {
+      state.count -= state.slots[drop]!.n;
+      drop++;
+    }
+    if (drop) state.slots.splice(0, drop);
+    if (state.slots.length === 0 && now - state.last >= minInterval) {
+      keys.delete(key);
+      return undefined;
+    }
+    return state;
+  }
+
+  function check(key: string, now: number): LimitRefusal | null {
+    const state = live(key, now);
+    if (!state) return null;
+    if (minInterval && now - state.last < minInterval) return { reason: "interval", retryMs: minInterval - (now - state.last) };
+    if (state.count >= rule.max) return { reason: "window", retryMs: rule.windowMs - (now - state.slots[0]!.t) };
+    return null;
+  }
+
+  function record(key: string, now: number): void {
+    const state = live(key, now) ?? { slots: [], count: 0, last: now };
+    const tail = state.slots[state.slots.length - 1];
+    if (tail && granularity && now - tail.t < granularity) tail.n++;
+    else state.slots.push({ t: now, n: 1 });
+    state.count++;
+    state.last = now;
+    // Re-inserting moves the key to the most recent end.
+    keys.delete(key);
+    keys.set(key, state);
+    while (keys.size > opts.maxKeys) {
+      const oldest = keys.keys().next();
+      if (oldest.done) break;
+      keys.delete(oldest.value);
+    }
+  }
+
+  return {
+    check,
+    record,
+    take(key, now) {
+      const refusal = check(key, now);
+      if (!refusal) record(key, now);
+      return refusal;
+    },
+    reset: () => keys.clear(),
+    get size() {
+      return keys.size;
+    },
+  };
+}