@tekmidian/pai 0.35.2 → 0.36.0

package/src/hooks/ts/lib/transcript-text.test.ts

@@ -0,0 +1,125 @@
+import { describe, it, expect } from "vitest";
+import { contentToText, isNoiseFilePath, preferCwdFiles } from "./transcript-text.js";
+
+// ---------------------------------------------------------------------------
+// contentToText — bug 1: "[object Object]" from a stringified nested array
+// ---------------------------------------------------------------------------
+
+describe("contentToText", () => {
+  it("returns a plain string unchanged", () => {
+    expect(contentToText("hello")).toBe("hello");
+  });
+
+  it("concatenates the text of type:text blocks", () => {
+    expect(
+      contentToText([
+        { type: "text", text: "first" },
+        { type: "text", text: "second" },
+      ])
+    ).toBe("first second");
+  });
+
+  it("BUG 1: ignores a tool_result block instead of stringifying its nested content array", () => {
+    // This is the exact shape a tool_result user-turn takes: content is an
+    // ARRAY of blocks, not a string or a .text field. The old code fell back
+    // to String(c.content) here, producing "[object Object],[object Object]".
+    const content = [
+      {
+        type: "tool_result",
+        tool_use_id: "abc",
+        content: [
+          { type: "text", text: "some tool output" },
+          { type: "text", text: "more tool output" },
+        ],
+      },
+    ];
+    const result = contentToText(content);
+    expect(result).not.toContain("[object Object]");
+    expect(result).toBe(""); // tool_result is ignored entirely, not partially stringified
+  });
+
+  it("ignores tool_use and image blocks", () => {
+    const content = [
+      { type: "tool_use", name: "Edit", input: { file_path: "/x.ts" } },
+      { type: "image", source: { data: "..." } },
+      { type: "text", text: "the actual request" },
+    ];
+    expect(contentToText(content)).toBe("the actual request");
+  });
+
+  it("returns empty string for non-string, non-array content", () => {
+    expect(contentToText(null)).toBe("");
+    expect(contentToText(undefined)).toBe("");
+    expect(contentToText(42)).toBe("");
+  });
+});
+
+// ---------------------------------------------------------------------------
+// isNoiseFilePath — bug 2: tmp/jobs paths crowding out real source files
+// ---------------------------------------------------------------------------
+
+describe("isNoiseFilePath", () => {
+  it("excludes a tmp/ path segment", () => {
+    expect(isNoiseFilePath("/Users/me/project/jobs/42/tmp/msg8.txt")).toBe(true);
+  });
+
+  it("excludes a jobs/ path segment even without a nested tmp/", () => {
+    expect(isNoiseFilePath("/Users/me/project/jobs/42/attachment.txt")).toBe(true);
+  });
+
+  it("excludes a scratchpad/ path segment", () => {
+    expect(isNoiseFilePath("/private/tmp/claude-501/proj/scratchpad/notes.md")).toBe(true);
+  });
+
+  it("excludes anything under an explicit scratchpad dir even without the word in the path", () => {
+    expect(isNoiseFilePath("/private/tmp/claude-501/proj/work/sess/draft.txt", "/private/tmp/claude-501/proj/work/sess")).toBe(true);
+  });
+
+  it("does not exclude a real source file", () => {
+    expect(isNoiseFilePath("/Users/me/project/src/hooks/ts/pre-compact/context-compression-hook.ts")).toBe(false);
+  });
+
+  it("does not false-positive on a directory merely containing 'tmp' as a substring", () => {
+    expect(isNoiseFilePath("/Users/me/project/src/attempt/file.ts")).toBe(false);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// preferCwdFiles — bug 2 continued: don't let noise crowd the last-10 slice
+// ---------------------------------------------------------------------------
+
+describe("preferCwdFiles", () => {
+  it("orders cwd files before non-cwd files", () => {
+    const files = [
+      "/other/repo/a.ts",
+      "/Users/me/project/src/real.ts",
+      "/other/repo/b.ts",
+      "/Users/me/project/src/other-real.ts",
+    ];
+    const result = preferCwdFiles(files, "/Users/me/project");
+    expect(result).toEqual([
+      "/Users/me/project/src/real.ts",
+      "/Users/me/project/src/other-real.ts",
+      "/other/repo/a.ts",
+      "/other/repo/b.ts",
+    ]);
+  });
+
+  it("returns the list unchanged when cwd is not provided", () => {
+    const files = ["/a.ts", "/b.ts"];
+    expect(preferCwdFiles(files)).toEqual(files);
+  });
+
+  it("real regression case: real source files survive a slice(-10) after noise filtering + cwd preference", () => {
+    // Reproduces the reported digest: 8 tmp/jobs paths plus 2 real source
+    // files. isNoiseFilePath already drops the 8 at collection time in the
+    // hook, so this only has the 2 real files left — they must not be pushed
+    // out of a 10-slot slice by anything.
+    const realFiles = [
+      "/Users/me/project/src/a.ts",
+      "/Users/me/project/src/b.ts",
+    ];
+    const result = preferCwdFiles(realFiles, "/Users/me/project").slice(-10);
+    expect(result).toEqual(realFiles);
+  });
+});

package/src/hooks/ts/lib/transcript-text.ts

@@ -0,0 +1,71 @@
+/**
+ * transcript-text.ts — small, pure pieces of transcript parsing that are
+ * worth testing in isolation.
+ *
+ * Split out of context-compression-hook.ts (a hook entrypoint that calls
+ * `main()` and `process.exit()` at import time, which makes it unsafe to
+ * import directly from a test) so the two bug fixes that live here can be
+ * covered by a real regression test rather than only a manual before/after
+ * digest.
+ */
+
+/**
+ * Turn Claude content (string or content block array) into plain text.
+ *
+ * BUG (found in ~1 in 4 injected digests on this machine): a "user" turn is
+ * not always something the user typed — a tool_result is also delivered as a
+ * user-role message, and its `content` block looks like
+ * `{ type: "tool_result", content: [...] }` where the inner `content` is
+ * itself an array of blocks, not a string. The old code fell back to
+ * `String(c.content)` for anything without a `.text` field, and `String()`
+ * on an array of objects produces literal `"[object Object],[object
+ * Object]"` — which is exactly what showed up under "Recent user requests".
+ * Only ever take the `.text` of a `type: "text"` block; everything else
+ * (tool_use, tool_result, image, and any nested content) is ignored rather
+ * than stringified.
+ */
+export function contentToText(content: unknown): string {
+  if (typeof content === 'string') return content;
+  if (Array.isArray(content)) {
+    return content
+      .map((c) => {
+        if (typeof c === 'string') return c;
+        if (c && typeof c === 'object' && c.type === 'text' && typeof c.text === 'string') {
+          return c.text;
+        }
+        return '';
+      })
+      .join(' ')
+      .trim();
+  }
+  return '';
+}
+
+/**
+ * True for a modified-file path that is noise for a handover, not signal.
+ *
+ * BUG: a real digest showed 8 of 10 "Files modified" slots filled with
+ * throwaway paths under a jobs/<id>/tmp/ directory (msg8.txt, msg9.txt …),
+ * crowding out the two real source files also edited that session. Anything
+ * under a `tmp/`, `jobs/`, or `scratchpad/` path segment is excluded — those
+ * are working scratch space, not the codebase the next session needs to
+ * remember it touched.
+ */
+export function isNoiseFilePath(filePath: string, scratchpadDir?: string): boolean {
+  if (scratchpadDir && filePath.startsWith(scratchpadDir)) return true;
+  const segments = filePath.split('/');
+  return segments.includes('tmp') || segments.includes('jobs') || segments.includes('scratchpad');
+}
+
+/**
+ * Order a modified-file list so entries inside `cwd` are preferred when a
+ * caller has to truncate to a fixed slot count — an unrelated repo touched
+ * in passing should not crowd out the files that actually matter for this
+ * session's working directory.
+ */
+export function preferCwdFiles(files: string[], cwd?: string): string[] {
+  if (!cwd) return files;
+  const inCwd = files.filter((f) => f.startsWith(cwd));
+  const outCwd = files.filter((f) => !f.startsWith(cwd));
+  return [...inCwd, ...outCwd];
+}

package/src/hooks/ts/pre-compact/context-compression-hook.ts

@@ -28,10 +28,12 @@ import {
   findNotesDir,
   renameSessionNote,
   updateTodoContinue,
-  calculateSessionTokens,
   isProbeSession,
   WorkItem,
 } from '../lib/project-utils';
+import { getContextFill, formatContextFill } from '../lib/context-fill.js';
+import { contentToText, isNoiseFilePath, preferCwdFiles } from '../lib/transcript-text.js';
+import { readContextHandoverCache } from '../lib/context-handover-cache.js';
 
 interface HookInput {
   session_id: string;
@@ -40,6 +42,10 @@ interface HookInput {
   hook_event_name: string;
   compact_type?: string;
   trigger?: string;
+  /** Session's scratchpad directory, when the harness provides one — used to
+   *  exclude scratch files from the "Files modified" list (see
+   *  isNoiseFilePath). */
+  scratchpad_dir?: string;
 }
 
 const DAEMON_SOCKET = process.env.PAI_SOCKET ?? '/tmp/pai.sock';
@@ -52,29 +58,21 @@ interface TranscriptData {
   captures: string[];
   lastCompleted: string;
   filesModified: string[];
+  /** Count of modified-file paths dropped as noise (tmp/jobs/scratchpad). */
+  filesExcluded: number;
   workItems: WorkItem[];
 }
 
 // ---------------------------------------------------------------------------
 // Helpers
 // ---------------------------------------------------------------------------
-
-/** Turn Claude content (string or content block array) into plain text. */
-function contentToText(content: unknown): string {
-  if (typeof content === 'string') return content;
-  if (Array.isArray(content)) {
-    return content
-      .map((c) => {
-        if (typeof c === 'string') return c;
-        if (c?.text) return c.text;
-        if (c?.content) return String(c.content);
-        return '';
-      })
-      .join(' ')
-      .trim();
-  }
-  return '';
-}
+//
+// contentToText, isNoiseFilePath, and preferCwdFiles live in
+// ../lib/transcript-text.ts, not here — this file calls `main()` and
+// `process.exit()` at import time, which makes it unsafe to import from a
+// test. Splitting the pure, bug-fixed logic out is what lets those two bugs
+// (the "[object Object]" content bug and the tmp/jobs noise-crowding bug)
+// carry a real regression test instead of only a manual before/after digest.
 
 function getTranscriptStats(transcriptPath: string): { messageCount: number; isLarge: boolean } {
   try {
@@ -101,13 +99,14 @@ function getTranscriptStats(transcriptPath: string): { messageCount: number; isL
 // Unified transcript parser — single pass extracts everything
 // ---------------------------------------------------------------------------
 
-function parseTranscript(transcriptPath: string): TranscriptData {
+function parseTranscript(transcriptPath: string, scratchpadDir?: string): TranscriptData {
   const data: TranscriptData = {
     userMessages: [],
     summaries: [],
     captures: [],
     lastCompleted: '',
     filesModified: [],
+    filesExcluded: 0,
     workItems: [],
   };
 
@@ -175,8 +174,11 @@ function parseTranscript(transcriptPath: string): TranscriptData {
             if (block.type === 'tool_use') {
               const tool = block.name;
               if ((tool === 'Edit' || tool === 'Write') && block.input?.file_path) {
-                if (!data.filesModified.includes(block.input.file_path)) {
-                  data.filesModified.push(block.input.file_path);
+                const fp = block.input.file_path;
+                if (isNoiseFilePath(fp, scratchpadDir)) {
+                  data.filesExcluded++;
+                } else if (!data.filesModified.includes(fp)) {
+                  data.filesModified.push(fp);
                 }
               }
             }
@@ -220,10 +222,19 @@ function formatSessionState(data: TranscriptData, cwd?: string): string | null {
     for (const c of recentCaptures) parts.push(`- ${c.slice(0, 150)}`);
   }
 
-  const files = data.filesModified.slice(-10);
+  // Noise (tmp/jobs/scratchpad) is already excluded at collection time — see
+  // isNoiseFilePath. Here, prefer files inside the session's own working
+  // directory when there are more than fit in the slice, so an unrelated
+  // repo touched in passing doesn't crowd out the files that matter.
+  const files = preferCwdFiles(data.filesModified, cwd).slice(-10);
   if (files.length > 0) {
     parts.push('\nFiles modified this session:');
     for (const f of files) parts.push(`- ${f}`);
+  } else if (data.filesExcluded > 0) {
+    parts.push(
+      `\nFiles modified this session: none in the working directory ` +
+      `(excluded ${data.filesExcluded} noise path(s) under tmp/jobs/scratchpad)`
+    );
   }
 
   if (data.lastCompleted) {
@@ -315,6 +326,7 @@ function loadCumulativeState(notesDir: string): TranscriptData | null {
       captures: raw.captures || [],
       lastCompleted: raw.lastCompleted || '',
       filesModified: raw.filesModified || [],
+      filesExcluded: raw.filesExcluded || 0,
       workItems: raw.workItems || [],
     };
   } catch {
@@ -322,6 +334,26 @@ function loadCumulativeState(notesDir: string): TranscriptData | null {
   }
 }
 
+/**
+ * Peek at `.compact-state.json`'s `lastUpdated` — the timestamp of THIS
+ * session's previous compaction — without pulling it into TranscriptData's
+ * contract. Read before `saveCumulativeState` overwrites the file for the
+ * current compaction, so callers can tell "a cached handover generated
+ * since the last compaction" from "a cached handover left over from before
+ * it, now stale". Returns null when there is no previous compaction (the
+ * first one this session) — treated as "any cached handover is fresh".
+ */
+function readPreviousCompactionTimestamp(notesDir: string): string | null {
+  try {
+    const filePath = join(notesDir, CUMULATIVE_STATE_FILE);
+    if (!existsSync(filePath)) return null;
+    const raw = JSON.parse(readFileSync(filePath, 'utf-8'));
+    return typeof raw.lastUpdated === 'string' ? raw.lastUpdated : null;
+  } catch {
+    return null;
+  }
+}
+
 function mergeTranscriptData(accumulated: TranscriptData | null, current: TranscriptData): TranscriptData {
   if (!accumulated) return current;
 
@@ -339,6 +371,7 @@ function mergeTranscriptData(accumulated: TranscriptData | null, current: Transc
     captures: mergeArrays(accumulated.captures, current.captures),
     lastCompleted: current.lastCompleted || accumulated.lastCompleted,
     filesModified: mergeArrays(accumulated.filesModified, current.filesModified),
+    filesExcluded: accumulated.filesExcluded + current.filesExcluded,
     workItems: [...accumulated.workItems, ...newWorkItems],
   };
 }
@@ -507,19 +540,33 @@ async function main() {
   }
 
   const compactType = hookInput?.compact_type || hookInput?.trigger || 'auto';
-  let tokenCount = 0;
+  let ntfyTokenDisplay: string | null = null;
 
   if (hookInput?.transcript_path) {
     const stats = getTranscriptStats(hookInput.transcript_path);
-    tokenCount = calculateSessionTokens(hookInput.transcript_path);
-    const tokenDisplay = tokenCount > 1000
-      ? `${Math.round(tokenCount / 1000)}k`
-      : String(tokenCount);
+
+    // BUG (real headers pulled from this machine's history): "~995073k
+    // tokens", "~2135399k tokens" — 995 million and 2.1 billion. The old
+    // source, calculateSessionTokens, sums usage across EVERY line in the
+    // transcript, which is a lifetime token-spend counter, not a context
+    // fill reading — it necessarily exceeds the window on any long session
+    // and is not what "context compression triggered at ~X tokens" means.
+    // getContextFill reads the most recent usage entry only (or the
+    // statusline's own reading), which is the actual current fill and can
+    // never legitimately exceed the window. formatContextFill still
+    // clamps-and-flags rather than trust that, and reports "unknown"
+    // instead of guessing when neither source has an answer.
+    const fill = getContextFill({
+      sessionId: hookInput.session_id,
+      transcriptPath: hookInput.transcript_path,
+    });
+    const tokenDisplay = formatContextFill(fill).text;
+    ntfyTokenDisplay = tokenDisplay;
 
     // -----------------------------------------------------------------
     // Single-pass transcript parsing + cumulative state merge
     // -----------------------------------------------------------------
-    const data = parseTranscript(hookInput.transcript_path);
+    const data = parseTranscript(hookInput.transcript_path, hookInput.scratchpad_dir);
 
     // Find notes directory early — needed for cumulative state
     let notesInfo: { path: string; isLocal: boolean };
@@ -533,6 +580,10 @@ async function main() {
 
     // Load accumulated state from previous compactions and merge
     const accumulated = loadCumulativeState(notesInfo.path);
+    // Read BEFORE saveCumulativeState (below) overwrites this file for the
+    // CURRENT compaction — this is the previous one's timestamp, the
+    // freshness line a cached handover has to clear (see the injection step).
+    const previousCompactionAt = readPreviousCompactionTimestamp(notesInfo.path);
     const merged = mergeTranscriptData(accumulated, data);
     const state = formatSessionState(merged, hookInput.cwd);
 
@@ -640,12 +691,43 @@ async function main() {
         ? `\nSESSION NOTE: ${notePath}\nIf this note still has a generic title (e.g. "New Session", "Context Compression"),\nrename it based on actual work done and add a rich summary.`
         : '';
 
+      // -------------------------------------------------------------------
+      // ADDITIVE: a model-written handover (decisions/reasoning/open threads
+      // — see the threshold-triggered context-handover-worker) sits ALONGSIDE
+      // the mechanical scrape above, never replaces it. The scrape is the
+      // floor and stays intact on every path, so this is never worse than
+      // the pre-existing behaviour — only sometimes better. "Fresh" means
+      // generated after this session's PREVIOUS compaction: a handover
+      // cached before that point describes state the last compaction
+      // already accounted for, not what has happened since.
+      // -------------------------------------------------------------------
+      const cachedHandover = readContextHandoverCache(hookInput.session_id);
+      const handoverIsFresh =
+        cachedHandover !== null &&
+        (!previousCompactionAt || new Date(cachedHandover.generatedAt) > new Date(previousCompactionAt));
+
+      const sourceLabel = handoverIsFresh
+        ? `model-written handover (${cachedHandover!.model}, generated ${cachedHandover!.generatedAt}, ` +
+          `threshold=${cachedHandover!.threshold}) + mechanical scrape`
+        : 'mechanical scrape only (no fresh model-written handover was available)';
+
+      const handoverBlock = handoverIsFresh
+        ? [
+            '',
+            '--- MODEL-WRITTEN HANDOVER (decisions, reasoning, open threads — not in the scrape above) ---',
+            cachedHandover!.summary,
+            '--- end model-written handover ---',
+          ].join('\n')
+        : '';
+
       const injection = [
         '<system-reminder>',
         `SESSION STATE RECOVERED AFTER COMPACTION (${compactType}, ~${tokenDisplay} tokens)`,
+        `HANDOVER SOURCE: ${sourceLabel}`,
         '',
         stateText,
         noteInfo,
+        handoverBlock,
         '',
         'IMPORTANT: This session state was captured before context compaction.',
         'Use it to maintain continuity. Continue the conversation from where',
@@ -687,8 +769,8 @@ async function main() {
   } catch { /* non-fatal */ }
 
   // Send ntfy.sh notification
-  const ntfyMessage = tokenCount > 0
-    ? `Auto-pause: ~${Math.round(tokenCount / 1000)}k tokens`
+  const ntfyMessage = ntfyTokenDisplay && ntfyTokenDisplay !== 'unknown'
+    ? `Auto-pause: ~${ntfyTokenDisplay} tokens`
     : 'Context compressing';
   await sendNtfyNotification(ntfyMessage);
 

package/statusline-command.sh

@@ -116,6 +116,22 @@ context_size=$(echo "$input" | jq -r '.context_window.context_window_size // 200
 context_used_k=$(( (context_pct * context_size / 100) / 1000 ))
 context_max_k=$((context_size / 1000))
 
+# Persist the authoritative context reading so hooks (which never receive
+# context_window on their own stdin) can read it instead of re-deriving an
+# estimate from the transcript. Best-effort only — a failure here must never
+# affect the status line itself.
+statusline_session_id=$(echo "$input" | jq -r '.session_id // empty' 2>/dev/null)
+if [ -n "$statusline_session_id" ]; then
+    context_state_file="${TMPDIR:-/tmp}/pai-context-${statusline_session_id}.json"
+    jq -n \
+        --argjson used_percentage "${context_pct:-0}" \
+        --argjson context_window_size "${context_size:-200000}" \
+        --arg session_id "$statusline_session_id" \
+        --argjson timestamp "$(date +%s000)" \
+        '{used_percentage: $used_percentage, context_window_size: $context_window_size, session_id: $session_id, timestamp: $timestamp}' \
+        > "$context_state_file" 2>/dev/null || true
+fi
+
 # Tokyo Night Storm Color Scheme
 BACKGROUND='\033[48;2;36;40;59m'
 BRIGHT_PURPLE='\033[38;2;187;154;247m'