@pi-unipi/unipi 2.6.1 → 2.6.2

package/packages/compactor/src/compaction/brief.ts

@@ -3,11 +3,16 @@
  */
 
 import type { NormalizedBlock } from "../types.js";
-import { clip, firstLine } from "./content.js";
+import { clip } from "./content.js";
 import { extractPath } from "./extract/files.js";
+import { collapseSkillText } from "./skill-collapse.js";
 
 const TRUNCATE_USER = 256;
-const TRUNCATE_ASSISTANT = 200;
+const SEGMENT_CLOSING_ASSISTANT_HEAD_WORDS = 120;
+const SEGMENT_CLOSING_ASSISTANT_TAIL_WORDS = 120;
+const ASSISTANT_HEAD_WORDS = 80;
+const ASSISTANT_TAIL_WORDS = 120;
+const MIN_SEGMENT_CLOSING_ASSISTANT_CHARS = 120;
 
 const SELF_TALK_PREFIX_RE =
   /^\s*(?:hmm|wait|actually|oh|okay|ok|well|so)[,.!\s-]+/i;
@@ -33,8 +38,15 @@ const STOP_WORDS = new Set([
   "if", "then", "than", "when", "where", "how", "just", "also",
 ]);
 
+const normalizeForTokenBudget = (text: string): string =>
+  text
+    .replace(/\r\n?/g, "\n")
+    .replace(/[^\S\n]+/g, " ")
+    .replace(/\n{3,}/g, "\n\n")
+    .trim();
+
 const truncateTokens = (text: string, limit: number): string => {
-  const flat = text.replace(/\s+/g, " ").trim();
+  const flat = normalizeForTokenBudget(text);
   let count = 0;
   let lastEnd = 0;
   for (const seg of segmenter.segment(flat)) {
@@ -51,17 +63,138 @@ const truncateTokens = (text: string, limit: number): string => {
   return flat;
 };
 
-const BASH_CAP = 120;
+/** Word spans excluding stop words — positions into the normalized flat text */
+const significantWordSpans = (flat: string): { start: number; end: number }[] => {
+  const words: { start: number; end: number }[] = [];
+  for (const seg of segmenter.segment(flat)) {
+    if (!isWord(seg)) continue;
+    if (STOP_WORDS.has(seg.segment.toLowerCase())) continue;
+    words.push({ start: seg.index, end: seg.index + seg.segment.length });
+  }
+  return words;
+};
+
+/** Keep the first headLimit and last tailLimit significant words; elide the middle */
+const truncateTokensHeadTail = (text: string, headLimit: number, tailLimit: number): string => {
+  const flat = normalizeForTokenBudget(text);
+  if (headLimit <= 0 || tailLimit <= 0) return flat;
+  const words = significantWordSpans(flat);
+  if (words.length <= headLimit + tailLimit) return flat;
+  const head = flat.slice(0, words[headLimit - 1].end).trimEnd();
+  const tail = flat.slice(words[words.length - tailLimit].start).trimStart();
+  return `${head}\n...(middle truncated)...\n${tail}`;
+};
+
+const nextRenderableBlock = (blocks: NormalizedBlock[], index: number): NormalizedBlock | undefined => {
+  for (let i = index + 1; i < blocks.length; i++) {
+    if (blocks[i].kind !== "tool_result") return blocks[i];
+  }
+  return undefined;
+};
+
+const isSegmentClosingAssistant = (blocks: NormalizedBlock[], index: number): boolean => {
+  if (blocks[index]?.kind !== "assistant") return false;
+  const next = nextRenderableBlock(blocks, index);
+  return !next || next.kind === "user";
+};
+
+// ── bash command compression ──
+
+const BASH_CAP = 240;
 const PIPE_TAIL_RE = /\s*\|\s*(?:head|tail|sort|wc|column|tr|cut|awk|uniq|python3|node|bun)(?:\s[^|]*)?$/;
+// Preamble/boilerplate lines that carry no durable fact on their own. Dropped
+// when a script has more informative lines, so `set -euo pipefail\ngit commit`
+// renders the commit rather than the shell option.
+const TRIVIAL_LINE_RE = /^(?:set\s+[-+]|cd\s+\S+$|export\s+\w+=|(?:source|\.)\s+\S+|pwd$|true$|:$|#)/;
+// A real heredoc opener: `<<` at a command boundary — not preceded by a word
+// char or `)`, so shift ops (`8 << 20`, `Rd<<8`) are not misread — with an
+// identifier terminator starting [A-Za-z_], so numeric `<< 10` is rejected too.
+export const HEREDOC_OPEN_RE = /(?<![\w)])<<-?\s*["']?([A-Za-z_]\w*)["']?/;
+// File-writer heredocs (`cat > f <<EOF`, tee, dd) already name their target, so
+// the opener alone is informative → body-only. EVERY other heredoc has a
+// content-free opener (interpreters python3/node, remote shells `ssh host <<CMD`,
+// sqlite3, ...) → we surface a one-line body preview.
+const FILEWRITER_HEREDOC_RE = /(?:^|[|&;]\s*)(?:cat|tee|dd)\b/;
+const HEREDOC_BODY_CAP = 80;
+// Body lines that are pure boilerplate and make a poor preview.
+const BODY_NOISE_RE = /^(?:import\s|from\s+\S+\s+import|require\(|const\s+\w+\s*=\s*require|#|\/\/|"""|'''|"use strict"|use\s+strict|<\?php)/;
 
-const compressBash = (raw: string): string => {
-  let cmd = raw.split("\n").map((l) => l.trim()).filter(Boolean)[0] ?? raw;
-  cmd = cmd.replace(/^cd\s+\S+\s*&&\s*/, "");
+/**
+ * If `lines[i]` opens a heredoc whose terminator actually appears on a later
+ * line, return that terminator's line index; otherwise -1. Callers use -1 to
+ * leave following lines intact instead of treating a stray `<<` (a shift op, a
+ * quoted string, or a truncated body) as a heredoc and skipping real commands.
+ */
+export const heredocCloseIndex = (lines: string[], i: number): number => {
+  const hd = lines[i].match(HEREDOC_OPEN_RE);
+  if (!hd) return -1;
+  const term = hd[1];
+  for (let j = i + 1; j < lines.length; j++) {
+    if (lines[j].trim() === term) return j;
+  }
+  return -1;
+};
+
+const stripCdPrefix = (line: string): string => line.replace(/^cd\s+\S+\s*&&\s*/, "").trim();
+const stripPipeTail = (line: string): string => {
+  let c = line;
   for (let i = 0; i < 3; i++) {
-    const stripped = cmd.replace(PIPE_TAIL_RE, "");
-    if (stripped === cmd) break;
-    cmd = stripped;
+    const stripped = c.replace(PIPE_TAIL_RE, "");
+    if (stripped === c) break;
+    c = stripped;
   }
+  return c.trim();
+};
+
+/**
+ * Semantic compression of a (possibly multi-line) bash command.
+ * 1. Drop heredoc BODIES (keep the opener line, e.g. `cat > f <<EOF` or
+ *    `python3 - <<PY`): the body is prose/script content that bloats the brief
+ *    without adding countable facts, while the opener still records what ran.
+ * 2. Drop trivial preamble lines (set -euo pipefail, cd-only, export, source,
+ *    comments) unless they are the ONLY line, so real work below them surfaces.
+ * 3. Strip `cd <path> &&` prefixes and pipe-tail formatting per line.
+ * 4. Join the remaining meaningful lines with `; ` and cap length.
+ */
+const compressBash = (raw: string): string => {
+  // Pass 1: keep heredoc opener lines, skip their bodies + terminators.
+  const rawLines = raw.split("\n");
+  const withoutHeredocBodies: string[] = [];
+  for (let i = 0; i < rawLines.length; i++) {
+    const line = rawLines[i];
+    withoutHeredocBodies.push(line);
+    // Only treat this as a heredoc when its terminator appears downstream; a
+    // stray `<<` (string/expression or truncated body) leaves later lines intact.
+    const close = heredocCloseIndex(rawLines, i);
+    if (close === -1) continue;
+    // File-writer heredocs (`cat > f <<EOF`) name their target → keep opener only.
+    // Every other heredoc has a content-free opener → grab the first meaningful
+    // body line as a preview (`python3 - <<PY`, `ssh host <<CMD`, `sqlite3 <<SQL`).
+    const wantPreview = !FILEWRITER_HEREDOC_RE.test(line);
+    let preview = "";
+    for (let j = i + 1; wantPreview && !preview && j < close; j++) {
+      const t = rawLines[j].trim();
+      if (t && !BODY_NOISE_RE.test(t)) preview = t;
+    }
+    if (preview) {
+      const clipped = preview.length > HEREDOC_BODY_CAP ? preview.slice(0, HEREDOC_BODY_CAP - 1) + "\u2026" : preview;
+      withoutHeredocBodies[withoutHeredocBodies.length - 1] = `${line.trim()} ${clipped}`;
+    }
+    i = close; // for-loop's i++ then skips past the terminator line
+  }
+
+  const lines = withoutHeredocBodies.map((l) => l.trim()).filter(Boolean);
+  if (lines.length === 0) return raw.trim();
+
+  const meaningful = lines
+    .filter((l) => !TRIVIAL_LINE_RE.test(l))
+    .map((l) => stripPipeTail(stripCdPrefix(l)))
+    .filter(Boolean);
+  // If everything was trivial (e.g. a bare `set -e` or `ls`), fall back to the
+  // first line so we never emit an empty marker.
+  const chosen = meaningful.length ? meaningful : [stripPipeTail(stripCdPrefix(lines[0]))].filter(Boolean);
+
+  const cmd = chosen.join("; ");
   if (cmd.length > BASH_CAP) {
     return cmd.slice(0, BASH_CAP - 3) + "...";
   }
@@ -97,15 +230,6 @@ export interface BriefLine {
   lines: string[];
 }
 
-export interface TranscriptEntry {
-  role: "user" | "assistant" | "tool_error";
-  text?: string;
-  tool?: string;
-  cmd?: string;
-  ref?: string;
-  count?: number;
-}
-
 export const buildBriefSections = (blocks: NormalizedBlock[]): BriefLine[] => {
   const sections: BriefLine[] = [];
   let lastHeader = "";
@@ -119,14 +243,23 @@ export const buildBriefSections = (blocks: NormalizedBlock[]): BriefLine[] => {
     lastHeader = header;
   };
 
-  for (const b of blocks) {
+  const pushText = (header: string, text: string, ref = "") => {
+    const lines = text.split("\n");
+    if (ref && lines.length > 0) {
+      lines[lines.length - 1] = `${lines[lines.length - 1]}${ref}`;
+    }
+    for (const line of lines) push(header, line);
+  };
+
+  for (let blockIndex = 0; blockIndex < blocks.length; blockIndex++) {
+    const b = blocks[blockIndex];
     switch (b.kind) {
       case "user": {
         if (!b.text.trim()) break;
-        const text = truncateTokens(b.text, TRUNCATE_USER);
+        const text = truncateTokens(collapseSkillText(b.text), TRUNCATE_USER);
         if (text) {
           const ref = b.sourceIndex != null ? ` (#${b.sourceIndex})` : "";
-          push("[user]", text + ref);
+          pushText("[user]", text, ref);
         }
         lastHeader = "[user]";
         break;
@@ -138,10 +271,12 @@ export const buildBriefSections = (blocks: NormalizedBlock[]): BriefLine[] => {
           if (stripped === raw) break;
           raw = stripped;
         }
-        const text = truncateTokens(raw, TRUNCATE_ASSISTANT);
+        const text = isSegmentClosingAssistant(blocks, blockIndex)
+          ? truncateTokensHeadTail(raw, SEGMENT_CLOSING_ASSISTANT_HEAD_WORDS, SEGMENT_CLOSING_ASSISTANT_TAIL_WORDS)
+          : truncateTokensHeadTail(raw, ASSISTANT_HEAD_WORDS, ASSISTANT_TAIL_WORDS);
         if (text) {
           const ref = b.sourceIndex != null ? ` (#${b.sourceIndex})` : "";
-          push("[assistant]", text + ref);
+          pushText("[assistant]", text, ref);
         }
         break;
       }
@@ -154,7 +289,7 @@ export const buildBriefSections = (blocks: NormalizedBlock[]): BriefLine[] => {
       }
       case "tool_result": {
         if (b.isError) {
-          const body = firstLine(b.text, 150);
+          const body = b.text.split("\n").map((l) => l.trim()).filter(Boolean)[0] ?? "";
           if (!body || body === "(no output)") break;
           const ref = b.sourceIndex != null ? ` (#${b.sourceIndex})` : "";
           const header = `[tool_error] ${b.name}${ref}`;
@@ -265,70 +400,6 @@ export const stringifyBrief = (sections: BriefLine[]): string => {
   return out.join("\n");
 };
 
-const parseToolLine = (line: string): { tool: string; cmd?: string; ref?: string; count?: number } | null => {
-  const m = line.match(/^\* (\S+)\s*(?:"([^"]*)")?\s*(?:\((#[\d, #]+)\))?\s*(?:x(\d+))?$/);
-  if (!m) return null;
-  return {
-    tool: m[1],
-    cmd: m[2] || undefined,
-    ref: m[3] || undefined,
-    count: m[4] ? parseInt(m[4]) : undefined,
-  };
-};
-
-const extractRef = (text: string): { clean: string; ref?: string } => {
-  const m = text.match(/\s*\(#(\d+)\)$/);
-  if (!m) return { clean: text };
-  return { clean: text.slice(0, m.index).trimEnd(), ref: `#${m[1]}` };
-};
-
-export const sectionsToTranscript = (sections: BriefLine[]): TranscriptEntry[] => {
-  const entries: TranscriptEntry[] = [];
-
-  for (const sec of sections) {
-    if (sec.header === "[user]") {
-      for (const line of sec.lines) {
-        const { clean, ref } = extractRef(line);
-        entries.push({ role: "user", text: clean, ...(ref && { ref }) });
-      }
-    } else if (sec.header === "[assistant]") {
-      for (const line of sec.lines) {
-        if (line.startsWith("* ")) {
-          const parsed = parseToolLine(line);
-          if (parsed) {
-            entries.push({
-              role: "assistant",
-              tool: parsed.tool,
-              ...(parsed.cmd && { cmd: parsed.cmd }),
-              ...(parsed.ref && { ref: parsed.ref }),
-              ...(parsed.count && { count: parsed.count }),
-            });
-          } else {
-            const { clean, ref } = extractRef(line.slice(2));
-            entries.push({ role: "assistant", text: clean, ...(ref && { ref }) });
-          }
-        } else {
-          const { clean, ref } = extractRef(line);
-          entries.push({ role: "assistant", text: clean, ...(ref && { ref }) });
-        }
-      }
-    } else if (sec.header.startsWith("[tool_error]")) {
-      const headerMatch = sec.header.match(/^\[tool_error\]\s+(\S+)\s*(?:\(#(\d+)\))?/);
-      const tool = headerMatch?.[1] ?? "unknown";
-      const ref = headerMatch?.[2] ? `#${headerMatch[2]}` : undefined;
-      for (const line of sec.lines) {
-        entries.push({
-          role: "tool_error",
-          tool,
-          text: line,
-          ...(ref && { ref }),
-        });
-      }
-    }
-  }
-
-  return entries;
-};
-
+/** Render a set of blocks to a brief transcript string (used by ranked selection cost calc) */
 export const compileBrief = (blocks: NormalizedBlock[]): string =>
   stringifyBrief(buildBriefSections(blocks));

package/packages/compactor/src/compaction/build-sections.ts

@@ -8,7 +8,7 @@ import { extractGoals } from "./extract/goals.js";
 import { extractFiles } from "./extract/files.js";
 import { extractPreferences, dedupPreferencesAgainstGoals } from "./extract/preferences.js";
 import { extractCommits, formatCommits } from "./extract/commits.js";
-import { buildBriefSections, sectionsToTranscript, stringifyBrief } from "./brief.js";
+import { buildBriefSections, stringifyBrief } from "./brief.js";
 import type { SectionData } from "../types.js";
 export type { SectionData } from "../types.js";
 
@@ -57,9 +57,9 @@ const formatFileActivity = (blocks: NormalizedBlock[]): string[] => {
   return lines;
 };
 
-export const buildSections = (input: { blocks: NormalizedBlock[] }): SectionData => {
+export const buildSections = (input: { blocks: NormalizedBlock[]; briefBlocks?: NormalizedBlock[] }): SectionData => {
   const { blocks } = input;
-  const briefSections = buildBriefSections(blocks);
+  const briefSections = buildBriefSections(input.briefBlocks ?? blocks);
   const sessionGoal = extractGoals(blocks);
   const userPreferences = dedupPreferencesAgainstGoals(
     extractPreferences(blocks),
@@ -72,6 +72,5 @@ export const buildSections = (input: { blocks: NormalizedBlock[] }): SectionData
     commits: formatCommits(extractCommits(blocks)),
     userPreferences,
     briefTranscript: stringifyBrief(briefSections),
-    transcriptEntries: sectionsToTranscript(briefSections),
   };
 };

package/packages/compactor/src/compaction/compact-args.ts

@@ -0,0 +1,86 @@
+/**
+ * Parse keep:N and follow-up prompt from compaction custom instructions
+ * (ported from pi-vcc compact-args.ts; marker constant stays ours: COMPACTOR_INSTRUCTION)
+ */
+
+import { COMPACTOR_INSTRUCTION } from "@pi-unipi/core";
+
+const KEEP_TOKEN_RE = /^keep:(\d+)$/;
+
+export interface ParsedCompactionArgs {
+  followUpPrompt: string;
+  keepUserTurns: number | null;
+  keepUserTurnsExplicit: boolean;
+}
+
+const parseKeepUserTurns = (raw: string): number => {
+  const value = Number(raw);
+  return Number.isSafeInteger(value) ? value : Number.MAX_SAFE_INTEGER;
+};
+
+export const parseKeepAndPrompt = (args?: string): ParsedCompactionArgs => {
+  const trimmed = args?.trim() ?? "";
+  if (!trimmed) return { followUpPrompt: "", keepUserTurns: null, keepUserTurnsExplicit: false };
+
+  const startMatch = trimmed.match(/^keep:(\d+)(?:\s+|$)([\s\S]*)$/);
+  if (startMatch) {
+    return {
+      followUpPrompt: startMatch[2].trim(),
+      keepUserTurns: parseKeepUserTurns(startMatch[1]),
+      keepUserTurnsExplicit: true,
+    };
+  }
+
+  const parts = trimmed.split(/\s+/);
+  const endMatch = parts[parts.length - 1].match(KEEP_TOKEN_RE);
+  if (endMatch) {
+    return {
+      followUpPrompt: trimmed.slice(0, trimmed.length - parts[parts.length - 1].length).trim(),
+      keepUserTurns: parseKeepUserTurns(endMatch[1]),
+      keepUserTurnsExplicit: true,
+    };
+  }
+
+  return { followUpPrompt: trimmed, keepUserTurns: null, keepUserTurnsExplicit: false };
+};
+
+export interface ParsedCompactionInstructions {
+  isCompactor: boolean;
+  keepUserTurns: number;
+  keepUserTurnsExplicit: boolean;
+  followUpPrompt: string | null;
+}
+
+/**
+ * Parse customInstructions arriving at session_before_compact.
+ * - Exactly COMPACTOR_INSTRUCTION → default path (keep 1, not explicit).
+ * - COMPACTOR_INSTRUCTION + args → parse keep:N / prompt after the marker.
+ * - Anything else → not ours (parse for a trailing keep:N anyway, pi-vcc parity).
+ */
+export const parseCompactionInstructions = (
+  customInstructions?: string,
+): ParsedCompactionInstructions => {
+  const trimmed = customInstructions?.trim();
+  if (trimmed === COMPACTOR_INSTRUCTION) {
+    return { isCompactor: true, keepUserTurns: 1, keepUserTurnsExplicit: false, followUpPrompt: null };
+  }
+
+  const keepPrefix = `${COMPACTOR_INSTRUCTION} `;
+  if (trimmed?.startsWith(keepPrefix)) {
+    const parsed = parseKeepAndPrompt(trimmed.slice(keepPrefix.length));
+    return {
+      isCompactor: true,
+      keepUserTurns: parsed.keepUserTurns ?? 1,
+      keepUserTurnsExplicit: parsed.keepUserTurnsExplicit,
+      followUpPrompt: null,
+    };
+  }
+
+  const parsed = parseKeepAndPrompt(customInstructions);
+  return {
+    isCompactor: false,
+    keepUserTurns: parsed.keepUserTurns ?? 1,
+    keepUserTurnsExplicit: parsed.keepUserTurnsExplicit,
+    followUpPrompt: parsed.followUpPrompt || null,
+  };
+};