@pify/shell-background 0.4.2 → 0.5.0

package/README.md

@@ -54,17 +54,18 @@ Put these in `.pi/shell-background.json` (project) or `<agentDir>/shell-backgrou
 {
   "autoBackgroundMs": 30000,
   "tailBytes": 65536,
+  "tailLines": 200,
   "maxBackground": 8
 }
 ```
 
-`autoBackgroundMs` is how long a foreground command may run before it auto-backgrounds; set it to `0` to disable auto-background (explicit `background: true` still works). `PIFY_SHELL_BG_MS` overrides it for one run or in CI. `tailBytes` bounds how much of a job's log a status result shows. `maxBackground` bounds how many jobs may be alive at once: over it, a `background: true` request is refused with a message naming the limit (the command can still run in the foreground), and a command that crosses the auto-background threshold simply stays in the foreground instead of moving — a command is never refused, only the decision to background it. Bad values fall back to the defaults with a warning rather than taking the tool down.
+`autoBackgroundMs` is how long a foreground command may run before it auto-backgrounds; set it to `0` to disable auto-background (explicit `background: true` still works). `PIFY_SHELL_BG_MS` overrides it for one run or in CI. `tailBytes` bounds how much of a job's log a status result reads; `tailLines` bounds how many lines of it the model is shown (0 disables the line shrink). `maxBackground` bounds how many jobs may be alive at once: over it, a `background: true` request is refused with a message naming the limit (the command can still run in the foreground), and a command that crosses the auto-background threshold simply stays in the foreground instead of moving — a command is never refused, only the decision to background it. Bad values fall back to the defaults with a warning rather than taking the tool down.
 
 Since pi 0.99 the bash tool also returns a machine-readable result — `structuredContent` — which codemode scripts and tools calling `ctx.executeTool("bash", …)` receive instead of the text. This package keeps pi's shape so such a script reads either tool unchanged: `output` (the whole log, up to 1 MiB, head and tail around an omission marker), `truncated`, `full_output_path`, `exit_code`, `wall_time_seconds` — plus `status` and `job_id`, because a command here can still be running when the tool returns; then `exit_code` is absent and `shell_status {id}` collects it, with the same structured result.
 
 A backgrounded command runs with no one to answer it, so its environment says so: `GIT_TERMINAL_PROMPT=0` and `GIT_EDITOR`/`EDITOR`/`VISUAL=true` are forced (a `git commit` without `-m` fails with "empty message" instead of waiting on an editor that never comes, and an https `git fetch` fails instead of prompting for a password), and pagers default to `cat` unless you set them. `CI` and `TERM` are left alone on purpose — dev servers and watchers read them.
 
-The tail a result carries is cleaned the way pi's own bash cleans what the model sees — ANSI escapes, control characters and carriage-return progress frames stripped — so a chatty build or dev server does not spend tokens on colour codes and thousands of overwritten progress lines. The log file on disk keeps every byte.
+The tail a result carries is cleaned the way pi's own bash cleans what the model sees — ANSI escapes and control characters stripped — with two improvements borrowed from caveman's log compressors. A carriage-return redraw is **resolved to its last frame** rather than having the `\r` deleted: pi's own cleaning keeps every frame's text, so a 2000-frame progress bar arrives as one line holding all 2000 frames; here it arrives as `1999/2000`. And a long result is **shrunk by lines, not bytes**: the first and last lines are kept, every line that names trouble (`error`, `FAILED`, `Traceback`, `at …`, `File "…"`, `path:12:5:`, `FooError`, `caused by`, `exit status 1`) is kept wherever it sits, and the rest collapses into markers that count what they replaced — `[… 412 lines omitted (INFO ×380, DEBUG ×29) …]` — computed from exactly those lines. A byte window kept the *end* of a build log, where the summary is; a line window with that regex keeps the failing step in the middle too. Runs under three lines are shown as they are (a marker in place of one line only reads as a hole), the markers are recognised on a later poll so a shrunk result re-serialises identically (the provider prefix cache depends on it), and one note names what was left out and where the whole log is. A log that cannot be read says `(log unavailable: …)` rather than `(no output)`, which would send the model polling for output that will never come. The log file on disk keeps every byte.
 
 ## Coexistence with @pify/pretty
 

package/extensions/shell-background.ts

@@ -208,7 +208,7 @@ export default function shellBackground(pi: ExtensionAPI) {
 
   function finished(job: Job): ToolResult {
     return {
-      content: [{ type: "text", text: formatResult(job, settings.tailBytes) }],
+      content: [{ type: "text", text: formatResult(job, settings.tailBytes, settings.tailLines) }],
       details: { id: job.id, status: job.status, exitCode: job.exitCode, signal: job.signal, background: false },
       structuredContent: structured(job),
       isError: job.status === "failed",
@@ -221,7 +221,7 @@ export default function shellBackground(pi: ExtensionAPI) {
     pi.sendMessage(
       {
         customType: DELIVERY_TYPE,
-        content: deliveryMessage(job.id, formatResult(job, settings.tailBytes)),
+        content: deliveryMessage(job.id, formatResult(job, settings.tailBytes, settings.tailLines)),
         display: true,
         details: { id: job.id, status: job.status, exitCode: job.exitCode },
       },
@@ -432,7 +432,7 @@ export default function shellBackground(pi: ExtensionAPI) {
             {
               type: "text",
               text:
-                formatResult(job, settings.tailBytes) +
+                formatResult(job, settings.tailBytes, settings.tailLines) +
                 `\n\n[kill signal sent; process had not exited after 3s — shell_status ${job.id} to confirm]`,
             },
           ],
@@ -446,7 +446,7 @@ export default function shellBackground(pi: ExtensionAPI) {
           ? `\n\n[killed: exceeded the ${params.timeout}s timeout]`
           : `\n\n[killed: the turn was aborted]`;
       return {
-        content: [{ type: "text", text: formatResult(job, settings.tailBytes) + note }],
+        content: [{ type: "text", text: formatResult(job, settings.tailBytes, settings.tailLines) + note }],
         details: { id: job.id, status: job.status },
         structuredContent: structured(job),
         isError: true,
@@ -559,7 +559,7 @@ export default function shellBackground(pi: ExtensionAPI) {
         registry.persist(job);
       }
       return {
-        content: [{ type: "text", text: formatResult(job, settings.tailBytes) }],
+        content: [{ type: "text", text: formatResult(job, settings.tailBytes, settings.tailLines) }],
         details: { id: job.id, status: job.status, exitCode: job.exitCode },
         structuredContent: structured(job),
       };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pify/shell-background",
-  "version": "0.4.2",
+  "version": "0.5.0",
   "description": "Long-running bash goes async: background: true launches detached, and any command still running after 30s auto-backgrounds and delivers its result when it finishes",
   "keywords": [
     "pi-package",

package/src/config.ts

@@ -14,6 +14,12 @@ export interface ShellBgSettings {
   autoBackgroundMs: number;
   /** Bytes of the log tail shown in a status/collect result. */
   tailBytes: number;
+  /**
+   * Lines a result may carry before it is shrunk: first and last lines kept,
+   * every error/warning line kept wherever it sits, the rest collapsed into
+   * counted markers. 0 disables shrinking (the byte tail is still the cap).
+   */
+  tailLines: number;
   /**
    * How many jobs may be alive at once before a request to background one
    * more is refused and an auto-background transition is skipped. Every live
@@ -28,6 +34,7 @@ export interface ShellBgSettings {
 export const DEFAULT_SETTINGS: ShellBgSettings = {
   autoBackgroundMs: 30_000,
   tailBytes: 64 * 1024,
+  tailLines: 200,
   maxBackground: 8,
 };
 
@@ -36,6 +43,7 @@ const LIMITS: Record<keyof ShellBgSettings, { min: number; max: number }> = {
   // does not make every command look long-running.
   autoBackgroundMs: { min: 0, max: 3_600_000 },
   tailBytes: { min: 1024, max: 4 * 1024 * 1024 },
+  tailLines: { min: 0, max: 5000 },
   maxBackground: { min: 1, max: 64 },
 };
 

package/src/format.ts

@@ -3,8 +3,13 @@
  * job and a tail, produce the string — the extension supplies both.
  */
 import type { Job } from "./types.ts";
-import { readTail, countLines } from "./tail.ts";
+import { readTail } from "./tail.ts";
 import { sanitizeOutput } from "./sanitize.ts";
+import { DEFAULT_SHRINK, shrinkLines } from "./shrink.ts";
+
+function kb(bytes: number): string {
+  return bytes < 1024 * 1024 ? `${Math.max(1, Math.round(bytes / 1024))}KB` : `${(bytes / (1024 * 1024)).toFixed(1)}MB`;
+}
 
 function secs(ms: number): string {
   if (ms < 1000) return `${ms}ms`;
@@ -34,22 +39,36 @@ export function header(job: Job): string {
 /**
  * The full result of a finished (or polled) job: the header, then the tail of
  * its output, with an honest note when output was truncated and where the whole
- * log lives.
+ * log lives. The tail is read as bytes (`tailBytes`), cleaned, and then shrunk
+ * by lines (`tailLines`): first and last lines kept, every error/warning line
+ * kept wherever it sits, the rest collapsed into counted markers — so a long
+ * build's failing step survives in the middle instead of only its summary at
+ * the end. The log on disk keeps every byte, and the note says where it is.
  */
-export function formatResult(job: Job, tailBytes: number): string {
+export function formatResult(job: Job, tailBytes: number, tailLines: number = DEFAULT_SHRINK.maxLines): string {
   const tail = readTail(job.logPath, tailBytes);
-  // Cleaned the way pi's own bash cleans its output for the model; the log on
-  // disk (which the truncation note points at) keeps every byte.
-  const text = sanitizeOutput(tail.text);
   const lines = [header(job)];
+  if (tail.missing) {
+    // A log that cannot be read is not a command that printed nothing: say so,
+    // or the model keeps polling for output that will never come.
+    lines.push(`(log unavailable: ${job.logPath} could not be read)`);
+    return lines.join("\n");
+  }
+  // Cleaned the way pi's own bash cleans its output for the model; the log on
+  // disk (which the note points at) keeps every byte.
+  const text = sanitizeOutput(tail.text).replace(/\n+$/, "");
   if (text.trim() === "") {
     lines.push(job.status === "running" ? "(no output yet)" : "(no output)");
-  } else {
-    if (tail.truncated) {
-      lines.push(`… showing the last ${countLines(text)} lines — full log: ${job.logPath}`);
-    }
-    lines.push(text.replace(/\n+$/, ""));
+    return lines.join("\n");
+  }
+  const shrunk = shrinkLines(text, { maxLines: tailLines, tail: Math.max(DEFAULT_SHRINK.tail, Math.floor(tailLines / 2)) });
+  const notes: string[] = [];
+  if (shrunk.omitted > 0) {
+    notes.push(`${shrunk.omitted} of ${shrunk.total} lines omitted, none with an error or warning (those are kept wherever they sit)`);
   }
+  if (tail.truncated) notes.push(`this is the last ${kb(Buffer.byteLength(tail.text, "utf8"))} of a ${kb(tail.bytes)} log`);
+  if (notes.length > 0) lines.push(`… ${notes.join("; ")} — full log: ${job.logPath}`);
+  lines.push(shrunk.text);
   return lines.join("\n");
 }
 

package/src/sanitize.ts

@@ -1,6 +1,10 @@
 /**
- * The same cleaning pi's own bash tool applies to what the model sees
- * (bash-executor.ts: `sanitizeBinaryOutput(stripAnsi(text)).replace(/\r/g, "")`).
+ * The cleaning pi's own bash tool applies to what the model sees
+ * (bash-executor.ts: `sanitizeBinaryOutput(stripAnsi(text)).replace(/\r/g, "")`),
+ * with one improvement: a carriage return is resolved, not deleted. pi drops
+ * the `\r` and keeps every progress frame's text, so a 2000-frame webpack bar
+ * arrives as one line holding all 2000 frames; here only the last frame of
+ * each line survives, which is what the terminal showed (see shrink.ts).
  *
  * This package re-registers `bash`, and a job's log is captured raw on disk on
  * purpose (the detached spawn writes straight to a file; no in-process pipes).
@@ -12,6 +16,8 @@
  * vendored here verbatim.
  */
 
+import { collapseCarriageReturns } from "./shrink.ts";
+
 // Valid string terminator sequences are BEL, ESC\, and 0x9c
 const ST = "(?:\\u0007|\\u001B\\u005C|\\u009C)";
 // OSC sequences only: ESC ] ... ST (non-greedy until the first ST)
@@ -44,7 +50,7 @@ export function sanitizeBinaryOutput(text: string): string {
     .join("");
 }
 
-/** Exactly what pi's bash gives the model: no ANSI, no control noise, no `\r`. */
+/** What pi's bash gives the model — no ANSI, no control noise — with `\r` redraws collapsed to their last frame. */
 export function sanitizeOutput(text: string): string {
-  return sanitizeBinaryOutput(stripAnsi(text)).replace(/\r/g, "");
+  return sanitizeBinaryOutput(collapseCarriageReturns(stripAnsi(text)));
 }

package/src/shrink.ts

@@ -0,0 +1,125 @@
+/**
+ * A long log, shrunk the way a person skims it: the first lines, the last
+ * lines, and every line that looks like trouble wherever it sits — the rest
+ * collapsed into counted markers. Adapted from caveman's log/terminal
+ * compressors (JuliusBrussee/caveman, engine/compressors/{log,terminal}.go):
+ * a byte window keeps the END of a build log, which is exactly where the
+ * summary is and not where the failing step was; a line window with an
+ * importance regex keeps both.
+ *
+ * Rules that matter, kept from the original:
+ * - an important line is never elided, no matter where it sits;
+ * - a run of fewer than 3 dropped lines is emitted verbatim — a marker that
+ *   replaces one line costs about what the line cost and reads as a hole;
+ * - a marker states facts computed from exactly the lines it replaces (their
+ *   log levels, counted), never a guess;
+ * - markers are recognised on re-entry and kept, so shrinking a shrunk text
+ *   is byte-identical — a result that re-serialises differently on a later
+ *   poll busts the provider's prefix cache.
+ * Pure; zero dependencies.
+ */
+
+/** Words that mark trouble, case-insensitive (`error:`, `FAILED`, `Traceback`). */
+const IMPORTANT_WORDS =
+  /\b(?:ERROR|FATAL|PANIC|EXCEPTION|TRACEBACK|FAIL|FAILED|FAILURE|WARN|WARNING|CRIT|CRITICAL|ALERT|EMERG|EMERGENCY)\b|^\s+at\s|^\s+File "|:\d+:\d+:|^\s*--->|caused by|\bexit (?:status|code) [1-9]/i;
+/** camelCase error types a bare ERROR never matches: AssertionError, ValueError, NullPointerException. Case-sensitive on purpose (not "terror"). */
+const CAMEL_ERROR = /\b[A-Za-z]+(?:Error|Exception)\b/;
+/** A log level token, for the marker's counts. */
+const LEVEL = /\b(TRACE|DEBUG|INFO|NOTICE|WARN|WARNING|ERROR|FATAL)\b/i;
+/** A marker this module wrote earlier; kept as-is on re-entry. */
+export const MARKER_RE = /^\[… \d+ lines omitted/;
+
+export function isImportantLine(line: string): boolean {
+  return IMPORTANT_WORDS.test(line) || CAMEL_ERROR.test(line) || MARKER_RE.test(line);
+}
+
+/**
+ * A bare carriage return rewinds the cursor: `10%\r20%\r100% done` only ever
+ * displayed `100% done`, and the earlier frames are visual history. Deleting
+ * the `\r` instead (what pi's own bash does) concatenates every frame into
+ * one enormous line. Keep the last non-empty segment of each line. A CRLF
+ * line ending leaves an empty final segment, so the content before it wins.
+ */
+export function collapseCarriageReturns(text: string): string {
+  if (!text.includes("\r")) return text;
+  return text
+    .split("\n")
+    .map((line) => {
+      if (!line.includes("\r")) return line;
+      let chosen = "";
+      for (const segment of line.split("\r")) if (segment.length > 0) chosen = segment;
+      return chosen;
+    })
+    .join("\n");
+}
+
+export interface ShrinkOptions {
+  /** Lines kept from the start. */
+  head?: number;
+  /** Lines kept from the end. */
+  tail?: number;
+  /** Texts of at most this many lines are returned untouched; 0 disables shrinking. */
+  maxLines?: number;
+}
+
+export interface Shrunk {
+  text: string;
+  /** Lines replaced by markers. */
+  omitted: number;
+  /** Lines in the input. */
+  total: number;
+}
+
+export const DEFAULT_SHRINK: Required<ShrinkOptions> = { head: 3, tail: 100, maxLines: 200 };
+
+function levelSummary(lines: string[]): string {
+  const counts = new Map<string, number>();
+  for (const line of lines) {
+    const m = LEVEL.exec(line);
+    if (!m) continue;
+    const key = m[1]!.toUpperCase();
+    counts.set(key, (counts.get(key) ?? 0) + 1);
+  }
+  if (counts.size === 0) return "";
+  // Every level seen, never a partial list; most frequent first, ties by name.
+  return [...counts.entries()]
+    .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
+    .map(([level, n]) => `${level} ×${n}`)
+    .join(", ");
+}
+
+function marker(run: number, dropped: string[]): string {
+  const summary = levelSummary(dropped);
+  return `[… ${run} lines omitted${summary ? ` (${summary})` : ""} …]`;
+}
+
+export function shrinkLines(text: string, opts: ShrinkOptions = {}): Shrunk {
+  const head = Math.max(0, opts.head ?? DEFAULT_SHRINK.head);
+  const tail = Math.max(0, opts.tail ?? DEFAULT_SHRINK.tail);
+  const maxLines = opts.maxLines ?? DEFAULT_SHRINK.maxLines;
+  const lines = text.split("\n");
+  const total = lines.length;
+  if (maxLines <= 0 || total <= maxLines) return { text, omitted: 0, total };
+  const keep = lines.map((line, i) => i < head || i >= total - tail || isImportantLine(line));
+  const out: string[] = [];
+  let omitted = 0;
+  let i = 0;
+  while (i < total) {
+    if (keep[i]) {
+      out.push(lines[i]!);
+      i++;
+      continue;
+    }
+    let j = i;
+    while (j < total && !keep[j]) j++;
+    const run = j - i;
+    if (run < 3) {
+      for (let k = i; k < j; k++) out.push(lines[k]!);
+    } else {
+      out.push(marker(run, lines.slice(i, j)));
+      omitted += run;
+    }
+    i = j;
+  }
+  return { text: out.join("\n"), omitted, total };
+}

package/src/tail.ts

@@ -20,6 +20,8 @@ export interface Tail {
   truncated: boolean;
   /** Total size of the file in bytes. */
   bytes: number;
+  /** True when the log could not be opened at all — distinct from a command that printed nothing. */
+  missing?: boolean;
 }
 
 export const DEFAULT_TAIL_BYTES = 64 * 1024;
@@ -29,7 +31,7 @@ export function readTail(path: string, maxBytes: number = DEFAULT_TAIL_BYTES): T
   try {
     fd = openSync(path, "r");
   } catch {
-    return { text: "", truncated: false, bytes: 0 };
+    return { text: "", truncated: false, bytes: 0, missing: true };
   }
   try {
     const size = fstatSync(fd).size;