@drewpayment/mink 0.13.0-beta.1 → 0.13.0-beta.3

package/src/core/compression.ts

@@ -0,0 +1,81 @@
+// Tool-output compression — configuration and decision logic (spec 21).
+//
+// This module is pure: it reads config and makes the eligibility / holdout /
+// min-savings decisions. It never touches the database or the tool payload, so
+// it is trivially testable. Phase 2 wires the actual compressors and the
+// reversible cache on top of these decisions; Phase 1 ships the measurement
+// instrument and leaves `enabled` off by default.
+
+import { resolveConfigValue } from "./global-config";
+import type { ConfigKey } from "../types/config";
+
+export interface CompressionConfig {
+  enabled: boolean;
+  thresholdTokens: number;
+  minSavingsRatio: number;
+  holdoutFraction: number;
+  retentionHours: number;
+}
+
+function numberValue(key: ConfigKey, fallback: number, min: number, max: number): number {
+  const raw = resolveConfigValue(key).value;
+  const n = Number(raw);
+  if (!Number.isFinite(n)) return fallback;
+  return Math.min(max, Math.max(min, n));
+}
+
+export function loadCompressionConfig(): CompressionConfig {
+  return {
+    enabled: resolveConfigValue("compression.enabled").value === "true",
+    thresholdTokens: numberValue("compression.threshold-tokens", 800, 0, Number.MAX_SAFE_INTEGER),
+    minSavingsRatio: numberValue("compression.min-savings-ratio", 0.25, 0, 1),
+    holdoutFraction: numberValue("compression.holdout-fraction", 0.1, 0, 1),
+    retentionHours: numberValue("compression.retention-hours", 168, 0, Number.MAX_SAFE_INTEGER),
+  };
+}
+
+// An output is eligible for compression only once it crosses the size threshold;
+// small outputs are never touched (spec 21 §Eligibility).
+export function isEligible(originalTokens: number, config: CompressionConfig): boolean {
+  return config.enabled && originalTokens >= config.thresholdTokens;
+}
+
+// A compression attempt is kept only if it saves at least the configured
+// fraction of tokens; otherwise the original is used (spec 21 §Thresholds).
+export function meetsMinSavings(
+  originalTokens: number,
+  compressedTokens: number,
+  config: CompressionConfig
+): boolean {
+  if (originalTokens <= 0) return false;
+  const ratio = (originalTokens - compressedTokens) / originalTokens;
+  return ratio >= config.minSavingsRatio;
+}
+
+export function measuredSavings(originalTokens: number, compressedTokens: number): number {
+  return Math.max(0, originalTokens - compressedTokens);
+}
+
+// Deterministic FNV-1a hash → a stable fraction in [0, 1) for a given key. Used
+// so holdout selection is stable per event: the same event always lands in the
+// same arm, which keeps measurement from being double-counted (spec 21 edge
+// case "Holdout selection must be stable for a given event").
+function hashUnitInterval(key: string): number {
+  let h = 0x811c9dc5;
+  for (let i = 0; i < key.length; i++) {
+    h ^= key.charCodeAt(i);
+    h = Math.imul(h, 0x01000193);
+  }
+  // Map the 32-bit unsigned result into [0, 1).
+  return (h >>> 0) / 0x100000000;
+}
+
+// Decide whether a given event is held out (left uncompressed as a control).
+// Selection is deterministic in `eventKey`, so callers must pass a key that is
+// stable for the event (e.g. a hash of the original output) and not, say, a
+// timestamp.
+export function selectHoldout(eventKey: string, fraction: number): boolean {
+  if (fraction <= 0) return false;
+  if (fraction >= 1) return true;
+  return hashUnitInterval(eventKey) < fraction;
+}

package/src/core/dashboard-api.ts

@@ -80,7 +80,10 @@ import type {
   WikiPanelPayload,
   WikiNotePayload,
   WikiTreeNode,
+  CompressionPayload,
 } from "../types/dashboard";
+import { TokenLedgerRepo } from "../repositories/token-ledger-repo";
+import { loadCompressionConfig } from "./compression";
 import { isDesignEvalReport } from "../types/design-eval";
 import type { DesignEvalReport } from "../types/design-eval";
 import type { FileIndex, FileIndexEntry } from "../types/file-index";
@@ -185,7 +188,7 @@ export function loadOverview(cwd: string): OverviewPayload {
     checkJsonFile("scheduler-manifest.json", schedulerManifestPath(cwd)),
   ];
 
-  return { project, daemon, summary, stateFiles };
+  return { project, daemon, summary, compression: ledger.compression, stateFiles };
 }
 
 export function loadTokenLedgerPanel(cwd: string): TokenLedgerPayload {
@@ -194,6 +197,22 @@ export function loadTokenLedgerPanel(cwd: string): TokenLedgerPayload {
     lifetime: ledger.lifetime,
     sessions: ledger.sessions,
     wasteFlags: ledger.wasteFlags ?? [],
+    compression: ledger.compression,
+  };
+}
+
+// Dedicated Compression panel (spec 21, phase 4). Reads the measured
+// compression aggregates, the holdout A/B split, per-kind/per-tool breakdowns,
+// and recent events, plus whether compression is currently enabled.
+export function loadCompressionPanel(cwd: string): CompressionPayload {
+  const repo = TokenLedgerRepo.for(cwd);
+  return {
+    enabled: loadCompressionConfig().enabled,
+    lifetime: repo.compressionLifetime(),
+    arms: repo.compressionArms(),
+    byKind: repo.compressionBreakdown("content_kind"),
+    byTool: repo.compressionBreakdown("tool_name"),
+    recent: repo.compressionEvents(50),
   };
 }
 

package/src/core/dashboard-server.ts

@@ -5,6 +5,7 @@ import { projectDir, designCapturesDir } from "./paths";
 import {
   loadOverview,
   loadTokenLedgerPanel,
+  loadCompressionPanel,
   loadFileIndexPanel,
   loadSchedulerPanel,
   loadLearningMemoryPanel,
@@ -525,6 +526,8 @@ export async function startDashboardServer(
               return jsonResponse(loadOverview(resolvedCwd));
             case "/api/token-ledger":
               return jsonResponse(loadTokenLedgerPanel(resolvedCwd));
+            case "/api/compression":
+              return jsonResponse(loadCompressionPanel(resolvedCwd));
             case "/api/file-index":
               return jsonResponse(loadFileIndexPanel(resolvedCwd));
             case "/api/scheduler":

package/src/core/hook-output.ts

@@ -0,0 +1,42 @@
+// Helpers for PostToolUse hooks that replace a tool's result (spec 21). The
+// replacement mechanism is Claude Code's `hookSpecificOutput.updatedToolOutput`
+// (verified against the hooks reference): whatever JSON we print to stdout here
+// substitutes the original output before the model sees it.
+
+import type { PostToolUseInput } from "../types/hook-input";
+
+// Best-effort extraction of the human-visible text from a PostToolUse payload,
+// across the shapes Claude Code uses for different tools (Bash stdout, Grep
+// content, MCP results). Returns null when no text is present, in which case the
+// caller must not compress (there is nothing to safely capture or replace).
+export function extractToolOutputText(input: PostToolUseInput): string | null {
+  const tr = input.tool_response as Record<string, unknown> | undefined;
+  if (tr) {
+    if (typeof tr.content === "string") return tr.content;
+    if (Array.isArray(tr.content)) {
+      const parts = (tr.content as Array<{ text?: unknown }>)
+        .map((p) => (p && typeof p.text === "string" ? p.text : ""))
+        .filter((s) => s.length > 0);
+      if (parts.length > 0) return parts.join("");
+    }
+    if (typeof tr.stdout === "string" && tr.stdout.length > 0) return tr.stdout;
+    if (typeof tr.text === "string") return tr.text;
+    const file = tr.file as { content?: unknown } | undefined;
+    if (file && typeof file.content === "string") return file.content;
+  }
+  const to = input.tool_output;
+  if (to && typeof to.content === "string") return to.content;
+  return null;
+}
+
+// Print the replacement so Claude Code swaps it in for the original output.
+export function emitUpdatedToolOutput(text: string): void {
+  process.stdout.write(
+    JSON.stringify({
+      hookSpecificOutput: {
+        hookEventName: "PostToolUse",
+        updatedToolOutput: text,
+      },
+    })
+  );
+}

package/src/core/output-compression.ts

@@ -0,0 +1,252 @@
+// Tool-output compression engine (spec 21 §Content-Aware Compression).
+//
+// Pure, deterministic, dependency-free. Each compressor takes a tool output
+// string and returns a smaller body plus a note of what was dropped, or null
+// when it has nothing worth substituting. No I/O, no DB, no token counting and
+// no retrieval-affordance text — the pipeline (compress-tool-output.ts) owns
+// eligibility, the holdout, the min-savings gate, the cache, and the
+// "mink retrieve" footer. Keeping this layer pure makes every strategy trivially
+// testable and prompt-cache-stable (identical input → identical output).
+//
+// The "file" strategy does line-based signature extraction; spec 21's phase 3
+// upgrades it to richer AST skeletons behind this same interface.
+
+import type { ContentKind, CompressionResult } from "../types/compression";
+import { extractCodeSkeleton } from "./code-skeleton";
+
+// Tuning constants. Fixed (not config) so output is deterministic and stable.
+const SEARCH_MAX_PER_FILE = 5;
+const LOG_HEAD = 40;
+const LOG_TAIL = 40;
+const TEXT_HEAD = 30;
+const TEXT_TAIL = 20;
+const JSON_ARRAY_HEAD = 20;
+const JSON_ARRAY_TAIL = 5;
+
+// Strip ANSI CSI escape sequences (colour, cursor moves) — pure noise in logs.
+const ANSI = /\[[0-9;?]*[ -/]*[@-~]/g;
+
+function stripAnsi(s: string): string {
+  return s.replace(ANSI, "");
+}
+
+function omittedMarker(n: number): string {
+  return `  … ${n} line${n === 1 ? "" : "s"} omitted — mink retrieve …`;
+}
+
+// Split into lines, dropping a single trailing empty line (from a final newline)
+// so counts and windows aren't skewed by it.
+function toLines(content: string): string[] {
+  const lines = content.split("\n");
+  if (lines.length > 0 && lines[lines.length - 1] === "") lines.pop();
+  return lines;
+}
+
+// ── Logs / command output ───────────────────────────────────────────────────
+// Strip ANSI, collapse runs of identical lines, then keep a head+tail window.
+function compressLog(content: string): { compressed: string; omittedNote: string } | null {
+  const lines = toLines(stripAnsi(content));
+
+  // Collapse consecutive duplicates into "<line>  (×N)".
+  const collapsed: string[] = [];
+  let i = 0;
+  while (i < lines.length) {
+    let run = 1;
+    while (i + run < lines.length && lines[i + run] === lines[i]) run++;
+    collapsed.push(run > 1 ? `${lines[i]}  (×${run})` : lines[i]);
+    i += run;
+  }
+
+  if (collapsed.length <= LOG_HEAD + LOG_TAIL) {
+    // Only worth substituting if collapsing actually removed lines.
+    if (collapsed.length === lines.length) return null;
+    return {
+      compressed: collapsed.join("\n"),
+      omittedNote: `collapsed ${lines.length - collapsed.length} repeated line(s)`,
+    };
+  }
+
+  const omitted = collapsed.length - LOG_HEAD - LOG_TAIL;
+  const head = collapsed.slice(0, LOG_HEAD);
+  const tail = collapsed.slice(collapsed.length - LOG_TAIL);
+  return {
+    compressed: [...head, omittedMarker(omitted), ...tail].join("\n"),
+    omittedNote: `${omitted} of ${collapsed.length} log line(s) omitted (middle)`,
+  };
+}
+
+// ── Search / match results ──────────────────────────────────────────────────
+// Dedup exact lines and cap matches per file (the file prefix before the first
+// colon), appending a per-file "+K more" tally.
+function compressSearch(content: string): { compressed: string; omittedNote: string } | null {
+  const lines = toLines(content);
+  const seen = new Set<string>();
+  const perFile = new Map<string, number>();
+  const omittedByFile = new Map<string, number>();
+  const out: string[] = [];
+
+  for (const line of lines) {
+    if (seen.has(line)) continue;
+    seen.add(line);
+    const colon = line.indexOf(":");
+    const file = colon > 0 ? line.slice(0, colon) : line;
+    const count = perFile.get(file) ?? 0;
+    if (count < SEARCH_MAX_PER_FILE) {
+      perFile.set(file, count + 1);
+      out.push(line);
+    } else {
+      omittedByFile.set(file, (omittedByFile.get(file) ?? 0) + 1);
+    }
+  }
+
+  let totalOmitted = 0;
+  for (const [file, n] of omittedByFile) {
+    totalOmitted += n;
+    out.push(`  … +${n} more match(es) in ${file} — mink retrieve …`);
+  }
+  const dedupRemoved = lines.length - seen.size;
+
+  // Nothing changed → not worth substituting.
+  if (totalOmitted === 0 && dedupRemoved === 0) return null;
+
+  const notes: string[] = [];
+  if (totalOmitted > 0) notes.push(`${totalOmitted} match(es) capped`);
+  if (dedupRemoved > 0) notes.push(`${dedupRemoved} duplicate(s) removed`);
+  return { compressed: out.join("\n"), omittedNote: notes.join("; ") };
+}
+
+// ── Large file reads ────────────────────────────────────────────────────────
+// Brace-aware structural skeleton (see code-skeleton.ts): declarations and class
+// members with bodies elided. Falls back to a generic text window when the
+// content has no recognisable structure.
+function compressFile(
+  filePath: string,
+  content: string
+): { compressed: string; omittedNote: string } | null {
+  const ext = filePath.slice(filePath.lastIndexOf(".")).toLowerCase();
+  const markdown = ext === ".md" || ext === ".mdx" || ext === ".markdown";
+  const skeleton = extractCodeSkeleton(content, { markdown });
+
+  if (!skeleton) {
+    // No recognisable structure — fall back to a generic text window.
+    return compressText(content);
+  }
+
+  const header =
+    `${filePath} — structural summary ` +
+    `(${skeleton.lines.length} signature(s) of ${skeleton.totalLines} lines)`;
+  return {
+    compressed: [header, ...skeleton.lines].join("\n"),
+    omittedNote: `bodies elided; ${skeleton.totalLines} lines available via mink retrieve`,
+  };
+}
+
+// ── Structured data ─────────────────────────────────────────────────────────
+// Recursively "crush" JSON: sample any over-long array (at any depth), recursing
+// into the elements that are kept. Records how many elements were dropped.
+function crush(value: unknown): { value: unknown; omitted: number } {
+  if (Array.isArray(value)) {
+    let omitted = 0;
+    const mapEl = (el: unknown): unknown => {
+      const r = crush(el);
+      omitted += r.omitted;
+      return r.value;
+    };
+    if (value.length <= JSON_ARRAY_HEAD + JSON_ARRAY_TAIL) {
+      return { value: value.map(mapEl), omitted };
+    }
+    const dropped = value.length - JSON_ARRAY_HEAD - JSON_ARRAY_TAIL;
+    omitted += dropped;
+    const out = [
+      ...value.slice(0, JSON_ARRAY_HEAD).map(mapEl),
+      `… ${dropped} element(s) omitted — mink retrieve …`,
+      ...value.slice(value.length - JSON_ARRAY_TAIL).map(mapEl),
+    ];
+    return { value: out, omitted };
+  }
+  if (value && typeof value === "object") {
+    let omitted = 0;
+    const out: Record<string, unknown> = {};
+    for (const [k, v] of Object.entries(value as Record<string, unknown>)) {
+      const r = crush(v);
+      omitted += r.omitted;
+      out[k] = r.value;
+    }
+    return { value: out, omitted };
+  }
+  return { value, omitted: 0 };
+}
+
+function compressJson(content: string): { compressed: string; omittedNote: string } | null {
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(content);
+  } catch {
+    return null;
+  }
+  const { value, omitted } = crush(parsed);
+  if (omitted === 0) return null;
+  return {
+    compressed: JSON.stringify(value, null, 2),
+    omittedNote: `${omitted} array element(s) sampled out`,
+  };
+}
+
+// ── Generic text ────────────────────────────────────────────────────────────
+function compressText(content: string): { compressed: string; omittedNote: string } | null {
+  const lines = toLines(content);
+  if (lines.length <= TEXT_HEAD + TEXT_TAIL) return null;
+  const omitted = lines.length - TEXT_HEAD - TEXT_TAIL;
+  const head = lines.slice(0, TEXT_HEAD);
+  const tail = lines.slice(lines.length - TEXT_TAIL);
+  return {
+    compressed: [...head, omittedMarker(omitted), ...tail].join("\n"),
+    omittedNote: `${omitted} of ${lines.length} line(s) omitted (middle)`,
+  };
+}
+
+// ── Routing ─────────────────────────────────────────────────────────────────
+
+export function detectContentKind(
+  toolName: string,
+  content: string,
+  filePath?: string
+): ContentKind {
+  const t = toolName.toLowerCase();
+  if (t === "read") return "file";
+  if (t === "grep" || t === "glob") return "search";
+  if (t === "bash") return "log";
+  // Generic / MCP output — sniff for JSON.
+  const head = content.trimStart()[0];
+  if (head === "{" || head === "[") {
+    try {
+      JSON.parse(content);
+      return "json";
+    } catch {
+      // not JSON — fall through
+    }
+  }
+  // A file path with no tool hint still implies a file read.
+  if (filePath) return "file";
+  return "text";
+}
+
+// Compress an output by its detected kind. Returns null when there is nothing
+// worth substituting; the caller then passes the original through unchanged.
+export function compressOutput(
+  toolName: string,
+  content: string,
+  filePath?: string
+): CompressionResult | null {
+  const kind = detectContentKind(toolName, content, filePath);
+  let result: { compressed: string; omittedNote: string } | null;
+  switch (kind) {
+    case "search": result = compressSearch(content); break;
+    case "log":    result = compressLog(content); break;
+    case "file":   result = compressFile(filePath ?? "file", content); break;
+    case "json":   result = compressJson(content); break;
+    case "text":   result = compressText(content); break;
+  }
+  if (!result) return null;
+  return { kind, compressed: result.compressed, omittedNote: result.omittedNote };
+}

package/src/core/token-estimate.ts

@@ -34,3 +34,43 @@ export function estimateTokens(content: string, filePath: string): number {
   }
   return Math.ceil(content.length / ratio);
 }
+
+// A deterministic, dependency-free token counter that segments text the way a
+// BPE tokenizer roughly would — on word, number, and punctuation boundaries —
+// rather than dividing by a single flat character ratio. It is more faithful
+// than `estimateTokens` (which exists for the file-index hot path and is pinned
+// by exact-ratio tests), and crucially it does not need a file extension, so it
+// can score arbitrary tool output (logs, search results, command output).
+//
+// It is intentionally NOT a real BPE vocabulary: Mink ships as a lean CLI with a
+// single runtime dependency, and the compression-measurement use only needs a
+// *consistent* estimator to compute an original-minus-compressed delta. The
+// signature is stable, so a real BPE library can be dropped in behind it later
+// without touching call sites.
+//
+// Segmentation: runs of ASCII letters and runs of digits each collapse to a
+// handful of sub-word tokens; every other character (punctuation, symbols,
+// non-ASCII, whitespace) is scored individually. Whitespace usually merges into
+// an adjacent token in real tokenizers, so spaces and tabs cost nothing and only
+// newlines count.
+export function countTokens(text: string): number {
+  if (!text) return 0;
+  const segments = text.match(/[A-Za-z]+|[0-9]+|[^A-Za-z0-9]/g);
+  if (!segments) return 0;
+  let tokens = 0;
+  for (const seg of segments) {
+    const first = seg.charCodeAt(0);
+    if ((first >= 65 && first <= 90) || (first >= 97 && first <= 122)) {
+      tokens += Math.ceil(seg.length / 4); // word splits into ~4-char sub-words
+    } else if (first >= 48 && first <= 57) {
+      tokens += Math.ceil(seg.length / 3); // digit runs tokenize more finely
+    } else if (seg === "\n") {
+      tokens += 1; // newlines are their own token
+    } else if (seg === " " || seg === "\t" || seg === "\r") {
+      // whitespace merges into the adjacent token — no extra cost
+    } else {
+      tokens += 1; // punctuation / symbol / non-ASCII char
+    }
+  }
+  return tokens;
+}

package/src/repositories/compression-cache-repo.ts

@@ -0,0 +1,97 @@
+// Reversible-compression cache repository (spec 21 §Reversibility). Stores the
+// byte-exact original of a compressed tool output keyed by a short retrieval
+// token, with a TTL. `get` treats an expired row as a miss and evicts it lazily,
+// so a stale token can never return partial or wrong content.
+//
+// This is a local cache — it is never injected into model context and is not
+// part of the cross-device sync surface. device_id is recorded for audit only.
+
+import { randomUUID } from "crypto";
+import type { DbDriver } from "../storage/driver";
+import type { CompressionCacheEntry, ContentKind } from "../types/compression";
+import { openProjectDb } from "../storage/db";
+import { getOrCreateDeviceId } from "../core/device";
+
+export interface StoreInput {
+  toolName: string;
+  contentKind: ContentKind;
+  content: string;
+  retentionHours: number;
+  token?: string;
+  now?: Date;
+}
+
+export class CompressionCacheRepo {
+  constructor(private readonly db: DbDriver) {}
+
+  static for(cwd: string): CompressionCacheRepo {
+    return new CompressionCacheRepo(openProjectDb(cwd));
+  }
+
+  // Short, unambiguous token the model can paste into `mink retrieve`.
+  static newToken(): string {
+    return `mc-${randomUUID().slice(0, 8)}`;
+  }
+
+  // Store an original and return its retrieval token.
+  store(input: StoreInput, deviceId: string = getOrCreateDeviceId()): string {
+    const token = input.token ?? CompressionCacheRepo.newToken();
+    const now = input.now ?? new Date();
+    const createdAt = now.toISOString();
+    const expiresAt = new Date(
+      now.getTime() + Math.max(0, input.retentionHours) * 3_600_000
+    ).toISOString();
+    this.db.prepare(`
+      INSERT OR REPLACE INTO compression_cache
+        (token, created_at, expires_at, tool_name, content_kind,
+         content, size_bytes, device_id)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+    `).run(
+      token, createdAt, expiresAt, input.toolName, input.contentKind,
+      input.content, Buffer.byteLength(input.content, "utf-8"), deviceId
+    );
+    return token;
+  }
+
+  // Return the stored original, or null if the token is unknown or expired.
+  // An expired row is deleted on the way out (lazy eviction).
+  get(token: string, now: Date = new Date()): CompressionCacheEntry | null {
+    const row = this.db
+      .prepare("SELECT * FROM compression_cache WHERE token = ?")
+      .get(token) as Record<string, unknown> | undefined;
+    if (!row) return null;
+    const expiresAt = String(row.expires_at);
+    if (expiresAt <= now.toISOString()) {
+      try {
+        this.db.prepare("DELETE FROM compression_cache WHERE token = ?").run(token);
+      } catch {
+        // best effort — a failed eviction still reports a miss below
+      }
+      return null;
+    }
+    return {
+      token: String(row.token),
+      createdAt: String(row.created_at),
+      expiresAt,
+      toolName: String(row.tool_name),
+      contentKind: String(row.content_kind) as ContentKind,
+      content: String(row.content),
+      sizeBytes: Number(row.size_bytes),
+    };
+  }
+
+  // Delete every row whose TTL has elapsed. Returns the count removed.
+  evictExpired(now: Date = new Date()): number {
+    const r = this.db
+      .prepare("DELETE FROM compression_cache WHERE expires_at <= ?")
+      .run(now.toISOString());
+    return Number(r.changes);
+  }
+
+  count(): number {
+    const row = this.db
+      .prepare("SELECT COUNT(*) AS n FROM compression_cache")
+      .get() as { n: number };
+    return Number(row.n);
+  }
+}