hillclimb 0.5.4 → 0.5.5

package/dist/cli.js

@@ -2419,6 +2419,48 @@ async function readFileContent(file) {
   }
 }
 
+// src/middleware/large-file.ts
+import { constants as bufferConstants } from "buffer";
+var MAX_STRINGIFIABLE_BYTES = bufferConstants.MAX_STRING_LENGTH;
+var largeFileThreshold = MAX_STRINGIFIABLE_BYTES;
+function getLargeFileThreshold() {
+  return largeFileThreshold;
+}
+var NEWLINE = 10;
+var OVERSIZED_LINE_PLACEHOLDER = '{"_hillclimb_omitted":"line exceeded max string length; dropped to allow redaction"}';
+function redactBufferByLine(buffer, redactLine, options = {}) {
+  const maxLineBytes = options.maxLineBytes ?? MAX_STRINGIFIABLE_BYTES;
+  const oversizedPlaceholder = options.oversizedPlaceholder ?? OVERSIZED_LINE_PLACEHOLDER;
+  const pieces = [];
+  const newlineBuf = Buffer.from([NEWLINE]);
+  let totalCount = 0;
+  let oversizedLines = 0;
+  let start = 0;
+  let first = true;
+  for (; ; ) {
+    const nl = buffer.indexOf(NEWLINE, start);
+    const end = nl === -1 ? buffer.length : nl;
+    const slice = buffer.subarray(start, end);
+    if (!first) pieces.push(newlineBuf);
+    first = false;
+    if (slice.length > maxLineBytes) {
+      oversizedLines++;
+      pieces.push(Buffer.from(oversizedPlaceholder, "utf-8"));
+    } else {
+      const { value, count } = redactLine(slice.toString("utf-8"));
+      totalCount += count;
+      pieces.push(Buffer.from(value, "utf-8"));
+    }
+    if (nl === -1) break;
+    start = nl + 1;
+  }
+  return {
+    content: Buffer.concat(pieces),
+    count: totalCount,
+    oversizedLines
+  };
+}
+
 // src/middleware/patterns.json
 var patterns_default = {
   patterns: [
@@ -10658,6 +10700,19 @@ function processFiles(files) {
 }
 
 // src/middleware/pattern-redact.ts
+function redactPatternLine(line, patterns, memo, isJsonl) {
+  if (isJsonl) {
+    if (!line.trim()) return { value: line, count: 0 };
+    try {
+      const parsed = JSON.parse(line);
+      const walked = walkAndRedactAll(parsed, patterns, memo);
+      return { value: JSON.stringify(walked.value), count: walked.count };
+    } catch {
+      return redactString(line, patterns);
+    }
+  }
+  return redactString(line, patterns);
+}
 var WORKER_COUNT = Math.min(
   os3.availableParallelism?.() ?? os3.cpus().length,
   4
@@ -10686,9 +10741,14 @@ var PatternRedactMiddleware = class {
     const tasks = [];
     const fileMap = /* @__PURE__ */ new Map();
     const passThrough = [];
+    const largeResults = /* @__PURE__ */ new Map();
     for (let i = 0; i < group.files.length; i++) {
       const file = group.files[i];
       this.stats.filesScanned++;
+      if (file.content && !file.isBinary && file.content.length > getLargeFileThreshold()) {
+        largeResults.set(i, this.redactLargeFile(file));
+        continue;
+      }
       const read = await readFileContent(file);
       if (read.kind === "binary") {
         this.stats.binaryFiles++;
@@ -10718,6 +10778,11 @@ var PatternRedactMiddleware = class {
     }
     const newFiles = [];
     for (let i = 0; i < group.files.length; i++) {
+      const large = largeResults.get(i);
+      if (large) {
+        newFiles.push(large);
+        continue;
+      }
       const pt = passThrough.find((p7) => p7.index === i);
       if (pt) {
         newFiles.push(pt.file);
@@ -10739,6 +10804,33 @@ var PatternRedactMiddleware = class {
     }
     return { ...group, files: newFiles };
   }
+  // Stream-redact an oversized file straight from its buffer, one line at a time,
+  // so we never build a string larger than a single line. Single-threaded (no
+  // worker pool) — it trades parallelism for bounded memory on the one giant file.
+  redactLargeFile(file) {
+    const buffer = file.content;
+    const isJsonl = file.absolutePath.endsWith(".jsonl");
+    const patterns = compilePatterns();
+    const memo = /* @__PURE__ */ new Map();
+    const { content, count, oversizedLines } = redactBufferByLine(
+      buffer,
+      (line) => redactPatternLine(line, patterns, memo, isJsonl)
+    );
+    if (oversizedLines > 0) {
+      appendLog(
+        "warn",
+        `pattern-redact: dropped ${oversizedLines} oversized line(s) in ${file.absolutePath} (exceeded max string length)`
+      );
+    }
+    if (count > 0) {
+      this.stats.filesRedacted++;
+      this.stats.totalRedactions += count;
+    }
+    if (count > 0 || oversizedLines > 0) {
+      return { ...file, content };
+    }
+    return file;
+  }
   async processWithWorkers(tasks) {
     const workerPath = resolveWorkerPath();
     const sorted = [...tasks].sort(
@@ -10814,6 +10906,10 @@ var RedactMiddleware = class {
     for (const file of group.files) {
       this.stats.filesScanned++;
       regex.lastIndex = 0;
+      if (file.content && !file.isBinary && file.content.length > getLargeFileThreshold()) {
+        newFiles.push(this.redactLargeFile(file, regex));
+        continue;
+      }
       const read = await readFileContent(file);
       if (read.kind === "binary") {
         this.stats.binaryFiles++;
@@ -10849,6 +10945,47 @@ var RedactMiddleware = class {
     }
     return { ...group, files: newFiles };
   }
+  // Stream-redact an oversized file straight from its buffer. Each line is
+  // redacted with the same JSON-aware (or raw) logic as redactJsonl, so the
+  // output matches the in-memory path exactly for files whose lines all fit in a
+  // string — which holds for real transcripts (largest line is a few MiB).
+  redactLargeFile(file, regex) {
+    const buffer = file.content;
+    const isJsonl = file.absolutePath.endsWith(".jsonl");
+    const { content, count, oversizedLines } = redactBufferByLine(
+      buffer,
+      (line) => this.redactLine(line, regex, isJsonl)
+    );
+    if (oversizedLines > 0) {
+      appendLog(
+        "warn",
+        `redact: dropped ${oversizedLines} oversized line(s) in ${file.absolutePath} (exceeded max string length)`
+      );
+    }
+    if (count > 0) {
+      this.stats.filesRedacted++;
+      this.stats.totalRedactions += count;
+    }
+    if (count > 0 || oversizedLines > 0) {
+      return { ...file, content };
+    }
+    return file;
+  }
+  redactLine(line, regex, isJsonl) {
+    if (isJsonl) {
+      if (!line.trim()) return { value: line, count: 0 };
+      try {
+        const parsed = JSON.parse(line);
+        const walked = walkAndRedact(parsed, regex);
+        return { value: JSON.stringify(walked.value), count: walked.count };
+      } catch {
+        const { result: result2, count: count2 } = this.redactString(line, regex);
+        return { value: result2, count: count2 };
+      }
+    }
+    const { result, count } = this.redactString(line, regex);
+    return { value: result, count };
+  }
   redactJsonl(content, regex) {
     let totalCount = 0;
     const lines = content.split("\n");
@@ -13351,10 +13488,10 @@ var NormalizeMiddleware = class {
         file.sourceName
       ))
         continue;
-      const content = file.content ? file.content.toString("utf-8") : null;
-      if (!content) continue;
       const sessionId = file.metadata?.sessionId ?? (file.sourceName === "codex" ? void 0 : path13.basename(file.absolutePath, ".jsonl"));
       try {
+        const content = file.content ? file.content.toString("utf-8") : null;
+        if (!content) continue;
         const trajectory = normalizeContent(
           file.sourceName,
           content,
@@ -13383,7 +13520,7 @@ var NormalizeMiddleware = class {
       } catch (err) {
         appendLog(
           "warn",
-          `normalize: failed for source=${file.sourceName} session=${sessionId ?? "<unknown>"} file=${file.absolutePath}: ${err instanceof Error ? err.message : String(err)}`
+          `normalize: skipped ATIF for source=${file.sourceName} session=${sessionId ?? "<unknown>"} file=${file.absolutePath}: ${err instanceof Error ? err.message : String(err)}`
         );
       }
     }
@@ -14085,7 +14222,7 @@ import { execFileSync as execFileSync3 } from "child_process";
 import path18 from "path";
 
 // src/git-traces/git-ops.ts
-import { execFileSync as execFileSync2 } from "child_process";
+import { execFileSync as execFileSync2, spawnSync } from "child_process";
 import fs12 from "fs";
 import os7 from "os";
 import path16 from "path";
@@ -14097,11 +14234,155 @@ var EXEC_OPTS = {
 };
 var MAX_ERROR_OUTPUT_CHARS = 2e3;
 var MAX_SNAPSHOT_FILE_BYTES = 10 * 1024 * 1024;
-var EXCLUDED_SNAPSHOT_EXTENSIONS = /* @__PURE__ */ new Set([".pdf"]);
+var MAX_BINARY_SNAPSHOT_FILE_BYTES = 1024 * 1024;
+var BINARY_SNIFF_BYTES = 8e3;
+var EXCLUDED_SNAPSHOT_EXTENSIONS = /* @__PURE__ */ new Set([
+  // Documents
+  ".pdf",
+  // Raster images
+  ".png",
+  ".jpg",
+  ".jpeg",
+  ".gif",
+  ".bmp",
+  ".tiff",
+  ".tif",
+  ".webp",
+  ".ico",
+  ".heic",
+  ".heif",
+  ".avif",
+  // Video
+  ".mp4",
+  ".mov",
+  ".avi",
+  ".mkv",
+  ".webm",
+  ".m4v",
+  ".mpg",
+  ".mpeg",
+  ".wmv",
+  ".flv",
+  // Audio
+  ".mp3",
+  ".wav",
+  ".flac",
+  ".ogg",
+  ".m4a",
+  ".aac",
+  ".aiff",
+  // Archives / compressed
+  ".zip",
+  ".tar",
+  ".gz",
+  ".tgz",
+  ".bz2",
+  ".tbz2",
+  ".xz",
+  ".7z",
+  ".rar",
+  ".zst",
+  ".lz4",
+  // Fonts
+  ".woff",
+  ".woff2",
+  ".ttf",
+  ".otf",
+  ".eot",
+  // Compiled / native artifacts
+  ".so",
+  ".dylib",
+  ".dll",
+  ".a",
+  ".o",
+  ".lib",
+  ".class",
+  ".jar",
+  ".wasm",
+  ".exe",
+  ".pyc",
+  ".pyd",
+  ".node",
+  // Databases
+  ".sqlite",
+  ".sqlite3",
+  ".db",
+  ".mdb",
+  // Office / design binaries
+  ".doc",
+  ".docx",
+  ".xls",
+  ".xlsx",
+  ".ppt",
+  ".pptx",
+  ".odt",
+  ".ods",
+  ".odp",
+  ".psd",
+  ".ai",
+  ".sketch",
+  ".fig",
+  ".blend"
+]);
+var EXCLUDED_SNAPSHOT_BASENAMES = /* @__PURE__ */ new Set([
+  ".DS_Store",
+  "Thumbs.db",
+  "desktop.ini"
+]);
 var EMPTY_TREE_SHA = "4b825dc642cb6eb9a060e54bf8d69288fbee4904";
-function hasExcludedSnapshotExtension(filePath) {
+function isExcludedSnapshotPath(filePath) {
+  if (EXCLUDED_SNAPSHOT_BASENAMES.has(path16.basename(filePath))) return true;
   return EXCLUDED_SNAPSHOT_EXTENSIONS.has(path16.extname(filePath).toLowerCase());
 }
+function isBinaryBuffer(buffer) {
+  return buffer.includes(0);
+}
+function readTreeBlobHead(repoRoot, sha) {
+  try {
+    const { stdout } = spawnSync("git", ["cat-file", "blob", sha], {
+      cwd: repoRoot,
+      timeout: GIT_COMMAND_TIMEOUT_MS,
+      maxBuffer: BINARY_SNIFF_BYTES
+    });
+    return stdout?.length ? stdout.subarray(0, BINARY_SNIFF_BYTES) : null;
+  } catch {
+    return null;
+  }
+}
+function readWorkingFileHead(absPath) {
+  let fd = null;
+  try {
+    fd = fs12.openSync(absPath, "r");
+    const buffer = Buffer.alloc(BINARY_SNIFF_BYTES);
+    const bytesRead = fs12.readSync(fd, buffer, 0, BINARY_SNIFF_BYTES, 0);
+    return buffer.subarray(0, bytesRead);
+  } catch {
+    return null;
+  } finally {
+    if (fd !== null) {
+      try {
+        fs12.closeSync(fd);
+      } catch {
+      }
+    }
+  }
+}
+function classifyOmission(filePath, sizeBytes, readHead) {
+  if (isExcludedSnapshotPath(filePath)) return "excluded-extension";
+  if (sizeBytes > MAX_SNAPSHOT_FILE_BYTES) return "file-over-limit";
+  if (sizeBytes > MAX_BINARY_SNAPSHOT_FILE_BYTES) {
+    const head = readHead();
+    if (!head) {
+      appendLog(
+        "warn",
+        `git-traces: binary sniff failed for ${filePath} (${sizeBytes} bytes); keeping it in the snapshot`
+      );
+      return null;
+    }
+    if (isBinaryBuffer(head)) return "binary-over-limit";
+  }
+  return null;
+}
 function quoteGitArg(arg) {
   if (/^[A-Za-z0-9_./:=@%+,-]+$/.test(arg)) return arg;
   return JSON.stringify(arg);
@@ -14205,7 +14486,7 @@ function recordOmittedSnapshotFile(omittedFiles, file, options = {}) {
   omittedFiles?.push(file);
   if (options.log === false) return;
   const action = file.tracked ? "omitting tracked" : "skipping untracked";
-  const reason = file.reason === "file-over-limit" ? `${file.sizeBytes} bytes > ${MAX_SNAPSHOT_FILE_BYTES} limit` : "excluded extension";
+  const reason = file.reason === "file-over-limit" ? `${file.sizeBytes} bytes > ${MAX_SNAPSHOT_FILE_BYTES} limit` : file.reason === "binary-over-limit" ? `binary ${file.sizeBytes} bytes > ${MAX_BINARY_SNAPSHOT_FILE_BYTES} binary limit` : "excluded extension";
   appendLog("warn", `git-traces: ${action} ${file.path} (${reason})`);
 }
 function parseLsTreeLongZ(output) {
@@ -14228,7 +14509,11 @@ function listOmittedTreeFiles(repoRoot, treeSha, options = {}) {
   const output = gitBuffer(repoRoot, ["ls-tree", "-r", "-l", "-z", treeSha]);
   const omitted = [];
   for (const entry of parseLsTreeLongZ(output)) {
-    const reason = hasExcludedSnapshotExtension(entry.path) ? "excluded-extension" : entry.sizeBytes > MAX_SNAPSHOT_FILE_BYTES ? "file-over-limit" : null;
+    const reason = classifyOmission(
+      entry.path,
+      entry.sizeBytes,
+      () => readTreeBlobHead(repoRoot, entry.sha)
+    );
     if (!reason) continue;
     const file = {
       path: entry.path,
@@ -14286,8 +14571,13 @@ function buildUntrackedTree(repoRoot, omittedFiles) {
   for (const relPath of list.split("\0")) {
     if (!relPath) continue;
     try {
-      const stat = fs12.lstatSync(path16.join(repoRoot, relPath));
-      const reason = hasExcludedSnapshotExtension(relPath) ? "excluded-extension" : stat.size > MAX_SNAPSHOT_FILE_BYTES ? "file-over-limit" : null;
+      const absPath = path16.join(repoRoot, relPath);
+      const stat = fs12.lstatSync(absPath);
+      const reason = classifyOmission(
+        relPath,
+        stat.size,
+        () => readWorkingFileHead(absPath)
+      );
       if (reason) {
         recordOmittedSnapshotFile(omittedFiles, {
           path: relPath,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hillclimb",
-  "version": "0.5.4",
+  "version": "0.5.5",
   "description": "Extract and export AI coding tool logs grouped by repo",
   "license": "MIT",
   "type": "module",