hillclimb 0.9.3 → 0.9.4

package/dist/main.js

@@ -4733,6 +4733,25 @@ async function readFileContent(file) {
     return { kind: "error" };
   }
 }
+async function readFileBuffer(file) {
+  if (file.isBinary) {
+    return { kind: "binary" };
+  }
+  if (file.content) {
+    return { kind: "bytes", content: file.content };
+  }
+  if (await checkBinary(file.absolutePath)) {
+    return { kind: "binary" };
+  }
+  try {
+    return {
+      kind: "bytes",
+      content: await fs13.promises.readFile(file.absolutePath)
+    };
+  } catch {
+    return { kind: "error" };
+  }
+}
 
 // src/middleware/large-file.ts
 import { constants as bufferConstants } from "buffer";
@@ -4743,36 +4762,307 @@ function getLargeFileThreshold() {
 }
 var NEWLINE = 10;
 var OVERSIZED_LINE_PLACEHOLDER = '{"_hillclimb_omitted":"line exceeded max string length; dropped to allow redaction"}';
-function redactBufferByLine(buffer, redactLine, options = {}) {
+function scanBufferLines(buffer, redactLine, options = {}) {
   const maxLineBytes = options.maxLineBytes ?? MAX_STRINGIFIABLE_BYTES;
   const oversizedPlaceholder = options.oversizedPlaceholder ?? OVERSIZED_LINE_PLACEHOLDER;
-  const pieces = [];
-  const newlineBuf = Buffer.from([NEWLINE]);
+  const end = options.end ?? buffer.length;
+  const ownsFinalLine = end === buffer.length;
+  const edits = [];
   let totalCount = 0;
   let oversizedLines = 0;
-  let start = 0;
-  let first = true;
-  for (; ; ) {
+  let start = options.start ?? 0;
+  while (ownsFinalLine || start < end) {
     const nl = buffer.indexOf(NEWLINE, start);
-    const end = nl === -1 ? buffer.length : nl;
-    const slice = buffer.subarray(start, end);
-    if (!first) pieces.push(newlineBuf);
-    first = false;
-    if (slice.length > maxLineBytes) {
+    const lineEnd = nl === -1 ? buffer.length : nl;
+    if (lineEnd - start > maxLineBytes) {
       oversizedLines++;
-      pieces.push(Buffer.from(oversizedPlaceholder, "utf-8"));
+      edits.push({ start, end: lineEnd, value: oversizedPlaceholder });
     } else {
-      const { value, count } = redactLine(slice.toString("utf-8"));
-      totalCount += count;
-      pieces.push(Buffer.from(value, "utf-8"));
+      const { value, count } = redactLine(
+        buffer.toString("utf-8", start, lineEnd)
+      );
+      if (count > 0) {
+        totalCount += count;
+        edits.push({ start, end: lineEnd, value });
+      }
     }
     if (nl === -1) break;
     start = nl + 1;
   }
+  return { edits, count: totalCount, oversizedLines };
+}
+function applyLineEdits(buffer, edits) {
+  if (edits.length === 0) return buffer;
+  const pieces = [];
+  let copied = 0;
+  for (const edit of edits) {
+    pieces.push(buffer.subarray(copied, edit.start));
+    pieces.push(Buffer.from(edit.value, "utf-8"));
+    copied = edit.end;
+  }
+  pieces.push(buffer.subarray(copied));
+  return Buffer.concat(pieces);
+}
+function redactBufferByLine(buffer, redactLine, options = {}) {
+  const { edits, count, oversizedLines } = scanBufferLines(
+    buffer,
+    redactLine,
+    options
+  );
+  return { content: applyLineEdits(buffer, edits), count, oversizedLines };
+}
+
+// src/middleware/pattern-prefilter.ts
+var MIN_LITERAL = 3;
+function score(req) {
+  if (!req) return -1;
+  let min = Number.POSITIVE_INFINITY;
+  for (const s of req) min = Math.min(min, s.length);
+  return min * 1e3 - req.length;
+}
+function better(a, b) {
+  return score(b) > score(a) ? b : a;
+}
+function isAsciiLiteral(ch) {
+  const c = ch.charCodeAt(0);
+  return c >= 32 && c < 127;
+}
+var UNKNOWN = { literals: null, minLength: 0 };
+function analyzeRegex(source, flags = "") {
+  if (flags.includes("v")) return UNKNOWN;
+  let i = 0;
+  function skipPast(close) {
+    const at = source.indexOf(close, i);
+    if (at === -1) throw new Error("unterminated escape");
+    i = at + 1;
+  }
+  function flush(seq) {
+    if (seq.run.length > 0) {
+      seq.req = better(seq.req, [seq.run.toLowerCase()]);
+      seq.run = "";
+    }
+  }
+  function quantifier() {
+    const ch = source[i];
+    let min = null;
+    if (ch === "*" || ch === "?") {
+      min = 0;
+      i++;
+    } else if (ch === "+") {
+      min = 1;
+      i++;
+    } else if (ch === "{") {
+      const m = /^\{(\d+)(?:,\d*)?\}/.exec(source.slice(i));
+      if (m) {
+        min = Number(m[1]);
+        i += m[0].length;
+      }
+    }
+    if (min !== null && source[i] === "?") i++;
+    return min;
+  }
+  function skipClass() {
+    i++;
+    while (i < source.length && source[i] !== "]") {
+      if (source[i] === "\\") i++;
+      i++;
+    }
+    i++;
+  }
+  function alternation() {
+    const branches = [];
+    let seq = { req: null, run: "", min: 0 };
+    while (i < source.length && source[i] !== ")") {
+      const ch = source[i];
+      if (ch === "|") {
+        flush(seq);
+        branches.push(seq);
+        seq = { req: null, run: "", min: 0 };
+        i++;
+        continue;
+      }
+      if (ch === "(") {
+        i++;
+        let lookaround = false;
+        if (source[i] === "?") {
+          const m = /^\?(?:[:=!]|<[=!]|<[A-Za-z_$][\w$]*>)/.exec(
+            source.slice(i)
+          );
+          if (!m) throw new Error(`unsupported group at ${i}`);
+          lookaround = m[0] !== "?:" && !m[0].endsWith(">");
+          i += m[0].length;
+        }
+        const inner = alternation();
+        if (source[i] !== ")") throw new Error("unbalanced group");
+        i++;
+        const min3 = quantifier();
+        flush(seq);
+        if (!lookaround) {
+          seq.min += inner.min * (min3 ?? 1);
+          if (min3 === null || min3 >= 1) seq.req = better(seq.req, inner.req);
+        }
+        continue;
+      }
+      if (ch === "[" || ch === ".") {
+        if (ch === "[") skipClass();
+        else i++;
+        seq.min += quantifier() ?? 1;
+        flush(seq);
+        continue;
+      }
+      if (ch === "^" || ch === "$") {
+        i++;
+        quantifier();
+        flush(seq);
+        continue;
+      }
+      let lit;
+      let width = 1;
+      if (ch === "\\") {
+        const e = source[i + 1];
+        i += 2;
+        if (/[A-Za-z0-9]/.test(e)) {
+          if (!/[dDwWsS]/.test(e)) width = 0;
+          if (e === "x") i += 2;
+          else if (e === "u") {
+            if (source[i] === "{") skipPast("}");
+            else i += 4;
+          } else if (e === "c") i += 1;
+          else if (e === "k" && source[i] === "<") skipPast(">");
+          else if ((e === "p" || e === "P") && source[i] === "{") skipPast("}");
+          lit = null;
+        } else {
+          lit = e;
+        }
+      } else {
+        lit = ch;
+        i++;
+      }
+      const min2 = quantifier();
+      seq.min += width * (min2 ?? 1);
+      if (lit === null || !isAsciiLiteral(lit)) {
+        flush(seq);
+      } else if (min2 === null) {
+        seq.run += lit;
+      } else {
+        if (min2 >= 1) seq.run += lit;
+        flush(seq);
+      }
+    }
+    flush(seq);
+    branches.push(seq);
+    let min = Number.POSITIVE_INFINITY;
+    let all = [];
+    for (const b of branches) {
+      min = Math.min(min, b.min);
+      if (!b.req) all = null;
+      else all?.push(...b.req);
+    }
+    return { req: all ? [...new Set(all)] : null, min };
+  }
+  try {
+    const { req, min } = alternation();
+    if (i !== source.length) return UNKNOWN;
+    const foldsBeyondAscii = flags.includes("i") && flags.includes("u");
+    const usable = req && !foldsBeyondAscii && !req.some((s) => s.length < MIN_LITERAL);
+    return { literals: usable ? req : null, minLength: min };
+  } catch {
+    return UNKNOWN;
+  }
+}
+function isAscii(text2) {
+  for (let k = 0; k < text2.length; k++) {
+    if (text2.charCodeAt(k) >= 128) return false;
+  }
+  return true;
+}
+function buildPrefilter(literalsByRule) {
+  const alwaysRun = [];
+  const goto = [/* @__PURE__ */ new Map()];
+  const outs = [[]];
+  literalsByRule.forEach((given, rule) => {
+    const lits = given?.map((lit) => lit.toLowerCase());
+    if (!lits || lits.some((lit) => lit === "" || !isAscii(lit))) {
+      alwaysRun.push(rule);
+      return;
+    }
+    for (const lit of lits) {
+      let s = 0;
+      for (let k = 0; k < lit.length; k++) {
+        const c = lit.charCodeAt(k);
+        let n = goto[s].get(c);
+        if (n === void 0) {
+          n = goto.length;
+          goto.push(/* @__PURE__ */ new Map());
+          outs.push([]);
+          goto[s].set(c, n);
+        }
+        s = n;
+      }
+      outs[s].push(rule);
+    }
+  });
+  const nStates = goto.length;
+  const next = new Int32Array(nStates << 7);
+  const fail = new Int32Array(nStates);
+  const queue = [];
+  for (const [c, n] of goto[0]) {
+    next[c] = n;
+    queue.push(n);
+  }
+  for (let q = 0; q < queue.length; q++) {
+    const s = queue[q];
+    const f = fail[s];
+    if (outs[f].length) outs[s] = [.../* @__PURE__ */ new Set([...outs[s], ...outs[f]])];
+    const base = s << 7;
+    const fbase = f << 7;
+    for (let c = 0; c < 128; c++) next[base | c] = next[fbase | c];
+    for (const [c, n] of goto[s]) {
+      fail[n] = next[fbase | c];
+      next[base | c] = n;
+      queue.push(n);
+    }
+  }
+  for (let s = 0; s < nStates; s++) {
+    const base = s << 7;
+    for (let c = 65; c <= 90; c++) next[base | c] = next[base | c + 32];
+  }
+  const hasOut = new Uint8Array(nStates);
+  const outRules = outs.map((o, s) => {
+    if (o.length === 0) return null;
+    hasOut[s] = 1;
+    return o;
+  });
+  const ruleStamp = new Uint32Array(literalsByRule.length);
+  const stateStamp = new Uint32Array(nStates);
+  let stamp = 0;
   return {
-    content: Buffer.concat(pieces),
-    count: totalCount,
-    oversizedLines
+    alwaysRun,
+    candidates(value, out) {
+      stamp++;
+      if (stamp === 4294967295) {
+        ruleStamp.fill(0);
+        stateStamp.fill(0);
+        stamp = 1;
+      }
+      let s = 0;
+      let found = false;
+      for (let k = 0, n = value.length; k < n; k++) {
+        const c = value.charCodeAt(k);
+        s = c < 128 ? next[s << 7 | c] : 0;
+        if (hasOut[s] === 1 && stateStamp[s] !== stamp) {
+          stateStamp[s] = stamp;
+          for (const r of outRules[s]) {
+            if (ruleStamp[r] !== stamp) {
+              ruleStamp[r] = stamp;
+              out.push(r);
+              found = true;
+            }
+          }
+        }
+      }
+      if (found) out.sort((a, b) => a - b);
+    }
   };
 }
 
@@ -13258,11 +13548,12 @@ function applyPattern(content, regex, groupCount, validate, prefixRun) {
   result += content.slice(lastIndex);
   return { value: result, count };
 }
-function redactString(value, patterns, owningKey = null) {
+function redactStringSlow(value, patterns, owningKey = null, from = 0) {
   let result = value;
   let totalCount = 0;
   let lower = null;
-  for (const p7 of patterns) {
+  for (let idx = from; idx < patterns.length; idx++) {
+    const p7 = patterns[idx];
     if (p7.precheck) {
       if (lower === null) lower = result.toLowerCase();
       let hit = false;
@@ -13290,6 +13581,69 @@ function redactString(value, patterns, owningKey = null) {
   }
   return { value: result, count: totalCount };
 }
+var engines = /* @__PURE__ */ new WeakMap();
+var candidateScratch = [];
+function engineFor(patterns) {
+  let engine = engines.get(patterns);
+  if (!engine) {
+    const minLength = new Int32Array(patterns.length);
+    const literals = patterns.map((p7, idx) => {
+      const facts = analyzeRegex(p7.regex.source, p7.regex.flags);
+      minLength[idx] = facts.minLength;
+      return p7.precheck ?? facts.literals;
+    });
+    engine = { prefilter: buildPrefilter(literals), minLength };
+    engines.set(patterns, engine);
+  }
+  return engine;
+}
+function redactString(value, patterns, owningKey = null) {
+  const { prefilter, minLength } = engineFor(patterns);
+  const hits = candidateScratch;
+  hits.length = 0;
+  prefilter.candidates(value, hits);
+  const always = prefilter.alwaysRun;
+  let lower = null;
+  let h = 0;
+  let a = 0;
+  while (h < hits.length || a < always.length) {
+    const idx = a >= always.length || h < hits.length && hits[h] < always[a] ? hits[h++] : always[a++];
+    if (value.length < minLength[idx]) continue;
+    const p7 = patterns[idx];
+    if (p7.precheck) {
+      if (lower === null) lower = value.toLowerCase();
+      let hit = false;
+      for (const needle of p7.precheck) {
+        if (lower.includes(needle)) {
+          hit = true;
+          break;
+        }
+      }
+      if (!hit) continue;
+    }
+    if (p7.context) {
+      const keyHit = owningKey !== null && p7.context.test(owningKey);
+      if (!keyHit && (p7.keyOnlyContext || !p7.context.test(value))) continue;
+    }
+    const applied = applyPattern(
+      value,
+      p7.regex,
+      p7.groupCount,
+      p7.validate,
+      p7.prefixRun
+    );
+    if (applied.count > 0) {
+      const rest = redactStringSlow(
+        applied.value,
+        patterns,
+        owningKey,
+        idx + 1
+      );
+      return { value: rest.value, count: applied.count + rest.count };
+    }
+  }
+  return { value, count: 0 };
+}
 var keyMaskCaches = /* @__PURE__ */ new WeakMap();
 var memoStringUnits = /* @__PURE__ */ new WeakMap();
 var MAX_MEMO_STRING_UNITS = 8 * 1024 * 1024;
@@ -13354,6 +13708,16 @@ function walkAndRedactAll(value, patterns, memo, owningKey = null) {
   }
   return { value, count: 0 };
 }
+function redactJsonlLine(line, patterns, memo) {
+  if (!line.trim()) return { value: line, count: 0 };
+  try {
+    const parsed = JSON.parse(line);
+    const walked = walkAndRedactAll(parsed, patterns, memo);
+    return { value: JSON.stringify(walked.value), count: walked.count };
+  } catch {
+    return redactString(line, patterns);
+  }
+}
 function processFiles(files) {
   const patterns = compilePatterns();
   const results = [];
@@ -13363,19 +13727,10 @@ function processFiles(files) {
     if (isJsonl) {
       const memo = /* @__PURE__ */ new Map();
       let count = 0;
-      const lines = content.split("\n");
-      const processed = lines.map((line) => {
-        if (!line.trim()) return line;
-        try {
-          const parsed = JSON.parse(line);
-          const walked = walkAndRedactAll(parsed, patterns, memo);
-          count += walked.count;
-          return JSON.stringify(walked.value);
-        } catch {
-          const r = redactString(line, patterns);
-          count += r.count;
-          return r.value;
-        }
+      const processed = content.split("\n").map((line) => {
+        const r = redactJsonlLine(line, patterns, memo);
+        count += r.count;
+        return r.count > 0 ? r.value : line;
       });
       totalCount = count;
       output = processed.join("\n");
@@ -13388,21 +13743,25 @@ function processFiles(files) {
   }
   return results;
 }
+function processRanges(ranges) {
+  const patterns = compilePatterns();
+  return ranges.map(({ index, bytes, start, end }) => {
+    const memo = /* @__PURE__ */ new Map();
+    const buffer = Buffer.from(
+      bytes.buffer,
+      bytes.byteOffset,
+      bytes.byteLength
+    );
+    const scanned = scanBufferLines(
+      buffer,
+      (line) => redactJsonlLine(line, patterns, memo),
+      { start, end }
+    );
+    return { index, ...scanned };
+  });
+}
 
 // src/middleware/pattern-redact.ts
-function redactPatternLine(line, patterns, memo, isJsonl) {
-  if (isJsonl) {
-    if (!line.trim()) return { value: line, count: 0 };
-    try {
-      const parsed = JSON.parse(line);
-      const walked = walkAndRedactAll(parsed, patterns, memo);
-      return { value: JSON.stringify(walked.value), count: walked.count };
-    } catch {
-      return redactString(line, patterns);
-    }
-  }
-  return redactString(line, patterns);
-}
 var WORKER_COUNT = Math.min(
   os4.availableParallelism?.() ?? os4.cpus().length,
   4
@@ -13410,6 +13769,31 @@ var WORKER_COUNT = Math.min(
 var WORKER_THRESHOLD = 4;
 var WORKER_STRING_THRESHOLD = 128 * 1024;
 var WORKER_TIMEOUT_MS = 12e4;
+var RANGE_MIN_BYTES = 8 * 1024 * 1024;
+var NEWLINE2 = 10;
+function shareBytes(source) {
+  if (source.buffer instanceof SharedArrayBuffer) return source;
+  const shared = new Uint8Array(new SharedArrayBuffer(source.length));
+  shared.set(source);
+  return shared;
+}
+function splitRanges(index, bytes) {
+  const parts = Math.min(
+    WORKER_COUNT,
+    Math.floor(bytes.length / RANGE_MIN_BYTES)
+  );
+  const ranges = [];
+  let start = 0;
+  for (let part = 1; part < parts; part++) {
+    const target = Math.max(start, Math.floor(bytes.length / parts * part));
+    const nl = bytes.indexOf(NEWLINE2, target);
+    if (nl === -1 || nl + 1 >= bytes.length) break;
+    ranges.push({ index, bytes, start, end: nl + 1 });
+    start = nl + 1;
+  }
+  ranges.push({ index, bytes, start, end: bytes.length });
+  return ranges;
+}
 function resolveWorkerPath() {
   return fileURLToPath(
     new URL(
@@ -13418,7 +13802,7 @@ function resolveWorkerPath() {
     )
   );
 }
-function runWorker(workerPath, files, signal, timeoutMs = WORKER_TIMEOUT_MS) {
+function runWorker(workerPath, job, signal, timeoutMs = WORKER_TIMEOUT_MS) {
   return new Promise((resolve, reject) => {
     if (signal?.aborted) {
       reject(
@@ -13426,7 +13810,7 @@ function runWorker(workerPath, files, signal, timeoutMs = WORKER_TIMEOUT_MS) {
       );
       return;
     }
-    const worker = new Worker(workerPath, { workerData: files });
+    const worker = new Worker(workerPath, { workerData: job });
     let settled = false;
     const finish = (error, results) => {
       if (settled) return;
@@ -13480,11 +13864,25 @@ var PatternRedactMiddleware = class {
     }
     const tasks = [];
     const fileMap = /* @__PURE__ */ new Map();
-    const passThrough = [];
+    const passThrough = /* @__PURE__ */ new Map();
     const largeResults = /* @__PURE__ */ new Map();
+    const jsonl = /* @__PURE__ */ new Map();
     for (let i = 0; i < group.files.length; i++) {
       const file = group.files[i];
       this.stats.filesScanned++;
+      if (file.absolutePath.endsWith(".jsonl")) {
+        const read2 = await readFileBuffer(file);
+        if (read2.kind === "binary") {
+          this.stats.binaryFiles++;
+          passThrough.set(i, file);
+        } else if (read2.kind === "error") {
+          this.stats.filesSkipped++;
+        } else {
+          jsonl.set(i, read2.content);
+          fileMap.set(i, file);
+        }
+        continue;
+      }
       if (file.content && !file.isBinary && file.content.length > getLargeFileThreshold()) {
         largeResults.set(i, this.redactLargeFile(file));
         continue;
@@ -13492,45 +13890,67 @@ var PatternRedactMiddleware = class {
       const read = await readFileContent(file);
       if (read.kind === "binary") {
         this.stats.binaryFiles++;
-        passThrough.push({ index: i, file });
+        passThrough.set(i, file);
         continue;
       }
       if (read.kind === "error") {
         this.stats.filesSkipped++;
         continue;
       }
-      tasks.push({
-        index: i,
-        content: read.content,
-        isJsonl: file.absolutePath.endsWith(".jsonl")
-      });
+      tasks.push({ index: i, content: read.content, isJsonl: false });
       fileMap.set(i, file);
     }
-    let results;
-    if (tasks.length >= WORKER_THRESHOLD || tasks.some((task) => task.content.length >= WORKER_STRING_THRESHOLD)) {
-      results = await this.processWithWorkers(tasks);
+    const sources2 = [...jsonl.values()];
+    let output;
+    if (tasks.length + sources2.length >= WORKER_THRESHOLD || tasks.some((task) => task.content.length >= WORKER_STRING_THRESHOLD) || sources2.some((source) => source.length >= WORKER_STRING_THRESHOLD)) {
+      const ranges = [];
+      for (const [index, source] of jsonl) {
+        ranges.push(...splitRanges(index, shareBytes(source)));
+      }
+      output = await this.processWithWorkers(tasks, ranges);
     } else {
-      results = processFiles(tasks);
+      output = {
+        files: processFiles(tasks),
+        ranges: processRanges(
+          [...jsonl].map(([index, bytes]) => ({
+            index,
+            bytes,
+            start: 0,
+            end: bytes.length
+          }))
+        )
+      };
     }
     const outputMap = /* @__PURE__ */ new Map();
-    for (const r of results) {
-      outputMap.set(r.index, { content: r.content, count: r.count });
+    for (const r of output.files) outputMap.set(r.index, r);
+    const scans = /* @__PURE__ */ new Map();
+    for (const { index, edits, count, oversizedLines } of output.ranges) {
+      const scan = scans.get(index);
+      if (!scan) {
+        scans.set(index, { edits, count, oversizedLines });
+        continue;
+      }
+      for (const edit of edits) scan.edits.push(edit);
+      scan.count += count;
+      scan.oversizedLines += oversizedLines;
     }
     const newFiles = [];
     for (let i = 0; i < group.files.length; i++) {
-      const large = largeResults.get(i);
-      if (large) {
-        newFiles.push(large);
+      const unchanged = largeResults.get(i) ?? passThrough.get(i);
+      if (unchanged) {
+        newFiles.push(unchanged);
         continue;
       }
-      const pt = passThrough.find((p7) => p7.index === i);
-      if (pt) {
-        newFiles.push(pt.file);
+      const file = fileMap.get(i);
+      if (!file) continue;
+      const source = jsonl.get(i);
+      const scan = scans.get(i);
+      if (source && scan) {
+        newFiles.push(this.applyScan(file, source, scan));
         continue;
       }
       const result = outputMap.get(i);
-      const file = fileMap.get(i);
-      if (!result || !file) continue;
+      if (!result) continue;
       if (result.count > 0) {
         this.stats.filesRedacted++;
         this.stats.totalRedactions += result.count;
@@ -13544,17 +13964,31 @@ var PatternRedactMiddleware = class {
     }
     return { ...group, files: newFiles };
   }
-  // Stream-redact an oversized file straight from its buffer, one line at a time,
-  // so we never build a string larger than a single line. Single-threaded (no
-  // worker pool) — it trades parallelism for bounded memory on the one giant file.
+  applyScan(file, source, scan) {
+    if (scan.oversizedLines > 0) {
+      appendLog(
+        "warn",
+        `pattern-redact: dropped ${scan.oversizedLines} oversized line(s) in ${file.absolutePath} (exceeded max string length)`
+      );
+    }
+    if (scan.count > 0) {
+      this.stats.filesRedacted++;
+      this.stats.totalRedactions += scan.count;
+    }
+    if (scan.count === 0 && scan.oversizedLines === 0) return file;
+    scan.edits.sort((a, b) => a.start - b.start);
+    return { ...file, content: applyLineEdits(source, scan.edits) };
+  }
+  // Stream-redact an oversized non-JSONL file straight from its buffer, one line
+  // at a time, so we never build a string larger than a single line.
+  // Single-threaded (no worker pool) — it trades parallelism for bounded memory
+  // on the one giant file.
   redactLargeFile(file) {
     const buffer = file.content;
-    const isJsonl = file.absolutePath.endsWith(".jsonl");
     const patterns = compilePatterns();
-    const memo = /* @__PURE__ */ new Map();
     const { content, count, oversizedLines } = redactBufferByLine(
       buffer,
-      (line) => redactPatternLine(line, patterns, memo, isJsonl)
+      (line) => redactString(line, patterns)
     );
     if (oversizedLines > 0) {
       appendLog(
@@ -13571,25 +14005,32 @@ var PatternRedactMiddleware = class {
     }
     return file;
   }
-  async processWithWorkers(tasks) {
+  async processWithWorkers(tasks, ranges) {
     const workerPath = resolveWorkerPath();
     const controller = new AbortController();
     const onAbort = () => controller.abort();
     this.signal?.addEventListener("abort", onAbort, { once: true });
     if (this.signal?.aborted) controller.abort();
-    const sorted = [...tasks].sort(
-      (a, b) => b.content.length - a.content.length
-    );
-    const buckets = Array.from(
-      { length: WORKER_COUNT },
-      () => []
-    );
-    for (let i = 0; i < sorted.length; i++) {
-      buckets[i % WORKER_COUNT].push(sorted[i]);
-    }
-    const pending = buckets.filter((b) => b.length > 0).map((chunk) => runWorker(workerPath, chunk, controller.signal));
+    const units = [
+      ...tasks.map((task) => ({ task, size: task.content.length })),
+      ...ranges.map((range) => ({ range, size: range.end - range.start }))
+    ].sort((a, b) => b.size - a.size);
+    const jobs = Array.from({ length: WORKER_COUNT }, () => ({
+      files: [],
+      ranges: []
+    }));
+    units.forEach((unit, i) => {
+      const job = jobs[i % WORKER_COUNT];
+      if ("task" in unit) job.files.push(unit.task);
+      else job.ranges.push(unit.range);
+    });
+    const pending = jobs.filter((job) => job.files.length > 0 || job.ranges.length > 0).map((job) => runWorker(workerPath, job, controller.signal));
     try {
-      return (await Promise.all(pending)).flat();
+      const outputs2 = await Promise.all(pending);
+      return {
+        files: outputs2.flatMap((o) => o.files),
+        ranges: outputs2.flatMap((o) => o.ranges)
+      };
     } catch (error) {
       controller.abort();
       await Promise.allSettled(pending);
@@ -13661,8 +14102,20 @@ var RedactMiddleware = class {
     for (const file of group.files) {
       this.stats.filesScanned++;
       regex.lastIndex = 0;
+      if (file.absolutePath.endsWith(".jsonl")) {
+        const read2 = await readFileBuffer(file);
+        if (read2.kind === "binary") {
+          this.stats.binaryFiles++;
+          newFiles.push(file);
+        } else if (read2.kind === "error") {
+          this.stats.filesSkipped++;
+        } else {
+          newFiles.push(this.redactBuffer(file, read2.content, regex, true));
+        }
+        continue;
+      }
       if (file.content && !file.isBinary && file.content.length > getLargeFileThreshold()) {
-        newFiles.push(this.redactLargeFile(file, regex));
+        newFiles.push(this.redactBuffer(file, file.content, regex, false));
         continue;
       }
       const read = await readFileContent(file);
@@ -13675,18 +14128,7 @@ var RedactMiddleware = class {
         this.stats.filesSkipped++;
         continue;
       }
-      const content = read.content;
-      let result;
-      let count;
-      if (file.absolutePath.endsWith(".jsonl")) {
-        const processed = this.redactJsonl(content, regex);
-        result = processed.result;
-        count = processed.count;
-      } else {
-        const processed = this.redactString(content, regex);
-        result = processed.result;
-        count = processed.count;
-      }
+      const { result, count } = this.redactString(read.content, regex);
       if (count > 0) {
         this.stats.filesRedacted++;
         this.stats.totalRedactions += count;
@@ -13700,13 +14142,9 @@ var RedactMiddleware = class {
     }
     return { ...group, files: newFiles };
   }
-  // Stream-redact an oversized file straight from its buffer. Each line is
-  // redacted with the same JSON-aware (or raw) logic as redactJsonl, so the
-  // output matches the in-memory path exactly for files whose lines all fit in a
-  // string — which holds for real transcripts (largest line is a few MiB).
-  redactLargeFile(file, regex) {
-    const buffer = file.content;
-    const isJsonl = file.absolutePath.endsWith(".jsonl");
+  // Redact a file line by line straight from its buffer. A line with no hit
+  // keeps its exact bytes; a JSONL line with one is re-serialized.
+  redactBuffer(file, buffer, regex, isJsonl) {
     const { content, count, oversizedLines } = redactBufferByLine(
       buffer,
       (line) => this.redactLine(line, regex, isJsonl)
@@ -13741,24 +14179,6 @@ var RedactMiddleware = class {
     const { result, count } = this.redactString(line, regex);
     return { value: result, count };
   }
-  redactJsonl(content, regex) {
-    let totalCount = 0;
-    const lines = content.split("\n");
-    const processed = lines.map((line) => {
-      if (!line.trim()) return line;
-      try {
-        const parsed = JSON.parse(line);
-        const walked = walkAndRedact(parsed, regex);
-        totalCount += walked.count;
-        return JSON.stringify(walked.value);
-      } catch {
-        const { result, count } = this.redactString(line, regex);
-        totalCount += count;
-        return result;
-      }
-    });
-    return { result: processed.join("\n"), count: totalCount };
-  }
   redactString(content, regex) {
     let count = 0;
     const result = content.replace(regex, () => {
@@ -16969,7 +17389,7 @@ async function uploadArtifact(client, contributionId, filename, mimeType, buffer
 }
 
 // package.json
-var version = "0.9.3";
+var version = "0.9.4";
 
 // src/version.ts
 var CLI_VERSION = version;

package/dist/pattern-worker.js

@@ -1,6 +1,295 @@
 // src/middleware/pattern-worker.ts
 import { parentPort, workerData } from "worker_threads";
 
+// src/middleware/large-file.ts
+import { constants as bufferConstants } from "buffer";
+var MAX_STRINGIFIABLE_BYTES = bufferConstants.MAX_STRING_LENGTH;
+var NEWLINE = 10;
+var OVERSIZED_LINE_PLACEHOLDER = '{"_hillclimb_omitted":"line exceeded max string length; dropped to allow redaction"}';
+function scanBufferLines(buffer, redactLine, options = {}) {
+  const maxLineBytes = options.maxLineBytes ?? MAX_STRINGIFIABLE_BYTES;
+  const oversizedPlaceholder = options.oversizedPlaceholder ?? OVERSIZED_LINE_PLACEHOLDER;
+  const end = options.end ?? buffer.length;
+  const ownsFinalLine = end === buffer.length;
+  const edits = [];
+  let totalCount = 0;
+  let oversizedLines = 0;
+  let start = options.start ?? 0;
+  while (ownsFinalLine || start < end) {
+    const nl = buffer.indexOf(NEWLINE, start);
+    const lineEnd = nl === -1 ? buffer.length : nl;
+    if (lineEnd - start > maxLineBytes) {
+      oversizedLines++;
+      edits.push({ start, end: lineEnd, value: oversizedPlaceholder });
+    } else {
+      const { value, count } = redactLine(
+        buffer.toString("utf-8", start, lineEnd)
+      );
+      if (count > 0) {
+        totalCount += count;
+        edits.push({ start, end: lineEnd, value });
+      }
+    }
+    if (nl === -1) break;
+    start = nl + 1;
+  }
+  return { edits, count: totalCount, oversizedLines };
+}
+
+// src/middleware/pattern-prefilter.ts
+var MIN_LITERAL = 3;
+function score(req) {
+  if (!req) return -1;
+  let min = Number.POSITIVE_INFINITY;
+  for (const s of req) min = Math.min(min, s.length);
+  return min * 1e3 - req.length;
+}
+function better(a, b) {
+  return score(b) > score(a) ? b : a;
+}
+function isAsciiLiteral(ch) {
+  const c = ch.charCodeAt(0);
+  return c >= 32 && c < 127;
+}
+var UNKNOWN = { literals: null, minLength: 0 };
+function analyzeRegex(source, flags = "") {
+  if (flags.includes("v")) return UNKNOWN;
+  let i = 0;
+  function skipPast(close) {
+    const at = source.indexOf(close, i);
+    if (at === -1) throw new Error("unterminated escape");
+    i = at + 1;
+  }
+  function flush(seq) {
+    if (seq.run.length > 0) {
+      seq.req = better(seq.req, [seq.run.toLowerCase()]);
+      seq.run = "";
+    }
+  }
+  function quantifier() {
+    const ch = source[i];
+    let min = null;
+    if (ch === "*" || ch === "?") {
+      min = 0;
+      i++;
+    } else if (ch === "+") {
+      min = 1;
+      i++;
+    } else if (ch === "{") {
+      const m = /^\{(\d+)(?:,\d*)?\}/.exec(source.slice(i));
+      if (m) {
+        min = Number(m[1]);
+        i += m[0].length;
+      }
+    }
+    if (min !== null && source[i] === "?") i++;
+    return min;
+  }
+  function skipClass() {
+    i++;
+    while (i < source.length && source[i] !== "]") {
+      if (source[i] === "\\") i++;
+      i++;
+    }
+    i++;
+  }
+  function alternation() {
+    const branches = [];
+    let seq = { req: null, run: "", min: 0 };
+    while (i < source.length && source[i] !== ")") {
+      const ch = source[i];
+      if (ch === "|") {
+        flush(seq);
+        branches.push(seq);
+        seq = { req: null, run: "", min: 0 };
+        i++;
+        continue;
+      }
+      if (ch === "(") {
+        i++;
+        let lookaround = false;
+        if (source[i] === "?") {
+          const m = /^\?(?:[:=!]|<[=!]|<[A-Za-z_$][\w$]*>)/.exec(
+            source.slice(i)
+          );
+          if (!m) throw new Error(`unsupported group at ${i}`);
+          lookaround = m[0] !== "?:" && !m[0].endsWith(">");
+          i += m[0].length;
+        }
+        const inner = alternation();
+        if (source[i] !== ")") throw new Error("unbalanced group");
+        i++;
+        const min3 = quantifier();
+        flush(seq);
+        if (!lookaround) {
+          seq.min += inner.min * (min3 ?? 1);
+          if (min3 === null || min3 >= 1) seq.req = better(seq.req, inner.req);
+        }
+        continue;
+      }
+      if (ch === "[" || ch === ".") {
+        if (ch === "[") skipClass();
+        else i++;
+        seq.min += quantifier() ?? 1;
+        flush(seq);
+        continue;
+      }
+      if (ch === "^" || ch === "$") {
+        i++;
+        quantifier();
+        flush(seq);
+        continue;
+      }
+      let lit;
+      let width = 1;
+      if (ch === "\\") {
+        const e = source[i + 1];
+        i += 2;
+        if (/[A-Za-z0-9]/.test(e)) {
+          if (!/[dDwWsS]/.test(e)) width = 0;
+          if (e === "x") i += 2;
+          else if (e === "u") {
+            if (source[i] === "{") skipPast("}");
+            else i += 4;
+          } else if (e === "c") i += 1;
+          else if (e === "k" && source[i] === "<") skipPast(">");
+          else if ((e === "p" || e === "P") && source[i] === "{") skipPast("}");
+          lit = null;
+        } else {
+          lit = e;
+        }
+      } else {
+        lit = ch;
+        i++;
+      }
+      const min2 = quantifier();
+      seq.min += width * (min2 ?? 1);
+      if (lit === null || !isAsciiLiteral(lit)) {
+        flush(seq);
+      } else if (min2 === null) {
+        seq.run += lit;
+      } else {
+        if (min2 >= 1) seq.run += lit;
+        flush(seq);
+      }
+    }
+    flush(seq);
+    branches.push(seq);
+    let min = Number.POSITIVE_INFINITY;
+    let all = [];
+    for (const b of branches) {
+      min = Math.min(min, b.min);
+      if (!b.req) all = null;
+      else all?.push(...b.req);
+    }
+    return { req: all ? [...new Set(all)] : null, min };
+  }
+  try {
+    const { req, min } = alternation();
+    if (i !== source.length) return UNKNOWN;
+    const foldsBeyondAscii = flags.includes("i") && flags.includes("u");
+    const usable = req && !foldsBeyondAscii && !req.some((s) => s.length < MIN_LITERAL);
+    return { literals: usable ? req : null, minLength: min };
+  } catch {
+    return UNKNOWN;
+  }
+}
+function isAscii(text) {
+  for (let k = 0; k < text.length; k++) {
+    if (text.charCodeAt(k) >= 128) return false;
+  }
+  return true;
+}
+function buildPrefilter(literalsByRule) {
+  const alwaysRun = [];
+  const goto = [/* @__PURE__ */ new Map()];
+  const outs = [[]];
+  literalsByRule.forEach((given, rule) => {
+    const lits = given?.map((lit) => lit.toLowerCase());
+    if (!lits || lits.some((lit) => lit === "" || !isAscii(lit))) {
+      alwaysRun.push(rule);
+      return;
+    }
+    for (const lit of lits) {
+      let s = 0;
+      for (let k = 0; k < lit.length; k++) {
+        const c = lit.charCodeAt(k);
+        let n = goto[s].get(c);
+        if (n === void 0) {
+          n = goto.length;
+          goto.push(/* @__PURE__ */ new Map());
+          outs.push([]);
+          goto[s].set(c, n);
+        }
+        s = n;
+      }
+      outs[s].push(rule);
+    }
+  });
+  const nStates = goto.length;
+  const next = new Int32Array(nStates << 7);
+  const fail = new Int32Array(nStates);
+  const queue = [];
+  for (const [c, n] of goto[0]) {
+    next[c] = n;
+    queue.push(n);
+  }
+  for (let q = 0; q < queue.length; q++) {
+    const s = queue[q];
+    const f = fail[s];
+    if (outs[f].length) outs[s] = [.../* @__PURE__ */ new Set([...outs[s], ...outs[f]])];
+    const base = s << 7;
+    const fbase = f << 7;
+    for (let c = 0; c < 128; c++) next[base | c] = next[fbase | c];
+    for (const [c, n] of goto[s]) {
+      fail[n] = next[fbase | c];
+      next[base | c] = n;
+      queue.push(n);
+    }
+  }
+  for (let s = 0; s < nStates; s++) {
+    const base = s << 7;
+    for (let c = 65; c <= 90; c++) next[base | c] = next[base | c + 32];
+  }
+  const hasOut = new Uint8Array(nStates);
+  const outRules = outs.map((o, s) => {
+    if (o.length === 0) return null;
+    hasOut[s] = 1;
+    return o;
+  });
+  const ruleStamp = new Uint32Array(literalsByRule.length);
+  const stateStamp = new Uint32Array(nStates);
+  let stamp = 0;
+  return {
+    alwaysRun,
+    candidates(value, out) {
+      stamp++;
+      if (stamp === 4294967295) {
+        ruleStamp.fill(0);
+        stateStamp.fill(0);
+        stamp = 1;
+      }
+      let s = 0;
+      let found = false;
+      for (let k = 0, n = value.length; k < n; k++) {
+        const c = value.charCodeAt(k);
+        s = c < 128 ? next[s << 7 | c] : 0;
+        if (hasOut[s] === 1 && stateStamp[s] !== stamp) {
+          stateStamp[s] = stamp;
+          for (const r of outRules[s]) {
+            if (ruleStamp[r] !== stamp) {
+              ruleStamp[r] = stamp;
+              out.push(r);
+              found = true;
+            }
+          }
+        }
+      }
+      if (found) out.sort((a, b) => a - b);
+    }
+  };
+}
+
 // src/middleware/patterns.json
 var patterns_default = {
   patterns: [
@@ -8483,11 +8772,12 @@ function applyPattern(content, regex, groupCount, validate, prefixRun) {
   result += content.slice(lastIndex);
   return { value: result, count };
 }
-function redactString(value, patterns, owningKey = null) {
+function redactStringSlow(value, patterns, owningKey = null, from = 0) {
   let result = value;
   let totalCount = 0;
   let lower = null;
-  for (const p of patterns) {
+  for (let idx = from; idx < patterns.length; idx++) {
+    const p = patterns[idx];
     if (p.precheck) {
       if (lower === null) lower = result.toLowerCase();
       let hit = false;
@@ -8515,6 +8805,69 @@ function redactString(value, patterns, owningKey = null) {
   }
   return { value: result, count: totalCount };
 }
+var engines = /* @__PURE__ */ new WeakMap();
+var candidateScratch = [];
+function engineFor(patterns) {
+  let engine = engines.get(patterns);
+  if (!engine) {
+    const minLength = new Int32Array(patterns.length);
+    const literals = patterns.map((p, idx) => {
+      const facts = analyzeRegex(p.regex.source, p.regex.flags);
+      minLength[idx] = facts.minLength;
+      return p.precheck ?? facts.literals;
+    });
+    engine = { prefilter: buildPrefilter(literals), minLength };
+    engines.set(patterns, engine);
+  }
+  return engine;
+}
+function redactString(value, patterns, owningKey = null) {
+  const { prefilter, minLength } = engineFor(patterns);
+  const hits = candidateScratch;
+  hits.length = 0;
+  prefilter.candidates(value, hits);
+  const always = prefilter.alwaysRun;
+  let lower = null;
+  let h = 0;
+  let a = 0;
+  while (h < hits.length || a < always.length) {
+    const idx = a >= always.length || h < hits.length && hits[h] < always[a] ? hits[h++] : always[a++];
+    if (value.length < minLength[idx]) continue;
+    const p = patterns[idx];
+    if (p.precheck) {
+      if (lower === null) lower = value.toLowerCase();
+      let hit = false;
+      for (const needle of p.precheck) {
+        if (lower.includes(needle)) {
+          hit = true;
+          break;
+        }
+      }
+      if (!hit) continue;
+    }
+    if (p.context) {
+      const keyHit = owningKey !== null && p.context.test(owningKey);
+      if (!keyHit && (p.keyOnlyContext || !p.context.test(value))) continue;
+    }
+    const applied = applyPattern(
+      value,
+      p.regex,
+      p.groupCount,
+      p.validate,
+      p.prefixRun
+    );
+    if (applied.count > 0) {
+      const rest = redactStringSlow(
+        applied.value,
+        patterns,
+        owningKey,
+        idx + 1
+      );
+      return { value: rest.value, count: applied.count + rest.count };
+    }
+  }
+  return { value, count: 0 };
+}
 var keyMaskCaches = /* @__PURE__ */ new WeakMap();
 var memoStringUnits = /* @__PURE__ */ new WeakMap();
 var MAX_MEMO_STRING_UNITS = 8 * 1024 * 1024;
@@ -8579,42 +8932,63 @@ function walkAndRedactAll(value, patterns, memo, owningKey = null) {
   }
   return { value, count: 0 };
 }
-function processFiles(files2) {
+function redactJsonlLine(line, patterns, memo) {
+  if (!line.trim()) return { value: line, count: 0 };
+  try {
+    const parsed = JSON.parse(line);
+    const walked = walkAndRedactAll(parsed, patterns, memo);
+    return { value: JSON.stringify(walked.value), count: walked.count };
+  } catch {
+    return redactString(line, patterns);
+  }
+}
+function processFiles(files) {
   const patterns = compilePatterns();
-  const results2 = [];
-  for (const { index, content, isJsonl } of files2) {
+  const results = [];
+  for (const { index, content, isJsonl } of files) {
     let totalCount;
-    let output;
+    let output2;
     if (isJsonl) {
       const memo = /* @__PURE__ */ new Map();
       let count = 0;
-      const lines = content.split("\n");
-      const processed = lines.map((line) => {
-        if (!line.trim()) return line;
-        try {
-          const parsed = JSON.parse(line);
-          const walked = walkAndRedactAll(parsed, patterns, memo);
-          count += walked.count;
-          return JSON.stringify(walked.value);
-        } catch {
-          const r = redactString(line, patterns);
-          count += r.count;
-          return r.value;
-        }
+      const processed = content.split("\n").map((line) => {
+        const r = redactJsonlLine(line, patterns, memo);
+        count += r.count;
+        return r.count > 0 ? r.value : line;
       });
       totalCount = count;
-      output = processed.join("\n");
+      output2 = processed.join("\n");
     } else {
       const r = redactString(content, patterns);
       totalCount = r.count;
-      output = r.value;
+      output2 = r.value;
     }
-    results2.push({ index, content: output, count: totalCount });
+    results.push({ index, content: output2, count: totalCount });
   }
-  return results2;
+  return results;
+}
+function processRanges(ranges) {
+  const patterns = compilePatterns();
+  return ranges.map(({ index, bytes, start, end }) => {
+    const memo = /* @__PURE__ */ new Map();
+    const buffer = Buffer.from(
+      bytes.buffer,
+      bytes.byteOffset,
+      bytes.byteLength
+    );
+    const scanned = scanBufferLines(
+      buffer,
+      (line) => redactJsonlLine(line, patterns, memo),
+      { start, end }
+    );
+    return { index, ...scanned };
+  });
 }
 
 // src/middleware/pattern-worker.ts
-var files = workerData;
-var results = processFiles(files);
-parentPort?.postMessage(results);
+var job = workerData;
+var output = {
+  files: processFiles(job.files),
+  ranges: processRanges(job.ranges)
+};
+parentPort?.postMessage(output);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hillclimb",
-  "version": "0.9.3",
+  "version": "0.9.4",
   "description": "Extract and export AI coding tool logs grouped by repo",
   "license": "MIT",
   "type": "module",
@@ -44,6 +44,7 @@
     "@types/archiver": "^6.0.3",
     "@types/node": "^22.0.0",
     "@typescript/native-preview": "^7.0.0-dev.20260330.1",
+    "randexp": "^0.5.3",
     "tsup": "^8.5.1",
     "typescript": "^5.9.3",
     "yaml": "^2.8.3"