hillclimb 0.9.2 → 0.9.4

package/dist/pattern-worker.js

@@ -1,6 +1,295 @@
 // src/middleware/pattern-worker.ts
 import { parentPort, workerData } from "worker_threads";
 
+// src/middleware/large-file.ts
+import { constants as bufferConstants } from "buffer";
+var MAX_STRINGIFIABLE_BYTES = bufferConstants.MAX_STRING_LENGTH;
+var NEWLINE = 10;
+var OVERSIZED_LINE_PLACEHOLDER = '{"_hillclimb_omitted":"line exceeded max string length; dropped to allow redaction"}';
+function scanBufferLines(buffer, redactLine, options = {}) {
+  const maxLineBytes = options.maxLineBytes ?? MAX_STRINGIFIABLE_BYTES;
+  const oversizedPlaceholder = options.oversizedPlaceholder ?? OVERSIZED_LINE_PLACEHOLDER;
+  const end = options.end ?? buffer.length;
+  const ownsFinalLine = end === buffer.length;
+  const edits = [];
+  let totalCount = 0;
+  let oversizedLines = 0;
+  let start = options.start ?? 0;
+  while (ownsFinalLine || start < end) {
+    const nl = buffer.indexOf(NEWLINE, start);
+    const lineEnd = nl === -1 ? buffer.length : nl;
+    if (lineEnd - start > maxLineBytes) {
+      oversizedLines++;
+      edits.push({ start, end: lineEnd, value: oversizedPlaceholder });
+    } else {
+      const { value, count } = redactLine(
+        buffer.toString("utf-8", start, lineEnd)
+      );
+      if (count > 0) {
+        totalCount += count;
+        edits.push({ start, end: lineEnd, value });
+      }
+    }
+    if (nl === -1) break;
+    start = nl + 1;
+  }
+  return { edits, count: totalCount, oversizedLines };
+}
+
+// src/middleware/pattern-prefilter.ts
+var MIN_LITERAL = 3;
+function score(req) {
+  if (!req) return -1;
+  let min = Number.POSITIVE_INFINITY;
+  for (const s of req) min = Math.min(min, s.length);
+  return min * 1e3 - req.length;
+}
+function better(a, b) {
+  return score(b) > score(a) ? b : a;
+}
+function isAsciiLiteral(ch) {
+  const c = ch.charCodeAt(0);
+  return c >= 32 && c < 127;
+}
+var UNKNOWN = { literals: null, minLength: 0 };
+function analyzeRegex(source, flags = "") {
+  if (flags.includes("v")) return UNKNOWN;
+  let i = 0;
+  function skipPast(close) {
+    const at = source.indexOf(close, i);
+    if (at === -1) throw new Error("unterminated escape");
+    i = at + 1;
+  }
+  function flush(seq) {
+    if (seq.run.length > 0) {
+      seq.req = better(seq.req, [seq.run.toLowerCase()]);
+      seq.run = "";
+    }
+  }
+  function quantifier() {
+    const ch = source[i];
+    let min = null;
+    if (ch === "*" || ch === "?") {
+      min = 0;
+      i++;
+    } else if (ch === "+") {
+      min = 1;
+      i++;
+    } else if (ch === "{") {
+      const m = /^\{(\d+)(?:,\d*)?\}/.exec(source.slice(i));
+      if (m) {
+        min = Number(m[1]);
+        i += m[0].length;
+      }
+    }
+    if (min !== null && source[i] === "?") i++;
+    return min;
+  }
+  function skipClass() {
+    i++;
+    while (i < source.length && source[i] !== "]") {
+      if (source[i] === "\\") i++;
+      i++;
+    }
+    i++;
+  }
+  function alternation() {
+    const branches = [];
+    let seq = { req: null, run: "", min: 0 };
+    while (i < source.length && source[i] !== ")") {
+      const ch = source[i];
+      if (ch === "|") {
+        flush(seq);
+        branches.push(seq);
+        seq = { req: null, run: "", min: 0 };
+        i++;
+        continue;
+      }
+      if (ch === "(") {
+        i++;
+        let lookaround = false;
+        if (source[i] === "?") {
+          const m = /^\?(?:[:=!]|<[=!]|<[A-Za-z_$][\w$]*>)/.exec(
+            source.slice(i)
+          );
+          if (!m) throw new Error(`unsupported group at ${i}`);
+          lookaround = m[0] !== "?:" && !m[0].endsWith(">");
+          i += m[0].length;
+        }
+        const inner = alternation();
+        if (source[i] !== ")") throw new Error("unbalanced group");
+        i++;
+        const min3 = quantifier();
+        flush(seq);
+        if (!lookaround) {
+          seq.min += inner.min * (min3 ?? 1);
+          if (min3 === null || min3 >= 1) seq.req = better(seq.req, inner.req);
+        }
+        continue;
+      }
+      if (ch === "[" || ch === ".") {
+        if (ch === "[") skipClass();
+        else i++;
+        seq.min += quantifier() ?? 1;
+        flush(seq);
+        continue;
+      }
+      if (ch === "^" || ch === "$") {
+        i++;
+        quantifier();
+        flush(seq);
+        continue;
+      }
+      let lit;
+      let width = 1;
+      if (ch === "\\") {
+        const e = source[i + 1];
+        i += 2;
+        if (/[A-Za-z0-9]/.test(e)) {
+          if (!/[dDwWsS]/.test(e)) width = 0;
+          if (e === "x") i += 2;
+          else if (e === "u") {
+            if (source[i] === "{") skipPast("}");
+            else i += 4;
+          } else if (e === "c") i += 1;
+          else if (e === "k" && source[i] === "<") skipPast(">");
+          else if ((e === "p" || e === "P") && source[i] === "{") skipPast("}");
+          lit = null;
+        } else {
+          lit = e;
+        }
+      } else {
+        lit = ch;
+        i++;
+      }
+      const min2 = quantifier();
+      seq.min += width * (min2 ?? 1);
+      if (lit === null || !isAsciiLiteral(lit)) {
+        flush(seq);
+      } else if (min2 === null) {
+        seq.run += lit;
+      } else {
+        if (min2 >= 1) seq.run += lit;
+        flush(seq);
+      }
+    }
+    flush(seq);
+    branches.push(seq);
+    let min = Number.POSITIVE_INFINITY;
+    let all = [];
+    for (const b of branches) {
+      min = Math.min(min, b.min);
+      if (!b.req) all = null;
+      else all?.push(...b.req);
+    }
+    return { req: all ? [...new Set(all)] : null, min };
+  }
+  try {
+    const { req, min } = alternation();
+    if (i !== source.length) return UNKNOWN;
+    const foldsBeyondAscii = flags.includes("i") && flags.includes("u");
+    const usable = req && !foldsBeyondAscii && !req.some((s) => s.length < MIN_LITERAL);
+    return { literals: usable ? req : null, minLength: min };
+  } catch {
+    return UNKNOWN;
+  }
+}
+function isAscii(text) {
+  for (let k = 0; k < text.length; k++) {
+    if (text.charCodeAt(k) >= 128) return false;
+  }
+  return true;
+}
+function buildPrefilter(literalsByRule) {
+  const alwaysRun = [];
+  const goto = [/* @__PURE__ */ new Map()];
+  const outs = [[]];
+  literalsByRule.forEach((given, rule) => {
+    const lits = given?.map((lit) => lit.toLowerCase());
+    if (!lits || lits.some((lit) => lit === "" || !isAscii(lit))) {
+      alwaysRun.push(rule);
+      return;
+    }
+    for (const lit of lits) {
+      let s = 0;
+      for (let k = 0; k < lit.length; k++) {
+        const c = lit.charCodeAt(k);
+        let n = goto[s].get(c);
+        if (n === void 0) {
+          n = goto.length;
+          goto.push(/* @__PURE__ */ new Map());
+          outs.push([]);
+          goto[s].set(c, n);
+        }
+        s = n;
+      }
+      outs[s].push(rule);
+    }
+  });
+  const nStates = goto.length;
+  const next = new Int32Array(nStates << 7);
+  const fail = new Int32Array(nStates);
+  const queue = [];
+  for (const [c, n] of goto[0]) {
+    next[c] = n;
+    queue.push(n);
+  }
+  for (let q = 0; q < queue.length; q++) {
+    const s = queue[q];
+    const f = fail[s];
+    if (outs[f].length) outs[s] = [.../* @__PURE__ */ new Set([...outs[s], ...outs[f]])];
+    const base = s << 7;
+    const fbase = f << 7;
+    for (let c = 0; c < 128; c++) next[base | c] = next[fbase | c];
+    for (const [c, n] of goto[s]) {
+      fail[n] = next[fbase | c];
+      next[base | c] = n;
+      queue.push(n);
+    }
+  }
+  for (let s = 0; s < nStates; s++) {
+    const base = s << 7;
+    for (let c = 65; c <= 90; c++) next[base | c] = next[base | c + 32];
+  }
+  const hasOut = new Uint8Array(nStates);
+  const outRules = outs.map((o, s) => {
+    if (o.length === 0) return null;
+    hasOut[s] = 1;
+    return o;
+  });
+  const ruleStamp = new Uint32Array(literalsByRule.length);
+  const stateStamp = new Uint32Array(nStates);
+  let stamp = 0;
+  return {
+    alwaysRun,
+    candidates(value, out) {
+      stamp++;
+      if (stamp === 4294967295) {
+        ruleStamp.fill(0);
+        stateStamp.fill(0);
+        stamp = 1;
+      }
+      let s = 0;
+      let found = false;
+      for (let k = 0, n = value.length; k < n; k++) {
+        const c = value.charCodeAt(k);
+        s = c < 128 ? next[s << 7 | c] : 0;
+        if (hasOut[s] === 1 && stateStamp[s] !== stamp) {
+          stateStamp[s] = stamp;
+          for (const r of outRules[s]) {
+            if (ruleStamp[r] !== stamp) {
+              ruleStamp[r] = stamp;
+              out.push(r);
+              found = true;
+            }
+          }
+        }
+      }
+      if (found) out.sort((a, b) => a - b);
+    }
+  };
+}
+
 // src/middleware/patterns.json
 var patterns_default = {
   patterns: [
@@ -8483,11 +8772,12 @@ function applyPattern(content, regex, groupCount, validate, prefixRun) {
   result += content.slice(lastIndex);
   return { value: result, count };
 }
-function redactString(value, patterns, owningKey = null) {
+function redactStringSlow(value, patterns, owningKey = null, from = 0) {
   let result = value;
   let totalCount = 0;
   let lower = null;
-  for (const p of patterns) {
+  for (let idx = from; idx < patterns.length; idx++) {
+    const p = patterns[idx];
     if (p.precheck) {
       if (lower === null) lower = result.toLowerCase();
       let hit = false;
@@ -8515,6 +8805,69 @@ function redactString(value, patterns, owningKey = null) {
   }
   return { value: result, count: totalCount };
 }
+var engines = /* @__PURE__ */ new WeakMap();
+var candidateScratch = [];
+function engineFor(patterns) {
+  let engine = engines.get(patterns);
+  if (!engine) {
+    const minLength = new Int32Array(patterns.length);
+    const literals = patterns.map((p, idx) => {
+      const facts = analyzeRegex(p.regex.source, p.regex.flags);
+      minLength[idx] = facts.minLength;
+      return p.precheck ?? facts.literals;
+    });
+    engine = { prefilter: buildPrefilter(literals), minLength };
+    engines.set(patterns, engine);
+  }
+  return engine;
+}
+function redactString(value, patterns, owningKey = null) {
+  const { prefilter, minLength } = engineFor(patterns);
+  const hits = candidateScratch;
+  hits.length = 0;
+  prefilter.candidates(value, hits);
+  const always = prefilter.alwaysRun;
+  let lower = null;
+  let h = 0;
+  let a = 0;
+  while (h < hits.length || a < always.length) {
+    const idx = a >= always.length || h < hits.length && hits[h] < always[a] ? hits[h++] : always[a++];
+    if (value.length < minLength[idx]) continue;
+    const p = patterns[idx];
+    if (p.precheck) {
+      if (lower === null) lower = value.toLowerCase();
+      let hit = false;
+      for (const needle of p.precheck) {
+        if (lower.includes(needle)) {
+          hit = true;
+          break;
+        }
+      }
+      if (!hit) continue;
+    }
+    if (p.context) {
+      const keyHit = owningKey !== null && p.context.test(owningKey);
+      if (!keyHit && (p.keyOnlyContext || !p.context.test(value))) continue;
+    }
+    const applied = applyPattern(
+      value,
+      p.regex,
+      p.groupCount,
+      p.validate,
+      p.prefixRun
+    );
+    if (applied.count > 0) {
+      const rest = redactStringSlow(
+        applied.value,
+        patterns,
+        owningKey,
+        idx + 1
+      );
+      return { value: rest.value, count: applied.count + rest.count };
+    }
+  }
+  return { value, count: 0 };
+}
 var keyMaskCaches = /* @__PURE__ */ new WeakMap();
 var memoStringUnits = /* @__PURE__ */ new WeakMap();
 var MAX_MEMO_STRING_UNITS = 8 * 1024 * 1024;
@@ -8579,42 +8932,63 @@ function walkAndRedactAll(value, patterns, memo, owningKey = null) {
   }
   return { value, count: 0 };
 }
-function processFiles(files2) {
+function redactJsonlLine(line, patterns, memo) {
+  if (!line.trim()) return { value: line, count: 0 };
+  try {
+    const parsed = JSON.parse(line);
+    const walked = walkAndRedactAll(parsed, patterns, memo);
+    return { value: JSON.stringify(walked.value), count: walked.count };
+  } catch {
+    return redactString(line, patterns);
+  }
+}
+function processFiles(files) {
   const patterns = compilePatterns();
-  const results2 = [];
-  for (const { index, content, isJsonl } of files2) {
+  const results = [];
+  for (const { index, content, isJsonl } of files) {
     let totalCount;
-    let output;
+    let output2;
     if (isJsonl) {
       const memo = /* @__PURE__ */ new Map();
       let count = 0;
-      const lines = content.split("\n");
-      const processed = lines.map((line) => {
-        if (!line.trim()) return line;
-        try {
-          const parsed = JSON.parse(line);
-          const walked = walkAndRedactAll(parsed, patterns, memo);
-          count += walked.count;
-          return JSON.stringify(walked.value);
-        } catch {
-          const r = redactString(line, patterns);
-          count += r.count;
-          return r.value;
-        }
+      const processed = content.split("\n").map((line) => {
+        const r = redactJsonlLine(line, patterns, memo);
+        count += r.count;
+        return r.count > 0 ? r.value : line;
       });
       totalCount = count;
-      output = processed.join("\n");
+      output2 = processed.join("\n");
     } else {
       const r = redactString(content, patterns);
       totalCount = r.count;
-      output = r.value;
+      output2 = r.value;
     }
-    results2.push({ index, content: output, count: totalCount });
+    results.push({ index, content: output2, count: totalCount });
   }
-  return results2;
+  return results;
+}
+function processRanges(ranges) {
+  const patterns = compilePatterns();
+  return ranges.map(({ index, bytes, start, end }) => {
+    const memo = /* @__PURE__ */ new Map();
+    const buffer = Buffer.from(
+      bytes.buffer,
+      bytes.byteOffset,
+      bytes.byteLength
+    );
+    const scanned = scanBufferLines(
+      buffer,
+      (line) => redactJsonlLine(line, patterns, memo),
+      { start, end }
+    );
+    return { index, ...scanned };
+  });
 }
 
 // src/middleware/pattern-worker.ts
-var files = workerData;
-var results = processFiles(files);
-parentPort?.postMessage(results);
+var job = workerData;
+var output = {
+  files: processFiles(job.files),
+  ranges: processRanges(job.ranges)
+};
+parentPort?.postMessage(output);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hillclimb",
-  "version": "0.9.2",
+  "version": "0.9.4",
   "description": "Extract and export AI coding tool logs grouped by repo",
   "license": "MIT",
   "type": "module",
@@ -44,6 +44,7 @@
     "@types/archiver": "^6.0.3",
     "@types/node": "^22.0.0",
     "@typescript/native-preview": "^7.0.0-dev.20260330.1",
+    "randexp": "^0.5.3",
     "tsup": "^8.5.1",
     "typescript": "^5.9.3",
     "yaml": "^2.8.3"