@socprime/logtotal-sanitizer 0.2.0-beta.3 → 0.2.1-beta.0

package/CHANGELOG.md

@@ -5,6 +5,37 @@ All notable changes to this project are documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.2.1-beta.0] - 2026-10-14
+
+### Fixed
+
+- JSON field redaction now applies to lines that are not a whole JSON document: pretty-printed
+  records, JSON embedded in a text line (`body="{\"password\":\"…\"}"`) and Python dict reprs.
+  Values of `"key": value` pairs whose key belongs to a rule are replaced by name, as in
+  single-line JSON. Previously, a pretty-printed record leaked `ssn`, `passport_number`, `iban`,
+  `cvv`, `diagnosis` and similar fields whose values fail the plain-text checksums.
+- Numeric values of sensitive keys (`"latitude": 48.1486`, `"pan": 4111111111111111`) are replaced
+  with a string token instead of being left as is.
+- Array items inherit the key of their array, so `"emails": [...]` and GeoJSON
+  `"coordinates": [...]` are redacted.
+- `hosts` rule: the reserved `.test`, `.invalid` and `.localhost` TLDs are recognized.
+
+### Changed
+
+- JSON key matching also ignores `.` and spaces, and tries a nested field together with its
+  parent's name (`user` + `name`, `client` + `ip`, `source` + `address`) before the field's own
+  name. Dotted flat keys (`"source.ip"`) resolve the same way.
+- In a pretty-printed record, a field named in `jsonKeys` is now replaced as a whole, the same
+  way the single-line form already was. For example, `"User": "CORP\\jdoe"` becomes one `USER`
+  token instead of a `HOST` and a `USER` token.
+- Key resolution is memoized per rule set, with a bounded cache.
+- Expanded `jsonKeys` and `jsonKeyContains` for every built-in rule with field names used by ECS,
+  AWS CloudTrail, Azure sign-in logs, Windows events, CEF, Datadog and common application logs:
+  cloud and API credentials, session and CSRF cookies, card, bank and wallet fields, national
+  identifiers (PESEL, rodné číslo, CPF, Aadhaar, РНОКПП and others), clinical fields, phone and
+  telephony fields, source, destination and forwarded IPs, workstation, SNI and DNS query hosts,
+  mail and actor identities, and street addresses.
+
 ## [0.2.0-beta.3] - 2026-09-23
 
 ### Fixed

package/README.md

@@ -98,7 +98,9 @@ Enabled in this order when you omit `rules`:
 
 `mask` mode (`<R:…>`) hides the kind of secret. `pseudo` mode keeps a type prefix so you can still read the timeline.
 
-JSON lines: if a line parses as a JSON object or array, fields listed in a rule's `jsonKeys` are redacted by name (case- and `-`/`_`-insensitive). Other string values still go through the regex pass.
+JSON lines: if a line parses as a JSON object or array, fields listed in a rule's `jsonKeys` (or whose name contains one of its `jsonKeyContains` fragments) are redacted by name. Matching ignores case, `-`, `_`, `.` and spaces. A nested field is also tried with its parent's name, so `{"user": {"name": …}}` and a flat `"user.name"` both resolve as `userName`, and `source.address` resolves as an IP rather than a street address. String and numeric values are replaced (a number becomes a string token); booleans, `null`, `""` and `"-"` are kept. Array items inherit the key of their array. Other string values still go through the regex pass.
+
+Lines that are not a whole JSON document (pretty-printed records, JSON embedded in a text line, JSON escaped inside another JSON string, Python dict reprs) get the same treatment for `"key": value`, `\"key\": \"value\"` and `'key': 'value'` pairs, without the parent-name lookup. A bare number replaced there is wrapped in quotes, so the line stays valid JSON. Set `json: false` to scan everything as plain text.
 
 ## Custom rules
 

package/{chunk-SGF4CYUV.js → chunk-5UDGYWP3.js}

@@ -1,4 +1,4 @@
-import { InvalidKeyError, InvalidOptionError, builtinRuleIds, SanitizationAbortedError, getBuiltinRule, UnknownRuleError, validateRule, countCapturingGroups, InvalidRuleError, deriveGateSource, lineHasAnyChar } from './chunk-VMPXBCSL.js';
+import { InvalidKeyError, InvalidOptionError, builtinRuleIds, SanitizationAbortedError, getBuiltinRule, UnknownRuleError, validateRule, countCapturingGroups, InvalidRuleError, deriveGateSource, lineHasAnyChar } from './chunk-X2QYRBY6.js';
 
 // src/core/constants.ts
 var DEFAULT_PREVIEW_BYTES = 256 * 1024;
@@ -459,6 +459,181 @@ function createLineSplitter(options = {}) {
   return { push, flush };
 }
 
+// src/core/jsonKeyMatcher.ts
+var RESOLVE_CACHE_LIMIT = 1e4;
+function normalizeKey2(key) {
+  return key.toLowerCase().replace(/[-_.\s]/g, "");
+}
+function shouldPreserveJsonValue(value) {
+  return value.length === 0 || value === "-";
+}
+function buildTables(rules) {
+  const exact = /* @__PURE__ */ new Map();
+  const contains = [];
+  for (const rule of rules) {
+    for (const key of rule.jsonKeys ?? []) {
+      const normalized = normalizeKey2(key);
+      if (!exact.has(normalized)) {
+        exact.set(normalized, rule);
+      }
+    }
+    for (const fragment of rule.jsonKeyContains ?? []) {
+      contains.push({ fragment: normalizeKey2(fragment), rule });
+    }
+  }
+  return { exact, contains };
+}
+function lastSegment(key) {
+  const dot = key.lastIndexOf(".");
+  return dot === -1 ? key : key.slice(dot + 1);
+}
+function resolveUncached(tables, key, parent) {
+  const dot = key.lastIndexOf(".");
+  const leaf = normalizeKey2(dot === -1 ? key : key.slice(dot + 1));
+  const parentRaw = dot === -1 ? parent : key.slice(0, dot);
+  if (leaf.length === 0) {
+    return null;
+  }
+  if (parentRaw !== void 0) {
+    const scoped = tables.exact.get(normalizeKey2(lastSegment(parentRaw)) + leaf);
+    if (scoped) {
+      return scoped;
+    }
+  }
+  const exact = tables.exact.get(leaf);
+  if (exact) {
+    return exact;
+  }
+  for (const entry of tables.contains) {
+    if (leaf.includes(entry.fragment)) {
+      return entry.rule;
+    }
+  }
+  return null;
+}
+function buildMatcher(rules) {
+  const tables = buildTables(rules);
+  const cache = /* @__PURE__ */ new Map();
+  return {
+    resolve(key, parent) {
+      const cacheKey = parent === void 0 ? key : `${parent}\0${key}`;
+      let rule = cache.get(cacheKey);
+      if (rule === void 0) {
+        rule = resolveUncached(tables, key, parent);
+        if (cache.size >= RESOLVE_CACHE_LIMIT) {
+          cache.clear();
+        }
+        cache.set(cacheKey, rule);
+      }
+      return rule ?? void 0;
+    }
+  };
+}
+var matcherCache = /* @__PURE__ */ new WeakMap();
+function jsonKeyMatcher(rules) {
+  const cached = matcherCache.get(rules);
+  if (cached) {
+    return cached;
+  }
+  const matcher = buildMatcher(rules);
+  matcherCache.set(rules, matcher);
+  return matcher;
+}
+
+// src/core/redactKeyValuePairs.ts
+var NUMBER = "(-?\\d+(?:\\.\\d+)?(?:[eE][+-]?\\d+)?)(?![\\w.:+-])";
+var DOUBLE_QUOTED = new RegExp(
+  `(?<![\\\\\\w])"((?:[^"\\\\\\r\\n]|\\\\.){1,128})"\\s*:\\s*(?:"((?:[^"\\\\]|\\\\.)*)"|${NUMBER})`,
+  "g"
+);
+var ESCAPED = new RegExp(
+  `\\\\"((?:[^"\\\\\\r\\n]|\\\\\\\\){1,128})\\\\"\\s*:\\s*(?:\\\\"((?:\\\\\\\\\\\\"|\\\\\\\\|\\\\[^"\\\\]|[^"\\\\])*)\\\\"|${NUMBER})`,
+  "g"
+);
+var SINGLE_QUOTED = new RegExp(
+  `(?<![\\\\\\w])'((?:[^'\\\\\\r\\n]|\\\\.){1,128})'\\s*:\\s*(?:'((?:[^'\\\\]|\\\\.)*)'|"((?:[^"\\\\]|\\\\.)*)"|${NUMBER})`,
+  "g"
+);
+function decodeJsonString(raw) {
+  if (!raw.includes("\\")) {
+    return raw;
+  }
+  try {
+    return JSON.parse(`"${raw}"`);
+  } catch {
+    return raw;
+  }
+}
+function decodeEscapedJsonString(raw) {
+  return decodeJsonString(decodeJsonString(raw));
+}
+function decodePythonString(raw) {
+  return raw.includes("\\") ? raw.replace(/\\(['"\\])/g, "$1") : raw;
+}
+function collect(line, regex, rules, decode, numberQuote, out) {
+  const matcher = jsonKeyMatcher(rules);
+  regex.lastIndex = 0;
+  for (let found = regex.exec(line); found !== null; found = regex.exec(line)) {
+    const rawKey = found[1];
+    let rawValue;
+    let quote = "";
+    for (let group = 2; group < found.length; group += 1) {
+      if (found[group] !== void 0) {
+        rawValue = found[group];
+        quote = group === found.length - 1 ? numberQuote : "";
+        break;
+      }
+    }
+    if (rawValue === void 0) {
+      continue;
+    }
+    const value = quote === "" ? decode(rawValue) : rawValue;
+    if (shouldPreserveJsonValue(value)) {
+      continue;
+    }
+    const rule = matcher.resolve(decode(rawKey));
+    if (!rule) {
+      continue;
+    }
+    const end = found.index + found[0].length - (quote === "" ? 1 : 0);
+    const valueEnd = quote === "" && regex === ESCAPED ? end - 1 : end;
+    out.push({
+      keyStart: found.index,
+      match: { start: valueEnd - rawValue.length, end: valueEnd, value, rule, quote }
+    });
+  }
+}
+function findKeyValueMatches(line, rules) {
+  const doubleQuoted = line.includes('":') || line.includes('" :');
+  const singleQuoted = line.includes("':") || line.includes("' :");
+  if (!doubleQuoted && !singleQuoted) {
+    return [];
+  }
+  const candidates = [];
+  if (doubleQuoted) {
+    collect(line, DOUBLE_QUOTED, rules, decodeJsonString, '"', candidates);
+    if (line.includes('\\"')) {
+      collect(line, ESCAPED, rules, decodeEscapedJsonString, '\\"', candidates);
+    }
+  }
+  if (singleQuoted) {
+    collect(line, SINGLE_QUOTED, rules, decodePythonString, "'", candidates);
+  }
+  if (candidates.length === 0) {
+    return [];
+  }
+  candidates.sort((left, right) => left.keyStart - right.keyStart);
+  const matches = [];
+  let floor = 0;
+  for (const { keyStart, match } of candidates) {
+    if (keyStart >= floor) {
+      matches.push(match);
+      floor = match.end;
+    }
+  }
+  return matches;
+}
+
 // src/core/redactLine.ts
 function sliceContext(text, index, length, chars) {
   return {
@@ -538,11 +713,21 @@ function pickNext(cursors) {
   }
   return best;
 }
+function blankOut(line, spans) {
+  let out = "";
+  let cursor = 0;
+  for (const span of spans) {
+    out += line.slice(cursor, span.start) + " ".repeat(span.end - span.start);
+    cursor = span.end;
+  }
+  return out + line.slice(cursor);
+}
 function redactLine(line, ctx, options = {}) {
   const { compiled, rulesById, allow, contextChars } = ctx;
   const withSegments = options.withSegments ?? false;
   const withMatches = options.withMatches ?? false;
-  if (compiled.length === 0) {
+  const keyed = ctx.json ? findKeyValueMatches(line, ctx.rules) : [];
+  if (compiled.length === 0 && keyed.length === 0) {
     return {
       output: line,
       counts: {},
@@ -550,18 +735,65 @@ function redactLine(line, ctx, options = {}) {
       segments: withSegments ? { before: [{ text: line, changed: false }], after: [{ text: line, changed: false }] } : void 0
     };
   }
-  const cursors = openCursors(line, compiled);
+  const scan = keyed.length === 0 ? line : blankOut(line, keyed);
+  const cursors = openCursors(scan, compiled);
   const counts = {};
   const matches = [];
   const before = [];
   const after = [];
   let output = "";
   let cursor = 0;
+  let keyedIndex = 0;
   for (; ; ) {
     const next = pickNext(cursors);
+    const pending = keyed[keyedIndex];
+    if (pending !== void 0 && (next === null || pending.start <= next.start)) {
+      keyedIndex += 1;
+      if (!isAllowed(allow, pending.rule.id, pending.value)) {
+        const token = buildReplacement(ctx, pending.rule.id, pending.value);
+        const raw = line.slice(pending.start, pending.end);
+        if (withSegments) {
+          if (pending.start > cursor) {
+            const gap = line.slice(cursor, pending.start);
+            before.push({ text: gap, changed: false });
+            after.push({ text: gap, changed: false });
+          }
+          before.push({ text: raw, changed: true });
+          if (pending.quote) {
+            after.push({ text: pending.quote, changed: false });
+          }
+          after.push({ text: token, changed: true });
+          if (pending.quote) {
+            after.push({ text: pending.quote, changed: false });
+          }
+        }
+        output += line.slice(cursor, pending.start) + pending.quote + token + pending.quote;
+        cursor = pending.end;
+        counts[pending.rule.id] = (counts[pending.rule.id] ?? 0) + 1;
+        if (withMatches) {
+          matches.push({
+            ruleId: pending.rule.id,
+            original: pending.value,
+            replacement: token,
+            index: pending.start,
+            ...contextChars > 0 ? sliceContext(line, pending.start, raw.length, contextChars) : {}
+          });
+        }
+      }
+      for (const candidate of cursors) {
+        if (candidate.active && candidate.start < pending.end) {
+          advance(candidate, scan, pending.end);
+        }
+      }
+      continue;
+    }
     if (next === null) {
       break;
     }
+    if (pending !== void 0 && next.start + next.original.length > pending.start) {
+      advance(next, scan, next.start + 1);
+      continue;
+    }
     const { start, original, ruleId } = next;
     const spanEnd = start + Math.max(original.length, 1);
     const rule = rulesById.get(ruleId);
@@ -592,7 +824,7 @@ function redactLine(line, ctx, options = {}) {
     }
     for (const candidate of cursors) {
       if (candidate.active && candidate.start < spanEnd) {
-        advance(candidate, line, spanEnd);
+        advance(candidate, scan, spanEnd);
       }
     }
   }
@@ -616,51 +848,6 @@ function looksLikeJson(line) {
   const trimmed = line.trim();
   return trimmed.length > 0 && (trimmed.startsWith("{") || trimmed.startsWith("["));
 }
-function normalizeKey2(key) {
-  return key.toLowerCase().replace(/[-_]/g, "");
-}
-function shouldPreserveJsonValue(value) {
-  return value.length === 0 || value === "-";
-}
-function buildJsonKeyMatcher(rules) {
-  const exact = /* @__PURE__ */ new Map();
-  const contains = [];
-  for (const rule of rules) {
-    for (const key of rule.jsonKeys ?? []) {
-      const normalized = normalizeKey2(key);
-      if (!exact.has(normalized)) {
-        exact.set(normalized, rule);
-      }
-    }
-    for (const fragment of rule.jsonKeyContains ?? []) {
-      contains.push({ fragment: normalizeKey2(fragment), rule });
-    }
-  }
-  return { exact, contains };
-}
-function resolveJsonKeyRule(matcher, key) {
-  const normalized = normalizeKey2(key);
-  const exact = matcher.exact.get(normalized);
-  if (exact) {
-    return exact;
-  }
-  for (const entry of matcher.contains) {
-    if (normalized.includes(entry.fragment)) {
-      return entry.rule;
-    }
-  }
-  return void 0;
-}
-var jsonKeyMatcherCache = /* @__PURE__ */ new WeakMap();
-function jsonKeyMatcher(rules) {
-  const cached = jsonKeyMatcherCache.get(rules);
-  if (cached) {
-    return cached;
-  }
-  const matcher = buildJsonKeyMatcher(rules);
-  jsonKeyMatcherCache.set(rules, matcher);
-  return matcher;
-}
 function mergeCounts(target, source) {
   for (const [id, count] of Object.entries(source)) {
     target[id] = (target[id] ?? 0) + count;
@@ -687,6 +874,15 @@ function highlightReplacements(text) {
 function jsonStringContent(value) {
   return JSON.stringify(value).slice(1, -1);
 }
+function locateJsonNumber(text, value, from = 0) {
+  const re = new RegExp(`:\\s*(${escapeRegExp(value)})(?![\\d.eE])`, "g");
+  re.lastIndex = from;
+  const match = re.exec(text);
+  if (match === null) {
+    return null;
+  }
+  return { index: match.index + match[0].length - value.length, length: value.length };
+}
 function locateQuotedJsonValue(text, value, from = 0) {
   const encoded = jsonStringContent(value);
   const quotedAt = text.indexOf(`"${encoded}"`, from);
@@ -696,6 +892,9 @@ function locateQuotedJsonValue(text, value, from = 0) {
   return { index: quotedAt + 1, length: encoded.length };
 }
 function findOriginal(text, original, from) {
+  if (original.number === true) {
+    return locateJsonNumber(text, original.value, from);
+  }
   const encoded = jsonStringContent(original.value);
   if (original.wholeValue) {
     const quoted = `"${encoded}"`;
@@ -778,32 +977,39 @@ function redactJsonLine(line, ctx, options = {}) {
     }
     return { contextBefore: "", contextAfter: "" };
   };
-  const redactValue = (value, key) => {
+  const redactField = (rule, value, number) => {
+    if (isAllowed(ctx.allow, rule.id, value)) {
+      return null;
+    }
+    const replacement = buildReplacement(ctx, rule.id, value);
+    counts[rule.id] = (counts[rule.id] ?? 0) + 1;
+    if (withSegments) {
+      originals.push({ value, wholeValue: true, number });
+    }
+    if (withMatches) {
+      const located = number ? locateJsonNumber(content, value) : locateQuotedJsonValue(content, value);
+      matches.push({
+        ruleId: rule.id,
+        original: value,
+        replacement,
+        index: located?.index ?? 0,
+        ...located && ctx.contextChars > 0 ? sliceContext(content, located.index, located.length, ctx.contextChars) : ctx.contextChars > 0 ? { contextBefore: "", contextAfter: "" } : {}
+      });
+    }
+    return replacement;
+  };
+  const redactValue = (value, key, parent) => {
+    if (typeof value === "number") {
+      const fieldRule = key === void 0 || !Number.isFinite(value) ? void 0 : keyMatcher.resolve(key, parent);
+      return fieldRule ? redactField(fieldRule, String(value), true) ?? value : value;
+    }
     if (typeof value === "string") {
       if (key !== void 0 && shouldPreserveJsonValue(value)) {
         return value;
       }
-      const fieldRule = key === void 0 ? void 0 : resolveJsonKeyRule(keyMatcher, key);
+      const fieldRule = key === void 0 ? void 0 : keyMatcher.resolve(key, parent);
       if (fieldRule) {
-        if (isAllowed(ctx.allow, fieldRule.id, value)) {
-          return value;
-        }
-        const replacement = buildReplacement(ctx, fieldRule.id, value);
-        counts[fieldRule.id] = (counts[fieldRule.id] ?? 0) + 1;
-        if (withSegments) {
-          originals.push({ value, wholeValue: true });
-        }
-        if (withMatches) {
-          const located = locateQuotedJsonValue(content, value);
-          matches.push({
-            ruleId: fieldRule.id,
-            original: value,
-            replacement,
-            index: located?.index ?? 0,
-            ...located && ctx.contextChars > 0 ? sliceContext(content, located.index, located.length, ctx.contextChars) : ctx.contextChars > 0 ? { contextBefore: "", contextAfter: "" } : {}
-          });
-        }
-        return replacement;
+        return redactField(fieldRule, value, false) ?? value;
       }
       const result = redactLine(value, ctx, { withMatches: collectOriginals });
       mergeCounts(counts, result.counts);
@@ -820,12 +1026,12 @@ function redactJsonLine(line, ctx, options = {}) {
       return result.output;
     }
     if (Array.isArray(value)) {
-      return value.map((item) => redactValue(item));
+      return value.map((item) => redactValue(item, key, parent));
     }
     if (value !== null && typeof value === "object") {
       const out = {};
       for (const [entryKey, entryValue] of Object.entries(value)) {
-        out[entryKey] = redactValue(entryValue, entryKey);
+        out[entryKey] = redactValue(entryValue, entryKey, key);
       }
       return out;
     }
@@ -1304,5 +1510,5 @@ function toNullSink() {
 }
 
 export { createSanitizer, fromBlob, fromString, fromWebStream, generateKey, mergeReports, planLineAlignedRanges, sanitizeStream, sanitizeText, toCallbackSink, toNullSink, toStringSink };
-//# sourceMappingURL=chunk-SGF4CYUV.js.map
-//# sourceMappingURL=chunk-SGF4CYUV.js.map
+//# sourceMappingURL=chunk-5UDGYWP3.js.map
+//# sourceMappingURL=chunk-5UDGYWP3.js.map