@link-foundation/sensitive-data-sanitizer 0.21.1 → 0.21.3

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## 0.21.3
+
+### Patch Changes
+
+- ee1ec36: Preserve harmless wrapped base64 binary within a bounded complete inspection, while retaining embedded credential detection and fail-closed handling of uninspected oversized payloads. Add regression coverage for complete JSON re-encoding after surrounding prose through sync, async and streaming sanitization.
+
+## 0.21.2
+
+### Patch Changes
+
+- b5bc2e1: Preserve prose after an existing credential mask on repeated plaintext and encoded sanitization passes, while continuing to detect later credentials and exact known secrets.
+
 ## 0.21.1
 
 ### Patch Changes

package/docs/API.md

@@ -365,7 +365,12 @@ status identifiers. Explicit `knownPersonal` values override these heuristics.
 
 Oversized encoded runs receive a conservative complete-run `ENCODED_LIMIT`
 redaction rather than aborting the record. Decoding probes are capped at 4,096
-characters; full decoding stays capped at 8,192 characters. A multi-line run is
+characters; ordinary text decoding stays capped at 8,192 characters. Uniform
+wrapped binary base64 up to 256 KiB of encoded input is decoded completely
+and its embedded text is inspected. Harmless binary bytes are preserved; a
+sensitive binary payload is masked as a whole because a text re-encoding
+cannot preserve its bytes. Larger uninspected wrapped binary runs still
+receive `ENCODED_LIMIT`. A multi-line run is
 treated as one wrapped payload only when its probe decodes as text or its lines
 share one base64 width (a multiple of four, standard alphabet, not plain hex);
 lists of identifiers or commit hashes are scanned line by line. Up to 4,096

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@link-foundation/sensitive-data-sanitizer",
-  "version": "0.21.1",
+  "version": "0.21.3",
   "description": "Offline sensitive-text sanitization with credential, multilingual personal-data and scanner union detection",
   "type": "module",
   "main": "src/index.js",

package/src/credential-spans.js

@@ -107,13 +107,25 @@ function narrowed(text, finding) {
   return parts ? parts.map((part) => ({ ...finding, ...part })) : [finding];
 }
 
+// A complete existing mask bounds context too: vendor rules no longer match
+// it on the next pass. Only strip sentence punctuation, never a glued suffix.
+function maskedPrefix(text, start, end) {
+  const token = /^[^\s]+/u.exec(text.slice(start, end))?.[0] ?? '';
+  let length = token.length;
+  while (length > 0 && /[.,;:!?]/.test(token[length - 1])) {
+    length--;
+  }
+  const value = token.slice(0, length);
+  return maskedValue(value) ? start + value.length : end;
+}
+
 // `token: ghp_… . Next sentence` — a context value that a vendor rule ends
 // earlier stops at that end when only punctuation or a space follows.
 function bounded(text, finding, findings) {
   if (!openEnded.has(finding.rule) || finding.category !== 'credential') {
     return finding;
   }
-  let end = finding.end;
+  let end = maskedPrefix(text, finding.start, finding.end);
   for (const other of findings) {
     if (
       other.category === 'credential' &&

package/src/credentials.js

@@ -56,7 +56,10 @@ export function templatedCredential(text, finding) {
   }
   // A template glued to literal characters (`<x>abc`, `[A]x9`) is not one.
   const after = text[finding.start + value.length] ?? '';
-  return !/[\w$%+/=.-]/.test(after) && templatedValue(value);
+  const sentenceEnd = /^[.,;:!?]+(?:\s|$)/.test(
+    text.slice(finding.start + value.length)
+  );
+  return (!/[\w$%+/=.-]/.test(after) || sentenceEnd) && templatedValue(value);
 }
 // Only variable references and the documented gh lookup are exempt. Arbitrary
 // shell commands and Actions expressions can contain literal credentials.

package/src/encoded.js

@@ -7,6 +7,9 @@ import { jsonStringRuns } from './json-string-runs.js';
 // remaining candidates as `encoded-limit` runs, without blocking the text.
 const WINDOW = 65536;
 const WINDOW_CANDIDATES = 4096;
+// Inspect a complete bounded binary payload, including any embedded text.
+// Ordinary text detection retains its 8 KiB cap.
+const WRAPPED_BINARY_LIMIT = 256 * 1024;
 
 const WRAPPED =
   /(?<![A-Za-z0-9_+/-])[A-Za-z0-9_+/-]{16,128}(?:\r?\n[A-Za-z0-9_+/-]{1,128})+={0,2}(?![A-Za-z0-9_+/-])/g;
@@ -56,7 +59,8 @@ function* matchRuns(match, wrapped, scan) {
   const start = match.index;
   const end = start + raw.length;
   const limited = { start, end, limited: true, depth: scan.depth };
-  if (raw.length > 8192) {
+  const candidates = runCandidates(raw, wrapped);
+  if (!candidates) {
     if (isLimitedTextRun(raw, wrapped)) {
       scan.covered.push(limited);
       yield limited;
@@ -64,7 +68,7 @@ function* matchRuns(match, wrapped, scan) {
     return;
   }
   const window = Math.floor(start / WINDOW);
-  for (const candidate of decodeCandidates(raw)) {
+  for (const candidate of candidates) {
     const { text: decoded, encoding } = candidate;
     const key = `${start}:${raw.length}:${decoded}`;
     if (scan.seen.has(key) || !decodedText(raw, decoded)) {
@@ -91,11 +95,48 @@ function* matchRuns(match, wrapped, scan) {
     };
     yield run;
     for (const inner of decodedRuns(decoded, scan.depth + 1)) {
-      yield { ...run, text: inner.text, depth: inner.depth };
+      yield inner.limited
+        ? { ...limited, depth: inner.depth }
+        : { ...run, text: inner.text, depth: inner.depth };
     }
   }
 }
 
+function runCandidates(raw, wrapped) {
+  if (raw.length <= 8192) {
+    return decodeCandidates(raw);
+  }
+  const binary = wrapped && boundedWrappedBinary(raw);
+  return binary ? [binary] : undefined;
+}
+
+function boundedWrappedBinary(raw) {
+  if (raw.length > WRAPPED_BINARY_LIMIT || !isWrappedBinary(raw)) {
+    return false;
+  }
+  const bytes = base64Bytes(raw.replace(/\s/g, ''));
+  if (!bytes) {
+    return false;
+  }
+  try {
+    const decoded = new TextDecoder('utf-8', { fatal: true }).decode(bytes);
+    if (decodedText(raw, decoded)) {
+      return undefined;
+    }
+  } catch {
+    /* Binary content still needs an embedded-text detection view. */
+  }
+  return {
+    encoding: 'binary',
+    // Preserve text offsets within the view while separating binary controls
+    // and invalid UTF-8. This view is never used to reconstruct binary bytes.
+    text: bytes
+      .toString('utf8')
+      // eslint-disable-next-line no-control-regex -- Replace binary controls in the text detection view.
+      .replace(/[\x00-\x08\x0e-\x1f\x7f\ufffd]/g, ' '),
+  };
+}
+
 function candidateMatches(text, pattern) {
   return pattern ? text.matchAll(pattern) : jsonStringRuns(text);
 }
@@ -211,11 +252,8 @@ function decodeCandidates(raw, partial) {
   } else {
     const compact = raw.replace(/\s/g, '');
     // Decode canonical base64 text only.
-    const bytes = Buffer.from(compact, 'base64url');
-    if (
-      bytes.toString('base64url') ===
-      compact.replace(/=+$/, '').replaceAll('+', '-').replaceAll('/', '_')
-    ) {
+    const bytes = base64Bytes(compact);
+    if (bytes) {
       try {
         candidates.push({
           encoding:
@@ -248,7 +286,18 @@ function decodeCandidates(raw, partial) {
   return candidates;
 }
 
+function base64Bytes(compact) {
+  const bytes = Buffer.from(compact, 'base64url');
+  return bytes.toString('base64url') ===
+    compact.replace(/=+$/, '').replaceAll('+', '-').replaceAll('/', '_')
+    ? bytes
+    : undefined;
+}
+
 export function encodeRun(run, text) {
+  if (run.encoding === 'binary') {
+    return undefined;
+  }
   const bytes = Buffer.from(text, 'utf8');
   let encoded;
   if (run.encoding === 'json-content') {