@crewhaus/tool-secure 0.7.0 → 0.8.0

package/dist/lib/pii.js

@@ -28,7 +28,8 @@
  * | `dob.labelled` / `date.calendar` | a calendar date. Only a nearby "date of birth"-style label, or a caller-supplied `referenceDate` that makes the age plausible, raises it above `possible`. |
  * | `address.us-street-suffix` / `address.us-city-state-zip` | a US-style street line or city/state/ZIP tail. US conventions only. |
  */
-import { matchAll, withPositions } from "./text";
+import { ipv6Candidates } from "./ipv6-scan";
+import { matchAll, matchAllSkipping, withPositions } from "./text";
 export const PII_TYPES = [
     "credit_card",
     "date_of_birth",
@@ -306,12 +307,74 @@ function ipv4Scope(octets) {
 // ---------------------------------------------------------------------------
 // patterns
 // ---------------------------------------------------------------------------
-const EMAIL = /[A-Za-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[A-Za-z0-9!#$%&'*+/=?^_`{|}~-]+)*@[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?(?:\.[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?)+/g;
+export const EMAIL = /[A-Za-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[A-Za-z0-9!#$%&'*+/=?^_`{|}~-]+)*@[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?(?:\.[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?)+/g;
+/** A character an email's local part may hold, besides the dots between its runs. */
+function isLocalPartUnit(c) {
+    return ((c >= 0x30 && c <= 0x39) ||
+        (c >= 0x41 && c <= 0x5a) ||
+        (c >= 0x61 && c <= 0x7a) ||
+        "!#$%&'*+/=?^_`{|}~-".includes(String.fromCharCode(c)));
+}
+/**
+ * Where a failed EMAIL try at `at` stops binding: the end of the dotted run
+ * of local-part characters that starts there. A later start inside it reads
+ * the same run to the same end and the same text after it, so it fails too.
+ */
+function localPartEnd(text, at) {
+    let i = at;
+    for (;;) {
+        while (i < text.length && isLocalPartUnit(text.charCodeAt(i)))
+            i++;
+        if (text.charCodeAt(i) !== 0x2e || !isLocalPartUnit(text.charCodeAt(i + 1)))
+            return i;
+        i++;
+    }
+}
+/** Where ADDRESS_CITY_STATE_ZIP may begin: an ASCII capital at a word boundary. */
+function startsCapitalWord(text, at) {
+    const c = text.charCodeAt(at);
+    if (c < 0x41 || c > 0x5a)
+        return false;
+    if (at === 0)
+        return true;
+    const p = text.charCodeAt(at - 1);
+    const word = (p >= 0x30 && p <= 0x39) || (p >= 0x41 && p <= 0x5a) || (p >= 0x61 && p <= 0x7a);
+    return !word && p !== 0x5f;
+}
+/** The end of the `[A-Za-z.'-]` run at `at`: a failed address try binds to it. */
+function addressWordEnd(text, at) {
+    let i = at;
+    for (; i < text.length; i++) {
+        const c = text.charCodeAt(i);
+        const ok = (c >= 0x41 && c <= 0x5a) ||
+            (c >= 0x61 && c <= 0x7a) ||
+            c === 0x2e ||
+            c === 0x27 ||
+            c === 0x2d;
+        if (!ok)
+            break;
+    }
+    return i;
+}
+/**
+ * EMAIL's matches. It opens with an unbounded local-part run, so it is tried
+ * only where one starts, and a failed try skips the rest of that run (see
+ * matchAllSkipping): 100K of base64 cost seconds as a global search.
+ */
+export function emailMatches(text) {
+    return matchAllSkipping(text, EMAIL, (t, at) => isLocalPartUnit(t.charCodeAt(at)), localPartEnd);
+}
+/**
+ * ADDRESS_CITY_STATE_ZIP's matches, tried where a capitalised word starts; a
+ * failed try skips the rest of its `[A-Za-z.'-]` run.
+ */
+export function cityStateZipMatches(text) {
+    return matchAllSkipping(text, ADDRESS_CITY_STATE_ZIP, startsCapitalWord, addressWordEnd);
+}
 const SSN = /\b(\d{3})-(\d{2})-(\d{4})\b/g;
 const IBAN_CANDIDATE = /\b[A-Z]{2}\d{2}[ ]?(?:[A-Z0-9]{4}[ ]?){2,7}[A-Z0-9]{1,4}\b/g;
 const CARD_RUN = /\b\d(?:[ -]?\d){11,}/g;
 const IPV4 = /\b\d{1,3}(?:\.\d{1,3}){3}\b/g;
-const IPV6_CANDIDATE = /\b(?=[0-9A-Fa-f:]*::|(?:[0-9A-Fa-f]{1,4}:){7})[0-9A-Fa-f:]{2,39}(?:\.\d{1,3}){0,3}/g;
 const PHONE_E164_RUN = /\+\d(?:[\s().-]?\d)+/g;
 const DATE_ISO = /\b(19|20)\d{2}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01])\b/g;
 const DATE_US = /\b(0?[1-9]|1[0-2])\/(0?[1-9]|[12]\d|3[01])\/((?:19|20)\d{2})\b/g;
@@ -320,7 +383,7 @@ const DATE_LONG = /\b(January|February|March|April|May|June|July|August|Septembe
 const DOB_LABEL = /(date of birth|d\.?o\.?b\.?|birth ?date|birthday|born(?:\s+on)?)\s*[:\-]?\s*$/i;
 const STREET_SUFFIX = "(?:Street|St|Avenue|Ave|Road|Rd|Boulevard|Blvd|Lane|Ln|Drive|Dr|Court|Ct|Way|Terrace|Ter|Place|Pl|Circle|Cir|Parkway|Pkwy|Highway|Hwy|Square|Sq|Trail|Trl)";
 const ADDRESS_STREET = new RegExp(`\\b\\d{1,6}\\s+(?:[A-Za-z0-9.'#-]+\\s+){0,4}${STREET_SUFFIX}\\b\\.?(?:\\s+(?:Apt|Suite|Ste|Unit|#)\\s*[A-Za-z0-9-]+)?`, "g");
-const ADDRESS_CITY_STATE_ZIP = /\b[A-Z][A-Za-z.'-]+(?:\s+[A-Z][A-Za-z.'-]+){0,3},\s*(A[KLRZ]|C[AOT]|D[CE]|FL|GA|HI|I[ADLN]|K[SY]|LA|M[ADEINOST]|N[CDEHJMVY]|O[HKR]|P[AR]|RI|S[CD]|T[NX]|UT|V[AIT]|W[AIVY])\s+\d{5}(?:-\d{4})?\b/g;
+export const ADDRESS_CITY_STATE_ZIP = /\b[A-Z][A-Za-z.'-]+(?:\s+[A-Z][A-Za-z.'-]+){0,3},\s*(A[KLRZ]|C[AOT]|D[CE]|FL|GA|HI|I[ADLN]|K[SY]|LA|M[ADEINOST]|N[CDEHJMVY]|O[HKR]|P[AR]|RI|S[CD]|T[NX]|UT|V[AIT]|W[AIVY])\s+\d{5}(?:-\d{4})?\b/g;
 const NATIONAL_PHONE = {
     // NANP: area and exchange codes both start 2-9.
     US: /(?:\(\d{3}\)|\b[2-9]\d{2})[\s.-]?[2-9]\d{2}[\s.-]?\d{4}\b/g,
@@ -503,7 +566,7 @@ export function scanPii(text, options = {}) {
         reference = parsed;
     }
     if (selected(options, "email")) {
-        for (const { index, match } of matchAll(text, EMAIL)) {
+        for (const { index, match } of emailMatches(text)) {
             raw.push({
                 type: "email",
                 rule: "email.addr-spec-subset",
@@ -601,8 +664,8 @@ export function scanPii(text, options = {}) {
                 detail: { version: 4, scope },
             });
         }
-        for (const { index, match } of matchAll(text, IPV6_CANDIDATE)) {
-            const candidate = match[0].replace(/[:.]+$/, "");
+        for (const { index, value } of ipv6Candidates(text)) {
+            const candidate = value.replace(/[:.]+$/, "");
             if (!candidate.includes(":"))
                 continue;
             const groups = parseIpv6(candidate);
@@ -720,7 +783,7 @@ export function scanPii(text, options = {}) {
                 detail: { convention: "US" },
             });
         }
-        for (const { index, match } of matchAll(text, ADDRESS_CITY_STATE_ZIP)) {
+        for (const { index, match } of cityStateZipMatches(text)) {
             raw.push({
                 type: "postal_address",
                 rule: "address.us-city-state-zip",

package/dist/lib/policy.d.ts

@@ -20,6 +20,11 @@ export type PolicyOutcome = {
     readonly message: string;
     readonly matches: ReadonlyArray<PolicyMatch>;
     readonly matchCount: number;
+    /**
+     * Set when the pattern could not be run to the end of the text, or more
+     * matches exist than were counted: `matchCount` is then "at least".
+     */
+    readonly matchCountIsLowerBound?: true;
 };
 export type PolicyResult = {
     /**
@@ -38,5 +43,22 @@ export declare const MAX_MATCHES_PER_RULE = 20;
 export declare class PolicyError extends Error {
     readonly name = "PolicyError";
 }
-/** Evaluate every rule against the text. One pass per rule, no early exit. */
-export declare function evaluatePolicy(text: string, rules: ReadonlyArray<PolicyRule>): PolicyResult;
+/** Wall-clock budget for all of one check's patterns, shared in rule order. */
+export declare const POLICY_PATTERN_DEADLINE_MS = 10000;
+/** Where a check's patterns run: the call's abort signal, and whose runaway workers they count as. */
+export type PolicyRunContext = {
+    readonly signal?: AbortSignal;
+    readonly runawayKey?: string;
+};
+/**
+ * Evaluate every rule against the text. One pass per rule, no early exit.
+ *
+ * Phrase rules are literals, matched on this thread by a search that passes
+ * each character of the text once, whatever the phrase ({@link LiteralSearch};
+ * 0.7.0's case-insensitive regex cost the text's length times the phrase's).
+ * Pattern rules run in the regex worker, one after another in one session,
+ * sharing {@link POLICY_PATTERN_DEADLINE_MS}.
+ */
+export declare function evaluatePolicy(text: string, rules: ReadonlyArray<PolicyRule>, ctx?: PolicyRunContext, options?: {
+    readonly deadlineMs?: number;
+}): Promise<PolicyResult>;

package/dist/lib/policy.js

@@ -17,19 +17,24 @@
  * alternatives — refusing the whole evaluation, or silently treating the rule
  * as no-match — either block work over a typo or quietly turn a policy off.
  *
- * ## The regex is the operator's, and it runs untimed
+ * ## Patterns run in the regex worker, and an unanswered one is an error
  *
- * `*_pattern` rules compile caller-supplied regex sources and run them over
- * text up to the package's 2,000,000-character cap. JavaScript's engine
- * backtracks, so a pattern like `(a+)+$` can take time exponential in the
- * length of the text it fails on. There is no timeout here, because there is
- * no way to interrupt a regex mid-match in this runtime. The rules are
- * operator-written, not attacker-written, which is what makes this
- * acceptable — but an operator who pastes a pattern from untrusted content
- * has handed that content a stall. Write anchored patterns with no nested
- * unbounded quantifier.
+ * The rules arrive in the tool's input, so a model (or text it read) writes
+ * them. 0.7.0 compiled each `*_pattern` and ran it on this thread over text
+ * of up to 2,000,000 characters: `(a+)+!$|guaranteed returns` held the
+ * process 0.75 s and then answered "forbidden text is absent" — JavaScriptCore
+ * gives up on a runaway match and reports it as no match — so a gate passed
+ * text it had not checked (C073). Patterns now run in
+ * `@crewhaus/tool-safety`'s regex worker under a deadline. A pattern the
+ * screen refuses (a shape that backtracks exponentially) is an `error`, like
+ * one that does not compile. A run that cannot finish is an `error` too,
+ * "not evaluated", unless the matches it found before stopping already
+ * decide the rule (a forbidden pattern that matched has failed); a count
+ * from a run that stopped early is a lower bound, and says so.
  */
-import { compareStrings, matchAll } from "./text";
+import { describeRegexOutcome, openRegexSession, } from "@crewhaus/tool-safety/regex";
+import { LiteralSearch } from "./literal";
+import { compareStrings } from "./text";
 import { lineStarts, locate } from "./text";
 export const POLICY_RULE_KINDS = [
     "forbidden_pattern",
@@ -46,11 +51,59 @@ const EXCERPT_CHARS = 120;
 export class PolicyError extends Error {
     name = "PolicyError";
 }
-function escapeRegex(literal) {
-    return literal.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+/** Wall-clock budget for all of one check's patterns, shared in rule order. */
+export const POLICY_PATTERN_DEADLINE_MS = 10_000;
+/** The pattern screen's limits for a rule. */
+const PATTERN_LIMITS = { maxPatternChars: 10_000 };
+/** Matches counted per pattern rule; past it the count is a lower bound. */
+const MAX_COUNTED_MATCHES = 10_000;
+/**
+ * Every match of a `*_pattern` rule's source, run in the regex worker: the
+ * matches, or why the pattern could not be run at all (`refused`).
+ */
+async function findPattern(session, text, source, flags, deadlineMs, ctx) {
+    const outcome = await session.run({
+        op: "matchAll",
+        pattern: source,
+        flags,
+        input: text,
+        maxMatches: MAX_COUNTED_MATCHES,
+        deadlineMs,
+        maxInputChars: Math.max(1, text.length),
+        limits: PATTERN_LIMITS,
+        ...(ctx.signal === undefined ? {} : { signal: ctx.signal }),
+        ...(ctx.runawayKey === undefined ? {} : { runawayKey: ctx.runawayKey }),
+    });
+    if (outcome.status === "rejected")
+        return { refused: describeRegexOutcome(outcome) };
+    if (outcome.status === "ok") {
+        const all = outcome.result.matches;
+        return {
+            matches: all.slice(0, MAX_MATCHES_PER_RULE),
+            count: all.length,
+            lowerBound: outcome.result.truncated,
+        };
+    }
+    const partial = outcome.status === "gave-up" || outcome.status === "timeout"
+        ? (outcome.partial?.matches ?? [])
+        : [];
+    return {
+        matches: partial.slice(0, MAX_MATCHES_PER_RULE),
+        count: partial.length,
+        stopped: describeRegexOutcome(outcome),
+        lowerBound: true,
+    };
 }
-/** Evaluate every rule against the text. One pass per rule, no early exit. */
-export function evaluatePolicy(text, rules) {
+/**
+ * Evaluate every rule against the text. One pass per rule, no early exit.
+ *
+ * Phrase rules are literals, matched on this thread by a search that passes
+ * each character of the text once, whatever the phrase ({@link LiteralSearch};
+ * 0.7.0's case-insensitive regex cost the text's length times the phrase's).
+ * Pattern rules run in the regex worker, one after another in one session,
+ * sharing {@link POLICY_PATTERN_DEADLINE_MS}.
+ */
+export async function evaluatePolicy(text, rules, ctx = {}, options = {}) {
     if (rules.length > MAX_POLICY_RULES) {
         throw new PolicyError(`${rules.length} rules, over the ${MAX_POLICY_RULES} limit`);
     }
@@ -63,72 +116,50 @@ export function evaluatePolicy(text, rules) {
     }
     const starts = lineStarts(text);
     const outcomes = [];
-    for (const rule of rules) {
-        const flags = rule.caseSensitive === true ? "g" : "gi";
-        const isPattern = rule.kind.endsWith("_pattern");
-        let re;
-        try {
-            re = new RegExp(isPattern ? rule.value : escapeRegex(rule.value), flags);
-        }
-        catch (err) {
-            outcomes.push({
-                id: rule.id,
-                kind: rule.kind,
-                status: "error",
-                message: `rule not evaluated: invalid regular expression /${rule.value}/ — ${err.message}`,
-                matches: [],
-                matchCount: 0,
-            });
-            continue;
-        }
-        const matches = [];
-        let matchCount = 0;
-        for (const { index, match } of matchAll(text, re)) {
-            matchCount += 1;
-            if (matches.length < MAX_MATCHES_PER_RULE) {
-                const { line, column } = locate(starts, index);
-                matches.push({ line, column, excerpt: match[0].slice(0, EXCERPT_CHARS) });
+    const deadlineMs = options.deadlineMs ?? POLICY_PATTERN_DEADLINE_MS;
+    const started = performance.now();
+    let session;
+    let literals;
+    try {
+        for (const rule of rules) {
+            const isPattern = rule.kind.endsWith("_pattern");
+            let found;
+            if (isPattern) {
+                const left = Math.floor(deadlineMs - (performance.now() - started));
+                session ??= openRegexSession();
+                const run = left < 1
+                    ? {
+                        matches: [],
+                        count: 0,
+                        stopped: `the check's ${deadlineMs} ms for running patterns ran out before this rule`,
+                        lowerBound: true,
+                    }
+                    : await findPattern(session, text, rule.value, rule.caseSensitive === true ? "" : "i", left, ctx);
+                if ("refused" in run) {
+                    outcomes.push({
+                        id: rule.id,
+                        kind: rule.kind,
+                        status: "error",
+                        message: `rule not evaluated: invalid regular expression /${rule.value}/ — ${run.refused}`,
+                        matches: [],
+                        matchCount: 0,
+                    });
+                    continue;
+                }
+                found = run;
             }
+            else {
+                // A literal: linear to search, so it runs here, counted in full and
+                // located for the first few.
+                literals ??= new LiteralSearch(text);
+                const { matches, count } = literals.find(rule.value, rule.caseSensitive === true, MAX_MATCHES_PER_RULE);
+                found = { matches, count, lowerBound: false };
+            }
+            outcomes.push(judge(rule, found, starts));
         }
-        const present = matchCount > 0;
-        const describe = rule.description === undefined ? "" : ` (${rule.description})`;
-        if (rule.kind === "required_phrase" || rule.kind === "required_pattern") {
-            outcomes.push({
-                id: rule.id,
-                kind: rule.kind,
-                status: present ? "pass" : "fail",
-                message: present
-                    ? `required text found ${matchCount} time(s)${describe}`
-                    : `required text is absent${describe}`,
-                // A required rule that passed needs no locations; one that failed has none.
-                matches: [],
-                matchCount,
-            });
-            continue;
-        }
-        if (rule.kind === "review_pattern") {
-            outcomes.push({
-                id: rule.id,
-                kind: rule.kind,
-                status: present ? "review" : "pass",
-                message: present
-                    ? `${matchCount} passage(s) need a human decision${describe}`
-                    : `nothing matched${describe}`,
-                matches,
-                matchCount,
-            });
-            continue;
-        }
-        outcomes.push({
-            id: rule.id,
-            kind: rule.kind,
-            status: present ? "fail" : "pass",
-            message: present
-                ? `forbidden text found ${matchCount} time(s)${describe}`
-                : `forbidden text is absent${describe}`,
-            matches,
-            matchCount,
-        });
+    }
+    finally {
+        session?.close();
     }
     const counts = { pass: 0, fail: 0, review: 0, error: 0 };
     for (const outcome of outcomes)
@@ -136,3 +167,69 @@ export function evaluatePolicy(text, rules) {
     outcomes.sort((a, b) => compareStrings(a.id, b.id));
     return { pass: counts.fail === 0 && counts.error === 0, outcomes, counts };
 }
+/**
+ * One rule's outcome from what its scan found. A scan that stopped early is
+ * a verdict only when what it found already decides the rule; otherwise the
+ * rule was not evaluated, and says why.
+ */
+function judge(rule, found, starts) {
+    const matches = [];
+    for (const { index, match } of found.matches) {
+        const { line, column } = locate(starts, index);
+        matches.push({ line, column, excerpt: match.slice(0, EXCERPT_CHARS) });
+    }
+    const matchCount = found.count;
+    const present = matchCount > 0;
+    const atLeast = found.lowerBound ? { matchCountIsLowerBound: true } : {};
+    const describe = rule.description === undefined ? "" : ` (${rule.description})`;
+    if (!present && found.stopped !== undefined) {
+        return {
+            id: rule.id,
+            kind: rule.kind,
+            status: "error",
+            message: `rule not evaluated: ${found.stopped} — nothing had matched before it stopped, which is not the same as nothing matching${describe}`,
+            matches: [],
+            matchCount: 0,
+            ...atLeast,
+        };
+    }
+    const count = found.lowerBound ? `at least ${matchCount}` : `${matchCount}`;
+    if (rule.kind === "required_phrase" || rule.kind === "required_pattern") {
+        return {
+            id: rule.id,
+            kind: rule.kind,
+            status: present ? "pass" : "fail",
+            message: present
+                ? `required text found ${count} time(s)${describe}`
+                : `required text is absent${describe}`,
+            // A required rule that passed needs no locations; one that failed has none.
+            matches: [],
+            matchCount,
+            ...atLeast,
+        };
+    }
+    if (rule.kind === "review_pattern") {
+        return {
+            id: rule.id,
+            kind: rule.kind,
+            status: present ? "review" : "pass",
+            message: present
+                ? `${count} passage(s) need a human decision${describe}`
+                : `nothing matched${describe}`,
+            matches,
+            matchCount,
+            ...atLeast,
+        };
+    }
+    return {
+        id: rule.id,
+        kind: rule.kind,
+        status: present ? "fail" : "pass",
+        message: present
+            ? `forbidden text found ${count} time(s)${describe}`
+            : `forbidden text is absent${describe}`,
+        matches,
+        matchCount,
+        ...atLeast,
+    };
+}

package/dist/lib/secrets.d.ts

@@ -40,6 +40,14 @@ export type SecretRule = {
     readonly pattern: RegExp;
     /** Which capture group holds the secret itself. 0 = the whole match. */
     readonly group?: number;
+    /**
+     * The pattern's matches when a plain global search would rescan a run;
+     * the same matches as `matchAll(text, pattern)`.
+     */
+    readonly matches?: (text: string) => Iterable<{
+        index: number;
+        match: RegExpExecArray;
+    }>;
 };
 /**
  * The rule table, in scan order. Ids are stable: an operator's allow-list of

package/dist/lib/secrets.js

@@ -30,7 +30,33 @@
  * scanner that cries wolf on them gets switched off.
  */
 import { classifyCharset, looksHighEntropy, roundBits, shannonEntropy, } from "./entropy";
-import { groupSpan, maskValue, matchAll, sortFindings, withPositions } from "./text";
+import { SpanIndex, groupSpan, maskValue, matchAll, matchAllSkipping, sortFindings, withPositions, } from "./text";
+const JWT_COMPACT = /\beyJ[A-Za-z0-9_-]{8,}\.eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b/g;
+/** A code unit `\w` matches. */
+function isWordUnit(c) {
+    return ((c >= 0x30 && c <= 0x39) || (c >= 0x41 && c <= 0x5a) || (c >= 0x61 && c <= 0x7a) || c === 0x5f);
+}
+/**
+ * JWT_COMPACT's matches. Its first segment reads to the end of a run of
+ * `[A-Za-z0-9_-]` and needs a `.` there, so a run with `eyJ` at many word
+ * boundaries (`eyJ-eyJ-…`) was read to its end again from every one:
+ * quadratic in the run, on the megabytes `SecretScan` and `RedactForExport`
+ * take. A later start in the same run meets the same end, with a shorter
+ * first segment and the same text after it, so a try that fails rules out
+ * the rest of the run.
+ */
+function jwtCompactMatches(text) {
+    return matchAllSkipping(text, JWT_COMPACT, (t, at) => t.charCodeAt(at) === 0x65 &&
+        t.charCodeAt(at + 1) === 0x79 &&
+        t.charCodeAt(at + 2) === 0x4a &&
+        (at === 0 || !isWordUnit(t.charCodeAt(at - 1))), (t, at) => {
+        let end = at;
+        while (end < t.length && (isWordUnit(t.charCodeAt(end)) || t.charCodeAt(end) === 0x2d)) {
+            end++;
+        }
+        return end;
+    });
+}
 /**
  * The rule table, in scan order. Ids are stable: an operator's allow-list of
  * accepted findings keys on them.
@@ -161,7 +187,8 @@ export const SECRET_RULES = [
         kind: "structural",
         description: "JWT in compact serialization; a bearer token unless it is a public id token",
         severity: "medium",
-        pattern: /\beyJ[A-Za-z0-9_-]{8,}\.eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b/g,
+        pattern: JWT_COMPACT,
+        matches: jwtCompactMatches,
     },
     {
         id: "url.credentials",
@@ -234,7 +261,7 @@ export function secretFindings(text, options = {}) {
     const rules = selectRules(options);
     const raw = [];
     for (const rule of rules) {
-        for (const { match } of matchAll(text, rule.pattern)) {
+        for (const { match } of rule.matches?.(text) ?? matchAll(text, rule.pattern)) {
             const group = rule.group ?? 0;
             const value = match[group];
             if (value === undefined || value.length === 0)
@@ -260,6 +287,7 @@ export function secretFindings(text, options = {}) {
     }
     if (options.highEntropy !== false) {
         const requireContext = options.requireContext !== false;
+        const candidates = [];
         for (const { index, match } of matchAll(text, HIGH_ENTROPY_CANDIDATE)) {
             const value = match[0];
             if (PLACEHOLDER.test(value))
@@ -272,8 +300,17 @@ export function secretFindings(text, options = {}) {
                 if (!CREDENTIAL_CONTEXT.test(before))
                     continue;
             }
-            // A vendor rule already covering this span is the better finding.
-            if (raw.some((f) => index < f.end && f.start < index + value.length))
+            candidates.push({ index, value });
+        }
+        // A vendor rule already covering this span is the better finding. Each
+        // candidate is checked against every finding pushed before it, as it
+        // always was, but through a `SpanIndex`: checking them one by one was
+        // quadratic in the findings on SecretScan's megabytes of text.
+        const covered = new SpanIndex(raw.map((f) => f.start).concat(candidates.map((c) => c.index)));
+        for (const f of raw)
+            covered.add(f.start, f.end);
+        for (const { index, value } of candidates) {
+            if (covered.overlaps(index, index + value.length))
                 continue;
             raw.push({
                 type: HIGH_ENTROPY_RULE,
@@ -284,6 +321,7 @@ export function secretFindings(text, options = {}) {
                 value,
                 detail: { severity: "low" },
             });
+            covered.add(index, index + value.length);
         }
     }
     return sortFindings(withPositions(text, raw));

package/dist/lib/text.d.ts

@@ -77,6 +77,37 @@ export declare function compareStrings(a: string, b: string): number;
  * that contains a live credential. It is tested directly.
  */
 export declare function maskValue(value: string, keep?: number): string;
+/**
+ * Spans, asked one question: does `[start, end)` overlap any of them?
+ *
+ * Overlap is `start < k.end && k.start < end`, the test `dedupeOverlaps` and
+ * `secretFindings` make, answered for any numbers, empty spans included.
+ * Among the spans that start before `end`, one overlaps exactly when the
+ * largest of their ends is past `start`. So the index keeps that largest end
+ * in a Fenwick tree with one slot per distinct start, and a question or an
+ * addition visits O(log n) slots. Checking every span kept so far made
+ * those two passes quadratic in the findings, on text a model or a web page
+ * supplies.
+ *
+ * Every start a span will be added with is named up front, because the
+ * slots are the distinct starts in order. `steps` counts the slots and
+ * probes the calls visit, so a test can hold the index to its bound.
+ */
+export declare class SpanIndex {
+    /** Tree slots and binary-search probes visited so far. */
+    steps: number;
+    /** The distinct starts, ascending. */
+    private readonly starts;
+    /** 1-based Fenwick tree of the largest end per prefix of `starts`. */
+    private readonly maxEnd;
+    constructor(starts: ReadonlyArray<number>);
+    /** How many distinct starts are below `bound`; none are below NaN. */
+    private below;
+    /** Whether some span added so far has `k.start < end && start < k.end`. */
+    overlaps(start: number, end: number): boolean;
+    /** Add `[start, end)`; `start` must be one the index was built with. */
+    add(start: number, end: number): void;
+}
 /**
  * Drop findings swallowed by a stronger one.
  *
@@ -85,6 +116,10 @@ export declare function maskValue(value: string, keep?: number): string;
  * survivor is chosen by confidence, then by length, then by earliest start,
  * then by rule name — a total order, so the same input always yields the
  * same set.
+ *
+ * Each candidate, in that order, is kept unless it overlaps one kept before
+ * it. A `SpanIndex` answers that, so the pass is O(k log k) in the findings
+ * rather than a check against every finding kept so far.
  */
 export declare function dedupeOverlaps(findings: ReadonlyArray<Finding>): Finding[];
 /** Reading order, with a deterministic tiebreak. */
@@ -94,8 +129,14 @@ export declare function withPositions(text: string, raw: ReadonlyArray<Omit<Find
 /**
  * Replace each span with the string `replace` returns for it.
  *
- * Spans must not overlap (`dedupeOverlaps` guarantees that); they are
- * applied right to left so earlier offsets stay valid.
+ * Spans must not overlap (`dedupeOverlaps` guarantees that). `replace` is
+ * called right to left, as it always has been, and the output is the text
+ * between the spans joined with their replacements: one copy of the text.
+ * Rebuilding the whole string for every span was O(spans × text), and a
+ * redaction pass over megabytes of findings-dense text has hundreds of
+ * thousands of spans. Spans that overlap, sit out of order or outside the
+ * text, or have fractional offsets keep the old right-to-left rebuild, which
+ * gives the same result on spans that do none of those things.
  */
 export declare function applySpans(text: string, findings: ReadonlyArray<Finding>, replace: (finding: Finding) => string): string;
 /** Tally findings by `type`, sorted by type so the record is stable. */
@@ -116,6 +157,24 @@ export declare function matchAll(text: string, pattern: RegExp): Generator<{
     index: number;
     match: RegExpExecArray;
 }>;
+/**
+ * `matchAll(text, pattern)`'s matches, in order, with the pattern tried
+ * (sticky) only where `canStart` says a match may begin, and, after a try
+ * that fails at `at`, not again before `failedTo(text, at)`.
+ *
+ * For a pattern that opens with an unbounded run (an email's local part, a
+ * word before a comma, the whitespace after a line start), a global search
+ * that fails at one start moves one character right and scans the same run
+ * again: quadratic in the run's length. Every later start inside such a run
+ * shares the run's end and reads the same text after it, so it fails too;
+ * `failedTo` names where that stops being true, and the matches are the
+ * global pattern's, start for start. After a match the next try is at its
+ * end (one further for an empty match), as `lastIndex` would be.
+ */
+export declare function matchAllSkipping(text: string, pattern: RegExp, canStart: (text: string, at: number) => boolean, failedTo: (text: string, at: number) => number): Generator<{
+    index: number;
+    match: RegExpExecArray;
+}>;
 /**
  * The absolute `[start, end)` of one capture group of a match produced by
  * `matchAll`, or `undefined` when the group did not participate.