@crewhaus/tool-secure 0.7.0

package/dist/lib/policy.d.ts

@@ -0,0 +1,42 @@
+export declare const POLICY_RULE_KINDS: readonly ["forbidden_pattern", "forbidden_phrase", "required_pattern", "required_phrase", "review_pattern"];
+export type PolicyRuleKind = (typeof POLICY_RULE_KINDS)[number];
+export type PolicyRule = {
+    readonly id: string;
+    readonly kind: PolicyRuleKind;
+    /** A literal for `*_phrase` rules; a JavaScript regex source for `*_pattern`. */
+    readonly value: string;
+    readonly caseSensitive?: boolean;
+    readonly description?: string;
+};
+export type PolicyMatch = {
+    readonly line: number;
+    readonly column: number;
+    readonly excerpt: string;
+};
+export type PolicyOutcome = {
+    readonly id: string;
+    readonly kind: PolicyRuleKind;
+    readonly status: "pass" | "fail" | "review" | "error";
+    readonly message: string;
+    readonly matches: ReadonlyArray<PolicyMatch>;
+    readonly matchCount: number;
+};
+export type PolicyResult = {
+    /**
+     * True when no rule produced `fail` AND none produced `error`. A rule that
+     * did not compile was not evaluated, so the check cannot claim to have
+     * passed. `review` outcomes do not fail; they queue for a person.
+     */
+    readonly pass: boolean;
+    readonly outcomes: ReadonlyArray<PolicyOutcome>;
+    readonly counts: Readonly<Record<PolicyOutcome["status"], number>>;
+};
+/** Bounds the rule set: a policy is a page of rules, not a corpus. */
+export declare const MAX_POLICY_RULES = 200;
+/** Locations reported per rule. The count is always exact. */
+export declare const MAX_MATCHES_PER_RULE = 20;
+export declare class PolicyError extends Error {
+    readonly name = "PolicyError";
+}
+/** Evaluate every rule against the text. One pass per rule, no early exit. */
+export declare function evaluatePolicy(text: string, rules: ReadonlyArray<PolicyRule>): PolicyResult;

package/dist/lib/policy.js

@@ -0,0 +1,138 @@
+/**
+ * Operator-declared content rules.
+ *
+ * Unlike every other module here, this one has no opinions: the rules come
+ * from the operator, and the evaluation is mechanical. "Does the disclaimer
+ * appear?", "does the word 'guaranteed' appear?", "does anything match this
+ * claim pattern?" — all answerable without judgement, which is exactly why
+ * they should not cost a model call.
+ *
+ * What it cannot do is decide whether a document COMPLIES with a policy.
+ * A required disclaimer can be present and wrong; a forbidden phrase can be
+ * absent while the same claim is made in other words. `review_pattern` is
+ * there for that: it is the rule kind that says "a person looks at this".
+ *
+ * A rule whose regular expression does not compile is reported as `error`
+ * for that rule alone, and an `error` outcome fails the check. The
+ * alternatives — refusing the whole evaluation, or silently treating the rule
+ * as no-match — either block work over a typo or quietly turn a policy off.
+ *
+ * ## The regex is the operator's, and it runs untimed
+ *
+ * `*_pattern` rules compile caller-supplied regex sources and run them over
+ * text up to the package's 2,000,000-character cap. JavaScript's engine
+ * backtracks, so a pattern like `(a+)+$` can take time exponential in the
+ * length of the text it fails on. There is no timeout here, because there is
+ * no way to interrupt a regex mid-match in this runtime. The rules are
+ * operator-written, not attacker-written, which is what makes this
+ * acceptable — but an operator who pastes a pattern from untrusted content
+ * has handed that content a stall. Write anchored patterns with no nested
+ * unbounded quantifier.
+ */
+import { compareStrings, matchAll } from "./text";
+import { lineStarts, locate } from "./text";
+export const POLICY_RULE_KINDS = [
+    "forbidden_pattern",
+    "forbidden_phrase",
+    "required_pattern",
+    "required_phrase",
+    "review_pattern",
+];
+/** Bounds the rule set: a policy is a page of rules, not a corpus. */
+export const MAX_POLICY_RULES = 200;
+/** Locations reported per rule. The count is always exact. */
+export const MAX_MATCHES_PER_RULE = 20;
+const EXCERPT_CHARS = 120;
+export class PolicyError extends Error {
+    name = "PolicyError";
+}
+function escapeRegex(literal) {
+    return literal.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+/** Evaluate every rule against the text. One pass per rule, no early exit. */
+export function evaluatePolicy(text, rules) {
+    if (rules.length > MAX_POLICY_RULES) {
+        throw new PolicyError(`${rules.length} rules, over the ${MAX_POLICY_RULES} limit`);
+    }
+    const seen = new Set();
+    for (const rule of rules) {
+        if (seen.has(rule.id)) {
+            throw new PolicyError(`duplicate rule id "${rule.id}" — ids identify a rule in the result`);
+        }
+        seen.add(rule.id);
+    }
+    const starts = lineStarts(text);
+    const outcomes = [];
+    for (const rule of rules) {
+        const flags = rule.caseSensitive === true ? "g" : "gi";
+        const isPattern = rule.kind.endsWith("_pattern");
+        let re;
+        try {
+            re = new RegExp(isPattern ? rule.value : escapeRegex(rule.value), flags);
+        }
+        catch (err) {
+            outcomes.push({
+                id: rule.id,
+                kind: rule.kind,
+                status: "error",
+                message: `rule not evaluated: invalid regular expression /${rule.value}/ — ${err.message}`,
+                matches: [],
+                matchCount: 0,
+            });
+            continue;
+        }
+        const matches = [];
+        let matchCount = 0;
+        for (const { index, match } of matchAll(text, re)) {
+            matchCount += 1;
+            if (matches.length < MAX_MATCHES_PER_RULE) {
+                const { line, column } = locate(starts, index);
+                matches.push({ line, column, excerpt: match[0].slice(0, EXCERPT_CHARS) });
+            }
+        }
+        const present = matchCount > 0;
+        const describe = rule.description === undefined ? "" : ` (${rule.description})`;
+        if (rule.kind === "required_phrase" || rule.kind === "required_pattern") {
+            outcomes.push({
+                id: rule.id,
+                kind: rule.kind,
+                status: present ? "pass" : "fail",
+                message: present
+                    ? `required text found ${matchCount} time(s)${describe}`
+                    : `required text is absent${describe}`,
+                // A required rule that passed needs no locations; one that failed has none.
+                matches: [],
+                matchCount,
+            });
+            continue;
+        }
+        if (rule.kind === "review_pattern") {
+            outcomes.push({
+                id: rule.id,
+                kind: rule.kind,
+                status: present ? "review" : "pass",
+                message: present
+                    ? `${matchCount} passage(s) need a human decision${describe}`
+                    : `nothing matched${describe}`,
+                matches,
+                matchCount,
+            });
+            continue;
+        }
+        outcomes.push({
+            id: rule.id,
+            kind: rule.kind,
+            status: present ? "fail" : "pass",
+            message: present
+                ? `forbidden text found ${matchCount} time(s)${describe}`
+                : `forbidden text is absent${describe}`,
+            matches,
+            matchCount,
+        });
+    }
+    const counts = { pass: 0, fail: 0, review: 0, error: 0 };
+    for (const outcome of outcomes)
+        counts[outcome.status] += 1;
+    outcomes.sort((a, b) => compareStrings(a.id, b.id));
+    return { pass: counts.fail === 0 && counts.error === 0, outcomes, counts };
+}

package/dist/lib/redact.d.ts

@@ -0,0 +1,41 @@
+import { type Finding } from "./text";
+/** Bounds the mapping tools: a mapping is a document, not a database. */
+export declare const MAX_MAPPING_ENTRIES = 5000;
+/** Bounds the alternation a mapping compiles to. */
+export declare const MAX_MAPPING_KEY_CHARS = 200000;
+export declare class MappingError extends Error {
+    readonly name = "MappingError";
+}
+/** Keyed, unlinkable-without-the-key token material. */
+export declare function derivePseudonym(key: string, type: string, canonical: string, length?: number): string;
+/** Unkeyed, content-addressed token. Reversible by enumeration — see above. */
+export declare function digestPseudonym(type: string, canonical: string, length?: number): string;
+/** `[EMAIL]`-style placeholder for a finding type. */
+export declare function placeholderFor(type: string): string;
+/** `[EMAIL:9f2b…]`-style pseudonym placeholder. */
+export declare function pseudonymPlaceholder(type: string, token: string): string;
+/** Anything this package emits as a token, for spotting leftovers. */
+export declare const TOKEN_SHAPE: RegExp;
+/** Replace every finding with `replace(finding)`; spans must not overlap. */
+export declare function redactFindings(text: string, findings: ReadonlyArray<Finding>, replace: (finding: Finding) => string): string;
+/** Escape a literal for embedding in a regular expression. */
+export declare function escapeRegex(literal: string): string;
+export type MappingApplication = {
+    readonly text: string;
+    /** Replacements per mapping key, sorted by key. Zero-count keys included. */
+    readonly counts: Record<string, number>;
+    readonly total: number;
+};
+/**
+ * Replace each key of `mapping` with its value, longest key first.
+ *
+ * Longest-first matters: with `{"Ann": "P1", "Anna": "P2"}` a shortest-first
+ * pass would turn "Anna" into "P1a". The pass is single, so a replacement's
+ * own text is never rescanned and tokens cannot cascade.
+ *
+ * `wholeWord` anchors each key between word boundaries, which is what you
+ * want for names and what you do NOT want for email addresses.
+ */
+export declare function applyMapping(text: string, mapping: Readonly<Record<string, string>>, wholeWord?: boolean): MappingApplication;
+/** Flip a mapping, refusing a value used for two different keys. */
+export declare function invertMapping(mapping: Readonly<Record<string, string>>): Record<string, string>;

package/dist/lib/redact.js

@@ -0,0 +1,121 @@
+/**
+ * Turning findings into removed text, and values into stable tokens.
+ *
+ * ## Two token schemes, with different properties
+ *
+ * KEYED (`derivePseudonym`) — `HMAC-SHA256(key, "<type>:<canonical>")`,
+ * truncated. The same value yields the same token in every document the same
+ * key is used for, so records can still be joined; without the key the token
+ * cannot be walked back to the value even by someone who guesses it, because
+ * the attacker cannot compute candidate tokens. This is the scheme to use.
+ *
+ * UNKEYED (`digestPseudonym`) — a plain SHA-256 prefix. Stable and
+ * convenient, and NOT unlinkable: an identifier space small enough to
+ * enumerate (every email at a company, every US SSN) can be exhaustively
+ * hashed and matched against the tokens. It is offered because a caller
+ * sometimes genuinely wants a content-addressed label, and it is labelled
+ * `reversibleByEnumeration: true` in every result that uses it so nobody
+ * mistakes it for anonymization.
+ *
+ * ## Truncation
+ *
+ * Tokens are truncated to keep documents readable. Truncation costs
+ * collision resistance: at n hex characters two different values collide
+ * with probability ~1 in 16^n. The default of 12 (48 bits) is comfortable
+ * for a document-sized corpus; the length is reported so a caller working at
+ * dataset scale can raise it.
+ */
+import { createHash, createHmac } from "node:crypto";
+import { applySpans, compareStrings } from "./text";
+/** Bounds the mapping tools: a mapping is a document, not a database. */
+export const MAX_MAPPING_ENTRIES = 5_000;
+/** Bounds the alternation a mapping compiles to. */
+export const MAX_MAPPING_KEY_CHARS = 200_000;
+export class MappingError extends Error {
+    name = "MappingError";
+}
+/** Keyed, unlinkable-without-the-key token material. */
+export function derivePseudonym(key, type, canonical, length = 12) {
+    const digest = createHmac("sha256", key).update(`${type}:${canonical}`, "utf8").digest("hex");
+    return digest.slice(0, Math.max(4, Math.min(64, length)));
+}
+/** Unkeyed, content-addressed token. Reversible by enumeration — see above. */
+export function digestPseudonym(type, canonical, length = 12) {
+    const digest = createHash("sha256").update(`${type}:${canonical}`, "utf8").digest("hex");
+    return digest.slice(0, Math.max(4, Math.min(64, length)));
+}
+/** `[EMAIL]`-style placeholder for a finding type. */
+export function placeholderFor(type) {
+    return `[${type.toUpperCase().replace(/[^A-Z0-9]+/g, "_")}]`;
+}
+/** `[EMAIL:9f2b…]`-style pseudonym placeholder. */
+export function pseudonymPlaceholder(type, token) {
+    return `[${type.toUpperCase().replace(/[^A-Z0-9]+/g, "_")}:${token}]`;
+}
+/** Anything this package emits as a token, for spotting leftovers. */
+export const TOKEN_SHAPE = /\[[A-Z0-9_]+(?::[0-9a-f]{4,64})?\]/g;
+/** Replace every finding with `replace(finding)`; spans must not overlap. */
+export function redactFindings(text, findings, replace) {
+    return applySpans(text, findings, replace);
+}
+/** Escape a literal for embedding in a regular expression. */
+export function escapeRegex(literal) {
+    return literal.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+/**
+ * Replace each key of `mapping` with its value, longest key first.
+ *
+ * Longest-first matters: with `{"Ann": "P1", "Anna": "P2"}` a shortest-first
+ * pass would turn "Anna" into "P1a". The pass is single, so a replacement's
+ * own text is never rescanned and tokens cannot cascade.
+ *
+ * `wholeWord` anchors each key between word boundaries, which is what you
+ * want for names and what you do NOT want for email addresses.
+ */
+export function applyMapping(text, mapping, wholeWord = false) {
+    const keys = Object.keys(mapping);
+    if (keys.length > MAX_MAPPING_ENTRIES) {
+        throw new MappingError(`mapping has ${keys.length} entries, over the ${MAX_MAPPING_ENTRIES} limit — split the dataset`);
+    }
+    const totalKeyChars = keys.reduce((sum, key) => sum + key.length, 0);
+    if (totalKeyChars > MAX_MAPPING_KEY_CHARS) {
+        throw new MappingError(`mapping keys total ${totalKeyChars} characters, over the ${MAX_MAPPING_KEY_CHARS} limit`);
+    }
+    const counts = {};
+    for (const key of [...keys].sort(compareStrings))
+        counts[key] = 0;
+    const empty = keys.filter((key) => key.length === 0);
+    if (empty.length > 0)
+        throw new MappingError("mapping contains an empty key");
+    if (keys.length === 0)
+        return { text, counts, total: 0 };
+    const ordered = [...keys].sort((a, b) => b.length - a.length || compareStrings(a, b));
+    const body = ordered.map(escapeRegex).join("|");
+    const source = wholeWord ? `(?<![\\w])(?:${body})(?![\\w])` : `(?:${body})`;
+    const re = new RegExp(source, "g");
+    let total = 0;
+    const out = text.replace(re, (match) => {
+        const replacement = mapping[match];
+        if (replacement === undefined)
+            return match;
+        counts[match] = (counts[match] ?? 0) + 1;
+        total += 1;
+        return replacement;
+    });
+    return { text: out, counts, total };
+}
+/** Flip a mapping, refusing a value used for two different keys. */
+export function invertMapping(mapping) {
+    const out = {};
+    for (const key of Object.keys(mapping).sort(compareStrings)) {
+        const token = mapping[key];
+        if (token === undefined)
+            continue;
+        const existing = out[token];
+        if (existing !== undefined && existing !== key) {
+            throw new MappingError(`token "${token}" maps to both "${existing}" and "${key}" — a mapping must be invertible to be rejoinable`);
+        }
+        out[token] = key;
+    }
+    return out;
+}

package/dist/lib/secrets.d.ts

@@ -0,0 +1,111 @@
+/**
+ * Credential detection.
+ *
+ * ## The one hard rule
+ *
+ * A finding never carries the secret. Every value that reaches a result goes
+ * through `maskValue` first, and the tests assert that the serialized result
+ * of scanning a known credential does not contain that credential. A scanner
+ * that prints what it found has moved the secret from a file into a
+ * transcript, a log and a model's context window — three more places it has
+ * to be rotated out of.
+ *
+ * ## What the rules are
+ *
+ * Two kinds. VENDOR rules key on a published, distinctive prefix
+ * (`AKIA…`, `ghp_…`, `sk-ant-…`); they are precise and they go stale — a
+ * provider that changes its key format silently stops being covered here.
+ * STRUCTURAL rules key on shape: a PEM block, a JWT, a URL with a password
+ * in the authority, an assignment to something called `password`.
+ *
+ * ## What it cannot do
+ *
+ * It cannot tell a live key from a revoked one, a real key from an example
+ * in documentation, or a missing rule from a clean file. `SecretScan`
+ * reports the rule list it ran precisely so that "no findings" reads as
+ * "none of THESE matched", which is the only claim it can support.
+ *
+ * The high-entropy rule is deliberately context-gated by default: commit
+ * hashes, UUIDs, base64 images and minified code are all high entropy, and a
+ * scanner that cries wolf on them gets switched off.
+ */
+import { type Charset } from "./entropy";
+import { type Finding } from "./text";
+export type Severity = "critical" | "high" | "medium" | "low";
+export type SecretRule = {
+    readonly id: string;
+    readonly kind: "vendor" | "structural";
+    readonly description: string;
+    readonly severity: Severity;
+    readonly pattern: RegExp;
+    /** Which capture group holds the secret itself. 0 = the whole match. */
+    readonly group?: number;
+};
+/**
+ * The rule table, in scan order. Ids are stable: an operator's allow-list of
+ * accepted findings keys on them.
+ */
+export declare const SECRET_RULES: ReadonlyArray<SecretRule>;
+export type SecretHit = {
+    readonly rule: string;
+    readonly kind: "vendor" | "structural";
+    readonly severity: Severity;
+    readonly description: string;
+    readonly line: number;
+    readonly column: number;
+    readonly start: number;
+    readonly length: number;
+    /** The ONLY view of the value that ever leaves this module. */
+    readonly masked: string;
+    readonly entropyBits: number;
+    readonly charset: Charset;
+};
+export type SecretScanOptions = {
+    /** Restrict to these rule ids. Unknown ids are an error, not a silent no-op. */
+    readonly rules?: ReadonlyArray<string>;
+    /** Also run the context-gated high-entropy rule. Default true. */
+    readonly highEntropy?: boolean;
+    /** Bits-per-character floor for the high-entropy rule. Default is per-charset. */
+    readonly minEntropy?: number;
+    /**
+     * Require a credential-ish word near a high-entropy string. Default true;
+     * turning it off surfaces every opaque blob, hashes and UUIDs included.
+     */
+    readonly requireContext?: boolean;
+};
+export declare class SecretRuleError extends Error {
+    readonly name = "SecretRuleError";
+}
+/** The id the context-gated entropy heuristic reports under. */
+export declare const HIGH_ENTROPY_RULE = "generic.high-entropy";
+/**
+ * Every rule id a given options object will report under, including the
+ * entropy heuristic when it is enabled. `SecretScan` prints this so an empty
+ * result reads as a scope rather than a verdict, and `secretFindings` below
+ * runs exactly this set — the list and the scan cannot drift apart.
+ */
+export declare function rulesRunFor(options?: SecretScanOptions): string[];
+/** The rules a given options object will actually run, in scan order. */
+export declare function selectRules(options?: SecretScanOptions): SecretRule[];
+/**
+ * Every located secret in the text, values included, in document order.
+ *
+ * This is the ONE implementation. `scanSecrets` masks it for a result and
+ * `secretSpans` hands it to a redactor unmasked; there is no second copy of
+ * the rule loop to fall out of step with this one, which is exactly what let
+ * `RedactForExport` report a high-entropy finding and then not remove it.
+ */
+export declare function secretFindings(text: string, options?: SecretScanOptions): Finding[];
+/** Scan text. The returned hits are already masked. */
+export declare function scanSecrets(text: string, options?: SecretScanOptions): SecretHit[];
+/**
+ * The spans a redactor should replace. Same rules, same options, same spans
+ * as `scanSecrets` — including the entropy heuristic, so a document cannot be
+ * reported as holding a secret and handed back with that secret still in it.
+ *
+ * `Finding`s carry their value because the caller is about to remove them; it
+ * is the scanner's OUTPUT that must never carry one.
+ */
+export declare function secretSpans(text: string, options?: SecretScanOptions): Finding[];
+/** Summarize hits by severity, highest first, for a one-line verdict. */
+export declare function severityCounts(hits: ReadonlyArray<SecretHit>): Record<Severity, number>;