@theoremjs/agents 0.3.0 → 0.3.2

package/esm/src/guardrails/sensitive.js

@@ -1,48 +1,89 @@
 import { blobAt, spansFromPatterns } from '../observability/spans.js';
+import { CREDENTIAL_RULES, GLOBAL_ALLOWLIST } from './credential-rules.js';
+import { credentialHit, credentialSpans } from './credential-scan.js';
 const SSN = /\b\d{3}-\d{2}-\d{4}\b/g;
-const SSN_CONTEXTUAL = /(?:SSN|social\s*security(?:\s*number)?)[:\s]+(?:\d[-\s]*){9}/gi;
+const SSN_CONTEXTUAL = /(?:SSN|social\s*security(?:\s*number)?)[:\s]+\d(?:[-\s]*\d){8}/gi;
 const ITIN = /\b9\d{2}-\d{2}-\d{4}\b/g;
 const EIN = /\b\d{2}-\d{7}\b/g;
 const IBAN = /\b[A-Z]{2}\d{2}[A-Z0-9]{13,30}\b/g;
 const IPV4 = /\b(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\b/g;
-const IPV6 = /\b(?:[0-9a-f]{1,4}:){7}[0-9a-f]{1,4}\b/gi;
-const AWS_ACCESS = /\bAKIA[0-9A-Z]{16,20}\b/g;
-const GOOGLE_API = /\bAIza[0-9A-Za-z_-]{35}\b/g;
-const OPENAI_KEY = /\bsk-\s*[A-Za-z0-9]{20,}\b/g;
-const ANTHROPIC_KEY = /\bsk-ant-[A-Za-z0-9_-]{20,}\b/g;
-const OPENROUTER_KEY = /\bsk-or-[A-Za-z0-9_-]{20,}\b/g;
-const GITHUB_PAT = /\bgithub_pat_[A-Za-z0-9_]{20,}\b/g;
-const GITHUB_TOKEN = /\bghp_[A-Za-z0-9]{36}\b/g;
-const SLACK_TOKEN = /\bxox[baprs]-[A-Za-z0-9-]{10,}\b/g;
-const BEARER = /\bBearer\s+[A-Za-z0-9._~+/-]+=*/gi;
-/** Bounded payload so PEM redaction cannot ReDoS on repeated BEGIN markers. */
-const PEM_KEY = /-----BEGIN (?:RSA )?PRIVATE KEY-----[\s\S]{0,16384}?-----END (?:RSA )?PRIVATE KEY-----/g;
+const HEX_GROUP = '[0-9a-f]{1,4}';
+const DOTTED_QUAD = '(?:(?:25[0-5]|2[0-4]\\d|[01]?\\d\\d?)\\.){3}(?:25[0-5]|2[0-4]\\d|[01]?\\d\\d?)';
+/** Every RFC 4291 text form: full, `::`-compressed, and with a dotted IPv4 tail. */
+const IPV6_FORMS = [
+    `(?:${HEX_GROUP}:){6}${DOTTED_QUAD}`,
+    `(?:${HEX_GROUP}:){1,5}:${DOTTED_QUAD}`,
+    `::(?:${HEX_GROUP}:){0,5}${DOTTED_QUAD}`,
+    `(?:${HEX_GROUP}:){7}${HEX_GROUP}`,
+    `(?:${HEX_GROUP}:){1,6}(?::${HEX_GROUP}){1,6}`,
+    `(?:${HEX_GROUP}:){1,7}:`,
+    `:(?::${HEX_GROUP}){1,7}`,
+];
+const IPV6 = new RegExp(`(?<![\\w:.])(?:${IPV6_FORMS.join('|')})(?![\\w:])`, 'gi');
+/** Credentials gitleaks' rules miss: keys cut short or spaced out, a short private key, and a bare bearer token. */
+const OWN_CREDENTIAL_RULES = [
+    {
+        id: 'openai-api-key-any',
+        pattern: /\bsk-\s*[A-Za-z0-9]{20,}\b/,
+        keywords: ['sk-'],
+        allowlists: [],
+    },
+    {
+        id: 'anthropic-api-key-any',
+        pattern: /\bsk-ant-[A-Za-z0-9_-]{20,}\b/,
+        keywords: ['sk-ant-'],
+        allowlists: [],
+    },
+    {
+        id: 'openrouter-api-key',
+        pattern: /\bsk-or-[A-Za-z0-9_-]{20,}\b/,
+        keywords: ['sk-or-'],
+        allowlists: [],
+    },
+    {
+        id: 'github-fine-grained-pat-any',
+        pattern: /\bgithub_pat_[A-Za-z0-9_]{20,}\b/,
+        keywords: ['github_pat_'],
+        allowlists: [],
+    },
+    {
+        id: 'slack-token-any',
+        pattern: /\bxox[baprs]-[A-Za-z0-9-]{10,}\b/,
+        keywords: ['xox'],
+        allowlists: [],
+    },
+    {
+        id: 'private-key-any',
+        pattern: /-----BEGIN (?:(?:RSA|DSA|EC|OPENSSH|ENCRYPTED|PGP) )?PRIVATE KEY(?: BLOCK)?-----[\s\S]{0,16384}?-----END (?:(?:RSA|DSA|EC|OPENSSH|ENCRYPTED|PGP) )?PRIVATE KEY(?: BLOCK)?-----/,
+        keywords: ['-----begin'],
+        allowlists: [],
+    },
+    {
+        id: 'bearer-token',
+        pattern: /\bBearer[ \t]+([A-Za-z0-9._~+/-]{16,}=*)/i,
+        entropy: 3,
+        keywords: ['bearer'],
+        allowlists: [],
+    },
+];
+/** Every credential rule: gitleaks', then Theorem's own. */
+const CREDENTIALS = [...CREDENTIAL_RULES, ...OWN_CREDENTIAL_RULES];
 const CARD_CANDIDATE = /\b(?:\d[\s.-]*?){13,19}\b/g;
-/**
- * The families of sensitive data, each switched on its own: a profile can
- * redact credentials and card numbers but leave network addresses alone.
- */
-const SENSITIVE_GROUPS = ['ids', 'financial', 'network', 'credentials'];
-/** The patterns each group matches. Card numbers are financial too, found by `cardSpans`. */
+/** The patterns each group matches. Card numbers are financial too, found by `cardSpans`; credentials by `credentialSpans`. */
 const GROUP_PATTERNS = {
     ids: [SSN, SSN_CONTEXTUAL, ITIN, EIN],
     financial: [IBAN],
     network: [IPV4, IPV6],
-    credentials: [
-        AWS_ACCESS,
-        GOOGLE_API,
-        OPENAI_KEY,
-        ANTHROPIC_KEY,
-        OPENROUTER_KEY,
-        GITHUB_PAT,
-        GITHUB_TOKEN,
-        SLACK_TOKEN,
-        BEARER,
-        PEM_KEY,
-    ],
 };
 /** Every pattern, with its group. */
-const SENSITIVE_PATTERNS = SENSITIVE_GROUPS.flatMap((group) => GROUP_PATTERNS[group].map((pattern) => ({ group, pattern })));
+const SENSITIVE_PATTERNS = [
+    ...['ids', 'financial', 'network'].flatMap((group) => GROUP_PATTERNS[group].map((pattern) => ({ group, pattern }))),
+    ...CREDENTIALS.map((rule) => ({
+        group: 'credentials',
+        pattern: new RegExp(rule.pattern.source, `${rule.pattern.flags}g`),
+        hit: (match, text, at) => credentialHit(rule, GLOBAL_ALLOWLIST, match, text, at),
+    })),
+];
 const ALL_GROUPS = {
     ids: true,
     financial: true,
@@ -58,15 +99,9 @@ function resolveSensitive(selection, defaults = ALL_GROUPS) {
     }
     return { ...defaults, ...selection };
 }
-/** Whether any group runs. */
-function anySensitive(groups) {
-    return SENSITIVE_GROUPS.some((group) => groups[group]);
-}
 const LUHN_DOUBLE = 2;
 const LUHN_NINE = 9;
 const LUHN_TEN = 10;
-const CARD_MIN_DIGITS = 13;
-const CARD_MAX_DIGITS = 19;
 function luhnOk(digits) {
     let sum = 0;
     let doubleIt = false;
@@ -87,17 +122,100 @@ function luhnOk(digits) {
     }
     return sum % LUHN_TEN === 0;
 }
-/** A card-number candidate that is a card: 13–19 digits passing the Luhn check. */
-function cardHit(blob) {
+const LENGTHS_16_TO_19 = [16, 17, 18, 19];
+/** Visa, Mastercard, American Express, Diners Club, Discover, JCB, UnionPay, Maestro, Mir, and RuPay, in that order. */
+const CARD_NETWORKS = [
+    { leading: [['4', '4']], lengths: [13, 16, 19] },
+    {
+        leading: [
+            ['51', '55'],
+            ['2221', '2720'],
+        ],
+        lengths: [16],
+    },
+    {
+        leading: [
+            ['34', '34'],
+            ['37', '37'],
+        ],
+        lengths: [15],
+    },
+    {
+        leading: [
+            ['300', '305'],
+            ['3095', '3095'],
+            ['36', '36'],
+            ['38', '39'],
+        ],
+        lengths: [14, 15, ...LENGTHS_16_TO_19],
+    },
+    {
+        leading: [
+            ['6011', '6011'],
+            ['644', '649'],
+            ['65', '65'],
+        ],
+        lengths: LENGTHS_16_TO_19,
+    },
+    { leading: [['3528', '3589']], lengths: LENGTHS_16_TO_19 },
+    { leading: [['62', '62']], lengths: LENGTHS_16_TO_19 },
+    {
+        leading: [
+            ['5018', '5018'],
+            ['5020', '5020'],
+            ['5038', '5038'],
+            ['5893', '5893'],
+            ['6304', '6304'],
+            ['6759', '6759'],
+            ['6761', '6763'],
+        ],
+        lengths: [13, 14, 15, ...LENGTHS_16_TO_19],
+    },
+    { leading: [['2200', '2204']], lengths: LENGTHS_16_TO_19 },
+    {
+        leading: [
+            ['60', '60'],
+            ['81', '82'],
+            ['508', '508'],
+            ['353', '353'],
+            ['356', '356'],
+        ],
+        lengths: [16],
+    },
+];
+/** Whether some card network issues numbers that start and run as long as `digits`. */
+function isIssued(digits) {
+    return CARD_NETWORKS.some((network) => network.lengths.includes(digits.length) &&
+        network.leading.some(([low, high]) => {
+            const start = digits.slice(0, low.length);
+            return start >= low && start <= high;
+        }));
+}
+const URL_SCHEME = /https?:\/\//i;
+const URL_LOOKBACK = 2048;
+/** Whether `at` of `text` is inside a web address: the unbroken run of characters before it holds `http://` or `https://`. */
+function isInUrl(text, at) {
+    const floor = Math.max(0, at - URL_LOOKBACK);
+    let from = at;
+    while (from > floor && !/\s/.test(text[from - 1] ?? ' '))
+        from -= 1;
+    return URL_SCHEME.test(text.slice(from, at));
+}
+/**
+ * A card-number candidate at `at` of `text` that is a card: digits a card
+ * network issues, passing the Luhn check, outside a web address. A run of
+ * digits in an address is the address's own identifier: one random 19-digit
+ * identifier in a hundred passes the first two tests.
+ */
+function cardHit(blob, text, at) {
     const digits = blob.replaceAll(/[^\d]/g, '');
-    const inRange = digits.length >= CARD_MIN_DIGITS && digits.length <= CARD_MAX_DIGITS;
-    return inRange && luhnOk(digits);
+    return isIssued(digits) && luhnOk(digits) && !isInUrl(text, at);
 }
 function cardSpans(text) {
     const spans = [];
     for (const match of text.matchAll(CARD_CANDIDATE)) {
         const found = blobAt(match);
-        if (found && cardHit(found.blob)) {
+        if (found && cardHit(found.blob, text, found.index)) {
             spans.push({ start: found.index, end: found.index + found.blob.length, kind: 'sensitive' });
         }
     }
@@ -106,8 +224,11 @@ function cardSpans(text) {
 /** Sensitive-data spans in `text`, from the groups `selection` runs (every group by default). */
 function sensitiveSpans(text, selection = true) {
     const groups = resolveSensitive(selection);
-    const patterns = SENSITIVE_GROUPS.flatMap((group) => groups[group] ? GROUP_PATTERNS[group] : []);
-    const spans = spansFromPatterns(text, patterns, 'sensitive');
-    return groups.financial ? [...spans, ...cardSpans(text)] : spans;
+    const patterns = ['ids', 'financial', 'network'].flatMap((group) => groups[group] ? GROUP_PATTERNS[group] : []);
+    return [
+        ...spansFromPatterns(text, patterns, 'sensitive'),
+        ...(groups.financial ? cardSpans(text) : []),
+        ...(groups.credentials ? credentialSpans(text, CREDENTIALS, GLOBAL_ALLOWLIST) : []),
+    ];
 }
-export { anySensitive, CARD_CANDIDATE, cardHit, resolveSensitive, SENSITIVE_GROUPS, SENSITIVE_PATTERNS, sensitiveSpans, };
+export { CARD_CANDIDATE, cardHit, SENSITIVE_PATTERNS, sensitiveSpans };

package/esm/src/guardrails/serialize.d.ts

@@ -1,4 +1,5 @@
 declare const CIRCULAR = "[circular]";
+/** A value rendered as text for a guardrail scan, and whether it could not be rendered. */
 export interface ScanText {
     text: string;
     /** True when the payload could not be rendered and was not inspected. */

package/esm/src/guardrails/testing.d.ts

@@ -1,9 +1,11 @@
 /**
- * Adversarial corpus, fuzz runners and live attack builders, kept off the production guardrails entry.
+ * Adversarial corpus, fuzz runners, and live attack builders, kept off the production guardrails entry.
  *
  * @module
  */
-/** lexicon-exempt-file: adversarial harness helpers — not runtime user or model copy (P2) */
+/** labels-exempt-file: adversarial harness helpers — not runtime user or model copy (P2) */
 import "../../_dnt.polyfills.js";
 export type { CanaryEgressAttack, CanaryEgressCatalogEntry, InboundFuzzPayload, InboundFuzzResult, LiveAttack, } from './corpus/mod.js';
 export { buildCanaryEgressAttacks, buildLiveAttacks, canaryEgressCatalog, FIXED_CANARY, FUZZ_SYSTEM, filterLiveAttacks, inboundFuzzPayloads, inboundPayloadByName, runInboundGuardrailFuzz, summarizeAttackBank, } from './corpus/mod.js';
+export { TEST_AWS_KEY, TEST_GITHUB_PAT, TEST_GOOGLE_KEY, TEST_OPENAI_KEY, TEST_PEM, TEST_SLACK, TEST_VISA, } from './corpus/secrets.js';
+export { INJ_IGNORE, INJ_IGNORE_REVEAL, injHexSpaced } from './corpus/strings.js';

package/esm/src/guardrails/testing.js

@@ -1,8 +1,10 @@
 /**
- * Adversarial corpus, fuzz runners and live attack builders, kept off the production guardrails entry.
+ * Adversarial corpus, fuzz runners, and live attack builders, kept off the production guardrails entry.
  *
  * @module
  */
-/** lexicon-exempt-file: adversarial harness helpers — not runtime user or model copy (P2) */
+/** labels-exempt-file: adversarial harness helpers — not runtime user or model copy (P2) */
 import "../../_dnt.polyfills.js";
 export { buildCanaryEgressAttacks, buildLiveAttacks, canaryEgressCatalog, FIXED_CANARY, FUZZ_SYSTEM, filterLiveAttacks, inboundFuzzPayloads, inboundPayloadByName, runInboundGuardrailFuzz, summarizeAttackBank, } from './corpus/mod.js';
+export { TEST_AWS_KEY, TEST_GITHUB_PAT, TEST_GOOGLE_KEY, TEST_OPENAI_KEY, TEST_PEM, TEST_SLACK, TEST_VISA, } from './corpus/secrets.js';
+export { INJ_IGNORE, INJ_IGNORE_REVEAL, injHexSpaced } from './corpus/strings.js';

package/esm/src/guardrails/theorem-error.d.ts

@@ -3,14 +3,17 @@ export type { ErrorCopies, ErrorCopy };
 /**
  * What kind of failure happened, decided where it happens. Both worlds read it:
  * the builder in code and traces (`errorKind`, `error.type`), the user through
- * the kind's wording (`error.<kind>` in the lexicon).
+ * the kind's wording (`error.<kind>` in the labels).
  */
-export declare const ERROR_KINDS: readonly ["config", "request", "input", "action", "auth", "rate_limit", "unsupported", "unavailable", "bad_response", "network", "timeout", "safety", "blocked", "declined", "failed", "cancelled", "internal"];
+export declare const ERROR_KINDS: readonly ["config", "request", "input", "action", "auth", "rate_limit", "busy", "unsupported", "unavailable", "bad_response", "network", "timeout", "safety", "blocked", "declined", "failed", "cancelled", "internal"];
+/** Why an operation failed; the user's wording and the HTTP status follow from it. */
 export type ErrorKind = (typeof ERROR_KINDS)[number];
+/** Options for a `TheoremError`: the cause and, optionally, wording for the user. */
 export interface TheoremErrorOptions extends ErrorOptions {
     /** User wording more specific than the kind's: one line, or one per problem found. */
     copy?: ErrorCopies;
 }
+/** An error that carries a `kind`, from which hosts choose a status and the user's wording. */
 export declare class TheoremError extends Error {
     readonly kind: ErrorKind;
     readonly copy?: ErrorCopies;

package/esm/src/guardrails/theorem-error.js

@@ -1,8 +1,8 @@
-// Its own module so `lexicon.ts` can throw it without importing `error.ts`, which reads the lexicon.
+// invariant: Its own module so `labels.ts` can throw it without importing `error.ts`, which reads the labels.
 /**
  * What kind of failure happened, decided where it happens. Both worlds read it:
  * the builder in code and traces (`errorKind`, `error.type`), the user through
- * the kind's wording (`error.<kind>` in the lexicon).
+ * the kind's wording (`error.<kind>` in the labels).
  */
 export const ERROR_KINDS = [
     /** The profile, tool, or schema is set up wrong. */
@@ -17,9 +17,11 @@ export const ERROR_KINDS = [
     'auth',
     /** Too many requests, or a quota used up. */
     'rate_limit',
+    /** The provider is up but too busy for the call; another key or a moment later may serve it. */
+    'busy',
     /** The model or route cannot serve this request. */
     'unsupported',
-    /** The provider is down or overloaded. */
+    /** The provider is down. */
     'unavailable',
     /** The provider answered with something that cannot be used. */
     'bad_response',
@@ -40,6 +42,7 @@ export const ERROR_KINDS = [
     /** A THEOREM invariant broke. */
     'internal',
 ];
+/** An error that carries a `kind`, from which hosts choose a status and the user's wording. */
 export class TheoremError extends Error {
     kind;
     copy;

package/esm/src/guardrails/thought-guard.d.ts

@@ -1,22 +1,25 @@
 /**
- * Thought guarding. A host that renders `thought` events shows the user what
+ * Thought guarding, under `guardrails.detect` at `thought`. A host that renders `thought` events shows the user what
  * they say and loads what they link, as it does a reply: the canary, the
  * system prompt and the user-data markers leak through a thought, and a URL
  * the model was not given carries data out. Nothing in a thought stops a turn
  * (`isGuardedOutput`); each leak is omitted and the rest of the thought
- * released.
+ * released, or under `block` omitted with the rest of the thought.
  *
  * @module
  */
-import { type ResolvedEgressChecks } from './egress.js';
+import { type Detection } from './detect-at.js';
+import type { UrlAllow } from './detectors.js';
 import { type GivenUrls } from './egress-urls.js';
-import { type LexiconOverrides } from './lexicon.js';
-import type { EgressEnforcer, GuardrailContext, GuardrailHit } from './types.js';
+import { type LabelOverrides } from './labels.js';
+import type { GuardrailContext, GuardrailHit, ResolvedGuardrailPolicy } from './types.js';
 /** What a thought guard lets through: the text to show, and what it omitted from it. */
 interface ThoughtRelease {
     text: string;
     /** One hit per rule that omitted something; empty when nothing was. */
     hits: GuardrailHit[];
+    /** What `guardrails.detect` found in the thought as it was read at `thought`. */
+    found?: Pick<Detection, 'action' | 'hits'>;
 }
 /** One provider call's (or Live cycle's) thought text, released as it clears. */
 interface ThoughtGuard {
@@ -27,14 +30,29 @@ interface ThoughtGuard {
     /** The tail of what was shown that a leak in the next call's thoughts could continue. */
     carryOut(): string;
 }
+/** What a thought loses besides the canary and the system instruction. */
+interface ThoughtOmits {
+    /** The kernel's markers and the words of the canary note (`marker_leak`). */
+    markers?: boolean;
+    /** Images to a URL the model was not given, but for what this lets through (`ungiven_images`). */
+    images?: UrlAllow;
+    /** Links to a URL the model was not given, but for what this lets through (`ungiven_links`). */
+    links?: UrlAllow;
+    /** Whether `ungiven_images` reads the thought, here or in front: a link then leaves an image's markup to it. */
+    imagesRead?: boolean;
+}
 interface ThoughtGuardOptions {
-    /** The URL and boundary checks thoughts run; the rest of a policy is for replies. */
-    checks?: ResolvedEgressChecks;
+    omit?: ThoughtOmits;
+    /** The canary to omit from a thought. */
     canary?: string;
-    /** Guarded against echo alongside the canary. */
+    /** The private stretches of the system instruction, whose words are omitted from a thought. */
     privateSystem?: readonly string[];
+    /** The canary the turn planted, when a thought is not read for it: the echo and the note still know its place. */
+    planted?: string;
+    /** Rules whose leak ends what is shown of the thought, where the others are omitted alone. */
+    ends?: ReadonlySet<string>;
     given?: GivenUrls;
-    lexicon?: LexiconOverrides;
+    labels?: LabelOverrides;
     /** Shown text an earlier thought ended on (`carryOut`), read in front and never shown again. */
     carry?: string;
 }
@@ -47,9 +65,12 @@ interface ThoughtGuardOptions {
  */
 declare function createThoughtGuard(options: ThoughtGuardOptions): ThoughtGuard;
 /**
- * A guard for a turn's thoughts: the canary and prompt echo whenever the turn
- * binds a canary, and the URL and boundary checks `enforce` runs.
+ * A guard for a turn's thoughts: the detectors `guardrails.detect` sets at
+ * `thought`. `context` carries the canary and the system instruction only
+ * while their detectors read somewhere (`leakScopeOf`); at `thought`, `redact`
+ * omits a match, `block` ends what is shown of the thought there, and `flag`
+ * reports it and shows it.
  */
-declare function thoughtGuardFor(enforce: EgressEnforcer | undefined, context: GuardrailContext, carry?: string): ThoughtGuard | undefined;
-export type { ThoughtGuard, ThoughtGuardOptions, ThoughtRelease };
+declare function thoughtGuardFor(policy: Pick<ResolvedGuardrailPolicy, 'detect' | 'allow'>, context: GuardrailContext, carry?: string): Promise<ThoughtGuard | undefined>;
+export type { ThoughtGuard, ThoughtGuardOptions, ThoughtOmits, ThoughtRelease };
 export { createThoughtGuard, thoughtGuardFor };