@event4u/agent-config 16.5.0 → 17.0.0

package/src/scripts/_lib/egress_rules_linearity.ts

@@ -0,0 +1,56 @@
+/**
+ * Why each rule of the egress table runs in linear time on any input — the
+ * argument the PR #2366 council review required per rule, reviewed with the
+ * pattern it explains.
+ *
+ * Kept beside the table rather than in it because no runtime path reads it:
+ * the table ships inside the egress engine chunk, and these sentences are
+ * prose for a reviewer. `egress_rules.test.ts` refuses a row without an entry
+ * and an entry without a row, so the contract is as binding as the field was.
+ */
+const PREFIX_RUN =
+    'Starts only where the literal prefix occurs; the single class after it either reaches its minimum and consumes the whole run (the global scan resumes past it) or fails within the minimum, so each character is read a bounded number of times.';
+const FIXED_RUN =
+    'Fixed-length class after a literal prefix: every attempt reads at most the prefix plus the fixed count.';
+
+export const LINEARITY: Readonly<Record<string, string>> = {
+    'anthropic-api-key': PREFIX_RUN,
+    'anthropic-project-key': PREFIX_RUN,
+    'aws-access-key': FIXED_RUN,
+    'google-api-key': FIXED_RUN,
+    'github-pat': FIXED_RUN,
+    'github-fine-grained-pat': PREFIX_RUN,
+    'openai-api-key': PREFIX_RUN,
+    'google-oauth-token': PREFIX_RUN,
+    'github-app-token': PREFIX_RUN,
+    'xai-api-key': PREFIX_RUN,
+    'perplexity-api-key': PREFIX_RUN,
+    'groq-api-key': PREFIX_RUN,
+    'huggingface-token': PREFIX_RUN,
+    'nvidia-api-key': PREFIX_RUN,
+    'replicate-api-token': PREFIX_RUN,
+    'jwt':
+        'One attempt per base64url run: the lookbehind admits a start only at the run boundary, and the lookahead-plus-backreference consumes the run up to its first eyJ atomically, so the engine never retries a later eyJ in the same run. That loses nothing: every eyJ in one run reaches the same run end, so the first one, which has the most characters after it, matches whenever any later one would. An attempt spans at most three dot-separated runs, so each character is read a bounded number of times.',
+    'stripe-secret-key': PREFIX_RUN,
+    'slack-token': PREFIX_RUN,
+    'pem-private-key':
+        'Starts only at the literal header; the label class excludes the dash, so label runs of different headers are disjoint, and backtracking over one run tries the fixed 16-character suffix at each position: linear with a constant factor of 16.',
+    'db-connection-uri':
+        'One attempt per scheme-character run: the lookbehind admits a start only at the run boundary, and the lookahead-plus-backreference consumes the run up to its first word-boundary letter atomically. Every such start in one run reaches the same :// at the same run end, so the first one matches whenever any later one would, and nothing the unbounded scheme matched is lost. The userinfo classes exclude : and /, so each userinfo run is read by one attempt per side.',
+    'iban':
+        'Starts only at a country-code boundary; every quantifier is bounded (at most seven four-character groups and a three-character tail), so an attempt reads at most 37 characters.',
+    'payment-card':
+        'Starts only at a digit boundary; each repetition reads one digit after an optional single separator, so a digit can be the end of only one repetition and an attempt reads at most 37 characters.',
+    'email':
+        'Starts only where no address character precedes; the local part is bounded at 64, each label at 63 and the labels at eight, and the label class must end on a dot, so an attempt reads a bounded window and backtracking inside it is bounded by those constants.',
+    'phone-international':
+        'Starts only at a plus sign; each digit group is either a parenthesised run of at most six digits or consumed atomically by a lookahead-plus-backreference of at most six, so groups split only at separators or the six-digit cap, and at most five groups follow: an attempt reads at most about 45 characters with no re-split.',
+    'phone-national':
+        'Starts only at a zero after a non-word character; the area code, bare or parenthesised, is bounded at five digits and each of at most three groups is consumed atomically by a lookahead-plus-backreference of at most eight digits, so an attempt reads at most about 35 characters with no re-split.',
+    'phone-nanp': 'Fixed shape: an optional country prefix of at most three characters, a three-digit area code bare or parenthesised, then three and four digits with optional single separators; every quantifier is bounded, so an attempt reads at most twenty characters.',
+    'de-tax-id':
+        'Starts only at a digit boundary; exactly ten repetitions of an optional space and one digit follow, so an attempt reads at most 21 characters.',
+    'us-ssn': 'Fixed-length digit runs between hyphens: every attempt reads at most eleven characters.',
+    'generic-assignment':
+        'Starts only at a keyword; the whitespace runs end at the first non-space and the value run at the first quote or space, and a match moves the global scan past the value, so no character is read by more than one successful attempt and a failed attempt stops at the first character after the whitespace.',
+};

package/src/scripts/_lib/egress_sender_scan.ts

@@ -0,0 +1,290 @@
+/**
+ * Sender census — every `.ts` and `.sh` module under {@link SCAN_ROOTS} that
+ * posts to a model-provider endpoint, derived from the tree rather than listed
+ * by hand. A `.js`/`.mjs` sender, or one under another root (skill scripts,
+ * `src/server`, templates, `internal/bench`), is not read, and the rendered
+ * table says so
+ * (road-to-one-detector-and-the-senders-we-own, step 0.4; contract
+ * docs/contracts/egress-placeholders-v1.md § This package's own senders).
+ *
+ * A row is a claim about source text, not about what leaves at runtime. Its
+ * worst failure is an absent row, so anything the scanner cannot settle is a
+ * row marked `unclassified` instead of silence: a provider endpoint named with
+ * no recognised send shape, a send shape aimed at a host on neither list, and a
+ * media adapter whose endpoint is configured rather than written down. The
+ * residual — a module whose endpoint is computed and whose send shape matches
+ * nothing below — is stated in the rendered table's own header.
+ */
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+
+export type SenderStatus = 'sender' | 'local' | 'unclassified';
+
+/** `undeclared` until the module carries an `egress-content:` marker. */
+export type SenderContent = 'session-content' | 'fixed-corpus' | 'undeclared';
+
+/**
+ * What the contract may publish for the module (step 3.4): `asserted` — its
+ * requests pass the replacement and the final check; `fixed-corpus` — it sends
+ * a corpus this package fixed, never session content; `uncovered` — it may send
+ * session content unchecked, for the stated reason. `undeclared` is no answer
+ * and fails the census.
+ */
+export type SenderCoverage = 'asserted' | 'fixed-corpus' | 'uncovered' | 'undeclared';
+
+export interface SenderRow {
+    /** Repo-relative, POSIX separators. */
+    readonly module: string;
+    /** Provider or other hosts the module names, sorted; `configured` when it names none. */
+    readonly endpoints: readonly string[];
+    readonly content: SenderContent;
+    /** True when the module imports the egress engine directly (one hop). */
+    readonly engine: boolean;
+    readonly status: SenderStatus;
+    readonly evidence: string;
+    readonly coverage: SenderCoverage;
+    /** Why the module has that coverage, as its marker states it; for `undeclared`, what is missing. */
+    readonly reason: string;
+}
+
+export const SCAN_ROOTS: readonly string[] = ['src/scripts', 'src/cli', 'src/install', 'src/shared'];
+
+/** Hosts that answer as a model or media-generation provider. */
+export const PROVIDER_HOSTS: readonly string[] = [
+    'api.anthropic.com',
+    'api.openai.com',
+    'generativelanguage.googleapis.com',
+    'api.x.ai',
+    'api.perplexity.ai',
+    'api.mistral.ai',
+    'openrouter.ai',
+    'api.deepseek.com',
+    'api.groq.com',
+    'api.together.xyz',
+    'api.cohere.com',
+    'api.replicate.com',
+    'queue.fal.run',
+    'fal.run',
+    'api.ideogram.ai',
+    'external.api.recraft.ai',
+    'api.bfl.ai',
+    'api.bfl.ml',
+    'platform.higgsfield.ai',
+    'api.klingai.com',
+    'api.sync.so',
+    'api.stability.ai',
+    'api.elevenlabs.io',
+    'api.lumalabs.ai',
+    'api.runwayml.com',
+];
+
+/** Hosts this package talks to that are not model providers; a send to one makes no row. */
+export const NON_PROVIDER_HOSTS: readonly string[] = [
+    'github.com',
+    'api.github.com',
+    'cli.github.com',
+    'codeload.github.com',
+    'objects.githubusercontent.com',
+    'raw.githubusercontent.com',
+    'registry.npmjs.org',
+    'www.npmjs.com',
+    'old.reddit.com',
+    'www.reddit.com',
+    'example.com',
+    'example.org',
+];
+
+const LOOPBACK_HOSTS: ReadonlySet<string> = new Set(['localhost', '127.0.0.1']);
+
+const SEND_SHAPES: readonly RegExp[] = [
+    /\bawait\s+fetch\s*\(/,
+    /\bawait\s+(?:this|self|[a-z_$][\w$]*)\s*\.\s*_?fetch\s*\(/i,
+    /\bhttps?\s*\.\s*request\s*\(/,
+    /\b(?:spawnSync|spawn|execFileSync|execFile|execSync)\s*\(\s*\[?\s*['"`]curl['"`]/,
+    /^\s*[^#\n]*\bcurl\s+-/m,
+];
+
+const URL_HOST = /https?:\/\/([a-z0-9-]+(?:\.[a-z0-9-]+)+|localhost)(?=[:/'"`\s)]|$)/gi;
+// `egress-content: fixed-corpus — <reason>`, or `egress-content: session-content
+// asserted|uncovered — <reason>`. The reason is the claim a reviewer checks.
+const CONTENT_MARKER =
+    /(?:\/\/|#)\s*egress-content:\s*(session-content|fixed-corpus)\b(?:[ \t]+(asserted|uncovered)\b)?(?:[ \t]*(?:—|--)[ \t]*(\S[^\n]*))?/;
+// `asserted` is a claim about the final check, so the module must reach it.
+const FINAL_CHECK_IMPORT = /import\s*\{[^}]*\bcouncilEgress\b[^}]*\}\s*from\s+['"][^'"]*council_egress(?:\.js)?['"]/;
+/** The check actually called, on a code line: an import alone, or the call named in a comment, is not it. */
+const FINAL_CHECK_CALL = /\bcouncilEgress\(\)\s*\.\s*check\s*\(/;
+
+function reachesFinalCheck(src: string): boolean {
+    return (
+        FINAL_CHECK_IMPORT.test(src) &&
+        src.split('\n').some((line) => !/^\s*(?:\/\/|\/?\*)/.test(line) && FINAL_CHECK_CALL.test(line))
+    );
+}
+// Passing the engine means importing a function that detects or replaces; a
+// module that reads only a table constant (a vendor key prefix) does not.
+const ENGINE_IMPORT =
+    /import\s*\{[^}]*\b(?:detect|findSpans|replaceSpans|replaceValues)\b[^}]*\}\s*from\s+['"][^'"]*egress_(?:token|rules)(?:\.js)?['"]/;
+const ADAPTER_DIR = /^src\/scripts\/ai-(?:image|video)\/adapters\/[^/]+\.sh$/;
+
+/** The scanner and its CLI quote every host above; a row for them would be the scanner reading itself. */
+const SELF_MODULES: ReadonlySet<string> = new Set([
+    'src/scripts/_lib/egress_sender_scan.ts',
+    'src/scripts/egress_sender_census.ts',
+]);
+
+const EXCLUDE = /(?:^|\/)(?:__tests__|node_modules)\/|\.test\.ts$|\.d\.ts$/;
+
+function escapeHost(h: string): string {
+    return h.replace(/[.]/g, '\\.');
+}
+
+const PROVIDER_RE = new RegExp(
+    `(?<![a-z0-9.-])(${[...PROVIDER_HOSTS].sort((a, b) => b.length - a.length).map(escapeHost).join('|')})(?![a-z0-9-])`,
+    'gi',
+);
+
+export class SenderScanError extends Error {
+    constructor(readonly dir: string, cause: unknown) {
+        super(`egress_sender_scan: cannot read ${dir} — the census is incomplete, not clean (${String(cause)})`);
+        this.name = 'SenderScanError';
+    }
+}
+
+function walk(abs: string, repoRoot: string, out: string[]): void {
+    let entries: fs.Dirent[];
+    try {
+        entries = fs.readdirSync(abs, { withFileTypes: true });
+    } catch (exc) {
+        throw new SenderScanError(abs, exc);
+    }
+    for (const e of entries) {
+        const child = path.join(abs, e.name);
+        const rel = path.relative(repoRoot, child).split(path.sep).join('/');
+        if (e.isDirectory()) {
+            if (!EXCLUDE.test(`${rel}/`)) walk(child, repoRoot, out);
+        } else if (e.isFile() && /\.(?:ts|sh)$/.test(rel) && !EXCLUDE.test(rel)) {
+            out.push(rel);
+        }
+    }
+}
+
+function coverageOf(
+    marker: RegExpExecArray | null,
+    reachesFinalCheck: boolean,
+): { coverage: SenderCoverage; reason: string } {
+    if (marker === null) return { coverage: 'undeclared', reason: 'no `egress-content:` marker' };
+    const reason = (marker[3] ?? '').trim();
+    if (reason === '') return { coverage: 'undeclared', reason: 'the marker states no reason' };
+    if (marker[1] === 'fixed-corpus') {
+        return marker[2] === undefined
+            ? { coverage: 'fixed-corpus', reason }
+            : { coverage: 'undeclared', reason: 'a fixed corpus takes no coverage word' };
+    }
+    if (marker[2] === 'uncovered') return { coverage: 'uncovered', reason };
+    if (marker[2] === 'asserted') {
+        return reachesFinalCheck
+            ? { coverage: 'asserted', reason }
+            : { coverage: 'undeclared', reason: 'claims `asserted` but does not import and call the final check' };
+    }
+    return { coverage: 'undeclared', reason: 'session content without `asserted` or `uncovered`' };
+}
+
+export function classifyModule(rel: string, src: string): SenderRow | null {
+    if (SELF_MODULES.has(rel)) return null;
+    const providers = new Set<string>();
+    for (const m of src.matchAll(PROVIDER_RE)) providers.add((m[1] as string).toLowerCase());
+    const unknown = new Set<string>();
+    const loopback = new Set<string>();
+    for (const m of src.matchAll(URL_HOST)) {
+        const host = (m[1] as string).toLowerCase();
+        if (providers.has(host) || PROVIDER_HOSTS.includes(host)) continue;
+        if (LOOPBACK_HOSTS.has(host)) loopback.add(host);
+        else if (!NON_PROVIDER_HOSTS.includes(host)) unknown.add(host);
+    }
+    const adapter = ADAPTER_DIR.test(rel);
+    const sends = adapter || SEND_SHAPES.some((re) => re.test(src));
+    const marker = CONTENT_MARKER.exec(src);
+    const content = (marker?.[1] as SenderContent | undefined) ?? 'undeclared';
+    const engine = ENGINE_IMPORT.test(src);
+    const { coverage, reason } = coverageOf(marker, reachesFinalCheck(src));
+    const row = (endpoints: Set<string>, status: SenderStatus, evidence: string): SenderRow => ({
+        module: rel,
+        endpoints: endpoints.size === 0 ? ['configured'] : [...endpoints].sort(),
+        content,
+        engine,
+        status,
+        evidence,
+        coverage,
+        reason,
+    });
+
+    if (providers.size > 0) {
+        return sends
+            ? row(providers, 'sender', adapter ? 'media adapter naming a provider endpoint' : 'names a provider endpoint and carries a send shape')
+            : row(providers, 'unclassified', 'names a provider endpoint with no recognised send shape — read the module');
+    }
+    if (sends && unknown.size > 0) {
+        return row(unknown, 'unclassified', 'send shape aimed at a host on neither list — read the module');
+    }
+    if (adapter) {
+        return loopback.size > 0
+            ? row(loopback, 'local', 'media adapter whose only endpoint is loopback')
+            : row(new Set(), 'unclassified', 'media adapter whose endpoint is configured, not written down');
+    }
+    return null;
+}
+
+export function scanSenders(repoRoot: string): SenderRow[] {
+    const files: string[] = [];
+    for (const root of SCAN_ROOTS) {
+        const abs = path.join(repoRoot, root);
+        if (fs.existsSync(abs)) walk(abs, repoRoot, files);
+    }
+    files.sort();
+    const rows: SenderRow[] = [];
+    for (const rel of files) {
+        const r = classifyModule(rel, fs.readFileSync(path.join(repoRoot, rel), 'utf-8'));
+        if (r !== null) rows.push(r);
+    }
+    return rows;
+}
+
+/** Rows the census refuses: a sender with no answer. */
+export function undeclared(rows: readonly SenderRow[]): SenderRow[] {
+    return rows.filter((r) => r.coverage === 'undeclared');
+}
+
+export function tally(rows: readonly SenderRow[]): Record<SenderStatus, number> {
+    const out: Record<SenderStatus, number> = { sender: 0, local: 0, unclassified: 0 };
+    for (const r of rows) out[r.status] += 1;
+    return out;
+}
+
+export function renderTable(rows: readonly SenderRow[]): string {
+    const lines = [
+        '| module | endpoint | content | engine | status | evidence | coverage | reason |',
+        '|---|---|---|---|---|---|---|---|',
+    ];
+    for (const r of rows) {
+        const endpoints = r.endpoints.map((e) => `\`${e}\``).join(', ');
+        lines.push(
+            `| \`${r.module}\` | ${endpoints} | ${r.content} | ${r.engine ? 'yes' : 'no'} | **${r.status}** | ${r.evidence} | ` +
+                `**${r.coverage}** | ${r.reason.replace(/\|/g, '\\|')} |`,
+        );
+    }
+    const t = tally(rows);
+    lines.push('');
+    const c = { asserted: 0, 'fixed-corpus': 0, uncovered: 0, undeclared: 0 };
+    for (const r of rows) c[r.coverage] += 1;
+    lines.push(
+        `${String(rows.length)} module(s): ${String(t.sender)} sender, ${String(t.local)} local, ` +
+            `${String(t.unclassified)} unclassified; coverage ${String(c.asserted)} asserted, ` +
+            `${String(c['fixed-corpus'])} fixed-corpus, ${String(c.uncovered)} uncovered, ` +
+            `${String(c.undeclared)} undeclared. ` +
+            `Scope: \`.ts\` and \`.sh\` under ${SCAN_ROOTS.map((r) => `\`${r}\``).join(', ')} only; ` +
+            '`asserted` means the module imports and calls the final check, not that every request path reaches it. ' +
+            // check_enforcement_denominator reads `N undeclared` beside the word as a rule-enforcement figure.
+            '<!-- enforcement-count-ok: egress coverage census of sender modules, not a rule-enforcement figure -->',
+    );
+    return lines.join('\n');
+}