@event4u/agent-config 16.4.0 → 17.0.0

package/src/scripts/_lib/workspace_secrets.ts

@@ -0,0 +1,273 @@
+/**
+ * Shared secret-detection + scrub primitives for the workspace stores.
+ *
+ * Lives under `src/scripts/_lib/` because a shipped module
+ * (`cross_repo_retrieve.ts`) reads it and `src/cli/` is not in the package;
+ * `src/cli/python/workspace_secrets.ts` re-exports it for the workspace stores.
+ * The contract — the exported names, the per-call match counts, `scrub_obj`'s
+ * depth and cycle guard and the `[SECRET]` placeholder — is pinned by
+ * `tests/scripts/cli/python/workspace_secrets.test.ts`, and what it detects by
+ * `egress_legacy_superset.test.ts`.
+ *
+ * Near-leaf module — its one import is the pure rule table (no I/O) — so the
+ * hot-path analytics `emit()` and the heavier `knowledge_ingest` can both
+ * depend on it cheaply without dragging in PII redaction, chunking, or file
+ * I/O. Every workspace store that persists text or arbitrary payloads at rest
+ * routes through this module before writing (Phase 8 Step 5 secret-hygiene
+ * sweep).
+ *
+ * Two confidence tiers:
+ *
+ * - **HIGH** — structurally unambiguous credentials: the credential rows of
+ *   the one rule table (vendor keys, tokens, PEM private-key block, a
+ *   connection URI with an inline password). Near-zero false-positive rate, so
+ *   safe to scrub destructively or to refuse a write over.
+ * - **FUZZY** — the generic `key/secret/token/password = <value>` assignment.
+ *   Fires on legitimate prose ("password reset token: see attached"), so it is
+ *   warn-only on user-authored content and only scrubbed on disposable,
+ *   machine-generated telemetry (sessions / analytics).
+ *
+ * The `[SECRET]` placeholder matches the knowledge-ingestion redactor so the
+ * two surfaces stay byte-consistent.
+ */
+
+import {
+    assignmentPattern,
+    detect,
+    EGRESS_RULES,
+    type EgressRule,
+    maskingEnd,
+    prefixesOf,
+    rawKeyPrefixPattern,
+} from './egress_rules.js';
+
+export const PLACEHOLDER = '[SECRET]';
+
+// --- HIGH-confidence tier: the credential rows of the one rule table -------
+//
+// Detection lives in `src/scripts/_lib/egress_rules.ts`, the only place a
+// credential pattern is defined (road-to-one-detector-and-the-senders-we-own
+// 2.2); this module keeps its own policy — the `[SECRET]` placeholder, the
+// HIGH/FUZZY tiers and the counts. `generic-assignment` is left to the FUZZY tier
+// below, which is this module's own, looser reading of the same shape.
+
+const HIGH_RULES: readonly EgressRule[] = EGRESS_RULES.filter(
+    (r) => r.class === 'credential' && r.rule !== 'generic-assignment',
+);
+
+// The finding names this module has always reported; a row without one reports
+// its rule id.
+const LEGACY_NAME: Readonly<Record<string, string>> = {
+    'pem-private-key': 'private_key',
+    'aws-access-key': 'aws_access_key',
+    'github-pat': 'github_pat',
+    'github-app-token': 'github_pat',
+    'openai-api-key': 'openai_key',
+};
+const LEGACY_ORDER = ['private_key', 'aws_access_key', 'github_pat', 'openai_key'];
+const rank = (name: string): number => {
+    const i = LEGACY_ORDER.indexOf(name);
+    return i < 0 ? LEGACY_ORDER.length : i;
+};
+
+// This module's own, looser readings of two shapes, kept so the move to the
+// table narrows nothing: the GitHub token family at any length and an OpenAI key
+// glued to a preceding word. The prefixes are the table's.
+const OWN_SHAPES: ReadonlyArray<readonly [string, RegExp]> = [
+    [
+        'github_pat',
+        rawKeyPrefixPattern(36, [...prefixesOf('github-pat'), ...prefixesOf('github-app-token')], {
+            boundary: 'never',
+            tail: 'A-Za-z0-9',
+        }),
+    ],
+    ['openai_key', rawKeyPrefixPattern(20, prefixesOf('openai-api-key'), { boundary: 'never', tail: 'A-Za-z0-9' })],
+];
+
+type Span = { name: string; start: number; end: number };
+
+/** HIGH spans in report order, each widened to the bytes a scrub must remove. */
+function highSpans(text: string): Span[] {
+    const spans: Span[] = detect(text, HIGH_RULES).map((s) => ({
+        name: LEGACY_NAME[s.rule] ?? s.rule,
+        start: s.start,
+        end: maskingEnd(text, s),
+    }));
+    for (const [name, re] of OWN_SHAPES) {
+        for (const m of text.matchAll(re)) spans.push({ name, start: m.index, end: m.index + m[0].length });
+    }
+    // One finding per value: a span inside another span of the same name is that span.
+    const kept = spans.filter(
+        (a, i) =>
+            !spans.some(
+                (b, j) =>
+                    j !== i &&
+                    b.name === a.name &&
+                    b.start <= a.start &&
+                    b.end >= a.end &&
+                    (b.end - b.start > a.end - a.start || j < i),
+            ),
+    );
+    return kept.sort((x, y) => rank(x.name) - rank(y.name) || x.start - y.start);
+}
+
+// --- FUZZY pattern (heuristic key/value assignment) -------------------------
+// Python `(?i)` inline flag → JS `i` flag. Character class is identical.
+// The keyword shape is the table's; the 12-character value is this module's threshold.
+const _RE_KV_SECRET = assignmentPattern(
+    {
+        keywords: ['apiKey', 'secret', 'token', 'password', 'passwd', 'bearer'],
+        value: `['"]?[A-Za-z0-9_\\-+/=]{12,}['"]?`,
+    },
+    'i',
+);
+
+type PatternEntry = readonly [string, RegExp];
+
+export const FUZZY: readonly PatternEntry[] = [['kv_secret', _RE_KV_SECRET]];
+
+// Recursion guard for `scrub_obj` — deep / cyclic structures degrade to a
+// controlled return instead of a RangeError on the never-raises hot path.
+const _MAX_DEPTH = 50;
+
+/** One secret match. Carries the pattern name + tier, never the value. */
+export interface Finding {
+    /** e.g. "aws_access_key" */
+    pattern: string;
+    /** "high" | "fuzzy" */
+    confidence: string;
+}
+
+/**
+ * Build a global-flagged clone of `pat` so `matchAll` / `replace` get
+ * left-to-right non-overlapping iteration matching Python `finditer` / `subn`,
+ * without mutating the shared stateless module-level pattern.
+ */
+function _global(pat: RegExp): RegExp {
+    const flags = pat.flags.includes('g') ? pat.flags : pat.flags + 'g';
+    return new RegExp(pat.source, flags);
+}
+
+/** Count of non-overlapping matches — mirrors `len(list(pat.finditer(text)))`. */
+function _finditerCount(pat: RegExp, text: string): number {
+    let n = 0;
+    for (const _m of text.matchAll(_global(pat))) n += 1;
+    return n;
+}
+
+/** `pat.subn(repl, text)` → `[clean, count]` (non-overlapping, left-to-right). */
+function _subn(pat: RegExp, repl: string, text: string): [string, number] {
+    let count = 0;
+    const clean = text.replace(_global(pat), () => {
+        count += 1;
+        return repl;
+    });
+    return [clean, count];
+}
+
+/**
+ * Non-destructive detection: one {@link Finding} per match.
+ *
+ * Never echoes the matched secret value — only the pattern name and tier,
+ * so a caller can warn or refuse without re-leaking the secret into a log.
+ */
+export function scan(text: unknown, opts?: { include_fuzzy?: boolean }): Finding[] {
+    const include_fuzzy = opts?.include_fuzzy ?? true;
+    if (typeof text !== 'string' || !text) {
+        return [];
+    }
+    const out: Finding[] = highSpans(text).map((s) => ({ pattern: s.name, confidence: 'high' }));
+    if (include_fuzzy) {
+        for (const [name, pat] of FUZZY) {
+            const n = _finditerCount(pat, text);
+            for (let i = 0; i < n; i += 1) out.push({ pattern: name, confidence: 'fuzzy' });
+        }
+    }
+    return out;
+}
+
+/** Replace every secret match with `[SECRET]`; return `[clean, count]`. */
+export function scrub(text: unknown, opts?: { include_fuzzy?: boolean }): [string, number] {
+    const include_fuzzy = opts?.include_fuzzy ?? true;
+    if (typeof text !== 'string' || !text) {
+        return [text as string, 0];
+    }
+    const spans = highSpans(text).sort((x, y) => x.start - y.start);
+    const merged: Array<{ start: number; end: number }> = [];
+    for (const sp of spans) {
+        const last = merged[merged.length - 1];
+        if (last !== undefined && sp.start < last.end) last.end = Math.max(last.end, sp.end);
+        else merged.push({ start: sp.start, end: sp.end });
+    }
+    let out = text;
+    for (let i = merged.length - 1; i >= 0; i -= 1) {
+        const m = merged[i] as { start: number; end: number };
+        out = out.slice(0, m.start) + PLACEHOLDER + out.slice(m.end);
+    }
+    let count = merged.length;
+    if (include_fuzzy) {
+        for (const [, pat] of FUZZY) {
+            const [next, n] = _subn(pat, PLACEHOLDER, out);
+            out = next;
+            count += n;
+        }
+    }
+    return [out, count];
+}
+
+/**
+ * Recursively scrub string leaves in object / array structures.
+ *
+ * Non-string leaves (number, boolean, null) pass through untouched. Recursion
+ * is bounded by a depth cap and a cycle guard so malformed or self-referential
+ * payloads degrade to a controlled return, never a stack overflow — callers on
+ * a never-raises hot path (`workspace_analytics.emit`) rely on this. Returns
+ * `[clean, count]`.
+ *
+ * Python distinguishes `list` (→ list) from `tuple` (→ tuple); JS has only
+ * `Array`, so an array round-trips as an array, matching the Python `list`
+ * branch (the workspace stores never feed a tuple through this path).
+ */
+export function scrub_obj(
+    obj: unknown,
+    opts?: { include_fuzzy?: boolean; _depth?: number; _seen?: Set<unknown> },
+): [unknown, number] {
+    const include_fuzzy = opts?.include_fuzzy ?? true;
+    const _depth = opts?._depth ?? 0;
+    const _seen = opts?._seen ?? new Set<unknown>();
+    if (_depth > _MAX_DEPTH) {
+        return [obj, 0];
+    }
+    if (typeof obj === 'string') {
+        return scrub(obj, { include_fuzzy });
+    }
+    if (Array.isArray(obj)) {
+        if (_seen.has(obj)) {
+            return [obj, 0];
+        }
+        _seen.add(obj);
+        let total = 0;
+        const items: unknown[] = [];
+        for (const value of obj) {
+            const [cv, n] = scrub_obj(value, { include_fuzzy, _depth: _depth + 1, _seen });
+            items.push(cv);
+            total += n;
+        }
+        return [items, total];
+    }
+    if (obj !== null && typeof obj === 'object') {
+        if (_seen.has(obj)) {
+            return [obj, 0];
+        }
+        _seen.add(obj);
+        let total = 0;
+        const clean: Record<string, unknown> = {};
+        for (const [key, value] of Object.entries(obj as Record<string, unknown>)) {
+            const [cv, n] = scrub_obj(value, { include_fuzzy, _depth: _depth + 1, _seen });
+            clean[key] = cv;
+            total += n;
+        }
+        return [clean, total];
+    }
+    return [obj, 0];
+}

package/src/scripts/_lib/worktree_git.ts

@@ -0,0 +1,67 @@
+/**
+ * The git and gh subprocess layer of the worktree cleanup: one deadline for a
+ * budgeted pass, and an environment that can neither refresh an index nor be
+ * redirected to another repository.
+ */
+
+import { execFileSync } from 'node:child_process';
+
+/**
+ * An absolute deadline for every git and gh subprocess while a budgeted pass
+ * runs. A budget checked only between candidates is not a budget: one slow
+ * `git status` would run past it. With a deadline set, each call gets only the
+ * time left, and a call that would start after it throws — so the pass fails
+ * closed instead of finishing late.
+ */
+let subprocessDeadline: number | null = null;
+
+export function withDeadline<T>(deadlineAt: number, fn: () => T): T {
+    const prev = subprocessDeadline;
+    subprocessDeadline = deadlineAt;
+    try {
+        return fn();
+    } finally {
+        subprocessDeadline = prev;
+    }
+}
+
+export function timeLeftMs(): number | undefined {
+    if (subprocessDeadline === null) return undefined;
+    const left = subprocessDeadline - Date.now();
+    if (left <= 0) throw new Error('budget spent');
+    return left;
+}
+
+export function git(cwd: string, args: string[], input?: string): string {
+    const timeout = timeLeftMs();
+    return execFileSync('git', ['-C', cwd, ...args], {
+        encoding: 'utf-8',
+        stdio: [input === undefined ? 'ignore' : 'pipe', 'pipe', 'pipe'],
+        ...(input === undefined ? {} : { input }),
+        ...(timeout === undefined ? {} : { timeout }),
+        // No git call made here may refresh an index: the inventory reads index
+        // mtimes as a liveness fallback, so a refresh would fake activity.
+        env: gitEnv(),
+    });
+}
+
+export function tryGit(cwd: string, args: string[], input?: string): string | null {
+    try {
+        return git(cwd, args, input);
+    } catch {
+        return null;
+    }
+}
+
+/**
+ * The caller's environment minus the variables that redirect `git -C`: a hook
+ * process inherits `GIT_DIR` / `GIT_WORK_TREE` / `GIT_INDEX_FILE`, and with any
+ * of them set every per-worktree call would read the wrong repository.
+ */
+export function gitEnv(): NodeJS.ProcessEnv {
+    const env: NodeJS.ProcessEnv = { ...process.env, GIT_OPTIONAL_LOCKS: '0' };
+    delete env.GIT_DIR;
+    delete env.GIT_WORK_TREE;
+    delete env.GIT_INDEX_FILE;
+    return env;
+}

package/src/scripts/_lib/worktree_ignored.ts

@@ -0,0 +1,116 @@
+/**
+ * Which ignored files a worktree removal would lose that a rebuild does not
+ * restore. `git worktree remove` deletes ignored files without refusing, so
+ * this is the check between a merged tree and a lost local `.env`.
+ */
+
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+
+import { tryGit } from './worktree_git.js';
+
+/**
+ * Ignored paths that rebuild themselves. `git worktree remove` without
+ * `--force` refuses a dirty tree but deletes ignored files without a word
+ * (reproduced 2026-10-21), so a local `.env` would vanish with a merged tree.
+ * A pattern counts as regenerable when it is one of these well-known build and
+ * dependency outputs, or when the `.gitignore` comment block directly above the
+ * matching pattern says the path is generated.
+ */
+export const REGENERABLE_IGNORED = [
+    // Built in only where a path is never hand-edited: dependency trees the
+    // package manager rebuilds from the lockfile, interpreter and tool caches,
+    // and OS litter. `dist/`, `build/`, `vendor/`, `.venv/` are deliberately NOT
+    // here — each can hold a patched dependency or an artefact someone kept
+    // (council 2026-10-21, both seats). A repository that knows they are
+    // generated says so in its own `.gitignore` comment, which this module reads.
+    'node_modules',
+    '__pycache__',
+    '.pytest_cache',
+    '.mypy_cache',
+    '.ruff_cache',
+    '.parcel-cache',
+    '.turbo',
+    '.cache',
+    '.DS_Store',
+] as const;
+
+const GENERATED_COMMENT = /generat|build output|rebuilt by|projection|byproduct/i;
+
+function isRegenerablePattern(pattern: string): boolean {
+    const bare = pattern.replace(/^!/, '').replace(/^\/+/, '').replace(/\/+$/, '');
+    const last = bare.split('/').pop() ?? bare;
+    return (REGENERABLE_IGNORED as readonly string[]).includes(last) || /\.tsbuildinfo$|\.pyc$/.test(bare);
+}
+
+/** A pattern naming a secret or a local override is never regenerable, whatever its comment says. */
+const NEVER_REGENERABLE = /\.env|secret|credential|token|\bkey|local|\.pem|\.p12|password/i;
+
+/**
+ * True when the comment group IMMEDIATELY above `line` (1-based) reads as
+ * "generated". Another pattern line in between ends the search: a comment
+ * vouches for the one pattern it sits on, never for a later rule appended
+ * under it (`# generated` / `dist/` / `.env` must not cover `.env`).
+ */
+function commentSaysGenerated(lines: string[], line: number, pattern: string): boolean {
+    if (NEVER_REGENERABLE.test(pattern)) return false;
+    let i = line - 2;
+    if (i < 0 || !lines[i]!.trim().startsWith('#')) return false;
+    for (; i >= 0 && lines[i]!.trim().startsWith('#'); i--) {
+        if (GENERATED_COMMENT.test(lines[i]!)) return true;
+    }
+    return false;
+}
+
+/**
+ * Ignored paths in `worktreePath` that are NOT regenerable. Empty means losing
+ * every ignored file with the tree costs nothing a rebuild would not restore.
+ */
+export function localOnlyIgnored(worktreePath: string, trunk: string | null = null): string[] {
+    const status = tryGit(worktreePath, ['status', '--ignored=matching', '--porcelain']);
+    if (status === null) return ['(ignored files could not be listed)'];
+    const ignored = status
+        .split('\n')
+        .filter((l) => l.startsWith('!! '))
+        .map((l) => l.slice(3));
+    if (ignored.length === 0) return [];
+    const verbose = tryGit(worktreePath, ['check-ignore', '-v', '--no-index', '--stdin'], ignored.join('\n'));
+    const sourceOf = new Map<string, { file: string; line: number; pattern: string }>();
+    for (const row of (verbose ?? '').split('\n')) {
+        const m = /^(.*?):(\d+):(.*)\t(.+)$/.exec(row);
+        if (m) sourceOf.set(m[4]!, { file: m[1]!, line: Number(m[2]), pattern: m[3]! });
+    }
+    const fileCache = new Map<string, string[]>();
+    const readLines = (file: string): string[] => {
+        const abs = path.isAbsolute(file) ? file : path.join(worktreePath, file);
+        if (!fileCache.has(abs)) {
+            try {
+                fileCache.set(abs, fs.readFileSync(abs, 'utf-8').split('\n'));
+            } catch {
+                fileCache.set(abs, []);
+            }
+        }
+        return fileCache.get(abs)!;
+    };
+    // The trunk's copy of the same ignore file speaks for an old branch too: a
+    // repository that declares a path generated today should not have to wait
+    // for every long-lived branch to pick the comment up.
+    const trunkCache = new Map<string, string[]>();
+    const trunkLines = (file: string): string[] => {
+        if (trunk === null || path.isAbsolute(file)) return [];
+        if (!trunkCache.has(file)) {
+            const text = tryGit(worktreePath, ['show', `${trunk}:${file}`]);
+            trunkCache.set(file, text === null ? [] : text.split('\n'));
+        }
+        return trunkCache.get(file)!;
+    };
+    return ignored.filter((p) => {
+        const src = sourceOf.get(p);
+        if (src === undefined) return true;
+        if (isRegenerablePattern(src.pattern)) return false;
+        if (commentSaysGenerated(readLines(src.file), src.line, src.pattern)) return false;
+        const t = trunkLines(src.file);
+        const idx = t.findIndex((l) => l.trim() === src.pattern.trim());
+        return !(idx >= 0 && commentSaysGenerated(t, idx + 1, src.pattern));
+    });
+}