@bigsteele/the-prospect 0.1.1 → 0.3.0

package/dist/walk.d.ts

@@ -1,3 +1,5 @@
+/** The path with any template suffix removed, so `x.ts.tmpl` reads as `x.ts`. */
+export declare function baseName(rel: string): string;
 export interface Repo {
     root: string;
     name: string;
@@ -29,8 +31,15 @@ export declare function grep(repo: Repo, files: string[], re: RegExp, limit?: nu
     file: string;
     count: number;
 }>>;
-/** Code files only: what the detectors read for behaviour. */
+/**
+ * Code files only: what the detectors read for behaviour.
+ *
+ * `isCode` is the one to call - it sees through a template suffix, so the
+ * adapter templates the generator ships are code, which they are. `CODE` stays
+ * exported because callers match raw paths with it.
+ */
 export declare const CODE: RegExp;
+export declare const isCode: (rel: string) => boolean;
 /** Test files, which describe behaviour but do not run in production. */
 export declare const TEST_FILE: RegExp;
 /**
@@ -42,3 +51,78 @@ export declare const TEST_FILE: RegExp;
 export declare const NOT_RUNTIME: RegExp;
 /** Files the detectors read for behaviour: code, in the runtime tree, not tests. */
 export declare function runtimeCode(files: string[]): string[];
+/**
+ * EXCLUSION IS PER QUESTION, NOT GLOBAL (0.2).
+ *
+ * One list used to decide what every detector saw, and it conflated two
+ * different things: "this is not our stack" and "this is not worth looking at".
+ * A fixture of somebody else's codebase is genuinely not our vendor evidence.
+ * It is still our file, and it can still hold a duplicate block or a dead
+ * module. Excluding it from every question at once is how 1,604 of 2,077 files
+ * went unexamined while the report claimed to have read the repository.
+ *
+ * It is also how the Auth0 false positive survived four rounds of hardening:
+ * `runtimeCode` filtered test files and `grepAny` did not, because each call
+ * site re-derived its own scope. A scope is now named once and asked for by
+ * name, so a detector cannot forget a term.
+ */
+export type Question = 
+/** Which vendors does this product use? Someone else's code is not evidence. */
+"vendor-presence"
+/** What did we build by hand? Tests and fixtures describe, they do not implement. */
+ | "implementation"
+/** What is duplicated? Fixtures and tests count: a copy is a copy. */
+ | "duplication"
+/** What does no entrypoint reach? Everything shipped counts. */
+ | "reachability"
+/** What does the database grant and to whom? Migrations, wherever they live. */
+ | "database"
+/** What did the people who built this decide, and write down? */
+ | "decisions"
+/** What vocabulary does this product speak? Someone else's domain is not ours. */
+ | "vocabulary";
+/** The files one question considers. Ask by name; never re-derive a scope inline. */
+export declare function scopeFor(files: string[], q: Question): string[];
+export interface Coverage {
+    walked: number;
+    analysed: number;
+    unreadable: number;
+    unaccounted: number;
+    excluded: Array<{
+        rule: string;
+        why: string;
+        files: number;
+        examples: string[];
+    }>;
+    by_extension: Record<string, {
+        walked: number;
+        analysed: number;
+    }>;
+    questions: Record<string, number>;
+    /**
+     * Readable file classes no question claims, biggest first.
+     *
+     * The difference between "we decided not to look at this" and "nothing looks
+     * at this" is the whole point of the ledger, and only this field can show the
+     * second. On the first repository it reported 146 `.sql` files - a platform
+     * whose authorisation model lives in migrations, analysed by nothing - which
+     * no amount of reading the detector list would have revealed. `.md` and `.png`
+     * appear here too and are correctly ignored; the list is for a reader to
+     * judge, not a defect on its own.
+     */
+    unclaimed: Array<{
+        ext: string;
+        files: number;
+        examples: string[];
+    }>;
+}
+/**
+ * Every file walked lands in exactly one bucket, and the bucket names its rule.
+ *
+ * This is the claim "no stone unturned" actually rests on. Before it, the scan
+ * could not answer "what did you not look at" - which is the question the claim
+ * is asserting an answer to. A file is ANALYSED if any question admits it,
+ * UNREADABLE if the walker holds no text for it, and otherwise EXCLUDED by the
+ * first rule that dropped it, named.
+ */
+export declare function coverageOf(repo: Repo): Promise<Coverage>;

package/dist/walk.js

@@ -2,12 +2,29 @@
 // deployable (family convention). The only place that touches the filesystem. Walks a repository, skips what is not the
 // project's own code, reads text files under a size cap, and never opens an env file.
 import { readdir, readFile, stat } from "node:fs/promises";
+import { DECISION_FILES } from "./decisions.js";
 import { join, relative, sep } from "node:path";
 const SKIP_DIRS = new Set(["node_modules", ".git", "dist", "build", "out", ".next", ".nuxt", ".svelte-kit", ".vercel", ".turbo", "coverage", "vendor", "dist.bak", ".cache", "__pycache__", ".venv", "venv", "target", "test-results", "playwright-report", ".chrome-debug", ".claude-browser"]);
 // Stylesheets are read too: `@import "tw-animate-css"` and Tailwind 4's `@plugin`
 // are how a whole class of packages is used, and a walker that listed .css files
 // but never read them flagged every one of those packages as unreferenced.
-const TEXT = /\.(ts|tsx|js|jsx|mjs|cjs|py|go|rs|rb|php|java|kt|swift|sql|json|jsonc|ya?ml|toml|md|mdx|txt|prisma|graphql|gql|env\.example|sh|css|scss|sass|less|pcss)$/i;
+// SHIPPED ARTIFACTS ARE CODE (0.2, "no stone unturned"). The first real
+// repository this met carried 137 `.tmpl` files - the adapter templates the
+// generator writes into a customer's codebase, which is to say the product -
+// and the walker never opened one. `.vue`, `.svelte` and `.html` were the same
+// omission at smaller scale. A scan blind to what you ship cannot claim to have
+// turned every stone, so a template suffix is stripped and the base extension
+// decides: `session-route.ts.tmpl` is TypeScript.
+const TEMPLATE_SUFFIX = /\.(tmpl|template|hbs|ejs|mustache|liquid|j2|jinja2?)$/i;
+const TEXT_BASE = /\.(ts|tsx|js|jsx|mjs|cjs|py|go|rs|rb|php|java|kt|swift|sql|json|jsonc|ya?ml|toml|md|mdx|txt|prisma|graphql|gql|env\.example|sh|css|scss|sass|less|pcss|vue|svelte|astro|html|htm)$/i;
+/** The path with any template suffix removed, so `x.ts.tmpl` reads as `x.ts`. */
+export function baseName(rel) {
+    let out = rel;
+    while (TEMPLATE_SUFFIX.test(out))
+        out = out.replace(TEMPLATE_SUFFIX, "");
+    return out;
+}
+const TEXT = (rel) => TEXT_BASE.test(baseName(rel));
 const ENV_FILE = /(^|\/)\.env(\.[a-z0-9_-]+)?$/i;
 const MAX_BYTES = 512 * 1024;
 const MAX_FILES = 25_000;
@@ -60,7 +77,7 @@ export async function openRepo(root) {
             if (cache.has(rel))
                 return cache.get(rel);
             let text = null;
-            if (TEXT.test(rel) && !ENV_FILE.test(rel)) {
+            if (TEXT(rel) && !ENV_FILE.test(rel)) {
                 try {
                     const s = await stat(join(root, rel));
                     if (s.size <= MAX_BYTES)
@@ -106,8 +123,15 @@ export async function grep(repo, files, re, limit = 4000) {
     }
     return out;
 }
-/** Code files only: what the detectors read for behaviour. */
-export const CODE = /\.(ts|tsx|js|jsx|mjs|cjs|py|go|rs|rb|php|java|kt|swift)$/i;
+/**
+ * Code files only: what the detectors read for behaviour.
+ *
+ * `isCode` is the one to call - it sees through a template suffix, so the
+ * adapter templates the generator ships are code, which they are. `CODE` stays
+ * exported because callers match raw paths with it.
+ */
+export const CODE = /\.(ts|tsx|js|jsx|mjs|cjs|py|go|rs|rb|php|java|kt|swift|vue|svelte|astro)$/i;
+export const isCode = (rel) => CODE.test(baseName(rel));
 /** Test files, which describe behaviour but do not run in production. */
 export const TEST_FILE = /(^|\.|_|\/)(test|spec|e2e)s?(\.|\/)|(^|\/)__tests__\//i;
 /**
@@ -119,5 +143,165 @@ export const TEST_FILE = /(^|\.|_|\/)(test|spec|e2e)s?(\.|\/)|(^|\/)__tests__\//
 export const NOT_RUNTIME = /(^|\/)(docs?|fixtures?|__fixtures__|__mocks__|golden|examples?|samples?|packages\/the-prospect)\/|(^|\/)\.(?!well-known\/)[^/]+\//i;
 /** Files the detectors read for behaviour: code, in the runtime tree, not tests. */
 export function runtimeCode(files) {
-    return files.filter((f) => CODE.test(f) && !TEST_FILE.test(f) && !NOT_RUNTIME.test(f));
+    return files.filter((f) => isCode(f) && !TEST_FILE.test(f) && !NOT_RUNTIME.test(f));
+}
+const DROP_NOT_RUNTIME = {
+    rule: "not-runtime",
+    why: "documentation, fixtures, golden files and dotfile trees: present in the repository, not the running software",
+    test: (f) => NOT_RUNTIME.test(f),
+};
+/**
+ * A generated artifact: code this repository WRITES INTO somebody else's, rather
+ * than code it runs. Reading these was the point of 0.2 - 137 of them on the
+ * first real repository, the product itself, previously unopened. But a vendor
+ * named inside one is the CUSTOMER'S vendor, not this product's: an adapter
+ * directory called `next-app-clerk` exists to support Clerk, and reading it as
+ * "you pay Clerk" told an owner to drop a vendor they never had. Same mistake as
+ * the fixtures, one level further in, and found by fixing the fixtures.
+ *
+ * So: read for duplication and reachability, never for vendor presence.
+ */
+const DROP_GENERATED_OUT = {
+    rule: "shipped-template",
+    why: "a template this repository writes into a customer's codebase: its imports are the customer's stack, not this product's",
+    test: (f) => /\.(tmpl|template|hbs|ejs|mustache|liquid|j2|jinja2?)$/i.test(f) || /(^|\/)(adapters?|shell-templates?|generators?)\//i.test(f),
+};
+const DROP_TESTS = {
+    rule: "test-file",
+    why: "a test describes behaviour and often quotes somebody else's code verbatim; it is not this product doing the thing",
+    test: (f) => TEST_FILE.test(f),
+};
+const SCOPES = {
+    "vendor-presence": {
+        why: "only code this product runs can prove which vendors it pays for",
+        admits: (f) => isCode(f) || /\.(toml|jsonc?|ya?ml)$/i.test(baseName(f)),
+        drops: [DROP_NOT_RUNTIME, DROP_TESTS, DROP_GENERATED_OUT],
+    },
+    implementation: {
+        why: "only code this product runs can show what it built by hand",
+        admits: isCode,
+        drops: [DROP_NOT_RUNTIME, DROP_TESTS, DROP_GENERATED_OUT],
+    },
+    duplication: {
+        why: "a copied block is a copied block wherever it lives, so only generated output and vendored trees are dropped",
+        admits: isCode,
+        drops: [],
+    },
+    reachability: {
+        why: "everything the repository ships is walked; only the auditor's own package is dropped",
+        admits: isCode,
+        drops: [
+            {
+                rule: "auditor-self",
+                why: "this auditor's own source, when it is audited from inside the repository that holds it",
+                test: (f) => /(^|\/)packages\/the-prospect\//.test(f),
+            },
+        ],
+    },
+    database: {
+        why: "the migrations that build THIS product's database, not a fixture of somebody else's",
+        admits: (f) => /\.sql$/i.test(baseName(f)),
+        // A fixture's migrations describe another product's database, and reporting
+        // `public.users has no row-level security` about a fixture of a customer's
+        // schema is the contamination this matrix exists to prevent - which it did,
+        // four findings out of five, on the detector's first real run.
+        drops: [DROP_GENERATED_OUT, DROP_NOT_RUNTIME],
+    },
+    decisions: {
+        why: "the files a person wrote to explain choices: DECISIONS, CLAUDE/AGENTS, ADRs, .planning, a prior audit. Read so a choice on record is not reported as drift",
+        admits: (f) => DECISION_FILES.test(f),
+        drops: [],
+    },
+    vocabulary: {
+        why: "the domain words of this product, not of a fixture describing another industry",
+        admits: isCode,
+        drops: [DROP_NOT_RUNTIME, DROP_TESTS, DROP_GENERATED_OUT],
+    },
+};
+/** The files one question considers. Ask by name; never re-derive a scope inline. */
+export function scopeFor(files, q) {
+    const sc = SCOPES[q];
+    return files.filter((f) => sc.admits(f) && !sc.drops.some((d) => d.test(f)));
+}
+/**
+ * Every file walked lands in exactly one bucket, and the bucket names its rule.
+ *
+ * This is the claim "no stone unturned" actually rests on. Before it, the scan
+ * could not answer "what did you not look at" - which is the question the claim
+ * is asserting an answer to. A file is ANALYSED if any question admits it,
+ * UNREADABLE if the walker holds no text for it, and otherwise EXCLUDED by the
+ * first rule that dropped it, named.
+ */
+export async function coverageOf(repo) {
+    const questions = Object.keys(SCOPES);
+    const admitted = new Set();
+    const perQuestion = {};
+    for (const q of questions) {
+        const got = scopeFor(repo.files, q);
+        perQuestion[q] = got.length;
+        for (const f of got)
+            admitted.add(f);
+    }
+    const excluded = new Map();
+    const byExt = {};
+    let unreadable = 0;
+    let unaccounted = 0;
+    for (const f of repo.files) {
+        // Keyed on the extension AS WALKED, not the parsed one. The ledger answers
+        // "did you look at my 137 .tmpl files", and an owner counts them as .tmpl.
+        // `baseName` still decides HOW each is parsed; that is a separate question.
+        const ext = (f.match(/\.[^./]+$/)?.[0] ?? "(none)").toLowerCase();
+        byExt[ext] ??= { walked: 0, analysed: 0 };
+        byExt[ext].walked++;
+        if (admitted.has(f)) {
+            byExt[ext].analysed++;
+            continue;
+        }
+        if ((await repo.read(f)) === null) {
+            unreadable++;
+            continue;
+        }
+        // Readable, and no question wanted it. Name the rule that dropped it, taking
+        // the first that matches so the count and the reason always agree.
+        const all = questions.flatMap((q) => SCOPES[q].drops);
+        const hit = all.find((d) => d.test(f));
+        const rule = hit ? hit.rule : isCode(f) ? "no-question-admits" : "not-code";
+        const why = hit
+            ? hit.why
+            : isCode(f)
+                ? "code that every question's scope dropped; if this list is long, a scope is too narrow"
+                : "not a code file: read for references, never analysed for behaviour";
+        const row = excluded.get(rule) ?? { rule, why, files: 0, examples: [] };
+        row.files++;
+        if (row.examples.length < 3)
+            row.examples.push(f);
+        excluded.set(rule, row);
+    }
+    const bucketed = admitted.size + [...excluded.values()].reduce((t, e) => t + e.files, 0) + unreadable;
+    unaccounted = repo.files.length - bucketed;
+    // A readable class nothing analysed. Binaries are not readable, so they never
+    // reach this list; anything here is text the scan could have read and did not.
+    const unclaimed = [];
+    for (const [ext, v] of Object.entries(byExt)) {
+        if (v.analysed > 0 || v.walked < 3)
+            continue;
+        const examples = repo.files.filter((f) => f.toLowerCase().endsWith(ext)).slice(0, 3);
+        let readable = false;
+        for (const f of examples)
+            if ((await repo.read(f)) !== null)
+                readable = true;
+        if (readable)
+            unclaimed.push({ ext, files: v.walked, examples });
+    }
+    unclaimed.sort((a, b) => b.files - a.files);
+    return {
+        walked: repo.files.length,
+        analysed: admitted.size,
+        unreadable,
+        unaccounted,
+        excluded: [...excluded.values()].sort((a, b) => b.files - a.files),
+        by_extension: byExt,
+        questions: perQuestion,
+        unclaimed,
+    };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bigsteele/the-prospect",
-  "version": "0.1.1",
+  "version": "0.3.0",
   "description": "A prospector's read of your codebase and your market. Digs the ground you own: every dependency that does no work, every vendor you pay twice, every subsystem you built by hand where a rail now exists. Then surveys the territory: what your industry ships by API that your code still does the hard way. Every suggestion stands on three legs - a fact read from your code, a fact researched from your market with a source and a date, and the thing your product exists to do. Standalone: one npx, no other scan required.",
   "type": "module",
   "license": "UNLICENSED",