@illuminis/comprism 0.1.0

package/out/graph/facts.d.ts

@@ -0,0 +1,47 @@
+/**
+ * The one shape both readers emit, and the only thing that ever leaves the
+ * machine.
+ *
+ * An object and a link, each carrying a file and a line. Nothing else. No file
+ * contents, no source text, no string a person wrote. That is what makes
+ * sending the map to the service a different act from sending the code: the
+ * service learns that a function called `price_of` exists at line 40 of a named
+ * file and what calls it, and it never learns what that function says.
+ */
+export interface GraphObject {
+    /** Unique within a project. A file is its own path; a symbol is `file::name`. */
+    id: string;
+    /** file, function, method, class, component, model, table, route. */
+    kind: string;
+    name: string;
+    file: string;
+    line: number;
+    [extra: string]: unknown;
+}
+export interface GraphLink {
+    from: string;
+    /** imports, defines, calls, renders, inherits, references, maps_to,
+     *  handled_by, declares, calls_endpoint, browser_route. */
+    kind: string;
+    /** Set when the reader could resolve the far end itself. */
+    to?: string;
+    /** Set when it could not, and the service joins it up. */
+    to_name?: string;
+    [extra: string]: unknown;
+}
+export interface FactSet {
+    objects: GraphObject[];
+    links: GraphLink[];
+    /** A file that declares routers, and what each is mounted under. Python only:
+     *  an endpoint's path is written relative to a router that is mounted with a
+     *  prefix in a different file, and without both halves the front end and the
+     *  back end never join up. */
+    routers?: {
+        prefixes: Record<string, string>;
+        mounts: {
+            target: string;
+            prefix: string;
+            line: number;
+        }[];
+    };
+}

package/out/graph/facts.js

@@ -0,0 +1,12 @@
+"use strict";
+/**
+ * The one shape both readers emit, and the only thing that ever leaves the
+ * machine.
+ *
+ * An object and a link, each carrying a file and a line. Nothing else. No file
+ * contents, no source text, no string a person wrote. That is what makes
+ * sending the map to the service a different act from sending the code: the
+ * service learns that a function called `price_of` exists at line 40 of a named
+ * file and what calls it, and it never learns what that function says.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });

package/out/graph/files.d.ts

@@ -0,0 +1,45 @@
+/** How many files one project may have indexed. A ceiling, not a guess: a
+ *  monorepo of forty thousand files would take minutes to read and produce a
+ *  map too large to be worth sending, and half a map that claims to be whole is
+ *  the failure this whole feature exists to avoid. Past this the map says
+ *  plainly that it covers part of the project. */
+export declare const MAX_FILES = 6000;
+/** How large a single file may be before it is skipped. A generated bundle or
+ *  a checked-in minified file is not code anybody navigates. */
+export declare const MAX_FILE_BYTES = 1500000;
+export declare const PY_EXT: Set<string>;
+export declare const TS_EXT: Set<string>;
+export interface IndexedFile {
+    /** Path relative to the project root, forward slashes, always. */
+    rel: string;
+    size: number;
+    mtimeMs: number;
+    lang: "python" | "typescript" | "translations";
+}
+/** Every file the map covers, in a stable order.
+ *
+ *  Sorted, and that is not cosmetic: the fingerprint below is a hash over this
+ *  list, so an unstable order would report the code had moved every time the
+ *  filesystem felt like returning entries differently. */
+export declare function indexedFiles(root: string): IndexedFile[];
+/** One short string per file that changes whenever that file does.
+ *
+ *  Size and modified time rather than a hash of the contents: reading six
+ *  thousand files to find out whether any of them changed costs more than
+ *  rebuilding the map. A file edited back to its original contents within the
+ *  same millisecond would be missed, which is a case that does not occur in
+ *  practice and is corrected by the next ordinary edit. */
+export declare function stampOf(file: IndexedFile): string;
+/** One short string for the whole project. Saying the code has not moved is a
+ *  claim, and this is the evidence for it. */
+export declare function fingerprint(files: IndexedFile[]): {
+    hash: string;
+    files: number;
+};
+/** The commit the project is standing on, when it is a repository at all.
+ *
+ *  Read with a plain file read rather than by running git, because this is
+ *  called at the start of every job and spawning a process there is a cost paid
+ *  on work that has nothing to do with the map. A project with no git still
+ *  gets a usable map; it simply cannot stamp an answer with a commit. */
+export declare function headCommit(root: string): string;

package/out/graph/files.js

@@ -0,0 +1,207 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TS_EXT = exports.PY_EXT = exports.MAX_FILE_BYTES = exports.MAX_FILES = void 0;
+exports.indexedFiles = indexedFiles;
+exports.stampOf = stampOf;
+exports.fingerprint = fingerprint;
+exports.headCommit = headCommit;
+/**
+ * What the code map covers, and how to tell in milliseconds whether the code
+ * has moved since it was last read.
+ *
+ * Everything here runs on the person's own machine, because it is the one place
+ * their source files exist. Nothing in this folder decides, prices or
+ * classifies anything: it reads files and produces facts. That is the division
+ * the whole product rests on, and a function here that formed an opinion would
+ * be the first crack in it.
+ */
+const crypto = __importStar(require("crypto"));
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const read_locales_1 = require("./read-locales");
+/** Folders never worth reading: somebody else's code, or build output. */
+const SKIP_DIRS = new Set([
+    "node_modules", ".git", "__pycache__", ".venv", "venv", "env",
+    "site-packages", "dist", "build", ".pytest_cache", ".mypy_cache",
+    "htmlcov", ".ruff_cache", "coverage", ".vite", ".next", ".turbo",
+    "playwright-report", "test-results", "vendor", "target", ".cache",
+    "scratchpad", ".comprism", ".completionprism",
+]);
+/** Folder names that are build output only when they sit beside a package
+ *  manifest. `out` is where TypeScript writes its JavaScript by convention, but
+ *  it is also a perfectly ordinary name for a source folder, so skipping it
+ *  everywhere would quietly remove real code from somebody's map. Next to a
+ *  package.json it is almost always generated.
+ *
+ *  Found by mapping this repository and getting 83 changed files from one edit:
+ *  the terminal client's own compiled output was in the map, so every rebuild
+ *  looked like eighty three source files moving. */
+const BUILT_BESIDE_MANIFEST = new Set(["out", "output", "lib", "es", "esm"]);
+/** How many files one project may have indexed. A ceiling, not a guess: a
+ *  monorepo of forty thousand files would take minutes to read and produce a
+ *  map too large to be worth sending, and half a map that claims to be whole is
+ *  the failure this whole feature exists to avoid. Past this the map says
+ *  plainly that it covers part of the project. */
+exports.MAX_FILES = 6000;
+/** How large a single file may be before it is skipped. A generated bundle or
+ *  a checked-in minified file is not code anybody navigates. */
+exports.MAX_FILE_BYTES = 1_500_000;
+exports.PY_EXT = new Set([".py"]);
+exports.TS_EXT = new Set([".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs"]);
+function isCode(rel, name) {
+    if (name.endsWith(".d.ts"))
+        return null;
+    if (name.endsWith(".min.js"))
+        return null;
+    const ext = path.extname(name);
+    if (exports.PY_EXT.has(ext))
+        return "python";
+    if (exports.TS_EXT.has(ext))
+        return "typescript";
+    // Translation files. Not code, and in the map for a reason no code parser can
+    // serve: the label a person sees is a key here and a sentence in ten language
+    // files, and "which languages are missing this one" is the question asked
+    // about it. See `read-locales.ts`.
+    if ((0, read_locales_1.looksLikeTranslations)(rel))
+        return "translations";
+    return null;
+}
+/** Every file the map covers, in a stable order.
+ *
+ *  Sorted, and that is not cosmetic: the fingerprint below is a hash over this
+ *  list, so an unstable order would report the code had moved every time the
+ *  filesystem felt like returning entries differently. */
+function indexedFiles(root) {
+    const out = [];
+    const walk = (dir) => {
+        if (out.length >= exports.MAX_FILES)
+            return;
+        let entries;
+        try {
+            entries = fs.readdirSync(dir, { withFileTypes: true });
+        }
+        catch {
+            return;
+        }
+        for (const entry of entries) {
+            if (out.length >= exports.MAX_FILES)
+                return;
+            const full = path.join(dir, entry.name);
+            if (entry.isDirectory()) {
+                if (entry.name.startsWith(".") && entry.name !== ".github")
+                    continue;
+                if (SKIP_DIRS.has(entry.name))
+                    continue;
+                if (BUILT_BESIDE_MANIFEST.has(entry.name)
+                    && fs.existsSync(path.join(dir, "package.json")))
+                    continue;
+                walk(full);
+                continue;
+            }
+            if (!entry.isFile())
+                continue;
+            const rel = path.relative(root, full).split(path.sep).join("/");
+            const lang = isCode(rel, entry.name);
+            if (!lang)
+                continue;
+            let stat;
+            try {
+                stat = fs.statSync(full);
+            }
+            catch {
+                continue;
+            }
+            if (stat.size > exports.MAX_FILE_BYTES)
+                continue;
+            out.push({
+                rel,
+                size: stat.size,
+                mtimeMs: Math.round(stat.mtimeMs),
+                lang,
+            });
+        }
+    };
+    walk(root);
+    out.sort((a, b) => (a.rel < b.rel ? -1 : a.rel > b.rel ? 1 : 0));
+    return out;
+}
+/** One short string per file that changes whenever that file does.
+ *
+ *  Size and modified time rather than a hash of the contents: reading six
+ *  thousand files to find out whether any of them changed costs more than
+ *  rebuilding the map. A file edited back to its original contents within the
+ *  same millisecond would be missed, which is a case that does not occur in
+ *  practice and is corrected by the next ordinary edit. */
+function stampOf(file) {
+    return `${file.size}:${file.mtimeMs}`;
+}
+/** One short string for the whole project. Saying the code has not moved is a
+ *  claim, and this is the evidence for it. */
+function fingerprint(files) {
+    const hash = crypto.createHash("sha1");
+    for (const file of files)
+        hash.update(`${file.rel}:${stampOf(file)}\n`);
+    return { hash: hash.digest("hex").slice(0, 16), files: files.length };
+}
+/** The commit the project is standing on, when it is a repository at all.
+ *
+ *  Read with a plain file read rather than by running git, because this is
+ *  called at the start of every job and spawning a process there is a cost paid
+ *  on work that has nothing to do with the map. A project with no git still
+ *  gets a usable map; it simply cannot stamp an answer with a commit. */
+function headCommit(root) {
+    try {
+        const head = fs.readFileSync(path.join(root, ".git", "HEAD"), "utf8").trim();
+        if (!head.startsWith("ref:"))
+            return head.slice(0, 40);
+        const ref = head.slice(4).trim();
+        const direct = path.join(root, ".git", ref);
+        if (fs.existsSync(direct))
+            return fs.readFileSync(direct, "utf8").trim().slice(0, 40);
+        const packed = path.join(root, ".git", "packed-refs");
+        if (fs.existsSync(packed)) {
+            for (const line of fs.readFileSync(packed, "utf8").split("\n")) {
+                const [sha, name] = line.trim().split(" ");
+                if (name === ref && sha)
+                    return sha.slice(0, 40);
+            }
+        }
+    }
+    catch {
+        /* not a repository, or an unusual layout. Neither is an error. */
+    }
+    return "";
+}

package/out/graph/read-locales.d.ts

@@ -0,0 +1,29 @@
+import type { FactSet } from "./facts";
+/** Is this JSON file a translation file?
+ *
+ *  Both conditions, not either: a `data/en.json` of sample records is not a
+ *  translation file, and a `locales/schema.json` is not one either. Indexing
+ *  the wrong JSON would fill the map with things nobody asked about. */
+export declare function looksLikeTranslations(rel: string): boolean;
+/** The language a translation file is for, from its own name. */
+export declare function languageOf(rel: string): string;
+interface KeyAt {
+    key: string;
+    line: number;
+}
+/**
+ * Every leaf key in a translation file, with the line it sits on.
+ *
+ * A single pass over the text, tracking strings, escapes, nesting and lines. It
+ * reads the structure rather than the values: what a translation SAYS never
+ * leaves the machine, only that the key exists and where.
+ */
+export declare function scanKeys(text: string): KeyAt[];
+export declare function readLocales(root: string, rels: string[]): {
+    facts: Record<string, FactSet>;
+    failures: {
+        file: string;
+        reason: string;
+    }[];
+};
+export {};

package/out/graph/read-locales.js

@@ -0,0 +1,246 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.looksLikeTranslations = looksLikeTranslations;
+exports.languageOf = languageOf;
+exports.scanKeys = scanKeys;
+exports.readLocales = readLocales;
+/**
+ * Read translation files into code map facts.
+ *
+ * ## Why these are in the map at all
+ *
+ * A label a person sees on screen is not written in the code. It is a key in
+ * the code and a sentence in ten language files, and the question people
+ * actually ask about it is one no code parser can answer: *where is this label,
+ * and which languages are missing it.* An agent asked to add a field to a form
+ * has to put the same line in ten files, and a map that indexes only Python and
+ * TypeScript sends it to search for them one at a time.
+ *
+ * ## One object per key, not one per key per file
+ *
+ * Ten language files times three thousand keys is thirty one thousand objects,
+ * and almost all of that is the same key repeated. Each file reports its own
+ * keys here, because facts are read per file and a file has to be re-readable
+ * on its own. The service then collapses them into ONE object per key carrying
+ * which languages have it and which do not, which is both far smaller and the
+ * answer somebody wanted.
+ *
+ * ## Line numbers are tracked by hand, and that is deliberate
+ *
+ * `JSON.parse` gives no positions, and "the key is in this file somewhere" is
+ * not a useful answer when the file is three thousand lines. So the text is
+ * scanned once, tracking the line of every key as it goes. It is a scanner, not
+ * a guess: the same file always produces the same lines.
+ */
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+/** Folders whose JSON files are translations. */
+const LOCALE_DIRS = new Set(["locales", "locale", "i18n", "lang", "translations"]);
+/** A file named like a language: `en`, `en-US`, `pt_BR`. */
+const LANGUAGE_NAME = /^[a-z]{2}([-_][A-Za-z]{2,4})?$/;
+/** Is this JSON file a translation file?
+ *
+ *  Both conditions, not either: a `data/en.json` of sample records is not a
+ *  translation file, and a `locales/schema.json` is not one either. Indexing
+ *  the wrong JSON would fill the map with things nobody asked about. */
+function looksLikeTranslations(rel) {
+    if (!rel.endsWith(".json"))
+        return false;
+    const parts = rel.split("/");
+    const base = path.basename(rel, ".json");
+    const parent = parts.length > 1 ? parts[parts.length - 2] ?? "" : "";
+    return LOCALE_DIRS.has(parent.toLowerCase()) && LANGUAGE_NAME.test(base);
+}
+/** The language a translation file is for, from its own name. */
+function languageOf(rel) {
+    return path.basename(rel, ".json");
+}
+/**
+ * Every leaf key in a translation file, with the line it sits on.
+ *
+ * A single pass over the text, tracking strings, escapes, nesting and lines. It
+ * reads the structure rather than the values: what a translation SAYS never
+ * leaves the machine, only that the key exists and where.
+ */
+function scanKeys(text) {
+    const out = [];
+    const stack = [];
+    let line = 1;
+    let i = 0;
+    /** The key whose value has not been seen yet. Until the value turns up we do
+     *  not know whether this key is a group or a leaf. */
+    let pending = null;
+    let pendingLine = 0;
+    const at = (n) => text[n] ?? "";
+    /** Consume a string starting at the opening quote, counting any lines in it. */
+    const readString = () => {
+        i += 1;
+        let value = "";
+        while (i < text.length) {
+            const ch = at(i);
+            if (ch === "\\") {
+                i += 2;
+                continue;
+            }
+            if (ch === '"') {
+                i += 1;
+                return value;
+            }
+            if (ch === "\n")
+                line += 1;
+            value += ch;
+            i += 1;
+        }
+        return value;
+    };
+    /** Consume an array, whatever is nested in it. Arrays hold no keys we want. */
+    const skipArray = () => {
+        let depth = 0;
+        while (i < text.length) {
+            const ch = at(i);
+            if (ch === '"') {
+                readString();
+                continue;
+            }
+            if (ch === "\n")
+                line += 1;
+            if (ch === "[")
+                depth += 1;
+            if (ch === "]") {
+                depth -= 1;
+                i += 1;
+                if (depth === 0)
+                    return;
+                continue;
+            }
+            i += 1;
+        }
+    };
+    while (i < text.length) {
+        const ch = at(i);
+        if (ch === "\n") {
+            line += 1;
+            i += 1;
+            continue;
+        }
+        if (ch === " " || ch === "\t" || ch === "\r" || ch === ",") {
+            i += 1;
+            continue;
+        }
+        // A key is waiting, so whatever comes next is its VALUE. This is checked
+        // first, and that is the whole correctness of the scanner: a string value
+        // looks exactly like a key until you know which position you are in.
+        if (pending !== null) {
+            if (ch === "{") {
+                stack.push(pending);
+                pending = null;
+                i += 1;
+                continue;
+            }
+            out.push({ key: [...stack, pending].join("."), line: pendingLine });
+            pending = null;
+            if (ch === '"') {
+                readString();
+                continue;
+            }
+            if (ch === "[") {
+                skipArray();
+                continue;
+            }
+            while (i < text.length && !/[,}\]\s]/.test(at(i)))
+                i += 1;
+            continue;
+        }
+        if (ch === '"') {
+            const startLine = line;
+            const name = readString();
+            let j = i;
+            while (j < text.length && /\s/.test(at(j))) {
+                if (at(j) === "\n")
+                    line += 1;
+                j += 1;
+            }
+            if (at(j) === ":") {
+                pending = name;
+                pendingLine = startLine;
+                i = j + 1;
+            }
+            continue;
+        }
+        if (ch === "}") {
+            stack.pop();
+            i += 1;
+            continue;
+        }
+        i += 1;
+    }
+    return out;
+}
+function readOne(root, rel) {
+    const text = fs.readFileSync(path.join(root, rel), "utf8");
+    const language = languageOf(rel);
+    const objects = [{
+            id: rel, kind: "file", name: path.basename(rel), file: rel, line: 1,
+            lang: "translations", language,
+            lines: text.split("\n").length,
+        }];
+    for (const { key, line } of scanKeys(text)) {
+        objects.push({
+            // Unique per file, because facts are stored per file. The service
+            // collapses every file's copy of one key into a single object.
+            id: `i18n:${key}@${rel}`,
+            kind: "translation_key",
+            name: key,
+            file: rel,
+            line,
+            language,
+        });
+    }
+    return { objects, links: [] };
+}
+function readLocales(root, rels) {
+    const facts = {};
+    const failures = [];
+    for (const rel of rels) {
+        try {
+            facts[rel] = readOne(root, rel);
+        }
+        catch (err) {
+            failures.push({ file: rel, reason: err?.name || "Error" });
+        }
+    }
+    return { facts, failures };
+}

package/out/graph/read-python.d.ts

@@ -0,0 +1,11 @@
+import type { FactSet } from "./facts";
+/** Which Python this machine has, or null. Asked once per process. */
+export declare function findPython(): string | null;
+export declare function readPython(root: string, rels: string[]): {
+    facts: Record<string, FactSet>;
+    failures: {
+        file: string;
+        reason: string;
+    }[];
+    available: boolean;
+};