@illuminis/comprism 0.1.4 → 0.1.6

package/out/machine/codemap/build.d.ts

@@ -0,0 +1,45 @@
+/**
+ * Turn a folder of source files into code map facts, on the machine that holds
+ * them.
+ *
+ * Reading is the only part of the map that happens here. The joining, the
+ * storing and the answering all happen in the service, because a client that
+ * joined the map would be a client holding an opinion, and because a map that
+ * lived on one laptop could not be read by the same person's editor an hour
+ * later.
+ */
+import type { FactSet } from "./facts";
+import { IndexedFile } from "./files";
+export interface ReadResult {
+    /** One entry per file successfully read. */
+    facts: Record<string, FactSet>;
+    failures: {
+        file: string;
+        reason: string;
+    }[];
+    /** Files present in the project that no reader on this machine can read. */
+    unread: {
+        python: number;
+        typescript: number;
+    };
+    seconds: number;
+}
+/** What this machine can read at all, said plainly and once. */
+export declare function readers(): {
+    python: boolean;
+    typescript: boolean;
+};
+/** Read the named files. Everything when `rels` is absent. */
+export declare function readFacts(root: string, rels?: string[], knownNames?: string[]): ReadResult;
+export interface ProjectState {
+    files: IndexedFile[];
+    stamps: Record<string, string>;
+    fingerprint: {
+        hash: string;
+        files: number;
+    };
+    commit: string;
+}
+/** What the project looks like right now: which files, and one short string
+ *  that changes the moment any of them does. Costs about ten milliseconds. */
+export declare function projectState(root: string): ProjectState;

package/out/machine/codemap/build.js

@@ -0,0 +1,91 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.readers = readers;
+exports.readFacts = readFacts;
+exports.projectState = projectState;
+const files_1 = require("./files");
+const read_locales_1 = require("./read-locales");
+const read_python_1 = require("./read-python");
+const read_typescript_1 = require("./read-typescript");
+/** What this machine can read at all, said plainly and once. */
+function readers() {
+    return {
+        python: Boolean(require("./read-python").findPython()),
+        typescript: Boolean((0, read_typescript_1.loadCompiler)()),
+    };
+}
+/**
+ * Drop the calls that could never resolve to anything in this project.
+ *
+ * Two thirds of the raw facts read from a real 1,816 file project were calls to
+ * names the project does not define: `len`, `push`, `map`, and every library
+ * function anybody imported. The service discards every one of them when it
+ * joins the map, so sending and storing them is pure waste, measured at 17.8
+ * megabytes of 26.6.
+ *
+ * `known` is what the project already defines, which the service sends back
+ * with the file stamps, plus whatever the files just read define themselves.
+ * Nothing is dropped that the service would have kept: the join applies the same
+ * rule, only later. Correctness never depends on this, only size.
+ */
+function prune(facts, known) {
+    const droppable = new Set(["calls", "references"]);
+    for (const rel of Object.keys(facts)) {
+        const entry = facts[rel];
+        if (!entry)
+            continue;
+        entry.links = entry.links.filter((link) => {
+            if (!droppable.has(link.kind))
+                return true;
+            const leaf = String(link.to_name ?? "").split(".").pop() ?? "";
+            return known.has(leaf);
+        });
+    }
+}
+/** Read the named files. Everything when `rels` is absent. */
+function readFacts(root, rels, knownNames) {
+    const started = Date.now();
+    const all = (0, files_1.indexedFiles)(root);
+    const wanted = rels ? new Set(rels) : null;
+    const chosen = wanted ? all.filter((f) => wanted.has(f.rel)) : all;
+    const py = chosen.filter((f) => f.lang === "python").map((f) => f.rel);
+    const ts = chosen.filter((f) => f.lang === "typescript").map((f) => f.rel);
+    const locales = chosen.filter((f) => f.lang === "translations").map((f) => f.rel);
+    const fromPython = (0, read_python_1.readPython)(root, py);
+    const fromTs = (0, read_typescript_1.readTypeScript)(root, ts);
+    // Translation files need no compiler and no interpreter, so they are always
+    // read: a machine missing Python still gets its labels mapped.
+    const fromLocales = (0, read_locales_1.readLocales)(root, locales);
+    const facts = { ...fromPython.facts, ...fromTs.facts, ...fromLocales.facts };
+    // Everything the files just read define, plus everything the service already
+    // knows this project defines. A name on neither list cannot resolve.
+    const known = new Set(knownNames ?? []);
+    for (const entry of Object.values(facts)) {
+        for (const object of entry.objects) {
+            if (object.kind === "file" || object.kind === "route" || object.kind === "table")
+                continue;
+            const leaf = String(object.name).split(".").pop();
+            if (leaf)
+                known.add(leaf);
+        }
+    }
+    prune(facts, known);
+    return {
+        facts,
+        failures: [...fromPython.failures, ...fromTs.failures, ...fromLocales.failures],
+        unread: {
+            python: fromPython.available ? 0 : py.length,
+            typescript: fromTs.available ? 0 : ts.length,
+        },
+        seconds: Math.round((Date.now() - started) / 100) / 10,
+    };
+}
+/** What the project looks like right now: which files, and one short string
+ *  that changes the moment any of them does. Costs about ten milliseconds. */
+function projectState(root) {
+    const files = (0, files_1.indexedFiles)(root);
+    const stamps = {};
+    for (const file of files)
+        stamps[file.rel] = (0, files_1.stampOf)(file);
+    return { files, stamps, fingerprint: (0, files_1.fingerprint)(files), commit: (0, files_1.headCommit)(root) };
+}

package/out/machine/codemap/facts.d.ts

@@ -0,0 +1,47 @@
+/**
+ * The one shape both readers emit, and the only thing that ever leaves the
+ * machine.
+ *
+ * An object and a link, each carrying a file and a line. Nothing else. No file
+ * contents, no source text, no string a person wrote. That is what makes
+ * sending the map to the service a different act from sending the code: the
+ * service learns that a function called `price_of` exists at line 40 of a named
+ * file and what calls it, and it never learns what that function says.
+ */
+export interface GraphObject {
+    /** Unique within a project. A file is its own path; a symbol is `file::name`. */
+    id: string;
+    /** file, function, method, class, component, model, table, route. */
+    kind: string;
+    name: string;
+    file: string;
+    line: number;
+    [extra: string]: unknown;
+}
+export interface GraphLink {
+    from: string;
+    /** imports, defines, calls, renders, inherits, references, maps_to,
+     *  handled_by, declares, calls_endpoint, browser_route. */
+    kind: string;
+    /** Set when the reader could resolve the far end itself. */
+    to?: string;
+    /** Set when it could not, and the service joins it up. */
+    to_name?: string;
+    [extra: string]: unknown;
+}
+export interface FactSet {
+    objects: GraphObject[];
+    links: GraphLink[];
+    /** A file that declares routers, and what each is mounted under. Python only:
+     *  an endpoint's path is written relative to a router that is mounted with a
+     *  prefix in a different file, and without both halves the front end and the
+     *  back end never join up. */
+    routers?: {
+        prefixes: Record<string, string>;
+        mounts: {
+            target: string;
+            prefix: string;
+            line: number;
+        }[];
+    };
+}

package/out/machine/codemap/facts.js

@@ -0,0 +1,12 @@
+"use strict";
+/**
+ * The one shape both readers emit, and the only thing that ever leaves the
+ * machine.
+ *
+ * An object and a link, each carrying a file and a line. Nothing else. No file
+ * contents, no source text, no string a person wrote. That is what makes
+ * sending the map to the service a different act from sending the code: the
+ * service learns that a function called `price_of` exists at line 40 of a named
+ * file and what calls it, and it never learns what that function says.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });

package/out/machine/codemap/files.d.ts

@@ -0,0 +1,45 @@
+/** How many files one project may have indexed. A ceiling, not a guess: a
+ *  monorepo of forty thousand files would take minutes to read and produce a
+ *  map too large to be worth sending, and half a map that claims to be whole is
+ *  the failure this whole feature exists to avoid. Past this the map says
+ *  plainly that it covers part of the project. */
+export declare const MAX_FILES = 6000;
+/** How large a single file may be before it is skipped. A generated bundle or
+ *  a checked-in minified file is not code anybody navigates. */
+export declare const MAX_FILE_BYTES = 1500000;
+export declare const PY_EXT: Set<string>;
+export declare const TS_EXT: Set<string>;
+export interface IndexedFile {
+    /** Path relative to the project root, forward slashes, always. */
+    rel: string;
+    size: number;
+    mtimeMs: number;
+    lang: "python" | "typescript" | "translations";
+}
+/** Every file the map covers, in a stable order.
+ *
+ *  Sorted, and that is not cosmetic: the fingerprint below is a hash over this
+ *  list, so an unstable order would report the code had moved every time the
+ *  filesystem felt like returning entries differently. */
+export declare function indexedFiles(root: string): IndexedFile[];
+/** One short string per file that changes whenever that file does.
+ *
+ *  Size and modified time rather than a hash of the contents: reading six
+ *  thousand files to find out whether any of them changed costs more than
+ *  rebuilding the map. A file edited back to its original contents within the
+ *  same millisecond would be missed, which is a case that does not occur in
+ *  practice and is corrected by the next ordinary edit. */
+export declare function stampOf(file: IndexedFile): string;
+/** One short string for the whole project. Saying the code has not moved is a
+ *  claim, and this is the evidence for it. */
+export declare function fingerprint(files: IndexedFile[]): {
+    hash: string;
+    files: number;
+};
+/** The commit the project is standing on, when it is a repository at all.
+ *
+ *  Read with a plain file read rather than by running git, because this is
+ *  called at the start of every job and spawning a process there is a cost paid
+ *  on work that has nothing to do with the map. A project with no git still
+ *  gets a usable map; it simply cannot stamp an answer with a commit. */
+export declare function headCommit(root: string): string;

package/out/machine/codemap/files.js

@@ -0,0 +1,207 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TS_EXT = exports.PY_EXT = exports.MAX_FILE_BYTES = exports.MAX_FILES = void 0;
+exports.indexedFiles = indexedFiles;
+exports.stampOf = stampOf;
+exports.fingerprint = fingerprint;
+exports.headCommit = headCommit;
+/**
+ * What the code map covers, and how to tell in milliseconds whether the code
+ * has moved since it was last read.
+ *
+ * Everything here runs on the person's own machine, because it is the one place
+ * their source files exist. Nothing in this folder decides, prices or
+ * classifies anything: it reads files and produces facts. That is the division
+ * the whole product rests on, and a function here that formed an opinion would
+ * be the first crack in it.
+ */
+const crypto = __importStar(require("crypto"));
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const read_locales_1 = require("./read-locales");
+/** Folders never worth reading: somebody else's code, or build output. */
+const SKIP_DIRS = new Set([
+    "node_modules", ".git", "__pycache__", ".venv", "venv", "env",
+    "site-packages", "dist", "build", ".pytest_cache", ".mypy_cache",
+    "htmlcov", ".ruff_cache", "coverage", ".vite", ".next", ".turbo",
+    "playwright-report", "test-results", "vendor", "target", ".cache",
+    "scratchpad", ".comprism", ".completionprism",
+]);
+/** Folder names that are build output only when they sit beside a package
+ *  manifest. `out` is where TypeScript writes its JavaScript by convention, but
+ *  it is also a perfectly ordinary name for a source folder, so skipping it
+ *  everywhere would quietly remove real code from somebody's map. Next to a
+ *  package.json it is almost always generated.
+ *
+ *  Found by mapping this repository and getting 83 changed files from one edit:
+ *  the terminal client's own compiled output was in the map, so every rebuild
+ *  looked like eighty three source files moving. */
+const BUILT_BESIDE_MANIFEST = new Set(["out", "output", "lib", "es", "esm"]);
+/** How many files one project may have indexed. A ceiling, not a guess: a
+ *  monorepo of forty thousand files would take minutes to read and produce a
+ *  map too large to be worth sending, and half a map that claims to be whole is
+ *  the failure this whole feature exists to avoid. Past this the map says
+ *  plainly that it covers part of the project. */
+exports.MAX_FILES = 6000;
+/** How large a single file may be before it is skipped. A generated bundle or
+ *  a checked-in minified file is not code anybody navigates. */
+exports.MAX_FILE_BYTES = 1_500_000;
+exports.PY_EXT = new Set([".py"]);
+exports.TS_EXT = new Set([".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs"]);
+function isCode(rel, name) {
+    if (name.endsWith(".d.ts"))
+        return null;
+    if (name.endsWith(".min.js"))
+        return null;
+    const ext = path.extname(name);
+    if (exports.PY_EXT.has(ext))
+        return "python";
+    if (exports.TS_EXT.has(ext))
+        return "typescript";
+    // Translation files. Not code, and in the map for a reason no code parser can
+    // serve: the label a person sees is a key here and a sentence in ten language
+    // files, and "which languages are missing this one" is the question asked
+    // about it. See `read-locales.ts`.
+    if ((0, read_locales_1.looksLikeTranslations)(rel))
+        return "translations";
+    return null;
+}
+/** Every file the map covers, in a stable order.
+ *
+ *  Sorted, and that is not cosmetic: the fingerprint below is a hash over this
+ *  list, so an unstable order would report the code had moved every time the
+ *  filesystem felt like returning entries differently. */
+function indexedFiles(root) {
+    const out = [];
+    const walk = (dir) => {
+        if (out.length >= exports.MAX_FILES)
+            return;
+        let entries;
+        try {
+            entries = fs.readdirSync(dir, { withFileTypes: true });
+        }
+        catch {
+            return;
+        }
+        for (const entry of entries) {
+            if (out.length >= exports.MAX_FILES)
+                return;
+            const full = path.join(dir, entry.name);
+            if (entry.isDirectory()) {
+                if (entry.name.startsWith(".") && entry.name !== ".github")
+                    continue;
+                if (SKIP_DIRS.has(entry.name))
+                    continue;
+                if (BUILT_BESIDE_MANIFEST.has(entry.name)
+                    && fs.existsSync(path.join(dir, "package.json")))
+                    continue;
+                walk(full);
+                continue;
+            }
+            if (!entry.isFile())
+                continue;
+            const rel = path.relative(root, full).split(path.sep).join("/");
+            const lang = isCode(rel, entry.name);
+            if (!lang)
+                continue;
+            let stat;
+            try {
+                stat = fs.statSync(full);
+            }
+            catch {
+                continue;
+            }
+            if (stat.size > exports.MAX_FILE_BYTES)
+                continue;
+            out.push({
+                rel,
+                size: stat.size,
+                mtimeMs: Math.round(stat.mtimeMs),
+                lang,
+            });
+        }
+    };
+    walk(root);
+    out.sort((a, b) => (a.rel < b.rel ? -1 : a.rel > b.rel ? 1 : 0));
+    return out;
+}
+/** One short string per file that changes whenever that file does.
+ *
+ *  Size and modified time rather than a hash of the contents: reading six
+ *  thousand files to find out whether any of them changed costs more than
+ *  rebuilding the map. A file edited back to its original contents within the
+ *  same millisecond would be missed, which is a case that does not occur in
+ *  practice and is corrected by the next ordinary edit. */
+function stampOf(file) {
+    return `${file.size}:${file.mtimeMs}`;
+}
+/** One short string for the whole project. Saying the code has not moved is a
+ *  claim, and this is the evidence for it. */
+function fingerprint(files) {
+    const hash = crypto.createHash("sha1");
+    for (const file of files)
+        hash.update(`${file.rel}:${stampOf(file)}\n`);
+    return { hash: hash.digest("hex").slice(0, 16), files: files.length };
+}
+/** The commit the project is standing on, when it is a repository at all.
+ *
+ *  Read with a plain file read rather than by running git, because this is
+ *  called at the start of every job and spawning a process there is a cost paid
+ *  on work that has nothing to do with the map. A project with no git still
+ *  gets a usable map; it simply cannot stamp an answer with a commit. */
+function headCommit(root) {
+    try {
+        const head = fs.readFileSync(path.join(root, ".git", "HEAD"), "utf8").trim();
+        if (!head.startsWith("ref:"))
+            return head.slice(0, 40);
+        const ref = head.slice(4).trim();
+        const direct = path.join(root, ".git", ref);
+        if (fs.existsSync(direct))
+            return fs.readFileSync(direct, "utf8").trim().slice(0, 40);
+        const packed = path.join(root, ".git", "packed-refs");
+        if (fs.existsSync(packed)) {
+            for (const line of fs.readFileSync(packed, "utf8").split("\n")) {
+                const [sha, name] = line.trim().split(" ");
+                if (name === ref && sha)
+                    return sha.slice(0, 40);
+            }
+        }
+    }
+    catch {
+        /* not a repository, or an unusual layout. Neither is an error. */
+    }
+    return "";
+}

package/out/machine/codemap/read-locales.d.ts

@@ -0,0 +1,29 @@
+import type { FactSet } from "./facts";
+/** Is this JSON file a translation file?
+ *
+ *  Both conditions, not either: a `data/en.json` of sample records is not a
+ *  translation file, and a `locales/schema.json` is not one either. Indexing
+ *  the wrong JSON would fill the map with things nobody asked about. */
+export declare function looksLikeTranslations(rel: string): boolean;
+/** The language a translation file is for, from its own name. */
+export declare function languageOf(rel: string): string;
+interface KeyAt {
+    key: string;
+    line: number;
+}
+/**
+ * Every leaf key in a translation file, with the line it sits on.
+ *
+ * A single pass over the text, tracking strings, escapes, nesting and lines. It
+ * reads the structure rather than the values: what a translation SAYS never
+ * leaves the machine, only that the key exists and where.
+ */
+export declare function scanKeys(text: string): KeyAt[];
+export declare function readLocales(root: string, rels: string[]): {
+    facts: Record<string, FactSet>;
+    failures: {
+        file: string;
+        reason: string;
+    }[];
+};
+export {};