@dmthepm/commune 0.1.0

package/lib/cli/related.js

@@ -0,0 +1,177 @@
+/**
+ * `commune graph related` — what a piece of text connects to.
+ *
+ * This is the primitive the connect step (#19) and the authoring skills (#10)
+ * are written against, so v1 is deliberately deterministic: two buckets, both
+ * computable from what the core already does, and no ranking.
+ *
+ *   `links`    — every edge the text actually contains, with its resolution.
+ *   `mentions` — entries whose title or alias appears as a whole word in the
+ *                prose, which is the "you wrote about this without linking it"
+ *                signal a connect step needs.
+ *
+ * Fuzzy and semantic similarity are out of scope on purpose. They are product
+ * direction, not a build decision, and inventing a ranking here would freeze it
+ * into the contract before anyone chose it.
+ */
+import { readFile, stat } from 'node:fs/promises';
+import path from 'node:path';
+import matter from 'gray-matter';
+import { buildGraph, buildLinkLookup, buildUrlLookup, extractLinks, loadContentEntries, resolveLink, stripCode, } from "../lib/graph.js";
+import { SCHEMA, writeJson, writeLines } from "./render.js";
+import { EXIT_OK, failure } from "./errors.js";
+/** Shortest name worth matching. Below this, a mention is noise. */
+const MIN_MENTION_LENGTH = 3;
+function toReference(entry) {
+    return {
+        urlPath: entry.urlPath,
+        title: entry.title,
+        collection: entry.collection,
+        file: entry.file,
+    };
+}
+/**
+ * Work out whether the positional is a path, stdin, or literal prose.
+ *
+ * A path is tried against the root before the cwd, because the root is the
+ * vault being asked about and the cwd is wherever the operator happens to be
+ * standing. `-` is stdin, which is how a draft that is not a file yet gets
+ * asked "what does this connect to".
+ */
+async function readSource(input, root) {
+    if (input === '-') {
+        const text = await readStdin();
+        const { content, data } = matter(text);
+        return { kind: 'stdin', text: content, frontmatter: data };
+    }
+    const candidates = path.isAbsolute(input)
+        ? [input]
+        : [path.join(root, input), path.resolve(input)];
+    for (const candidate of candidates) {
+        if (!(await isFile(candidate)))
+            continue;
+        let source;
+        try {
+            source = await readFile(candidate, 'utf8');
+        }
+        catch (error) {
+            throw failure('EPARSE', `cannot read ${candidate}: ${error.message}`);
+        }
+        let parsed;
+        try {
+            parsed = matter(source);
+        }
+        catch (error) {
+            throw failure('EPARSE', `cannot parse frontmatter in ${candidate}: ${error.message}`);
+        }
+        const relative = path.relative(root, candidate).split(path.sep).join('/');
+        return {
+            kind: 'file',
+            file: relative,
+            text: parsed.content,
+            frontmatter: parsed.data,
+        };
+    }
+    return { kind: 'text', text: input, frontmatter: {} };
+}
+async function isFile(candidate) {
+    try {
+        return (await stat(candidate)).isFile();
+    }
+    catch {
+        return false;
+    }
+}
+async function readStdin() {
+    const chunks = [];
+    for await (const chunk of process.stdin)
+        chunks.push(chunk);
+    return Buffer.concat(chunks).toString('utf8');
+}
+/** Escape a title so it can be matched literally: real titles contain `,`, `'`, `(`. */
+function escapeRegExp(value) {
+    return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+/**
+ * Count whole-word occurrences of `name` in `text`.
+ *
+ * Lookarounds rather than `\b`, because `\b` is defined relative to the
+ * character next to it: a title ending in `?` or `)` puts a non-word character
+ * where `\b` expects a word one, and the match silently stops happening.
+ */
+function countMentions(text, name) {
+    const pattern = new RegExp(`(?<!\\w)${escapeRegExp(name)}(?!\\w)`, 'gi');
+    return text.match(pattern)?.length ?? 0;
+}
+export async function relatedCommand(root, input, json) {
+    const entries = await loadContentEntries({ root });
+    const graph = buildGraph(entries);
+    const byName = buildLinkLookup(entries);
+    const byUrl = buildUrlLookup(entries);
+    const byUrlPath = new Map(entries.map((entry) => [entry.urlPath, entry]));
+    const source = await readSource(input, root);
+    // A file that happens to be a graph entry gets its inbound links too; a
+    // draft outside the vault, or raw text, has none by definition.
+    const self = source.file ? entries.find((entry) => entry.file === source.file) : undefined;
+    const links = extractLinks(source.text, source.frontmatter).map((link) => {
+        const target = resolveLink(link, byName, byUrl);
+        const entry = target ? byUrlPath.get(target.urlPath) : undefined;
+        return {
+            kind: link.kind,
+            target: link.target,
+            resolved: entry ? toReference(entry) : null,
+        };
+    });
+    const prose = stripCode(source.text);
+    const mentions = entries
+        .filter((entry) => entry.urlPath !== self?.urlPath)
+        .map((entry) => {
+        const names = [entry.title, ...entry.aliases].filter((name) => name.length >= MIN_MENTION_LENGTH);
+        let count = 0;
+        let matched;
+        for (const name of names) {
+            const hits = countMentions(prose, name);
+            if (!hits)
+                continue;
+            count += hits;
+            matched ??= name.toLowerCase();
+        }
+        return { ...toReference(entry), matched, count };
+    })
+        .filter((mention) => mention.count > 0)
+        .sort((a, b) => b.count - a.count || a.title.localeCompare(b.title) || a.urlPath.localeCompare(b.urlPath));
+    const inbound = (self ? graph.nodes[self.urlPath].inbound : [])
+        .map((urlPath) => byUrlPath.get(urlPath))
+        .filter((entry) => Boolean(entry))
+        .map(toReference);
+    const summary = {
+        links: links.length,
+        resolved: links.filter((link) => link.resolved).length,
+        unresolved: links.filter((link) => !link.resolved).length,
+        mentions: mentions.length,
+        inbound: inbound.length,
+    };
+    if (json) {
+        writeJson({
+            schema: SCHEMA,
+            root,
+            source: {
+                kind: source.kind,
+                ...(source.file ? { file: source.file } : {}),
+                ...(self ? { urlPath: self.urlPath } : {}),
+            },
+            links,
+            mentions,
+            inbound,
+            summary,
+        });
+        return EXIT_OK;
+    }
+    writeLines([
+        ...links.map((link) => `link\t${link.target}\t${link.resolved ? link.resolved.urlPath : 'UNRESOLVED'}`),
+        ...mentions.map((mention) => `mention\t${mention.urlPath}\t${mention.count}`),
+        ...inbound.map((entry) => `inbound\t${entry.urlPath}`),
+        `${summary.links} links (${summary.resolved} resolved), ${summary.mentions} mentions, ${summary.inbound} inbound`,
+    ]);
+    return EXIT_OK;
+}

package/lib/cli/render.d.ts

@@ -0,0 +1,20 @@
+/**
+ * How a command's result reaches the caller.
+ *
+ * One rule, and everything else follows from it: in `--json` mode stdout holds
+ * exactly one JSON document and nothing else. Progress, warnings and errors go
+ * to stderr, so `commune check --json > findings.json` yields a parseable file
+ * even on the run that fails.
+ */
+import type { CliError } from './errors.ts';
+/** Every payload carries the contract version, so #10's skills can pin it. */
+export declare const SCHEMA = 1;
+export declare function writeJson(payload: unknown): void;
+export declare function writeLines(lines: string[]): void;
+/**
+ * Render a failure, on stderr, in whichever mode the caller asked for.
+ *
+ * `json` is read from the raw argv rather than the parsed options, because the
+ * errors most worth rendering as JSON are the ones thrown while parsing.
+ */
+export declare function writeError(error: CliError, json: boolean): void;

package/lib/cli/render.js

@@ -0,0 +1,32 @@
+/**
+ * How a command's result reaches the caller.
+ *
+ * One rule, and everything else follows from it: in `--json` mode stdout holds
+ * exactly one JSON document and nothing else. Progress, warnings and errors go
+ * to stderr, so `commune check --json > findings.json` yields a parseable file
+ * even on the run that fails.
+ */
+/** Every payload carries the contract version, so #10's skills can pin it. */
+export const SCHEMA = 1;
+export function writeJson(payload) {
+    process.stdout.write(JSON.stringify(payload, null, 2) + '\n');
+}
+export function writeLines(lines) {
+    if (lines.length)
+        process.stdout.write(lines.join('\n') + '\n');
+}
+/**
+ * Render a failure, on stderr, in whichever mode the caller asked for.
+ *
+ * `json` is read from the raw argv rather than the parsed options, because the
+ * errors most worth rendering as JSON are the ones thrown while parsing.
+ */
+export function writeError(error, json) {
+    if (json) {
+        process.stderr.write(JSON.stringify({ error: { code: error.code, message: error.message } }, null, 2) + '\n');
+        return;
+    }
+    process.stderr.write(`commune: ${error.message}\n`);
+    if (error.detail)
+        process.stderr.write(`${error.detail}\n`);
+}

package/lib/cli/root.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Resolving `--root`.
+ *
+ * `--root` names the *project* root: the directory that contains `src/content`.
+ * Not `src/content` itself — every slug and every `file` value in the graph is
+ * derived by stripping `src/content/<collection>/` off the front of a path, so
+ * a root one level too deep produces a complete, confident, wrong answer with
+ * no error anywhere. Requiring `src/content` to exist under the root turns that
+ * silent corruption into an exit 1 on the first command.
+ */
+export declare function resolveRoot(value: string | undefined): Promise<string>;

package/lib/cli/root.js

@@ -0,0 +1,28 @@
+/**
+ * Resolving `--root`.
+ *
+ * `--root` names the *project* root: the directory that contains `src/content`.
+ * Not `src/content` itself — every slug and every `file` value in the graph is
+ * derived by stripping `src/content/<collection>/` off the front of a path, so
+ * a root one level too deep produces a complete, confident, wrong answer with
+ * no error anywhere. Requiring `src/content` to exist under the root turns that
+ * silent corruption into an exit 1 on the first command.
+ */
+import { stat } from 'node:fs/promises';
+import path from 'node:path';
+import { failure } from "./errors.js";
+export async function resolveRoot(value) {
+    const root = path.resolve(value ?? process.cwd());
+    const content = path.join(root, 'src', 'content');
+    let stats;
+    try {
+        stats = await stat(content);
+    }
+    catch {
+        throw failure('ENOCONTENT', `${root} is not a project root: no src/content directory. --root names the directory that contains src/content, not src/content itself.`);
+    }
+    if (!stats.isDirectory()) {
+        throw failure('ENOCONTENT', `${content} is not a directory`);
+    }
+    return root;
+}

package/lib/cli/usage.d.ts

@@ -0,0 +1,3 @@
+/** Usage text. Hand-written: `parseArgs` generates none, which is its one real cost. */
+export declare const USAGE = "commune \u2014 query the content graph without an Astro process\n\nUsage:\n  commune [--root <dir>] graph query   [filters] [--json]\n  commune [--root <dir>] graph related <path|text|-> [--json]\n  commune [--root <dir>] check         [--json]\n  commune [--root <dir>] gate          [--dist <dir>] [--json]\n  commune --version\n\nGlobal options:\n  --root <dir>   Project root: the directory containing src/content. Default: cwd.\n  --json         Emit one JSON document on stdout. Everything else goes to stderr.\n  --help         Show this text.\n  --version      Print the version of the installed package and exit.\n\ngraph query filters (any-of within a flag, all-of across flags):\n  --collection <notes|research|pages>   Repeatable.\n  --tag <tag>                           Repeatable.\n  --status <status>\n  --orphans                             Zero inbound and zero outbound.\n  --deadends                            Zero outbound.\n\ngate options:\n  --dist <dir>   The built site to check, relative to --root. Default: dist.\n\nExit codes:\n  0  finished, findings or not\n  1  could not finish\n  2  invalid invocation\n\n  gate is the one exception, and the only verb whose exit code encodes a\n  finding: it exits 1 when the build it checked is wrong. That is what a gate\n  is for \u2014 a build stops on a non-zero exit \u2014 so gate cannot report a finding\n  the way every other verb does, in the payload with exit 0.";
+export declare const COMMAND_USAGE: Record<string, string>;

package/lib/cli/usage.js

@@ -0,0 +1,46 @@
+/** Usage text. Hand-written: `parseArgs` generates none, which is its one real cost. */
+export const USAGE = `commune — query the content graph without an Astro process
+
+Usage:
+  commune [--root <dir>] graph query   [filters] [--json]
+  commune [--root <dir>] graph related <path|text|-> [--json]
+  commune [--root <dir>] check         [--json]
+  commune [--root <dir>] gate          [--dist <dir>] [--json]
+  commune --version
+
+Global options:
+  --root <dir>   Project root: the directory containing src/content. Default: cwd.
+  --json         Emit one JSON document on stdout. Everything else goes to stderr.
+  --help         Show this text.
+  --version      Print the version of the installed package and exit.
+
+graph query filters (any-of within a flag, all-of across flags):
+  --collection <notes|research|pages>   Repeatable.
+  --tag <tag>                           Repeatable.
+  --status <status>
+  --orphans                             Zero inbound and zero outbound.
+  --deadends                            Zero outbound.
+
+gate options:
+  --dist <dir>   The built site to check, relative to --root. Default: dist.
+
+Exit codes:
+  0  finished, findings or not
+  1  could not finish
+  2  invalid invocation
+
+  gate is the one exception, and the only verb whose exit code encodes a
+  finding: it exits 1 when the build it checked is wrong. That is what a gate
+  is for — a build stops on a non-zero exit — so gate cannot report a finding
+  the way every other verb does, in the payload with exit 0.`;
+export const COMMAND_USAGE = {
+    'graph query': 'Usage: commune [--root <dir>] graph query [--collection <c>]... [--tag <t>]... [--status <s>] [--orphans] [--deadends] [--json]',
+    'graph related': 'Usage: commune [--root <dir>] graph related <path|text|-> [--json]',
+    check: 'Usage: commune [--root <dir>] check [--json]',
+    gate: `Usage: commune [--root <dir>] gate [--dist <dir>] [--json]
+
+Run after a build. Asserts that every standalone page is in the search index,
+that every resolving WikiLink uses its target's exact title, and that WikiLinks
+to standalone pages rendered as hrefs. Exits 1 if any of that is false — the
+one verb whose exit code encodes a finding.`,
+};

package/lib/cli/version.d.ts

@@ -0,0 +1,24 @@
+/**
+ * The version this build answers to.
+ *
+ * Read out of `package.json` at run time rather than baked in at build time.
+ * There is nothing in the build that could bake it in — `tsc` compiles types
+ * away, not values — and release-please bumps exactly one file when it cuts a
+ * release. A second copy of the number would be a second thing to forget.
+ *
+ * `../../package.json` survives compilation because `src/cli/` and `lib/cli/`
+ * sit at the same depth: `tsconfig.build.json` sets `rootDir: src`, so this
+ * file becomes `lib/cli/version.js` and two levels up is the package root
+ * exactly as two levels up from `src/cli/version.ts` is the repo root. Only the
+ * `lib/` layout is ever exercised — nothing imports `src/cli/*.ts` directly,
+ * and the tests reach this code the way a consumer does, by spawning
+ * `bin/commune.mjs`, which imports `../lib/cli/main.js`. The `src/` half of
+ * that sentence is a property of the build, not a path anything walks.
+ *
+ * It resolves inside an installed `node_modules/@dmthepm/commune` too, and for
+ * a reason worth writing down: npm packs `package.json` into every tarball
+ * regardless of the `files` allowlist, so it is there even though `files` never
+ * names it.
+ */
+/** The `version` field of the package this file was loaded from. */
+export declare function readVersion(): Promise<string>;

package/lib/cli/version.js

@@ -0,0 +1,29 @@
+/**
+ * The version this build answers to.
+ *
+ * Read out of `package.json` at run time rather than baked in at build time.
+ * There is nothing in the build that could bake it in — `tsc` compiles types
+ * away, not values — and release-please bumps exactly one file when it cuts a
+ * release. A second copy of the number would be a second thing to forget.
+ *
+ * `../../package.json` survives compilation because `src/cli/` and `lib/cli/`
+ * sit at the same depth: `tsconfig.build.json` sets `rootDir: src`, so this
+ * file becomes `lib/cli/version.js` and two levels up is the package root
+ * exactly as two levels up from `src/cli/version.ts` is the repo root. Only the
+ * `lib/` layout is ever exercised — nothing imports `src/cli/*.ts` directly,
+ * and the tests reach this code the way a consumer does, by spawning
+ * `bin/commune.mjs`, which imports `../lib/cli/main.js`. The `src/` half of
+ * that sentence is a property of the build, not a path anything walks.
+ *
+ * It resolves inside an installed `node_modules/@dmthepm/commune` too, and for
+ * a reason worth writing down: npm packs `package.json` into every tarball
+ * regardless of the `files` allowlist, so it is there even though `files` never
+ * names it.
+ */
+import { readFile } from 'node:fs/promises';
+/** The `version` field of the package this file was loaded from. */
+export async function readVersion() {
+    const manifest = new URL('../../package.json', import.meta.url);
+    const { version } = JSON.parse(await readFile(manifest, 'utf8'));
+    return version;
+}

package/lib/integration.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Astro integration for building backlinks and graph data at build time, and
+ * for writing each entry's source markdown beside its rendered page.
+ *
+ * Thin by design: it owns *when* the graph is built and *where* the artifacts
+ * are written, and nothing else. Which content exists, how links resolve, how
+ * stars are ranked and what counts as a finding are all decided by the graph
+ * core in `src/lib/graph.ts`, so the `commune` CLI produces the same graph
+ * without an Astro process anywhere in sight.
+ */
+import type { AstroIntegration } from 'astro';
+/**
+ * Options for the integration.
+ *
+ * Empty, and deliberately so: everything the graph needs is already in the
+ * consumer's `defineConfig` — the root to read content from, the public
+ * directory to write the index to — and reading it from there is what stops a
+ * second, drifting copy of Astro's own configuration existing. The parameter
+ * is here so that the day something *is* configurable, `commune({ … })` is
+ * already the spelling in every consumer's config.
+ */
+export interface CommuneOptions {
+}
+export default function commune(_options?: CommuneOptions): AstroIntegration;

package/lib/integration.js

@@ -0,0 +1,111 @@
+/**
+ * Astro integration for building backlinks and graph data at build time, and
+ * for writing each entry's source markdown beside its rendered page.
+ *
+ * Thin by design: it owns *when* the graph is built and *where* the artifacts
+ * are written, and nothing else. Which content exists, how links resolve, how
+ * stars are ranked and what counts as a finding are all decided by the graph
+ * core in `src/lib/graph.ts`, so the `commune` CLI produces the same graph
+ * without an Astro process anywhere in sight.
+ */
+import { copyFile, writeFile, mkdir } from 'node:fs/promises';
+import { fileURLToPath } from 'node:url';
+import path from 'node:path';
+import { buildGraph, formatDiagnostic, loadContentEntries, toBacklinksJson, toMarkdownPath, } from "./lib/graph.js";
+function buildBacklinksGraph(entries, logger) {
+    const graph = buildGraph(entries);
+    logger.info(`📝 Found ${Object.keys(graph.nodes).length} public content entries`);
+    for (const diagnostic of graph.diagnostics) {
+        logger.warn(formatDiagnostic(diagnostic));
+    }
+    const starred = Object.values(graph.nodes).filter((note) => note.isStarred).length;
+    if (starred > 0) {
+        logger.info(`⭐ ${starred} notes starred (top 5%)`);
+    }
+    return graph;
+}
+async function writeBacklinksFile(filePath, graph) {
+    await mkdir(path.dirname(filePath), { recursive: true });
+    await writeFile(filePath, JSON.stringify(graph, null, 2) + '\n');
+}
+/**
+ * Copy every entry's source file next to its rendered page, at `<url>.md`.
+ *
+ * `copyFile` rather than a re-serialization on purpose: the promise of the `.md`
+ * URL is that it returns *the file*, frontmatter and `[[wikilinks]]` untouched,
+ * so anything that reformats on the way through would break it. The graph
+ * already decided which entries exist and where each one lives; this only moves
+ * bytes.
+ */
+async function writeMarkdownFiles(entries, root, outDir) {
+    for (const entry of entries) {
+        const destination = path.join(outDir, toMarkdownPath(entry.urlPath));
+        await mkdir(path.dirname(destination), { recursive: true });
+        // `entry.file` is relative to the project root — the graph's promise —
+        // so the read is joined to that root and not left to the cwd. They are
+        // the same directory when someone runs `astro build` in their project
+        // and different the moment they do not.
+        await copyFile(path.join(root, entry.file), destination);
+    }
+    return entries.length;
+}
+function summarize(graph) {
+    return `📊 ${graph.totalBacklinks} total backlinks across ${Object.keys(graph.nodes).length} entries`;
+}
+export default function commune(_options = {}) {
+    // `astro:build:done` is not handed the resolved config, so the two paths
+    // the graph needs are captured in `astro:config:setup` — which Astro always
+    // runs first — and closed over. Per-instance rather than module-level: one
+    // `commune()` call belongs to one Astro config, so two projects built in
+    // one process do not overwrite each other's roots.
+    let root;
+    let publicBacklinks;
+    return {
+        name: 'commune-backlinks',
+        hooks: {
+            // Write the public artifact before pages render. Note pages import
+            // backlinks.json at build time, so it has to be fresh on disk first —
+            // otherwise the build bakes in whatever the previous run left behind.
+            'astro:config:setup': async ({ config, logger }) => {
+                logger.info('🔗 Building public backlinks index...');
+                // Both are URLs, and `.pathname` percent-encodes: a project
+                // checked out under a path with a space would read from a
+                // literal `%20` directory and find no content at all.
+                // `fileURLToPath` is what Astro's own docs use.
+                root = fileURLToPath(config.root);
+                publicBacklinks = fileURLToPath(new URL('./backlinks.json', config.publicDir));
+                try {
+                    const graph = buildBacklinksGraph(await loadContentEntries({ root }), logger);
+                    await writeBacklinksFile(publicBacklinks, toBacklinksJson(graph));
+                    logger.info(`✅ Backlinks index written to ${path.relative(root, publicBacklinks)}`);
+                    logger.info(summarize(graph));
+                }
+                catch (error) {
+                    logger.error('❌ Failed to build public backlinks index:');
+                    logger.error(String(error));
+                    throw error;
+                }
+            },
+            'astro:build:done': async ({ dir, logger }) => {
+                logger.info('🔗 Building backlinks index...');
+                try {
+                    const entries = await loadContentEntries({ root });
+                    const graph = buildBacklinksGraph(entries, logger);
+                    const json = toBacklinksJson(graph);
+                    await writeBacklinksFile(fileURLToPath(new URL('./backlinks.json', dir)), json);
+                    // Also written to the public directory, for dev server parity.
+                    await writeBacklinksFile(publicBacklinks, json);
+                    const written = await writeMarkdownFiles(entries, root, fileURLToPath(dir));
+                    logger.info(`✅ Backlinks index written to /backlinks.json (dist + public)`);
+                    logger.info(`📄 ${written} source files written as .md alongside their pages`);
+                    logger.info(summarize(graph));
+                }
+                catch (error) {
+                    logger.error('❌ Failed to build backlinks index:');
+                    logger.error(String(error));
+                    throw error;
+                }
+            },
+        },
+    };
+}