@plebeiantech/lat.md 0.12.3-fork.4 → 0.12.3-fork.6

package/dist/src/fork/upstream-guard.js

@@ -0,0 +1,445 @@
+import { spawnSync } from 'node:child_process';
+import { existsSync, readFileSync, writeFileSync } from 'node:fs';
+import { isAbsolute, resolve } from 'node:path';
+// @lat: [[fork#The upstream guard]]
+export const SYNC_POINT_FILE = 'fork-upstream-sync-point';
+/**
+ * The commit this fork was originally cut from. It is not the guard's
+ * baseline — see [[fork#The upstream guard#The sync point]] for why a frozen
+ * one cannot tell the fork's changes from upstream's.
+ */
+export const ORIGINAL_FORK_POINT = '65d2f5b220853b2b10dca3de18d511f575bc12c7';
+// @lat: [[fork#The upstream guard#The allowlist]]
+export const ALLOWLIST_FILE = 'fork-upstream-allowlist.tsv';
+export const REASON_PLACEHOLDER = 'TODO';
+const ALLOWLIST_HEADER = [
+    '# Upstream files this fork is permitted to modify.',
+    '#',
+    '# Generated by `node dist/src/fork/upstream-guard-cli.js --regenerate`.',
+    '# One entry per line: the repo-relative path, a TAB, then the reason the',
+    '# fork must diverge in that file. A placeholder reason fails the check —',
+    '# regenerate writes the marker, a person replaces it with the real reason.',
+    '#',
+    '# Paths are relative to the sync point in fork-upstream-sync-point, so an',
+    '# upstream merge moves this list. Regenerate in the same commit.',
+    '#',
+    '# Deleting an upstream file is never allowlistable. See lat.md/fork.md.',
+    '',
+];
+export function parseAllowlist(text) {
+    const entries = [];
+    const errors = [];
+    const seen = new Map();
+    text.split('\n').forEach((raw, index) => {
+        const line = index + 1;
+        const stripped = raw.replace(/\r$/, '');
+        if (stripped.trim() === '' || stripped.trimStart().startsWith('#'))
+            return;
+        const tab = stripped.indexOf('\t');
+        const split = tab >= 0 ? tab : stripped.search(/\s/);
+        if (split < 0) {
+            errors.push(`${ALLOWLIST_FILE}:${line}: entry has no reason: ${stripped}`);
+            return;
+        }
+        const path = stripped.slice(0, split).trim();
+        const reason = stripped.slice(split + 1).trim();
+        if (path === '') {
+            errors.push(`${ALLOWLIST_FILE}:${line}: entry has no path`);
+            return;
+        }
+        if (reason === '') {
+            errors.push(`${ALLOWLIST_FILE}:${line}: entry has no reason: ${path}`);
+            return;
+        }
+        const first = seen.get(path);
+        if (first !== undefined) {
+            errors.push(`${ALLOWLIST_FILE}:${line}: duplicate entry for ${path} ` +
+                `(first at line ${first})`);
+            return;
+        }
+        seen.set(path, line);
+        entries.push({ path, reason, line });
+    });
+    return { entries, errors };
+}
+export function renderAllowlist(entries) {
+    const sorted = [...entries].sort((a, b) => (a.path < b.path ? -1 : 1));
+    const body = sorted.map((e) => `${e.path}\t${e.reason}`);
+    return [...ALLOWLIST_HEADER, ...body, ''].join('\n');
+}
+export function needsReason(reason) {
+    return reason.trimStart().toUpperCase().startsWith(REASON_PLACEHOLDER);
+}
+export function parseNameStatus(raw) {
+    const fields = raw.split('\0').filter((f) => f !== '');
+    const out = [];
+    for (let i = 0; i + 1 < fields.length; i += 2) {
+        out.push({ status: fields[i], path: fields[i + 1] });
+    }
+    return out;
+}
+export function classify(input) {
+    const findings = [];
+    const touched = [];
+    for (const entry of input.diff) {
+        if (!input.upstream.has(entry.path))
+            continue;
+        touched.push(entry.path);
+        if (entry.status.startsWith('D')) {
+            findings.push({
+                kind: 'deletion',
+                path: entry.path,
+                message: `${entry.path}: deletes an upstream file.\n` +
+                    '    No allowlist entry can permit this. A file this fork deletes ' +
+                    'conflicts on\n' +
+                    '    every future merge that touches it upstream; a file this fork ' +
+                    'modifies\n' +
+                    '    conflicts once and stays resolved. Leave the file in place and ' +
+                    'change\n' +
+                    '    what it does, or turn it off out of band.',
+            });
+            continue;
+        }
+        const reason = input.allowlist.get(entry.path);
+        if (reason === undefined) {
+            findings.push({
+                kind: 'unallowlisted',
+                path: entry.path,
+                message: `${entry.path}: modifies an upstream file that is not allowlisted.\n` +
+                    '    Move the change into a new fork-owned file, or add the path to ' +
+                    `${ALLOWLIST_FILE}\n` +
+                    '    with the reason the fork must diverge here.',
+            });
+            continue;
+        }
+        if (needsReason(reason)) {
+            findings.push({
+                kind: 'missing-reason',
+                path: entry.path,
+                message: `${entry.path}: allowlisted with a placeholder reason.\n` +
+                    `    Replace the ${REASON_PLACEHOLDER} in ${ALLOWLIST_FILE} with why ` +
+                    'this file must diverge.',
+            });
+        }
+    }
+    const touchedSet = new Set(touched);
+    const warnings = [];
+    for (const path of input.allowlist.keys()) {
+        if (!touchedSet.has(path)) {
+            warnings.push(`${path}: allowlisted but no longer modified — regenerate to drop it.`);
+        }
+    }
+    return { findings, warnings, touched };
+}
+function git(repo, args, stdin) {
+    const result = spawnSync('git', ['-C', repo, ...args], {
+        input: stdin,
+        encoding: 'utf-8',
+        maxBuffer: 64 * 1024 * 1024,
+    });
+    if (result.error)
+        throw result.error;
+    return result;
+}
+export function repoRoot(from) {
+    const result = git(from, ['rev-parse', '--show-toplevel']);
+    if (result.status !== 0) {
+        throw new Error(`not a git repository: ${from}`);
+    }
+    return result.stdout.trim();
+}
+export function hasCommit(repo, rev) {
+    return git(repo, ['cat-file', '-e', `${rev}^{commit}`]).status === 0;
+}
+export function diffAgainst(repo, rev) {
+    const result = git(repo, [
+        'diff',
+        '--no-renames',
+        '--name-status',
+        '-z',
+        rev,
+        '--',
+    ]);
+    if (result.status !== 0) {
+        throw new Error(`git diff against ${rev} failed: ${result.stderr.trim()}`);
+    }
+    return parseNameStatus(result.stdout);
+}
+export function upstreamPaths(repo, rev, paths) {
+    const out = new Set();
+    if (paths.length === 0)
+        return out;
+    const batchable = paths.filter((p) => !p.includes('\n'));
+    const input = batchable.map((p) => `${rev}:${p}`).join('\n') + '\n';
+    const result = git(repo, ['cat-file', '--batch-check'], input);
+    const lines = result.stdout.split('\n').filter((l) => l !== '');
+    const aligned = lines.length === batchable.length;
+    if (aligned) {
+        lines.forEach((line, index) => {
+            if (/ blob \d+$/.test(line))
+                out.add(batchable[index]);
+        });
+    }
+    const leftover = aligned ? paths.filter((p) => p.includes('\n')) : paths;
+    for (const path of leftover) {
+        const probe = git(repo, ['cat-file', '-t', `${rev}:${path}`]);
+        if (probe.status === 0 && probe.stdout.trim() === 'blob')
+            out.add(path);
+    }
+    return out;
+}
+// @lat: [[fork#The upstream guard#The sync point]]
+export function parseSyncPoint(text) {
+    for (const raw of text.split('\n')) {
+        const line = raw.replace(/\r$/, '').trim();
+        if (line === '' || line.startsWith('#'))
+            continue;
+        return line.split(/\s/)[0];
+    }
+    return undefined;
+}
+export function renderSyncPoint(previous, rev) {
+    const kept = previous
+        .split('\n')
+        .filter((raw) => {
+        const line = raw.replace(/\r$/, '').trim();
+        return line === '' || line.startsWith('#');
+    })
+        .join('\n')
+        .replace(/\n+$/, '');
+    return `${kept}\n\n${rev}\n`;
+}
+export function isAncestorOfHead(repo, rev) {
+    return git(repo, ['merge-base', '--is-ancestor', rev, 'HEAD']).status === 0;
+}
+export function isShallow(repo) {
+    const result = git(repo, ['rev-parse', '--is-shallow-repository']);
+    return result.stdout.trim() === 'true';
+}
+// @lat: [[fork#The upstream guard#Running it in CI]]
+export function fetchSyncPoint(repo, rev) {
+    const result = git(repo, ['fetch', '--no-tags', '--depth=1', 'origin', rev]);
+    return result.status === 0 && hasCommit(repo, rev);
+}
+function pathFor(repo, file) {
+    return isAbsolute(file) ? file : resolve(repo, file);
+}
+function allowlistPathFor(options) {
+    return pathFor(options.repo, options.allowlistFile);
+}
+function syncPointPathFor(options) {
+    return pathFor(options.repo, options.syncPointFile ?? SYNC_POINT_FILE);
+}
+function loadAllowlist(file) {
+    if (!existsSync(file))
+        return { entries: [], errors: [] };
+    return parseAllowlist(readFileSync(file, 'utf-8'));
+}
+function fail(report) {
+    return { ok: false, report, findings: [], warnings: [] };
+}
+export function runGuard(options) {
+    const root = repoRoot(options.repo);
+    const opts = { ...options, repo: root };
+    const syncFile = syncPointPathFor(opts);
+    if (opts.setSyncPoint !== undefined) {
+        return recordSyncPoint(root, syncFile, opts.setSyncPoint);
+    }
+    let syncPoint = opts.syncPoint;
+    if (syncPoint === undefined) {
+        if (!existsSync(syncFile)) {
+            return fail(`upstream-guard: no sync point recorded at ${syncFile}\n` +
+                '    The guard measures everything from the last upstream commit ' +
+                'merged\n' +
+                '    into this fork. Record it:\n' +
+                '      node dist/src/fork/upstream-guard-cli.js --set-sync-point ' +
+                '<rev>\n');
+        }
+        syncPoint = parseSyncPoint(readFileSync(syncFile, 'utf-8'));
+        if (syncPoint === undefined || syncPoint === '') {
+            return fail(`upstream-guard: ${syncFile} holds no revision.\n` +
+                '    It must contain one commit id on a line of its own.\n');
+        }
+    }
+    const present = hasCommit(root, syncPoint) ||
+        (opts.fetch && fetchSyncPoint(root, syncPoint));
+    if (!present) {
+        return fail(`upstream-guard: sync point ${syncPoint} is not in this repository.\n` +
+            '    A shallow clone does not carry it. Fetch it first:\n' +
+            `      git fetch --no-tags --depth=1 origin ${syncPoint}\n`);
+    }
+    // A sync point off the current history makes every number below
+    // meaningless — the diff would report upstream's own commits as the fork's.
+    // Truncated history cannot answer the question, so shallow clones skip it.
+    if (!isShallow(root) && !isAncestorOfHead(root, syncPoint)) {
+        return fail(`upstream-guard: sync point ${syncPoint.slice(0, 7)} is not an ancestor ` +
+            'of HEAD.\n' +
+            '    Merge that upstream commit before recording it, or record the ' +
+            'one\n' +
+            '    this branch actually descends from.\n');
+    }
+    const file = allowlistPathFor(opts);
+    const diff = diffAgainst(root, syncPoint);
+    const upstream = upstreamPaths(root, syncPoint, diff.map((d) => d.path));
+    if (opts.regenerate) {
+        return regenerateInto(file, diff, upstream);
+    }
+    const parsed = loadAllowlist(file);
+    if (!existsSync(file)) {
+        return fail(`upstream-guard: allowlist not found at ${file}\n`);
+    }
+    const map = new Map(parsed.entries.map((e) => [e.path, e.reason]));
+    const result = classify({ diff, upstream, allowlist: map });
+    const findings = [
+        ...parsed.errors.map((message) => ({ kind: 'allowlist-error', path: file, message })),
+        ...result.findings,
+    ];
+    const lines = [];
+    if (findings.length > 0) {
+        lines.push(`upstream-guard: ${findings.length} violation(s) against sync point ` +
+            `${syncPoint.slice(0, 7)}`, '');
+        for (const finding of findings)
+            lines.push(`  ${finding.message}`, '');
+        lines.push('The fork keeps its diff small: new behaviour goes in new fork-owned ' +
+            'files,', 'and an upstream file takes an import plus one call line. See ' +
+            'lat.md/fork.md.');
+    }
+    else {
+        lines.push(`upstream-guard: ok — ${result.touched.length} upstream file(s) ` +
+            'modified, all allowlisted.');
+    }
+    for (const warning of result.warnings)
+        lines.push(`  warning: ${warning}`);
+    return {
+        ok: findings.length === 0,
+        report: lines.join('\n') + '\n',
+        findings,
+        warnings: result.warnings,
+    };
+}
+// @lat: [[fork#The upstream guard#The sync point]]
+function recordSyncPoint(repo, file, rev) {
+    const resolved = git(repo, ['rev-parse', `${rev}^{commit}`]);
+    if (resolved.status !== 0) {
+        return fail(`upstream-guard: ${rev} is not a commit in this repository.\n`);
+    }
+    const sha = resolved.stdout.trim();
+    if (!isShallow(repo) && !isAncestorOfHead(repo, sha)) {
+        return fail(`upstream-guard: ${sha.slice(0, 7)} is not an ancestor of HEAD.\n` +
+            '    Record a sync point only once its merge has landed.\n');
+    }
+    const previous = existsSync(file) ? readFileSync(file, 'utf-8') : '';
+    writeFileSync(file, renderSyncPoint(previous, sha), 'utf-8');
+    return {
+        ok: true,
+        report: `upstream-guard: sync point is now ${sha}\n` +
+            '  regenerate the allowlist next — the set of files the fork ' +
+            'diverges in has moved.\n',
+        findings: [],
+        warnings: [],
+    };
+}
+function regenerateInto(file, diff, upstream) {
+    const existing = new Map(loadAllowlist(file).entries.map((e) => [e.path, e.reason]));
+    const entries = [];
+    const deletions = [];
+    const fresh = [];
+    for (const entry of diff) {
+        if (!upstream.has(entry.path))
+            continue;
+        if (entry.status.startsWith('D')) {
+            deletions.push({
+                kind: 'deletion',
+                path: entry.path,
+                message: `${entry.path}: deletes an upstream file, which is never ` +
+                    'allowlistable — left out of the regenerated list.',
+            });
+            continue;
+        }
+        const reason = existing.get(entry.path) ??
+            `${REASON_PLACEHOLDER}: why must this upstream file diverge?`;
+        if (!existing.has(entry.path))
+            fresh.push(entry.path);
+        entries.push({ path: entry.path, reason, line: 0 });
+    }
+    writeFileSync(file, renderAllowlist(entries), 'utf-8');
+    const lines = [
+        `upstream-guard: wrote ${entries.length} entr(ies) to ${file}`,
+    ];
+    for (const path of fresh) {
+        lines.push(`  new entry needs a reason: ${path}`);
+    }
+    for (const finding of deletions)
+        lines.push(`  ${finding.message}`);
+    return {
+        ok: deletions.length === 0,
+        report: lines.join('\n') + '\n',
+        findings: deletions,
+        warnings: [],
+    };
+}
+const USAGE = `Usage: node dist/src/fork/upstream-guard-cli.js [options]
+
+  --regenerate            rewrite the allowlist from the current diff
+  --fetch                 fetch the sync point from origin if it is absent
+  --set-sync-point <rev>  record <rev> as the last merged upstream commit
+  --sync-point <rev>      check against <rev> instead of the recorded one
+  --repo <dir>            default the current working directory
+  --allowlist <file>      default <repo>/${ALLOWLIST_FILE}
+  --sync-point-file <f>   default <repo>/${SYNC_POINT_FILE}
+  -h, --help              this text
+`;
+export function parseArgs(argv) {
+    const options = {
+        repo: process.cwd(),
+        allowlistFile: ALLOWLIST_FILE,
+        regenerate: false,
+        fetch: false,
+    };
+    for (let i = 0; i < argv.length; i += 1) {
+        const arg = argv[i];
+        if (arg === '-h' || arg === '--help')
+            return { help: true };
+        else if (arg === '--regenerate' || arg === '--write')
+            options.regenerate = true;
+        else if (arg === '--fetch')
+            options.fetch = true;
+        else if (arg === '--set-sync-point')
+            options.setSyncPoint = argv[++i] ?? '';
+        else if (arg === '--sync-point')
+            options.syncPoint = argv[++i] ?? '';
+        else if (arg === '--sync-point-file')
+            options.syncPointFile = argv[++i] ?? '';
+        else if (arg === '--repo')
+            options.repo = argv[++i] ?? '';
+        else if (arg === '--allowlist')
+            options.allowlistFile = argv[++i] ?? '';
+        else
+            throw new Error(`unknown argument: ${arg}`);
+    }
+    return options;
+}
+export function main(argv) {
+    let options;
+    try {
+        options = parseArgs(argv);
+    }
+    catch (error) {
+        process.stderr.write(`${error.message}\n\n${USAGE}`);
+        return 2;
+    }
+    if ('help' in options) {
+        process.stdout.write(USAGE);
+        return 0;
+    }
+    let result;
+    try {
+        result = runGuard(options);
+    }
+    catch (error) {
+        process.stderr.write(`upstream-guard: ${error.message}\n`);
+        return 2;
+    }
+    process.stdout.write(result.report);
+    return result.ok ? 0 : 1;
+}

package/dist/src/init-version.d.ts

@@ -3,7 +3,7 @@
  * requires users to re-run it (e.g. new hooks, generated files, MCP config, or
  * a configuration migration).
  */
-export declare const INIT_VERSION = 5;
+export declare const INIT_VERSION = 6;
 export declare function readInitVersion(latDir: string): number | null;
 export declare function readFileHash(latDir: string, relPath: string): string | null;
 export declare function contentHash(content: string): string;

package/dist/src/init-version.js

@@ -6,7 +6,7 @@ import { join } from 'node:path';
  * requires users to re-run it (e.g. new hooks, generated files, MCP config, or
  * a configuration migration).
  */
-export const INIT_VERSION = 5;
+export const INIT_VERSION = 6;
 function cachePath(latDir) {
     return join(latDir, '.cache', 'lat_init.json');
 }

package/dist/src/lattice.d.ts

@@ -38,7 +38,7 @@ export type MdLink = {
  * edit — the trap this list guards against is silent, so it must not depend on
  * whoever adds the next field remembering to register it twice.
  */
-export declare const LAT_FIELDS: readonly ["require-code-mention", "mode", "status", "reviewed-hash", "tags"];
+export declare const LAT_FIELDS: readonly ["require-code-mention", "mode", "require-mode", "status", "reviewed-hash", "tags"];
 /**
  * Something wrong with the frontmatter itself, carried out of the parser for a
  * caller to report. Both kinds fail OPEN if they stay quiet: a misplaced or
@@ -101,6 +101,7 @@ export declare function resolveRef(target: string, sectionIds: Set<string>, file
 export type SectionMatch = {
     section: Section;
     reason: string;
+    score?: number;
 };
 export declare function findSections(sections: Section[], query: string): SectionMatch[];
 export declare function extractRefs(filePath: string, content: string, projectRoot?: string, tree?: Root): Ref[];

package/dist/src/lattice.js

@@ -15,6 +15,7 @@ import { visit } from 'unist-util-visit';
 export const LAT_FIELDS = [
     'require-code-mention',
     'mode',
+    'require-mode',
     'status',
     'reviewed-hash',
     'tags',

package/dist/src/search/search.d.ts

@@ -5,5 +5,6 @@ export type SearchResult = {
     file: string;
     heading: string;
     content: string;
+    score: number;
 };
 export declare function searchSections(db: Client, query: string, embedder: Embedder, limit?: number): Promise<SearchResult[]>;

package/dist/src/search/search.js

@@ -2,15 +2,21 @@ export async function searchSections(db, query, embedder, limit = 5) {
     const [queryVec] = await embedder.embed([query]);
     const vecJson = JSON.stringify(queryVec);
     const rows = await db.execute({
-        sql: `SELECT s.id, s.file, s.heading, s.content
+        sql: `SELECT s.id, s.file, s.heading, s.content,
+                 1.0 - vector_distance_cos(s.embedding, vector(?)) AS score
           FROM vector_top_k('sections_vec_idx', vector(?), ?) AS v
-          JOIN sections AS s ON s.rowid = v.id`,
-        args: [vecJson, limit],
+          JOIN sections AS s ON s.rowid = v.id
+          ORDER BY score DESC`,
+        args: [vecJson, vecJson, limit],
+    });
+    return rows.rows.map((row) => {
+        const score = Number(row.score);
+        return {
+            id: row.id,
+            file: row.file,
+            heading: row.heading,
+            content: row.content,
+            score: Number.isFinite(score) ? score : 0,
+        };
     });
-    return rows.rows.map((row) => ({
-        id: row.id,
-        file: row.file,
-        heading: row.heading,
-        content: row.content,
-    }));
 }