@plebeiantech/lat.md 0.12.3-fork.1

package/dist/src/knowledge/ranking.js

@@ -0,0 +1,62 @@
+// Runs of anything that is not an ASCII/Unicode letter or digit. This is a
+// splitter, not a validator, so it stays permissive about what counts as a
+// "word" — the length-3 filter below does the real filtering.
+const WORD_BREAK = /[^\p{L}\p{N}]+/u;
+/**
+ * Turn a frontmatter `tags:` value into search terms.
+ *
+ * Two terms by default: each store call costs a subprocess or a query, and
+ * two terms is the least that still lets matched-term-count ranking mean
+ * anything (a hit that matched both terms outranks one that matched one).
+ */
+export function tagsToTerms(value, maxTerms = 2) {
+    let tags;
+    if (typeof value === 'string') {
+        tags = [value];
+    }
+    else if (Array.isArray(value)) {
+        tags = value.filter((v) => typeof v === 'string');
+    }
+    else {
+        return [];
+    }
+    const seenLower = new Set();
+    const terms = [];
+    // One term per tag, taken BEFORE the maxTerms cap is applied. A single
+    // hyphenated tag splits into several words; if the cap were applied to
+    // that per-word list, one tag could exhaust the whole budget and starve
+    // every other authored tag (see the worked example above the cap).
+    // Picking one representative word per tag first means the cap trims
+    // across tags instead of within one.
+    for (const tag of tags) {
+        const words = tag.split(WORD_BREAK).filter((w) => w.length >= 3);
+        if (words.length === 0)
+            continue;
+        const word = words[0];
+        const lower = word.toLowerCase();
+        if (seenLower.has(lower))
+            continue;
+        seenLower.add(lower);
+        terms.push(word);
+    }
+    // NEVER sort. Tags are search terms the author chose and ordered; the shell
+    // implementation piped them through `sort -u` for the dedupe and the sort
+    // silently reordered them too, so cutting to maxTerms took the two
+    // alphabetically-first fragments instead of the two the author actually
+    // wrote first. `tags: [run-pin, carry, query-param]` searched "carry" and
+    // "param" and dropped the primary tag entirely. Authored order must survive
+    // to the slice below.
+    return terms.slice(0, maxTerms);
+}
+/**
+ * Stable sort by score, descending, then cap. Scores are only ever compared
+ * within one store's hits — a store that cannot attribute a match to
+ * individual terms and reports one flat score should never be capped
+ * alongside a store that scores per-term, so callers must call this once per
+ * store's hit list, not once over a merged list.
+ */
+export function capHits(hits, limit) {
+    // Array#sort is stable per spec (Node/V8 for years now), so ties keep
+    // their input order without any extra bookkeeping.
+    return [...hits].sort((a, b) => b.score - a.score).slice(0, limit);
+}

package/dist/src/knowledge/session.d.ts

@@ -0,0 +1,46 @@
+/**
+ * How long a document stays in `attemptedEmpty` across prompts. Short and
+ * deliberately so: this set means "found nothing last time, worth another
+ * look once a store has something to say", not "never look again". A long
+ * or missing expiry would let a document that happened to catch the stores
+ * cold on prompt one stay unreachable for the rest of the session — the
+ * exact starvation bug the attempt-budget rotation exists to prevent.
+ */
+export declare const ATTEMPTED_EMPTY_TTL_MS: number;
+export type SessionMarkers = {
+    seen: Set<string>;
+    attemptedEmpty: Set<string>;
+};
+/**
+ * A loaded session's markers plus enough state to save them back correctly.
+ * `federateTags` mutates `markers.seen` / `markers.attemptedEmpty` in place;
+ * pass the same handle to `saveSessionMarkers` afterward so an id that was
+ * already in `attemptedEmpty` before this call keeps its original recorded
+ * time instead of having its TTL silently refreshed on every prompt.
+ */
+export type SessionHandle = {
+    sessionId: string | undefined;
+    markers: SessionMarkers;
+    /** doc id -> original timestamp, for entries loaded from disk. */
+    attemptedEmptySince: Map<string, number>;
+};
+/**
+ * Load persisted markers for a session. Returns a handle with fresh, empty
+ * Sets — never throws, never blocks the caller — whenever the marker path is
+ * missing, unreadable, unsafe, or corrupt. An empty result just means
+ * federation runs as if this were the first prompt of the session: dedupe is
+ * lost for this call, but the lookup itself still happens.
+ *
+ * No session id means no persistence: the caller gets fresh Sets and
+ * `saveSessionMarkers` becomes a no-op, never falling back to any shared or
+ * global file.
+ */
+export declare function loadSessionMarkers(sessionId: string | undefined): SessionHandle;
+/**
+ * Persist a handle's markers after a `federateTags` call has mutated them.
+ * An id already present at load time keeps the timestamp it was loaded
+ * with, so its TTL counts from when it first came back empty, not from
+ * every subsequent prompt that re-skips it for free. An id that is new this
+ * call is stamped with the current time.
+ */
+export declare function saveSessionMarkers(handle: SessionHandle): void;

package/dist/src/knowledge/session.js

@@ -0,0 +1,203 @@
+// Session-scoped persistence for federation dedupe markers.
+//
+// `lat hook` runs as a fresh process per prompt, so the in-process `seen` and
+// `attemptedEmpty` Sets that `federateTags` accepts start empty every time
+// unless something persists them across invocations. This module is that
+// something: a small on-disk marker store keyed on the hook payload's
+// session id.
+//
+// The marker path is predictable from the session id (a hash of it), so
+// another local user who can predict or observe the session id could plant a
+// symlink there ahead of us. The checks below port the `private_dir` pattern
+// from the shell hooks this ports from: verify no symlink, create with mode
+// 0700, then re-verify ownership and mode AFTER creation (mkdir -p succeeds
+// silently on a path that's already a symlink to a directory). Every open
+// additionally passes O_NOFOLLOW so a link planted in the gap between our
+// checks and the actual read/write still can't be followed.
+//
+// Failing any of this must never fail the prompt and must never suppress a
+// lookup: on any doubt about the marker path's safety, we drop back to fresh,
+// empty Sets. Federation still runs (with no cross-prompt memory for this
+// call) rather than either crashing or letting a hostile file decide whether
+// federation fires.
+import { constants as fsConstants, closeSync, fstatSync, lstatSync, mkdirSync, openSync, readFileSync, writeFileSync, } from 'node:fs';
+import { createHash } from 'node:crypto';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+/**
+ * How long a document stays in `attemptedEmpty` across prompts. Short and
+ * deliberately so: this set means "found nothing last time, worth another
+ * look once a store has something to say", not "never look again". A long
+ * or missing expiry would let a document that happened to catch the stores
+ * cold on prompt one stay unreachable for the rest of the session — the
+ * exact starvation bug the attempt-budget rotation exists to prevent.
+ */
+export const ATTEMPTED_EMPTY_TTL_MS = 5 * 60 * 1000;
+function markerDir(sessionId) {
+    const hash = createHash('sha256')
+        .update(sessionId)
+        .digest('hex')
+        .slice(0, 32);
+    return join(tmpdir(), `lat-session-${hash}`);
+}
+function markerFile(sessionId) {
+    return join(markerDir(sessionId), 'markers.json');
+}
+/**
+ * Verify (and if needed, tighten) that `dir` is a private directory we own,
+ * creating it if absent. Returns false on anything that looks even slightly
+ * off — a symlink, a directory owned by someone else, permissions we can't
+ * correct — rather than trying to be clever about partial trust.
+ */
+function ensurePrivateDir(dir) {
+    try {
+        // Refuse a pre-existing symlink outright, before mkdir touches it.
+        try {
+            const preStat = lstatSync(dir);
+            if (preStat.isSymbolicLink())
+                return false;
+        }
+        catch {
+            // Doesn't exist yet — fine, mkdir below creates it.
+        }
+        mkdirSync(dir, { recursive: true, mode: 0o700 });
+        // Re-check AFTER mkdir: `mkdir` with `recursive: true` succeeds silently
+        // on a path that is already a symlink to a directory, so the check above
+        // alone isn't enough — an attacker could win the race between it and the
+        // mkdir call.
+        const st = lstatSync(dir);
+        if (st.isSymbolicLink())
+            return false;
+        if (!st.isDirectory())
+            return false;
+        if (typeof st.uid === 'number' && typeof process.getuid === 'function') {
+            if (st.uid !== process.getuid())
+                return false;
+        }
+        // Mode must be exactly 0700: group/world access on a directory holding
+        // doc summaries and store hits is a leak, not just a corruption risk.
+        if ((st.mode & 0o777) !== 0o700)
+            return false;
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Open with O_NOFOLLOW so a symlink planted after ensurePrivateDir's checks
+ * still can't be followed for the actual read/write.
+ */
+function openNoFollow(path, flags, mode) {
+    try {
+        return openSync(path, flags, mode);
+    }
+    catch {
+        return null;
+    }
+}
+function emptyHandle(sessionId) {
+    return {
+        sessionId,
+        markers: { seen: new Set(), attemptedEmpty: new Set() },
+        attemptedEmptySince: new Map(),
+    };
+}
+/**
+ * Load persisted markers for a session. Returns a handle with fresh, empty
+ * Sets — never throws, never blocks the caller — whenever the marker path is
+ * missing, unreadable, unsafe, or corrupt. An empty result just means
+ * federation runs as if this were the first prompt of the session: dedupe is
+ * lost for this call, but the lookup itself still happens.
+ *
+ * No session id means no persistence: the caller gets fresh Sets and
+ * `saveSessionMarkers` becomes a no-op, never falling back to any shared or
+ * global file.
+ */
+export function loadSessionMarkers(sessionId) {
+    // `sessionId` typically arrives via `JSON.parse` on untyped hook input, so
+    // a malformed or differently-shaped payload can hand this a non-string.
+    // Treat that exactly like "no session id" rather than letting it reach
+    // createHash() and throw past this function's callers.
+    if (typeof sessionId !== 'string' || sessionId.length === 0) {
+        return emptyHandle(undefined);
+    }
+    const dir = markerDir(sessionId);
+    if (!ensurePrivateDir(dir))
+        return emptyHandle(sessionId);
+    const file = markerFile(sessionId);
+    const fd = openNoFollow(file, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW);
+    if (fd === null)
+        return emptyHandle(sessionId);
+    try {
+        const st = fstatSync(fd);
+        if (!st.isFile())
+            return emptyHandle(sessionId);
+        const raw = readFileSync(fd, 'utf8');
+        const parsed = JSON.parse(raw);
+        const seen = new Set(Array.isArray(parsed.seen) ? parsed.seen : []);
+        const now = Date.now();
+        const attemptedEmpty = new Set();
+        const attemptedEmptySince = new Map();
+        if (parsed.attemptedEmpty && typeof parsed.attemptedEmpty === 'object') {
+            for (const [id, ts] of Object.entries(parsed.attemptedEmpty)) {
+                if (typeof ts === 'number' && now - ts < ATTEMPTED_EMPTY_TTL_MS) {
+                    attemptedEmpty.add(id);
+                    attemptedEmptySince.set(id, ts);
+                }
+            }
+        }
+        return {
+            sessionId,
+            markers: { seen, attemptedEmpty },
+            attemptedEmptySince,
+        };
+    }
+    catch {
+        return emptyHandle(sessionId);
+    }
+    finally {
+        closeSync(fd);
+    }
+}
+/**
+ * Persist a handle's markers after a `federateTags` call has mutated them.
+ * An id already present at load time keeps the timestamp it was loaded
+ * with, so its TTL counts from when it first came back empty, not from
+ * every subsequent prompt that re-skips it for free. An id that is new this
+ * call is stamped with the current time.
+ */
+export function saveSessionMarkers(handle) {
+    const { sessionId, markers, attemptedEmptySince } = handle;
+    if (!sessionId)
+        return;
+    const dir = markerDir(sessionId);
+    if (!ensurePrivateDir(dir))
+        return;
+    const now = Date.now();
+    const attemptedEmpty = {};
+    for (const id of markers.attemptedEmpty) {
+        attemptedEmpty[id] = attemptedEmptySince.get(id) ?? now;
+    }
+    const payload = {
+        seen: [...markers.seen],
+        attemptedEmpty,
+    };
+    const file = markerFile(sessionId);
+    const fd = openNoFollow(file, fsConstants.O_WRONLY |
+        fsConstants.O_CREAT |
+        fsConstants.O_TRUNC |
+        fsConstants.O_NOFOLLOW, 0o600);
+    if (fd === null)
+        return;
+    try {
+        writeFileSync(fd, JSON.stringify(payload));
+    }
+    catch {
+        // Best-effort; a failed write just means this call's markers aren't
+        // persisted, same as if the session had no marker store at all.
+    }
+    finally {
+        closeSync(fd);
+    }
+}

package/dist/src/knowledge/types.d.ts

@@ -0,0 +1,47 @@
+export type StoreName = 'cq' | 'bd' | 'claude-memory';
+/** One result from one store. Text is RAW here — the caller quotes it. */
+export type KnowledgeHit = {
+    store: StoreName;
+    /**
+     * Stable identity, used for dedupe and for match counting: a bd memory key,
+     * an absolute memory-file path, a cq row identity. Never shown to a reader.
+     */
+    key: string;
+    /** Short label shown first. A bd key, a memory `name:`, a cq summary. */
+    title: string;
+    /** Longer text shown after the title. May be empty. */
+    detail: string;
+    /**
+     * How many query terms matched this hit. Higher sorts first. A store that
+     * ranks internally (cq, by bm25) reports the number of terms it was given.
+     */
+    score: number;
+};
+/** Everything a store needs for one lookup. */
+export type StoreQuery = {
+    /**
+     * Search terms in the order the author wrote the tags. NEVER re-sort this
+     * array: the order is the author's ranking of the terms, and sorting makes
+     * the primary tag lose to an alphabetical accident.
+     */
+    terms: string[];
+    /** Absolute path to the project root (the parent of lat.md/). */
+    projectRoot: string;
+    /** Most hits this store may return for this one query. */
+    limit: number;
+};
+/**
+ * A knowledge store. `query` resolves to [] when the store is unavailable for
+ * any reason. It must never throw and must never write to stdout or stderr.
+ */
+export type Store = {
+    name: StoreName;
+    query(q: StoreQuery): Promise<KnowledgeHit[]>;
+};
+/** A lat.md document that carries `lat: { tags: [...] }` in its frontmatter. */
+export type TaggedDoc = {
+    /** Section or document id, used as the dedupe key and as the block heading. */
+    id: string;
+    /** Tags exactly as authored, in the authored order. */
+    tags: string[];
+};

package/dist/src/knowledge/types.js

@@ -0,0 +1,10 @@
+// The contract every knowledge store implements. This file is the integration
+// point between the three store modules and the federation entry point, so it
+// is deliberately the only file in src/knowledge/ that none of them owns.
+//
+// The governing rule for the whole directory: a store is OPTIONAL. Absent
+// binary, absent database, absent directory, malformed row, non-zero exit,
+// timeout — every one of those is an empty result, never a thrown error and
+// never a message on stderr. `lat hook` must not fail a user's prompt because
+// a lookup that was never guaranteed to find anything found nothing.
+export {};

package/dist/src/lattice.d.ts

@@ -0,0 +1,108 @@
+import type { Root } from 'mdast';
+export type Section = {
+    id: string;
+    heading: string;
+    depth: number;
+    file: string;
+    filePath: string;
+    children: Section[];
+    startLine: number;
+    endLine: number;
+    firstParagraph: string;
+    /** GitHub-compatible anchor generated from the rendered heading text. */
+    githubSlug?: string;
+};
+export type Ref = {
+    target: string;
+    fromSection: string;
+    file: string;
+    line: number;
+};
+/** An ordinary markdown destination or an undefined reference-style link. */
+export type MdLink = {
+    /** Destination exactly as authored, percent-escapes and all. */
+    url: string;
+    kind: 'link' | 'image' | 'definition';
+    line: number;
+} | {
+    /** Authored label that should have a matching definition. */
+    identifier: string;
+    /** Full reference syntax exactly as authored. */
+    source: string;
+    kind: 'linkReference' | 'imageReference';
+    line: number;
+};
+/**
+ * Every field this build reads out of the `lat:` mapping. Kept in one place so
+ * a field added later is covered by the misplacement check without a second
+ * edit — the trap this list guards against is silent, so it must not depend on
+ * whoever adds the next field remembering to register it twice.
+ */
+export declare const LAT_FIELDS: readonly ["require-code-mention", "mode", "status", "reviewed-hash", "tags"];
+/**
+ * Something wrong with the frontmatter itself, carried out of the parser for a
+ * caller to report. Both kinds fail OPEN if they stay quiet: a misplaced or
+ * unparseable `require-code-mention` turns a validation off and `lat check`
+ * then reports success on a file that is no longer being validated.
+ */
+export type FrontmatterProblem = {
+    kind: 'root-level-field';
+    field: string;
+} | {
+    kind: 'parse-error';
+    message: string;
+};
+export type LatFrontmatter = {
+    /** Every key under the `lat:` mapping, parsed but not mapped to a field. */
+    raw: Record<string, unknown>;
+    requireCodeMention?: boolean;
+    /** Populated only when something is wrong; absent on a healthy document. */
+    problems?: FrontmatterProblem[];
+};
+export declare function parseFrontmatter(content: string): LatFrontmatter;
+export declare function findLatticeDir(from?: string): string | null;
+export declare function findProjectRoot(from?: string): string | null;
+export declare function listLatticeFiles(latticeDir: string): Promise<string[]>;
+export declare function parseSections(filePath: string, content: string, projectRoot?: string, tree?: Root): Section[];
+export declare function loadAllSections(latticeDir: string, projectRoot?: string): Promise<Section[]>;
+export declare function flattenSections(sections: Section[]): Section[];
+/**
+ * Build an index mapping path suffixes to their full vault-relative paths.
+ * Used by resolveRef to allow short references when a suffix is unambiguous.
+ *
+ * For a file like `lat.md/guides/setup`, indexes both `guides/setup` and `setup`.
+ * This ensures backward-compatible short refs after the vault root moved to the
+ * project root (so section IDs now include the `lat.md/` prefix).
+ */
+export declare function buildFileIndex(sections: Section[]): Map<string, string[]>;
+export type SectionSlugIndex = ReadonlyMap<string, string>;
+/**
+ * Map GitHub-slugged heading paths back to canonical, literal-heading ids.
+ * Every path segment accepts either representation so an implicit literal h1
+ * can still prefix slugged child headings during short-ref resolution.
+ */
+export declare function buildSectionSlugIndex(sections: Section[]): Map<string, string>;
+export type ResolveResult = {
+    resolved: string;
+    ambiguous: string[] | null;
+    /** When ambiguous but exactly one candidate has the section, suggest it. */
+    suggested: string | null;
+};
+/**
+ * Resolve a potentially short reference to its canonical full-path form.
+ * If the file segment of the ref is a bare stem that uniquely maps to one
+ * full path, expands it. Otherwise returns the ref unchanged.
+ *
+ * When ambiguous (multiple files share the stem), returns all candidates.
+ * If exactly one candidate actually contains the referenced section,
+ * `suggested` is set to that candidate so the caller can propose a fix.
+ */
+export declare function resolveRef(target: string, sectionIds: Set<string>, fileIndex: Map<string, string[]>, slugIndex?: SectionSlugIndex): ResolveResult;
+export type SectionMatch = {
+    section: Section;
+    reason: string;
+};
+export declare function findSections(sections: Section[], query: string): SectionMatch[];
+export declare function extractRefs(filePath: string, content: string, projectRoot?: string, tree?: Root): Ref[];
+/** Extract link destinations and undefined full/collapsed references. */
+export declare function extractLinks(content: string, tree?: Root): MdLink[];