melchizedek-agents 0.9.0

package/dist/lib/wiki/dive.js

@@ -0,0 +1,208 @@
+/**
+ * lib/wiki/dive.ts — search scoring and "repo dive" reading paths.
+ *
+ * WHY this file exists:
+ *   The two navigation questions an agent asks a wiki are "which documents
+ *   mention X?" (search) and "I have a TASK — what should I read, in what
+ *   order, within a budget?" (dive). Both are answered here with plain
+ *   lexical scoring plus graph propagation — deliberately NO model call and
+ *   NO embeddings: navigation must be instant, deterministic, and free, so
+ *   an agent can call it dozens of times while working. Semantic answering
+ *   belongs to wiki_query (an agent using these primitives), not here.
+ *
+ *   Dive = scored seeds + link expansion. Seeds are the best lexical
+ *   matches; the knowledge graph then pulls in what the seeds are wired to
+ *   (score decaying with link distance), because in a well-gardened bundle
+ *   PROXIMITY IN THE GRAPH IS RELEVANCE. The result is an ordered reading
+ *   plan — orientation first (indexes), then matched concepts, then linked
+ *   context — trimmed to a word budget.
+ */
+import { buildGraph } from "./graph.js";
+import { conceptDocs } from "./vault.js";
+// ── Query tokenisation ───────────────────────────────────────────────────────
+const STOPWORDS = new Set([
+    'a', 'an', 'and', 'are', 'as', 'at', 'be', 'by', 'do', 'does', 'for',
+    'from', 'how', 'i', 'in', 'is', 'it', 'my', 'of', 'on', 'or', 'that',
+    'the', 'this', 'to', 'us', 'was', 'we', 'what', 'when', 'where', 'which', 'why',
+    'with', 'you',
+]);
+export function tokenize(text) {
+    return [
+        ...new Set(text
+            .toLowerCase()
+            .split(/[^a-z0-9_-]+/)
+            .filter((t) => t.length > 1 && !STOPWORDS.has(t))),
+    ];
+}
+function fieldTokens(doc) {
+    return {
+        title: new Set(tokenize(doc.title)),
+        tags: new Set((doc.fm?.tags ?? []).flatMap(tokenize)),
+        description: new Set(tokenize(doc.fm?.description ?? '')),
+        headings: new Set(doc.parsed.headings.flatMap((h) => tokenize(h.text))),
+        path: new Set(tokenize(doc.bundlePath.replace(/\.md$/, ''))),
+    };
+}
+function findSnippet(doc, tokens) {
+    const lines = doc.parsed.raw.split('\n');
+    for (let i = doc.parsed.bodyStartLine - 1; i < lines.length; i++) {
+        const line = lines[i].trim();
+        if (line === '' || line.startsWith('#') || line.startsWith('<!--'))
+            continue;
+        const lower = line.toLowerCase();
+        if (tokens.some((t) => lower.includes(t))) {
+            return line.length > 180 ? `${line.slice(0, 177)}...` : line;
+        }
+    }
+    return doc.fm?.description ?? '';
+}
+export function scoreDocs(vault, query) {
+    const tokens = tokenize(query);
+    if (tokens.length === 0)
+        return [];
+    const results = [];
+    for (const doc of conceptDocs(vault)) {
+        const fields = fieldTokens(doc);
+        const bodyLower = doc.parsed.raw.toLowerCase();
+        let score = 0;
+        const matched = new Set();
+        for (const token of tokens) {
+            let hit = false;
+            if (fields.title.has(token)) {
+                score += 5;
+                hit = true;
+            }
+            if (fields.tags.has(token)) {
+                score += 4;
+                hit = true;
+            }
+            if (fields.description.has(token)) {
+                score += 3;
+                hit = true;
+            }
+            if (fields.headings.has(token)) {
+                score += 2;
+                hit = true;
+            }
+            if (fields.path.has(token)) {
+                score += 2;
+                hit = true;
+            }
+            if (doc.fm?.type === token) {
+                score += 3;
+                hit = true;
+            }
+            if (!hit) {
+                // Body-only hits count, weakly, capped so length can't dominate.
+                let count = 0;
+                let idx = bodyLower.indexOf(token);
+                while (idx !== -1 && count < 5) {
+                    count++;
+                    idx = bodyLower.indexOf(token, idx + token.length);
+                }
+                if (count > 0) {
+                    score += count * 0.5;
+                    hit = true;
+                }
+            }
+            if (hit)
+                matched.add(token);
+        }
+        // Multi-token queries reward docs matching MORE distinct tokens.
+        if (matched.size > 1)
+            score *= 1 + 0.25 * (matched.size - 1);
+        if (score > 0) {
+            results.push({
+                doc,
+                score: Math.round(score * 100) / 100,
+                matched: [...matched],
+                snippet: findSnippet(doc, tokens),
+            });
+        }
+    }
+    return results.sort((a, b) => b.score - a.score);
+}
+/**
+ * Assemble an ordered reading plan for a task. Deterministic and
+ * model-free; the caller (an agent on a knowledge task) does the reading.
+ */
+export function repoDive(vault, task, budgetWords = 6000, graph = buildGraph(vault)) {
+    const scored = scoreDocs(vault, task);
+    const seeds = scored.slice(0, 5);
+    // Propagate seed scores along resolved links, decaying by distance.
+    const propagated = new Map();
+    for (const seed of seeds) {
+        let frontier = [seed.doc.bundlePath];
+        const seen = new Set(frontier);
+        for (let dist = 1; dist <= 2; dist++) {
+            const next = [];
+            for (const path of frontier) {
+                const edges = [
+                    ...(graph.outbound.get(path) ?? []),
+                    ...(graph.inbound.get(path) ?? []).map((e) => ({ ...e, target: e.source })),
+                ];
+                for (const edge of edges) {
+                    if (!edge.resolved || seen.has(edge.target))
+                        continue;
+                    seen.add(edge.target);
+                    const gain = seed.score * Math.pow(0.4, dist);
+                    const prev = propagated.get(edge.target);
+                    if (!prev || prev.score < gain) {
+                        propagated.set(edge.target, { score: gain, via: seed.doc.bundlePath, dist });
+                    }
+                    next.push(edge.target);
+                }
+            }
+            frontier = next;
+        }
+    }
+    const stops = [];
+    const included = new Set();
+    let totalWords = 0;
+    const overflow = [];
+    const push = (doc, role, reason) => {
+        if (included.has(doc.bundlePath))
+            return true;
+        if (totalWords + doc.wordCount > budgetWords && stops.length > 0) {
+            overflow.push({ path: doc.bundlePath, title: doc.title });
+            return false;
+        }
+        included.add(doc.bundlePath);
+        totalWords += doc.wordCount;
+        stops.push({
+            order: stops.length + 1,
+            path: doc.bundlePath,
+            title: doc.title,
+            wordCount: doc.wordCount,
+            reason,
+            role,
+        });
+        return true;
+    };
+    // 1. Orient: the root index, then the indexes of directories holding seeds.
+    const rootIndex = vault.docs.get('/index.md');
+    if (rootIndex)
+        push(rootIndex, 'orient', 'bundle map — start here');
+    const seedDirs = new Set(seeds.map((s) => s.doc.bundlePath.replace(/\/[^/]+$/, '/index.md')));
+    for (const dirIndex of seedDirs) {
+        const doc = vault.docs.get(dirIndex);
+        if (doc && dirIndex !== '/index.md') {
+            push(doc, 'orient', 'directory of matched concepts');
+        }
+    }
+    // 2. Core: the matched concepts, best first.
+    for (const seed of seeds) {
+        push(seed.doc, 'core', `matched: ${seed.matched.join(', ')}`);
+    }
+    // 3. Context: what the core is wired to, best propagated score first.
+    const contextRanked = [...propagated.entries()]
+        .filter(([path]) => !included.has(path))
+        .sort((a, b) => b[1].score - a[1].score);
+    for (const [path, info] of contextRanked) {
+        const doc = vault.docs.get(path);
+        if (!doc || doc.kind === 'log')
+            continue;
+        push(doc, 'context', `linked ${info.dist} hop(s) from ${info.via}`);
+    }
+    return { task, stops, totalWords, budgetWords, overflow };
+}

package/dist/lib/wiki/entities.d.ts

@@ -0,0 +1,220 @@
+/**
+ * lib/wiki/entities.ts — the entity layer: a typed graph over repo truth.
+ *
+ * WHY this file exists:
+ *   lib/wiki/graph.ts derives ONE graph: documents joined by markdown
+ *   links. That graph answers "what should I read next", but it cannot
+ *   answer "which syndicates call web_search", "what breaks without
+ *   XAI_API_KEY", or "which decision constrains the export pipeline" —
+ *   because the things those questions name (agents, tools, models,
+ *   tables, env vars, decisions) are not documents. They are ENTITIES,
+ *   and the relations between them are typed, not hyperlinks.
+ *
+ *   This module is that second layer, and it keeps the same discipline as
+ *   the first: nodes and edges are DERIVED from repo truth wherever a
+ *   parser can see them, and merely ASSERTED where only judgment can.
+ *   The two tiers are labelled, never mixed:
+ *
+ *     extracted — a parser read it out of a YAML, a contract, DDL, an
+ *                 import statement, a link. Rebuildable, disposable,
+ *                 always regenerated by `wiki:build --graph`.
+ *     inferred  — a human or an agent read prose and asserted it, with
+ *                 evidence and an actor id. Never machine-overwritten;
+ *                 lives in .graph/relations.json beside the snapshot.
+ *
+ *   Everything here is bundle-agnostic. WHAT to extract from (this repo's
+ *   agent YAMLs, its SQL, its package.json) is the caller's business —
+ *   scripts/wiki/build.ts does that privately. This module owns the
+ *   vocabulary, the graph mechanics, the two stores, and the lint rules,
+ *   including the same private-closure invariant the document lint
+ *   enforces (wiki/decisions/0003): a public node may never point at a
+ *   private one, so any public slice of this graph is closed by
+ *   construction.
+ */
+import { z } from 'zod';
+import type { WikiGraph } from './graph.ts';
+/**
+ * What kinds of thing this graph knows about. A document keeps its OKF
+ * identity (the bundle path); every other node is `kind:name`, so the two
+ * namespaces can never collide — a doc id always starts with '/'.
+ */
+export declare const NODE_KINDS: Record<string, string>;
+export declare function entityId(kind: string, name: string): string;
+export declare function kindOf(id: string): string;
+export declare function nameOf(id: string): string;
+export type RelationTier = 'extracted' | 'inferred';
+export interface RelationSpec {
+    tier: RelationTier;
+    /** Reads as `<from> <phrase> <to>`. */
+    phrase: string;
+    gloss: string;
+    /** Node kinds this relation may start from (extracted tier only). */
+    from?: string[];
+    /** Node kinds this relation may point at (extracted tier only). */
+    to?: string[];
+}
+/**
+ * The whole relation vocabulary. Extracted relations carry kind
+ * constraints because a parser produces them and a violation is a bug;
+ * inferred relations do not, because judgment legitimately connects
+ * anything to anything — the evidence field is what disciplines them.
+ */
+export declare const RELATIONS: Record<string, RelationSpec>;
+export declare function relationsByTier(tier: RelationTier): string[];
+export interface EntityNode {
+    id: string;
+    kind: string;
+    label: string;
+    /** Repo-relative path or bundle path where this node's truth lives. */
+    source?: string;
+    /** True when the node belongs to the never-exported annex. */
+    private?: boolean;
+    attrs?: Record<string, unknown>;
+}
+export interface EntityEdge {
+    from: string;
+    to: string;
+    rel: string;
+    tier: RelationTier;
+    /** Where the claim comes from: a repo path, a line, or a quotation. */
+    evidence?: string;
+    /** Actor id — required on inferred edges. */
+    by?: string;
+    at?: string;
+}
+export interface EntityGraph {
+    nodes: Map<string, EntityNode>;
+    edges: EntityEdge[];
+    outbound: Map<string, EntityEdge[]>;
+    inbound: Map<string, EntityEdge[]>;
+}
+export declare function edgeKey(edge: Pick<EntityEdge, 'from' | 'rel' | 'to'>): string;
+export declare function buildEntityGraph(nodes: EntityNode[], edges: EntityEdge[]): EntityGraph;
+/** The document layer, restated as entities so both layers share one graph. */
+export declare function docLayer(docGraph: WikiGraph): {
+    nodes: EntityNode[];
+    edges: EntityEdge[];
+};
+export interface NeighborStep {
+    node: EntityNode;
+    rel: string;
+    direction: 'out' | 'in';
+    distance: number;
+    via: string;
+    evidence?: string;
+    tier: RelationTier;
+}
+export interface NeighborOptions {
+    depth?: number;
+    direction?: 'out' | 'in' | 'both';
+    /** Restrict the walk to these relation ids. */
+    relations?: string[];
+}
+export declare function neighbors(graph: EntityGraph, originId: string, options?: NeighborOptions): NeighborStep[];
+export interface PathStep {
+    node: EntityNode;
+    /** The relation that reached this node ('' for the origin). */
+    rel: string;
+    direction: 'out' | 'in' | 'origin';
+}
+/** Shortest connection between two nodes, walking edges in either direction. */
+export declare function shortestPath(graph: EntityGraph, fromId: string, toId: string, maxDepth?: number): PathStep[] | null;
+export interface NodeMatch {
+    node: EntityNode;
+    score: number;
+}
+/** Lexical node lookup: exact id, then id/label substring, then attrs. */
+export declare function findNodes(graph: EntityGraph, query: string, limit?: number): NodeMatch[];
+export declare function nodesOfKind(graph: EntityGraph, kind: string): EntityNode[];
+export interface EntityStats {
+    nodes: number;
+    edges: number;
+    byKind: Record<string, number>;
+    byRelation: Record<string, number>;
+    byTier: Record<RelationTier, number>;
+    isolated: number;
+}
+export declare function entityStats(graph: EntityGraph): EntityStats;
+export interface GraphFinding {
+    severity: 'error' | 'warning' | 'info';
+    rule: string;
+    message: string;
+}
+/** True for nodes that must never be referenced from a public one. */
+export declare function isPrivateNode(node: EntityNode): boolean;
+export declare function lintEntityGraph(graph: EntityGraph): GraphFinding[];
+/**
+ * Both stores live in a dot-directory inside the bundle: the vault walker
+ * skips dotfiles, so they are invisible to every document operation, they
+ * travel with WIKI_ROOT, and the export pipeline (which copies *.md only)
+ * cannot ship them by accident.
+ */
+export declare function graphDir(wikiRoot: string): string;
+export declare function snapshotPath(wikiRoot: string): string;
+export declare function relationsPath(wikiRoot: string): string;
+export declare const relationRecordSchema: z.ZodObject<{
+    from: z.ZodString;
+    to: z.ZodString;
+    rel: z.ZodString;
+    evidence: z.ZodString;
+    by: z.ZodString;
+    at: z.ZodOptional<z.ZodString>;
+    note: z.ZodOptional<z.ZodString>;
+}, z.core.$loose>;
+export type RelationRecord = z.infer<typeof relationRecordSchema>;
+export declare const relationStoreSchema: z.ZodObject<{
+    version: z.ZodDefault<z.ZodNumber>;
+    relations: z.ZodDefault<z.ZodArray<z.ZodObject<{
+        from: z.ZodString;
+        to: z.ZodString;
+        rel: z.ZodString;
+        evidence: z.ZodString;
+        by: z.ZodString;
+        at: z.ZodOptional<z.ZodString>;
+        note: z.ZodOptional<z.ZodString>;
+    }, z.core.$loose>>>;
+}, z.core.$loose>;
+export type RelationStore = z.infer<typeof relationStoreSchema>;
+export interface LoadedRelations {
+    records: RelationRecord[];
+    /** Malformed entries are reported, never thrown — one bad row can't blind the graph. */
+    issues: string[];
+}
+export declare function loadRelations(wikiRoot: string): LoadedRelations;
+export declare function relationEdges(records: RelationRecord[]): EntityEdge[];
+export declare function saveRelations(wikiRoot: string, records: RelationRecord[]): void;
+export interface GraphSnapshot {
+    version: number;
+    generated: {
+        by: string;
+        at: string;
+    };
+    stats: EntityStats;
+    nodes: EntityNode[];
+    /** Extracted edges only — inferred ones are read live from relations.json. */
+    edges: EntityEdge[];
+}
+export declare function writeSnapshot(wikiRoot: string, graph: EntityGraph, generated: {
+    by: string;
+    at: string;
+}): string;
+export declare function readSnapshot(wikiRoot: string): GraphSnapshot | null;
+export interface LoadedGraph {
+    graph: EntityGraph;
+    snapshot: GraphSnapshot | null;
+    /** Problems reading the asserted store — surfaced, never fatal. */
+    issues: string[];
+    inferredCount: number;
+}
+/**
+ * The read path every consumer uses: derived snapshot (structure, as of the
+ * last build) + asserted relations (read live, so a fresh assertion is
+ * visible immediately). Missing snapshot yields an empty graph plus the
+ * asserted edges, which lint will flag as dangling — honest, not silent.
+ */
+export declare function loadEntityGraph(wikiRoot: string): LoadedGraph;
+/** Documents the snapshot has not seen yet — the staleness signal. */
+export declare function snapshotDrift(snapshot: GraphSnapshot | null, currentDocPaths: string[]): {
+    missing: string[];
+    removed: string[];
+};