@nusoft/nuos-build-catalogue 0.10.0 → 0.10.2

package/dist/embedder/vertex.d.ts

@@ -0,0 +1,41 @@
+/**
+ * Vertex AI embedder — text-embedding-005 (768 dims).
+ *
+ * Auth: GOOGLE_APPLICATION_CREDENTIALS env var pointing at a service
+ * account JSON, or any other ADC mechanism Google accepts.
+ *
+ * Chosen as the default because: it matches Sensight's production
+ * embedder; UK data residency is available; the README example
+ * documents 768 dimensions (matches text-embedding-005); per D010 the
+ * choice is the consumer's.
+ *
+ * Implementation note — Vertex's REST API does not bundle nicely without
+ * the Google auth library. Rather than vendor a heavyweight SDK in
+ * this small CLI, this implementation expects either:
+ *   - GOOGLE_VERTEX_ACCESS_TOKEN env var (a short-lived OAuth token,
+ *     refreshable via `gcloud auth print-access-token`), or
+ *   - GOOGLE_APPLICATION_CREDENTIALS pointing at a service account JSON,
+ *     in which case it shells out to `gcloud` to mint a token.
+ *
+ * The shell-out path is used by Sensight's local dev environment too.
+ * For production use of this CLI a future revision can adopt
+ * @google-cloud/aiplatform; not needed for Phase 0.
+ */
+import type { Embedder } from './types.js';
+interface VertexConfig {
+    project: string;
+    location: string;
+    accessToken: string;
+}
+export declare class VertexEmbedder implements Embedder {
+    private readonly config;
+    readonly dimensions = 768;
+    readonly modelId = "text-embedding-005";
+    constructor(config: VertexConfig);
+    static fromEnv(): VertexEmbedder;
+    private get endpoint();
+    embed(texts: string[]): Promise<Float32Array[]>;
+    private embedBatch;
+    dispose(): Promise<void>;
+}
+export {};

package/dist/embedder/vertex.js

@@ -0,0 +1,94 @@
+/**
+ * Vertex AI embedder — text-embedding-005 (768 dims).
+ *
+ * Auth: GOOGLE_APPLICATION_CREDENTIALS env var pointing at a service
+ * account JSON, or any other ADC mechanism Google accepts.
+ *
+ * Chosen as the default because: it matches Sensight's production
+ * embedder; UK data residency is available; the README example
+ * documents 768 dimensions (matches text-embedding-005); per D010 the
+ * choice is the consumer's.
+ *
+ * Implementation note — Vertex's REST API does not bundle nicely without
+ * the Google auth library. Rather than vendor a heavyweight SDK in
+ * this small CLI, this implementation expects either:
+ *   - GOOGLE_VERTEX_ACCESS_TOKEN env var (a short-lived OAuth token,
+ *     refreshable via `gcloud auth print-access-token`), or
+ *   - GOOGLE_APPLICATION_CREDENTIALS pointing at a service account JSON,
+ *     in which case it shells out to `gcloud` to mint a token.
+ *
+ * The shell-out path is used by Sensight's local dev environment too.
+ * For production use of this CLI a future revision can adopt
+ * @google-cloud/aiplatform; not needed for Phase 0.
+ */
+import { execSync } from 'node:child_process';
+const MODEL_ID = 'text-embedding-005';
+const DIMENSIONS = 768;
+const DEFAULT_LOCATION = 'us-central1';
+export class VertexEmbedder {
+    config;
+    dimensions = DIMENSIONS;
+    modelId = MODEL_ID;
+    constructor(config) {
+        this.config = config;
+    }
+    static fromEnv() {
+        const project = process.env.GOOGLE_CLOUD_PROJECT ?? process.env.GCP_PROJECT;
+        if (!project) {
+            throw new Error('GOOGLE_CLOUD_PROJECT (or GCP_PROJECT) is required for the vertex embedder.');
+        }
+        const location = process.env.GOOGLE_CLOUD_LOCATION ?? DEFAULT_LOCATION;
+        let accessToken = process.env.GOOGLE_VERTEX_ACCESS_TOKEN;
+        if (!accessToken) {
+            try {
+                accessToken = execSync('gcloud auth print-access-token', {
+                    encoding: 'utf8',
+                }).trim();
+            }
+            catch (err) {
+                throw new Error('Could not obtain a Vertex access token. Set GOOGLE_VERTEX_ACCESS_TOKEN, ' +
+                    'or run `gcloud auth application-default login` and ensure `gcloud` is on PATH. ' +
+                    'Original error: ' +
+                    (err instanceof Error ? err.message : String(err)));
+            }
+        }
+        return new VertexEmbedder({ project, location, accessToken });
+    }
+    get endpoint() {
+        const { project, location } = this.config;
+        return `https://${location}-aiplatform.googleapis.com/v1/projects/${project}/locations/${location}/publishers/google/models/${MODEL_ID}:predict`;
+    }
+    async embed(texts) {
+        if (texts.length === 0)
+            return [];
+        // Vertex enforces a per-request batch limit — chunk to be safe
+        const BATCH = 5;
+        const out = [];
+        for (let i = 0; i < texts.length; i += BATCH) {
+            const slice = texts.slice(i, i + BATCH);
+            const embeddings = await this.embedBatch(slice);
+            out.push(...embeddings);
+        }
+        return out;
+    }
+    async embedBatch(texts) {
+        const res = await fetch(this.endpoint, {
+            method: 'POST',
+            headers: {
+                'content-type': 'application/json',
+                authorization: `Bearer ${this.config.accessToken}`,
+            },
+            body: JSON.stringify({
+                instances: texts.map((content) => ({ content, task_type: 'RETRIEVAL_DOCUMENT' })),
+            }),
+        });
+        if (!res.ok) {
+            const body = await res.text().catch(() => '<unreadable body>');
+            throw new Error(`Vertex embed call failed (${res.status}): ${body}`);
+        }
+        const json = (await res.json());
+        return json.predictions.map((p) => new Float32Array(p.embeddings.values));
+    }
+    // Cloud embedder — nothing to release on the local machine.
+    async dispose() { }
+}

package/dist/indexer/chunk.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Markdown-aware chunker.
+ *
+ * Splits a file on H1/H2/H3 boundaries. Code fences (``` ... ```) are
+ * preserved intact — we never break a chunk inside one. Each chunk gets
+ * a deterministic id of the form `<relPath>#<heading-slug-path>` so
+ * re-indexing the same file produces stable IDs.
+ *
+ * Token budget: estimated as ~4 chars per token (rough but adequate for
+ * routing decisions; the actual cost is at the embedder, which has its
+ * own per-call limits we respect there).
+ */
+export interface Chunk {
+    id: string;
+    text: string;
+    headings: string[];
+    startLine: number;
+    endLine: number;
+}
+export declare function chunkMarkdown(relativePath: string, content: string): Chunk[];

package/dist/indexer/chunk.js

@@ -0,0 +1,196 @@
+/**
+ * Markdown-aware chunker.
+ *
+ * Splits a file on H1/H2/H3 boundaries. Code fences (``` ... ```) are
+ * preserved intact — we never break a chunk inside one. Each chunk gets
+ * a deterministic id of the form `<relPath>#<heading-slug-path>` so
+ * re-indexing the same file produces stable IDs.
+ *
+ * Token budget: estimated as ~4 chars per token (rough but adequate for
+ * routing decisions; the actual cost is at the embedder, which has its
+ * own per-call limits we respect there).
+ */
+const MAX_CHUNK_CHARS = 600 * 4; // ~600 tokens
+const OVERLAP_CHARS = 50 * 4; // ~50 tokens overlap when splitting
+/**
+ * Minimum body length (the section text MINUS its heading line) for a
+ * chunk to be embedded as its own unit. Sections shorter than this are
+ * merged forward into the next non-empty sibling so the embedding has
+ * something substantive to anchor on. Without this, headings like
+ * `## Scope` with no body until the next sub-heading produce single-line
+ * chunks that all match the same generic queries at the same similarity,
+ * crowding real content out of search results.
+ */
+const MIN_BODY_CHARS = 80;
+export function chunkMarkdown(relativePath, content) {
+    const lines = content.split('\n');
+    const rawSections = splitOnHeadings(lines);
+    const sections = mergeTinySections(rawSections);
+    const chunks = [];
+    for (const section of sections) {
+        const sectionText = section.lines.join('\n').trim();
+        if (sectionText.length === 0)
+            continue;
+        if (sectionText.length <= MAX_CHUNK_CHARS) {
+            chunks.push({
+                id: makeChunkId(relativePath, section.headings, 0),
+                text: sectionText,
+                headings: section.headings,
+                startLine: section.startLine,
+                endLine: section.endLine,
+            });
+        }
+        else {
+            const slices = sliceLong(sectionText);
+            slices.forEach((slice, i) => {
+                chunks.push({
+                    id: makeChunkId(relativePath, section.headings, i),
+                    text: slice,
+                    headings: section.headings,
+                    startLine: section.startLine,
+                    endLine: section.endLine,
+                });
+            });
+        }
+    }
+    return chunks;
+}
+/**
+ * Merge sections whose body (everything after the heading line) is
+ * under MIN_BODY_CHARS into the next sibling. The merged section keeps
+ * the heading hierarchy of the upstream tiny section so navigation
+ * still works, but the embedded text now has substantive content.
+ *
+ * If a tiny section is the LAST section, it merges backward into the
+ * previous one. This catches files that end on a near-empty heading.
+ */
+function mergeTinySections(sections) {
+    if (sections.length <= 1)
+        return sections;
+    const merged = [];
+    let i = 0;
+    while (i < sections.length) {
+        const current = sections[i];
+        const body = bodyOfSection(current);
+        if (body.length >= MIN_BODY_CHARS) {
+            merged.push(current);
+            i += 1;
+            continue;
+        }
+        // tiny section — merge forward into the next sibling
+        if (i + 1 < sections.length) {
+            const next = sections[i + 1];
+            merged.push({
+                // Keep the LATER (more specific) heading hierarchy so search
+                // results point at the section the user actually wants.
+                headings: next.headings.length >= current.headings.length ? next.headings : current.headings,
+                lines: [...current.lines, ...next.lines],
+                startLine: current.startLine,
+                endLine: next.endLine,
+            });
+            i += 2;
+            continue;
+        }
+        // tiny section is the last — merge backward into the previous one
+        if (merged.length > 0) {
+            const prev = merged[merged.length - 1];
+            merged[merged.length - 1] = {
+                headings: prev.headings,
+                lines: [...prev.lines, ...current.lines],
+                startLine: prev.startLine,
+                endLine: current.endLine,
+            };
+        }
+        else {
+            // Lone tiny section in the file — keep it.
+            merged.push(current);
+        }
+        i += 1;
+    }
+    return merged;
+}
+function bodyOfSection(section) {
+    // The first line is the heading itself; "body" is everything after.
+    const bodyLines = section.lines.slice(1);
+    return bodyLines.join('\n').trim();
+}
+function splitOnHeadings(lines) {
+    const sections = [];
+    let inFence = false;
+    const stack = []; // current heading hierarchy
+    let current = {
+        headings: [...stack],
+        lines: [],
+        startLine: 1,
+    };
+    const flush = (endLine) => {
+        if (current.lines.length > 0 || current.headings.length > 0) {
+            sections.push({
+                headings: current.headings,
+                lines: current.lines,
+                startLine: current.startLine,
+                endLine,
+            });
+        }
+    };
+    lines.forEach((line, i) => {
+        const lineNum = i + 1;
+        if (line.trim().startsWith('```')) {
+            inFence = !inFence;
+            current.lines.push(line);
+            return;
+        }
+        if (!inFence) {
+            const m = /^(#{1,3})\s+(.+?)\s*$/u.exec(line);
+            if (m) {
+                // close the current section before the heading line
+                flush(lineNum - 1);
+                const depth = m[1].length;
+                const text = m[2];
+                // Truncate to depth-1, padding any holes left by missing parent
+                // levels (e.g. a file that starts at H3 with no preceding H1/H2).
+                stack.length = Math.max(0, depth - 1);
+                for (let s = 0; s < stack.length; s++) {
+                    if (stack[s] === undefined)
+                        stack[s] = '';
+                }
+                stack.push(text);
+                current = {
+                    headings: stack.filter((h) => h && h.length > 0),
+                    lines: [line],
+                    startLine: lineNum,
+                };
+                return;
+            }
+        }
+        current.lines.push(line);
+    });
+    flush(lines.length);
+    return sections;
+}
+function sliceLong(text) {
+    const out = [];
+    let pos = 0;
+    while (pos < text.length) {
+        const end = Math.min(pos + MAX_CHUNK_CHARS, text.length);
+        out.push(text.slice(pos, end));
+        if (end >= text.length)
+            break;
+        pos = end - OVERLAP_CHARS;
+        if (pos <= 0)
+            pos = end;
+    }
+    return out;
+}
+function makeChunkId(relPath, headings, sliceIdx) {
+    const slug = headings
+        .map((h) => h
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/gu, '-')
+        .replace(/^-+|-+$/gu, '')
+        .slice(0, 60))
+        .filter(Boolean)
+        .join('/');
+    const tail = sliceIdx > 0 ? `~${sliceIdx}` : '';
+    return slug ? `${relPath}#${slug}${tail}` : `${relPath}#root${tail}`;
+}

package/dist/indexer/crawl.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Crawler — walks the NuOS catalogue tree picking up indexable .md files.
+ *
+ * Per WU 110 spec:
+ *   - includes: docs/build/**, docs/contracts/**, docs/philosophy/**,
+ *     docs/guides/**, plus top-level docs/build/STATE.md, BUILD-ORDER.md,
+ *     README.md, reference-index.md
+ *   - skips: _index.md (derived; adds noise), done/, archive/, superseded/
+ *     subdirs (opt-in via includeArchived)
+ *   - skips: .excalidraw, binary
+ */
+export interface CrawlOptions {
+    catalogueRoot: string;
+    includeArchived?: boolean;
+}
+export interface CrawledFile {
+    absolutePath: string;
+    relativePath: string;
+}
+export declare function crawl(options: CrawlOptions): Promise<CrawledFile[]>;

package/dist/indexer/crawl.js

@@ -0,0 +1,66 @@
+/**
+ * Crawler — walks the NuOS catalogue tree picking up indexable .md files.
+ *
+ * Per WU 110 spec:
+ *   - includes: docs/build/**, docs/contracts/**, docs/philosophy/**,
+ *     docs/guides/**, plus top-level docs/build/STATE.md, BUILD-ORDER.md,
+ *     README.md, reference-index.md
+ *   - skips: _index.md (derived; adds noise), done/, archive/, superseded/
+ *     subdirs (opt-in via includeArchived)
+ *   - skips: .excalidraw, binary
+ */
+import { readdir, stat } from 'node:fs/promises';
+import path from 'node:path';
+const TOP_LEVEL_INCLUDES = ['build', 'contracts', 'philosophy', 'guides'];
+const SKIPPED_DIR_NAMES = new Set(['node_modules', '.git', '.nuos-catalogue']);
+const ARCHIVED_DIR_NAMES = new Set(['done', 'archive', 'superseded']);
+const INDEX_FILENAMES = new Set(['_index.md']);
+export async function crawl(options) {
+    const out = [];
+    for (const top of TOP_LEVEL_INCLUDES) {
+        const start = path.join(options.catalogueRoot, top);
+        if (await exists(start)) {
+            await walkDir(start, options, out);
+        }
+    }
+    return out.sort((a, b) => a.relativePath.localeCompare(b.relativePath));
+}
+async function walkDir(dir, options, out) {
+    let entries;
+    try {
+        entries = await readdir(dir, { withFileTypes: true });
+    }
+    catch {
+        return;
+    }
+    for (const entry of entries) {
+        const full = path.join(dir, entry.name);
+        if (entry.isDirectory()) {
+            if (SKIPPED_DIR_NAMES.has(entry.name))
+                continue;
+            if (!options.includeArchived && ARCHIVED_DIR_NAMES.has(entry.name))
+                continue;
+            await walkDir(full, options, out);
+            continue;
+        }
+        if (!entry.isFile())
+            continue;
+        if (!entry.name.endsWith('.md'))
+            continue;
+        if (INDEX_FILENAMES.has(entry.name))
+            continue;
+        out.push({
+            absolutePath: full,
+            relativePath: path.relative(options.catalogueRoot, full),
+        });
+    }
+}
+async function exists(p) {
+    try {
+        await stat(p);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}

package/dist/indexer/metadata.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Per-file metadata extraction.
+ *
+ * Returns a structured FileMeta record from the file path and content.
+ * Per-kind resolvers handle the variation between work-units, decisions,
+ * sessions, etc.
+ *
+ * Cross-references parse markdown links of the form `(D040)`, `(WU 110)`,
+ * `(Q015)`, `[D040](D040-...)`, etc. — so a future query like "what
+ * references D040" can be answered by metadata filter alone.
+ */
+export type FileKind = 'work_unit' | 'decision' | 'session' | 'open_question' | 'risk' | 'contract' | 'philosophy' | 'guide' | 'map' | 'state' | 'build_order' | 'reference' | 'readme' | 'unknown';
+export interface FileMeta {
+    path: string;
+    kind: FileKind;
+    idInKind: string | null;
+    status: string | null;
+    date: string | null;
+    crossRefs: string[];
+}
+export declare function extractMetadata(absolutePath: string, relativePath: string, content: string): Promise<FileMeta>;

package/dist/indexer/metadata.js

@@ -0,0 +1,126 @@
+/**
+ * Per-file metadata extraction.
+ *
+ * Returns a structured FileMeta record from the file path and content.
+ * Per-kind resolvers handle the variation between work-units, decisions,
+ * sessions, etc.
+ *
+ * Cross-references parse markdown links of the form `(D040)`, `(WU 110)`,
+ * `(Q015)`, `[D040](D040-...)`, etc. — so a future query like "what
+ * references D040" can be answered by metadata filter alone.
+ */
+import { stat } from 'node:fs/promises';
+export async function extractMetadata(absolutePath, relativePath, content) {
+    const kind = classifyKind(relativePath);
+    const idInKind = extractIdInKind(kind, relativePath, content);
+    const status = extractStatus(content);
+    const date = await extractDate(absolutePath, content);
+    const crossRefs = extractCrossRefs(content);
+    return {
+        path: relativePath,
+        kind,
+        idInKind,
+        status,
+        date,
+        crossRefs,
+    };
+}
+function classifyKind(relPath) {
+    const p = relPath.replace(/\\/g, '/');
+    if (p === 'build/STATE.md')
+        return 'state';
+    if (p === 'build/BUILD-ORDER.md')
+        return 'build_order';
+    if (p === 'build/README.md')
+        return 'readme';
+    if (p === 'build/reference-index.md')
+        return 'reference';
+    if (p.startsWith('build/work-units/'))
+        return 'work_unit';
+    if (p.startsWith('build/decisions/'))
+        return 'decision';
+    if (p.startsWith('build/sessions/'))
+        return 'session';
+    if (p.startsWith('build/open-questions/'))
+        return 'open_question';
+    if (p.startsWith('build/risks/'))
+        return 'risk';
+    if (p.startsWith('build/maps/'))
+        return 'map';
+    if (p.startsWith('contracts/'))
+        return 'contract';
+    if (p.startsWith('philosophy/'))
+        return 'philosophy';
+    if (p.startsWith('guides/'))
+        return 'guide';
+    return 'unknown';
+}
+function extractIdInKind(kind, relPath, content) {
+    const file = relPath.split('/').pop() ?? '';
+    if (kind === 'work_unit') {
+        const m = /^(\d{3})/.exec(file);
+        return m ? `WU ${m[1]}` : null;
+    }
+    if (kind === 'decision') {
+        const m = /^(D\d+)/.exec(file);
+        return m ? m[1] : null;
+    }
+    if (kind === 'open_question') {
+        const m = /^(Q\d+)/.exec(file);
+        return m ? m[1] : null;
+    }
+    if (kind === 'risk') {
+        const m = /^(R\d+)/.exec(file);
+        return m ? m[1] : null;
+    }
+    if (kind === 'session') {
+        // Session files are dated; pull the leading H1 if present
+        const h = /^#\s+(.+?)\s*$/m.exec(content);
+        return h ? h[1] : file.replace(/\.md$/, '');
+    }
+    return null;
+}
+function extractStatus(content) {
+    const m = /^\*\*Status:\*\*\s*(.+?)\s*$/m.exec(content);
+    if (m)
+        return m[1].replace(/\s+/g, ' ').trim();
+    return null;
+}
+async function extractDate(absolutePath, content) {
+    // Look for "**Date:** 2026-05-08" or "Date: ..." in frontmatter style
+    const m = /^\*\*Date:\*\*\s*(\d{4}-\d{2}-\d{2})/m.exec(content) ||
+        /^Date:\s*(\d{4}-\d{2}-\d{2})/m.exec(content);
+    if (m)
+        return m[1];
+    try {
+        const s = await stat(absolutePath);
+        return s.mtime.toISOString().slice(0, 10);
+    }
+    catch {
+        return null;
+    }
+}
+const REF_PATTERNS = [
+    { regex: /\bD\d{3}\b/gu, canonical: (m) => m[0] },
+    { regex: /\bQ\d{3}\b/gu, canonical: (m) => m[0] },
+    { regex: /\bR\d{3}\b/gu, canonical: (m) => m[0] },
+    { regex: /\bWU\s*0?\d{2,3}[a-z]?\b/giu, canonical: (m) => normaliseWu(m[0]) },
+];
+function normaliseWu(raw) {
+    const m = /WU\s*0?(\d{2,3})([a-z]?)/i.exec(raw);
+    if (!m)
+        return raw.toUpperCase();
+    const num = m[1].padStart(3, '0');
+    const tail = m[2] ? m[2].toLowerCase() : '';
+    return `WU ${num}${tail}`;
+}
+function extractCrossRefs(content) {
+    const found = new Set();
+    for (const { regex, canonical } of REF_PATTERNS) {
+        let m;
+        while ((m = regex.exec(content)) !== null) {
+            found.add(canonical(m));
+        }
+    }
+    return [...found].sort();
+}

package/dist/indexer/upsert.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Index orchestrator — crawl + chunk + extract metadata + embed + upsert.
+ *
+ * Hash-based incremental: a separate `.nuos-catalogue/hashes.json` tracks
+ * the last-indexed content hash per file. Unchanged files are skipped.
+ * Deleted files are removed from the index.
+ */
+import type { NuVector } from '@nusoft/nuvector';
+import type { Embedder } from '../embedder/types.js';
+export interface IndexConfig {
+    catalogueRoot: string;
+    hashFilePath: string;
+    store: NuVector;
+    embedder: Embedder;
+    force?: boolean;
+    dryRun?: boolean;
+}
+export interface IndexReport {
+    indexed: number;
+    updated: number;
+    deleted: number;
+    unchanged: number;
+    chunks: number;
+    durationMs: number;
+}
+export declare function runIndex(config: IndexConfig): Promise<IndexReport>;