@nusoft/nuos-build-catalogue 0.10.0 → 0.10.1

package/dist/indexer/upsert.js

@@ -0,0 +1,152 @@
+/**
+ * Index orchestrator — crawl + chunk + extract metadata + embed + upsert.
+ *
+ * Hash-based incremental: a separate `.nuos-catalogue/hashes.json` tracks
+ * the last-indexed content hash per file. Unchanged files are skipped.
+ * Deleted files are removed from the index.
+ */
+import { readFile, mkdir, writeFile } from 'node:fs/promises';
+import { existsSync } from 'node:fs';
+import { createHash } from 'node:crypto';
+import path from 'node:path';
+import { crawl } from './crawl.js';
+import { chunkMarkdown } from './chunk.js';
+import { extractMetadata } from './metadata.js';
+import { TENANT } from '../store/open.js';
+export async function runIndex(config) {
+    const startedAt = Date.now();
+    const files = await crawl({ catalogueRoot: config.catalogueRoot });
+    const previous = await loadHashes(config.hashFilePath);
+    const next = {};
+    let indexed = 0;
+    let updated = 0;
+    let unchanged = 0;
+    let totalChunks = 0;
+    // Collect chunks for files that need (re)indexing
+    const pending = [];
+    for (const file of files) {
+        const content = await readFile(file.absolutePath, 'utf8');
+        const sha = sha256(content);
+        const prev = previous[file.relativePath];
+        if (!config.force && prev && prev.sha === sha) {
+            unchanged += 1;
+            next[file.relativePath] = prev;
+            continue;
+        }
+        const meta = await extractMetadata(file.absolutePath, file.relativePath, content);
+        const chunks = chunkMarkdown(file.relativePath, content);
+        pending.push({ file, chunks, meta, sha });
+        totalChunks += chunks.length;
+        if (prev)
+            updated += 1;
+        else
+            indexed += 1;
+    }
+    // Embed and upsert pending chunks in batches
+    if (pending.length > 0 && !config.dryRun) {
+        const allUpserts = [];
+        for (const item of pending) {
+            const texts = item.chunks.map((c) => c.text);
+            const embeddings = await config.embedder.embed(texts);
+            item.chunks.forEach((c, i) => {
+                allUpserts.push({
+                    chunkId: c.id,
+                    text: c.text,
+                    embedding: embeddings[i],
+                    fileMeta: item.meta,
+                    headings: c.headings,
+                    startLine: c.startLine,
+                    endLine: c.endLine,
+                });
+            });
+            // Remove any old chunk ids that are no longer present
+            const oldIds = new Set(previous[item.file.relativePath]?.chunkIds ?? []);
+            const currentIds = new Set(item.chunks.map((c) => c.id));
+            for (const stale of oldIds) {
+                if (!currentIds.has(stale)) {
+                    await safeDelete(config.store, stale);
+                }
+            }
+            next[item.file.relativePath] = {
+                sha: item.sha,
+                chunkIds: item.chunks.map((c) => c.id),
+            };
+        }
+        for (const u of allUpserts) {
+            // Indexed as document_chunk because that's what these are: chunks
+            // of standalone markdown documents, not NuWiki articles with
+            // section/citation/graph structure. Search uses retrieveContext()
+            // (not searchKnowledge, which is the NuWiki four-layer entry point).
+            await config.store.upsert({
+                id: u.chunkId,
+                kind: 'document_chunk',
+                embedding: u.embedding,
+                text: u.text,
+                tenant: TENANT,
+                metadata: {
+                    path: u.fileMeta.path,
+                    file_kind: u.fileMeta.kind,
+                    id_in_kind: u.fileMeta.idInKind ?? '',
+                    status: u.fileMeta.status ?? '',
+                    date: u.fileMeta.date ?? '',
+                    headings: u.headings.join(' / '),
+                    start_line: u.startLine,
+                    end_line: u.endLine,
+                    cross_refs: u.fileMeta.crossRefs.join(','),
+                },
+            });
+        }
+    }
+    // Detect and remove files that vanished
+    let deleted = 0;
+    for (const oldRelPath of Object.keys(previous)) {
+        if (!files.some((f) => f.relativePath === oldRelPath)) {
+            deleted += 1;
+            if (!config.dryRun) {
+                for (const id of previous[oldRelPath].chunkIds) {
+                    await safeDelete(config.store, id);
+                }
+            }
+        }
+    }
+    if (!config.dryRun) {
+        await saveHashes(config.hashFilePath, next);
+    }
+    return {
+        indexed,
+        updated,
+        deleted,
+        unchanged,
+        chunks: totalChunks,
+        durationMs: Date.now() - startedAt,
+    };
+}
+function sha256(content) {
+    return createHash('sha256').update(content).digest('hex');
+}
+async function loadHashes(filePath) {
+    if (!existsSync(filePath))
+        return {};
+    try {
+        const buf = await readFile(filePath, 'utf8');
+        return JSON.parse(buf);
+    }
+    catch {
+        return {};
+    }
+}
+async function saveHashes(filePath, table) {
+    await mkdir(path.dirname(filePath), { recursive: true });
+    await writeFile(filePath, JSON.stringify(table, null, 2) + '\n', 'utf8');
+}
+async function safeDelete(store, id) {
+    try {
+        await store.delete({
+            ids: [id],
+            tenant: TENANT,
+        });
+    }
+    catch {
+        // NuVector v0.1.0 delete API shape may vary; failure here is non-fatal
+    }
+}

package/dist/migrate/parsers.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Per-register markdown parsers.
+ *
+ * Each parser tolerates the pre-D046 shape — fields that may or may not
+ * be present default to null. The shared shape extracted is title +
+ * status; the rest of the file body is preserved verbatim in
+ * `rawMarkdown`.
+ */
+import type { MigratedRecord, Register } from './types.js';
+export interface ParseFileInput {
+    absolutePath: string;
+    relativePath: string;
+    content: string;
+    register: Register;
+}
+export declare function parseFile(input: ParseFileInput): Promise<MigratedRecord>;
+export declare function registerForRelativePath(relativePath: string): Register | null;

package/dist/migrate/parsers.js

@@ -0,0 +1,123 @@
+/**
+ * Per-register markdown parsers.
+ *
+ * Each parser tolerates the pre-D046 shape — fields that may or may not
+ * be present default to null. The shared shape extracted is title +
+ * status; the rest of the file body is preserved verbatim in
+ * `rawMarkdown`.
+ */
+import { stat } from 'node:fs/promises';
+import path from 'node:path';
+// WU filenames may carry a single lowercase-letter suffix to denote
+// sub-WUs that share a parent number (e.g. 030g-..., 072a-..., 072b-...).
+// The integer `number` is the digit portion; the `handle` carries the
+// suffix verbatim so wu-030 and wu-030g remain distinct records.
+const FILENAME_PATTERNS = {
+    work_unit: /^(?<num>\d{1,4})(?<suffix>[a-z]?)-(?<slug>.+)\.md$/,
+    decision: /^D(?<num>\d{3,})-(?<slug>.+)\.md$/,
+    open_question: /^Q(?<num>\d{3,})-(?<slug>.+)\.md$/,
+    persona: /^P(?<num>\d{3,})-(?<slug>.+)\.md$/,
+};
+export async function parseFile(input) {
+    const { absolutePath, relativePath, content, register } = input;
+    const filename = path.basename(relativePath);
+    // Skip _index.md and template files in the parent walker — but if they
+    // ever reach here, fail loudly so we don't silently migrate noise.
+    if (filename.startsWith('_') || filename.includes('template')) {
+        throw new Error(`parseFile: ${relativePath} looks like a non-artefact file (index/template); the walker should have skipped it`);
+    }
+    const pattern = FILENAME_PATTERNS[register];
+    const match = pattern.exec(filename);
+    if (!match || !match.groups) {
+        throw new Error(`parseFile: ${relativePath} does not match the ${register} filename pattern ${pattern}`);
+    }
+    const number = parseInt(match.groups.num, 10);
+    if (!Number.isInteger(number) || number < 1) {
+        throw new Error(`parseFile: ${relativePath} produced invalid number ${match.groups.num}`);
+    }
+    const slug = match.groups.slug;
+    const suffix = match.groups.suffix ?? '';
+    const handle = formatHandle(register, number, suffix);
+    const title = extractTitle(content) ?? slugToTitle(slug);
+    const status = extractStatus(content);
+    let fileModifiedAt;
+    try {
+        const s = await stat(absolutePath);
+        fileModifiedAt = s.mtime.toISOString();
+    }
+    catch {
+        fileModifiedAt = new Date().toISOString();
+    }
+    const record = {
+        handle,
+        number,
+        register,
+        title,
+        status,
+        slug,
+        sourcePath: relativePath,
+        rawMarkdown: content,
+        fileModifiedAt,
+        migratedAt: new Date().toISOString(),
+        migratedFrom: 'markdown',
+    };
+    return record;
+}
+function formatHandle(register, n, suffix = '') {
+    switch (register) {
+        case 'work_unit':
+            return `wu-${String(n).padStart(3, '0')}${suffix}`;
+        case 'decision':
+            return `D${String(n).padStart(3, '0')}`;
+        case 'open_question':
+            return `Q${String(n).padStart(3, '0')}`;
+        case 'persona':
+            return `P${String(n).padStart(3, '0')}`;
+    }
+}
+function extractTitle(content) {
+    // First H1 wins. Leading whitespace and trailing whitespace trimmed.
+    const m = /^#\s+(.+?)\s*$/m.exec(content);
+    if (!m)
+        return null;
+    return m[1].trim();
+}
+function extractStatus(content) {
+    // Recognise both "**Status:** ..." (decisions, sessions) and
+    // "| Status | ... |" pipe-table rows (some WUs use a metadata table).
+    // Returns the raw status string for downstream interpretation; we
+    // don't normalise to a typed enum here because the pre-D046 shape
+    // includes states like "🟢 ready" with emoji prefixes.
+    const bold = /^\*\*Status:\*\*\s*(.+?)\s*$/m.exec(content);
+    if (bold)
+        return bold[1].trim();
+    const table = /^\|\s*Status\s*\|\s*(.+?)\s*\|\s*$/m.exec(content);
+    if (table)
+        return table[1].trim();
+    return null;
+}
+function slugToTitle(slug) {
+    return slug
+        .split('-')
+        .map((part) => (part.length > 0 ? part[0].toUpperCase() + part.slice(1) : part))
+        .join(' ');
+}
+export function registerForRelativePath(relativePath) {
+    // The walker uses this to assign each file to a register based on its
+    // top-level directory. Subdirectories like work-units/done/ and
+    // decisions/superseded/ map to the parent register.
+    const normalised = relativePath.replace(/\\/g, '/');
+    const top = normalised.split('/')[0];
+    switch (top) {
+        case 'work-units':
+            return 'work_unit';
+        case 'decisions':
+            return 'decision';
+        case 'open-questions':
+            return 'open_question';
+        case 'personas':
+            return 'persona';
+        default:
+            return null;
+    }
+}

package/dist/migrate/run.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Migration runner.
+ *
+ * Walks the four register directories under the catalogue root, parses
+ * each artefact file, and writes a `MigratedRecord` to the JSON-backed
+ * workflow store. Idempotent: re-running on a clean catalogue produces
+ * zero new records on the second pass.
+ *
+ * Per the WU 111 spec, this runner does NOT use the NuFlow runtime
+ * lifecycle. Migration is bulk back-fill, not a series of build-
+ * maintainer decisions; using the runtime would force every legacy
+ * artefact through propose → confirm → approve → commit, which is
+ * neither honest nor scalable.
+ */
+import type { MigrationReport } from './types.js';
+import type { WorkflowStore } from './store.js';
+export interface RunMigrateConfig {
+    catalogueRoot: string;
+    store: WorkflowStore;
+    dryRun?: boolean;
+}
+export declare function runMigrate(config: RunMigrateConfig): Promise<MigrationReport>;

package/dist/migrate/run.js

@@ -0,0 +1,142 @@
+/**
+ * Migration runner.
+ *
+ * Walks the four register directories under the catalogue root, parses
+ * each artefact file, and writes a `MigratedRecord` to the JSON-backed
+ * workflow store. Idempotent: re-running on a clean catalogue produces
+ * zero new records on the second pass.
+ *
+ * Per the WU 111 spec, this runner does NOT use the NuFlow runtime
+ * lifecycle. Migration is bulk back-fill, not a series of build-
+ * maintainer decisions; using the runtime would force every legacy
+ * artefact through propose → confirm → approve → commit, which is
+ * neither honest nor scalable.
+ */
+import { readdir, readFile } from 'node:fs/promises';
+import path from 'node:path';
+import { parseFile, registerForRelativePath } from './parsers.js';
+const REGISTER_DIRS = ['work_unit', 'decision', 'open_question', 'persona'];
+const REGISTER_TO_DIRNAME = {
+    work_unit: 'work-units',
+    decision: 'decisions',
+    open_question: 'open-questions',
+    persona: 'personas',
+};
+export async function runMigrate(config) {
+    const startedAt = Date.now();
+    const byRegister = {
+        work_unit: { scanned: 0, migrated: 0, skipped: 0 },
+        decision: { scanned: 0, migrated: 0, skipped: 0 },
+        open_question: { scanned: 0, migrated: 0, skipped: 0 },
+        persona: { scanned: 0, migrated: 0, skipped: 0 },
+    };
+    let scanned = 0;
+    let migrated = 0;
+    let skipped = 0;
+    const conflicts = [];
+    for (const register of REGISTER_DIRS) {
+        const dirName = REGISTER_TO_DIRNAME[register];
+        const baseDir = path.join(config.catalogueRoot, dirName);
+        const files = await collectArtefactFiles(baseDir, dirName);
+        for (const relPath of files) {
+            const inferredRegister = registerForRelativePath(relPath);
+            if (inferredRegister !== register) {
+                // Defensive: shouldn't happen given the walker scoping above.
+                throw new Error(`runMigrate: register mismatch for ${relPath} (expected ${register}, got ${inferredRegister})`);
+            }
+            const absolutePath = path.join(config.catalogueRoot, relPath);
+            const content = await readFile(absolutePath, 'utf8');
+            const record = await parseFile({
+                absolutePath,
+                relativePath: relPath,
+                content,
+                register,
+            });
+            scanned += 1;
+            byRegister[register].scanned += 1;
+            if (config.store.has(record.handle)) {
+                const existing = config.store.get(record.handle);
+                if (existing && existing.sourcePath !== record.sourcePath) {
+                    // A different file claimed this handle. This is a real
+                    // catalogue-discipline issue (e.g. two WUs sharing the same
+                    // number prefix); surface it rather than silently dropping.
+                    conflicts.push({
+                        handle: record.handle,
+                        winnerSourcePath: existing.sourcePath,
+                        loserSourcePath: record.sourcePath,
+                    });
+                }
+                skipped += 1;
+                byRegister[register].skipped += 1;
+                continue;
+            }
+            if (!config.dryRun) {
+                config.store.put(record);
+            }
+            migrated += 1;
+            byRegister[register].migrated += 1;
+        }
+    }
+    if (!config.dryRun) {
+        await config.store.flush();
+    }
+    return {
+        scanned,
+        migrated,
+        skipped,
+        conflicts,
+        byRegister,
+        durationMs: Date.now() - startedAt,
+    };
+}
+/**
+ * After in-memory store-puts, we still need to detect within-pass
+ * conflicts. The block above handles that via `config.store.has()` +
+ * `get()`; the result lands in the report's `conflicts` array.
+ *
+ * Collect markdown artefact files under a register directory, including
+ * one level of subdirectory (e.g. work-units/done, decisions/superseded).
+ * Skips index files (`_index.md`), templates (filename includes
+ * 'template'), and any non-.md files.
+ */
+async function collectArtefactFiles(baseDir, registerDirName) {
+    let entries;
+    try {
+        entries = (await readdir(baseDir, { withFileTypes: true }));
+    }
+    catch {
+        // Register directory may not exist (e.g. personas/ before any
+        // persona is authored). Treat as empty.
+        return [];
+    }
+    const files = [];
+    for (const entry of entries) {
+        const entryName = entry.name;
+        if (entry.isFile() && isArtefactFile(entryName)) {
+            files.push(`${registerDirName}/${entryName}`);
+        }
+        else if (entry.isDirectory()) {
+            // Recurse one level for done/, superseded/, etc.
+            const subPath = path.join(baseDir, entryName);
+            const subEntries = (await readdir(subPath, { withFileTypes: true }));
+            for (const sub of subEntries) {
+                const subName = sub.name;
+                if (sub.isFile() && isArtefactFile(subName)) {
+                    files.push(`${registerDirName}/${entryName}/${subName}`);
+                }
+            }
+        }
+    }
+    return files;
+}
+function isArtefactFile(filename) {
+    if (!filename.endsWith('.md'))
+        return false;
+    if (filename.startsWith('_'))
+        return false; // _index.md, _template, etc.
+    if (filename.toLowerCase().includes('template'))
+        return false;
+    if (filename.toLowerCase().endsWith('-template.md'))
+        return false;
+    return true;
+}

package/dist/migrate/store.d.ts

@@ -0,0 +1,20 @@
+/**
+ * JSON-backed workflow record store.
+ *
+ * Phase G uses a flat JSON file at `.nuos-catalogue/workflows.json` for
+ * the migrated workflow records. Simple, inspectable, and sets up
+ * Phase I cleanly (markdown regeneration reads from the same file).
+ *
+ * NuVector cutover is a deliberate follow-up. The store interface is
+ * intentionally narrow (read by handle, write, list) so a NuVector
+ * adapter can be substituted later without changing call sites.
+ */
+import type { MigratedRecord } from './types.js';
+export interface WorkflowStore {
+    has(handle: string): boolean;
+    get(handle: string): MigratedRecord | null;
+    put(record: MigratedRecord): void;
+    list(): MigratedRecord[];
+    flush(): Promise<void>;
+}
+export declare function openWorkflowStore(filePath: string): Promise<WorkflowStore>;

package/dist/migrate/store.js

@@ -0,0 +1,52 @@
+/**
+ * JSON-backed workflow record store.
+ *
+ * Phase G uses a flat JSON file at `.nuos-catalogue/workflows.json` for
+ * the migrated workflow records. Simple, inspectable, and sets up
+ * Phase I cleanly (markdown regeneration reads from the same file).
+ *
+ * NuVector cutover is a deliberate follow-up. The store interface is
+ * intentionally narrow (read by handle, write, list) so a NuVector
+ * adapter can be substituted later without changing call sites.
+ */
+import { readFile, writeFile, mkdir } from 'node:fs/promises';
+import { existsSync } from 'node:fs';
+import path from 'node:path';
+export async function openWorkflowStore(filePath) {
+    const data = await load(filePath);
+    return {
+        has(handle) {
+            return Object.prototype.hasOwnProperty.call(data.records, handle);
+        },
+        get(handle) {
+            return data.records[handle] ?? null;
+        },
+        put(record) {
+            data.records[record.handle] = record;
+        },
+        list() {
+            return Object.values(data.records);
+        },
+        async flush() {
+            await persist(filePath, data);
+        },
+    };
+}
+async function load(filePath) {
+    if (!existsSync(filePath)) {
+        return { schemaVersion: 1, records: {} };
+    }
+    const raw = await readFile(filePath, 'utf8');
+    if (raw.trim().length === 0) {
+        return { schemaVersion: 1, records: {} };
+    }
+    const parsed = JSON.parse(raw);
+    if (parsed.schemaVersion !== 1 || typeof parsed.records !== 'object') {
+        throw new Error(`openWorkflowStore: ${filePath} has unrecognised shape (expected { schemaVersion: 1, records: {} })`);
+    }
+    return parsed;
+}
+async function persist(filePath, data) {
+    await mkdir(path.dirname(filePath), { recursive: true });
+    await writeFile(filePath, JSON.stringify(data, null, 2) + '\n', 'utf8');
+}

package/dist/migrate/types.d.ts

@@ -0,0 +1,57 @@
+/**
+ * Minimal shapes the migrate runner produces. Phase G ships count-parity
+ * across the four registers (work_unit, decision, open_question,
+ * persona); rich field-level fidelity is deferred to the post-cutover
+ * authoring path (workflows write the new shape directly).
+ *
+ * The migration is a back-fill: the live catalogue's pre-D046 WUs and
+ * decisions don't have the new fields in their markdown, so the parser
+ * preserves the title, handle, number, status, slug, source path, raw
+ * markdown, and file mtime. Future authoring-via-workflow operations
+ * fill in the richer shape organically.
+ */
+export type Register = 'work_unit' | 'decision' | 'open_question' | 'persona';
+export interface MigratedRecord {
+    /** wu-NNN | D### | Q### | P### */
+    handle: string;
+    /** Numeric portion of the handle (e.g. 111 from wu-111). */
+    number: number;
+    /** Which register this record belongs to. */
+    register: Register;
+    /** First H1 heading from the file, or filename-derived fallback. */
+    title: string;
+    /** Status string parsed from the file, or null if not surfaced. */
+    status: string | null;
+    /** Filename slug (the kebab-cased portion after the number prefix). */
+    slug: string;
+    /** Path relative to the catalogue root (e.g. work-units/done/111-...). */
+    sourcePath: string;
+    /** Full markdown body, preserved for future re-parsing. */
+    rawMarkdown: string;
+    /** ISO timestamp — file mtime; preserves "original timestamp" per spec. */
+    fileModifiedAt: string;
+    /** ISO timestamp — when this record was migrated. */
+    migratedAt: string;
+    /** Always 'markdown' for Phase G; future phases may add other origins. */
+    migratedFrom: 'markdown';
+}
+export interface HandleConflict {
+    handle: string;
+    /** The file that won (was already in the store or scanned first). */
+    winnerSourcePath: string;
+    /** The file that was dropped because its handle was already taken. */
+    loserSourcePath: string;
+}
+export interface MigrationReport {
+    scanned: number;
+    migrated: number;
+    skipped: number;
+    /** Files dropped because a different file already claimed the same handle. */
+    conflicts: HandleConflict[];
+    byRegister: Record<Register, {
+        scanned: number;
+        migrated: number;
+        skipped: number;
+    }>;
+    durationMs: number;
+}

package/dist/migrate/types.js

@@ -0,0 +1,13 @@
+/**
+ * Minimal shapes the migrate runner produces. Phase G ships count-parity
+ * across the four registers (work_unit, decision, open_question,
+ * persona); rich field-level fidelity is deferred to the post-cutover
+ * authoring path (workflows write the new shape directly).
+ *
+ * The migration is a back-fill: the live catalogue's pre-D046 WUs and
+ * decisions don't have the new fields in their markdown, so the parser
+ * preserves the title, handle, number, status, slug, source path, raw
+ * markdown, and file mtime. Future authoring-via-workflow operations
+ * fill in the richer shape organically.
+ */
+export {};

package/dist/regenerate/check.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Drift detection: walk the workflow store, compare each record's
+ * stored `rawMarkdown` to its source file, report differences.
+ */
+import type { WorkflowStore } from '../migrate/store.js';
+import type { DriftReport, RegenerateConfig } from './types.js';
+export interface CheckRegenerateConfig extends RegenerateConfig {
+    catalogueRoot: string;
+    store: WorkflowStore;
+}
+export declare function runRegenerate(config: CheckRegenerateConfig): Promise<DriftReport>;