@nusoft/nuos-build-catalogue 0.33.1 → 0.33.3

package/dist/commands/memory.d.ts

@@ -4,8 +4,9 @@
  *
  * Cross-agent memory: every agent in a swarm can write findings here and
  * any future agent (in this run or a later one) can retrieve them by
- * semantic query. Uses the same NuVector store as the catalogue index,
- * distinguished by kind: 'agent_memory'.
+ * semantic query. Uses its own NuVector store file (`memory.nv`), separate
+ * from the doc-search index (`index.nv`), so that the ~40s background
+ * reindex never locks out memory writes. See D131.
  *
  * CLI:
  *   memory store  --value="..."  [--wu=wu-007] [--agent=architect] [--key="label"]
@@ -18,6 +19,9 @@ export interface MemoryStoreOptions {
     key?: string;
     cwd?: string;
     buildRoot?: string | boolean;
+    /** Override for the memory store path (defaults to `<index-dir>/memory.nv`). */
+    memory?: string | boolean;
+    /** @deprecated Kept for callers that pass `index` — resolved as `memory` for memory commands. */
     index?: string | boolean;
 }
 export interface MemorySearchOptions {
@@ -27,6 +31,9 @@ export interface MemorySearchOptions {
     agent?: string;
     cwd?: string;
     buildRoot?: string | boolean;
+    /** Override for the memory store path (defaults to `<index-dir>/memory.nv`). */
+    memory?: string | boolean;
+    /** @deprecated Kept for callers that pass `index` — resolved as `memory` for memory commands. */
     index?: string | boolean;
 }
 export interface MemoryHit {

package/dist/commands/memory.js

@@ -4,21 +4,167 @@
  *
  * Cross-agent memory: every agent in a swarm can write findings here and
  * any future agent (in this run or a later one) can retrieve them by
- * semantic query. Uses the same NuVector store as the catalogue index,
- * distinguished by kind: 'agent_memory'.
+ * semantic query. Uses its own NuVector store file (`memory.nv`), separate
+ * from the doc-search index (`index.nv`), so that the ~40s background
+ * reindex never locks out memory writes. See D131.
  *
  * CLI:
  *   memory store  --value="..."  [--wu=wu-007] [--agent=architect] [--key="label"]
  *   memory search --query="..."  [--limit=N]   [--wu=wu-007]       [--agent=architect]
  */
 import { randomUUID } from 'node:crypto';
-import { resolveBuildRoot, resolveIndexPath } from '../path-resolution.js';
+import { existsSync, unlinkSync, writeFileSync } from 'node:fs';
+import { resolveBuildRoot, resolveIndexPath, resolveMemoryPath } from '../path-resolution.js';
+// resolveIndexPath is used only as the migration *source* (legacy index.nv),
+// not as the live memory path (which is resolved via resolveMemoryPath).
 // NuVector's MemoryRecordKind union doesn't include a swarm-specific kind yet.
 // 'workflow_provenance' is the closest semantic match — agent memories are
 // provenance of the swarm workflow. NuFlow isn't wired (harness.runtime.nuflow
 // is null) so there's no collision today; records are further distinguished by
 // the presence of an `agent_role` metadata field (absent on NuFlow provenance).
 const MEMORY_KIND = 'workflow_provenance';
+/**
+ * One-time idempotent migration: copy existing agent-memory records
+ * (kind `workflow_provenance` with an `agent_role` metadata field) from
+ * the legacy `index.nv` into the new `memory.nv`. Triggered lazily the
+ * first time a memory command opens the store (i.e. when `memory.nv` does
+ * not yet exist). Once `memory.nv` exists this function is a no-op.
+ *
+ * Decision on delete-vs-leave: we leave migrated records in `index.nv`.
+ * They are dead weight there — `memory search` reads only `memory.nv`,
+ * and the doc reindex upserts only doc-kind records — so leaving them
+ * causes no observable problem. Deletion via the store's `DeletionQuery`
+ * API would need the id list; the extra complexity buys nothing for a
+ * handful of records.
+ *
+ * Embeddings are copied verbatim via `fetch(ids)` — no re-embedding.
+ * If `index.nv` does not exist yet (fresh project), migration is skipped.
+ *
+ * Atomicity: uses a sentinel file (`memory.nv.migrating`) written before the
+ * migration opens `memory.nv` and deleted after a successful close. If the
+ * process dies mid-migration, the next run sees both files and retries.
+ *
+ * INVARIANT — never `unlinkSync(memoryPath)` then `openStore(memoryPath)` in
+ * the same process. NuVector's NAPI in-process inode registry tracks handles
+ * by inode; a same-process unlink+reopen materialises the store in-memory only
+ * (the file never appears on disk), silently losing all data on process exit.
+ * The only permitted `unlinkSync(memoryPath)` is the corrupt-open-failure guard
+ * at the bottom, which always re-throws immediately — the store is never
+ * reopened in the same process after that unlink.
+ *
+ * In the interrupted-migration path (memory.nv + sentinel both present) we
+ * therefore open the existing partial `memory.nv` directly. `upsertBatch` is
+ * idempotent by id, so re-writing the same records into a partial store just
+ * completes it, with no phantom-materialisation risk.
+ */
+async function migrateMemoryRecordsIfNeeded(indexPath, memoryPath, dimensions) {
+    const sentinelPath = `${memoryPath}.migrating`;
+    // Complete gate: memory.nv exists with no sentinel → done (either a clean
+    // migration or a store created by a normal memory write). Early return.
+    if (existsSync(memoryPath) && !existsSync(sentinelPath))
+        return;
+    // Fresh project: no legacy index to migrate from. Clear any stray sentinel
+    // (shouldn't exist, but be tidy) and return; the caller's openStore will
+    // create memory.nv fresh on its own write.
+    if (!existsSync(indexPath)) {
+        if (existsSync(sentinelPath)) {
+            try {
+                unlinkSync(sentinelPath);
+            }
+            catch { /* ignore — best-effort */ }
+        }
+        return;
+    }
+    const { openStore, TENANT } = await import('../store/open.js');
+    // Write the sentinel before opening memory.nv. If the process dies after
+    // this point, the next run sees both files (or just the sentinel) and
+    // falls through to the (re)migration path below.
+    try {
+        writeFileSync(sentinelPath, '');
+    }
+    catch { /* non-fatal; best-effort */ }
+    try {
+        // Read from index.nv. Hold the store open for both retrieveContext and
+        // fetch — a single open avoids a close→reopen timing window.
+        const srcStore = await openStore({ storagePath: indexPath, dimensions });
+        let fullRecords;
+        try {
+            const zeroEmbedding = new Float32Array(dimensions);
+            const result = await srcStore.retrieveContext({
+                embedding: zeroEmbedding,
+                tenant: TENANT,
+                topK: 10_000,
+                filters: { kind: MEMORY_KIND },
+                scoreThreshold: 0,
+            });
+            const items = (result?.items ?? []);
+            // Filter to agent-memory records (presence of `agent_role` metadata).
+            const agentMemoryRefs = items
+                .filter((item) => {
+                const meta = item.metadata;
+                return meta !== undefined && 'agent_role' in meta;
+            })
+                .map((item) => item.ref);
+            fullRecords = agentMemoryRefs.length > 0
+                ? await srcStore.fetch(agentMemoryRefs)
+                : [];
+        }
+        finally {
+            await srcStore.close();
+        }
+        // Open memory.nv — create fresh (first run) or open the existing partial
+        // file (interrupted run). Do NOT unlink first: same-process unlink+reopen
+        // triggers the NAPI phantom-materialisation bug (see invariant above).
+        // upsertBatch is idempotent by id, so replaying into a partial store is safe.
+        let dstStore;
+        try {
+            dstStore = await openStore({ storagePath: memoryPath, dimensions });
+        }
+        catch (openErr) {
+            // openStore itself threw — the partial file is genuinely corrupt.
+            // Unlink it so a future process gets a clean create, leave the sentinel
+            // so that future run still enters the (re)migration path, then rethrow.
+            // NEVER reopen memoryPath in this process after this unlink.
+            if (existsSync(memoryPath)) {
+                try {
+                    unlinkSync(memoryPath);
+                }
+                catch { /* ignore */ }
+            }
+            throw openErr;
+        }
+        try {
+            if (fullRecords.length > 0) {
+                await dstStore.upsertBatch(fullRecords);
+            }
+            // If there are no agent-memory records, the store is opened-and-closed
+            // empty. That materialises memory.nv on disk so existsSync is true and
+            // the gate is stable — memory search never falls through to re-read
+            // index.nv on subsequent calls.
+        }
+        finally {
+            await dstStore.close();
+        }
+        // Migration complete. Remove sentinel so the gate sees memory.nv alone.
+        try {
+            unlinkSync(sentinelPath);
+        }
+        catch { /* ignore — best-effort */ }
+    }
+    catch (err) {
+        // Any failure other than the corrupt-open case above (e.g. F2 lock on
+        // index.nv): clean up the sentinel so the next call retries from scratch.
+        // Do NOT unlink memoryPath here — if it was opened successfully before the
+        // failure, it's a valid partial store that the next run can complete via
+        // upsertBatch. Unlinking it would trigger the phantom-materialisation bug
+        // on re-entry in the same process.
+        try {
+            unlinkSync(sentinelPath);
+        }
+        catch { /* ignore */ }
+        throw err;
+    }
+}
 export async function cmdMemoryStore(opts) {
     const { value, wu, agent, key } = opts;
     if (!value || value.trim().length === 0) {
@@ -28,9 +174,16 @@ export async function cmdMemoryStore(opts) {
     const { selectEmbedderFromEnv } = await import('../embedder/select.js');
     const { openStore, TENANT } = await import('../store/open.js');
     const buildRoot = resolveBuildRoot(opts.buildRoot, { cwd: opts.cwd ?? process.cwd() });
-    const indexPath = resolveIndexPath(buildRoot, opts.index);
+    // Resolve the memory-specific path (memory.nv), falling back to the
+    // legacy `index` flag for callers that pass it, then the default.
+    const memoryFlag = opts.memory ?? opts.index;
+    const memoryPath = resolveMemoryPath(buildRoot, memoryFlag);
+    const indexPath = resolveIndexPath(buildRoot, undefined);
     const embedder = await selectEmbedderFromEnv();
-    const store = await openStore({ storagePath: indexPath, dimensions: embedder.dimensions });
+    // Lazy one-time migration: move existing agent-memory records from
+    // index.nv into memory.nv on the first memory command run.
+    await migrateMemoryRecordsIfNeeded(indexPath, memoryPath, embedder.dimensions);
+    const store = await openStore({ storagePath: memoryPath, dimensions: embedder.dimensions });
     const [embedding] = await embedder.embed([value]);
     await store.upsert({
         id: randomUUID(),
@@ -59,9 +212,16 @@ export async function cmdMemorySearch(opts) {
     const { selectEmbedderFromEnv } = await import('../embedder/select.js');
     const { openStore, TENANT } = await import('../store/open.js');
     const buildRoot = resolveBuildRoot(opts.buildRoot, { cwd: opts.cwd ?? process.cwd() });
-    const indexPath = resolveIndexPath(buildRoot, opts.index);
+    // Resolve the memory-specific path (memory.nv), falling back to the
+    // legacy `index` flag for callers that pass it, then the default.
+    const memoryFlag = opts.memory ?? opts.index;
+    const memoryPath = resolveMemoryPath(buildRoot, memoryFlag);
+    const indexPath = resolveIndexPath(buildRoot, undefined);
     const embedder = await selectEmbedderFromEnv();
-    const store = await openStore({ storagePath: indexPath, dimensions: embedder.dimensions });
+    // Lazy one-time migration: move existing agent-memory records from
+    // index.nv into memory.nv on the first memory command run.
+    await migrateMemoryRecordsIfNeeded(indexPath, memoryPath, embedder.dimensions);
+    const store = await openStore({ storagePath: memoryPath, dimensions: embedder.dimensions });
     const [queryEmbedding] = await embedder.embed([query]);
     const result = await store.retrieveContext({
         embedding: queryEmbedding,

package/dist/path-resolution.d.ts

@@ -52,6 +52,15 @@ export declare function resolveCatalogueRoot(flag: string | boolean | undefined,
 export declare function resolveIndexDir(buildRoot: string, ctx?: ResolutionContext): string;
 export declare function resolveWorkflowsPath(buildRoot: string, flag: string | boolean | undefined, ctx?: ResolutionContext): string;
 export declare function resolveIndexPath(buildRoot: string, flag: string | boolean | undefined, ctx?: ResolutionContext): string;
+/**
+ * Resolve the cross-agent memory store path. Always co-located with the
+ * doc-index in the same `.nuos-catalogue/` directory, but in a separate
+ * file (`memory.nv`) so that the doc-index reindex (which holds an
+ * exclusive lock on `index.nv`) never contends with memory writes.
+ * Resolves `NUOS_CATALOGUE_MEMORY_PATH` env var when set; otherwise
+ * derives from `resolveIndexDir`. See D131.
+ */
+export declare function resolveMemoryPath(buildRoot: string, flag: string | boolean | undefined, ctx?: ResolutionContext): string;
 export declare function resolveHashPath(buildRoot: string, flag: string | boolean | undefined, ctx?: ResolutionContext): string;
 /**
  * Soft warning surfaced after a `migrate` or `regenerate` run: if the

package/dist/path-resolution.js

@@ -108,6 +108,22 @@ export function resolveIndexPath(buildRoot, flag, ctx) {
         return path.resolve(flag);
     return path.join(resolveIndexDir(buildRoot, ctx), 'index.nv');
 }
+/**
+ * Resolve the cross-agent memory store path. Always co-located with the
+ * doc-index in the same `.nuos-catalogue/` directory, but in a separate
+ * file (`memory.nv`) so that the doc-index reindex (which holds an
+ * exclusive lock on `index.nv`) never contends with memory writes.
+ * Resolves `NUOS_CATALOGUE_MEMORY_PATH` env var when set; otherwise
+ * derives from `resolveIndexDir`. See D131.
+ */
+export function resolveMemoryPath(buildRoot, flag, ctx) {
+    if (typeof flag === 'string' && flag.length > 0)
+        return path.resolve(flag);
+    const env = ctxEnv(ctx);
+    if (env.NUOS_CATALOGUE_MEMORY_PATH)
+        return path.resolve(env.NUOS_CATALOGUE_MEMORY_PATH);
+    return path.join(resolveIndexDir(buildRoot, ctx), 'memory.nv');
+}
 export function resolveHashPath(buildRoot, flag, ctx) {
     if (typeof flag === 'string' && flag.length > 0)
         return path.resolve(flag);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nusoft/nuos-build-catalogue",
-  "version": "0.33.1",
+  "version": "0.33.3",
   "description": "NuOS build-catalogue tooling: semantic search (WU 110) + migration runner that lifts markdown artefacts into JSON-backed workflow records (WU 111, Phase G).",
   "type": "module",
   "bin": {
@@ -19,7 +19,7 @@
     "build": "rm -rf dist && tsc && chmod +x dist/cli.js",
     "prepublishOnly": "npm run build",
     "verify-storage": "tsx scripts/verify-persistence.ts",
-    "test": "tsx --test tests/chunk.test.ts tests/metadata.test.ts tests/crawl.test.ts tests/migrate.test.ts tests/commands-read.test.ts tests/regenerate.test.ts tests/commands-write.test.ts tests/ac-parse.test.ts tests/create.test.ts tests/init.test.ts tests/wu-111-soak-findings.test.ts tests/plan.test.ts tests/mode.test.ts tests/render.test.ts tests/swarm.test.ts tests/setup-progress-bar.test.ts tests/setup-ollama-pull.test.ts tests/setup-run-llm-setup.test.ts tests/wu-active.test.ts tests/install-claude-hooks.test.ts tests/protocols-in-sync.test.ts tests/end-of-session.test.ts",
+    "test": "tsx --test tests/chunk.test.ts tests/metadata.test.ts tests/crawl.test.ts tests/migrate.test.ts tests/commands-read.test.ts tests/regenerate.test.ts tests/commands-write.test.ts tests/ac-parse.test.ts tests/create.test.ts tests/init.test.ts tests/wu-111-soak-findings.test.ts tests/plan.test.ts tests/mode.test.ts tests/render.test.ts tests/swarm.test.ts tests/setup-progress-bar.test.ts tests/setup-ollama-pull.test.ts tests/setup-run-llm-setup.test.ts tests/wu-active.test.ts tests/install-claude-hooks.test.ts tests/protocols-in-sync.test.ts tests/end-of-session.test.ts tests/hooks-in-sync.test.ts tests/memory-store-separation.test.ts",
     "typecheck": "tsc --noEmit",
     "index": "tsx src/cli.ts index",
     "search": "tsx src/cli.ts search"

package/scripts/hooks/pre-commit

@@ -134,14 +134,38 @@ fi
 # ---------- Rule 2: active-decision modification block (WU 111 ship) ---
 
 dim "[nuos:pre-commit] active-decision modification check"
-modified_decisions=$(git diff --cached --name-only --diff-filter=M \
+#
+# "Immutable once accepted" — this blocks edits only to a decision whose
+# status *in HEAD* is already a locked state (`accepted` or `active`).
+# Editing a still-`proposed` decision is allowed: promoting it to
+# accepted/active is the sanctioned lifecycle step, not a violation, and
+# proposed decisions are in-flight by design. New decision files are
+# additions (excluded by --diff-filter=M), so a decision born `accepted`
+# is never blocked on creation. The locked-status check uses the HEAD
+# pre-image, so flipping an accepted decision back to `proposed` to sneak
+# a substantive edit is still caught.
+candidate_decisions=$(git diff --cached --name-only --diff-filter=M \
   | grep -E '^docs/build/decisions/D[0-9]+.*\.md$' \
   | grep -v '/superseded/' \
   || true)
 
-if [[ -n "$modified_decisions" ]]; then
+locked_decisions=""
+if [[ -n "$candidate_decisions" ]]; then
+  while IFS= read -r f; do
+    [[ -z "$f" ]] && continue
+    head_status=$(git show "HEAD:$f" 2>/dev/null \
+      | grep -m1 -E '^\*\*Status:\*\*' \
+      | sed -E 's/^\*\*Status:\*\*[[:space:]]*//' \
+      | awk '{print tolower($1)}')
+    case "$head_status" in
+      accepted|active) locked_decisions+="${f}"$'\n' ;;
+    esac
+  done <<< "$candidate_decisions"
+fi
+
+if [[ -n "$locked_decisions" ]]; then
   red "✖ active-decision modification — BLOCKED (WU 111 enforcement):"
-  while IFS= read -r f; do echo "    — $f"; done <<< "$modified_decisions"
+  while IFS= read -r f; do [[ -n "$f" ]] && echo "    — $f"; done <<< "$locked_decisions"
   red "  Decisions are immutable once accepted. The discipline is to write a"
   red "  superseding D-NNN+1 and link forward. Use:"
   red "    nuos-catalogue decision supersede <target> --by=<new-D> --reason=\"...\""
@@ -149,7 +173,7 @@ if [[ -n "$modified_decisions" ]]; then
   red "  If this edit is a non-substantive typo fix or link cleanup that does"
   red "  not change the decision's meaning, you may bypass this block with"
   red "  --no-verify. CLAUDE.md prohibits --no-verify for substantive changes."
-  log_event "active-decision-block" "$(echo "$modified_decisions" | tr '\n' ',')"
+  log_event "active-decision-block" "$(echo "$locked_decisions" | tr '\n' ',')"
   EXIT_CODE=1
 fi
 

package/templates/hooks/pre-commit

@@ -134,14 +134,38 @@ fi
 # ---------- Rule 2: active-decision modification block (WU 111 ship) ---
 
 dim "[nuos:pre-commit] active-decision modification check"
-modified_decisions=$(git diff --cached --name-only --diff-filter=M \
+#
+# "Immutable once accepted" — this blocks edits only to a decision whose
+# status *in HEAD* is already a locked state (`accepted` or `active`).
+# Editing a still-`proposed` decision is allowed: promoting it to
+# accepted/active is the sanctioned lifecycle step, not a violation, and
+# proposed decisions are in-flight by design. New decision files are
+# additions (excluded by --diff-filter=M), so a decision born `accepted`
+# is never blocked on creation. The locked-status check uses the HEAD
+# pre-image, so flipping an accepted decision back to `proposed` to sneak
+# a substantive edit is still caught.
+candidate_decisions=$(git diff --cached --name-only --diff-filter=M \
   | grep -E '^docs/build/decisions/D[0-9]+.*\.md$' \
   | grep -v '/superseded/' \
   || true)
 
-if [[ -n "$modified_decisions" ]]; then
+locked_decisions=""
+if [[ -n "$candidate_decisions" ]]; then
+  while IFS= read -r f; do
+    [[ -z "$f" ]] && continue
+    head_status=$(git show "HEAD:$f" 2>/dev/null \
+      | grep -m1 -E '^\*\*Status:\*\*' \
+      | sed -E 's/^\*\*Status:\*\*[[:space:]]*//' \
+      | awk '{print tolower($1)}')
+    case "$head_status" in
+      accepted|active) locked_decisions+="${f}"$'\n' ;;
+    esac
+  done <<< "$candidate_decisions"
+fi
+
+if [[ -n "$locked_decisions" ]]; then
   red "✖ active-decision modification — BLOCKED (WU 111 enforcement):"
-  while IFS= read -r f; do echo "    — $f"; done <<< "$modified_decisions"
+  while IFS= read -r f; do [[ -n "$f" ]] && echo "    — $f"; done <<< "$locked_decisions"
   red "  Decisions are immutable once accepted. The discipline is to write a"
   red "  superseding D-NNN+1 and link forward. Use:"
   red "    nuos-catalogue decision supersede <target> --by=<new-D> --reason=\"...\""
@@ -149,7 +173,7 @@ if [[ -n "$modified_decisions" ]]; then
   red "  If this edit is a non-substantive typo fix or link cleanup that does"
   red "  not change the decision's meaning, you may bypass this block with"
   red "  --no-verify. CLAUDE.md prohibits --no-verify for substantive changes."
-  log_event "active-decision-block" "$(echo "$modified_decisions" | tr '\n' ',')"
+  log_event "active-decision-block" "$(echo "$locked_decisions" | tr '\n' ',')"
   EXIT_CODE=1
 fi