@zosmaai/pi-llm-wiki 0.12.1 → 0.12.2

package/dist/mcp/operations.js

@@ -9,13 +9,14 @@ import { join } from "node:path";
 import { bootstrapVault } from "../extensions/llm-wiki/lib/bootstrap.js";
 import { applyWikilinkGate, buildWikilinkIndex, } from "../extensions/llm-wiki/lib/knowledge-links.js";
 import { rebuildMetadata } from "../extensions/llm-wiki/lib/metadata.js";
+import { reindexQmdVault } from "../extensions/llm-wiki/lib/qmd-indexing.js";
 import { searchWikiLayered } from "../extensions/llm-wiki/lib/recall.js";
 import { saveInsight } from "../extensions/llm-wiki/lib/retro.js";
 import { captureFile, captureText, captureUrl } from "../extensions/llm-wiki/lib/source-packet.js";
 import { resolveWikilinkValidation } from "../extensions/llm-wiki/lib/task-config.js";
 import { readJson } from "../extensions/llm-wiki/lib/utils.js";
 import { inspectVaultFormat, inspectWritableVault, VaultWriteError, } from "../extensions/llm-wiki/lib/vault-format.js";
-import { getWikiStatus, searchRegistry } from "../extensions/llm-wiki/lib/wiki-service.js";
+import { getWikiStatus, reindexWiki, searchRegistry, } from "../extensions/llm-wiki/lib/wiki-service.js";
 function projectionOutcome(projection) {
     return projection.ok
         ? { ok: true }
@@ -24,6 +25,22 @@ function projectionOutcome(projection) {
             diagnostics: projection.diagnostics.map(({ code, message }) => ({ code, message })),
         };
 }
+/**
+ * Enqueue a post-projection lexical QMD pass. Model-free and repairable; never
+ * fails the authoritative write. Goes through the per-vault in-process queue.
+ */
+async function scheduleLexicalQmd(paths) {
+    try {
+        await reindexQmdVault(paths, {
+            scope: "changed",
+            components: ["lexical"],
+            force: false,
+        });
+    }
+    catch {
+        // Generated QMD state is repairable; an authoritative write must not fail.
+    }
+}
 /**
  * Shared bootstrap operation: create (or update) the vault at `paths`.
  *
@@ -76,9 +93,13 @@ export async function searchOperation(paths, query, type) {
         diagnostics: result.diagnostics.map((d) => ({ code: d.code, message: d.message })),
     };
 }
+/** Shared reindex operation: delegates to the shared reindexWiki operation. */
+export async function reindexOperation(paths, input) {
+    return reindexWiki(paths, input);
+}
 /** Shared status operation: delegates directly to wiki-service. */
 export async function statusOperation(paths) {
-    const status = getWikiStatus(paths);
+    const status = await getWikiStatus(paths);
     return {
         knowledgeFormat: status.knowledgeFormat,
         totalPages: status.totalPages,
@@ -88,6 +109,7 @@ export async function statusOperation(paths) {
             message: d.message,
         })),
         lastUpdated: status.lastUpdated,
+        qmd: status.qmd,
     };
 }
 /** Shared retro operation: validates vault then delegates to saveInsight. */
@@ -119,6 +141,7 @@ export async function retroOperation(paths, slug, title, body, category, wikilin
         const projection = projectionOutcome(rebuildMetadata(paths));
         if (!projection.ok)
             return projection;
+        await scheduleLexicalQmd(paths);
         return { ok: true, slug: result.slug, sourcePagePath: result.sourcePagePath };
     }
     catch (error) {
@@ -171,6 +194,7 @@ export async function captureSourceOperation(paths, input, execApi) {
         const projection = projectionOutcome(rebuildMetadata(paths));
         if (!projection.ok)
             return projection;
+        await scheduleLexicalQmd(paths);
         return { ok: true, sourceId };
     }
     catch (error) {

package/docs/api.md

@@ -154,7 +154,7 @@ scoring, no PRF, no vault layering. Use for lookups when you already know what y
 
 | Name | Type | Required | Description |
 |------|------|----------|-------------|
-| `query` | `string` | ✅ | Search term matched against page IDs, titles, and types |
+| `query` | `string` | ✅ | Search term matched against page IDs, titles, types, states, statuses, categories, domains, tags, aliases, and recall triggers |
 | `type` | `string` | — | Filter results to a specific page type (e.g. `"concept"`, `"entity"`) |
 
 **Returns**
@@ -302,6 +302,48 @@ details: { pageCount: number }
 
 ---
 
+## wiki_reindex
+
+Rebuild or repair the generated QMD search index under `meta/qmd`. Validated, parser-checked mirrors of
+`.llm-wiki/wiki/**` are written to `meta/qmd/documents/{canonical,evidence}/**`, and a manifest maps them
+back to stable `(vault_id, page_id)` identities. The live store lives at `meta/qmd/current/index.sqlite`
+and is replaced atomically via a recoverable copy-on-write swap. **Active recall still uses the legacy
+heuristic until Phase 3**; QMD indexing is independently observable and repairable but no recall path
+depends on it yet.
+
+**Parameters**
+
+```
+wiki_reindex(
+  scope: "changed" | "all" = "changed",
+  components: ("lexical" | "vectors")[] = ["lexical", "vectors"],
+  force: boolean = false,
+  vault: "active" | "personal" | "project" | "all" = "active"
+)
+```
+
+- `scope`: `changed` hashes and skips unchanged mirrors; `all` rewrites every accepted file (unchanged pages are still counted as `unchanged`).
+- `components`: `vectors` first refreshes documents; `lexical`-only never loads a model. Default: `components: ["lexical", "vectors"]`.
+- `force`: applies only to the selected components (a forced lexical rebuild starts from an empty staging store).
+- `vault`: which vaults to reindex. `all` reports each vault independently.
+
+Selecting `vectors` may download approximately 2 GB of models on first use. Cancellation and failures retain
+the last usable `current` store; stale/error/recovering state is repaired by re-running this tool. The swap
+journal is write-ahead intent — each phase is published before the destructive rename it covers, and recovery
+also checks filesystem state, so interrupted promotions are restored or rolled back rather than guessed. Do not copy,
+partially restore, or edit individual SQLite/WAL/SHM files inside `current` — restore the whole directory or rebuild.
+
+**Returns**
+
+```
+details: {
+  scope, components, vault,
+  results: Array<{ root, label, result: QmdReindexResult }>
+}
+```
+
+---
+
 ## wiki_log_event
 
 Append a structured event to the authoritative, append-only `meta/events.jsonl` stream and regenerate available log projections. Every event is timestamped automatically. The event stream must be preserved in full-vault backups; generated Markdown logs cannot reconstruct it.

package/docs/architecture.md

@@ -75,10 +75,38 @@ WIKI_ROOT/
 | `.llm-wiki/wiki/**`   | Model + user             | Editable knowledge pages |
 | `.llm-wiki/meta/events.jsonl` | Extension tools | Authoritative, append-only; preserve in full-vault backups |
 | `.llm-wiki/meta/**` except `events.jsonl` | Extension | Generated projections |
+| `.llm-wiki/meta/qmd/**` | Extension | Generated QMD search index (mirrors + SQLite); local, rebuildable with `wiki_reindex` |
 | `.llm-wiki/` | Human + explicit request | Operating rules          |
 
 `events.jsonl` records selected extension operations, not every filesystem edit. `meta/log.md` and OKF-mode `wiki/log.md` are one-way projections; neither can recover the event stream.
 
+## Generated QMD Search Index (phase 2)
+
+`.llm-wiki/meta/qmd/**` is extension-owned, generated, local, and **generated and rebuildable** via `wiki_reindex`. It is a validated, independently repairable search index that no recall path depends on yet (active recall stays the legacy heuristic until Phase 3).
+
+```
+meta/qmd/
+  manifest.json             # maps generated paths -> (vault_id, page_id)
+  documents/
+    canonical/**/*.md        # parser-valid concept/entity/analysis/synthesis/requirement/skill/case
+    evidence/**/*.md         # parser-valid source/unknown types
+  current/index.sqlite       # live QMD store (copied, never edited in place)
+  index.lock/                # cross-process lock (owner.json)
+  swap.json                  # journal for crash-safe promotion
+```
+
+- QMD never scans `.llm-wiki/wiki/**` directly; it reads only the validated mirrors QMD owns.
+- `manifest.json` maps validated mirrors back to stable `(vault_id, page_id)` identities.
+- Canonical and evidence collections never overlap.
+- Ordinary write-triggered updates are **lexical and model-free**; `vectors` may download ~2 GB on first use.
+- The live store is replaced atomically via a recoverable copy-on-write swap; failures retain the last usable `current`.
+- `swap.json` is a **write-ahead journal**: each phase is published before the destructive rename it covers, and recovery re-checks the filesystem, so every crash window restores a usable `current` or an explicit missing/error state.
+- Stale `staging-<uuid>` directories left by failed or cancelled pre-journal work are extension-owned and swept while the per-vault lock is held; recovery never touches arbitrary names under `meta/qmd`.
+- Malformed generated artifacts (`swap.json`, `manifest.json`, `index-state.json`) report `error`, never `missing` or `ready`.
+- Generated status exposes `repairComponents` (valid tool component values) so lint can suggest an exact `wiki_reindex` command. `vectors` refreshes documents before embedding, so `components=["vectors"]` repairs stale vectors and their document index together.
+- Do not copy, partially restore, or edit individual SQLite/WAL/SHM files inside `current` — restore the whole directory or rebuild.
+- Full-vault backups include generated searchable text; OKF-only exports do not.
+
 ## Source Packet Format
 
 Each captured source becomes a packet:

package/docs/commands.md

@@ -41,6 +41,7 @@ off by default** (issue #80) — registered only when `llm-wiki.trajectories` is
 | `wiki_status`         | Instant stats                               |
 | `wiki_observe` | Record a timestamped observation from the current session |
 | `wiki_rebuild_meta`   | Force metadata rebuild                      |
+| `wiki_reindex`        | Rebuild/repair the QMD search index at `meta/qmd` (lexical model-free; vectors may download ~2 GB) |
 | `wiki_reindex_embeddings` | Refresh semantic embeddings (no-op when no embedding provider) |
 | `wiki_log_event`      | Record custom event                         |
 | `wiki_watch`          | Schedule auto-updates                       |

package/docs/qmd-compatibility.md

@@ -0,0 +1,47 @@
+# QMD Compatibility
+
+pi-llm-wiki's next major pins `@tobilu/qmd` **2.5.3**, the latest version published to npm when Phase 1 was planned.
+
+## Runtime
+
+- Node.js: `>=22.0.0`
+- TypeScript development peer: `^5.9.3`
+- Package manager: pnpm 9
+
+Users requiring Node.js 18 must remain on the previous pi-llm-wiki major.
+
+## Native compatibility
+
+Clean-install CI covers:
+
+- Linux x64
+- macOS arm64
+- Windows x64
+
+QMD brings `better-sqlite3`, `sqlite-vec`, and `node-llama-cpp`. Failure to install required native packages is an installation failure, not a runtime lexical fallback.
+
+## Model-free contract
+
+`createStore`, `update`, `searchLex`, `getStatus`, and `close` must work without downloading or loading an embedding, expansion, or reranking model. Ordinary CI tests this path with `QMD_FORCE_CPU=1`.
+
+## Model-backed contract
+
+The scheduled/manual model smoke exercises:
+
+- `embed`
+- `searchVector`
+- typed hybrid `search` with reranking disabled
+- `expandQuery`
+- expanded/reranked `search`
+
+QMD stores default models under `~/.cache/qmd/models`. First use downloads roughly 2 GB across embedding, reranking, and expansion models. CI caches that directory. `QMD_FORCE_CPU=1` avoids GPU probing in compatibility jobs.
+
+## Upgrade rule
+
+Do not widen the QMD version range. A QMD upgrade requires:
+
+1. exact-version lock update
+2. SDK contract and clean-install CI passing
+3. model smoke passing
+4. retrieval benchmark comparison before production use
+5. updated model and native-support documentation

package/docs/retrieval-benchmark.md

@@ -0,0 +1,47 @@
+# Retrieval Benchmark
+
+The Phase 1 benchmark records current heuristic recall quality before QMD powers production retrieval.
+
+## Corpus
+
+`test/fixtures/retrieval-benchmark/fixture.ts` contains 22 sanitized Markdown pages and 60 graded queries:
+
+- 45 train queries
+- 15 immutable held-out queries
+- exact lookup, aliases, paraphrase, vague recollection, concepts, graph scope, evidence, time, conflicts, conclusions, synthesis, and unrelated negatives
+- English, Chinese, and mixed-language examples
+
+The fixture must not contain raw home paths, email addresses, credentials, customer identifiers, or copied private notes. Sanitize representative phrasing before committing it.
+
+The CJK query `什么是卡片盒笔记法` (zettelkasten-concept #4) is an intentional guaranteed miss for the current heuristic engine: the page body is English-only, so lexical score is zero by construction. Do not reword the page to make it pass; it exists to prove that only CJK-aware lexical handling (QMD's normalized FTS) can recover it.
+
+## Judgments
+
+Grades are:
+
+- `3`: directly answers the query
+- `2`: useful supporting evidence or secondary answer
+- `1`: relevant context
+
+Roles are `canonical` or `evidence`. Contradiction queries list every claim that must appear together.
+
+## Metrics
+
+The report records candidate Recall@20, MRR, nDCG@5, nDCG@10, canonical@3, evidence Recall@20, contradiction coverage, and automatic-recall false-positive rate. Scores are computed from ranked page IDs, never raw engine scores.
+
+## Commands
+
+Verify the committed baseline:
+
+```bash
+pnpm benchmark:retrieval
+```
+
+Regenerate after an intentional fixture or baseline-engine change:
+
+```bash
+pnpm benchmark:retrieval:update
+pnpm benchmark:retrieval
+```
+
+Never hand-edit `docs/superpowers/benchmarks/phase-1-current-baseline.json`. Every update must explain why the benchmark or baseline engine changed. Later phases may tune against `train`, but must not inspect or alter held-out judgments while tuning.

package/docs/superpowers/benchmarks/phase-1-current-baseline.json

@@ -0,0 +1,53 @@
+{
+  "schema": 1,
+  "fixtureVersion": 1,
+  "engine": "current-heuristic",
+  "productionRecallChanged": false,
+  "packageContract": {
+    "node": ">=22.0.0",
+    "qmd": "2.5.3"
+  },
+  "queryCounts": {
+    "all": 60,
+    "train": 45,
+    "heldout": 15
+  },
+  "metrics": {
+    "all": {
+      "queryCount": 60,
+      "judgedQueryCount": 55,
+      "candidateRecall20": 0.872727,
+      "mrr": 0.785455,
+      "ndcg5": 0.748766,
+      "ndcg10": 0.759226,
+      "canonicalAt3": 0.836364,
+      "evidenceRecall20": 0.8,
+      "contradictionCoverage": 0.6,
+      "autoFalsePositiveRate": 0.2
+    },
+    "train": {
+      "queryCount": 45,
+      "judgedQueryCount": 40,
+      "candidateRecall20": 0.925,
+      "mrr": 0.827917,
+      "ndcg5": 0.775036,
+      "ndcg10": 0.787218,
+      "canonicalAt3": 0.875,
+      "evidenceRecall20": 0.84,
+      "contradictionCoverage": 0,
+      "autoFalsePositiveRate": 0.2
+    },
+    "heldout": {
+      "queryCount": 15,
+      "judgedQueryCount": 15,
+      "candidateRecall20": 0.733333,
+      "mrr": 0.672222,
+      "ndcg5": 0.678714,
+      "ndcg10": 0.684582,
+      "canonicalAt3": 0.733333,
+      "evidenceRecall20": 0.6,
+      "contradictionCoverage": 0.6,
+      "autoFalsePositiveRate": 0
+    }
+  }
+}