@dikolab/kbdb 0.2.2 → 0.3.1

package/docs/overview.md

@@ -1,187 +1,169 @@
 # @dikolab/kbdb -- Knowledge Base Database
 
-*A persistent, file-based knowledge store and MCP server purpose-built for AI agents -- your second brain, on disk.*
+*A file-based knowledge store and MCP server for
+AI agents -- your second brain, on disk.*
 
-## What Is @dikolab/kbdb?
+## What Is kbdb?
 
-AI agents operate in ephemeral sessions. They lose context between conversations, forget prior research, and have no built-in way to persist or retrieve knowledge across runs. When an agent needs to recall documentation, search a codebase's API surface, or query a corpus of notes, there is no lightweight, local solution optimized for how agents actually retrieve information -- keyword-driven, section-level, ranked.
+AI agents forget everything between sessions. kbdb
+gives them a searchable, persistent memory.
 
-@dikolab/kbdb is a CLI tool and MCP server that imports documents and code documentation, extracts keywords, builds indexes, and enables ranked search -- all backed by a directory-based file database. No database server, no cloud dependency. A folder on disk is your entire knowledge base.
+Import your documents, and kbdb indexes them
+automatically. Search returns ranked results
+instantly. No database server to install, no cloud
+account to create -- your knowledge base is just a
+folder on disk.
 
-The computationally intensive work -- keyword extraction, indexing, ranking -- is offloaded to a Rust engine compiled to WebAssembly, delivering near-native performance inside a Deno or Node.js runtime. The TypeScript CLI provides the developer-facing interface and MCP server; Rust/WASM does the heavy lifting underneath.
+kbdb works as both a command-line tool and an
+[MCP](https://modelcontextprotocol.io/) server.
+Humans use it from the terminal; AI agents connect
+to it as a tool.
 
 ## Who Is This For?
 
-- **AI agent developers** -- Anyone building agentic workflows (with Claude, LangChain, or custom harnesses) who needs persistent, searchable knowledge across sessions.
-- **MCP tool authors** -- Developers creating Model Context Protocol servers who need a local knowledge backend their tools can query.
-- **Knowledge workers using AI assistants** -- Power users who want their AI to have a second brain -- a searchable, structured store of documentation, research, and reference material that survives session boundaries.
-- **CLI-first developers** -- Those who prefer composable command-line tools over GUI applications for managing knowledge.
-
-## Core Features
-
-### CLI
-
-1. **Markdown import and indexing** -- Import `*.md` files, extract keywords and sections, and store them in the file system database.
-2. **Codebase documentation scanning** -- Scan a codebase and import code documentation (TypeDoc, JSDoc, PHPDoc), index it, and store it in the file system database.
-3. **Ranked search** -- Search and display a ranked list of results by section or by whole document, with output in `.md` or `.json` format.
-
-### MCP
-
-1. **MCP service endpoints** -- Expose the knowledge base as MCP service endpoints for Claude and other AI agents to query, search, and retrieve knowledge programmatically.
-
-### Distribution
-
-This tool is available via **NPM** for Node.js or **JSR** for Deno.
-
-- **Node.js**: `npm install -g @dikolab/kbdb`
-- **Deno**: `deno add @dikolab/kbdb` (via [jsr.io](https://jsr.io))
-
-## High-Level Architecture
-
-The system is organized in five layers. Data flows
-downward from user commands or agent requests through
-the worker client and daemon into the WASM engine,
-which performs heavy processing and manages the file
-database.
-
-![System Architecture](goals/architecture.svg)
-
-**The kbdb Command** -- A single executable with two
-run modes. CLI mode (`kbdb learn`, `kbdb search`,
-`kbdb unlearn`) handles human and script invocation --
-each command is a short-lived process. MCP mode
-(`kbdb mcp`) starts a long-lived MCP server that AI
-agents connect to over stdio.
-
-**Worker Client** -- Shared TypeScript module imported
-by both run modes. Discovers or spawns the worker
-daemon and provides a typed async API over IPC.
-
-**Worker Daemon** -- A detached background process
-that loads WASM modules, manages an in-memory cache,
-and serves JSON-RPC 2.0 requests over local IPC (Unix
-sockets or Windows named pipes). The daemon has no
-knowledge of its callers.
-
-**Rust/WASM Engine** -- Three WASM modules:
-`query-parser.wasm` (tokenization, query parsing,
-ranking), `fs-database.wasm` (file database I/O), and
-`kb-worker.wasm` (imports both, orchestrates). Rust
-provides memory safety and performance; WASM provides
-portability across runtimes.
-
-**File Database** -- A structured directory on disk.
-No external database process required. Documents,
-indexes, keywords, and metadata are stored in a
-defined directory layout. The database is portable,
-version-controllable, and human-inspectable.
-
-## Technology Stack
-
-| Layer | Technology | Role |
-|-------|-----------|------|
-| Runtime | Deno (primary), Node.js (compatible) | Executes the CLI, MCP server, and TypeScript orchestration |
-| Language | TypeScript | CLI, command parsing, MCP protocol, I/O coordination |
-| Engine | Rust compiled to WebAssembly | Keyword processing, indexing, ranking, file DB operations |
-| Protocol | MCP (Model Context Protocol) | AI agent integration interface |
-| Distribution | NPM (Node.js), JSR (Deno) | Package distribution for both ecosystems |
-| Storage | Directory-based file database | Persistent, portable, server-less data storage |
+- **AI agent developers** -- persistent, searchable
+  knowledge across agent sessions
+- **MCP tool authors** -- a local knowledge backend
+  for Model Context Protocol servers
+- **Knowledge workers** -- a second brain your AI
+  assistant can search across conversations
+- **CLI-first developers** -- composable commands
+  for managing knowledge from the terminal
+
+## What Can It Do?
+
+- **Import** Markdown files with automatic keyword
+  extraction and indexing
+- **Search** with ranked results, snippets, and
+  paginated output
+- **Recall** sections with progressive context
+  (depth 0--3)
+- **Skills** -- reusable prompt templates with
+  placeholder arguments
+- **Agents** -- AI profiles combining a persona
+  with skills
+- **Export** the database as a portable snapshot
+- **Verify** database integrity and clean up stale
+  data
+- **MCP server** with 20 tools, prompts, resources,
+  and completions
+
+### Install
+
+```sh
+# Node.js
+npm install -g @dikolab/kbdb
+
+# Deno
+deno add @dikolab/kbdb
+```
+
+## Architecture
+
+Five layers, top to bottom:
+
+```
+User / AI Agent
+  -> kbdb command (CLI or MCP mode)
+    -> Worker Client (TypeScript IPC)
+      -> Worker Daemon (background process)
+        -> Rust/WASM Engine
+          -> File Database (folder on disk)
+```
+
+**kbdb command** -- One executable, two modes. CLI
+mode runs short-lived commands (`learn`, `search`,
+`recall`). MCP mode starts a long-lived server for
+AI agents over stdio.
+
+**Worker Client** -- TypeScript module that discovers
+or spawns the daemon and provides a typed async API.
+
+**Worker Daemon** -- Background process that loads
+WASM modules, caches data in memory, and serves
+JSON-RPC 2.0 requests over local IPC.
+
+**Rust/WASM Engine** -- Three WASM modules handle
+tokenization, indexing, ranking, and file I/O. Rust
+provides speed and memory safety; WASM runs on any
+runtime.
+
+**File Database** -- Structured directory on disk.
+No server process. Portable, version-controllable,
+and human-readable.
+
+### Technology Stack
+
+| Layer | Technology | Purpose |
+|-------|-----------|---------|
+| Runtime | Node.js, Deno | Runs CLI, MCP server, orchestration |
+| Language | TypeScript | CLI, MCP protocol, I/O |
+| Engine | Rust -> WebAssembly | Indexing, ranking, file operations |
+| Protocol | MCP | AI agent integration |
+| Distribution | NPM, JSR | Package delivery |
+| Storage | File database | Persistent, portable data |
 
 ## Design Principles
 
-1. **Offline-first, server-less** -- No database server, no cloud dependency. A directory on disk is the entire database. Works anywhere a filesystem exists.
-2. **AI-agent-native** -- Designed from the ground up to be invoked by AI agents, not just humans. Commands, output formats, and the MCP interface prioritize machine consumption.
-3. **MCP-first integration** -- The MCP server is a primary interface, not an afterthought. AI agents connect to the knowledge base as naturally as they connect to any other MCP service.
-4. **Performance through Rust/WASM** -- Computationally intensive work runs in compiled Rust via WebAssembly, keeping the TypeScript layer thin and the heavy processing fast.
-5. **Portability** -- Runs on Deno or Node.js. The file database is a plain directory. No platform lock-in at any layer.
-6. **Section-level precision** -- Documents are decomposed into sections. Search returns the specific passage that matters, not an entire document.
-7. **Incremental design** -- The project ships working slices. Storage format and engine capabilities will evolve; the architecture accommodates change without requiring rewrites.
-
-## Details
-
-The following documents describe the intended behavior and
-design goals for each major subsystem. They serve as
-implementation specifications, not descriptions of current
-functionality.
-
-- **[Database](goals/database.md)** -- Filesystem storage
-  engine (`fs-database.wasm`). The `.kbdb` directory
-  structure, section and document CRUD operations, index
-  management, and storage design.
-- **[Query Parser](goals/query-parser.md)** -- Query
-  processing and tokenization (`query-parser.wasm`).
-  Standalone WASM module for tokenization, query
-  parsing, keyword extraction, and ranking.
-- **[Document](goals/document.md)** -- Text content
-  format specification. Markdown body text with
-  embedding markers and cross-reference link markers.
+1. **No server needed** -- A folder is your whole
+   database. Works anywhere.
+2. **Built for AI agents** -- Commands and output
+   are optimized for machine use.
+3. **MCP-first** -- The MCP server is a primary
+   interface, not an add-on.
+4. **Fast** -- Heavy work runs in compiled Rust
+   via WebAssembly.
+5. **Portable** -- Runs on Node.js or Deno. The
+   database is a plain directory.
+6. **Section-level search** -- Returns the specific
+   passage that matters, not a whole document.
+
+## Design Documents
+
+Detailed specifications for each subsystem:
+
+- **[Database](goals/database.md)** -- File storage
+  engine, directory structure, section and document
+  operations
+- **[Query Parser](goals/query-parser.md)** --
+  Tokenization, query parsing, keyword extraction,
+  ranking
+- **[Document](goals/document.md)** -- Content format
+  with markers and cross-references
 - **[Content Parser](goals/content-parser.md)** --
-  Marker extraction engine (`kbdb-shared` rlib).
-  Scans section content for `$(type: value)` markers
-  and returns structured reference lists.
+  Marker extraction from section content
 - **[Content Composer](goals/content-composer.md)** --
-  Content composition engine (`kbdb-shared` rlib).
-  Resolves sections and markers, produces rendered
-  Markdown documents from kbid and docid identifiers.
-- **[CLI](goals/cli.md)** -- The `kbdb` command and its
-  two run modes. CLI mode commands (`learn`, `unlearn`,
-  `search`), MCP mode dispatch, output formats, and the
-  pass-through execution model.
-- **[MCP](goals/mcp.md)** -- MCP server mode
-  (`kbdb mcp`). Invocation, database path
-  configuration, stdio transport, MCP tools and
-  resources, agent wiring examples, and relationship
-  to CLI mode.
-- **[Worker Daemon](goals/worker-daemon.md)** -- Cache
-  daemon (`kb-worker.wasm` + `src/worker.ts`).
-  The detached TypeScript daemon, WASM module loading,
-  caching, IPC protocol, lifecycle, and configuration.
-- **[Worker Client](goals/worker-client.md)** -- Shared
-  TypeScript client module (`src/shared/worker-client/`).
-  Daemon discovery, spawning, IPC connection, typed
-  API, and the sole entry point used by `src/cli.ts`.
+  Rendered Markdown composition from identifiers
+- **[CLI](goals/cli.md)** -- Commands, run modes,
+  output formats
+- **[MCP](goals/mcp.md)** -- MCP server, tools,
+  resources, agent wiring
+- **[Worker Daemon](goals/worker-daemon.md)** --
+  Background daemon, WASM loading, caching, IPC
+- **[Worker Client](goals/worker-client.md)** --
+  Client module, daemon discovery, typed API
 
 ## Project Status
 
-@dikolab/kbdb is in active development. The full
-five-layer stack (CLI/MCP -> Worker Client -> Worker
-Daemon -> WASM Engine -> File Database) is wired and
-working end-to-end for all core operations: learn,
-search, recall, content retrieval, skills, agents,
-and maintenance commands.
-
-E2E smoke tests run on both Node.js and Deno via
-Docker, covering 380+ assertions across CLI commands,
-18 MCP tools, MCP protocol features, and worker IPC
-methods.
-
-### Completed
-
-- CLI with 18 commands across 6 groups (tested)
-- MCP server with 18 tools, prompts, resources,
-  and completions (tested)
-- Worker client with typed async IPC API (tested)
-- Worker daemon with WASM module loading (tested)
-- Rust/WASM engine: search, recall, indexing,
-  reference graph, KWIC snippets, pagination
-- Paginated search with rich metadata (heading,
-  type, snippet, matched_terms, docids)
+All layers are wired and working end-to-end. Every
+core operation -- learn, search, recall, content
+retrieval, skills, agents, export, and maintenance
+-- runs through the full stack.
+
+**548 e2e assertions** pass on both Node.js and Deno
+with zero failures and zero skips.
+
+- 18 CLI commands across 6 groups
+- 20 MCP tools with prompts, resources, completions
+- Sequential MCP request processing
+- Worker daemon with fully implemented WASM host I/O
+- Paginated search with rich metadata
 - Progressive recall at depths 0--3
-- Bidirectional reference graph (references.idx)
-- Skills (template-based prompts) and agents
-  (persona + skills compositions)
-- esbuild bundling pipeline (ESM + CJS + .d.ts)
-- Docker build containers (Deno + Node + Rust +
-  wasm-pack)
-- npm and JSR packaging with documentation
-
-### Remaining
-
-- Complete WASM host I/O imports (some operations
-  still hit stubs: document removal, agent document
-  persistence)
-- Database format versioning and migration
-  infrastructure
+- Bidirectional reference graph
+- Skills and agent document management
+- esbuild bundling (ESM + CJS + .d.ts)
+- Docker build containers
+- npm and JSR packages with documentation
 
 ## Links
 

package/docs/release-notes/0.2.0.md

@@ -62,7 +62,7 @@ at once.
   scores are kept in CLI and IPC output for
   debugging.
 
-- **Tool annotations** -- All 18 MCP tool
+- **Tool annotations** -- All 20 MCP tool
   definitions now include `openWorldHint: false`,
   `readOnlyHint`, `destructiveHint`, and
   `idempotentHint` annotations per the MCP spec.
@@ -84,8 +84,8 @@ at once.
 
 ## Testing
 
-- **383 e2e assertions (Node.js), 378 (Deno)** --
-  up from ~300. All pass with 0 failures.
+- **548 e2e assertions** on both Node.js and Deno --
+  all passing with 0 failures and 0 skips.
 
 - **Recall depth testing** -- Cross-referenced
   test data (sections with `$(link: ...)` markers)

package/docs/release-notes/0.3.0.md

@@ -0,0 +1,89 @@
+# 0.3.0
+
+*Minor release -- 2026-06-23*
+
+Adds supersession, relaxed queries, database export,
+tags, and confidence scoring. The Rust engine gains
+source tracking, near-duplicate detection, and
+synonym expansion.
+
+---
+
+## New Features
+
+- **Supersession** -- Re-learning from the same
+  source automatically replaces outdated sections
+  instead of duplicating them. Source key identity
+  tracked via SHA-256.
+
+- **Relaxed OR fallback** -- When an exact AND
+  query returns no results, kbdb automatically
+  retries with relaxed OR matching so you always
+  get the best available results.
+
+- **Database export** -- New `export` CLI command
+  and MCP tool. Copies sections, documents, and
+  catalog as a self-contained directory snapshot.
+
+- **Tags and replace on learn** -- `--tags` attaches
+  labels to imported sections for scoped queries.
+  `--replace` supersedes existing sections from the
+  same source in one step.
+
+- **Confidence scoring** -- Search results include
+  a normalized BM25F confidence score and
+  `matched_fields` (heading, body, code) so agents
+  can tell strong matches from weak ones.
+
+- **Near-duplicate detection** -- Jaccard similarity
+  warns when imported content closely matches an
+  existing section, preventing silent drift.
+
+- **Default skills** -- `db init` installs
+  `query-guidance` and `memory-protocol` skills
+  that teach agents how to search and what to store.
+
+---
+
+## Improvements
+
+- **Synonym expansion pipeline** in the Rust engine
+  broadens keyword coverage automatically.
+
+- **Matched field tracking** -- Results show which
+  fields (heading, body, code) matched the query.
+
+- **`created_at` timestamps** in search results.
+
+- **Divergent content detection** via integrity
+  check catches contradictions between versions.
+
+- **GC fix** -- Garbage collection now only removes
+  superseded sections, not active ones.
+
+- **Reference graph rebuild** included in `rebuild`
+  command.
+
+- **Writer lock** prevents concurrent WASM writes.
+
+- **Worker daemon startup health check** verifies
+  WASM modules load correctly.
+
+---
+
+## Testing
+
+- **517 e2e assertions (Node.js), 514 (Deno)** --
+  all pass with 0 failures.
+
+---
+
+## Upgrade
+
+```sh
+# Node.js
+npm install -g @dikolab/kbdb@0.3.0
+
+# Deno
+deno install -Agf jsr:@dikolab/kbdb/cli
+```

package/docs/release-notes/0.3.1.md

@@ -0,0 +1,111 @@
+# 0.3.1
+
+*Patch release -- 2026-06-23*
+
+Fixes MCP request processing reliability, agent
+document persistence, WASM memory safety, and
+eliminates all e2e test skips. Adds the 20th MCP
+tool (`auto-capture-review`).
+
+---
+
+## Bug Fixes
+
+- **MCP sequential request processing** -- The MCP
+  server previously dispatched JSON-RPC requests
+  concurrently (fire-and-forget). When multiple
+  requests arrived in a batch, later requests could
+  execute before earlier ones finished -- for
+  example, reading an agent document before it was
+  fully written. Requests are now processed
+  sequentially via a promise chain, ensuring each
+  completes before the next begins.
+
+- **Agent document type persistence** -- Agent
+  documents created via `agent-create` were missing
+  their `doc_type` field, so `listDocumentsByType`
+  could not find them. `handleAgentCreate` now
+  passes `'agent'` as the document type.
+
+- **Agent prompt lookup** -- `prompts/get` for
+  agent prompts treated the agent title as a
+  document ID, which failed because document IDs
+  are content hashes. Now looks up agents by title
+  via `listDocumentsByType('agent')`.
+
+- **WASM `worker_remove_grouping` double-free** --
+  The WASM export returned success without calling
+  `set_return`, leaving stale memory in the return
+  slot. The next `readReturnSlot` call freed
+  already-freed memory, causing a WASM trap. Now
+  sets the return slot to null before returning.
+
+- **WASM defensive `set_return` calls** -- Added
+  null return slot initialization to `worker_init`
+  and `worker_init_directory` to prevent the same
+  stale-memory pattern from surfacing in edge cases.
+
+---
+
+## New Features
+
+- **`auto-capture-review` MCP tool** -- Review and
+  approve pending auto-captured knowledge before
+  storage. Accepts `action` parameter with values
+  `list`, `approve`, or `reject`. This brings the
+  MCP server to 20 tools.
+
+---
+
+## Testing
+
+- **548 e2e assertions** on both Node.js and Deno,
+  all passing with **0 failures and 0 skips** (up
+  from 517/514 with some skips in 0.3.0).
+
+- **E2E test restructuring** -- Test databases
+  isolated by category (CLI, MCP, Worker) instead
+  of sharing a flat directory. Worker recall tests
+  now seed their own cross-reference data instead
+  of depending on CLI test data.
+
+- **Pre-built WASM in Docker** -- E2E test
+  container caches WASM binaries at build time,
+  reducing test startup from ~30 seconds to
+  sub-second.
+
+---
+
+## Documentation
+
+- **README.md** reorganized for clarity -- shorter
+  intro, labeled paragraphs ("How search works",
+  "Knowledge stays fresh"), accessible to both
+  technical and non-technical readers.
+
+- **Architecture overview** rewritten -- simplified
+  prose, ASCII architecture diagram, removed stale
+  "Remaining" section (WASM host I/O is fully
+  implemented).
+
+- **MCP server guide** updated with
+  `auto-capture-review` tool in reference tables
+  and annotations summary.
+
+- **Tool counts corrected** across all docs
+  (19 -> 20).
+
+- **Assertion counts updated** across all docs
+  (380+/383 -> 548).
+
+---
+
+## Upgrade
+
+```sh
+# Node.js
+npm install -g @dikolab/kbdb@0.3.1
+
+# Deno
+deno install -Agf jsr:@dikolab/kbdb/cli
+```

package/docs/release-notes/README.md

@@ -2,6 +2,8 @@
 
 ## Releases
 
+- [0.3.1](0.3.1.md) -- MCP reliability fixes, 20 tools, zero test skips
+- [0.3.0](0.3.0.md) -- Supersession, relaxed queries, export, tags, confidence
 - [0.2.0](0.2.0.md) -- Recall, paginated search, reference graph
 - [0.1.3](0.1.3.md) -- Include WASM binaries in JSR package
 - [0.1.2](0.1.2.md) -- Fix WASM loading from JSR

package/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@dikolab/kbdb",
-   "version": "0.2.2",
+   "version": "0.3.1",
    "description": "CLI tool and MCP server for a file-based knowledge base database",
    "publishConfig": {
       "access": "public"

package/src/shared/auto-capture/classes/auto-capture-controller.class.ts

@@ -0,0 +1,62 @@
+import type { AutoCaptureConfig } from '../typings/auto-capture-config.interface.ts';
+import type { AutoCaptureState } from '../typings/auto-capture-state.interface.ts';
+import type { AutoCaptureTrigger } from '../typings/auto-capture-trigger.interface.ts';
+import { evaluateTrigger } from '../functions/evaluate-trigger.function.ts';
+
+/**
+ * Controls the auto-capture lifecycle.
+ *
+ * Tracks tool call counts, evaluates trigger conditions, and
+ * advances session state when a capture fires. Reset state on
+ * disconnect or session end via `reset()`.
+ */
+export class AutoCaptureController {
+   private toolCallsSinceLastCapture = 0;
+   private lastCaptureTimestamp = 0;
+   private totalCapturesThisSession = 0;
+   private readonly config: AutoCaptureConfig;
+
+   /** @param config - the auto-capture configuration to use */
+   constructor(config: AutoCaptureConfig) {
+      this.config = config;
+   }
+
+   /** Record one tool call for trigger counting. */
+   recordToolCall(): void {
+      this.toolCallsSinceLastCapture += 1;
+   }
+
+   /**
+    * Evaluates the current state against the trigger criteria.
+    *
+    * When the trigger fires, advances the session counters so the
+    * next evaluation respects rate limiting and the session cap.
+    *
+    * @returns the trigger decision with a human-readable reason
+    */
+   evaluate(): AutoCaptureTrigger {
+      const trigger = evaluateTrigger(this.config, this.state());
+      if (trigger.shouldCapture) {
+         this.totalCapturesThisSession += 1;
+         this.lastCaptureTimestamp = Date.now();
+         this.toolCallsSinceLastCapture = 0;
+      }
+      return trigger;
+   }
+
+   /** Reset all counters, e.g. on client disconnect. */
+   reset(): void {
+      this.toolCallsSinceLastCapture = 0;
+      this.lastCaptureTimestamp = 0;
+      this.totalCapturesThisSession = 0;
+   }
+
+   /** Returns a snapshot of the current controller state. */
+   state(): AutoCaptureState {
+      return {
+         toolCallsSinceLastCapture: this.toolCallsSinceLastCapture,
+         lastCaptureTimestamp: this.lastCaptureTimestamp,
+         totalCapturesThisSession: this.totalCapturesThisSession,
+      };
+   }
+}

package/src/shared/auto-capture/constants/auto-capture-defaults.constant.ts

@@ -0,0 +1,15 @@
+import type { AutoCaptureConfig } from '../typings/auto-capture-config.interface.ts';
+
+/**
+ * Default configuration for the auto-capture feature.
+ *
+ * Auto-capture is disabled by default. Enable it in
+ * `worker.toml` under `[auto_capture]`.
+ */
+export const AUTO_CAPTURE_DEFAULTS: AutoCaptureConfig = {
+   enabled: false,
+   triggerAfterToolCalls: 5,
+   minIntervalMs: 60_000,
+   maxCapturesPerSession: 10,
+   dryRun: false,
+};