clawvault 3.2.0 → 3.3.0

package/README.md

@@ -59,7 +59,7 @@ Unlike vector databases or cloud-based memory solutions, ClawVault is:
 │        ▼                                    ▼                               │
 │   ┌──────────┐    ┌──────────┐    ┌──────────┐    ┌──────────┐             │
 │   │  wake    │◀──▶│ context  │◀──▶│  Graph   │◀──▶│  Search  │             │
-│   │  sleep   │    │ profiles │    │ Traversal│    │(qmd/vec) │             │
+│   │  sleep   │    │ profiles │    │ Traversal│    │(hybrid)  │             │
 │   │checkpoint│    └──────────┘    └──────────┘    └──────────┘             │
 │   └──────────┘                                                              │
 │                                                                             │
@@ -97,8 +97,11 @@ These primitives map directly to CLI commands and vault structure, creating a co
 # Install ClawVault CLI
 npm install -g clawvault
 
-# Install qmd (required for search/context features)
+# Optional: install qmd for backward-compatible fallback paths
 npm install -g github:tobi/qmd
+
+# Quick verification
+clawvault doctor
 ```
 
 ### Initialize Your Vault
@@ -131,12 +134,18 @@ clawvault sleep "finished auth rollout" --next "implement migration"
 ### Search and Context
 
 ```bash
-# Keyword search
+# In-process hybrid search (BM25 + semantic reranking)
 clawvault search "postgresql"
 
-# Semantic search
+# Semantic/vector search commands (requires hosted embeddings configured)
 clawvault vsearch "what did we decide about storage"
 
+# Configure hosted embeddings (OpenAI/Gemini/Ollama)
+clawvault config set search.embeddings.provider openai
+clawvault config set search.embeddings.model text-embedding-3-small
+clawvault config set search.embeddings.apiKey "$OPENAI_API_KEY"
+clawvault rebuild-embeddings
+
 # Get context for a task
 clawvault context "database migration"
 clawvault context --profile planning "Q1 roadmap"
@@ -163,6 +172,12 @@ ClawVault v3 adds **write-time fact extraction** and **entity graphs** to the co
 
 ## Features
 
+### v3.3 Highlights
+
+- **In-process hybrid search engine** — BM25 + hosted semantic embeddings + cross-encoder reranking, with `qmd` now optional.
+- **Python SDK (`clawvault-py`)** — PyPI package with a `Vault` class, BM25 search, and checkpoint/wake lifecycle helpers.
+- **Inbox + background workers** — `clawvault inbox add` and `clawvault maintain` with Curator, Janitor, Distiller, and Surveyor workers.
+
 ### Memory Graph
 
 ClawVault builds a typed knowledge graph from wiki-links, tags, and frontmatter:
@@ -215,6 +230,20 @@ clawvault inject "How should we handle the deployment?"
 clawvault inject --enable-llm "What's our pricing strategy?"
 ```
 
+### Python SDK (clawvault-py)
+
+```bash
+pip install clawvault-py
+```
+
+```python
+from clawvault import Vault
+
+vault = Vault("~/memory")
+results = vault.search_bm25("postgresql decision")
+vault.checkpoint("working on auth rollout")
+```
+
 ---
 
 ## Obsidian Integration
@@ -269,10 +298,14 @@ See [docs/openclaw-plugin-usage.md](docs/openclaw-plugin-usage.md) for detailed
 
 ---
 
-## Requirements
+## System Requirements
+
+- Node.js 18+ (22+ recommended)
+- npm 9+
+- Linux, macOS, Windows, or WSL
+- `qmd` is optional (used only for backward-compatible fallback paths; in-process search is built in)
 
-- Node.js 18+
-- `qmd` installed and available on `PATH` (for search/context features)
+For Linux-specific install and PATH guidance, see [docs/getting-started/installation.md](docs/getting-started/installation.md).
 
 ## LLM Providers
 
@@ -444,14 +477,21 @@ vault/
 
 ## Troubleshooting
 
-- Hook not found after enable:
-  - run `openclaw hooks install clawvault` first
-  - then `openclaw hooks enable clawvault`
-  - restart gateway
+- First-line diagnostic:
+  - run `clawvault doctor` after install or environment changes
+- Global install fails with `EACCES` / permission denied:
+  - run `npm config set prefix ~/.npm-global`
+  - add `export PATH="$HOME/.npm-global/bin:$PATH"` to your shell rc and reload shell
+- `clawvault: command not found` after install:
+  - check `npm config get prefix`
+  - ensure `<prefix>/bin` is present in your `PATH`
+- `qmd` fallback errors:
+  - `qmd` is optional; in-process BM25 search is available without it
+  - if you want fallback compatibility, ensure `qmd --version` works in the same shell
+- Hook/plugin not active in OpenClaw:
+  - run `openclaw hooks install clawvault`
+  - run `openclaw hooks enable clawvault`
   - verify with `openclaw hooks list --verbose`
-- `qmd` errors:
-  - ensure `qmd --version` works from same shell
-  - rerun `clawvault setup` after qmd install
 - OpenClaw integration drift:
   - run `clawvault compat`
 - Session transcript corruption:

package/bin/clawvault.js

@@ -23,7 +23,6 @@ import { registerProjectCommands } from './register-project-commands.js';
 
 import { registerTaskCommands } from './register-task-commands.js';
 
-import { registerWorkgraphCommands } from './register-workgraph-commands.js';
 import { registerTailscaleCommands } from './register-tailscale-commands.js';
 import {
   getVault,
@@ -109,7 +108,6 @@ registerProjectCommands(program, {
   resolveVaultPath
 });
 
-registerWorkgraphCommands(program, { chalk, resolveVaultPath });
 registerTailscaleCommands(program, { chalk });
 registerConfigCommands(program, { chalk, resolveVaultPath });
 registerRouteCommands(program, { chalk, resolveVaultPath });

package/bin/command-registration.test.js

@@ -35,7 +35,7 @@ describe('CLI command registration modules', () => {
     });
 
     const names = listCommandNames(program);
-    expect(names).toEqual(expect.arrayContaining(['init', 'setup', 'store', 'capture']));
+    expect(names).toEqual(expect.arrayContaining(['init', 'setup', 'store', 'patch', 'capture', 'inbox']));
   });
 
   it('registers query commands with profile option', () => {
@@ -121,6 +121,8 @@ describe('CLI command registration modules', () => {
     const names = listCommandNames(program);
     expect(names).toEqual(expect.arrayContaining([
       'doctor',
+      'benchmark',
+      'maintain',
       'embed',
       'compat',
       'graph',
@@ -163,4 +165,14 @@ describe('CLI command registration modules', () => {
     const unique = new Set(names);
     expect(unique.size).toBe(names.length);
   });
+
+  it('does not register removed workgraph commands', () => {
+    const program = registerAllCommandModules(new Command());
+    const names = listCommandNames(program);
+
+    expect(names).not.toContain('wg');
+    expect(names).not.toContain('thread');
+    expect(names).not.toContain('primitive');
+    expect(names).not.toContain('ledger');
+  });
 });

package/bin/help-contract.test.js

@@ -8,7 +8,11 @@ describe('CLI help contract', () => {
     expect(help).toContain('context');
     expect(help).toContain('inject');
     expect(help).toContain('doctor');
+    expect(help).toContain('benchmark');
+    expect(help).toContain('maintain');
     expect(help).toContain('embed');
+    expect(help).toContain('inbox');
+    expect(help).toContain('patch');
     expect(help).toContain('compat');
     expect(help).toContain('graph');
     expect(help).toContain('reflect');
@@ -36,4 +40,14 @@ describe('CLI help contract', () => {
     expect(projectListHelp).toContain('archived projects are hidden');
     expect(projectBoardHelp).toContain('default: status');
   });
+
+  it('does not advertise removed workgraph commands', () => {
+    const program = registerAllCommandModules();
+    const names = program.commands.map((command) => command.name());
+
+    expect(names).not.toContain('wg');
+    expect(names).not.toContain('thread');
+    expect(names).not.toContain('primitive');
+    expect(names).not.toContain('ledger');
+  });
 });

package/bin/register-core-commands.js

@@ -212,6 +212,66 @@ export function registerCoreCommands(
       }
     });
 
+  // === CAPTURE ===
+  program
+    .command('patch <idOrPath>')
+    .description('Patch an existing memory document')
+    .option('--append <text>', 'Append text to the document body (or target section)')
+    .option('--replace <text>', 'Text to replace')
+    .option('--with <text>', 'Replacement text used with --replace')
+    .option('--section <heading>', 'Limit patching to a markdown section heading')
+    .option('--content <text>', 'Replace document body (or section body) with text')
+    .option('-v, --vault <path>', 'Vault path (default: find nearest)')
+    .action(async (idOrPath, options) => {
+      try {
+        const modeFlags = [
+          typeof options.append === 'string',
+          typeof options.replace === 'string',
+          typeof options.content === 'string'
+        ];
+        const selectedModes = modeFlags.filter(Boolean).length;
+        if (selectedModes !== 1) {
+          throw new Error('Select exactly one patch mode: --append, --replace/--with, or --content.');
+        }
+
+        if (typeof options.with === 'string' && typeof options.replace !== 'string') {
+          throw new Error('--with can only be used together with --replace.');
+        }
+
+        const vault = await getVault(options.vault);
+        const patchOptions = {
+          idOrPath,
+          mode: 'content'
+        };
+
+        if (typeof options.append === 'string') {
+          patchOptions.mode = 'append';
+          patchOptions.append = options.append;
+        } else if (typeof options.replace === 'string') {
+          if (typeof options.with !== 'string') {
+            throw new Error('--replace requires --with.');
+          }
+          patchOptions.mode = 'replace';
+          patchOptions.replace = options.replace;
+          patchOptions.with = options.with;
+        } else if (typeof options.content === 'string') {
+          patchOptions.mode = 'content';
+          patchOptions.content = options.content;
+        }
+
+        if (typeof options.section === 'string') {
+          patchOptions.section = options.section;
+        }
+
+        const doc = await vault.patch(patchOptions);
+        console.log(chalk.green(`✓ Patched: ${doc.id}`));
+        console.log(chalk.dim(`  Path: ${doc.path}`));
+      } catch (err) {
+        console.error(chalk.red(`Error: ${err.message}`));
+        process.exit(1);
+      }
+    });
+
   // === CAPTURE ===
   program
     .command('capture <note>')
@@ -234,4 +294,32 @@ export function registerCoreCommands(
         process.exit(1);
       }
     });
+
+  // === INBOX ===
+  const inbox = program
+    .command('inbox')
+    .description('Manage raw captures in the inbox');
+
+  inbox
+    .command('add [content]')
+    .description('Add content to inbox (or pipe stdin)')
+    .option('-t, --title <title>', 'Capture title')
+    .option('--source <source>', 'Capture source label')
+    .option('--stdin', 'Read content from stdin')
+    .option('-v, --vault <path>', 'Vault path')
+    .action(async (content, options) => {
+      try {
+        const { inboxAddCommand } = await import('../dist/commands/inbox.js');
+        await inboxAddCommand({
+          vaultPath: options.vault,
+          content,
+          title: options.title,
+          source: options.source,
+          stdin: options.stdin || !process.stdin.isTTY
+        });
+      } catch (err) {
+        console.error(chalk.red(`Error: ${err.message}`));
+        process.exit(1);
+      }
+    });
 }

package/bin/register-core-commands.test.js

@@ -0,0 +1,80 @@
+import { describe, expect, it, vi } from 'vitest';
+import { Command } from 'commander';
+import * as fs from 'fs';
+import * as path from 'path';
+import { registerCoreCommands } from './register-core-commands.js';
+import { chalkStub } from './test-helpers/cli-command-fixtures.js';
+
+function buildProgram(patchImpl) {
+  const program = new Command();
+  registerCoreCommands(program, {
+    chalk: chalkStub,
+    path,
+    fs,
+    createVault: async () => ({ getCategories: () => [], getQmdRoot: () => '', getQmdCollection: () => '' }),
+    getVault: async () => ({ patch: patchImpl }),
+    runQmd: async () => {}
+  });
+  return program;
+}
+
+describe('register-core-commands patch command', () => {
+  it('exposes patch command mode flags', () => {
+    const program = buildProgram(async () => ({ id: 'decisions/example', path: '/vault/decisions/example.md' }));
+    const patchCommand = program.commands.find((command) => command.name() === 'patch');
+    expect(patchCommand).toBeDefined();
+    const optionFlags = patchCommand?.options.map((option) => option.flags) ?? [];
+    expect(optionFlags).toEqual(expect.arrayContaining([
+      '--append <text>',
+      '--replace <text>',
+      '--with <text>',
+      '--section <heading>',
+      '--content <text>',
+      '-v, --vault <path>'
+    ]));
+  });
+
+  it('forwards append mode payload to vault.patch', async () => {
+    const patchMock = vi.fn(async () => ({ id: 'decisions/example', path: '/vault/decisions/example.md' }));
+    const program = buildProgram(patchMock);
+    const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {});
+    const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {});
+
+    try {
+      await program.parseAsync(['patch', 'decisions/example', '--append', 'new line'], { from: 'user' });
+      expect(patchMock).toHaveBeenCalledWith({
+        idOrPath: 'decisions/example',
+        mode: 'append',
+        append: 'new line'
+      });
+      expect(errorSpy).not.toHaveBeenCalled();
+    } finally {
+      logSpy.mockRestore();
+      errorSpy.mockRestore();
+    }
+  });
+
+  it('forwards section/content mode payload to vault.patch', async () => {
+    const patchMock = vi.fn(async () => ({ id: 'decisions/example', path: '/vault/decisions/example.md' }));
+    const program = buildProgram(patchMock);
+    const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {});
+    const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {});
+
+    try {
+      await program.parseAsync(
+        ['patch', 'decisions/example', '--section', 'Notes', '--content', 'updated notes'],
+        { from: 'user' }
+      );
+      expect(patchMock).toHaveBeenCalledWith({
+        idOrPath: 'decisions/example',
+        mode: 'content',
+        content: 'updated notes',
+        section: 'Notes'
+      });
+      expect(errorSpy).not.toHaveBeenCalled();
+    } finally {
+      logSpy.mockRestore();
+      errorSpy.mockRestore();
+    }
+  });
+});

package/bin/register-maintenance-commands.js

@@ -4,10 +4,33 @@
  */
 
 export function registerMaintenanceCommands(program, { chalk }) {
+  // === MAINTAIN ===
+  program
+    .command('maintain')
+    .description('Run background inbox maintenance workers')
+    .option('--worker <name>', 'Run a single worker (curator|janitor|distiller|surveyor)')
+    .option('--limit <n>', 'Limit inbox items processed per worker', (value) => Number.parseInt(value, 10))
+    .option('--dry-run', 'Preview actions without writing files')
+    .option('-v, --vault <path>', 'Vault path')
+    .action(async (options) => {
+      try {
+        const { maintainCommand } = await import('../dist/commands/maintain.js');
+        await maintainCommand({
+          vaultPath: options.vault,
+          worker: options.worker,
+          limit: options.limit,
+          dryRun: options.dryRun
+        });
+      } catch (err) {
+        console.error(chalk.red(`Error: ${err.message}`));
+        process.exit(1);
+      }
+    });
+
   // === DOCTOR (health check) ===
   program
     .command('doctor')
-    .description('Diagnose vault health and optionally apply fixes')
+    .description('Run installation and environment diagnostics')
     .option('-v, --vault <path>', 'Vault path')
     .option('--fix', 'Apply safe auto-fixes for qmd index, embeddings, and dead collections')
     .option('--json', 'Output machine-readable JSON')
@@ -24,10 +47,11 @@ export function registerMaintenanceCommands(program, { chalk }) {
           return;
         }
 
-        console.log(chalk.cyan('\n🩺 ClawVault Health Check\n'));
-        console.log(chalk.dim(`Vault: ${report.vaultPath}`));
-        console.log(chalk.dim(`qmd collection: ${report.qmdCollection}`));
-        console.log(chalk.dim(`qmd root: ${report.qmdRoot}`));
+        console.log(chalk.cyan('\n🩺 ClawVault Doctor Report\n'));
+        if (report.vaultPath) {
+          console.log(chalk.dim(`Vault: ${report.vaultPath}`));
+        }
+        console.log(chalk.dim(`Generated: ${report.generatedAt}`));
         console.log();
 
         for (const check of report.checks) {
@@ -36,7 +60,7 @@ export function registerMaintenanceCommands(program, { chalk }) {
             : check.status === 'warn'
               ? chalk.yellow('⚠')
               : chalk.red('✗');
-          const line = `${check.label}: ${check.detail}`;
+          const line = check.detail ? `${check.label}: ${check.detail}` : check.label;
           const renderedLine = check.status === 'ok'
             ? chalk.green(line)
             : check.status === 'warn'
@@ -298,4 +322,31 @@ export function registerMaintenanceCommands(program, { chalk }) {
         process.exit(1);
       }
     });
+
+  // === BENCHMARK ===
+  const benchmark = program
+    .command('benchmark')
+    .description('Run benchmark harnesses');
+
+  benchmark
+    .command('observer')
+    .description('Evaluate observer output quality against annotated transcripts')
+    .option('--fixtures-dir <path>', 'Fixture root directory (default: testdata/observer-benchmark)')
+    .option('--provider <provider>', 'Compression provider (mock|anthropic|openai|gemini|xai|openai-compatible|ollama|minimax|zai)', 'mock')
+    .option('--model <model>', 'Model override for live provider runs')
+    .option('--report-format <format>', 'Report output format (json|text)', 'text')
+    .action(async (options) => {
+      try {
+        const { benchmarkObserverCommand } = await import('../dist/commands/benchmark.js');
+        await benchmarkObserverCommand({
+          fixturesDir: options.fixturesDir,
+          provider: options.provider,
+          model: options.model,
+          reportFormat: options.reportFormat
+        });
+      } catch (err) {
+        console.error(chalk.red(`Error: ${err.message}`));
+        process.exit(1);
+      }
+    });
 }

package/bin/register-query-commands.js

@@ -17,7 +17,7 @@ export function registerQueryCommands(
   // === SEARCH ===
   program
     .command('search <query>')
-    .description('Search the vault via qmd (BM25), optionally with semantic hybrid search')
+    .description('Search the vault using in-process hybrid retrieval (BM25 + semantic when configured)')
     .option('-n, --limit <n>', 'Max results (default: 10)', '10')
     .option('-c, --category <category>', 'Filter by category')
     .option('--tags <tags>', 'Filter by tags (comma-separated)')
@@ -25,8 +25,8 @@ export function registerQueryCommands(
     .option('--full', 'Include full content in results')
     .option('-v, --vault <path>', 'Vault path')
     .option('--json', 'Output as JSON')
-    .option('--semantic', 'Enable hybrid search (BM25 + semantic with RRF)')
-    .option('--rebuild-embeddings', 'Rebuild the embedding cache before searching')
+    .option('--semantic', 'Legacy alias. Hybrid retrieval is already default when embeddings are configured.')
+    .option('--rebuild-embeddings', 'Rebuild hosted embedding cache before searching')
     .action(async (query, options) => {
       try {
         const vaultPath = resolveVaultPath(options.vault);
@@ -45,34 +45,16 @@ export function registerQueryCommands(
           console.log();
         }
 
-        // Get BM25 results
-        const bm25Results = await vault.find(query, {
-          limit: options.semantic ? 50 : parseInt(options.limit, 10),
+        const results = await vault.find(query, {
+          limit: parseInt(options.limit, 10),
           category: options.category,
           tags: options.tags?.split(',').map((value) => value.trim()),
           fullContent: options.full,
           temporalBoost: options.recent
         });
-
-        let results = bm25Results;
-        let searchMode = 'BM25';
-
-        // Apply hybrid search if --semantic flag is set
-        if (options.semantic) {
-          const { EmbeddingCache, hybridSearch } = await import('../dist/lib/hybrid-search.js');
-          const cache = new EmbeddingCache(vaultPath);
-          cache.load();
-
-          if (cache.size === 0) {
-            console.log(chalk.yellow('Warning: No embeddings found. Run with --rebuild-embeddings to build the cache.'));
-          } else {
-            results = await hybridSearch(query, bm25Results, cache, {
-              topK: parseInt(options.limit, 10),
-              rrfK: 60
-            });
-            searchMode = 'Hybrid (BM25 + Semantic)';
-          }
-        }
+        const searchMode = options.semantic
+          ? 'Hybrid (legacy flag acknowledged)'
+          : 'Hybrid (in-process)';
 
         if (options.json) {
           console.log(JSON.stringify({ searchMode, results }, null, 2));
@@ -84,7 +66,7 @@ export function registerQueryCommands(
           return;
         }
 
-        const icon = options.semantic ? '🔍🧠' : '🔍';
+        const icon = '🔍🧠';
         console.log(chalk.cyan(`\n${icon} Found ${results.length} result(s) for "${query}" [${searchMode}]:\n`));
 
         for (const result of results) {
@@ -114,7 +96,7 @@ export function registerQueryCommands(
   // === VSEARCH ===
   program
     .command('vsearch <query>')
-    .description('Semantic search via qmd (requires qmd installed)')
+    .description('Semantic search via hosted embeddings (qmd fallback when available)')
     .option('-n, --limit <n>', 'Max results (default: 5)', '5')
     .option('-c, --category <category>', 'Filter by category')
     .option('--tags <tags>', 'Filter by tags (comma-separated)')

package/bin/test-helpers/cli-command-fixtures.js

@@ -30,6 +30,7 @@ export function stubResolveVaultPath(value) {
 export function createVaultStub(overrides = {}) {
   return {
     store: async () => ({}),
+    patch: async () => ({}),
     capture: async () => ({}),
     find: async () => [],
     vsearch: async () => [],