@hasna/knowledge 0.2.13 → 0.2.15

package/docs/architecture/ai-native-knowledge-base.md

@@ -215,6 +215,17 @@ Local mode should start with SQLite FTS and a local vector-index option. Hosted
 mode can use Postgres with pgvector or a managed vector index. Permission
 filters must be applied before agent context is assembled.
 
+The first local semantic-search implementation indexes derived chunks with
+`open-knowledge embeddings index` and queries them with
+`open-knowledge search --semantic` or the lower-level
+`open-knowledge embeddings search`. It stores OpenAI embedding vectors as
+generated metadata rows, not raw source bytes, and pins each row to `open-files`
+provenance: source ref/URI, revision/hash, chunk offsets, token count, provider,
+model, dimensions, status, and timestamps. The structured `search` contract
+merges keyword FTS, wiki/index catalog hits, generated wiki chunks, and optional
+vector results so the local SQLite index can later move to pgvector or a managed
+hosted vector store without changing CLI/MCP result shape.
+
 ## Agent Workflow
 
 The target user flow is:

package/docs/architecture/hybrid-semantic-search.md

@@ -32,6 +32,9 @@ Local mode starts with SQLite:
 - `chunks_fts` provides keyword search.
 - `chunk_embeddings` stores embedding vectors as JSON until a local vector
   extension is chosen.
+- `vector_index_entries` stores searchable embedding rows with provider/model,
+  dimensions, source revision/hash, chunk offsets, status, timestamps, and
+  provenance metadata.
 - `wiki_pages`, `wiki_backlinks`, and `citations` provide graph and provenance
   signals.
 - `knowledge_indexes` tracks generated machine-readable shards.
@@ -41,6 +44,21 @@ implementation. The retrieval interface should hide it so a later vector
 extension or pgvector backend can replace storage without changing CLI/MCP
 contracts.
 
+The current local command surface is:
+
+```bash
+open-knowledge search "company wiki policy" --scope project --json
+open-knowledge search "company wiki policy" --scope project --semantic --json
+open-knowledge embeddings index --scope project --model openai:text-embedding-3-small
+open-knowledge embeddings search "company wiki policy" --scope project --json
+```
+
+`search` is the structured hybrid layer for agents. `embeddings search` is the
+lower-level vector-only command. MCP exposes the same capability through
+`ok_search`, `ok_embeddings_status`, `ok_embeddings_index`, and
+`ok_semantic_search`. Deterministic `--fake` embeddings exist for tests and
+offline verification only.
+
 ## Hosted Indexes
 
 Hosted mode may use:
@@ -58,16 +76,18 @@ unauthorized content.
 
 1. Normalize the query.
 2. Embed the query if a semantic-capable provider is configured.
-3. Run keyword FTS over source chunks and wiki chunks.
-4. Run vector search over source chunks and wiki pages.
-5. Expand candidate pages through backlinks and citations.
-6. Drop stale candidates whose source revision/hash no longer matches
+3. Run keyword FTS over source chunks and generated wiki chunks.
+4. Search wiki page and machine-readable index catalog rows.
+5. Run vector search over source chunks and wiki pages when semantic mode is
+   requested.
+6. Expand candidate pages through backlinks and citations.
+7. Drop stale candidates whose source revision/hash no longer matches
    `open-files`.
-7. Apply permission filters.
-8. Merge and dedupe by source revision, wiki page, citation, and text hash.
-9. Rerank by relevance, exact-match score, semantic score, freshness, citation
+8. Apply permission filters.
+9. Merge and dedupe by source revision, wiki page, citation, and text hash.
+10. Rerank by relevance, exact-match score, semantic score, freshness, citation
    quality, and wiki authority.
-10. Return structured results with source refs, citation spans, page refs,
+11. Return structured results with source refs, citation spans, page refs,
     scores, and reason codes.
 
 ## Result Shape
@@ -120,6 +140,9 @@ Reindexing is driven by source revisions:
   stale.
 - If a source is deleted or access changes, affected chunks must be hidden or
   removed before future retrieval.
+- Local outbox consumption deletes stale `chunk_embeddings` and
+  `vector_index_entries` for deleted revisions, so semantic search cannot return
+  removed source chunks.
 - Wiki pages should track the source revisions they cite so lint can flag stale
   pages.
 - Embedding refresh jobs should be idempotent and checkpointed in `runs` and

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hasna/knowledge",
-  "version": "0.2.13",
+  "version": "0.2.15",
   "description": "Agent-friendly local knowledge CLI with JSON output, pagination, and safe destructive actions",
   "type": "module",
   "bin": {

package/src/cli.ts

@@ -47,10 +47,14 @@ interface Flags {
   format?: string;
   completions?: string;
   purpose?: string;
+  model?: string;
+  dimensions?: number;
+  semantic?: boolean;
   noColor?: boolean;
   scope?: string;
   olderThan?: number;
   empty?: boolean;
+  fake?: boolean;
   archived?: boolean;
   includeArchived?: boolean;
 }
@@ -60,7 +64,7 @@ interface ParseResult {
   flags: Flags;
 }
 
-const COMMANDS = ['add', 'list', 'get', 'delete', 'update', 'archive', 'restore', 'upsert', 'untag', 'export', 'prune', 'dedupe', 'stats', 'paths', 'storage', 'db', 'wiki', 'source', 'ingest', 'reindex', 'providers', 'safety', 'help'];
+const COMMANDS = ['add', 'list', 'get', 'delete', 'update', 'archive', 'restore', 'upsert', 'untag', 'export', 'prune', 'dedupe', 'stats', 'paths', 'storage', 'db', 'wiki', 'source', 'ingest', 'reindex', 'search', 'embeddings', 'providers', 'safety', 'help'];
 const COMMAND_ALIASES: Record<string, string> = {
   ls: 'list',
   rm: 'delete',
@@ -96,6 +100,10 @@ function parseArgs(argv: string[]): ParseResult {
       case '--format': flags.format = argv[i + 1]; i += 1; break;
       case '--completions': flags.completions = argv[i + 1]; i += 1; break;
       case '--purpose': flags.purpose = argv[i + 1]; i += 1; break;
+      case '--model': flags.model = argv[i + 1]; i += 1; break;
+      case '--dimensions': flags.dimensions = Number(argv[i + 1]); i += 1; break;
+      case '--semantic': flags.semantic = true; break;
+      case '--fake': flags.fake = true; break;
       case '--no-color': flags.noColor = true; break;
       case '--scope': flags.scope = argv[i + 1]; i += 1; break;
       case '--older-than': flags.olderThan = Number(argv[i + 1]); i += 1; break;
@@ -169,6 +177,8 @@ Commands:
   ingest manifest <file|s3://> Ingest an open-files manifest into knowledge.db
   ingest source <source-ref>   Ingest a read-only source ref into knowledge.db
   reindex outbox <file|s3://>  Consume open-files change events and invalidate chunks
+  search <query>               Hybrid search sources, wiki pages, and indexes
+  embeddings status|index|search Build/query local vector embeddings
   providers status|models|check Inspect AI SDK provider config and credentials
   safety status|check|approve|audit|redact
   help [command]               Show help
@@ -177,6 +187,10 @@ Global Options:
   --json                      Output JSON
   --store <path>              Override store path
   --purpose <name>            Read-only source purpose (default: knowledge_answer)
+  --model <provider:model>     AI/embedding model ref
+  --dimensions <n>             Embedding dimensions for local/fake providers
+  --semantic                   Include vector semantic results in search
+  --fake                       Use deterministic fake embeddings for local tests
   --scope local|global|project  Store scope (default: global ~/.hasna/apps/knowledge/)
   --no-color                  Disable color output
   --completions <shell>       Output completions for bash|zsh|fish
@@ -237,6 +251,8 @@ function printCommandHelp(command: string): void {
   if (command === 'source') { console.log('Usage: open-knowledge source resolve <source-ref> [--purpose knowledge_answer|knowledge_index] [--limit <n>] [--scope local|global|project] [--json]'); return; }
   if (command === 'ingest') { console.log('Usage: open-knowledge ingest manifest <file|s3://bucket/key> | source <source-ref> [--purpose knowledge_index] [--scope local|global|project] [--json]'); return; }
   if (command === 'reindex') { console.log('Usage: open-knowledge reindex outbox <file|s3://bucket/key> [--scope local|global|project] [--json]'); return; }
+  if (command === 'search') { console.log('Usage: open-knowledge search <query> [--semantic] [--model openai:text-embedding-3-small] [--limit <n>] [--dimensions <n>] [--fake] [--scope local|global|project] [--json]'); return; }
+  if (command === 'embeddings') { console.log('Usage: open-knowledge embeddings status|index|search [query] [--model openai:text-embedding-3-small] [--limit <n>] [--dimensions <n>] [--fake] [--scope local|global|project] [--json]'); return; }
   if (command === 'providers') { console.log('Usage: open-knowledge providers status|models|check [provider|model-alias] [--scope local|global|project] [--json]'); return; }
   if (command === 'safety') { console.log('Usage: open-knowledge safety status|check|approve|audit|redact [args] [--scope local|global|project] [--json]'); return; }
   printGlobalHelp();
@@ -283,11 +299,11 @@ async function run(argv: string[]): Promise<void> {
   if (flags.completions) {
     const shell = flags.completions;
     if (shell === 'bash') {
-      console.log(`_open_knowledge() { local cur; cur="${"$"}{COMP_WORDS[COMP_CWORD]}"; COMPREPLY=($(compgen -W "add list get update archive restore upsert untag delete export prune dedupe stats paths storage db wiki source ingest reindex providers safety help ls rm edit unarchive --json --yes --help --version --desc --page --limit --search --sort --id --store --title --content --url --tag --format --completions --purpose --no-color --scope --archived --include-archived" -- "$cur")); }; complete -F _open_knowledge open-knowledge`);
+      console.log(`_open_knowledge() { local cur; cur="${"$"}{COMP_WORDS[COMP_CWORD]}"; COMPREPLY=($(compgen -W "add list get update archive restore upsert untag delete export prune dedupe stats paths storage db wiki source ingest reindex search embeddings providers safety help ls rm edit unarchive --json --yes --help --version --desc --page --limit --search --sort --id --store --title --content --url --tag --format --completions --purpose --model --dimensions --semantic --fake --no-color --scope --archived --include-archived" -- "$cur")); }; complete -F _open_knowledge open-knowledge`);
     } else if (shell === 'zsh') {
-      console.log(`#compdef open-knowledge\n_open_knowledge() { _arguments -C "1: :(add list get update archive restore upsert untag delete export prune dedupe stats paths storage db wiki source ingest reindex providers safety help ls rm edit unarchive)" "(--json)--json" "(--yes)-y" "(--help)--help" "(--version)--version" "(--desc)--desc" "(--archived)--archived" "(--include-archived)--include-archived" "(-p --page)"{-p,--page}"[page number]:number:" "(-l --limit)"{-l,--limit}"[items per page]:number:" "(-s --search)"{-s,--search}"[search text]:text:" "(--sort)--sort"\{created,title\}:" "(--id)--id[item id]:id:" "(--store)--store[store path]:path:" "(--title)--title[new title]:" "(--content)--content[new content]:" "(--url)--url[source url]:" "(-t --tag)"{-t,--tag}"[tag]:tag:" "(--format)--format[json|jsonl]:" "(--completions)--completions[output completions]:shell:(bash zsh fish):" "(--purpose)--purpose[purpose]:" "(--no-color)--no-color[disable color]" "(--scope)--scope"\{local,global,project\}:" }; _open_knowledge`);
+      console.log(`#compdef open-knowledge\n_open_knowledge() { _arguments -C "1: :(add list get update archive restore upsert untag delete export prune dedupe stats paths storage db wiki source ingest reindex search embeddings providers safety help ls rm edit unarchive)" "(--json)--json" "(--yes)-y" "(--help)--help" "(--version)--version" "(--desc)--desc" "(--archived)--archived" "(--include-archived)--include-archived" "(--semantic)--semantic" "(--fake)--fake" "(-p --page)"{-p,--page}"[page number]:number:" "(-l --limit)"{-l,--limit}"[items per page]:number:" "(-s --search)"{-s,--search}"[search text]:text:" "(--sort)--sort"\{created,title\}:" "(--id)--id[item id]:id:" "(--store)--store[store path]:path:" "(--title)--title[new title]:" "(--content)--content[new content]:" "(--url)--url[source url]:" "(-t --tag)"{-t,--tag}"[tag]:tag:" "(--format)--format[json|jsonl]:" "(--completions)--completions[output completions]:shell:(bash zsh fish):" "(--purpose)--purpose[purpose]:" "(--model)--model[model ref]:" "(--dimensions)--dimensions[embedding dimensions]:number:" "(--no-color)--no-color[disable color]" "(--scope)--scope"\{local,global,project\}:" }; _open_knowledge`);
     } else if (shell === 'fish') {
-      console.log(`complete -c open-knowledge -f; complete -c open-knowledge -a "add list get update archive restore upsert untag delete export prune dedupe stats paths storage db wiki source ingest reindex providers safety help ls rm edit unarchive"; complete -c open-knowledge -l json; complete -c open-knowledge -l yes -s y; complete -c open-knowledge -l help -s h; complete -c open-knowledge -l version -s v; complete -c open-knowledge -l desc; complete -c open-knowledge -l archived; complete -c open-knowledge -l include-archived; complete -c open-knowledge -s p -l page; complete -c open-knowledge -s l -l limit; complete -c open-knowledge -s s -l search; complete -c open-knowledge -l sort; complete -c open-knowledge -l id; complete -c open-knowledge -l store; complete -c open-knowledge -l title; complete -c open-knowledge -l content; complete -c open-knowledge -l url; complete -c open-knowledge -s t -l tag; complete -c open-knowledge -l format; complete -c open-knowledge -l completions; complete -c open-knowledge -l purpose; complete -c open-knowledge -l no-color; complete -c open-knowledge -l scope -a "local global project"`);
+      console.log(`complete -c open-knowledge -f; complete -c open-knowledge -a "add list get update archive restore upsert untag delete export prune dedupe stats paths storage db wiki source ingest reindex search embeddings providers safety help ls rm edit unarchive"; complete -c open-knowledge -l json; complete -c open-knowledge -l yes -s y; complete -c open-knowledge -l help -s h; complete -c open-knowledge -l version -s v; complete -c open-knowledge -l desc; complete -c open-knowledge -l archived; complete -c open-knowledge -l include-archived; complete -c open-knowledge -l semantic; complete -c open-knowledge -l fake; complete -c open-knowledge -s p -l page; complete -c open-knowledge -s l -l limit; complete -c open-knowledge -s s -l search; complete -c open-knowledge -l sort; complete -c open-knowledge -l id; complete -c open-knowledge -l store; complete -c open-knowledge -l title; complete -c open-knowledge -l content; complete -c open-knowledge -l url; complete -c open-knowledge -s t -l tag; complete -c open-knowledge -l format; complete -c open-knowledge -l completions; complete -c open-knowledge -l purpose; complete -c open-knowledge -l model; complete -c open-knowledge -l dimensions; complete -c open-knowledge -l no-color; complete -c open-knowledge -l scope -a "local global project"`);
     } else {
       throw new Error("Invalid --completions value. Use 'bash', 'zsh', or 'fish'.");
     }
@@ -534,6 +550,54 @@ async function run(argv: string[]): Promise<void> {
     return;
   }
 
+  if (command === 'embeddings') {
+    const action = positional[1] ?? 'status';
+    if (action === 'status') {
+      const result = service.embeddingStatus();
+      output({ ok: true, ...result, message: `${result.total_vector_entries} vector index entries` }, flags.json);
+      return;
+    }
+    if (action === 'index') {
+      const result = await service.indexEmbeddings({
+        limit: flags.limit,
+        modelRef: flags.model,
+        dimensions: flags.dimensions,
+        fake: flags.fake,
+      });
+      output({ ok: true, ...result, message: `Embedded ${result.chunks_embedded} chunk(s)` }, flags.json);
+      return;
+    }
+    if (action === 'search') {
+      const query = positional.slice(2).join(' ');
+      if (!query) throw new Error('Usage: open-knowledge embeddings search <query>');
+      const result = await service.semanticSearch({
+        query,
+        limit: flags.limit,
+        modelRef: flags.model,
+        dimensions: flags.dimensions,
+        fake: flags.fake,
+      });
+      output({ ok: true, ...result, message: `${result.results.length} semantic result(s)` }, flags.json);
+      return;
+    }
+    throw new Error("Invalid embeddings action. Use 'status', 'index', or 'search'.");
+  }
+
+  if (command === 'search') {
+    const query = positional.slice(1).join(' ');
+    if (!query) throw new Error('Usage: open-knowledge search <query>');
+    const result = await service.search({
+      query,
+      limit: flags.limit,
+      semantic: flags.semantic,
+      modelRef: flags.model,
+      dimensions: flags.dimensions,
+      fake: flags.fake,
+    });
+    output({ ok: true, ...result, message: `${result.results.length} search result(s)` }, flags.json);
+    return;
+  }
+
   if (command === 'providers') {
     const action = positional[1] ?? 'status';
     if (action === 'status') {