@hasna/knowledge 0.2.12 → 0.2.14

package/docs/architecture/ai-native-knowledge-base.md

@@ -89,6 +89,21 @@ file revisions, hashes, extraction state, permissions, and storage metadata.
 Direct `s3://`, `file://`, and `https://` refs are useful for bootstrap and
 interop, but should be normalized into source records when possible.
 
+## Provenance Contract
+
+Every durable search/wiki artifact should carry a provenance object in metadata:
+`source_owner`, `source_ref`, `source_uri`, `source_kind`, `source_revision_id`,
+`revision`, `hash`, optional `chunk_id`, offsets, `read_only`,
+`citation_required`, resolver name, and stale status. For generated artifacts
+that are not source-backed yet, metadata still records that `open-files` owns
+source bytes and that citations are required before durable facts are filed.
+
+`wiki init` now catalogs the starter `wiki/README.md` and `indexes/root.md`
+records with generated-artifact provenance. Source ingestion stores source
+provenance on every chunk, and source resolution returns that provenance with
+chunks and citations so semantic search can pass through trustworthy evidence
+without reconstructing it later.
+
 ## Resolver Boundary
 
 The local resolver is exposed through:
@@ -200,6 +215,15 @@ Local mode should start with SQLite FTS and a local vector-index option. Hosted
 mode can use Postgres with pgvector or a managed vector index. Permission
 filters must be applied before agent context is assembled.
 
+The first local semantic-search implementation indexes derived chunks with
+`open-knowledge embeddings index` and queries them with
+`open-knowledge embeddings search`. It stores OpenAI embedding vectors as
+generated metadata rows, not raw source bytes, and pins each row to
+`open-files` provenance: source ref/URI, revision/hash, chunk offsets, token
+count, provider, model, dimensions, status, and timestamps. This lets the local
+SQLite index later move to pgvector or a managed hosted vector store without
+changing the CLI/MCP result contract.
+
 ## Agent Workflow
 
 The target user flow is:

package/docs/architecture/hybrid-semantic-search.md

@@ -32,6 +32,9 @@ Local mode starts with SQLite:
 - `chunks_fts` provides keyword search.
 - `chunk_embeddings` stores embedding vectors as JSON until a local vector
   extension is chosen.
+- `vector_index_entries` stores searchable embedding rows with provider/model,
+  dimensions, source revision/hash, chunk offsets, status, timestamps, and
+  provenance metadata.
 - `wiki_pages`, `wiki_backlinks`, and `citations` provide graph and provenance
   signals.
 - `knowledge_indexes` tracks generated machine-readable shards.
@@ -41,6 +44,17 @@ implementation. The retrieval interface should hide it so a later vector
 extension or pgvector backend can replace storage without changing CLI/MCP
 contracts.
 
+The current local command surface is:
+
+```bash
+open-knowledge embeddings index --scope project --model openai:text-embedding-3-small
+open-knowledge embeddings search "company wiki policy" --scope project --json
+```
+
+MCP exposes the same capability through `ok_embeddings_status`,
+`ok_embeddings_index`, and `ok_semantic_search`. Deterministic `--fake`
+embeddings exist for tests and offline verification only.
+
 ## Hosted Indexes
 
 Hosted mode may use:
@@ -120,6 +134,9 @@ Reindexing is driven by source revisions:
   stale.
 - If a source is deleted or access changes, affected chunks must be hidden or
   removed before future retrieval.
+- Local outbox consumption deletes stale `chunk_embeddings` and
+  `vector_index_entries` for deleted revisions, so semantic search cannot return
+  removed source chunks.
 - Wiki pages should track the source revisions they cite so lint can flag stale
   pages.
 - Embedding refresh jobs should be idempotent and checkpointed in `runs` and

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hasna/knowledge",
-  "version": "0.2.12",
+  "version": "0.2.14",
   "description": "Agent-friendly local knowledge CLI with JSON output, pagination, and safe destructive actions",
   "type": "module",
   "bin": {

package/src/cli.ts

@@ -47,10 +47,13 @@ interface Flags {
   format?: string;
   completions?: string;
   purpose?: string;
+  model?: string;
+  dimensions?: number;
   noColor?: boolean;
   scope?: string;
   olderThan?: number;
   empty?: boolean;
+  fake?: boolean;
   archived?: boolean;
   includeArchived?: boolean;
 }
@@ -60,7 +63,7 @@ interface ParseResult {
   flags: Flags;
 }
 
-const COMMANDS = ['add', 'list', 'get', 'delete', 'update', 'archive', 'restore', 'upsert', 'untag', 'export', 'prune', 'dedupe', 'stats', 'paths', 'storage', 'db', 'wiki', 'source', 'ingest', 'reindex', 'providers', 'safety', 'help'];
+const COMMANDS = ['add', 'list', 'get', 'delete', 'update', 'archive', 'restore', 'upsert', 'untag', 'export', 'prune', 'dedupe', 'stats', 'paths', 'storage', 'db', 'wiki', 'source', 'ingest', 'reindex', 'embeddings', 'providers', 'safety', 'help'];
 const COMMAND_ALIASES: Record<string, string> = {
   ls: 'list',
   rm: 'delete',
@@ -96,6 +99,9 @@ function parseArgs(argv: string[]): ParseResult {
       case '--format': flags.format = argv[i + 1]; i += 1; break;
       case '--completions': flags.completions = argv[i + 1]; i += 1; break;
       case '--purpose': flags.purpose = argv[i + 1]; i += 1; break;
+      case '--model': flags.model = argv[i + 1]; i += 1; break;
+      case '--dimensions': flags.dimensions = Number(argv[i + 1]); i += 1; break;
+      case '--fake': flags.fake = true; break;
       case '--no-color': flags.noColor = true; break;
       case '--scope': flags.scope = argv[i + 1]; i += 1; break;
       case '--older-than': flags.olderThan = Number(argv[i + 1]); i += 1; break;
@@ -169,6 +175,7 @@ Commands:
   ingest manifest <file|s3://> Ingest an open-files manifest into knowledge.db
   ingest source <source-ref>   Ingest a read-only source ref into knowledge.db
   reindex outbox <file|s3://>  Consume open-files change events and invalidate chunks
+  embeddings status|index|search Build/query local vector embeddings
   providers status|models|check Inspect AI SDK provider config and credentials
   safety status|check|approve|audit|redact
   help [command]               Show help
@@ -177,6 +184,9 @@ Global Options:
   --json                      Output JSON
   --store <path>              Override store path
   --purpose <name>            Read-only source purpose (default: knowledge_answer)
+  --model <provider:model>     AI/embedding model ref
+  --dimensions <n>             Embedding dimensions for local/fake providers
+  --fake                       Use deterministic fake embeddings for local tests
   --scope local|global|project  Store scope (default: global ~/.hasna/apps/knowledge/)
   --no-color                  Disable color output
   --completions <shell>       Output completions for bash|zsh|fish
@@ -237,6 +247,7 @@ function printCommandHelp(command: string): void {
   if (command === 'source') { console.log('Usage: open-knowledge source resolve <source-ref> [--purpose knowledge_answer|knowledge_index] [--limit <n>] [--scope local|global|project] [--json]'); return; }
   if (command === 'ingest') { console.log('Usage: open-knowledge ingest manifest <file|s3://bucket/key> | source <source-ref> [--purpose knowledge_index] [--scope local|global|project] [--json]'); return; }
   if (command === 'reindex') { console.log('Usage: open-knowledge reindex outbox <file|s3://bucket/key> [--scope local|global|project] [--json]'); return; }
+  if (command === 'embeddings') { console.log('Usage: open-knowledge embeddings status|index|search [query] [--model openai:text-embedding-3-small] [--limit <n>] [--dimensions <n>] [--fake] [--scope local|global|project] [--json]'); return; }
   if (command === 'providers') { console.log('Usage: open-knowledge providers status|models|check [provider|model-alias] [--scope local|global|project] [--json]'); return; }
   if (command === 'safety') { console.log('Usage: open-knowledge safety status|check|approve|audit|redact [args] [--scope local|global|project] [--json]'); return; }
   printGlobalHelp();
@@ -283,11 +294,11 @@ async function run(argv: string[]): Promise<void> {
   if (flags.completions) {
     const shell = flags.completions;
     if (shell === 'bash') {
-      console.log(`_open_knowledge() { local cur; cur="${"$"}{COMP_WORDS[COMP_CWORD]}"; COMPREPLY=($(compgen -W "add list get update archive restore upsert untag delete export prune dedupe stats paths storage db wiki source ingest reindex providers safety help ls rm edit unarchive --json --yes --help --version --desc --page --limit --search --sort --id --store --title --content --url --tag --format --completions --purpose --no-color --scope --archived --include-archived" -- "$cur")); }; complete -F _open_knowledge open-knowledge`);
+      console.log(`_open_knowledge() { local cur; cur="${"$"}{COMP_WORDS[COMP_CWORD]}"; COMPREPLY=($(compgen -W "add list get update archive restore upsert untag delete export prune dedupe stats paths storage db wiki source ingest reindex embeddings providers safety help ls rm edit unarchive --json --yes --help --version --desc --page --limit --search --sort --id --store --title --content --url --tag --format --completions --purpose --model --dimensions --fake --no-color --scope --archived --include-archived" -- "$cur")); }; complete -F _open_knowledge open-knowledge`);
     } else if (shell === 'zsh') {
-      console.log(`#compdef open-knowledge\n_open_knowledge() { _arguments -C "1: :(add list get update archive restore upsert untag delete export prune dedupe stats paths storage db wiki source ingest reindex providers safety help ls rm edit unarchive)" "(--json)--json" "(--yes)-y" "(--help)--help" "(--version)--version" "(--desc)--desc" "(--archived)--archived" "(--include-archived)--include-archived" "(-p --page)"{-p,--page}"[page number]:number:" "(-l --limit)"{-l,--limit}"[items per page]:number:" "(-s --search)"{-s,--search}"[search text]:text:" "(--sort)--sort"\{created,title\}:" "(--id)--id[item id]:id:" "(--store)--store[store path]:path:" "(--title)--title[new title]:" "(--content)--content[new content]:" "(--url)--url[source url]:" "(-t --tag)"{-t,--tag}"[tag]:tag:" "(--format)--format[json|jsonl]:" "(--completions)--completions[output completions]:shell:(bash zsh fish):" "(--purpose)--purpose[purpose]:" "(--no-color)--no-color[disable color]" "(--scope)--scope"\{local,global,project\}:" }; _open_knowledge`);
+      console.log(`#compdef open-knowledge\n_open_knowledge() { _arguments -C "1: :(add list get update archive restore upsert untag delete export prune dedupe stats paths storage db wiki source ingest reindex embeddings providers safety help ls rm edit unarchive)" "(--json)--json" "(--yes)-y" "(--help)--help" "(--version)--version" "(--desc)--desc" "(--archived)--archived" "(--include-archived)--include-archived" "(--fake)--fake" "(-p --page)"{-p,--page}"[page number]:number:" "(-l --limit)"{-l,--limit}"[items per page]:number:" "(-s --search)"{-s,--search}"[search text]:text:" "(--sort)--sort"\{created,title\}:" "(--id)--id[item id]:id:" "(--store)--store[store path]:path:" "(--title)--title[new title]:" "(--content)--content[new content]:" "(--url)--url[source url]:" "(-t --tag)"{-t,--tag}"[tag]:tag:" "(--format)--format[json|jsonl]:" "(--completions)--completions[output completions]:shell:(bash zsh fish):" "(--purpose)--purpose[purpose]:" "(--model)--model[model ref]:" "(--dimensions)--dimensions[embedding dimensions]:number:" "(--no-color)--no-color[disable color]" "(--scope)--scope"\{local,global,project\}:" }; _open_knowledge`);
     } else if (shell === 'fish') {
-      console.log(`complete -c open-knowledge -f; complete -c open-knowledge -a "add list get update archive restore upsert untag delete export prune dedupe stats paths storage db wiki source ingest reindex providers safety help ls rm edit unarchive"; complete -c open-knowledge -l json; complete -c open-knowledge -l yes -s y; complete -c open-knowledge -l help -s h; complete -c open-knowledge -l version -s v; complete -c open-knowledge -l desc; complete -c open-knowledge -l archived; complete -c open-knowledge -l include-archived; complete -c open-knowledge -s p -l page; complete -c open-knowledge -s l -l limit; complete -c open-knowledge -s s -l search; complete -c open-knowledge -l sort; complete -c open-knowledge -l id; complete -c open-knowledge -l store; complete -c open-knowledge -l title; complete -c open-knowledge -l content; complete -c open-knowledge -l url; complete -c open-knowledge -s t -l tag; complete -c open-knowledge -l format; complete -c open-knowledge -l completions; complete -c open-knowledge -l purpose; complete -c open-knowledge -l no-color; complete -c open-knowledge -l scope -a "local global project"`);
+      console.log(`complete -c open-knowledge -f; complete -c open-knowledge -a "add list get update archive restore upsert untag delete export prune dedupe stats paths storage db wiki source ingest reindex embeddings providers safety help ls rm edit unarchive"; complete -c open-knowledge -l json; complete -c open-knowledge -l yes -s y; complete -c open-knowledge -l help -s h; complete -c open-knowledge -l version -s v; complete -c open-knowledge -l desc; complete -c open-knowledge -l archived; complete -c open-knowledge -l include-archived; complete -c open-knowledge -l fake; complete -c open-knowledge -s p -l page; complete -c open-knowledge -s l -l limit; complete -c open-knowledge -s s -l search; complete -c open-knowledge -l sort; complete -c open-knowledge -l id; complete -c open-knowledge -l store; complete -c open-knowledge -l title; complete -c open-knowledge -l content; complete -c open-knowledge -l url; complete -c open-knowledge -s t -l tag; complete -c open-knowledge -l format; complete -c open-knowledge -l completions; complete -c open-knowledge -l purpose; complete -c open-knowledge -l model; complete -c open-knowledge -l dimensions; complete -c open-knowledge -l no-color; complete -c open-knowledge -l scope -a "local global project"`);
     } else {
       throw new Error("Invalid --completions value. Use 'bash', 'zsh', or 'fish'.");
     }
@@ -534,6 +545,39 @@ async function run(argv: string[]): Promise<void> {
     return;
   }
 
+  if (command === 'embeddings') {
+    const action = positional[1] ?? 'status';
+    if (action === 'status') {
+      const result = service.embeddingStatus();
+      output({ ok: true, ...result, message: `${result.total_vector_entries} vector index entries` }, flags.json);
+      return;
+    }
+    if (action === 'index') {
+      const result = await service.indexEmbeddings({
+        limit: flags.limit,
+        modelRef: flags.model,
+        dimensions: flags.dimensions,
+        fake: flags.fake,
+      });
+      output({ ok: true, ...result, message: `Embedded ${result.chunks_embedded} chunk(s)` }, flags.json);
+      return;
+    }
+    if (action === 'search') {
+      const query = positional.slice(2).join(' ');
+      if (!query) throw new Error('Usage: open-knowledge embeddings search <query>');
+      const result = await service.semanticSearch({
+        query,
+        limit: flags.limit,
+        modelRef: flags.model,
+        dimensions: flags.dimensions,
+        fake: flags.fake,
+      });
+      output({ ok: true, ...result, message: `${result.results.length} semantic result(s)` }, flags.json);
+      return;
+    }
+    throw new Error("Invalid embeddings action. Use 'status', 'index', or 'search'.");
+  }
+
   if (command === 'providers') {
     const action = positional[1] ?? 'status';
     if (action === 'status') {