@hasna/knowledge 0.2.11 → 0.2.13

package/docs/architecture/ai-native-knowledge-base.md

@@ -89,6 +89,21 @@ file revisions, hashes, extraction state, permissions, and storage metadata.
 Direct `s3://`, `file://`, and `https://` refs are useful for bootstrap and
 interop, but should be normalized into source records when possible.
 
+## Provenance Contract
+
+Every durable search/wiki artifact should carry a provenance object in metadata:
+`source_owner`, `source_ref`, `source_uri`, `source_kind`, `source_revision_id`,
+`revision`, `hash`, optional `chunk_id`, offsets, `read_only`,
+`citation_required`, resolver name, and stale status. For generated artifacts
+that are not source-backed yet, metadata still records that `open-files` owns
+source bytes and that citations are required before durable facts are filed.
+
+`wiki init` now catalogs the starter `wiki/README.md` and `indexes/root.md`
+records with generated-artifact provenance. Source ingestion stores source
+provenance on every chunk, and source resolution returns that provenance with
+chunks and citations so semantic search can pass through trustworthy evidence
+without reconstructing it later.
+
 ## Resolver Boundary
 
 The local resolver is exposed through:
@@ -133,6 +148,18 @@ Raw files still route through `open-files`. Knowledge S3 storage is for derived
 artifacts such as wiki pages, index shards, schema versions, logs, exports, and
 run outputs.
 
+The storage contract is inspectable through:
+
+```bash
+open-knowledge storage status --scope project --json
+```
+
+That contract names the local app path, SQLite catalog, generated artifact
+classes, S3 bucket/prefix when configured, and the source ownership rule that
+raw source bytes stay in `open-files`. The `storage_objects` table catalogs
+generated artifacts by URI, kind, hash, size, and metadata so local mode and
+remote/S3 mode share the same DB-facing shape.
+
 ## Wiki Model
 
 The Karpathy-style wiki pattern is implemented as scalable artifacts, not three

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hasna/knowledge",
-  "version": "0.2.11",
+  "version": "0.2.13",
   "description": "Agent-friendly local knowledge CLI with JSON output, pagination, and safe destructive actions",
   "type": "module",
   "bin": {

package/src/cli.ts

@@ -60,7 +60,7 @@ interface ParseResult {
   flags: Flags;
 }
 
-const COMMANDS = ['add', 'list', 'get', 'delete', 'update', 'archive', 'restore', 'upsert', 'untag', 'export', 'prune', 'dedupe', 'stats', 'paths', 'db', 'wiki', 'source', 'ingest', 'reindex', 'providers', 'safety', 'help'];
+const COMMANDS = ['add', 'list', 'get', 'delete', 'update', 'archive', 'restore', 'upsert', 'untag', 'export', 'prune', 'dedupe', 'stats', 'paths', 'storage', 'db', 'wiki', 'source', 'ingest', 'reindex', 'providers', 'safety', 'help'];
 const COMMAND_ALIASES: Record<string, string> = {
   ls: 'list',
   rm: 'delete',
@@ -162,6 +162,7 @@ Commands:
   dedupe                       Remove duplicate items by title+content (requires --yes)
   stats                        Show knowledge base statistics
   paths                        Show resolved workspace/store paths
+  storage status|validate      Inspect local/S3 artifact storage contract
   db init|stats                Initialize or inspect local knowledge.db
   wiki init                    Initialize scalable wiki/schema/index/log artifacts
   source resolve <source-ref>  Resolve read-only source content and citation evidence
@@ -230,6 +231,7 @@ function printCommandHelp(command: string): void {
   if (command === 'dedupe') { console.log('Usage: open-knowledge dedupe --yes [--json]'); return; }
   if (command === 'stats') { console.log('Usage: open-knowledge stats [--json]'); return; }
   if (command === 'paths') { console.log('Usage: open-knowledge paths [--scope local|global|project] [--json]'); return; }
+  if (command === 'storage') { console.log('Usage: open-knowledge storage status|validate [--scope local|global|project] [--json]'); return; }
   if (command === 'db') { console.log('Usage: open-knowledge db init|stats [--scope local|global|project] [--json]'); return; }
   if (command === 'wiki') { console.log('Usage: open-knowledge wiki init [--scope local|global|project] [--json]'); return; }
   if (command === 'source') { console.log('Usage: open-knowledge source resolve <source-ref> [--purpose knowledge_answer|knowledge_index] [--limit <n>] [--scope local|global|project] [--json]'); return; }
@@ -281,11 +283,11 @@ async function run(argv: string[]): Promise<void> {
   if (flags.completions) {
     const shell = flags.completions;
     if (shell === 'bash') {
-      console.log(`_open_knowledge() { local cur; cur="${"$"}{COMP_WORDS[COMP_CWORD]}"; COMPREPLY=($(compgen -W "add list get update archive restore upsert untag delete export prune dedupe stats paths db wiki source ingest reindex providers safety help ls rm edit unarchive --json --yes --help --version --desc --page --limit --search --sort --id --store --title --content --url --tag --format --completions --purpose --no-color --scope --archived --include-archived" -- "$cur")); }; complete -F _open_knowledge open-knowledge`);
+      console.log(`_open_knowledge() { local cur; cur="${"$"}{COMP_WORDS[COMP_CWORD]}"; COMPREPLY=($(compgen -W "add list get update archive restore upsert untag delete export prune dedupe stats paths storage db wiki source ingest reindex providers safety help ls rm edit unarchive --json --yes --help --version --desc --page --limit --search --sort --id --store --title --content --url --tag --format --completions --purpose --no-color --scope --archived --include-archived" -- "$cur")); }; complete -F _open_knowledge open-knowledge`);
     } else if (shell === 'zsh') {
-      console.log(`#compdef open-knowledge\n_open_knowledge() { _arguments -C "1: :(add list get update archive restore upsert untag delete export prune dedupe stats paths db wiki source ingest reindex providers safety help ls rm edit unarchive)" "(--json)--json" "(--yes)-y" "(--help)--help" "(--version)--version" "(--desc)--desc" "(--archived)--archived" "(--include-archived)--include-archived" "(-p --page)"{-p,--page}"[page number]:number:" "(-l --limit)"{-l,--limit}"[items per page]:number:" "(-s --search)"{-s,--search}"[search text]:text:" "(--sort)--sort"\{created,title\}:" "(--id)--id[item id]:id:" "(--store)--store[store path]:path:" "(--title)--title[new title]:" "(--content)--content[new content]:" "(--url)--url[source url]:" "(-t --tag)"{-t,--tag}"[tag]:tag:" "(--format)--format[json|jsonl]:" "(--completions)--completions[output completions]:shell:(bash zsh fish):" "(--purpose)--purpose[purpose]:" "(--no-color)--no-color[disable color]" "(--scope)--scope"\{local,global,project\}:" }; _open_knowledge`);
+      console.log(`#compdef open-knowledge\n_open_knowledge() { _arguments -C "1: :(add list get update archive restore upsert untag delete export prune dedupe stats paths storage db wiki source ingest reindex providers safety help ls rm edit unarchive)" "(--json)--json" "(--yes)-y" "(--help)--help" "(--version)--version" "(--desc)--desc" "(--archived)--archived" "(--include-archived)--include-archived" "(-p --page)"{-p,--page}"[page number]:number:" "(-l --limit)"{-l,--limit}"[items per page]:number:" "(-s --search)"{-s,--search}"[search text]:text:" "(--sort)--sort"\{created,title\}:" "(--id)--id[item id]:id:" "(--store)--store[store path]:path:" "(--title)--title[new title]:" "(--content)--content[new content]:" "(--url)--url[source url]:" "(-t --tag)"{-t,--tag}"[tag]:tag:" "(--format)--format[json|jsonl]:" "(--completions)--completions[output completions]:shell:(bash zsh fish):" "(--purpose)--purpose[purpose]:" "(--no-color)--no-color[disable color]" "(--scope)--scope"\{local,global,project\}:" }; _open_knowledge`);
     } else if (shell === 'fish') {
-      console.log(`complete -c open-knowledge -f; complete -c open-knowledge -a "add list get update archive restore upsert untag delete export prune dedupe stats paths db wiki source ingest reindex providers safety help ls rm edit unarchive"; complete -c open-knowledge -l json; complete -c open-knowledge -l yes -s y; complete -c open-knowledge -l help -s h; complete -c open-knowledge -l version -s v; complete -c open-knowledge -l desc; complete -c open-knowledge -l archived; complete -c open-knowledge -l include-archived; complete -c open-knowledge -s p -l page; complete -c open-knowledge -s l -l limit; complete -c open-knowledge -s s -l search; complete -c open-knowledge -l sort; complete -c open-knowledge -l id; complete -c open-knowledge -l store; complete -c open-knowledge -l title; complete -c open-knowledge -l content; complete -c open-knowledge -l url; complete -c open-knowledge -s t -l tag; complete -c open-knowledge -l format; complete -c open-knowledge -l completions; complete -c open-knowledge -l purpose; complete -c open-knowledge -l no-color; complete -c open-knowledge -l scope -a "local global project"`);
+      console.log(`complete -c open-knowledge -f; complete -c open-knowledge -a "add list get update archive restore upsert untag delete export prune dedupe stats paths storage db wiki source ingest reindex providers safety help ls rm edit unarchive"; complete -c open-knowledge -l json; complete -c open-knowledge -l yes -s y; complete -c open-knowledge -l help -s h; complete -c open-knowledge -l version -s v; complete -c open-knowledge -l desc; complete -c open-knowledge -l archived; complete -c open-knowledge -l include-archived; complete -c open-knowledge -s p -l page; complete -c open-knowledge -s l -l limit; complete -c open-knowledge -s s -l search; complete -c open-knowledge -l sort; complete -c open-knowledge -l id; complete -c open-knowledge -l store; complete -c open-knowledge -l title; complete -c open-knowledge -l content; complete -c open-knowledge -l url; complete -c open-knowledge -s t -l tag; complete -c open-knowledge -l format; complete -c open-knowledge -l completions; complete -c open-knowledge -l purpose; complete -c open-knowledge -l no-color; complete -c open-knowledge -l scope -a "local global project"`);
     } else {
       throw new Error("Invalid --completions value. Use 'bash', 'zsh', or 'fish'.");
     }
@@ -311,6 +313,31 @@ async function run(argv: string[]): Promise<void> {
     return;
   }
 
+  if (command === 'storage') {
+    const action = positional[1] ?? 'status';
+    if (action === 'status') {
+      const contract = service.storageContract();
+      const validation = service.validateStorage();
+      output({
+        ok: validation.ok,
+        ...contract,
+        validation,
+        message: `${contract.storage_type} artifact storage at ${contract.artifact_store.uri_prefix}`,
+      }, flags.json);
+      return;
+    }
+    if (action === 'validate') {
+      const validation = service.validateStorage();
+      output({
+        ok: validation.ok,
+        validation,
+        message: validation.ok ? 'Storage contract valid' : `Storage contract invalid: ${validation.errors.join('; ')}`,
+      }, flags.json);
+      return;
+    }
+    throw new Error("Invalid storage action. Use 'status' or 'validate'.");
+  }
+
   if (command === 'db') {
     const action = positional[1] ?? 'init';
     if (action !== 'init' && action !== 'stats') {

package/src/knowledge-db.ts

@@ -16,6 +16,7 @@ export interface KnowledgeDbStats {
   redaction_findings: number;
   audit_events: number;
   approval_gates: number;
+  storage_objects: number;
 }
 
 const MIGRATION_1 = `
@@ -239,6 +240,7 @@ export function openKnowledgeDb(path: string): Database {
   ensureParentDir(path);
   const db = new Database(path);
   db.exec('PRAGMA foreign_keys = ON;');
+  db.exec('PRAGMA busy_timeout = 5000;');
   return db;
 }
 
@@ -280,6 +282,7 @@ export function getKnowledgeDbStats(path: string): KnowledgeDbStats {
       redaction_findings: count(db, 'redaction_findings'),
       audit_events: count(db, 'audit_events'),
       approval_gates: count(db, 'approval_gates'),
+      storage_objects: count(db, 'storage_objects'),
     };
   } finally {
     db.close();

package/src/manifest-ingest.ts

@@ -4,6 +4,7 @@ import { basename } from 'node:path';
 import type { Database } from 'bun:sqlite';
 import { migrateKnowledgeDb, openKnowledgeDb } from './knowledge-db';
 import { parseSourceRef, type SourceRef } from './source-ref';
+import { sourceProvenance, withProvenance } from './provenance';
 import type { KnowledgeConfig } from './workspace';
 import {
   assertS3ReadAllowed,
@@ -382,15 +383,31 @@ function insertChunks(db: Database, sourceRevisionId: string, item: NormalizedMa
   const chunks = chunkText(redacted.text, maxChars, overlapChars);
   for (const chunk of chunks) {
     const chunkId = stableId('chk', `${sourceRevisionId}\u0000${chunk.ordinal}\u0000${chunk.text}`);
-    const metadata = {
+    const provenance = sourceProvenance({
       source_ref: item.sourceRef,
       source_uri: item.sourceUri,
+      source_kind: item.kind,
+      source_revision_id: sourceRevisionId,
+      revision: item.revision,
+      hash: item.hash,
+      chunk_id: chunkId,
+      start_offset: chunk.startOffset,
+      end_offset: chunk.endOffset,
+      status: item.status,
+      resolver: 'open-files-read-only',
+    });
+    const metadata = withProvenance({
+      source_ref: item.sourceRef,
+      source_uri: item.sourceUri,
+      source_kind: item.kind,
+      source_revision_id: sourceRevisionId,
+      revision: item.revision,
       hash: item.hash,
       status: item.status,
       path: asString(item.raw.path) ?? null,
       mime: asString(item.raw.mime) ?? asString(item.raw.content_type) ?? null,
       size: asNumber(item.raw.size) ?? null,
-    };
+    }, provenance);
     db.run(
       `INSERT INTO chunks (id, source_revision_id, kind, ordinal, text, token_count, start_offset, end_offset, metadata_json, created_at)
        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,

package/src/mcp.js

@@ -77,6 +77,18 @@ export function buildServer() {
     return jsonText(createKnowledgeService({ scope }).paths());
   });
 
+  registerTool(server, 'ok_storage_status', 'Knowledge storage status', 'Inspect local/S3 artifact storage, source ownership, and scalability contract', {
+    scope: scopeField,
+  }, async ({ scope }) => {
+    const service = createKnowledgeService({ scope });
+    const validation = service.validateStorage();
+    return jsonText({
+      ok: validation.ok,
+      ...service.storageContract(),
+      validation,
+    });
+  });
+
   registerTool(server, 'ok_parse_source_ref', 'Parse source reference', 'Parse and validate an open-files, S3, file, or web source ref', {
     uri: z.string().describe('Source reference URI'),
   }, async ({ uri }) => {

package/src/provenance.ts

@@ -0,0 +1,93 @@
+export interface KnowledgeProvenance {
+  source_owner: 'open-files';
+  source_ref: string | null;
+  source_uri: string | null;
+  source_kind: string | null;
+  source_revision_id: string | null;
+  revision: string | null;
+  hash: string | null;
+  chunk_id: string | null;
+  start_offset: number | null;
+  end_offset: number | null;
+  status: string | null;
+  read_only: true;
+  citation_required: boolean;
+  resolver: string | null;
+  stale: boolean;
+}
+
+export interface GeneratedArtifactProvenance {
+  source_owner: 'open-files';
+  generated_from: string;
+  artifact_key: string;
+  source_refs: string[];
+  read_only_sources: true;
+  citation_required: boolean;
+  raw_source_bytes_stored_in_open_knowledge: false;
+}
+
+export interface SourceProvenanceInput {
+  source_ref?: string | null;
+  source_uri?: string | null;
+  source_kind?: string | null;
+  source_revision_id?: string | null;
+  revision?: string | null;
+  hash?: string | null;
+  chunk_id?: string | null;
+  start_offset?: number | null;
+  end_offset?: number | null;
+  status?: string | null;
+  resolver?: string | null;
+}
+
+export function isStaleStatus(status: string | null | undefined): boolean {
+  return ['deleted', 'stale', 'invalidated', 'reindex_required'].includes((status ?? '').toLowerCase());
+}
+
+export function sourceProvenance(input: SourceProvenanceInput): KnowledgeProvenance {
+  const status = input.status ?? null;
+  return {
+    source_owner: 'open-files',
+    source_ref: input.source_ref ?? null,
+    source_uri: input.source_uri ?? null,
+    source_kind: input.source_kind ?? null,
+    source_revision_id: input.source_revision_id ?? null,
+    revision: input.revision ?? null,
+    hash: input.hash ?? null,
+    chunk_id: input.chunk_id ?? null,
+    start_offset: input.start_offset ?? null,
+    end_offset: input.end_offset ?? null,
+    status,
+    read_only: true,
+    citation_required: true,
+    resolver: input.resolver ?? null,
+    stale: isStaleStatus(status),
+  };
+}
+
+export function generatedArtifactProvenance(input: {
+  generated_from: string;
+  artifact_key: string;
+  source_refs?: string[];
+  citation_required?: boolean;
+}): GeneratedArtifactProvenance {
+  return {
+    source_owner: 'open-files',
+    generated_from: input.generated_from,
+    artifact_key: input.artifact_key,
+    source_refs: input.source_refs ?? [],
+    read_only_sources: true,
+    citation_required: input.citation_required ?? true,
+    raw_source_bytes_stored_in_open_knowledge: false,
+  };
+}
+
+export function withProvenance<T extends Record<string, unknown>>(
+  metadata: T,
+  provenance: KnowledgeProvenance | GeneratedArtifactProvenance,
+): T & { provenance: KnowledgeProvenance | GeneratedArtifactProvenance } {
+  return {
+    ...metadata,
+    provenance,
+  };
+}

package/src/service.ts

@@ -1,12 +1,19 @@
 import { createArtifactStore } from './artifact-store';
 import { consumeOpenFilesOutbox } from './outbox-consume';
-import { getKnowledgeDbStats, migrateKnowledgeDb } from './knowledge-db';
+import { getKnowledgeDbStats, migrateKnowledgeDb, openKnowledgeDb } from './knowledge-db';
 import { ingestOpenFilesManifest } from './manifest-ingest';
 import { ingestSourceRef } from './source-ingest';
 import { resolveOpenFilesSource } from './source-resolver';
 import { providerStatus, listModelRegistry, type ProviderStatusResult, type ModelRegistryEntry } from './providers';
 import { resolveSafetyPolicy } from './safety';
-import { initializeWikiLayout } from './wiki-layout';
+import {
+  recordStorageObjects,
+  resolveStorageContract,
+  validateStorageConfig,
+  type StorageContract,
+  type StorageValidationResult,
+} from './storage-contract';
+import { initializeWikiLayout, recordWikiLayoutCatalog } from './wiki-layout';
 import {
   ensureKnowledgeWorkspace,
   readKnowledgeConfig,
@@ -76,6 +83,14 @@ export class KnowledgeService {
     return createArtifactStore(this.config(), this.ensureWorkspace());
   }
 
+  storageContract(): StorageContract {
+    return resolveStorageContract(this.config(), this.ensureWorkspace(), this.scope);
+  }
+
+  validateStorage(): StorageValidationResult {
+    return validateStorageConfig(this.config(), this.ensureWorkspace());
+  }
+
   paths(): KnowledgePathsResult {
     const workspace = this.ensureWorkspace();
     return {
@@ -107,7 +122,17 @@ export class KnowledgeService {
   }
 
   async initWiki() {
-    return initializeWikiLayout(this.artifactStore());
+    const workspace = this.ensureWorkspace();
+    migrateKnowledgeDb(workspace.knowledgeDbPath);
+    const result = await initializeWikiLayout(this.artifactStore());
+    const db = openKnowledgeDb(workspace.knowledgeDbPath);
+    try {
+      recordStorageObjects(db, result.artifacts);
+      recordWikiLayoutCatalog(db, result.artifacts);
+    } finally {
+      db.close();
+    }
+    return result;
   }
 
   async ingestManifest(input: string) {

package/src/source-resolver.ts

@@ -1,5 +1,6 @@
 import type { Database } from 'bun:sqlite';
 import { migrateKnowledgeDb, openKnowledgeDb } from './knowledge-db';
+import { sourceProvenance, type KnowledgeProvenance } from './provenance';
 import { catalogSourceUriForRef, parseSourceRef, revisionIdForSourceRef } from './source-ref';
 import { assertWriteAllowed, recordAuditEvent, type SafetyPolicy } from './safety';
 
@@ -38,6 +39,7 @@ export interface ResolvedSourceChunk {
   end_offset: number | null;
   metadata: Record<string, unknown>;
   evidence: SourceResolverEvidence;
+  provenance: KnowledgeProvenance;
 }
 
 export interface ResolvedSourceCitation {
@@ -48,6 +50,7 @@ export interface ResolvedSourceCitation {
   start_offset: number | null;
   end_offset: number | null;
   evidence: SourceResolverEvidence;
+  provenance: KnowledgeProvenance;
 }
 
 export interface SourceResolveResult {
@@ -326,6 +329,19 @@ export async function resolveOpenFilesSource(options: SourceResolveOptions): Pro
           end_offset: row.end_offset,
           resolved_at: resolvedAt,
         };
+        const provenance = sourceProvenance({
+          source_ref: evidence.source_ref,
+          source_uri: evidence.source_uri,
+          source_kind: source.kind,
+          source_revision_id: evidence.source_revision_id,
+          revision: evidence.revision,
+          hash: evidence.hash,
+          chunk_id: row.id,
+          start_offset: row.start_offset,
+          end_offset: row.end_offset,
+          status: metadataString(metadata, ['status']),
+          resolver: evidence.resolver,
+        });
         return {
           id: row.id,
           kind: row.kind,
@@ -336,6 +352,7 @@ export async function resolveOpenFilesSource(options: SourceResolveOptions): Pro
           end_offset: row.end_offset,
           metadata,
           evidence,
+          provenance,
         };
       });
 
@@ -347,6 +364,7 @@ export async function resolveOpenFilesSource(options: SourceResolveOptions): Pro
         start_offset: chunk.start_offset,
         end_offset: chunk.end_offset,
         evidence: chunk.evidence,
+        provenance: chunk.provenance,
       }));
 
       recordAuditEvent(db, {