@hasna/knowledge 0.2.11 → 0.2.12

package/src/cli.ts

@@ -60,7 +60,7 @@ interface ParseResult {
   flags: Flags;
 }
 
-const COMMANDS = ['add', 'list', 'get', 'delete', 'update', 'archive', 'restore', 'upsert', 'untag', 'export', 'prune', 'dedupe', 'stats', 'paths', 'db', 'wiki', 'source', 'ingest', 'reindex', 'providers', 'safety', 'help'];
+const COMMANDS = ['add', 'list', 'get', 'delete', 'update', 'archive', 'restore', 'upsert', 'untag', 'export', 'prune', 'dedupe', 'stats', 'paths', 'storage', 'db', 'wiki', 'source', 'ingest', 'reindex', 'providers', 'safety', 'help'];
 const COMMAND_ALIASES: Record<string, string> = {
   ls: 'list',
   rm: 'delete',
@@ -162,6 +162,7 @@ Commands:
   dedupe                       Remove duplicate items by title+content (requires --yes)
   stats                        Show knowledge base statistics
   paths                        Show resolved workspace/store paths
+  storage status|validate      Inspect local/S3 artifact storage contract
   db init|stats                Initialize or inspect local knowledge.db
   wiki init                    Initialize scalable wiki/schema/index/log artifacts
   source resolve <source-ref>  Resolve read-only source content and citation evidence
@@ -230,6 +231,7 @@ function printCommandHelp(command: string): void {
   if (command === 'dedupe') { console.log('Usage: open-knowledge dedupe --yes [--json]'); return; }
   if (command === 'stats') { console.log('Usage: open-knowledge stats [--json]'); return; }
   if (command === 'paths') { console.log('Usage: open-knowledge paths [--scope local|global|project] [--json]'); return; }
+  if (command === 'storage') { console.log('Usage: open-knowledge storage status|validate [--scope local|global|project] [--json]'); return; }
   if (command === 'db') { console.log('Usage: open-knowledge db init|stats [--scope local|global|project] [--json]'); return; }
   if (command === 'wiki') { console.log('Usage: open-knowledge wiki init [--scope local|global|project] [--json]'); return; }
   if (command === 'source') { console.log('Usage: open-knowledge source resolve <source-ref> [--purpose knowledge_answer|knowledge_index] [--limit <n>] [--scope local|global|project] [--json]'); return; }
@@ -281,11 +283,11 @@ async function run(argv: string[]): Promise<void> {
   if (flags.completions) {
     const shell = flags.completions;
     if (shell === 'bash') {
-      console.log(`_open_knowledge() { local cur; cur="${"$"}{COMP_WORDS[COMP_CWORD]}"; COMPREPLY=($(compgen -W "add list get update archive restore upsert untag delete export prune dedupe stats paths db wiki source ingest reindex providers safety help ls rm edit unarchive --json --yes --help --version --desc --page --limit --search --sort --id --store --title --content --url --tag --format --completions --purpose --no-color --scope --archived --include-archived" -- "$cur")); }; complete -F _open_knowledge open-knowledge`);
+      console.log(`_open_knowledge() { local cur; cur="${"$"}{COMP_WORDS[COMP_CWORD]}"; COMPREPLY=($(compgen -W "add list get update archive restore upsert untag delete export prune dedupe stats paths storage db wiki source ingest reindex providers safety help ls rm edit unarchive --json --yes --help --version --desc --page --limit --search --sort --id --store --title --content --url --tag --format --completions --purpose --no-color --scope --archived --include-archived" -- "$cur")); }; complete -F _open_knowledge open-knowledge`);
     } else if (shell === 'zsh') {
-      console.log(`#compdef open-knowledge\n_open_knowledge() { _arguments -C "1: :(add list get update archive restore upsert untag delete export prune dedupe stats paths db wiki source ingest reindex providers safety help ls rm edit unarchive)" "(--json)--json" "(--yes)-y" "(--help)--help" "(--version)--version" "(--desc)--desc" "(--archived)--archived" "(--include-archived)--include-archived" "(-p --page)"{-p,--page}"[page number]:number:" "(-l --limit)"{-l,--limit}"[items per page]:number:" "(-s --search)"{-s,--search}"[search text]:text:" "(--sort)--sort"\{created,title\}:" "(--id)--id[item id]:id:" "(--store)--store[store path]:path:" "(--title)--title[new title]:" "(--content)--content[new content]:" "(--url)--url[source url]:" "(-t --tag)"{-t,--tag}"[tag]:tag:" "(--format)--format[json|jsonl]:" "(--completions)--completions[output completions]:shell:(bash zsh fish):" "(--purpose)--purpose[purpose]:" "(--no-color)--no-color[disable color]" "(--scope)--scope"\{local,global,project\}:" }; _open_knowledge`);
+      console.log(`#compdef open-knowledge\n_open_knowledge() { _arguments -C "1: :(add list get update archive restore upsert untag delete export prune dedupe stats paths storage db wiki source ingest reindex providers safety help ls rm edit unarchive)" "(--json)--json" "(--yes)-y" "(--help)--help" "(--version)--version" "(--desc)--desc" "(--archived)--archived" "(--include-archived)--include-archived" "(-p --page)"{-p,--page}"[page number]:number:" "(-l --limit)"{-l,--limit}"[items per page]:number:" "(-s --search)"{-s,--search}"[search text]:text:" "(--sort)--sort"\{created,title\}:" "(--id)--id[item id]:id:" "(--store)--store[store path]:path:" "(--title)--title[new title]:" "(--content)--content[new content]:" "(--url)--url[source url]:" "(-t --tag)"{-t,--tag}"[tag]:tag:" "(--format)--format[json|jsonl]:" "(--completions)--completions[output completions]:shell:(bash zsh fish):" "(--purpose)--purpose[purpose]:" "(--no-color)--no-color[disable color]" "(--scope)--scope"\{local,global,project\}:" }; _open_knowledge`);
     } else if (shell === 'fish') {
-      console.log(`complete -c open-knowledge -f; complete -c open-knowledge -a "add list get update archive restore upsert untag delete export prune dedupe stats paths db wiki source ingest reindex providers safety help ls rm edit unarchive"; complete -c open-knowledge -l json; complete -c open-knowledge -l yes -s y; complete -c open-knowledge -l help -s h; complete -c open-knowledge -l version -s v; complete -c open-knowledge -l desc; complete -c open-knowledge -l archived; complete -c open-knowledge -l include-archived; complete -c open-knowledge -s p -l page; complete -c open-knowledge -s l -l limit; complete -c open-knowledge -s s -l search; complete -c open-knowledge -l sort; complete -c open-knowledge -l id; complete -c open-knowledge -l store; complete -c open-knowledge -l title; complete -c open-knowledge -l content; complete -c open-knowledge -l url; complete -c open-knowledge -s t -l tag; complete -c open-knowledge -l format; complete -c open-knowledge -l completions; complete -c open-knowledge -l purpose; complete -c open-knowledge -l no-color; complete -c open-knowledge -l scope -a "local global project"`);
+      console.log(`complete -c open-knowledge -f; complete -c open-knowledge -a "add list get update archive restore upsert untag delete export prune dedupe stats paths storage db wiki source ingest reindex providers safety help ls rm edit unarchive"; complete -c open-knowledge -l json; complete -c open-knowledge -l yes -s y; complete -c open-knowledge -l help -s h; complete -c open-knowledge -l version -s v; complete -c open-knowledge -l desc; complete -c open-knowledge -l archived; complete -c open-knowledge -l include-archived; complete -c open-knowledge -s p -l page; complete -c open-knowledge -s l -l limit; complete -c open-knowledge -s s -l search; complete -c open-knowledge -l sort; complete -c open-knowledge -l id; complete -c open-knowledge -l store; complete -c open-knowledge -l title; complete -c open-knowledge -l content; complete -c open-knowledge -l url; complete -c open-knowledge -s t -l tag; complete -c open-knowledge -l format; complete -c open-knowledge -l completions; complete -c open-knowledge -l purpose; complete -c open-knowledge -l no-color; complete -c open-knowledge -l scope -a "local global project"`);
     } else {
       throw new Error("Invalid --completions value. Use 'bash', 'zsh', or 'fish'.");
     }
@@ -311,6 +313,31 @@ async function run(argv: string[]): Promise<void> {
     return;
   }
 
+  if (command === 'storage') {
+    const action = positional[1] ?? 'status';
+    if (action === 'status') {
+      const contract = service.storageContract();
+      const validation = service.validateStorage();
+      output({
+        ok: validation.ok,
+        ...contract,
+        validation,
+        message: `${contract.storage_type} artifact storage at ${contract.artifact_store.uri_prefix}`,
+      }, flags.json);
+      return;
+    }
+    if (action === 'validate') {
+      const validation = service.validateStorage();
+      output({
+        ok: validation.ok,
+        validation,
+        message: validation.ok ? 'Storage contract valid' : `Storage contract invalid: ${validation.errors.join('; ')}`,
+      }, flags.json);
+      return;
+    }
+    throw new Error("Invalid storage action. Use 'status' or 'validate'.");
+  }
+
   if (command === 'db') {
     const action = positional[1] ?? 'init';
     if (action !== 'init' && action !== 'stats') {

package/src/knowledge-db.ts

@@ -16,6 +16,7 @@ export interface KnowledgeDbStats {
   redaction_findings: number;
   audit_events: number;
   approval_gates: number;
+  storage_objects: number;
 }
 
 const MIGRATION_1 = `
@@ -280,6 +281,7 @@ export function getKnowledgeDbStats(path: string): KnowledgeDbStats {
       redaction_findings: count(db, 'redaction_findings'),
       audit_events: count(db, 'audit_events'),
       approval_gates: count(db, 'approval_gates'),
+      storage_objects: count(db, 'storage_objects'),
     };
   } finally {
     db.close();

package/src/mcp.js

@@ -77,6 +77,18 @@ export function buildServer() {
     return jsonText(createKnowledgeService({ scope }).paths());
   });
 
+  registerTool(server, 'ok_storage_status', 'Knowledge storage status', 'Inspect local/S3 artifact storage, source ownership, and scalability contract', {
+    scope: scopeField,
+  }, async ({ scope }) => {
+    const service = createKnowledgeService({ scope });
+    const validation = service.validateStorage();
+    return jsonText({
+      ok: validation.ok,
+      ...service.storageContract(),
+      validation,
+    });
+  });
+
   registerTool(server, 'ok_parse_source_ref', 'Parse source reference', 'Parse and validate an open-files, S3, file, or web source ref', {
     uri: z.string().describe('Source reference URI'),
   }, async ({ uri }) => {

package/src/service.ts

@@ -1,11 +1,18 @@
 import { createArtifactStore } from './artifact-store';
 import { consumeOpenFilesOutbox } from './outbox-consume';
-import { getKnowledgeDbStats, migrateKnowledgeDb } from './knowledge-db';
+import { getKnowledgeDbStats, migrateKnowledgeDb, openKnowledgeDb } from './knowledge-db';
 import { ingestOpenFilesManifest } from './manifest-ingest';
 import { ingestSourceRef } from './source-ingest';
 import { resolveOpenFilesSource } from './source-resolver';
 import { providerStatus, listModelRegistry, type ProviderStatusResult, type ModelRegistryEntry } from './providers';
 import { resolveSafetyPolicy } from './safety';
+import {
+  recordStorageObjects,
+  resolveStorageContract,
+  validateStorageConfig,
+  type StorageContract,
+  type StorageValidationResult,
+} from './storage-contract';
 import { initializeWikiLayout } from './wiki-layout';
 import {
   ensureKnowledgeWorkspace,
@@ -76,6 +83,14 @@ export class KnowledgeService {
     return createArtifactStore(this.config(), this.ensureWorkspace());
   }
 
+  storageContract(): StorageContract {
+    return resolveStorageContract(this.config(), this.ensureWorkspace(), this.scope);
+  }
+
+  validateStorage(): StorageValidationResult {
+    return validateStorageConfig(this.config(), this.ensureWorkspace());
+  }
+
   paths(): KnowledgePathsResult {
     const workspace = this.ensureWorkspace();
     return {
@@ -107,7 +122,16 @@ export class KnowledgeService {
   }
 
   async initWiki() {
-    return initializeWikiLayout(this.artifactStore());
+    const workspace = this.ensureWorkspace();
+    migrateKnowledgeDb(workspace.knowledgeDbPath);
+    const result = await initializeWikiLayout(this.artifactStore());
+    const db = openKnowledgeDb(workspace.knowledgeDbPath);
+    try {
+      recordStorageObjects(db, result.artifacts);
+    } finally {
+      db.close();
+    }
+    return result;
   }
 
   async ingestManifest(input: string) {

package/src/storage-contract.ts

@@ -0,0 +1,265 @@
+import { createHash, randomUUID } from 'node:crypto';
+import type { Database } from 'bun:sqlite';
+import type { KnowledgeConfig, KnowledgeWorkspace } from './workspace';
+import { HASNA_KNOWLEDGE_APP_PATH } from './workspace';
+
+export interface StorageArtifactClass {
+  kind: string;
+  prefix: string;
+  description: string;
+}
+
+export interface StorageContract {
+  scope: string;
+  mode: KnowledgeConfig['mode'];
+  storage_type: KnowledgeConfig['storage']['type'];
+  workspace_home: string;
+  local_layout: {
+    app_path: string;
+    config_path: string;
+    json_store_path: string;
+    knowledge_db_path: string;
+    directories: Record<string, string>;
+  };
+  artifact_store: {
+    type: KnowledgeConfig['storage']['type'];
+    artifacts_root: string;
+    uri_prefix: string;
+    s3: {
+      bucket: string;
+      prefix: string;
+      region: string | null;
+      profile: string | null;
+      server_side_encryption: string | null;
+      kms_key_configured: boolean;
+    } | null;
+  };
+  source_ownership: {
+    owner: 'open-files';
+    preferred_ref: string;
+    allowed_schemes: string[];
+    raw_source_bytes_stored_in_open_knowledge: false;
+    stores: string[];
+    does_not_store: string[];
+  };
+  generated_artifacts: StorageArtifactClass[];
+  scalability: {
+    catalog: string;
+    indexes: string;
+    logs: string;
+    markdown: string;
+  };
+  warnings: string[];
+}
+
+export interface StorageValidationResult {
+  ok: boolean;
+  errors: string[];
+  warnings: string[];
+}
+
+export interface GeneratedStorageObject {
+  uri: string;
+  key: string;
+  kind: string;
+  content_type?: string;
+  hash?: string;
+  size_bytes?: number;
+  metadata?: Record<string, unknown>;
+}
+
+const GENERATED_ARTIFACTS: StorageArtifactClass[] = [
+  {
+    kind: 'schema',
+    prefix: 'schemas/',
+    description: 'Machine-readable agent schemas and source rules.',
+  },
+  {
+    kind: 'index',
+    prefix: 'indexes/',
+    description: 'Small orientation indexes and future shard manifests.',
+  },
+  {
+    kind: 'log',
+    prefix: 'logs/',
+    description: 'Append-only JSONL run and wiki-maintenance log partitions.',
+  },
+  {
+    kind: 'run',
+    prefix: 'runs/',
+    description: 'Prompt/tool/cost ledgers and generated output records.',
+  },
+  {
+    kind: 'wiki_page',
+    prefix: 'wiki/',
+    description: 'Generated cited Markdown pages, not raw source files.',
+  },
+  {
+    kind: 'export',
+    prefix: 'exports/',
+    description: 'Portable exports and snapshots of derived knowledge state.',
+  },
+];
+
+export function hashArtifactBody(body: string | Uint8Array): { hash: string; size_bytes: number } {
+  const bytes = typeof body === 'string' ? Buffer.from(body) : Buffer.from(body);
+  return {
+    hash: `sha256:${createHash('sha256').update(bytes).digest('hex')}`,
+    size_bytes: bytes.byteLength,
+  };
+}
+
+export function artifactKindForKey(key: string): string {
+  const match = GENERATED_ARTIFACTS.find((entry) => key.startsWith(entry.prefix));
+  return match?.kind ?? 'artifact';
+}
+
+export function resolveStorageContract(
+  config: KnowledgeConfig,
+  workspace: KnowledgeWorkspace,
+  scope = 'global',
+): StorageContract {
+  const validation = validateStorageConfig(config, workspace);
+  const s3 = config.storage.s3 ?? null;
+  const prefix = s3?.prefix?.replace(/^\/+|\/+$/g, '') ?? '';
+  const s3UriPrefix = s3 ? `s3://${s3.bucket}/${prefix ? `${prefix}/` : ''}` : '';
+
+  return {
+    scope,
+    mode: config.mode,
+    storage_type: config.storage.type,
+    workspace_home: workspace.home,
+    local_layout: {
+      app_path: HASNA_KNOWLEDGE_APP_PATH,
+      config_path: workspace.configPath,
+      json_store_path: workspace.jsonStorePath,
+      knowledge_db_path: workspace.knowledgeDbPath,
+      directories: {
+        artifacts: workspace.artifactsDir,
+        cache: workspace.cacheDir,
+        exports: workspace.exportsDir,
+        indexes: workspace.indexesDir,
+        logs: workspace.logsDir,
+        runs: workspace.runsDir,
+        schemas: workspace.schemasDir,
+        wiki: workspace.wikiDir,
+      },
+    },
+    artifact_store: {
+      type: config.storage.type,
+      artifacts_root: config.storage.artifacts_root,
+      uri_prefix: config.storage.type === 's3' ? s3UriPrefix : `file://${workspace.artifactsDir}/`,
+      s3: s3
+        ? {
+            bucket: s3.bucket,
+            prefix,
+            region: s3.region ?? null,
+            profile: s3.profile ?? null,
+            server_side_encryption: s3.server_side_encryption ?? null,
+            kms_key_configured: Boolean(s3.kms_key_id),
+          }
+        : null,
+    },
+    source_ownership: {
+      owner: 'open-files',
+      preferred_ref: config.sources.preferred_ref,
+      allowed_schemes: config.sources.allowed_schemes,
+      raw_source_bytes_stored_in_open_knowledge: false,
+      stores: [
+        'source refs',
+        'source revisions and hashes',
+        'citation spans',
+        'redacted extracted chunks',
+        'embeddings',
+        'generated wiki artifacts',
+        'indexes',
+        'run ledgers',
+      ],
+      does_not_store: [
+        'raw open-files bytes',
+        'S3 object credentials',
+        'connector secrets',
+        'hosted tenant ownership state',
+      ],
+    },
+    generated_artifacts: GENERATED_ARTIFACTS,
+    scalability: {
+      catalog: 'knowledge.db tracks sources, revisions, chunks, citations, indexes, runs, and storage_objects.',
+      indexes: 'Indexes are cataloged DB rows plus sharded artifacts, not one giant index.md.',
+      logs: 'Logs use dated JSONL partitions under logs/yyyy/mm/dd.jsonl.',
+      markdown: 'Markdown pages are the readable wiki layer over DB/object-store state.',
+    },
+    warnings: validation.warnings,
+  };
+}
+
+export function validateStorageConfig(config: KnowledgeConfig, workspace: KnowledgeWorkspace): StorageValidationResult {
+  const errors: string[] = [];
+  const warnings: string[] = [];
+
+  if (!workspace.home.endsWith(HASNA_KNOWLEDGE_APP_PATH)) {
+    warnings.push(`Workspace home does not end with ${HASNA_KNOWLEDGE_APP_PATH}: ${workspace.home}`);
+  }
+
+  if (config.storage.type === 's3') {
+    if (!config.storage.s3?.bucket) errors.push('storage.s3.bucket is required when storage.type is s3.');
+    if (!config.storage.s3?.prefix) warnings.push('storage.s3.prefix is empty; generated knowledge artifacts will be written at the bucket root.');
+    if (config.mode === 'local') warnings.push('storage.type is s3 while mode is local; this is valid for BYO S3, but hosted wrappers should set mode to hosted.');
+  }
+
+  if (config.storage.type === 'local' && config.storage.s3) {
+    warnings.push('storage.s3 is configured but ignored while storage.type is local.');
+  }
+
+  if (config.sources.preferred_ref !== 'open-files') {
+    warnings.push('sources.preferred_ref should stay open-files for durable company knowledge.');
+  }
+
+  if (!config.sources.allowed_schemes.includes('open-files')) {
+    errors.push('sources.allowed_schemes must include open-files.');
+  }
+
+  return {
+    ok: errors.length === 0,
+    errors,
+    warnings,
+  };
+}
+
+export function recordStorageObjects(db: Database, objects: GeneratedStorageObject[], now = new Date()): void {
+  const timestamp = now.toISOString();
+  const statement = db.prepare(`
+    INSERT INTO storage_objects (
+      id, artifact_uri, kind, content_type, hash, size_bytes, metadata_json, created_at, updated_at
+    )
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+    ON CONFLICT(artifact_uri) DO UPDATE SET
+      kind = excluded.kind,
+      content_type = excluded.content_type,
+      hash = excluded.hash,
+      size_bytes = excluded.size_bytes,
+      metadata_json = excluded.metadata_json,
+      updated_at = excluded.updated_at
+  `);
+
+  const insert = db.transaction((entries: GeneratedStorageObject[]) => {
+    for (const entry of entries) {
+      statement.run(
+        randomUUID(),
+        entry.uri,
+        entry.kind,
+        entry.content_type ?? null,
+        entry.hash ?? null,
+        entry.size_bytes ?? null,
+        JSON.stringify({
+          key: entry.key,
+          ...(entry.metadata ?? {}),
+        }),
+        timestamp,
+        timestamp,
+      );
+    }
+  });
+
+  insert(objects);
+}

package/src/wiki-layout.ts

@@ -1,10 +1,16 @@
 import type { ArtifactStore } from './artifact-store';
+import {
+  artifactKindForKey,
+  hashArtifactBody,
+  type GeneratedStorageObject,
+} from './storage-contract';
 
 export interface WikiLayoutInitResult {
   schema_key: string;
   root_index_key: string;
   wiki_readme_key: string;
   log_key: string;
+  artifacts: GeneratedStorageObject[];
   written: string[];
 }
 
@@ -86,19 +92,29 @@ export async function initializeWikiLayout(store: ArtifactStore, now = new Date(
     wiki_readme_key: wikiReadmeKey,
   };
 
-  const writes = [
-    store.put({ key: schemaKey, body: agentSchemaTemplate(), content_type: 'text/markdown' }),
-    store.put({ key: rootIndexKey, body: rootIndexTemplate(), content_type: 'text/markdown' }),
-    store.put({ key: wikiReadmeKey, body: wikiReadmeTemplate(), content_type: 'text/markdown' }),
-    store.put({ key: logKey, body: `${JSON.stringify(event)}\n`, content_type: 'application/x-ndjson' }),
+  const entries = [
+    { key: schemaKey, body: agentSchemaTemplate(), content_type: 'text/markdown' },
+    { key: rootIndexKey, body: rootIndexTemplate(), content_type: 'text/markdown' },
+    { key: wikiReadmeKey, body: wikiReadmeTemplate(), content_type: 'text/markdown' },
+    { key: logKey, body: `${JSON.stringify(event)}\n`, content_type: 'application/x-ndjson' },
   ];
 
-  await Promise.all(writes);
+  const artifacts = await Promise.all(entries.map(async (entry) => {
+    const result = await store.put(entry);
+    return {
+      key: result.key,
+      uri: result.uri,
+      kind: artifactKindForKey(entry.key),
+      content_type: entry.content_type,
+      ...hashArtifactBody(entry.body),
+    };
+  }));
   return {
     schema_key: schemaKey,
     root_index_key: rootIndexKey,
     wiki_readme_key: wikiReadmeKey,
     log_key: logKey,
+    artifacts,
     written: [schemaKey, rootIndexKey, wikiReadmeKey, logKey],
   };
 }