@hasna/knowledge 0.2.11 → 0.2.13

package/README.md

@@ -62,6 +62,9 @@ open-knowledge export --format jsonl
 # Show resolved workspace paths
 open-knowledge paths --scope project --json
 
+# Inspect local/S3 artifact storage and source ownership
+open-knowledge storage status --scope project --json
+
 # Initialize the project SQLite catalog
 open-knowledge db init --scope project
 
@@ -168,6 +171,18 @@ open-knowledge paths [--scope global|project|local] [--json]
 Show the resolved Hasna app workspace, JSON compatibility store, SQLite path,
 artifact directories, and config.
 
+### storage
+```bash
+open-knowledge storage status [--scope project] [--json]
+open-knowledge storage validate [--scope project] [--json]
+```
+Show the storage contract for local or S3-backed generated artifacts. Local mode
+uses `.hasna/apps/knowledge` for config, SQLite, indexes, wiki artifacts, logs,
+runs, and exports. S3 mode stores generated artifacts under the configured
+knowledge bucket/prefix while `open-files` remains the source of truth for raw
+source bytes. The command also reports artifact classes, allowed source ref
+schemes, and warnings for non-scalable or unsafe config.
+
 ### db
 ```bash
 open-knowledge db init [--scope project]
@@ -277,8 +292,9 @@ open-knowledge-mcp
 The MCP server exposes item tools (`ok_add`, `ok_list`, `ok_get`, `ok_update`,
 `ok_delete`, `ok_archive`, `ok_restore`, `ok_upsert`, `ok_untag`,
 `ok_bulk_delete`, `ok_prune`, `ok_dedupe`, `ok_stats`, `ok_export`,
-`ok_import`, `ok_batch`), workspace inspection (`ok_paths`), and source-ref
-parsing/resolution (`ok_parse_source_ref`, `ok_resolve_source`).
+`ok_import`, `ok_batch`), workspace/storage inspection (`ok_paths`,
+`ok_storage_status`), and source-ref parsing/resolution
+(`ok_parse_source_ref`, `ok_resolve_source`).
 
 ## Source And Artifact Boundary
 
@@ -298,6 +314,12 @@ source ref. It does not copy raw files into the knowledge workspace; local file,
 S3, web, and open-files inputs are converted into redacted chunks with offsets,
 hashes, revision metadata, and FTS rows.
 
+Chunks, resolver results, generated wiki pages, and index records carry
+provenance metadata: source owner, source ref/URI, revision/hash, chunk offsets,
+read-only status, citation requirements, and stale-source status. This keeps
+future semantic search and wiki compile flows tied back to `open-files` instead
+of detached Markdown.
+
 AI provider configuration is local/BYOK by default. `open-knowledge` declares
 AI SDK v6 provider support through `ai`, `@ai-sdk/openai`,
 `@ai-sdk/anthropic`, and `@ai-sdk/deepseek`, but does not call providers until a

package/bin/open-knowledge-mcp.js

@@ -13660,7 +13660,7 @@ import { existsSync as existsSync7, readFileSync as readFileSync7, writeFileSync
 // package.json
 var package_default = {
   name: "@hasna/knowledge",
-  version: "0.2.11",
+  version: "0.2.13",
   description: "Agent-friendly local knowledge CLI with JSON output, pagination, and safe destructive actions",
   type: "module",
   bin: {
@@ -14353,6 +14353,7 @@ function openKnowledgeDb(path) {
   ensureParentDir(path);
   const db = new Database(path);
   db.exec("PRAGMA foreign_keys = ON;");
+  db.exec("PRAGMA busy_timeout = 5000;");
   return db;
 }
 function migrateKnowledgeDb(path) {
@@ -14391,7 +14392,8 @@ function getKnowledgeDbStats(path) {
       run_events: count(db, "run_events"),
       redaction_findings: count(db, "redaction_findings"),
       audit_events: count(db, "audit_events"),
-      approval_gates: count(db, "approval_gates")
+      approval_gates: count(db, "approval_gates"),
+      storage_objects: count(db, "storage_objects")
     };
   } finally {
     db.close();
@@ -14898,6 +14900,50 @@ async function consumeOpenFilesOutbox(options) {
 import { createHash as createHash3 } from "crypto";
 import { existsSync as existsSync5, readFileSync as readFileSync5 } from "fs";
 import { basename as basename2 } from "path";
+
+// src/provenance.ts
+function isStaleStatus(status) {
+  return ["deleted", "stale", "invalidated", "reindex_required"].includes((status ?? "").toLowerCase());
+}
+function sourceProvenance(input) {
+  const status = input.status ?? null;
+  return {
+    source_owner: "open-files",
+    source_ref: input.source_ref ?? null,
+    source_uri: input.source_uri ?? null,
+    source_kind: input.source_kind ?? null,
+    source_revision_id: input.source_revision_id ?? null,
+    revision: input.revision ?? null,
+    hash: input.hash ?? null,
+    chunk_id: input.chunk_id ?? null,
+    start_offset: input.start_offset ?? null,
+    end_offset: input.end_offset ?? null,
+    status,
+    read_only: true,
+    citation_required: true,
+    resolver: input.resolver ?? null,
+    stale: isStaleStatus(status)
+  };
+}
+function generatedArtifactProvenance(input) {
+  return {
+    source_owner: "open-files",
+    generated_from: input.generated_from,
+    artifact_key: input.artifact_key,
+    source_refs: input.source_refs ?? [],
+    read_only_sources: true,
+    citation_required: input.citation_required ?? true,
+    raw_source_bytes_stored_in_open_knowledge: false
+  };
+}
+function withProvenance(metadata, provenance) {
+  return {
+    ...metadata,
+    provenance
+  };
+}
+
+// src/manifest-ingest.ts
 function stableId2(prefix, value) {
   return `${prefix}_${createHash3("sha256").update(value).digest("hex").slice(0, 20)}`;
 }
@@ -15186,15 +15232,31 @@ function insertChunks(db, sourceRevisionId, item, now, maxChars, overlapChars, s
   const chunks = chunkText(redacted.text, maxChars, overlapChars);
   for (const chunk of chunks) {
     const chunkId = stableId2("chk", `${sourceRevisionId}\x00${chunk.ordinal}\x00${chunk.text}`);
-    const metadata = {
+    const provenance = sourceProvenance({
       source_ref: item.sourceRef,
       source_uri: item.sourceUri,
+      source_kind: item.kind,
+      source_revision_id: sourceRevisionId,
+      revision: item.revision,
+      hash: item.hash,
+      chunk_id: chunkId,
+      start_offset: chunk.startOffset,
+      end_offset: chunk.endOffset,
+      status: item.status,
+      resolver: "open-files-read-only"
+    });
+    const metadata = withProvenance({
+      source_ref: item.sourceRef,
+      source_uri: item.sourceUri,
+      source_kind: item.kind,
+      source_revision_id: sourceRevisionId,
+      revision: item.revision,
       hash: item.hash,
       status: item.status,
       path: asString2(item.raw.path) ?? null,
       mime: asString2(item.raw.mime) ?? asString2(item.raw.content_type) ?? null,
       size: asNumber(item.raw.size) ?? null
-    };
+    }, provenance);
     db.run(`INSERT INTO chunks (id, source_revision_id, kind, ordinal, text, token_count, start_offset, end_offset, metadata_json, created_at)
        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
       chunkId,
@@ -15487,6 +15549,19 @@ async function resolveOpenFilesSource(options) {
           end_offset: row.end_offset,
           resolved_at: resolvedAt
         };
+        const provenance = sourceProvenance({
+          source_ref: evidence.source_ref,
+          source_uri: evidence.source_uri,
+          source_kind: source.kind,
+          source_revision_id: evidence.source_revision_id,
+          revision: evidence.revision,
+          hash: evidence.hash,
+          chunk_id: row.id,
+          start_offset: row.start_offset,
+          end_offset: row.end_offset,
+          status: metadataString(metadata, ["status"]),
+          resolver: evidence.resolver
+        });
         return {
           id: row.id,
           kind: row.kind,
@@ -15496,7 +15571,8 @@ async function resolveOpenFilesSource(options) {
           start_offset: row.start_offset,
           end_offset: row.end_offset,
           metadata,
-          evidence
+          evidence,
+          provenance
         };
       });
       const citations = chunks.map((chunk) => ({
@@ -15506,7 +15582,8 @@ async function resolveOpenFilesSource(options) {
         quote: chunk.text.slice(0, 500),
         start_offset: chunk.start_offset,
         end_offset: chunk.end_offset,
-        evidence: chunk.evidence
+        evidence: chunk.evidence,
+        provenance: chunk.provenance
       }));
       recordAuditEvent(db, {
         event_type: "source_read",
@@ -15922,13 +15999,188 @@ function providerStatus(config2, env = process.env) {
   };
 }
 
+// src/storage-contract.ts
+import { createHash as createHash5, randomUUID as randomUUID4 } from "crypto";
+var GENERATED_ARTIFACTS = [
+  {
+    kind: "schema",
+    prefix: "schemas/",
+    description: "Machine-readable agent schemas and source rules."
+  },
+  {
+    kind: "index",
+    prefix: "indexes/",
+    description: "Small orientation indexes and future shard manifests."
+  },
+  {
+    kind: "log",
+    prefix: "logs/",
+    description: "Append-only JSONL run and wiki-maintenance log partitions."
+  },
+  {
+    kind: "run",
+    prefix: "runs/",
+    description: "Prompt/tool/cost ledgers and generated output records."
+  },
+  {
+    kind: "wiki_page",
+    prefix: "wiki/",
+    description: "Generated cited Markdown pages, not raw source files."
+  },
+  {
+    kind: "export",
+    prefix: "exports/",
+    description: "Portable exports and snapshots of derived knowledge state."
+  }
+];
+function hashArtifactBody(body) {
+  const bytes = typeof body === "string" ? Buffer.from(body) : Buffer.from(body);
+  return {
+    hash: `sha256:${createHash5("sha256").update(bytes).digest("hex")}`,
+    size_bytes: bytes.byteLength
+  };
+}
+function artifactKindForKey(key) {
+  const match = GENERATED_ARTIFACTS.find((entry) => key.startsWith(entry.prefix));
+  return match?.kind ?? "artifact";
+}
+function resolveStorageContract(config2, workspace, scope = "global") {
+  const validation = validateStorageConfig(config2, workspace);
+  const s3 = config2.storage.s3 ?? null;
+  const prefix = s3?.prefix?.replace(/^\/+|\/+$/g, "") ?? "";
+  const s3UriPrefix = s3 ? `s3://${s3.bucket}/${prefix ? `${prefix}/` : ""}` : "";
+  return {
+    scope,
+    mode: config2.mode,
+    storage_type: config2.storage.type,
+    workspace_home: workspace.home,
+    local_layout: {
+      app_path: HASNA_KNOWLEDGE_APP_PATH,
+      config_path: workspace.configPath,
+      json_store_path: workspace.jsonStorePath,
+      knowledge_db_path: workspace.knowledgeDbPath,
+      directories: {
+        artifacts: workspace.artifactsDir,
+        cache: workspace.cacheDir,
+        exports: workspace.exportsDir,
+        indexes: workspace.indexesDir,
+        logs: workspace.logsDir,
+        runs: workspace.runsDir,
+        schemas: workspace.schemasDir,
+        wiki: workspace.wikiDir
+      }
+    },
+    artifact_store: {
+      type: config2.storage.type,
+      artifacts_root: config2.storage.artifacts_root,
+      uri_prefix: config2.storage.type === "s3" ? s3UriPrefix : `file://${workspace.artifactsDir}/`,
+      s3: s3 ? {
+        bucket: s3.bucket,
+        prefix,
+        region: s3.region ?? null,
+        profile: s3.profile ?? null,
+        server_side_encryption: s3.server_side_encryption ?? null,
+        kms_key_configured: Boolean(s3.kms_key_id)
+      } : null
+    },
+    source_ownership: {
+      owner: "open-files",
+      preferred_ref: config2.sources.preferred_ref,
+      allowed_schemes: config2.sources.allowed_schemes,
+      raw_source_bytes_stored_in_open_knowledge: false,
+      stores: [
+        "source refs",
+        "source revisions and hashes",
+        "citation spans",
+        "redacted extracted chunks",
+        "embeddings",
+        "generated wiki artifacts",
+        "indexes",
+        "run ledgers"
+      ],
+      does_not_store: [
+        "raw open-files bytes",
+        "S3 object credentials",
+        "connector secrets",
+        "hosted tenant ownership state"
+      ]
+    },
+    generated_artifacts: GENERATED_ARTIFACTS,
+    scalability: {
+      catalog: "knowledge.db tracks sources, revisions, chunks, citations, indexes, runs, and storage_objects.",
+      indexes: "Indexes are cataloged DB rows plus sharded artifacts, not one giant index.md.",
+      logs: "Logs use dated JSONL partitions under logs/yyyy/mm/dd.jsonl.",
+      markdown: "Markdown pages are the readable wiki layer over DB/object-store state."
+    },
+    warnings: validation.warnings
+  };
+}
+function validateStorageConfig(config2, workspace) {
+  const errors3 = [];
+  const warnings = [];
+  if (!workspace.home.endsWith(HASNA_KNOWLEDGE_APP_PATH)) {
+    warnings.push(`Workspace home does not end with ${HASNA_KNOWLEDGE_APP_PATH}: ${workspace.home}`);
+  }
+  if (config2.storage.type === "s3") {
+    if (!config2.storage.s3?.bucket)
+      errors3.push("storage.s3.bucket is required when storage.type is s3.");
+    if (!config2.storage.s3?.prefix)
+      warnings.push("storage.s3.prefix is empty; generated knowledge artifacts will be written at the bucket root.");
+    if (config2.mode === "local")
+      warnings.push("storage.type is s3 while mode is local; this is valid for BYO S3, but hosted wrappers should set mode to hosted.");
+  }
+  if (config2.storage.type === "local" && config2.storage.s3) {
+    warnings.push("storage.s3 is configured but ignored while storage.type is local.");
+  }
+  if (config2.sources.preferred_ref !== "open-files") {
+    warnings.push("sources.preferred_ref should stay open-files for durable company knowledge.");
+  }
+  if (!config2.sources.allowed_schemes.includes("open-files")) {
+    errors3.push("sources.allowed_schemes must include open-files.");
+  }
+  return {
+    ok: errors3.length === 0,
+    errors: errors3,
+    warnings
+  };
+}
+function recordStorageObjects(db, objects, now = new Date) {
+  const timestamp = now.toISOString();
+  const statement = db.prepare(`
+    INSERT INTO storage_objects (
+      id, artifact_uri, kind, content_type, hash, size_bytes, metadata_json, created_at, updated_at
+    )
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+    ON CONFLICT(artifact_uri) DO UPDATE SET
+      kind = excluded.kind,
+      content_type = excluded.content_type,
+      hash = excluded.hash,
+      size_bytes = excluded.size_bytes,
+      metadata_json = excluded.metadata_json,
+      updated_at = excluded.updated_at
+  `);
+  const insert = db.transaction((entries) => {
+    for (const entry of entries) {
+      statement.run(randomUUID4(), entry.uri, entry.kind, entry.content_type ?? null, entry.hash ?? null, entry.size_bytes ?? null, JSON.stringify({
+        key: entry.key,
+        ...entry.metadata ?? {}
+      }), timestamp, timestamp);
+    }
+  });
+  insert(objects);
+}
+
 // src/wiki-layout.ts
+import { createHash as createHash6 } from "crypto";
 function todayParts(now) {
   const year = String(now.getUTCFullYear());
   const month = String(now.getUTCMonth() + 1).padStart(2, "0");
   const day = String(now.getUTCDate()).padStart(2, "0");
   return { year, month, day };
 }
+function stableId3(prefix, value) {
+  return `${prefix}_${createHash6("sha256").update(value).digest("hex").slice(0, 20)}`;
+}
 function agentSchemaTemplate() {
   return `# Knowledge Agent Schema v1
 
@@ -15996,22 +16248,99 @@ async function initializeWikiLayout(store, now = new Date) {
     root_index_key: rootIndexKey,
     wiki_readme_key: wikiReadmeKey
   };
-  const writes = [
-    store.put({ key: schemaKey, body: agentSchemaTemplate(), content_type: "text/markdown" }),
-    store.put({ key: rootIndexKey, body: rootIndexTemplate(), content_type: "text/markdown" }),
-    store.put({ key: wikiReadmeKey, body: wikiReadmeTemplate(), content_type: "text/markdown" }),
-    store.put({ key: logKey, body: `${JSON.stringify(event)}
-`, content_type: "application/x-ndjson" })
+  const entries = [
+    { key: schemaKey, body: agentSchemaTemplate(), content_type: "text/markdown" },
+    { key: rootIndexKey, body: rootIndexTemplate(), content_type: "text/markdown" },
+    { key: wikiReadmeKey, body: wikiReadmeTemplate(), content_type: "text/markdown" },
+    { key: logKey, body: `${JSON.stringify(event)}
+`, content_type: "application/x-ndjson" }
   ];
-  await Promise.all(writes);
+  const artifacts = await Promise.all(entries.map(async (entry) => {
+    const result = await store.put(entry);
+    return {
+      key: result.key,
+      uri: result.uri,
+      kind: artifactKindForKey(entry.key),
+      content_type: entry.content_type,
+      metadata: {
+        provenance: generatedArtifactProvenance({
+          generated_from: "wiki_layout_init",
+          artifact_key: entry.key,
+          citation_required: entry.key.startsWith("wiki/") || entry.key.startsWith("indexes/")
+        })
+      },
+      ...hashArtifactBody(entry.body)
+    };
+  }));
   return {
     schema_key: schemaKey,
     root_index_key: rootIndexKey,
     wiki_readme_key: wikiReadmeKey,
     log_key: logKey,
+    artifacts,
     written: [schemaKey, rootIndexKey, wikiReadmeKey, logKey]
   };
 }
+function provenanceFor(artifact) {
+  const existing = artifact.metadata?.provenance;
+  if (existing && typeof existing === "object" && !Array.isArray(existing)) {
+    return existing;
+  }
+  return generatedArtifactProvenance({
+    generated_from: "wiki_layout_init",
+    artifact_key: artifact.key
+  });
+}
+function recordWikiLayoutCatalog(db, artifacts, now = new Date) {
+  const timestamp = now.toISOString();
+  const rootIndex = artifacts.find((artifact) => artifact.key.endsWith("indexes/root.md"));
+  const wikiReadme = artifacts.find((artifact) => artifact.key.endsWith("wiki/README.md"));
+  if (rootIndex) {
+    db.run(`INSERT INTO knowledge_indexes (id, kind, name, artifact_uri, shard_key, metadata_json, created_at, updated_at)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+       ON CONFLICT(kind, name, shard_key) DO UPDATE SET
+         artifact_uri = excluded.artifact_uri,
+         metadata_json = excluded.metadata_json,
+         updated_at = excluded.updated_at`, [
+      stableId3("idx", "root:indexes/root.md"),
+      "root",
+      "root",
+      rootIndex.uri,
+      "root",
+      JSON.stringify({
+        artifact_key: rootIndex.key,
+        content_hash: rootIndex.hash ?? null,
+        provenance: provenanceFor(rootIndex)
+      }),
+      timestamp,
+      timestamp
+    ]);
+  }
+  if (wikiReadme) {
+    db.run(`INSERT INTO wiki_pages (id, path, title, artifact_uri, content_hash, status, metadata_json, created_at, updated_at)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+       ON CONFLICT(path) DO UPDATE SET
+         title = excluded.title,
+         artifact_uri = excluded.artifact_uri,
+         content_hash = excluded.content_hash,
+         status = excluded.status,
+         metadata_json = excluded.metadata_json,
+         updated_at = excluded.updated_at`, [
+      stableId3("wiki", "wiki/README.md"),
+      "wiki/README.md",
+      "Wiki",
+      wikiReadme.uri,
+      wikiReadme.hash ?? null,
+      "active",
+      JSON.stringify({
+        artifact_key: wikiReadme.key,
+        provenance: provenanceFor(wikiReadme)
+      }),
+      timestamp,
+      timestamp
+    ]);
+  }
+}
 
 // src/service.ts
 class KnowledgeService {
@@ -16048,6 +16377,12 @@ class KnowledgeService {
   artifactStore() {
     return createArtifactStore(this.config(), this.ensureWorkspace());
   }
+  storageContract() {
+    return resolveStorageContract(this.config(), this.ensureWorkspace(), this.scope);
+  }
+  validateStorage() {
+    return validateStorageConfig(this.config(), this.ensureWorkspace());
+  }
   paths() {
     const workspace = this.ensureWorkspace();
     return {
@@ -16076,7 +16411,17 @@ class KnowledgeService {
     return getKnowledgeDbStats(workspace.knowledgeDbPath);
   }
   async initWiki() {
-    return initializeWikiLayout(this.artifactStore());
+    const workspace = this.ensureWorkspace();
+    migrateKnowledgeDb(workspace.knowledgeDbPath);
+    const result = await initializeWikiLayout(this.artifactStore());
+    const db = openKnowledgeDb(workspace.knowledgeDbPath);
+    try {
+      recordStorageObjects(db, result.artifacts);
+      recordWikiLayoutCatalog(db, result.artifacts);
+    } finally {
+      db.close();
+    }
+    return result;
   }
   async ingestManifest(input) {
     const workspace = this.ensureWorkspace();
@@ -16187,6 +16532,17 @@ function buildServer() {
   }, async ({ scope }) => {
     return jsonText(createKnowledgeService({ scope }).paths());
   });
+  registerTool(server, "ok_storage_status", "Knowledge storage status", "Inspect local/S3 artifact storage, source ownership, and scalability contract", {
+    scope: scopeField
+  }, async ({ scope }) => {
+    const service = createKnowledgeService({ scope });
+    const validation = service.validateStorage();
+    return jsonText({
+      ok: validation.ok,
+      ...service.storageContract(),
+      validation
+    });
+  });
   registerTool(server, "ok_parse_source_ref", "Parse source reference", "Parse and validate an open-files, S3, file, or web source ref", {
     uri: exports_external.string().describe("Source reference URI")
   }, async ({ uri }) => {