@hasna/knowledge 0.2.11 → 0.2.13

package/src/storage-contract.ts

@@ -0,0 +1,265 @@
+import { createHash, randomUUID } from 'node:crypto';
+import type { Database } from 'bun:sqlite';
+import type { KnowledgeConfig, KnowledgeWorkspace } from './workspace';
+import { HASNA_KNOWLEDGE_APP_PATH } from './workspace';
+
+export interface StorageArtifactClass {
+  kind: string;
+  prefix: string;
+  description: string;
+}
+
+export interface StorageContract {
+  scope: string;
+  mode: KnowledgeConfig['mode'];
+  storage_type: KnowledgeConfig['storage']['type'];
+  workspace_home: string;
+  local_layout: {
+    app_path: string;
+    config_path: string;
+    json_store_path: string;
+    knowledge_db_path: string;
+    directories: Record<string, string>;
+  };
+  artifact_store: {
+    type: KnowledgeConfig['storage']['type'];
+    artifacts_root: string;
+    uri_prefix: string;
+    s3: {
+      bucket: string;
+      prefix: string;
+      region: string | null;
+      profile: string | null;
+      server_side_encryption: string | null;
+      kms_key_configured: boolean;
+    } | null;
+  };
+  source_ownership: {
+    owner: 'open-files';
+    preferred_ref: string;
+    allowed_schemes: string[];
+    raw_source_bytes_stored_in_open_knowledge: false;
+    stores: string[];
+    does_not_store: string[];
+  };
+  generated_artifacts: StorageArtifactClass[];
+  scalability: {
+    catalog: string;
+    indexes: string;
+    logs: string;
+    markdown: string;
+  };
+  warnings: string[];
+}
+
+export interface StorageValidationResult {
+  ok: boolean;
+  errors: string[];
+  warnings: string[];
+}
+
+export interface GeneratedStorageObject {
+  uri: string;
+  key: string;
+  kind: string;
+  content_type?: string;
+  hash?: string;
+  size_bytes?: number;
+  metadata?: Record<string, unknown>;
+}
+
+const GENERATED_ARTIFACTS: StorageArtifactClass[] = [
+  {
+    kind: 'schema',
+    prefix: 'schemas/',
+    description: 'Machine-readable agent schemas and source rules.',
+  },
+  {
+    kind: 'index',
+    prefix: 'indexes/',
+    description: 'Small orientation indexes and future shard manifests.',
+  },
+  {
+    kind: 'log',
+    prefix: 'logs/',
+    description: 'Append-only JSONL run and wiki-maintenance log partitions.',
+  },
+  {
+    kind: 'run',
+    prefix: 'runs/',
+    description: 'Prompt/tool/cost ledgers and generated output records.',
+  },
+  {
+    kind: 'wiki_page',
+    prefix: 'wiki/',
+    description: 'Generated cited Markdown pages, not raw source files.',
+  },
+  {
+    kind: 'export',
+    prefix: 'exports/',
+    description: 'Portable exports and snapshots of derived knowledge state.',
+  },
+];
+
+export function hashArtifactBody(body: string | Uint8Array): { hash: string; size_bytes: number } {
+  const bytes = typeof body === 'string' ? Buffer.from(body) : Buffer.from(body);
+  return {
+    hash: `sha256:${createHash('sha256').update(bytes).digest('hex')}`,
+    size_bytes: bytes.byteLength,
+  };
+}
+
+export function artifactKindForKey(key: string): string {
+  const match = GENERATED_ARTIFACTS.find((entry) => key.startsWith(entry.prefix));
+  return match?.kind ?? 'artifact';
+}
+
+export function resolveStorageContract(
+  config: KnowledgeConfig,
+  workspace: KnowledgeWorkspace,
+  scope = 'global',
+): StorageContract {
+  const validation = validateStorageConfig(config, workspace);
+  const s3 = config.storage.s3 ?? null;
+  const prefix = s3?.prefix?.replace(/^\/+|\/+$/g, '') ?? '';
+  const s3UriPrefix = s3 ? `s3://${s3.bucket}/${prefix ? `${prefix}/` : ''}` : '';
+
+  return {
+    scope,
+    mode: config.mode,
+    storage_type: config.storage.type,
+    workspace_home: workspace.home,
+    local_layout: {
+      app_path: HASNA_KNOWLEDGE_APP_PATH,
+      config_path: workspace.configPath,
+      json_store_path: workspace.jsonStorePath,
+      knowledge_db_path: workspace.knowledgeDbPath,
+      directories: {
+        artifacts: workspace.artifactsDir,
+        cache: workspace.cacheDir,
+        exports: workspace.exportsDir,
+        indexes: workspace.indexesDir,
+        logs: workspace.logsDir,
+        runs: workspace.runsDir,
+        schemas: workspace.schemasDir,
+        wiki: workspace.wikiDir,
+      },
+    },
+    artifact_store: {
+      type: config.storage.type,
+      artifacts_root: config.storage.artifacts_root,
+      uri_prefix: config.storage.type === 's3' ? s3UriPrefix : `file://${workspace.artifactsDir}/`,
+      s3: s3
+        ? {
+            bucket: s3.bucket,
+            prefix,
+            region: s3.region ?? null,
+            profile: s3.profile ?? null,
+            server_side_encryption: s3.server_side_encryption ?? null,
+            kms_key_configured: Boolean(s3.kms_key_id),
+          }
+        : null,
+    },
+    source_ownership: {
+      owner: 'open-files',
+      preferred_ref: config.sources.preferred_ref,
+      allowed_schemes: config.sources.allowed_schemes,
+      raw_source_bytes_stored_in_open_knowledge: false,
+      stores: [
+        'source refs',
+        'source revisions and hashes',
+        'citation spans',
+        'redacted extracted chunks',
+        'embeddings',
+        'generated wiki artifacts',
+        'indexes',
+        'run ledgers',
+      ],
+      does_not_store: [
+        'raw open-files bytes',
+        'S3 object credentials',
+        'connector secrets',
+        'hosted tenant ownership state',
+      ],
+    },
+    generated_artifacts: GENERATED_ARTIFACTS,
+    scalability: {
+      catalog: 'knowledge.db tracks sources, revisions, chunks, citations, indexes, runs, and storage_objects.',
+      indexes: 'Indexes are cataloged DB rows plus sharded artifacts, not one giant index.md.',
+      logs: 'Logs use dated JSONL partitions under logs/yyyy/mm/dd.jsonl.',
+      markdown: 'Markdown pages are the readable wiki layer over DB/object-store state.',
+    },
+    warnings: validation.warnings,
+  };
+}
+
+export function validateStorageConfig(config: KnowledgeConfig, workspace: KnowledgeWorkspace): StorageValidationResult {
+  const errors: string[] = [];
+  const warnings: string[] = [];
+
+  if (!workspace.home.endsWith(HASNA_KNOWLEDGE_APP_PATH)) {
+    warnings.push(`Workspace home does not end with ${HASNA_KNOWLEDGE_APP_PATH}: ${workspace.home}`);
+  }
+
+  if (config.storage.type === 's3') {
+    if (!config.storage.s3?.bucket) errors.push('storage.s3.bucket is required when storage.type is s3.');
+    if (!config.storage.s3?.prefix) warnings.push('storage.s3.prefix is empty; generated knowledge artifacts will be written at the bucket root.');
+    if (config.mode === 'local') warnings.push('storage.type is s3 while mode is local; this is valid for BYO S3, but hosted wrappers should set mode to hosted.');
+  }
+
+  if (config.storage.type === 'local' && config.storage.s3) {
+    warnings.push('storage.s3 is configured but ignored while storage.type is local.');
+  }
+
+  if (config.sources.preferred_ref !== 'open-files') {
+    warnings.push('sources.preferred_ref should stay open-files for durable company knowledge.');
+  }
+
+  if (!config.sources.allowed_schemes.includes('open-files')) {
+    errors.push('sources.allowed_schemes must include open-files.');
+  }
+
+  return {
+    ok: errors.length === 0,
+    errors,
+    warnings,
+  };
+}
+
+export function recordStorageObjects(db: Database, objects: GeneratedStorageObject[], now = new Date()): void {
+  const timestamp = now.toISOString();
+  const statement = db.prepare(`
+    INSERT INTO storage_objects (
+      id, artifact_uri, kind, content_type, hash, size_bytes, metadata_json, created_at, updated_at
+    )
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+    ON CONFLICT(artifact_uri) DO UPDATE SET
+      kind = excluded.kind,
+      content_type = excluded.content_type,
+      hash = excluded.hash,
+      size_bytes = excluded.size_bytes,
+      metadata_json = excluded.metadata_json,
+      updated_at = excluded.updated_at
+  `);
+
+  const insert = db.transaction((entries: GeneratedStorageObject[]) => {
+    for (const entry of entries) {
+      statement.run(
+        randomUUID(),
+        entry.uri,
+        entry.kind,
+        entry.content_type ?? null,
+        entry.hash ?? null,
+        entry.size_bytes ?? null,
+        JSON.stringify({
+          key: entry.key,
+          ...(entry.metadata ?? {}),
+        }),
+        timestamp,
+        timestamp,
+      );
+    }
+  });
+
+  insert(objects);
+}

package/src/wiki-layout.ts

@@ -1,13 +1,29 @@
+import { createHash } from 'node:crypto';
+import type { Database } from 'bun:sqlite';
 import type { ArtifactStore } from './artifact-store';
+import { generatedArtifactProvenance, type GeneratedArtifactProvenance } from './provenance';
+import {
+  artifactKindForKey,
+  hashArtifactBody,
+  type GeneratedStorageObject,
+} from './storage-contract';
 
 export interface WikiLayoutInitResult {
   schema_key: string;
   root_index_key: string;
   wiki_readme_key: string;
   log_key: string;
+  artifacts: GeneratedStorageObject[];
   written: string[];
 }
 
+interface CatalogArtifact {
+  key: string;
+  uri: string;
+  hash?: string;
+  metadata?: Record<string, unknown>;
+}
+
 function todayParts(now: Date): { year: string; month: string; day: string } {
   const year = String(now.getUTCFullYear());
   const month = String(now.getUTCMonth() + 1).padStart(2, '0');
@@ -15,6 +31,10 @@ function todayParts(now: Date): { year: string; month: string; day: string } {
   return { year, month, day };
 }
 
+function stableId(prefix: string, value: string): string {
+  return `${prefix}_${createHash('sha256').update(value).digest('hex').slice(0, 20)}`;
+}
+
 export function agentSchemaTemplate(): string {
   return `# Knowledge Agent Schema v1
 
@@ -86,19 +106,106 @@ export async function initializeWikiLayout(store: ArtifactStore, now = new Date(
     wiki_readme_key: wikiReadmeKey,
   };
 
-  const writes = [
-    store.put({ key: schemaKey, body: agentSchemaTemplate(), content_type: 'text/markdown' }),
-    store.put({ key: rootIndexKey, body: rootIndexTemplate(), content_type: 'text/markdown' }),
-    store.put({ key: wikiReadmeKey, body: wikiReadmeTemplate(), content_type: 'text/markdown' }),
-    store.put({ key: logKey, body: `${JSON.stringify(event)}\n`, content_type: 'application/x-ndjson' }),
+  const entries = [
+    { key: schemaKey, body: agentSchemaTemplate(), content_type: 'text/markdown' },
+    { key: rootIndexKey, body: rootIndexTemplate(), content_type: 'text/markdown' },
+    { key: wikiReadmeKey, body: wikiReadmeTemplate(), content_type: 'text/markdown' },
+    { key: logKey, body: `${JSON.stringify(event)}\n`, content_type: 'application/x-ndjson' },
   ];
 
-  await Promise.all(writes);
+  const artifacts = await Promise.all(entries.map(async (entry) => {
+    const result = await store.put(entry);
+    return {
+      key: result.key,
+      uri: result.uri,
+      kind: artifactKindForKey(entry.key),
+      content_type: entry.content_type,
+      metadata: {
+        provenance: generatedArtifactProvenance({
+          generated_from: 'wiki_layout_init',
+          artifact_key: entry.key,
+          citation_required: entry.key.startsWith('wiki/') || entry.key.startsWith('indexes/'),
+        }),
+      },
+      ...hashArtifactBody(entry.body),
+    };
+  }));
   return {
     schema_key: schemaKey,
     root_index_key: rootIndexKey,
     wiki_readme_key: wikiReadmeKey,
     log_key: logKey,
+    artifacts,
     written: [schemaKey, rootIndexKey, wikiReadmeKey, logKey],
   };
 }
+
+function provenanceFor(artifact: CatalogArtifact): GeneratedArtifactProvenance {
+  const existing = artifact.metadata?.provenance;
+  if (existing && typeof existing === 'object' && !Array.isArray(existing)) {
+    return existing as GeneratedArtifactProvenance;
+  }
+  return generatedArtifactProvenance({
+    generated_from: 'wiki_layout_init',
+    artifact_key: artifact.key,
+  });
+}
+
+export function recordWikiLayoutCatalog(db: Database, artifacts: CatalogArtifact[], now = new Date()): void {
+  const timestamp = now.toISOString();
+  const rootIndex = artifacts.find((artifact) => artifact.key.endsWith('indexes/root.md'));
+  const wikiReadme = artifacts.find((artifact) => artifact.key.endsWith('wiki/README.md'));
+
+  if (rootIndex) {
+    db.run(
+      `INSERT INTO knowledge_indexes (id, kind, name, artifact_uri, shard_key, metadata_json, created_at, updated_at)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+       ON CONFLICT(kind, name, shard_key) DO UPDATE SET
+         artifact_uri = excluded.artifact_uri,
+         metadata_json = excluded.metadata_json,
+         updated_at = excluded.updated_at`,
+      [
+        stableId('idx', 'root:indexes/root.md'),
+        'root',
+        'root',
+        rootIndex.uri,
+        'root',
+        JSON.stringify({
+          artifact_key: rootIndex.key,
+          content_hash: rootIndex.hash ?? null,
+          provenance: provenanceFor(rootIndex),
+        }),
+        timestamp,
+        timestamp,
+      ],
+    );
+  }
+
+  if (wikiReadme) {
+    db.run(
+      `INSERT INTO wiki_pages (id, path, title, artifact_uri, content_hash, status, metadata_json, created_at, updated_at)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+       ON CONFLICT(path) DO UPDATE SET
+         title = excluded.title,
+         artifact_uri = excluded.artifact_uri,
+         content_hash = excluded.content_hash,
+         status = excluded.status,
+         metadata_json = excluded.metadata_json,
+         updated_at = excluded.updated_at`,
+      [
+        stableId('wiki', 'wiki/README.md'),
+        'wiki/README.md',
+        'Wiki',
+        wikiReadme.uri,
+        wikiReadme.hash ?? null,
+        'active',
+        JSON.stringify({
+          artifact_key: wikiReadme.key,
+          provenance: provenanceFor(wikiReadme),
+        }),
+        timestamp,
+        timestamp,
+      ],
+    );
+  }
+}