@hasna/knowledge 0.2.12 → 0.2.14

package/src/outbox-consume.ts

@@ -30,6 +30,7 @@ export interface OutboxConsumeResult {
   deleted_sources: number;
   moved_sources: number;
   permission_updates: number;
+  vector_entries_deleted: number;
 }
 
 interface NormalizedOutboxEvent {
@@ -289,12 +290,16 @@ function revisionIdsForEvent(db: Database, sourceId: string, event: NormalizedOu
   ).all(sourceId).map((row) => row.id);
 }
 
-function invalidateRevision(db: Database, revisionId: string): { chunksDeleted: number; embeddingsDeleted: number } {
+function invalidateRevision(db: Database, revisionId: string): { chunksDeleted: number; embeddingsDeleted: number; vectorEntriesDeleted: number } {
   const chunks = db.query<{ id: string }, [string]>('SELECT id FROM chunks WHERE source_revision_id = ?').all(revisionId);
   let embeddingsDeleted = 0;
+  let vectorEntriesDeleted = 0;
   for (const chunk of chunks) {
     const row = db.query<{ n: number }, [string]>('SELECT COUNT(*) AS n FROM chunk_embeddings WHERE chunk_id = ?').get(chunk.id);
     embeddingsDeleted += row?.n ?? 0;
+    const vectorRow = db.query<{ n: number }, [string]>('SELECT COUNT(*) AS n FROM vector_index_entries WHERE chunk_id = ?').get(chunk.id);
+    vectorEntriesDeleted += vectorRow?.n ?? 0;
+    db.run('DELETE FROM vector_index_entries WHERE chunk_id = ?', [chunk.id]);
     db.run('DELETE FROM chunk_embeddings WHERE chunk_id = ?', [chunk.id]);
     db.run('DELETE FROM chunks_fts WHERE chunk_id = ?', [chunk.id]);
   }
@@ -304,7 +309,7 @@ function invalidateRevision(db: Database, revisionId: string): { chunksDeleted:
     'UPDATE source_revisions SET metadata_json = ? WHERE id = ?',
     [mergeJson(revision?.metadata_json, { reindex_required: true, invalidated_at: new Date().toISOString() }), revisionId],
   );
-  return { chunksDeleted: chunks.length, embeddingsDeleted };
+  return { chunksDeleted: chunks.length, embeddingsDeleted, vectorEntriesDeleted };
 }
 
 function isDeleteEvent(eventType: string, status: string | null): boolean {
@@ -349,6 +354,7 @@ export async function consumeOpenFilesOutbox(options: OutboxConsumeOptions): Pro
       const revisionsTouched = new Set<string>();
       let chunksDeleted = 0;
       let embeddingsDeleted = 0;
+      let vectorEntriesDeleted = 0;
       let staleRevisions = 0;
       let deletedSources = 0;
       let movedSources = 0;
@@ -376,6 +382,7 @@ export async function consumeOpenFilesOutbox(options: OutboxConsumeOptions): Pro
           const invalidation = invalidateRevision(db, revisionId);
           chunksDeleted += invalidation.chunksDeleted;
           embeddingsDeleted += invalidation.embeddingsDeleted;
+          vectorEntriesDeleted += invalidation.vectorEntriesDeleted;
           staleRevisions += 1;
         }
 
@@ -429,6 +436,7 @@ export async function consumeOpenFilesOutbox(options: OutboxConsumeOptions): Pro
           revisions: revisionsTouched.size,
           chunks_deleted: chunksDeleted,
           embeddings_deleted: embeddingsDeleted,
+          vector_entries_deleted: vectorEntriesDeleted,
         },
         created_at: now,
       });
@@ -442,6 +450,7 @@ export async function consumeOpenFilesOutbox(options: OutboxConsumeOptions): Pro
         revisions_touched: revisionsTouched.size,
         chunks_deleted: chunksDeleted,
         embeddings_deleted: embeddingsDeleted,
+        vector_entries_deleted: vectorEntriesDeleted,
         stale_revisions: staleRevisions,
         deleted_sources: deletedSources,
         moved_sources: movedSources,

package/src/provenance.ts

@@ -0,0 +1,93 @@
+export interface KnowledgeProvenance {
+  source_owner: 'open-files';
+  source_ref: string | null;
+  source_uri: string | null;
+  source_kind: string | null;
+  source_revision_id: string | null;
+  revision: string | null;
+  hash: string | null;
+  chunk_id: string | null;
+  start_offset: number | null;
+  end_offset: number | null;
+  status: string | null;
+  read_only: true;
+  citation_required: boolean;
+  resolver: string | null;
+  stale: boolean;
+}
+
+export interface GeneratedArtifactProvenance {
+  source_owner: 'open-files';
+  generated_from: string;
+  artifact_key: string;
+  source_refs: string[];
+  read_only_sources: true;
+  citation_required: boolean;
+  raw_source_bytes_stored_in_open_knowledge: false;
+}
+
+export interface SourceProvenanceInput {
+  source_ref?: string | null;
+  source_uri?: string | null;
+  source_kind?: string | null;
+  source_revision_id?: string | null;
+  revision?: string | null;
+  hash?: string | null;
+  chunk_id?: string | null;
+  start_offset?: number | null;
+  end_offset?: number | null;
+  status?: string | null;
+  resolver?: string | null;
+}
+
+export function isStaleStatus(status: string | null | undefined): boolean {
+  return ['deleted', 'stale', 'invalidated', 'reindex_required'].includes((status ?? '').toLowerCase());
+}
+
+export function sourceProvenance(input: SourceProvenanceInput): KnowledgeProvenance {
+  const status = input.status ?? null;
+  return {
+    source_owner: 'open-files',
+    source_ref: input.source_ref ?? null,
+    source_uri: input.source_uri ?? null,
+    source_kind: input.source_kind ?? null,
+    source_revision_id: input.source_revision_id ?? null,
+    revision: input.revision ?? null,
+    hash: input.hash ?? null,
+    chunk_id: input.chunk_id ?? null,
+    start_offset: input.start_offset ?? null,
+    end_offset: input.end_offset ?? null,
+    status,
+    read_only: true,
+    citation_required: true,
+    resolver: input.resolver ?? null,
+    stale: isStaleStatus(status),
+  };
+}
+
+export function generatedArtifactProvenance(input: {
+  generated_from: string;
+  artifact_key: string;
+  source_refs?: string[];
+  citation_required?: boolean;
+}): GeneratedArtifactProvenance {
+  return {
+    source_owner: 'open-files',
+    generated_from: input.generated_from,
+    artifact_key: input.artifact_key,
+    source_refs: input.source_refs ?? [],
+    read_only_sources: true,
+    citation_required: input.citation_required ?? true,
+    raw_source_bytes_stored_in_open_knowledge: false,
+  };
+}
+
+export function withProvenance<T extends Record<string, unknown>>(
+  metadata: T,
+  provenance: KnowledgeProvenance | GeneratedArtifactProvenance,
+): T & { provenance: KnowledgeProvenance | GeneratedArtifactProvenance } {
+  return {
+    ...metadata,
+    provenance,
+  };
+}

package/src/service.ts

@@ -1,4 +1,11 @@
 import { createArtifactStore } from './artifact-store';
+import {
+  embeddingIndexStatus,
+  indexKnowledgeEmbeddings,
+  searchVectorIndex,
+  type EmbeddingIndexOptions,
+  type EmbeddingSearchOptions,
+} from './embeddings';
 import { consumeOpenFilesOutbox } from './outbox-consume';
 import { getKnowledgeDbStats, migrateKnowledgeDb, openKnowledgeDb } from './knowledge-db';
 import { ingestOpenFilesManifest } from './manifest-ingest';
@@ -13,7 +20,7 @@ import {
   type StorageContract,
   type StorageValidationResult,
 } from './storage-contract';
-import { initializeWikiLayout } from './wiki-layout';
+import { initializeWikiLayout, recordWikiLayoutCatalog } from './wiki-layout';
 import {
   ensureKnowledgeWorkspace,
   readKnowledgeConfig,
@@ -128,6 +135,7 @@ export class KnowledgeService {
     const db = openKnowledgeDb(workspace.knowledgeDbPath);
     try {
       recordStorageObjects(db, result.artifacts);
+      recordWikiLayoutCatalog(db, result.artifacts);
     } finally {
       db.close();
     }
@@ -183,6 +191,29 @@ export class KnowledgeService {
   modelRegistry(): ModelRegistryEntry[] {
     return listModelRegistry(this.config());
   }
+
+  embeddingStatus() {
+    const workspace = this.ensureWorkspace();
+    return embeddingIndexStatus(workspace.knowledgeDbPath);
+  }
+
+  async indexEmbeddings(options: Omit<EmbeddingIndexOptions, 'dbPath' | 'config'> = {}) {
+    const workspace = this.ensureWorkspace();
+    return indexKnowledgeEmbeddings({
+      ...options,
+      dbPath: workspace.knowledgeDbPath,
+      config: this.config(),
+    });
+  }
+
+  async semanticSearch(options: Omit<EmbeddingSearchOptions, 'dbPath' | 'config'>) {
+    const workspace = this.ensureWorkspace();
+    return searchVectorIndex({
+      ...options,
+      dbPath: workspace.knowledgeDbPath,
+      config: this.config(),
+    });
+  }
 }
 
 export function createKnowledgeService(options: KnowledgeServiceOptions = {}): KnowledgeService {

package/src/source-resolver.ts

@@ -1,5 +1,6 @@
 import type { Database } from 'bun:sqlite';
 import { migrateKnowledgeDb, openKnowledgeDb } from './knowledge-db';
+import { sourceProvenance, type KnowledgeProvenance } from './provenance';
 import { catalogSourceUriForRef, parseSourceRef, revisionIdForSourceRef } from './source-ref';
 import { assertWriteAllowed, recordAuditEvent, type SafetyPolicy } from './safety';
 
@@ -38,6 +39,7 @@ export interface ResolvedSourceChunk {
   end_offset: number | null;
   metadata: Record<string, unknown>;
   evidence: SourceResolverEvidence;
+  provenance: KnowledgeProvenance;
 }
 
 export interface ResolvedSourceCitation {
@@ -48,6 +50,7 @@ export interface ResolvedSourceCitation {
   start_offset: number | null;
   end_offset: number | null;
   evidence: SourceResolverEvidence;
+  provenance: KnowledgeProvenance;
 }
 
 export interface SourceResolveResult {
@@ -326,6 +329,19 @@ export async function resolveOpenFilesSource(options: SourceResolveOptions): Pro
           end_offset: row.end_offset,
           resolved_at: resolvedAt,
         };
+        const provenance = sourceProvenance({
+          source_ref: evidence.source_ref,
+          source_uri: evidence.source_uri,
+          source_kind: source.kind,
+          source_revision_id: evidence.source_revision_id,
+          revision: evidence.revision,
+          hash: evidence.hash,
+          chunk_id: row.id,
+          start_offset: row.start_offset,
+          end_offset: row.end_offset,
+          status: metadataString(metadata, ['status']),
+          resolver: evidence.resolver,
+        });
         return {
           id: row.id,
           kind: row.kind,
@@ -336,6 +352,7 @@ export async function resolveOpenFilesSource(options: SourceResolveOptions): Pro
           end_offset: row.end_offset,
           metadata,
           evidence,
+          provenance,
         };
       });
 
@@ -347,6 +364,7 @@ export async function resolveOpenFilesSource(options: SourceResolveOptions): Pro
         start_offset: chunk.start_offset,
         end_offset: chunk.end_offset,
         evidence: chunk.evidence,
+        provenance: chunk.provenance,
       }));
 
       recordAuditEvent(db, {

package/src/wiki-layout.ts

@@ -1,4 +1,7 @@
+import { createHash } from 'node:crypto';
+import type { Database } from 'bun:sqlite';
 import type { ArtifactStore } from './artifact-store';
+import { generatedArtifactProvenance, type GeneratedArtifactProvenance } from './provenance';
 import {
   artifactKindForKey,
   hashArtifactBody,
@@ -14,6 +17,13 @@ export interface WikiLayoutInitResult {
   written: string[];
 }
 
+interface CatalogArtifact {
+  key: string;
+  uri: string;
+  hash?: string;
+  metadata?: Record<string, unknown>;
+}
+
 function todayParts(now: Date): { year: string; month: string; day: string } {
   const year = String(now.getUTCFullYear());
   const month = String(now.getUTCMonth() + 1).padStart(2, '0');
@@ -21,6 +31,10 @@ function todayParts(now: Date): { year: string; month: string; day: string } {
   return { year, month, day };
 }
 
+function stableId(prefix: string, value: string): string {
+  return `${prefix}_${createHash('sha256').update(value).digest('hex').slice(0, 20)}`;
+}
+
 export function agentSchemaTemplate(): string {
   return `# Knowledge Agent Schema v1
 
@@ -106,6 +120,13 @@ export async function initializeWikiLayout(store: ArtifactStore, now = new Date(
       uri: result.uri,
       kind: artifactKindForKey(entry.key),
       content_type: entry.content_type,
+      metadata: {
+        provenance: generatedArtifactProvenance({
+          generated_from: 'wiki_layout_init',
+          artifact_key: entry.key,
+          citation_required: entry.key.startsWith('wiki/') || entry.key.startsWith('indexes/'),
+        }),
+      },
       ...hashArtifactBody(entry.body),
     };
   }));
@@ -118,3 +139,73 @@ export async function initializeWikiLayout(store: ArtifactStore, now = new Date(
     written: [schemaKey, rootIndexKey, wikiReadmeKey, logKey],
   };
 }
+
+function provenanceFor(artifact: CatalogArtifact): GeneratedArtifactProvenance {
+  const existing = artifact.metadata?.provenance;
+  if (existing && typeof existing === 'object' && !Array.isArray(existing)) {
+    return existing as GeneratedArtifactProvenance;
+  }
+  return generatedArtifactProvenance({
+    generated_from: 'wiki_layout_init',
+    artifact_key: artifact.key,
+  });
+}
+
+export function recordWikiLayoutCatalog(db: Database, artifacts: CatalogArtifact[], now = new Date()): void {
+  const timestamp = now.toISOString();
+  const rootIndex = artifacts.find((artifact) => artifact.key.endsWith('indexes/root.md'));
+  const wikiReadme = artifacts.find((artifact) => artifact.key.endsWith('wiki/README.md'));
+
+  if (rootIndex) {
+    db.run(
+      `INSERT INTO knowledge_indexes (id, kind, name, artifact_uri, shard_key, metadata_json, created_at, updated_at)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+       ON CONFLICT(kind, name, shard_key) DO UPDATE SET
+         artifact_uri = excluded.artifact_uri,
+         metadata_json = excluded.metadata_json,
+         updated_at = excluded.updated_at`,
+      [
+        stableId('idx', 'root:indexes/root.md'),
+        'root',
+        'root',
+        rootIndex.uri,
+        'root',
+        JSON.stringify({
+          artifact_key: rootIndex.key,
+          content_hash: rootIndex.hash ?? null,
+          provenance: provenanceFor(rootIndex),
+        }),
+        timestamp,
+        timestamp,
+      ],
+    );
+  }
+
+  if (wikiReadme) {
+    db.run(
+      `INSERT INTO wiki_pages (id, path, title, artifact_uri, content_hash, status, metadata_json, created_at, updated_at)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+       ON CONFLICT(path) DO UPDATE SET
+         title = excluded.title,
+         artifact_uri = excluded.artifact_uri,
+         content_hash = excluded.content_hash,
+         status = excluded.status,
+         metadata_json = excluded.metadata_json,
+         updated_at = excluded.updated_at`,
+      [
+        stableId('wiki', 'wiki/README.md'),
+        'wiki/README.md',
+        'Wiki',
+        wikiReadme.uri,
+        wikiReadme.hash ?? null,
+        'active',
+        JSON.stringify({
+          artifact_key: wikiReadme.key,
+          provenance: provenanceFor(wikiReadme),
+        }),
+        timestamp,
+        timestamp,
+      ],
+    );
+  }
+}

package/src/workspace.ts

@@ -39,6 +39,12 @@ export interface KnowledgeConfig {
     preferred_ref: 'open-files';
     allowed_schemes: string[];
   };
+  embeddings?: {
+    default_model?: string;
+    dimensions?: number;
+    batch_size?: number;
+    max_parallel_calls?: number;
+  };
   providers?: {
     default_model?: string;
     aliases?: Record<string, string>;
@@ -136,6 +142,12 @@ export function defaultKnowledgeConfig(): KnowledgeConfig {
         default_model: 'deepseek-chat',
       },
     },
+    embeddings: {
+      default_model: 'openai:text-embedding-3-small',
+      dimensions: 1536,
+      batch_size: 64,
+      max_parallel_calls: 4,
+    },
     safety: {
       network: {
         web_search_enabled: false,