@hasna/knowledge 0.2.13 → 0.2.14

package/src/embeddings.ts

@@ -0,0 +1,516 @@
+import { createHash } from 'node:crypto';
+import type { Database } from 'bun:sqlite';
+import { migrateKnowledgeDb, openKnowledgeDb } from './knowledge-db';
+import { assertProviderCredentials, parseModelRef, providerSettings, type AiProviderId } from './providers';
+import { sourceProvenance, type KnowledgeProvenance } from './provenance';
+import type { KnowledgeConfig } from './workspace';
+
+export interface EmbeddingRuntimeOptions {
+  config?: KnowledgeConfig;
+  env?: Record<string, string | undefined>;
+  modelRef?: string;
+  dimensions?: number;
+  fake?: boolean;
+  batchSize?: number;
+  maxParallelCalls?: number;
+}
+
+export interface EmbeddingIndexOptions extends EmbeddingRuntimeOptions {
+  dbPath: string;
+  limit?: number;
+  sourceRevisionId?: string;
+  now?: Date;
+}
+
+export interface EmbeddingSearchOptions extends EmbeddingRuntimeOptions {
+  dbPath: string;
+  query: string;
+  limit?: number;
+}
+
+export interface EmbeddingUsage {
+  input_tokens: number;
+}
+
+export interface EmbeddingVectorResult {
+  provider: AiProviderId;
+  model: string;
+  dimensions: number;
+  vectors: number[][];
+  usage: EmbeddingUsage;
+}
+
+export interface EmbeddingIndexResult {
+  provider: AiProviderId;
+  model: string;
+  dimensions: number;
+  chunks_seen: number;
+  chunks_embedded: number;
+  embeddings_upserted: number;
+  vector_entries_upserted: number;
+  usage: EmbeddingUsage;
+}
+
+export interface EmbeddingStatusResult {
+  total_embeddings: number;
+  total_vector_entries: number;
+  indexes: Array<{
+    provider: string;
+    model: string;
+    dimensions: number;
+    entries: number;
+    updated_at: string | null;
+  }>;
+}
+
+export interface SemanticSearchResult {
+  provider: AiProviderId;
+  model: string;
+  dimensions: number;
+  query: string;
+  results: Array<{
+    chunk_id: string;
+    score: number;
+    text: string;
+    source_uri: string | null;
+    source_ref: string | null;
+    revision: string | null;
+    hash: string | null;
+    provenance: KnowledgeProvenance | null;
+  }>;
+}
+
+interface CandidateChunk {
+  id: string;
+  text: string;
+  token_count: number | null;
+  start_offset: number | null;
+  end_offset: number | null;
+  metadata_json: string;
+  source_revision_id: string | null;
+  revision: string | null;
+  hash: string | null;
+  source_uri: string | null;
+  source_kind: string | null;
+}
+
+interface VectorRow {
+  chunk_id: string;
+  text: string;
+  vector_json: string;
+  vector_norm: number;
+  source_uri: string | null;
+  source_ref: string | null;
+  revision: string | null;
+  hash: string | null;
+  metadata_json: string;
+}
+
+export const DEFAULT_EMBEDDING_MODEL_REF = 'openai:text-embedding-3-small';
+export const DEFAULT_EMBEDDING_DIMENSIONS = 1536;
+
+function embeddingConfig(config?: KnowledgeConfig) {
+  return (config as KnowledgeConfig & {
+    embeddings?: {
+      default_model?: string;
+      dimensions?: number;
+      batch_size?: number;
+      max_parallel_calls?: number;
+    };
+  } | undefined)?.embeddings ?? {};
+}
+
+function stableId(prefix: string, value: string): string {
+  return `${prefix}_${createHash('sha256').update(value).digest('hex').slice(0, 20)}`;
+}
+
+function parseJsonObject(value: string | null | undefined): Record<string, unknown> {
+  if (!value) return {};
+  try {
+    const parsed = JSON.parse(value);
+    return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? parsed as Record<string, unknown> : {};
+  } catch {
+    return {};
+  }
+}
+
+function metadataString(metadata: Record<string, unknown>, keys: string[]): string | null {
+  for (const key of keys) {
+    const value = metadata[key];
+    if (typeof value === 'string' && value.length > 0) return value;
+  }
+  return null;
+}
+
+function metadataNumber(metadata: Record<string, unknown>, keys: string[]): number | null {
+  for (const key of keys) {
+    const value = metadata[key];
+    if (typeof value === 'number' && Number.isFinite(value)) return value;
+  }
+  return null;
+}
+
+function vectorNorm(vector: number[]): number {
+  return Math.sqrt(vector.reduce((sum, value) => sum + value * value, 0));
+}
+
+function cosineSimilarity(a: number[], b: number[], bNorm = vectorNorm(b)): number {
+  const aNorm = vectorNorm(a);
+  if (aNorm === 0 || bNorm === 0) return 0;
+  const length = Math.min(a.length, b.length);
+  let dot = 0;
+  for (let i = 0; i < length; i += 1) dot += a[i] * b[i];
+  return dot / (aNorm * bNorm);
+}
+
+function deterministicVector(text: string, dimensions: number): number[] {
+  const bytes = createHash('sha256').update(text).digest();
+  return Array.from({ length: dimensions }, (_, index) => {
+    const value = bytes[index % bytes.length] / 255;
+    return Number((value * 2 - 1).toFixed(6));
+  });
+}
+
+async function openAiEmbeddingModel(model: string, config?: KnowledgeConfig, env: Record<string, string | undefined> = process.env): Promise<unknown> {
+  assertProviderCredentials('openai', config, env);
+  const settings = providerSettings(config, 'openai');
+  const { createOpenAI } = await import('@ai-sdk/openai');
+  const openai = createOpenAI({
+    apiKey: env[settings.api_key_env],
+    baseURL: settings.base_url,
+  }) as unknown as {
+    embeddingModel?: (modelId: string) => unknown;
+    textEmbedding?: (modelId: string) => unknown;
+    textEmbeddingModel?: (modelId: string) => unknown;
+  };
+  if (openai.embeddingModel) return openai.embeddingModel(model);
+  if (openai.textEmbedding) return openai.textEmbedding(model);
+  if (openai.textEmbeddingModel) return openai.textEmbeddingModel(model);
+  throw new Error('OpenAI provider does not expose an embedding model factory.');
+}
+
+export function resolveEmbeddingModelRef(modelRef?: string, config?: KnowledgeConfig): string {
+  if (!modelRef || modelRef === 'default' || modelRef === 'embedding') {
+    return embeddingConfig(config).default_model ?? DEFAULT_EMBEDDING_MODEL_REF;
+  }
+  return modelRef;
+}
+
+export async function embedTexts(texts: string[], options: EmbeddingRuntimeOptions = {}): Promise<EmbeddingVectorResult> {
+  const modelRef = resolveEmbeddingModelRef(options.modelRef, options.config);
+  const parsed = parseModelRef(modelRef);
+  if (parsed.provider !== 'openai') {
+    throw new Error(`Embedding provider ${parsed.provider} is not supported yet. Use openai:text-embedding-3-small.`);
+  }
+  const dimensions = options.dimensions ?? embeddingConfig(options.config).dimensions ?? DEFAULT_EMBEDDING_DIMENSIONS;
+
+  if (options.fake) {
+    return {
+      provider: parsed.provider,
+      model: parsed.model,
+      dimensions,
+      vectors: texts.map((text) => deterministicVector(text, dimensions)),
+      usage: { input_tokens: texts.reduce((sum, text) => sum + Math.max(1, Math.ceil(text.split(/\s+/).filter(Boolean).length * 1.25)), 0) },
+    };
+  }
+
+  const { embedMany } = await import('ai');
+  const model = await openAiEmbeddingModel(parsed.model, options.config, options.env);
+  const result = await embedMany({
+    model: model as never,
+    values: texts,
+    maxParallelCalls: options.maxParallelCalls ?? embeddingConfig(options.config).max_parallel_calls,
+    providerOptions: {
+      openai: {
+        dimensions,
+      },
+    },
+  });
+  const vectors = result.embeddings as number[][];
+  return {
+    provider: parsed.provider,
+    model: parsed.model,
+    dimensions: vectors[0]?.length ?? dimensions,
+    vectors,
+    usage: { input_tokens: result.usage?.tokens ?? 0 },
+  };
+}
+
+function selectCandidateChunks(db: Database, options: {
+  provider: AiProviderId;
+  model: string;
+  limit: number;
+  sourceRevisionId?: string;
+}): CandidateChunk[] {
+  const baseQuery =
+    `SELECT
+       c.id,
+       c.text,
+       c.token_count,
+       c.start_offset,
+       c.end_offset,
+       c.metadata_json,
+       c.source_revision_id,
+       sr.revision,
+       sr.hash,
+       s.uri AS source_uri,
+       s.kind AS source_kind
+     FROM chunks c
+     LEFT JOIN source_revisions sr ON sr.id = c.source_revision_id
+     LEFT JOIN sources s ON s.id = sr.source_id
+     LEFT JOIN vector_index_entries v
+       ON v.chunk_id = c.id AND v.provider = ? AND v.model = ?
+     WHERE v.id IS NULL`;
+  const suffix = `
+     ORDER BY c.created_at ASC, c.ordinal ASC
+     LIMIT ?`;
+  if (options.sourceRevisionId) {
+    return db.query<CandidateChunk, [string, string, string, number]>(
+      `${baseQuery} AND c.source_revision_id = ?${suffix}`,
+    ).all(options.provider, options.model, options.sourceRevisionId, options.limit);
+  }
+  return db.query<CandidateChunk, [string, string, number]>(
+    `${baseQuery}${suffix}`,
+  ).all(options.provider, options.model, options.limit);
+}
+
+function provenanceForChunk(row: CandidateChunk): KnowledgeProvenance {
+  const metadata = parseJsonObject(row.metadata_json);
+  const existing = metadata.provenance;
+  if (existing && typeof existing === 'object' && !Array.isArray(existing)) return existing as KnowledgeProvenance;
+  return sourceProvenance({
+    source_ref: metadataString(metadata, ['source_ref']),
+    source_uri: row.source_uri ?? metadataString(metadata, ['source_uri']),
+    source_kind: row.source_kind ?? metadataString(metadata, ['source_kind']),
+    source_revision_id: row.source_revision_id,
+    revision: row.revision ?? metadataString(metadata, ['revision']),
+    hash: row.hash ?? metadataString(metadata, ['hash']),
+    chunk_id: row.id,
+    start_offset: row.start_offset ?? metadataNumber(metadata, ['start_offset']),
+    end_offset: row.end_offset ?? metadataNumber(metadata, ['end_offset']),
+    status: metadataString(metadata, ['status']),
+    resolver: 'open-files-read-only',
+  });
+}
+
+function upsertVectors(db: Database, rows: CandidateChunk[], embedding: EmbeddingVectorResult, now: string): number {
+  const insertEmbedding = db.prepare(`
+    INSERT INTO chunk_embeddings (id, chunk_id, provider, model, dimensions, vector_json, created_at)
+    VALUES (?, ?, ?, ?, ?, ?, ?)
+    ON CONFLICT(chunk_id, provider, model) DO UPDATE SET
+      dimensions = excluded.dimensions,
+      vector_json = excluded.vector_json,
+      created_at = excluded.created_at
+  `);
+  const insertVector = db.prepare(`
+    INSERT INTO vector_index_entries (
+      id, chunk_id, source_revision_id, provider, model, dimensions, vector_json, vector_norm,
+      source_uri, source_ref, revision, hash, start_offset, end_offset, token_count, status,
+      metadata_json, created_at, updated_at
+    )
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    ON CONFLICT(chunk_id, provider, model) DO UPDATE SET
+      source_revision_id = excluded.source_revision_id,
+      dimensions = excluded.dimensions,
+      vector_json = excluded.vector_json,
+      vector_norm = excluded.vector_norm,
+      source_uri = excluded.source_uri,
+      source_ref = excluded.source_ref,
+      revision = excluded.revision,
+      hash = excluded.hash,
+      start_offset = excluded.start_offset,
+      end_offset = excluded.end_offset,
+      token_count = excluded.token_count,
+      status = excluded.status,
+      metadata_json = excluded.metadata_json,
+      updated_at = excluded.updated_at
+  `);
+
+  const write = db.transaction(() => {
+    for (let index = 0; index < rows.length; index += 1) {
+      const row = rows[index];
+      const vector = embedding.vectors[index];
+      if (!vector) continue;
+      const metadata = parseJsonObject(row.metadata_json);
+      const provenance = provenanceForChunk(row);
+      const sourceRef = provenance.source_ref ?? metadataString(metadata, ['source_ref']);
+      const sourceUri = provenance.source_uri ?? row.source_uri ?? metadataString(metadata, ['source_uri']);
+      const revision = provenance.revision ?? row.revision ?? metadataString(metadata, ['revision']);
+      const hash = provenance.hash ?? row.hash ?? metadataString(metadata, ['hash']);
+      const status = provenance.status ?? metadataString(metadata, ['status']) ?? 'active';
+      const vectorJson = JSON.stringify(vector);
+      insertEmbedding.run(
+        stableId('emb', `${row.id}\u0000${embedding.provider}\u0000${embedding.model}`),
+        row.id,
+        embedding.provider,
+        embedding.model,
+        embedding.dimensions,
+        vectorJson,
+        now,
+      );
+      insertVector.run(
+        stableId('vec', `${row.id}\u0000${embedding.provider}\u0000${embedding.model}`),
+        row.id,
+        row.source_revision_id,
+        embedding.provider,
+        embedding.model,
+        embedding.dimensions,
+        vectorJson,
+        vectorNorm(vector),
+        sourceUri,
+        sourceRef,
+        revision,
+        hash,
+        provenance.start_offset,
+        provenance.end_offset,
+        row.token_count,
+        status,
+        JSON.stringify({
+          ...metadata,
+          provenance,
+          embedded_at: now,
+        }),
+        now,
+        now,
+      );
+    }
+  });
+  write();
+  return rows.length;
+}
+
+export async function indexKnowledgeEmbeddings(options: EmbeddingIndexOptions): Promise<EmbeddingIndexResult> {
+  const modelRef = resolveEmbeddingModelRef(options.modelRef, options.config);
+  const parsed = parseModelRef(modelRef);
+  if (parsed.provider !== 'openai') throw new Error(`Embedding provider ${parsed.provider} is not supported yet.`);
+  const now = (options.now ?? new Date()).toISOString();
+  const limit = Math.max(1, Math.min(options.limit ?? 100, 1000));
+  migrateKnowledgeDb(options.dbPath);
+  const readDb = openKnowledgeDb(options.dbPath);
+  let rows: CandidateChunk[];
+  try {
+    rows = selectCandidateChunks(readDb, {
+      provider: parsed.provider,
+      model: parsed.model,
+      limit,
+      sourceRevisionId: options.sourceRevisionId,
+    });
+  } finally {
+    readDb.close();
+  }
+
+  if (rows.length === 0) {
+    return {
+      provider: parsed.provider,
+      model: parsed.model,
+      dimensions: options.dimensions ?? embeddingConfig(options.config).dimensions ?? DEFAULT_EMBEDDING_DIMENSIONS,
+      chunks_seen: 0,
+      chunks_embedded: 0,
+      embeddings_upserted: 0,
+      vector_entries_upserted: 0,
+      usage: { input_tokens: 0 },
+    };
+  }
+
+  const embedding = await embedTexts(rows.map((row) => row.text), options);
+  const writeDb = openKnowledgeDb(options.dbPath);
+  try {
+    const upserted = upsertVectors(writeDb, rows, embedding, now);
+    return {
+      provider: embedding.provider,
+      model: embedding.model,
+      dimensions: embedding.dimensions,
+      chunks_seen: rows.length,
+      chunks_embedded: rows.length,
+      embeddings_upserted: upserted,
+      vector_entries_upserted: upserted,
+      usage: embedding.usage,
+    };
+  } finally {
+    writeDb.close();
+  }
+}
+
+export function embeddingIndexStatus(dbPath: string): EmbeddingStatusResult {
+  migrateKnowledgeDb(dbPath);
+  const db = openKnowledgeDb(dbPath);
+  try {
+    const totalEmbeddings = db.query<{ n: number }, []>('SELECT COUNT(*) AS n FROM chunk_embeddings').get()?.n ?? 0;
+    const totalVectorEntries = db.query<{ n: number }, []>('SELECT COUNT(*) AS n FROM vector_index_entries').get()?.n ?? 0;
+    const indexes = db.query<{
+      provider: string;
+      model: string;
+      dimensions: number;
+      entries: number;
+      updated_at: string | null;
+    }, []>(
+      `SELECT provider, model, dimensions, COUNT(*) AS entries, MAX(updated_at) AS updated_at
+       FROM vector_index_entries
+       GROUP BY provider, model, dimensions
+       ORDER BY provider, model`,
+    ).all();
+    return {
+      total_embeddings: totalEmbeddings,
+      total_vector_entries: totalVectorEntries,
+      indexes,
+    };
+  } finally {
+    db.close();
+  }
+}
+
+export async function searchVectorIndex(options: EmbeddingSearchOptions): Promise<SemanticSearchResult> {
+  const modelRef = resolveEmbeddingModelRef(options.modelRef, options.config);
+  const parsed = parseModelRef(modelRef);
+  const limit = Math.max(1, Math.min(options.limit ?? 10, 100));
+  const embedded = await embedTexts([options.query], options);
+  const queryVector = embedded.vectors[0] ?? [];
+
+  migrateKnowledgeDb(options.dbPath);
+  const db = openKnowledgeDb(options.dbPath);
+  try {
+    const rows = db.query<VectorRow, [string, string]>(
+      `SELECT
+         v.chunk_id,
+         c.text,
+         v.vector_json,
+         v.vector_norm,
+         v.source_uri,
+         v.source_ref,
+         v.revision,
+         v.hash,
+         v.metadata_json
+       FROM vector_index_entries v
+       JOIN chunks c ON c.id = v.chunk_id
+       WHERE v.provider = ? AND v.model = ? AND v.status = 'active'`,
+    ).all(parsed.provider, parsed.model);
+
+    const scored = rows.map((row) => {
+      const vector = JSON.parse(row.vector_json) as number[];
+      const metadata = parseJsonObject(row.metadata_json);
+      const provenance = metadata.provenance && typeof metadata.provenance === 'object' && !Array.isArray(metadata.provenance)
+        ? metadata.provenance as KnowledgeProvenance
+        : null;
+      return {
+        chunk_id: row.chunk_id,
+        score: cosineSimilarity(queryVector, vector, row.vector_norm),
+        text: row.text,
+        source_uri: row.source_uri,
+        source_ref: row.source_ref,
+        revision: row.revision,
+        hash: row.hash,
+        provenance,
+      };
+    }).sort((a, b) => b.score - a.score).slice(0, limit);
+
+    return {
+      provider: parsed.provider,
+      model: parsed.model,
+      dimensions: embedded.dimensions,
+      query: options.query,
+      results: scored,
+    };
+  } finally {
+    db.close();
+  }
+}

package/src/knowledge-db.ts

@@ -1,7 +1,7 @@
 import { Database } from 'bun:sqlite';
 import { ensureParentDir } from './workspace';
 
-export const CURRENT_SCHEMA_VERSION = 3;
+export const CURRENT_SCHEMA_VERSION = 4;
 
 export interface KnowledgeDbStats {
   schema_version: number;
@@ -17,6 +17,8 @@ export interface KnowledgeDbStats {
   audit_events: number;
   approval_gates: number;
   storage_objects: number;
+  embeddings: number;
+  vector_entries: number;
 }
 
 const MIGRATION_1 = `
@@ -236,6 +238,39 @@ INSERT OR IGNORE INTO schema_versions(version, applied_at)
 VALUES (3, datetime('now'));
 `;
 
+const MIGRATION_4 = `
+CREATE TABLE IF NOT EXISTS vector_index_entries (
+  id TEXT PRIMARY KEY,
+  chunk_id TEXT NOT NULL REFERENCES chunks(id) ON DELETE CASCADE,
+  source_revision_id TEXT REFERENCES source_revisions(id) ON DELETE CASCADE,
+  provider TEXT NOT NULL,
+  model TEXT NOT NULL,
+  dimensions INTEGER NOT NULL,
+  vector_json TEXT NOT NULL,
+  vector_norm REAL NOT NULL,
+  source_uri TEXT,
+  source_ref TEXT,
+  revision TEXT,
+  hash TEXT,
+  start_offset INTEGER,
+  end_offset INTEGER,
+  token_count INTEGER,
+  status TEXT NOT NULL DEFAULT 'active',
+  metadata_json TEXT NOT NULL DEFAULT '{}',
+  created_at TEXT NOT NULL,
+  updated_at TEXT NOT NULL,
+  UNIQUE(chunk_id, provider, model)
+);
+
+CREATE INDEX IF NOT EXISTS idx_vector_index_provider_model ON vector_index_entries(provider, model);
+CREATE INDEX IF NOT EXISTS idx_vector_index_source_revision ON vector_index_entries(source_revision_id);
+CREATE INDEX IF NOT EXISTS idx_vector_index_source_uri ON vector_index_entries(source_uri);
+CREATE INDEX IF NOT EXISTS idx_vector_index_status ON vector_index_entries(status);
+
+INSERT OR IGNORE INTO schema_versions(version, applied_at)
+VALUES (4, datetime('now'));
+`;
+
 export function openKnowledgeDb(path: string): Database {
   ensureParentDir(path);
   const db = new Database(path);
@@ -250,6 +285,7 @@ export function migrateKnowledgeDb(path: string): { path: string; schema_version
     db.exec(MIGRATION_1);
     if (getSchemaVersion(db) < 2) db.exec(MIGRATION_2);
     if (getSchemaVersion(db) < 3) db.exec(MIGRATION_3);
+    if (getSchemaVersion(db) < 4) db.exec(MIGRATION_4);
     return { path, schema_version: getSchemaVersion(db) };
   } finally {
     db.close();
@@ -283,6 +319,8 @@ export function getKnowledgeDbStats(path: string): KnowledgeDbStats {
       audit_events: count(db, 'audit_events'),
       approval_gates: count(db, 'approval_gates'),
       storage_objects: count(db, 'storage_objects'),
+      embeddings: count(db, 'chunk_embeddings'),
+      vector_entries: count(db, 'vector_index_entries'),
     };
   } finally {
     db.close();

package/src/mcp.js

@@ -131,6 +131,44 @@ export function buildServer() {
     return jsonText({ ok: true, models: service.modelRegistry() });
   });
 
+  registerTool(server, 'ok_embeddings_status', 'Embedding index status', 'Inspect local embedding/vector index counts by provider and model', {
+    scope: scopeField,
+  }, async ({ scope }) => {
+    const service = createKnowledgeService({ scope });
+    return jsonText({ ok: true, ...service.embeddingStatus() });
+  });
+
+  registerTool(server, 'ok_embeddings_index', 'Index embeddings', 'Embed unindexed knowledge chunks into the local vector index', {
+    scope: scopeField,
+    limit: z.number().optional().describe('Maximum chunks to embed'),
+    model: z.string().optional().describe('Embedding model ref, default openai:text-embedding-3-small'),
+    dimensions: z.number().optional().describe('Embedding dimensions for deterministic fake mode'),
+    fake: z.boolean().optional().describe('Use deterministic fake embeddings for local tests'),
+  }, async ({ scope, limit, model, dimensions, fake }) => {
+    const service = createKnowledgeService({ scope });
+    try {
+      return jsonText({ ok: true, ...await service.indexEmbeddings({ limit, modelRef: model, dimensions, fake }) });
+    } catch (error) {
+      return errorText(error instanceof Error ? error.message : String(error));
+    }
+  });
+
+  registerTool(server, 'ok_semantic_search', 'Semantic search', 'Search the local vector index and return cited chunks with provenance', {
+    scope: scopeField,
+    query: z.string().describe('Semantic query'),
+    limit: z.number().optional().describe('Maximum results'),
+    model: z.string().optional().describe('Embedding model ref, default openai:text-embedding-3-small'),
+    dimensions: z.number().optional().describe('Embedding dimensions for deterministic fake mode'),
+    fake: z.boolean().optional().describe('Use deterministic fake embeddings for local tests'),
+  }, async ({ scope, query, limit, model, dimensions, fake }) => {
+    const service = createKnowledgeService({ scope });
+    try {
+      return jsonText({ ok: true, ...await service.semanticSearch({ query, limit, modelRef: model, dimensions, fake }) });
+    } catch (error) {
+      return errorText(error instanceof Error ? error.message : String(error));
+    }
+  });
+
   registerTool(server, 'ok_add', 'Add a knowledge item', 'Add a new item to the knowledge store', {
     title: z.string().describe('Item title'),
     content: z.string().describe('Item content/body'),

package/src/outbox-consume.ts

@@ -30,6 +30,7 @@ export interface OutboxConsumeResult {
   deleted_sources: number;
   moved_sources: number;
   permission_updates: number;
+  vector_entries_deleted: number;
 }
 
 interface NormalizedOutboxEvent {
@@ -289,12 +290,16 @@ function revisionIdsForEvent(db: Database, sourceId: string, event: NormalizedOu
   ).all(sourceId).map((row) => row.id);
 }
 
-function invalidateRevision(db: Database, revisionId: string): { chunksDeleted: number; embeddingsDeleted: number } {
+function invalidateRevision(db: Database, revisionId: string): { chunksDeleted: number; embeddingsDeleted: number; vectorEntriesDeleted: number } {
   const chunks = db.query<{ id: string }, [string]>('SELECT id FROM chunks WHERE source_revision_id = ?').all(revisionId);
   let embeddingsDeleted = 0;
+  let vectorEntriesDeleted = 0;
   for (const chunk of chunks) {
     const row = db.query<{ n: number }, [string]>('SELECT COUNT(*) AS n FROM chunk_embeddings WHERE chunk_id = ?').get(chunk.id);
     embeddingsDeleted += row?.n ?? 0;
+    const vectorRow = db.query<{ n: number }, [string]>('SELECT COUNT(*) AS n FROM vector_index_entries WHERE chunk_id = ?').get(chunk.id);
+    vectorEntriesDeleted += vectorRow?.n ?? 0;
+    db.run('DELETE FROM vector_index_entries WHERE chunk_id = ?', [chunk.id]);
     db.run('DELETE FROM chunk_embeddings WHERE chunk_id = ?', [chunk.id]);
     db.run('DELETE FROM chunks_fts WHERE chunk_id = ?', [chunk.id]);
   }
@@ -304,7 +309,7 @@ function invalidateRevision(db: Database, revisionId: string): { chunksDeleted:
     'UPDATE source_revisions SET metadata_json = ? WHERE id = ?',
     [mergeJson(revision?.metadata_json, { reindex_required: true, invalidated_at: new Date().toISOString() }), revisionId],
   );
-  return { chunksDeleted: chunks.length, embeddingsDeleted };
+  return { chunksDeleted: chunks.length, embeddingsDeleted, vectorEntriesDeleted };
 }
 
 function isDeleteEvent(eventType: string, status: string | null): boolean {
@@ -349,6 +354,7 @@ export async function consumeOpenFilesOutbox(options: OutboxConsumeOptions): Pro
       const revisionsTouched = new Set<string>();
       let chunksDeleted = 0;
       let embeddingsDeleted = 0;
+      let vectorEntriesDeleted = 0;
       let staleRevisions = 0;
       let deletedSources = 0;
       let movedSources = 0;
@@ -376,6 +382,7 @@ export async function consumeOpenFilesOutbox(options: OutboxConsumeOptions): Pro
           const invalidation = invalidateRevision(db, revisionId);
           chunksDeleted += invalidation.chunksDeleted;
           embeddingsDeleted += invalidation.embeddingsDeleted;
+          vectorEntriesDeleted += invalidation.vectorEntriesDeleted;
           staleRevisions += 1;
         }
 
@@ -429,6 +436,7 @@ export async function consumeOpenFilesOutbox(options: OutboxConsumeOptions): Pro
           revisions: revisionsTouched.size,
           chunks_deleted: chunksDeleted,
           embeddings_deleted: embeddingsDeleted,
+          vector_entries_deleted: vectorEntriesDeleted,
         },
         created_at: now,
       });
@@ -442,6 +450,7 @@ export async function consumeOpenFilesOutbox(options: OutboxConsumeOptions): Pro
         revisions_touched: revisionsTouched.size,
         chunks_deleted: chunksDeleted,
         embeddings_deleted: embeddingsDeleted,
+        vector_entries_deleted: vectorEntriesDeleted,
         stale_revisions: staleRevisions,
         deleted_sources: deletedSources,
         moved_sources: movedSources,