@semiont/make-meaning 0.4.18 → 0.4.20

package/dist/index.js

@@ -10000,7 +10000,7 @@ import { WorkingTreeStore } from "@semiont/content";
 var import_rxjs = __toESM(require_cjs(), 1);
 var import_operators = __toESM(require_operators(), 1);
 import { EventQuery } from "@semiont/event-sourcing";
-import { didToAgent, burstBuffer } from "@semiont/core";
+import { didToAgent, burstBuffer, errField } from "@semiont/core";
 import { resourceId as makeResourceId, annotationId as makeAnnotationId, findBodyItem } from "@semiont/core";
 
 // src/batch-utils.ts
@@ -10109,7 +10109,7 @@ var GraphDBConsumer = class _GraphDBConsumer {
       this.logger.error("Failed to apply event to graph", {
         eventType: storedEvent.type,
         resourceId: storedEvent.resourceId,
-        error
+        error: errField(error)
       });
     }
   }
@@ -10148,7 +10148,7 @@ var GraphDBConsumer = class _GraphDBConsumer {
         this.logger.error("Failed to process batch run", {
           eventType: run[0].type,
           runSize: run.length,
-          error
+          error: errField(error)
         });
       }
       const last = run[run.length - 1];
@@ -10218,7 +10218,8 @@ var GraphDBConsumer = class _GraphDBConsumer {
       archived: false,
       dateCreated: (/* @__PURE__ */ new Date()).toISOString(),
       wasAttributedTo: didToAgent(event.userId),
-      creationMethod: event.payload.creationMethod
+      creationMethod: event.payload.creationMethod,
+      ...event.payload.storageUri ? { storageUri: event.payload.storageUri } : {}
     };
   }
   /**
@@ -10304,8 +10305,7 @@ var GraphDBConsumer = class _GraphDBConsumer {
         } catch (error) {
           this.logger.error("Error in annotation.body.updated handler", {
             annotationId: event.payload.annotationId,
-            error,
-            stack: error instanceof Error ? error.stack : void 0
+            error: errField(error)
           });
         }
         break;
@@ -10417,23 +10417,25 @@ var GraphDBConsumer = class _GraphDBConsumer {
 // src/smelter.ts
 var import_rxjs2 = __toESM(require_cjs(), 1);
 var import_operators2 = __toESM(require_operators(), 1);
-import { EventQuery as EventQuery2 } from "@semiont/event-sourcing";
-import { burstBuffer as burstBuffer2 } from "@semiont/core";
+import { burstBuffer as burstBuffer2, errField as errField2 } from "@semiont/core";
 import { resourceId as makeResourceId2, annotationId as makeAnnotationId2 } from "@semiont/core";
 import { chunkText, DEFAULT_CHUNKING_CONFIG } from "@semiont/vectors";
 import { getExactText, getTargetSelector } from "@semiont/api-client";
 var Smelter = class _Smelter {
-  constructor(eventStore, eventBus, vectorStore, embeddingProvider, contentStore, logger, chunkingConfig) {
-    this.eventStore = eventStore;
+  constructor(_eventStore, eventBus, vectorStore, embeddingProvider, contentStore, embeddingStore, viewStorage, logger, chunkingConfig) {
     this.eventBus = eventBus;
     this.vectorStore = vectorStore;
     this.embeddingProvider = embeddingProvider;
     this.contentStore = contentStore;
+    this.embeddingStore = embeddingStore;
+    this.viewStorage = viewStorage;
     this.logger = logger;
     this.chunkingConfig = chunkingConfig ?? DEFAULT_CHUNKING_CONFIG;
   }
   static SMELTER_RELEVANT_EVENTS = /* @__PURE__ */ new Set([
     "yield:created",
+    "yield:updated",
+    "yield:representation-added",
     "mark:archived",
     "mark:added",
     "mark:removed"
@@ -10485,68 +10487,121 @@ var Smelter = class _Smelter {
     this.logger.info("Smelter actor stopped");
   }
   /**
-   * Rebuild the vector store from persisted embedding events in the event log.
-   * Reads all embedding:computed / embedding:deleted events and replays them.
-   * Bypasses the live pipeline — reads directly from the event store.
+   * Rebuild the vector store from the EmbeddingStore (.semiont/embeddings/).
+   *
+   * For each stored file, checks whether the model matches the configured
+   * provider. On mismatch, re-embeds from the stored text and overwrites the
+   * file before upserting into Qdrant. On match, loads the stored vectors
+   * directly — no embedding provider calls needed.
    */
   async rebuildAll() {
-    this.logger.info("Rebuilding vector store from events");
+    this.logger.info("Rebuilding vector store from EmbeddingStore");
     await this.vectorStore.clearAll();
-    const allResourceIds = await this.eventStore.log.getAllResourceIds();
-    this.logger.info("Found resources to scan", { count: allResourceIds.length });
-    const query = new EventQuery2(this.eventStore.log.storage);
-    let indexed = 0;
-    for (const rid of allResourceIds) {
-      const events = await query.getResourceEvents(makeResourceId2(rid));
-      const embeddingEvents = events.filter(
-        (e) => e.type === "embedding:computed" || e.type === "embedding:deleted"
-      );
-      if (embeddingEvents.length === 0) continue;
-      const lastEvent = embeddingEvents[embeddingEvents.length - 1];
-      if (lastEvent.type === "embedding:deleted" && !lastEvent.payload.annotationId) {
-        continue;
-      }
-      const deletedAnnotations = /* @__PURE__ */ new Set();
-      for (const e of embeddingEvents) {
-        if (e.type === "embedding:deleted") {
-          const payload = e.payload;
-          if (payload.annotationId) deletedAnnotations.add(String(payload.annotationId));
-        }
-      }
-      const resourceChunks = [];
-      for (const e of embeddingEvents) {
-        if (e.type !== "embedding:computed") continue;
-        const payload = e.payload;
-        if (payload.annotationId) {
-          if (deletedAnnotations.has(String(payload.annotationId))) continue;
-          await this.vectorStore.upsertAnnotationVector(
-            makeAnnotationId2(String(payload.annotationId)),
-            payload.embedding,
-            {
-              annotationId: makeAnnotationId2(String(payload.annotationId)),
-              resourceId: makeResourceId2(e.resourceId),
-              motivation: "",
-              entityTypes: [],
-              exactText: payload.chunkText
-            }
-          );
-        } else {
-          resourceChunks.push({
-            chunkIndex: payload.chunkIndex,
-            text: payload.chunkText,
-            embedding: payload.embedding
-          });
-        }
-      }
-      if (resourceChunks.length > 0) {
-        await this.vectorStore.upsertResourceVectors(
-          makeResourceId2(rid),
-          resourceChunks
+    const currentModel = this.embeddingProvider.model();
+    const currentDimensions = this.embeddingProvider.dimensions();
+    const resourceIds = await this.embeddingStore.getAllResourceIds();
+    this.logger.info("Found resource embedding files", { count: resourceIds.length });
+    let resourcesIndexed = 0;
+    for (const rid of resourceIds) {
+      const resourceId7 = makeResourceId2(rid);
+      const stored = await this.embeddingStore.readResourceEmbeddings(resourceId7);
+      if (!stored || stored.chunks.length === 0) continue;
+      let chunks;
+      if (stored.model !== currentModel) {
+        this.logger.info("Re-embedding resource (model mismatch)", {
+          resourceId: rid,
+          storedModel: stored.model,
+          currentModel
+        });
+        const texts = stored.chunks.map((c) => c.text);
+        const embeddings = await this.embeddingProvider.embedBatch(texts);
+        chunks = stored.chunks.map((c, i) => ({
+          chunkIndex: c.chunkIndex,
+          text: c.text,
+          embedding: embeddings[i]
+        }));
+        await this.embeddingStore.writeResourceChunks(resourceId7, currentModel, currentDimensions, chunks);
+      } else {
+        chunks = stored.chunks;
+      }
+      await this.vectorStore.upsertResourceVectors(resourceId7, chunks);
+      resourcesIndexed++;
+    }
+    const annotationIds = await this.embeddingStore.getAllAnnotationIds();
+    this.logger.info("Found annotation embedding files", { count: annotationIds.length });
+    let annotationsIndexed = 0;
+    for (const aid of annotationIds) {
+      const annotationId3 = makeAnnotationId2(aid);
+      const stored = await this.embeddingStore.readAnnotationEmbedding(annotationId3);
+      if (!stored) continue;
+      let embedding;
+      if (stored.model !== currentModel) {
+        this.logger.info("Re-embedding annotation (model mismatch)", {
+          annotationId: aid,
+          storedModel: stored.model,
+          currentModel
+        });
+        embedding = await this.embeddingProvider.embed(stored.text);
+        await this.embeddingStore.writeAnnotationEmbedding(
+          annotationId3,
+          makeResourceId2(stored.resourceId),
+          currentModel,
+          currentDimensions,
+          stored.text,
+          embedding,
+          stored.motivation,
+          stored.entityTypes
         );
+      } else {
+        embedding = stored.embedding;
       }
-      indexed++;
+      const payload = {
+        annotationId: annotationId3,
+        resourceId: makeResourceId2(stored.resourceId),
+        motivation: stored.motivation,
+        entityTypes: stored.entityTypes,
+        exactText: stored.text
+      };
+      await this.vectorStore.upsertAnnotationVector(annotationId3, embedding, payload);
+      annotationsIndexed++;
+    }
+    const storedResourceIdSet = new Set(resourceIds);
+    const allViews = await this.viewStorage.getAll();
+    let backfilled = 0;
+    for (const view of allViews) {
+      const ridStr = view.resource["@id"];
+      if (storedResourceIdSet.has(ridStr)) continue;
+      if (view.resource.archived) continue;
+      if (!view.resource.storageUri) continue;
+      let content;
+      try {
+        content = await this.contentStore.retrieve(view.resource.storageUri);
+      } catch (err) {
+        this.logger.warn("Smelter back-fill skipped \u2014 content missing", {
+          resourceId: ridStr,
+          storageUri: view.resource.storageUri,
+          error: err instanceof Error ? err.message : String(err)
+        });
+        continue;
+      }
+      const text = new TextDecoder().decode(content);
+      if (!text.trim()) continue;
+      const chunks = chunkText(text, this.chunkingConfig);
+      if (chunks.length === 0) continue;
+      const rid = makeResourceId2(ridStr);
+      const embeddings = await this.embeddingProvider.embedBatch(chunks);
+      const embeddingChunks = chunks.map((chunkText2, i) => ({
+        chunkIndex: i,
+        text: chunkText2,
+        embedding: embeddings[i]
+      }));
+      await this.embeddingStore.writeResourceChunks(rid, currentModel, currentDimensions, embeddingChunks);
+      await this.vectorStore.upsertResourceVectors(rid, embeddingChunks);
+      backfilled++;
+      resourcesIndexed++;
+      this.logger.info("Smelter back-filled missing resource embedding", { resourceId: ridStr });
     }
-    this.logger.info("Vector store rebuild complete", { resourcesIndexed: indexed });
+    this.logger.info("Vector store rebuild complete", { resourcesIndexed, annotationsIndexed, backfilled });
   }
   async processBatch(events) {
     const runs = partitionByType(events);
@@ -10561,15 +10616,13 @@ var Smelter = class _Smelter {
         this.logger.error("Smelter failed to process batch run", {
           eventType: run[0].type,
           runSize: run.length,
-          error
+          error: errField2(error)
         });
       }
     }
   }
   /**
    * Batch-optimized processing for consecutive events of the same type.
-   * Collects all texts across events, embeds in a single embedBatch() call,
-   * then distributes results back to their respective resources/annotations.
    */
   async applyBatchByType(events) {
     const type = events[0].type;
@@ -10587,8 +10640,8 @@ var Smelter = class _Smelter {
     }
   }
   /**
-   * Batch-embed chunks from multiple resource.created events in a single
-   * embedBatch() call, then emit events and index per resource.
+   * Batch-embed chunks from multiple yield:created events in a single
+   * embedBatch() call, then write to EmbeddingStore and index per resource.
    */
   async batchResourceCreated(events) {
     const resourceData = [];
@@ -10613,18 +10666,12 @@ var Smelter = class _Smelter {
     const dimensions = this.embeddingProvider.dimensions();
     let offset = 0;
     for (const { rid, chunks } of resourceData) {
-      const embeddingChunks = chunks.map((text, i) => {
-        const embedding = allEmbeddings[offset + i];
-        this.eventBus.get("embedding:compute").next({
-          resourceId: rid,
-          chunkIndex: i,
-          chunkText: text,
-          embedding,
-          model,
-          dimensions
-        });
-        return { chunkIndex: i, text, embedding };
-      });
+      const embeddingChunks = chunks.map((text, i) => ({
+        chunkIndex: i,
+        text,
+        embedding: allEmbeddings[offset + i]
+      }));
+      await this.embeddingStore.writeResourceChunks(rid, model, dimensions, embeddingChunks);
       await this.vectorStore.upsertResourceVectors(rid, embeddingChunks);
       this.logger.debug("Smelter batch-indexed resource", {
         resourceId: String(rid),
@@ -10634,8 +10681,8 @@ var Smelter = class _Smelter {
     }
   }
   /**
-   * Batch-embed exact texts from multiple annotation.added events in a
-   * single embedBatch() call, then emit events and index per annotation.
+   * Batch-embed exact texts from multiple mark:added events in a single
+   * embedBatch() call, then write to EmbeddingStore and index per annotation.
    */
   async batchAnnotationAdded(events) {
     const annotationData = [];
@@ -10650,30 +10697,34 @@ var Smelter = class _Smelter {
         rid: makeResourceId2(event.resourceId),
         aid: makeAnnotationId2(annotation.id),
         exactText,
-        annotation
+        motivation: annotation.motivation ?? "",
+        entityTypes: annotation.entityTypes ?? []
       });
     }
     if (annotationData.length === 0) return;
     const allEmbeddings = await this.embeddingProvider.embedBatch(
       annotationData.map((a) => a.exactText)
     );
+    const model = this.embeddingProvider.model();
+    const dimensions = this.embeddingProvider.dimensions();
     for (let i = 0; i < annotationData.length; i++) {
-      const { rid, aid, exactText, annotation } = annotationData[i];
+      const { rid, aid, exactText, motivation, entityTypes } = annotationData[i];
       const embedding = allEmbeddings[i];
-      this.eventBus.get("embedding:compute").next({
-        resourceId: rid,
-        annotationId: aid,
-        chunkIndex: 0,
-        chunkText: exactText,
+      await this.embeddingStore.writeAnnotationEmbedding(
+        aid,
+        rid,
+        model,
+        dimensions,
+        exactText,
         embedding,
-        model: this.embeddingProvider.model(),
-        dimensions: this.embeddingProvider.dimensions()
-      });
+        motivation,
+        entityTypes
+      );
       const payload = {
         annotationId: aid,
         resourceId: rid,
-        motivation: annotation.motivation ?? "",
-        entityTypes: annotation.entityTypes ?? [],
+        motivation,
+        entityTypes,
         exactText
       };
       await this.vectorStore.upsertAnnotationVector(aid, embedding, payload);
@@ -10696,19 +10747,24 @@ var Smelter = class _Smelter {
     }
   }
   async processEvent(storedEvent) {
-    const event = storedEvent;
-    switch (event.type) {
+    switch (storedEvent.type) {
       case "yield:created":
-        await this.handleResourceCreated(event);
+        await this.handleResourceCreated(storedEvent);
+        break;
+      case "yield:updated":
+        await this.handleResourceUpdated(storedEvent);
+        break;
+      case "yield:representation-added":
+        await this.handleRepresentationAdded(storedEvent);
         break;
       case "mark:archived":
-        await this.handleResourceArchived(event);
+        await this.handleResourceArchived(storedEvent);
         break;
       case "mark:added":
-        await this.handleAnnotationAdded(event);
+        await this.handleAnnotationAdded(storedEvent);
         break;
       case "mark:removed":
-        await this.handleAnnotationRemoved(event);
+        await this.handleAnnotationRemoved(storedEvent);
         break;
     }
   }
@@ -10743,18 +10799,13 @@ var Smelter = class _Smelter {
       dimensions,
       heapMB: Math.round(process.memoryUsage().heapUsed / 1024 / 1024)
     });
-    const embeddingChunks = chunks.map((text2, i) => {
-      this.eventBus.get("embedding:compute").next({
-        resourceId: rid,
-        chunkIndex: i,
-        chunkText: text2,
-        embedding: embeddings[i],
-        model,
-        dimensions
-      });
-      return { chunkIndex: i, text: text2, embedding: embeddings[i] };
-    });
-    this.logger.info("Smelter emitted events", {
+    const embeddingChunks = chunks.map((text2, i) => ({
+      chunkIndex: i,
+      text: text2,
+      embedding: embeddings[i]
+    }));
+    await this.embeddingStore.writeResourceChunks(rid, model, dimensions, embeddingChunks);
+    this.logger.info("Smelter wrote resource embeddings to store", {
       resourceId: String(rid),
       chunkCount: embeddingChunks.length,
       heapMB: Math.round(process.memoryUsage().heapUsed / 1024 / 1024)
@@ -10766,13 +10817,51 @@ var Smelter = class _Smelter {
       heapMB: Math.round(process.memoryUsage().heapUsed / 1024 / 1024)
     });
   }
+  /**
+   * Re-embed a resource whose content has changed in-place.
+   *
+   * Used by yield:updated and yield:representation-added handlers. Reads the
+   * current storageUri from the materialized view (which is updated before the
+   * EventBus fires), deletes stale Qdrant vectors, and overwrites the
+   * EmbeddingStore file with fresh chunks.
+   */
+  async reembedResource(rid) {
+    const view = await this.viewStorage.get(rid);
+    const storageUri = view?.resource.storageUri;
+    if (!storageUri) return;
+    const content = await this.contentStore.retrieve(storageUri);
+    if (!content) return;
+    const text = new TextDecoder().decode(content);
+    if (!text.trim()) return;
+    const chunks = chunkText(text, this.chunkingConfig);
+    if (chunks.length === 0) return;
+    const embeddings = await this.embeddingProvider.embedBatch(chunks);
+    const model = this.embeddingProvider.model();
+    const dimensions = this.embeddingProvider.dimensions();
+    const embeddingChunks = chunks.map((chunkText2, i) => ({
+      chunkIndex: i,
+      text: chunkText2,
+      embedding: embeddings[i]
+    }));
+    await this.embeddingStore.writeResourceChunks(rid, model, dimensions, embeddingChunks);
+    await this.vectorStore.deleteResourceVectors(rid);
+    await this.vectorStore.upsertResourceVectors(rid, embeddingChunks);
+    this.logger.debug("Smelter re-embedded resource", {
+      resourceId: String(rid),
+      chunks: embeddingChunks.length
+    });
+  }
+  async handleResourceUpdated(event) {
+    await this.reembedResource(makeResourceId2(event.resourceId));
+  }
+  async handleRepresentationAdded(event) {
+    await this.reembedResource(makeResourceId2(event.resourceId));
+  }
   async handleResourceArchived(event) {
     const rid = makeResourceId2(event.resourceId);
     await this.vectorStore.deleteResourceVectors(rid);
-    this.eventBus.get("embedding:delete").next({ resourceId: rid });
-    this.logger.debug("Smelter deleted resource vectors", {
-      resourceId: String(rid)
-    });
+    await this.embeddingStore.deleteResourceEmbeddings(rid);
+    this.logger.debug("Smelter deleted resource vectors", { resourceId: String(rid) });
   }
   async handleAnnotationAdded(event) {
     await new Promise((resolve2) => setTimeout(resolve2, 0));
@@ -10790,20 +10879,25 @@ var Smelter = class _Smelter {
       heapMB: Math.round(process.memoryUsage().heapUsed / 1024 / 1024)
     });
     const embedding = await this.embeddingProvider.embed(exactText);
-    this.eventBus.get("embedding:compute").next({
-      resourceId: rid,
-      annotationId: aid,
-      chunkIndex: 0,
-      chunkText: exactText,
+    const model = this.embeddingProvider.model();
+    const dimensions = this.embeddingProvider.dimensions();
+    const motivation = annotation.motivation ?? "";
+    const entityTypes = annotation.entityTypes ?? [];
+    await this.embeddingStore.writeAnnotationEmbedding(
+      aid,
+      rid,
+      model,
+      dimensions,
+      exactText,
       embedding,
-      model: this.embeddingProvider.model(),
-      dimensions: this.embeddingProvider.dimensions()
-    });
+      motivation,
+      entityTypes
+    );
     const payload = {
       annotationId: aid,
       resourceId: rid,
-      motivation: annotation.motivation ?? "",
-      entityTypes: annotation.entityTypes ?? [],
+      motivation,
+      entityTypes,
       exactText
     };
     await this.vectorStore.upsertAnnotationVector(aid, embedding, payload);
@@ -10815,16 +10909,134 @@ var Smelter = class _Smelter {
   async handleAnnotationRemoved(event) {
     const annotationId3 = String(event.payload.annotationId);
     if (!annotationId3) return;
-    const rid = makeResourceId2(event.resourceId);
     const aid = makeAnnotationId2(annotationId3);
     await this.vectorStore.deleteAnnotationVector(aid);
-    this.eventBus.get("embedding:delete").next({
-      resourceId: rid,
-      annotationId: aid
-    });
-    this.logger.debug("Smelter deleted annotation vector", {
-      annotationId: String(aid)
-    });
+    await this.embeddingStore.deleteAnnotationEmbedding(aid);
+    this.logger.debug("Smelter deleted annotation vector", { annotationId: String(aid) });
+  }
+};
+
+// src/embedding-store.ts
+import { promises as fs } from "fs";
+import * as path from "path";
+import { getShardPath } from "@semiont/event-sourcing";
+var EmbeddingStore = class {
+  constructor(project) {
+    this.project = project;
+  }
+  // ── Path helpers ────────────────────────────────────────────────────────────
+  resourceFilePath(resourceId7) {
+    const [ab, cd] = getShardPath(String(resourceId7));
+    return path.join(this.project.embeddingsDir, ab, cd, `${String(resourceId7)}.jsonl`);
+  }
+  annotationFilePath(annotationId3) {
+    const [ab, cd] = getShardPath(String(annotationId3));
+    return path.join(this.project.embeddingsDir, ab, cd, `${String(annotationId3)}.json`);
+  }
+  // ── Resource embeddings ─────────────────────────────────────────────────────
+  async writeResourceChunks(resourceId7, model, dimensions, chunks) {
+    const filePath = this.resourceFilePath(resourceId7);
+    await fs.mkdir(path.dirname(filePath), { recursive: true });
+    const header = JSON.stringify({ model, dimensions });
+    const lines = chunks.map(
+      (c) => JSON.stringify({ chunkIndex: c.chunkIndex, text: c.text, embedding: c.embedding })
+    );
+    await fs.writeFile(filePath, [header, ...lines].join("\n") + "\n", "utf-8");
+  }
+  async readResourceEmbeddings(resourceId7) {
+    const filePath = this.resourceFilePath(resourceId7);
+    try {
+      const content = await fs.readFile(filePath, "utf-8");
+      const lines = content.trim().split("\n").filter((l) => l.trim());
+      if (lines.length === 0) return null;
+      const header = JSON.parse(lines[0]);
+      const chunks = lines.slice(1).map((l) => JSON.parse(l));
+      return { model: header.model, dimensions: header.dimensions, chunks };
+    } catch (err) {
+      if (err.code === "ENOENT") return null;
+      throw err;
+    }
+  }
+  async deleteResourceEmbeddings(resourceId7) {
+    const filePath = this.resourceFilePath(resourceId7);
+    try {
+      await fs.unlink(filePath);
+    } catch (err) {
+      if (err.code !== "ENOENT") throw err;
+    }
+  }
+  // ── Annotation embeddings ───────────────────────────────────────────────────
+  async writeAnnotationEmbedding(annotationId3, resourceId7, model, dimensions, text, embedding, motivation, entityTypes) {
+    const filePath = this.annotationFilePath(annotationId3);
+    await fs.mkdir(path.dirname(filePath), { recursive: true });
+    const data = {
+      model,
+      dimensions,
+      resourceId: String(resourceId7),
+      text,
+      embedding,
+      motivation,
+      entityTypes
+    };
+    await fs.writeFile(filePath, JSON.stringify(data), "utf-8");
+  }
+  async readAnnotationEmbedding(annotationId3) {
+    const filePath = this.annotationFilePath(annotationId3);
+    try {
+      const content = await fs.readFile(filePath, "utf-8");
+      return JSON.parse(content);
+    } catch (err) {
+      if (err.code === "ENOENT") return null;
+      throw err;
+    }
+  }
+  async deleteAnnotationEmbedding(annotationId3) {
+    const filePath = this.annotationFilePath(annotationId3);
+    try {
+      await fs.unlink(filePath);
+    } catch (err) {
+      if (err.code !== "ENOENT") throw err;
+    }
+  }
+  // ── Scan ────────────────────────────────────────────────────────────────────
+  /**
+   * Scan embeddings directory and return all resource IDs (from *.jsonl files).
+   */
+  async getAllResourceIds() {
+    return this.scanIds((name) => name.endsWith(".jsonl"), ".jsonl");
+  }
+  /**
+   * Scan embeddings directory and return all annotation IDs (from *.json files).
+   */
+  async getAllAnnotationIds() {
+    return this.scanIds((name) => name.endsWith(".json"), ".json");
+  }
+  async scanIds(filter, ext) {
+    const base = this.project.embeddingsDir;
+    try {
+      await fs.access(base);
+    } catch {
+      return [];
+    }
+    const results = [];
+    const scan = async (dir) => {
+      let entries;
+      try {
+        entries = await fs.readdir(dir, { withFileTypes: true });
+      } catch {
+        return;
+      }
+      for (const entry of entries) {
+        const full = path.join(dir, entry.name);
+        if (entry.isDirectory()) {
+          await scan(full);
+        } else if (filter(entry.name)) {
+          results.push(entry.name.slice(0, -ext.length));
+        }
+      }
+    };
+    await scan(base);
+    return results;
   }
 };
 
@@ -10856,12 +11068,15 @@ async function createKnowledgeBase(eventStore, project, graphDb, eventBus, logge
   };
   if (options?.vectorStore && options?.embeddingProvider) {
     kb.vectors = options.vectorStore;
+    const embeddingStore = new EmbeddingStore(project);
     kb.smelter = new Smelter(
       eventStore,
       eventBus,
       options.vectorStore,
       options.embeddingProvider,
       content,
+      embeddingStore,
+      views,
       logger.child({ component: "smelter" }),
       options.chunkingConfig
     );
@@ -10876,7 +11091,7 @@ async function createKnowledgeBase(eventStore, project, graphDb, eventBus, logge
 // src/gatherer.ts
 var import_rxjs3 = __toESM(require_cjs(), 1);
 var import_operators3 = __toESM(require_operators(), 1);
-import { annotationId as makeAnnotationId3, resourceId } from "@semiont/core";
+import { annotationId as makeAnnotationId3, resourceId, errField as errField3 } from "@semiont/core";
 
 // src/generation/resource-generation.ts
 async function generateResourceSummary(resourceName, content, entityTypes, client) {
@@ -11664,7 +11879,7 @@ var Gatherer = class {
     } catch (error) {
       this.logger.error("Gather annotation context failed", {
         annotationId: event.annotationId,
-        error
+        error: errField3(error)
       });
       resultBus.get("gather:failed").next({
         correlationId: event.correlationId,
@@ -11693,7 +11908,7 @@ var Gatherer = class {
     } catch (error) {
       this.logger.error("Gather resource context failed", {
         resourceId: event.resourceId,
-        error
+        error: errField3(error)
       });
       resultBus.get("gather:resource-failed").next({
         correlationId: event.correlationId,
@@ -11722,7 +11937,7 @@ var Gatherer = class {
 // src/matcher.ts
 var import_rxjs4 = __toESM(require_cjs(), 1);
 var import_operators4 = __toESM(require_operators(), 1);
-import { resourceId as resourceId2 } from "@semiont/core";
+import { resourceId as resourceId2, errField as errField4 } from "@semiont/core";
 import { getResourceId as getResourceId4, getResourceEntityTypes as getResourceEntityTypes4 } from "@semiont/api-client";
 var Matcher = class {
   constructor(kb, eventBus, logger, inferenceClient, embeddingProvider) {
@@ -11771,7 +11986,7 @@ var Matcher = class {
     } catch (error) {
       this.logger.error("Bind search failed", {
         referenceId: event.referenceId,
-        error
+        error: errField4(error)
       });
       this.eventBus.scope(event.resourceId).get("match:search-failed").next({
         correlationId: event.correlationId,
@@ -12041,7 +12256,7 @@ For each candidate, output a line with the number and score, like:
 // src/stower.ts
 var import_rxjs5 = __toESM(require_cjs(), 1);
 var import_operators5 = __toESM(require_operators(), 1);
-import { promises as fs } from "fs";
+import { promises as fs2 } from "fs";
 import { resourceId as resourceId3, userId as makeUserId, annotationId as makeAnnotationId4, CREATION_METHODS, generateUuid } from "@semiont/core";
 import { resolveStorageUri } from "@semiont/event-sourcing";
 var Stower = class {
@@ -12069,9 +12284,7 @@ var Stower = class {
       pipe("job:start", (e) => this.handleJobStart(e)),
       pipe("job:report-progress", (e) => this.handleJobReportProgress(e)),
       pipe("job:complete", (e) => this.handleJobComplete(e)),
-      pipe("job:fail", (e) => this.handleJobFail(e)),
-      pipe("embedding:compute", (e) => this.handleEmbeddingComputed(e)),
-      pipe("embedding:delete", (e) => this.handleEmbeddingDeleted(e))
+      pipe("job:fail", (e) => this.handleJobFail(e))
     ).subscribe({
       error: (err) => this.logger.error("Stower pipeline error", { error: err })
     });
@@ -12275,7 +12488,7 @@ var Stower = class {
     if (event.storageUri) {
       const absPath = this.kb.content.resolveUri(event.storageUri);
       try {
-        await fs.access(absPath);
+        await fs2.access(absPath);
       } catch {
         this.logger.warn("Unarchive failed: file not found at storageUri", { storageUri: event.storageUri });
         return;
@@ -12375,33 +12588,6 @@ var Stower = class {
       }
     });
   }
-  async handleEmbeddingComputed(event) {
-    await this.kb.eventStore.appendEvent({
-      type: "embedding:computed",
-      resourceId: resourceId3(event.resourceId),
-      userId: makeUserId("did:web:system:smelter"),
-      version: 1,
-      payload: {
-        annotationId: event.annotationId,
-        chunkIndex: event.chunkIndex,
-        chunkText: event.chunkText,
-        embedding: event.embedding,
-        model: event.model,
-        dimensions: event.dimensions
-      }
-    });
-  }
-  async handleEmbeddingDeleted(event) {
-    await this.kb.eventStore.appendEvent({
-      type: "embedding:deleted",
-      resourceId: resourceId3(event.resourceId),
-      userId: makeUserId("did:web:system:smelter"),
-      version: 1,
-      payload: {
-        annotationId: event.annotationId
-      }
-    });
-  }
   async stop() {
     this.subscription?.unsubscribe();
     this.subscription = null;
@@ -12412,25 +12598,25 @@ var Stower = class {
 // src/browser.ts
 var import_rxjs6 = __toESM(require_cjs(), 1);
 var import_operators6 = __toESM(require_operators(), 1);
-import { promises as fs3 } from "fs";
-import * as path2 from "path";
-import { resourceId as resourceId4, annotationId } from "@semiont/core";
+import { promises as fs4 } from "fs";
+import * as path3 from "path";
+import { resourceId as resourceId4, annotationId, errField as errField5 } from "@semiont/core";
 import { getExactText as getExactText2, getTargetSource as getTargetSource2, getTargetSelector as getTargetSelector3, getResourceEntityTypes as getResourceEntityTypes5, getBodySource as getBodySource2 } from "@semiont/api-client";
-import { EventQuery as EventQuery3 } from "@semiont/event-sourcing";
+import { EventQuery as EventQuery2 } from "@semiont/event-sourcing";
 import { getEntityTypes as getEntityTypes2 } from "@semiont/ontology";
 
 // src/views/entity-types-reader.ts
-import { promises as fs2 } from "fs";
-import * as path from "path";
+import { promises as fs3 } from "fs";
+import * as path2 from "path";
 async function readEntityTypesProjection(project) {
-  const entityTypesPath = path.join(
+  const entityTypesPath = path2.join(
     project.stateDir,
     "projections",
     "__system__",
     "entitytypes.json"
   );
   try {
-    const content = await fs2.readFile(entityTypesPath, "utf-8");
+    const content = await fs3.readFile(entityTypesPath, "utf-8");
     const projection = JSON.parse(content);
     return projection.entityTypes || [];
   } catch (error) {
@@ -12473,7 +12659,7 @@ var Browser = class {
   // ========================================================================
   async handleBrowseResource(event) {
     try {
-      const eventQuery = new EventQuery3(this.kb.eventStore.log.storage);
+      const eventQuery = new EventQuery2(this.kb.eventStore.log.storage);
       const events = await eventQuery.getResourceEvents(resourceId4(event.resourceId));
       const stored = await this.kb.eventStore.views.materializer.materialize(events, resourceId4(event.resourceId));
       if (!stored) {
@@ -12497,7 +12683,7 @@ var Browser = class {
         }
       });
     } catch (error) {
-      this.logger.error("Browse resource failed", { resourceId: event.resourceId, error });
+      this.logger.error("Browse resource failed", { resourceId: event.resourceId, error: errField5(error) });
       this.eventBus.get("browse:resource-failed").next({
         correlationId: event.correlationId,
         message: error instanceof Error ? error.message : String(error)
@@ -12527,7 +12713,7 @@ var Browser = class {
         }
       });
     } catch (error) {
-      this.logger.error("Browse resources failed", { error });
+      this.logger.error("Browse resources failed", { error: errField5(error) });
       this.eventBus.get("browse:resources-failed").next({
         correlationId: event.correlationId,
         message: error instanceof Error ? error.message : String(error)
@@ -12545,7 +12731,7 @@ var Browser = class {
         }
       });
     } catch (error) {
-      this.logger.error("Browse annotations failed", { resourceId: event.resourceId, error });
+      this.logger.error("Browse annotations failed", { resourceId: event.resourceId, error: errField5(error) });
       this.eventBus.get("browse:annotations-failed").next({
         correlationId: event.correlationId,
         message: error instanceof Error ? error.message : String(error)
@@ -12577,7 +12763,7 @@ var Browser = class {
         }
       });
     } catch (error) {
-      this.logger.error("Browse annotation failed", { resourceId: event.resourceId, annotationId: event.annotationId, error });
+      this.logger.error("Browse annotation failed", { resourceId: event.resourceId, annotationId: event.annotationId, error: errField5(error) });
       this.eventBus.get("browse:annotation-failed").next({
         correlationId: event.correlationId,
         message: error instanceof Error ? error.message : String(error)
@@ -12586,7 +12772,7 @@ var Browser = class {
   }
   async handleBrowseEvents(event) {
     try {
-      const eventQuery = new EventQuery3(this.kb.eventStore.log.storage);
+      const eventQuery = new EventQuery2(this.kb.eventStore.log.storage);
       const filters = {
         resourceId: resourceId4(event.resourceId)
       };
@@ -12609,7 +12795,7 @@ var Browser = class {
         }
       });
     } catch (error) {
-      this.logger.error("Browse events failed", { resourceId: event.resourceId, error });
+      this.logger.error("Browse events failed", { resourceId: event.resourceId, error: errField5(error) });
       this.eventBus.get("browse:events-failed").next({
         correlationId: event.correlationId,
         message: error instanceof Error ? error.message : String(error)
@@ -12626,7 +12812,7 @@ var Browser = class {
         });
         return;
       }
-      const eventQuery = new EventQuery3(this.kb.eventStore.log.storage);
+      const eventQuery = new EventQuery2(this.kb.eventStore.log.storage);
       const allEvents = await eventQuery.queryEvents({ resourceId: resourceId4(event.resourceId) });
       const annotationEvents = allEvents.filter((stored) => {
         const p = stored.payload;
@@ -12645,7 +12831,7 @@ var Browser = class {
         }
       });
     } catch (error) {
-      this.logger.error("Browse annotation history failed", { resourceId: event.resourceId, annotationId: event.annotationId, error });
+      this.logger.error("Browse annotation history failed", { resourceId: event.resourceId, annotationId: event.annotationId, error: errField5(error) });
       this.eventBus.get("browse:annotation-history-failed").next({
         correlationId: event.correlationId,
         message: error instanceof Error ? error.message : String(error)
@@ -12687,7 +12873,7 @@ var Browser = class {
         response: { referencedBy }
       });
     } catch (error) {
-      this.logger.error("Referenced-by query failed", { resourceId: event.resourceId, error });
+      this.logger.error("Referenced-by query failed", { resourceId: event.resourceId, error: errField5(error) });
       this.eventBus.get("browse:referenced-by-failed").next({
         correlationId: event.correlationId,
         message: error instanceof Error ? error.message : String(error)
@@ -12702,7 +12888,7 @@ var Browser = class {
         response: { entityTypes }
       });
     } catch (error) {
-      this.logger.error("Entity types read failed", { error });
+      this.logger.error("Entity types read failed", { error: errField5(error) });
       this.eventBus.get("browse:entity-types-failed").next({
         correlationId: event.correlationId,
         message: error instanceof Error ? error.message : String(error)
@@ -12715,8 +12901,8 @@ var Browser = class {
   async handleBrowseDirectory(event) {
     const { correlationId, path: reqPath, sort = "name" } = event;
     const projectRoot = this.project.root;
-    const resolved = path2.resolve(projectRoot, reqPath);
-    if (!resolved.startsWith(projectRoot + path2.sep) && resolved !== projectRoot) {
+    const resolved = path3.resolve(projectRoot, reqPath);
+    if (!resolved.startsWith(projectRoot + path3.sep) && resolved !== projectRoot) {
       this.eventBus.get("browse:directory-failed").next({
         correlationId,
         path: reqPath,
@@ -12726,7 +12912,7 @@ var Browser = class {
     }
     let dirents;
     try {
-      dirents = await fs3.readdir(resolved, { withFileTypes: true, encoding: "utf8" });
+      dirents = await fs4.readdir(resolved, { withFileTypes: true, encoding: "utf8" });
     } catch (err) {
       const msg = err.code === "ENOENT" ? "path not found" : String(err);
       this.eventBus.get("browse:directory-failed").next({
@@ -12740,16 +12926,16 @@ var Browser = class {
     const allViews = await this.views.getAll();
     const prefix = `file://${resolved}`;
     const viewsByUri = new Map(
-      allViews.filter((v) => v.resource.storageUri?.startsWith(prefix + "/") || v.resource.storageUri?.startsWith(prefix + path2.sep)).map((v) => [v.resource.storageUri, v])
+      allViews.filter((v) => v.resource.storageUri?.startsWith(prefix + "/") || v.resource.storageUri?.startsWith(prefix + path3.sep)).map((v) => [v.resource.storageUri, v])
     );
     const entries = [];
     for (const dirent of visible) {
-      const entryPath = path2.join(resolved, dirent.name);
-      const relPath = path2.relative(projectRoot, entryPath);
+      const entryPath = path3.join(resolved, dirent.name);
+      const relPath = path3.relative(projectRoot, entryPath);
       if (dirent.isDirectory()) {
         let mtime = (/* @__PURE__ */ new Date(0)).toISOString();
         try {
-          const stat = await fs3.stat(entryPath);
+          const stat = await fs4.stat(entryPath);
           mtime = stat.mtime.toISOString();
         } catch {
         }
@@ -12759,7 +12945,7 @@ var Browser = class {
         let size = 0;
         let mtime = (/* @__PURE__ */ new Date(0)).toISOString();
         try {
-          const stat = await fs3.stat(entryPath);
+          const stat = await fs4.stat(entryPath);
           size = stat.size;
           mtime = stat.mtime.toISOString();
         } catch {
@@ -13430,9 +13616,7 @@ async function exportBackup(options, output) {
   for (const [streamId, events] of streamData) {
     streamSummaries.push({
       stream: streamId,
-      eventCount: events.length,
-      firstChecksum: events[0].metadata.checksum || "",
-      lastChecksum: events[events.length - 1].metadata.checksum || ""
+      eventCount: events.length
     });
   }
   const manifestHeader = {
@@ -13501,26 +13685,11 @@ async function replayEventStream(jsonl, eventBus, resolveBlob, contentStore, log
     annotationsCreated: 0,
     entityTypesAdded: 0
   };
-  let hashChainValid = true;
-  for (let i = 1; i < storedEvents.length; i++) {
-    const prev = storedEvents[i - 1];
-    const curr = storedEvents[i];
-    if (curr.metadata.prevEventHash && prev.metadata.checksum) {
-      if (curr.metadata.prevEventHash !== prev.metadata.checksum) {
-        logger?.warn("Hash chain break", {
-          index: i,
-          expected: prev.metadata.checksum,
-          got: curr.metadata.prevEventHash
-        });
-        hashChainValid = false;
-      }
-    }
-  }
   for (const stored of storedEvents) {
     await replayEvent(stored, eventBus, resolveBlob, contentStore, stats, logger);
     stats.eventsReplayed++;
   }
-  return { stats, hashChainValid };
+  return { stats };
 }
 async function replayEvent(event, eventBus, resolveBlob, contentStore, stats, logger) {
   switch (event.type) {
@@ -13749,7 +13918,6 @@ async function importBackup(archive, options) {
   const resolveBlob = buildBlobResolver(entries);
   const systemData = entries.get(".semiont/events/__system__.jsonl");
   let stats = { eventsReplayed: 0, resourcesCreated: 0, annotationsCreated: 0, entityTypesAdded: 0 };
-  let hashChainValid = true;
   if (systemData) {
     const result = await replayEventStream(
       systemData.toString("utf8"),
@@ -13759,7 +13927,6 @@ async function importBackup(archive, options) {
       logger
     );
     stats = mergeStats(stats, result.stats);
-    if (!result.hashChainValid) hashChainValid = false;
   }
   for (const summary of streamSummaries) {
     if (summary.stream === "__system__") continue;
@@ -13776,10 +13943,9 @@ async function importBackup(archive, options) {
       logger
     );
     stats = mergeStats(stats, result.stats);
-    if (!result.hashChainValid) hashChainValid = false;
   }
-  logger?.info("Backup import complete", { ...stats, hashChainValid });
-  return { manifest: header, stats, hashChainValid };
+  logger?.info("Backup import complete", { ...stats });
+  return { manifest: header, stats };
 }
 function mergeStats(a, b) {
   return {