@semiont/make-meaning 0.5.11 → 0.5.13

package/dist/smelter-main.js

@@ -1,4 +1,4 @@
-import { createTomlConfigLoader, accessToken, baseUrl as baseUrl$1, burstBuffer, errField, resourceId, textExtractionOf, decodeRepresentation, busRequest, getResourceEntityTypes, getTargetSelector, getExactText, annotationId, getPrimaryMediaType, getPrimaryRepresentation } from '@semiont/core';
+import { createTomlConfigLoader, accessToken, baseUrl as baseUrl$1, retryWithBackoff, isTransientFetchError, STARTUP_FETCH_RETRY, burstBuffer, errField, resourceId, textExtractionOf, decodeRepresentation, busRequest, getResourceEntityTypes, getTargetSelector, getExactText, annotationId, getPrimaryMediaType, getPrimaryRepresentation } from '@semiont/core';
 import { calculateChecksum } from '@semiont/content';
 import { createEmbeddingProvider, createVectorStore, chunkText } from '@semiont/vectors';
 import { registerVectorIndexSizeProvider, withActorSpan } from '@semiont/observability';
@@ -9732,8 +9732,11 @@ var SMELTER_CHANNELS = [
   "yield:updated",
   "yield:representation-added",
   "mark:archived",
+  "mark:unarchived",
   "mark:added",
-  "mark:removed"
+  "mark:removed",
+  "mark:entity-tag-added",
+  "mark:entity-tag-removed"
 ];
 function createSmelterActorStateUnit(options) {
   const { bus } = options;
@@ -9782,8 +9785,20 @@ function partitionByType(events) {
 }
 
 // src/smelter.ts
+function sameStringSet(a, b) {
+  if (a.length !== b.length) return false;
+  const set = new Set(a);
+  return b.every((t) => set.has(t));
+}
+var WORK_ITEM_TYPES = /* @__PURE__ */ new Set([
+  "smelt:embed",
+  "smelt:restamp",
+  "smelt:purge",
+  "smelt:embed-annotation",
+  "smelt:purge-annotation"
+]);
 function isWorkItem(input) {
-  return input.type.startsWith("smelt:");
+  return WORK_ITEM_TYPES.has(input.type);
 }
 var Smelter = class _Smelter {
   constructor(events$, vectorStore, embeddingProvider, content, bus, chunkingConfig2, timing, logger2) {
@@ -9947,16 +9962,26 @@ var Smelter = class _Smelter {
       case "mark:archived":
         await this.handleResourceArchived(event);
         break;
+      case "mark:unarchived":
+        await this.handleResourceUnarchived(event);
+        break;
       case "mark:added":
         await this.handleAnnotationAdded(event);
         break;
       case "mark:removed":
         await this.handleAnnotationRemoved(event);
         break;
+      case "mark:entity-tag-added":
+      case "mark:entity-tag-removed":
+        await this.restampResource(event);
+        break;
       // Reconcile work items — same handlers, distinct provenance.
       case "smelt:embed":
         await this.embedResource(event, "Reconcile-indexed resource");
         break;
+      case "smelt:restamp":
+        await this.restampResource(event);
+        break;
       case "smelt:purge":
         await this.handleResourcePurge(event);
         break;
@@ -9968,6 +9993,21 @@ var Smelter = class _Smelter {
         break;
     }
   }
+  /**
+   * Payload-only stamp refresh: re-read the resource's CURRENT entity types
+   * (one code path — `resolveEntityTypes` — so any prior drift self-corrects
+   * on first touch) and rewrite the stamp on its existing points. Never calls
+   * the embedding provider (S13): content is unchanged by definition on every
+   * path that lands here. A resource with no points is a no-op — the stamp
+   * rides the next embed.
+   */
+  async restampResource(event) {
+    const rid = event.resourceId;
+    if (!rid) return;
+    const entityTypes = await this.resolveEntityTypes(rid);
+    await this.vectorStore.updateResourceEntityTypes(resourceId(rid), entityTypes);
+    this.logger.info("Restamped resource entity types", { resourceId: rid, entityTypes });
+  }
   async handleResourcePurge(event) {
     const rid = event.resourceId;
     if (!rid) return;
@@ -9985,16 +10025,30 @@ var Smelter = class _Smelter {
   async fetchEmbeddableText(resourceId$1) {
     try {
       const { data, contentType } = await this.content.getBinary(resourceId(resourceId$1));
+      const bytes = Buffer.from(data);
+      const checksum = calculateChecksum(bytes);
       if (textExtractionOf(contentType) !== "decode") {
         this.logger.debug("Skipping resource that does not decode as text", { resourceId: resourceId$1, contentType });
-        return null;
+        return { kind: "skipped", checksum };
       }
-      const bytes = Buffer.from(data);
       const text = decodeRepresentation(bytes, contentType);
-      return text.trim() ? { text, checksum: calculateChecksum(bytes) } : null;
+      return text.trim() ? { kind: "text", text, checksum } : { kind: "skipped", checksum };
     } catch (error) {
       this.logger.warn("Content unavailable for embedding", { resourceId: resourceId$1, error: errField(error) });
-      return null;
+      return { kind: "unavailable" };
+    }
+  }
+  /**
+   * The Smelter's single outbound signal (SMELTER-AXIOMS D3 as amended by
+   * SMELTER-INDEX-SYNC): a per-resource decision report for the barrier
+   * fold. Best-effort — waiters degrade to their bounded timeout; a signal
+   * failure must never fail the embed.
+   */
+  async emitSettled(resourceId, contentChecksum, outcome) {
+    try {
+      await this.bus.emit("smelt:settled", { resourceId, contentChecksum, outcome });
+    } catch (error) {
+      this.logger.warn("Failed to emit smelt:settled", { resourceId, outcome, error: errField(error) });
     }
   }
   /**
@@ -10018,9 +10072,16 @@ var Smelter = class _Smelter {
     const rid = event.resourceId;
     if (!rid) return;
     const fetched = await this.fetchEmbeddableText(rid);
-    if (!fetched) return;
+    if (fetched.kind === "unavailable") return;
+    if (fetched.kind === "skipped") {
+      await this.emitSettled(rid, fetched.checksum, "skipped");
+      return;
+    }
     const chunks = chunkText(fetched.text, this.chunkingConfig);
-    if (chunks.length === 0) return;
+    if (chunks.length === 0) {
+      await this.emitSettled(rid, fetched.checksum, "skipped");
+      return;
+    }
     const entityTypes = await this.resolveEntityTypes(rid);
     const embeddings = await this.embeddingProvider.embedBatch(chunks);
     const embeddingChunks = chunks.map((t, i) => ({
@@ -10029,6 +10090,7 @@ var Smelter = class _Smelter {
       embedding: embeddings[i]
     }));
     await this.vectorStore.upsertResourceVectors(resourceId(rid), embeddingChunks, fetched.checksum, entityTypes);
+    await this.emitSettled(rid, fetched.checksum, "indexed");
     this.logger.info(logMessage, { resourceId: rid, chunks: chunks.length });
   }
   async handleResourceArchived(event) {
@@ -10038,11 +10100,34 @@ var Smelter = class _Smelter {
     await this.vectorStore.deleteAnnotationVectorsForResource(resourceId(rid));
     this.logger.info("Deleted vectors for archived resource", { resourceId: rid });
   }
+  /**
+   * Restore what `handleResourceArchived` deleted, from CURRENT state: the
+   * resource's vectors (media-gated, full-replace) and its current exact-text
+   * annotations — the same catalog read `reconcile()` uses, so the live path
+   * and a restart agree (bugs/smelter-misses-unarchive.md).
+   */
+  async handleResourceUnarchived(event) {
+    const rid = event.resourceId;
+    if (!rid) return;
+    await this.embedResource(event, "Re-embedded unarchived resource");
+    const { annotations } = await busRequest(
+      this.bus,
+      "browse:annotations-requested",
+      { resourceId: rid }
+    );
+    for (const annotation of annotations) {
+      await this.indexAnnotation(rid, annotation);
+    }
+  }
   async handleAnnotationAdded(event) {
     const annotation = event.payload.annotation;
     if (!annotation?.id) return;
     const rid = event.resourceId;
     if (!rid) return;
+    await this.indexAnnotation(rid, annotation);
+  }
+  async indexAnnotation(rid, annotation) {
+    if (!annotation.id) return;
     const selector = getTargetSelector(annotation.target);
     const exactText = getExactText(selector);
     if (!exactText?.trim()) return;
@@ -10076,9 +10161,16 @@ var Smelter = class _Smelter {
       const rid = event.resourceId;
       if (!rid) continue;
       const fetched = await this.fetchEmbeddableText(rid);
-      if (!fetched) continue;
+      if (fetched.kind === "unavailable") continue;
+      if (fetched.kind === "skipped") {
+        await this.emitSettled(rid, fetched.checksum, "skipped");
+        continue;
+      }
       const chunks = chunkText(fetched.text, this.chunkingConfig);
-      if (chunks.length === 0) continue;
+      if (chunks.length === 0) {
+        await this.emitSettled(rid, fetched.checksum, "skipped");
+        continue;
+      }
       const entityTypes = await this.resolveEntityTypes(rid);
       resourceData.push({ rid: resourceId(rid), chunks, checksum: fetched.checksum, entityTypes });
       allChunks.push(...chunks);
@@ -10093,6 +10185,7 @@ var Smelter = class _Smelter {
         embedding: allEmbeddings[offset + i]
       }));
       await this.vectorStore.upsertResourceVectors(rid, embeddingChunks, checksum, entityTypes);
+      await this.emitSettled(String(rid), checksum, "indexed");
       this.logger.info("Batch-indexed resource", { resourceId: String(rid), chunks: chunks.length });
       offset += chunks.length;
     }
@@ -10165,7 +10258,7 @@ var Smelter = class _Smelter {
     this._reconcileState = { phase: "running" };
     try {
       const [indexedResources, indexedAnnotations] = await Promise.all([
-        this.vectorStore.listResourceChecksums(),
+        this.vectorStore.listResourceStamps(),
         this.vectorStore.listAnnotationIds()
       ]);
       const resources = await this.listAllResources();
@@ -10178,18 +10271,24 @@ var Smelter = class _Smelter {
       for (const resource of resources) {
         const mediaType = getPrimaryMediaType(resource);
         if (resource["@id"] && mediaType && textExtractionOf(mediaType) === "decode") {
-          embeddable.set(resource["@id"], getPrimaryRepresentation(resource)?.checksum);
+          embeddable.set(resource["@id"], {
+            checksum: getPrimaryRepresentation(resource)?.checksum,
+            entityTypes: getResourceEntityTypes(resource)
+          });
         }
       }
       const work = [];
       for (const rid of indexedResources.keys()) {
         if (!embeddable.has(rid)) work.push({ type: "smelt:purge", resourceId: rid, payload: {} });
       }
-      for (const [rid, catalogChecksum] of embeddable) {
-        if (!indexedResources.has(rid)) {
+      for (const [rid, catalog] of embeddable) {
+        const indexed = indexedResources.get(rid);
+        if (!indexed) {
           work.push({ type: "smelt:embed", resourceId: rid, payload: {} });
-        } else if (catalogChecksum !== void 0 && indexedResources.get(rid) !== catalogChecksum) {
+        } else if (catalog.checksum !== void 0 && indexed.contentChecksum !== catalog.checksum) {
           work.push({ type: "smelt:embed", resourceId: rid, payload: {} });
+        } else if (!sameStringSet(indexed.entityTypes, catalog.entityTypes)) {
+          work.push({ type: "smelt:restamp", resourceId: rid, payload: {} });
         }
       }
       const liveAnnotationIds = /* @__PURE__ */ new Set();
@@ -10218,6 +10317,7 @@ var Smelter = class _Smelter {
       await this.drain(work);
       const summary = {
         resourcesEmbedded: work.filter((w) => w.type === "smelt:embed").length,
+        resourcesRestamped: work.filter((w) => w.type === "smelt:restamp").length,
         resourceVectorsDeleted: work.filter((w) => w.type === "smelt:purge").length,
         annotationsEmbedded: work.filter((w) => w.type === "smelt:embed-annotation").length,
         annotationVectorsDeleted: work.filter((w) => w.type === "smelt:purge-annotation").length
@@ -10306,20 +10406,34 @@ async function authenticate() {
     logger.warn("No SEMIONT_WORKER_SECRET set \u2014 using empty token");
     return "";
   }
-  const response = await fetch(`${baseUrl}/api/tokens/agent`, {
-    method: "POST",
-    headers: { "Content-Type": "application/json" },
-    body: JSON.stringify({
-      secret: workerSecret,
-      provider: embeddingType,
-      model: embeddingModel
-    })
-  });
-  if (!response.ok) {
-    throw new Error(`Authentication failed: ${response.status} ${response.statusText}`);
-  }
-  const { token } = await response.json();
-  return token;
+  return retryWithBackoff(
+    async () => {
+      const response = await fetch(`${baseUrl}/api/tokens/agent`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({
+          secret: workerSecret,
+          provider: embeddingType,
+          model: embeddingModel
+        })
+      });
+      if (!response.ok) {
+        throw new Error(`Authentication failed: ${response.status} ${response.statusText}`);
+      }
+      const { token } = await response.json();
+      return token;
+    },
+    isTransientFetchError,
+    STARTUP_FETCH_RETRY,
+    ({ attempt, attempts, delayMs, error }) => {
+      logger.warn("Backend unreachable, retrying authentication", {
+        attempt,
+        attempts,
+        retryInMs: delayMs,
+        error: error instanceof Error ? error.message : String(error)
+      });
+    }
+  );
 }
 async function main() {
   const { initObservabilityNode } = await import('@semiont/observability/node');