@semiont/make-meaning 0.5.32 → 0.5.34

package/dist/index.d.ts

@@ -230,10 +230,23 @@ interface KnowledgeBase {
 /** The lifecycle half of the working tree (GATEWAY.md D4a): the Archivist
  *  accessions, moves, removes and resolves — it never serves bytes. */
 type ContentLifecycle = Pick<WorkingTreeStore, 'register' | 'move' | 'remove' | 'resolveUri'>;
-/** The record's single write seam. `Stower` is the only appendEvent caller
- *  anywhere in make-meaning or the gateway (post-#1252): single-owner by
- *  construction. A second caller is a design smell, not a wiring chore. */
-type EventAppends = Pick<EventStore, 'appendEvent'>;
+/**
+ * The record's single write seam. `Stower` is the only appendEvent caller
+ * anywhere in make-meaning or the gateway (post-#1252): single-owner by
+ * construction. A second caller is a design smell, not a wiring chore.
+ *
+ * It carries a read — `viewStorage.get`, narrowed to `get` — because one write
+ * path is at-least-once and must not duplicate the log
+ * (COMMIT-ACK-FALSE-FAILURE F3): `mark:commit` diffs its batch against what the
+ * resource already holds. This does NOT reverse JOB-RESTART-SAFETY HD1, which
+ * rejected read-before-write for a WORKER reading a REMOTE store mid-recovery
+ * — "the thing it would read is exactly what is down". This read is inside the
+ * Archivist, against the store it is about to write, and cannot be down
+ * relative to itself.
+ */
+type EventAppends = Pick<EventStore, 'appendEvent'> & {
+    readonly viewStorage: Pick<ViewStorage, 'get'>;
+};
 /** Read-only reach into the event store: the log for queries, the
  *  materializer for on-demand view assembly (`assembleResourceGraph`). */
 interface EventStoreReads {
@@ -350,10 +363,22 @@ declare class Stower {
      *
      * Appends are sequential, not concurrent: the event log is the system of
      * record and a batch that half-lands under concurrency is harder to reason
-     * about than one that stops at the first failure. A partial batch is
-     * reported as a failure and the worker retries the WHOLE unit, which is safe
-     * because ids are deterministic (P3) and the annotation fold is idempotent
-     * by id — re-appending what already landed changes nothing.
+     * about than one that stops at the first failure.
+     *
+     * This channel is AT-LEAST-ONCE, and the log must not grow on a repeat
+     * (COMMIT-ACK-FALSE-FAILURE F3). Two paths re-send a batch that already
+     * landed: an acknowledgement lost after a successful append (the unit is
+     * never checkpointed, so the retry re-runs exactly the unit that landed), and
+     * a partial batch, reported as a failure and retried whole. Deterministic ids
+     * (JOB-RESTART-SAFETY P3) made those safe for the PROJECTIONS — the resource
+     * view and the graph both refuse a duplicate id — but a projection's guard
+     * says nothing about the log, which appends whatever it is handed. The result
+     * was a green graph over a doubled log: silent, and not undoable.
+     *
+     * So the batch is diffed against what the resource already holds. ONE view
+     * read per commit, never per annotation: the view for a 1,673-annotation
+     * resource is ~3 MB, and re-reading it per append would cost gigabytes of
+     * parsing for a single job.
      */
     private handleMarkCommit;
     private handleMarkDelete;
@@ -1650,6 +1675,9 @@ declare class Smelter {
      */
     private classifyEmbeddable;
     /** Page through `browse:resources-requested` until the catalog is exhausted. */
+    /** Shared with the weaver since 2026-09-09 — see `browse-resources.ts` for why the
+     *  retry lives at the page rather than around the pass. `archived: false` is
+     *  the smelter's own filter and the one difference between the two loops. */
     private listAllResources;
 }
 

package/dist/index.js

@@ -1,6 +1,6 @@
 import { STALL_THRESHOLD_MS, FsJobQueue } from '@semiont/jobs';
 import { FilesystemViewStorage, resolveStorageUri, EventQuery, createEventStore } from '@semiont/event-sourcing';
-import { getResourceEntityTypes, getResourceId, getTargetSource, resourceId, decodeRepresentation, getBodySource, getStorageUri, getPrimaryRepresentation, getTargetSelector, deriveViews, getTextPositionSelector, annotationId, errField, userId, generateUuid, getExactText, busRequest, cloneToken, assembleAnnotation, applyBodyOperations, didToAgent, isGenerationJobParams, jobId, entityType, baseUrl, busLog, BRIDGED_CHANNELS, burstBuffer, textSourceOf, chunkText, getPrimaryMediaType, yieldsGeometryOf, isAnnotatable, softwareToAgent } from '@semiont/core';
+import { getResourceEntityTypes, getResourceId, getTargetSource, resourceId, decodeRepresentation, getBodySource, getStorageUri, getPrimaryRepresentation, getTargetSelector, deriveViews, getTextPositionSelector, annotationId, errField, userId, generateUuid, getExactText, busRequest, cloneToken, assembleAnnotation, applyBodyOperations, didToAgent, isGenerationJobParams, jobId, entityType, baseUrl, busLog, BRIDGED_CHANNELS, burstBuffer, textSourceOf, chunkText, getPrimaryMediaType, yieldsGeometryOf, isAnnotatable, retryWithBackoff, isPeerUnavailable, withDeadline, softwareToAgent } from '@semiont/core';
 import { recordGatherDegrade, withActorSpan, recordBusEmit, withSpan, SpanKind, registerJobQueueProvider, registerVectorIndexSizeProvider } from '@semiont/observability';
 import { createInferenceClient } from '@semiont/inference';
 import { compareByRecencyThenId, getGraphDatabase } from '@semiont/graph';
@@ -11574,36 +11574,56 @@ var Stower = class {
    *
    * Appends are sequential, not concurrent: the event log is the system of
    * record and a batch that half-lands under concurrency is harder to reason
-   * about than one that stops at the first failure. A partial batch is
-   * reported as a failure and the worker retries the WHOLE unit, which is safe
-   * because ids are deterministic (P3) and the annotation fold is idempotent
-   * by id — re-appending what already landed changes nothing.
+   * about than one that stops at the first failure.
+   *
+   * This channel is AT-LEAST-ONCE, and the log must not grow on a repeat
+   * (COMMIT-ACK-FALSE-FAILURE F3). Two paths re-send a batch that already
+   * landed: an acknowledgement lost after a successful append (the unit is
+   * never checkpointed, so the retry re-runs exactly the unit that landed), and
+   * a partial batch, reported as a failure and retried whole. Deterministic ids
+   * (JOB-RESTART-SAFETY P3) made those safe for the PROJECTIONS — the resource
+   * view and the graph both refuse a duplicate id — but a projection's guard
+   * says nothing about the log, which appends whatever it is handed. The result
+   * was a green graph over a doubled log: silent, and not undoable.
+   *
+   * So the batch is diffed against what the resource already holds. ONE view
+   * read per commit, never per annotation: the view for a 1,673-annotation
+   * resource is ~3 MB, and re-reading it per append would cost gigabytes of
+   * parsing for a single job.
    */
   async handleMarkCommit(event) {
     if (!event._userId) {
       throw new Error("mark:commit missing _userId (gateway injection)");
     }
     const annotations = event.annotations ?? [];
+    const rid = resourceId(event.resourceId);
     try {
-      let persisted = 0;
+      const view = await this.stores.eventStore.viewStorage.get(rid);
+      const present = new Set((view?.annotations.annotations ?? []).map((a) => String(a.id)));
       for (const annotation of annotations) {
+        if (present.has(String(annotation.id))) continue;
         await this.stores.eventStore.appendEvent({
           type: "mark:added",
-          resourceId: resourceId(event.resourceId),
+          resourceId: rid,
           userId: userId(event._userId),
           version: 1,
           payload: { annotation }
         });
-        persisted++;
+        present.add(String(annotation.id));
       }
       this.logger.debug("Committed annotation batch", {
         correlationId: event.correlationId,
         resourceId: event.resourceId,
-        persisted
+        persisted: annotations.length
       });
       this.eventBus.get("mark:commit-ok").next({
         correlationId: event.correlationId,
-        response: { persisted, annotationIds: annotations.map((a) => String(a.id)) }
+        // The DURABLE count, which is what the acknowledgement means ("every
+        // annotation named by the command is in the event log"). Not an append
+        // tally: a retry whose annotations are all already present has
+        // succeeded, and must be indistinguishable from the first commit or the
+        // caller would have to interpret a 0 that means "all good".
+        response: { persisted: annotations.length, annotationIds: annotations.map((a) => String(a.id)) }
       });
     } catch (error) {
       this.logger.error("Failed to commit annotation batch", {
@@ -11829,7 +11849,11 @@ var Stower = class {
         jobId: event.jobId,
         jobType: event.jobType,
         ...event.annotationId ? { annotationId: event.annotationId } : {},
-        result: event.result
+        result: event.result,
+        // How durability was ESTABLISHED (COMMIT-ACK-FALSE-FAILURE). An
+        // acknowledged batch and one inferred from a probe are different
+        // claims; absent means the question never arose.
+        ...event.durability !== void 0 ? { durability: event.durability } : {}
       }
     });
   }
@@ -11846,7 +11870,19 @@ var Stower = class {
         jobId: event.jobId,
         jobType: event.jobType,
         ...event.annotationId ? { annotationId: event.annotationId } : {},
-        error: event.error
+        error: event.error,
+        // The worker's JUDGMENTS, not just its message. Both are computed where
+        // the error is still typed and are unrecoverable here — the only other
+        // witness in the log is `error`, a flattened English string. Spread
+        // conditionally: absent `failureClass` means UNRECOGNISED, a different
+        // claim from 'transient', and defaulting either would write a judgment
+        // nobody made into a log nobody can rewrite.
+        ...event.failureClass !== void 0 ? { failureClass: event.failureClass } : {},
+        ...event.willRetry !== void 0 ? { willRetry: event.willRetry } : {},
+        // How durability was ESTABLISHED (COMMIT-ACK-FALSE-FAILURE). An
+        // acknowledged batch and one inferred from a probe are different
+        // claims; absent means the question never arose.
+        ...event.durability !== void 0 ? { durability: event.durability } : {}
       }
     });
   }
@@ -13384,43 +13420,36 @@ async function createJobQueue(state, eventBus, logger) {
   return { jobQueue, jobStatusSubscription };
 }
 var STARTUP_CONNECT_TIMEOUT_MS = 6e4;
-async function withStartupTimeout(what, work) {
-  let timer;
-  try {
-    return await Promise.race([
-      work,
-      new Promise((_resolve, reject) => {
-        timer = setTimeout(
-          () => reject(
-            new Error(
-              `${what} did not become available within ${STARTUP_CONNECT_TIMEOUT_MS / 1e3}s. Exiting so the container restart policy can retry \u2014 it is normal for a dependency to be slow when every service restarts at once.`
-            )
-          ),
-          STARTUP_CONNECT_TIMEOUT_MS
-        );
-      })
-    ]);
-  } finally {
-    if (timer !== void 0) clearTimeout(timer);
-  }
-}
+var RESTART_HINT = "Exiting so the container restart policy can retry \u2014 it is normal for a dependency to be slow when every service restarts at once.";
 async function connectStores(project, config, eventBus, logger, skipRebuild) {
   const graphConfig = config.services.graph;
   logger.info("Connecting to graph database", { type: graphConfig.type });
-  const graphDb = await withStartupTimeout("Graph database", getGraphDatabase(graphConfig));
+  const graphDb = await withDeadline(
+    "Graph database",
+    STARTUP_CONNECT_TIMEOUT_MS,
+    () => getGraphDatabase(graphConfig),
+    RESTART_HINT
+  );
   const eventStore = createEventStore(project, eventBus, logger.child({ component: "event-store" }));
   const vectorsConfig = config.services.vectors;
   const embeddingConfig = config.services.embedding;
   const { createVectorStore, createEmbeddingProvider } = await import('@semiont/vectors');
   logger.info("Connecting to embedding provider", { type: embeddingConfig.type, model: embeddingConfig.model });
-  const embeddingProvider = await withStartupTimeout(
+  const embeddingProvider = await withDeadline(
     "Embedding provider",
-    createEmbeddingProvider(embeddingConfig)
+    STARTUP_CONNECT_TIMEOUT_MS,
+    () => createEmbeddingProvider(embeddingConfig),
+    RESTART_HINT
   );
   logger.info("Connecting to vector store", { type: vectorsConfig.type });
-  const vectorStore = await withStartupTimeout(
+  const vectorStore = await withDeadline(
     "Vector store",
-    createVectorStore({
+    STARTUP_CONNECT_TIMEOUT_MS,
+    (signal) => createVectorStore({
+      // The deadline this call is already being raced against, handed down so
+      // the dimension-probe retry stops when it fires instead of being abandoned
+      // mid-flight. This is the whole point of the signal parameter.
+      signal,
       type: vectorsConfig.type,
       host: vectorsConfig.host,
       port: vectorsConfig.port,
@@ -13432,7 +13461,8 @@ async function connectStores(project, config, eventBus, logger, skipRebuild) {
       // network round-trip a precondition of booting. A `memory` store, or a
       // Qdrant whose collections already exist, never consults the provider.
       dimensions: () => embeddingProvider.dimensions()
-    })
+    }),
+    RESTART_HINT
   );
   if (vectorsConfig.type === "memory") {
     logger.info("memory vector store: the index rebuilds from the event log on every restart (reconcile re-embeds)");
@@ -13789,6 +13819,28 @@ function partitionByType(events) {
   if (currentRun.length > 0) runs.push(currentRun);
   return runs;
 }
+var RESOURCE_LISTING_RETRY = {
+  attempts: 17,
+  initialDelayMs: 1e3,
+  maxDelayMs: 3e4
+};
+var RESOURCES_CHANNEL = "browse:resources-requested";
+async function browseAllResources(bus, options) {
+  const all = [];
+  for (; ; ) {
+    const page = await retryWithBackoff(
+      () => busRequest(bus, RESOURCES_CHANNEL, {
+        ...options.archived !== void 0 ? { archived: options.archived } : {},
+        offset: all.length,
+        limit: options.limit
+      }),
+      isPeerUnavailable,
+      RESOURCE_LISTING_RETRY
+    );
+    all.push(...page.resources);
+    if (page.resources.length === 0 || all.length >= page.total) return all;
+  }
+}
 
 // src/smelter.ts
 function sameStringSet(a, b) {
@@ -14544,17 +14596,11 @@ var Smelter = class _Smelter {
     return embeddable;
   }
   /** Page through `browse:resources-requested` until the catalog is exhausted. */
-  async listAllResources() {
-    const all = [];
-    for (; ; ) {
-      const page = await busRequest(
-        this.bus,
-        "browse:resources-requested",
-        { archived: false, offset: all.length, limit: _Smelter.RECONCILE_PAGE_SIZE }
-      );
-      all.push(...page.resources);
-      if (page.resources.length === 0 || all.length >= page.total) return all;
-    }
+  /** Shared with the weaver since 2026-09-09 — see `browse-resources.ts` for why the
+   *  retry lives at the page rather than around the pass. `archived: false` is
+   *  the smelter's own filter and the one difference between the two loops. */
+  listAllResources() {
+    return browseAllResources(this.bus, { limit: _Smelter.RECONCILE_PAGE_SIZE, archived: false });
   }
 };