@semiont/make-meaning 0.5.29 → 0.5.31

package/dist/index.d.ts

@@ -1,6 +1,6 @@
 import { JobQueue } from '@semiont/jobs';
 import { SemiontProject, SemiontState } from '@semiont/core/node';
-import { GraphServiceConfig, VectorsServiceConfig, EmbeddingServiceConfig, ArchivistServiceConfig, EnvironmentConfig, StateUnit, EventBus, Logger, ResourceId, ResourceDescriptor, components, AnnotationId, GatheredContext, ResourceAnnotations, Annotation, AnnotationCategory, ITransport, BaseUrl, ConnectionState, SemiontError, UserDID, EventMap, IContentTransport, PutBinaryRequest, PutBinaryOptions, ExtractionOutcome, AccessToken, BusRequestPrimitive, ChunkingConfig, UserId } from '@semiont/core';
+import { GraphServiceConfig, VectorsServiceConfig, EmbeddingServiceConfig, ArchivistServiceConfig, EnvironmentConfig, StateUnit, EventBus, Logger, ResourceId, ResourceDescriptor, components, AnnotationId, GatheredContext, ResourceAnnotations, Annotation, AnnotationCategory, ITransport, BaseUrl, ConnectionState, SemiontError, UserDID, EventMap, IContentTransport, PutBinaryRequest, PutBinaryOptions, AccessToken, BusRequestPrimitive, ChunkingConfig, UserId } from '@semiont/core';
 import { EventReadStorage, ViewMaterializer, EventStore, ViewStorage, EventLog } from '@semiont/event-sourcing';
 import { WorkingTreeStore, AnchoredTextStore, ContentReads } from '@semiont/content';
 import { GraphDatabase } from '@semiont/graph';
@@ -337,6 +337,25 @@ declare class Stower {
     private handleYieldUpdate;
     private handleYieldMv;
     private handleMarkCreate;
+    /**
+     * Persist a detection unit's annotations as ONE acknowledged batch, then
+     * answer (JOB-RESTART-SAFETY P6).
+     *
+     * The difference from `mark:create` is the reply, and it is the whole point.
+     * `mark:create` is fire-and-forget: the worker's emit resolves when the bus
+     * accepts it, which says nothing about the event log, so a down Stower loses
+     * a unit silently and a flapping one hangs the worker forever. This answers
+     * only after every append has returned, so the worker can gate unit
+     * completion — and its checkpoint — on durability.
+     *
+     * Appends are sequential, not concurrent: the event log is the system of
+     * record and a batch that half-lands under concurrency is harder to reason
+     * about than one that stops at the first failure. A partial batch is
+     * reported as a failure and the worker retries the WHOLE unit, which is safe
+     * because ids are deterministic (P3) and the annotation fold is idempotent
+     * by id — re-appending what already landed changes nothing.
+     */
+    private handleMarkCommit;
     private handleMarkDelete;
     private handleMarkUpdateBody;
     private handleMarkArchive;
@@ -722,8 +741,6 @@ interface LimitsDiscovery {
  *
  * Handles:
  * - browse:resource-requested — single resource metadata (materialized from events)
- * - browse:anchored-text-by-checksum-requested — the detection workers' read-through
- *   cache consult: a stored extraction outcome by content identity (ANCHORED-TEXT-TO-SMELTER P2)
  * - browse:resources-requested — list resources
  * - browse:annotations-requested — all annotations for a resource
  * - browse:annotation-requested — single annotation with resolved resource
@@ -794,7 +811,6 @@ declare class Browser {
      * parser nor engine. Read-only over the wire: the Smelter is the sole
      * writer and never answers here.
      */
-    private handleAnchoredTextByChecksum;
     private handleBrowseResource;
     private handleBrowseResources;
     private handleBrowseAnnotations;
@@ -1133,36 +1149,6 @@ declare class LocalContentTransport implements IContentTransport {
     putBinary(_request: PutBinaryRequest, _options?: PutBinaryOptions): Promise<{
         resourceId: ResourceId;
     }>;
-    /**
-     * Store a derived coordinate map under the content checksum the producer
-     * read (PERSIST-ANCHORS decision A — see the interface doc for why the
-     * producer supplies the key). In local mode this is the same store the
-     * HTTP route writes to — one storage authority, reached the same way from
-     * every process (ANCHORED-TEXT-CACHE Lane 5).
-     */
-    putAnchoredText(checksum: string, outcome: ExtractionOutcome, _options?: {
-        auth?: AccessToken;
-    }): Promise<void>;
-    /** The stored outcome, or null when nothing has derived one — the common case. */
-    getAnchoredText(resourceId: ResourceId, _options?: {
-        auth?: AccessToken;
-    }): Promise<ExtractionOutcome | null>;
-    /**
-     * The cache-consult read (PERSIST-ANCHORS P2c), straight from the store —
-     * checksum-addressed, so no view resolution and no settle barrier: the
-     * caller holds the content identity already.
-     */
-    getAnchoredTextByChecksum(checksum: string, _options?: {
-        auth?: AccessToken;
-    }): Promise<ExtractionOutcome | null>;
-    /**
-     * The store's would-hit keys, straight from the store — planning data for
-     * the reconcile diff (PERSIST-ANCHORS P0), so no settle barrier applies:
-     * presence is being asked, not content at a moment.
-     */
-    listAnchoredTextKeys(_options?: {
-        auth?: AccessToken;
-    }): Promise<string[]>;
     getBinary(resourceId: ResourceId, _options?: {
         auth?: AccessToken;
     }): Promise<{
@@ -1654,10 +1640,12 @@ declare class Smelter {
      * representation's checksum (the bytes the smelter would read), the
      * current entity-type set (the discriminator the stamps must carry), and
      * whether the media type's extractor derives geometry (whether an
-     * anchored-text artifact should exist). Embeddable ⇔ an extractor exists
-     * for the media type's strategy — the same registry the live fetch
-     * resolves, and `yieldsGeometry` is declared on the extractor itself, so
-     * every gate here and the live fetch's behavior are twins by construction.
+     * anchored-text artifact should exist). Both answers are core's, keyed by the
+     * media type's strategy: embeddable ⇔ the type has any text-reading strategy
+     * at all, and geometry ⇔ that strategy derives it. Until READ-VS-EXTRACT P2
+     * embeddability was asked as `EXTRACTORS[strategy] !== null` — true, but a
+     * second statement of `strategy !== 'none'`, answered by resolving an
+     * implementation to learn a fact about a media type.
      * Shared by `reconcile()` and the `smelt:rebuild-anchors` planner.
      */
     private classifyEmbeddable;

package/dist/index.js

@@ -1,10 +1,10 @@
 import { STALL_THRESHOLD_MS, FsJobQueue } from '@semiont/jobs';
 import { FilesystemViewStorage, resolveStorageUri, EventQuery, createEventStore } from '@semiont/event-sourcing';
-import { getResourceEntityTypes, getResourceId, getTargetSource, resourceId, decodeRepresentation, getBodySource, getStorageUri, getPrimaryRepresentation, getTargetSelector, deriveViews, getTextPositionSelector, annotationId, errField, userId, generateUuid, getExactText, busRequest, cloneToken, assembleAnnotation, applyBodyOperations, didToAgent, isGenerationJobParams, jobId, entityType, baseUrl, busLog, BRIDGED_CHANNELS, burstBuffer, textExtractionOf, chunkText, getPrimaryMediaType, isAnnotatable, softwareToAgent } from '@semiont/core';
+import { getResourceEntityTypes, getResourceId, getTargetSource, resourceId, decodeRepresentation, getBodySource, getStorageUri, getPrimaryRepresentation, getTargetSelector, deriveViews, getTextPositionSelector, annotationId, errField, userId, generateUuid, getExactText, busRequest, cloneToken, assembleAnnotation, applyBodyOperations, didToAgent, isGenerationJobParams, jobId, entityType, baseUrl, busLog, BRIDGED_CHANNELS, burstBuffer, textSourceOf, chunkText, getPrimaryMediaType, yieldsGeometryOf, isAnnotatable, softwareToAgent } from '@semiont/core';
 import { recordGatherDegrade, withActorSpan, recordBusEmit, withSpan, SpanKind, registerJobQueueProvider, registerVectorIndexSizeProvider } from '@semiont/observability';
 import { createInferenceClient } from '@semiont/inference';
 import { compareByRecencyThenId, getGraphDatabase } from '@semiont/graph';
-import { WorkingTreeStore, createAnchoredTextStore, RepresentationMissing, ChecksumMismatchError, EXTRACTORS, calculateChecksum } from '@semiont/content';
+import { WorkingTreeStore, createAnchoredTextStore, RepresentationMissing, ChecksumMismatchError, derivingExtractorFor, calculateChecksum } from '@semiont/content';
 import { getEntityTypes, DEFAULT_ENTITY_TYPES } from '@semiont/ontology';
 import { mergeByResource } from '@semiont/vectors';
 import { promises } from 'fs';
@@ -11335,6 +11335,7 @@ var Stower = class {
       pipe("yield:update", (e) => this.handleYieldUpdate(e)),
       pipe("yield:mv", (e) => this.handleYieldMv(e)),
       pipe("mark:create", (e) => this.handleMarkCreate(e)),
+      pipe("mark:commit", (e) => this.handleMarkCommit(e)),
       pipe("mark:delete", (e) => this.handleMarkDelete(e)),
       pipe("mark:update-body", (e) => this.handleMarkUpdateBody(e)),
       pipe("frame:add-entity-type", (e) => this.handleAddEntityType(e)),
@@ -11560,6 +11561,61 @@ var Stower = class {
       });
     }
   }
+  /**
+   * Persist a detection unit's annotations as ONE acknowledged batch, then
+   * answer (JOB-RESTART-SAFETY P6).
+   *
+   * The difference from `mark:create` is the reply, and it is the whole point.
+   * `mark:create` is fire-and-forget: the worker's emit resolves when the bus
+   * accepts it, which says nothing about the event log, so a down Stower loses
+   * a unit silently and a flapping one hangs the worker forever. This answers
+   * only after every append has returned, so the worker can gate unit
+   * completion — and its checkpoint — on durability.
+   *
+   * Appends are sequential, not concurrent: the event log is the system of
+   * record and a batch that half-lands under concurrency is harder to reason
+   * about than one that stops at the first failure. A partial batch is
+   * reported as a failure and the worker retries the WHOLE unit, which is safe
+   * because ids are deterministic (P3) and the annotation fold is idempotent
+   * by id — re-appending what already landed changes nothing.
+   */
+  async handleMarkCommit(event) {
+    if (!event._userId) {
+      throw new Error("mark:commit missing _userId (gateway injection)");
+    }
+    const annotations = event.annotations ?? [];
+    try {
+      let persisted = 0;
+      for (const annotation of annotations) {
+        await this.stores.eventStore.appendEvent({
+          type: "mark:added",
+          resourceId: resourceId(event.resourceId),
+          userId: userId(event._userId),
+          version: 1,
+          payload: { annotation }
+        });
+        persisted++;
+      }
+      this.logger.debug("Committed annotation batch", {
+        correlationId: event.correlationId,
+        resourceId: event.resourceId,
+        persisted
+      });
+      this.eventBus.get("mark:commit-ok").next({
+        correlationId: event.correlationId,
+        response: { persisted, annotationIds: annotations.map((a) => String(a.id)) }
+      });
+    } catch (error) {
+      this.logger.error("Failed to commit annotation batch", {
+        correlationId: event.correlationId,
+        error: errField(error)
+      });
+      this.eventBus.get("mark:commit-failed").next({
+        correlationId: event.correlationId,
+        message: error instanceof Error ? error.message : String(error)
+      });
+    }
+  }
   async handleMarkDelete(event) {
     if (!event._userId) {
       throw new Error("mark:delete missing _userId (gateway injection)");
@@ -11805,17 +11861,19 @@ var Stower = class {
 var import_rxjs4 = __toESM(require_cjs());
 var import_operators4 = __toESM(require_operators());
 var ANCHORED_TEXT_SETTLE_TIMEOUT_MS = 15e3;
-async function readAnchoredText(kb, resourceId10) {
+async function readAnchoredText(kb, resourceId10, settleTimeoutMs = ANCHORED_TEXT_SETTLE_TIMEOUT_MS) {
   const view = await kb.views.get(resourceId(resourceId10));
   const checksum = getPrimaryRepresentation(view?.resource)?.checksum;
-  if (!checksum) return null;
+  if (!checksum) return { kind: "unknown" };
   const hit = await kb.anchoredText.read(checksum);
   if (hit) return hit;
   try {
-    const outcome = await kb.smeltProgress.whenSettled(resourceId10, checksum, ANCHORED_TEXT_SETTLE_TIMEOUT_MS);
-    return outcome === "indexed" ? kb.anchoredText.read(checksum) : null;
+    const outcome = await kb.smeltProgress.whenSettled(resourceId10, checksum, settleTimeoutMs);
+    if (outcome === "skipped") return { kind: "no-map" };
+    if (outcome === "inert") return { kind: "not-yet" };
+    return await kb.anchoredText.read(checksum) ?? { kind: "not-yet" };
   } catch (error) {
-    if (error instanceof SmeltProgressTimeout) return null;
+    if (error instanceof SmeltProgressTimeout) return { kind: "not-yet" };
     throw error;
   }
 }
@@ -11947,7 +12005,6 @@ var Browser = class {
     this.subscriptions.push(
       pipe("browse:resource-requested", (e) => this.handleBrowseResource(e)).subscribe({ error: errorHandler }),
       pipe("browse:anchored-text-requested", (e) => this.handleAnchoredText(e)).subscribe({ error: errorHandler }),
-      pipe("browse:anchored-text-by-checksum-requested", (e) => this.handleAnchoredTextByChecksum(e)).subscribe({ error: errorHandler }),
       pipe("browse:resources-requested", (e) => this.handleBrowseResources(e)).subscribe({ error: errorHandler }),
       pipe("browse:annotations-requested", (e) => this.handleBrowseAnnotations(e)).subscribe({ error: errorHandler }),
       pipe("browse:annotation-requested", (e) => this.handleBrowseAnnotation(e)).subscribe({ error: errorHandler }),
@@ -12001,20 +12058,6 @@ var Browser = class {
    * parser nor engine. Read-only over the wire: the Smelter is the sole
    * writer and never answers here.
    */
-  async handleAnchoredTextByChecksum(event) {
-    try {
-      this.eventBus.get("browse:anchored-text-by-checksum-result").next({
-        correlationId: event.correlationId,
-        response: await this.kb.anchoredText.read(event.checksum)
-      });
-    } catch (error) {
-      this.logger.error("Browse anchored text by checksum failed", { checksum: event.checksum, error: errField(error) });
-      this.eventBus.get("browse:anchored-text-by-checksum-failed").next({
-        correlationId: event.correlationId,
-        message: error instanceof Error ? error.message : String(error)
-      });
-    }
-  }
   async handleBrowseResource(event) {
     try {
       const response = await assembleResourceGraph(this.kb, resourceId(event.resourceId));
@@ -13194,7 +13237,7 @@ function registerJobCommandHandlers(eventBus, jobQueue, state, parentLogger) {
   });
   eventBus.get("job:fail").subscribe(async (event) => {
     try {
-      const outcome = await jobQueue.failJob(jobId(event.jobId), event.error);
+      const outcome = await jobQueue.failJob(jobId(event.jobId), event.error, event.completedUnits, event.failureClass);
       if (outcome === "retried") {
         logger.info("Job re-queued for retry", { jobId: event.jobId });
       } else if (outcome === null) {
@@ -13220,16 +13263,45 @@ function registerJobCommandHandlers(eventBus, jobQueue, state, parentLogger) {
       });
     }
   });
+  eventBus.get("job:checkpoint").subscribe(async (event) => {
+    try {
+      await jobQueue.checkpointUnits(jobId(event.jobId), event.completedUnits);
+    } catch (error) {
+      logger.error("Failed to checkpoint job units", {
+        jobId: event.jobId,
+        error: error.message
+      });
+    }
+  });
   eventBus.get("job:cancel-requested").subscribe(async (event) => {
     try {
-      const cancelled = await jobQueue.cancelPendingJobs(event.jobType);
-      logger.info("Cancel requested", { jobType: event.jobType, cancelled });
+      let cancelled;
+      if (event.jobId) {
+        const target = await jobQueue.getJob(jobId(event.jobId));
+        if (!target) {
+          cancelled = 0;
+        } else if (target.status === "pending") {
+          cancelled = await jobQueue.cancelJob(jobId(event.jobId)) ? 1 : 0;
+        } else if (target.status === "running") {
+          logger.info("Cancel of running job delegated to its worker", { jobId: event.jobId });
+          cancelled = 1;
+        } else {
+          cancelled = 0;
+        }
+        logger.info("Cancel requested", { jobId: event.jobId, cancelled });
+      } else if (event.jobType) {
+        cancelled = await jobQueue.cancelPendingJobs(event.jobType);
+        logger.info("Cancel requested", { jobType: event.jobType, cancelled });
+      } else {
+        cancelled = 0;
+      }
       eventBus.get("job:cancel-ok").next({
         correlationId: event.correlationId,
         response: { cancelled }
       });
     } catch (error) {
-      logger.error("Failed to cancel pending jobs", {
+      logger.error("Failed to cancel jobs", {
+        jobId: event.jobId,
         jobType: event.jobType,
         error: error.message
       });
@@ -13239,6 +13311,17 @@ function registerJobCommandHandlers(eventBus, jobQueue, state, parentLogger) {
       });
     }
   });
+  eventBus.get("job:cancel").subscribe(async (event) => {
+    try {
+      await jobQueue.cancelJob(jobId(event.jobId));
+      logger.info("Job cancelled by its worker", { jobId: event.jobId });
+    } catch (error) {
+      logger.error("Failed to cancel job", {
+        jobId: event.jobId,
+        error: error.message
+      });
+    }
+  });
 }
 
 // src/handlers/index.ts
@@ -13636,40 +13719,6 @@ var LocalContentTransport = class {
       "LocalContentTransport does not support putBinary() \u2014 create resources via bus emits (mark/yield namespaces) in local mode"
     );
   }
-  /**
-   * Store a derived coordinate map under the content checksum the producer
-   * read (PERSIST-ANCHORS decision A — see the interface doc for why the
-   * producer supplies the key). In local mode this is the same store the
-   * HTTP route writes to — one storage authority, reached the same way from
-   * every process (ANCHORED-TEXT-CACHE Lane 5).
-   */
-  async putAnchoredText(checksum, outcome, _options) {
-    busLog("PUT", "anchored-text", { checksum });
-    await this.kb.anchoredText.write(checksum, outcome);
-  }
-  /** The stored outcome, or null when nothing has derived one — the common case. */
-  async getAnchoredText(resourceId10, _options) {
-    busLog("GET", "anchored-text", { resourceId: resourceId10 });
-    return readAnchoredText(this.kb, resourceId10);
-  }
-  /**
-   * The cache-consult read (PERSIST-ANCHORS P2c), straight from the store —
-   * checksum-addressed, so no view resolution and no settle barrier: the
-   * caller holds the content identity already.
-   */
-  async getAnchoredTextByChecksum(checksum, _options) {
-    busLog("GET", "anchored-text-by-checksum", { checksum });
-    return this.kb.anchoredText.read(checksum);
-  }
-  /**
-   * The store's would-hit keys, straight from the store — planning data for
-   * the reconcile diff (PERSIST-ANCHORS P0), so no settle barrier applies:
-   * presence is being asked, not content at a moment.
-   */
-  async listAnchoredTextKeys(_options) {
-    busLog("GET", "anchored-text-keys", {});
-    return this.kb.anchoredText.list();
-  }
   async getBinary(resourceId10, _options) {
     busLog("GET", "content", { resourceId: resourceId10 });
     return withSpan(
@@ -14011,8 +14060,8 @@ var Smelter = class _Smelter {
     if (!rid) return;
     const { data, contentType } = await this.content.getBinary(resourceId(rid));
     const bytes = Buffer.from(data);
-    const extractor = EXTRACTORS[textExtractionOf(contentType)];
-    if (!extractor?.yieldsGeometry) {
+    const extractor = derivingExtractorFor(contentType);
+    if (!extractor) {
       this.logger.info("Re-anchor found no geometry-capable extractor", { resourceId: rid, contentType });
       return;
     }
@@ -14051,15 +14100,12 @@ var Smelter = class _Smelter {
       const { data, contentType } = await this.content.getBinary(resourceId(resourceId10));
       const bytes = Buffer.from(data);
       const checksum = calculateChecksum(bytes);
-      const extractor = EXTRACTORS[textExtractionOf(contentType)];
-      if (!extractor) {
-        this.logger.debug("Skipping resource with no extractor for its media type", { resourceId: resourceId10, contentType });
+      const extractor = derivingExtractorFor(contentType);
+      if (!extractor && textSourceOf(contentType) === "none") {
+        this.logger.debug("Skipping resource with no way to read its media type", { resourceId: resourceId10, contentType });
         return { kind: "skipped", checksum, reason: "no-extractor" };
       }
-      const extracted = await extractor.extract(bytes, contentType, {
-        key: checksum,
-        store: this.anchoredStore
-      });
+      const extracted = extractor ? await extractor.extract(bytes, contentType, { key: checksum, store: this.anchoredStore }) : { kind: "extracted", text: decodeRepresentation(bytes, contentType), method: "text-passthrough" };
       if (extracted.kind === "declined") {
         this.logger.debug("Extractor declined", { resourceId: resourceId10, contentType, reason: extracted.declined });
         return { kind: "skipped", checksum, reason: extracted.declined };
@@ -14474,22 +14520,24 @@ var Smelter = class _Smelter {
    * representation's checksum (the bytes the smelter would read), the
    * current entity-type set (the discriminator the stamps must carry), and
    * whether the media type's extractor derives geometry (whether an
-   * anchored-text artifact should exist). Embeddable ⇔ an extractor exists
-   * for the media type's strategy — the same registry the live fetch
-   * resolves, and `yieldsGeometry` is declared on the extractor itself, so
-   * every gate here and the live fetch's behavior are twins by construction.
+   * anchored-text artifact should exist). Both answers are core's, keyed by the
+   * media type's strategy: embeddable ⇔ the type has any text-reading strategy
+   * at all, and geometry ⇔ that strategy derives it. Until READ-VS-EXTRACT P2
+   * embeddability was asked as `EXTRACTORS[strategy] !== null` — true, but a
+   * second statement of `strategy !== 'none'`, answered by resolving an
+   * implementation to learn a fact about a media type.
    * Shared by `reconcile()` and the `smelt:rebuild-anchors` planner.
    */
   classifyEmbeddable(resources) {
     const embeddable = /* @__PURE__ */ new Map();
     for (const resource of resources) {
       const mediaType = getPrimaryMediaType(resource);
-      const extractor = mediaType ? EXTRACTORS[textExtractionOf(mediaType)] : null;
-      if (resource["@id"] && extractor) {
+      const readable = mediaType !== void 0 && textSourceOf(mediaType) !== "none";
+      if (resource["@id"] && mediaType && readable) {
         embeddable.set(resource["@id"], {
           checksum: getPrimaryRepresentation(resource)?.checksum,
           entityTypes: getResourceEntityTypes(resource),
-          yieldsGeometry: extractor.yieldsGeometry
+          yieldsGeometry: yieldsGeometryOf(mediaType)
         });
       }
     }