@semiont/jobs 0.5.32 → 0.5.34

package/dist/worker-main.js

@@ -1,4 +1,4 @@
-import { replyChannelsFor, createTomlConfigLoader, didToAgent, baseUrl, STARTUP_FETCH_RETRY, retryWithBackoff, isTransientFetchError, isObject, isString, isNumber, busRequest, isArray, resourceId, getPrimaryMediaType, isGenerationJobParams, capabilitiesOf, assembleAnnotation, findClaimSpan, textSourceOf, yieldsGeometryOf, decodeRepresentation, GENERATABLE_MEDIA_TYPES, locate, createFragmentSelector, deriveViews, estimateTokens, chunkText, reconcileSelector, getLocaleEnglishName } from '@semiont/core';
+import { replyChannelsFor, createTomlConfigLoader, didToAgent, baseUrl, STARTUP_FETCH_RETRY, retryWithBackoff, isTransientFetchError, isObject, isString, isNumber, busRequest, isArray, resourceId, getPrimaryMediaType, isGenerationJobParams, capabilitiesOf, assembleAnnotation, findClaimSpan, textSourceOf, yieldsGeometryOf, decodeRepresentation, GENERATABLE_MEDIA_TYPES, BusRequestError, locate, createFragmentSelector, deriveViews, annotationId, estimateTokens, chunkText, reconcileSelector, getLocaleEnglishName } from '@semiont/core';
 import { archivistContentReads, extractPdfTextLayer, withinByteBudget, MAX_PDF_BYTES } from '@semiont/content';
 import { withSpan, SpanKind, recordJobOutcome, recordAnchorOutcome, recordDetectionCall } from '@semiont/observability';
 import { createInferenceClient, StructuredReadError } from '@semiont/inference';
@@ -9476,12 +9476,16 @@ var YieldCollapseError = class extends DeterministicJobError {
   /** What the flagged extraction DID find — every span write-time-verified,
    * so discarding it at the floor would add loss on top of the under-report.
    * Carried on the error because the flag site cannot know whether descent
-   * remains possible; the floor is the subdivider's knowledge. */
-  constructor(message, salvage = []) {
+   * remains possible; the floor is the subdivider's knowledge. The verdict
+   * rides beside it so an acceptance can report evidence, not a message
+   * string. */
+  constructor(message, salvage, verdict) {
     super(message);
     this.salvage = salvage;
+    this.verdict = verdict;
   }
   salvage;
+  verdict;
   name = "YieldCollapseError";
 };
 function deriveDetectionBudget(limits, scaffoldTokens, typesPerCall) {
@@ -9535,7 +9539,7 @@ function outcomeOf(error) {
   if (error instanceof InferenceTimeoutError) return "timeout";
   return "error";
 }
-async function callChunkSubdividing(label, chunk, chunking, call, logger2) {
+async function callChunkSubdividing(label, chunk, chunking, call, logger2, onUnderReport, onCounted) {
   async function recorded(piece, depth, reroll) {
     const start = performance.now();
     try {
@@ -9550,6 +9554,7 @@ async function callChunkSubdividing(label, chunk, chunking, call, logger2) {
         outcome: "success",
         ...result.usage ? { inputTokens: result.usage.inputTokens, outputTokens: result.usage.outputTokens } : {}
       });
+      if (result.counted !== void 0) onCounted?.(result.counted);
       return result;
     } catch (error) {
       recordDetectionCall({
@@ -9580,6 +9585,8 @@ async function callChunkSubdividing(label, chunk, chunking, call, logger2) {
             salvaged: error.salvage.length,
             error: error.message
           });
+          onUnderReport?.(error.verdict);
+          onCounted?.(error.verdict.counted);
           return error.salvage;
         }
         if (!truncation(error)) throw error;
@@ -10103,7 +10110,7 @@ function logAnchorMethod(motivation, exact, anchorMethod) {
 }
 
 // src/workers/annotation-detection.ts
-async function detectInChunks(client, content, buildPrompt, motivation, elementSchema, parse, onActivity) {
+async function detectInChunks(client, content, buildPrompt, motivation, elementSchema, parse, onActivity, onChunkResults) {
   const limits = await client.limits();
   const scaffoldTokens = estimateTokens(buildPrompt(""));
   const { chunking, outputBudget } = deriveDetectionBudget(limits, scaffoldTokens, 1);
@@ -10123,7 +10130,9 @@ async function detectInChunks(client, content, buildPrompt, motivation, elementS
       assertNotTruncated(response, `${motivation} detection`, i + 1, chunks.length, outputBudget);
       return { items: response.items, ...response.usage ? { usage: response.usage } : {} };
     });
-    collected.push(...parse(items));
+    const fromChunk = parse(items);
+    collected.push(...fromChunk);
+    await onChunkResults?.(fromChunk);
     if (i < chunks.length - 1) {
       onActivity?.(i + 1, chunks.length);
     }
@@ -10139,7 +10148,7 @@ var AnnotationDetection = class {
    * (source-resource locale). See `types.ts` "Locale conventions" for the
    * full discussion.
    */
-  static async detectComments(content, client, instructions, tone, density, language, sourceLanguage, onActivity) {
+  static async detectComments(content, client, instructions, tone, density, language, sourceLanguage, onActivity, onChunkResults) {
     return detectInChunks(
       client,
       content,
@@ -10147,7 +10156,8 @@ var AnnotationDetection = class {
       "comment",
       COMMENT_ELEMENT_SCHEMA,
       (items) => MotivationParsers.parseComments(items, content),
-      onActivity
+      onActivity,
+      onChunkResults
     );
   }
   /**
@@ -10157,7 +10167,7 @@ var AnnotationDetection = class {
    * applies, used in the prompt so the LLM analyzes non-English source
    * correctly.
    */
-  static async detectHighlights(content, client, instructions, density, sourceLanguage, onActivity) {
+  static async detectHighlights(content, client, instructions, density, sourceLanguage, onActivity, onChunkResults) {
     return detectInChunks(
       client,
       content,
@@ -10165,7 +10175,8 @@ var AnnotationDetection = class {
       "highlight",
       HIGHLIGHT_ELEMENT_SCHEMA,
       (items) => MotivationParsers.parseHighlights(items, content),
-      onActivity
+      onActivity,
+      onChunkResults
     );
   }
   /**
@@ -10175,7 +10186,7 @@ var AnnotationDetection = class {
    * (annotation body locale). `sourceLanguage` is the locale of the content
    * being analyzed (source-resource locale).
    */
-  static async detectAssessments(content, client, instructions, tone, density, language, sourceLanguage, onActivity) {
+  static async detectAssessments(content, client, instructions, tone, density, language, sourceLanguage, onActivity, onChunkResults) {
     return detectInChunks(
       client,
       content,
@@ -10183,7 +10194,8 @@ var AnnotationDetection = class {
       "assessment",
       ASSESSMENT_ELEMENT_SCHEMA,
       (items) => MotivationParsers.parseAssessments(items, content),
-      onActivity
+      onActivity,
+      onChunkResults
     );
   }
   /**
@@ -10198,7 +10210,7 @@ var AnnotationDetection = class {
    * identifiers, not LLM-generated text — so it's consumed at the body-stamp
    * site, not here.
    */
-  static async detectTags(content, client, schema, category, sourceLanguage, onActivity) {
+  static async detectTags(content, client, schema, category, sourceLanguage, onActivity, onChunkResults) {
     const categoryInfo = schema.tags.find((t) => t.name === category);
     if (!categoryInfo) {
       throw new Error(`Invalid category "${category}" for schema ${schema.id}`);
@@ -10219,7 +10231,8 @@ var AnnotationDetection = class {
       "tag",
       TAG_ELEMENT_SCHEMA,
       (items) => MotivationParsers.parseTags(items),
-      onActivity
+      onActivity,
+      onChunkResults ? async (raw) => onChunkResults(MotivationParsers.validateTagOffsets(raw, content, category)) : void 0
     );
     return MotivationParsers.validateTagOffsets(parsedTags, content, category);
   }
@@ -10256,20 +10269,22 @@ ${piece}
       pieceChars: piece.length,
       error: err instanceof Error ? err.message : String(err)
     });
-    return;
+    return void 0;
   }
   if (counted === void 0) {
     logger2.warn("Count-verifier answer carried no number \u2014 yield check skipped for this chunk", { pieceChars: piece.length });
-    return;
+    return void 0;
   }
   if (items.length * YIELD_COLLAPSE_BAND < counted) {
     throw new YieldCollapseError(
       `Extraction found ${items.length} entities where a count call reports ~${counted} mentions (band \xD7${YIELD_COLLAPSE_BAND}) on a ${piece.length}-char chunk \u2014 silent yield collapse (F7): deterministic \u2014 a same-size retry returns the identical under-report.`,
-      [...items]
+      [...items],
+      { found: items.length, counted, pieceChars: piece.length }
     );
   }
+  return counted;
 }
-async function extractEntities(exact, entityTypes, client, includeDescriptiveReferences, logger2, sourceLanguage, onActivity) {
+async function extractEntities(exact, entityTypes, client, includeDescriptiveReferences, logger2, sourceLanguage, onActivity, onUnderReport, onCounted, onChunkResults) {
   const entityTypesDescription = entityTypes.map((et) => {
     if (typeof et === "string") {
       return et;
@@ -10350,14 +10365,17 @@ Example output:
         items: response.items.length
       });
       assertNotTruncated(response, "Entity extraction", i + 1, chunks.length, outputBudget);
-      if (verifyYield) {
-        await assertYieldNotCollapsed(client, piece, response.items, entityTypesDescription, logger2);
-      }
-      return { items: response.items, ...response.usage ? { usage: response.usage } : {} };
-    }, logger2);
+      const counted = verifyYield ? await assertYieldNotCollapsed(client, piece, response.items, entityTypesDescription, logger2) : void 0;
+      return {
+        items: response.items,
+        ...response.usage ? { usage: response.usage } : {},
+        ...counted !== void 0 ? { counted } : {}
+      };
+    }, logger2, onUnderReport, onCounted);
+    const fromChunk = [];
     for (const e of items) {
       if (isObject(e) && isString(e.exact) && isString(e.entityType)) {
-        collected.push({
+        fromChunk.push({
           exact: e.exact,
           entityType: e.entityType,
           ...isString(e.prefix) ? { prefix: e.prefix } : {},
@@ -10367,6 +10385,8 @@ Example output:
         logger2.debug("Dropped malformed LLM entity", { entity: e });
       }
     }
+    collected.push(...fromChunk);
+    await onChunkResults?.(fromChunk);
     if (i < chunks.length - 1) {
       onActivity?.(i + 1, chunks.length);
     }
@@ -10725,16 +10745,18 @@ function annotationDedupeKey(ann) {
   }
   return [ann.motivation, anchor, JSON.stringify(ann.body ?? null)].join("|");
 }
-function dedupeAnnotations(annotations) {
+function makeSpanDeduper() {
   const seen = /* @__PURE__ */ new Set();
-  const out = [];
-  for (const ann of annotations) {
-    const key = annotationDedupeKey(ann);
-    if (seen.has(key)) continue;
-    seen.add(key);
-    out.push(ann);
-  }
-  return out;
+  return (annotations) => {
+    const out = [];
+    for (const ann of annotations) {
+      const key = annotationDedupeKey(ann);
+      if (seen.has(key)) continue;
+      seen.add(key);
+      out.push(ann);
+    }
+    return out;
+  };
 }
 function buildTextAnnotation(content, resourceId, userId, generator, motivation, match, body) {
   if (content.substring(match.start, match.end) !== match.exact) {
@@ -10792,7 +10814,12 @@ function buildPdfAnnotation(anchored, resourceId, userId, generator, motivation,
     Math.max(...overlap.map((i) => i.end))
   ) : "";
   const normalize = (s) => s.replace(/\s+/g, " ").trim();
-  if (rects.length === 0 || !normalize(coveredText).includes(normalize(match.exact))) {
+  if (rects.length === 0) {
+    throw new Error(
+      `buildPdfAnnotation invariant: no rects located for offsets ${match.start}-${match.end} for resource ${resourceId}, motivation ${motivation}`
+    );
+  }
+  if (!normalize(coveredText).includes(normalize(match.exact))) {
     throw new Error(
       `buildPdfAnnotation invariant: covered text does not contain exact for resource ${resourceId}, motivation ${motivation}`
     );
@@ -10828,27 +10855,32 @@ function buildPdfAnnotation(anchored, resourceId, userId, generator, motivation,
     ...body !== void 0 ? { body } : {}
   };
 }
-async function processHighlightJob(content, inferenceClient, params, buildAnnotation, onProgress) {
+async function processHighlightJob(content, inferenceClient, params, buildAnnotation, onProgress, onChunkComplete) {
   const echo = detectionEcho(params);
   onProgress(10, { code: "loading" }, echo);
   onProgress(30, { code: "analyzing" }, echo);
-  const highlights = await AnnotationDetection.detectHighlights(
+  const dedupe = makeSpanDeduper();
+  let found = 0;
+  let created = 0;
+  await AnnotationDetection.detectHighlights(
     content,
     inferenceClient,
     params.instructions,
     params.density,
     params.sourceLanguage,
     // Liveness (chunk boundaries + in-flight heartbeat): 30–60 band.
-    (completed, total) => onProgress(30 + Math.round(completed / total * 30), { code: "analyzing" }, echo)
+    (completed, total) => onProgress(30 + Math.round(completed / total * 30), { code: "analyzing" }, echo),
+    async (matches) => {
+      found += matches.length;
+      const fresh = dedupe(matches.map((h) => buildAnnotation("highlighting", h)));
+      created += fresh.length;
+      onProgress(60, { code: "creating-annotations", count: created }, echo);
+      await onChunkComplete(fresh);
+    }
   );
-  onProgress(60, { code: "creating-annotations", count: highlights.length }, echo);
-  const annotations = dedupeAnnotations(highlights.map(
-    (h) => buildAnnotation("highlighting", h)
-  ));
-  onProgress(100, { code: "complete-created", count: annotations.length, kind: "highlight" }, echo);
+  onProgress(100, { code: "complete-created", count: created, kind: "highlight" }, echo);
   return {
-    annotations,
-    result: { kind: "highlight-annotation", highlightsFound: highlights.length, highlightsCreated: annotations.length }
+    result: { kind: "highlight-annotation", highlightsFound: found, highlightsCreated: created }
   };
 }
 function detectionEcho(p) {
@@ -10858,11 +10890,15 @@ function detectionEcho(p) {
   if (p.density !== void 0) requestParams.push({ label: "density", value: String(p.density) });
   return requestParams.length > 0 ? { requestParams } : {};
 }
-async function processCommentJob(content, inferenceClient, params, buildAnnotation, onProgress) {
+async function processCommentJob(content, inferenceClient, params, buildAnnotation, onProgress, onChunkComplete) {
   const echo = detectionEcho(params);
   onProgress(10, { code: "loading" }, echo);
   onProgress(30, { code: "analyzing" }, echo);
-  const comments = await AnnotationDetection.detectComments(
+  const bodyLanguage = params.language ?? "en";
+  const dedupe = makeSpanDeduper();
+  let found = 0;
+  let created = 0;
+  await AnnotationDetection.detectComments(
     content,
     inferenceClient,
     params.instructions,
@@ -10871,31 +10907,38 @@ async function processCommentJob(content, inferenceClient, params, buildAnnotati
     params.language,
     params.sourceLanguage,
     // Liveness (chunk boundaries + in-flight heartbeat): 30–60 band.
-    (completed, total) => onProgress(30 + Math.round(completed / total * 30), { code: "analyzing" }, echo)
+    (completed, total) => onProgress(30 + Math.round(completed / total * 30), { code: "analyzing" }, echo),
+    async (comments) => {
+      found += comments.length;
+      const fresh = dedupe(comments.map(
+        (c) => (
+          // Match the pre-#651 CommentAnnotationWorker: include format and
+          // language on the body TextualBody. Optional in the schema, but
+          // consumers that do language-aware rendering rely on them.
+          buildAnnotation("commenting", c, [
+            { type: "TextualBody", value: c.comment, purpose: "commenting", format: "text/plain", language: bodyLanguage }
+          ])
+        )
+      ));
+      created += fresh.length;
+      onProgress(60, { code: "creating-annotations", count: created }, echo);
+      await onChunkComplete(fresh);
+    }
   );
-  onProgress(60, { code: "creating-annotations", count: comments.length }, echo);
-  const bodyLanguage = params.language ?? "en";
-  const annotations = dedupeAnnotations(comments.map(
-    (c) => (
-      // Match the pre-#651 CommentAnnotationWorker: include format and
-      // language on the body TextualBody. Optional in the schema, but
-      // consumers that do language-aware rendering rely on them.
-      buildAnnotation("commenting", c, [
-        { type: "TextualBody", value: c.comment, purpose: "commenting", format: "text/plain", language: bodyLanguage }
-      ])
-    )
-  ));
-  onProgress(100, { code: "complete-created", count: annotations.length, kind: "comment" }, echo);
+  onProgress(100, { code: "complete-created", count: created, kind: "comment" }, echo);
   return {
-    annotations,
-    result: { kind: "comment-annotation", commentsFound: comments.length, commentsCreated: annotations.length }
+    result: { kind: "comment-annotation", commentsFound: found, commentsCreated: created }
   };
 }
-async function processAssessmentJob(content, inferenceClient, params, buildAnnotation, onProgress) {
+async function processAssessmentJob(content, inferenceClient, params, buildAnnotation, onProgress, onChunkComplete) {
   const echo = detectionEcho(params);
   onProgress(10, { code: "loading" }, echo);
   onProgress(30, { code: "analyzing" }, echo);
-  const assessments = await AnnotationDetection.detectAssessments(
+  const bodyLanguage = params.language ?? "en";
+  const dedupe = makeSpanDeduper();
+  let found = 0;
+  let created = 0;
+  await AnnotationDetection.detectAssessments(
     content,
     inferenceClient,
     params.instructions,
@@ -10904,40 +10947,45 @@ async function processAssessmentJob(content, inferenceClient, params, buildAnnot
     params.language,
     params.sourceLanguage,
     // Liveness (chunk boundaries + in-flight heartbeat): 30–60 band.
-    (completed, total) => onProgress(30 + Math.round(completed / total * 30), { code: "analyzing" }, echo)
+    (completed, total) => onProgress(30 + Math.round(completed / total * 30), { code: "analyzing" }, echo),
+    async (assessments) => {
+      found += assessments.length;
+      const fresh = dedupe(assessments.map(
+        (a) => (
+          // Single-object body with purpose aligned to motivation, matching the
+          // pre-#651 AssessmentAnnotationWorker's shape and the majority of
+          // persisted assessments. Do not switch to an array or to
+          // purpose='describing' — that loses the "this is an assessment, not
+          // a description" signal and breaks existing readers that access
+          // `body.value` directly on the object.
+          buildAnnotation("assessing", a, {
+            type: "TextualBody",
+            value: a.assessment,
+            purpose: "assessing",
+            format: "text/plain",
+            language: bodyLanguage
+          })
+        )
+      ));
+      created += fresh.length;
+      onProgress(60, { code: "creating-annotations", count: created }, echo);
+      await onChunkComplete(fresh);
+    }
   );
-  onProgress(60, { code: "creating-annotations", count: assessments.length }, echo);
-  const bodyLanguage = params.language ?? "en";
-  const annotations = dedupeAnnotations(assessments.map(
-    (a) => (
-      // Single-object body with purpose aligned to motivation, matching the
-      // pre-#651 AssessmentAnnotationWorker's shape and the majority of
-      // persisted assessments. Do not switch to an array or to
-      // purpose='describing' — that loses the "this is an assessment, not
-      // a description" signal and breaks existing readers that access
-      // `body.value` directly on the object.
-      buildAnnotation("assessing", a, {
-        type: "TextualBody",
-        value: a.assessment,
-        purpose: "assessing",
-        format: "text/plain",
-        language: bodyLanguage
-      })
-    )
-  ));
-  onProgress(100, { code: "complete-created", count: annotations.length, kind: "assessment" }, echo);
+  onProgress(100, { code: "complete-created", count: created, kind: "assessment" }, echo);
   return {
-    annotations,
-    result: { kind: "assessment-annotation", assessmentsFound: assessments.length, assessmentsCreated: annotations.length }
+    result: { kind: "assessment-annotation", assessmentsFound: found, assessmentsCreated: created }
   };
 }
-async function processReferenceJob(content, inferenceClient, params, buildAnnotation, onProgress, logger2, onUnitComplete, signal) {
+async function processReferenceJob(content, inferenceClient, params, buildAnnotation, onProgress, logger2, onUnitComplete, signal, onChunkComplete) {
   const entityTypeNames = params.entityTypes.map(String);
   const requestParams = [{ label: "entity-types", value: entityTypeNames.join(", ") }];
   const completedItems = [];
   let totalFound = 0;
   let totalEmitted = 0;
   let errors = 0;
+  let totalUnderReportedPieces = 0;
+  let totalExpected = 0;
   onProgress(10, { code: "loading" }, { requestParams });
   const bodyLanguage = params.language ?? "en";
   let completed = 0;
@@ -10949,6 +10997,7 @@ async function processReferenceJob(content, inferenceClient, params, buildAnnota
       processed: completed,
       total,
       entitiesFound: totalFound,
+      ...totalExpected > 0 ? { entitiesExpected: totalExpected } : {},
       entitiesEmitted: totalEmitted,
       completedItems: [...completedItems],
       requestParams
@@ -10958,7 +11007,14 @@ async function processReferenceJob(content, inferenceClient, params, buildAnnota
     if (!entityTypeName) return;
     if (signal?.aborted) return;
     emitTypeProgress(entityTypeName);
-    const extractedEntities = await extractEntities(
+    const unresolvedBody = [
+      { type: "TextualBody", value: entityTypeName, purpose: "tagging", format: "text/plain", language: bodyLanguage }
+    ];
+    const dedupe = makeSpanDeduper();
+    let unitFound = 0;
+    let unitPersisted = 0;
+    let underReported;
+    await extractEntities(
       content,
       [entityTypeName],
       inferenceClient,
@@ -10970,54 +11026,80 @@ async function processReferenceJob(content, inferenceClient, params, buildAnnota
       // not silent. It repeats the current position rather than inventing an
       // advance — the stall watchdog, janitor and client timeout need a signal,
       // not a monotone.
-      () => emitTypeProgress(entityTypeName)
-    );
-    const unresolvedBody = [
-      { type: "TextualBody", value: entityTypeName, purpose: "tagging", format: "text/plain", language: bodyLanguage }
-    ];
-    const built = [];
-    for (const entity of extractedEntities) {
-      const reconciled = reconcileSelector(content, {
-        exact: entity.exact,
-        ...entity.prefix !== void 0 ? { prefix: entity.prefix } : {},
-        ...entity.suffix !== void 0 ? { suffix: entity.suffix } : {}
-      });
-      if (!reconciled) {
-        logger2.error("Entity dropped \u2014 text not found in source", {
-          text: entity.exact,
-          entityType: entity.entityType
-        });
-        errors++;
-        continue;
+      () => emitTypeProgress(entityTypeName),
+      (verdict) => {
+        underReported = {
+          pieces: (underReported?.pieces ?? 0) + 1,
+          found: (underReported?.found ?? 0) + verdict.found,
+          counted: (underReported?.counted ?? 0) + verdict.counted
+        };
+      },
+      (counted) => {
+        totalExpected += counted;
+        emitTypeProgress(entityTypeName);
+      },
+      async (chunkEntities) => {
+        const built = [];
+        for (const entity of chunkEntities) {
+          const reconciled = reconcileSelector(content, {
+            exact: entity.exact,
+            ...entity.prefix !== void 0 ? { prefix: entity.prefix } : {},
+            ...entity.suffix !== void 0 ? { suffix: entity.suffix } : {}
+          });
+          if (!reconciled) {
+            logger2.error("Entity dropped \u2014 text not found in source", {
+              text: entity.exact,
+              entityType: entity.entityType
+            });
+            errors++;
+            continue;
+          }
+          noteAnchor("reference", entity.exact, reconciled.anchorMethod, logger2);
+          built.push(buildAnnotation("linking", toMatch(reconciled), unresolvedBody));
+        }
+        const fresh = dedupe(built);
+        await onChunkComplete?.(fresh);
+        unitFound += chunkEntities.length;
+        unitPersisted += fresh.length;
+        totalFound += chunkEntities.length;
+        totalEmitted += fresh.length;
+        emitTypeProgress(entityTypeName);
       }
-      noteAnchor("reference", entity.exact, reconciled.anchorMethod, logger2);
-      const ann = buildAnnotation("linking", toMatch(reconciled), unresolvedBody);
-      built.push(ann);
-    }
-    const unitAnnotations = dedupeAnnotations(built);
-    await onUnitComplete(entityTypeName, unitAnnotations);
-    totalEmitted += unitAnnotations.length;
-    totalFound += extractedEntities.length;
+    );
+    await onUnitComplete(entityTypeName);
     completedItems.push({
       value: entityTypeName,
-      foundCount: extractedEntities.length,
-      persistedCount: unitAnnotations.length
+      foundCount: unitFound,
+      persistedCount: unitPersisted,
+      ...underReported ? { underReported } : {}
     });
+    if (underReported) totalUnderReportedPieces += underReported.pieces;
     completed++;
     emitTypeProgress(entityTypeName);
   });
   onProgress(100, { code: "complete-created", count: totalEmitted, kind: "reference" }, {
+    ...totalExpected > 0 ? { entitiesExpected: totalExpected } : {},
     completedItems: [...completedItems],
     requestParams
   });
   return {
-    result: { kind: "reference-annotation", totalFound, totalEmitted, errors }
+    result: {
+      kind: "reference-annotation",
+      totalFound,
+      totalEmitted,
+      errors,
+      ...totalUnderReportedPieces > 0 ? { underReportedPieces: totalUnderReportedPieces } : {}
+    }
   };
 }
-async function processTagJob(content, inferenceClient, params, buildAnnotation, onProgress) {
+async function processTagJob(content, inferenceClient, params, buildAnnotation, onProgress, onChunkComplete) {
   onProgress(10, { code: "loading" });
   onProgress(30, { code: "analyzing-tags" });
-  const allTags = [];
+  const bodyLanguage = params.language ?? "en";
+  const dedupe = makeSpanDeduper();
+  let found = 0;
+  let created = 0;
+  const byCategory = {};
   const completedItems = [];
   for (let c = 0; c < params.categories.length; c++) {
     const category = params.categories[c];
@@ -11032,7 +11114,8 @@ async function processTagJob(content, inferenceClient, params, buildAnnotation,
       { code: "analyzing-tags" },
       position()
     );
-    const categoryTags = await AnnotationDetection.detectTags(
+    let categoryFound = 0;
+    await AnnotationDetection.detectTags(
       content,
       inferenceClient,
       params.schema,
@@ -11044,31 +11127,32 @@ async function processTagJob(content, inferenceClient, params, buildAnnotation,
         30 + Math.round((c + completed / total) / params.categories.length * 30),
         { code: "analyzing-tags" },
         position()
-      )
+      ),
+      async (matches) => {
+        categoryFound += matches.length;
+        const fresh = dedupe(matches.map((t) => {
+          const cat = t.category ?? "unknown";
+          return buildAnnotation("tagging", t, [
+            { type: "TextualBody", value: cat, purpose: "tagging", format: "text/plain", language: bodyLanguage },
+            { type: "TextualBody", value: params.schema.id, purpose: "classifying", format: "text/plain" }
+          ]);
+        }));
+        created += fresh.length;
+        for (const ann of fresh) {
+          const body = ann.body;
+          const cat = Array.isArray(body) && typeof body[0]?.value === "string" ? body[0].value : "unknown";
+          byCategory[cat] = (byCategory[cat] ?? 0) + 1;
+        }
+        onProgress(60, { code: "creating-tag-annotations", count: created });
+        await onChunkComplete(fresh);
+      }
     );
-    completedItems.push({ value: category, foundCount: categoryTags.length });
-    allTags.push(...categoryTags);
-  }
-  const tags = allTags;
-  onProgress(60, { code: "creating-tag-annotations", count: tags.length });
-  const bodyLanguage = params.language ?? "en";
-  const annotations = dedupeAnnotations(tags.map((t) => {
-    const category = t.category ?? "unknown";
-    return buildAnnotation("tagging", t, [
-      { type: "TextualBody", value: category, purpose: "tagging", format: "text/plain", language: bodyLanguage },
-      { type: "TextualBody", value: params.schema.id, purpose: "classifying", format: "text/plain" }
-    ]);
-  }));
-  const byCategory = {};
-  for (const ann of annotations) {
-    const body = ann.body;
-    const category = Array.isArray(body) && typeof body[0]?.value === "string" ? body[0].value : "unknown";
-    byCategory[category] = (byCategory[category] ?? 0) + 1;
+    found += categoryFound;
+    completedItems.push({ value: category, foundCount: categoryFound });
   }
-  onProgress(100, { code: "complete-created", count: annotations.length, kind: "tag" });
+  onProgress(100, { code: "complete-created", count: created, kind: "tag" });
   return {
-    annotations,
-    result: { kind: "tag-annotation", tagsFound: tags.length, tagsCreated: annotations.length, byCategory }
+    result: { kind: "tag-annotation", tagsFound: found, tagsCreated: created, byCategory }
   };
 }
 function assertWithinOutputBudget(byteLength) {
@@ -11269,13 +11353,40 @@ function referenceIdOf(job) {
 }
 var MARK_COMMIT_TIMEOUT_MS = 6e4;
 async function commitAnnotations(session, resourceId, annotations) {
-  if (annotations.length === 0) return;
-  await busRequest(
-    workerBusAsPrimitive(session.client.transport.actor),
-    "mark:commit",
-    { resourceId, annotations },
-    MARK_COMMIT_TIMEOUT_MS
-  );
+  if (annotations.length === 0) return void 0;
+  try {
+    await busRequest(
+      workerBusAsPrimitive(session.client.transport.actor),
+      "mark:commit",
+      { resourceId, annotations },
+      MARK_COMMIT_TIMEOUT_MS
+    );
+    return "acknowledged";
+  } catch (error) {
+    if (!(error instanceof BusRequestError) || error.code !== "bus.timeout") throw error;
+    const evidence = await probeDurability(session, resourceId, annotations);
+    if (evidence === "probe-confirmed") return evidence;
+    throw new CommitDurabilityError(error.message, evidence, error);
+  }
+}
+var CommitDurabilityError = class extends Error {
+  constructor(message, durability, cause) {
+    super(message, { cause });
+    this.durability = durability;
+  }
+  durability;
+  name = "CommitDurabilityError";
+};
+async function probeDurability(session, resourceId$1, annotations) {
+  const last = annotations[annotations.length - 1];
+  if (!last) return "probe-unreachable";
+  try {
+    await session.client.browse.annotation(resourceId(resourceId$1), annotationId(String(last.id))).fresh();
+    return "probe-confirmed";
+  } catch (error) {
+    const code = isObject(error) && isString(error.code) ? error.code : void 0;
+    return code === "bus.rejected" ? "probe-refused" : "probe-unreachable";
+  }
 }
 async function emitEvent(session, channel, payload) {
   await session.client.transport.emit(channel, payload);
@@ -11320,6 +11431,10 @@ function startWorkerProcess(config) {
           error: message,
           ...completedUnits && completedUnits.length > 0 ? { completedUnits } : {},
           ...failureClass !== void 0 ? { failureClass } : {},
+          // What the commit path OBSERVED about durability, when the failure
+          // came from a commit at all. Present only on that path: absent means
+          // the question never arose, never that durability was ruled out.
+          ...error instanceof CommitDurabilityError ? { durability: error.durability } : {},
           // Whether this failure is the END, answered by the same predicate
           // the queue applies at failJob (JOB-RESTART-SAFETY P5). Without
           // it a client cannot tell a recovering run from a dead one: it
@@ -11376,6 +11491,12 @@ async function handleJobInner(adapter, config, job, completedUnitsByJob, signal)
     jobType,
     ...annotationId ? { annotationId } : {}
   };
+  let durability;
+  const record = (evidence) => {
+    if (evidence === void 0) return;
+    if (durability === void 0 || durability === "acknowledged") durability = evidence;
+  };
+  const terminalBase = () => ({ ...lifecycleBase, ...durability ? { durability } : {} });
   await emitEvent(session, "job:start", lifecycleBase);
   if (!config.jobTypes.includes(jobType)) {
     adapter.failJob(jobId, `Worker not configured for job type: ${jobType}`);
@@ -11401,7 +11522,7 @@ async function handleJobInner(adapter, config, job, completedUnitsByJob, signal)
         throw new DeterministicJobError(`Cannot run ${jobType} on resource ${resourceId$1}: anchored-text consult returned '${source.declined}'`);
       }
       await emitEvent(session, "job:complete", {
-        ...lifecycleBase,
+        ...terminalBase(),
         result: {
           kind: "declined",
           declined: true,
@@ -11416,7 +11537,7 @@ async function handleJobInner(adapter, config, job, completedUnitsByJob, signal)
   const onProgress = (percentage, message, extra) => {
     adapter.touchActivity();
     emitEvent(session, "job:report-progress", {
-      ...lifecycleBase,
+      ...terminalBase(),
       percentage,
       progress: {
         percentage,
@@ -11428,44 +11549,56 @@ async function handleJobInner(adapter, config, job, completedUnitsByJob, signal)
     });
   };
   if (jobType === "highlight-annotation") {
-    const { annotations, result } = await processHighlightJob(
+    const { result } = await processHighlightJob(
       ready.text,
       inferenceClient,
       asJobParams(job.params),
       ready.buildAnnotation,
-      onProgress
+      onProgress,
+      // The durability write, per chunk, awaited; folds into the terminal
+      // durability evidence like every commit.
+      async (annotations) => {
+        record(await commitAnnotations(session, String(resourceId$1), annotations));
+      }
     );
-    await commitAnnotations(session, String(resourceId$1), annotations);
     await emitEvent(session, "job:complete", {
-      ...lifecycleBase,
+      ...terminalBase(),
       result
     });
     adapter.completeJob();
   } else if (jobType === "comment-annotation") {
-    const { annotations, result } = await processCommentJob(
+    const { result } = await processCommentJob(
       ready.text,
       inferenceClient,
       asJobParams(job.params),
       ready.buildAnnotation,
-      onProgress
+      onProgress,
+      // The durability write, per chunk, awaited; folds into the terminal
+      // durability evidence like every commit.
+      async (annotations) => {
+        record(await commitAnnotations(session, String(resourceId$1), annotations));
+      }
     );
-    await commitAnnotations(session, String(resourceId$1), annotations);
     await emitEvent(session, "job:complete", {
-      ...lifecycleBase,
+      ...terminalBase(),
       result
     });
     adapter.completeJob();
   } else if (jobType === "assessment-annotation") {
-    const { annotations, result } = await processAssessmentJob(
+    const { result } = await processAssessmentJob(
       ready.text,
       inferenceClient,
       asJobParams(job.params),
       ready.buildAnnotation,
-      onProgress
+      onProgress,
+      // The durability write, per chunk, awaited; folds into the terminal
+      // durability evidence like every commit.
+      async (annotations) => {
+        record(await commitAnnotations(session, String(resourceId$1), annotations));
+      }
     );
-    await commitAnnotations(session, String(resourceId$1), annotations);
     await emitEvent(session, "job:complete", {
-      ...lifecycleBase,
+      ...terminalBase(),
       result
     });
     adapter.completeJob();
@@ -11485,40 +11618,48 @@ async function handleJobInner(adapter, config, job, completedUnitsByJob, signal)
       ready.buildAnnotation,
       onProgress,
       config.logger,
-      async (unit, annotations) => {
-        await commitAnnotations(session, String(resourceId$1), annotations);
+      async (unit) => {
         committed.push(unit);
         await emitEvent(session, "job:checkpoint", {
           jobId: job.jobId,
           completedUnits: [...committed]
         });
       },
-      signal
+      signal,
+      // The durability write, per chunk, awaited; folds into the terminal
+      // durability evidence like every commit.
+      async (annotations) => {
+        record(await commitAnnotations(session, String(resourceId$1), annotations));
+      }
     );
     if (signal?.aborted) {
       await emitEvent(session, "job:cancel", {
-        ...lifecycleBase,
+        ...terminalBase(),
         ...committed.length > 0 ? { completedUnits: [...committed] } : {}
       });
       adapter.completeJob();
       return;
     }
     await emitEvent(session, "job:complete", {
-      ...lifecycleBase,
+      ...terminalBase(),
       result
     });
     adapter.completeJob();
   } else if (jobType === "tag-annotation") {
-    const { annotations, result } = await processTagJob(
+    const { result } = await processTagJob(
       ready.text,
       inferenceClient,
       asJobParams(job.params),
       ready.buildAnnotation,
-      onProgress
+      onProgress,
+      // The durability write, per chunk, awaited; folds into the terminal
+      // durability evidence like every commit.
+      async (annotations) => {
+        record(await commitAnnotations(session, String(resourceId$1), annotations));
+      }
     );
-    await commitAnnotations(session, String(resourceId$1), annotations);
     await emitEvent(session, "job:complete", {
-      ...lifecycleBase,
+      ...terminalBase(),
       result
     });
     adapter.completeJob();
@@ -11567,7 +11708,7 @@ async function handleJobInner(adapter, config, job, completedUnitsByJob, signal)
         },
         generator
       );
-      await commitAnnotations(session, String(resourceId$1), [provenanceRef]);
+      record(await commitAnnotations(session, String(resourceId$1), [provenanceRef]));
     }
     const citationRefs = [];
     if (genResult.format === "application/pdf" && genResult.citations.length > 0) {
@@ -11621,9 +11762,9 @@ async function handleJobInner(adapter, config, job, completedUnitsByJob, signal)
         citationRefs.push(citationRef);
       }
     }
-    await commitAnnotations(session, String(newResourceId), citationRefs);
+    record(await commitAnnotations(session, String(newResourceId), citationRefs));
     await emitEvent(session, "job:complete", {
-      ...lifecycleBase,
+      ...terminalBase(),
       result: { kind: "generation", resourceId: newResourceId, resourceName: genResult.title, truncated: genResult.result.truncated }
     });
     adapter.completeJob();
@@ -11684,7 +11825,12 @@ var WORKER_AWAITED_OPERATIONS = [
   // P6). The worker AWAITS this one — a unit may not advance until its
   // annotations are in the event log — so its replies must be in the narrow
   // channel set or every commit fails fast with `bus.unsubscribed`.
-  "mark:commit"
+  "mark:commit",
+  // The durability probe for a commit whose acknowledgement never routed
+  // (COMMIT-ACK-FALSE-FAILURE F1). SINGULAR by design: the annotation LIST
+  // channel is the multi-MB fan-out this narrowing exists to keep out, and a
+  // rare error path is no reason to let it back in.
+  "browse:annotation-requested"
 ];
 var WORKER_CHANNELS = replyChannelsFor(WORKER_AWAITED_OPERATIONS);
 function parseGatewayUrl(url) {
@@ -11888,8 +12034,9 @@ for (const jobType of ALL_JOB_TYPES) {
   group.jobTypes.push(jobType);
 }
 async function main() {
-  const { initObservabilityNode } = await import('@semiont/observability/node');
+  const { initObservabilityNode, registerSupervisorRestartCount } = await import('@semiont/observability/node');
   initObservabilityNode({ serviceName: "semiont-worker" });
+  registerSupervisorRestartCount();
   logger.info("Starting agents", {
     baseUrl: gatewayBaseUrl,
     agents: Array.from(groups.values()).map((g) => ({