@semiont/jobs 0.5.28 → 0.5.30

package/dist/index.js

@@ -1,8 +1,9 @@
 import { promises, mkdtempSync, writeFileSync, readFileSync, rmSync } from 'fs';
 import * as path from 'path';
 import { join } from 'path';
-import { jobId, deriveViews, reconcileSelector, GENERATABLE_MEDIA_TYPES, estimateTokens, chunkText, isObject, isString, getLocaleEnglishName } from '@semiont/core';
-import { withSpan } from '@semiont/observability';
+import { replyChannelsFor, jobId, deriveViews, GENERATABLE_MEDIA_TYPES, estimateTokens, chunkText, isObject, isString, reconcileSelector, getLocaleEnglishName } from '@semiont/core';
+import { withSpan, recordAnchorOutcome, recordDetectionCall } from '@semiont/observability';
+import { StructuredReadError } from '@semiont/inference';
 import { execFileSync } from 'child_process';
 import { tmpdir } from 'os';
 import { withinByteBudget, MAX_PDF_BYTES } from '@semiont/content';
@@ -10,6 +11,13 @@ import '@semiont/event-sourcing';
 import '@semiont/sdk';
 import '@semiont/http-transport';
 
+// src/fs-job-queue.ts
+
+// src/will-retry.ts
+function willRetryAfter(metadata, failureClass) {
+  return failureClass !== "deterministic" && metadata.retryCount < metadata.maxRetries;
+}
+
 // src/fs-job-queue.ts
 var REANNOUNCE_INTERVAL_MS = 3e4;
 var STALE_RUNNING_MS = 30 * 6e4;
@@ -17,9 +25,9 @@ var PROGRESS_WRITE_MIN_INTERVAL_MS = 5e3;
 var RETENTION_HOURS = 24;
 var CLEANUP_INTERVAL_MS = 36e5;
 var FsJobQueue = class {
-  constructor(project, logger, eventBus) {
+  constructor(state, logger, eventBus) {
     this.eventBus = eventBus;
-    this.jobsDir = project.jobsDir;
+    this.jobsDir = state.jobsDir;
     this.logger = logger;
   }
   eventBus;
@@ -194,24 +202,29 @@ var FsJobQueue = class {
    * re-announced); after that it lands in `failed` with the error.
    * Returns null (and changes nothing) if the job isn't running.
    */
-  async failJob(jobId, error) {
+  async failJob(jobId, error, completedUnits, failureClass) {
     const job = await this.getJob(jobId);
     if (!job || job.status !== "running") {
       return null;
     }
     this.lastProgressWrite.delete(jobId);
-    if (job.metadata.retryCount < job.metadata.maxRetries) {
+    const checkpoint = [.../* @__PURE__ */ new Set([...job.metadata.completedUnits ?? [], ...completedUnits ?? []])];
+    const metadata = checkpoint.length > 0 ? { ...job.metadata, completedUnits: checkpoint } : job.metadata;
+    if (willRetryAfter(job.metadata, failureClass)) {
       const retried = {
         status: "pending",
-        metadata: { ...job.metadata, retryCount: job.metadata.retryCount + 1 },
+        metadata: { ...metadata, retryCount: job.metadata.retryCount + 1 },
         params: job.params
       };
       await this.updateJob(retried, "running");
       return "retried";
     }
+    if (failureClass === "deterministic" && job.metadata.retryCount < job.metadata.maxRetries) {
+      this.logger.info("Job failed without retry \u2014 deterministic failure", { jobId, error });
+    }
     const failed = {
       status: "failed",
-      metadata: job.metadata,
+      metadata,
       params: job.params,
       startedAt: job.startedAt,
       completedAt: (/* @__PURE__ */ new Date()).toISOString(),
@@ -220,6 +233,30 @@ var FsJobQueue = class {
     await this.updateJob(failed, "running");
     return "failed";
   }
+  /**
+   * Persist a running job's completed-unit checkpoint at unit completion
+   * (JOB-RESTART-SAFETY P2). `failJob` writes this checkpoint on a clean
+   * failure, but a worker that dies without emitting `job:fail` would lose
+   * its finished units — so the janitor's recovery (which re-queues with
+   * whatever `metadata.completedUnits` the running file holds) would redo
+   * them. Writing it HERE, as each unit lands, makes a crash lose at most
+   * the in-flight unit. Unioned with any existing checkpoint, unthrottled
+   * (a unit completion is rare and must never be dropped), a no-op for
+   * non-running jobs. Written directly, like `recordProgress`, so it also
+   * refreshes the mtime heartbeat.
+   */
+  async checkpointUnits(jobId, completedUnits) {
+    const job = await this.getJob(jobId);
+    if (!job || job.status !== "running") {
+      return;
+    }
+    const merged = [.../* @__PURE__ */ new Set([...job.metadata.completedUnits ?? [], ...completedUnits])];
+    const updated = {
+      ...job,
+      metadata: { ...job.metadata, completedUnits: merged }
+    };
+    await promises.writeFile(this.getJobPath(jobId, "running"), JSON.stringify(updated, null, 2), "utf-8");
+  }
   /**
    * Write progress into a running job's file. Throttled per job, and
    * a no-op for jobs that aren't running. Beyond surfacing live
@@ -427,6 +464,9 @@ function isCancelledJob(job) {
   return job.status === "cancelled";
 }
 var INFERENCE_TIMEOUT_MS = 10 * 6e4;
+var InferenceTimeoutError = class extends Error {
+  name = "InferenceTimeoutError";
+};
 var INFERENCE_HEARTBEAT_MS = 15e3;
 function spanned(client, kind, maxTokens, work) {
   return withSpan(`inference:${kind}`, work, {
@@ -437,12 +477,20 @@ function spanned(client, kind, maxTokens, work) {
     }
   });
 }
-async function withTimeout(work, label, onHeartbeat) {
+async function withTimeout(work, meta, onHeartbeat, logger) {
+  const controller = new AbortController();
   let timer;
   const timedOut = new Promise((_, reject) => {
     timer = setTimeout(() => {
-      reject(new Error(
-        `Inference call timed out after ${INFERENCE_TIMEOUT_MS / 6e4} minutes (${label}) \u2014 failing the job to keep the claim loop live`
+      logger?.warn("Aborting in-flight inference call at the timeout bound", {
+        provider: meta.provider,
+        model: meta.model,
+        label: meta.label,
+        boundMs: INFERENCE_TIMEOUT_MS
+      });
+      controller.abort();
+      reject(new InferenceTimeoutError(
+        `Inference call timed out after ${INFERENCE_TIMEOUT_MS / 6e4} minutes (${meta.label}) \u2014 failing the job to keep the claim loop live`
       ));
     }, INFERENCE_TIMEOUT_MS);
     timer.unref?.();
@@ -457,10 +505,11 @@ async function withTimeout(work, label, onHeartbeat) {
     }, INFERENCE_HEARTBEAT_MS);
     heartbeat.unref?.();
   }
+  const pending = work(controller.signal);
   try {
-    return await Promise.race([work, timedOut]);
+    return await Promise.race([pending, timedOut]);
   } catch (err) {
-    work.catch(() => {
+    pending.catch(() => {
     });
     throw err;
   } finally {
@@ -468,28 +517,55 @@ async function withTimeout(work, label, onHeartbeat) {
     if (heartbeat) clearInterval(heartbeat);
   }
 }
-function boundedGenerateWithMetadata(client, prompt, maxTokens, temperature, onHeartbeat) {
+function boundedGenerateWithMetadata(client, prompt, maxTokens, temperature, onHeartbeat, logger) {
   return spanned(client, "text", maxTokens, () => withTimeout(
-    client.generateTextWithMetadata(prompt, maxTokens, temperature),
-    `${client.type}:${client.modelId}`,
-    onHeartbeat
+    (signal) => client.generateTextWithMetadata(prompt, maxTokens, temperature, signal),
+    { provider: client.type, model: client.modelId, label: `${client.type}:${client.modelId}` },
+    onHeartbeat,
+    logger
   ));
 }
-function boundedGenerateStructured(client, prompt, maxTokens, temperature, elementSchema, onHeartbeat) {
+function boundedGenerateStructured(client, prompt, maxTokens, temperature, elementSchema, onHeartbeat, logger) {
   return spanned(client, "structured", maxTokens, () => withTimeout(
-    client.generateStructured(prompt, maxTokens, temperature, elementSchema),
-    `${client.type}:${client.modelId}`,
-    onHeartbeat
+    (signal) => client.generateStructured(prompt, maxTokens, temperature, elementSchema, signal),
+    { provider: client.type, model: client.modelId, label: `${client.type}:${client.modelId}` },
+    onHeartbeat,
+    logger
   ));
 }
+var DeterministicJobError = class extends Error {
+  // Typed string, not the literal: subclasses (YieldCollapseError) carry
+  // their own name — classification is instanceof, never name-matching.
+  name = "DeterministicJobError";
+};
 
 // src/workers/detection/detection-chunking.ts
+function assertNotTruncated(response, label, chunk, totalChunks, outputBudget) {
+  if (response.stopReason === "max_tokens") {
+    throw new DeterministicJobError(`${label} response truncated (max_tokens) on chunk ${chunk}/${totalChunks} despite the derived output budget of ${outputBudget} tokens \u2014 failing the job rather than under-reporting annotations.`);
+  }
+}
 var SELECTOR_CONTEXT_CHARS = 64;
 var OVERLAP_CHARS = SELECTOR_CONTEXT_CHARS + // prefix
 SELECTOR_CONTEXT_CHARS + // suffix
 2 * SELECTOR_CONTEXT_CHARS;
 var OVERLAP_TOKENS = Math.ceil(OVERLAP_CHARS / 4);
-function deriveDetectionBudget(limits, scaffoldTokens) {
+var DETECTION_TEMPERATURE = 0;
+var ASSUMED_OUTPUT_TOKENS_PER_HOUR = 108e3;
+var YIELD_COLLAPSE_BAND = 2;
+var YieldCollapseError = class extends DeterministicJobError {
+  /** What the flagged extraction DID find — every span write-time-verified,
+   * so discarding it at the floor would add loss on top of the under-report.
+   * Carried on the error because the flag site cannot know whether descent
+   * remains possible; the floor is the subdivider's knowledge. */
+  constructor(message, salvage = []) {
+    super(message);
+    this.salvage = salvage;
+  }
+  salvage;
+  name = "YieldCollapseError";
+};
+function deriveDetectionBudget(limits, scaffoldTokens, typesPerCall) {
   const { contextTokens, maxOutputTokens } = limits;
   const available = contextTokens - scaffoldTokens;
   let inputBudget;
@@ -505,6 +581,15 @@ function deriveDetectionBudget(limits, scaffoldTokens) {
       outputBudget = available - inputBudget;
     }
   }
+  const outputTokensPerHour = limits.outputTokensPerHour ?? ASSUMED_OUTPUT_TOKENS_PER_HOUR;
+  {
+    const durationSafeOutput = Math.floor(outputTokensPerHour * (INFERENCE_TIMEOUT_MS / 2) / 36e5);
+    if (outputBudget > durationSafeOutput) {
+      inputBudget = Math.floor(inputBudget * (durationSafeOutput / outputBudget));
+      outputBudget = durationSafeOutput;
+    }
+  }
+  inputBudget = Math.min(inputBudget, Math.floor(outputBudget / (2 * typesPerCall)));
   if (inputBudget <= OVERLAP_TOKENS) {
     throw new Error(
       `Inference window too small for detection: context ${contextTokens} tokens minus scaffold ${scaffoldTokens} leaves an input budget of ${inputBudget} (need > ${OVERLAP_TOKENS}). Use a model with a larger context window or reduce the prompt scaffold.`
@@ -515,6 +600,91 @@ function deriveDetectionBudget(limits, scaffoldTokens) {
     outputBudget
   };
 }
+var MAX_SUBDIVISION_DEPTH = 2;
+function unknownUnreadable(error) {
+  return error instanceof StructuredReadError && error.stopReason === "unknown";
+}
+function subdividable(error) {
+  return error instanceof InferenceTimeoutError || truncation(error) || unknownUnreadable(error);
+}
+function truncation(error) {
+  return error instanceof DeterministicJobError || error instanceof StructuredReadError && error.stopReason === "max_tokens";
+}
+function outcomeOf(error) {
+  if (error instanceof YieldCollapseError) return "collapsed";
+  if (truncation(error)) return "truncated";
+  if (error instanceof InferenceTimeoutError) return "timeout";
+  return "error";
+}
+async function callChunkSubdividing(label, chunk, chunking, call, logger) {
+  async function recorded(piece, depth, reroll) {
+    const start = performance.now();
+    try {
+      const result = await call(piece);
+      recordDetectionCall({
+        label,
+        pieceChars: piece.length,
+        durationMs: performance.now() - start,
+        items: result.items.length,
+        depth,
+        reroll,
+        outcome: "success",
+        ...result.usage ? { inputTokens: result.usage.inputTokens, outputTokens: result.usage.outputTokens } : {}
+      });
+      return result;
+    } catch (error) {
+      recordDetectionCall({
+        label,
+        pieceChars: piece.length,
+        durationMs: performance.now() - start,
+        items: 0,
+        depth,
+        reroll,
+        outcome: outcomeOf(error)
+      });
+      throw error;
+    }
+  }
+  async function attempt(piece, chunkSize, depth) {
+    try {
+      return (await recorded(piece, depth, false)).items;
+    } catch (error) {
+      if (!subdividable(error)) throw error;
+      const half = Math.floor(chunkSize / 2);
+      const pieces = chunkText(piece, { chunkSize: half, overlap: chunking.overlap });
+      const shrinks = pieces.length > 1 || pieces[0] !== piece;
+      const canDescend = shrinks && (truncation(error) || unknownUnreadable(error) ? half > 2 * OVERLAP_TOKENS : depth < MAX_SUBDIVISION_DEPTH);
+      if (!canDescend) {
+        if (error instanceof YieldCollapseError) {
+          logger?.warn("Floor-size piece still flagged as collapsed \u2014 accepting its under-reported salvage and continuing", {
+            pieceChars: piece.length,
+            salvaged: error.salvage.length,
+            error: error.message
+          });
+          return error.salvage;
+        }
+        if (!truncation(error)) throw error;
+        logger?.warn("Floor-size piece truncated \u2014 re-rolling once before giving up", {
+          pieceChars: piece.length,
+          error: error instanceof Error ? error.message : String(error)
+        });
+        return (await recorded(piece, depth, true)).items;
+      }
+      logger?.warn("Chunk call failed at a size-shaped bound \u2014 subdividing and retrying smaller", {
+        depth: depth + 1,
+        pieceChars: piece.length,
+        nextChunkSizeTokens: half,
+        error: error instanceof Error ? error.message : String(error)
+      });
+      const collected = [];
+      for (const p of pieces) {
+        collected.push(...await attempt(p, half, depth + 1));
+      }
+      return collected;
+    }
+  }
+  return attempt(chunk, chunking.chunkSize, 0);
+}
 function languageName(tag) {
   return getLocaleEnglishName(tag) || tag;
 }
@@ -812,6 +982,16 @@ Example format:
     return prompt;
   }
 };
+var DEGRADED = /* @__PURE__ */ new Set(["first-of-many", "fuzzy-match"]);
+function noteAnchor(label, exact, method, logger) {
+  recordAnchorOutcome(label, method);
+  if (!DEGRADED.has(method)) return;
+  const detail = { text: exact, anchorMethod: method };
+  if (logger) logger.warn("Annotation anchored via degraded method", { label, ...detail });
+  else console.warn(`[${label}] anchored via ${method}: "${exact}"`);
+}
+
+// src/workers/detection/motivation-parsers.ts
 var COMMENT_ELEMENT_SCHEMA = {
   type: "object",
   properties: {
@@ -1000,35 +1180,31 @@ var MotivationParsers = class {
   }
 };
 function logAnchorMethod(motivation, exact, anchorMethod) {
-  if (anchorMethod === "first-of-many" || anchorMethod === "fuzzy-match") {
-    console.warn(`[MotivationParsers] ${motivation} anchored via ${anchorMethod}: "${exact}"`);
-  }
+  noteAnchor(motivation, exact, anchorMethod);
 }
 
 // src/workers/annotation-detection.ts
-function assertNotTruncated(response, motivation, chunk, totalChunks, outputBudget) {
-  if (response.stopReason === "max_tokens") {
-    throw new Error(`${motivation} detection response truncated (max_tokens) on chunk ${chunk}/${totalChunks} despite the derived output budget of ${outputBudget} tokens \u2014 failing the job rather than under-reporting annotations.`);
-  }
-}
-async function detectInChunks(client, content, buildPrompt, temperature, motivation, elementSchema, parse, onActivity) {
+async function detectInChunks(client, content, buildPrompt, motivation, elementSchema, parse, onActivity) {
   const limits = await client.limits();
   const scaffoldTokens = estimateTokens(buildPrompt(""));
-  const { chunking, outputBudget } = deriveDetectionBudget(limits, scaffoldTokens);
+  const { chunking, outputBudget } = deriveDetectionBudget(limits, scaffoldTokens, 1);
   const chunks = chunkText(content, chunking);
   const collected = [];
   for (let i = 0; i < chunks.length; i++) {
-    const response = await boundedGenerateStructured(
-      client,
-      buildPrompt(chunks[i]),
-      outputBudget,
-      temperature,
-      elementSchema,
-      // Still alive, same position (a long single call is otherwise silent).
-      () => onActivity?.(i, chunks.length)
-    );
-    assertNotTruncated(response, motivation, i + 1, chunks.length, outputBudget);
-    collected.push(...parse(response.items));
+    const items = await callChunkSubdividing(motivation, chunks[i], chunking, async (piece) => {
+      const response = await boundedGenerateStructured(
+        client,
+        buildPrompt(piece),
+        outputBudget,
+        DETECTION_TEMPERATURE,
+        elementSchema,
+        // Still alive, same position (a long single call is otherwise silent).
+        () => onActivity?.(i, chunks.length)
+      );
+      assertNotTruncated(response, `${motivation} detection`, i + 1, chunks.length, outputBudget);
+      return { items: response.items, ...response.usage ? { usage: response.usage } : {} };
+    });
+    collected.push(...parse(items));
     if (i < chunks.length - 1) {
       onActivity?.(i + 1, chunks.length);
     }
@@ -1049,7 +1225,6 @@ var AnnotationDetection = class {
       client,
       content,
       (chunk) => MotivationPrompts.buildCommentPrompt(chunk, instructions, tone, density, language, sourceLanguage),
-      0.4,
       "comment",
       COMMENT_ELEMENT_SCHEMA,
       (items) => MotivationParsers.parseComments(items, content),
@@ -1068,7 +1243,6 @@ var AnnotationDetection = class {
       client,
       content,
       (chunk) => MotivationPrompts.buildHighlightPrompt(chunk, instructions, density, sourceLanguage),
-      0.3,
       "highlight",
       HIGHLIGHT_ELEMENT_SCHEMA,
       (items) => MotivationParsers.parseHighlights(items, content),
@@ -1087,7 +1261,6 @@ var AnnotationDetection = class {
       client,
       content,
       (chunk) => MotivationPrompts.buildAssessmentPrompt(chunk, instructions, tone, density, language, sourceLanguage),
-      0.3,
       "assessment",
       ASSESSMENT_ELEMENT_SCHEMA,
       (items) => MotivationParsers.parseAssessments(items, content),
@@ -1124,7 +1297,6 @@ var AnnotationDetection = class {
         categoryInfo.examples,
         sourceLanguage
       ),
-      0.2,
       "tag",
       TAG_ELEMENT_SCHEMA,
       (items) => MotivationParsers.parseTags(items),
@@ -1144,6 +1316,40 @@ var ENTITY_ELEMENT_SCHEMA = {
   required: ["exact", "entityType"],
   additionalProperties: false
 };
+var COUNT_MAX_TOKENS = 16;
+function parseCount(text) {
+  const m = text.trim().match(/\d+/);
+  return m ? Number(m[0]) : void 0;
+}
+async function assertYieldNotCollapsed(client, piece, items, entityTypesDescription, logger) {
+  const prompt = `Count every mention of: ${entityTypesDescription} in the following text. Repeated mentions of the same entity count separately. Respond with only the number.
+
+Text:
+"""
+${piece}
+"""`;
+  let counted;
+  try {
+    const response = await boundedGenerateWithMetadata(client, prompt, COUNT_MAX_TOKENS, DETECTION_TEMPERATURE, void 0, logger);
+    counted = parseCount(response.text);
+  } catch (err) {
+    logger.warn("Count-verifier call failed \u2014 yield check skipped for this chunk", {
+      pieceChars: piece.length,
+      error: err instanceof Error ? err.message : String(err)
+    });
+    return;
+  }
+  if (counted === void 0) {
+    logger.warn("Count-verifier answer carried no number \u2014 yield check skipped for this chunk", { pieceChars: piece.length });
+    return;
+  }
+  if (items.length * YIELD_COLLAPSE_BAND < counted) {
+    throw new YieldCollapseError(
+      `Extraction found ${items.length} entities where a count call reports ~${counted} mentions (band \xD7${YIELD_COLLAPSE_BAND}) on a ${piece.length}-char chunk \u2014 silent yield collapse (F7): deterministic \u2014 a same-size retry returns the identical under-report.`,
+      [...items]
+    );
+  }
+}
 async function extractEntities(exact, entityTypes, client, includeDescriptiveReferences, logger, sourceLanguage, onActivity) {
   const entityTypesDescription = entityTypes.map((et) => {
     if (typeof et === "string") {
@@ -1194,8 +1400,9 @@ If no entities are found, respond with an empty array [].
 Example output:
 [{"exact":"Alice","entityType":"Person","prefix":"","suffix":" went to"},{"exact":"Paris","entityType":"Location","prefix":"went to ","suffix":" yesterday"}]`;
   const limits = await client.limits();
+  const verifyYield = client.verifyDetectionYield;
   const scaffoldTokens = estimateTokens(buildPrompt(""));
-  const { chunking, outputBudget } = deriveDetectionBudget(limits, scaffoldTokens);
+  const { chunking, outputBudget } = deriveDetectionBudget(limits, scaffoldTokens, entityTypes.length);
   const chunks = chunkText(exact, chunking);
   logger.debug("Sending entity extraction request", {
     entityTypes: entityTypesDescription,
@@ -1205,28 +1412,31 @@ Example output:
   });
   const collected = [];
   for (let i = 0; i < chunks.length; i++) {
-    const response = await boundedGenerateStructured(
-      client,
-      buildPrompt(chunks[i]),
-      outputBudget,
-      0.3,
-      // Lower temperature for more consistent extraction
-      ENTITY_ELEMENT_SCHEMA,
-      // Still alive, same position: a long single call would otherwise emit
-      // nothing at all between start and finish.
-      () => onActivity?.(i, chunks.length)
-    );
-    logger.debug("Got entity extraction response", {
-      chunk: i + 1,
-      chunks: chunks.length,
-      items: response.items.length
-    });
-    if (response.stopReason === "max_tokens") {
-      const errorMsg = `Entity extraction response truncated (max_tokens) on chunk ${i + 1}/${chunks.length} despite the derived output budget of ${outputBudget} tokens \u2014 failing the job rather than dropping annotations.`;
-      logger.error(errorMsg, { items: response.items.length });
-      throw new Error(errorMsg);
-    }
-    for (const e of response.items) {
+    const items = await callChunkSubdividing("reference", chunks[i], chunking, async (piece) => {
+      const response = await boundedGenerateStructured(
+        client,
+        buildPrompt(piece),
+        outputBudget,
+        DETECTION_TEMPERATURE,
+        ENTITY_ELEMENT_SCHEMA,
+        // Still alive, same position: a long single call would otherwise emit
+        // nothing at all between start and finish.
+        () => onActivity?.(i, chunks.length),
+        logger
+      );
+      logger.debug("Got entity extraction response", {
+        chunk: i + 1,
+        chunks: chunks.length,
+        pieceChars: piece.length,
+        items: response.items.length
+      });
+      assertNotTruncated(response, "Entity extraction", i + 1, chunks.length, outputBudget);
+      if (verifyYield) {
+        await assertYieldNotCollapsed(client, piece, response.items, entityTypesDescription, logger);
+      }
+      return { items: response.items, ...response.usage ? { usage: response.usage } : {} };
+    }, logger);
+    for (const e of items) {
       if (isObject(e) && isString(e.exact) && isString(e.entityType)) {
         collected.push({
           exact: e.exact,
@@ -1457,7 +1667,7 @@ ${formatRequirements}`;
     temperature: finalTemperature,
     maxTokens: finalMaxTokens
   });
-  const response = await boundedGenerateWithMetadata(client, prompt, finalMaxTokens, finalTemperature);
+  const response = await boundedGenerateWithMetadata(client, prompt, finalMaxTokens, finalTemperature, void 0, logger);
   logger.debug("Got response from inference", { responseLength: response.text.length, stopReason: response.stopReason });
   const result = parseResponse(response.text);
   logger.debug("Parsed response", {
@@ -1552,6 +1762,24 @@ function resolveCitationTokens(content, validResourceIds, logger) {
   clean += content.slice(last);
   return { content: clean, citations };
 }
+
+// src/workers/detection/bounded-concurrency.ts
+async function runBounded(items, limit, worker) {
+  const results = new Array(items.length);
+  let next = 0;
+  async function pump() {
+    while (true) {
+      const i = next++;
+      if (i >= items.length) return;
+      results[i] = await worker(items[i], i);
+    }
+  }
+  const poolSize = Math.max(1, Math.min(limit, items.length));
+  await Promise.all(Array.from({ length: poolSize }, () => pump()));
+  return results;
+}
+
+// src/processors.ts
 function toMatch(r) {
   return {
     exact: r.exact,
@@ -1689,31 +1917,33 @@ async function processAssessmentJob(content, inferenceClient, params, buildAnnot
     result: { kind: "assessment-annotation", assessmentsFound: assessments.length, assessmentsCreated: annotations.length }
   };
 }
-async function processReferenceJob(content, inferenceClient, params, buildAnnotation, onProgress, logger) {
+async function processReferenceJob(content, inferenceClient, params, buildAnnotation, onProgress, logger, onUnitComplete, signal) {
   const entityTypeNames = params.entityTypes.map(String);
   const requestParams = [{ label: "entity-types", value: entityTypeNames.join(", ") }];
   const completedItems = [];
   let totalFound = 0;
   let totalEmitted = 0;
   let errors = 0;
-  const allAnnotations = [];
   onProgress(10, { code: "loading" }, { requestParams });
   const bodyLanguage = params.language ?? "en";
-  for (let i = 0; i < entityTypeNames.length; i++) {
-    const entityTypeName = entityTypeNames[i];
-    if (!entityTypeName) continue;
-    const pct = 20 + Math.round(i / entityTypeNames.length * 60);
+  let completed = 0;
+  const total = entityTypeNames.length;
+  const emitTypeProgress = (entityTypeName) => {
+    const pct = 20 + Math.round(completed / total * 60);
     onProgress(pct, { code: "detecting-entities", entityType: entityTypeName }, {
-      // One vocabulary for "what is in flight" (CLEAN-PROGRESS D2): the entity
-      // type is KB data, `kind` is the code the client localizes around it.
       current: { kind: "entity-type", value: entityTypeName },
-      processed: i,
-      total: entityTypeNames.length,
+      processed: completed,
+      total,
       entitiesFound: totalFound,
       entitiesEmitted: totalEmitted,
       completedItems: [...completedItems],
       requestParams
     });
+  };
+  await runBounded(entityTypeNames, inferenceClient.maxConcurrency, async (entityTypeName) => {
+    if (!entityTypeName) return;
+    if (signal?.aborted) return;
+    emitTypeProgress(entityTypeName);
     const extractedEntities = await extractEntities(
       content,
       [entityTypeName],
@@ -1721,31 +1951,17 @@ async function processReferenceJob(content, inferenceClient, params, buildAnnota
       params.includeDescriptiveReferences ?? false,
       logger,
       params.sourceLanguage,
-      // Liveness: fires at chunk boundaries AND every ~15 s while a single
-      // inference call is in flight (DETECTION-HEARTBEAT). Progress feeds the
-      // stall watchdog, the janitor, AND the client's inter-emission timeout,
-      // so a long single-chunk call must not be silent. Percentage
-      // interpolates within this entity type's band of the 20–80 range; a
-      // heartbeat repeats the current position rather than inventing an
-      // advance.
-      (completed, total) => {
-        const interpolated = 20 + Math.round((i + completed / total) / entityTypeNames.length * 60);
-        onProgress(interpolated, { code: "detecting-entities", entityType: entityTypeName }, {
-          current: { kind: "entity-type", value: entityTypeName },
-          processed: i,
-          total: entityTypeNames.length,
-          entitiesFound: totalFound,
-          entitiesEmitted: totalEmitted,
-          completedItems: [...completedItems],
-          requestParams
-        });
-      }
+      // Liveness heartbeat (DETECTION-HEARTBEAT): fires at chunk boundaries and
+      // every ~15 s while a call is in flight, so a long single-chunk call is
+      // not silent. It repeats the current position rather than inventing an
+      // advance — the stall watchdog, janitor and client timeout need a signal,
+      // not a monotone.
+      () => emitTypeProgress(entityTypeName)
     );
-    totalFound += extractedEntities.length;
-    completedItems.push({ value: entityTypeName, foundCount: extractedEntities.length });
     const unresolvedBody = [
       { type: "TextualBody", value: entityTypeName, purpose: "tagging", format: "text/plain", language: bodyLanguage }
     ];
+    const built = [];
     for (const entity of extractedEntities) {
       const reconciled = reconcileSelector(content, {
         exact: entity.exact,
@@ -1760,23 +1976,28 @@ async function processReferenceJob(content, inferenceClient, params, buildAnnota
         errors++;
         continue;
       }
-      if (reconciled.anchorMethod === "first-of-many" || reconciled.anchorMethod === "fuzzy-match") {
-        logger.warn("Entity anchored via degraded method", {
-          text: entity.exact,
-          entityType: entity.entityType,
-          anchorMethod: reconciled.anchorMethod
-        });
-      }
+      noteAnchor("reference", entity.exact, reconciled.anchorMethod, logger);
       const ann = buildAnnotation("linking", toMatch(reconciled), unresolvedBody);
-      allAnnotations.push(ann);
-      totalEmitted++;
+      built.push(ann);
     }
-  }
-  const annotations = dedupeAnnotations(allAnnotations);
-  onProgress(100, { code: "complete-created", count: annotations.length, kind: "reference" }, { requestParams });
+    const unitAnnotations = dedupeAnnotations(built);
+    await onUnitComplete(entityTypeName, unitAnnotations);
+    totalEmitted += unitAnnotations.length;
+    totalFound += extractedEntities.length;
+    completedItems.push({
+      value: entityTypeName,
+      foundCount: extractedEntities.length,
+      persistedCount: unitAnnotations.length
+    });
+    completed++;
+    emitTypeProgress(entityTypeName);
+  });
+  onProgress(100, { code: "complete-created", count: totalEmitted, kind: "reference" }, {
+    completedItems: [...completedItems],
+    requestParams
+  });
   return {
-    annotations,
-    result: { kind: "reference-annotation", totalFound, totalEmitted: annotations.length, errors }
+    result: { kind: "reference-annotation", totalFound, totalEmitted, errors }
   };
 }
 async function processTagJob(content, inferenceClient, params, buildAnnotation, onProgress) {
@@ -1980,6 +2201,17 @@ async function processGenerationJob(inferenceClient, params, onProgress, logger)
   };
 }
 var STALL_THRESHOLD_MS = 15 * 6e4;
+var WORKER_AWAITED_OPERATIONS = [
+  "job:claim",
+  "browse:resource-requested",
+  "browse:anchored-text-by-checksum-requested",
+  // Durability acknowledgement for a unit's annotations (JOB-RESTART-SAFETY
+  // P6). The worker AWAITS this one — a unit may not advance until its
+  // annotations are in the event log — so its replies must be in the narrow
+  // channel set or every commit fails fast with `bus.unsubscribed`.
+  "mark:commit"
+];
+replyChannelsFor(WORKER_AWAITED_OPERATIONS);
 
 export { AnnotationDetection, FsJobQueue, STALL_THRESHOLD_MS, generateResourceFromTopic, isCancelledJob, isCompleteJob, isFailedJob, isPendingJob, isRunningJob, processAssessmentJob, processCommentJob, processGenerationJob, processHighlightJob, processReferenceJob, processTagJob };
 //# sourceMappingURL=index.js.map