@semiont/jobs 0.5.25 → 0.5.27

package/dist/worker-main.js

@@ -1,11 +1,11 @@
-import { createTomlConfigLoader, didToAgent, baseUrl, STARTUP_FETCH_RETRY, retryWithBackoff, isTransientFetchError, busRequest, resourceId, getPrimaryMediaType, assembleAnnotation, findClaimSpan, textExtractionOf, reconcileSelector, GENERATABLE_MEDIA_TYPES, locate, createFragmentSelector, getLocaleEnglishName, estimateTokens, chunkText, isArray, isObject, isString, deriveViews } from '@semiont/core';
+import { createTomlConfigLoader, didToAgent, baseUrl, STARTUP_FETCH_RETRY, retryWithBackoff, isTransientFetchError, busRequest, resourceId, getPrimaryMediaType, isGenerationJobParams, assembleAnnotation, findClaimSpan, textExtractionOf, reconcileSelector, GENERATABLE_MEDIA_TYPES, locate, createFragmentSelector, getLocaleEnglishName, estimateTokens, chunkText, isObject, isString, deriveViews } from '@semiont/core';
 import { anchoredTextStoreOverTransport, deriveStorageUri, extractPdfTextLayer, EXTRACTORS, calculateChecksum, withinByteBudget, MAX_PDF_BYTES } from '@semiont/content';
+import { withSpan, SpanKind, recordJobOutcome } from '@semiont/observability';
 import { execFileSync } from 'child_process';
 import { existsSync, readFileSync, mkdtempSync, writeFileSync, rmSync } from 'fs';
 import { homedir, hostname, tmpdir } from 'os';
 import { join } from 'path';
 import { generateAnnotationId } from '@semiont/event-sourcing';
-import { withSpan, SpanKind, recordJobOutcome } from '@semiont/observability';
 import { InMemorySessionStorage, setStoredSession, kbBackendUrl, SemiontClient, SemiontSession } from '@semiont/sdk';
 import { HttpTransport, HttpContentTransport } from '@semiont/http-transport';
 import { createInferenceClient } from '@semiont/inference';
@@ -3766,9 +3766,9 @@ var require_mapOneOrManyArgs = __commonJS({
     Object.defineProperty(exports, "__esModule", { value: true });
     exports.mapOneOrManyArgs = void 0;
     var map_1 = require_map();
-    var isArray2 = Array.isArray;
+    var isArray = Array.isArray;
     function callOrApply(fn, args) {
-      return isArray2(args) ? fn.apply(void 0, __spreadArray([], __read(args))) : fn(args);
+      return isArray(args) ? fn.apply(void 0, __spreadArray([], __read(args))) : fn(args);
     }
     function mapOneOrManyArgs(fn) {
       return map_1.map(function(args) {
@@ -3913,14 +3913,14 @@ var require_argsArgArrayOrObject = __commonJS({
   "../../node_modules/rxjs/dist/cjs/internal/util/argsArgArrayOrObject.js"(exports) {
     Object.defineProperty(exports, "__esModule", { value: true });
     exports.argsArgArrayOrObject = void 0;
-    var isArray2 = Array.isArray;
+    var isArray = Array.isArray;
     var getPrototypeOf = Object.getPrototypeOf;
     var objectProto = Object.prototype;
     var getKeys = Object.keys;
     function argsArgArrayOrObject(args) {
       if (args.length === 1) {
         var first_1 = args[0];
-        if (isArray2(first_1)) {
+        if (isArray(first_1)) {
           return { args: first_1, keys: null };
         }
         if (isPOJO(first_1)) {
@@ -4668,9 +4668,9 @@ var require_argsOrArgArray = __commonJS({
   "../../node_modules/rxjs/dist/cjs/internal/util/argsOrArgArray.js"(exports) {
     Object.defineProperty(exports, "__esModule", { value: true });
     exports.argsOrArgArray = void 0;
-    var isArray2 = Array.isArray;
+    var isArray = Array.isArray;
     function argsOrArgArray(args) {
-      return args.length === 1 && isArray2(args[0]) ? args[0] : args;
+      return args.length === 1 && isArray(args[0]) ? args[0] : args;
     }
     exports.argsOrArgArray = argsOrArgArray;
   }
@@ -9348,10 +9348,18 @@ function asJobParams(params) {
   }
   return params;
 }
-
-// src/workers/inference-call.ts
 var INFERENCE_TIMEOUT_MS = 10 * 6e4;
-async function withTimeout(work, label) {
+var INFERENCE_HEARTBEAT_MS = 15e3;
+function spanned(client, kind, maxTokens, work) {
+  return withSpan(`inference:${kind}`, work, {
+    attrs: {
+      "inference.provider": client.type,
+      "inference.model": client.modelId,
+      "inference.max_tokens": maxTokens
+    }
+  });
+}
+async function withTimeout(work, label, onHeartbeat) {
   let timer;
   const timedOut = new Promise((_, reject) => {
     timer = setTimeout(() => {
@@ -9361,6 +9369,16 @@ async function withTimeout(work, label) {
     }, INFERENCE_TIMEOUT_MS);
     timer.unref?.();
   });
+  let heartbeat;
+  if (onHeartbeat) {
+    heartbeat = setInterval(() => {
+      try {
+        onHeartbeat();
+      } catch {
+      }
+    }, INFERENCE_HEARTBEAT_MS);
+    heartbeat.unref?.();
+  }
   try {
     return await Promise.race([work, timedOut]);
   } catch (err) {
@@ -9369,19 +9387,22 @@ async function withTimeout(work, label) {
     throw err;
   } finally {
     clearTimeout(timer);
+    if (heartbeat) clearInterval(heartbeat);
   }
 }
-function boundedGenerate(client, prompt, maxTokens, temperature, options) {
-  return withTimeout(
-    client.generateText(prompt, maxTokens, temperature, options),
-    `${client.type}:${client.modelId}`
-  );
+function boundedGenerate(client, prompt, maxTokens, temperature, onHeartbeat) {
+  return spanned(client, "text", maxTokens, () => withTimeout(
+    client.generateText(prompt, maxTokens, temperature),
+    `${client.type}:${client.modelId}`,
+    onHeartbeat
+  ));
 }
-function boundedGenerateWithMetadata(client, prompt, maxTokens, temperature, options) {
-  return withTimeout(
-    client.generateTextWithMetadata(prompt, maxTokens, temperature, options),
-    `${client.type}:${client.modelId}`
-  );
+function boundedGenerateStructured(client, prompt, maxTokens, temperature, elementSchema, onHeartbeat) {
+  return spanned(client, "structured", maxTokens, () => withTimeout(
+    client.generateStructured(prompt, maxTokens, temperature, elementSchema),
+    `${client.type}:${client.modelId}`,
+    onHeartbeat
+  ));
 }
 
 // src/workers/detection/detection-chunking.ts
@@ -9713,32 +9734,57 @@ Example format:
     return prompt;
   }
 };
-function parseJsonArray(response, motivation) {
-  let parsed;
-  try {
-    parsed = JSON.parse(response.trim());
-  } catch (error) {
-    console.error(`[MotivationParsers] Failed to parse AI ${motivation} response:`, error);
-    console.error("Raw response:", response);
-    throw error instanceof Error ? error : new Error(String(error));
-  }
-  if (!Array.isArray(parsed)) {
-    console.error(`[MotivationParsers] Expected a JSON array for ${motivation} detection, got ${typeof parsed}:`, response);
-    throw new Error(`Expected a JSON array for ${motivation} detection, got ${typeof parsed}`);
-  }
-  return parsed;
-}
+var COMMENT_ELEMENT_SCHEMA = {
+  type: "object",
+  properties: {
+    exact: { type: "string" },
+    prefix: { type: "string" },
+    suffix: { type: "string" },
+    comment: { type: "string" }
+  },
+  required: ["exact", "comment"],
+  additionalProperties: false
+};
+var HIGHLIGHT_ELEMENT_SCHEMA = {
+  type: "object",
+  properties: {
+    exact: { type: "string" },
+    prefix: { type: "string" },
+    suffix: { type: "string" }
+  },
+  required: ["exact"],
+  additionalProperties: false
+};
+var ASSESSMENT_ELEMENT_SCHEMA = {
+  type: "object",
+  properties: {
+    exact: { type: "string" },
+    prefix: { type: "string" },
+    suffix: { type: "string" },
+    assessment: { type: "string" }
+  },
+  required: ["exact", "assessment"],
+  additionalProperties: false
+};
+var TAG_ELEMENT_SCHEMA = {
+  type: "object",
+  properties: {
+    exact: { type: "string" },
+    prefix: { type: "string" },
+    suffix: { type: "string" }
+  },
+  required: ["exact"],
+  additionalProperties: false
+};
 var MotivationParsers = class {
   /**
-   * Parse and validate AI response for comment detection
+   * Validate and reconcile structured comment elements.
    *
-   * @param response - Raw AI response text (a JSON array)
+   * @param parsed - Already-parsed elements from the structured surface
    * @param content - Original content to validate offsets against
    * @returns Array of validated comment matches
-   * @throws if the response is not a parseable JSON array
    */
-  static parseComments(response, content) {
-    const parsed = parseJsonArray(response, "comment");
+  static parseComments(parsed, content) {
     const valid = parsed.filter(
       (c) => isObject(c) && isString(c.exact) && isString(c.comment) && c.comment.trim().length > 0
     );
@@ -9767,15 +9813,13 @@ var MotivationParsers = class {
     return validatedComments;
   }
   /**
-   * Parse and validate AI response for highlight detection
+   * Validate and reconcile structured highlight elements.
    *
-   * @param response - Raw AI response text (a JSON array)
+   * @param parsed - Already-parsed elements from the structured surface
    * @param content - Original content to validate offsets against
    * @returns Array of validated highlight matches
-   * @throws if the response is not a parseable JSON array
    */
-  static parseHighlights(response, content) {
-    const parsed = parseJsonArray(response, "highlight");
+  static parseHighlights(parsed, content) {
     const highlights = parsed.filter(
       (h) => isObject(h) && isString(h.exact)
     );
@@ -9802,15 +9846,13 @@ var MotivationParsers = class {
     return validatedHighlights;
   }
   /**
-   * Parse and validate AI response for assessment detection
+   * Validate and reconcile structured assessment elements.
    *
-   * @param response - Raw AI response text (a JSON array)
+   * @param parsed - Already-parsed elements from the structured surface
    * @param content - Original content to validate offsets against
    * @returns Array of validated assessment matches
-   * @throws if the response is not a parseable JSON array
    */
-  static parseAssessments(response, content) {
-    const parsed = parseJsonArray(response, "assessment");
+  static parseAssessments(parsed, content) {
     const assessments = parsed.filter(
       (a) => isObject(a) && isString(a.exact) && isString(a.assessment)
     );
@@ -9838,14 +9880,13 @@ var MotivationParsers = class {
     return validatedAssessments;
   }
   /**
-   * Parse the LLM's tag response into raw, pre-reconciliation tag inputs.
+   * Validate structured tag elements into raw, pre-reconciliation tag inputs.
    * Reconciliation happens in `validateTagOffsets`, which adds `start`/`end`
    * by anchoring `exact` against the source content.
    *
-   * @throws if the response is not a parseable JSON array
+   * @param parsed - Already-parsed elements from the structured surface
    */
-  static parseTags(response) {
-    const parsed = parseJsonArray(response, "tag");
+  static parseTags(parsed) {
     const valid = parsed.filter(
       (t) => isObject(t) && isString(t.exact) && t.exact.trim().length > 0
     );
@@ -9892,24 +9933,26 @@ function assertNotTruncated(response, motivation, chunk, totalChunks, outputBudg
     throw new Error(`${motivation} detection response truncated (max_tokens) on chunk ${chunk}/${totalChunks} despite the derived output budget of ${outputBudget} tokens \u2014 failing the job rather than under-reporting annotations.`);
   }
 }
-async function detectInChunks(client, content, buildPrompt, temperature, motivation, parse, onChunk) {
+async function detectInChunks(client, content, buildPrompt, temperature, motivation, elementSchema, parse, onActivity) {
   const limits = await client.limits();
   const scaffoldTokens = estimateTokens(buildPrompt(""));
   const { chunking, outputBudget } = deriveDetectionBudget(limits, scaffoldTokens);
   const chunks = chunkText(content, chunking);
   const collected = [];
   for (let i = 0; i < chunks.length; i++) {
-    const response = await boundedGenerateWithMetadata(
+    const response = await boundedGenerateStructured(
       client,
       buildPrompt(chunks[i]),
       outputBudget,
       temperature,
-      { format: "json" }
+      elementSchema,
+      // Still alive, same position (a long single call is otherwise silent).
+      () => onActivity?.(i, chunks.length)
     );
     assertNotTruncated(response, motivation, i + 1, chunks.length, outputBudget);
-    collected.push(...parse(response.text));
+    collected.push(...parse(response.items));
     if (i < chunks.length - 1) {
-      onChunk?.(i + 1, chunks.length);
+      onActivity?.(i + 1, chunks.length);
     }
   }
   return collected;
@@ -9923,15 +9966,16 @@ var AnnotationDetection = class {
    * (source-resource locale). See `types.ts` "Locale conventions" for the
    * full discussion.
    */
-  static async detectComments(content, client, instructions, tone, density, language, sourceLanguage, onChunk) {
+  static async detectComments(content, client, instructions, tone, density, language, sourceLanguage, onActivity) {
     return detectInChunks(
       client,
       content,
       (chunk) => MotivationPrompts.buildCommentPrompt(chunk, instructions, tone, density, language, sourceLanguage),
       0.4,
       "comment",
-      (text) => MotivationParsers.parseComments(text, content),
-      onChunk
+      COMMENT_ELEMENT_SCHEMA,
+      (items) => MotivationParsers.parseComments(items, content),
+      onActivity
     );
   }
   /**
@@ -9941,15 +9985,16 @@ var AnnotationDetection = class {
    * applies, used in the prompt so the LLM analyzes non-English source
    * correctly.
    */
-  static async detectHighlights(content, client, instructions, density, sourceLanguage, onChunk) {
+  static async detectHighlights(content, client, instructions, density, sourceLanguage, onActivity) {
     return detectInChunks(
       client,
       content,
       (chunk) => MotivationPrompts.buildHighlightPrompt(chunk, instructions, density, sourceLanguage),
       0.3,
       "highlight",
-      (text) => MotivationParsers.parseHighlights(text, content),
-      onChunk
+      HIGHLIGHT_ELEMENT_SCHEMA,
+      (items) => MotivationParsers.parseHighlights(items, content),
+      onActivity
     );
   }
   /**
@@ -9959,15 +10004,16 @@ var AnnotationDetection = class {
    * (annotation body locale). `sourceLanguage` is the locale of the content
    * being analyzed (source-resource locale).
    */
-  static async detectAssessments(content, client, instructions, tone, density, language, sourceLanguage, onChunk) {
+  static async detectAssessments(content, client, instructions, tone, density, language, sourceLanguage, onActivity) {
     return detectInChunks(
       client,
       content,
       (chunk) => MotivationPrompts.buildAssessmentPrompt(chunk, instructions, tone, density, language, sourceLanguage),
       0.3,
       "assessment",
-      (text) => MotivationParsers.parseAssessments(text, content),
-      onChunk
+      ASSESSMENT_ELEMENT_SCHEMA,
+      (items) => MotivationParsers.parseAssessments(items, content),
+      onActivity
     );
   }
   /**
@@ -9982,7 +10028,7 @@ var AnnotationDetection = class {
    * identifiers, not LLM-generated text — so it's consumed at the body-stamp
    * site, not here.
    */
-  static async detectTags(content, client, schema, category, sourceLanguage, onChunk) {
+  static async detectTags(content, client, schema, category, sourceLanguage, onActivity) {
     const categoryInfo = schema.tags.find((t) => t.name === category);
     if (!categoryInfo) {
       throw new Error(`Invalid category "${category}" for schema ${schema.id}`);
@@ -10002,13 +10048,25 @@ var AnnotationDetection = class {
       ),
       0.2,
       "tag",
-      (text) => MotivationParsers.parseTags(text),
-      onChunk
+      TAG_ELEMENT_SCHEMA,
+      (items) => MotivationParsers.parseTags(items),
+      onActivity
     );
     return MotivationParsers.validateTagOffsets(parsedTags, content, category);
   }
 };
-async function extractEntities(exact, entityTypes, client, includeDescriptiveReferences, logger2, sourceLanguage, onChunk) {
+var ENTITY_ELEMENT_SCHEMA = {
+  type: "object",
+  properties: {
+    exact: { type: "string" },
+    entityType: { type: "string" },
+    prefix: { type: "string" },
+    suffix: { type: "string" }
+  },
+  required: ["exact", "entityType"],
+  additionalProperties: false
+};
+async function extractEntities(exact, entityTypes, client, includeDescriptiveReferences, logger2, sourceLanguage, onActivity) {
   const entityTypesDescription = entityTypes.map((et) => {
     if (typeof et === "string") {
       return et;
@@ -10069,50 +10127,28 @@ Example output:
   });
   const collected = [];
   for (let i = 0; i < chunks.length; i++) {
-    const response = await boundedGenerateWithMetadata(
+    const response = await boundedGenerateStructured(
       client,
       buildPrompt(chunks[i]),
       outputBudget,
       0.3,
       // Lower temperature for more consistent extraction
-      // Force grammar-constrained JSON output. Without this, Ollama models
-      // periodically emit malformed JSON (truncated brackets, mid-token
-      // breaks at higher token counts) which silently parse-fails into
-      // [] downstream. The prompt's schema (which keys, what types) still
-      // governs *what* the JSON contains; `format: 'json'` governs that
-      // it's syntactically valid.
-      { format: "json" }
+      ENTITY_ELEMENT_SCHEMA,
+      // Still alive, same position: a long single call would otherwise emit
+      // nothing at all between start and finish.
+      () => onActivity?.(i, chunks.length)
     );
     logger2.debug("Got entity extraction response", {
       chunk: i + 1,
       chunks: chunks.length,
-      responseLength: response.text.length
+      items: response.items.length
     });
     if (response.stopReason === "max_tokens") {
       const errorMsg = `Entity extraction response truncated (max_tokens) on chunk ${i + 1}/${chunks.length} despite the derived output budget of ${outputBudget} tokens \u2014 failing the job rather than dropping annotations.`;
-      logger2.error(errorMsg, { responseLength: response.text.length });
+      logger2.error(errorMsg, { items: response.items.length });
       throw new Error(errorMsg);
     }
-    let entities;
-    try {
-      entities = JSON.parse(response.text.trim());
-    } catch (error) {
-      logger2.error("Failed to parse entity extraction response", {
-        error: error instanceof Error ? error.message : String(error),
-        response: response.text.slice(0, 500)
-      });
-      throw new Error("Failed to parse entity extraction response", {
-        cause: error instanceof Error ? error : new Error(String(error))
-      });
-    }
-    if (!isArray(entities)) {
-      logger2.error("Failed to parse entity extraction response: expected a JSON array", {
-        response: response.text.slice(0, 500)
-      });
-      throw new Error("Failed to parse entity extraction response: expected a JSON array");
-    }
-    logger2.debug("Parsed entities from AI response", { chunk: i + 1, count: entities.length });
-    for (const e of entities) {
+    for (const e of response.items) {
       if (isObject(e) && isString(e.exact) && isString(e.entityType)) {
         collected.push({
           exact: e.exact,
@@ -10125,7 +10161,7 @@ Example output:
       }
     }
     if (i < chunks.length - 1) {
-      onChunk?.(i + 1, chunks.length);
+      onActivity?.(i + 1, chunks.length);
     }
   }
   return collected;
@@ -10561,30 +10597,39 @@ function buildPdfAnnotation(anchored, resourceId, userId, generator, motivation,
   };
 }
 async function processHighlightJob(content, inferenceClient, params, buildAnnotation, onProgress) {
-  onProgress(10, "Loading resource...", "analyzing");
-  onProgress(30, "Analyzing text...", "analyzing");
+  const echo = detectionEcho(params);
+  onProgress(10, { code: "loading" }, echo);
+  onProgress(30, { code: "analyzing" }, echo);
   const highlights = await AnnotationDetection.detectHighlights(
     content,
     inferenceClient,
     params.instructions,
     params.density,
     params.sourceLanguage,
-    // Chunk-boundary heartbeat (liveness): interpolate within the 30–60 band.
-    (completed, total) => onProgress(30 + Math.round(completed / total * 30), "Analyzing text...", "analyzing")
+    // Liveness (chunk boundaries + in-flight heartbeat): 30–60 band.
+    (completed, total) => onProgress(30 + Math.round(completed / total * 30), { code: "analyzing" }, echo)
   );
-  onProgress(60, `Creating ${highlights.length} annotations...`, "creating");
+  onProgress(60, { code: "creating-annotations", count: highlights.length }, echo);
   const annotations = dedupeAnnotations(highlights.map(
     (h) => buildAnnotation("highlighting", h)
   ));
-  onProgress(100, `Complete! Created ${annotations.length} highlights`, "creating");
+  onProgress(100, { code: "complete-created", count: annotations.length, kind: "highlight" }, echo);
   return {
     annotations,
     result: { highlightsFound: highlights.length, highlightsCreated: annotations.length }
   };
 }
+function detectionEcho(p) {
+  const requestParams = [];
+  if (p.instructions?.trim()) requestParams.push({ label: "instructions", value: p.instructions.trim() });
+  if (p.tone?.trim()) requestParams.push({ label: "tone", value: p.tone.trim() });
+  if (p.density !== void 0) requestParams.push({ label: "density", value: String(p.density) });
+  return requestParams.length > 0 ? { requestParams } : {};
+}
 async function processCommentJob(content, inferenceClient, params, buildAnnotation, onProgress) {
-  onProgress(10, "Loading resource...", "analyzing");
-  onProgress(30, "Analyzing text...", "analyzing");
+  const echo = detectionEcho(params);
+  onProgress(10, { code: "loading" }, echo);
+  onProgress(30, { code: "analyzing" }, echo);
   const comments = await AnnotationDetection.detectComments(
     content,
     inferenceClient,
@@ -10593,10 +10638,10 @@ async function processCommentJob(content, inferenceClient, params, buildAnnotati
     params.density,
     params.language,
     params.sourceLanguage,
-    // Chunk-boundary heartbeat (liveness): interpolate within the 30–60 band.
-    (completed, total) => onProgress(30 + Math.round(completed / total * 30), "Analyzing text...", "analyzing")
+    // Liveness (chunk boundaries + in-flight heartbeat): 30–60 band.
+    (completed, total) => onProgress(30 + Math.round(completed / total * 30), { code: "analyzing" }, echo)
   );
-  onProgress(60, `Creating ${comments.length} annotations...`, "creating");
+  onProgress(60, { code: "creating-annotations", count: comments.length }, echo);
   const bodyLanguage = params.language ?? "en";
   const annotations = dedupeAnnotations(comments.map(
     (c) => (
@@ -10608,15 +10653,16 @@ async function processCommentJob(content, inferenceClient, params, buildAnnotati
       ])
     )
   ));
-  onProgress(100, `Complete! Created ${annotations.length} comments`, "creating");
+  onProgress(100, { code: "complete-created", count: annotations.length, kind: "comment" }, echo);
   return {
     annotations,
     result: { commentsFound: comments.length, commentsCreated: annotations.length }
   };
 }
 async function processAssessmentJob(content, inferenceClient, params, buildAnnotation, onProgress) {
-  onProgress(10, "Loading resource...", "analyzing");
-  onProgress(30, "Analyzing text...", "analyzing");
+  const echo = detectionEcho(params);
+  onProgress(10, { code: "loading" }, echo);
+  onProgress(30, { code: "analyzing" }, echo);
   const assessments = await AnnotationDetection.detectAssessments(
     content,
     inferenceClient,
@@ -10625,10 +10671,10 @@ async function processAssessmentJob(content, inferenceClient, params, buildAnnot
     params.density,
     params.language,
     params.sourceLanguage,
-    // Chunk-boundary heartbeat (liveness): interpolate within the 30–60 band.
-    (completed, total) => onProgress(30 + Math.round(completed / total * 30), "Analyzing text...", "analyzing")
+    // Liveness (chunk boundaries + in-flight heartbeat): 30–60 band.
+    (completed, total) => onProgress(30 + Math.round(completed / total * 30), { code: "analyzing" }, echo)
   );
-  onProgress(60, `Creating ${assessments.length} annotations...`, "creating");
+  onProgress(60, { code: "creating-annotations", count: assessments.length }, echo);
   const bodyLanguage = params.language ?? "en";
   const annotations = dedupeAnnotations(assessments.map(
     (a) => (
@@ -10647,7 +10693,7 @@ async function processAssessmentJob(content, inferenceClient, params, buildAnnot
       })
     )
   ));
-  onProgress(100, `Complete! Created ${annotations.length} assessments`, "creating");
+  onProgress(100, { code: "complete-created", count: annotations.length, kind: "assessment" }, echo);
   return {
     annotations,
     result: { assessmentsFound: assessments.length, assessmentsCreated: annotations.length }
@@ -10655,25 +10701,27 @@ async function processAssessmentJob(content, inferenceClient, params, buildAnnot
 }
 async function processReferenceJob(content, inferenceClient, params, buildAnnotation, onProgress, logger2) {
   const entityTypeNames = params.entityTypes.map(String);
-  const requestParams = [{ label: "Entity types", value: entityTypeNames.join(", ") }];
-  const completedEntityTypes = [];
+  const requestParams = [{ label: "entity-types", value: entityTypeNames.join(", ") }];
+  const completedItems = [];
   let totalFound = 0;
   let totalEmitted = 0;
   let errors = 0;
   const allAnnotations = [];
-  onProgress(10, "Loading resource...", "analyzing", { requestParams });
+  onProgress(10, { code: "loading" }, { requestParams });
   const bodyLanguage = params.language ?? "en";
   for (let i = 0; i < entityTypeNames.length; i++) {
     const entityTypeName = entityTypeNames[i];
     if (!entityTypeName) continue;
     const pct = 20 + Math.round(i / entityTypeNames.length * 60);
-    onProgress(pct, `Detecting ${entityTypeName} entities...`, "analyzing", {
-      currentEntityType: entityTypeName,
-      processedEntityTypes: i,
-      totalEntityTypes: entityTypeNames.length,
+    onProgress(pct, { code: "detecting-entities", entityType: entityTypeName }, {
+      // One vocabulary for "what is in flight" (CLEAN-PROGRESS D2): the entity
+      // type is KB data, `kind` is the code the client localizes around it.
+      current: { kind: "entity-type", value: entityTypeName },
+      processed: i,
+      total: entityTypeNames.length,
       entitiesFound: totalFound,
       entitiesEmitted: totalEmitted,
-      completedEntityTypes: [...completedEntityTypes],
+      completedItems: [...completedItems],
       requestParams
     });
     const extractedEntities = await extractEntities(
@@ -10683,25 +10731,28 @@ async function processReferenceJob(content, inferenceClient, params, buildAnnota
       params.includeDescriptiveReferences ?? false,
       logger2,
       params.sourceLanguage,
-      // Chunk-boundary heartbeat: progress is the worker's liveness signal
-      // (stall watchdog + backend janitor), so multi-chunk extraction must
-      // emit between inference calls. Percentage interpolates within this
-      // entity type's band of the 20–80 range.
+      // Liveness: fires at chunk boundaries AND every ~15 s while a single
+      // inference call is in flight (DETECTION-HEARTBEAT). Progress feeds the
+      // stall watchdog, the janitor, AND the client's inter-emission timeout,
+      // so a long single-chunk call must not be silent. Percentage
+      // interpolates within this entity type's band of the 20–80 range; a
+      // heartbeat repeats the current position rather than inventing an
+      // advance.
       (completed, total) => {
         const interpolated = 20 + Math.round((i + completed / total) / entityTypeNames.length * 60);
-        onProgress(interpolated, `Detecting ${entityTypeName} entities...`, "analyzing", {
-          currentEntityType: entityTypeName,
-          processedEntityTypes: i,
-          totalEntityTypes: entityTypeNames.length,
+        onProgress(interpolated, { code: "detecting-entities", entityType: entityTypeName }, {
+          current: { kind: "entity-type", value: entityTypeName },
+          processed: i,
+          total: entityTypeNames.length,
           entitiesFound: totalFound,
           entitiesEmitted: totalEmitted,
-          completedEntityTypes: [...completedEntityTypes],
+          completedItems: [...completedItems],
           requestParams
         });
       }
     );
     totalFound += extractedEntities.length;
-    completedEntityTypes.push({ entityType: entityTypeName, foundCount: extractedEntities.length });
+    completedItems.push({ value: entityTypeName, foundCount: extractedEntities.length });
     const unresolvedBody = [
       { type: "TextualBody", value: entityTypeName, purpose: "tagging", format: "text/plain", language: bodyLanguage }
     ];
@@ -10732,36 +10783,49 @@ async function processReferenceJob(content, inferenceClient, params, buildAnnota
     }
   }
   const annotations = dedupeAnnotations(allAnnotations);
-  onProgress(100, `Complete! Created ${annotations.length} references`, "creating");
+  onProgress(100, { code: "complete-created", count: annotations.length, kind: "reference" }, { requestParams });
   return {
     annotations,
     result: { totalFound, totalEmitted: annotations.length, errors }
   };
 }
 async function processTagJob(content, inferenceClient, params, buildAnnotation, onProgress) {
-  onProgress(10, "Loading resource...", "analyzing");
-  onProgress(30, "Analyzing text for tags...", "analyzing");
+  onProgress(10, { code: "loading" });
+  onProgress(30, { code: "analyzing-tags" });
   const allTags = [];
+  const completedItems = [];
   for (let c = 0; c < params.categories.length; c++) {
     const category = params.categories[c];
+    const position = () => ({
+      current: { kind: "category", value: category },
+      processed: c,
+      total: params.categories.length,
+      completedItems: [...completedItems]
+    });
+    onProgress(
+      30 + Math.round(c / params.categories.length * 30),
+      { code: "analyzing-tags" },
+      position()
+    );
     const categoryTags = await AnnotationDetection.detectTags(
       content,
       inferenceClient,
       params.schema,
       category,
       params.sourceLanguage,
-      // Chunk-boundary heartbeat (liveness): interpolate within this
-      // category's slice of the 30–60 band.
+      // Liveness (chunk boundaries + in-flight heartbeat): this category's
+      // slice of the 30–60 band.
       (completed, total) => onProgress(
         30 + Math.round((c + completed / total) / params.categories.length * 30),
-        "Analyzing text for tags...",
-        "analyzing"
+        { code: "analyzing-tags" },
+        position()
       )
     );
+    completedItems.push({ value: category, foundCount: categoryTags.length });
     allTags.push(...categoryTags);
   }
   const tags = allTags;
-  onProgress(60, `Creating ${tags.length} tag annotations...`, "creating");
+  onProgress(60, { code: "creating-tag-annotations", count: tags.length });
   const bodyLanguage = params.language ?? "en";
   const annotations = dedupeAnnotations(tags.map((t) => {
     const category = t.category ?? "unknown";
@@ -10776,7 +10840,7 @@ async function processTagJob(content, inferenceClient, params, buildAnnotation,
     const category = Array.isArray(body) && typeof body[0]?.value === "string" ? body[0].value : "unknown";
     byCategory[category] = (byCategory[category] ?? 0) + 1;
   }
-  onProgress(100, `Complete! Created ${annotations.length} tags`, "creating");
+  onProgress(100, { code: "complete-created", count: annotations.length, kind: "tag" });
   return {
     annotations,
     result: { tagsFound: tags.length, tagsCreated: annotations.length, byCategory }
@@ -10799,7 +10863,7 @@ async function processGenerationJob(inferenceClient, params, onProgress, logger2
   const title = params.title ?? "Untitled";
   const entityTypes = (params.entityTypes ?? []).map(String);
   if (outputMediaType === "application/pdf") {
-    onProgress(5, "Generating resource...", "generating");
+    onProgress(5, { code: "generating-resource" });
     const validIds = params.cite === true ? collectContextResourceIds(params.context) : null;
     let generated2 = await generateResourceFromTopic(
       title,
@@ -10864,7 +10928,7 @@ async function processGenerationJob(inferenceClient, params, onProgress, logger2
       );
     }
     assertWithinOutputBudget(compiled.pdf.byteLength);
-    onProgress(95, "Creating resource...", "creating");
+    onProgress(95, { code: "creating-resource" });
     return {
       content: compiled.pdf,
       title: generated2.title ?? title,
@@ -10876,7 +10940,7 @@ async function processGenerationJob(inferenceClient, params, onProgress, logger2
       }
     };
   }
-  onProgress(5, "Generating resource...", "generating");
+  onProgress(5, { code: "generating-resource" });
   const generated = await generateResourceFromTopic(
     title,
     entityTypes,
@@ -10900,7 +10964,7 @@ async function processGenerationJob(inferenceClient, params, onProgress, logger2
     content = resolved.content;
     citations = resolved.citations;
   }
-  onProgress(95, "Creating resource...", "creating");
+  onProgress(95, { code: "creating-resource" });
   const artifact = new TextEncoder().encode(content);
   assertWithinOutputBudget(artifact.byteLength);
   return {
@@ -10940,13 +11004,18 @@ async function prepareDetection(mediaType, session, resourceId, userId, generato
     buildAnnotation: (motivation, match, body) => buildTextAnnotation(extracted.text, resourceId, userId, generator, motivation, match, body)
   };
 }
-var DECLINE_MESSAGES = {
-  "no-text-layer": "This PDF is a scan whose text could not be recognized; there is nothing to detect over.",
-  "encrypted": "This PDF is password-protected, so its text cannot be read.",
-  "corrupt": "This PDF could not be parsed \u2014 the file may be damaged or truncated.",
-  "too-large": "This document is too large to extract text from.",
-  "empty": "This document contains no text to detect over."
-};
+function referenceIdOf(job) {
+  if (job.type === "generation") {
+    const context = job.params.context;
+    const focus = context?.focus;
+    if (focus?.kind === "annotation" && typeof focus.annotation?.id === "string") {
+      return focus.annotation.id;
+    }
+    return void 0;
+  }
+  const ref = job.params.referenceId;
+  return typeof ref === "string" ? ref : void 0;
+}
 async function emitEvent(session, channel, payload) {
   await session.client.transport.emit(channel, payload);
 }
@@ -10963,7 +11032,7 @@ function startWorkerProcess(config) {
     handleJob(adapter, config, job).catch((error) => {
       const message = error instanceof Error ? error.message : String(error);
       logger2.error("Job failed", { jobId: job.jobId, error: message, stack: error instanceof Error ? error.stack : void 0 });
-      const failAnnotationId = job.params.referenceId;
+      const failAnnotationId = referenceIdOf(job);
       if (isJobType(job.type)) {
         emitEvent(session, "job:fail", {
           resourceId: job.resourceId,
@@ -11012,7 +11081,7 @@ async function handleJobInner(adapter, config, job) {
   }
   const jobType = job.type;
   const resourceId$1 = resourceId(job.resourceId);
-  const annotationId = job.params.referenceId;
+  const annotationId = referenceIdOf(job);
   const lifecycleBase = {
     resourceId: resourceId$1,
     jobId,
@@ -11028,7 +11097,11 @@ async function handleJobInner(adapter, config, job) {
   if (jobType !== "generation") {
     const descriptor = await session.client.browse.resource(resourceId$1).fresh();
     const mediaType = getPrimaryMediaType(descriptor);
-    const source = await prepareDetection(mediaType ?? "", session, resourceId$1, userId, generator, config.anchoredTextStore);
+    const source = await withSpan(
+      "detection:prepare",
+      () => prepareDetection(mediaType ?? "", session, resourceId$1, userId, generator, config.anchoredTextStore),
+      { attrs: { "resource.id": resourceId$1, "media.type": mediaType ?? "unknown" } }
+    );
     if ("declined" in source) {
       if (source.declined === "no-extractor") {
         throw new Error(`Cannot run ${jobType} on resource ${resourceId$1}: media type '${mediaType ?? "unknown"}' has no extractable text to analyze`);
@@ -11037,8 +11110,7 @@ async function handleJobInner(adapter, config, job) {
         ...lifecycleBase,
         result: {
           declined: true,
-          reason: source.declined,
-          message: DECLINE_MESSAGES[source.declined]
+          reason: source.declined
         }
       });
       adapter.completeJob();
@@ -11046,13 +11118,12 @@ async function handleJobInner(adapter, config, job) {
     }
     ready = source;
   }
-  const onProgress = (percentage, message, stage, extra) => {
+  const onProgress = (percentage, message, extra) => {
     adapter.touchActivity();
     emitEvent(session, "job:report-progress", {
       ...lifecycleBase,
       percentage,
       progress: {
-        stage,
         percentage,
         message,
         ...annotationId ? { annotationId } : {},
@@ -11143,6 +11214,11 @@ async function handleJobInner(adapter, config, job) {
     });
     adapter.completeJob();
   } else if (jobType === "generation") {
+    if (!isGenerationJobParams(job.params)) {
+      throw new Error(
+        `generation job ${job.jobId}: params do not satisfy GenerationJobParams (title, storageUri, and context are required)`
+      );
+    }
     const genResult = await processGenerationJob(
       inferenceClient,
       job.params,
@@ -11150,6 +11226,7 @@ async function handleJobInner(adapter, config, job) {
       config.logger
     );
     const genParams = job.params;
+    const genReferenceId = referenceIdOf(job);
     const storageUri = deriveStorageUri(genResult.title, genResult.format);
     const { resourceId: newResourceId } = await session.client.yield.resource({
       name: genResult.title,
@@ -11157,13 +11234,13 @@ async function handleJobInner(adapter, config, job) {
       format: genResult.format,
       storageUri,
       sourceResourceId: resourceId$1,
-      ...genParams.referenceId ? { sourceAnnotationId: genParams.referenceId } : {},
+      ...genReferenceId ? { sourceAnnotationId: genReferenceId } : {},
       ...genParams.prompt ? { generationPrompt: genParams.prompt } : {},
       ...genParams.language ? { language: genParams.language } : {},
       ...genParams.entityTypes && genParams.entityTypes.length > 0 ? { entityTypes: genParams.entityTypes } : {},
       generator
     });
-    if (!genParams.referenceId) {
+    if (!genReferenceId) {
       const { annotation: provenanceRef } = assembleAnnotation(
         {
           motivation: "linking",