@semiont/jobs 0.5.5 → 0.5.7

package/dist/worker-main.js

@@ -1,4 +1,4 @@
-import { createTomlConfigLoader, softwareToAgent, baseUrl, RESOURCE_BROADCAST_TYPES, resourceId, validateAndCorrectOffsets, didToAgent, getLocaleEnglishName } from '@semiont/core';
+import { createTomlConfigLoader, softwareToAgent, baseUrl, getPrimaryMediaType, textExtractionOf, reconcileSelector, didToAgent, getLocaleEnglishName } from '@semiont/core';
 import { deriveStorageUri } from '@semiont/content';
 import { withSpan, SpanKind, recordJobOutcome } from '@semiont/observability';
 import { generateAnnotationId } from '@semiont/event-sourcing';
@@ -8,7 +8,7 @@ import { existsSync, readFileSync } from 'fs';
 import { homedir, hostname } from 'os';
 import { join } from 'path';
 import { InMemorySessionStorage, setStoredSession, kbBackendUrl, SemiontClient, SemiontSession } from '@semiont/sdk';
-import { HttpTransport, HttpContentTransport } from '@semiont/api-client';
+import { HttpTransport, HttpContentTransport } from '@semiont/http-transport';
 import { createProcessLogger } from '@semiont/observability/process-logger';
 
 var __create = Object.create;
@@ -9337,17 +9337,15 @@ ${content.substring(0, 8e3)}
 
 Return a JSON array of comments. Each comment must have:
 - "exact": the exact text passage being commented on (quoted verbatim from source)
-- "start": character offset where the passage starts
-- "end": character offset where the passage ends
-- "prefix": up to 32 characters of text immediately before the passage
-- "suffix": up to 32 characters of text immediately after the passage
+- "prefix": up to 64 characters of text immediately before the passage
+- "suffix": up to 64 characters of text immediately after the passage
 - "comment": your comment following the instructions above
 
 Respond with a valid JSON array.
 
 Example:
 [
-  {"exact": "the quarterly review meeting", "start": 142, "end": 169, "prefix": "We need to schedule ", "suffix": " for next month.", "comment": "Who will lead this? Should we invite the external auditors?"}
+  {"exact": "the quarterly review meeting", "prefix": "We need to schedule ", "suffix": " for next month.", "comment": "Who will lead this? Should we invite the external auditors?"}
 ]`;
     } else {
       const toneGuidance = tone ? `
@@ -9373,17 +9371,15 @@ ${content.substring(0, 8e3)}
 
 Return a JSON array of comments. Each comment should have:
 - "exact": the exact text passage being commented on (quoted verbatim from source)
-- "start": character offset where the passage starts
-- "end": character offset where the passage ends
-- "prefix": up to 32 characters of text immediately before the passage
-- "suffix": up to 32 characters of text immediately after the passage
+- "prefix": up to 64 characters of text immediately before the passage
+- "suffix": up to 64 characters of text immediately after the passage
 - "comment": your explanatory comment (1-3 sentences, provide context/background/clarification)
 
 Respond with a valid JSON array.
 
 Example format:
 [
-  {"exact": "Ouranos", "start": 52, "end": 59, "prefix": "In the beginning, ", "suffix": " ruled the universe", "comment": "Ouranos (also spelled Uranus) is the primordial Greek deity personifying the sky. In Hesiod's Theogony, he is the son and husband of Gaia (Earth) and father of the Titans."}
+  {"exact": "Ouranos", "prefix": "In the beginning, ", "suffix": " ruled the universe", "comment": "Ouranos (also spelled Uranus) is the primordial Greek deity personifying the sky. In Hesiod's Theogony, he is the son and husband of Gaia (Earth) and father of the Titans."}
 ]`;
     }
     return prompt;
@@ -9414,16 +9410,14 @@ ${content.substring(0, 8e3)}
 
 Return a JSON array of highlights. Each highlight must have:
 - "exact": the exact text passage to highlight (quoted verbatim from source)
-- "start": character offset where the passage starts
-- "end": character offset where the passage ends
-- "prefix": up to 32 characters of text immediately before the passage
-- "suffix": up to 32 characters of text immediately after the passage
+- "prefix": up to 64 characters of text immediately before the passage
+- "suffix": up to 64 characters of text immediately after the passage
 
 Respond with a valid JSON array.
 
 Example:
 [
-  {"exact": "revenue grew 45% year-over-year", "start": 142, "end": 174, "prefix": "In Q3 2024, ", "suffix": ", exceeding all forecasts."}
+  {"exact": "revenue grew 45% year-over-year", "prefix": "In Q3 2024, ", "suffix": ", exceeding all forecasts."}
 ]`;
     } else {
       const densityGuidance = density ? `
@@ -9447,16 +9441,14 @@ ${content.substring(0, 8e3)}
 
 Return a JSON array of highlights. Each highlight should have:
 - "exact": the exact text passage to highlight (quoted verbatim from source)
-- "start": character offset where the passage starts
-- "end": character offset where the passage ends
-- "prefix": up to 32 characters of text immediately before the passage
-- "suffix": up to 32 characters of text immediately after the passage
+- "prefix": up to 64 characters of text immediately before the passage
+- "suffix": up to 64 characters of text immediately after the passage
 
 Respond with a valid JSON array.
 
 Example format:
 [
-  {"exact": "we will discontinue support for legacy systems by March 2025", "start": 52, "end": 113, "prefix": "After careful consideration, ", "suffix": ". This decision affects"}
+  {"exact": "we will discontinue support for legacy systems by March 2025", "prefix": "After careful consideration, ", "suffix": ". This decision affects"}
 ]`;
     }
     return prompt;
@@ -9490,17 +9482,15 @@ ${content.substring(0, 8e3)}
 
 Return a JSON array of assessments. Each assessment must have:
 - "exact": the exact text passage being assessed (quoted verbatim from source)
-- "start": character offset where the passage starts
-- "end": character offset where the passage ends
-- "prefix": up to 32 characters of text immediately before the passage
-- "suffix": up to 32 characters of text immediately after the passage
+- "prefix": up to 64 characters of text immediately before the passage
+- "suffix": up to 64 characters of text immediately after the passage
 - "assessment": your assessment following the instructions above
 
 Respond with a valid JSON array.
 
 Example:
 [
-  {"exact": "the quarterly revenue target", "start": 142, "end": 169, "prefix": "We established ", "suffix": " for Q4 2024.", "assessment": "This target seems ambitious given market conditions. Consider revising based on recent trends."}
+  {"exact": "the quarterly revenue target", "prefix": "We established ", "suffix": " for Q4 2024.", "assessment": "This target seems ambitious given market conditions. Consider revising based on recent trends."}
 ]`;
     } else {
       const toneGuidance = tone ? `
@@ -9526,17 +9516,15 @@ ${content.substring(0, 8e3)}
 
 Return a JSON array of assessments. Each assessment should have:
 - "exact": the exact text passage being assessed (quoted verbatim from source)
-- "start": character offset where the passage starts
-- "end": character offset where the passage ends
-- "prefix": up to 32 characters of text immediately before the passage
-- "suffix": up to 32 characters of text immediately after the passage
+- "prefix": up to 64 characters of text immediately before the passage
+- "suffix": up to 64 characters of text immediately after the passage
 - "assessment": your analytical assessment (1-3 sentences, evaluate validity/strength/implications)
 
 Respond with a valid JSON array.
 
 Example format:
 [
-  {"exact": "AI will replace most jobs by 2030", "start": 52, "end": 89, "prefix": "Many experts predict that ", "suffix": ", fundamentally reshaping", "assessment": "This claim lacks nuance and supporting evidence. Employment patterns historically show job transformation rather than wholesale replacement. The timeline appears speculative without specific sector analysis."}
+  {"exact": "AI will replace most jobs by 2030", "prefix": "Many experts predict that ", "suffix": ", fundamentally reshaping", "assessment": "This claim lacks nuance and supporting evidence. Employment patterns historically show job transformation rather than wholesale replacement. The timeline appears speculative without specific sector analysis."}
 ]`;
     }
     return prompt;
@@ -9582,17 +9570,15 @@ ${content}
 
 Return a JSON array of tags. Each tag should have:
 - "exact": the exact text passage (quoted verbatim from source)
-- "start": character offset where the passage starts
-- "end": character offset where the passage ends
-- "prefix": up to 32 characters of text immediately before the passage
-- "suffix": up to 32 characters of text immediately after the passage
+- "prefix": up to 64 characters of text immediately before the passage
+- "suffix": up to 64 characters of text immediately after the passage
 
 Respond with a valid JSON array.
 
 Example format:
 [
-  {"exact": "What duty did the defendant owe?", "start": 142, "end": 175, "prefix": "The central question is: ", "suffix": " This question must be"},
-  {"exact": "In tort law, a duty of care is established when...", "start": 412, "end": 520, "prefix": "Legal framework:\\n", "suffix": "\\n\\nApplying this standard"}
+  {"exact": "What duty did the defendant owe?", "prefix": "The central question is: ", "suffix": " This question must be"},
+  {"exact": "In tort law, a duty of care is established when...", "prefix": "Legal framework:\\n", "suffix": "\\n\\nApplying this standard"}
 ]`;
     return prompt;
   }
@@ -9660,23 +9646,29 @@ var MotivationParsers = class {
     try {
       const parsed = extractObjectsFromArray(response);
       const valid = parsed.filter(
-        (c) => !!c && typeof c === "object" && typeof c.exact === "string" && typeof c.start === "number" && typeof c.end === "number" && typeof c.comment === "string" && c.comment.trim().length > 0
+        (c) => !!c && typeof c === "object" && typeof c.exact === "string" && typeof c.comment === "string" && c.comment.trim().length > 0
       );
       console.log(`[MotivationParsers] Parsed ${valid.length} valid comments from ${parsed.length} total`);
       const validatedComments = [];
       for (const comment of valid) {
-        try {
-          const validated = validateAndCorrectOffsets(content, comment.start, comment.end, comment.exact);
-          validatedComments.push({
-            ...comment,
-            start: validated.start,
-            end: validated.end,
-            prefix: validated.prefix,
-            suffix: validated.suffix
-          });
-        } catch (error) {
-          console.warn(`[MotivationParsers] Skipping invalid comment "${comment.exact}":`, error);
-        }
+        const reconciled = reconcileSelector(content, {
+          exact: comment.exact,
+          ...typeof comment.prefix === "string" ? { prefix: comment.prefix } : {},
+          ...typeof comment.suffix === "string" ? { suffix: comment.suffix } : {}
+        });
+        if (!reconciled) {
+          console.warn(`[MotivationParsers] Dropped hallucinated comment "${comment.exact}"`);
+          continue;
+        }
+        logAnchorMethod("comment", comment.exact, reconciled.anchorMethod);
+        validatedComments.push({
+          comment: comment.comment,
+          exact: reconciled.exact,
+          start: reconciled.start,
+          end: reconciled.end,
+          ...reconciled.prefix !== void 0 ? { prefix: reconciled.prefix } : {},
+          ...reconciled.suffix !== void 0 ? { suffix: reconciled.suffix } : {}
+        });
       }
       return validatedComments;
     } catch (error) {
@@ -9695,22 +9687,27 @@ var MotivationParsers = class {
     try {
       const parsed = extractObjectsFromArray(response);
       const highlights = parsed.filter(
-        (h) => !!h && typeof h === "object" && typeof h.exact === "string" && typeof h.start === "number" && typeof h.end === "number"
+        (h) => !!h && typeof h === "object" && typeof h.exact === "string"
       );
       const validatedHighlights = [];
       for (const highlight of highlights) {
-        try {
-          const validated = validateAndCorrectOffsets(content, highlight.start, highlight.end, highlight.exact);
-          validatedHighlights.push({
-            ...highlight,
-            start: validated.start,
-            end: validated.end,
-            prefix: validated.prefix,
-            suffix: validated.suffix
-          });
-        } catch (error) {
-          console.warn(`[MotivationParsers] Skipping invalid highlight "${highlight.exact}":`, error);
-        }
+        const reconciled = reconcileSelector(content, {
+          exact: highlight.exact,
+          ...typeof highlight.prefix === "string" ? { prefix: highlight.prefix } : {},
+          ...typeof highlight.suffix === "string" ? { suffix: highlight.suffix } : {}
+        });
+        if (!reconciled) {
+          console.warn(`[MotivationParsers] Dropped hallucinated highlight "${highlight.exact}"`);
+          continue;
+        }
+        logAnchorMethod("highlight", highlight.exact, reconciled.anchorMethod);
+        validatedHighlights.push({
+          exact: reconciled.exact,
+          start: reconciled.start,
+          end: reconciled.end,
+          ...reconciled.prefix !== void 0 ? { prefix: reconciled.prefix } : {},
+          ...reconciled.suffix !== void 0 ? { suffix: reconciled.suffix } : {}
+        });
       }
       return validatedHighlights;
     } catch (error) {
@@ -9730,22 +9727,28 @@ var MotivationParsers = class {
     try {
       const parsed = extractObjectsFromArray(response);
       const assessments = parsed.filter(
-        (a) => !!a && typeof a === "object" && typeof a.exact === "string" && typeof a.start === "number" && typeof a.end === "number" && typeof a.assessment === "string"
+        (a) => !!a && typeof a === "object" && typeof a.exact === "string" && typeof a.assessment === "string"
       );
       const validatedAssessments = [];
       for (const assessment of assessments) {
-        try {
-          const validated = validateAndCorrectOffsets(content, assessment.start, assessment.end, assessment.exact);
-          validatedAssessments.push({
-            ...assessment,
-            start: validated.start,
-            end: validated.end,
-            prefix: validated.prefix,
-            suffix: validated.suffix
-          });
-        } catch (error) {
-          console.warn(`[MotivationParsers] Skipping invalid assessment "${assessment.exact}":`, error);
-        }
+        const reconciled = reconcileSelector(content, {
+          exact: assessment.exact,
+          ...typeof assessment.prefix === "string" ? { prefix: assessment.prefix } : {},
+          ...typeof assessment.suffix === "string" ? { suffix: assessment.suffix } : {}
+        });
+        if (!reconciled) {
+          console.warn(`[MotivationParsers] Dropped hallucinated assessment "${assessment.exact}"`);
+          continue;
+        }
+        logAnchorMethod("assessment", assessment.exact, reconciled.anchorMethod);
+        validatedAssessments.push({
+          assessment: assessment.assessment,
+          exact: reconciled.exact,
+          start: reconciled.start,
+          end: reconciled.end,
+          ...reconciled.prefix !== void 0 ? { prefix: reconciled.prefix } : {},
+          ...reconciled.suffix !== void 0 ? { suffix: reconciled.suffix } : {}
+        });
       }
       return validatedAssessments;
     } catch (error) {
@@ -9755,17 +9758,15 @@ var MotivationParsers = class {
     }
   }
   /**
-   * Parse and validate AI response for tag detection
-   * Note: Does NOT validate offsets - caller must do that with content
-   *
-   * @param response - Raw AI response string (may include markdown code fences)
-   * @returns Array of tag matches (offsets not yet validated)
+   * Parse the LLM's tag response into raw, pre-reconciliation tag inputs.
+   * Reconciliation happens in `validateTagOffsets`, which adds `start`/`end`
+   * by anchoring `exact` against the source content.
    */
   static parseTags(response) {
     try {
       const parsed = extractObjectsFromArray(response);
       const valid = parsed.filter(
-        (t) => !!t && typeof t === "object" && typeof t.exact === "string" && typeof t.start === "number" && typeof t.end === "number" && t.exact.trim().length > 0
+        (t) => !!t && typeof t === "object" && typeof t.exact === "string" && t.exact.trim().length > 0
       );
       console.log(`[MotivationParsers] Parsed ${valid.length} valid tags from ${parsed.length} total`);
       return valid;
@@ -9775,52 +9776,41 @@ var MotivationParsers = class {
     }
   }
   /**
-   * Validate tag offsets against content and add category
-   * Helper for tag detection after initial parsing
-   *
-   * @param tags - Parsed tags without validated offsets
-   * @param content - Original content to validate against
-   * @param category - Category to assign to validated tags
-   * @returns Array of validated tag matches
+   * Anchor raw tag inputs against source content and add category.
    */
   static validateTagOffsets(tags, content, category) {
     const validatedTags = [];
     for (const tag of tags) {
-      try {
-        const validated = validateAndCorrectOffsets(content, tag.start, tag.end, tag.exact);
-        validatedTags.push({
-          ...tag,
-          category,
-          start: validated.start,
-          end: validated.end,
-          prefix: validated.prefix,
-          suffix: validated.suffix
-        });
-      } catch (error) {
-        console.warn(`[MotivationParsers] Skipping invalid tag for category "${category}":`, error);
-      }
+      const reconciled = reconcileSelector(content, {
+        exact: tag.exact,
+        ...typeof tag.prefix === "string" ? { prefix: tag.prefix } : {},
+        ...typeof tag.suffix === "string" ? { suffix: tag.suffix } : {}
+      });
+      if (!reconciled) {
+        console.warn(`[MotivationParsers] Dropped hallucinated tag "${tag.exact}" for category "${category}"`);
+        continue;
+      }
+      logAnchorMethod("tag", tag.exact, reconciled.anchorMethod);
+      validatedTags.push({
+        category,
+        exact: reconciled.exact,
+        start: reconciled.start,
+        end: reconciled.end,
+        ...reconciled.prefix !== void 0 ? { prefix: reconciled.prefix } : {},
+        ...reconciled.suffix !== void 0 ? { suffix: reconciled.suffix } : {}
+      });
     }
     return validatedTags;
   }
 };
+function logAnchorMethod(motivation, exact, anchorMethod) {
+  if (anchorMethod === "first-of-many" || anchorMethod === "fuzzy-match") {
+    console.warn(`[MotivationParsers] ${motivation} anchored via ${anchorMethod}: "${exact}"`);
+  }
+}
 
 // src/workers/annotation-detection.ts
 var AnnotationDetection = class {
-  /**
-   * Fetch content from a ContentFetcher and read the stream to a string.
-   * Shared helper for all workers.
-   */
-  static async fetchContent(contentFetcher, resourceId) {
-    const stream = await contentFetcher(resourceId);
-    if (!stream) {
-      throw new Error(`Could not load content for resource ${resourceId}`);
-    }
-    const chunks = [];
-    for await (const chunk of stream) {
-      chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
-    }
-    return Buffer.concat(chunks).toString("utf-8");
-  }
   /**
    * Detect comments in content.
    *
@@ -9930,17 +9920,15 @@ ${exact}
 """
 
 Respond with a JSON array of entities found. Each entity should have:
-- exact: the exact text span from the input
+- exact: the exact text span from the input (quoted verbatim \u2014 character-for-character)
 - entityType: one of the provided entity types
-- startOffset: character position where the entity starts (0-indexed)
-- endOffset: character position where the entity ends
-- prefix: up to 32 characters of text immediately before the entity (helps identify correct occurrence)
-- suffix: up to 32 characters of text immediately after the entity (helps identify correct occurrence)
+- prefix: up to 64 characters of text immediately before the entity (used to disambiguate when the same text appears more than once)
+- suffix: up to 64 characters of text immediately after the entity (same purpose)
 
 If no entities are found, respond with an empty array [].
 
 Example output:
-[{"exact":"Alice","entityType":"Person","startOffset":0,"endOffset":5,"prefix":"","suffix":" went to"},{"exact":"Paris","entityType":"Location","startOffset":20,"endOffset":25,"prefix":"went to ","suffix":" yesterday"}]`;
+[{"exact":"Alice","entityType":"Person","prefix":"","suffix":" went to"},{"exact":"Paris","entityType":"Location","prefix":"went to ","suffix":" yesterday"}]`;
   logger2.debug("Sending entity extraction request", { entityTypes: entityTypesDescription });
   const response = await client.generateTextWithMetadata(
     prompt,
@@ -9969,151 +9957,18 @@ Example output:
       logger2.error(errorMsg);
       throw new Error(errorMsg);
     }
-    return entities.map((entity, idx) => {
-      let start = entity.startOffset;
-      let end = entity.endOffset;
-      logger2.debug("Processing entity", {
-        index: idx + 1,
-        total: entities.length,
-        type: entity.entityType,
-        text: entity.exact,
-        offsetsFromAI: `[${start}:${end}]`
-      });
-      const extractedText = exact.substring(start, end);
-      let anchorMethod;
-      if (extractedText === entity.exact) {
-        anchorMethod = "llm-exact";
-        logger2.debug("Entity anchored", {
-          text: entity.exact,
-          entityType: entity.entityType,
-          anchorMethod
-        });
-      } else {
-        logger2.debug("LLM offsets mismatch \u2014 attempting re-anchor", {
-          expected: entity.exact,
-          llmOffsets: `[${start}:${end}]`,
-          foundAtLlmOffsets: extractedText
-        });
-        let occurrenceCount = 0;
-        let firstOccurrence = -1;
-        let searchPos = 0;
-        while ((searchPos = exact.indexOf(entity.exact, searchPos)) !== -1) {
-          if (firstOccurrence === -1) firstOccurrence = searchPos;
-          occurrenceCount++;
-          searchPos++;
-        }
-        if (occurrenceCount === 0) {
-          anchorMethod = "dropped";
-          logger2.error("Entity text not found in resource \u2014 dropping", {
-            text: entity.exact,
-            entityType: entity.entityType,
-            llmOffsets: `[${start}:${end}]`,
-            anchorMethod,
-            resourceStart: exact.substring(0, 200)
-          });
-          return null;
-        }
-        let recoveredOffset = -1;
-        if (entity.prefix || entity.suffix) {
-          let p = 0;
-          while ((p = exact.indexOf(entity.exact, p)) !== -1) {
-            const candidatePrefix = exact.substring(Math.max(0, p - 32), p);
-            const candidateSuffix = exact.substring(
-              p + entity.exact.length,
-              Math.min(exact.length, p + entity.exact.length + 32)
-            );
-            const prefixMatch = !entity.prefix || candidatePrefix.endsWith(entity.prefix);
-            const suffixMatch = !entity.suffix || candidateSuffix.startsWith(entity.suffix);
-            if (prefixMatch && suffixMatch) {
-              recoveredOffset = p;
-              break;
-            }
-            p++;
-          }
-        }
-        if (recoveredOffset !== -1) {
-          anchorMethod = "context-recovered";
-          start = recoveredOffset;
-          end = recoveredOffset + entity.exact.length;
-          logger2.debug("Entity anchored", {
-            text: entity.exact,
-            entityType: entity.entityType,
-            anchorMethod,
-            offsetDiff: recoveredOffset - entity.startOffset
-          });
-        } else if (occurrenceCount === 1) {
-          anchorMethod = "unique-match";
-          start = firstOccurrence;
-          end = firstOccurrence + entity.exact.length;
-          logger2.debug("Entity anchored", {
-            text: entity.exact,
-            entityType: entity.entityType,
-            anchorMethod,
-            offsetDiff: firstOccurrence - entity.startOffset
-          });
-        } else {
-          anchorMethod = "first-of-many";
-          start = firstOccurrence;
-          end = firstOccurrence + entity.exact.length;
-          logger2.warn("Entity anchored at first of multiple occurrences \u2014 may be wrong", {
-            text: entity.exact,
-            entityType: entity.entityType,
-            anchorMethod,
-            occurrenceCount,
-            chosenOffset: firstOccurrence,
-            llmOffsets: `[${entity.startOffset}:${entity.endOffset}]`,
-            hasPrefix: !!entity.prefix,
-            hasSuffix: !!entity.suffix
-          });
-        }
-      }
-      return {
-        exact: entity.exact,
-        entityType: entity.entityType,
-        start,
-        end,
-        prefix: entity.prefix,
-        suffix: entity.suffix
-      };
-    }).filter((entity) => {
-      if (entity === null) {
-        logger2.debug("Filtered entity: null");
-        return false;
-      }
-      if (entity.start === void 0 || entity.end === void 0) {
-        logger2.warn("Filtered entity: missing offsets", { text: entity.exact });
-        return false;
-      }
-      if (entity.start < 0) {
-        logger2.warn("Filtered entity: negative start", {
-          text: entity.exact,
-          start: entity.start
-        });
-        return false;
-      }
-      if (entity.end > exact.length) {
-        logger2.warn("Filtered entity: end exceeds text length", {
-          text: entity.exact,
-          end: entity.end,
-          textLength: exact.length
-        });
-        return false;
-      }
-      const extractedText = exact.substring(entity.start, entity.end);
-      if (extractedText !== entity.exact) {
-        logger2.warn("Filtered entity: offset mismatch", {
-          expected: entity.exact,
-          got: extractedText,
-          offsets: `[${entity.start}:${entity.end}]`
-        });
-        return false;
-      }
-      logger2.debug("Accepted entity", {
-        text: entity.exact,
-        offsets: `[${entity.start}:${entity.end}]`
-      });
-      return true;
-    });
+    return entities.filter((e) => {
+      const ok = e && typeof e === "object" && typeof e.exact === "string" && typeof e.entityType === "string";
+      if (!ok) {
+        logger2.debug("Dropped malformed LLM entity", { entity: e });
+      }
+      return ok;
+    }).map((entity) => ({
+      exact: entity.exact,
+      entityType: entity.entityType,
+      ...typeof entity.prefix === "string" ? { prefix: entity.prefix } : {},
+      ...typeof entity.suffix === "string" ? { suffix: entity.suffix } : {}
+    }));
   } catch (error) {
     logger2.error("Failed to parse entity extraction response", {
       error: error instanceof Error ? error.message : String(error)
@@ -10249,7 +10104,59 @@ Requirements:
   });
   return result;
 }
-function buildTextAnnotation(resourceId, userId, generator, motivation, match, body) {
+function toMatch(r) {
+  return {
+    exact: r.exact,
+    start: r.start,
+    end: r.end,
+    ...r.prefix !== void 0 ? { prefix: r.prefix } : {},
+    ...r.suffix !== void 0 ? { suffix: r.suffix } : {}
+  };
+}
+function annotationDedupeKey(ann) {
+  const target = ann.target;
+  const selectors = Array.isArray(target?.selector) ? target.selector : [];
+  const pos = selectors.find((s) => s.type === "TextPositionSelector");
+  return [
+    ann.motivation,
+    pos?.start ?? "?",
+    pos?.end ?? "?",
+    JSON.stringify(ann.body ?? null)
+  ].join("|");
+}
+function dedupeAnnotations(annotations) {
+  const seen = /* @__PURE__ */ new Set();
+  const out = [];
+  for (const ann of annotations) {
+    const key = annotationDedupeKey(ann);
+    if (seen.has(key)) continue;
+    seen.add(key);
+    out.push(ann);
+  }
+  return out;
+}
+function buildTextAnnotation(content, resourceId, userId, generator, motivation, match, body) {
+  if (content.substring(match.start, match.end) !== match.exact) {
+    throw new Error(
+      `buildTextAnnotation invariant: content.substring(${match.start}, ${match.end}) !== exact for resource ${resourceId}, motivation ${motivation}`
+    );
+  }
+  if (match.prefix !== void 0) {
+    const actualPrefix = content.substring(Math.max(0, match.start - match.prefix.length), match.start);
+    if (actualPrefix !== match.prefix) {
+      throw new Error(
+        `buildTextAnnotation invariant: content prefix-slice !== prefix for resource ${resourceId}, motivation ${motivation}`
+      );
+    }
+  }
+  if (match.suffix !== void 0) {
+    const actualSuffix = content.substring(match.end, Math.min(content.length, match.end + match.suffix.length));
+    if (actualSuffix !== match.suffix) {
+      throw new Error(
+        `buildTextAnnotation invariant: content suffix-slice !== suffix for resource ${resourceId}, motivation ${motivation}`
+      );
+    }
+  }
   const creator = didToAgent(userId);
   const wasAttributedTo = creator["@id"] === generator["@id"] ? [generator] : [creator, generator];
   return {
@@ -10288,9 +10195,9 @@ async function processHighlightJob(content, inferenceClient, params, userId, gen
     params.sourceLanguage
   );
   onProgress(60, `Creating ${highlights.length} annotations...`, "creating");
-  const annotations = highlights.map(
-    (h) => buildTextAnnotation(params.resourceId, userId, generator, "highlighting", h)
-  );
+  const annotations = dedupeAnnotations(highlights.map(
+    (h) => buildTextAnnotation(content, params.resourceId, userId, generator, "highlighting", h)
+  ));
   onProgress(100, `Complete! Created ${annotations.length} highlights`, "creating");
   return {
     annotations,
@@ -10311,16 +10218,16 @@ async function processCommentJob(content, inferenceClient, params, userId, gener
   );
   onProgress(60, `Creating ${comments.length} annotations...`, "creating");
   const bodyLanguage = params.language ?? "en";
-  const annotations = comments.map(
+  const annotations = dedupeAnnotations(comments.map(
     (c) => (
       // Match the pre-#651 CommentAnnotationWorker: include format and
       // language on the body TextualBody. Optional in the schema, but
       // consumers that do language-aware rendering rely on them.
-      buildTextAnnotation(params.resourceId, userId, generator, "commenting", c, [
+      buildTextAnnotation(content, params.resourceId, userId, generator, "commenting", c, [
         { type: "TextualBody", value: c.comment, purpose: "commenting", format: "text/plain", language: bodyLanguage }
       ])
     )
-  );
+  ));
   onProgress(100, `Complete! Created ${annotations.length} comments`, "creating");
   return {
     annotations,
@@ -10341,7 +10248,7 @@ async function processAssessmentJob(content, inferenceClient, params, userId, ge
   );
   onProgress(60, `Creating ${assessments.length} annotations...`, "creating");
   const bodyLanguage = params.language ?? "en";
-  const annotations = assessments.map(
+  const annotations = dedupeAnnotations(assessments.map(
     (a) => (
       // Single-object body with purpose aligned to motivation, matching the
       // pre-#651 AssessmentAnnotationWorker's shape and the majority of
@@ -10349,7 +10256,7 @@ async function processAssessmentJob(content, inferenceClient, params, userId, ge
       // purpose='describing' — that loses the "this is an assessment, not
       // a description" signal and breaks existing readers that access
       // `body.value` directly on the object.
-      buildTextAnnotation(params.resourceId, userId, generator, "assessing", a, {
+      buildTextAnnotation(content, params.resourceId, userId, generator, "assessing", a, {
         type: "TextualBody",
         value: a.assessment,
         purpose: "assessing",
@@ -10357,7 +10264,7 @@ async function processAssessmentJob(content, inferenceClient, params, userId, ge
         language: bodyLanguage
       })
     )
-  );
+  ));
   onProgress(100, `Complete! Created ${annotations.length} assessments`, "creating");
   return {
     annotations,
@@ -10401,27 +10308,44 @@ async function processReferenceJob(content, inferenceClient, params, userId, gen
       { type: "TextualBody", value: entityTypeName, purpose: "tagging", format: "text/plain", language: bodyLanguage }
     ];
     for (const entity of extractedEntities) {
-      try {
-        const validated = validateAndCorrectOffsets(content, entity.start, entity.end, entity.exact);
-        const ann = buildTextAnnotation(
-          params.resourceId,
-          userId,
-          generator,
-          "linking",
-          validated,
-          unresolvedBody
-        );
-        allAnnotations.push(ann);
-        totalEmitted++;
-      } catch {
+      const reconciled = reconcileSelector(content, {
+        exact: entity.exact,
+        ...entity.prefix !== void 0 ? { prefix: entity.prefix } : {},
+        ...entity.suffix !== void 0 ? { suffix: entity.suffix } : {}
+      });
+      if (!reconciled) {
+        logger2.error("Entity dropped \u2014 text not found in source", {
+          text: entity.exact,
+          entityType: entity.entityType
+        });
         errors++;
+        continue;
+      }
+      if (reconciled.anchorMethod === "first-of-many" || reconciled.anchorMethod === "fuzzy-match") {
+        logger2.warn("Entity anchored via degraded method", {
+          text: entity.exact,
+          entityType: entity.entityType,
+          anchorMethod: reconciled.anchorMethod
+        });
       }
+      const ann = buildTextAnnotation(
+        content,
+        params.resourceId,
+        userId,
+        generator,
+        "linking",
+        toMatch(reconciled),
+        unresolvedBody
+      );
+      allAnnotations.push(ann);
+      totalEmitted++;
     }
   }
-  onProgress(100, `Complete! Created ${totalEmitted} references`, "creating");
+  const annotations = dedupeAnnotations(allAnnotations);
+  onProgress(100, `Complete! Created ${annotations.length} references`, "creating");
   return {
-    annotations: allAnnotations,
-    result: { totalFound, totalEmitted, errors }
+    annotations,
+    result: { totalFound, totalEmitted: annotations.length, errors }
   };
 }
 async function processTagJob(content, inferenceClient, params, userId, generator, onProgress) {
@@ -10441,15 +10365,19 @@ async function processTagJob(content, inferenceClient, params, userId, generator
   const tags = allTags;
   onProgress(60, `Creating ${tags.length} tag annotations...`, "creating");
   const bodyLanguage = params.language ?? "en";
-  const byCategory = {};
-  const annotations = tags.map((t) => {
+  const annotations = dedupeAnnotations(tags.map((t) => {
     const category = t.category ?? "unknown";
-    byCategory[category] = (byCategory[category] ?? 0) + 1;
-    return buildTextAnnotation(params.resourceId, userId, generator, "tagging", t, [
+    return buildTextAnnotation(content, params.resourceId, userId, generator, "tagging", t, [
       { type: "TextualBody", value: category, purpose: "tagging", format: "text/plain", language: bodyLanguage },
       { type: "TextualBody", value: params.schema.id, purpose: "classifying", format: "text/plain" }
     ]);
-  });
+  }));
+  const byCategory = {};
+  for (const ann of annotations) {
+    const body = ann.body;
+    const category = Array.isArray(body) && typeof body[0]?.value === "string" ? body[0].value : "unknown";
+    byCategory[category] = (byCategory[category] ?? 0) + 1;
+  }
   onProgress(100, `Complete! Created ${annotations.length} tags`, "creating");
   return {
     annotations,
@@ -10487,10 +10415,7 @@ async function processGenerationJob(inferenceClient, params, onProgress, logger2
 
 // src/worker-process.ts
 async function emitEvent(session, channel, payload) {
-  const isBroadcast = RESOURCE_BROADCAST_TYPES.includes(channel);
-  const rawScope = isBroadcast ? payload.resourceId : void 0;
-  const resourceScope = rawScope ? resourceId(rawScope) : void 0;
-  await session.client.transport.emit(channel, payload, resourceScope);
+  await session.client.transport.emit(channel, payload);
 }
 function startWorkerProcess(config) {
   const { session, logger: logger2 } = config;
@@ -10560,6 +10485,17 @@ async function handleJobInner(adapter, config, job) {
     adapter.failJob(jobId, `Worker not configured for job type: ${jobType}`);
     return;
   }
+  if (jobType !== "generation") {
+    const descriptor = await session.client.browse.resource(resourceId);
+    const mediaType = getPrimaryMediaType(descriptor);
+    const extraction = mediaType ? textExtractionOf(mediaType) : "none";
+    if (extraction === "pdf-text-layer") {
+      throw new Error(`Cannot run ${jobType} on resource ${resourceId}: PDF text-layer detection is not yet supported`);
+    }
+    if (extraction !== "decode") {
+      throw new Error(`Cannot run ${jobType} on resource ${resourceId}: media type '${mediaType ?? "unknown"}' has no extractable text to analyze`);
+    }
+  }
   const onProgress = (percentage, message, stage, extra) => {
     emitEvent(session, "job:report-progress", {
       ...lifecycleBase,