@semiont/jobs 0.5.23 → 0.5.25

package/dist/worker-main.js

@@ -1,14 +1,15 @@
-import { createTomlConfigLoader, didToAgent, baseUrl, STARTUP_FETCH_RETRY, retryWithBackoff, isTransientFetchError, busRequest, getPrimaryMediaType, textExtractionOf, assembleAnnotation, reconcileSelector, createFragmentSelector, getLocaleEnglishName, isArray, isObject, isString, deriveViews } from '@semiont/core';
-import { deriveStorageUri, extractPdfTextLayer, locate } from '@semiont/content';
+import { createTomlConfigLoader, didToAgent, baseUrl, STARTUP_FETCH_RETRY, retryWithBackoff, isTransientFetchError, busRequest, resourceId, getPrimaryMediaType, assembleAnnotation, findClaimSpan, textExtractionOf, reconcileSelector, GENERATABLE_MEDIA_TYPES, locate, createFragmentSelector, getLocaleEnglishName, estimateTokens, chunkText, isArray, isObject, isString, deriveViews } from '@semiont/core';
+import { anchoredTextStoreOverTransport, deriveStorageUri, extractPdfTextLayer, EXTRACTORS, calculateChecksum, withinByteBudget, MAX_PDF_BYTES } from '@semiont/content';
+import { execFileSync } from 'child_process';
+import { existsSync, readFileSync, mkdtempSync, writeFileSync, rmSync } from 'fs';
+import { homedir, hostname, tmpdir } from 'os';
+import { join } from 'path';
 import { generateAnnotationId } from '@semiont/event-sourcing';
 import { withSpan, SpanKind, recordJobOutcome } from '@semiont/observability';
-import { homedir, hostname } from 'os';
 import { InMemorySessionStorage, setStoredSession, kbBackendUrl, SemiontClient, SemiontSession } from '@semiont/sdk';
 import { HttpTransport, HttpContentTransport } from '@semiont/http-transport';
 import { createInferenceClient } from '@semiont/inference';
 import { createServer } from 'http';
-import { existsSync, readFileSync } from 'fs';
-import { join } from 'path';
 import { createProcessLogger } from '@semiont/observability/process-logger';
 
 var __create = Object.create;
@@ -9329,6 +9330,25 @@ function createJobClaimAdapter(options) {
   };
 }
 
+// src/types.ts
+var JOB_TYPES = /* @__PURE__ */ new Set([
+  "reference-annotation",
+  "generation",
+  "highlight-annotation",
+  "assessment-annotation",
+  "comment-annotation",
+  "tag-annotation"
+]);
+function isJobType(value) {
+  return JOB_TYPES.has(value);
+}
+function asJobParams(params) {
+  if (typeof params.resourceId !== "string") {
+    throw new Error("Job params are missing a resourceId");
+  }
+  return params;
+}
+
 // src/workers/inference-call.ts
 var INFERENCE_TIMEOUT_MS = 10 * 6e4;
 async function withTimeout(work, label) {
@@ -9363,6 +9383,39 @@ function boundedGenerateWithMetadata(client, prompt, maxTokens, temperature, opt
     `${client.type}:${client.modelId}`
   );
 }
+
+// src/workers/detection/detection-chunking.ts
+var SELECTOR_CONTEXT_CHARS = 64;
+var OVERLAP_CHARS = SELECTOR_CONTEXT_CHARS + // prefix
+SELECTOR_CONTEXT_CHARS + // suffix
+2 * SELECTOR_CONTEXT_CHARS;
+var OVERLAP_TOKENS = Math.ceil(OVERLAP_CHARS / 4);
+function deriveDetectionBudget(limits, scaffoldTokens) {
+  const { contextTokens, maxOutputTokens } = limits;
+  const available = contextTokens - scaffoldTokens;
+  let inputBudget;
+  let outputBudget;
+  if (maxOutputTokens >= contextTokens) {
+    inputBudget = Math.floor(available / 3);
+    outputBudget = available - inputBudget;
+  } else {
+    outputBudget = maxOutputTokens;
+    inputBudget = contextTokens - outputBudget - scaffoldTokens;
+    if (inputBudget <= 0) {
+      inputBudget = Math.floor(available / 3);
+      outputBudget = available - inputBudget;
+    }
+  }
+  if (inputBudget <= OVERLAP_TOKENS) {
+    throw new Error(
+      `Inference window too small for detection: context ${contextTokens} tokens minus scaffold ${scaffoldTokens} leaves an input budget of ${inputBudget} (need > ${OVERLAP_TOKENS}). Use a model with a larger context window or reduce the prompt scaffold.`
+    );
+  }
+  return {
+    chunking: { chunkSize: inputBudget, overlap: OVERLAP_TOKENS },
+    outputBudget
+  };
+}
 function languageName(tag) {
   return getLocaleEnglishName(tag) || tag;
 }
@@ -9382,7 +9435,9 @@ var MotivationPrompts = class {
   /**
    * Build a prompt for detecting comment-worthy passages
    *
-   * @param content - The text content to analyze (will be truncated to 8000 chars)
+   * @param content - The text content to analyze — a chunk sized by the
+   *   caller from derived provider limits; NEVER re-truncated here (a
+   *   builder-level clip is silent input loss — see #738)
    * @param instructions - Optional user-provided instructions
    * @param tone - Optional tone guidance (e.g., "academic", "conversational")
    * @param density - Optional target number of comments per 2000 words
@@ -9403,7 +9458,7 @@ ${instructions}${toneGuidance}${densityGuidance}${sourceLang}${bodyLang}
 
 Text to analyze:
 ---
-${content.substring(0, 8e3)}
+${content}
 ---
 
 Return a JSON array of comments. Each comment must have:
@@ -9437,7 +9492,7 @@ Guidelines:
 
 Text to analyze:
 ---
-${content.substring(0, 8e3)}
+${content}
 ---
 
 Return a JSON array of comments. Each comment should have:
@@ -9458,7 +9513,9 @@ Example format:
   /**
    * Build a prompt for detecting highlight-worthy passages
    *
-   * @param content - The text content to analyze (will be truncated to 8000 chars)
+   * @param content - The text content to analyze — a chunk sized by the
+   *   caller from derived provider limits; NEVER re-truncated here (a
+   *   builder-level clip is silent input loss — see #738)
    * @param instructions - Optional user-provided instructions
    * @param density - Optional target number of highlights per 2000 words
    * @returns Formatted prompt string
@@ -9476,7 +9533,7 @@ ${instructions}${densityGuidance}${sourceLang}
 
 Text to analyze:
 ---
-${content.substring(0, 8e3)}
+${content}
 ---
 
 Return a JSON array of highlights. Each highlight must have:
@@ -9507,7 +9564,7 @@ Guidelines:
 
 Text to analyze:
 ---
-${content.substring(0, 8e3)}
+${content}
 ---
 
 Return a JSON array of highlights. Each highlight should have:
@@ -9527,7 +9584,9 @@ Example format:
   /**
    * Build a prompt for detecting assessment-worthy passages
    *
-   * @param content - The text content to analyze (will be truncated to 8000 chars)
+   * @param content - The text content to analyze — a chunk sized by the
+   *   caller from derived provider limits; NEVER re-truncated here (a
+   *   builder-level clip is silent input loss — see #738)
    * @param instructions - Optional user-provided instructions
    * @param tone - Optional tone guidance (e.g., "critical", "supportive")
    * @param density - Optional target number of assessments per 2000 words
@@ -9548,7 +9607,7 @@ ${instructions}${toneGuidance}${densityGuidance}${sourceLang}${bodyLang}
 
 Text to analyze:
 ---
-${content.substring(0, 8e3)}
+${content}
 ---
 
 Return a JSON array of assessments. Each assessment must have:
@@ -9582,7 +9641,7 @@ Guidelines:
 
 Text to analyze:
 ---
-${content.substring(0, 8e3)}
+${content}
 ---
 
 Return a JSON array of assessments. Each assessment should have:
@@ -9828,10 +9887,32 @@ function logAnchorMethod(motivation, exact, anchorMethod) {
 }
 
 // src/workers/annotation-detection.ts
-function assertNotTruncated(response, motivation) {
+function assertNotTruncated(response, motivation, chunk, totalChunks, outputBudget) {
   if (response.stopReason === "max_tokens") {
-    throw new Error(`${motivation} detection response truncated (max_tokens) \u2014 increase max_tokens or reduce resource size; failing the job rather than under-reporting annotations.`);
+    throw new Error(`${motivation} detection response truncated (max_tokens) on chunk ${chunk}/${totalChunks} despite the derived output budget of ${outputBudget} tokens \u2014 failing the job rather than under-reporting annotations.`);
+  }
+}
+async function detectInChunks(client, content, buildPrompt, temperature, motivation, parse, onChunk) {
+  const limits = await client.limits();
+  const scaffoldTokens = estimateTokens(buildPrompt(""));
+  const { chunking, outputBudget } = deriveDetectionBudget(limits, scaffoldTokens);
+  const chunks = chunkText(content, chunking);
+  const collected = [];
+  for (let i = 0; i < chunks.length; i++) {
+    const response = await boundedGenerateWithMetadata(
+      client,
+      buildPrompt(chunks[i]),
+      outputBudget,
+      temperature,
+      { format: "json" }
+    );
+    assertNotTruncated(response, motivation, i + 1, chunks.length, outputBudget);
+    collected.push(...parse(response.text));
+    if (i < chunks.length - 1) {
+      onChunk?.(i + 1, chunks.length);
+    }
   }
+  return collected;
 }
 var AnnotationDetection = class {
   /**
@@ -9842,11 +9923,16 @@ var AnnotationDetection = class {
    * (source-resource locale). See `types.ts` "Locale conventions" for the
    * full discussion.
    */
-  static async detectComments(content, client, instructions, tone, density, language, sourceLanguage) {
-    const prompt = MotivationPrompts.buildCommentPrompt(content, instructions, tone, density, language, sourceLanguage);
-    const response = await boundedGenerateWithMetadata(client, prompt, 3e3, 0.4, { format: "json" });
-    assertNotTruncated(response, "comment");
-    return MotivationParsers.parseComments(response.text, content);
+  static async detectComments(content, client, instructions, tone, density, language, sourceLanguage, onChunk) {
+    return detectInChunks(
+      client,
+      content,
+      (chunk) => MotivationPrompts.buildCommentPrompt(chunk, instructions, tone, density, language, sourceLanguage),
+      0.4,
+      "comment",
+      (text) => MotivationParsers.parseComments(text, content),
+      onChunk
+    );
   }
   /**
    * Detect highlights in content.
@@ -9855,11 +9941,16 @@ var AnnotationDetection = class {
    * applies, used in the prompt so the LLM analyzes non-English source
    * correctly.
    */
-  static async detectHighlights(content, client, instructions, density, sourceLanguage) {
-    const prompt = MotivationPrompts.buildHighlightPrompt(content, instructions, density, sourceLanguage);
-    const response = await boundedGenerateWithMetadata(client, prompt, 2e3, 0.3, { format: "json" });
-    assertNotTruncated(response, "highlight");
-    return MotivationParsers.parseHighlights(response.text, content);
+  static async detectHighlights(content, client, instructions, density, sourceLanguage, onChunk) {
+    return detectInChunks(
+      client,
+      content,
+      (chunk) => MotivationPrompts.buildHighlightPrompt(chunk, instructions, density, sourceLanguage),
+      0.3,
+      "highlight",
+      (text) => MotivationParsers.parseHighlights(text, content),
+      onChunk
+    );
   }
   /**
    * Detect assessments in content.
@@ -9868,11 +9959,16 @@ var AnnotationDetection = class {
    * (annotation body locale). `sourceLanguage` is the locale of the content
    * being analyzed (source-resource locale).
    */
-  static async detectAssessments(content, client, instructions, tone, density, language, sourceLanguage) {
-    const prompt = MotivationPrompts.buildAssessmentPrompt(content, instructions, tone, density, language, sourceLanguage);
-    const response = await boundedGenerateWithMetadata(client, prompt, 3e3, 0.3, { format: "json" });
-    assertNotTruncated(response, "assessment");
-    return MotivationParsers.parseAssessments(response.text, content);
+  static async detectAssessments(content, client, instructions, tone, density, language, sourceLanguage, onChunk) {
+    return detectInChunks(
+      client,
+      content,
+      (chunk) => MotivationPrompts.buildAssessmentPrompt(chunk, instructions, tone, density, language, sourceLanguage),
+      0.3,
+      "assessment",
+      (text) => MotivationParsers.parseAssessments(text, content),
+      onChunk
+    );
   }
   /**
    * Detect tags in content for a specific category.
@@ -9886,28 +9982,33 @@ var AnnotationDetection = class {
    * identifiers, not LLM-generated text — so it's consumed at the body-stamp
    * site, not here.
    */
-  static async detectTags(content, client, schema, category, sourceLanguage) {
+  static async detectTags(content, client, schema, category, sourceLanguage, onChunk) {
     const categoryInfo = schema.tags.find((t) => t.name === category);
     if (!categoryInfo) {
       throw new Error(`Invalid category "${category}" for schema ${schema.id}`);
     }
-    const prompt = MotivationPrompts.buildTagPrompt(
+    const parsedTags = await detectInChunks(
+      client,
       content,
-      category,
-      schema.name,
-      schema.description,
-      schema.domain,
-      categoryInfo.description,
-      categoryInfo.examples,
-      sourceLanguage
+      (chunk) => MotivationPrompts.buildTagPrompt(
+        chunk,
+        category,
+        schema.name,
+        schema.description,
+        schema.domain,
+        categoryInfo.description,
+        categoryInfo.examples,
+        sourceLanguage
+      ),
+      0.2,
+      "tag",
+      (text) => MotivationParsers.parseTags(text),
+      onChunk
     );
-    const response = await boundedGenerateWithMetadata(client, prompt, 4e3, 0.2, { format: "json" });
-    assertNotTruncated(response, "tag");
-    const parsedTags = MotivationParsers.parseTags(response.text);
     return MotivationParsers.validateTagOffsets(parsedTags, content, category);
   }
 };
-async function extractEntities(exact, entityTypes, client, includeDescriptiveReferences, logger2, sourceLanguage) {
+async function extractEntities(exact, entityTypes, client, includeDescriptiveReferences, logger2, sourceLanguage, onChunk) {
   const entityTypesDescription = entityTypes.map((et) => {
     if (typeof et === "string") {
       return et;
@@ -9939,11 +10040,11 @@ Find direct mentions only (names, proper nouns). Do not include pronouns or desc
   const sourceLangGuidance = sourceLanguage ? `
 Source text language: ${getLocaleEnglishName(sourceLanguage) || sourceLanguage}.
 ` : "";
-  const prompt = `Identify entity references in the following text. Look for mentions of: ${entityTypesDescription}.
+  const buildPrompt = (text) => `Identify entity references in the following text. Look for mentions of: ${entityTypesDescription}.
 ${descriptiveReferenceGuidance}${sourceLangGuidance}
 Text to analyze:
 """
-${exact}
+${text}
 """
 
 Respond with a JSON array of entities found. Each entity should have:
@@ -9956,59 +10057,78 @@ If no entities are found, respond with an empty array [].
 
 Example output:
 [{"exact":"Alice","entityType":"Person","prefix":"","suffix":" went to"},{"exact":"Paris","entityType":"Location","prefix":"went to ","suffix":" yesterday"}]`;
-  logger2.debug("Sending entity extraction request", { entityTypes: entityTypesDescription });
-  const response = await boundedGenerateWithMetadata(
-    client,
-    prompt,
-    4e3,
-    // Increased to handle many entities without truncation
-    0.3,
-    // Lower temperature for more consistent extraction
-    // Force grammar-constrained JSON output. Without this, Ollama models
-    // periodically emit malformed JSON (truncated brackets, mid-token
-    // breaks at higher token counts) which silently parse-fails into
-    // [] downstream. The prompt's schema (which keys, what types) still
-    // governs *what* the JSON contains; `format: 'json'` governs that
-    // it's syntactically valid.
-    { format: "json" }
-  );
-  logger2.debug("Got entity extraction response", { responseLength: response.text.length });
-  if (response.stopReason === "max_tokens") {
-    const errorMsg = "Entity extraction response truncated (max_tokens) \u2014 increase max_tokens or reduce resource size; failing the job rather than dropping annotations.";
-    logger2.error(errorMsg, { responseLength: response.text.length });
-    throw new Error(errorMsg);
-  }
-  let entities;
-  try {
-    entities = JSON.parse(response.text.trim());
-  } catch (error) {
-    logger2.error("Failed to parse entity extraction response", {
-      error: error instanceof Error ? error.message : String(error),
-      response: response.text.slice(0, 500)
-    });
-    throw new Error("Failed to parse entity extraction response", {
-      cause: error instanceof Error ? error : new Error(String(error))
+  const limits = await client.limits();
+  const scaffoldTokens = estimateTokens(buildPrompt(""));
+  const { chunking, outputBudget } = deriveDetectionBudget(limits, scaffoldTokens);
+  const chunks = chunkText(exact, chunking);
+  logger2.debug("Sending entity extraction request", {
+    entityTypes: entityTypesDescription,
+    chunks: chunks.length,
+    chunkSizeTokens: chunking.chunkSize,
+    outputBudget
+  });
+  const collected = [];
+  for (let i = 0; i < chunks.length; i++) {
+    const response = await boundedGenerateWithMetadata(
+      client,
+      buildPrompt(chunks[i]),
+      outputBudget,
+      0.3,
+      // Lower temperature for more consistent extraction
+      // Force grammar-constrained JSON output. Without this, Ollama models
+      // periodically emit malformed JSON (truncated brackets, mid-token
+      // breaks at higher token counts) which silently parse-fails into
+      // [] downstream. The prompt's schema (which keys, what types) still
+      // governs *what* the JSON contains; `format: 'json'` governs that
+      // it's syntactically valid.
+      { format: "json" }
+    );
+    logger2.debug("Got entity extraction response", {
+      chunk: i + 1,
+      chunks: chunks.length,
+      responseLength: response.text.length
     });
+    if (response.stopReason === "max_tokens") {
+      const errorMsg = `Entity extraction response truncated (max_tokens) on chunk ${i + 1}/${chunks.length} despite the derived output budget of ${outputBudget} tokens \u2014 failing the job rather than dropping annotations.`;
+      logger2.error(errorMsg, { responseLength: response.text.length });
+      throw new Error(errorMsg);
+    }
+    let entities;
+    try {
+      entities = JSON.parse(response.text.trim());
+    } catch (error) {
+      logger2.error("Failed to parse entity extraction response", {
+        error: error instanceof Error ? error.message : String(error),
+        response: response.text.slice(0, 500)
+      });
+      throw new Error("Failed to parse entity extraction response", {
+        cause: error instanceof Error ? error : new Error(String(error))
+      });
+    }
+    if (!isArray(entities)) {
+      logger2.error("Failed to parse entity extraction response: expected a JSON array", {
+        response: response.text.slice(0, 500)
+      });
+      throw new Error("Failed to parse entity extraction response: expected a JSON array");
+    }
+    logger2.debug("Parsed entities from AI response", { chunk: i + 1, count: entities.length });
+    for (const e of entities) {
+      if (isObject(e) && isString(e.exact) && isString(e.entityType)) {
+        collected.push({
+          exact: e.exact,
+          entityType: e.entityType,
+          ...isString(e.prefix) ? { prefix: e.prefix } : {},
+          ...isString(e.suffix) ? { suffix: e.suffix } : {}
+        });
+      } else {
+        logger2.debug("Dropped malformed LLM entity", { entity: e });
+      }
+    }
+    if (i < chunks.length - 1) {
+      onChunk?.(i + 1, chunks.length);
+    }
   }
-  if (!isArray(entities)) {
-    logger2.error("Failed to parse entity extraction response: expected a JSON array", {
-      response: response.text.slice(0, 500)
-    });
-    throw new Error("Failed to parse entity extraction response: expected a JSON array");
-  }
-  logger2.debug("Parsed entities from AI response", { count: entities.length });
-  return entities.filter((e) => {
-    const ok = isObject(e) && isString(e.exact) && isString(e.entityType);
-    if (!ok) {
-      logger2.debug("Dropped malformed LLM entity", { entity: e });
-    }
-    return ok;
-  }).map((entity) => ({
-    exact: entity.exact,
-    entityType: entity.entityType,
-    ...isString(entity.prefix) ? { prefix: entity.prefix } : {},
-    ...isString(entity.suffix) ? { suffix: entity.suffix } : {}
-  }));
+  return collected;
 }
 function getLanguageName(locale) {
   return getLocaleEnglishName(locale) || locale;
@@ -10019,7 +10139,7 @@ var SEMANTIC_MATCH_CHARS = 240;
 function idLabel(resourceId, annotationId) {
   return `[${resourceId}${annotationId ? `/${annotationId}` : ""}]`;
 }
-async function generateResourceFromTopic(topic, entityTypes, client, logger2, userPrompt, locale, context, temperature, maxTokens, sourceLanguage, outputMediaType = "text/markdown", task = "resource", structure, cite = false) {
+async function generateResourceFromTopic(topic, entityTypes, client, logger2, userPrompt, locale, context, temperature, maxTokens, sourceLanguage, outputMediaType = "text/markdown", task = "resource", structure, cite = false, repair) {
   logger2.debug("Generating resource from topic", {
     topicPreview: topic.substring(0, 100),
     entityTypes,
@@ -10135,11 +10255,12 @@ ${parts.join("\n")}`;
   let semanticContextSection = "";
   const similar = context?.semanticContext?.similar ?? [];
   if (similar.length > 0) {
-    const lines = [...similar].sort((a, b) => b.score - a.score).slice(0, SEMANTIC_MATCH_LIMIT).map((m) => `- ${idLabel(m.resourceId, m.annotationId)} (${m.score.toFixed(2)}) ${m.text.slice(0, SEMANTIC_MATCH_CHARS)}`);
+    const lines = [...similar].sort((a, b) => b.score - a.score).slice(0, SEMANTIC_MATCH_LIMIT).map((m) => `- ${idLabel(m.resourceId, m.annotationId)} (${m.score.toFixed(2)})${m.machineRead ? " [OCR]" : ""} ${m.text.slice(0, SEMANTIC_MATCH_CHARS)}`);
+    const ocrNote = similar.some((m) => m.machineRead) ? "\nPassages marked [OCR] were read from scanned images by character recognition; treat their exact wording and numbers as uncertain, and say so if you rely on one." : "";
     semanticContextSection = `
 
 Related passages from the knowledge base:
-${lines.join("\n")}`;
+${lines.join("\n")}${ocrNote}`;
   }
   let leadLine;
   if (task === "resource") {
@@ -10154,11 +10275,12 @@ ${lines.join("\n")}`;
 Topic: "${topic}"`;
   }
   const isPlainText = outputMediaType === "text/plain";
+  const isPdf = outputMediaType === "application/pdf";
   let structureRequirement = "";
   let titleRequirement = "";
   if (structure === "sections") {
-    structureRequirement = isPlainText ? "\n- Organize the content into titled sections with well-structured paragraphs" : "\n- Organize the content into titled sections (## Section) with well-structured paragraphs";
-    if (!isPlainText) {
+    structureRequirement = isPdf ? "\n- Organize the content into titled sections (= Heading) with well-structured paragraphs" : isPlainText ? "\n- Organize the content into titled sections with well-structured paragraphs" : "\n- Organize the content into titled sections (## Section) with well-structured paragraphs";
+    if (!isPlainText && !isPdf) {
       titleRequirement = "\n- Start with a clear heading (# Title)";
     }
   } else if (structure === "prose") {
@@ -10171,11 +10293,20 @@ Topic: "${topic}"`;
 - Organize the output as: ${structure}`;
   }
   const citeRequirement = cite ? "\n- Ground every claim in the provided context. Immediately after each claim, cite its source by emitting [[<id>]], where <id> is an id shown in square brackets in the context above (for a passage labeled [abc], emit [[abc]]). Cite only ids that appear in the context." : "";
-  const formatRequirements = isPlainText ? `- Write the response as plain text \u2014 no formatting markup (no #, *, backticks, headings, or links)
+  const formatRequirements = isPdf ? `- Write the response as Typst markup (the Typst typesetting language \u2014 not markdown, not LaTeX)
+- Headings are written as = Heading (deeper levels == Subheading); everything else is plain prose paragraphs
+- Do not emit markdown syntax or code fences` : isPlainText ? `- Write the response as plain text \u2014 no formatting markup (no #, *, backticks, headings, or links)
 - Begin with the title on its own first line` : `- Use markdown formatting
 - Write the response as markdown`;
+  const repairSection = repair ? `
+
+Your previous attempt failed to compile. Fix the error and return the complete corrected document \u2014 full source, not a diff.
+Compile error:
+${repair.error}
+Previous source:
+${repair.source}` : "";
   const prompt = `${leadLine}
-${userPrompt ? `Instruction: ${userPrompt}` : ""}
+${userPrompt ? `Instruction: ${userPrompt}` : ""}${repairSection}
 ${entityTypes.length > 0 ? `Focus on these entity types: ${entityTypes.join(", ")}.` : ""}${annotationSection}${contextSection}${resourceSection}${graphSection}${semanticContextSection}${sourceLanguageInstruction}${languageInstruction}
 
 Requirements:
@@ -10184,7 +10315,7 @@ Requirements:
 ${formatRequirements}`;
   const parseResponse = (response2) => {
     let content = response2.trim();
-    if (content.startsWith("```markdown") || content.startsWith("```md")) {
+    if (content.startsWith("```markdown") || content.startsWith("```md") || content.startsWith("```typst")) {
       content = content.slice(content.indexOf("\n") + 1);
       const endIndex = content.lastIndexOf("```");
       if (endIndex !== -1) {
@@ -10219,6 +10350,29 @@ ${formatRequirements}`;
   });
   return result;
 }
+var PINNED_CREATION_TIMESTAMP = 17e8;
+var MAX_COMPILE_REPAIRS = 2;
+function compileTypst(source) {
+  const dir = mkdtempSync(join(tmpdir(), "typst-"));
+  try {
+    const inFile = join(dir, "doc.typ");
+    const outFile = join(dir, "doc.pdf");
+    writeFileSync(inFile, source);
+    try {
+      execFileSync(
+        "typst",
+        ["compile", "--creation-timestamp", String(PINNED_CREATION_TIMESTAMP), inFile, outFile],
+        { stdio: ["ignore", "pipe", "pipe"] }
+      );
+    } catch (err) {
+      const stderr = err.stderr;
+      return { error: stderr?.length ? stderr.toString("utf8") : String(err) };
+    }
+    return { pdf: new Uint8Array(readFileSync(outFile)) };
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+}
 
 // src/workers/generation/citation-resolver.ts
 var CITATION_TOKEN = /\[\[([^\s[\]/]+)(?:\/([^\s[\]/]+))?\]\]/g;
@@ -10363,9 +10517,9 @@ function buildTextAnnotation(content, resourceId, userId, generator, motivation,
     ...body !== void 0 ? { body } : {}
   };
 }
-function buildPdfAnnotation(layer, resourceId, userId, generator, motivation, match, body) {
-  const { rects, overlap } = locate(layer, match.start, match.end);
-  const coveredText = overlap.length ? layer.text.substring(
+function buildPdfAnnotation(anchored, resourceId, userId, generator, motivation, match, body) {
+  const { rects, overlap } = locate(anchored, match.start, match.end);
+  const coveredText = overlap.length ? anchored.text.substring(
     Math.min(...overlap.map((i) => i.start)),
     Math.max(...overlap.map((i) => i.end))
   ) : "";
@@ -10414,7 +10568,9 @@ async function processHighlightJob(content, inferenceClient, params, buildAnnota
     inferenceClient,
     params.instructions,
     params.density,
-    params.sourceLanguage
+    params.sourceLanguage,
+    // Chunk-boundary heartbeat (liveness): interpolate within the 30–60 band.
+    (completed, total) => onProgress(30 + Math.round(completed / total * 30), "Analyzing text...", "analyzing")
   );
   onProgress(60, `Creating ${highlights.length} annotations...`, "creating");
   const annotations = dedupeAnnotations(highlights.map(
@@ -10436,7 +10592,9 @@ async function processCommentJob(content, inferenceClient, params, buildAnnotati
     params.tone,
     params.density,
     params.language,
-    params.sourceLanguage
+    params.sourceLanguage,
+    // Chunk-boundary heartbeat (liveness): interpolate within the 30–60 band.
+    (completed, total) => onProgress(30 + Math.round(completed / total * 30), "Analyzing text...", "analyzing")
   );
   onProgress(60, `Creating ${comments.length} annotations...`, "creating");
   const bodyLanguage = params.language ?? "en";
@@ -10466,7 +10624,9 @@ async function processAssessmentJob(content, inferenceClient, params, buildAnnot
     params.tone,
     params.density,
     params.language,
-    params.sourceLanguage
+    params.sourceLanguage,
+    // Chunk-boundary heartbeat (liveness): interpolate within the 30–60 band.
+    (completed, total) => onProgress(30 + Math.round(completed / total * 30), "Analyzing text...", "analyzing")
   );
   onProgress(60, `Creating ${assessments.length} annotations...`, "creating");
   const bodyLanguage = params.language ?? "en";
@@ -10522,7 +10682,23 @@ async function processReferenceJob(content, inferenceClient, params, buildAnnota
       inferenceClient,
       params.includeDescriptiveReferences ?? false,
       logger2,
-      params.sourceLanguage
+      params.sourceLanguage,
+      // Chunk-boundary heartbeat: progress is the worker's liveness signal
+      // (stall watchdog + backend janitor), so multi-chunk extraction must
+      // emit between inference calls. Percentage interpolates within this
+      // entity type's band of the 20–80 range.
+      (completed, total) => {
+        const interpolated = 20 + Math.round((i + completed / total) / entityTypeNames.length * 60);
+        onProgress(interpolated, `Detecting ${entityTypeName} entities...`, "analyzing", {
+          currentEntityType: entityTypeName,
+          processedEntityTypes: i,
+          totalEntityTypes: entityTypeNames.length,
+          entitiesFound: totalFound,
+          entitiesEmitted: totalEmitted,
+          completedEntityTypes: [...completedEntityTypes],
+          requestParams
+        });
+      }
     );
     totalFound += extractedEntities.length;
     completedEntityTypes.push({ entityType: entityTypeName, foundCount: extractedEntities.length });
@@ -10566,13 +10742,21 @@ async function processTagJob(content, inferenceClient, params, buildAnnotation,
   onProgress(10, "Loading resource...", "analyzing");
   onProgress(30, "Analyzing text for tags...", "analyzing");
   const allTags = [];
-  for (const category of params.categories) {
+  for (let c = 0; c < params.categories.length; c++) {
+    const category = params.categories[c];
     const categoryTags = await AnnotationDetection.detectTags(
       content,
       inferenceClient,
       params.schema,
       category,
-      params.sourceLanguage
+      params.sourceLanguage,
+      // Chunk-boundary heartbeat (liveness): interpolate within this
+      // category's slice of the 30–60 band.
+      (completed, total) => onProgress(
+        30 + Math.round((c + completed / total) / params.categories.length * 30),
+        "Analyzing text for tags...",
+        "analyzing"
+      )
     );
     allTags.push(...categoryTags);
   }
@@ -10598,8 +10782,14 @@ async function processTagJob(content, inferenceClient, params, buildAnnotation,
     result: { tagsFound: tags.length, tagsCreated: annotations.length, byCategory }
   };
 }
+function assertWithinOutputBudget(byteLength) {
+  if (!withinByteBudget(byteLength)) {
+    throw new Error(
+      `Generated artifact exceeds the output byte budget: ${byteLength} bytes > ${MAX_PDF_BYTES}. Refusing a runaway generation.`
+    );
+  }
+}
 async function processGenerationJob(inferenceClient, params, onProgress, logger2) {
-  const GENERATABLE_MEDIA_TYPES = ["text/markdown", "text/plain"];
   const outputMediaType = params.outputMediaType ?? "text/markdown";
   if (!GENERATABLE_MEDIA_TYPES.includes(outputMediaType)) {
     throw new Error(
@@ -10608,6 +10798,84 @@ async function processGenerationJob(inferenceClient, params, onProgress, logger2
   }
   const title = params.title ?? "Untitled";
   const entityTypes = (params.entityTypes ?? []).map(String);
+  if (outputMediaType === "application/pdf") {
+    onProgress(5, "Generating resource...", "generating");
+    const validIds = params.cite === true ? collectContextResourceIds(params.context) : null;
+    let generated2 = await generateResourceFromTopic(
+      title,
+      entityTypes,
+      inferenceClient,
+      logger2,
+      params.prompt,
+      params.language,
+      params.context,
+      params.temperature,
+      params.maxTokens,
+      params.sourceLanguage,
+      outputMediaType,
+      params.task,
+      params.structure,
+      params.cite
+    );
+    let source = generated2.content;
+    let citations2 = [];
+    if (validIds) {
+      const resolved = resolveCitationTokens(generated2.content, validIds, logger2);
+      source = resolved.content;
+      citations2 = resolved.citations;
+    }
+    let compiled = compileTypst(source);
+    let repairs = 0;
+    while ("error" in compiled && repairs < MAX_COMPILE_REPAIRS) {
+      repairs++;
+      logger2.warn("Typst compile failed \u2014 feeding the error back for repair", {
+        attempt: repairs,
+        error: compiled.error.slice(0, 500)
+      });
+      generated2 = await generateResourceFromTopic(
+        title,
+        entityTypes,
+        inferenceClient,
+        logger2,
+        params.prompt,
+        params.language,
+        params.context,
+        params.temperature,
+        params.maxTokens,
+        params.sourceLanguage,
+        outputMediaType,
+        params.task,
+        params.structure,
+        params.cite,
+        { source, error: compiled.error }
+      );
+      if (validIds) {
+        const resolved = resolveCitationTokens(generated2.content, validIds, logger2);
+        source = resolved.content;
+        citations2 = resolved.citations;
+      } else {
+        source = generated2.content;
+      }
+      compiled = compileTypst(source);
+    }
+    if ("error" in compiled) {
+      throw new Error(
+        `Typst compilation failed after ${MAX_COMPILE_REPAIRS} repair attempts: ${compiled.error}`
+      );
+    }
+    assertWithinOutputBudget(compiled.pdf.byteLength);
+    onProgress(95, "Creating resource...", "creating");
+    return {
+      content: compiled.pdf,
+      title: generated2.title ?? title,
+      format: outputMediaType,
+      citations: citations2,
+      result: {
+        resourceId: "",
+        resourceName: generated2.title ?? title
+      }
+    };
+  }
   onProgress(5, "Generating resource...", "generating");
   const generated = await generateResourceFromTopic(
     title,
@@ -10633,8 +10901,10 @@ async function processGenerationJob(inferenceClient, params, onProgress, logger2
     citations = resolved.citations;
   }
   onProgress(95, "Creating resource...", "creating");
+  const artifact = new TextEncoder().encode(content);
+  assertWithinOutputBudget(artifact.byteLength);
   return {
-    content,
+    content: artifact,
     title: generated.title ?? title,
     format: outputMediaType,
     citations,
@@ -10646,28 +10916,37 @@ async function processGenerationJob(inferenceClient, params, onProgress, logger2
 }
 
 // src/workers/detection/prepare-detection.ts
-async function prepareDetection(strategy, session, resourceId, userId, generator) {
-  switch (strategy) {
-    case "decode": {
-      const text = await session.client.browse.resourceContent(resourceId);
-      return {
-        text,
-        buildAnnotation: (motivation, match, body) => buildTextAnnotation(text, resourceId, userId, generator, motivation, match, body)
-      };
-    }
-    case "pdf-text-layer": {
-      const { data } = await session.client.browse.resourceRepresentation(resourceId);
-      const layer = await extractPdfTextLayer(new Uint8Array(data));
-      if (!layer) return null;
-      return {
-        text: layer.text,
-        buildAnnotation: (motivation, match, body) => buildPdfAnnotation(layer, resourceId, userId, generator, motivation, match, body)
-      };
-    }
-    case "none":
-      return null;
+async function prepareDetection(mediaType, session, resourceId, userId, generator, store) {
+  const extractor = EXTRACTORS[textExtractionOf(mediaType)];
+  if (!extractor) return { declined: "no-extractor" };
+  const { data } = await session.client.browse.resourceRepresentation(resourceId);
+  const bytes = Buffer.from(data);
+  const extracted = await extractor.extract(bytes, mediaType, {
+    key: calculateChecksum(bytes),
+    store
+  });
+  if ("declined" in extracted) return extracted;
+  if (!extracted.text.trim()) return { declined: "empty" };
+  const items = extracted.items;
+  if (items && items.length > 0) {
+    const anchored = { text: extracted.text, items };
+    return {
+      text: extracted.text,
+      buildAnnotation: (motivation, match, body) => buildPdfAnnotation(anchored, resourceId, userId, generator, motivation, match, body)
+    };
   }
+  return {
+    text: extracted.text,
+    buildAnnotation: (motivation, match, body) => buildTextAnnotation(extracted.text, resourceId, userId, generator, motivation, match, body)
+  };
 }
+var DECLINE_MESSAGES = {
+  "no-text-layer": "This PDF is a scan whose text could not be recognized; there is nothing to detect over.",
+  "encrypted": "This PDF is password-protected, so its text cannot be read.",
+  "corrupt": "This PDF could not be parsed \u2014 the file may be damaged or truncated.",
+  "too-large": "This document is too large to extract text from.",
+  "empty": "This document contains no text to detect over."
+};
 async function emitEvent(session, channel, payload) {
   await session.client.transport.emit(channel, payload);
 }
@@ -10685,15 +10964,16 @@ function startWorkerProcess(config) {
       const message = error instanceof Error ? error.message : String(error);
       logger2.error("Job failed", { jobId: job.jobId, error: message, stack: error instanceof Error ? error.stack : void 0 });
       const failAnnotationId = job.params.referenceId;
-      emitEvent(session, "job:fail", {
-        resourceId: job.resourceId,
-        userId: job.userId,
-        jobId: job.jobId,
-        jobType: job.type,
-        ...failAnnotationId ? { annotationId: failAnnotationId } : {},
-        error: message
-      }).catch(() => {
-      });
+      if (isJobType(job.type)) {
+        emitEvent(session, "job:fail", {
+          resourceId: job.resourceId,
+          jobId: job.jobId,
+          jobType: job.type,
+          ...failAnnotationId ? { annotationId: failAnnotationId } : {},
+          error: message
+        }).catch(() => {
+        });
+      }
       adapter.failJob(job.jobId, message);
     });
   });
@@ -10725,11 +11005,16 @@ async function handleJob(adapter, config, job) {
 }
 async function handleJobInner(adapter, config, job) {
   const { session, inferenceClient, generator } = config;
-  const { resourceId, userId, jobId, type: jobType } = job;
+  const { userId, jobId } = job;
+  if (!isJobType(job.type)) {
+    adapter.failJob(jobId, `Unrecognized job type: ${job.type}`);
+    return;
+  }
+  const jobType = job.type;
+  const resourceId$1 = resourceId(job.resourceId);
   const annotationId = job.params.referenceId;
   const lifecycleBase = {
-    resourceId,
-    userId,
+    resourceId: resourceId$1,
     jobId,
     jobType,
     ...annotationId ? { annotationId } : {}
@@ -10739,23 +11024,27 @@ async function handleJobInner(adapter, config, job) {
     adapter.failJob(jobId, `Worker not configured for job type: ${jobType}`);
     return;
   }
-  let source = null;
+  let ready = null;
   if (jobType !== "generation") {
-    const descriptor = await session.client.browse.resource(resourceId).fresh();
+    const descriptor = await session.client.browse.resource(resourceId$1).fresh();
     const mediaType = getPrimaryMediaType(descriptor);
-    const strategy = mediaType ? textExtractionOf(mediaType) : "none";
-    if (strategy === "none") {
-      throw new Error(`Cannot run ${jobType} on resource ${resourceId}: media type '${mediaType ?? "unknown"}' has no extractable text to analyze`);
-    }
-    source = await prepareDetection(strategy, session, resourceId, userId, generator);
-    if (!source) {
+    const source = await prepareDetection(mediaType ?? "", session, resourceId$1, userId, generator, config.anchoredTextStore);
+    if ("declined" in source) {
+      if (source.declined === "no-extractor") {
+        throw new Error(`Cannot run ${jobType} on resource ${resourceId$1}: media type '${mediaType ?? "unknown"}' has no extractable text to analyze`);
+      }
       await emitEvent(session, "job:complete", {
         ...lifecycleBase,
-        result: { declined: true, reason: "no-text-layer", message: "This PDF has no extractable text layer (scanned or image-only); detection is not supported." }
+        result: {
+          declined: true,
+          reason: source.declined,
+          message: DECLINE_MESSAGES[source.declined]
+        }
       });
       adapter.completeJob();
       return;
     }
+    ready = source;
   }
   const onProgress = (percentage, message, stage, extra) => {
     adapter.touchActivity();
@@ -10774,14 +11063,14 @@ async function handleJobInner(adapter, config, job) {
   };
   if (jobType === "highlight-annotation") {
     const { annotations, result } = await processHighlightJob(
-      source.text,
+      ready.text,
       inferenceClient,
-      job.params,
-      source.buildAnnotation,
+      asJobParams(job.params),
+      ready.buildAnnotation,
       onProgress
     );
     for (const ann of annotations) {
-      await emitEvent(session, "mark:create", { annotation: ann, userId, resourceId });
+      await emitEvent(session, "mark:create", { annotation: ann, resourceId: resourceId$1 });
     }
     await emitEvent(session, "job:complete", {
       ...lifecycleBase,
@@ -10790,14 +11079,14 @@ async function handleJobInner(adapter, config, job) {
     adapter.completeJob();
   } else if (jobType === "comment-annotation") {
     const { annotations, result } = await processCommentJob(
-      source.text,
+      ready.text,
       inferenceClient,
-      job.params,
-      source.buildAnnotation,
+      asJobParams(job.params),
+      ready.buildAnnotation,
       onProgress
     );
     for (const ann of annotations) {
-      await emitEvent(session, "mark:create", { annotation: ann, userId, resourceId });
+      await emitEvent(session, "mark:create", { annotation: ann, resourceId: resourceId$1 });
     }
     await emitEvent(session, "job:complete", {
       ...lifecycleBase,
@@ -10806,14 +11095,14 @@ async function handleJobInner(adapter, config, job) {
     adapter.completeJob();
   } else if (jobType === "assessment-annotation") {
     const { annotations, result } = await processAssessmentJob(
-      source.text,
+      ready.text,
       inferenceClient,
-      job.params,
-      source.buildAnnotation,
+      asJobParams(job.params),
+      ready.buildAnnotation,
       onProgress
     );
     for (const ann of annotations) {
-      await emitEvent(session, "mark:create", { annotation: ann, userId, resourceId });
+      await emitEvent(session, "mark:create", { annotation: ann, resourceId: resourceId$1 });
     }
     await emitEvent(session, "job:complete", {
       ...lifecycleBase,
@@ -10822,15 +11111,15 @@ async function handleJobInner(adapter, config, job) {
     adapter.completeJob();
   } else if (jobType === "reference-annotation") {
     const { annotations, result } = await processReferenceJob(
-      source.text,
+      ready.text,
       inferenceClient,
-      job.params,
-      source.buildAnnotation,
+      asJobParams(job.params),
+      ready.buildAnnotation,
       onProgress,
       config.logger
     );
     for (const ann of annotations) {
-      await emitEvent(session, "mark:create", { annotation: ann, userId, resourceId });
+      await emitEvent(session, "mark:create", { annotation: ann, resourceId: resourceId$1 });
     }
     await emitEvent(session, "job:complete", {
       ...lifecycleBase,
@@ -10839,14 +11128,14 @@ async function handleJobInner(adapter, config, job) {
     adapter.completeJob();
   } else if (jobType === "tag-annotation") {
     const { annotations, result } = await processTagJob(
-      source.text,
+      ready.text,
       inferenceClient,
-      job.params,
-      source.buildAnnotation,
+      asJobParams(job.params),
+      ready.buildAnnotation,
       onProgress
     );
     for (const ann of annotations) {
-      await emitEvent(session, "mark:create", { annotation: ann, userId, resourceId });
+      await emitEvent(session, "mark:create", { annotation: ann, resourceId: resourceId$1 });
     }
     await emitEvent(session, "job:complete", {
       ...lifecycleBase,
@@ -10867,7 +11156,7 @@ async function handleJobInner(adapter, config, job) {
       file: Buffer.from(genResult.content),
       format: genResult.format,
       storageUri,
-      sourceResourceId: resourceId,
+      sourceResourceId: resourceId$1,
       ...genParams.referenceId ? { sourceAnnotationId: genParams.referenceId } : {},
       ...genParams.prompt ? { generationPrompt: genParams.prompt } : {},
       ...genParams.language ? { language: genParams.language } : {},
@@ -10878,29 +11167,63 @@ async function handleJobInner(adapter, config, job) {
       const { annotation: provenanceRef } = assembleAnnotation(
         {
           motivation: "linking",
-          target: { source: String(resourceId) },
+          target: { source: String(resourceId$1) },
           body: { type: "SpecificResource", source: String(newResourceId), purpose: "linking" }
         },
         generator
       );
-      await emitEvent(session, "mark:create", { annotation: provenanceRef, userId, resourceId });
-    }
-    for (const citation of genResult.citations) {
-      const { annotation: citationRef } = assembleAnnotation(
-        {
-          motivation: "linking",
-          target: {
-            source: String(newResourceId),
-            selector: [
-              { type: "TextPositionSelector", start: citation.start, end: citation.end },
-              { type: "TextQuoteSelector", exact: citation.exact }
-            ]
+      await emitEvent(session, "mark:create", { annotation: provenanceRef, resourceId: resourceId$1 });
+    }
+    if (genResult.format === "application/pdf" && genResult.citations.length > 0) {
+      const layer = await extractPdfTextLayer(genResult.content);
+      if (!layer) {
+        config.logger.warn("PDF citations dropped \u2014 the generated artifact yielded no text layer", {
+          jobId,
+          resourceId: newResourceId,
+          citations: genResult.citations.length
+        });
+      } else {
+        for (const citation of genResult.citations) {
+          const span = findClaimSpan(layer, citation.exact);
+          if (!span) {
+            config.logger.warn("PDF citation dropped \u2014 claim not found in the rendered text layer", {
+              jobId,
+              resourceId: newResourceId,
+              citedResourceId: citation.resourceId,
+              exactPreview: citation.exact.slice(0, 80)
+            });
+            continue;
+          }
+          const citationRef = buildPdfAnnotation(
+            layer,
+            resourceId(String(newResourceId)),
+            userId,
+            generator,
+            "linking",
+            { exact: layer.text.slice(span.start, span.end), start: span.start, end: span.end },
+            { type: "SpecificResource", source: citation.resourceId, purpose: "linking" }
+          );
+          await emitEvent(session, "mark:create", { annotation: citationRef, resourceId: newResourceId });
+        }
+      }
+    } else {
+      for (const citation of genResult.citations) {
+        const { annotation: citationRef } = assembleAnnotation(
+          {
+            motivation: "linking",
+            target: {
+              source: String(newResourceId),
+              selector: [
+                { type: "TextPositionSelector", start: citation.start, end: citation.end },
+                { type: "TextQuoteSelector", exact: citation.exact }
+              ]
+            },
+            body: { type: "SpecificResource", source: citation.resourceId, purpose: "linking" }
           },
-          body: { type: "SpecificResource", source: citation.resourceId, purpose: "linking" }
-        },
-        generator
-      );
-      await emitEvent(session, "mark:create", { annotation: citationRef, userId, resourceId: newResourceId });
+          generator
+        );
+        await emitEvent(session, "mark:create", { annotation: citationRef, resourceId: newResourceId });
+      }
     }
     await emitEvent(session, "job:complete", {
       ...lifecycleBase,
@@ -11052,6 +11375,10 @@ async function startAgentWorker(opts) {
     jobTypes: group.jobTypes,
     inferenceClient: group.client,
     generator,
+    // The extraction seam's cache, over this worker's content transport
+    // (PERSIST-ANCHORS P2d). Built here because the client keeps its
+    // content transport private — this is where it is in hand.
+    anchoredTextStore: anchoredTextStoreOverTransport(content, logger2),
     logger: logger2
   });
   logger2.info("Agent ready", {