@semiont/jobs 0.5.24 → 0.5.26

package/dist/index.js

@@ -1,8 +1,11 @@
-import { promises } from 'fs';
+import { promises, mkdtempSync, writeFileSync, readFileSync, rmSync } from 'fs';
 import * as path from 'path';
-import { jobId, deriveViews, reconcileSelector, isObject, isString, getLocaleEnglishName, isArray } from '@semiont/core';
+import { join } from 'path';
+import { jobId, deriveViews, reconcileSelector, GENERATABLE_MEDIA_TYPES, estimateTokens, chunkText, isObject, isString, getLocaleEnglishName } from '@semiont/core';
+import { execFileSync } from 'child_process';
+import { tmpdir } from 'os';
+import { withinByteBudget, MAX_PDF_BYTES } from '@semiont/content';
 import '@semiont/event-sourcing';
-import '@semiont/content';
 import '@semiont/observability';
 import '@semiont/sdk';
 import '@semiont/http-transport';
@@ -446,18 +449,51 @@ async function withTimeout(work, label) {
     clearTimeout(timer);
   }
 }
-function boundedGenerate(client, prompt, maxTokens, temperature, options) {
+function boundedGenerate(client, prompt, maxTokens, temperature) {
   return withTimeout(
-    client.generateText(prompt, maxTokens, temperature, options),
+    client.generateText(prompt, maxTokens, temperature),
     `${client.type}:${client.modelId}`
   );
 }
-function boundedGenerateWithMetadata(client, prompt, maxTokens, temperature, options) {
+function boundedGenerateStructured(client, prompt, maxTokens, temperature, elementSchema) {
   return withTimeout(
-    client.generateTextWithMetadata(prompt, maxTokens, temperature, options),
+    client.generateStructured(prompt, maxTokens, temperature, elementSchema),
     `${client.type}:${client.modelId}`
   );
 }
+
+// src/workers/detection/detection-chunking.ts
+var SELECTOR_CONTEXT_CHARS = 64;
+var OVERLAP_CHARS = SELECTOR_CONTEXT_CHARS + // prefix
+SELECTOR_CONTEXT_CHARS + // suffix
+2 * SELECTOR_CONTEXT_CHARS;
+var OVERLAP_TOKENS = Math.ceil(OVERLAP_CHARS / 4);
+function deriveDetectionBudget(limits, scaffoldTokens) {
+  const { contextTokens, maxOutputTokens } = limits;
+  const available = contextTokens - scaffoldTokens;
+  let inputBudget;
+  let outputBudget;
+  if (maxOutputTokens >= contextTokens) {
+    inputBudget = Math.floor(available / 3);
+    outputBudget = available - inputBudget;
+  } else {
+    outputBudget = maxOutputTokens;
+    inputBudget = contextTokens - outputBudget - scaffoldTokens;
+    if (inputBudget <= 0) {
+      inputBudget = Math.floor(available / 3);
+      outputBudget = available - inputBudget;
+    }
+  }
+  if (inputBudget <= OVERLAP_TOKENS) {
+    throw new Error(
+      `Inference window too small for detection: context ${contextTokens} tokens minus scaffold ${scaffoldTokens} leaves an input budget of ${inputBudget} (need > ${OVERLAP_TOKENS}). Use a model with a larger context window or reduce the prompt scaffold.`
+    );
+  }
+  return {
+    chunking: { chunkSize: inputBudget, overlap: OVERLAP_TOKENS },
+    outputBudget
+  };
+}
 function languageName(tag) {
   return getLocaleEnglishName(tag) || tag;
 }
@@ -477,7 +513,9 @@ var MotivationPrompts = class {
   /**
    * Build a prompt for detecting comment-worthy passages
    *
-   * @param content - The text content to analyze (will be truncated to 8000 chars)
+   * @param content - The text content to analyze — a chunk sized by the
+   *   caller from derived provider limits; NEVER re-truncated here (a
+   *   builder-level clip is silent input loss — see #738)
    * @param instructions - Optional user-provided instructions
    * @param tone - Optional tone guidance (e.g., "academic", "conversational")
    * @param density - Optional target number of comments per 2000 words
@@ -498,7 +536,7 @@ ${instructions}${toneGuidance}${densityGuidance}${sourceLang}${bodyLang}
 
 Text to analyze:
 ---
-${content.substring(0, 8e3)}
+${content}
 ---
 
 Return a JSON array of comments. Each comment must have:
@@ -532,7 +570,7 @@ Guidelines:
 
 Text to analyze:
 ---
-${content.substring(0, 8e3)}
+${content}
 ---
 
 Return a JSON array of comments. Each comment should have:
@@ -553,7 +591,9 @@ Example format:
   /**
    * Build a prompt for detecting highlight-worthy passages
    *
-   * @param content - The text content to analyze (will be truncated to 8000 chars)
+   * @param content - The text content to analyze — a chunk sized by the
+   *   caller from derived provider limits; NEVER re-truncated here (a
+   *   builder-level clip is silent input loss — see #738)
    * @param instructions - Optional user-provided instructions
    * @param density - Optional target number of highlights per 2000 words
    * @returns Formatted prompt string
@@ -571,7 +611,7 @@ ${instructions}${densityGuidance}${sourceLang}
 
 Text to analyze:
 ---
-${content.substring(0, 8e3)}
+${content}
 ---
 
 Return a JSON array of highlights. Each highlight must have:
@@ -602,7 +642,7 @@ Guidelines:
 
 Text to analyze:
 ---
-${content.substring(0, 8e3)}
+${content}
 ---
 
 Return a JSON array of highlights. Each highlight should have:
@@ -622,7 +662,9 @@ Example format:
   /**
    * Build a prompt for detecting assessment-worthy passages
    *
-   * @param content - The text content to analyze (will be truncated to 8000 chars)
+   * @param content - The text content to analyze — a chunk sized by the
+   *   caller from derived provider limits; NEVER re-truncated here (a
+   *   builder-level clip is silent input loss — see #738)
    * @param instructions - Optional user-provided instructions
    * @param tone - Optional tone guidance (e.g., "critical", "supportive")
    * @param density - Optional target number of assessments per 2000 words
@@ -643,7 +685,7 @@ ${instructions}${toneGuidance}${densityGuidance}${sourceLang}${bodyLang}
 
 Text to analyze:
 ---
-${content.substring(0, 8e3)}
+${content}
 ---
 
 Return a JSON array of assessments. Each assessment must have:
@@ -677,7 +719,7 @@ Guidelines:
 
 Text to analyze:
 ---
-${content.substring(0, 8e3)}
+${content}
 ---
 
 Return a JSON array of assessments. Each assessment should have:
@@ -749,32 +791,57 @@ Example format:
     return prompt;
   }
 };
-function parseJsonArray(response, motivation) {
-  let parsed;
-  try {
-    parsed = JSON.parse(response.trim());
-  } catch (error) {
-    console.error(`[MotivationParsers] Failed to parse AI ${motivation} response:`, error);
-    console.error("Raw response:", response);
-    throw error instanceof Error ? error : new Error(String(error));
-  }
-  if (!Array.isArray(parsed)) {
-    console.error(`[MotivationParsers] Expected a JSON array for ${motivation} detection, got ${typeof parsed}:`, response);
-    throw new Error(`Expected a JSON array for ${motivation} detection, got ${typeof parsed}`);
-  }
-  return parsed;
-}
+var COMMENT_ELEMENT_SCHEMA = {
+  type: "object",
+  properties: {
+    exact: { type: "string" },
+    prefix: { type: "string" },
+    suffix: { type: "string" },
+    comment: { type: "string" }
+  },
+  required: ["exact", "comment"],
+  additionalProperties: false
+};
+var HIGHLIGHT_ELEMENT_SCHEMA = {
+  type: "object",
+  properties: {
+    exact: { type: "string" },
+    prefix: { type: "string" },
+    suffix: { type: "string" }
+  },
+  required: ["exact"],
+  additionalProperties: false
+};
+var ASSESSMENT_ELEMENT_SCHEMA = {
+  type: "object",
+  properties: {
+    exact: { type: "string" },
+    prefix: { type: "string" },
+    suffix: { type: "string" },
+    assessment: { type: "string" }
+  },
+  required: ["exact", "assessment"],
+  additionalProperties: false
+};
+var TAG_ELEMENT_SCHEMA = {
+  type: "object",
+  properties: {
+    exact: { type: "string" },
+    prefix: { type: "string" },
+    suffix: { type: "string" }
+  },
+  required: ["exact"],
+  additionalProperties: false
+};
 var MotivationParsers = class {
   /**
-   * Parse and validate AI response for comment detection
+   * Validate and reconcile structured comment elements.
    *
-   * @param response - Raw AI response text (a JSON array)
+   * @param parsed - Already-parsed elements from the structured surface
    * @param content - Original content to validate offsets against
    * @returns Array of validated comment matches
-   * @throws if the response is not a parseable JSON array
    */
-  static parseComments(response, content) {
-    const parsed = parseJsonArray(response, "comment");
+  static parseComments(parsed, content) {
     const valid = parsed.filter(
       (c) => isObject(c) && isString(c.exact) && isString(c.comment) && c.comment.trim().length > 0
     );
@@ -803,15 +870,13 @@ var MotivationParsers = class {
     return validatedComments;
   }
   /**
-   * Parse and validate AI response for highlight detection
+   * Validate and reconcile structured highlight elements.
    *
-   * @param response - Raw AI response text (a JSON array)
+   * @param parsed - Already-parsed elements from the structured surface
    * @param content - Original content to validate offsets against
    * @returns Array of validated highlight matches
-   * @throws if the response is not a parseable JSON array
    */
-  static parseHighlights(response, content) {
-    const parsed = parseJsonArray(response, "highlight");
+  static parseHighlights(parsed, content) {
     const highlights = parsed.filter(
       (h) => isObject(h) && isString(h.exact)
     );
@@ -838,15 +903,13 @@ var MotivationParsers = class {
     return validatedHighlights;
   }
   /**
-   * Parse and validate AI response for assessment detection
+   * Validate and reconcile structured assessment elements.
    *
-   * @param response - Raw AI response text (a JSON array)
+   * @param parsed - Already-parsed elements from the structured surface
    * @param content - Original content to validate offsets against
    * @returns Array of validated assessment matches
-   * @throws if the response is not a parseable JSON array
    */
-  static parseAssessments(response, content) {
-    const parsed = parseJsonArray(response, "assessment");
+  static parseAssessments(parsed, content) {
     const assessments = parsed.filter(
       (a) => isObject(a) && isString(a.exact) && isString(a.assessment)
     );
@@ -874,14 +937,13 @@ var MotivationParsers = class {
     return validatedAssessments;
   }
   /**
-   * Parse the LLM's tag response into raw, pre-reconciliation tag inputs.
+   * Validate structured tag elements into raw, pre-reconciliation tag inputs.
    * Reconciliation happens in `validateTagOffsets`, which adds `start`/`end`
    * by anchoring `exact` against the source content.
    *
-   * @throws if the response is not a parseable JSON array
+   * @param parsed - Already-parsed elements from the structured surface
    */
-  static parseTags(response) {
-    const parsed = parseJsonArray(response, "tag");
+  static parseTags(parsed) {
     const valid = parsed.filter(
       (t) => isObject(t) && isString(t.exact) && t.exact.trim().length > 0
     );
@@ -923,10 +985,32 @@ function logAnchorMethod(motivation, exact, anchorMethod) {
 }
 
 // src/workers/annotation-detection.ts
-function assertNotTruncated(response, motivation) {
+function assertNotTruncated(response, motivation, chunk, totalChunks, outputBudget) {
   if (response.stopReason === "max_tokens") {
-    throw new Error(`${motivation} detection response truncated (max_tokens) \u2014 increase max_tokens or reduce resource size; failing the job rather than under-reporting annotations.`);
+    throw new Error(`${motivation} detection response truncated (max_tokens) on chunk ${chunk}/${totalChunks} despite the derived output budget of ${outputBudget} tokens \u2014 failing the job rather than under-reporting annotations.`);
+  }
+}
+async function detectInChunks(client, content, buildPrompt, temperature, motivation, elementSchema, parse, onChunk) {
+  const limits = await client.limits();
+  const scaffoldTokens = estimateTokens(buildPrompt(""));
+  const { chunking, outputBudget } = deriveDetectionBudget(limits, scaffoldTokens);
+  const chunks = chunkText(content, chunking);
+  const collected = [];
+  for (let i = 0; i < chunks.length; i++) {
+    const response = await boundedGenerateStructured(
+      client,
+      buildPrompt(chunks[i]),
+      outputBudget,
+      temperature,
+      elementSchema
+    );
+    assertNotTruncated(response, motivation, i + 1, chunks.length, outputBudget);
+    collected.push(...parse(response.items));
+    if (i < chunks.length - 1) {
+      onChunk?.(i + 1, chunks.length);
+    }
   }
+  return collected;
 }
 var AnnotationDetection = class {
   /**
@@ -937,11 +1021,17 @@ var AnnotationDetection = class {
    * (source-resource locale). See `types.ts` "Locale conventions" for the
    * full discussion.
    */
-  static async detectComments(content, client, instructions, tone, density, language, sourceLanguage) {
-    const prompt = MotivationPrompts.buildCommentPrompt(content, instructions, tone, density, language, sourceLanguage);
-    const response = await boundedGenerateWithMetadata(client, prompt, 3e3, 0.4, { format: "json" });
-    assertNotTruncated(response, "comment");
-    return MotivationParsers.parseComments(response.text, content);
+  static async detectComments(content, client, instructions, tone, density, language, sourceLanguage, onChunk) {
+    return detectInChunks(
+      client,
+      content,
+      (chunk) => MotivationPrompts.buildCommentPrompt(chunk, instructions, tone, density, language, sourceLanguage),
+      0.4,
+      "comment",
+      COMMENT_ELEMENT_SCHEMA,
+      (items) => MotivationParsers.parseComments(items, content),
+      onChunk
+    );
   }
   /**
    * Detect highlights in content.
@@ -950,11 +1040,17 @@ var AnnotationDetection = class {
    * applies, used in the prompt so the LLM analyzes non-English source
    * correctly.
    */
-  static async detectHighlights(content, client, instructions, density, sourceLanguage) {
-    const prompt = MotivationPrompts.buildHighlightPrompt(content, instructions, density, sourceLanguage);
-    const response = await boundedGenerateWithMetadata(client, prompt, 2e3, 0.3, { format: "json" });
-    assertNotTruncated(response, "highlight");
-    return MotivationParsers.parseHighlights(response.text, content);
+  static async detectHighlights(content, client, instructions, density, sourceLanguage, onChunk) {
+    return detectInChunks(
+      client,
+      content,
+      (chunk) => MotivationPrompts.buildHighlightPrompt(chunk, instructions, density, sourceLanguage),
+      0.3,
+      "highlight",
+      HIGHLIGHT_ELEMENT_SCHEMA,
+      (items) => MotivationParsers.parseHighlights(items, content),
+      onChunk
+    );
   }
   /**
    * Detect assessments in content.
@@ -963,11 +1059,17 @@ var AnnotationDetection = class {
    * (annotation body locale). `sourceLanguage` is the locale of the content
    * being analyzed (source-resource locale).
    */
-  static async detectAssessments(content, client, instructions, tone, density, language, sourceLanguage) {
-    const prompt = MotivationPrompts.buildAssessmentPrompt(content, instructions, tone, density, language, sourceLanguage);
-    const response = await boundedGenerateWithMetadata(client, prompt, 3e3, 0.3, { format: "json" });
-    assertNotTruncated(response, "assessment");
-    return MotivationParsers.parseAssessments(response.text, content);
+  static async detectAssessments(content, client, instructions, tone, density, language, sourceLanguage, onChunk) {
+    return detectInChunks(
+      client,
+      content,
+      (chunk) => MotivationPrompts.buildAssessmentPrompt(chunk, instructions, tone, density, language, sourceLanguage),
+      0.3,
+      "assessment",
+      ASSESSMENT_ELEMENT_SCHEMA,
+      (items) => MotivationParsers.parseAssessments(items, content),
+      onChunk
+    );
   }
   /**
    * Detect tags in content for a specific category.
@@ -981,28 +1083,45 @@ var AnnotationDetection = class {
    * identifiers, not LLM-generated text — so it's consumed at the body-stamp
    * site, not here.
    */
-  static async detectTags(content, client, schema, category, sourceLanguage) {
+  static async detectTags(content, client, schema, category, sourceLanguage, onChunk) {
     const categoryInfo = schema.tags.find((t) => t.name === category);
     if (!categoryInfo) {
       throw new Error(`Invalid category "${category}" for schema ${schema.id}`);
     }
-    const prompt = MotivationPrompts.buildTagPrompt(
+    const parsedTags = await detectInChunks(
+      client,
       content,
-      category,
-      schema.name,
-      schema.description,
-      schema.domain,
-      categoryInfo.description,
-      categoryInfo.examples,
-      sourceLanguage
+      (chunk) => MotivationPrompts.buildTagPrompt(
+        chunk,
+        category,
+        schema.name,
+        schema.description,
+        schema.domain,
+        categoryInfo.description,
+        categoryInfo.examples,
+        sourceLanguage
+      ),
+      0.2,
+      "tag",
+      TAG_ELEMENT_SCHEMA,
+      (items) => MotivationParsers.parseTags(items),
+      onChunk
     );
-    const response = await boundedGenerateWithMetadata(client, prompt, 4e3, 0.2, { format: "json" });
-    assertNotTruncated(response, "tag");
-    const parsedTags = MotivationParsers.parseTags(response.text);
     return MotivationParsers.validateTagOffsets(parsedTags, content, category);
   }
 };
-async function extractEntities(exact, entityTypes, client, includeDescriptiveReferences, logger, sourceLanguage) {
+var ENTITY_ELEMENT_SCHEMA = {
+  type: "object",
+  properties: {
+    exact: { type: "string" },
+    entityType: { type: "string" },
+    prefix: { type: "string" },
+    suffix: { type: "string" }
+  },
+  required: ["exact", "entityType"],
+  additionalProperties: false
+};
+async function extractEntities(exact, entityTypes, client, includeDescriptiveReferences, logger, sourceLanguage, onChunk) {
   const entityTypesDescription = entityTypes.map((et) => {
     if (typeof et === "string") {
       return et;
@@ -1034,11 +1153,11 @@ Find direct mentions only (names, proper nouns). Do not include pronouns or desc
   const sourceLangGuidance = sourceLanguage ? `
 Source text language: ${getLocaleEnglishName(sourceLanguage) || sourceLanguage}.
 ` : "";
-  const prompt = `Identify entity references in the following text. Look for mentions of: ${entityTypesDescription}.
+  const buildPrompt = (text) => `Identify entity references in the following text. Look for mentions of: ${entityTypesDescription}.
 ${descriptiveReferenceGuidance}${sourceLangGuidance}
 Text to analyze:
 """
-${exact}
+${text}
 """
 
 Respond with a JSON array of entities found. Each entity should have:
@@ -1051,59 +1170,53 @@ If no entities are found, respond with an empty array [].
 
 Example output:
 [{"exact":"Alice","entityType":"Person","prefix":"","suffix":" went to"},{"exact":"Paris","entityType":"Location","prefix":"went to ","suffix":" yesterday"}]`;
-  logger.debug("Sending entity extraction request", { entityTypes: entityTypesDescription });
-  const response = await boundedGenerateWithMetadata(
-    client,
-    prompt,
-    4e3,
-    // Increased to handle many entities without truncation
-    0.3,
-    // Lower temperature for more consistent extraction
-    // Force grammar-constrained JSON output. Without this, Ollama models
-    // periodically emit malformed JSON (truncated brackets, mid-token
-    // breaks at higher token counts) which silently parse-fails into
-    // [] downstream. The prompt's schema (which keys, what types) still
-    // governs *what* the JSON contains; `format: 'json'` governs that
-    // it's syntactically valid.
-    { format: "json" }
-  );
-  logger.debug("Got entity extraction response", { responseLength: response.text.length });
-  if (response.stopReason === "max_tokens") {
-    const errorMsg = "Entity extraction response truncated (max_tokens) \u2014 increase max_tokens or reduce resource size; failing the job rather than dropping annotations.";
-    logger.error(errorMsg, { responseLength: response.text.length });
-    throw new Error(errorMsg);
-  }
-  let entities;
-  try {
-    entities = JSON.parse(response.text.trim());
-  } catch (error) {
-    logger.error("Failed to parse entity extraction response", {
-      error: error instanceof Error ? error.message : String(error),
-      response: response.text.slice(0, 500)
-    });
-    throw new Error("Failed to parse entity extraction response", {
-      cause: error instanceof Error ? error : new Error(String(error))
-    });
-  }
-  if (!isArray(entities)) {
-    logger.error("Failed to parse entity extraction response: expected a JSON array", {
-      response: response.text.slice(0, 500)
+  const limits = await client.limits();
+  const scaffoldTokens = estimateTokens(buildPrompt(""));
+  const { chunking, outputBudget } = deriveDetectionBudget(limits, scaffoldTokens);
+  const chunks = chunkText(exact, chunking);
+  logger.debug("Sending entity extraction request", {
+    entityTypes: entityTypesDescription,
+    chunks: chunks.length,
+    chunkSizeTokens: chunking.chunkSize,
+    outputBudget
+  });
+  const collected = [];
+  for (let i = 0; i < chunks.length; i++) {
+    const response = await boundedGenerateStructured(
+      client,
+      buildPrompt(chunks[i]),
+      outputBudget,
+      0.3,
+      // Lower temperature for more consistent extraction
+      ENTITY_ELEMENT_SCHEMA
+    );
+    logger.debug("Got entity extraction response", {
+      chunk: i + 1,
+      chunks: chunks.length,
+      items: response.items.length
     });
-    throw new Error("Failed to parse entity extraction response: expected a JSON array");
-  }
-  logger.debug("Parsed entities from AI response", { count: entities.length });
-  return entities.filter((e) => {
-    const ok = isObject(e) && isString(e.exact) && isString(e.entityType);
-    if (!ok) {
-      logger.debug("Dropped malformed LLM entity", { entity: e });
+    if (response.stopReason === "max_tokens") {
+      const errorMsg = `Entity extraction response truncated (max_tokens) on chunk ${i + 1}/${chunks.length} despite the derived output budget of ${outputBudget} tokens \u2014 failing the job rather than dropping annotations.`;
+      logger.error(errorMsg, { items: response.items.length });
+      throw new Error(errorMsg);
     }
-    return ok;
-  }).map((entity) => ({
-    exact: entity.exact,
-    entityType: entity.entityType,
-    ...isString(entity.prefix) ? { prefix: entity.prefix } : {},
-    ...isString(entity.suffix) ? { suffix: entity.suffix } : {}
-  }));
+    for (const e of response.items) {
+      if (isObject(e) && isString(e.exact) && isString(e.entityType)) {
+        collected.push({
+          exact: e.exact,
+          entityType: e.entityType,
+          ...isString(e.prefix) ? { prefix: e.prefix } : {},
+          ...isString(e.suffix) ? { suffix: e.suffix } : {}
+        });
+      } else {
+        logger.debug("Dropped malformed LLM entity", { entity: e });
+      }
+    }
+    if (i < chunks.length - 1) {
+      onChunk?.(i + 1, chunks.length);
+    }
+  }
+  return collected;
 }
 function getLanguageName(locale) {
   return getLocaleEnglishName(locale) || locale;
@@ -1114,7 +1227,7 @@ var SEMANTIC_MATCH_CHARS = 240;
 function idLabel(resourceId, annotationId) {
   return `[${resourceId}${annotationId ? `/${annotationId}` : ""}]`;
 }
-async function generateResourceFromTopic(topic, entityTypes, client, logger, userPrompt, locale, context, temperature, maxTokens, sourceLanguage, outputMediaType = "text/markdown", task = "resource", structure, cite = false) {
+async function generateResourceFromTopic(topic, entityTypes, client, logger, userPrompt, locale, context, temperature, maxTokens, sourceLanguage, outputMediaType = "text/markdown", task = "resource", structure, cite = false, repair) {
   logger.debug("Generating resource from topic", {
     topicPreview: topic.substring(0, 100),
     entityTypes,
@@ -1230,11 +1343,12 @@ ${parts.join("\n")}`;
   let semanticContextSection = "";
   const similar = context?.semanticContext?.similar ?? [];
   if (similar.length > 0) {
-    const lines = [...similar].sort((a, b) => b.score - a.score).slice(0, SEMANTIC_MATCH_LIMIT).map((m) => `- ${idLabel(m.resourceId, m.annotationId)} (${m.score.toFixed(2)}) ${m.text.slice(0, SEMANTIC_MATCH_CHARS)}`);
+    const lines = [...similar].sort((a, b) => b.score - a.score).slice(0, SEMANTIC_MATCH_LIMIT).map((m) => `- ${idLabel(m.resourceId, m.annotationId)} (${m.score.toFixed(2)})${m.machineRead ? " [OCR]" : ""} ${m.text.slice(0, SEMANTIC_MATCH_CHARS)}`);
+    const ocrNote = similar.some((m) => m.machineRead) ? "\nPassages marked [OCR] were read from scanned images by character recognition; treat their exact wording and numbers as uncertain, and say so if you rely on one." : "";
     semanticContextSection = `
 
 Related passages from the knowledge base:
-${lines.join("\n")}`;
+${lines.join("\n")}${ocrNote}`;
   }
   let leadLine;
   if (task === "resource") {
@@ -1249,11 +1363,12 @@ ${lines.join("\n")}`;
 Topic: "${topic}"`;
   }
   const isPlainText = outputMediaType === "text/plain";
+  const isPdf = outputMediaType === "application/pdf";
   let structureRequirement = "";
   let titleRequirement = "";
   if (structure === "sections") {
-    structureRequirement = isPlainText ? "\n- Organize the content into titled sections with well-structured paragraphs" : "\n- Organize the content into titled sections (## Section) with well-structured paragraphs";
-    if (!isPlainText) {
+    structureRequirement = isPdf ? "\n- Organize the content into titled sections (= Heading) with well-structured paragraphs" : isPlainText ? "\n- Organize the content into titled sections with well-structured paragraphs" : "\n- Organize the content into titled sections (## Section) with well-structured paragraphs";
+    if (!isPlainText && !isPdf) {
       titleRequirement = "\n- Start with a clear heading (# Title)";
     }
   } else if (structure === "prose") {
@@ -1266,11 +1381,20 @@ Topic: "${topic}"`;
 - Organize the output as: ${structure}`;
   }
   const citeRequirement = cite ? "\n- Ground every claim in the provided context. Immediately after each claim, cite its source by emitting [[<id>]], where <id> is an id shown in square brackets in the context above (for a passage labeled [abc], emit [[abc]]). Cite only ids that appear in the context." : "";
-  const formatRequirements = isPlainText ? `- Write the response as plain text \u2014 no formatting markup (no #, *, backticks, headings, or links)
+  const formatRequirements = isPdf ? `- Write the response as Typst markup (the Typst typesetting language \u2014 not markdown, not LaTeX)
+- Headings are written as = Heading (deeper levels == Subheading); everything else is plain prose paragraphs
+- Do not emit markdown syntax or code fences` : isPlainText ? `- Write the response as plain text \u2014 no formatting markup (no #, *, backticks, headings, or links)
 - Begin with the title on its own first line` : `- Use markdown formatting
 - Write the response as markdown`;
+  const repairSection = repair ? `
+
+Your previous attempt failed to compile. Fix the error and return the complete corrected document \u2014 full source, not a diff.
+Compile error:
+${repair.error}
+Previous source:
+${repair.source}` : "";
   const prompt = `${leadLine}
-${userPrompt ? `Instruction: ${userPrompt}` : ""}
+${userPrompt ? `Instruction: ${userPrompt}` : ""}${repairSection}
 ${entityTypes.length > 0 ? `Focus on these entity types: ${entityTypes.join(", ")}.` : ""}${annotationSection}${contextSection}${resourceSection}${graphSection}${semanticContextSection}${sourceLanguageInstruction}${languageInstruction}
 
 Requirements:
@@ -1279,7 +1403,7 @@ Requirements:
 ${formatRequirements}`;
   const parseResponse = (response2) => {
     let content = response2.trim();
-    if (content.startsWith("```markdown") || content.startsWith("```md")) {
+    if (content.startsWith("```markdown") || content.startsWith("```md") || content.startsWith("```typst")) {
       content = content.slice(content.indexOf("\n") + 1);
       const endIndex = content.lastIndexOf("```");
       if (endIndex !== -1) {
@@ -1314,6 +1438,29 @@ ${formatRequirements}`;
   });
   return result;
 }
+var PINNED_CREATION_TIMESTAMP = 17e8;
+var MAX_COMPILE_REPAIRS = 2;
+function compileTypst(source) {
+  const dir = mkdtempSync(join(tmpdir(), "typst-"));
+  try {
+    const inFile = join(dir, "doc.typ");
+    const outFile = join(dir, "doc.pdf");
+    writeFileSync(inFile, source);
+    try {
+      execFileSync(
+        "typst",
+        ["compile", "--creation-timestamp", String(PINNED_CREATION_TIMESTAMP), inFile, outFile],
+        { stdio: ["ignore", "pipe", "pipe"] }
+      );
+    } catch (err) {
+      const stderr = err.stderr;
+      return { error: stderr?.length ? stderr.toString("utf8") : String(err) };
+    }
+    return { pdf: new Uint8Array(readFileSync(outFile)) };
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+}
 
 // src/workers/generation/citation-resolver.ts
 var CITATION_TOKEN = /\[\[([^\s[\]/]+)(?:\/([^\s[\]/]+))?\]\]/g;
@@ -1417,7 +1564,9 @@ async function processHighlightJob(content, inferenceClient, params, buildAnnota
     inferenceClient,
     params.instructions,
     params.density,
-    params.sourceLanguage
+    params.sourceLanguage,
+    // Chunk-boundary heartbeat (liveness): interpolate within the 30–60 band.
+    (completed, total) => onProgress(30 + Math.round(completed / total * 30), "Analyzing text...", "analyzing")
   );
   onProgress(60, `Creating ${highlights.length} annotations...`, "creating");
   const annotations = dedupeAnnotations(highlights.map(
@@ -1439,7 +1588,9 @@ async function processCommentJob(content, inferenceClient, params, buildAnnotati
     params.tone,
     params.density,
     params.language,
-    params.sourceLanguage
+    params.sourceLanguage,
+    // Chunk-boundary heartbeat (liveness): interpolate within the 30–60 band.
+    (completed, total) => onProgress(30 + Math.round(completed / total * 30), "Analyzing text...", "analyzing")
   );
   onProgress(60, `Creating ${comments.length} annotations...`, "creating");
   const bodyLanguage = params.language ?? "en";
@@ -1469,7 +1620,9 @@ async function processAssessmentJob(content, inferenceClient, params, buildAnnot
     params.tone,
     params.density,
     params.language,
-    params.sourceLanguage
+    params.sourceLanguage,
+    // Chunk-boundary heartbeat (liveness): interpolate within the 30–60 band.
+    (completed, total) => onProgress(30 + Math.round(completed / total * 30), "Analyzing text...", "analyzing")
   );
   onProgress(60, `Creating ${assessments.length} annotations...`, "creating");
   const bodyLanguage = params.language ?? "en";
@@ -1525,7 +1678,23 @@ async function processReferenceJob(content, inferenceClient, params, buildAnnota
       inferenceClient,
       params.includeDescriptiveReferences ?? false,
       logger,
-      params.sourceLanguage
+      params.sourceLanguage,
+      // Chunk-boundary heartbeat: progress is the worker's liveness signal
+      // (stall watchdog + backend janitor), so multi-chunk extraction must
+      // emit between inference calls. Percentage interpolates within this
+      // entity type's band of the 20–80 range.
+      (completed, total) => {
+        const interpolated = 20 + Math.round((i + completed / total) / entityTypeNames.length * 60);
+        onProgress(interpolated, `Detecting ${entityTypeName} entities...`, "analyzing", {
+          currentEntityType: entityTypeName,
+          processedEntityTypes: i,
+          totalEntityTypes: entityTypeNames.length,
+          entitiesFound: totalFound,
+          entitiesEmitted: totalEmitted,
+          completedEntityTypes: [...completedEntityTypes],
+          requestParams
+        });
+      }
     );
     totalFound += extractedEntities.length;
     completedEntityTypes.push({ entityType: entityTypeName, foundCount: extractedEntities.length });
@@ -1569,13 +1738,21 @@ async function processTagJob(content, inferenceClient, params, buildAnnotation,
   onProgress(10, "Loading resource...", "analyzing");
   onProgress(30, "Analyzing text for tags...", "analyzing");
   const allTags = [];
-  for (const category of params.categories) {
+  for (let c = 0; c < params.categories.length; c++) {
+    const category = params.categories[c];
     const categoryTags = await AnnotationDetection.detectTags(
       content,
       inferenceClient,
       params.schema,
       category,
-      params.sourceLanguage
+      params.sourceLanguage,
+      // Chunk-boundary heartbeat (liveness): interpolate within this
+      // category's slice of the 30–60 band.
+      (completed, total) => onProgress(
+        30 + Math.round((c + completed / total) / params.categories.length * 30),
+        "Analyzing text for tags...",
+        "analyzing"
+      )
     );
     allTags.push(...categoryTags);
   }
@@ -1601,8 +1778,14 @@ async function processTagJob(content, inferenceClient, params, buildAnnotation,
     result: { tagsFound: tags.length, tagsCreated: annotations.length, byCategory }
   };
 }
+function assertWithinOutputBudget(byteLength) {
+  if (!withinByteBudget(byteLength)) {
+    throw new Error(
+      `Generated artifact exceeds the output byte budget: ${byteLength} bytes > ${MAX_PDF_BYTES}. Refusing a runaway generation.`
+    );
+  }
+}
 async function processGenerationJob(inferenceClient, params, onProgress, logger) {
-  const GENERATABLE_MEDIA_TYPES = ["text/markdown", "text/plain"];
   const outputMediaType = params.outputMediaType ?? "text/markdown";
   if (!GENERATABLE_MEDIA_TYPES.includes(outputMediaType)) {
     throw new Error(
@@ -1611,6 +1794,84 @@ async function processGenerationJob(inferenceClient, params, onProgress, logger)
   }
   const title = params.title ?? "Untitled";
   const entityTypes = (params.entityTypes ?? []).map(String);
+  if (outputMediaType === "application/pdf") {
+    onProgress(5, "Generating resource...", "generating");
+    const validIds = params.cite === true ? collectContextResourceIds(params.context) : null;
+    let generated2 = await generateResourceFromTopic(
+      title,
+      entityTypes,
+      inferenceClient,
+      logger,
+      params.prompt,
+      params.language,
+      params.context,
+      params.temperature,
+      params.maxTokens,
+      params.sourceLanguage,
+      outputMediaType,
+      params.task,
+      params.structure,
+      params.cite
+    );
+    let source = generated2.content;
+    let citations2 = [];
+    if (validIds) {
+      const resolved = resolveCitationTokens(generated2.content, validIds, logger);
+      source = resolved.content;
+      citations2 = resolved.citations;
+    }
+    let compiled = compileTypst(source);
+    let repairs = 0;
+    while ("error" in compiled && repairs < MAX_COMPILE_REPAIRS) {
+      repairs++;
+      logger.warn("Typst compile failed \u2014 feeding the error back for repair", {
+        attempt: repairs,
+        error: compiled.error.slice(0, 500)
+      });
+      generated2 = await generateResourceFromTopic(
+        title,
+        entityTypes,
+        inferenceClient,
+        logger,
+        params.prompt,
+        params.language,
+        params.context,
+        params.temperature,
+        params.maxTokens,
+        params.sourceLanguage,
+        outputMediaType,
+        params.task,
+        params.structure,
+        params.cite,
+        { source, error: compiled.error }
+      );
+      if (validIds) {
+        const resolved = resolveCitationTokens(generated2.content, validIds, logger);
+        source = resolved.content;
+        citations2 = resolved.citations;
+      } else {
+        source = generated2.content;
+      }
+      compiled = compileTypst(source);
+    }
+    if ("error" in compiled) {
+      throw new Error(
+        `Typst compilation failed after ${MAX_COMPILE_REPAIRS} repair attempts: ${compiled.error}`
+      );
+    }
+    assertWithinOutputBudget(compiled.pdf.byteLength);
+    onProgress(95, "Creating resource...", "creating");
+    return {
+      content: compiled.pdf,
+      title: generated2.title ?? title,
+      format: outputMediaType,
+      citations: citations2,
+      result: {
+        resourceId: "",
+        resourceName: generated2.title ?? title
+      }
+    };
+  }
   onProgress(5, "Generating resource...", "generating");
   const generated = await generateResourceFromTopic(
     title,
@@ -1636,8 +1897,10 @@ async function processGenerationJob(inferenceClient, params, onProgress, logger)
     citations = resolved.citations;
   }
   onProgress(95, "Creating resource...", "creating");
+  const artifact = new TextEncoder().encode(content);
+  assertWithinOutputBudget(artifact.byteLength);
   return {
-    content,
+    content: artifact,
     title: generated.title ?? title,
     format: outputMediaType,
     citations,