@semiont/jobs 0.5.25 → 0.5.26

package/dist/worker-main.js

@@ -1,4 +1,4 @@
-import { createTomlConfigLoader, didToAgent, baseUrl, STARTUP_FETCH_RETRY, retryWithBackoff, isTransientFetchError, busRequest, resourceId, getPrimaryMediaType, assembleAnnotation, findClaimSpan, textExtractionOf, reconcileSelector, GENERATABLE_MEDIA_TYPES, locate, createFragmentSelector, getLocaleEnglishName, estimateTokens, chunkText, isArray, isObject, isString, deriveViews } from '@semiont/core';
+import { createTomlConfigLoader, didToAgent, baseUrl, STARTUP_FETCH_RETRY, retryWithBackoff, isTransientFetchError, busRequest, resourceId, getPrimaryMediaType, assembleAnnotation, findClaimSpan, textExtractionOf, reconcileSelector, GENERATABLE_MEDIA_TYPES, locate, createFragmentSelector, getLocaleEnglishName, estimateTokens, chunkText, isObject, isString, deriveViews } from '@semiont/core';
 import { anchoredTextStoreOverTransport, deriveStorageUri, extractPdfTextLayer, EXTRACTORS, calculateChecksum, withinByteBudget, MAX_PDF_BYTES } from '@semiont/content';
 import { execFileSync } from 'child_process';
 import { existsSync, readFileSync, mkdtempSync, writeFileSync, rmSync } from 'fs';
@@ -3766,9 +3766,9 @@ var require_mapOneOrManyArgs = __commonJS({
     Object.defineProperty(exports, "__esModule", { value: true });
     exports.mapOneOrManyArgs = void 0;
     var map_1 = require_map();
-    var isArray2 = Array.isArray;
+    var isArray = Array.isArray;
     function callOrApply(fn, args) {
-      return isArray2(args) ? fn.apply(void 0, __spreadArray([], __read(args))) : fn(args);
+      return isArray(args) ? fn.apply(void 0, __spreadArray([], __read(args))) : fn(args);
     }
     function mapOneOrManyArgs(fn) {
       return map_1.map(function(args) {
@@ -3913,14 +3913,14 @@ var require_argsArgArrayOrObject = __commonJS({
   "../../node_modules/rxjs/dist/cjs/internal/util/argsArgArrayOrObject.js"(exports) {
     Object.defineProperty(exports, "__esModule", { value: true });
     exports.argsArgArrayOrObject = void 0;
-    var isArray2 = Array.isArray;
+    var isArray = Array.isArray;
     var getPrototypeOf = Object.getPrototypeOf;
     var objectProto = Object.prototype;
     var getKeys = Object.keys;
     function argsArgArrayOrObject(args) {
       if (args.length === 1) {
         var first_1 = args[0];
-        if (isArray2(first_1)) {
+        if (isArray(first_1)) {
           return { args: first_1, keys: null };
         }
         if (isPOJO(first_1)) {
@@ -4668,9 +4668,9 @@ var require_argsOrArgArray = __commonJS({
   "../../node_modules/rxjs/dist/cjs/internal/util/argsOrArgArray.js"(exports) {
     Object.defineProperty(exports, "__esModule", { value: true });
     exports.argsOrArgArray = void 0;
-    var isArray2 = Array.isArray;
+    var isArray = Array.isArray;
     function argsOrArgArray(args) {
-      return args.length === 1 && isArray2(args[0]) ? args[0] : args;
+      return args.length === 1 && isArray(args[0]) ? args[0] : args;
     }
     exports.argsOrArgArray = argsOrArgArray;
   }
@@ -9371,15 +9371,15 @@ async function withTimeout(work, label) {
     clearTimeout(timer);
   }
 }
-function boundedGenerate(client, prompt, maxTokens, temperature, options) {
+function boundedGenerate(client, prompt, maxTokens, temperature) {
   return withTimeout(
-    client.generateText(prompt, maxTokens, temperature, options),
+    client.generateText(prompt, maxTokens, temperature),
     `${client.type}:${client.modelId}`
   );
 }
-function boundedGenerateWithMetadata(client, prompt, maxTokens, temperature, options) {
+function boundedGenerateStructured(client, prompt, maxTokens, temperature, elementSchema) {
   return withTimeout(
-    client.generateTextWithMetadata(prompt, maxTokens, temperature, options),
+    client.generateStructured(prompt, maxTokens, temperature, elementSchema),
     `${client.type}:${client.modelId}`
   );
 }
@@ -9713,32 +9713,57 @@ Example format:
     return prompt;
   }
 };
-function parseJsonArray(response, motivation) {
-  let parsed;
-  try {
-    parsed = JSON.parse(response.trim());
-  } catch (error) {
-    console.error(`[MotivationParsers] Failed to parse AI ${motivation} response:`, error);
-    console.error("Raw response:", response);
-    throw error instanceof Error ? error : new Error(String(error));
-  }
-  if (!Array.isArray(parsed)) {
-    console.error(`[MotivationParsers] Expected a JSON array for ${motivation} detection, got ${typeof parsed}:`, response);
-    throw new Error(`Expected a JSON array for ${motivation} detection, got ${typeof parsed}`);
-  }
-  return parsed;
-}
+var COMMENT_ELEMENT_SCHEMA = {
+  type: "object",
+  properties: {
+    exact: { type: "string" },
+    prefix: { type: "string" },
+    suffix: { type: "string" },
+    comment: { type: "string" }
+  },
+  required: ["exact", "comment"],
+  additionalProperties: false
+};
+var HIGHLIGHT_ELEMENT_SCHEMA = {
+  type: "object",
+  properties: {
+    exact: { type: "string" },
+    prefix: { type: "string" },
+    suffix: { type: "string" }
+  },
+  required: ["exact"],
+  additionalProperties: false
+};
+var ASSESSMENT_ELEMENT_SCHEMA = {
+  type: "object",
+  properties: {
+    exact: { type: "string" },
+    prefix: { type: "string" },
+    suffix: { type: "string" },
+    assessment: { type: "string" }
+  },
+  required: ["exact", "assessment"],
+  additionalProperties: false
+};
+var TAG_ELEMENT_SCHEMA = {
+  type: "object",
+  properties: {
+    exact: { type: "string" },
+    prefix: { type: "string" },
+    suffix: { type: "string" }
+  },
+  required: ["exact"],
+  additionalProperties: false
+};
 var MotivationParsers = class {
   /**
-   * Parse and validate AI response for comment detection
+   * Validate and reconcile structured comment elements.
    *
-   * @param response - Raw AI response text (a JSON array)
+   * @param parsed - Already-parsed elements from the structured surface
    * @param content - Original content to validate offsets against
    * @returns Array of validated comment matches
-   * @throws if the response is not a parseable JSON array
    */
-  static parseComments(response, content) {
-    const parsed = parseJsonArray(response, "comment");
+  static parseComments(parsed, content) {
     const valid = parsed.filter(
       (c) => isObject(c) && isString(c.exact) && isString(c.comment) && c.comment.trim().length > 0
     );
@@ -9767,15 +9792,13 @@ var MotivationParsers = class {
     return validatedComments;
   }
   /**
-   * Parse and validate AI response for highlight detection
+   * Validate and reconcile structured highlight elements.
    *
-   * @param response - Raw AI response text (a JSON array)
+   * @param parsed - Already-parsed elements from the structured surface
    * @param content - Original content to validate offsets against
    * @returns Array of validated highlight matches
-   * @throws if the response is not a parseable JSON array
    */
-  static parseHighlights(response, content) {
-    const parsed = parseJsonArray(response, "highlight");
+  static parseHighlights(parsed, content) {
     const highlights = parsed.filter(
       (h) => isObject(h) && isString(h.exact)
     );
@@ -9802,15 +9825,13 @@ var MotivationParsers = class {
     return validatedHighlights;
   }
   /**
-   * Parse and validate AI response for assessment detection
+   * Validate and reconcile structured assessment elements.
    *
-   * @param response - Raw AI response text (a JSON array)
+   * @param parsed - Already-parsed elements from the structured surface
    * @param content - Original content to validate offsets against
    * @returns Array of validated assessment matches
-   * @throws if the response is not a parseable JSON array
    */
-  static parseAssessments(response, content) {
-    const parsed = parseJsonArray(response, "assessment");
+  static parseAssessments(parsed, content) {
     const assessments = parsed.filter(
       (a) => isObject(a) && isString(a.exact) && isString(a.assessment)
     );
@@ -9838,14 +9859,13 @@ var MotivationParsers = class {
     return validatedAssessments;
   }
   /**
-   * Parse the LLM's tag response into raw, pre-reconciliation tag inputs.
+   * Validate structured tag elements into raw, pre-reconciliation tag inputs.
    * Reconciliation happens in `validateTagOffsets`, which adds `start`/`end`
    * by anchoring `exact` against the source content.
    *
-   * @throws if the response is not a parseable JSON array
+   * @param parsed - Already-parsed elements from the structured surface
    */
-  static parseTags(response) {
-    const parsed = parseJsonArray(response, "tag");
+  static parseTags(parsed) {
     const valid = parsed.filter(
       (t) => isObject(t) && isString(t.exact) && t.exact.trim().length > 0
     );
@@ -9892,22 +9912,22 @@ function assertNotTruncated(response, motivation, chunk, totalChunks, outputBudg
     throw new Error(`${motivation} detection response truncated (max_tokens) on chunk ${chunk}/${totalChunks} despite the derived output budget of ${outputBudget} tokens \u2014 failing the job rather than under-reporting annotations.`);
   }
 }
-async function detectInChunks(client, content, buildPrompt, temperature, motivation, parse, onChunk) {
+async function detectInChunks(client, content, buildPrompt, temperature, motivation, elementSchema, parse, onChunk) {
   const limits = await client.limits();
   const scaffoldTokens = estimateTokens(buildPrompt(""));
   const { chunking, outputBudget } = deriveDetectionBudget(limits, scaffoldTokens);
   const chunks = chunkText(content, chunking);
   const collected = [];
   for (let i = 0; i < chunks.length; i++) {
-    const response = await boundedGenerateWithMetadata(
+    const response = await boundedGenerateStructured(
       client,
       buildPrompt(chunks[i]),
       outputBudget,
       temperature,
-      { format: "json" }
+      elementSchema
     );
     assertNotTruncated(response, motivation, i + 1, chunks.length, outputBudget);
-    collected.push(...parse(response.text));
+    collected.push(...parse(response.items));
     if (i < chunks.length - 1) {
       onChunk?.(i + 1, chunks.length);
     }
@@ -9930,7 +9950,8 @@ var AnnotationDetection = class {
       (chunk) => MotivationPrompts.buildCommentPrompt(chunk, instructions, tone, density, language, sourceLanguage),
       0.4,
       "comment",
-      (text) => MotivationParsers.parseComments(text, content),
+      COMMENT_ELEMENT_SCHEMA,
+      (items) => MotivationParsers.parseComments(items, content),
       onChunk
     );
   }
@@ -9948,7 +9969,8 @@ var AnnotationDetection = class {
       (chunk) => MotivationPrompts.buildHighlightPrompt(chunk, instructions, density, sourceLanguage),
       0.3,
       "highlight",
-      (text) => MotivationParsers.parseHighlights(text, content),
+      HIGHLIGHT_ELEMENT_SCHEMA,
+      (items) => MotivationParsers.parseHighlights(items, content),
       onChunk
     );
   }
@@ -9966,7 +9988,8 @@ var AnnotationDetection = class {
       (chunk) => MotivationPrompts.buildAssessmentPrompt(chunk, instructions, tone, density, language, sourceLanguage),
       0.3,
       "assessment",
-      (text) => MotivationParsers.parseAssessments(text, content),
+      ASSESSMENT_ELEMENT_SCHEMA,
+      (items) => MotivationParsers.parseAssessments(items, content),
       onChunk
     );
   }
@@ -10002,12 +10025,24 @@ var AnnotationDetection = class {
       ),
       0.2,
       "tag",
-      (text) => MotivationParsers.parseTags(text),
+      TAG_ELEMENT_SCHEMA,
+      (items) => MotivationParsers.parseTags(items),
       onChunk
     );
     return MotivationParsers.validateTagOffsets(parsedTags, content, category);
   }
 };
+var ENTITY_ELEMENT_SCHEMA = {
+  type: "object",
+  properties: {
+    exact: { type: "string" },
+    entityType: { type: "string" },
+    prefix: { type: "string" },
+    suffix: { type: "string" }
+  },
+  required: ["exact", "entityType"],
+  additionalProperties: false
+};
 async function extractEntities(exact, entityTypes, client, includeDescriptiveReferences, logger2, sourceLanguage, onChunk) {
   const entityTypesDescription = entityTypes.map((et) => {
     if (typeof et === "string") {
@@ -10069,50 +10104,25 @@ Example output:
   });
   const collected = [];
   for (let i = 0; i < chunks.length; i++) {
-    const response = await boundedGenerateWithMetadata(
+    const response = await boundedGenerateStructured(
       client,
       buildPrompt(chunks[i]),
       outputBudget,
       0.3,
       // Lower temperature for more consistent extraction
-      // Force grammar-constrained JSON output. Without this, Ollama models
-      // periodically emit malformed JSON (truncated brackets, mid-token
-      // breaks at higher token counts) which silently parse-fails into
-      // [] downstream. The prompt's schema (which keys, what types) still
-      // governs *what* the JSON contains; `format: 'json'` governs that
-      // it's syntactically valid.
-      { format: "json" }
+      ENTITY_ELEMENT_SCHEMA
     );
     logger2.debug("Got entity extraction response", {
       chunk: i + 1,
       chunks: chunks.length,
-      responseLength: response.text.length
+      items: response.items.length
     });
     if (response.stopReason === "max_tokens") {
       const errorMsg = `Entity extraction response truncated (max_tokens) on chunk ${i + 1}/${chunks.length} despite the derived output budget of ${outputBudget} tokens \u2014 failing the job rather than dropping annotations.`;
-      logger2.error(errorMsg, { responseLength: response.text.length });
+      logger2.error(errorMsg, { items: response.items.length });
       throw new Error(errorMsg);
     }
-    let entities;
-    try {
-      entities = JSON.parse(response.text.trim());
-    } catch (error) {
-      logger2.error("Failed to parse entity extraction response", {
-        error: error instanceof Error ? error.message : String(error),
-        response: response.text.slice(0, 500)
-      });
-      throw new Error("Failed to parse entity extraction response", {
-        cause: error instanceof Error ? error : new Error(String(error))
-      });
-    }
-    if (!isArray(entities)) {
-      logger2.error("Failed to parse entity extraction response: expected a JSON array", {
-        response: response.text.slice(0, 500)
-      });
-      throw new Error("Failed to parse entity extraction response: expected a JSON array");
-    }
-    logger2.debug("Parsed entities from AI response", { chunk: i + 1, count: entities.length });
-    for (const e of entities) {
+    for (const e of response.items) {
       if (isObject(e) && isString(e.exact) && isString(e.entityType)) {
         collected.push({
           exact: e.exact,