@semiont/jobs 0.5.7 → 0.5.9

package/dist/worker-main.js

@@ -1,4 +1,4 @@
-import { createTomlConfigLoader, softwareToAgent, baseUrl, getPrimaryMediaType, textExtractionOf, reconcileSelector, didToAgent, getLocaleEnglishName } from '@semiont/core';
+import { createTomlConfigLoader, softwareToAgent, baseUrl, getPrimaryMediaType, textExtractionOf, reconcileSelector, didToAgent, getLocaleEnglishName, isArray, isObject, isString } from '@semiont/core';
 import { deriveStorageUri } from '@semiont/content';
 import { withSpan, SpanKind, recordJobOutcome } from '@semiont/observability';
 import { generateAnnotationId } from '@semiont/event-sourcing';
@@ -18,7 +18,11 @@ var __getOwnPropNames = Object.getOwnPropertyNames;
 var __getProtoOf = Object.getPrototypeOf;
 var __hasOwnProp = Object.prototype.hasOwnProperty;
 var __commonJS = (cb, mod) => function __require() {
-  return mod || (0, cb[__getOwnPropNames(cb)[0]])((mod = { exports: {} }).exports, mod), mod.exports;
+  try {
+    return mod || (0, cb[__getOwnPropNames(cb)[0]])((mod = { exports: {} }).exports, mod), mod.exports;
+  } catch (e) {
+    throw mod = 0, e;
+  }
 };
 var __copyProps = (to, from, except, desc) => {
   if (from && typeof from === "object" || typeof from === "function") {
@@ -3761,9 +3765,9 @@ var require_mapOneOrManyArgs = __commonJS({
     Object.defineProperty(exports, "__esModule", { value: true });
     exports.mapOneOrManyArgs = void 0;
     var map_1 = require_map();
-    var isArray = Array.isArray;
+    var isArray2 = Array.isArray;
     function callOrApply(fn, args) {
-      return isArray(args) ? fn.apply(void 0, __spreadArray([], __read(args))) : fn(args);
+      return isArray2(args) ? fn.apply(void 0, __spreadArray([], __read(args))) : fn(args);
     }
     function mapOneOrManyArgs(fn) {
       return map_1.map(function(args) {
@@ -3908,14 +3912,14 @@ var require_argsArgArrayOrObject = __commonJS({
   "../../node_modules/rxjs/dist/cjs/internal/util/argsArgArrayOrObject.js"(exports) {
     Object.defineProperty(exports, "__esModule", { value: true });
     exports.argsArgArrayOrObject = void 0;
-    var isArray = Array.isArray;
+    var isArray2 = Array.isArray;
     var getPrototypeOf = Object.getPrototypeOf;
     var objectProto = Object.prototype;
     var getKeys = Object.keys;
     function argsArgArrayOrObject(args) {
       if (args.length === 1) {
         var first_1 = args[0];
-        if (isArray(first_1)) {
+        if (isArray2(first_1)) {
           return { args: first_1, keys: null };
         }
         if (isPOJO(first_1)) {
@@ -4663,9 +4667,9 @@ var require_argsOrArgArray = __commonJS({
   "../../node_modules/rxjs/dist/cjs/internal/util/argsOrArgArray.js"(exports) {
     Object.defineProperty(exports, "__esModule", { value: true });
     exports.argsOrArgArray = void 0;
-    var isArray = Array.isArray;
+    var isArray2 = Array.isArray;
     function argsOrArgArray(args) {
-      return args.length === 1 && isArray(args[0]) ? args[0] : args;
+      return args.length === 1 && isArray2(args[0]) ? args[0] : args;
     }
     exports.argsOrArgArray = argsOrArgArray;
   }
@@ -9583,197 +9587,144 @@ Example format:
     return prompt;
   }
 };
-function extractObjectsFromArray(response) {
-  let cleaned = response.trim();
-  if (cleaned.startsWith("```")) {
-    cleaned = cleaned.replace(/^```(?:json)?\s*\n?/, "").replace(/\n?```\s*$/, "");
-  }
+function parseJsonArray(response, motivation) {
+  let parsed;
   try {
-    const parsed = JSON.parse(cleaned);
-    return Array.isArray(parsed) ? parsed : [];
-  } catch {
-  }
-  const start = cleaned.indexOf("[");
-  if (start === -1) return [];
-  const endBracket = cleaned.lastIndexOf("]");
-  const end = endBracket > start ? endBracket : cleaned.length;
-  const inner = cleaned.slice(start + 1, end);
-  const objects = [];
-  let depth = 0;
-  let objStart = -1;
-  let inString = false;
-  let escape = false;
-  for (let i = 0; i < inner.length; i++) {
-    const ch = inner[i];
-    if (escape) {
-      escape = false;
-      continue;
-    }
-    if (ch === "\\") {
-      escape = true;
-      continue;
-    }
-    if (ch === '"') {
-      inString = !inString;
-      continue;
-    }
-    if (inString) continue;
-    if (ch === "{") {
-      if (depth === 0) objStart = i;
-      depth++;
-    } else if (ch === "}") {
-      depth--;
-      if (depth === 0 && objStart !== -1) {
-        try {
-          objects.push(JSON.parse(inner.slice(objStart, i + 1)));
-        } catch {
-        }
-        objStart = -1;
-      }
-    }
+    parsed = JSON.parse(response.trim());
+  } catch (error) {
+    console.error(`[MotivationParsers] Failed to parse AI ${motivation} response:`, error);
+    console.error("Raw response:", response);
+    throw error instanceof Error ? error : new Error(String(error));
+  }
+  if (!Array.isArray(parsed)) {
+    console.error(`[MotivationParsers] Expected a JSON array for ${motivation} detection, got ${typeof parsed}:`, response);
+    throw new Error(`Expected a JSON array for ${motivation} detection, got ${typeof parsed}`);
   }
-  return objects;
+  return parsed;
 }
 var MotivationParsers = class {
   /**
    * Parse and validate AI response for comment detection
    *
-   * @param response - Raw AI response string (may include markdown code fences)
+   * @param response - Raw AI response text (a JSON array)
    * @param content - Original content to validate offsets against
    * @returns Array of validated comment matches
+   * @throws if the response is not a parseable JSON array
    */
   static parseComments(response, content) {
-    try {
-      const parsed = extractObjectsFromArray(response);
-      const valid = parsed.filter(
-        (c) => !!c && typeof c === "object" && typeof c.exact === "string" && typeof c.comment === "string" && c.comment.trim().length > 0
-      );
-      console.log(`[MotivationParsers] Parsed ${valid.length} valid comments from ${parsed.length} total`);
-      const validatedComments = [];
-      for (const comment of valid) {
-        const reconciled = reconcileSelector(content, {
-          exact: comment.exact,
-          ...typeof comment.prefix === "string" ? { prefix: comment.prefix } : {},
-          ...typeof comment.suffix === "string" ? { suffix: comment.suffix } : {}
-        });
-        if (!reconciled) {
-          console.warn(`[MotivationParsers] Dropped hallucinated comment "${comment.exact}"`);
-          continue;
-        }
-        logAnchorMethod("comment", comment.exact, reconciled.anchorMethod);
-        validatedComments.push({
-          comment: comment.comment,
-          exact: reconciled.exact,
-          start: reconciled.start,
-          end: reconciled.end,
-          ...reconciled.prefix !== void 0 ? { prefix: reconciled.prefix } : {},
-          ...reconciled.suffix !== void 0 ? { suffix: reconciled.suffix } : {}
-        });
+    const parsed = parseJsonArray(response, "comment");
+    const valid = parsed.filter(
+      (c) => isObject(c) && isString(c.exact) && isString(c.comment) && c.comment.trim().length > 0
+    );
+    console.log(`[MotivationParsers] Parsed ${valid.length} valid comments from ${parsed.length} total`);
+    const validatedComments = [];
+    for (const comment of valid) {
+      const reconciled = reconcileSelector(content, {
+        exact: comment.exact,
+        ...typeof comment.prefix === "string" ? { prefix: comment.prefix } : {},
+        ...typeof comment.suffix === "string" ? { suffix: comment.suffix } : {}
+      });
+      if (!reconciled) {
+        console.warn(`[MotivationParsers] Dropped hallucinated comment "${comment.exact}"`);
+        continue;
       }
-      return validatedComments;
-    } catch (error) {
-      console.error("[MotivationParsers] Failed to parse AI comment response:", error);
-      return [];
+      logAnchorMethod("comment", comment.exact, reconciled.anchorMethod);
+      validatedComments.push({
+        comment: comment.comment,
+        exact: reconciled.exact,
+        start: reconciled.start,
+        end: reconciled.end,
+        ...reconciled.prefix !== void 0 ? { prefix: reconciled.prefix } : {},
+        ...reconciled.suffix !== void 0 ? { suffix: reconciled.suffix } : {}
+      });
     }
+    return validatedComments;
   }
   /**
    * Parse and validate AI response for highlight detection
    *
-   * @param response - Raw AI response string (may include markdown code fences)
+   * @param response - Raw AI response text (a JSON array)
    * @param content - Original content to validate offsets against
    * @returns Array of validated highlight matches
+   * @throws if the response is not a parseable JSON array
    */
   static parseHighlights(response, content) {
-    try {
-      const parsed = extractObjectsFromArray(response);
-      const highlights = parsed.filter(
-        (h) => !!h && typeof h === "object" && typeof h.exact === "string"
-      );
-      const validatedHighlights = [];
-      for (const highlight of highlights) {
-        const reconciled = reconcileSelector(content, {
-          exact: highlight.exact,
-          ...typeof highlight.prefix === "string" ? { prefix: highlight.prefix } : {},
-          ...typeof highlight.suffix === "string" ? { suffix: highlight.suffix } : {}
-        });
-        if (!reconciled) {
-          console.warn(`[MotivationParsers] Dropped hallucinated highlight "${highlight.exact}"`);
-          continue;
-        }
-        logAnchorMethod("highlight", highlight.exact, reconciled.anchorMethod);
-        validatedHighlights.push({
-          exact: reconciled.exact,
-          start: reconciled.start,
-          end: reconciled.end,
-          ...reconciled.prefix !== void 0 ? { prefix: reconciled.prefix } : {},
-          ...reconciled.suffix !== void 0 ? { suffix: reconciled.suffix } : {}
-        });
+    const parsed = parseJsonArray(response, "highlight");
+    const highlights = parsed.filter(
+      (h) => isObject(h) && isString(h.exact)
+    );
+    const validatedHighlights = [];
+    for (const highlight of highlights) {
+      const reconciled = reconcileSelector(content, {
+        exact: highlight.exact,
+        ...typeof highlight.prefix === "string" ? { prefix: highlight.prefix } : {},
+        ...typeof highlight.suffix === "string" ? { suffix: highlight.suffix } : {}
+      });
+      if (!reconciled) {
+        console.warn(`[MotivationParsers] Dropped hallucinated highlight "${highlight.exact}"`);
+        continue;
       }
-      return validatedHighlights;
-    } catch (error) {
-      console.error("[MotivationParsers] Failed to parse AI highlight response:", error);
-      console.error("Raw response:", response);
-      return [];
+      logAnchorMethod("highlight", highlight.exact, reconciled.anchorMethod);
+      validatedHighlights.push({
+        exact: reconciled.exact,
+        start: reconciled.start,
+        end: reconciled.end,
+        ...reconciled.prefix !== void 0 ? { prefix: reconciled.prefix } : {},
+        ...reconciled.suffix !== void 0 ? { suffix: reconciled.suffix } : {}
+      });
     }
+    return validatedHighlights;
   }
   /**
    * Parse and validate AI response for assessment detection
    *
-   * @param response - Raw AI response string (may include markdown code fences)
+   * @param response - Raw AI response text (a JSON array)
    * @param content - Original content to validate offsets against
    * @returns Array of validated assessment matches
+   * @throws if the response is not a parseable JSON array
    */
   static parseAssessments(response, content) {
-    try {
-      const parsed = extractObjectsFromArray(response);
-      const assessments = parsed.filter(
-        (a) => !!a && typeof a === "object" && typeof a.exact === "string" && typeof a.assessment === "string"
-      );
-      const validatedAssessments = [];
-      for (const assessment of assessments) {
-        const reconciled = reconcileSelector(content, {
-          exact: assessment.exact,
-          ...typeof assessment.prefix === "string" ? { prefix: assessment.prefix } : {},
-          ...typeof assessment.suffix === "string" ? { suffix: assessment.suffix } : {}
-        });
-        if (!reconciled) {
-          console.warn(`[MotivationParsers] Dropped hallucinated assessment "${assessment.exact}"`);
-          continue;
-        }
-        logAnchorMethod("assessment", assessment.exact, reconciled.anchorMethod);
-        validatedAssessments.push({
-          assessment: assessment.assessment,
-          exact: reconciled.exact,
-          start: reconciled.start,
-          end: reconciled.end,
-          ...reconciled.prefix !== void 0 ? { prefix: reconciled.prefix } : {},
-          ...reconciled.suffix !== void 0 ? { suffix: reconciled.suffix } : {}
-        });
+    const parsed = parseJsonArray(response, "assessment");
+    const assessments = parsed.filter(
+      (a) => isObject(a) && isString(a.exact) && isString(a.assessment)
+    );
+    const validatedAssessments = [];
+    for (const assessment of assessments) {
+      const reconciled = reconcileSelector(content, {
+        exact: assessment.exact,
+        ...typeof assessment.prefix === "string" ? { prefix: assessment.prefix } : {},
+        ...typeof assessment.suffix === "string" ? { suffix: assessment.suffix } : {}
+      });
+      if (!reconciled) {
+        console.warn(`[MotivationParsers] Dropped hallucinated assessment "${assessment.exact}"`);
+        continue;
       }
-      return validatedAssessments;
-    } catch (error) {
-      console.error("[MotivationParsers] Failed to parse AI assessment response:", error);
-      console.error("Raw response:", response);
-      return [];
+      logAnchorMethod("assessment", assessment.exact, reconciled.anchorMethod);
+      validatedAssessments.push({
+        assessment: assessment.assessment,
+        exact: reconciled.exact,
+        start: reconciled.start,
+        end: reconciled.end,
+        ...reconciled.prefix !== void 0 ? { prefix: reconciled.prefix } : {},
+        ...reconciled.suffix !== void 0 ? { suffix: reconciled.suffix } : {}
+      });
     }
+    return validatedAssessments;
   }
   /**
    * Parse the LLM's tag response into raw, pre-reconciliation tag inputs.
    * Reconciliation happens in `validateTagOffsets`, which adds `start`/`end`
    * by anchoring `exact` against the source content.
+   *
+   * @throws if the response is not a parseable JSON array
    */
   static parseTags(response) {
-    try {
-      const parsed = extractObjectsFromArray(response);
-      const valid = parsed.filter(
-        (t) => !!t && typeof t === "object" && typeof t.exact === "string" && t.exact.trim().length > 0
-      );
-      console.log(`[MotivationParsers] Parsed ${valid.length} valid tags from ${parsed.length} total`);
-      return valid;
-    } catch (error) {
-      console.error("[MotivationParsers] Failed to parse AI tag response:", error);
-      return [];
-    }
+    const parsed = parseJsonArray(response, "tag");
+    const valid = parsed.filter(
+      (t) => isObject(t) && isString(t.exact) && t.exact.trim().length > 0
+    );
+    console.log(`[MotivationParsers] Parsed ${valid.length} valid tags from ${parsed.length} total`);
+    return valid;
   }
   /**
    * Anchor raw tag inputs against source content and add category.
@@ -9810,6 +9761,11 @@ function logAnchorMethod(motivation, exact, anchorMethod) {
 }
 
 // src/workers/annotation-detection.ts
+function assertNotTruncated(response, motivation) {
+  if (response.stopReason === "max_tokens") {
+    throw new Error(`${motivation} detection response truncated (max_tokens) \u2014 increase max_tokens or reduce resource size; failing the job rather than under-reporting annotations.`);
+  }
+}
 var AnnotationDetection = class {
   /**
    * Detect comments in content.
@@ -9821,8 +9777,9 @@ var AnnotationDetection = class {
    */
   static async detectComments(content, client, instructions, tone, density, language, sourceLanguage) {
     const prompt = MotivationPrompts.buildCommentPrompt(content, instructions, tone, density, language, sourceLanguage);
-    const response = await client.generateText(prompt, 3e3, 0.4, { format: "json" });
-    return MotivationParsers.parseComments(response, content);
+    const response = await client.generateTextWithMetadata(prompt, 3e3, 0.4, { format: "json" });
+    assertNotTruncated(response, "comment");
+    return MotivationParsers.parseComments(response.text, content);
   }
   /**
    * Detect highlights in content.
@@ -9833,8 +9790,9 @@ var AnnotationDetection = class {
    */
   static async detectHighlights(content, client, instructions, density, sourceLanguage) {
     const prompt = MotivationPrompts.buildHighlightPrompt(content, instructions, density, sourceLanguage);
-    const response = await client.generateText(prompt, 2e3, 0.3, { format: "json" });
-    return MotivationParsers.parseHighlights(response, content);
+    const response = await client.generateTextWithMetadata(prompt, 2e3, 0.3, { format: "json" });
+    assertNotTruncated(response, "highlight");
+    return MotivationParsers.parseHighlights(response.text, content);
   }
   /**
    * Detect assessments in content.
@@ -9845,8 +9803,9 @@ var AnnotationDetection = class {
    */
   static async detectAssessments(content, client, instructions, tone, density, language, sourceLanguage) {
     const prompt = MotivationPrompts.buildAssessmentPrompt(content, instructions, tone, density, language, sourceLanguage);
-    const response = await client.generateText(prompt, 3e3, 0.3, { format: "json" });
-    return MotivationParsers.parseAssessments(response, content);
+    const response = await client.generateTextWithMetadata(prompt, 3e3, 0.3, { format: "json" });
+    assertNotTruncated(response, "assessment");
+    return MotivationParsers.parseAssessments(response.text, content);
   }
   /**
    * Detect tags in content for a specific category.
@@ -9875,8 +9834,9 @@ var AnnotationDetection = class {
       categoryInfo.examples,
       sourceLanguage
     );
-    const response = await client.generateText(prompt, 4e3, 0.2, { format: "json" });
-    const parsedTags = MotivationParsers.parseTags(response);
+    const response = await client.generateTextWithMetadata(prompt, 4e3, 0.2, { format: "json" });
+    assertNotTruncated(response, "tag");
+    const parsedTags = MotivationParsers.parseTags(response.text);
     return MotivationParsers.validateTagOffsets(parsedTags, content, category);
   }
 };
@@ -9945,36 +9905,42 @@ Example output:
     { format: "json" }
   );
   logger2.debug("Got entity extraction response", { responseLength: response.text.length });
+  if (response.stopReason === "max_tokens") {
+    const errorMsg = "Entity extraction response truncated (max_tokens) \u2014 increase max_tokens or reduce resource size; failing the job rather than dropping annotations.";
+    logger2.error(errorMsg, { responseLength: response.text.length });
+    throw new Error(errorMsg);
+  }
+  let entities;
   try {
-    let jsonStr = response.text.trim();
-    if (jsonStr.startsWith("```")) {
-      jsonStr = jsonStr.replace(/^```(?:json)?\n?/, "").replace(/\n?```$/, "");
-    }
-    const entities = JSON.parse(jsonStr);
-    logger2.debug("Parsed entities from AI response", { count: entities.length });
-    if (response.stopReason === "max_tokens") {
-      const errorMsg = `AI response truncated: Found ${entities.length} entities but response hit max_tokens limit. Increase max_tokens or reduce resource size.`;
-      logger2.error(errorMsg);
-      throw new Error(errorMsg);
-    }
-    return entities.filter((e) => {
-      const ok = e && typeof e === "object" && typeof e.exact === "string" && typeof e.entityType === "string";
-      if (!ok) {
-        logger2.debug("Dropped malformed LLM entity", { entity: e });
-      }
-      return ok;
-    }).map((entity) => ({
-      exact: entity.exact,
-      entityType: entity.entityType,
-      ...typeof entity.prefix === "string" ? { prefix: entity.prefix } : {},
-      ...typeof entity.suffix === "string" ? { suffix: entity.suffix } : {}
-    }));
+    entities = JSON.parse(response.text.trim());
   } catch (error) {
     logger2.error("Failed to parse entity extraction response", {
-      error: error instanceof Error ? error.message : String(error)
+      error: error instanceof Error ? error.message : String(error),
+      response: response.text.slice(0, 500)
+    });
+    throw new Error("Failed to parse entity extraction response", {
+      cause: error instanceof Error ? error : new Error(String(error))
     });
-    return [];
   }
+  if (!isArray(entities)) {
+    logger2.error("Failed to parse entity extraction response: expected a JSON array", {
+      response: response.text.slice(0, 500)
+    });
+    throw new Error("Failed to parse entity extraction response: expected a JSON array");
+  }
+  logger2.debug("Parsed entities from AI response", { count: entities.length });
+  return entities.filter((e) => {
+    const ok = isObject(e) && isString(e.exact) && isString(e.entityType);
+    if (!ok) {
+      logger2.debug("Dropped malformed LLM entity", { entity: e });
+    }
+    return ok;
+  }).map((entity) => ({
+    exact: entity.exact,
+    entityType: entity.entityType,
+    ...isString(entity.prefix) ? { prefix: entity.prefix } : {},
+    ...isString(entity.suffix) ? { suffix: entity.suffix } : {}
+  }));
 }
 function getLanguageName(locale) {
   return getLocaleEnglishName(locale) || locale;
@@ -10056,10 +10022,19 @@ Knowledge graph context:
 ${parts.join("\n")}`;
     }
   }
+  let semanticContextSection = "";
+  const similar = context?.semanticContext?.similar ?? [];
+  if (similar.length > 0) {
+    const lines = [...similar].sort((a, b) => b.score - a.score).slice(0, 3).map((m) => `- (${m.score.toFixed(2)}) ${m.text.slice(0, 240)}`);
+    semanticContextSection = `
+
+Related passages from the knowledge base:
+${lines.join("\n")}`;
+  }
   const structureGuidance = finalMaxTokens >= 1e3 ? "organized into titled sections (## Section) with well-structured paragraphs" : "organized into well-structured paragraphs";
   const prompt = `Generate a concise, informative resource about "${topic}".
 ${entityTypes.length > 0 ? `Focus on these entity types: ${entityTypes.join(", ")}.` : ""}
-${userPrompt ? `Additional context: ${userPrompt}` : ""}${annotationSection}${contextSection}${graphContextSection}${sourceLanguageInstruction}${languageInstruction}
+${userPrompt ? `Additional context: ${userPrompt}` : ""}${annotationSection}${contextSection}${graphContextSection}${semanticContextSection}${sourceLanguageInstruction}${languageInstruction}
 
 Requirements:
 - Start with a clear heading (# Title)