@semiont/jobs 0.5.7 → 0.5.9

package/dist/index.d.ts

@@ -503,6 +503,16 @@ declare function processGenerationJob(inferenceClient: InferenceClient, params:
     result: GenerationResult;
 }>;
 
+/**
+ * Response parsers for annotation detection motivations
+ *
+ * Provides static methods to parse and validate AI responses for each motivation type.
+ * Includes offset validation and correction logic.
+ * Extracted from worker implementations to centralize parsing logic.
+ *
+ * NOTE: These are static utility methods without logger access.
+ * Console statements kept for debugging - consider adding logger parameter in future.
+ */
 /**
  * Represents a detected comment with validated position
  */

package/dist/index.js

@@ -1,6 +1,6 @@
 import { promises } from 'fs';
 import * as path from 'path';
-import { jobId, reconcileSelector, getLocaleEnglishName, didToAgent } from '@semiont/core';
+import { jobId, reconcileSelector, isObject, isString, getLocaleEnglishName, didToAgent, isArray } from '@semiont/core';
 import { generateAnnotationId } from '@semiont/event-sourcing';
 
 // src/fs-job-queue.ts
@@ -710,197 +710,144 @@ Example format:
     return prompt;
   }
 };
-function extractObjectsFromArray(response) {
-  let cleaned = response.trim();
-  if (cleaned.startsWith("```")) {
-    cleaned = cleaned.replace(/^```(?:json)?\s*\n?/, "").replace(/\n?```\s*$/, "");
-  }
+function parseJsonArray(response, motivation) {
+  let parsed;
   try {
-    const parsed = JSON.parse(cleaned);
-    return Array.isArray(parsed) ? parsed : [];
-  } catch {
+    parsed = JSON.parse(response.trim());
+  } catch (error) {
+    console.error(`[MotivationParsers] Failed to parse AI ${motivation} response:`, error);
+    console.error("Raw response:", response);
+    throw error instanceof Error ? error : new Error(String(error));
   }
-  const start = cleaned.indexOf("[");
-  if (start === -1) return [];
-  const endBracket = cleaned.lastIndexOf("]");
-  const end = endBracket > start ? endBracket : cleaned.length;
-  const inner = cleaned.slice(start + 1, end);
-  const objects = [];
-  let depth = 0;
-  let objStart = -1;
-  let inString = false;
-  let escape = false;
-  for (let i = 0; i < inner.length; i++) {
-    const ch = inner[i];
-    if (escape) {
-      escape = false;
-      continue;
-    }
-    if (ch === "\\") {
-      escape = true;
-      continue;
-    }
-    if (ch === '"') {
-      inString = !inString;
-      continue;
-    }
-    if (inString) continue;
-    if (ch === "{") {
-      if (depth === 0) objStart = i;
-      depth++;
-    } else if (ch === "}") {
-      depth--;
-      if (depth === 0 && objStart !== -1) {
-        try {
-          objects.push(JSON.parse(inner.slice(objStart, i + 1)));
-        } catch {
-        }
-        objStart = -1;
-      }
-    }
+  if (!Array.isArray(parsed)) {
+    console.error(`[MotivationParsers] Expected a JSON array for ${motivation} detection, got ${typeof parsed}:`, response);
+    throw new Error(`Expected a JSON array for ${motivation} detection, got ${typeof parsed}`);
   }
-  return objects;
+  return parsed;
 }
 var MotivationParsers = class {
   /**
    * Parse and validate AI response for comment detection
    *
-   * @param response - Raw AI response string (may include markdown code fences)
+   * @param response - Raw AI response text (a JSON array)
    * @param content - Original content to validate offsets against
    * @returns Array of validated comment matches
+   * @throws if the response is not a parseable JSON array
    */
   static parseComments(response, content) {
-    try {
-      const parsed = extractObjectsFromArray(response);
-      const valid = parsed.filter(
-        (c) => !!c && typeof c === "object" && typeof c.exact === "string" && typeof c.comment === "string" && c.comment.trim().length > 0
-      );
-      console.log(`[MotivationParsers] Parsed ${valid.length} valid comments from ${parsed.length} total`);
-      const validatedComments = [];
-      for (const comment of valid) {
-        const reconciled = reconcileSelector(content, {
-          exact: comment.exact,
-          ...typeof comment.prefix === "string" ? { prefix: comment.prefix } : {},
-          ...typeof comment.suffix === "string" ? { suffix: comment.suffix } : {}
-        });
-        if (!reconciled) {
-          console.warn(`[MotivationParsers] Dropped hallucinated comment "${comment.exact}"`);
-          continue;
-        }
-        logAnchorMethod("comment", comment.exact, reconciled.anchorMethod);
-        validatedComments.push({
-          comment: comment.comment,
-          exact: reconciled.exact,
-          start: reconciled.start,
-          end: reconciled.end,
-          ...reconciled.prefix !== void 0 ? { prefix: reconciled.prefix } : {},
-          ...reconciled.suffix !== void 0 ? { suffix: reconciled.suffix } : {}
-        });
+    const parsed = parseJsonArray(response, "comment");
+    const valid = parsed.filter(
+      (c) => isObject(c) && isString(c.exact) && isString(c.comment) && c.comment.trim().length > 0
+    );
+    console.log(`[MotivationParsers] Parsed ${valid.length} valid comments from ${parsed.length} total`);
+    const validatedComments = [];
+    for (const comment of valid) {
+      const reconciled = reconcileSelector(content, {
+        exact: comment.exact,
+        ...typeof comment.prefix === "string" ? { prefix: comment.prefix } : {},
+        ...typeof comment.suffix === "string" ? { suffix: comment.suffix } : {}
+      });
+      if (!reconciled) {
+        console.warn(`[MotivationParsers] Dropped hallucinated comment "${comment.exact}"`);
+        continue;
       }
-      return validatedComments;
-    } catch (error) {
-      console.error("[MotivationParsers] Failed to parse AI comment response:", error);
-      return [];
+      logAnchorMethod("comment", comment.exact, reconciled.anchorMethod);
+      validatedComments.push({
+        comment: comment.comment,
+        exact: reconciled.exact,
+        start: reconciled.start,
+        end: reconciled.end,
+        ...reconciled.prefix !== void 0 ? { prefix: reconciled.prefix } : {},
+        ...reconciled.suffix !== void 0 ? { suffix: reconciled.suffix } : {}
+      });
     }
+    return validatedComments;
   }
   /**
    * Parse and validate AI response for highlight detection
    *
-   * @param response - Raw AI response string (may include markdown code fences)
+   * @param response - Raw AI response text (a JSON array)
    * @param content - Original content to validate offsets against
    * @returns Array of validated highlight matches
+   * @throws if the response is not a parseable JSON array
    */
   static parseHighlights(response, content) {
-    try {
-      const parsed = extractObjectsFromArray(response);
-      const highlights = parsed.filter(
-        (h) => !!h && typeof h === "object" && typeof h.exact === "string"
-      );
-      const validatedHighlights = [];
-      for (const highlight of highlights) {
-        const reconciled = reconcileSelector(content, {
-          exact: highlight.exact,
-          ...typeof highlight.prefix === "string" ? { prefix: highlight.prefix } : {},
-          ...typeof highlight.suffix === "string" ? { suffix: highlight.suffix } : {}
-        });
-        if (!reconciled) {
-          console.warn(`[MotivationParsers] Dropped hallucinated highlight "${highlight.exact}"`);
-          continue;
-        }
-        logAnchorMethod("highlight", highlight.exact, reconciled.anchorMethod);
-        validatedHighlights.push({
-          exact: reconciled.exact,
-          start: reconciled.start,
-          end: reconciled.end,
-          ...reconciled.prefix !== void 0 ? { prefix: reconciled.prefix } : {},
-          ...reconciled.suffix !== void 0 ? { suffix: reconciled.suffix } : {}
-        });
+    const parsed = parseJsonArray(response, "highlight");
+    const highlights = parsed.filter(
+      (h) => isObject(h) && isString(h.exact)
+    );
+    const validatedHighlights = [];
+    for (const highlight of highlights) {
+      const reconciled = reconcileSelector(content, {
+        exact: highlight.exact,
+        ...typeof highlight.prefix === "string" ? { prefix: highlight.prefix } : {},
+        ...typeof highlight.suffix === "string" ? { suffix: highlight.suffix } : {}
+      });
+      if (!reconciled) {
+        console.warn(`[MotivationParsers] Dropped hallucinated highlight "${highlight.exact}"`);
+        continue;
       }
-      return validatedHighlights;
-    } catch (error) {
-      console.error("[MotivationParsers] Failed to parse AI highlight response:", error);
-      console.error("Raw response:", response);
-      return [];
+      logAnchorMethod("highlight", highlight.exact, reconciled.anchorMethod);
+      validatedHighlights.push({
+        exact: reconciled.exact,
+        start: reconciled.start,
+        end: reconciled.end,
+        ...reconciled.prefix !== void 0 ? { prefix: reconciled.prefix } : {},
+        ...reconciled.suffix !== void 0 ? { suffix: reconciled.suffix } : {}
+      });
     }
+    return validatedHighlights;
   }
   /**
    * Parse and validate AI response for assessment detection
    *
-   * @param response - Raw AI response string (may include markdown code fences)
+   * @param response - Raw AI response text (a JSON array)
    * @param content - Original content to validate offsets against
    * @returns Array of validated assessment matches
+   * @throws if the response is not a parseable JSON array
    */
   static parseAssessments(response, content) {
-    try {
-      const parsed = extractObjectsFromArray(response);
-      const assessments = parsed.filter(
-        (a) => !!a && typeof a === "object" && typeof a.exact === "string" && typeof a.assessment === "string"
-      );
-      const validatedAssessments = [];
-      for (const assessment of assessments) {
-        const reconciled = reconcileSelector(content, {
-          exact: assessment.exact,
-          ...typeof assessment.prefix === "string" ? { prefix: assessment.prefix } : {},
-          ...typeof assessment.suffix === "string" ? { suffix: assessment.suffix } : {}
-        });
-        if (!reconciled) {
-          console.warn(`[MotivationParsers] Dropped hallucinated assessment "${assessment.exact}"`);
-          continue;
-        }
-        logAnchorMethod("assessment", assessment.exact, reconciled.anchorMethod);
-        validatedAssessments.push({
-          assessment: assessment.assessment,
-          exact: reconciled.exact,
-          start: reconciled.start,
-          end: reconciled.end,
-          ...reconciled.prefix !== void 0 ? { prefix: reconciled.prefix } : {},
-          ...reconciled.suffix !== void 0 ? { suffix: reconciled.suffix } : {}
-        });
+    const parsed = parseJsonArray(response, "assessment");
+    const assessments = parsed.filter(
+      (a) => isObject(a) && isString(a.exact) && isString(a.assessment)
+    );
+    const validatedAssessments = [];
+    for (const assessment of assessments) {
+      const reconciled = reconcileSelector(content, {
+        exact: assessment.exact,
+        ...typeof assessment.prefix === "string" ? { prefix: assessment.prefix } : {},
+        ...typeof assessment.suffix === "string" ? { suffix: assessment.suffix } : {}
+      });
+      if (!reconciled) {
+        console.warn(`[MotivationParsers] Dropped hallucinated assessment "${assessment.exact}"`);
+        continue;
       }
-      return validatedAssessments;
-    } catch (error) {
-      console.error("[MotivationParsers] Failed to parse AI assessment response:", error);
-      console.error("Raw response:", response);
-      return [];
+      logAnchorMethod("assessment", assessment.exact, reconciled.anchorMethod);
+      validatedAssessments.push({
+        assessment: assessment.assessment,
+        exact: reconciled.exact,
+        start: reconciled.start,
+        end: reconciled.end,
+        ...reconciled.prefix !== void 0 ? { prefix: reconciled.prefix } : {},
+        ...reconciled.suffix !== void 0 ? { suffix: reconciled.suffix } : {}
+      });
     }
+    return validatedAssessments;
   }
   /**
    * Parse the LLM's tag response into raw, pre-reconciliation tag inputs.
    * Reconciliation happens in `validateTagOffsets`, which adds `start`/`end`
    * by anchoring `exact` against the source content.
+   *
+   * @throws if the response is not a parseable JSON array
    */
   static parseTags(response) {
-    try {
-      const parsed = extractObjectsFromArray(response);
-      const valid = parsed.filter(
-        (t) => !!t && typeof t === "object" && typeof t.exact === "string" && t.exact.trim().length > 0
-      );
-      console.log(`[MotivationParsers] Parsed ${valid.length} valid tags from ${parsed.length} total`);
-      return valid;
-    } catch (error) {
-      console.error("[MotivationParsers] Failed to parse AI tag response:", error);
-      return [];
-    }
+    const parsed = parseJsonArray(response, "tag");
+    const valid = parsed.filter(
+      (t) => isObject(t) && isString(t.exact) && t.exact.trim().length > 0
+    );
+    console.log(`[MotivationParsers] Parsed ${valid.length} valid tags from ${parsed.length} total`);
+    return valid;
   }
   /**
    * Anchor raw tag inputs against source content and add category.
@@ -937,6 +884,11 @@ function logAnchorMethod(motivation, exact, anchorMethod) {
 }
 
 // src/workers/annotation-detection.ts
+function assertNotTruncated(response, motivation) {
+  if (response.stopReason === "max_tokens") {
+    throw new Error(`${motivation} detection response truncated (max_tokens) \u2014 increase max_tokens or reduce resource size; failing the job rather than under-reporting annotations.`);
+  }
+}
 var AnnotationDetection = class {
   /**
    * Detect comments in content.
@@ -948,8 +900,9 @@ var AnnotationDetection = class {
    */
   static async detectComments(content, client, instructions, tone, density, language, sourceLanguage) {
     const prompt = MotivationPrompts.buildCommentPrompt(content, instructions, tone, density, language, sourceLanguage);
-    const response = await client.generateText(prompt, 3e3, 0.4, { format: "json" });
-    return MotivationParsers.parseComments(response, content);
+    const response = await client.generateTextWithMetadata(prompt, 3e3, 0.4, { format: "json" });
+    assertNotTruncated(response, "comment");
+    return MotivationParsers.parseComments(response.text, content);
   }
   /**
    * Detect highlights in content.
@@ -960,8 +913,9 @@ var AnnotationDetection = class {
    */
   static async detectHighlights(content, client, instructions, density, sourceLanguage) {
     const prompt = MotivationPrompts.buildHighlightPrompt(content, instructions, density, sourceLanguage);
-    const response = await client.generateText(prompt, 2e3, 0.3, { format: "json" });
-    return MotivationParsers.parseHighlights(response, content);
+    const response = await client.generateTextWithMetadata(prompt, 2e3, 0.3, { format: "json" });
+    assertNotTruncated(response, "highlight");
+    return MotivationParsers.parseHighlights(response.text, content);
   }
   /**
    * Detect assessments in content.
@@ -972,8 +926,9 @@ var AnnotationDetection = class {
    */
   static async detectAssessments(content, client, instructions, tone, density, language, sourceLanguage) {
     const prompt = MotivationPrompts.buildAssessmentPrompt(content, instructions, tone, density, language, sourceLanguage);
-    const response = await client.generateText(prompt, 3e3, 0.3, { format: "json" });
-    return MotivationParsers.parseAssessments(response, content);
+    const response = await client.generateTextWithMetadata(prompt, 3e3, 0.3, { format: "json" });
+    assertNotTruncated(response, "assessment");
+    return MotivationParsers.parseAssessments(response.text, content);
   }
   /**
    * Detect tags in content for a specific category.
@@ -1002,8 +957,9 @@ var AnnotationDetection = class {
       categoryInfo.examples,
       sourceLanguage
     );
-    const response = await client.generateText(prompt, 4e3, 0.2, { format: "json" });
-    const parsedTags = MotivationParsers.parseTags(response);
+    const response = await client.generateTextWithMetadata(prompt, 4e3, 0.2, { format: "json" });
+    assertNotTruncated(response, "tag");
+    const parsedTags = MotivationParsers.parseTags(response.text);
     return MotivationParsers.validateTagOffsets(parsedTags, content, category);
   }
 };
@@ -1072,36 +1028,42 @@ Example output:
     { format: "json" }
   );
   logger.debug("Got entity extraction response", { responseLength: response.text.length });
+  if (response.stopReason === "max_tokens") {
+    const errorMsg = "Entity extraction response truncated (max_tokens) \u2014 increase max_tokens or reduce resource size; failing the job rather than dropping annotations.";
+    logger.error(errorMsg, { responseLength: response.text.length });
+    throw new Error(errorMsg);
+  }
+  let entities;
   try {
-    let jsonStr = response.text.trim();
-    if (jsonStr.startsWith("```")) {
-      jsonStr = jsonStr.replace(/^```(?:json)?\n?/, "").replace(/\n?```$/, "");
-    }
-    const entities = JSON.parse(jsonStr);
-    logger.debug("Parsed entities from AI response", { count: entities.length });
-    if (response.stopReason === "max_tokens") {
-      const errorMsg = `AI response truncated: Found ${entities.length} entities but response hit max_tokens limit. Increase max_tokens or reduce resource size.`;
-      logger.error(errorMsg);
-      throw new Error(errorMsg);
-    }
-    return entities.filter((e) => {
-      const ok = e && typeof e === "object" && typeof e.exact === "string" && typeof e.entityType === "string";
-      if (!ok) {
-        logger.debug("Dropped malformed LLM entity", { entity: e });
-      }
-      return ok;
-    }).map((entity) => ({
-      exact: entity.exact,
-      entityType: entity.entityType,
-      ...typeof entity.prefix === "string" ? { prefix: entity.prefix } : {},
-      ...typeof entity.suffix === "string" ? { suffix: entity.suffix } : {}
-    }));
+    entities = JSON.parse(response.text.trim());
   } catch (error) {
     logger.error("Failed to parse entity extraction response", {
-      error: error instanceof Error ? error.message : String(error)
+      error: error instanceof Error ? error.message : String(error),
+      response: response.text.slice(0, 500)
+    });
+    throw new Error("Failed to parse entity extraction response", {
+      cause: error instanceof Error ? error : new Error(String(error))
+    });
+  }
+  if (!isArray(entities)) {
+    logger.error("Failed to parse entity extraction response: expected a JSON array", {
+      response: response.text.slice(0, 500)
     });
-    return [];
+    throw new Error("Failed to parse entity extraction response: expected a JSON array");
   }
+  logger.debug("Parsed entities from AI response", { count: entities.length });
+  return entities.filter((e) => {
+    const ok = isObject(e) && isString(e.exact) && isString(e.entityType);
+    if (!ok) {
+      logger.debug("Dropped malformed LLM entity", { entity: e });
+    }
+    return ok;
+  }).map((entity) => ({
+    exact: entity.exact,
+    entityType: entity.entityType,
+    ...isString(entity.prefix) ? { prefix: entity.prefix } : {},
+    ...isString(entity.suffix) ? { suffix: entity.suffix } : {}
+  }));
 }
 function getLanguageName(locale) {
   return getLocaleEnglishName(locale) || locale;
@@ -1183,10 +1145,19 @@ Knowledge graph context:
 ${parts.join("\n")}`;
     }
   }
+  let semanticContextSection = "";
+  const similar = context?.semanticContext?.similar ?? [];
+  if (similar.length > 0) {
+    const lines = [...similar].sort((a, b) => b.score - a.score).slice(0, 3).map((m) => `- (${m.score.toFixed(2)}) ${m.text.slice(0, 240)}`);
+    semanticContextSection = `
+
+Related passages from the knowledge base:
+${lines.join("\n")}`;
+  }
   const structureGuidance = finalMaxTokens >= 1e3 ? "organized into titled sections (## Section) with well-structured paragraphs" : "organized into well-structured paragraphs";
   const prompt = `Generate a concise, informative resource about "${topic}".
 ${entityTypes.length > 0 ? `Focus on these entity types: ${entityTypes.join(", ")}.` : ""}
-${userPrompt ? `Additional context: ${userPrompt}` : ""}${annotationSection}${contextSection}${graphContextSection}${sourceLanguageInstruction}${languageInstruction}
+${userPrompt ? `Additional context: ${userPrompt}` : ""}${annotationSection}${contextSection}${graphContextSection}${semanticContextSection}${sourceLanguageInstruction}${languageInstruction}
 
 Requirements:
 - Start with a clear heading (# Title)