@semiont/jobs 0.5.3 → 0.5.4

package/dist/index.js

@@ -974,7 +974,7 @@ var AnnotationDetection = class {
    */
   static async detectComments(content, client, instructions, tone, density, language, sourceLanguage) {
     const prompt = MotivationPrompts.buildCommentPrompt(content, instructions, tone, density, language, sourceLanguage);
-    const response = await client.generateText(prompt, 3e3, 0.4);
+    const response = await client.generateText(prompt, 3e3, 0.4, { format: "json" });
     return MotivationParsers.parseComments(response, content);
   }
   /**
@@ -986,7 +986,7 @@ var AnnotationDetection = class {
    */
   static async detectHighlights(content, client, instructions, density, sourceLanguage) {
     const prompt = MotivationPrompts.buildHighlightPrompt(content, instructions, density, sourceLanguage);
-    const response = await client.generateText(prompt, 2e3, 0.3);
+    const response = await client.generateText(prompt, 2e3, 0.3, { format: "json" });
     return MotivationParsers.parseHighlights(response, content);
   }
   /**
@@ -998,7 +998,7 @@ var AnnotationDetection = class {
    */
   static async detectAssessments(content, client, instructions, tone, density, language, sourceLanguage) {
     const prompt = MotivationPrompts.buildAssessmentPrompt(content, instructions, tone, density, language, sourceLanguage);
-    const response = await client.generateText(prompt, 3e3, 0.3);
+    const response = await client.generateText(prompt, 3e3, 0.3, { format: "json" });
     return MotivationParsers.parseAssessments(response, content);
   }
   /**
@@ -1028,12 +1028,12 @@ var AnnotationDetection = class {
       categoryInfo.examples,
       sourceLanguage
     );
-    const response = await client.generateText(prompt, 4e3, 0.2);
+    const response = await client.generateText(prompt, 4e3, 0.2, { format: "json" });
     const parsedTags = MotivationParsers.parseTags(response);
     return MotivationParsers.validateTagOffsets(parsedTags, content, category);
   }
 };
-async function extractEntities(exact, entityTypes, client, includeDescriptiveReferences = false, logger, sourceLanguage) {
+async function extractEntities(exact, entityTypes, client, includeDescriptiveReferences, logger, sourceLanguage) {
   const entityTypesDescription = entityTypes.map((et) => {
     if (typeof et === "string") {
       return et;
@@ -1084,50 +1084,57 @@ If no entities are found, respond with an empty array [].
 
 Example output:
 [{"exact":"Alice","entityType":"Person","startOffset":0,"endOffset":5,"prefix":"","suffix":" went to"},{"exact":"Paris","entityType":"Location","startOffset":20,"endOffset":25,"prefix":"went to ","suffix":" yesterday"}]`;
-  logger?.debug("Sending entity extraction request", { entityTypes: entityTypesDescription });
+  logger.debug("Sending entity extraction request", { entityTypes: entityTypesDescription });
   const response = await client.generateTextWithMetadata(
     prompt,
     4e3,
     // Increased to handle many entities without truncation
-    0.3
+    0.3,
     // Lower temperature for more consistent extraction
+    // Force grammar-constrained JSON output. Without this, Ollama models
+    // periodically emit malformed JSON (truncated brackets, mid-token
+    // breaks at higher token counts) which silently parse-fails into
+    // [] downstream. The prompt's schema (which keys, what types) still
+    // governs *what* the JSON contains; `format: 'json'` governs that
+    // it's syntactically valid.
+    { format: "json" }
   );
-  logger?.debug("Got entity extraction response", { responseLength: response.text.length });
+  logger.debug("Got entity extraction response", { responseLength: response.text.length });
   try {
     let jsonStr = response.text.trim();
     if (jsonStr.startsWith("```")) {
       jsonStr = jsonStr.replace(/^```(?:json)?\n?/, "").replace(/\n?```$/, "");
     }
     const entities = JSON.parse(jsonStr);
-    logger?.debug("Parsed entities from AI response", { count: entities.length });
+    logger.debug("Parsed entities from AI response", { count: entities.length });
     if (response.stopReason === "max_tokens") {
       const errorMsg = `AI response truncated: Found ${entities.length} entities but response hit max_tokens limit. Increase max_tokens or reduce resource size.`;
-      logger?.error(errorMsg);
+      logger.error(errorMsg);
       throw new Error(errorMsg);
     }
     return entities.map((entity, idx) => {
-      let startOffset = entity.startOffset;
-      let endOffset = entity.endOffset;
-      logger?.debug("Processing entity", {
+      let start = entity.startOffset;
+      let end = entity.endOffset;
+      logger.debug("Processing entity", {
         index: idx + 1,
         total: entities.length,
         type: entity.entityType,
         text: entity.exact,
-        offsetsFromAI: `[${startOffset}:${endOffset}]`
+        offsetsFromAI: `[${start}:${end}]`
       });
-      const extractedText = exact.substring(startOffset, endOffset);
+      const extractedText = exact.substring(start, end);
       let anchorMethod;
       if (extractedText === entity.exact) {
         anchorMethod = "llm-exact";
-        logger?.debug("Entity anchored", {
+        logger.debug("Entity anchored", {
           text: entity.exact,
           entityType: entity.entityType,
           anchorMethod
         });
       } else {
-        logger?.debug("LLM offsets mismatch \u2014 attempting re-anchor", {
+        logger.debug("LLM offsets mismatch \u2014 attempting re-anchor", {
           expected: entity.exact,
-          llmOffsets: `[${startOffset}:${endOffset}]`,
+          llmOffsets: `[${start}:${end}]`,
           foundAtLlmOffsets: extractedText
         });
         let occurrenceCount = 0;
@@ -1140,10 +1147,10 @@ Example output:
         }
         if (occurrenceCount === 0) {
           anchorMethod = "dropped";
-          logger?.error("Entity text not found in resource \u2014 dropping", {
+          logger.error("Entity text not found in resource \u2014 dropping", {
             text: entity.exact,
             entityType: entity.entityType,
-            llmOffsets: `[${startOffset}:${endOffset}]`,
+            llmOffsets: `[${start}:${end}]`,
             anchorMethod,
             resourceStart: exact.substring(0, 200)
           });
@@ -1169,9 +1176,9 @@ Example output:
         }
         if (recoveredOffset !== -1) {
           anchorMethod = "context-recovered";
-          startOffset = recoveredOffset;
-          endOffset = recoveredOffset + entity.exact.length;
-          logger?.debug("Entity anchored", {
+          start = recoveredOffset;
+          end = recoveredOffset + entity.exact.length;
+          logger.debug("Entity anchored", {
             text: entity.exact,
             entityType: entity.entityType,
             anchorMethod,
@@ -1179,9 +1186,9 @@ Example output:
           });
         } else if (occurrenceCount === 1) {
           anchorMethod = "unique-match";
-          startOffset = firstOccurrence;
-          endOffset = firstOccurrence + entity.exact.length;
-          logger?.debug("Entity anchored", {
+          start = firstOccurrence;
+          end = firstOccurrence + entity.exact.length;
+          logger.debug("Entity anchored", {
             text: entity.exact,
             entityType: entity.entityType,
             anchorMethod,
@@ -1189,9 +1196,9 @@ Example output:
           });
         } else {
           anchorMethod = "first-of-many";
-          startOffset = firstOccurrence;
-          endOffset = firstOccurrence + entity.exact.length;
-          logger?.warn("Entity anchored at first of multiple occurrences \u2014 may be wrong", {
+          start = firstOccurrence;
+          end = firstOccurrence + entity.exact.length;
+          logger.warn("Entity anchored at first of multiple occurrences \u2014 may be wrong", {
             text: entity.exact,
             entityType: entity.entityType,
             anchorMethod,
@@ -1206,52 +1213,52 @@ Example output:
       return {
         exact: entity.exact,
         entityType: entity.entityType,
-        startOffset,
-        endOffset,
+        start,
+        end,
         prefix: entity.prefix,
         suffix: entity.suffix
       };
     }).filter((entity) => {
       if (entity === null) {
-        logger?.debug("Filtered entity: null");
+        logger.debug("Filtered entity: null");
         return false;
       }
-      if (entity.startOffset === void 0 || entity.endOffset === void 0) {
-        logger?.warn("Filtered entity: missing offsets", { text: entity.exact });
+      if (entity.start === void 0 || entity.end === void 0) {
+        logger.warn("Filtered entity: missing offsets", { text: entity.exact });
         return false;
       }
-      if (entity.startOffset < 0) {
-        logger?.warn("Filtered entity: negative startOffset", {
+      if (entity.start < 0) {
+        logger.warn("Filtered entity: negative start", {
           text: entity.exact,
-          startOffset: entity.startOffset
+          start: entity.start
         });
         return false;
       }
-      if (entity.endOffset > exact.length) {
-        logger?.warn("Filtered entity: endOffset exceeds text length", {
+      if (entity.end > exact.length) {
+        logger.warn("Filtered entity: end exceeds text length", {
           text: entity.exact,
-          endOffset: entity.endOffset,
+          end: entity.end,
           textLength: exact.length
         });
         return false;
       }
-      const extractedText = exact.substring(entity.startOffset, entity.endOffset);
+      const extractedText = exact.substring(entity.start, entity.end);
       if (extractedText !== entity.exact) {
-        logger?.warn("Filtered entity: offset mismatch", {
+        logger.warn("Filtered entity: offset mismatch", {
           expected: entity.exact,
           got: extractedText,
-          offsets: `[${entity.startOffset}:${entity.endOffset}]`
+          offsets: `[${entity.start}:${entity.end}]`
         });
         return false;
       }
-      logger?.debug("Accepted entity", {
+      logger.debug("Accepted entity", {
         text: entity.exact,
-        offsets: `[${entity.startOffset}:${entity.endOffset}]`
+        offsets: `[${entity.start}:${entity.end}]`
       });
       return true;
     });
   } catch (error) {
-    logger?.error("Failed to parse entity extraction response", {
+    logger.error("Failed to parse entity extraction response", {
       error: error instanceof Error ? error.message : String(error)
     });
     return [];
@@ -1260,8 +1267,8 @@ Example output:
 function getLanguageName(locale) {
   return getLocaleEnglishName(locale) || locale;
 }
-async function generateResourceFromTopic(topic, entityTypes, client, userPrompt, locale, context, temperature, maxTokens, logger, sourceLanguage) {
-  logger?.debug("Generating resource from topic", {
+async function generateResourceFromTopic(topic, entityTypes, client, logger, userPrompt, locale, context, temperature, maxTokens, sourceLanguage) {
+  logger.debug("Generating resource from topic", {
     topicPreview: topic.substring(0, 100),
     entityTypes,
     hasUserPrompt: !!userPrompt,
@@ -1369,15 +1376,15 @@ Requirements:
       content
     };
   };
-  logger?.debug("Sending prompt to inference", {
+  logger.debug("Sending prompt to inference", {
     promptLength: prompt.length,
     temperature: finalTemperature,
     maxTokens: finalMaxTokens
   });
   const response = await client.generateText(prompt, finalMaxTokens, finalTemperature);
-  logger?.debug("Got response from inference", { responseLength: response.length });
+  logger.debug("Got response from inference", { responseLength: response.length });
   const result = parseResponse(response);
-  logger?.debug("Parsed response", {
+  logger.debug("Parsed response", {
     hasTitle: !!result.title,
     titleLength: result.title?.length,
     hasContent: !!result.content,
@@ -1386,13 +1393,16 @@ Requirements:
   return result;
 }
 function buildTextAnnotation(resourceId, userId, generator, motivation, match, body) {
+  const creator = didToAgent(userId);
+  const wasAttributedTo = creator["@id"] === generator["@id"] ? [generator] : [creator, generator];
   return {
     "@context": "http://www.w3.org/ns/anno.jsonld",
     "type": "Annotation",
     "id": generateAnnotationId(),
     motivation,
-    creator: didToAgent(userId),
+    creator,
     generator,
+    wasAttributedTo,
     created: (/* @__PURE__ */ new Date()).toISOString(),
     target: {
       type: "SpecificResource",
@@ -1535,7 +1545,7 @@ async function processReferenceJob(content, inferenceClient, params, userId, gen
     ];
     for (const entity of extractedEntities) {
       try {
-        const validated = validateAndCorrectOffsets(content, entity.startOffset, entity.endOffset, entity.exact);
+        const validated = validateAndCorrectOffsets(content, entity.start, entity.end, entity.exact);
         const ann = buildTextAnnotation(
           params.resourceId,
           userId,
@@ -1589,7 +1599,7 @@ async function processTagJob(content, inferenceClient, params, userId, generator
     result: { tagsFound: tags.length, tagsCreated: annotations.length, byCategory }
   };
 }
-async function processGenerationJob(inferenceClient, params, onProgress) {
+async function processGenerationJob(inferenceClient, params, onProgress, logger) {
   onProgress(20, "Fetching context...", "fetching");
   const title = params.title ?? "Untitled";
   const entityTypes = (params.entityTypes ?? []).map(String);
@@ -1598,13 +1608,12 @@ async function processGenerationJob(inferenceClient, params, onProgress) {
     title,
     entityTypes,
     inferenceClient,
+    logger,
     params.prompt,
     params.language,
     params.context,
     params.temperature,
     params.maxTokens,
-    void 0,
-    // logger
     params.sourceLanguage
   );
   onProgress(85, "Creating resource...", "creating");