@semiont/jobs 0.5.2 → 0.5.4

package/dist/index.js

@@ -947,159 +947,6 @@ var MotivationParsers = class {
   }
 };
 
-// ../ontology/dist/index.js
-var TAG_SCHEMAS = {
-  "legal-irac": {
-    id: "legal-irac",
-    name: "Legal Analysis (IRAC)",
-    description: "Issue, Rule, Application, Conclusion framework for legal reasoning",
-    domain: "legal",
-    tags: [
-      {
-        name: "Issue",
-        description: "The legal question or problem to be resolved",
-        examples: [
-          "What is the central legal question?",
-          "What must the court decide?",
-          "What is the dispute about?"
-        ]
-      },
-      {
-        name: "Rule",
-        description: "The relevant law, statute, or legal principle",
-        examples: [
-          "What law applies?",
-          "What is the legal standard?",
-          "What statute governs this case?"
-        ]
-      },
-      {
-        name: "Application",
-        description: "How the rule applies to the specific facts",
-        examples: [
-          "How does the law apply to these facts?",
-          "Analysis of the case",
-          "How do the facts satisfy the legal standard?"
-        ]
-      },
-      {
-        name: "Conclusion",
-        description: "The resolution or outcome based on the analysis",
-        examples: [
-          "What is the court's decision?",
-          "What is the final judgment?",
-          "What is the holding?"
-        ]
-      }
-    ]
-  },
-  "scientific-imrad": {
-    id: "scientific-imrad",
-    name: "Scientific Paper (IMRAD)",
-    description: "Introduction, Methods, Results, Discussion structure for research papers",
-    domain: "scientific",
-    tags: [
-      {
-        name: "Introduction",
-        description: "Background, context, and research question",
-        examples: [
-          "What is the research question?",
-          "Why is this important?",
-          "What is the hypothesis?"
-        ]
-      },
-      {
-        name: "Methods",
-        description: "Experimental design and procedures",
-        examples: [
-          "How was the study conducted?",
-          "What methods were used?",
-          "What was the experimental design?"
-        ]
-      },
-      {
-        name: "Results",
-        description: "Findings and observations",
-        examples: [
-          "What did the study find?",
-          "What are the data?",
-          "What were the observations?"
-        ]
-      },
-      {
-        name: "Discussion",
-        description: "Interpretation and implications of results",
-        examples: [
-          "What do the results mean?",
-          "What are the implications?",
-          "How do these findings relate to prior work?"
-        ]
-      }
-    ]
-  },
-  "argument-toulmin": {
-    id: "argument-toulmin",
-    name: "Argument Structure (Toulmin)",
-    description: "Claim, Evidence, Warrant, Counterargument, Rebuttal framework for argumentation",
-    domain: "general",
-    tags: [
-      {
-        name: "Claim",
-        description: "The main assertion or thesis",
-        examples: [
-          "What is being argued?",
-          "What is the main point?",
-          "What position is being taken?"
-        ]
-      },
-      {
-        name: "Evidence",
-        description: "Data or facts supporting the claim",
-        examples: [
-          "What supports this claim?",
-          "What are the facts?",
-          "What data is provided?"
-        ]
-      },
-      {
-        name: "Warrant",
-        description: "Reasoning connecting evidence to claim",
-        examples: [
-          "Why does this evidence support the claim?",
-          "What is the logic?",
-          "How does this reasoning work?"
-        ]
-      },
-      {
-        name: "Counterargument",
-        description: "Opposing viewpoints or objections",
-        examples: [
-          "What are the objections?",
-          "What do critics say?",
-          "What are alternative views?"
-        ]
-      },
-      {
-        name: "Rebuttal",
-        description: "Response to counterarguments",
-        examples: [
-          "How is the objection addressed?",
-          "Why is the counterargument wrong?",
-          "How is the criticism answered?"
-        ]
-      }
-    ]
-  }
-};
-function getTagSchema(schemaId) {
-  return TAG_SCHEMAS[schemaId] || null;
-}
-function getSchemaCategory(schemaId, categoryName) {
-  const schema = getTagSchema(schemaId);
-  if (!schema) return null;
-  return schema.tags.find((tag) => tag.name === categoryName) || null;
-}
-
 // src/workers/annotation-detection.ts
 var AnnotationDetection = class {
   /**
@@ -1127,7 +974,7 @@ var AnnotationDetection = class {
    */
   static async detectComments(content, client, instructions, tone, density, language, sourceLanguage) {
     const prompt = MotivationPrompts.buildCommentPrompt(content, instructions, tone, density, language, sourceLanguage);
-    const response = await client.generateText(prompt, 3e3, 0.4);
+    const response = await client.generateText(prompt, 3e3, 0.4, { format: "json" });
     return MotivationParsers.parseComments(response, content);
   }
   /**
@@ -1139,7 +986,7 @@ var AnnotationDetection = class {
    */
   static async detectHighlights(content, client, instructions, density, sourceLanguage) {
     const prompt = MotivationPrompts.buildHighlightPrompt(content, instructions, density, sourceLanguage);
-    const response = await client.generateText(prompt, 2e3, 0.3);
+    const response = await client.generateText(prompt, 2e3, 0.3, { format: "json" });
     return MotivationParsers.parseHighlights(response, content);
   }
   /**
@@ -1151,25 +998,25 @@ var AnnotationDetection = class {
    */
   static async detectAssessments(content, client, instructions, tone, density, language, sourceLanguage) {
     const prompt = MotivationPrompts.buildAssessmentPrompt(content, instructions, tone, density, language, sourceLanguage);
-    const response = await client.generateText(prompt, 3e3, 0.3);
+    const response = await client.generateText(prompt, 3e3, 0.3, { format: "json" });
     return MotivationParsers.parseAssessments(response, content);
   }
   /**
    * Detect tags in content for a specific category.
    *
+   * The full `TagSchema` is supplied by the dispatcher (resolved against
+   * the per-KB tag-schema projection at job-creation time) so the worker
+   * is independent of the registry.
+   *
    * `sourceLanguage` is the locale of the content being analyzed. Body-locale
    * (`language`) doesn't influence the tag prompt — categories are schema
    * identifiers, not LLM-generated text — so it's consumed at the body-stamp
    * site, not here.
    */
-  static async detectTags(content, client, schemaId, category, sourceLanguage) {
-    const schema = getTagSchema(schemaId);
-    if (!schema) {
-      throw new Error(`Invalid tag schema: ${schemaId}`);
-    }
-    const categoryInfo = getSchemaCategory(schemaId, category);
+  static async detectTags(content, client, schema, category, sourceLanguage) {
+    const categoryInfo = schema.tags.find((t) => t.name === category);
     if (!categoryInfo) {
-      throw new Error(`Invalid category "${category}" for schema ${schemaId}`);
+      throw new Error(`Invalid category "${category}" for schema ${schema.id}`);
     }
     const prompt = MotivationPrompts.buildTagPrompt(
       content,
@@ -1181,12 +1028,12 @@ var AnnotationDetection = class {
       categoryInfo.examples,
       sourceLanguage
     );
-    const response = await client.generateText(prompt, 4e3, 0.2);
+    const response = await client.generateText(prompt, 4e3, 0.2, { format: "json" });
     const parsedTags = MotivationParsers.parseTags(response);
     return MotivationParsers.validateTagOffsets(parsedTags, content, category);
   }
 };
-async function extractEntities(exact, entityTypes, client, includeDescriptiveReferences = false, logger, sourceLanguage) {
+async function extractEntities(exact, entityTypes, client, includeDescriptiveReferences, logger, sourceLanguage) {
   const entityTypesDescription = entityTypes.map((et) => {
     if (typeof et === "string") {
       return et;
@@ -1237,50 +1084,57 @@ If no entities are found, respond with an empty array [].
 
 Example output:
 [{"exact":"Alice","entityType":"Person","startOffset":0,"endOffset":5,"prefix":"","suffix":" went to"},{"exact":"Paris","entityType":"Location","startOffset":20,"endOffset":25,"prefix":"went to ","suffix":" yesterday"}]`;
-  logger?.debug("Sending entity extraction request", { entityTypes: entityTypesDescription });
+  logger.debug("Sending entity extraction request", { entityTypes: entityTypesDescription });
   const response = await client.generateTextWithMetadata(
     prompt,
     4e3,
     // Increased to handle many entities without truncation
-    0.3
+    0.3,
     // Lower temperature for more consistent extraction
+    // Force grammar-constrained JSON output. Without this, Ollama models
+    // periodically emit malformed JSON (truncated brackets, mid-token
+    // breaks at higher token counts) which silently parse-fails into
+    // [] downstream. The prompt's schema (which keys, what types) still
+    // governs *what* the JSON contains; `format: 'json'` governs that
+    // it's syntactically valid.
+    { format: "json" }
   );
-  logger?.debug("Got entity extraction response", { responseLength: response.text.length });
+  logger.debug("Got entity extraction response", { responseLength: response.text.length });
   try {
     let jsonStr = response.text.trim();
     if (jsonStr.startsWith("```")) {
       jsonStr = jsonStr.replace(/^```(?:json)?\n?/, "").replace(/\n?```$/, "");
     }
     const entities = JSON.parse(jsonStr);
-    logger?.debug("Parsed entities from AI response", { count: entities.length });
+    logger.debug("Parsed entities from AI response", { count: entities.length });
     if (response.stopReason === "max_tokens") {
       const errorMsg = `AI response truncated: Found ${entities.length} entities but response hit max_tokens limit. Increase max_tokens or reduce resource size.`;
-      logger?.error(errorMsg);
+      logger.error(errorMsg);
       throw new Error(errorMsg);
     }
     return entities.map((entity, idx) => {
-      let startOffset = entity.startOffset;
-      let endOffset = entity.endOffset;
-      logger?.debug("Processing entity", {
+      let start = entity.startOffset;
+      let end = entity.endOffset;
+      logger.debug("Processing entity", {
         index: idx + 1,
         total: entities.length,
         type: entity.entityType,
         text: entity.exact,
-        offsetsFromAI: `[${startOffset}:${endOffset}]`
+        offsetsFromAI: `[${start}:${end}]`
       });
-      const extractedText = exact.substring(startOffset, endOffset);
+      const extractedText = exact.substring(start, end);
       let anchorMethod;
       if (extractedText === entity.exact) {
         anchorMethod = "llm-exact";
-        logger?.debug("Entity anchored", {
+        logger.debug("Entity anchored", {
           text: entity.exact,
           entityType: entity.entityType,
           anchorMethod
         });
       } else {
-        logger?.debug("LLM offsets mismatch \u2014 attempting re-anchor", {
+        logger.debug("LLM offsets mismatch \u2014 attempting re-anchor", {
           expected: entity.exact,
-          llmOffsets: `[${startOffset}:${endOffset}]`,
+          llmOffsets: `[${start}:${end}]`,
           foundAtLlmOffsets: extractedText
         });
         let occurrenceCount = 0;
@@ -1293,10 +1147,10 @@ Example output:
         }
         if (occurrenceCount === 0) {
           anchorMethod = "dropped";
-          logger?.error("Entity text not found in resource \u2014 dropping", {
+          logger.error("Entity text not found in resource \u2014 dropping", {
             text: entity.exact,
             entityType: entity.entityType,
-            llmOffsets: `[${startOffset}:${endOffset}]`,
+            llmOffsets: `[${start}:${end}]`,
             anchorMethod,
             resourceStart: exact.substring(0, 200)
           });
@@ -1322,9 +1176,9 @@ Example output:
         }
         if (recoveredOffset !== -1) {
           anchorMethod = "context-recovered";
-          startOffset = recoveredOffset;
-          endOffset = recoveredOffset + entity.exact.length;
-          logger?.debug("Entity anchored", {
+          start = recoveredOffset;
+          end = recoveredOffset + entity.exact.length;
+          logger.debug("Entity anchored", {
             text: entity.exact,
             entityType: entity.entityType,
             anchorMethod,
@@ -1332,9 +1186,9 @@ Example output:
           });
         } else if (occurrenceCount === 1) {
           anchorMethod = "unique-match";
-          startOffset = firstOccurrence;
-          endOffset = firstOccurrence + entity.exact.length;
-          logger?.debug("Entity anchored", {
+          start = firstOccurrence;
+          end = firstOccurrence + entity.exact.length;
+          logger.debug("Entity anchored", {
             text: entity.exact,
             entityType: entity.entityType,
             anchorMethod,
@@ -1342,9 +1196,9 @@ Example output:
           });
         } else {
           anchorMethod = "first-of-many";
-          startOffset = firstOccurrence;
-          endOffset = firstOccurrence + entity.exact.length;
-          logger?.warn("Entity anchored at first of multiple occurrences \u2014 may be wrong", {
+          start = firstOccurrence;
+          end = firstOccurrence + entity.exact.length;
+          logger.warn("Entity anchored at first of multiple occurrences \u2014 may be wrong", {
             text: entity.exact,
             entityType: entity.entityType,
             anchorMethod,
@@ -1359,52 +1213,52 @@ Example output:
       return {
         exact: entity.exact,
         entityType: entity.entityType,
-        startOffset,
-        endOffset,
+        start,
+        end,
         prefix: entity.prefix,
         suffix: entity.suffix
       };
     }).filter((entity) => {
       if (entity === null) {
-        logger?.debug("Filtered entity: null");
+        logger.debug("Filtered entity: null");
         return false;
       }
-      if (entity.startOffset === void 0 || entity.endOffset === void 0) {
-        logger?.warn("Filtered entity: missing offsets", { text: entity.exact });
+      if (entity.start === void 0 || entity.end === void 0) {
+        logger.warn("Filtered entity: missing offsets", { text: entity.exact });
         return false;
       }
-      if (entity.startOffset < 0) {
-        logger?.warn("Filtered entity: negative startOffset", {
+      if (entity.start < 0) {
+        logger.warn("Filtered entity: negative start", {
           text: entity.exact,
-          startOffset: entity.startOffset
+          start: entity.start
         });
         return false;
       }
-      if (entity.endOffset > exact.length) {
-        logger?.warn("Filtered entity: endOffset exceeds text length", {
+      if (entity.end > exact.length) {
+        logger.warn("Filtered entity: end exceeds text length", {
           text: entity.exact,
-          endOffset: entity.endOffset,
+          end: entity.end,
           textLength: exact.length
         });
         return false;
       }
-      const extractedText = exact.substring(entity.startOffset, entity.endOffset);
+      const extractedText = exact.substring(entity.start, entity.end);
       if (extractedText !== entity.exact) {
-        logger?.warn("Filtered entity: offset mismatch", {
+        logger.warn("Filtered entity: offset mismatch", {
           expected: entity.exact,
           got: extractedText,
-          offsets: `[${entity.startOffset}:${entity.endOffset}]`
+          offsets: `[${entity.start}:${entity.end}]`
         });
         return false;
       }
-      logger?.debug("Accepted entity", {
+      logger.debug("Accepted entity", {
         text: entity.exact,
-        offsets: `[${entity.startOffset}:${entity.endOffset}]`
+        offsets: `[${entity.start}:${entity.end}]`
       });
       return true;
     });
   } catch (error) {
-    logger?.error("Failed to parse entity extraction response", {
+    logger.error("Failed to parse entity extraction response", {
       error: error instanceof Error ? error.message : String(error)
     });
     return [];
@@ -1413,8 +1267,8 @@ Example output:
 function getLanguageName(locale) {
   return getLocaleEnglishName(locale) || locale;
 }
-async function generateResourceFromTopic(topic, entityTypes, client, userPrompt, locale, context, temperature, maxTokens, logger, sourceLanguage) {
-  logger?.debug("Generating resource from topic", {
+async function generateResourceFromTopic(topic, entityTypes, client, logger, userPrompt, locale, context, temperature, maxTokens, sourceLanguage) {
+  logger.debug("Generating resource from topic", {
     topicPreview: topic.substring(0, 100),
     entityTypes,
     hasUserPrompt: !!userPrompt,
@@ -1522,15 +1376,15 @@ Requirements:
       content
     };
   };
-  logger?.debug("Sending prompt to inference", {
+  logger.debug("Sending prompt to inference", {
     promptLength: prompt.length,
     temperature: finalTemperature,
     maxTokens: finalMaxTokens
   });
   const response = await client.generateText(prompt, finalMaxTokens, finalTemperature);
-  logger?.debug("Got response from inference", { responseLength: response.length });
+  logger.debug("Got response from inference", { responseLength: response.length });
   const result = parseResponse(response);
-  logger?.debug("Parsed response", {
+  logger.debug("Parsed response", {
     hasTitle: !!result.title,
     titleLength: result.title?.length,
     hasContent: !!result.content,
@@ -1539,13 +1393,16 @@ Requirements:
   return result;
 }
 function buildTextAnnotation(resourceId, userId, generator, motivation, match, body) {
+  const creator = didToAgent(userId);
+  const wasAttributedTo = creator["@id"] === generator["@id"] ? [generator] : [creator, generator];
   return {
     "@context": "http://www.w3.org/ns/anno.jsonld",
     "type": "Annotation",
     "id": generateAnnotationId(),
     motivation,
-    creator: didToAgent(userId),
+    creator,
     generator,
+    wasAttributedTo,
     created: (/* @__PURE__ */ new Date()).toISOString(),
     target: {
       type: "SpecificResource",
@@ -1688,7 +1545,7 @@ async function processReferenceJob(content, inferenceClient, params, userId, gen
     ];
     for (const entity of extractedEntities) {
       try {
-        const validated = validateAndCorrectOffsets(content, entity.startOffset, entity.endOffset, entity.exact);
+        const validated = validateAndCorrectOffsets(content, entity.start, entity.end, entity.exact);
         const ann = buildTextAnnotation(
           params.resourceId,
           userId,
@@ -1718,7 +1575,7 @@ async function processTagJob(content, inferenceClient, params, userId, generator
     const categoryTags = await AnnotationDetection.detectTags(
       content,
       inferenceClient,
-      params.schemaId,
+      params.schema,
       category,
       params.sourceLanguage
     );
@@ -1733,7 +1590,7 @@ async function processTagJob(content, inferenceClient, params, userId, generator
     byCategory[category] = (byCategory[category] ?? 0) + 1;
     return buildTextAnnotation(params.resourceId, userId, generator, "tagging", t, [
       { type: "TextualBody", value: category, purpose: "tagging", format: "text/plain", language: bodyLanguage },
-      { type: "TextualBody", value: params.schemaId, purpose: "classifying", format: "text/plain" }
+      { type: "TextualBody", value: params.schema.id, purpose: "classifying", format: "text/plain" }
     ]);
   });
   onProgress(100, `Complete! Created ${annotations.length} tags`, "creating");
@@ -1742,7 +1599,7 @@ async function processTagJob(content, inferenceClient, params, userId, generator
     result: { tagsFound: tags.length, tagsCreated: annotations.length, byCategory }
   };
 }
-async function processGenerationJob(inferenceClient, params, onProgress) {
+async function processGenerationJob(inferenceClient, params, onProgress, logger) {
   onProgress(20, "Fetching context...", "fetching");
   const title = params.title ?? "Untitled";
   const entityTypes = (params.entityTypes ?? []).map(String);
@@ -1751,13 +1608,12 @@ async function processGenerationJob(inferenceClient, params, onProgress) {
     title,
     entityTypes,
     inferenceClient,
+    logger,
     params.prompt,
     params.language,
     params.context,
     params.temperature,
     params.maxTokens,
-    void 0,
-    // logger
     params.sourceLanguage
   );
   onProgress(85, "Creating resource...", "creating");