@semiont/jobs 0.5.3 → 0.5.4

package/dist/worker-main.js

@@ -1,4 +1,4 @@
-import { createTomlConfigLoader, baseUrl, RESOURCE_BROADCAST_TYPES, resourceId, validateAndCorrectOffsets, didToAgent, getLocaleEnglishName } from '@semiont/core';
+import { createTomlConfigLoader, softwareToAgent, baseUrl, RESOURCE_BROADCAST_TYPES, resourceId, validateAndCorrectOffsets, didToAgent, getLocaleEnglishName } from '@semiont/core';
 import { deriveStorageUri } from '@semiont/content';
 import { withSpan, SpanKind, recordJobOutcome } from '@semiont/observability';
 import { generateAnnotationId } from '@semiont/event-sourcing';
@@ -6388,9 +6388,9 @@ var require_groupBy = __commonJS({
         } else {
           duration = elementOrOptions.duration, element = elementOrOptions.element, connector = elementOrOptions.connector;
         }
-        var groups = /* @__PURE__ */ new Map();
+        var groups2 = /* @__PURE__ */ new Map();
         var notify = function(cb) {
-          groups.forEach(cb);
+          groups2.forEach(cb);
           cb(subscriber);
         };
         var handleError = function(err) {
@@ -6403,9 +6403,9 @@ var require_groupBy = __commonJS({
         var groupBySourceSubscriber = new OperatorSubscriber_1.OperatorSubscriber(subscriber, function(value) {
           try {
             var key_1 = keySelector(value);
-            var group_1 = groups.get(key_1);
+            var group_1 = groups2.get(key_1);
             if (!group_1) {
-              groups.set(key_1, group_1 = connector ? connector() : new Subject_1.Subject());
+              groups2.set(key_1, group_1 = connector ? connector() : new Subject_1.Subject());
               var grouped = createGroupedObservable(key_1, group_1);
               subscriber.next(grouped);
               if (duration) {
@@ -6413,7 +6413,7 @@ var require_groupBy = __commonJS({
                   group_1.complete();
                   durationSubscriber_1 === null || durationSubscriber_1 === void 0 ? void 0 : durationSubscriber_1.unsubscribe();
                 }, void 0, void 0, function() {
-                  return groups.delete(key_1);
+                  return groups2.delete(key_1);
                 });
                 groupBySourceSubscriber.add(innerFrom_1.innerFrom(duration(grouped)).subscribe(durationSubscriber_1));
               }
@@ -6427,7 +6427,7 @@ var require_groupBy = __commonJS({
             return consumer.complete();
           });
         }, handleError, function() {
-          return groups.clear();
+          return groups2.clear();
         }, function() {
           teardownAttempted = true;
           return activeGroups === 0;
@@ -9831,7 +9831,7 @@ var AnnotationDetection = class {
    */
   static async detectComments(content, client, instructions, tone, density, language, sourceLanguage) {
     const prompt = MotivationPrompts.buildCommentPrompt(content, instructions, tone, density, language, sourceLanguage);
-    const response = await client.generateText(prompt, 3e3, 0.4);
+    const response = await client.generateText(prompt, 3e3, 0.4, { format: "json" });
     return MotivationParsers.parseComments(response, content);
   }
   /**
@@ -9843,7 +9843,7 @@ var AnnotationDetection = class {
    */
   static async detectHighlights(content, client, instructions, density, sourceLanguage) {
     const prompt = MotivationPrompts.buildHighlightPrompt(content, instructions, density, sourceLanguage);
-    const response = await client.generateText(prompt, 2e3, 0.3);
+    const response = await client.generateText(prompt, 2e3, 0.3, { format: "json" });
     return MotivationParsers.parseHighlights(response, content);
   }
   /**
@@ -9855,7 +9855,7 @@ var AnnotationDetection = class {
    */
   static async detectAssessments(content, client, instructions, tone, density, language, sourceLanguage) {
     const prompt = MotivationPrompts.buildAssessmentPrompt(content, instructions, tone, density, language, sourceLanguage);
-    const response = await client.generateText(prompt, 3e3, 0.3);
+    const response = await client.generateText(prompt, 3e3, 0.3, { format: "json" });
     return MotivationParsers.parseAssessments(response, content);
   }
   /**
@@ -9885,12 +9885,12 @@ var AnnotationDetection = class {
       categoryInfo.examples,
       sourceLanguage
     );
-    const response = await client.generateText(prompt, 4e3, 0.2);
+    const response = await client.generateText(prompt, 4e3, 0.2, { format: "json" });
     const parsedTags = MotivationParsers.parseTags(response);
     return MotivationParsers.validateTagOffsets(parsedTags, content, category);
   }
 };
-async function extractEntities(exact, entityTypes, client, includeDescriptiveReferences = false, logger2, sourceLanguage) {
+async function extractEntities(exact, entityTypes, client, includeDescriptiveReferences, logger2, sourceLanguage) {
   const entityTypesDescription = entityTypes.map((et) => {
     if (typeof et === "string") {
       return et;
@@ -9941,48 +9941,57 @@ If no entities are found, respond with an empty array [].
 
 Example output:
 [{"exact":"Alice","entityType":"Person","startOffset":0,"endOffset":5,"prefix":"","suffix":" went to"},{"exact":"Paris","entityType":"Location","startOffset":20,"endOffset":25,"prefix":"went to ","suffix":" yesterday"}]`;
+  logger2.debug("Sending entity extraction request", { entityTypes: entityTypesDescription });
   const response = await client.generateTextWithMetadata(
     prompt,
     4e3,
     // Increased to handle many entities without truncation
-    0.3
+    0.3,
     // Lower temperature for more consistent extraction
+    // Force grammar-constrained JSON output. Without this, Ollama models
+    // periodically emit malformed JSON (truncated brackets, mid-token
+    // breaks at higher token counts) which silently parse-fails into
+    // [] downstream. The prompt's schema (which keys, what types) still
+    // governs *what* the JSON contains; `format: 'json'` governs that
+    // it's syntactically valid.
+    { format: "json" }
   );
+  logger2.debug("Got entity extraction response", { responseLength: response.text.length });
   try {
     let jsonStr = response.text.trim();
     if (jsonStr.startsWith("```")) {
       jsonStr = jsonStr.replace(/^```(?:json)?\n?/, "").replace(/\n?```$/, "");
     }
     const entities = JSON.parse(jsonStr);
-    logger2?.debug("Parsed entities from AI response", { count: entities.length });
+    logger2.debug("Parsed entities from AI response", { count: entities.length });
     if (response.stopReason === "max_tokens") {
       const errorMsg = `AI response truncated: Found ${entities.length} entities but response hit max_tokens limit. Increase max_tokens or reduce resource size.`;
-      logger2?.error(errorMsg);
+      logger2.error(errorMsg);
       throw new Error(errorMsg);
     }
     return entities.map((entity, idx) => {
-      let startOffset = entity.startOffset;
-      let endOffset = entity.endOffset;
-      logger2?.debug("Processing entity", {
+      let start = entity.startOffset;
+      let end = entity.endOffset;
+      logger2.debug("Processing entity", {
         index: idx + 1,
         total: entities.length,
         type: entity.entityType,
         text: entity.exact,
-        offsetsFromAI: `[${startOffset}:${endOffset}]`
+        offsetsFromAI: `[${start}:${end}]`
       });
-      const extractedText = exact.substring(startOffset, endOffset);
+      const extractedText = exact.substring(start, end);
       let anchorMethod;
       if (extractedText === entity.exact) {
         anchorMethod = "llm-exact";
-        logger2?.debug("Entity anchored", {
+        logger2.debug("Entity anchored", {
           text: entity.exact,
           entityType: entity.entityType,
           anchorMethod
         });
       } else {
-        logger2?.debug("LLM offsets mismatch \u2014 attempting re-anchor", {
+        logger2.debug("LLM offsets mismatch \u2014 attempting re-anchor", {
           expected: entity.exact,
-          llmOffsets: `[${startOffset}:${endOffset}]`,
+          llmOffsets: `[${start}:${end}]`,
           foundAtLlmOffsets: extractedText
         });
         let occurrenceCount = 0;
@@ -9995,10 +10004,10 @@ Example output:
         }
         if (occurrenceCount === 0) {
           anchorMethod = "dropped";
-          logger2?.error("Entity text not found in resource \u2014 dropping", {
+          logger2.error("Entity text not found in resource \u2014 dropping", {
             text: entity.exact,
             entityType: entity.entityType,
-            llmOffsets: `[${startOffset}:${endOffset}]`,
+            llmOffsets: `[${start}:${end}]`,
             anchorMethod,
             resourceStart: exact.substring(0, 200)
           });
@@ -10024,9 +10033,9 @@ Example output:
         }
         if (recoveredOffset !== -1) {
           anchorMethod = "context-recovered";
-          startOffset = recoveredOffset;
-          endOffset = recoveredOffset + entity.exact.length;
-          logger2?.debug("Entity anchored", {
+          start = recoveredOffset;
+          end = recoveredOffset + entity.exact.length;
+          logger2.debug("Entity anchored", {
             text: entity.exact,
             entityType: entity.entityType,
             anchorMethod,
@@ -10034,9 +10043,9 @@ Example output:
           });
         } else if (occurrenceCount === 1) {
           anchorMethod = "unique-match";
-          startOffset = firstOccurrence;
-          endOffset = firstOccurrence + entity.exact.length;
-          logger2?.debug("Entity anchored", {
+          start = firstOccurrence;
+          end = firstOccurrence + entity.exact.length;
+          logger2.debug("Entity anchored", {
             text: entity.exact,
             entityType: entity.entityType,
             anchorMethod,
@@ -10044,9 +10053,9 @@ Example output:
           });
         } else {
           anchorMethod = "first-of-many";
-          startOffset = firstOccurrence;
-          endOffset = firstOccurrence + entity.exact.length;
-          logger2?.warn("Entity anchored at first of multiple occurrences \u2014 may be wrong", {
+          start = firstOccurrence;
+          end = firstOccurrence + entity.exact.length;
+          logger2.warn("Entity anchored at first of multiple occurrences \u2014 may be wrong", {
             text: entity.exact,
             entityType: entity.entityType,
             anchorMethod,
@@ -10061,58 +10070,71 @@ Example output:
       return {
         exact: entity.exact,
         entityType: entity.entityType,
-        startOffset,
-        endOffset,
+        start,
+        end,
         prefix: entity.prefix,
         suffix: entity.suffix
       };
     }).filter((entity) => {
       if (entity === null) {
-        logger2?.debug("Filtered entity: null");
+        logger2.debug("Filtered entity: null");
         return false;
       }
-      if (entity.startOffset === void 0 || entity.endOffset === void 0) {
-        logger2?.warn("Filtered entity: missing offsets", { text: entity.exact });
+      if (entity.start === void 0 || entity.end === void 0) {
+        logger2.warn("Filtered entity: missing offsets", { text: entity.exact });
         return false;
       }
-      if (entity.startOffset < 0) {
-        logger2?.warn("Filtered entity: negative startOffset", {
+      if (entity.start < 0) {
+        logger2.warn("Filtered entity: negative start", {
           text: entity.exact,
-          startOffset: entity.startOffset
+          start: entity.start
         });
         return false;
       }
-      if (entity.endOffset > exact.length) {
-        logger2?.warn("Filtered entity: endOffset exceeds text length", {
+      if (entity.end > exact.length) {
+        logger2.warn("Filtered entity: end exceeds text length", {
           text: entity.exact,
-          endOffset: entity.endOffset,
+          end: entity.end,
           textLength: exact.length
         });
         return false;
       }
-      const extractedText = exact.substring(entity.startOffset, entity.endOffset);
+      const extractedText = exact.substring(entity.start, entity.end);
       if (extractedText !== entity.exact) {
-        logger2?.warn("Filtered entity: offset mismatch", {
+        logger2.warn("Filtered entity: offset mismatch", {
           expected: entity.exact,
           got: extractedText,
-          offsets: `[${entity.startOffset}:${entity.endOffset}]`
+          offsets: `[${entity.start}:${entity.end}]`
         });
         return false;
       }
-      logger2?.debug("Accepted entity", {
+      logger2.debug("Accepted entity", {
         text: entity.exact,
-        offsets: `[${entity.startOffset}:${entity.endOffset}]`
+        offsets: `[${entity.start}:${entity.end}]`
       });
       return true;
     });
   } catch (error) {
+    logger2.error("Failed to parse entity extraction response", {
+      error: error instanceof Error ? error.message : String(error)
+    });
     return [];
   }
 }
 function getLanguageName(locale) {
   return getLocaleEnglishName(locale) || locale;
 }
-async function generateResourceFromTopic(topic, entityTypes, client, userPrompt, locale, context, temperature, maxTokens, logger2, sourceLanguage) {
+async function generateResourceFromTopic(topic, entityTypes, client, logger2, userPrompt, locale, context, temperature, maxTokens, sourceLanguage) {
+  logger2.debug("Generating resource from topic", {
+    topicPreview: topic.substring(0, 100),
+    entityTypes,
+    hasUserPrompt: !!userPrompt,
+    locale,
+    sourceLanguage,
+    hasContext: !!context,
+    temperature,
+    maxTokens
+  });
   const finalTemperature = temperature ?? 0.7;
   const finalMaxTokens = maxTokens ?? 500;
   const languageInstruction = locale && locale !== "en" ? `
@@ -10211,18 +10233,33 @@ Requirements:
       content
     };
   };
+  logger2.debug("Sending prompt to inference", {
+    promptLength: prompt.length,
+    temperature: finalTemperature,
+    maxTokens: finalMaxTokens
+  });
   const response = await client.generateText(prompt, finalMaxTokens, finalTemperature);
+  logger2.debug("Got response from inference", { responseLength: response.length });
   const result = parseResponse(response);
+  logger2.debug("Parsed response", {
+    hasTitle: !!result.title,
+    titleLength: result.title?.length,
+    hasContent: !!result.content,
+    contentLength: result.content?.length
+  });
   return result;
 }
 function buildTextAnnotation(resourceId, userId, generator, motivation, match, body) {
+  const creator = didToAgent(userId);
+  const wasAttributedTo = creator["@id"] === generator["@id"] ? [generator] : [creator, generator];
   return {
     "@context": "http://www.w3.org/ns/anno.jsonld",
     "type": "Annotation",
     "id": generateAnnotationId(),
     motivation,
-    creator: didToAgent(userId),
+    creator,
     generator,
+    wasAttributedTo,
     created: (/* @__PURE__ */ new Date()).toISOString(),
     target: {
       type: "SpecificResource",
@@ -10365,7 +10402,7 @@ async function processReferenceJob(content, inferenceClient, params, userId, gen
     ];
     for (const entity of extractedEntities) {
       try {
-        const validated = validateAndCorrectOffsets(content, entity.startOffset, entity.endOffset, entity.exact);
+        const validated = validateAndCorrectOffsets(content, entity.start, entity.end, entity.exact);
         const ann = buildTextAnnotation(
           params.resourceId,
           userId,
@@ -10419,7 +10456,7 @@ async function processTagJob(content, inferenceClient, params, userId, generator
     result: { tagsFound: tags.length, tagsCreated: annotations.length, byCategory }
   };
 }
-async function processGenerationJob(inferenceClient, params, onProgress) {
+async function processGenerationJob(inferenceClient, params, onProgress, logger2) {
   onProgress(20, "Fetching context...", "fetching");
   const title = params.title ?? "Untitled";
   const entityTypes = (params.entityTypes ?? []).map(String);
@@ -10428,13 +10465,12 @@ async function processGenerationJob(inferenceClient, params, onProgress) {
     title,
     entityTypes,
     inferenceClient,
+    logger2,
     params.prompt,
     params.language,
     params.context,
     params.temperature,
     params.maxTokens,
-    void 0,
-    // logger
     params.sourceLanguage
   );
   onProgress(85, "Creating resource...", "creating");
@@ -10509,7 +10545,7 @@ async function handleJob(adapter, config, job) {
   }
 }
 async function handleJobInner(adapter, config, job) {
-  const { session } = config;
+  const { session, inferenceClient, generator } = config;
   const { resourceId, userId, jobId, type: jobType } = job;
   const annotationId = job.params.referenceId;
   const lifecycleBase = {
@@ -10520,12 +10556,10 @@ async function handleJobInner(adapter, config, job) {
     ...annotationId ? { annotationId } : {}
   };
   await emitEvent(session, "job:start", lifecycleBase);
-  const engine = config.engines[jobType];
-  if (!engine) {
-    adapter.failJob(jobId, `No inference engine configured for job type: ${jobType}`);
+  if (!config.jobTypes.includes(jobType)) {
+    adapter.failJob(jobId, `Worker not configured for job type: ${jobType}`);
     return;
   }
-  const { inferenceClient, generator } = engine;
   const onProgress = (percentage, message, stage, extra) => {
     emitEvent(session, "job:report-progress", {
       ...lifecycleBase,
@@ -10605,7 +10639,8 @@ async function handleJobInner(adapter, config, job) {
       job.params,
       userId,
       generator,
-      onProgress
+      onProgress,
+      config.logger
     );
     for (const ann of annotations) {
       await emitEvent(session, "mark:create", { annotation: ann, userId, resourceId });
@@ -10637,7 +10672,8 @@ async function handleJobInner(adapter, config, job) {
     const genResult = await processGenerationJob(
       inferenceClient,
       job.params,
-      onProgress
+      onProgress,
+      config.logger
     );
     const genParams = job.params;
     const storageUri = deriveStorageUri(genResult.title, genResult.format);
@@ -10646,7 +10682,6 @@ async function handleJobInner(adapter, config, job) {
       file: Buffer.from(genResult.content),
       format: genResult.format,
       storageUri,
-      creationMethod: "generated",
       sourceResourceId: resourceId,
       ...genParams.referenceId ? { sourceAnnotationId: genParams.referenceId } : {},
       ...genParams.prompt ? { generationPrompt: genParams.prompt } : {},
@@ -10718,24 +10753,20 @@ function toClientConfig(w) {
     ...w.apiKey && { apiKey: w.apiKey }
   };
 }
-var clientCache = /* @__PURE__ */ new Map();
-var engines = {};
+var groups = /* @__PURE__ */ new Map();
 for (const jobType of ALL_JOB_TYPES) {
-  const w = resolveWorker(jobType);
-  const key = clientKey(w);
-  let client = clientCache.get(key);
-  if (!client) {
-    client = createInferenceClient(toClientConfig(w), logger);
-    clientCache.set(key, client);
-  }
-  const generator = {
-    "@type": "SoftwareAgent",
-    name: `worker-pool / ${w.type} ${w.model}`,
-    worker: "worker-pool",
-    inferenceProvider: w.type,
-    model: w.model
-  };
-  engines[jobType] = { inferenceClient: client, generator };
+  const inference = resolveWorker(jobType);
+  const key = clientKey(inference);
+  let group = groups.get(key);
+  if (!group) {
+    group = {
+      inference,
+      jobTypes: [],
+      client: createInferenceClient(toClientConfig(inference), logger)
+    };
+    groups.set(key, group);
+  }
+  group.jobTypes.push(jobType);
 }
 function parseBackendUrl(url) {
   const parsed = new URL(url);
@@ -10744,35 +10775,35 @@ function parseBackendUrl(url) {
   const port = parsed.port ? Number(parsed.port) : protocol === "https" ? 443 : 80;
   return { protocol, host, port };
 }
-async function authenticate() {
+async function authenticateAgent(provider, model) {
   if (!workerSecret) {
-    logger.warn("No SEMIONT_WORKER_SECRET set \u2014 using empty token");
-    return "";
+    throw new Error("SEMIONT_WORKER_SECRET is required to authenticate worker agents");
   }
-  const response = await fetch(`${backendBaseUrl}/api/tokens/worker`, {
+  const response = await fetch(`${backendBaseUrl}/api/tokens/agent`, {
     method: "POST",
     headers: { "Content-Type": "application/json" },
-    body: JSON.stringify({ secret: workerSecret })
+    body: JSON.stringify({ secret: workerSecret, provider, model })
   });
   if (!response.ok) {
-    throw new Error(`Authentication failed: ${response.status} ${response.statusText}`);
+    throw new Error(`Agent authentication failed for ${provider}:${model}: ${response.status} ${response.statusText}`);
   }
-  const { token } = await response.json();
-  return token;
+  return await response.json();
 }
-async function main() {
-  const { initObservabilityNode } = await import('@semiont/observability/node');
-  initObservabilityNode({ serviceName: "semiont-worker" });
-  logger.info("Authenticating", { baseUrl: backendBaseUrl });
-  const initialToken = await authenticate();
-  logger.info("Authenticated");
+async function startAgentWorker(group) {
+  const { inference } = group;
   const { protocol, host, port } = parseBackendUrl(backendBaseUrl);
-  const kbId = `worker-${hostname()}`;
+  const { token: initialToken, did } = await authenticateAgent(inference.type, inference.model);
+  const generator = softwareToAgent({
+    domain: host,
+    provider: inference.type,
+    model: inference.model
+  });
+  const kbId = `agent-${inference.type}-${inference.model}-${hostname()}`;
   const endpoint = { kind: "http", host, port, protocol };
   const kb = {
     id: kbId,
-    label: `Worker pool @ ${host}`,
-    email: `worker-pool@${host}`,
+    label: `${inference.type} / ${inference.model} @ ${host}`,
+    email: `agent@${host}`,
     endpoint
   };
   const storage = new InMemorySessionStorage();
@@ -10793,37 +10824,58 @@ async function main() {
     token$,
     refresh: async () => {
       try {
-        return await authenticate();
+        const { token } = await authenticateAgent(inference.type, inference.model);
+        return token;
       } catch (err) {
-        logger.error("Worker token refresh failed", {
-          error: err instanceof Error ? err.message : String(err)
+        logger.error("Agent token refresh failed", {
+          error: err instanceof Error ? err.message : String(err),
+          agent: did
         });
         return null;
       }
     },
-    // No validate callback — workers are service principals with no
-    // user record to fetch. `session.user$` stays null.
     onError: (err) => {
-      logger.error("Session error", { code: err.code, message: err.message });
+      logger.error("Session error", { code: err.code, message: err.message, agent: did });
     }
   });
   await session.ready;
-  const workerVm = startWorkerProcess({
+  const adapter = startWorkerProcess({
     session,
-    jobTypes: ALL_JOB_TYPES,
-    engines,
+    jobTypes: group.jobTypes,
+    inferenceClient: group.client,
+    generator,
     logger
   });
-  logger.info("Connected", {
+  logger.info("Agent ready", {
+    did,
+    provider: inference.type,
+    model: inference.model,
+    jobTypes: group.jobTypes
+  });
+  return {
+    session,
+    dispose: async () => {
+      adapter.dispose();
+      await session.dispose();
+    }
+  };
+}
+async function main() {
+  const { initObservabilityNode } = await import('@semiont/observability/node');
+  initObservabilityNode({ serviceName: "semiont-worker" });
+  logger.info("Starting agents", {
     baseUrl: backendBaseUrl,
-    engines: Object.fromEntries(
-      Object.entries(engines).map(([jt, e]) => [jt, `${e.generator.inferenceProvider} / ${e.generator.model}`])
-    )
+    agents: Array.from(groups.values()).map((g) => ({
+      provider: g.inference.type,
+      model: g.inference.model,
+      jobTypes: g.jobTypes
+    }))
   });
+  const workers = await Promise.all(Array.from(groups.values()).map(startAgentWorker));
   const health = createServer((req, res) => {
     if (req.url === "/health") {
       res.writeHead(200, { "Content-Type": "application/json" });
-      res.end(JSON.stringify({ status: "ok" }));
+      res.end(JSON.stringify({ status: "ok", agents: workers.length }));
     } else {
       res.writeHead(404);
       res.end();
@@ -10834,8 +10886,7 @@ async function main() {
   });
   const shutdown = async () => {
     logger.info("Shutting down");
-    workerVm.dispose();
-    await session.dispose();
+    await Promise.all(workers.map((w) => w.dispose()));
     health.close();
     process.exit(0);
   };