@semiont/jobs 0.5.11 → 0.5.13

package/dist/worker-main.js

@@ -1,4 +1,4 @@
-import { createTomlConfigLoader, didToAgent, baseUrl, busRequest, getPrimaryMediaType, textExtractionOf, assembleAnnotation, reconcileSelector, getLocaleEnglishName, isArray, isObject, isString, deriveViews } from '@semiont/core';
+import { createTomlConfigLoader, didToAgent, baseUrl, STARTUP_FETCH_RETRY, retryWithBackoff, isTransientFetchError, busRequest, getPrimaryMediaType, textExtractionOf, assembleAnnotation, reconcileSelector, getLocaleEnglishName, isArray, isObject, isString, deriveViews } from '@semiont/core';
 import { deriveStorageUri } from '@semiont/content';
 import { withSpan, SpanKind, recordJobOutcome } from '@semiont/observability';
 import { generateAnnotationId } from '@semiont/event-sourcing';
@@ -9226,6 +9226,12 @@ function createJobClaimAdapter(options) {
   const errors$ = new import_rxjs.Subject();
   let jobSubscription = null;
   let started = false;
+  let lastQueuedEventAt = null;
+  let lastClaimAt = null;
+  let lastFinishedAt = null;
+  let lastActivityAt = null;
+  let activeSince = null;
+  const iso = (t) => t === null ? null : new Date(t).toISOString();
   const claimJob = async (assignment) => {
     try {
       const job = await busRequest(requestBus, "job:claim", { jobId: assignment.jobId }, 1e4);
@@ -9250,12 +9256,17 @@ function createJobClaimAdapter(options) {
       started = true;
       bus.addChannels?.(["job:queued"]);
       jobSubscription = bus.on$("job:queued").subscribe((event) => {
+        lastQueuedEventAt = Date.now();
         const jobType = event.jobType;
         if (jobTypes.length > 0 && !jobTypes.includes(jobType)) return;
         if (isProcessing$.getValue()) return;
         isProcessing$.next(true);
         claimJob({ jobId: event.jobId, type: jobType, resourceId: event.resourceId }).then((job) => {
           if (job) {
+            const now = Date.now();
+            lastClaimAt = now;
+            lastActivityAt = now;
+            activeSince = now;
             activeJob$.next(job);
           } else {
             isProcessing$.next(false);
@@ -9271,15 +9282,37 @@ function createJobClaimAdapter(options) {
       started = false;
     },
     completeJob: () => {
+      const now = Date.now();
+      lastFinishedAt = now;
+      lastActivityAt = now;
+      activeSince = null;
       activeJob$.next(null);
       isProcessing$.next(false);
       jobsCompleted$.next(jobsCompleted$.getValue() + 1);
     },
     failJob: (jid, error) => {
+      const now = Date.now();
+      lastFinishedAt = now;
+      lastActivityAt = now;
+      activeSince = null;
       activeJob$.next(null);
       isProcessing$.next(false);
       errors$.next({ jobId: jid, error });
     },
+    vitals: () => {
+      const active = activeJob$.getValue();
+      return {
+        lastQueuedEventAt: iso(lastQueuedEventAt),
+        lastClaimAt: iso(lastClaimAt),
+        lastFinishedAt: iso(lastFinishedAt),
+        lastActivityAt: iso(lastActivityAt),
+        activeJob: active && activeSince !== null ? { jobId: active.jobId, type: active.type, since: iso(activeSince) } : null,
+        jobsCompleted: jobsCompleted$.getValue()
+      };
+    },
+    touchActivity: () => {
+      lastActivityAt = Date.now();
+    },
     dispose: () => {
       jobSubscription?.unsubscribe();
       jobSubscription = null;
@@ -9291,6 +9324,41 @@ function createJobClaimAdapter(options) {
     }
   };
 }
+
+// src/workers/inference-call.ts
+var INFERENCE_TIMEOUT_MS = 10 * 6e4;
+async function withTimeout(work, label) {
+  let timer;
+  const timedOut = new Promise((_, reject) => {
+    timer = setTimeout(() => {
+      reject(new Error(
+        `Inference call timed out after ${INFERENCE_TIMEOUT_MS / 6e4} minutes (${label}) \u2014 failing the job to keep the claim loop live`
+      ));
+    }, INFERENCE_TIMEOUT_MS);
+    timer.unref?.();
+  });
+  try {
+    return await Promise.race([work, timedOut]);
+  } catch (err) {
+    work.catch(() => {
+    });
+    throw err;
+  } finally {
+    clearTimeout(timer);
+  }
+}
+function boundedGenerate(client, prompt, maxTokens, temperature, options) {
+  return withTimeout(
+    client.generateText(prompt, maxTokens, temperature, options),
+    `${client.type}:${client.modelId}`
+  );
+}
+function boundedGenerateWithMetadata(client, prompt, maxTokens, temperature, options) {
+  return withTimeout(
+    client.generateTextWithMetadata(prompt, maxTokens, temperature, options),
+    `${client.type}:${client.modelId}`
+  );
+}
 function languageName(tag) {
   return getLocaleEnglishName(tag) || tag;
 }
@@ -9772,7 +9840,7 @@ var AnnotationDetection = class {
    */
   static async detectComments(content, client, instructions, tone, density, language, sourceLanguage) {
     const prompt = MotivationPrompts.buildCommentPrompt(content, instructions, tone, density, language, sourceLanguage);
-    const response = await client.generateTextWithMetadata(prompt, 3e3, 0.4, { format: "json" });
+    const response = await boundedGenerateWithMetadata(client, prompt, 3e3, 0.4, { format: "json" });
     assertNotTruncated(response, "comment");
     return MotivationParsers.parseComments(response.text, content);
   }
@@ -9785,7 +9853,7 @@ var AnnotationDetection = class {
    */
   static async detectHighlights(content, client, instructions, density, sourceLanguage) {
     const prompt = MotivationPrompts.buildHighlightPrompt(content, instructions, density, sourceLanguage);
-    const response = await client.generateTextWithMetadata(prompt, 2e3, 0.3, { format: "json" });
+    const response = await boundedGenerateWithMetadata(client, prompt, 2e3, 0.3, { format: "json" });
     assertNotTruncated(response, "highlight");
     return MotivationParsers.parseHighlights(response.text, content);
   }
@@ -9798,7 +9866,7 @@ var AnnotationDetection = class {
    */
   static async detectAssessments(content, client, instructions, tone, density, language, sourceLanguage) {
     const prompt = MotivationPrompts.buildAssessmentPrompt(content, instructions, tone, density, language, sourceLanguage);
-    const response = await client.generateTextWithMetadata(prompt, 3e3, 0.3, { format: "json" });
+    const response = await boundedGenerateWithMetadata(client, prompt, 3e3, 0.3, { format: "json" });
     assertNotTruncated(response, "assessment");
     return MotivationParsers.parseAssessments(response.text, content);
   }
@@ -9829,7 +9897,7 @@ var AnnotationDetection = class {
       categoryInfo.examples,
       sourceLanguage
     );
-    const response = await client.generateTextWithMetadata(prompt, 4e3, 0.2, { format: "json" });
+    const response = await boundedGenerateWithMetadata(client, prompt, 4e3, 0.2, { format: "json" });
     assertNotTruncated(response, "tag");
     const parsedTags = MotivationParsers.parseTags(response.text);
     return MotivationParsers.validateTagOffsets(parsedTags, content, category);
@@ -9885,7 +9953,8 @@ If no entities are found, respond with an empty array [].
 Example output:
 [{"exact":"Alice","entityType":"Person","prefix":"","suffix":" went to"},{"exact":"Paris","entityType":"Location","prefix":"went to ","suffix":" yesterday"}]`;
   logger2.debug("Sending entity extraction request", { entityTypes: entityTypesDescription });
-  const response = await client.generateTextWithMetadata(
+  const response = await boundedGenerateWithMetadata(
+    client,
     prompt,
     4e3,
     // Increased to handle many entities without truncation
@@ -10135,7 +10204,7 @@ ${formatRequirements}`;
     temperature: finalTemperature,
     maxTokens: finalMaxTokens
   });
-  const response = await client.generateText(prompt, finalMaxTokens, finalTemperature);
+  const response = await boundedGenerate(client, prompt, finalMaxTokens, finalTemperature);
   logger2.debug("Got response from inference", { responseLength: response.length });
   const result = parseResponse(response);
   logger2.debug("Parsed response", {
@@ -10495,10 +10564,9 @@ async function processGenerationJob(inferenceClient, params, onProgress, logger2
       `Unsupported outputMediaType for generation: ${outputMediaType}. Generation produces ${GENERATABLE_MEDIA_TYPES.join(" or ")}.`
     );
   }
-  onProgress(20, "Fetching context...", "fetching");
   const title = params.title ?? "Untitled";
   const entityTypes = (params.entityTypes ?? []).map(String);
-  onProgress(40, "Generating resource...", "generating");
+  onProgress(5, "Generating resource...", "generating");
   const generated = await generateResourceFromTopic(
     title,
     entityTypes,
@@ -10522,7 +10590,7 @@ async function processGenerationJob(inferenceClient, params, onProgress, logger2
     content = resolved.content;
     citations = resolved.citations;
   }
-  onProgress(85, "Creating resource...", "creating");
+  onProgress(95, "Creating resource...", "creating");
   return {
     content,
     title: generated.title ?? title,
@@ -10619,6 +10687,7 @@ async function handleJobInner(adapter, config, job) {
     }
   }
   const onProgress = (percentage, message, stage, extra) => {
+    adapter.touchActivity();
     emitEvent(session, "job:report-progress", {
       ...lifecycleBase,
       percentage,
@@ -10787,6 +10856,43 @@ async function handleJobInner(adapter, config, job) {
 
 // src/worker-runtime.ts
 var import_rxjs2 = __toESM(require_cjs());
+function buildHealthPayload(workers) {
+  return {
+    status: "ok",
+    agents: workers.length,
+    workers: workers.map((w) => w.vitals())
+  };
+}
+var STALL_THRESHOLD_MS = 15 * 6e4;
+var STALL_CHECK_INTERVAL_MS = 6e4;
+function startStallWatchdog(opts) {
+  const { workers, logger: logger2, exit = (code) => process.exit(code) } = opts;
+  const timer = setInterval(() => {
+    const now = Date.now();
+    for (const worker of workers) {
+      const v = worker.vitals();
+      if (!v.activeJob || !v.lastActivityAt) continue;
+      const silentForMs = now - Date.parse(v.lastActivityAt);
+      if (silentForMs <= STALL_THRESHOLD_MS) continue;
+      logger2.error("Worker stalled \u2014 exiting for restart", {
+        provider: v.provider,
+        model: v.model,
+        did: v.did,
+        jobId: v.activeJob.jobId,
+        jobType: v.activeJob.type,
+        processingSince: v.activeJob.since,
+        lastActivityAt: v.lastActivityAt,
+        silentForMs,
+        thresholdMs: STALL_THRESHOLD_MS
+      });
+      clearInterval(timer);
+      exit(1);
+      return;
+    }
+  }, STALL_CHECK_INTERVAL_MS);
+  timer.unref?.();
+  return { dispose: () => clearInterval(timer) };
+}
 function parseBackendUrl(url) {
   const parsed = new URL(url);
   const protocol = parsed.protocol.replace(":", "") === "https" ? "https" : "http";
@@ -10795,19 +10901,34 @@ function parseBackendUrl(url) {
   return { protocol, host, port };
 }
 async function authenticateAgent(opts) {
-  const { backendBaseUrl: backendBaseUrl2, workerSecret: workerSecret2, provider, model } = opts;
+  const { backendBaseUrl: backendBaseUrl2, workerSecret: workerSecret2, provider, model, logger: logger2, retry = STARTUP_FETCH_RETRY } = opts;
   if (!workerSecret2) {
     throw new Error("SEMIONT_WORKER_SECRET is required to authenticate worker agents");
   }
-  const response = await fetch(`${backendBaseUrl2}/api/tokens/agent`, {
-    method: "POST",
-    headers: { "Content-Type": "application/json" },
-    body: JSON.stringify({ secret: workerSecret2, provider, model })
-  });
-  if (!response.ok) {
-    throw new Error(`Agent authentication failed for ${provider}:${model}: ${response.status} ${response.statusText}`);
-  }
-  return await response.json();
+  return retryWithBackoff(
+    async () => {
+      const response = await fetch(`${backendBaseUrl2}/api/tokens/agent`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({ secret: workerSecret2, provider, model })
+      });
+      if (!response.ok) {
+        throw new Error(`Agent authentication failed for ${provider}:${model}: ${response.status} ${response.statusText}`);
+      }
+      return await response.json();
+    },
+    isTransientFetchError,
+    retry,
+    ({ attempt, attempts, delayMs, error }) => {
+      logger2?.warn("Backend unreachable, retrying agent authentication", {
+        agent: `${provider}:${model}`,
+        attempt,
+        attempts,
+        retryInMs: delayMs,
+        error: error instanceof Error ? error.message : String(error)
+      });
+    }
+  );
 }
 async function startAgentWorker(opts) {
   const { group, backendBaseUrl: backendBaseUrl2, workerSecret: workerSecret2, logger: logger2 } = opts;
@@ -10817,7 +10938,8 @@ async function startAgentWorker(opts) {
     backendBaseUrl: backendBaseUrl2,
     workerSecret: workerSecret2,
     provider: inference.type,
-    model: inference.model
+    model: inference.model,
+    logger: logger2
   });
   const generator = didToAgent(did);
   const kbId = `agent-${inference.type}-${inference.model}-${hostname()}`;
@@ -10850,7 +10972,8 @@ async function startAgentWorker(opts) {
           backendBaseUrl: backendBaseUrl2,
           workerSecret: workerSecret2,
           provider: inference.type,
-          model: inference.model
+          model: inference.model,
+          logger: logger2
         });
         return token;
       } catch (err) {
@@ -10881,6 +11004,13 @@ async function startAgentWorker(opts) {
   });
   return {
     session,
+    vitals: () => ({
+      provider: inference.type,
+      model: inference.model,
+      did,
+      jobTypes: group.jobTypes,
+      ...adapter.vitals()
+    }),
     dispose: async () => {
       adapter.dispose();
       await session.dispose();
@@ -10973,7 +11103,7 @@ async function main() {
   const health = createServer((req, res) => {
     if (req.url === "/health") {
       res.writeHead(200, { "Content-Type": "application/json" });
-      res.end(JSON.stringify({ status: "ok", agents: workers.length }));
+      res.end(JSON.stringify(buildHealthPayload(workers)));
     } else {
       res.writeHead(404);
       res.end();
@@ -10982,8 +11112,10 @@ async function main() {
   health.listen(healthPort, () => {
     logger.info("Health endpoint ready", { port: healthPort });
   });
+  const watchdog = startStallWatchdog({ workers, logger });
   const shutdown = async () => {
     logger.info("Shutting down");
+    watchdog.dispose();
     await Promise.all(workers.map((w) => w.dispose()));
     health.close();
     process.exit(0);