@semiont/jobs 0.5.27 → 0.5.29

package/dist/worker-main.js

@@ -1,12 +1,12 @@
-import { createTomlConfigLoader, didToAgent, baseUrl, STARTUP_FETCH_RETRY, retryWithBackoff, isTransientFetchError, busRequest, resourceId, getPrimaryMediaType, isGenerationJobParams, assembleAnnotation, findClaimSpan, textExtractionOf, reconcileSelector, GENERATABLE_MEDIA_TYPES, locate, createFragmentSelector, getLocaleEnglishName, estimateTokens, chunkText, isObject, isString, deriveViews } from '@semiont/core';
-import { anchoredTextStoreOverTransport, deriveStorageUri, extractPdfTextLayer, EXTRACTORS, calculateChecksum, withinByteBudget, MAX_PDF_BYTES } from '@semiont/content';
+import { createTomlConfigLoader, didToAgent, baseUrl, STARTUP_FETCH_RETRY, retryWithBackoff, isTransientFetchError, busRequest, resourceId, getPrimaryMediaType, isGenerationJobParams, capabilitiesOf, assembleAnnotation, findClaimSpan, textExtractionOf, reconcileSelector, GENERATABLE_MEDIA_TYPES, locate, createFragmentSelector, getLocaleEnglishName, estimateTokens, chunkText, isObject, isString, deriveViews } from '@semiont/core';
+import { archivistContentReads, extractPdfTextLayer, EXTRACTORS, calculateChecksum, withinByteBudget, MAX_PDF_BYTES } from '@semiont/content';
 import { withSpan, SpanKind, recordJobOutcome } from '@semiont/observability';
 import { execFileSync } from 'child_process';
 import { existsSync, readFileSync, mkdtempSync, writeFileSync, rmSync } from 'fs';
 import { homedir, hostname, tmpdir } from 'os';
 import { join } from 'path';
 import { generateAnnotationId } from '@semiont/event-sourcing';
-import { InMemorySessionStorage, setStoredSession, kbBackendUrl, SemiontClient, SemiontSession } from '@semiont/sdk';
+import { InMemorySessionStorage, setStoredSession, kbGatewayUrl, SemiontClient, SemiontSession } from '@semiont/sdk';
 import { HttpTransport, HttpContentTransport } from '@semiont/http-transport';
 import { createInferenceClient } from '@semiont/inference';
 import { createServer } from 'http';
@@ -9390,9 +9390,9 @@ async function withTimeout(work, label, onHeartbeat) {
     if (heartbeat) clearInterval(heartbeat);
   }
 }
-function boundedGenerate(client, prompt, maxTokens, temperature, onHeartbeat) {
+function boundedGenerateWithMetadata(client, prompt, maxTokens, temperature, onHeartbeat) {
   return spanned(client, "text", maxTokens, () => withTimeout(
-    client.generateText(prompt, maxTokens, temperature),
+    client.generateTextWithMetadata(prompt, maxTokens, temperature),
     `${client.type}:${client.modelId}`,
     onHeartbeat
   ));
@@ -10175,6 +10175,7 @@ var SEMANTIC_MATCH_CHARS = 240;
 function idLabel(resourceId, annotationId) {
   return `[${resourceId}${annotationId ? `/${annotationId}` : ""}]`;
 }
+var DEFAULT_MAX_TOKENS = 500;
 async function generateResourceFromTopic(topic, entityTypes, client, logger2, userPrompt, locale, context, temperature, maxTokens, sourceLanguage, outputMediaType = "text/markdown", task = "resource", structure, cite = false, repair) {
   logger2.debug("Generating resource from topic", {
     topicPreview: topic.substring(0, 100),
@@ -10190,7 +10191,7 @@ async function generateResourceFromTopic(topic, entityTypes, client, logger2, us
     structure
   });
   const finalTemperature = temperature ?? 0.7;
-  const finalMaxTokens = maxTokens ?? 500;
+  const finalMaxTokens = maxTokens ?? DEFAULT_MAX_TOKENS;
   const languageInstruction = locale && locale !== "en" ? `
 
 IMPORTANT: Write the entire resource in ${getLanguageName(locale)}.` : "";
@@ -10217,6 +10218,9 @@ The source resource and embedded context are in ${getLanguageName(sourceLanguage
           parts.push(`- ${label}: ${bodyItem.value}`);
         }
       }
+      if (focus.userHint) {
+        parts.push(`- User hint (steers what to generate): ${focus.userHint}`);
+      }
       annotationSection = `
 
 Annotation context:
@@ -10375,16 +10379,16 @@ ${formatRequirements}`;
     temperature: finalTemperature,
     maxTokens: finalMaxTokens
   });
-  const response = await boundedGenerate(client, prompt, finalMaxTokens, finalTemperature);
-  logger2.debug("Got response from inference", { responseLength: response.length });
-  const result = parseResponse(response);
+  const response = await boundedGenerateWithMetadata(client, prompt, finalMaxTokens, finalTemperature);
+  logger2.debug("Got response from inference", { responseLength: response.text.length, stopReason: response.stopReason });
+  const result = parseResponse(response.text);
   logger2.debug("Parsed response", {
     hasTitle: !!result.title,
     titleLength: result.title?.length,
     hasContent: !!result.content,
     contentLength: result.content?.length
   });
-  return result;
+  return { ...result, truncated: response.stopReason === "max_tokens" };
 }
 var PINNED_CREATION_TIMESTAMP = 17e8;
 var MAX_COMPILE_REPAIRS = 2;
@@ -10616,7 +10620,7 @@ async function processHighlightJob(content, inferenceClient, params, buildAnnota
   onProgress(100, { code: "complete-created", count: annotations.length, kind: "highlight" }, echo);
   return {
     annotations,
-    result: { highlightsFound: highlights.length, highlightsCreated: annotations.length }
+    result: { kind: "highlight-annotation", highlightsFound: highlights.length, highlightsCreated: annotations.length }
   };
 }
 function detectionEcho(p) {
@@ -10656,7 +10660,7 @@ async function processCommentJob(content, inferenceClient, params, buildAnnotati
   onProgress(100, { code: "complete-created", count: annotations.length, kind: "comment" }, echo);
   return {
     annotations,
-    result: { commentsFound: comments.length, commentsCreated: annotations.length }
+    result: { kind: "comment-annotation", commentsFound: comments.length, commentsCreated: annotations.length }
   };
 }
 async function processAssessmentJob(content, inferenceClient, params, buildAnnotation, onProgress) {
@@ -10696,7 +10700,7 @@ async function processAssessmentJob(content, inferenceClient, params, buildAnnot
   onProgress(100, { code: "complete-created", count: annotations.length, kind: "assessment" }, echo);
   return {
     annotations,
-    result: { assessmentsFound: assessments.length, assessmentsCreated: annotations.length }
+    result: { kind: "assessment-annotation", assessmentsFound: assessments.length, assessmentsCreated: annotations.length }
   };
 }
 async function processReferenceJob(content, inferenceClient, params, buildAnnotation, onProgress, logger2) {
@@ -10786,7 +10790,7 @@ async function processReferenceJob(content, inferenceClient, params, buildAnnota
   onProgress(100, { code: "complete-created", count: annotations.length, kind: "reference" }, { requestParams });
   return {
     annotations,
-    result: { totalFound, totalEmitted: annotations.length, errors }
+    result: { kind: "reference-annotation", totalFound, totalEmitted: annotations.length, errors }
   };
 }
 async function processTagJob(content, inferenceClient, params, buildAnnotation, onProgress) {
@@ -10843,7 +10847,7 @@ async function processTagJob(content, inferenceClient, params, buildAnnotation,
   onProgress(100, { code: "complete-created", count: annotations.length, kind: "tag" });
   return {
     annotations,
-    result: { tagsFound: tags.length, tagsCreated: annotations.length, byCategory }
+    result: { kind: "tag-annotation", tagsFound: tags.length, tagsCreated: annotations.length, byCategory }
   };
 }
 function assertWithinOutputBudget(byteLength) {
@@ -10890,7 +10894,17 @@ async function processGenerationJob(inferenceClient, params, onProgress, logger2
     }
     let compiled = compileTypst(source);
     let repairs = 0;
-    while ("error" in compiled && repairs < MAX_COMPILE_REPAIRS) {
+    while ("error" in compiled) {
+      if (generated2.truncated) {
+        throw new Error(
+          `Generation stopped at the maxTokens ceiling (${params.maxTokens ?? DEFAULT_MAX_TOKENS} tokens) and the cut-off Typst source does not compile \u2014 repair cannot help; raise maxTokens. Compile error: ${compiled.error}`
+        );
+      }
+      if (repairs >= MAX_COMPILE_REPAIRS) {
+        throw new Error(
+          `Typst compilation failed after ${MAX_COMPILE_REPAIRS} repair attempts: ${compiled.error}`
+        );
+      }
       repairs++;
       logger2.warn("Typst compile failed \u2014 feeding the error back for repair", {
         attempt: repairs,
@@ -10922,21 +10936,19 @@ async function processGenerationJob(inferenceClient, params, onProgress, logger2
       }
       compiled = compileTypst(source);
     }
-    if ("error" in compiled) {
-      throw new Error(
-        `Typst compilation failed after ${MAX_COMPILE_REPAIRS} repair attempts: ${compiled.error}`
-      );
-    }
     assertWithinOutputBudget(compiled.pdf.byteLength);
     onProgress(95, { code: "creating-resource" });
+    onProgress(100, { code: "complete-generated", truncated: generated2.truncated });
     return {
       content: compiled.pdf,
-      title: generated2.title ?? title,
+      title,
       format: outputMediaType,
       citations: citations2,
       result: {
+        kind: "generation",
         resourceId: "",
-        resourceName: generated2.title ?? title
+        resourceName: title,
+        truncated: generated2.truncated
       }
     };
   }
@@ -10967,29 +10979,32 @@ async function processGenerationJob(inferenceClient, params, onProgress, logger2
   onProgress(95, { code: "creating-resource" });
   const artifact = new TextEncoder().encode(content);
   assertWithinOutputBudget(artifact.byteLength);
+  onProgress(100, { code: "complete-generated", truncated: generated.truncated });
   return {
     content: artifact,
-    title: generated.title ?? title,
+    title,
     format: outputMediaType,
     citations,
     result: {
+      kind: "generation",
       resourceId: "",
-      resourceName: generated.title ?? title
+      resourceName: title,
+      truncated: generated.truncated
     }
   };
 }
 
 // src/workers/detection/prepare-detection.ts
-async function prepareDetection(mediaType, session, resourceId, userId, generator, store) {
+async function prepareDetection(mediaType, content, resourceId, userId, generator, store) {
   const extractor = EXTRACTORS[textExtractionOf(mediaType)];
   if (!extractor) return { declined: "no-extractor" };
-  const { data } = await session.client.browse.resourceRepresentation(resourceId);
+  const { data } = await content.getBinary(resourceId);
   const bytes = Buffer.from(data);
   const extracted = await extractor.extract(bytes, mediaType, {
     key: calculateChecksum(bytes),
     store
   });
-  if ("declined" in extracted) return extracted;
+  if (extracted.kind === "declined") return extracted;
   if (!extracted.text.trim()) return { declined: "empty" };
   const items = extracted.items;
   if (items && items.length > 0) {
@@ -11099,7 +11114,7 @@ async function handleJobInner(adapter, config, job) {
     const mediaType = getPrimaryMediaType(descriptor);
     const source = await withSpan(
       "detection:prepare",
-      () => prepareDetection(mediaType ?? "", session, resourceId$1, userId, generator, config.anchoredTextStore),
+      () => prepareDetection(mediaType ?? "", config.contentReads, resourceId$1, userId, generator, config.anchoredTextStore),
       { attrs: { "resource.id": resourceId$1, "media.type": mediaType ?? "unknown" } }
     );
     if ("declined" in source) {
@@ -11109,6 +11124,7 @@ async function handleJobInner(adapter, config, job) {
       await emitEvent(session, "job:complete", {
         ...lifecycleBase,
         result: {
+          kind: "declined",
           declined: true,
           reason: source.declined
         }
@@ -11227,7 +11243,16 @@ async function handleJobInner(adapter, config, job) {
     );
     const genParams = job.params;
     const genReferenceId = referenceIdOf(job);
-    const storageUri = deriveStorageUri(genResult.title, genResult.format);
+    const storageUri = job.params.storageUri;
+    const expectedExtension = capabilitiesOf(genResult.format)?.extension;
+    if (expectedExtension && !storageUri.toLowerCase().endsWith(expectedExtension)) {
+      config.logger.warn("Storage URI extension does not match the generated format \u2014 writing it as requested", {
+        jobId,
+        storageUri,
+        format: genResult.format,
+        expectedExtension
+      });
+    }
     const { resourceId: newResourceId } = await session.client.yield.resource({
       name: genResult.title,
       file: Buffer.from(genResult.content),
@@ -11304,7 +11329,7 @@ async function handleJobInner(adapter, config, job) {
     }
     await emitEvent(session, "job:complete", {
       ...lifecycleBase,
-      result: { resourceId: newResourceId, resourceName: genResult.title }
+      result: { kind: "generation", resourceId: newResourceId, resourceName: genResult.title, truncated: genResult.result.truncated }
     });
     adapter.completeJob();
   } else {
@@ -11312,6 +11337,29 @@ async function handleJobInner(adapter, config, job) {
   }
 }
 
+// src/anchored-text-over-bus.ts
+function anchoredTextOverBus(client, logger2) {
+  return {
+    async read(key) {
+      try {
+        return await client.browse.anchoredTextByChecksum(key);
+      } catch (error) {
+        logger2?.debug("Anchored-text consult failed \u2014 treating as miss", {
+          key,
+          reason: error instanceof Error ? error.message : String(error)
+        });
+        return null;
+      }
+    },
+    async write(key) {
+      logger2?.debug("Anchored-text write skipped \u2014 the Smelter is the sole writer", { key });
+    },
+    async list() {
+      return [];
+    }
+  };
+}
+
 // src/worker-runtime.ts
 var import_rxjs2 = __toESM(require_cjs());
 function buildHealthPayload(workers) {
@@ -11351,7 +11399,7 @@ function startStallWatchdog(opts) {
   timer.unref?.();
   return { dispose: () => clearInterval(timer) };
 }
-function parseBackendUrl(url) {
+function parseGatewayUrl(url) {
   const parsed = new URL(url);
   const protocol = parsed.protocol.replace(":", "") === "https" ? "https" : "http";
   const host = parsed.hostname;
@@ -11359,13 +11407,13 @@ function parseBackendUrl(url) {
   return { protocol, host, port };
 }
 async function authenticateAgent(opts) {
-  const { backendBaseUrl: backendBaseUrl2, workerSecret: workerSecret2, provider, model, logger: logger2, retry = STARTUP_FETCH_RETRY } = opts;
+  const { gatewayBaseUrl: gatewayBaseUrl2, workerSecret: workerSecret2, provider, model, logger: logger2, retry = STARTUP_FETCH_RETRY } = opts;
   if (!workerSecret2) {
     throw new Error("SEMIONT_WORKER_SECRET is required to authenticate worker agents");
   }
   return retryWithBackoff(
     async () => {
-      const response = await fetch(`${backendBaseUrl2}/api/tokens/agent`, {
+      const response = await fetch(`${gatewayBaseUrl2}/api/tokens/agent`, {
         method: "POST",
         headers: { "Content-Type": "application/json" },
         body: JSON.stringify({ secret: workerSecret2, provider, model })
@@ -11378,7 +11426,7 @@ async function authenticateAgent(opts) {
     isTransientFetchError,
     retry,
     ({ attempt, attempts, delayMs, error }) => {
-      logger2?.warn("Backend unreachable, retrying agent authentication", {
+      logger2?.warn("Gateway unreachable, retrying agent authentication", {
         agent: `${provider}:${model}`,
         attempt,
         attempts,
@@ -11389,11 +11437,11 @@ async function authenticateAgent(opts) {
   );
 }
 async function startAgentWorker(opts) {
-  const { group, backendBaseUrl: backendBaseUrl2, workerSecret: workerSecret2, logger: logger2 } = opts;
+  const { group, gatewayBaseUrl: gatewayBaseUrl2, workerSecret: workerSecret2, contentReads: contentReads2, logger: logger2 } = opts;
   const { inference } = group;
-  const { protocol, host, port } = parseBackendUrl(backendBaseUrl2);
+  const { protocol, host, port } = parseGatewayUrl(gatewayBaseUrl2);
   const { token: initialToken, did } = await authenticateAgent({
-    backendBaseUrl: backendBaseUrl2,
+    gatewayBaseUrl: gatewayBaseUrl2,
     workerSecret: workerSecret2,
     provider: inference.type,
     model: inference.model,
@@ -11413,7 +11461,7 @@ async function startAgentWorker(opts) {
   const token$ = new import_rxjs2.BehaviorSubject(null);
   let session;
   const transport = new HttpTransport({
-    baseUrl: baseUrl(kbBackendUrl(endpoint)),
+    baseUrl: baseUrl(kbGatewayUrl(endpoint)),
     token$,
     tokenRefresher: () => session.refresh().then((t) => t ?? null)
   });
@@ -11427,7 +11475,7 @@ async function startAgentWorker(opts) {
     refresh: async () => {
       try {
         const { token } = await authenticateAgent({
-          backendBaseUrl: backendBaseUrl2,
+          gatewayBaseUrl: gatewayBaseUrl2,
           workerSecret: workerSecret2,
           provider: inference.type,
           model: inference.model,
@@ -11452,10 +11500,12 @@ async function startAgentWorker(opts) {
     jobTypes: group.jobTypes,
     inferenceClient: group.client,
     generator,
-    // The extraction seam's cache, over this worker's content transport
-    // (PERSIST-ANCHORS P2d). Built here because the client keeps its
-    // content transport private — this is where it is in hand.
-    anchoredTextStore: anchoredTextStoreOverTransport(content, logger2),
+    // The extraction seam's cache, consulted over the bus and never written
+    // (ANCHORED-TEXT-TO-SMELTER D2): the Smelter owns this store, the
+    // Archivist answers the checksum-addressed read, and a worker that
+    // misses extracts locally and discards.
+    anchoredTextStore: anchoredTextOverBus(client, logger2),
+    contentReads: contentReads2,
     logger: logger2
   });
   logger2.info("Agent ready", {
@@ -11511,12 +11561,13 @@ function resolveWorker(jobType) {
     `No inference config for worker '${jobType}' and no workers.default in ~/.semiontconfig.`
   );
 }
-var backendPublicURL = envConfig.services?.backend?.publicURL;
-if (!backendPublicURL) {
-  throw new Error("services.backend.publicURL is required in ~/.semiontconfig");
+var gatewayPublicURL = envConfig.services?.gateway?.publicURL;
+if (!gatewayPublicURL) {
+  throw new Error("services.gateway.publicURL is required in ~/.semiontconfig");
 }
-var backendBaseUrl = backendPublicURL;
+var gatewayBaseUrl = gatewayPublicURL;
 var workerSecret = process.env.SEMIONT_WORKER_SECRET ?? "";
+var contentReads = archivistContentReads(envConfig);
 var healthPort = 9090;
 var logger = createProcessLogger("worker");
 function clientKey(w) {
@@ -11550,7 +11601,7 @@ async function main() {
   const { initObservabilityNode } = await import('@semiont/observability/node');
   initObservabilityNode({ serviceName: "semiont-worker" });
   logger.info("Starting agents", {
-    baseUrl: backendBaseUrl,
+    baseUrl: gatewayBaseUrl,
     agents: Array.from(groups.values()).map((g) => ({
       provider: g.inference.type,
       model: g.inference.model,
@@ -11559,7 +11610,7 @@ async function main() {
   });
   const workers = await Promise.all(
     Array.from(groups.values()).map(
-      (group) => startAgentWorker({ group, backendBaseUrl, workerSecret, logger })
+      (group) => startAgentWorker({ group, gatewayBaseUrl, workerSecret, contentReads, logger })
     )
   );
   const health = createServer((req, res) => {