@semiont/jobs 0.5.28 → 0.5.29

package/dist/worker-main.js

@@ -1,12 +1,12 @@
-import { createTomlConfigLoader, didToAgent, baseUrl, STARTUP_FETCH_RETRY, retryWithBackoff, isTransientFetchError, busRequest, resourceId, getPrimaryMediaType, isGenerationJobParams, assembleAnnotation, findClaimSpan, textExtractionOf, reconcileSelector, GENERATABLE_MEDIA_TYPES, locate, createFragmentSelector, getLocaleEnglishName, estimateTokens, chunkText, isObject, isString, deriveViews } from '@semiont/core';
-import { anchoredTextStoreOverTransport, deriveStorageUri, extractPdfTextLayer, EXTRACTORS, calculateChecksum, withinByteBudget, MAX_PDF_BYTES } from '@semiont/content';
+import { createTomlConfigLoader, didToAgent, baseUrl, STARTUP_FETCH_RETRY, retryWithBackoff, isTransientFetchError, busRequest, resourceId, getPrimaryMediaType, isGenerationJobParams, capabilitiesOf, assembleAnnotation, findClaimSpan, textExtractionOf, reconcileSelector, GENERATABLE_MEDIA_TYPES, locate, createFragmentSelector, getLocaleEnglishName, estimateTokens, chunkText, isObject, isString, deriveViews } from '@semiont/core';
+import { archivistContentReads, extractPdfTextLayer, EXTRACTORS, calculateChecksum, withinByteBudget, MAX_PDF_BYTES } from '@semiont/content';
 import { withSpan, SpanKind, recordJobOutcome } from '@semiont/observability';
 import { execFileSync } from 'child_process';
 import { existsSync, readFileSync, mkdtempSync, writeFileSync, rmSync } from 'fs';
 import { homedir, hostname, tmpdir } from 'os';
 import { join } from 'path';
 import { generateAnnotationId } from '@semiont/event-sourcing';
-import { InMemorySessionStorage, setStoredSession, kbBackendUrl, SemiontClient, SemiontSession } from '@semiont/sdk';
+import { InMemorySessionStorage, setStoredSession, kbGatewayUrl, SemiontClient, SemiontSession } from '@semiont/sdk';
 import { HttpTransport, HttpContentTransport } from '@semiont/http-transport';
 import { createInferenceClient } from '@semiont/inference';
 import { createServer } from 'http';
@@ -10995,10 +10995,10 @@ async function processGenerationJob(inferenceClient, params, onProgress, logger2
 }
 
 // src/workers/detection/prepare-detection.ts
-async function prepareDetection(mediaType, session, resourceId, userId, generator, store) {
+async function prepareDetection(mediaType, content, resourceId, userId, generator, store) {
   const extractor = EXTRACTORS[textExtractionOf(mediaType)];
   if (!extractor) return { declined: "no-extractor" };
-  const { data } = await session.client.browse.resourceRepresentation(resourceId);
+  const { data } = await content.getBinary(resourceId);
   const bytes = Buffer.from(data);
   const extracted = await extractor.extract(bytes, mediaType, {
     key: calculateChecksum(bytes),
@@ -11114,7 +11114,7 @@ async function handleJobInner(adapter, config, job) {
     const mediaType = getPrimaryMediaType(descriptor);
     const source = await withSpan(
       "detection:prepare",
-      () => prepareDetection(mediaType ?? "", session, resourceId$1, userId, generator, config.anchoredTextStore),
+      () => prepareDetection(mediaType ?? "", config.contentReads, resourceId$1, userId, generator, config.anchoredTextStore),
       { attrs: { "resource.id": resourceId$1, "media.type": mediaType ?? "unknown" } }
     );
     if ("declined" in source) {
@@ -11243,7 +11243,16 @@ async function handleJobInner(adapter, config, job) {
     );
     const genParams = job.params;
     const genReferenceId = referenceIdOf(job);
-    const storageUri = deriveStorageUri(genResult.title, genResult.format);
+    const storageUri = job.params.storageUri;
+    const expectedExtension = capabilitiesOf(genResult.format)?.extension;
+    if (expectedExtension && !storageUri.toLowerCase().endsWith(expectedExtension)) {
+      config.logger.warn("Storage URI extension does not match the generated format \u2014 writing it as requested", {
+        jobId,
+        storageUri,
+        format: genResult.format,
+        expectedExtension
+      });
+    }
     const { resourceId: newResourceId } = await session.client.yield.resource({
       name: genResult.title,
       file: Buffer.from(genResult.content),
@@ -11328,6 +11337,29 @@ async function handleJobInner(adapter, config, job) {
   }
 }
 
+// src/anchored-text-over-bus.ts
+function anchoredTextOverBus(client, logger2) {
+  return {
+    async read(key) {
+      try {
+        return await client.browse.anchoredTextByChecksum(key);
+      } catch (error) {
+        logger2?.debug("Anchored-text consult failed \u2014 treating as miss", {
+          key,
+          reason: error instanceof Error ? error.message : String(error)
+        });
+        return null;
+      }
+    },
+    async write(key) {
+      logger2?.debug("Anchored-text write skipped \u2014 the Smelter is the sole writer", { key });
+    },
+    async list() {
+      return [];
+    }
+  };
+}
+
 // src/worker-runtime.ts
 var import_rxjs2 = __toESM(require_cjs());
 function buildHealthPayload(workers) {
@@ -11367,7 +11399,7 @@ function startStallWatchdog(opts) {
   timer.unref?.();
   return { dispose: () => clearInterval(timer) };
 }
-function parseBackendUrl(url) {
+function parseGatewayUrl(url) {
   const parsed = new URL(url);
   const protocol = parsed.protocol.replace(":", "") === "https" ? "https" : "http";
   const host = parsed.hostname;
@@ -11375,13 +11407,13 @@ function parseBackendUrl(url) {
   return { protocol, host, port };
 }
 async function authenticateAgent(opts) {
-  const { backendBaseUrl: backendBaseUrl2, workerSecret: workerSecret2, provider, model, logger: logger2, retry = STARTUP_FETCH_RETRY } = opts;
+  const { gatewayBaseUrl: gatewayBaseUrl2, workerSecret: workerSecret2, provider, model, logger: logger2, retry = STARTUP_FETCH_RETRY } = opts;
   if (!workerSecret2) {
     throw new Error("SEMIONT_WORKER_SECRET is required to authenticate worker agents");
   }
   return retryWithBackoff(
     async () => {
-      const response = await fetch(`${backendBaseUrl2}/api/tokens/agent`, {
+      const response = await fetch(`${gatewayBaseUrl2}/api/tokens/agent`, {
         method: "POST",
         headers: { "Content-Type": "application/json" },
         body: JSON.stringify({ secret: workerSecret2, provider, model })
@@ -11394,7 +11426,7 @@ async function authenticateAgent(opts) {
     isTransientFetchError,
     retry,
     ({ attempt, attempts, delayMs, error }) => {
-      logger2?.warn("Backend unreachable, retrying agent authentication", {
+      logger2?.warn("Gateway unreachable, retrying agent authentication", {
         agent: `${provider}:${model}`,
         attempt,
         attempts,
@@ -11405,11 +11437,11 @@ async function authenticateAgent(opts) {
   );
 }
 async function startAgentWorker(opts) {
-  const { group, backendBaseUrl: backendBaseUrl2, workerSecret: workerSecret2, logger: logger2 } = opts;
+  const { group, gatewayBaseUrl: gatewayBaseUrl2, workerSecret: workerSecret2, contentReads: contentReads2, logger: logger2 } = opts;
   const { inference } = group;
-  const { protocol, host, port } = parseBackendUrl(backendBaseUrl2);
+  const { protocol, host, port } = parseGatewayUrl(gatewayBaseUrl2);
   const { token: initialToken, did } = await authenticateAgent({
-    backendBaseUrl: backendBaseUrl2,
+    gatewayBaseUrl: gatewayBaseUrl2,
     workerSecret: workerSecret2,
     provider: inference.type,
     model: inference.model,
@@ -11429,7 +11461,7 @@ async function startAgentWorker(opts) {
   const token$ = new import_rxjs2.BehaviorSubject(null);
   let session;
   const transport = new HttpTransport({
-    baseUrl: baseUrl(kbBackendUrl(endpoint)),
+    baseUrl: baseUrl(kbGatewayUrl(endpoint)),
     token$,
     tokenRefresher: () => session.refresh().then((t) => t ?? null)
   });
@@ -11443,7 +11475,7 @@ async function startAgentWorker(opts) {
     refresh: async () => {
       try {
         const { token } = await authenticateAgent({
-          backendBaseUrl: backendBaseUrl2,
+          gatewayBaseUrl: gatewayBaseUrl2,
           workerSecret: workerSecret2,
           provider: inference.type,
           model: inference.model,
@@ -11468,10 +11500,12 @@ async function startAgentWorker(opts) {
     jobTypes: group.jobTypes,
     inferenceClient: group.client,
     generator,
-    // The extraction seam's cache, over this worker's content transport
-    // (PERSIST-ANCHORS P2d). Built here because the client keeps its
-    // content transport private — this is where it is in hand.
-    anchoredTextStore: anchoredTextStoreOverTransport(content, logger2),
+    // The extraction seam's cache, consulted over the bus and never written
+    // (ANCHORED-TEXT-TO-SMELTER D2): the Smelter owns this store, the
+    // Archivist answers the checksum-addressed read, and a worker that
+    // misses extracts locally and discards.
+    anchoredTextStore: anchoredTextOverBus(client, logger2),
+    contentReads: contentReads2,
     logger: logger2
   });
   logger2.info("Agent ready", {
@@ -11527,12 +11561,13 @@ function resolveWorker(jobType) {
     `No inference config for worker '${jobType}' and no workers.default in ~/.semiontconfig.`
   );
 }
-var backendPublicURL = envConfig.services?.backend?.publicURL;
-if (!backendPublicURL) {
-  throw new Error("services.backend.publicURL is required in ~/.semiontconfig");
+var gatewayPublicURL = envConfig.services?.gateway?.publicURL;
+if (!gatewayPublicURL) {
+  throw new Error("services.gateway.publicURL is required in ~/.semiontconfig");
 }
-var backendBaseUrl = backendPublicURL;
+var gatewayBaseUrl = gatewayPublicURL;
 var workerSecret = process.env.SEMIONT_WORKER_SECRET ?? "";
+var contentReads = archivistContentReads(envConfig);
 var healthPort = 9090;
 var logger = createProcessLogger("worker");
 function clientKey(w) {
@@ -11566,7 +11601,7 @@ async function main() {
   const { initObservabilityNode } = await import('@semiont/observability/node');
   initObservabilityNode({ serviceName: "semiont-worker" });
   logger.info("Starting agents", {
-    baseUrl: backendBaseUrl,
+    baseUrl: gatewayBaseUrl,
     agents: Array.from(groups.values()).map((g) => ({
       provider: g.inference.type,
       model: g.inference.model,
@@ -11575,7 +11610,7 @@ async function main() {
   });
   const workers = await Promise.all(
     Array.from(groups.values()).map(
-      (group) => startAgentWorker({ group, backendBaseUrl, workerSecret, logger })
+      (group) => startAgentWorker({ group, gatewayBaseUrl, workerSecret, contentReads, logger })
     )
   );
   const health = createServer((req, res) => {