@semiont/jobs 0.5.38 → 0.6.0

package/dist/worker-main.js

@@ -1,4 +1,4 @@
-import { replyChannelsFor, createTomlConfigLoader, didToAgent, baseUrl, STARTUP_FETCH_RETRY, retryWithBackoff, isTransientFetchError, isObject, isString, isNumber, RETRY_RULES, busRequest, isArray, resourceId, getPrimaryMediaType, isGenerationJobParams, capabilitiesOf, assembleAnnotation, findClaimSpan, textSourceOf, yieldsGeometryOf, decodeRepresentation, GENERATABLE_MEDIA_TYPES, BusRequestError, locate, createFragmentSelector, deriveViews, annotationId, estimateTokens, reconcileSelector, getLocaleEnglishName, cutChunk, chunkText } from '@semiont/core';
+import { replyChannelsFor, createTomlConfigLoader, didToAgent, baseUrl, STARTUP_FETCH_RETRY, retryWithBackoff, serviceAccountToken, isTransientFetchError, isObject, isString, isNumber, RETRY_RULES, resourceId, getPrimaryMediaType, isGenerationJobParams, capabilitiesOf, assembleAnnotation, findClaimSpan, busRequest, BusRequestError, isArray, textSourceOf, yieldsGeometryOf, decodeRepresentation, GENERATABLE_MEDIA_TYPES, locate, createFragmentSelector, deriveViews, annotationId, estimateTokens, reconcileSelector, getLocaleEnglishName, cutChunk, chunkText } from '@semiont/core';
 import { archivistContentReads, extractPdfTextLayer, withinByteBudget, MAX_PDF_BYTES } from '@semiont/content';
 import { withSpan, SpanKind, recordJobOutcome, recordAnchorOutcome, recordDetectionCall } from '@semiont/observability';
 import { createInferenceClient, StructuredReadError } from '@semiont/inference';
@@ -9223,13 +9223,15 @@ function readUnitCursors(raw, completedUnits) {
 }
 function createJobClaimAdapter(options) {
   const { bus, jobTypes } = options;
-  const requestBus = bus;
   const activeJob$ = new import_rxjs.BehaviorSubject(null);
   const isProcessing$ = new import_rxjs.BehaviorSubject(false);
   const jobsCompleted$ = new import_rxjs.BehaviorSubject(0);
   const errors$ = new import_rxjs.Subject();
-  let jobSubscription = null;
+  const refused$ = new import_rxjs.Subject();
+  let subscriptions = [];
   let started = false;
+  let claimInFlight = false;
+  let wakePending = false;
   let lastQueuedEventAt = null;
   let lastClaimAt = null;
   let lastFinishedAt = null;
@@ -9237,62 +9239,98 @@ function createJobClaimAdapter(options) {
   let activeSince = null;
   const iso = (t) => t === null ? null : new Date(t).toISOString();
   const claimNext = async () => {
+    let record;
     try {
-      const job = await busRequest(requestBus, "job:claim", { types: jobTypes }, 1e4);
-      if (!isString(job.metadata?.id) || !isString(job.metadata?.type)) return null;
-      const completedUnits = isArray(job.metadata?.completedUnits) ? job.metadata.completedUnits.filter(isString) : [];
-      const unitCursors = readUnitCursors(job.metadata?.unitCursors, completedUnits);
-      const params = job.params ?? {};
-      return {
-        jobId: job.metadata.id,
-        type: job.metadata.type,
+      record = await busRequest(bus, "job:claim", { types: jobTypes }, 1e4);
+    } catch (error) {
+      if (error instanceof BusRequestError) {
+        if (error.code === "bus.none-pending") return { declined: true };
+        return { refused: { code: error.code, message: error.message } };
+      }
+      return { refused: { code: null, message: error instanceof Error ? error.message : String(error) } };
+    }
+    if (!isString(record.metadata?.id) || !isString(record.metadata?.type)) {
+      return { refused: { code: null, message: "claimed record carries no job id or type" } };
+    }
+    const completedUnits = isArray(record.metadata?.completedUnits) ? record.metadata.completedUnits.filter(isString) : [];
+    const unitCursors = readUnitCursors(record.metadata?.unitCursors, completedUnits);
+    const params = record.params ?? {};
+    return {
+      job: {
+        jobId: record.metadata.id,
+        type: record.metadata.type,
         resourceId: isString(params.resourceId) ? params.resourceId : "",
-        userId: job.metadata?.userId ?? "",
+        userId: record.metadata?.userId ?? "",
         params,
         completedUnits,
         unitCursors,
         // Absent or malformed metadata reads as "no budget left" — a worker
         // that cannot see the budget must not claim a retry is coming.
-        retryCount: isNumber(job.metadata?.retryCount) ? job.metadata.retryCount : 0,
-        maxRetries: isNumber(job.metadata?.maxRetries) ? job.metadata.maxRetries : 0
-      };
-    } catch {
-      return null;
+        retryCount: isNumber(record.metadata?.retryCount) ? record.metadata.retryCount : 0,
+        maxRetries: isNumber(record.metadata?.maxRetries) ? record.metadata.maxRetries : 0
+      }
+    };
+  };
+  const pull = () => {
+    if (!started) return;
+    if (activeJob$.getValue() !== null) return;
+    if (claimInFlight) {
+      wakePending = true;
+      return;
     }
+    claimInFlight = true;
+    wakePending = false;
+    isProcessing$.next(true);
+    void claimNext().then((outcome) => {
+      claimInFlight = false;
+      if ("job" in outcome) {
+        const now = Date.now();
+        lastClaimAt = now;
+        lastActivityAt = now;
+        activeSince = now;
+        wakePending = false;
+        activeJob$.next(outcome.job);
+        return;
+      }
+      if ("refused" in outcome) refused$.next(outcome.refused);
+      isProcessing$.next(false);
+      if (wakePending) {
+        wakePending = false;
+        pull();
+      }
+    });
   };
   return {
     activeJob$: activeJob$.asObservable(),
     isProcessing$: isProcessing$.asObservable(),
     jobsCompleted$: jobsCompleted$.asObservable(),
     errors$: errors$.asObservable(),
+    refused$: refused$.asObservable(),
     start: () => {
       if (started) return;
       started = true;
-      bus.addChannels?.(["job:queued"]);
-      jobSubscription = bus.stream("job:queued").subscribe((event) => {
-        lastQueuedEventAt = Date.now();
-        const jobType = event.jobType;
-        if (jobTypes.length > 0 && !jobTypes.includes(jobType)) return;
-        if (isProcessing$.getValue()) return;
-        isProcessing$.next(true);
-        claimNext().then((job) => {
-          if (job) {
-            const now = Date.now();
-            lastClaimAt = now;
-            lastActivityAt = now;
-            activeSince = now;
-            activeJob$.next(job);
-          } else {
-            isProcessing$.next(false);
-          }
-        }).catch(() => {
-          isProcessing$.next(false);
-        });
-      });
+      subscriptions.push(
+        bus.stream("job:queued").subscribe((event) => {
+          lastQueuedEventAt = Date.now();
+          if (jobTypes.length > 0 && !jobTypes.includes(event.jobType)) return;
+          pull();
+        })
+      );
+      let observed = false;
+      let wasOpen = false;
+      subscriptions.push(
+        bus.state$.subscribe((state) => {
+          const open = state === "open";
+          if (observed && open && !wasOpen) pull();
+          observed = true;
+          wasOpen = open;
+        })
+      );
+      if (wasOpen) pull();
     },
     stop: () => {
-      jobSubscription?.unsubscribe();
-      jobSubscription = null;
+      for (const s of subscriptions) s.unsubscribe();
+      subscriptions = [];
       started = false;
     },
     completeJob: () => {
@@ -9303,6 +9341,7 @@ function createJobClaimAdapter(options) {
       activeJob$.next(null);
       isProcessing$.next(false);
       jobsCompleted$.next(jobsCompleted$.getValue() + 1);
+      pull();
     },
     failJob: (jid, error) => {
       const now = Date.now();
@@ -9312,6 +9351,7 @@ function createJobClaimAdapter(options) {
       activeJob$.next(null);
       isProcessing$.next(false);
       errors$.next({ jobId: jid, error });
+      pull();
     },
     vitals: () => {
       const active = activeJob$.getValue();
@@ -9328,13 +9368,14 @@ function createJobClaimAdapter(options) {
       lastActivityAt = Date.now();
     },
     dispose: () => {
-      jobSubscription?.unsubscribe();
-      jobSubscription = null;
+      for (const s of subscriptions) s.unsubscribe();
+      subscriptions = [];
       started = false;
       activeJob$.complete();
       isProcessing$.complete();
       jobsCompleted$.complete();
       errors$.complete();
+      refused$.complete();
     }
   };
 }
@@ -10838,7 +10879,7 @@ function makeSpanDeduper() {
     return out;
   };
 }
-function buildTextAnnotation(content, resourceId, userId, generator, motivation, match, body) {
+function buildTextAnnotation(content, resourceId, generator, motivation, match, body) {
   if (content.substring(match.start, match.end) !== match.exact) {
     throw new Error(
       `buildTextAnnotation invariant: content.substring(${match.start}, ${match.end}) !== exact for resource ${resourceId}, motivation ${motivation}`
@@ -10860,16 +10901,12 @@ function buildTextAnnotation(content, resourceId, userId, generator, motivation,
       );
     }
   }
-  const creator = didToAgent(userId);
-  const wasAttributedTo = creator["@id"] === generator["@id"] ? [generator] : [creator, generator];
   return {
     "@context": "http://www.w3.org/ns/anno.jsonld",
     "type": "Annotation",
     "id": annotationIdFor({ resourceId, motivation, anchor: spanAnchor(match), body }),
     motivation,
-    creator,
     generator,
-    wasAttributedTo,
     created: (/* @__PURE__ */ new Date()).toISOString(),
     target: {
       type: "SpecificResource",
@@ -10887,7 +10924,7 @@ function buildTextAnnotation(content, resourceId, userId, generator, motivation,
     ...body !== void 0 ? { body } : {}
   };
 }
-function buildPdfAnnotation(anchored, resourceId, userId, generator, motivation, match, body) {
+function buildPdfAnnotation(anchored, resourceId, generator, motivation, match, body) {
   const { rects, overlap } = locate(anchored, match.start, match.end);
   const coveredText = overlap.length ? anchored.text.substring(
     Math.min(...overlap.map((i) => i.start)),
@@ -10904,16 +10941,12 @@ function buildPdfAnnotation(anchored, resourceId, userId, generator, motivation,
       `buildPdfAnnotation invariant: covered text does not contain exact for resource ${resourceId}, motivation ${motivation}`
     );
   }
-  const creator = didToAgent(userId);
-  const wasAttributedTo = creator["@id"] === generator["@id"] ? [generator] : [creator, generator];
   return {
     "@context": "http://www.w3.org/ns/anno.jsonld",
     "type": "Annotation",
     "id": annotationIdFor({ resourceId, motivation, anchor: spanAnchor(match), body }),
     motivation,
-    creator,
     generator,
-    wasAttributedTo,
     created: (/* @__PURE__ */ new Date()).toISOString(),
     target: {
       type: "SpecificResource",
@@ -11400,7 +11433,7 @@ async function processGenerationJob(inferenceClient, params, onProgress, logger2
 }
 
 // src/workers/detection/prepare-detection.ts
-async function prepareDetection(mediaType, content, resourceId, userId, generator, consult) {
+async function prepareDetection(mediaType, content, resourceId, generator, consult) {
   if (textSourceOf(mediaType) === "none") return { declined: "no-extractor" };
   if (yieldsGeometryOf(mediaType)) {
     const answer = await consult(resourceId);
@@ -11410,7 +11443,7 @@ async function prepareDetection(mediaType, content, resourceId, userId, generato
         const anchored = { text: answer.text, items: answer.items ?? [] };
         return {
           text: answer.text,
-          buildAnnotation: (motivation, match, body) => buildPdfAnnotation(anchored, resourceId, userId, generator, motivation, match, body)
+          buildAnnotation: (motivation, match, body) => buildPdfAnnotation(anchored, resourceId, generator, motivation, match, body)
         };
       }
       // The Smelter's own decline (encrypted, corrupt) — passed through by name.
@@ -11436,7 +11469,7 @@ async function prepareDetection(mediaType, content, resourceId, userId, generato
   if (!text.trim()) return { declined: "empty" };
   return {
     text,
-    buildAnnotation: (motivation, match, body) => buildTextAnnotation(text, resourceId, userId, generator, motivation, match, body)
+    buildAnnotation: (motivation, match, body) => buildTextAnnotation(text, resourceId, generator, motivation, match, body)
   };
 }
 function referenceIdOf(job) {
@@ -11452,13 +11485,15 @@ function referenceIdOf(job) {
   return typeof ref === "string" ? ref : void 0;
 }
 var MARK_COMMIT_TIMEOUT_MS = 6e4;
-async function commitAnnotations(session, resourceId, annotations) {
+async function commitAnnotations(session, resourceId, annotations, jobId) {
   if (annotations.length === 0) return void 0;
   try {
     await busRequest(
       session.client.transport.actor,
       "mark:commit",
-      { resourceId, annotations },
+      // The batch cites the job it fulfils. Who asked for these annotations is
+      // derived downstream from that job's own events; the worker never says.
+      { resourceId, annotations, jobId },
       MARK_COMMIT_TIMEOUT_MS
     );
     return "acknowledged";
@@ -11498,10 +11533,22 @@ function startWorkerProcess(config) {
     bus: httpTransport.actor,
     jobTypes: config.jobTypes
   });
+  const exit = config.exit ?? ((code) => process.exit(code));
+  adapter.refused$.subscribe(({ code, message }) => {
+    if (code === "bus.unauthorized") {
+      logger2.error("Claim refused: this worker is not authorized to claim jobs \u2014 exiting for restart", {
+        code,
+        message,
+        jobTypes: config.jobTypes
+      });
+      exit(1);
+      return;
+    }
+    logger2.warn("Claim declined; parked until the next wake-up", { code, message });
+  });
   const completedUnitsByJob = /* @__PURE__ */ new Map();
   const unitCursorsByJob = /* @__PURE__ */ new Map();
   let activeCancel = null;
-  httpTransport.actor.addChannels?.(["job:cancel-requested"]);
   httpTransport.on("job:cancel-requested", (event) => {
     const targetId = event.jobId;
     if (targetId && activeCancel?.jobId === targetId) {
@@ -11585,7 +11632,7 @@ async function handleJob(adapter, config, job, completedUnitsByJob = /* @__PURE_
 }
 async function handleJobInner(adapter, config, job, completedUnitsByJob, signal, unitCursorsByJob = /* @__PURE__ */ new Map()) {
   const { session, inferenceClient, generator } = config;
-  const { userId, jobId } = job;
+  const { jobId } = job;
   if (!isJobType(job.type)) {
     adapter.failJob(jobId, `Unrecognized job type: ${job.type}`);
     return;
@@ -11618,7 +11665,7 @@ async function handleJobInner(adapter, config, job, completedUnitsByJob, signal,
     const mediaType = getPrimaryMediaType(descriptor);
     const source = await withSpan(
       "detection:prepare",
-      () => prepareDetection(mediaType ?? "", config.contentReads, resourceId$1, userId, generator, (rid) => session.client.browse.resourceAnchoredText(rid)),
+      () => prepareDetection(mediaType ?? "", config.contentReads, resourceId$1, generator, (rid) => session.client.browse.resourceAnchoredText(rid)),
       { attrs: { "resource.id": resourceId$1, "media.type": mediaType ?? "unknown" } }
     );
     if ("declined" in source) {
@@ -11660,7 +11707,7 @@ async function handleJobInner(adapter, config, job, completedUnitsByJob, signal,
   };
   const unitCursors = /* @__PURE__ */ new Map();
   const commitChunk = async (annotations, checkpoint) => {
-    record(await commitAnnotations(session, String(resourceId$1), annotations));
+    record(await commitAnnotations(session, String(resourceId$1), annotations, jobId));
     unitCursors.set(checkpoint.unit, checkpoint.cursor);
     unitCursorsByJob.set(job.jobId, Object.fromEntries(unitCursors));
     await emitEvent(session, "job:checkpoint", {
@@ -11826,7 +11873,10 @@ async function handleJobInner(adapter, config, job, completedUnitsByJob, signal,
       ...genParams.prompt ? { generationPrompt: genParams.prompt } : {},
       ...genParams.language ? { language: genParams.language } : {},
       ...genParams.entityTypes && genParams.entityTypes.length > 0 ? { entityTypes: genParams.entityTypes } : {},
-      generator
+      generator,
+      // The resource cites the job it fulfils; who asked for it is derived
+      // downstream from that job's events, never stated here.
+      jobId
     });
     if (!genReferenceId) {
       const { annotation: provenanceRef } = assembleAnnotation(
@@ -11837,7 +11887,7 @@ async function handleJobInner(adapter, config, job, completedUnitsByJob, signal,
         },
         generator
       );
-      record(await commitAnnotations(session, String(resourceId$1), [provenanceRef]));
+      record(await commitAnnotations(session, String(resourceId$1), [provenanceRef], jobId));
     }
     const citationRefs = [];
     if (genResult.format === "application/pdf" && genResult.citations.length > 0) {
@@ -11863,7 +11913,6 @@ async function handleJobInner(adapter, config, job, completedUnitsByJob, signal,
           const citationRef = buildPdfAnnotation(
             layer,
             resourceId(String(newResourceId)),
-            userId,
             generator,
             "linking",
             { exact: layer.text.slice(span.start, span.end), start: span.start, end: span.end },
@@ -11891,7 +11940,7 @@ async function handleJobInner(adapter, config, job, completedUnitsByJob, signal,
         citationRefs.push(citationRef);
       }
     }
-    record(await commitAnnotations(session, String(newResourceId), citationRefs));
+    record(await commitAnnotations(session, String(newResourceId), citationRefs, jobId));
     await emitEvent(session, "job:complete", {
       ...terminalBase(),
       result: { kind: "generation", resourceId: newResourceId, resourceName: genResult.title, truncated: genResult.result.truncated }
@@ -11961,7 +12010,16 @@ var WORKER_AWAITED_OPERATIONS = [
   // rare error path is no reason to let it back in.
   "browse:annotation-requested"
 ];
-var WORKER_CHANNELS = replyChannelsFor(WORKER_AWAITED_OPERATIONS);
+var WORKER_CONSUMED_BROADCASTS = [
+  // The queue announcement the claim adapter races for.
+  "job:queued",
+  // Cooperative cancellation of the ACTIVE job (JOB-RESTART-SAFETY P4).
+  "job:cancel-requested"
+];
+var WORKER_CHANNELS = [
+  ...replyChannelsFor(WORKER_AWAITED_OPERATIONS),
+  ...WORKER_CONSUMED_BROADCASTS
+];
 function parseGatewayUrl(url) {
   const parsed = new URL(url);
   const protocol = parsed.protocol.replace(":", "") === "https" ? "https" : "http";
@@ -11970,16 +12028,17 @@ function parseGatewayUrl(url) {
   return { protocol, host, port };
 }
 async function authenticateAgent(opts) {
-  const { gatewayBaseUrl: gatewayBaseUrl2, workerSecret: workerSecret2, provider, model, logger: logger2, retry = STARTUP_FETCH_RETRY } = opts;
-  if (!workerSecret2) {
-    throw new Error("SEMIONT_WORKER_SECRET is required to authenticate worker agents");
-  }
+  const { gatewayBaseUrl: gatewayBaseUrl2, credential: credential2, provider, model, logger: logger2, retry = STARTUP_FETCH_RETRY } = opts;
   return retryWithBackoff(
     async () => {
+      const caller = await serviceAccountToken(credential2);
       const response = await fetch(`${gatewayBaseUrl2}/api/tokens/agent`, {
         method: "POST",
-        headers: { "Content-Type": "application/json" },
-        body: JSON.stringify({ secret: workerSecret2, provider, model })
+        headers: {
+          "Content-Type": "application/json",
+          Authorization: `Bearer ${caller}`
+        },
+        body: JSON.stringify({ provider, model })
       });
       if (!response.ok) {
         throw new Error(`Agent authentication failed for ${provider}:${model}: ${response.status} ${response.statusText}`);
@@ -12000,12 +12059,12 @@ async function authenticateAgent(opts) {
   );
 }
 async function startAgentWorker(opts) {
-  const { group, gatewayBaseUrl: gatewayBaseUrl2, workerSecret: workerSecret2, contentReads: contentReads2, logger: logger2 } = opts;
+  const { group, gatewayBaseUrl: gatewayBaseUrl2, credential: credential2, contentReads: contentReads2, logger: logger2 } = opts;
   const { inference } = group;
   const { protocol, host, port } = parseGatewayUrl(gatewayBaseUrl2);
   const { token: initialToken, did } = await authenticateAgent({
     gatewayBaseUrl: gatewayBaseUrl2,
-    workerSecret: workerSecret2,
+    credential: credential2,
     provider: inference.type,
     model: inference.model,
     logger: logger2
@@ -12016,11 +12075,15 @@ async function startAgentWorker(opts) {
   const kb = {
     id: kbId,
     label: `${inference.type} / ${inference.model} @ ${host}`,
-    email: `agent@${host}`,
     endpoint
   };
   const storage = new InMemorySessionStorage();
-  setStoredSession(storage, kbId, { access: initialToken, refresh: "" });
+  setStoredSession(storage, kbId, {
+    access: initialToken,
+    refresh: "",
+    clientId: credential2.clientId,
+    tokenEndpoint: credential2.issuer
+  });
   const token$ = new import_rxjs2.BehaviorSubject(null);
   let session;
   const transport = new HttpTransport({
@@ -12042,7 +12105,7 @@ async function startAgentWorker(opts) {
       try {
         const { token } = await authenticateAgent({
           gatewayBaseUrl: gatewayBaseUrl2,
-          workerSecret: workerSecret2,
+          credential: credential2,
           provider: inference.type,
           model: inference.model,
           logger: logger2
@@ -12131,8 +12194,17 @@ if (!gatewayPublicURL) {
   throw new Error("services.gateway.publicURL is required in ~/.semiontconfig");
 }
 var gatewayBaseUrl = gatewayPublicURL;
-var workerSecret = process.env.SEMIONT_WORKER_SECRET ?? "";
-var contentReads = archivistContentReads(envConfig);
+var issuerUrl = envConfig.services?.identity?.issuer;
+if (!issuerUrl) {
+  throw new Error("services.identity.issuer is required: a worker authenticates at the knowledge base's issuer");
+}
+var clientId = process.env.SEMIONT_OIDC_CLIENT_ID;
+var clientSecret = process.env.SEMIONT_OIDC_CLIENT_SECRET;
+if (!clientId || !clientSecret) {
+  throw new Error("SEMIONT_OIDC_CLIENT_ID and SEMIONT_OIDC_CLIENT_SECRET are required to authenticate as a service account");
+}
+var credential = { issuer: issuerUrl, clientId, clientSecret };
+var contentReads = archivistContentReads(envConfig, credential);
 var healthPort = 24100;
 var logger = createProcessLogger("worker");
 function clientKey(w) {
@@ -12176,7 +12248,7 @@ async function main() {
   });
   const workers = await Promise.all(
     Array.from(groups.values()).map(
-      (group) => startAgentWorker({ group, gatewayBaseUrl, workerSecret, contentReads, logger })
+      (group) => startAgentWorker({ group, gatewayBaseUrl, credential, contentReads, logger })
     )
   );
   const health = createServer((req, res) => {