@semiont/jobs 0.6.7 → 0.6.9

package/dist/worker-main.js

@@ -1,4 +1,4 @@
-import { replyChannelsFor, createTomlConfigLoader, didToAgent, baseUrl, STARTUP_FETCH_RETRY, retryWithBackoff, serviceAccountToken, isTransientFetchError, isObject, isString, isNumber, RETRY_RULES, resourceId, getPrimaryMediaType, isGenerationJobParams, capabilitiesOf, assembleAnnotation, findClaimSpan, busRequest, BusRequestError, isArray, textSourceOf, yieldsGeometryOf, decodeRepresentation, GENERATABLE_MEDIA_TYPES, locate, createFragmentSelector, deriveViews, annotationId, estimateTokens, reconcileSelector, getLocaleEnglishName, cutChunk, chunkText } from '@semiont/core';
+import { replyChannelsFor, createTomlConfigLoader, JOB_TYPES, didToAgent, baseUrl, STARTUP_FETCH_RETRY, retryWithBackoff, serviceAccountToken, isTransientFetchError, isObject, isString, isNumber, RETRY_RULES, getPrimaryMediaType, isGenerationJobParams, capabilitiesOf, assembleAnnotation, findClaimSpan, busRequest, BusRequestError, textSourceOf, yieldsGeometryOf, decodeRepresentation, GENERATABLE_MEDIA_TYPES, locate, createFragmentSelector, deriveViews, isResourceId, estimateTokens, reconcileSelector, getLocaleEnglishName, cutChunk, chunkText } from '@semiont/core';
 import { archivistContentReads, extractPdfTextLayer, withinByteBudget, MAX_PDF_BYTES } from '@semiont/content';
 import { withSpan, SpanKind, recordJobOutcome, recordAnchorOutcome, recordDetectionCall } from '@semiont/observability';
 import { createInferenceClient, answerLimitsRequests, StructuredReadError } from '@semiont/inference';
@@ -3766,9 +3766,9 @@ var require_mapOneOrManyArgs = __commonJS({
     Object.defineProperty(exports, "__esModule", { value: true });
     exports.mapOneOrManyArgs = void 0;
     var map_1 = require_map();
-    var isArray2 = Array.isArray;
+    var isArray = Array.isArray;
     function callOrApply(fn, args) {
-      return isArray2(args) ? fn.apply(void 0, __spreadArray([], __read(args))) : fn(args);
+      return isArray(args) ? fn.apply(void 0, __spreadArray([], __read(args))) : fn(args);
     }
     function mapOneOrManyArgs(fn) {
       return map_1.map(function(args) {
@@ -3913,14 +3913,14 @@ var require_argsArgArrayOrObject = __commonJS({
   "../../node_modules/rxjs/dist/cjs/internal/util/argsArgArrayOrObject.js"(exports) {
     Object.defineProperty(exports, "__esModule", { value: true });
     exports.argsArgArrayOrObject = void 0;
-    var isArray2 = Array.isArray;
+    var isArray = Array.isArray;
     var getPrototypeOf = Object.getPrototypeOf;
     var objectProto = Object.prototype;
     var getKeys = Object.keys;
     function argsArgArrayOrObject(args) {
       if (args.length === 1) {
         var first_1 = args[0];
-        if (isArray2(first_1)) {
+        if (isArray(first_1)) {
           return { args: first_1, keys: null };
         }
         if (isPOJO(first_1)) {
@@ -4668,9 +4668,9 @@ var require_argsOrArgArray = __commonJS({
   "../../node_modules/rxjs/dist/cjs/internal/util/argsOrArgArray.js"(exports) {
     Object.defineProperty(exports, "__esModule", { value: true });
     exports.argsOrArgArray = void 0;
-    var isArray2 = Array.isArray;
+    var isArray = Array.isArray;
     function argsOrArgArray(args) {
-      return args.length === 1 && isArray2(args[0]) ? args[0] : args;
+      return args.length === 1 && isArray(args[0]) ? args[0] : args;
     }
     exports.argsOrArgArray = argsOrArgArray;
   }
@@ -9208,19 +9208,6 @@ var require_cjs = __commonJS({
 
 // src/job-claim-adapter.ts
 var import_rxjs = __toESM(require_cjs());
-function readUnitCursors(raw, completedUnits) {
-  if (!isObject(raw)) return {};
-  const done = new Set(completedUnits);
-  const cursors = {};
-  for (const [unit, value] of Object.entries(raw)) {
-    if (done.has(unit) || !isObject(value)) continue;
-    const { next, size, found, emitted } = value;
-    if (!isNumber(next) || !isNumber(size) || next < 0 || size < 1) continue;
-    if (!isNumber(found) || !isNumber(emitted) || found < 0 || emitted < 0) continue;
-    cursors[unit] = { next, size, found, emitted };
-  }
-  return cursors;
-}
 function createJobClaimAdapter(options) {
   const { bus, jobTypes } = options;
   const activeJob$ = new import_rxjs.BehaviorSubject(null);
@@ -9239,9 +9226,9 @@ function createJobClaimAdapter(options) {
   let activeSince = null;
   const iso = (t) => t === null ? null : new Date(t).toISOString();
   const claimNext = async () => {
-    let record;
+    let claimed;
     try {
-      record = await busRequest(bus, "job:claim", { types: jobTypes }, 1e4);
+      claimed = await busRequest(bus, "job:claim", { types: jobTypes }, 1e4);
     } catch (error) {
       if (error instanceof BusRequestError) {
         if (error.code === "bus.none-pending") return { declined: true };
@@ -9249,25 +9236,19 @@ function createJobClaimAdapter(options) {
       }
       return { refused: { code: null, message: error instanceof Error ? error.message : String(error) } };
     }
-    if (!isString(record.metadata?.id) || !isString(record.metadata?.type)) {
-      return { refused: { code: null, message: "claimed record carries no job id or type" } };
-    }
-    const completedUnits = isArray(record.metadata?.completedUnits) ? record.metadata.completedUnits.filter(isString) : [];
-    const unitCursors = readUnitCursors(record.metadata?.unitCursors, completedUnits);
-    const params = record.params ?? {};
+    const { metadata, params } = claimed;
     return {
       job: {
-        jobId: record.metadata.id,
-        type: record.metadata.type,
-        resourceId: isString(params.resourceId) ? params.resourceId : "",
-        userId: record.metadata?.userId ?? "",
+        jobId: metadata.id,
+        type: metadata.type,
+        resourceId: params.resourceId,
         params,
-        completedUnits,
-        unitCursors,
-        // Absent or malformed metadata reads as "no budget left" — a worker
-        // that cannot see the budget must not claim a retry is coming.
-        retryCount: isNumber(record.metadata?.retryCount) ? record.metadata.retryCount : 0,
-        maxRetries: isNumber(record.metadata?.maxRetries) ? record.metadata.maxRetries : 0
+        // Both appear on the record once an attempt has checkpointed, and a
+        // finished unit has no cursor: the dispatcher drops it at the merge.
+        completedUnits: metadata.completedUnits ?? [],
+        unitCursors: metadata.unitCursors ?? {},
+        retryCount: metadata.retryCount,
+        maxRetries: metadata.maxRetries
       }
     };
   };
@@ -9386,17 +9367,6 @@ function willRetryAfter(metadata, failureClass) {
 }
 
 // src/types.ts
-var JOB_TYPES = /* @__PURE__ */ new Set([
-  "reference-annotation",
-  "generation",
-  "highlight-annotation",
-  "assessment-annotation",
-  "comment-annotation",
-  "tag-annotation"
-]);
-function isJobType(value) {
-  return JOB_TYPES.has(value);
-}
 function asJobParams(params) {
   if (typeof params.resourceId !== "string") {
     throw new Error("Job params are missing a resourceId");
@@ -10761,8 +10731,6 @@ function compileTypst(source) {
     rmSync(dir, { recursive: true, force: true });
   }
 }
-
-// src/workers/generation/citation-resolver.ts
 var CITATION_TOKEN = /\[\[([^\s[\]/]+)(?:\/([^\s[\]/]+))?\]\]/g;
 function collectContextResourceIds(context) {
   const ids = /* @__PURE__ */ new Set();
@@ -10788,7 +10756,7 @@ function resolveCitationTokens(content, validResourceIds, logger2) {
     const annotationId = match[2];
     clean += content.slice(last, match.index).replace(/[ \t]+$/, "");
     last = match.index + token.length;
-    if (!validResourceIds.has(citedResourceId)) {
+    if (!isResourceId(citedResourceId) || !validResourceIds.has(citedResourceId)) {
       logger2.warn("Citation token references an id absent from the provided context \u2014 dropped", {
         resourceId: citedResourceId
       });
@@ -11382,12 +11350,7 @@ async function processGenerationJob(inferenceClient, params, onProgress, logger2
       title,
       format: outputMediaType,
       citations: citations2,
-      result: {
-        kind: "generation",
-        resourceId: "",
-        resourceName: title,
-        truncated: generated2.truncated
-      }
+      truncated: generated2.truncated
     };
   }
   onProgress(5, { code: "generating-resource" });
@@ -11423,12 +11386,7 @@ async function processGenerationJob(inferenceClient, params, onProgress, logger2
     title,
     format: outputMediaType,
     citations,
-    result: {
-      kind: "generation",
-      resourceId: "",
-      resourceName: title,
-      truncated: generated.truncated
-    }
+    truncated: generated.truncated
   };
 }
 
@@ -11512,11 +11470,11 @@ var CommitDurabilityError = class extends Error {
   durability;
   name = "CommitDurabilityError";
 };
-async function probeDurability(session, resourceId$1, annotations) {
+async function probeDurability(session, resourceId, annotations) {
   const last = annotations[annotations.length - 1];
   if (!last) return "probe-unreachable";
   try {
-    await session.client.browse.annotation(resourceId(resourceId$1), annotationId(String(last.id))).fresh();
+    await session.client.browse.annotation(resourceId, last.id).fresh();
     return "probe-confirmed";
   } catch (error) {
     const code = isObject(error) && isString(error.code) ? error.code : void 0;
@@ -11573,32 +11531,30 @@ function startWorkerProcess(config) {
       completedUnitsByJob.delete(job.jobId);
       unitCursorsByJob.delete(job.jobId);
       const failAnnotationId = referenceIdOf(job);
-      if (isJobType(job.type)) {
-        emitEvent(session, "job:fail", {
-          resourceId: job.resourceId,
-          jobId: job.jobId,
-          jobType: job.type,
-          ...failAnnotationId ? { annotationId: failAnnotationId } : {},
-          error: message,
-          ...completedUnits && completedUnits.length > 0 ? { completedUnits } : {},
-          // Where each unfinished unit got to. Absent rather than `{}` when
-          // nothing was reached: an empty object would claim units were
-          // tracked and none progressed.
-          ...unitCursors && Object.keys(unitCursors).length > 0 ? { unitCursors } : {},
-          ...failureClass !== void 0 ? { failureClass } : {},
-          // What the commit path OBSERVED about durability, when the failure
-          // came from a commit at all. Present only on that path: absent means
-          // the question never arose, never that durability was ruled out.
-          ...error instanceof CommitDurabilityError ? { durability: error.durability } : {},
-          // Whether this failure is the END, answered by the same predicate
-          // the queue applies at failJob (JOB-RESTART-SAFETY P5). Without
-          // it a client cannot tell a recovering run from a dead one: it
-          // sees job:fail either way and would end its stream on a job the
-          // queue is about to re-run.
-          willRetry: willRetryAfter(job, failureClass)
-        }).catch(() => {
-        });
-      }
+      emitEvent(session, "job:fail", {
+        resourceId: job.resourceId,
+        jobId: job.jobId,
+        jobType: job.type,
+        ...failAnnotationId ? { annotationId: failAnnotationId } : {},
+        error: message,
+        ...completedUnits && completedUnits.length > 0 ? { completedUnits } : {},
+        // Where each unfinished unit got to. Absent rather than `{}` when
+        // nothing was reached: an empty object would claim units were
+        // tracked and none progressed.
+        ...unitCursors && Object.keys(unitCursors).length > 0 ? { unitCursors } : {},
+        ...failureClass !== void 0 ? { failureClass } : {},
+        // What the commit path OBSERVED about durability, when the failure
+        // came from a commit at all. Present only on that path: absent means
+        // the question never arose, never that durability was ruled out.
+        ...error instanceof CommitDurabilityError ? { durability: error.durability } : {},
+        // Whether this failure is the END, answered by the same predicate
+        // the queue applies at failJob (JOB-RESTART-SAFETY P5). Without
+        // it a client cannot tell a recovering run from a dead one: it
+        // sees job:fail either way and would end its stream on a job the
+        // queue is about to re-run.
+        willRetry: willRetryAfter(job, failureClass)
+      }).catch(() => {
+      });
       adapter.failJob(job.jobId, message);
     }).finally(() => {
       if (activeCancel?.jobId === job.jobId) activeCancel = null;
@@ -11632,17 +11588,11 @@ async function handleJob(adapter, config, job, completedUnitsByJob = /* @__PURE_
 }
 async function handleJobInner(adapter, config, job, completedUnitsByJob, signal, unitCursorsByJob = /* @__PURE__ */ new Map()) {
   const { session, inferenceClient, generator } = config;
-  const { jobId } = job;
-  if (!isJobType(job.type)) {
-    adapter.failJob(jobId, `Unrecognized job type: ${job.type}`);
-    return;
-  }
-  const jobType = job.type;
-  const resourceId$1 = resourceId(job.resourceId);
+  const { jobId, type: jobType, resourceId } = job;
   const annotationId = referenceIdOf(job);
   const attempt = job.retryCount + 1;
   const lifecycleBase = {
-    resourceId: resourceId$1,
+    resourceId,
     jobId,
     jobType,
     attempt,
@@ -11661,22 +11611,22 @@ async function handleJobInner(adapter, config, job, completedUnitsByJob, signal,
   }
   let ready = null;
   if (jobType !== "generation") {
-    const descriptor = await session.client.browse.resource(resourceId$1).fresh();
+    const descriptor = await session.client.browse.resource(resourceId).fresh();
     const mediaType = getPrimaryMediaType(descriptor);
     const source = await withSpan(
       "detection:prepare",
-      () => prepareDetection(mediaType ?? "", config.contentReads, resourceId$1, generator, (rid) => session.client.browse.resourceAnchoredText(rid)),
-      { attrs: { "resource.id": resourceId$1, "media.type": mediaType ?? "unknown" } }
+      () => prepareDetection(mediaType ?? "", config.contentReads, resourceId, generator, (rid) => session.client.browse.resourceAnchoredText(rid)),
+      { attrs: { "resource.id": resourceId, "media.type": mediaType ?? "unknown" } }
     );
     if ("declined" in source) {
       if (source.declined === "not-yet") {
-        throw new Error(`Anchored text not yet derived for resource ${resourceId$1} \u2014 Smelter has not settled; retrying`);
+        throw new Error(`Anchored text not yet derived for resource ${resourceId} \u2014 Smelter has not settled; retrying`);
       }
       if (source.declined === "no-extractor") {
-        throw new DeterministicJobError(`Cannot run ${jobType} on resource ${resourceId$1}: media type '${mediaType ?? "unknown"}' has no extractable text to analyze`);
+        throw new DeterministicJobError(`Cannot run ${jobType} on resource ${resourceId}: media type '${mediaType ?? "unknown"}' has no extractable text to analyze`);
       }
       if (source.declined === "no-map" || source.declined === "unknown") {
-        throw new DeterministicJobError(`Cannot run ${jobType} on resource ${resourceId$1}: anchored-text consult returned '${source.declined}'`);
+        throw new DeterministicJobError(`Cannot run ${jobType} on resource ${resourceId}: anchored-text consult returned '${source.declined}'`);
       }
       await emitEvent(session, "job:complete", {
         ...terminalBase(),
@@ -11707,7 +11657,7 @@ async function handleJobInner(adapter, config, job, completedUnitsByJob, signal,
   };
   const unitCursors = /* @__PURE__ */ new Map();
   const commitChunk = async (annotations, checkpoint) => {
-    record(await commitAnnotations(session, String(resourceId$1), annotations, jobId));
+    record(await commitAnnotations(session, resourceId, annotations, jobId));
     unitCursors.set(checkpoint.unit, checkpoint.cursor);
     unitCursorsByJob.set(job.jobId, Object.fromEntries(unitCursors));
     await emitEvent(session, "job:checkpoint", {
@@ -11868,7 +11818,7 @@ async function handleJobInner(adapter, config, job, completedUnitsByJob, signal,
       file: Buffer.from(genResult.content),
       format: genResult.format,
       storageUri,
-      sourceResourceId: resourceId$1,
+      sourceResourceId: resourceId,
       ...genReferenceId ? { sourceAnnotationId: genReferenceId } : {},
       ...genParams.prompt ? { generationPrompt: genParams.prompt } : {},
       ...genParams.language ? { language: genParams.language } : {},
@@ -11882,12 +11832,12 @@ async function handleJobInner(adapter, config, job, completedUnitsByJob, signal,
       const { annotation: provenanceRef } = assembleAnnotation(
         {
           motivation: "linking",
-          target: { source: String(resourceId$1) },
-          body: { type: "SpecificResource", source: String(newResourceId), purpose: "linking" }
+          target: { source: resourceId },
+          body: { type: "SpecificResource", source: newResourceId, purpose: "linking" }
         },
         generator
       );
-      record(await commitAnnotations(session, String(resourceId$1), [provenanceRef], jobId));
+      record(await commitAnnotations(session, resourceId, [provenanceRef], jobId));
     }
     const citationRefs = [];
     if (genResult.format === "application/pdf" && genResult.citations.length > 0) {
@@ -11912,7 +11862,7 @@ async function handleJobInner(adapter, config, job, completedUnitsByJob, signal,
           }
           const citationRef = buildPdfAnnotation(
             layer,
-            resourceId(String(newResourceId)),
+            newResourceId,
             generator,
             "linking",
             { exact: layer.text.slice(span.start, span.end), start: span.start, end: span.end },
@@ -11927,7 +11877,7 @@ async function handleJobInner(adapter, config, job, completedUnitsByJob, signal,
           {
             motivation: "linking",
             target: {
-              source: String(newResourceId),
+              source: newResourceId,
               selector: [
                 { type: "TextPositionSelector", start: citation.start, end: citation.end },
                 { type: "TextQuoteSelector", exact: citation.exact }
@@ -11940,10 +11890,10 @@ async function handleJobInner(adapter, config, job, completedUnitsByJob, signal,
         citationRefs.push(citationRef);
       }
     }
-    record(await commitAnnotations(session, String(newResourceId), citationRefs, jobId));
+    record(await commitAnnotations(session, newResourceId, citationRefs, jobId));
     await emitEvent(session, "job:complete", {
       ...terminalBase(),
-      result: { kind: "generation", resourceId: newResourceId, resourceName: genResult.title, truncated: genResult.result.truncated }
+      result: { kind: "generation", resourceId: newResourceId, resourceName: genResult.title, truncated: genResult.truncated }
     });
     adapter.completeJob();
   } else {
@@ -12163,14 +12113,6 @@ async function startAgentWorker(opts) {
     }
   };
 }
-var ALL_JOB_TYPES = [
-  "reference-annotation",
-  "generation",
-  "highlight-annotation",
-  "assessment-annotation",
-  "comment-annotation",
-  "tag-annotation"
-];
 var configPath = join(homedir(), ".semiontconfig");
 var tomlReader = {
   readIfExists: (p) => existsSync(p) ? readFileSync(p, "utf-8") : null
@@ -12227,7 +12169,7 @@ function toClientConfig(w) {
   };
 }
 var groups = /* @__PURE__ */ new Map();
-for (const jobType of ALL_JOB_TYPES) {
+for (const jobType of JOB_TYPES) {
   const inference = resolveWorker(jobType);
   const key = clientKey(inference);
   let group = groups.get(key);