@semiont/jobs 0.5.19 → 0.5.21

package/dist/worker-main.js

@@ -1,7 +1,7 @@
-import { createTomlConfigLoader, didToAgent, baseUrl, STARTUP_FETCH_RETRY, retryWithBackoff, isTransientFetchError, busRequest, getPrimaryMediaType, textExtractionOf, assembleAnnotation, reconcileSelector, getLocaleEnglishName, isArray, isObject, isString, deriveViews } from '@semiont/core';
-import { deriveStorageUri } from '@semiont/content';
-import { withSpan, SpanKind, recordJobOutcome } from '@semiont/observability';
+import { createTomlConfigLoader, didToAgent, baseUrl, STARTUP_FETCH_RETRY, retryWithBackoff, isTransientFetchError, busRequest, getPrimaryMediaType, textExtractionOf, assembleAnnotation, reconcileSelector, createFragmentSelector, getLocaleEnglishName, isArray, isObject, isString, deriveViews } from '@semiont/core';
+import { deriveStorageUri, extractPdfTextLayer, locate } from '@semiont/content';
 import { generateAnnotationId } from '@semiont/event-sourcing';
+import { withSpan, SpanKind, recordJobOutcome } from '@semiont/observability';
 import { homedir, hostname } from 'os';
 import { InMemorySessionStorage, setStoredSession, kbBackendUrl, SemiontClient, SemiontSession } from '@semiont/sdk';
 import { HttpTransport, HttpContentTransport } from '@semiont/http-transport';
@@ -10289,12 +10289,15 @@ function annotationDedupeKey(ann) {
   const target = ann.target;
   const selectors = Array.isArray(target?.selector) ? target.selector : [];
   const pos = selectors.find((s) => s.type === "TextPositionSelector");
-  return [
-    ann.motivation,
-    pos?.start ?? "?",
-    pos?.end ?? "?",
-    JSON.stringify(ann.body ?? null)
-  ].join("|");
+  let anchor;
+  if (pos) {
+    anchor = `pos:${pos.start ?? "?"}:${pos.end ?? "?"}`;
+  } else {
+    const frags = selectors.filter((s) => s.type === "FragmentSelector").map((s) => s.value ?? "").join(",");
+    const quote = selectors.find((s) => s.type === "TextQuoteSelector");
+    anchor = `frag:${frags}|quote:${quote?.exact ?? ""}:${quote?.prefix ?? ""}:${quote?.suffix ?? ""}`;
+  }
+  return [ann.motivation, anchor, JSON.stringify(ann.body ?? null)].join("|");
 }
 function dedupeAnnotations(annotations) {
   const seen = /* @__PURE__ */ new Set();
@@ -10356,7 +10359,50 @@ function buildTextAnnotation(content, resourceId, userId, generator, motivation,
     ...body !== void 0 ? { body } : {}
   };
 }
-async function processHighlightJob(content, inferenceClient, params, userId, generator, onProgress) {
+function buildPdfAnnotation(layer, resourceId, userId, generator, motivation, match, body) {
+  const { rects, overlap } = locate(layer, match.start, match.end);
+  const coveredText = overlap.length ? layer.text.substring(
+    Math.min(...overlap.map((i) => i.start)),
+    Math.max(...overlap.map((i) => i.end))
+  ) : "";
+  const normalize = (s) => s.replace(/\s+/g, " ").trim();
+  if (rects.length === 0 || !normalize(coveredText).includes(normalize(match.exact))) {
+    throw new Error(
+      `buildPdfAnnotation invariant: covered text does not contain exact for resource ${resourceId}, motivation ${motivation}`
+    );
+  }
+  const creator = didToAgent(userId);
+  const wasAttributedTo = creator["@id"] === generator["@id"] ? [generator] : [creator, generator];
+  return {
+    "@context": "http://www.w3.org/ns/anno.jsonld",
+    "type": "Annotation",
+    "id": generateAnnotationId(),
+    motivation,
+    creator,
+    generator,
+    wasAttributedTo,
+    created: (/* @__PURE__ */ new Date()).toISOString(),
+    target: {
+      type: "SpecificResource",
+      source: resourceId,
+      selector: [
+        ...rects.map((coord) => ({
+          type: "FragmentSelector",
+          conformsTo: "http://tools.ietf.org/rfc/rfc3778",
+          value: createFragmentSelector(coord)
+        })),
+        {
+          type: "TextQuoteSelector",
+          exact: match.exact,
+          ...match.prefix && { prefix: match.prefix },
+          ...match.suffix && { suffix: match.suffix }
+        }
+      ]
+    },
+    ...body !== void 0 ? { body } : {}
+  };
+}
+async function processHighlightJob(content, inferenceClient, params, buildAnnotation, onProgress) {
   onProgress(10, "Loading resource...", "analyzing");
   onProgress(30, "Analyzing text...", "analyzing");
   const highlights = await AnnotationDetection.detectHighlights(
@@ -10368,7 +10414,7 @@ async function processHighlightJob(content, inferenceClient, params, userId, gen
   );
   onProgress(60, `Creating ${highlights.length} annotations...`, "creating");
   const annotations = dedupeAnnotations(highlights.map(
-    (h) => buildTextAnnotation(content, params.resourceId, userId, generator, "highlighting", h)
+    (h) => buildAnnotation("highlighting", h)
   ));
   onProgress(100, `Complete! Created ${annotations.length} highlights`, "creating");
   return {
@@ -10376,7 +10422,7 @@ async function processHighlightJob(content, inferenceClient, params, userId, gen
     result: { highlightsFound: highlights.length, highlightsCreated: annotations.length }
   };
 }
-async function processCommentJob(content, inferenceClient, params, userId, generator, onProgress) {
+async function processCommentJob(content, inferenceClient, params, buildAnnotation, onProgress) {
   onProgress(10, "Loading resource...", "analyzing");
   onProgress(30, "Analyzing text...", "analyzing");
   const comments = await AnnotationDetection.detectComments(
@@ -10395,7 +10441,7 @@ async function processCommentJob(content, inferenceClient, params, userId, gener
       // Match the pre-#651 CommentAnnotationWorker: include format and
       // language on the body TextualBody. Optional in the schema, but
       // consumers that do language-aware rendering rely on them.
-      buildTextAnnotation(content, params.resourceId, userId, generator, "commenting", c, [
+      buildAnnotation("commenting", c, [
         { type: "TextualBody", value: c.comment, purpose: "commenting", format: "text/plain", language: bodyLanguage }
       ])
     )
@@ -10406,7 +10452,7 @@ async function processCommentJob(content, inferenceClient, params, userId, gener
     result: { commentsFound: comments.length, commentsCreated: annotations.length }
   };
 }
-async function processAssessmentJob(content, inferenceClient, params, userId, generator, onProgress) {
+async function processAssessmentJob(content, inferenceClient, params, buildAnnotation, onProgress) {
   onProgress(10, "Loading resource...", "analyzing");
   onProgress(30, "Analyzing text...", "analyzing");
   const assessments = await AnnotationDetection.detectAssessments(
@@ -10428,7 +10474,7 @@ async function processAssessmentJob(content, inferenceClient, params, userId, ge
       // purpose='describing' — that loses the "this is an assessment, not
       // a description" signal and breaks existing readers that access
       // `body.value` directly on the object.
-      buildTextAnnotation(content, params.resourceId, userId, generator, "assessing", a, {
+      buildAnnotation("assessing", a, {
         type: "TextualBody",
         value: a.assessment,
         purpose: "assessing",
@@ -10443,7 +10489,7 @@ async function processAssessmentJob(content, inferenceClient, params, userId, ge
     result: { assessmentsFound: assessments.length, assessmentsCreated: annotations.length }
   };
 }
-async function processReferenceJob(content, inferenceClient, params, userId, generator, onProgress, logger2) {
+async function processReferenceJob(content, inferenceClient, params, buildAnnotation, onProgress, logger2) {
   const entityTypeNames = params.entityTypes.map(String);
   const requestParams = [{ label: "Entity types", value: entityTypeNames.join(", ") }];
   const completedEntityTypes = [];
@@ -10500,15 +10546,7 @@ async function processReferenceJob(content, inferenceClient, params, userId, gen
           anchorMethod: reconciled.anchorMethod
         });
       }
-      const ann = buildTextAnnotation(
-        content,
-        params.resourceId,
-        userId,
-        generator,
-        "linking",
-        toMatch(reconciled),
-        unresolvedBody
-      );
+      const ann = buildAnnotation("linking", toMatch(reconciled), unresolvedBody);
       allAnnotations.push(ann);
       totalEmitted++;
     }
@@ -10520,7 +10558,7 @@ async function processReferenceJob(content, inferenceClient, params, userId, gen
     result: { totalFound, totalEmitted: annotations.length, errors }
   };
 }
-async function processTagJob(content, inferenceClient, params, userId, generator, onProgress) {
+async function processTagJob(content, inferenceClient, params, buildAnnotation, onProgress) {
   onProgress(10, "Loading resource...", "analyzing");
   onProgress(30, "Analyzing text for tags...", "analyzing");
   const allTags = [];
@@ -10539,7 +10577,7 @@ async function processTagJob(content, inferenceClient, params, userId, generator
   const bodyLanguage = params.language ?? "en";
   const annotations = dedupeAnnotations(tags.map((t) => {
     const category = t.category ?? "unknown";
-    return buildTextAnnotation(content, params.resourceId, userId, generator, "tagging", t, [
+    return buildAnnotation("tagging", t, [
       { type: "TextualBody", value: category, purpose: "tagging", format: "text/plain", language: bodyLanguage },
       { type: "TextualBody", value: params.schema.id, purpose: "classifying", format: "text/plain" }
     ]);
@@ -10603,7 +10641,29 @@ async function processGenerationJob(inferenceClient, params, onProgress, logger2
   };
 }
 
-// src/worker-process.ts
+// src/workers/detection/prepare-detection.ts
+async function prepareDetection(strategy, session, resourceId, userId, generator) {
+  switch (strategy) {
+    case "decode": {
+      const text = await session.client.browse.resourceContent(resourceId);
+      return {
+        text,
+        buildAnnotation: (motivation, match, body) => buildTextAnnotation(text, resourceId, userId, generator, motivation, match, body)
+      };
+    }
+    case "pdf-text-layer": {
+      const { data } = await session.client.browse.resourceRepresentation(resourceId);
+      const layer = await extractPdfTextLayer(new Uint8Array(data));
+      if (!layer) return null;
+      return {
+        text: layer.text,
+        buildAnnotation: (motivation, match, body) => buildPdfAnnotation(layer, resourceId, userId, generator, motivation, match, body)
+      };
+    }
+    case "none":
+      return null;
+  }
+}
 async function emitEvent(session, channel, payload) {
   await session.client.transport.emit(channel, payload);
 }
@@ -10675,16 +10735,23 @@ async function handleJobInner(adapter, config, job) {
     adapter.failJob(jobId, `Worker not configured for job type: ${jobType}`);
     return;
   }
+  let source = null;
   if (jobType !== "generation") {
     const descriptor = await session.client.browse.resource(resourceId);
     const mediaType = getPrimaryMediaType(descriptor);
-    const extraction = mediaType ? textExtractionOf(mediaType) : "none";
-    if (extraction === "pdf-text-layer") {
-      throw new Error(`Cannot run ${jobType} on resource ${resourceId}: PDF text-layer detection is not yet supported`);
-    }
-    if (extraction !== "decode") {
+    const strategy = mediaType ? textExtractionOf(mediaType) : "none";
+    if (strategy === "none") {
       throw new Error(`Cannot run ${jobType} on resource ${resourceId}: media type '${mediaType ?? "unknown"}' has no extractable text to analyze`);
     }
+    source = await prepareDetection(strategy, session, resourceId, userId, generator);
+    if (!source) {
+      await emitEvent(session, "job:complete", {
+        ...lifecycleBase,
+        result: { declined: true, reason: "no-text-layer", message: "This PDF has no extractable text layer (scanned or image-only); detection is not supported." }
+      });
+      adapter.completeJob();
+      return;
+    }
   }
   const onProgress = (percentage, message, stage, extra) => {
     adapter.touchActivity();
@@ -10701,17 +10768,12 @@ async function handleJobInner(adapter, config, job) {
     }).catch(() => {
     });
   };
-  const fetchContent = async () => {
-    return session.client.browse.resourceContent(resourceId);
-  };
   if (jobType === "highlight-annotation") {
-    const content = await fetchContent();
     const { annotations, result } = await processHighlightJob(
-      content,
+      source.text,
       inferenceClient,
       job.params,
-      userId,
-      generator,
+      source.buildAnnotation,
       onProgress
     );
     for (const ann of annotations) {
@@ -10723,13 +10785,11 @@ async function handleJobInner(adapter, config, job) {
     });
     adapter.completeJob();
   } else if (jobType === "comment-annotation") {
-    const content = await fetchContent();
     const { annotations, result } = await processCommentJob(
-      content,
+      source.text,
       inferenceClient,
       job.params,
-      userId,
-      generator,
+      source.buildAnnotation,
       onProgress
     );
     for (const ann of annotations) {
@@ -10741,13 +10801,11 @@ async function handleJobInner(adapter, config, job) {
     });
     adapter.completeJob();
   } else if (jobType === "assessment-annotation") {
-    const content = await fetchContent();
     const { annotations, result } = await processAssessmentJob(
-      content,
+      source.text,
       inferenceClient,
       job.params,
-      userId,
-      generator,
+      source.buildAnnotation,
       onProgress
     );
     for (const ann of annotations) {
@@ -10759,13 +10817,11 @@ async function handleJobInner(adapter, config, job) {
     });
     adapter.completeJob();
   } else if (jobType === "reference-annotation") {
-    const content = await fetchContent();
     const { annotations, result } = await processReferenceJob(
-      content,
+      source.text,
       inferenceClient,
       job.params,
-      userId,
-      generator,
+      source.buildAnnotation,
       onProgress,
       config.logger
     );
@@ -10778,13 +10834,11 @@ async function handleJobInner(adapter, config, job) {
     });
     adapter.completeJob();
   } else if (jobType === "tag-annotation") {
-    const content = await fetchContent();
     const { annotations, result } = await processTagJob(
-      content,
+      source.text,
       inferenceClient,
       job.params,
-      userId,
-      generator,
+      source.buildAnnotation,
       onProgress
     );
     for (const ann of annotations) {