@semiont/jobs 0.5.8 → 0.5.10

package/dist/worker-main.js

@@ -1,4 +1,4 @@
-import { createTomlConfigLoader, softwareToAgent, baseUrl, getPrimaryMediaType, textExtractionOf, reconcileSelector, didToAgent, getLocaleEnglishName } from '@semiont/core';
+import { createTomlConfigLoader, softwareToAgent, baseUrl, busRequest, getPrimaryMediaType, textExtractionOf, assembleAnnotation, reconcileSelector, didToAgent, getLocaleEnglishName, isArray, isObject, isString, deriveViews } from '@semiont/core';
 import { deriveStorageUri } from '@semiont/content';
 import { withSpan, SpanKind, recordJobOutcome } from '@semiont/observability';
 import { generateAnnotationId } from '@semiont/event-sourcing';
@@ -326,16 +326,16 @@ var require_timeoutProvider = __commonJS({
     Object.defineProperty(exports, "__esModule", { value: true });
     exports.timeoutProvider = void 0;
     exports.timeoutProvider = {
-      setTimeout: function(handler, timeout2) {
+      setTimeout: function(handler, timeout) {
         var args = [];
         for (var _i = 2; _i < arguments.length; _i++) {
           args[_i - 2] = arguments[_i];
         }
         var delegate = exports.timeoutProvider.delegate;
         if (delegate === null || delegate === void 0 ? void 0 : delegate.setTimeout) {
-          return delegate.setTimeout.apply(delegate, __spreadArray([handler, timeout2], __read(args)));
+          return delegate.setTimeout.apply(delegate, __spreadArray([handler, timeout], __read(args)));
         }
-        return setTimeout.apply(void 0, __spreadArray([handler, timeout2], __read(args)));
+        return setTimeout.apply(void 0, __spreadArray([handler, timeout], __read(args)));
       },
       clearTimeout: function(handle) {
         var delegate = exports.timeoutProvider.delegate;
@@ -1659,16 +1659,16 @@ var require_intervalProvider = __commonJS({
     Object.defineProperty(exports, "__esModule", { value: true });
     exports.intervalProvider = void 0;
     exports.intervalProvider = {
-      setInterval: function(handler, timeout2) {
+      setInterval: function(handler, timeout) {
         var args = [];
         for (var _i = 2; _i < arguments.length; _i++) {
           args[_i - 2] = arguments[_i];
         }
         var delegate = exports.intervalProvider.delegate;
         if (delegate === null || delegate === void 0 ? void 0 : delegate.setInterval) {
-          return delegate.setInterval.apply(delegate, __spreadArray([handler, timeout2], __read(args)));
+          return delegate.setInterval.apply(delegate, __spreadArray([handler, timeout], __read(args)));
         }
-        return setInterval.apply(void 0, __spreadArray([handler, timeout2], __read(args)));
+        return setInterval.apply(void 0, __spreadArray([handler, timeout], __read(args)));
       },
       clearInterval: function(handle) {
         var delegate = exports.intervalProvider.delegate;
@@ -3566,7 +3566,7 @@ var require_firstValueFrom = __commonJS({
     exports.firstValueFrom = void 0;
     var EmptyError_1 = require_EmptyError();
     var Subscriber_1 = require_Subscriber();
-    function firstValueFrom2(source, config) {
+    function firstValueFrom(source, config) {
       var hasConfig = typeof config === "object";
       return new Promise(function(resolve, reject) {
         var subscriber = new Subscriber_1.SafeSubscriber({
@@ -3586,7 +3586,7 @@ var require_firstValueFrom = __commonJS({
         source.subscribe(subscriber);
       });
     }
-    exports.firstValueFrom = firstValueFrom2;
+    exports.firstValueFrom = firstValueFrom;
   }
 });
 
@@ -3673,7 +3673,7 @@ var require_timeout = __commonJS({
         this.info = info;
       };
     });
-    function timeout2(config, schedulerArg) {
+    function timeout(config, schedulerArg) {
       var _a = isDate_1.isValidDate(config) ? { first: config } : typeof config === "number" ? { each: config } : config, first = _a.first, each = _a.each, _b = _a.with, _with = _b === void 0 ? timeoutErrorFactory : _b, _c = _a.scheduler, scheduler = _c === void 0 ? schedulerArg !== null && schedulerArg !== void 0 ? schedulerArg : async_1.asyncScheduler : _c, _d = _a.meta, meta = _d === void 0 ? null : _d;
       if (first == null && each == null) {
         throw new TypeError("No timeout provided.");
@@ -3711,7 +3711,7 @@ var require_timeout = __commonJS({
         !seen && startTimer(first != null ? typeof first === "number" ? first : +first - scheduler.now() : each);
       });
     }
-    exports.timeout = timeout2;
+    exports.timeout = timeout;
     function timeoutErrorFactory(info) {
       throw new exports.TimeoutError(info);
     }
@@ -3725,7 +3725,7 @@ var require_map = __commonJS({
     exports.map = void 0;
     var lift_1 = require_lift();
     var OperatorSubscriber_1 = require_OperatorSubscriber();
-    function map2(project, thisArg) {
+    function map(project, thisArg) {
       return lift_1.operate(function(source, subscriber) {
         var index = 0;
         source.subscribe(OperatorSubscriber_1.createOperatorSubscriber(subscriber, function(value) {
@@ -3733,7 +3733,7 @@ var require_map = __commonJS({
         }));
       });
     }
-    exports.map = map2;
+    exports.map = map;
   }
 });
 
@@ -3765,9 +3765,9 @@ var require_mapOneOrManyArgs = __commonJS({
     Object.defineProperty(exports, "__esModule", { value: true });
     exports.mapOneOrManyArgs = void 0;
     var map_1 = require_map();
-    var isArray = Array.isArray;
+    var isArray2 = Array.isArray;
     function callOrApply(fn, args) {
-      return isArray(args) ? fn.apply(void 0, __spreadArray([], __read(args))) : fn(args);
+      return isArray2(args) ? fn.apply(void 0, __spreadArray([], __read(args))) : fn(args);
     }
     function mapOneOrManyArgs(fn) {
       return map_1.map(function(args) {
@@ -3912,14 +3912,14 @@ var require_argsArgArrayOrObject = __commonJS({
   "../../node_modules/rxjs/dist/cjs/internal/util/argsArgArrayOrObject.js"(exports) {
     Object.defineProperty(exports, "__esModule", { value: true });
     exports.argsArgArrayOrObject = void 0;
-    var isArray = Array.isArray;
+    var isArray2 = Array.isArray;
     var getPrototypeOf = Object.getPrototypeOf;
     var objectProto = Object.prototype;
     var getKeys = Object.keys;
     function argsArgArrayOrObject(args) {
       if (args.length === 1) {
         var first_1 = args[0];
-        if (isArray(first_1)) {
+        if (isArray2(first_1)) {
           return { args: first_1, keys: null };
         }
         if (isPOJO(first_1)) {
@@ -4633,7 +4633,7 @@ var require_merge = __commonJS({
     var empty_1 = require_empty();
     var args_1 = require_args();
     var from_1 = require_from();
-    function merge2() {
+    function merge() {
       var args = [];
       for (var _i = 0; _i < arguments.length; _i++) {
         args[_i] = arguments[_i];
@@ -4643,7 +4643,7 @@ var require_merge = __commonJS({
       var sources = args;
       return !sources.length ? empty_1.EMPTY : sources.length === 1 ? innerFrom_1.innerFrom(sources[0]) : mergeAll_1.mergeAll(concurrent)(from_1.from(sources, scheduler));
     }
-    exports.merge = merge2;
+    exports.merge = merge;
   }
 });
 
@@ -4667,9 +4667,9 @@ var require_argsOrArgArray = __commonJS({
   "../../node_modules/rxjs/dist/cjs/internal/util/argsOrArgArray.js"(exports) {
     Object.defineProperty(exports, "__esModule", { value: true });
     exports.argsOrArgArray = void 0;
-    var isArray = Array.isArray;
+    var isArray2 = Array.isArray;
     function argsOrArgArray(args) {
-      return args.length === 1 && isArray(args[0]) ? args[0] : args;
+      return args.length === 1 && isArray2(args[0]) ? args[0] : args;
     }
     exports.argsOrArgArray = argsOrArgArray;
   }
@@ -4750,7 +4750,7 @@ var require_filter = __commonJS({
     exports.filter = void 0;
     var lift_1 = require_lift();
     var OperatorSubscriber_1 = require_OperatorSubscriber();
-    function filter2(predicate, thisArg) {
+    function filter(predicate, thisArg) {
       return lift_1.operate(function(source, subscriber) {
         var index = 0;
         source.subscribe(OperatorSubscriber_1.createOperatorSubscriber(subscriber, function(value) {
@@ -4758,7 +4758,7 @@ var require_filter = __commonJS({
         }));
       });
     }
-    exports.filter = filter2;
+    exports.filter = filter;
   }
 });
 
@@ -5885,7 +5885,7 @@ var require_take = __commonJS({
     var empty_1 = require_empty();
     var lift_1 = require_lift();
     var OperatorSubscriber_1 = require_OperatorSubscriber();
-    function take2(count) {
+    function take(count) {
       return count <= 0 ? function() {
         return empty_1.EMPTY;
       } : lift_1.operate(function(source, subscriber) {
@@ -5900,7 +5900,7 @@ var require_take = __commonJS({
         }));
       });
     }
-    exports.take = take2;
+    exports.take = take;
   }
 });
 
@@ -6692,7 +6692,7 @@ var require_merge2 = __commonJS({
     var mergeAll_1 = require_mergeAll();
     var args_1 = require_args();
     var from_1 = require_from();
-    function merge2() {
+    function merge() {
       var args = [];
       for (var _i = 0; _i < arguments.length; _i++) {
         args[_i] = arguments[_i];
@@ -6703,7 +6703,7 @@ var require_merge2 = __commonJS({
         mergeAll_1.mergeAll(concurrent)(from_1.from(__spreadArray([source], __read(args)), scheduler)).subscribe(subscriber);
       });
     }
-    exports.merge = merge2;
+    exports.merge = merge;
   }
 });
 
@@ -9207,9 +9207,19 @@ var require_cjs = __commonJS({
 
 // src/job-claim-adapter.ts
 var import_rxjs = __toESM(require_cjs());
-var import_rxjs2 = __toESM(require_cjs());
+function workerBusAsPrimitive(bus) {
+  return {
+    emit(channel, payload) {
+      return bus.emit(channel, payload);
+    },
+    stream(channel) {
+      return bus.on$(channel);
+    }
+  };
+}
 function createJobClaimAdapter(options) {
   const { bus, jobTypes } = options;
+  const requestBus = workerBusAsPrimitive(bus);
   const activeJob$ = new import_rxjs.BehaviorSubject(null);
   const isProcessing$ = new import_rxjs.BehaviorSubject(false);
   const jobsCompleted$ = new import_rxjs.BehaviorSubject(0);
@@ -9218,22 +9228,7 @@ function createJobClaimAdapter(options) {
   let started = false;
   const claimJob = async (assignment) => {
     try {
-      const correlationId = crypto.randomUUID();
-      const result$ = (0, import_rxjs2.merge)(
-        bus.on$("job:claimed").pipe(
-          (0, import_rxjs2.filter)((e) => e.correlationId === correlationId),
-          (0, import_rxjs2.map)((e) => ({ ok: true, response: e.response }))
-        ),
-        bus.on$("job:claim-failed").pipe(
-          (0, import_rxjs2.filter)((e) => e.correlationId === correlationId),
-          (0, import_rxjs2.map)(() => ({ ok: false }))
-        )
-      ).pipe((0, import_rxjs2.take)(1), (0, import_rxjs2.timeout)(1e4));
-      const resultPromise = (0, import_rxjs2.firstValueFrom)(result$);
-      await bus.emit("job:claim", { correlationId, jobId: assignment.jobId });
-      const result = await resultPromise;
-      if (!result.ok) return null;
-      const job = result.response;
+      const job = await busRequest(requestBus, "job:claim", { jobId: assignment.jobId }, 1e4);
       return {
         jobId: assignment.jobId,
         type: assignment.type,
@@ -9587,197 +9582,144 @@ Example format:
     return prompt;
   }
 };
-function extractObjectsFromArray(response) {
-  let cleaned = response.trim();
-  if (cleaned.startsWith("```")) {
-    cleaned = cleaned.replace(/^```(?:json)?\s*\n?/, "").replace(/\n?```\s*$/, "");
-  }
+function parseJsonArray(response, motivation) {
+  let parsed;
   try {
-    const parsed = JSON.parse(cleaned);
-    return Array.isArray(parsed) ? parsed : [];
-  } catch {
-  }
-  const start = cleaned.indexOf("[");
-  if (start === -1) return [];
-  const endBracket = cleaned.lastIndexOf("]");
-  const end = endBracket > start ? endBracket : cleaned.length;
-  const inner = cleaned.slice(start + 1, end);
-  const objects = [];
-  let depth = 0;
-  let objStart = -1;
-  let inString = false;
-  let escape = false;
-  for (let i = 0; i < inner.length; i++) {
-    const ch = inner[i];
-    if (escape) {
-      escape = false;
-      continue;
-    }
-    if (ch === "\\") {
-      escape = true;
-      continue;
-    }
-    if (ch === '"') {
-      inString = !inString;
-      continue;
-    }
-    if (inString) continue;
-    if (ch === "{") {
-      if (depth === 0) objStart = i;
-      depth++;
-    } else if (ch === "}") {
-      depth--;
-      if (depth === 0 && objStart !== -1) {
-        try {
-          objects.push(JSON.parse(inner.slice(objStart, i + 1)));
-        } catch {
-        }
-        objStart = -1;
-      }
-    }
+    parsed = JSON.parse(response.trim());
+  } catch (error) {
+    console.error(`[MotivationParsers] Failed to parse AI ${motivation} response:`, error);
+    console.error("Raw response:", response);
+    throw error instanceof Error ? error : new Error(String(error));
+  }
+  if (!Array.isArray(parsed)) {
+    console.error(`[MotivationParsers] Expected a JSON array for ${motivation} detection, got ${typeof parsed}:`, response);
+    throw new Error(`Expected a JSON array for ${motivation} detection, got ${typeof parsed}`);
   }
-  return objects;
+  return parsed;
 }
 var MotivationParsers = class {
   /**
    * Parse and validate AI response for comment detection
    *
-   * @param response - Raw AI response string (may include markdown code fences)
+   * @param response - Raw AI response text (a JSON array)
    * @param content - Original content to validate offsets against
    * @returns Array of validated comment matches
+   * @throws if the response is not a parseable JSON array
    */
   static parseComments(response, content) {
-    try {
-      const parsed = extractObjectsFromArray(response);
-      const valid = parsed.filter(
-        (c) => !!c && typeof c === "object" && typeof c.exact === "string" && typeof c.comment === "string" && c.comment.trim().length > 0
-      );
-      console.log(`[MotivationParsers] Parsed ${valid.length} valid comments from ${parsed.length} total`);
-      const validatedComments = [];
-      for (const comment of valid) {
-        const reconciled = reconcileSelector(content, {
-          exact: comment.exact,
-          ...typeof comment.prefix === "string" ? { prefix: comment.prefix } : {},
-          ...typeof comment.suffix === "string" ? { suffix: comment.suffix } : {}
-        });
-        if (!reconciled) {
-          console.warn(`[MotivationParsers] Dropped hallucinated comment "${comment.exact}"`);
-          continue;
-        }
-        logAnchorMethod("comment", comment.exact, reconciled.anchorMethod);
-        validatedComments.push({
-          comment: comment.comment,
-          exact: reconciled.exact,
-          start: reconciled.start,
-          end: reconciled.end,
-          ...reconciled.prefix !== void 0 ? { prefix: reconciled.prefix } : {},
-          ...reconciled.suffix !== void 0 ? { suffix: reconciled.suffix } : {}
-        });
+    const parsed = parseJsonArray(response, "comment");
+    const valid = parsed.filter(
+      (c) => isObject(c) && isString(c.exact) && isString(c.comment) && c.comment.trim().length > 0
+    );
+    console.log(`[MotivationParsers] Parsed ${valid.length} valid comments from ${parsed.length} total`);
+    const validatedComments = [];
+    for (const comment of valid) {
+      const reconciled = reconcileSelector(content, {
+        exact: comment.exact,
+        ...typeof comment.prefix === "string" ? { prefix: comment.prefix } : {},
+        ...typeof comment.suffix === "string" ? { suffix: comment.suffix } : {}
+      });
+      if (!reconciled) {
+        console.warn(`[MotivationParsers] Dropped hallucinated comment "${comment.exact}"`);
+        continue;
       }
-      return validatedComments;
-    } catch (error) {
-      console.error("[MotivationParsers] Failed to parse AI comment response:", error);
-      return [];
+      logAnchorMethod("comment", comment.exact, reconciled.anchorMethod);
+      validatedComments.push({
+        comment: comment.comment,
+        exact: reconciled.exact,
+        start: reconciled.start,
+        end: reconciled.end,
+        ...reconciled.prefix !== void 0 ? { prefix: reconciled.prefix } : {},
+        ...reconciled.suffix !== void 0 ? { suffix: reconciled.suffix } : {}
+      });
     }
+    return validatedComments;
   }
   /**
    * Parse and validate AI response for highlight detection
    *
-   * @param response - Raw AI response string (may include markdown code fences)
+   * @param response - Raw AI response text (a JSON array)
    * @param content - Original content to validate offsets against
    * @returns Array of validated highlight matches
+   * @throws if the response is not a parseable JSON array
    */
   static parseHighlights(response, content) {
-    try {
-      const parsed = extractObjectsFromArray(response);
-      const highlights = parsed.filter(
-        (h) => !!h && typeof h === "object" && typeof h.exact === "string"
-      );
-      const validatedHighlights = [];
-      for (const highlight of highlights) {
-        const reconciled = reconcileSelector(content, {
-          exact: highlight.exact,
-          ...typeof highlight.prefix === "string" ? { prefix: highlight.prefix } : {},
-          ...typeof highlight.suffix === "string" ? { suffix: highlight.suffix } : {}
-        });
-        if (!reconciled) {
-          console.warn(`[MotivationParsers] Dropped hallucinated highlight "${highlight.exact}"`);
-          continue;
-        }
-        logAnchorMethod("highlight", highlight.exact, reconciled.anchorMethod);
-        validatedHighlights.push({
-          exact: reconciled.exact,
-          start: reconciled.start,
-          end: reconciled.end,
-          ...reconciled.prefix !== void 0 ? { prefix: reconciled.prefix } : {},
-          ...reconciled.suffix !== void 0 ? { suffix: reconciled.suffix } : {}
-        });
+    const parsed = parseJsonArray(response, "highlight");
+    const highlights = parsed.filter(
+      (h) => isObject(h) && isString(h.exact)
+    );
+    const validatedHighlights = [];
+    for (const highlight of highlights) {
+      const reconciled = reconcileSelector(content, {
+        exact: highlight.exact,
+        ...typeof highlight.prefix === "string" ? { prefix: highlight.prefix } : {},
+        ...typeof highlight.suffix === "string" ? { suffix: highlight.suffix } : {}
+      });
+      if (!reconciled) {
+        console.warn(`[MotivationParsers] Dropped hallucinated highlight "${highlight.exact}"`);
+        continue;
       }
-      return validatedHighlights;
-    } catch (error) {
-      console.error("[MotivationParsers] Failed to parse AI highlight response:", error);
-      console.error("Raw response:", response);
-      return [];
+      logAnchorMethod("highlight", highlight.exact, reconciled.anchorMethod);
+      validatedHighlights.push({
+        exact: reconciled.exact,
+        start: reconciled.start,
+        end: reconciled.end,
+        ...reconciled.prefix !== void 0 ? { prefix: reconciled.prefix } : {},
+        ...reconciled.suffix !== void 0 ? { suffix: reconciled.suffix } : {}
+      });
     }
+    return validatedHighlights;
   }
   /**
    * Parse and validate AI response for assessment detection
    *
-   * @param response - Raw AI response string (may include markdown code fences)
+   * @param response - Raw AI response text (a JSON array)
    * @param content - Original content to validate offsets against
    * @returns Array of validated assessment matches
+   * @throws if the response is not a parseable JSON array
    */
   static parseAssessments(response, content) {
-    try {
-      const parsed = extractObjectsFromArray(response);
-      const assessments = parsed.filter(
-        (a) => !!a && typeof a === "object" && typeof a.exact === "string" && typeof a.assessment === "string"
-      );
-      const validatedAssessments = [];
-      for (const assessment of assessments) {
-        const reconciled = reconcileSelector(content, {
-          exact: assessment.exact,
-          ...typeof assessment.prefix === "string" ? { prefix: assessment.prefix } : {},
-          ...typeof assessment.suffix === "string" ? { suffix: assessment.suffix } : {}
-        });
-        if (!reconciled) {
-          console.warn(`[MotivationParsers] Dropped hallucinated assessment "${assessment.exact}"`);
-          continue;
-        }
-        logAnchorMethod("assessment", assessment.exact, reconciled.anchorMethod);
-        validatedAssessments.push({
-          assessment: assessment.assessment,
-          exact: reconciled.exact,
-          start: reconciled.start,
-          end: reconciled.end,
-          ...reconciled.prefix !== void 0 ? { prefix: reconciled.prefix } : {},
-          ...reconciled.suffix !== void 0 ? { suffix: reconciled.suffix } : {}
-        });
+    const parsed = parseJsonArray(response, "assessment");
+    const assessments = parsed.filter(
+      (a) => isObject(a) && isString(a.exact) && isString(a.assessment)
+    );
+    const validatedAssessments = [];
+    for (const assessment of assessments) {
+      const reconciled = reconcileSelector(content, {
+        exact: assessment.exact,
+        ...typeof assessment.prefix === "string" ? { prefix: assessment.prefix } : {},
+        ...typeof assessment.suffix === "string" ? { suffix: assessment.suffix } : {}
+      });
+      if (!reconciled) {
+        console.warn(`[MotivationParsers] Dropped hallucinated assessment "${assessment.exact}"`);
+        continue;
       }
-      return validatedAssessments;
-    } catch (error) {
-      console.error("[MotivationParsers] Failed to parse AI assessment response:", error);
-      console.error("Raw response:", response);
-      return [];
+      logAnchorMethod("assessment", assessment.exact, reconciled.anchorMethod);
+      validatedAssessments.push({
+        assessment: assessment.assessment,
+        exact: reconciled.exact,
+        start: reconciled.start,
+        end: reconciled.end,
+        ...reconciled.prefix !== void 0 ? { prefix: reconciled.prefix } : {},
+        ...reconciled.suffix !== void 0 ? { suffix: reconciled.suffix } : {}
+      });
     }
+    return validatedAssessments;
   }
   /**
    * Parse the LLM's tag response into raw, pre-reconciliation tag inputs.
    * Reconciliation happens in `validateTagOffsets`, which adds `start`/`end`
    * by anchoring `exact` against the source content.
+   *
+   * @throws if the response is not a parseable JSON array
    */
   static parseTags(response) {
-    try {
-      const parsed = extractObjectsFromArray(response);
-      const valid = parsed.filter(
-        (t) => !!t && typeof t === "object" && typeof t.exact === "string" && t.exact.trim().length > 0
-      );
-      console.log(`[MotivationParsers] Parsed ${valid.length} valid tags from ${parsed.length} total`);
-      return valid;
-    } catch (error) {
-      console.error("[MotivationParsers] Failed to parse AI tag response:", error);
-      return [];
-    }
+    const parsed = parseJsonArray(response, "tag");
+    const valid = parsed.filter(
+      (t) => isObject(t) && isString(t.exact) && t.exact.trim().length > 0
+    );
+    console.log(`[MotivationParsers] Parsed ${valid.length} valid tags from ${parsed.length} total`);
+    return valid;
   }
   /**
    * Anchor raw tag inputs against source content and add category.
@@ -9814,6 +9756,11 @@ function logAnchorMethod(motivation, exact, anchorMethod) {
 }
 
 // src/workers/annotation-detection.ts
+function assertNotTruncated(response, motivation) {
+  if (response.stopReason === "max_tokens") {
+    throw new Error(`${motivation} detection response truncated (max_tokens) \u2014 increase max_tokens or reduce resource size; failing the job rather than under-reporting annotations.`);
+  }
+}
 var AnnotationDetection = class {
   /**
    * Detect comments in content.
@@ -9825,8 +9772,9 @@ var AnnotationDetection = class {
    */
   static async detectComments(content, client, instructions, tone, density, language, sourceLanguage) {
     const prompt = MotivationPrompts.buildCommentPrompt(content, instructions, tone, density, language, sourceLanguage);
-    const response = await client.generateText(prompt, 3e3, 0.4, { format: "json" });
-    return MotivationParsers.parseComments(response, content);
+    const response = await client.generateTextWithMetadata(prompt, 3e3, 0.4, { format: "json" });
+    assertNotTruncated(response, "comment");
+    return MotivationParsers.parseComments(response.text, content);
   }
   /**
    * Detect highlights in content.
@@ -9837,8 +9785,9 @@ var AnnotationDetection = class {
    */
   static async detectHighlights(content, client, instructions, density, sourceLanguage) {
     const prompt = MotivationPrompts.buildHighlightPrompt(content, instructions, density, sourceLanguage);
-    const response = await client.generateText(prompt, 2e3, 0.3, { format: "json" });
-    return MotivationParsers.parseHighlights(response, content);
+    const response = await client.generateTextWithMetadata(prompt, 2e3, 0.3, { format: "json" });
+    assertNotTruncated(response, "highlight");
+    return MotivationParsers.parseHighlights(response.text, content);
   }
   /**
    * Detect assessments in content.
@@ -9849,8 +9798,9 @@ var AnnotationDetection = class {
    */
   static async detectAssessments(content, client, instructions, tone, density, language, sourceLanguage) {
     const prompt = MotivationPrompts.buildAssessmentPrompt(content, instructions, tone, density, language, sourceLanguage);
-    const response = await client.generateText(prompt, 3e3, 0.3, { format: "json" });
-    return MotivationParsers.parseAssessments(response, content);
+    const response = await client.generateTextWithMetadata(prompt, 3e3, 0.3, { format: "json" });
+    assertNotTruncated(response, "assessment");
+    return MotivationParsers.parseAssessments(response.text, content);
   }
   /**
    * Detect tags in content for a specific category.
@@ -9879,8 +9829,9 @@ var AnnotationDetection = class {
       categoryInfo.examples,
       sourceLanguage
     );
-    const response = await client.generateText(prompt, 4e3, 0.2, { format: "json" });
-    const parsedTags = MotivationParsers.parseTags(response);
+    const response = await client.generateTextWithMetadata(prompt, 4e3, 0.2, { format: "json" });
+    assertNotTruncated(response, "tag");
+    const parsedTags = MotivationParsers.parseTags(response.text);
     return MotivationParsers.validateTagOffsets(parsedTags, content, category);
   }
 };
@@ -9949,41 +9900,47 @@ Example output:
     { format: "json" }
   );
   logger2.debug("Got entity extraction response", { responseLength: response.text.length });
+  if (response.stopReason === "max_tokens") {
+    const errorMsg = "Entity extraction response truncated (max_tokens) \u2014 increase max_tokens or reduce resource size; failing the job rather than dropping annotations.";
+    logger2.error(errorMsg, { responseLength: response.text.length });
+    throw new Error(errorMsg);
+  }
+  let entities;
   try {
-    let jsonStr = response.text.trim();
-    if (jsonStr.startsWith("```")) {
-      jsonStr = jsonStr.replace(/^```(?:json)?\n?/, "").replace(/\n?```$/, "");
-    }
-    const entities = JSON.parse(jsonStr);
-    logger2.debug("Parsed entities from AI response", { count: entities.length });
-    if (response.stopReason === "max_tokens") {
-      const errorMsg = `AI response truncated: Found ${entities.length} entities but response hit max_tokens limit. Increase max_tokens or reduce resource size.`;
-      logger2.error(errorMsg);
-      throw new Error(errorMsg);
-    }
-    return entities.filter((e) => {
-      const ok = e && typeof e === "object" && typeof e.exact === "string" && typeof e.entityType === "string";
-      if (!ok) {
-        logger2.debug("Dropped malformed LLM entity", { entity: e });
-      }
-      return ok;
-    }).map((entity) => ({
-      exact: entity.exact,
-      entityType: entity.entityType,
-      ...typeof entity.prefix === "string" ? { prefix: entity.prefix } : {},
-      ...typeof entity.suffix === "string" ? { suffix: entity.suffix } : {}
-    }));
+    entities = JSON.parse(response.text.trim());
   } catch (error) {
     logger2.error("Failed to parse entity extraction response", {
-      error: error instanceof Error ? error.message : String(error)
+      error: error instanceof Error ? error.message : String(error),
+      response: response.text.slice(0, 500)
+    });
+    throw new Error("Failed to parse entity extraction response", {
+      cause: error instanceof Error ? error : new Error(String(error))
     });
-    return [];
   }
+  if (!isArray(entities)) {
+    logger2.error("Failed to parse entity extraction response: expected a JSON array", {
+      response: response.text.slice(0, 500)
+    });
+    throw new Error("Failed to parse entity extraction response: expected a JSON array");
+  }
+  logger2.debug("Parsed entities from AI response", { count: entities.length });
+  return entities.filter((e) => {
+    const ok = isObject(e) && isString(e.exact) && isString(e.entityType);
+    if (!ok) {
+      logger2.debug("Dropped malformed LLM entity", { entity: e });
+    }
+    return ok;
+  }).map((entity) => ({
+    exact: entity.exact,
+    entityType: entity.entityType,
+    ...isString(entity.prefix) ? { prefix: entity.prefix } : {},
+    ...isString(entity.suffix) ? { suffix: entity.suffix } : {}
+  }));
 }
 function getLanguageName(locale) {
   return getLocaleEnglishName(locale) || locale;
 }
-async function generateResourceFromTopic(topic, entityTypes, client, logger2, userPrompt, locale, context, temperature, maxTokens, sourceLanguage) {
+async function generateResourceFromTopic(topic, entityTypes, client, logger2, userPrompt, locale, context, temperature, maxTokens, sourceLanguage, outputMediaType = "text/markdown") {
   logger2.debug("Generating resource from topic", {
     topicPreview: topic.substring(0, 100),
     entityTypes,
@@ -10003,61 +9960,95 @@ IMPORTANT: Write the entire resource in ${getLanguageName(locale)}.` : "";
 
 The source resource and embedded context are in ${getLanguageName(sourceLanguage)}.` : "";
   let annotationSection = "";
+  let contextSection = "";
+  let resourceSection = "";
+  let graphSection = "";
   if (context) {
-    const parts = [];
-    parts.push(`- Annotation motivation: ${context.annotation.motivation}`);
-    parts.push(`- Source resource: ${context.sourceResource.name}`);
-    const { motivation, body } = context.annotation;
-    if (motivation === "commenting" || motivation === "assessing") {
-      const bodyItem = Array.isArray(body) ? body[0] : body;
-      if (bodyItem && "value" in bodyItem && bodyItem.value) {
-        const label = motivation === "commenting" ? "Comment" : "Assessment";
-        parts.push(`- ${label}: ${bodyItem.value}`);
-      }
-    }
-    annotationSection = `
+    const { focus } = context;
+    const mainResourceId = focus.kind === "annotation" ? focus.sourceResource["@id"] : focus.resource["@id"];
+    const focalAnnotationId = focus.kind === "annotation" ? focus.annotation.id : void 0;
+    if (focus.kind === "annotation") {
+      const parts = [];
+      parts.push(`- Annotation motivation: ${focus.annotation.motivation}`);
+      parts.push(`- Source resource: ${focus.sourceResource.name}`);
+      const { motivation, body } = focus.annotation;
+      if (motivation === "commenting" || motivation === "assessing") {
+        const bodyItem = Array.isArray(body) ? body[0] : body;
+        if (bodyItem && "value" in bodyItem && bodyItem.value) {
+          const label = motivation === "commenting" ? "Comment" : "Assessment";
+          parts.push(`- ${label}: ${bodyItem.value}`);
+        }
+      }
+      annotationSection = `
 
 Annotation context:
 ${parts.join("\n")}`;
-  }
-  let contextSection = "";
-  if (context?.sourceContext) {
-    const { before, selected, after } = context.sourceContext;
-    contextSection = `
+      if (focus.selected) {
+        const { before, text, after } = focus.selected;
+        contextSection = `
 
 Source document context:
 ---
 ${before ? `...${before}` : ""}
-**[${selected}]**
+**[${text}]**
 ${after ? `${after}...` : ""}
 ---
 `;
-  }
-  let graphContextSection = "";
-  if (context?.graphContext) {
-    const gc = context.graphContext;
-    const connections = gc.connections ?? [];
-    const citedBy = gc.citedBy ?? [];
-    const parts = [];
-    if (connections.length > 0) {
-      const connList = connections.map((c) => `${c.resourceName}${c.entityTypes?.length ? ` (${c.entityTypes.join(", ")})` : ""}`).join(", ");
-      parts.push(`- Connected resources: ${connList}`);
-    }
-    if (gc.citedByCount && gc.citedByCount > 0) {
-      const citedNames = citedBy.map((c) => c.resourceName).join(", ");
-      parts.push(`- This resource is cited by ${gc.citedByCount} other resource${gc.citedByCount > 1 ? "s" : ""}${citedNames ? `: ${citedNames}` : ""}`);
-    }
-    if (gc.siblingEntityTypes && gc.siblingEntityTypes.length > 0) {
-      parts.push(`- Related entity types in this document: ${gc.siblingEntityTypes.join(", ")}`);
-    }
-    if (gc.inferredRelationshipSummary) {
-      parts.push(`- Relationship summary: ${gc.inferredRelationshipSummary}`);
+      }
+    } else {
+      const RESOURCE_CONTENT_CAP = 4e3;
+      const parts = [`- Resource: ${focus.resource.name}`];
+      if (focus.summary) parts.push(`- Summary: ${focus.summary}`);
+      if (focus.suggestedReferences && focus.suggestedReferences.length > 0) {
+        parts.push(`- Suggested references: ${focus.suggestedReferences.join(", ")}`);
+      }
+      resourceSection = `
+
+Resource context:
+${parts.join("\n")}`;
+      if (focus.content?.main) {
+        resourceSection += `
+
+Resource content:
+---
+${focus.content.main.slice(0, RESOURCE_CONTENT_CAP)}
+---`;
+      }
+      const related = Object.entries(focus.content?.related ?? {});
+      if (related.length > 0) {
+        const blocks = related.map(([id, text]) => `[${id}]
+${text.slice(0, RESOURCE_CONTENT_CAP)}`).join("\n\n");
+        resourceSection += `
+
+Related resource content:
+---
+${blocks}
+---`;
+      }
     }
-    if (parts.length > 0) {
-      graphContextSection = `
+    if (mainResourceId) {
+      const views = deriveViews(context.graph, mainResourceId, focalAnnotationId);
+      const parts = [];
+      if (views.connections.length > 0) {
+        const connList = views.connections.map((c) => `${c.resourceName}${c.entityTypes.length ? ` (${c.entityTypes.join(", ")})` : ""}`).join(", ");
+        parts.push(`- Connected resources: ${connList}`);
+      }
+      if (views.citedByCount > 0) {
+        const citedNames = views.citedBy.map((c) => c.resourceName).join(", ");
+        parts.push(`- This resource is cited by ${views.citedByCount} other resource${views.citedByCount > 1 ? "s" : ""}${citedNames ? `: ${citedNames}` : ""}`);
+      }
+      if (views.siblingEntityTypes.length > 0) {
+        parts.push(`- Related entity types in this document: ${views.siblingEntityTypes.join(", ")}`);
+      }
+      if (context.inferredRelationshipSummary) {
+        parts.push(`- Relationship summary: ${context.inferredRelationshipSummary}`);
+      }
+      if (parts.length > 0) {
+        graphSection = `
 
 Knowledge graph context:
 ${parts.join("\n")}`;
+      }
     }
   }
   let semanticContextSection = "";
@@ -10069,17 +10060,20 @@ ${parts.join("\n")}`;
 Related passages from the knowledge base:
 ${lines.join("\n")}`;
   }
-  const structureGuidance = finalMaxTokens >= 1e3 ? "organized into titled sections (## Section) with well-structured paragraphs" : "organized into well-structured paragraphs";
+  const isPlainText = outputMediaType === "text/plain";
+  const structureGuidance = !isPlainText && finalMaxTokens >= 1e3 ? "organized into titled sections (## Section) with well-structured paragraphs" : "organized into well-structured paragraphs";
+  const formatRequirements = isPlainText ? `- Write the response as plain text \u2014 no formatting markup (no #, *, backticks, headings, or links)
+- Begin with the title on its own first line` : `- Start with a clear heading (# Title)
+- Use markdown formatting
+- Write the response as markdown`;
   const prompt = `Generate a concise, informative resource about "${topic}".
 ${entityTypes.length > 0 ? `Focus on these entity types: ${entityTypes.join(", ")}.` : ""}
-${userPrompt ? `Additional context: ${userPrompt}` : ""}${annotationSection}${contextSection}${graphContextSection}${semanticContextSection}${sourceLanguageInstruction}${languageInstruction}
+${userPrompt ? `Additional context: ${userPrompt}` : ""}${annotationSection}${contextSection}${resourceSection}${graphSection}${semanticContextSection}${sourceLanguageInstruction}${languageInstruction}
 
 Requirements:
-- Start with a clear heading (# Title)
 - Aim for approximately ${finalMaxTokens} tokens of content, ${structureGuidance}
 - Be factual and informative
-- Use markdown formatting
-- Write the response as markdown`;
+${formatRequirements}`;
   const parseResponse = (response2) => {
     let content = response2.trim();
     if (content.startsWith("```markdown") || content.startsWith("```md")) {
@@ -10398,6 +10392,13 @@ async function processTagJob(content, inferenceClient, params, userId, generator
   };
 }
 async function processGenerationJob(inferenceClient, params, onProgress, logger2) {
+  const GENERATABLE_MEDIA_TYPES = ["text/markdown", "text/plain"];
+  const outputMediaType = params.outputMediaType ?? "text/markdown";
+  if (!GENERATABLE_MEDIA_TYPES.includes(outputMediaType)) {
+    throw new Error(
+      `Unsupported outputMediaType for generation: ${outputMediaType}. Generation produces ${GENERATABLE_MEDIA_TYPES.join(" or ")}.`
+    );
+  }
   onProgress(20, "Fetching context...", "fetching");
   const title = params.title ?? "Untitled";
   const entityTypes = (params.entityTypes ?? []).map(String);
@@ -10412,13 +10413,14 @@ async function processGenerationJob(inferenceClient, params, onProgress, logger2
     params.context,
     params.temperature,
     params.maxTokens,
-    params.sourceLanguage
+    params.sourceLanguage,
+    outputMediaType
   );
   onProgress(85, "Creating resource...", "creating");
   return {
     content: generated.content,
     title: generated.title ?? title,
-    format: "text/markdown",
+    format: outputMediaType,
     result: {
       resourceId: "",
       resourceName: generated.title ?? title
@@ -10638,6 +10640,17 @@ async function handleJobInner(adapter, config, job) {
       ...genParams.entityTypes && genParams.entityTypes.length > 0 ? { entityTypes: genParams.entityTypes } : {},
       generator
     });
+    if (!genParams.referenceId) {
+      const { annotation: provenanceRef } = assembleAnnotation(
+        {
+          motivation: "linking",
+          target: { source: String(resourceId) },
+          body: { type: "SpecificResource", source: String(newResourceId), purpose: "linking" }
+        },
+        generator
+      );
+      await emitEvent(session, "mark:create", { annotation: provenanceRef, userId, resourceId });
+    }
     await emitEvent(session, "job:complete", {
       ...lifecycleBase,
       result: { resourceId: newResourceId, resourceName: genResult.title }
@@ -10649,7 +10662,7 @@ async function handleJobInner(adapter, config, job) {
 }
 
 // src/worker-main.ts
-var import_rxjs3 = __toESM(require_cjs());
+var import_rxjs2 = __toESM(require_cjs());
 var ALL_JOB_TYPES = [
   "reference-annotation",
   "generation",
@@ -10757,7 +10770,7 @@ async function startAgentWorker(group) {
   };
   const storage = new InMemorySessionStorage();
   setStoredSession(storage, kbId, { access: initialToken, refresh: "" });
-  const token$ = new import_rxjs3.BehaviorSubject(null);
+  const token$ = new import_rxjs2.BehaviorSubject(null);
   let session;
   const transport = new HttpTransport({
     baseUrl: baseUrl(kbBackendUrl(endpoint)),