@semiont/jobs 0.5.9 → 0.5.11

package/dist/worker-main.js

@@ -1,14 +1,14 @@
-import { createTomlConfigLoader, softwareToAgent, baseUrl, getPrimaryMediaType, textExtractionOf, reconcileSelector, didToAgent, getLocaleEnglishName, isArray, isObject, isString } from '@semiont/core';
+import { createTomlConfigLoader, didToAgent, baseUrl, busRequest, getPrimaryMediaType, textExtractionOf, assembleAnnotation, reconcileSelector, getLocaleEnglishName, isArray, isObject, isString, deriveViews } from '@semiont/core';
 import { deriveStorageUri } from '@semiont/content';
 import { withSpan, SpanKind, recordJobOutcome } from '@semiont/observability';
 import { generateAnnotationId } from '@semiont/event-sourcing';
+import { homedir, hostname } from 'os';
+import { InMemorySessionStorage, setStoredSession, kbBackendUrl, SemiontClient, SemiontSession } from '@semiont/sdk';
+import { HttpTransport, HttpContentTransport } from '@semiont/http-transport';
 import { createInferenceClient } from '@semiont/inference';
 import { createServer } from 'http';
 import { existsSync, readFileSync } from 'fs';
-import { homedir, hostname } from 'os';
 import { join } from 'path';
-import { InMemorySessionStorage, setStoredSession, kbBackendUrl, SemiontClient, SemiontSession } from '@semiont/sdk';
-import { HttpTransport, HttpContentTransport } from '@semiont/http-transport';
 import { createProcessLogger } from '@semiont/observability/process-logger';
 
 var __create = Object.create;
@@ -326,16 +326,16 @@ var require_timeoutProvider = __commonJS({
     Object.defineProperty(exports, "__esModule", { value: true });
     exports.timeoutProvider = void 0;
     exports.timeoutProvider = {
-      setTimeout: function(handler, timeout2) {
+      setTimeout: function(handler, timeout) {
         var args = [];
         for (var _i = 2; _i < arguments.length; _i++) {
           args[_i - 2] = arguments[_i];
         }
         var delegate = exports.timeoutProvider.delegate;
         if (delegate === null || delegate === void 0 ? void 0 : delegate.setTimeout) {
-          return delegate.setTimeout.apply(delegate, __spreadArray([handler, timeout2], __read(args)));
+          return delegate.setTimeout.apply(delegate, __spreadArray([handler, timeout], __read(args)));
         }
-        return setTimeout.apply(void 0, __spreadArray([handler, timeout2], __read(args)));
+        return setTimeout.apply(void 0, __spreadArray([handler, timeout], __read(args)));
       },
       clearTimeout: function(handle) {
         var delegate = exports.timeoutProvider.delegate;
@@ -1659,16 +1659,16 @@ var require_intervalProvider = __commonJS({
     Object.defineProperty(exports, "__esModule", { value: true });
     exports.intervalProvider = void 0;
     exports.intervalProvider = {
-      setInterval: function(handler, timeout2) {
+      setInterval: function(handler, timeout) {
         var args = [];
         for (var _i = 2; _i < arguments.length; _i++) {
           args[_i - 2] = arguments[_i];
         }
         var delegate = exports.intervalProvider.delegate;
         if (delegate === null || delegate === void 0 ? void 0 : delegate.setInterval) {
-          return delegate.setInterval.apply(delegate, __spreadArray([handler, timeout2], __read(args)));
+          return delegate.setInterval.apply(delegate, __spreadArray([handler, timeout], __read(args)));
         }
-        return setInterval.apply(void 0, __spreadArray([handler, timeout2], __read(args)));
+        return setInterval.apply(void 0, __spreadArray([handler, timeout], __read(args)));
       },
       clearInterval: function(handle) {
         var delegate = exports.intervalProvider.delegate;
@@ -3566,7 +3566,7 @@ var require_firstValueFrom = __commonJS({
     exports.firstValueFrom = void 0;
     var EmptyError_1 = require_EmptyError();
     var Subscriber_1 = require_Subscriber();
-    function firstValueFrom2(source, config) {
+    function firstValueFrom(source, config) {
       var hasConfig = typeof config === "object";
       return new Promise(function(resolve, reject) {
         var subscriber = new Subscriber_1.SafeSubscriber({
@@ -3586,7 +3586,7 @@ var require_firstValueFrom = __commonJS({
         source.subscribe(subscriber);
       });
     }
-    exports.firstValueFrom = firstValueFrom2;
+    exports.firstValueFrom = firstValueFrom;
   }
 });
 
@@ -3673,7 +3673,7 @@ var require_timeout = __commonJS({
         this.info = info;
       };
     });
-    function timeout2(config, schedulerArg) {
+    function timeout(config, schedulerArg) {
       var _a = isDate_1.isValidDate(config) ? { first: config } : typeof config === "number" ? { each: config } : config, first = _a.first, each = _a.each, _b = _a.with, _with = _b === void 0 ? timeoutErrorFactory : _b, _c = _a.scheduler, scheduler = _c === void 0 ? schedulerArg !== null && schedulerArg !== void 0 ? schedulerArg : async_1.asyncScheduler : _c, _d = _a.meta, meta = _d === void 0 ? null : _d;
       if (first == null && each == null) {
         throw new TypeError("No timeout provided.");
@@ -3711,7 +3711,7 @@ var require_timeout = __commonJS({
         !seen && startTimer(first != null ? typeof first === "number" ? first : +first - scheduler.now() : each);
       });
     }
-    exports.timeout = timeout2;
+    exports.timeout = timeout;
     function timeoutErrorFactory(info) {
       throw new exports.TimeoutError(info);
     }
@@ -3725,7 +3725,7 @@ var require_map = __commonJS({
     exports.map = void 0;
     var lift_1 = require_lift();
     var OperatorSubscriber_1 = require_OperatorSubscriber();
-    function map2(project, thisArg) {
+    function map(project, thisArg) {
       return lift_1.operate(function(source, subscriber) {
         var index = 0;
         source.subscribe(OperatorSubscriber_1.createOperatorSubscriber(subscriber, function(value) {
@@ -3733,7 +3733,7 @@ var require_map = __commonJS({
         }));
       });
     }
-    exports.map = map2;
+    exports.map = map;
   }
 });
 
@@ -4633,7 +4633,7 @@ var require_merge = __commonJS({
     var empty_1 = require_empty();
     var args_1 = require_args();
     var from_1 = require_from();
-    function merge2() {
+    function merge() {
       var args = [];
       for (var _i = 0; _i < arguments.length; _i++) {
         args[_i] = arguments[_i];
@@ -4643,7 +4643,7 @@ var require_merge = __commonJS({
       var sources = args;
       return !sources.length ? empty_1.EMPTY : sources.length === 1 ? innerFrom_1.innerFrom(sources[0]) : mergeAll_1.mergeAll(concurrent)(from_1.from(sources, scheduler));
     }
-    exports.merge = merge2;
+    exports.merge = merge;
   }
 });
 
@@ -4750,7 +4750,7 @@ var require_filter = __commonJS({
     exports.filter = void 0;
     var lift_1 = require_lift();
     var OperatorSubscriber_1 = require_OperatorSubscriber();
-    function filter2(predicate, thisArg) {
+    function filter(predicate, thisArg) {
       return lift_1.operate(function(source, subscriber) {
         var index = 0;
         source.subscribe(OperatorSubscriber_1.createOperatorSubscriber(subscriber, function(value) {
@@ -4758,7 +4758,7 @@ var require_filter = __commonJS({
         }));
       });
     }
-    exports.filter = filter2;
+    exports.filter = filter;
   }
 });
 
@@ -5885,7 +5885,7 @@ var require_take = __commonJS({
     var empty_1 = require_empty();
     var lift_1 = require_lift();
     var OperatorSubscriber_1 = require_OperatorSubscriber();
-    function take2(count) {
+    function take(count) {
       return count <= 0 ? function() {
         return empty_1.EMPTY;
       } : lift_1.operate(function(source, subscriber) {
@@ -5900,7 +5900,7 @@ var require_take = __commonJS({
         }));
       });
     }
-    exports.take = take2;
+    exports.take = take;
   }
 });
 
@@ -6692,7 +6692,7 @@ var require_merge2 = __commonJS({
     var mergeAll_1 = require_mergeAll();
     var args_1 = require_args();
     var from_1 = require_from();
-    function merge2() {
+    function merge() {
       var args = [];
       for (var _i = 0; _i < arguments.length; _i++) {
         args[_i] = arguments[_i];
@@ -6703,7 +6703,7 @@ var require_merge2 = __commonJS({
         mergeAll_1.mergeAll(concurrent)(from_1.from(__spreadArray([source], __read(args)), scheduler)).subscribe(subscriber);
       });
     }
-    exports.merge = merge2;
+    exports.merge = merge;
   }
 });
 
@@ -9207,9 +9207,19 @@ var require_cjs = __commonJS({
 
 // src/job-claim-adapter.ts
 var import_rxjs = __toESM(require_cjs());
-var import_rxjs2 = __toESM(require_cjs());
+function workerBusAsPrimitive(bus) {
+  return {
+    emit(channel, payload) {
+      return bus.emit(channel, payload);
+    },
+    stream(channel) {
+      return bus.on$(channel);
+    }
+  };
+}
 function createJobClaimAdapter(options) {
   const { bus, jobTypes } = options;
+  const requestBus = workerBusAsPrimitive(bus);
   const activeJob$ = new import_rxjs.BehaviorSubject(null);
   const isProcessing$ = new import_rxjs.BehaviorSubject(false);
   const jobsCompleted$ = new import_rxjs.BehaviorSubject(0);
@@ -9218,22 +9228,7 @@ function createJobClaimAdapter(options) {
   let started = false;
   const claimJob = async (assignment) => {
     try {
-      const correlationId = crypto.randomUUID();
-      const result$ = (0, import_rxjs2.merge)(
-        bus.on$("job:claimed").pipe(
-          (0, import_rxjs2.filter)((e) => e.correlationId === correlationId),
-          (0, import_rxjs2.map)((e) => ({ ok: true, response: e.response }))
-        ),
-        bus.on$("job:claim-failed").pipe(
-          (0, import_rxjs2.filter)((e) => e.correlationId === correlationId),
-          (0, import_rxjs2.map)(() => ({ ok: false }))
-        )
-      ).pipe((0, import_rxjs2.take)(1), (0, import_rxjs2.timeout)(1e4));
-      const resultPromise = (0, import_rxjs2.firstValueFrom)(result$);
-      await bus.emit("job:claim", { correlationId, jobId: assignment.jobId });
-      const result = await resultPromise;
-      if (!result.ok) return null;
-      const job = result.response;
+      const job = await busRequest(requestBus, "job:claim", { jobId: assignment.jobId }, 1e4);
       return {
         jobId: assignment.jobId,
         type: assignment.type,
@@ -9945,7 +9940,13 @@ Example output:
 function getLanguageName(locale) {
   return getLocaleEnglishName(locale) || locale;
 }
-async function generateResourceFromTopic(topic, entityTypes, client, logger2, userPrompt, locale, context, temperature, maxTokens, sourceLanguage) {
+var RESOURCE_CONTENT_CAP = 4e3;
+var SEMANTIC_MATCH_LIMIT = 3;
+var SEMANTIC_MATCH_CHARS = 240;
+function idLabel(resourceId, annotationId) {
+  return `[${resourceId}${annotationId ? `/${annotationId}` : ""}]`;
+}
+async function generateResourceFromTopic(topic, entityTypes, client, logger2, userPrompt, locale, context, temperature, maxTokens, sourceLanguage, outputMediaType = "text/markdown", task = "resource", structure, cite = false) {
   logger2.debug("Generating resource from topic", {
     topicPreview: topic.substring(0, 100),
     entityTypes,
@@ -9954,7 +9955,10 @@ async function generateResourceFromTopic(topic, entityTypes, client, logger2, us
     sourceLanguage,
     hasContext: !!context,
     temperature,
-    maxTokens
+    maxTokens,
+    outputMediaType,
+    task,
+    structure
   });
   const finalTemperature = temperature ?? 0.7;
   const finalMaxTokens = maxTokens ?? 500;
@@ -9965,83 +9969,146 @@ IMPORTANT: Write the entire resource in ${getLanguageName(locale)}.` : "";
 
 The source resource and embedded context are in ${getLanguageName(sourceLanguage)}.` : "";
   let annotationSection = "";
+  let contextSection = "";
+  let resourceSection = "";
+  let graphSection = "";
   if (context) {
-    const parts = [];
-    parts.push(`- Annotation motivation: ${context.annotation.motivation}`);
-    parts.push(`- Source resource: ${context.sourceResource.name}`);
-    const { motivation, body } = context.annotation;
-    if (motivation === "commenting" || motivation === "assessing") {
-      const bodyItem = Array.isArray(body) ? body[0] : body;
-      if (bodyItem && "value" in bodyItem && bodyItem.value) {
-        const label = motivation === "commenting" ? "Comment" : "Assessment";
-        parts.push(`- ${label}: ${bodyItem.value}`);
-      }
-    }
-    annotationSection = `
+    const { focus } = context;
+    const mainResourceId = focus.kind === "annotation" ? focus.sourceResource["@id"] : focus.resource["@id"];
+    const focalAnnotationId = focus.kind === "annotation" ? focus.annotation.id : void 0;
+    if (focus.kind === "annotation") {
+      const parts = [];
+      parts.push(`- Annotation motivation: ${focus.annotation.motivation}`);
+      parts.push(`- Source resource: ${focus.sourceResource.name} ${idLabel(mainResourceId)}`);
+      const { motivation, body } = focus.annotation;
+      if (motivation === "commenting" || motivation === "assessing") {
+        const bodyItem = Array.isArray(body) ? body[0] : body;
+        if (bodyItem && "value" in bodyItem && bodyItem.value) {
+          const label = motivation === "commenting" ? "Comment" : "Assessment";
+          parts.push(`- ${label}: ${bodyItem.value}`);
+        }
+      }
+      annotationSection = `
 
 Annotation context:
 ${parts.join("\n")}`;
-  }
-  let contextSection = "";
-  if (context?.sourceContext) {
-    const { before, selected, after } = context.sourceContext;
-    contextSection = `
+      if (focus.selected) {
+        const { before, text, after } = focus.selected;
+        contextSection = `
 
 Source document context:
 ---
 ${before ? `...${before}` : ""}
-**[${selected}]**
+**[${text}]**
 ${after ? `${after}...` : ""}
 ---
 `;
-  }
-  let graphContextSection = "";
-  if (context?.graphContext) {
-    const gc = context.graphContext;
-    const connections = gc.connections ?? [];
-    const citedBy = gc.citedBy ?? [];
-    const parts = [];
-    if (connections.length > 0) {
-      const connList = connections.map((c) => `${c.resourceName}${c.entityTypes?.length ? ` (${c.entityTypes.join(", ")})` : ""}`).join(", ");
-      parts.push(`- Connected resources: ${connList}`);
-    }
-    if (gc.citedByCount && gc.citedByCount > 0) {
-      const citedNames = citedBy.map((c) => c.resourceName).join(", ");
-      parts.push(`- This resource is cited by ${gc.citedByCount} other resource${gc.citedByCount > 1 ? "s" : ""}${citedNames ? `: ${citedNames}` : ""}`);
-    }
-    if (gc.siblingEntityTypes && gc.siblingEntityTypes.length > 0) {
-      parts.push(`- Related entity types in this document: ${gc.siblingEntityTypes.join(", ")}`);
-    }
-    if (gc.inferredRelationshipSummary) {
-      parts.push(`- Relationship summary: ${gc.inferredRelationshipSummary}`);
+      }
+    } else {
+      const parts = [`- Resource: ${focus.resource.name} ${idLabel(mainResourceId)}`];
+      if (focus.summary) parts.push(`- Summary: ${focus.summary}`);
+      if (focus.suggestedReferences && focus.suggestedReferences.length > 0) {
+        parts.push(`- Suggested references: ${focus.suggestedReferences.join(", ")}`);
+      }
+      resourceSection = `
+
+Resource context:
+${parts.join("\n")}`;
+      if (focus.content?.main) {
+        resourceSection += `
+
+Resource content:
+---
+${focus.content.main.slice(0, RESOURCE_CONTENT_CAP)}
+---`;
+      }
+      const related = Object.entries(focus.content?.related ?? {});
+      if (related.length > 0) {
+        const blocks = related.map(([id, text]) => `[${id}]
+${text.slice(0, RESOURCE_CONTENT_CAP)}`).join("\n\n");
+        resourceSection += `
+
+Related resource content:
+---
+${blocks}
+---`;
+      }
     }
-    if (parts.length > 0) {
-      graphContextSection = `
+    if (mainResourceId) {
+      const views = deriveViews(context.graph, mainResourceId, focalAnnotationId);
+      const parts = [];
+      if (views.connections.length > 0) {
+        const connList = views.connections.map((c) => `${c.resourceName}${c.entityTypes.length ? ` (${c.entityTypes.join(", ")})` : ""} ${idLabel(c.resourceId)}`).join(", ");
+        parts.push(`- Connected resources: ${connList}`);
+      }
+      if (views.citedByCount > 0) {
+        const citedNames = views.citedBy.map((c) => `${c.resourceName} ${idLabel(c.resourceId)}`).join(", ");
+        parts.push(`- This resource is cited by ${views.citedByCount} other resource${views.citedByCount > 1 ? "s" : ""}${citedNames ? `: ${citedNames}` : ""}`);
+      }
+      if (views.siblingEntityTypes.length > 0) {
+        parts.push(`- Related entity types in this document: ${views.siblingEntityTypes.join(", ")}`);
+      }
+      if (context.inferredRelationshipSummary) {
+        parts.push(`- Relationship summary: ${context.inferredRelationshipSummary}`);
+      }
+      if (parts.length > 0) {
+        graphSection = `
 
 Knowledge graph context:
 ${parts.join("\n")}`;
+      }
     }
   }
   let semanticContextSection = "";
   const similar = context?.semanticContext?.similar ?? [];
   if (similar.length > 0) {
-    const lines = [...similar].sort((a, b) => b.score - a.score).slice(0, 3).map((m) => `- (${m.score.toFixed(2)}) ${m.text.slice(0, 240)}`);
+    const lines = [...similar].sort((a, b) => b.score - a.score).slice(0, SEMANTIC_MATCH_LIMIT).map((m) => `- ${idLabel(m.resourceId, m.annotationId)} (${m.score.toFixed(2)}) ${m.text.slice(0, SEMANTIC_MATCH_CHARS)}`);
     semanticContextSection = `
 
 Related passages from the knowledge base:
 ${lines.join("\n")}`;
   }
-  const structureGuidance = finalMaxTokens >= 1e3 ? "organized into titled sections (## Section) with well-structured paragraphs" : "organized into well-structured paragraphs";
-  const prompt = `Generate a concise, informative resource about "${topic}".
-${entityTypes.length > 0 ? `Focus on these entity types: ${entityTypes.join(", ")}.` : ""}
-${userPrompt ? `Additional context: ${userPrompt}` : ""}${annotationSection}${contextSection}${graphContextSection}${semanticContextSection}${sourceLanguageInstruction}${languageInstruction}
+  let leadLine;
+  if (task === "resource") {
+    leadLine = `Generate a concise, informative resource about "${topic}".`;
+  } else if (task === "answer") {
+    leadLine = `Answer the following question directly and concisely, grounded in the provided context: "${topic}"`;
+  } else if (task === "summary") {
+    leadLine = `Write a concise summary of "${topic}".`;
+  } else {
+    logger2.warn("Unknown task \u2014 using it verbatim as the framing instruction", { task });
+    leadLine = `${task}
+Topic: "${topic}"`;
+  }
+  const isPlainText = outputMediaType === "text/plain";
+  let structureRequirement = "";
+  let titleRequirement = "";
+  if (structure === "sections") {
+    structureRequirement = isPlainText ? "\n- Organize the content into titled sections with well-structured paragraphs" : "\n- Organize the content into titled sections (## Section) with well-structured paragraphs";
+    if (!isPlainText) {
+      titleRequirement = "\n- Start with a clear heading (# Title)";
+    }
+  } else if (structure === "prose") {
+    structureRequirement = "\n- Write flowing, well-structured paragraphs with no section headings";
+  } else if (structure === "chat") {
+    structureRequirement = "\n- Structure the content as a conversational chat transcript \u2014 a sequence of alternating, speaker-labeled turns (no section headings)";
+  } else if (structure) {
+    logger2.warn("Unknown structure \u2014 passing it through as freeform organization guidance", { structure });
+    structureRequirement = `
+- Organize the output as: ${structure}`;
+  }
+  const citeRequirement = cite ? "\n- Ground every claim in the provided context. Immediately after each claim, cite its source by emitting [[<id>]], where <id> is an id shown in square brackets in the context above (for a passage labeled [abc], emit [[abc]]). Cite only ids that appear in the context." : "";
+  const formatRequirements = isPlainText ? `- Write the response as plain text \u2014 no formatting markup (no #, *, backticks, headings, or links)
+- Begin with the title on its own first line` : `- Use markdown formatting
+- Write the response as markdown`;
+  const prompt = `${leadLine}
+${userPrompt ? `Instruction: ${userPrompt}` : ""}
+${entityTypes.length > 0 ? `Focus on these entity types: ${entityTypes.join(", ")}.` : ""}${annotationSection}${contextSection}${resourceSection}${graphSection}${semanticContextSection}${sourceLanguageInstruction}${languageInstruction}
 
 Requirements:
-- Start with a clear heading (# Title)
-- Aim for approximately ${finalMaxTokens} tokens of content, ${structureGuidance}
-- Be factual and informative
-- Use markdown formatting
-- Write the response as markdown`;
+- Aim for approximately ${finalMaxTokens} tokens of content
+- Be factual and informative${structureRequirement}${titleRequirement}${citeRequirement}
+${formatRequirements}`;
   const parseResponse = (response2) => {
     let content = response2.trim();
     if (content.startsWith("```markdown") || content.startsWith("```md")) {
@@ -10079,6 +10146,67 @@ Requirements:
   });
   return result;
 }
+
+// src/workers/generation/citation-resolver.ts
+var CITATION_TOKEN = /\[\[([^\s[\]/]+)(?:\/([^\s[\]/]+))?\]\]/g;
+function collectContextResourceIds(context) {
+  const ids = /* @__PURE__ */ new Set();
+  if (!context) return ids;
+  const { focus } = context;
+  ids.add(focus.kind === "annotation" ? focus.sourceResource["@id"] : focus.resource["@id"]);
+  for (const node of context.graph.nodes) {
+    if (node.type === "resource") ids.add(node.id);
+  }
+  for (const m of context.semanticContext?.similar ?? []) ids.add(m.resourceId);
+  if (focus.kind === "resource") {
+    for (const id of Object.keys(focus.content?.related ?? {})) ids.add(id);
+  }
+  return ids;
+}
+function resolveCitationTokens(content, validResourceIds, logger2) {
+  const citations = [];
+  let clean = "";
+  let last = 0;
+  for (const match of content.matchAll(CITATION_TOKEN)) {
+    const token = match[0];
+    const citedResourceId = match[1];
+    const annotationId = match[2];
+    clean += content.slice(last, match.index).replace(/[ \t]+$/, "");
+    last = match.index + token.length;
+    if (!validResourceIds.has(citedResourceId)) {
+      logger2.warn("Citation token references an id absent from the provided context \u2014 dropped", {
+        resourceId: citedResourceId
+      });
+      continue;
+    }
+    const end = clean.length;
+    let start = 0;
+    for (let i = end - 2; i >= 0; i--) {
+      const ch = clean[i];
+      if (ch === "." || ch === "!" || ch === "?" || ch === "\n") {
+        start = i + 1;
+        break;
+      }
+    }
+    while (start < end && /\s/.test(clean[start])) start++;
+    const exact = clean.slice(start, end);
+    if (exact.length === 0) {
+      logger2.warn("Citation token has no preceding claim text \u2014 dropped", {
+        resourceId: citedResourceId
+      });
+      continue;
+    }
+    citations.push({
+      resourceId: citedResourceId,
+      ...annotationId ? { annotationId } : {},
+      start,
+      end,
+      exact
+    });
+  }
+  clean += content.slice(last);
+  return { content: clean, citations };
+}
 function toMatch(r) {
   return {
     exact: r.exact,
@@ -10360,6 +10488,13 @@ async function processTagJob(content, inferenceClient, params, userId, generator
   };
 }
 async function processGenerationJob(inferenceClient, params, onProgress, logger2) {
+  const GENERATABLE_MEDIA_TYPES = ["text/markdown", "text/plain"];
+  const outputMediaType = params.outputMediaType ?? "text/markdown";
+  if (!GENERATABLE_MEDIA_TYPES.includes(outputMediaType)) {
+    throw new Error(
+      `Unsupported outputMediaType for generation: ${outputMediaType}. Generation produces ${GENERATABLE_MEDIA_TYPES.join(" or ")}.`
+    );
+  }
   onProgress(20, "Fetching context...", "fetching");
   const title = params.title ?? "Untitled";
   const entityTypes = (params.entityTypes ?? []).map(String);
@@ -10374,13 +10509,25 @@ async function processGenerationJob(inferenceClient, params, onProgress, logger2
     params.context,
     params.temperature,
     params.maxTokens,
-    params.sourceLanguage
+    params.sourceLanguage,
+    outputMediaType,
+    params.task,
+    params.structure,
+    params.cite
   );
+  let content = generated.content;
+  let citations = [];
+  if (params.cite === true) {
+    const resolved = resolveCitationTokens(content, collectContextResourceIds(params.context), logger2);
+    content = resolved.content;
+    citations = resolved.citations;
+  }
   onProgress(85, "Creating resource...", "creating");
   return {
-    content: generated.content,
+    content,
     title: generated.title ?? title,
-    format: "text/markdown",
+    format: outputMediaType,
+    citations,
     result: {
       resourceId: "",
       resourceName: generated.title ?? title
@@ -10600,6 +10747,34 @@ async function handleJobInner(adapter, config, job) {
       ...genParams.entityTypes && genParams.entityTypes.length > 0 ? { entityTypes: genParams.entityTypes } : {},
       generator
     });
+    if (!genParams.referenceId) {
+      const { annotation: provenanceRef } = assembleAnnotation(
+        {
+          motivation: "linking",
+          target: { source: String(resourceId) },
+          body: { type: "SpecificResource", source: String(newResourceId), purpose: "linking" }
+        },
+        generator
+      );
+      await emitEvent(session, "mark:create", { annotation: provenanceRef, userId, resourceId });
+    }
+    for (const citation of genResult.citations) {
+      const { annotation: citationRef } = assembleAnnotation(
+        {
+          motivation: "linking",
+          target: {
+            source: String(newResourceId),
+            selector: [
+              { type: "TextPositionSelector", start: citation.start, end: citation.end },
+              { type: "TextQuoteSelector", exact: citation.exact }
+            ]
+          },
+          body: { type: "SpecificResource", source: citation.resourceId, purpose: "linking" }
+        },
+        generator
+      );
+      await emitEvent(session, "mark:create", { annotation: citationRef, userId, resourceId: newResourceId });
+    }
     await emitEvent(session, "job:complete", {
       ...lifecycleBase,
       result: { resourceId: newResourceId, resourceName: genResult.title }
@@ -10610,75 +10785,8 @@ async function handleJobInner(adapter, config, job) {
   }
 }
 
-// src/worker-main.ts
-var import_rxjs3 = __toESM(require_cjs());
-var ALL_JOB_TYPES = [
-  "reference-annotation",
-  "generation",
-  "highlight-annotation",
-  "assessment-annotation",
-  "comment-annotation",
-  "tag-annotation"
-];
-var configPath = join(homedir(), ".semiontconfig");
-var tomlReader = {
-  readIfExists: (p) => existsSync(p) ? readFileSync(p, "utf-8") : null
-};
-var envConfig = createTomlConfigLoader(
-  tomlReader,
-  configPath,
-  process.env
-)(null, "local");
-var workerInferenceMap = envConfig._metadata?.workers;
-if (!workerInferenceMap || Object.keys(workerInferenceMap).length === 0) {
-  throw new Error(
-    'No worker inference config found in ~/.semiontconfig. Add at least [environments.<env>.workers.default.inference] with type = "..." and model = "...".'
-  );
-}
-function resolveWorker(jobType) {
-  const specific = workerInferenceMap[jobType];
-  if (specific) return specific;
-  const def = workerInferenceMap["default"];
-  if (def) return def;
-  throw new Error(
-    `No inference config for worker '${jobType}' and no workers.default in ~/.semiontconfig.`
-  );
-}
-var backendPublicURL = envConfig.services?.backend?.publicURL;
-if (!backendPublicURL) {
-  throw new Error("services.backend.publicURL is required in ~/.semiontconfig");
-}
-var backendBaseUrl = backendPublicURL;
-var workerSecret = process.env.SEMIONT_WORKER_SECRET ?? "";
-var healthPort = 9090;
-var logger = createProcessLogger("worker");
-function clientKey(w) {
-  return [w.type, w.model, w.apiKey ?? "", w.endpoint ?? "", w.baseURL ?? ""].join("|");
-}
-function toClientConfig(w) {
-  return {
-    type: w.type,
-    model: w.model,
-    ...w.endpoint && { endpoint: w.endpoint },
-    ...w.baseURL && { baseURL: w.baseURL },
-    ...w.apiKey && { apiKey: w.apiKey }
-  };
-}
-var groups = /* @__PURE__ */ new Map();
-for (const jobType of ALL_JOB_TYPES) {
-  const inference = resolveWorker(jobType);
-  const key = clientKey(inference);
-  let group = groups.get(key);
-  if (!group) {
-    group = {
-      inference,
-      jobTypes: [],
-      client: createInferenceClient(toClientConfig(inference), logger)
-    };
-    groups.set(key, group);
-  }
-  group.jobTypes.push(jobType);
-}
+// src/worker-runtime.ts
+var import_rxjs2 = __toESM(require_cjs());
 function parseBackendUrl(url) {
   const parsed = new URL(url);
   const protocol = parsed.protocol.replace(":", "") === "https" ? "https" : "http";
@@ -10686,29 +10794,32 @@ function parseBackendUrl(url) {
   const port = parsed.port ? Number(parsed.port) : protocol === "https" ? 443 : 80;
   return { protocol, host, port };
 }
-async function authenticateAgent(provider, model) {
-  if (!workerSecret) {
+async function authenticateAgent(opts) {
+  const { backendBaseUrl: backendBaseUrl2, workerSecret: workerSecret2, provider, model } = opts;
+  if (!workerSecret2) {
     throw new Error("SEMIONT_WORKER_SECRET is required to authenticate worker agents");
   }
-  const response = await fetch(`${backendBaseUrl}/api/tokens/agent`, {
+  const response = await fetch(`${backendBaseUrl2}/api/tokens/agent`, {
     method: "POST",
     headers: { "Content-Type": "application/json" },
-    body: JSON.stringify({ secret: workerSecret, provider, model })
+    body: JSON.stringify({ secret: workerSecret2, provider, model })
   });
   if (!response.ok) {
     throw new Error(`Agent authentication failed for ${provider}:${model}: ${response.status} ${response.statusText}`);
   }
   return await response.json();
 }
-async function startAgentWorker(group) {
+async function startAgentWorker(opts) {
+  const { group, backendBaseUrl: backendBaseUrl2, workerSecret: workerSecret2, logger: logger2 } = opts;
   const { inference } = group;
-  const { protocol, host, port } = parseBackendUrl(backendBaseUrl);
-  const { token: initialToken, did } = await authenticateAgent(inference.type, inference.model);
-  const generator = softwareToAgent({
-    domain: host,
+  const { protocol, host, port } = parseBackendUrl(backendBaseUrl2);
+  const { token: initialToken, did } = await authenticateAgent({
+    backendBaseUrl: backendBaseUrl2,
+    workerSecret: workerSecret2,
     provider: inference.type,
     model: inference.model
   });
+  const generator = didToAgent(did);
   const kbId = `agent-${inference.type}-${inference.model}-${hostname()}`;
   const endpoint = { kind: "http", host, port, protocol };
   const kb = {
@@ -10719,7 +10830,7 @@ async function startAgentWorker(group) {
   };
   const storage = new InMemorySessionStorage();
   setStoredSession(storage, kbId, { access: initialToken, refresh: "" });
-  const token$ = new import_rxjs3.BehaviorSubject(null);
+  const token$ = new import_rxjs2.BehaviorSubject(null);
   let session;
   const transport = new HttpTransport({
     baseUrl: baseUrl(kbBackendUrl(endpoint)),
@@ -10735,10 +10846,15 @@ async function startAgentWorker(group) {
     token$,
     refresh: async () => {
       try {
-        const { token } = await authenticateAgent(inference.type, inference.model);
+        const { token } = await authenticateAgent({
+          backendBaseUrl: backendBaseUrl2,
+          workerSecret: workerSecret2,
+          provider: inference.type,
+          model: inference.model
+        });
         return token;
       } catch (err) {
-        logger.error("Agent token refresh failed", {
+        logger2.error("Agent token refresh failed", {
           error: err instanceof Error ? err.message : String(err),
           agent: did
         });
@@ -10746,7 +10862,7 @@ async function startAgentWorker(group) {
       }
     },
     onError: (err) => {
-      logger.error("Session error", { code: err.code, message: err.message, agent: did });
+      logger2.error("Session error", { code: err.code, message: err.message, agent: did });
     }
   });
   await session.ready;
@@ -10755,9 +10871,9 @@ async function startAgentWorker(group) {
     jobTypes: group.jobTypes,
     inferenceClient: group.client,
     generator,
-    logger
+    logger: logger2
   });
-  logger.info("Agent ready", {
+  logger2.info("Agent ready", {
     did,
     provider: inference.type,
     model: inference.model,
@@ -10771,6 +10887,73 @@ async function startAgentWorker(group) {
     }
   };
 }
+var ALL_JOB_TYPES = [
+  "reference-annotation",
+  "generation",
+  "highlight-annotation",
+  "assessment-annotation",
+  "comment-annotation",
+  "tag-annotation"
+];
+var configPath = join(homedir(), ".semiontconfig");
+var tomlReader = {
+  readIfExists: (p) => existsSync(p) ? readFileSync(p, "utf-8") : null
+};
+var envConfig = createTomlConfigLoader(
+  tomlReader,
+  configPath,
+  process.env
+)(null, "local");
+var workerInferenceMap = envConfig._metadata?.workers;
+if (!workerInferenceMap || Object.keys(workerInferenceMap).length === 0) {
+  throw new Error(
+    'No worker inference config found in ~/.semiontconfig. Add at least [environments.<env>.workers.default.inference] with type = "..." and model = "...".'
+  );
+}
+function resolveWorker(jobType) {
+  const specific = workerInferenceMap[jobType];
+  if (specific) return specific;
+  const def = workerInferenceMap["default"];
+  if (def) return def;
+  throw new Error(
+    `No inference config for worker '${jobType}' and no workers.default in ~/.semiontconfig.`
+  );
+}
+var backendPublicURL = envConfig.services?.backend?.publicURL;
+if (!backendPublicURL) {
+  throw new Error("services.backend.publicURL is required in ~/.semiontconfig");
+}
+var backendBaseUrl = backendPublicURL;
+var workerSecret = process.env.SEMIONT_WORKER_SECRET ?? "";
+var healthPort = 9090;
+var logger = createProcessLogger("worker");
+function clientKey(w) {
+  return [w.type, w.model, w.apiKey ?? "", w.endpoint ?? "", w.baseURL ?? ""].join("|");
+}
+function toClientConfig(w) {
+  return {
+    type: w.type,
+    model: w.model,
+    ...w.endpoint && { endpoint: w.endpoint },
+    ...w.baseURL && { baseURL: w.baseURL },
+    ...w.apiKey && { apiKey: w.apiKey }
+  };
+}
+var groups = /* @__PURE__ */ new Map();
+for (const jobType of ALL_JOB_TYPES) {
+  const inference = resolveWorker(jobType);
+  const key = clientKey(inference);
+  let group = groups.get(key);
+  if (!group) {
+    group = {
+      inference,
+      jobTypes: [],
+      client: createInferenceClient(toClientConfig(inference), logger)
+    };
+    groups.set(key, group);
+  }
+  group.jobTypes.push(jobType);
+}
 async function main() {
   const { initObservabilityNode } = await import('@semiont/observability/node');
   initObservabilityNode({ serviceName: "semiont-worker" });
@@ -10782,7 +10965,11 @@ async function main() {
       jobTypes: g.jobTypes
     }))
   });
-  const workers = await Promise.all(Array.from(groups.values()).map(startAgentWorker));
+  const workers = await Promise.all(
+    Array.from(groups.values()).map(
+      (group) => startAgentWorker({ group, backendBaseUrl, workerSecret, logger })
+    )
+  );
   const health = createServer((req, res) => {
     if (req.url === "/health") {
       res.writeHead(200, { "Content-Type": "application/json" });