@semiont/jobs 0.5.10 → 0.5.12

package/dist/worker-main.js

@@ -1,14 +1,14 @@
-import { createTomlConfigLoader, softwareToAgent, baseUrl, busRequest, getPrimaryMediaType, textExtractionOf, assembleAnnotation, reconcileSelector, didToAgent, getLocaleEnglishName, isArray, isObject, isString, deriveViews } from '@semiont/core';
+import { createTomlConfigLoader, didToAgent, baseUrl, busRequest, getPrimaryMediaType, textExtractionOf, assembleAnnotation, reconcileSelector, getLocaleEnglishName, isArray, isObject, isString, deriveViews } from '@semiont/core';
 import { deriveStorageUri } from '@semiont/content';
 import { withSpan, SpanKind, recordJobOutcome } from '@semiont/observability';
 import { generateAnnotationId } from '@semiont/event-sourcing';
+import { homedir, hostname } from 'os';
+import { InMemorySessionStorage, setStoredSession, kbBackendUrl, SemiontClient, SemiontSession } from '@semiont/sdk';
+import { HttpTransport, HttpContentTransport } from '@semiont/http-transport';
 import { createInferenceClient } from '@semiont/inference';
 import { createServer } from 'http';
 import { existsSync, readFileSync } from 'fs';
-import { homedir, hostname } from 'os';
 import { join } from 'path';
-import { InMemorySessionStorage, setStoredSession, kbBackendUrl, SemiontClient, SemiontSession } from '@semiont/sdk';
-import { HttpTransport, HttpContentTransport } from '@semiont/http-transport';
 import { createProcessLogger } from '@semiont/observability/process-logger';
 
 var __create = Object.create;
@@ -9940,7 +9940,13 @@ Example output:
 function getLanguageName(locale) {
   return getLocaleEnglishName(locale) || locale;
 }
-async function generateResourceFromTopic(topic, entityTypes, client, logger2, userPrompt, locale, context, temperature, maxTokens, sourceLanguage, outputMediaType = "text/markdown") {
+var RESOURCE_CONTENT_CAP = 4e3;
+var SEMANTIC_MATCH_LIMIT = 3;
+var SEMANTIC_MATCH_CHARS = 240;
+function idLabel(resourceId, annotationId) {
+  return `[${resourceId}${annotationId ? `/${annotationId}` : ""}]`;
+}
+async function generateResourceFromTopic(topic, entityTypes, client, logger2, userPrompt, locale, context, temperature, maxTokens, sourceLanguage, outputMediaType = "text/markdown", task = "resource", structure, cite = false) {
   logger2.debug("Generating resource from topic", {
     topicPreview: topic.substring(0, 100),
     entityTypes,
@@ -9949,7 +9955,10 @@ async function generateResourceFromTopic(topic, entityTypes, client, logger2, us
     sourceLanguage,
     hasContext: !!context,
     temperature,
-    maxTokens
+    maxTokens,
+    outputMediaType,
+    task,
+    structure
   });
   const finalTemperature = temperature ?? 0.7;
   const finalMaxTokens = maxTokens ?? 500;
@@ -9970,7 +9979,7 @@ The source resource and embedded context are in ${getLanguageName(sourceLanguage
     if (focus.kind === "annotation") {
       const parts = [];
       parts.push(`- Annotation motivation: ${focus.annotation.motivation}`);
-      parts.push(`- Source resource: ${focus.sourceResource.name}`);
+      parts.push(`- Source resource: ${focus.sourceResource.name} ${idLabel(mainResourceId)}`);
       const { motivation, body } = focus.annotation;
       if (motivation === "commenting" || motivation === "assessing") {
         const bodyItem = Array.isArray(body) ? body[0] : body;
@@ -9996,8 +10005,7 @@ ${after ? `${after}...` : ""}
 `;
       }
     } else {
-      const RESOURCE_CONTENT_CAP = 4e3;
-      const parts = [`- Resource: ${focus.resource.name}`];
+      const parts = [`- Resource: ${focus.resource.name} ${idLabel(mainResourceId)}`];
       if (focus.summary) parts.push(`- Summary: ${focus.summary}`);
       if (focus.suggestedReferences && focus.suggestedReferences.length > 0) {
         parts.push(`- Suggested references: ${focus.suggestedReferences.join(", ")}`);
@@ -10030,11 +10038,11 @@ ${blocks}
       const views = deriveViews(context.graph, mainResourceId, focalAnnotationId);
       const parts = [];
       if (views.connections.length > 0) {
-        const connList = views.connections.map((c) => `${c.resourceName}${c.entityTypes.length ? ` (${c.entityTypes.join(", ")})` : ""}`).join(", ");
+        const connList = views.connections.map((c) => `${c.resourceName}${c.entityTypes.length ? ` (${c.entityTypes.join(", ")})` : ""} ${idLabel(c.resourceId)}`).join(", ");
         parts.push(`- Connected resources: ${connList}`);
       }
       if (views.citedByCount > 0) {
-        const citedNames = views.citedBy.map((c) => c.resourceName).join(", ");
+        const citedNames = views.citedBy.map((c) => `${c.resourceName} ${idLabel(c.resourceId)}`).join(", ");
         parts.push(`- This resource is cited by ${views.citedByCount} other resource${views.citedByCount > 1 ? "s" : ""}${citedNames ? `: ${citedNames}` : ""}`);
       }
       if (views.siblingEntityTypes.length > 0) {
@@ -10054,25 +10062,52 @@ ${parts.join("\n")}`;
   let semanticContextSection = "";
   const similar = context?.semanticContext?.similar ?? [];
   if (similar.length > 0) {
-    const lines = [...similar].sort((a, b) => b.score - a.score).slice(0, 3).map((m) => `- (${m.score.toFixed(2)}) ${m.text.slice(0, 240)}`);
+    const lines = [...similar].sort((a, b) => b.score - a.score).slice(0, SEMANTIC_MATCH_LIMIT).map((m) => `- ${idLabel(m.resourceId, m.annotationId)} (${m.score.toFixed(2)}) ${m.text.slice(0, SEMANTIC_MATCH_CHARS)}`);
     semanticContextSection = `
 
 Related passages from the knowledge base:
 ${lines.join("\n")}`;
+  }
+  let leadLine;
+  if (task === "resource") {
+    leadLine = `Generate a concise, informative resource about "${topic}".`;
+  } else if (task === "answer") {
+    leadLine = `Answer the following question directly and concisely, grounded in the provided context: "${topic}"`;
+  } else if (task === "summary") {
+    leadLine = `Write a concise summary of "${topic}".`;
+  } else {
+    logger2.warn("Unknown task \u2014 using it verbatim as the framing instruction", { task });
+    leadLine = `${task}
+Topic: "${topic}"`;
   }
   const isPlainText = outputMediaType === "text/plain";
-  const structureGuidance = !isPlainText && finalMaxTokens >= 1e3 ? "organized into titled sections (## Section) with well-structured paragraphs" : "organized into well-structured paragraphs";
+  let structureRequirement = "";
+  let titleRequirement = "";
+  if (structure === "sections") {
+    structureRequirement = isPlainText ? "\n- Organize the content into titled sections with well-structured paragraphs" : "\n- Organize the content into titled sections (## Section) with well-structured paragraphs";
+    if (!isPlainText) {
+      titleRequirement = "\n- Start with a clear heading (# Title)";
+    }
+  } else if (structure === "prose") {
+    structureRequirement = "\n- Write flowing, well-structured paragraphs with no section headings";
+  } else if (structure === "chat") {
+    structureRequirement = "\n- Structure the content as a conversational chat transcript \u2014 a sequence of alternating, speaker-labeled turns (no section headings)";
+  } else if (structure) {
+    logger2.warn("Unknown structure \u2014 passing it through as freeform organization guidance", { structure });
+    structureRequirement = `
+- Organize the output as: ${structure}`;
+  }
+  const citeRequirement = cite ? "\n- Ground every claim in the provided context. Immediately after each claim, cite its source by emitting [[<id>]], where <id> is an id shown in square brackets in the context above (for a passage labeled [abc], emit [[abc]]). Cite only ids that appear in the context." : "";
   const formatRequirements = isPlainText ? `- Write the response as plain text \u2014 no formatting markup (no #, *, backticks, headings, or links)
-- Begin with the title on its own first line` : `- Start with a clear heading (# Title)
-- Use markdown formatting
+- Begin with the title on its own first line` : `- Use markdown formatting
 - Write the response as markdown`;
-  const prompt = `Generate a concise, informative resource about "${topic}".
-${entityTypes.length > 0 ? `Focus on these entity types: ${entityTypes.join(", ")}.` : ""}
-${userPrompt ? `Additional context: ${userPrompt}` : ""}${annotationSection}${contextSection}${resourceSection}${graphSection}${semanticContextSection}${sourceLanguageInstruction}${languageInstruction}
+  const prompt = `${leadLine}
+${userPrompt ? `Instruction: ${userPrompt}` : ""}
+${entityTypes.length > 0 ? `Focus on these entity types: ${entityTypes.join(", ")}.` : ""}${annotationSection}${contextSection}${resourceSection}${graphSection}${semanticContextSection}${sourceLanguageInstruction}${languageInstruction}
 
 Requirements:
-- Aim for approximately ${finalMaxTokens} tokens of content, ${structureGuidance}
-- Be factual and informative
+- Aim for approximately ${finalMaxTokens} tokens of content
+- Be factual and informative${structureRequirement}${titleRequirement}${citeRequirement}
 ${formatRequirements}`;
   const parseResponse = (response2) => {
     let content = response2.trim();
@@ -10111,6 +10146,67 @@ ${formatRequirements}`;
   });
   return result;
 }
+
+// src/workers/generation/citation-resolver.ts
+var CITATION_TOKEN = /\[\[([^\s[\]/]+)(?:\/([^\s[\]/]+))?\]\]/g;
+function collectContextResourceIds(context) {
+  const ids = /* @__PURE__ */ new Set();
+  if (!context) return ids;
+  const { focus } = context;
+  ids.add(focus.kind === "annotation" ? focus.sourceResource["@id"] : focus.resource["@id"]);
+  for (const node of context.graph.nodes) {
+    if (node.type === "resource") ids.add(node.id);
+  }
+  for (const m of context.semanticContext?.similar ?? []) ids.add(m.resourceId);
+  if (focus.kind === "resource") {
+    for (const id of Object.keys(focus.content?.related ?? {})) ids.add(id);
+  }
+  return ids;
+}
+function resolveCitationTokens(content, validResourceIds, logger2) {
+  const citations = [];
+  let clean = "";
+  let last = 0;
+  for (const match of content.matchAll(CITATION_TOKEN)) {
+    const token = match[0];
+    const citedResourceId = match[1];
+    const annotationId = match[2];
+    clean += content.slice(last, match.index).replace(/[ \t]+$/, "");
+    last = match.index + token.length;
+    if (!validResourceIds.has(citedResourceId)) {
+      logger2.warn("Citation token references an id absent from the provided context \u2014 dropped", {
+        resourceId: citedResourceId
+      });
+      continue;
+    }
+    const end = clean.length;
+    let start = 0;
+    for (let i = end - 2; i >= 0; i--) {
+      const ch = clean[i];
+      if (ch === "." || ch === "!" || ch === "?" || ch === "\n") {
+        start = i + 1;
+        break;
+      }
+    }
+    while (start < end && /\s/.test(clean[start])) start++;
+    const exact = clean.slice(start, end);
+    if (exact.length === 0) {
+      logger2.warn("Citation token has no preceding claim text \u2014 dropped", {
+        resourceId: citedResourceId
+      });
+      continue;
+    }
+    citations.push({
+      resourceId: citedResourceId,
+      ...annotationId ? { annotationId } : {},
+      start,
+      end,
+      exact
+    });
+  }
+  clean += content.slice(last);
+  return { content: clean, citations };
+}
 function toMatch(r) {
   return {
     exact: r.exact,
@@ -10399,10 +10495,9 @@ async function processGenerationJob(inferenceClient, params, onProgress, logger2
       `Unsupported outputMediaType for generation: ${outputMediaType}. Generation produces ${GENERATABLE_MEDIA_TYPES.join(" or ")}.`
     );
   }
-  onProgress(20, "Fetching context...", "fetching");
   const title = params.title ?? "Untitled";
   const entityTypes = (params.entityTypes ?? []).map(String);
-  onProgress(40, "Generating resource...", "generating");
+  onProgress(5, "Generating resource...", "generating");
   const generated = await generateResourceFromTopic(
     title,
     entityTypes,
@@ -10414,13 +10509,24 @@ async function processGenerationJob(inferenceClient, params, onProgress, logger2
     params.temperature,
     params.maxTokens,
     params.sourceLanguage,
-    outputMediaType
+    outputMediaType,
+    params.task,
+    params.structure,
+    params.cite
   );
-  onProgress(85, "Creating resource...", "creating");
+  let content = generated.content;
+  let citations = [];
+  if (params.cite === true) {
+    const resolved = resolveCitationTokens(content, collectContextResourceIds(params.context), logger2);
+    content = resolved.content;
+    citations = resolved.citations;
+  }
+  onProgress(95, "Creating resource...", "creating");
   return {
-    content: generated.content,
+    content,
     title: generated.title ?? title,
     format: outputMediaType,
+    citations,
     result: {
       resourceId: "",
       resourceName: generated.title ?? title
@@ -10651,6 +10757,23 @@ async function handleJobInner(adapter, config, job) {
       );
       await emitEvent(session, "mark:create", { annotation: provenanceRef, userId, resourceId });
     }
+    for (const citation of genResult.citations) {
+      const { annotation: citationRef } = assembleAnnotation(
+        {
+          motivation: "linking",
+          target: {
+            source: String(newResourceId),
+            selector: [
+              { type: "TextPositionSelector", start: citation.start, end: citation.end },
+              { type: "TextQuoteSelector", exact: citation.exact }
+            ]
+          },
+          body: { type: "SpecificResource", source: citation.resourceId, purpose: "linking" }
+        },
+        generator
+      );
+      await emitEvent(session, "mark:create", { annotation: citationRef, userId, resourceId: newResourceId });
+    }
     await emitEvent(session, "job:complete", {
       ...lifecycleBase,
       result: { resourceId: newResourceId, resourceName: genResult.title }
@@ -10661,75 +10784,8 @@ async function handleJobInner(adapter, config, job) {
   }
 }
 
-// src/worker-main.ts
+// src/worker-runtime.ts
 var import_rxjs2 = __toESM(require_cjs());
-var ALL_JOB_TYPES = [
-  "reference-annotation",
-  "generation",
-  "highlight-annotation",
-  "assessment-annotation",
-  "comment-annotation",
-  "tag-annotation"
-];
-var configPath = join(homedir(), ".semiontconfig");
-var tomlReader = {
-  readIfExists: (p) => existsSync(p) ? readFileSync(p, "utf-8") : null
-};
-var envConfig = createTomlConfigLoader(
-  tomlReader,
-  configPath,
-  process.env
-)(null, "local");
-var workerInferenceMap = envConfig._metadata?.workers;
-if (!workerInferenceMap || Object.keys(workerInferenceMap).length === 0) {
-  throw new Error(
-    'No worker inference config found in ~/.semiontconfig. Add at least [environments.<env>.workers.default.inference] with type = "..." and model = "...".'
-  );
-}
-function resolveWorker(jobType) {
-  const specific = workerInferenceMap[jobType];
-  if (specific) return specific;
-  const def = workerInferenceMap["default"];
-  if (def) return def;
-  throw new Error(
-    `No inference config for worker '${jobType}' and no workers.default in ~/.semiontconfig.`
-  );
-}
-var backendPublicURL = envConfig.services?.backend?.publicURL;
-if (!backendPublicURL) {
-  throw new Error("services.backend.publicURL is required in ~/.semiontconfig");
-}
-var backendBaseUrl = backendPublicURL;
-var workerSecret = process.env.SEMIONT_WORKER_SECRET ?? "";
-var healthPort = 9090;
-var logger = createProcessLogger("worker");
-function clientKey(w) {
-  return [w.type, w.model, w.apiKey ?? "", w.endpoint ?? "", w.baseURL ?? ""].join("|");
-}
-function toClientConfig(w) {
-  return {
-    type: w.type,
-    model: w.model,
-    ...w.endpoint && { endpoint: w.endpoint },
-    ...w.baseURL && { baseURL: w.baseURL },
-    ...w.apiKey && { apiKey: w.apiKey }
-  };
-}
-var groups = /* @__PURE__ */ new Map();
-for (const jobType of ALL_JOB_TYPES) {
-  const inference = resolveWorker(jobType);
-  const key = clientKey(inference);
-  let group = groups.get(key);
-  if (!group) {
-    group = {
-      inference,
-      jobTypes: [],
-      client: createInferenceClient(toClientConfig(inference), logger)
-    };
-    groups.set(key, group);
-  }
-  group.jobTypes.push(jobType);
-}
 function parseBackendUrl(url) {
   const parsed = new URL(url);
   const protocol = parsed.protocol.replace(":", "") === "https" ? "https" : "http";
@@ -10737,29 +10793,32 @@ function parseBackendUrl(url) {
   const port = parsed.port ? Number(parsed.port) : protocol === "https" ? 443 : 80;
   return { protocol, host, port };
 }
-async function authenticateAgent(provider, model) {
-  if (!workerSecret) {
+async function authenticateAgent(opts) {
+  const { backendBaseUrl: backendBaseUrl2, workerSecret: workerSecret2, provider, model } = opts;
+  if (!workerSecret2) {
     throw new Error("SEMIONT_WORKER_SECRET is required to authenticate worker agents");
   }
-  const response = await fetch(`${backendBaseUrl}/api/tokens/agent`, {
+  const response = await fetch(`${backendBaseUrl2}/api/tokens/agent`, {
     method: "POST",
     headers: { "Content-Type": "application/json" },
-    body: JSON.stringify({ secret: workerSecret, provider, model })
+    body: JSON.stringify({ secret: workerSecret2, provider, model })
   });
   if (!response.ok) {
     throw new Error(`Agent authentication failed for ${provider}:${model}: ${response.status} ${response.statusText}`);
   }
   return await response.json();
 }
-async function startAgentWorker(group) {
+async function startAgentWorker(opts) {
+  const { group, backendBaseUrl: backendBaseUrl2, workerSecret: workerSecret2, logger: logger2 } = opts;
   const { inference } = group;
-  const { protocol, host, port } = parseBackendUrl(backendBaseUrl);
-  const { token: initialToken, did } = await authenticateAgent(inference.type, inference.model);
-  const generator = softwareToAgent({
-    domain: host,
+  const { protocol, host, port } = parseBackendUrl(backendBaseUrl2);
+  const { token: initialToken, did } = await authenticateAgent({
+    backendBaseUrl: backendBaseUrl2,
+    workerSecret: workerSecret2,
     provider: inference.type,
     model: inference.model
   });
+  const generator = didToAgent(did);
   const kbId = `agent-${inference.type}-${inference.model}-${hostname()}`;
   const endpoint = { kind: "http", host, port, protocol };
   const kb = {
@@ -10786,10 +10845,15 @@ async function startAgentWorker(group) {
     token$,
     refresh: async () => {
       try {
-        const { token } = await authenticateAgent(inference.type, inference.model);
+        const { token } = await authenticateAgent({
+          backendBaseUrl: backendBaseUrl2,
+          workerSecret: workerSecret2,
+          provider: inference.type,
+          model: inference.model
+        });
         return token;
       } catch (err) {
-        logger.error("Agent token refresh failed", {
+        logger2.error("Agent token refresh failed", {
           error: err instanceof Error ? err.message : String(err),
           agent: did
         });
@@ -10797,7 +10861,7 @@ async function startAgentWorker(group) {
       }
     },
     onError: (err) => {
-      logger.error("Session error", { code: err.code, message: err.message, agent: did });
+      logger2.error("Session error", { code: err.code, message: err.message, agent: did });
     }
   });
   await session.ready;
@@ -10806,9 +10870,9 @@ async function startAgentWorker(group) {
     jobTypes: group.jobTypes,
     inferenceClient: group.client,
     generator,
-    logger
+    logger: logger2
   });
-  logger.info("Agent ready", {
+  logger2.info("Agent ready", {
     did,
     provider: inference.type,
     model: inference.model,
@@ -10822,6 +10886,73 @@ async function startAgentWorker(group) {
     }
   };
 }
+var ALL_JOB_TYPES = [
+  "reference-annotation",
+  "generation",
+  "highlight-annotation",
+  "assessment-annotation",
+  "comment-annotation",
+  "tag-annotation"
+];
+var configPath = join(homedir(), ".semiontconfig");
+var tomlReader = {
+  readIfExists: (p) => existsSync(p) ? readFileSync(p, "utf-8") : null
+};
+var envConfig = createTomlConfigLoader(
+  tomlReader,
+  configPath,
+  process.env
+)(null, "local");
+var workerInferenceMap = envConfig._metadata?.workers;
+if (!workerInferenceMap || Object.keys(workerInferenceMap).length === 0) {
+  throw new Error(
+    'No worker inference config found in ~/.semiontconfig. Add at least [environments.<env>.workers.default.inference] with type = "..." and model = "...".'
+  );
+}
+function resolveWorker(jobType) {
+  const specific = workerInferenceMap[jobType];
+  if (specific) return specific;
+  const def = workerInferenceMap["default"];
+  if (def) return def;
+  throw new Error(
+    `No inference config for worker '${jobType}' and no workers.default in ~/.semiontconfig.`
+  );
+}
+var backendPublicURL = envConfig.services?.backend?.publicURL;
+if (!backendPublicURL) {
+  throw new Error("services.backend.publicURL is required in ~/.semiontconfig");
+}
+var backendBaseUrl = backendPublicURL;
+var workerSecret = process.env.SEMIONT_WORKER_SECRET ?? "";
+var healthPort = 9090;
+var logger = createProcessLogger("worker");
+function clientKey(w) {
+  return [w.type, w.model, w.apiKey ?? "", w.endpoint ?? "", w.baseURL ?? ""].join("|");
+}
+function toClientConfig(w) {
+  return {
+    type: w.type,
+    model: w.model,
+    ...w.endpoint && { endpoint: w.endpoint },
+    ...w.baseURL && { baseURL: w.baseURL },
+    ...w.apiKey && { apiKey: w.apiKey }
+  };
+}
+var groups = /* @__PURE__ */ new Map();
+for (const jobType of ALL_JOB_TYPES) {
+  const inference = resolveWorker(jobType);
+  const key = clientKey(inference);
+  let group = groups.get(key);
+  if (!group) {
+    group = {
+      inference,
+      jobTypes: [],
+      client: createInferenceClient(toClientConfig(inference), logger)
+    };
+    groups.set(key, group);
+  }
+  group.jobTypes.push(jobType);
+}
 async function main() {
   const { initObservabilityNode } = await import('@semiont/observability/node');
   initObservabilityNode({ serviceName: "semiont-worker" });
@@ -10833,7 +10964,11 @@ async function main() {
       jobTypes: g.jobTypes
     }))
   });
-  const workers = await Promise.all(Array.from(groups.values()).map(startAgentWorker));
+  const workers = await Promise.all(
+    Array.from(groups.values()).map(
+      (group) => startAgentWorker({ group, backendBaseUrl, workerSecret, logger })
+    )
+  );
   const health = createServer((req, res) => {
     if (req.url === "/health") {
       res.writeHead(200, { "Content-Type": "application/json" });