@semiont/jobs 0.5.9 → 0.5.11

package/dist/index.d.ts

@@ -1,4 +1,4 @@
-import { JobId, UserId, ResourceId, EntityType, AnnotationId, Annotation, GatheredContext, TagSchema, Logger, EventBus, components, SupportedMediaType } from '@semiont/core';
+import { JobId, UserId, ResourceId, EntityType, AnnotationId, GatheredContext, SupportedMediaType, TagSchema, Logger, EventBus, components } from '@semiont/core';
 import { SemiontProject } from '@semiont/core/node';
 import { InferenceClient } from '@semiont/inference';
 
@@ -74,10 +74,13 @@ interface DetectionParams {
  * Generation job parameters
  */
 interface GenerationParams {
-    referenceId: AnnotationId;
-    sourceResourceId: ResourceId;
-    sourceResourceName: string;
-    annotation: Annotation;
+    /**
+     * The unresolved reference an annotation-focus generation was triggered from.
+     * Absent for resource-focus generation (`yield.fromResource`). When present, the
+     * worker auto-binds the new resource to it; when absent, provenance is a minted
+     * source→derived reference annotation (Fork 2b).
+     */
+    referenceId?: AnnotationId;
     prompt?: string;
     title?: string;
     entityTypes?: EntityType[];
@@ -93,6 +96,36 @@ interface GenerationParams {
     temperature?: number;
     maxTokens?: number;
     storageUri?: string;
+    /**
+     * Requested media type of the generated resource's content. Default `text/markdown`.
+     * The generation worker produces `text/markdown` and `text/plain`; any other value
+     * fails the job (no silent fallback) — Semiont is multi-modal at the core, but
+     * generation coverage is text-only for now and the gap is surfaced, not hidden.
+     */
+    outputMediaType?: SupportedMediaType;
+    /**
+     * What the model is asked to DO — the prompt's framing verb. Canonical values map
+     * to tested framings; any other string is used verbatim as the framing instruction
+     * (loud degrade: the worker warns, never silently falls back to 'resource').
+     * Default 'resource'. See .plans/YIELD-STRUCTURE.md.
+     */
+    task?: 'resource' | 'answer' | 'summary' | (string & {});
+    /**
+     * How the output is internally segmented — text-bearing shape, subordinate to
+     * `outputMediaType` (never its peer). Canonical values map to tested guidance; any
+     * other string becomes a freeform "Organize the output as: …" instruction (loud
+     * degrade). UNSET means NO structure directive is emitted — the task framing and
+     * the model determine shape. See .plans/YIELD-STRUCTURE.md D2/D5.
+     */
+    structure?: 'prose' | 'sections' | 'chat' | (string & {});
+    /**
+     * Ask the model to cite: emit `[[<id>]]` transport tokens after each claim, using
+     * the ids the context embedding provides (CONTEXT-IDENTIFIERS). The worker
+     * validates each id against the embedded context (unknown ids are dropped
+     * loudly), strips the tokens from the stored content, and mints W3C linking
+     * annotations on the derived resource. See .plans/INLINE-CITATIONS.md.
+     */
+    cite?: boolean;
 }
 /**
  * Highlight detection job parameters
@@ -466,6 +499,30 @@ declare class FsJobQueue implements JobQueue {
     }>;
 }
 
+/**
+ * Citation-token resolver (INLINE-CITATIONS P1).
+ *
+ * Under `cite`, the generation prompt asks the model to emit `[[<resourceId>]]` /
+ * `[[<resourceId>/<annotationId>]]` transport tokens after each claim, citing ids
+ * the context embedding put in front of it (CONTEXT-IDENTIFIERS). Tokens are
+ * TRANSPORT, not content: this resolver parses them, validates each id against the
+ * ids actually present in the embedded context (the hallucination guard — an
+ * unknown id is dropped loudly, never silently linked), STRIPS them from the
+ * content before upload, and returns claim-span citations the worker mints as W3C
+ * linking annotations on the derived resource. See .plans/INLINE-CITATIONS.md.
+ */
+
+interface GenerationCitation {
+    /** The cited source resource (validated present in the embedded context). */
+    resourceId: string;
+    /** The contributing annotation, when the cited excerpt was annotation-derived. */
+    annotationId?: string;
+    /** Claim span in the FINAL (token-stripped) content; substring(start, end) === exact. */
+    start: number;
+    end: number;
+    exact: string;
+}
+
 /**
  * Job Processors
  *
@@ -500,6 +557,7 @@ declare function processGenerationJob(inferenceClient: InferenceClient, params:
     content: string;
     title: string;
     format: SupportedMediaType;
+    citations: GenerationCitation[];
     result: GenerationResult;
 }>;
 
@@ -627,7 +685,7 @@ declare class AnnotationDetection {
  * generate German content from an English source resource. See
  * `types.ts` "Locale conventions" for the full discussion.
  */
-declare function generateResourceFromTopic(topic: string, entityTypes: string[], client: InferenceClient, logger: Logger, userPrompt?: string, locale?: string, context?: GatheredContext, temperature?: number, maxTokens?: number, sourceLanguage?: string): Promise<{
+declare function generateResourceFromTopic(topic: string, entityTypes: string[], client: InferenceClient, logger: Logger, userPrompt?: string, locale?: string, context?: GatheredContext, temperature?: number, maxTokens?: number, sourceLanguage?: string, outputMediaType?: SupportedMediaType, task?: string, structure?: string, cite?: boolean): Promise<{
     title: string;
     content: string;
 }>;

package/dist/index.js

@@ -1,6 +1,6 @@
 import { promises } from 'fs';
 import * as path from 'path';
-import { jobId, reconcileSelector, isObject, isString, getLocaleEnglishName, didToAgent, isArray } from '@semiont/core';
+import { jobId, deriveViews, reconcileSelector, isObject, isString, getLocaleEnglishName, didToAgent, isArray } from '@semiont/core';
 import { generateAnnotationId } from '@semiont/event-sourcing';
 
 // src/fs-job-queue.ts
@@ -1068,7 +1068,13 @@ Example output:
 function getLanguageName(locale) {
   return getLocaleEnglishName(locale) || locale;
 }
-async function generateResourceFromTopic(topic, entityTypes, client, logger, userPrompt, locale, context, temperature, maxTokens, sourceLanguage) {
+var RESOURCE_CONTENT_CAP = 4e3;
+var SEMANTIC_MATCH_LIMIT = 3;
+var SEMANTIC_MATCH_CHARS = 240;
+function idLabel(resourceId, annotationId) {
+  return `[${resourceId}${annotationId ? `/${annotationId}` : ""}]`;
+}
+async function generateResourceFromTopic(topic, entityTypes, client, logger, userPrompt, locale, context, temperature, maxTokens, sourceLanguage, outputMediaType = "text/markdown", task = "resource", structure, cite = false) {
   logger.debug("Generating resource from topic", {
     topicPreview: topic.substring(0, 100),
     entityTypes,
@@ -1077,7 +1083,10 @@ async function generateResourceFromTopic(topic, entityTypes, client, logger, use
     sourceLanguage,
     hasContext: !!context,
     temperature,
-    maxTokens
+    maxTokens,
+    outputMediaType,
+    task,
+    structure
   });
   const finalTemperature = temperature ?? 0.7;
   const finalMaxTokens = maxTokens ?? 500;
@@ -1088,83 +1097,146 @@ IMPORTANT: Write the entire resource in ${getLanguageName(locale)}.` : "";
 
 The source resource and embedded context are in ${getLanguageName(sourceLanguage)}.` : "";
   let annotationSection = "";
+  let contextSection = "";
+  let resourceSection = "";
+  let graphSection = "";
   if (context) {
-    const parts = [];
-    parts.push(`- Annotation motivation: ${context.annotation.motivation}`);
-    parts.push(`- Source resource: ${context.sourceResource.name}`);
-    const { motivation, body } = context.annotation;
-    if (motivation === "commenting" || motivation === "assessing") {
-      const bodyItem = Array.isArray(body) ? body[0] : body;
-      if (bodyItem && "value" in bodyItem && bodyItem.value) {
-        const label = motivation === "commenting" ? "Comment" : "Assessment";
-        parts.push(`- ${label}: ${bodyItem.value}`);
+    const { focus } = context;
+    const mainResourceId = focus.kind === "annotation" ? focus.sourceResource["@id"] : focus.resource["@id"];
+    const focalAnnotationId = focus.kind === "annotation" ? focus.annotation.id : void 0;
+    if (focus.kind === "annotation") {
+      const parts = [];
+      parts.push(`- Annotation motivation: ${focus.annotation.motivation}`);
+      parts.push(`- Source resource: ${focus.sourceResource.name} ${idLabel(mainResourceId)}`);
+      const { motivation, body } = focus.annotation;
+      if (motivation === "commenting" || motivation === "assessing") {
+        const bodyItem = Array.isArray(body) ? body[0] : body;
+        if (bodyItem && "value" in bodyItem && bodyItem.value) {
+          const label = motivation === "commenting" ? "Comment" : "Assessment";
+          parts.push(`- ${label}: ${bodyItem.value}`);
+        }
       }
-    }
-    annotationSection = `
+      annotationSection = `
 
 Annotation context:
 ${parts.join("\n")}`;
-  }
-  let contextSection = "";
-  if (context?.sourceContext) {
-    const { before, selected, after } = context.sourceContext;
-    contextSection = `
+      if (focus.selected) {
+        const { before, text, after } = focus.selected;
+        contextSection = `
 
 Source document context:
 ---
 ${before ? `...${before}` : ""}
-**[${selected}]**
+**[${text}]**
 ${after ? `${after}...` : ""}
 ---
 `;
-  }
-  let graphContextSection = "";
-  if (context?.graphContext) {
-    const gc = context.graphContext;
-    const connections = gc.connections ?? [];
-    const citedBy = gc.citedBy ?? [];
-    const parts = [];
-    if (connections.length > 0) {
-      const connList = connections.map((c) => `${c.resourceName}${c.entityTypes?.length ? ` (${c.entityTypes.join(", ")})` : ""}`).join(", ");
-      parts.push(`- Connected resources: ${connList}`);
-    }
-    if (gc.citedByCount && gc.citedByCount > 0) {
-      const citedNames = citedBy.map((c) => c.resourceName).join(", ");
-      parts.push(`- This resource is cited by ${gc.citedByCount} other resource${gc.citedByCount > 1 ? "s" : ""}${citedNames ? `: ${citedNames}` : ""}`);
-    }
-    if (gc.siblingEntityTypes && gc.siblingEntityTypes.length > 0) {
-      parts.push(`- Related entity types in this document: ${gc.siblingEntityTypes.join(", ")}`);
-    }
-    if (gc.inferredRelationshipSummary) {
-      parts.push(`- Relationship summary: ${gc.inferredRelationshipSummary}`);
+      }
+    } else {
+      const parts = [`- Resource: ${focus.resource.name} ${idLabel(mainResourceId)}`];
+      if (focus.summary) parts.push(`- Summary: ${focus.summary}`);
+      if (focus.suggestedReferences && focus.suggestedReferences.length > 0) {
+        parts.push(`- Suggested references: ${focus.suggestedReferences.join(", ")}`);
+      }
+      resourceSection = `
+
+Resource context:
+${parts.join("\n")}`;
+      if (focus.content?.main) {
+        resourceSection += `
+
+Resource content:
+---
+${focus.content.main.slice(0, RESOURCE_CONTENT_CAP)}
+---`;
+      }
+      const related = Object.entries(focus.content?.related ?? {});
+      if (related.length > 0) {
+        const blocks = related.map(([id, text]) => `[${id}]
+${text.slice(0, RESOURCE_CONTENT_CAP)}`).join("\n\n");
+        resourceSection += `
+
+Related resource content:
+---
+${blocks}
+---`;
+      }
     }
-    if (parts.length > 0) {
-      graphContextSection = `
+    if (mainResourceId) {
+      const views = deriveViews(context.graph, mainResourceId, focalAnnotationId);
+      const parts = [];
+      if (views.connections.length > 0) {
+        const connList = views.connections.map((c) => `${c.resourceName}${c.entityTypes.length ? ` (${c.entityTypes.join(", ")})` : ""} ${idLabel(c.resourceId)}`).join(", ");
+        parts.push(`- Connected resources: ${connList}`);
+      }
+      if (views.citedByCount > 0) {
+        const citedNames = views.citedBy.map((c) => `${c.resourceName} ${idLabel(c.resourceId)}`).join(", ");
+        parts.push(`- This resource is cited by ${views.citedByCount} other resource${views.citedByCount > 1 ? "s" : ""}${citedNames ? `: ${citedNames}` : ""}`);
+      }
+      if (views.siblingEntityTypes.length > 0) {
+        parts.push(`- Related entity types in this document: ${views.siblingEntityTypes.join(", ")}`);
+      }
+      if (context.inferredRelationshipSummary) {
+        parts.push(`- Relationship summary: ${context.inferredRelationshipSummary}`);
+      }
+      if (parts.length > 0) {
+        graphSection = `
 
 Knowledge graph context:
 ${parts.join("\n")}`;
+      }
     }
   }
   let semanticContextSection = "";
   const similar = context?.semanticContext?.similar ?? [];
   if (similar.length > 0) {
-    const lines = [...similar].sort((a, b) => b.score - a.score).slice(0, 3).map((m) => `- (${m.score.toFixed(2)}) ${m.text.slice(0, 240)}`);
+    const lines = [...similar].sort((a, b) => b.score - a.score).slice(0, SEMANTIC_MATCH_LIMIT).map((m) => `- ${idLabel(m.resourceId, m.annotationId)} (${m.score.toFixed(2)}) ${m.text.slice(0, SEMANTIC_MATCH_CHARS)}`);
     semanticContextSection = `
 
 Related passages from the knowledge base:
 ${lines.join("\n")}`;
   }
-  const structureGuidance = finalMaxTokens >= 1e3 ? "organized into titled sections (## Section) with well-structured paragraphs" : "organized into well-structured paragraphs";
-  const prompt = `Generate a concise, informative resource about "${topic}".
-${entityTypes.length > 0 ? `Focus on these entity types: ${entityTypes.join(", ")}.` : ""}
-${userPrompt ? `Additional context: ${userPrompt}` : ""}${annotationSection}${contextSection}${graphContextSection}${semanticContextSection}${sourceLanguageInstruction}${languageInstruction}
+  let leadLine;
+  if (task === "resource") {
+    leadLine = `Generate a concise, informative resource about "${topic}".`;
+  } else if (task === "answer") {
+    leadLine = `Answer the following question directly and concisely, grounded in the provided context: "${topic}"`;
+  } else if (task === "summary") {
+    leadLine = `Write a concise summary of "${topic}".`;
+  } else {
+    logger.warn("Unknown task \u2014 using it verbatim as the framing instruction", { task });
+    leadLine = `${task}
+Topic: "${topic}"`;
+  }
+  const isPlainText = outputMediaType === "text/plain";
+  let structureRequirement = "";
+  let titleRequirement = "";
+  if (structure === "sections") {
+    structureRequirement = isPlainText ? "\n- Organize the content into titled sections with well-structured paragraphs" : "\n- Organize the content into titled sections (## Section) with well-structured paragraphs";
+    if (!isPlainText) {
+      titleRequirement = "\n- Start with a clear heading (# Title)";
+    }
+  } else if (structure === "prose") {
+    structureRequirement = "\n- Write flowing, well-structured paragraphs with no section headings";
+  } else if (structure === "chat") {
+    structureRequirement = "\n- Structure the content as a conversational chat transcript \u2014 a sequence of alternating, speaker-labeled turns (no section headings)";
+  } else if (structure) {
+    logger.warn("Unknown structure \u2014 passing it through as freeform organization guidance", { structure });
+    structureRequirement = `
+- Organize the output as: ${structure}`;
+  }
+  const citeRequirement = cite ? "\n- Ground every claim in the provided context. Immediately after each claim, cite its source by emitting [[<id>]], where <id> is an id shown in square brackets in the context above (for a passage labeled [abc], emit [[abc]]). Cite only ids that appear in the context." : "";
+  const formatRequirements = isPlainText ? `- Write the response as plain text \u2014 no formatting markup (no #, *, backticks, headings, or links)
+- Begin with the title on its own first line` : `- Use markdown formatting
+- Write the response as markdown`;
+  const prompt = `${leadLine}
+${userPrompt ? `Instruction: ${userPrompt}` : ""}
+${entityTypes.length > 0 ? `Focus on these entity types: ${entityTypes.join(", ")}.` : ""}${annotationSection}${contextSection}${resourceSection}${graphSection}${semanticContextSection}${sourceLanguageInstruction}${languageInstruction}
 
 Requirements:
-- Start with a clear heading (# Title)
-- Aim for approximately ${finalMaxTokens} tokens of content, ${structureGuidance}
-- Be factual and informative
-- Use markdown formatting
-- Write the response as markdown`;
+- Aim for approximately ${finalMaxTokens} tokens of content
+- Be factual and informative${structureRequirement}${titleRequirement}${citeRequirement}
+${formatRequirements}`;
   const parseResponse = (response2) => {
     let content = response2.trim();
     if (content.startsWith("```markdown") || content.startsWith("```md")) {
@@ -1202,6 +1274,67 @@ Requirements:
   });
   return result;
 }
+
+// src/workers/generation/citation-resolver.ts
+var CITATION_TOKEN = /\[\[([^\s[\]/]+)(?:\/([^\s[\]/]+))?\]\]/g;
+function collectContextResourceIds(context) {
+  const ids = /* @__PURE__ */ new Set();
+  if (!context) return ids;
+  const { focus } = context;
+  ids.add(focus.kind === "annotation" ? focus.sourceResource["@id"] : focus.resource["@id"]);
+  for (const node of context.graph.nodes) {
+    if (node.type === "resource") ids.add(node.id);
+  }
+  for (const m of context.semanticContext?.similar ?? []) ids.add(m.resourceId);
+  if (focus.kind === "resource") {
+    for (const id of Object.keys(focus.content?.related ?? {})) ids.add(id);
+  }
+  return ids;
+}
+function resolveCitationTokens(content, validResourceIds, logger) {
+  const citations = [];
+  let clean = "";
+  let last = 0;
+  for (const match of content.matchAll(CITATION_TOKEN)) {
+    const token = match[0];
+    const citedResourceId = match[1];
+    const annotationId = match[2];
+    clean += content.slice(last, match.index).replace(/[ \t]+$/, "");
+    last = match.index + token.length;
+    if (!validResourceIds.has(citedResourceId)) {
+      logger.warn("Citation token references an id absent from the provided context \u2014 dropped", {
+        resourceId: citedResourceId
+      });
+      continue;
+    }
+    const end = clean.length;
+    let start = 0;
+    for (let i = end - 2; i >= 0; i--) {
+      const ch = clean[i];
+      if (ch === "." || ch === "!" || ch === "?" || ch === "\n") {
+        start = i + 1;
+        break;
+      }
+    }
+    while (start < end && /\s/.test(clean[start])) start++;
+    const exact = clean.slice(start, end);
+    if (exact.length === 0) {
+      logger.warn("Citation token has no preceding claim text \u2014 dropped", {
+        resourceId: citedResourceId
+      });
+      continue;
+    }
+    citations.push({
+      resourceId: citedResourceId,
+      ...annotationId ? { annotationId } : {},
+      start,
+      end,
+      exact
+    });
+  }
+  clean += content.slice(last);
+  return { content: clean, citations };
+}
 function toMatch(r) {
   return {
     exact: r.exact,
@@ -1483,6 +1616,13 @@ async function processTagJob(content, inferenceClient, params, userId, generator
   };
 }
 async function processGenerationJob(inferenceClient, params, onProgress, logger) {
+  const GENERATABLE_MEDIA_TYPES = ["text/markdown", "text/plain"];
+  const outputMediaType = params.outputMediaType ?? "text/markdown";
+  if (!GENERATABLE_MEDIA_TYPES.includes(outputMediaType)) {
+    throw new Error(
+      `Unsupported outputMediaType for generation: ${outputMediaType}. Generation produces ${GENERATABLE_MEDIA_TYPES.join(" or ")}.`
+    );
+  }
   onProgress(20, "Fetching context...", "fetching");
   const title = params.title ?? "Untitled";
   const entityTypes = (params.entityTypes ?? []).map(String);
@@ -1497,13 +1637,25 @@ async function processGenerationJob(inferenceClient, params, onProgress, logger)
     params.context,
     params.temperature,
     params.maxTokens,
-    params.sourceLanguage
+    params.sourceLanguage,
+    outputMediaType,
+    params.task,
+    params.structure,
+    params.cite
   );
+  let content = generated.content;
+  let citations = [];
+  if (params.cite === true) {
+    const resolved = resolveCitationTokens(content, collectContextResourceIds(params.context), logger);
+    content = resolved.content;
+    citations = resolved.citations;
+  }
   onProgress(85, "Creating resource...", "creating");
   return {
-    content: generated.content,
+    content,
     title: generated.title ?? title,
-    format: "text/markdown",
+    format: outputMediaType,
+    citations,
     result: {
       resourceId: "",
       resourceName: generated.title ?? title