@semiont/jobs 0.5.10 → 0.5.12

package/dist/index.d.ts

@@ -103,6 +103,29 @@ interface GenerationParams {
      * generation coverage is text-only for now and the gap is surfaced, not hidden.
      */
     outputMediaType?: SupportedMediaType;
+    /**
+     * What the model is asked to DO — the prompt's framing verb. Canonical values map
+     * to tested framings; any other string is used verbatim as the framing instruction
+     * (loud degrade: the worker warns, never silently falls back to 'resource').
+     * Default 'resource'. See .plans/YIELD-STRUCTURE.md.
+     */
+    task?: 'resource' | 'answer' | 'summary' | (string & {});
+    /**
+     * How the output is internally segmented — text-bearing shape, subordinate to
+     * `outputMediaType` (never its peer). Canonical values map to tested guidance; any
+     * other string becomes a freeform "Organize the output as: …" instruction (loud
+     * degrade). UNSET means NO structure directive is emitted — the task framing and
+     * the model determine shape. See .plans/YIELD-STRUCTURE.md D2/D5.
+     */
+    structure?: 'prose' | 'sections' | 'chat' | (string & {});
+    /**
+     * Ask the model to cite: emit `[[<id>]]` transport tokens after each claim, using
+     * the ids the context embedding provides (CONTEXT-IDENTIFIERS). The worker
+     * validates each id against the embedded context (unknown ids are dropped
+     * loudly), strips the tokens from the stored content, and mints W3C linking
+     * annotations on the derived resource. See .plans/INLINE-CITATIONS.md.
+     */
+    cite?: boolean;
 }
 /**
  * Highlight detection job parameters
@@ -179,7 +202,8 @@ interface DetectionResult {
  * Generation job progress
  */
 interface YieldProgress {
-    stage: 'fetching' | 'generating' | 'creating' | 'linking';
+    /** The two real generation transitions — LLM call running, then persisting. */
+    stage: 'generating' | 'creating';
     percentage: number;
     message?: string;
 }
@@ -476,6 +500,30 @@ declare class FsJobQueue implements JobQueue {
     }>;
 }
 
+/**
+ * Citation-token resolver (INLINE-CITATIONS P1).
+ *
+ * Under `cite`, the generation prompt asks the model to emit `[[<resourceId>]]` /
+ * `[[<resourceId>/<annotationId>]]` transport tokens after each claim, citing ids
+ * the context embedding put in front of it (CONTEXT-IDENTIFIERS). Tokens are
+ * TRANSPORT, not content: this resolver parses them, validates each id against the
+ * ids actually present in the embedded context (the hallucination guard — an
+ * unknown id is dropped loudly, never silently linked), STRIPS them from the
+ * content before upload, and returns claim-span citations the worker mints as W3C
+ * linking annotations on the derived resource. See .plans/INLINE-CITATIONS.md.
+ */
+
+interface GenerationCitation {
+    /** The cited source resource (validated present in the embedded context). */
+    resourceId: string;
+    /** The contributing annotation, when the cited excerpt was annotation-derived. */
+    annotationId?: string;
+    /** Claim span in the FINAL (token-stripped) content; substring(start, end) === exact. */
+    start: number;
+    end: number;
+    exact: string;
+}
+
 /**
  * Job Processors
  *
@@ -510,6 +558,7 @@ declare function processGenerationJob(inferenceClient: InferenceClient, params:
     content: string;
     title: string;
     format: SupportedMediaType;
+    citations: GenerationCitation[];
     result: GenerationResult;
 }>;
 
@@ -637,7 +686,7 @@ declare class AnnotationDetection {
  * generate German content from an English source resource. See
  * `types.ts` "Locale conventions" for the full discussion.
  */
-declare function generateResourceFromTopic(topic: string, entityTypes: string[], client: InferenceClient, logger: Logger, userPrompt?: string, locale?: string, context?: GatheredContext, temperature?: number, maxTokens?: number, sourceLanguage?: string, outputMediaType?: SupportedMediaType): Promise<{
+declare function generateResourceFromTopic(topic: string, entityTypes: string[], client: InferenceClient, logger: Logger, userPrompt?: string, locale?: string, context?: GatheredContext, temperature?: number, maxTokens?: number, sourceLanguage?: string, outputMediaType?: SupportedMediaType, task?: string, structure?: string, cite?: boolean): Promise<{
     title: string;
     content: string;
 }>;

package/dist/index.js

@@ -1068,7 +1068,13 @@ Example output:
 function getLanguageName(locale) {
   return getLocaleEnglishName(locale) || locale;
 }
-async function generateResourceFromTopic(topic, entityTypes, client, logger, userPrompt, locale, context, temperature, maxTokens, sourceLanguage, outputMediaType = "text/markdown") {
+var RESOURCE_CONTENT_CAP = 4e3;
+var SEMANTIC_MATCH_LIMIT = 3;
+var SEMANTIC_MATCH_CHARS = 240;
+function idLabel(resourceId, annotationId) {
+  return `[${resourceId}${annotationId ? `/${annotationId}` : ""}]`;
+}
+async function generateResourceFromTopic(topic, entityTypes, client, logger, userPrompt, locale, context, temperature, maxTokens, sourceLanguage, outputMediaType = "text/markdown", task = "resource", structure, cite = false) {
   logger.debug("Generating resource from topic", {
     topicPreview: topic.substring(0, 100),
     entityTypes,
@@ -1077,7 +1083,10 @@ async function generateResourceFromTopic(topic, entityTypes, client, logger, use
     sourceLanguage,
     hasContext: !!context,
     temperature,
-    maxTokens
+    maxTokens,
+    outputMediaType,
+    task,
+    structure
   });
   const finalTemperature = temperature ?? 0.7;
   const finalMaxTokens = maxTokens ?? 500;
@@ -1098,7 +1107,7 @@ The source resource and embedded context are in ${getLanguageName(sourceLanguage
     if (focus.kind === "annotation") {
       const parts = [];
       parts.push(`- Annotation motivation: ${focus.annotation.motivation}`);
-      parts.push(`- Source resource: ${focus.sourceResource.name}`);
+      parts.push(`- Source resource: ${focus.sourceResource.name} ${idLabel(mainResourceId)}`);
       const { motivation, body } = focus.annotation;
       if (motivation === "commenting" || motivation === "assessing") {
         const bodyItem = Array.isArray(body) ? body[0] : body;
@@ -1124,8 +1133,7 @@ ${after ? `${after}...` : ""}
 `;
       }
     } else {
-      const RESOURCE_CONTENT_CAP = 4e3;
-      const parts = [`- Resource: ${focus.resource.name}`];
+      const parts = [`- Resource: ${focus.resource.name} ${idLabel(mainResourceId)}`];
       if (focus.summary) parts.push(`- Summary: ${focus.summary}`);
       if (focus.suggestedReferences && focus.suggestedReferences.length > 0) {
         parts.push(`- Suggested references: ${focus.suggestedReferences.join(", ")}`);
@@ -1158,11 +1166,11 @@ ${blocks}
       const views = deriveViews(context.graph, mainResourceId, focalAnnotationId);
       const parts = [];
       if (views.connections.length > 0) {
-        const connList = views.connections.map((c) => `${c.resourceName}${c.entityTypes.length ? ` (${c.entityTypes.join(", ")})` : ""}`).join(", ");
+        const connList = views.connections.map((c) => `${c.resourceName}${c.entityTypes.length ? ` (${c.entityTypes.join(", ")})` : ""} ${idLabel(c.resourceId)}`).join(", ");
         parts.push(`- Connected resources: ${connList}`);
       }
       if (views.citedByCount > 0) {
-        const citedNames = views.citedBy.map((c) => c.resourceName).join(", ");
+        const citedNames = views.citedBy.map((c) => `${c.resourceName} ${idLabel(c.resourceId)}`).join(", ");
         parts.push(`- This resource is cited by ${views.citedByCount} other resource${views.citedByCount > 1 ? "s" : ""}${citedNames ? `: ${citedNames}` : ""}`);
       }
       if (views.siblingEntityTypes.length > 0) {
@@ -1182,25 +1190,52 @@ ${parts.join("\n")}`;
   let semanticContextSection = "";
   const similar = context?.semanticContext?.similar ?? [];
   if (similar.length > 0) {
-    const lines = [...similar].sort((a, b) => b.score - a.score).slice(0, 3).map((m) => `- (${m.score.toFixed(2)}) ${m.text.slice(0, 240)}`);
+    const lines = [...similar].sort((a, b) => b.score - a.score).slice(0, SEMANTIC_MATCH_LIMIT).map((m) => `- ${idLabel(m.resourceId, m.annotationId)} (${m.score.toFixed(2)}) ${m.text.slice(0, SEMANTIC_MATCH_CHARS)}`);
     semanticContextSection = `
 
 Related passages from the knowledge base:
 ${lines.join("\n")}`;
+  }
+  let leadLine;
+  if (task === "resource") {
+    leadLine = `Generate a concise, informative resource about "${topic}".`;
+  } else if (task === "answer") {
+    leadLine = `Answer the following question directly and concisely, grounded in the provided context: "${topic}"`;
+  } else if (task === "summary") {
+    leadLine = `Write a concise summary of "${topic}".`;
+  } else {
+    logger.warn("Unknown task \u2014 using it verbatim as the framing instruction", { task });
+    leadLine = `${task}
+Topic: "${topic}"`;
   }
   const isPlainText = outputMediaType === "text/plain";
-  const structureGuidance = !isPlainText && finalMaxTokens >= 1e3 ? "organized into titled sections (## Section) with well-structured paragraphs" : "organized into well-structured paragraphs";
+  let structureRequirement = "";
+  let titleRequirement = "";
+  if (structure === "sections") {
+    structureRequirement = isPlainText ? "\n- Organize the content into titled sections with well-structured paragraphs" : "\n- Organize the content into titled sections (## Section) with well-structured paragraphs";
+    if (!isPlainText) {
+      titleRequirement = "\n- Start with a clear heading (# Title)";
+    }
+  } else if (structure === "prose") {
+    structureRequirement = "\n- Write flowing, well-structured paragraphs with no section headings";
+  } else if (structure === "chat") {
+    structureRequirement = "\n- Structure the content as a conversational chat transcript \u2014 a sequence of alternating, speaker-labeled turns (no section headings)";
+  } else if (structure) {
+    logger.warn("Unknown structure \u2014 passing it through as freeform organization guidance", { structure });
+    structureRequirement = `
+- Organize the output as: ${structure}`;
+  }
+  const citeRequirement = cite ? "\n- Ground every claim in the provided context. Immediately after each claim, cite its source by emitting [[<id>]], where <id> is an id shown in square brackets in the context above (for a passage labeled [abc], emit [[abc]]). Cite only ids that appear in the context." : "";
   const formatRequirements = isPlainText ? `- Write the response as plain text \u2014 no formatting markup (no #, *, backticks, headings, or links)
-- Begin with the title on its own first line` : `- Start with a clear heading (# Title)
-- Use markdown formatting
+- Begin with the title on its own first line` : `- Use markdown formatting
 - Write the response as markdown`;
-  const prompt = `Generate a concise, informative resource about "${topic}".
-${entityTypes.length > 0 ? `Focus on these entity types: ${entityTypes.join(", ")}.` : ""}
-${userPrompt ? `Additional context: ${userPrompt}` : ""}${annotationSection}${contextSection}${resourceSection}${graphSection}${semanticContextSection}${sourceLanguageInstruction}${languageInstruction}
+  const prompt = `${leadLine}
+${userPrompt ? `Instruction: ${userPrompt}` : ""}
+${entityTypes.length > 0 ? `Focus on these entity types: ${entityTypes.join(", ")}.` : ""}${annotationSection}${contextSection}${resourceSection}${graphSection}${semanticContextSection}${sourceLanguageInstruction}${languageInstruction}
 
 Requirements:
-- Aim for approximately ${finalMaxTokens} tokens of content, ${structureGuidance}
-- Be factual and informative
+- Aim for approximately ${finalMaxTokens} tokens of content
+- Be factual and informative${structureRequirement}${titleRequirement}${citeRequirement}
 ${formatRequirements}`;
   const parseResponse = (response2) => {
     let content = response2.trim();
@@ -1239,6 +1274,67 @@ ${formatRequirements}`;
   });
   return result;
 }
+
+// src/workers/generation/citation-resolver.ts
+var CITATION_TOKEN = /\[\[([^\s[\]/]+)(?:\/([^\s[\]/]+))?\]\]/g;
+function collectContextResourceIds(context) {
+  const ids = /* @__PURE__ */ new Set();
+  if (!context) return ids;
+  const { focus } = context;
+  ids.add(focus.kind === "annotation" ? focus.sourceResource["@id"] : focus.resource["@id"]);
+  for (const node of context.graph.nodes) {
+    if (node.type === "resource") ids.add(node.id);
+  }
+  for (const m of context.semanticContext?.similar ?? []) ids.add(m.resourceId);
+  if (focus.kind === "resource") {
+    for (const id of Object.keys(focus.content?.related ?? {})) ids.add(id);
+  }
+  return ids;
+}
+function resolveCitationTokens(content, validResourceIds, logger) {
+  const citations = [];
+  let clean = "";
+  let last = 0;
+  for (const match of content.matchAll(CITATION_TOKEN)) {
+    const token = match[0];
+    const citedResourceId = match[1];
+    const annotationId = match[2];
+    clean += content.slice(last, match.index).replace(/[ \t]+$/, "");
+    last = match.index + token.length;
+    if (!validResourceIds.has(citedResourceId)) {
+      logger.warn("Citation token references an id absent from the provided context \u2014 dropped", {
+        resourceId: citedResourceId
+      });
+      continue;
+    }
+    const end = clean.length;
+    let start = 0;
+    for (let i = end - 2; i >= 0; i--) {
+      const ch = clean[i];
+      if (ch === "." || ch === "!" || ch === "?" || ch === "\n") {
+        start = i + 1;
+        break;
+      }
+    }
+    while (start < end && /\s/.test(clean[start])) start++;
+    const exact = clean.slice(start, end);
+    if (exact.length === 0) {
+      logger.warn("Citation token has no preceding claim text \u2014 dropped", {
+        resourceId: citedResourceId
+      });
+      continue;
+    }
+    citations.push({
+      resourceId: citedResourceId,
+      ...annotationId ? { annotationId } : {},
+      start,
+      end,
+      exact
+    });
+  }
+  clean += content.slice(last);
+  return { content: clean, citations };
+}
 function toMatch(r) {
   return {
     exact: r.exact,
@@ -1527,10 +1623,9 @@ async function processGenerationJob(inferenceClient, params, onProgress, logger)
       `Unsupported outputMediaType for generation: ${outputMediaType}. Generation produces ${GENERATABLE_MEDIA_TYPES.join(" or ")}.`
     );
   }
-  onProgress(20, "Fetching context...", "fetching");
   const title = params.title ?? "Untitled";
   const entityTypes = (params.entityTypes ?? []).map(String);
-  onProgress(40, "Generating resource...", "generating");
+  onProgress(5, "Generating resource...", "generating");
   const generated = await generateResourceFromTopic(
     title,
     entityTypes,
@@ -1542,13 +1637,24 @@ async function processGenerationJob(inferenceClient, params, onProgress, logger)
     params.temperature,
     params.maxTokens,
     params.sourceLanguage,
-    outputMediaType
+    outputMediaType,
+    params.task,
+    params.structure,
+    params.cite
   );
-  onProgress(85, "Creating resource...", "creating");
+  let content = generated.content;
+  let citations = [];
+  if (params.cite === true) {
+    const resolved = resolveCitationTokens(content, collectContextResourceIds(params.context), logger);
+    content = resolved.content;
+    citations = resolved.citations;
+  }
+  onProgress(95, "Creating resource...", "creating");
   return {
-    content: generated.content,
+    content,
     title: generated.title ?? title,
     format: outputMediaType,
+    citations,
     result: {
       resourceId: "",
       resourceName: generated.title ?? title