@semiont/jobs 0.5.0 → 0.5.2

package/README.md

@@ -10,7 +10,7 @@ Job queue, worker infrastructure, and annotation workers for [Semiont](https://g
 
 ## Architecture Context
 
-Workers run in a separate separate process and connect to the Knowledge System (KS) over HTTP/SSE using `WorkerVM` from `@semiont/api-client`. Workers receive job assignments via SSE push, claim jobs atomically, and emit domain events back to the KS via HTTP. The KS ingests these events onto its EventBus for SSE delivery to the frontend.
+Workers run in a separate separate process and connect to the Knowledge System (KS) over HTTP/SSE using `WorkerStateUnit` from `@semiont/api-client`. Workers receive job assignments via SSE push, claim jobs atomically, and emit domain events back to the KS via HTTP. The KS ingests these events onto its EventBus for SSE delivery to the frontend.
 
 ## Installation
 

package/dist/index.d.ts

@@ -39,6 +39,26 @@ interface JobMetadata {
     retryCount: number;
     maxRetries: number;
 }
+/**
+ * Locale conventions for detection/generation params.
+ *
+ * Two independent locales flow through these jobs:
+ *
+ *   - `language` — *annotation body* locale. The BCP-47 tag the LLM should
+ *     write generated body text in (comment text, assessment text, generated
+ *     resource content, tag category label). Sourced from the user's UI
+ *     locale. Stamped onto the W3C `TextualBody.language` field.
+ *
+ *   - `sourceLanguage` — *source resource* locale. The BCP-47 tag of the
+ *     content being analyzed. Sourced from `ResourceDescriptor` (carried as
+ *     `Representation.language` on the primary representation). Used in
+ *     prompts so the LLM analyzes non-English source correctly even when
+ *     the user's UI locale differs.
+ *
+ * Examples: a German user analyzing an English document → `language='de'`,
+ * `sourceLanguage='en'`. An English user detecting entities in a French
+ * document → `language='en'` (unused for entity references), `sourceLanguage='fr'`.
+ */
 /**
  * Detection job parameters
  */
@@ -46,6 +66,10 @@ interface DetectionParams {
     resourceId: ResourceId;
     entityTypes: EntityType[];
     includeDescriptiveReferences?: boolean;
+    /** Annotation body locale — see locale conventions above. */
+    language?: string;
+    /** Source-resource locale — see locale conventions above. */
+    sourceLanguage?: string;
 }
 /**
  * Generation job parameters
@@ -58,7 +82,14 @@ interface GenerationParams {
     prompt?: string;
     title?: string;
     entityTypes?: EntityType[];
+    /** Annotation body locale — language the *generated resource* is written in. */
     language?: string;
+    /**
+     * Source-resource locale — language of the resource being referenced.
+     * Used in the prompt so the LLM understands the embedded source-context
+     * snippet correctly when source ≠ target language.
+     */
+    sourceLanguage?: string;
     context?: GatheredContext;
     temperature?: number;
     maxTokens?: number;
@@ -71,6 +102,8 @@ interface HighlightDetectionParams {
     resourceId: ResourceId;
     instructions?: string;
     density?: number;
+    /** Source-resource locale — see locale conventions above. */
+    sourceLanguage?: string;
 }
 /**
  * Assessment detection job parameters
@@ -80,7 +113,10 @@ interface AssessmentDetectionParams {
     instructions?: string;
     tone?: 'analytical' | 'critical' | 'balanced' | 'constructive';
     density?: number;
+    /** Annotation body locale — see locale conventions above. */
     language?: string;
+    /** Source-resource locale — see locale conventions above. */
+    sourceLanguage?: string;
 }
 /**
  * Comment detection job parameters
@@ -90,7 +126,10 @@ interface CommentDetectionParams {
     instructions?: string;
     tone?: 'scholarly' | 'explanatory' | 'conversational' | 'technical';
     density?: number;
+    /** Annotation body locale — see locale conventions above. */
     language?: string;
+    /** Source-resource locale — see locale conventions above. */
+    sourceLanguage?: string;
 }
 /**
  * Tag detection job parameters
@@ -99,6 +138,10 @@ interface TagDetectionParams {
     resourceId: ResourceId;
     schemaId: string;
     categories: string[];
+    /** Annotation body locale — see locale conventions above. */
+    language?: string;
+    /** Source-resource locale — see locale conventions above. */
+    sourceLanguage?: string;
 }
 /**
  * Detection job progress
@@ -521,21 +564,39 @@ declare class AnnotationDetection {
      */
     static fetchContent(contentFetcher: ContentFetcher, resourceId: ResourceId): Promise<string>;
     /**
-     * Detect comments in content
+     * Detect comments in content.
+     *
+     * `language` is the locale the LLM should write comment text in (annotation
+     * body locale). `sourceLanguage` is the locale of the content being analyzed
+     * (source-resource locale). See `types.ts` "Locale conventions" for the
+     * full discussion.
      */
-    static detectComments(content: string, client: InferenceClient, instructions?: string, tone?: string, density?: number): Promise<CommentMatch[]>;
+    static detectComments(content: string, client: InferenceClient, instructions?: string, tone?: string, density?: number, language?: string, sourceLanguage?: string): Promise<CommentMatch[]>;
     /**
-     * Detect highlights in content
+     * Detect highlights in content.
+     *
+     * Highlights have no body — only `sourceLanguage` (source-resource locale)
+     * applies, used in the prompt so the LLM analyzes non-English source
+     * correctly.
      */
-    static detectHighlights(content: string, client: InferenceClient, instructions?: string, density?: number): Promise<HighlightMatch[]>;
+    static detectHighlights(content: string, client: InferenceClient, instructions?: string, density?: number, sourceLanguage?: string): Promise<HighlightMatch[]>;
     /**
-     * Detect assessments in content
+     * Detect assessments in content.
+     *
+     * `language` is the locale the LLM should write assessment text in
+     * (annotation body locale). `sourceLanguage` is the locale of the content
+     * being analyzed (source-resource locale).
      */
-    static detectAssessments(content: string, client: InferenceClient, instructions?: string, tone?: string, density?: number): Promise<AssessmentMatch[]>;
+    static detectAssessments(content: string, client: InferenceClient, instructions?: string, tone?: string, density?: number, language?: string, sourceLanguage?: string): Promise<AssessmentMatch[]>;
     /**
-     * Detect tags in content for a specific category
+     * Detect tags in content for a specific category.
+     *
+     * `sourceLanguage` is the locale of the content being analyzed. Body-locale
+     * (`language`) doesn't influence the tag prompt — categories are schema
+     * identifiers, not LLM-generated text — so it's consumed at the body-stamp
+     * site, not here.
      */
-    static detectTags(content: string, client: InferenceClient, schemaId: string, category: string): Promise<TagMatch[]>;
+    static detectTags(content: string, client: InferenceClient, schemaId: string, category: string, sourceLanguage?: string): Promise<TagMatch[]>;
 }
 
 /**
@@ -545,9 +606,17 @@ declare class AnnotationDetection {
  */
 
 /**
- * Generate resource content using inference
- */
-declare function generateResourceFromTopic(topic: string, entityTypes: string[], client: InferenceClient, userPrompt?: string, locale?: string, context?: GatheredContext, temperature?: number, maxTokens?: number, logger?: Logger): Promise<{
+ * Generate resource content using inference.
+ *
+ * Locale parameters: `locale` is the *body* locale — the language the
+ * generated resource should be written in (sourced from the user's UI
+ * locale). `sourceLanguage` is the *source* locale — the language of the
+ * referenced resource whose context (selected passage, surrounding text)
+ * is embedded into the prompt. They're independent: a German user can
+ * generate German content from an English source resource. See
+ * `types.ts` "Locale conventions" for the full discussion.
+ */
+declare function generateResourceFromTopic(topic: string, entityTypes: string[], client: InferenceClient, userPrompt?: string, locale?: string, context?: GatheredContext, temperature?: number, maxTokens?: number, logger?: Logger, sourceLanguage?: string): Promise<{
     title: string;
     content: string;
 }>;

package/dist/index.js

@@ -435,8 +435,21 @@ function isFailedJob(job) {
 function isCancelledJob(job) {
   return job.status === "cancelled";
 }
+function languageName(tag) {
+  return getLocaleEnglishName(tag) || tag;
+}
+function sourceLanguageGuidance(sourceLanguage) {
+  if (!sourceLanguage) return "";
+  return `
+
+Source text language: ${languageName(sourceLanguage)}.`;
+}
+function bodyLanguageGuidance(language, kind) {
+  if (!language || language === "en") return "";
+  return `
 
-// src/workers/detection/motivation-prompts.ts
+IMPORTANT: Write your ${kind} in ${languageName(language)}.`;
+}
 var MotivationPrompts = class {
   /**
    * Build a prompt for detecting comment-worthy passages
@@ -447,8 +460,10 @@ var MotivationPrompts = class {
    * @param density - Optional target number of comments per 2000 words
    * @returns Formatted prompt string
    */
-  static buildCommentPrompt(content, instructions, tone, density) {
+  static buildCommentPrompt(content, instructions, tone, density, language, sourceLanguage) {
     let prompt;
+    const sourceLang = sourceLanguageGuidance(sourceLanguage);
+    const bodyLang = bodyLanguageGuidance(language, "comments");
     if (instructions) {
       const toneGuidance = tone ? ` Use a ${tone} tone.` : "";
       const densityGuidance = density ? `
@@ -456,7 +471,7 @@ var MotivationPrompts = class {
 Aim for approximately ${density} comments per 2000 words of text.` : "";
       prompt = `Add comments to passages in this text following these instructions:
 
-${instructions}${toneGuidance}${densityGuidance}
+${instructions}${toneGuidance}${densityGuidance}${sourceLang}${bodyLang}
 
 Text to analyze:
 ---
@@ -492,7 +507,7 @@ Guidelines:
 - Provide comments that ADD VALUE beyond restating the text
 - Focus on explanation, background, or connections to other ideas
 - Avoid obvious or trivial comments
-- Keep comments concise (1-3 sentences typically)${densityGuidance}
+- Keep comments concise (1-3 sentences typically)${densityGuidance}${sourceLang}${bodyLang}
 
 Text to analyze:
 ---
@@ -524,15 +539,16 @@ Example format:
    * @param density - Optional target number of highlights per 2000 words
    * @returns Formatted prompt string
    */
-  static buildHighlightPrompt(content, instructions, density) {
+  static buildHighlightPrompt(content, instructions, density, sourceLanguage) {
     let prompt;
+    const sourceLang = sourceLanguageGuidance(sourceLanguage);
     if (instructions) {
       const densityGuidance = density ? `
 
 Aim for approximately ${density} highlights per 2000 words of text.` : "";
       prompt = `Identify passages in this text to highlight following these instructions:
 
-${instructions}${densityGuidance}
+${instructions}${densityGuidance}${sourceLang}
 
 Text to analyze:
 ---
@@ -565,7 +581,7 @@ Guidelines:
 - Highlight notable quotes or particularly striking statements
 - Highlight critical decisions, action items, or turning points
 - Select passages that are SIGNIFICANT, not just interesting
-- Avoid trivial or obvious content${densityGuidance}
+- Avoid trivial or obvious content${densityGuidance}${sourceLang}
 
 Text to analyze:
 ---
@@ -597,8 +613,10 @@ Example format:
    * @param density - Optional target number of assessments per 2000 words
    * @returns Formatted prompt string
    */
-  static buildAssessmentPrompt(content, instructions, tone, density) {
+  static buildAssessmentPrompt(content, instructions, tone, density, language, sourceLanguage) {
     let prompt;
+    const sourceLang = sourceLanguageGuidance(sourceLanguage);
+    const bodyLang = bodyLanguageGuidance(language, "assessments");
     if (instructions) {
       const toneGuidance = tone ? ` Use a ${tone} tone.` : "";
       const densityGuidance = density ? `
@@ -606,7 +624,7 @@ Example format:
 Aim for approximately ${density} assessments per 2000 words of text.` : "";
       prompt = `Assess passages in this text following these instructions:
 
-${instructions}${toneGuidance}${densityGuidance}
+${instructions}${toneGuidance}${densityGuidance}${sourceLang}${bodyLang}
 
 Text to analyze:
 ---
@@ -642,7 +660,7 @@ Guidelines:
 - Assess evidence quality, logical soundness, or practical implications
 - Provide assessments that ADD INSIGHT beyond restating the text
 - Focus on passages where evaluation would help readers form judgments
-- Keep assessments concise yet substantive (1-3 sentences typically)${densityGuidance}
+- Keep assessments concise yet substantive (1-3 sentences typically)${densityGuidance}${sourceLang}${bodyLang}
 
 Text to analyze:
 ---
@@ -678,7 +696,8 @@ Example format:
    * @param categoryExamples - Example questions/guidance for this category
    * @returns Formatted prompt string
    */
-  static buildTagPrompt(content, category, schemaName, schemaDescription, schemaDomain, categoryDescription, categoryExamples) {
+  static buildTagPrompt(content, category, schemaName, schemaDescription, schemaDomain, categoryDescription, categoryExamples, sourceLanguage) {
+    const sourceLang = sourceLanguageGuidance(sourceLanguage);
     const prompt = `You are analyzing a text using the ${schemaName} framework.
 
 Schema: ${schemaDescription}
@@ -697,7 +716,7 @@ Guidelines:
 - Look for passages that explicitly fulfill this role
 - Passages can be sentences, paragraphs, or sections
 - Aim for precision - only tag passages that clearly serve this structural role
-- Typical documents have 1-5 instances of each category (some may have 0)
+- Typical documents have 1-5 instances of each category (some may have 0)${sourceLang}
 
 Text to analyze:
 ---
@@ -1099,33 +1118,51 @@ var AnnotationDetection = class {
     return Buffer.concat(chunks).toString("utf-8");
   }
   /**
-   * Detect comments in content
+   * Detect comments in content.
+   *
+   * `language` is the locale the LLM should write comment text in (annotation
+   * body locale). `sourceLanguage` is the locale of the content being analyzed
+   * (source-resource locale). See `types.ts` "Locale conventions" for the
+   * full discussion.
    */
-  static async detectComments(content, client, instructions, tone, density) {
-    const prompt = MotivationPrompts.buildCommentPrompt(content, instructions, tone, density);
+  static async detectComments(content, client, instructions, tone, density, language, sourceLanguage) {
+    const prompt = MotivationPrompts.buildCommentPrompt(content, instructions, tone, density, language, sourceLanguage);
     const response = await client.generateText(prompt, 3e3, 0.4);
     return MotivationParsers.parseComments(response, content);
   }
   /**
-   * Detect highlights in content
+   * Detect highlights in content.
+   *
+   * Highlights have no body — only `sourceLanguage` (source-resource locale)
+   * applies, used in the prompt so the LLM analyzes non-English source
+   * correctly.
    */
-  static async detectHighlights(content, client, instructions, density) {
-    const prompt = MotivationPrompts.buildHighlightPrompt(content, instructions, density);
+  static async detectHighlights(content, client, instructions, density, sourceLanguage) {
+    const prompt = MotivationPrompts.buildHighlightPrompt(content, instructions, density, sourceLanguage);
     const response = await client.generateText(prompt, 2e3, 0.3);
     return MotivationParsers.parseHighlights(response, content);
   }
   /**
-   * Detect assessments in content
+   * Detect assessments in content.
+   *
+   * `language` is the locale the LLM should write assessment text in
+   * (annotation body locale). `sourceLanguage` is the locale of the content
+   * being analyzed (source-resource locale).
    */
-  static async detectAssessments(content, client, instructions, tone, density) {
-    const prompt = MotivationPrompts.buildAssessmentPrompt(content, instructions, tone, density);
+  static async detectAssessments(content, client, instructions, tone, density, language, sourceLanguage) {
+    const prompt = MotivationPrompts.buildAssessmentPrompt(content, instructions, tone, density, language, sourceLanguage);
     const response = await client.generateText(prompt, 3e3, 0.3);
     return MotivationParsers.parseAssessments(response, content);
   }
   /**
-   * Detect tags in content for a specific category
+   * Detect tags in content for a specific category.
+   *
+   * `sourceLanguage` is the locale of the content being analyzed. Body-locale
+   * (`language`) doesn't influence the tag prompt — categories are schema
+   * identifiers, not LLM-generated text — so it's consumed at the body-stamp
+   * site, not here.
    */
-  static async detectTags(content, client, schemaId, category) {
+  static async detectTags(content, client, schemaId, category, sourceLanguage) {
     const schema = getTagSchema(schemaId);
     if (!schema) {
       throw new Error(`Invalid tag schema: ${schemaId}`);
@@ -1141,16 +1178,15 @@ var AnnotationDetection = class {
       schema.description,
       schema.domain,
       categoryInfo.description,
-      categoryInfo.examples
+      categoryInfo.examples,
+      sourceLanguage
     );
     const response = await client.generateText(prompt, 4e3, 0.2);
     const parsedTags = MotivationParsers.parseTags(response);
     return MotivationParsers.validateTagOffsets(parsedTags, content, category);
   }
 };
-
-// src/workers/detection/entity-extractor.ts
-async function extractEntities(exact, entityTypes, client, includeDescriptiveReferences = false, logger) {
+async function extractEntities(exact, entityTypes, client, includeDescriptiveReferences = false, logger, sourceLanguage) {
   const entityTypesDescription = entityTypes.map((et) => {
     if (typeof et === "string") {
       return et;
@@ -1179,8 +1215,11 @@ Examples:
 ` : `
 Find direct mentions only (names, proper nouns). Do not include pronouns or descriptive references.
 `;
+  const sourceLangGuidance = sourceLanguage ? `
+Source text language: ${getLocaleEnglishName(sourceLanguage) || sourceLanguage}.
+` : "";
   const prompt = `Identify entity references in the following text. Look for mentions of: ${entityTypesDescription}.
-${descriptiveReferenceGuidance}
+${descriptiveReferenceGuidance}${sourceLangGuidance}
 Text to analyze:
 """
 ${exact}
@@ -1374,12 +1413,13 @@ Example output:
 function getLanguageName(locale) {
   return getLocaleEnglishName(locale) || locale;
 }
-async function generateResourceFromTopic(topic, entityTypes, client, userPrompt, locale, context, temperature, maxTokens, logger) {
+async function generateResourceFromTopic(topic, entityTypes, client, userPrompt, locale, context, temperature, maxTokens, logger, sourceLanguage) {
   logger?.debug("Generating resource from topic", {
     topicPreview: topic.substring(0, 100),
     entityTypes,
     hasUserPrompt: !!userPrompt,
     locale,
+    sourceLanguage,
     hasContext: !!context,
     temperature,
     maxTokens
@@ -1389,6 +1429,9 @@ async function generateResourceFromTopic(topic, entityTypes, client, userPrompt,
   const languageInstruction = locale && locale !== "en" ? `
 
 IMPORTANT: Write the entire resource in ${getLanguageName(locale)}.` : "";
+  const sourceLanguageInstruction = sourceLanguage ? `
+
+The source resource and embedded context are in ${getLanguageName(sourceLanguage)}.` : "";
   let annotationSection = "";
   if (context) {
     const parts = [];
@@ -1450,7 +1493,7 @@ ${parts.join("\n")}`;
   const structureGuidance = finalMaxTokens >= 1e3 ? "organized into titled sections (## Section) with well-structured paragraphs" : "organized into well-structured paragraphs";
   const prompt = `Generate a concise, informative resource about "${topic}".
 ${entityTypes.length > 0 ? `Focus on these entity types: ${entityTypes.join(", ")}.` : ""}
-${userPrompt ? `Additional context: ${userPrompt}` : ""}${annotationSection}${contextSection}${graphContextSection}${languageInstruction}
+${userPrompt ? `Additional context: ${userPrompt}` : ""}${annotationSection}${contextSection}${graphContextSection}${sourceLanguageInstruction}${languageInstruction}
 
 Requirements:
 - Start with a clear heading (# Title)
@@ -1527,7 +1570,8 @@ async function processHighlightJob(content, inferenceClient, params, userId, gen
     content,
     inferenceClient,
     params.instructions,
-    params.density
+    params.density,
+    params.sourceLanguage
   );
   onProgress(60, `Creating ${highlights.length} annotations...`, "creating");
   const annotations = highlights.map(
@@ -1547,16 +1591,19 @@ async function processCommentJob(content, inferenceClient, params, userId, gener
     inferenceClient,
     params.instructions,
     params.tone,
-    params.density
+    params.density,
+    params.language,
+    params.sourceLanguage
   );
   onProgress(60, `Creating ${comments.length} annotations...`, "creating");
+  const bodyLanguage = params.language ?? "en";
   const annotations = comments.map(
     (c) => (
       // Match the pre-#651 CommentAnnotationWorker: include format and
       // language on the body TextualBody. Optional in the schema, but
       // consumers that do language-aware rendering rely on them.
       buildTextAnnotation(params.resourceId, userId, generator, "commenting", c, [
-        { type: "TextualBody", value: c.comment, purpose: "commenting", format: "text/plain", language: "en" }
+        { type: "TextualBody", value: c.comment, purpose: "commenting", format: "text/plain", language: bodyLanguage }
       ])
     )
   );
@@ -1574,9 +1621,12 @@ async function processAssessmentJob(content, inferenceClient, params, userId, ge
     inferenceClient,
     params.instructions,
     params.tone,
-    params.density
+    params.density,
+    params.language,
+    params.sourceLanguage
   );
   onProgress(60, `Creating ${assessments.length} annotations...`, "creating");
+  const bodyLanguage = params.language ?? "en";
   const annotations = assessments.map(
     (a) => (
       // Single-object body with purpose aligned to motivation, matching the
@@ -1590,7 +1640,7 @@ async function processAssessmentJob(content, inferenceClient, params, userId, ge
         value: a.assessment,
         purpose: "assessing",
         format: "text/plain",
-        language: "en"
+        language: bodyLanguage
       })
     )
   );
@@ -1609,6 +1659,7 @@ async function processReferenceJob(content, inferenceClient, params, userId, gen
   let errors = 0;
   const allAnnotations = [];
   onProgress(10, "Loading resource...", "analyzing", { requestParams });
+  const bodyLanguage = params.language ?? "en";
   for (let i = 0; i < entityTypeNames.length; i++) {
     const entityTypeName = entityTypeNames[i];
     if (!entityTypeName) continue;
@@ -1627,11 +1678,14 @@ async function processReferenceJob(content, inferenceClient, params, userId, gen
       [entityTypeName],
       inferenceClient,
       params.includeDescriptiveReferences ?? false,
-      logger
+      logger,
+      params.sourceLanguage
     );
     totalFound += extractedEntities.length;
     completedEntityTypes.push({ entityType: entityTypeName, foundCount: extractedEntities.length });
-    const unresolvedBody = [{ type: "TextualBody", value: entityTypeName, purpose: "tagging" }];
+    const unresolvedBody = [
+      { type: "TextualBody", value: entityTypeName, purpose: "tagging", format: "text/plain", language: bodyLanguage }
+    ];
     for (const entity of extractedEntities) {
       try {
         const validated = validateAndCorrectOffsets(content, entity.startOffset, entity.endOffset, entity.exact);
@@ -1665,18 +1719,20 @@ async function processTagJob(content, inferenceClient, params, userId, generator
       content,
       inferenceClient,
       params.schemaId,
-      category
+      category,
+      params.sourceLanguage
     );
     allTags.push(...categoryTags);
   }
   const tags = allTags;
   onProgress(60, `Creating ${tags.length} tag annotations...`, "creating");
+  const bodyLanguage = params.language ?? "en";
   const byCategory = {};
   const annotations = tags.map((t) => {
     const category = t.category ?? "unknown";
     byCategory[category] = (byCategory[category] ?? 0) + 1;
     return buildTextAnnotation(params.resourceId, userId, generator, "tagging", t, [
-      { type: "TextualBody", value: category, purpose: "tagging", format: "text/plain", language: "en" },
+      { type: "TextualBody", value: category, purpose: "tagging", format: "text/plain", language: bodyLanguage },
       { type: "TextualBody", value: params.schemaId, purpose: "classifying", format: "text/plain" }
     ]);
   });
@@ -1699,7 +1755,10 @@ async function processGenerationJob(inferenceClient, params, onProgress) {
     params.language,
     params.context,
     params.temperature,
-    params.maxTokens
+    params.maxTokens,
+    void 0,
+    // logger
+    params.sourceLanguage
   );
   onProgress(85, "Creating resource...", "creating");
   return {