@semiont/jobs 0.5.27 → 0.5.29

package/README.md

@@ -10,7 +10,7 @@ Job queue, worker infrastructure, and annotation workers for [Semiont](https://g
 
 ## Architecture Context
 
-Workers run in a separate process and connect to the Knowledge System (KS) over HTTP/SSE using a `SemiontSession` (from `@semiont/sdk`) driven by a `JobClaimAdapter`. Workers receive job assignments via an SSE `job:queued` subscription, claim jobs atomically, and emit domain events back to the KS via `session.client.transport.emit(...)`. The KS ingests these events onto its EventBus for SSE delivery to the frontend.
+Workers run in a separate process and connect to the Knowledge System (KS) over HTTP/SSE using a `SemiontSession` (from `@semiont/sdk`) driven by a `JobClaimAdapter`. Workers receive job assignments via an SSE `job:queued` subscription, claim jobs atomically, and emit domain events back to the KS via `session.client.transport.emit(...)`. The KS ingests these events onto its EventBus for SSE delivery to the Browser.
 
 ## Installation
 
@@ -36,7 +36,9 @@ import { SemiontProject } from '@semiont/core/node';
 
 // Initialize — jobs are stored under project.jobsDir
 const eventBus = new EventBus();
-const project = new SemiontProject('/path/to/project');
+const project = new SemiontProject('/path/to/project', {
+  anchoredTextDir: process.env.SEMIONT_ANCHORED_TEXT_DIR!,
+});
 const jobQueue = new FsJobQueue(project, logger, eventBus);
 await jobQueue.initialize();
 

package/dist/index.d.ts

@@ -1,5 +1,5 @@
 import { JobId, UserId, ResourceId, EntityType, GenerationJobParams, TagSchema, Logger, EventBus, components, Annotation, SupportedMediaType, GatheredContext } from '@semiont/core';
-import { SemiontProject } from '@semiont/core/node';
+import { SemiontState } from '@semiont/core/node';
 import { InferenceClient } from '@semiont/inference';
 
 /**
@@ -136,6 +136,7 @@ interface DetectionProgress {
  * Detection job result
  */
 interface DetectionResult {
+    kind: 'reference-annotation';
     totalFound: number;
     totalEmitted: number;
     errors: number;
@@ -153,8 +154,11 @@ interface YieldProgress {
  * Generation job result
  */
 interface GenerationResult {
+    kind: 'generation';
     resourceId: ResourceId;
     resourceName: string;
+    /** True when the model stopped at the maxTokens ceiling — the artifact is cut off, not complete (GENERATE-FROM-RESOURCE D6). */
+    truncated: boolean;
 }
 /**
  * Highlight detection job progress
@@ -168,6 +172,7 @@ interface HighlightDetectionProgress {
  * Highlight detection job result
  */
 interface HighlightDetectionResult {
+    kind: 'highlight-annotation';
     highlightsFound: number;
     highlightsCreated: number;
 }
@@ -183,6 +188,7 @@ interface AssessmentDetectionProgress {
  * Assessment detection job result
  */
 interface AssessmentDetectionResult {
+    kind: 'assessment-annotation';
     assessmentsFound: number;
     assessmentsCreated: number;
 }
@@ -198,6 +204,7 @@ interface CommentDetectionProgress {
  * Comment detection job result
  */
 interface CommentDetectionResult {
+    kind: 'comment-annotation';
     commentsFound: number;
     commentsCreated: number;
 }
@@ -216,6 +223,7 @@ interface TagDetectionProgress {
  * Tag detection job result
  */
 interface TagDetectionResult {
+    kind: 'tag-annotation';
     tagsFound: number;
     tagsCreated: number;
     byCategory: Record<string, number>;
@@ -345,7 +353,10 @@ declare class FsJobQueue implements JobQueue {
     private cleanupTimer;
     /** Per-job timestamp of the last progress write, for throttling. */
     private lastProgressWrite;
-    constructor(project: SemiontProject, logger: Logger, eventBus?: EventBus | undefined);
+    constructor(
+    /** `SemiontState` and not `SemiontProject`: the queue reads ONE path,
+     *  and the gateway that owns it mounts no KB tree (SINGLE-KB-MOUNT P5). */
+    state: SemiontState, logger: Logger, eventBus?: EventBus | undefined);
     /**
      * Initialize job queue directories, announce any pending backlog,
      * and start the re-announce interval. Idempotent.
@@ -654,6 +665,7 @@ declare function generateResourceFromTopic(topic: string, entityTypes: string[],
 }): Promise<{
     title: string;
     content: string;
+    truncated: boolean;
 }>;
 
 /**
@@ -683,7 +695,7 @@ declare function generateResourceFromTopic(topic: string, entityTypes: string[],
  *
  * Thresholds are fixed by design (no env knobs) and deliberately
  * layered: inference timeout (10 min, P2) fires first; this watchdog
- * (15 min) catches the failure modes nobody predicted; the backend's
+ * (15 min) catches the failure modes nobody predicted; the gateway's
  * dead-worker janitor (30 min) re-queues the job regardless.
  */
 declare const STALL_THRESHOLD_MS: number;

package/dist/index.js

@@ -17,9 +17,9 @@ var PROGRESS_WRITE_MIN_INTERVAL_MS = 5e3;
 var RETENTION_HOURS = 24;
 var CLEANUP_INTERVAL_MS = 36e5;
 var FsJobQueue = class {
-  constructor(project, logger, eventBus) {
+  constructor(state, logger, eventBus) {
     this.eventBus = eventBus;
-    this.jobsDir = project.jobsDir;
+    this.jobsDir = state.jobsDir;
     this.logger = logger;
   }
   eventBus;
@@ -468,9 +468,9 @@ async function withTimeout(work, label, onHeartbeat) {
     if (heartbeat) clearInterval(heartbeat);
   }
 }
-function boundedGenerate(client, prompt, maxTokens, temperature, onHeartbeat) {
+function boundedGenerateWithMetadata(client, prompt, maxTokens, temperature, onHeartbeat) {
   return spanned(client, "text", maxTokens, () => withTimeout(
-    client.generateText(prompt, maxTokens, temperature),
+    client.generateTextWithMetadata(prompt, maxTokens, temperature),
     `${client.type}:${client.modelId}`,
     onHeartbeat
   ));
@@ -1253,6 +1253,7 @@ var SEMANTIC_MATCH_CHARS = 240;
 function idLabel(resourceId, annotationId) {
   return `[${resourceId}${annotationId ? `/${annotationId}` : ""}]`;
 }
+var DEFAULT_MAX_TOKENS = 500;
 async function generateResourceFromTopic(topic, entityTypes, client, logger, userPrompt, locale, context, temperature, maxTokens, sourceLanguage, outputMediaType = "text/markdown", task = "resource", structure, cite = false, repair) {
   logger.debug("Generating resource from topic", {
     topicPreview: topic.substring(0, 100),
@@ -1268,7 +1269,7 @@ async function generateResourceFromTopic(topic, entityTypes, client, logger, use
     structure
   });
   const finalTemperature = temperature ?? 0.7;
-  const finalMaxTokens = maxTokens ?? 500;
+  const finalMaxTokens = maxTokens ?? DEFAULT_MAX_TOKENS;
   const languageInstruction = locale && locale !== "en" ? `
 
 IMPORTANT: Write the entire resource in ${getLanguageName(locale)}.` : "";
@@ -1295,6 +1296,9 @@ The source resource and embedded context are in ${getLanguageName(sourceLanguage
           parts.push(`- ${label}: ${bodyItem.value}`);
         }
       }
+      if (focus.userHint) {
+        parts.push(`- User hint (steers what to generate): ${focus.userHint}`);
+      }
       annotationSection = `
 
 Annotation context:
@@ -1453,16 +1457,16 @@ ${formatRequirements}`;
     temperature: finalTemperature,
     maxTokens: finalMaxTokens
   });
-  const response = await boundedGenerate(client, prompt, finalMaxTokens, finalTemperature);
-  logger.debug("Got response from inference", { responseLength: response.length });
-  const result = parseResponse(response);
+  const response = await boundedGenerateWithMetadata(client, prompt, finalMaxTokens, finalTemperature);
+  logger.debug("Got response from inference", { responseLength: response.text.length, stopReason: response.stopReason });
+  const result = parseResponse(response.text);
   logger.debug("Parsed response", {
     hasTitle: !!result.title,
     titleLength: result.title?.length,
     hasContent: !!result.content,
     contentLength: result.content?.length
   });
-  return result;
+  return { ...result, truncated: response.stopReason === "max_tokens" };
 }
 var PINNED_CREATION_TIMESTAMP = 17e8;
 var MAX_COMPILE_REPAIRS = 2;
@@ -1602,7 +1606,7 @@ async function processHighlightJob(content, inferenceClient, params, buildAnnota
   onProgress(100, { code: "complete-created", count: annotations.length, kind: "highlight" }, echo);
   return {
     annotations,
-    result: { highlightsFound: highlights.length, highlightsCreated: annotations.length }
+    result: { kind: "highlight-annotation", highlightsFound: highlights.length, highlightsCreated: annotations.length }
   };
 }
 function detectionEcho(p) {
@@ -1642,7 +1646,7 @@ async function processCommentJob(content, inferenceClient, params, buildAnnotati
   onProgress(100, { code: "complete-created", count: annotations.length, kind: "comment" }, echo);
   return {
     annotations,
-    result: { commentsFound: comments.length, commentsCreated: annotations.length }
+    result: { kind: "comment-annotation", commentsFound: comments.length, commentsCreated: annotations.length }
   };
 }
 async function processAssessmentJob(content, inferenceClient, params, buildAnnotation, onProgress) {
@@ -1682,7 +1686,7 @@ async function processAssessmentJob(content, inferenceClient, params, buildAnnot
   onProgress(100, { code: "complete-created", count: annotations.length, kind: "assessment" }, echo);
   return {
     annotations,
-    result: { assessmentsFound: assessments.length, assessmentsCreated: annotations.length }
+    result: { kind: "assessment-annotation", assessmentsFound: assessments.length, assessmentsCreated: annotations.length }
   };
 }
 async function processReferenceJob(content, inferenceClient, params, buildAnnotation, onProgress, logger) {
@@ -1772,7 +1776,7 @@ async function processReferenceJob(content, inferenceClient, params, buildAnnota
   onProgress(100, { code: "complete-created", count: annotations.length, kind: "reference" }, { requestParams });
   return {
     annotations,
-    result: { totalFound, totalEmitted: annotations.length, errors }
+    result: { kind: "reference-annotation", totalFound, totalEmitted: annotations.length, errors }
   };
 }
 async function processTagJob(content, inferenceClient, params, buildAnnotation, onProgress) {
@@ -1829,7 +1833,7 @@ async function processTagJob(content, inferenceClient, params, buildAnnotation,
   onProgress(100, { code: "complete-created", count: annotations.length, kind: "tag" });
   return {
     annotations,
-    result: { tagsFound: tags.length, tagsCreated: annotations.length, byCategory }
+    result: { kind: "tag-annotation", tagsFound: tags.length, tagsCreated: annotations.length, byCategory }
   };
 }
 function assertWithinOutputBudget(byteLength) {
@@ -1876,7 +1880,17 @@ async function processGenerationJob(inferenceClient, params, onProgress, logger)
     }
     let compiled = compileTypst(source);
     let repairs = 0;
-    while ("error" in compiled && repairs < MAX_COMPILE_REPAIRS) {
+    while ("error" in compiled) {
+      if (generated2.truncated) {
+        throw new Error(
+          `Generation stopped at the maxTokens ceiling (${params.maxTokens ?? DEFAULT_MAX_TOKENS} tokens) and the cut-off Typst source does not compile \u2014 repair cannot help; raise maxTokens. Compile error: ${compiled.error}`
+        );
+      }
+      if (repairs >= MAX_COMPILE_REPAIRS) {
+        throw new Error(
+          `Typst compilation failed after ${MAX_COMPILE_REPAIRS} repair attempts: ${compiled.error}`
+        );
+      }
       repairs++;
       logger.warn("Typst compile failed \u2014 feeding the error back for repair", {
         attempt: repairs,
@@ -1908,21 +1922,19 @@ async function processGenerationJob(inferenceClient, params, onProgress, logger)
       }
       compiled = compileTypst(source);
     }
-    if ("error" in compiled) {
-      throw new Error(
-        `Typst compilation failed after ${MAX_COMPILE_REPAIRS} repair attempts: ${compiled.error}`
-      );
-    }
     assertWithinOutputBudget(compiled.pdf.byteLength);
     onProgress(95, { code: "creating-resource" });
+    onProgress(100, { code: "complete-generated", truncated: generated2.truncated });
     return {
       content: compiled.pdf,
-      title: generated2.title ?? title,
+      title,
       format: outputMediaType,
       citations: citations2,
       result: {
+        kind: "generation",
         resourceId: "",
-        resourceName: generated2.title ?? title
+        resourceName: title,
+        truncated: generated2.truncated
       }
     };
   }
@@ -1953,14 +1965,17 @@ async function processGenerationJob(inferenceClient, params, onProgress, logger)
   onProgress(95, { code: "creating-resource" });
   const artifact = new TextEncoder().encode(content);
   assertWithinOutputBudget(artifact.byteLength);
+  onProgress(100, { code: "complete-generated", truncated: generated.truncated });
   return {
     content: artifact,
-    title: generated.title ?? title,
+    title,
     format: outputMediaType,
     citations,
     result: {
+      kind: "generation",
       resourceId: "",
-      resourceName: generated.title ?? title
+      resourceName: title,
+      truncated: generated.truncated
     }
   };
 }