@semiont/jobs 0.4.20 → 0.4.22

package/README.md

@@ -6,13 +6,11 @@
 [![npm downloads](https://img.shields.io/npm/dm/@semiont/jobs.svg)](https://www.npmjs.com/package/@semiont/jobs)
 [![License](https://img.shields.io/npm/l/@semiont/jobs.svg)](https://github.com/The-AI-Alliance/semiont/blob/main/LICENSE)
 
-Filesystem-based job queue, worker infrastructure, and annotation workers for [Semiont](https://github.com/The-AI-Alliance/semiont).
+Job queue, worker infrastructure, and annotation workers for [Semiont](https://github.com/The-AI-Alliance/semiont).
 
 ## Architecture Context
 
-In production, the job queue and workers are created by `@semiont/make-meaning`'s `startMakeMeaning()` function. Workers emit commands on the **EventBus** — the **Stower** actor (in @semiont/make-meaning) handles all persistence to the Knowledge Base.
-
-Workers are **not** actors. They use a polling loop, not RxJS subscriptions. But they emit the same EventBus commands as any other caller in the system.
+Workers run in a separate separate process and connect to the Knowledge System (KS) over HTTP/SSE using `WorkerVM` from `@semiont/api-client`. Workers receive job assignments via SSE push, claim jobs atomically, and emit domain events back to the KS via HTTP. The KS ingests these events onto its EventBus for SSE delivery to the frontend.
 
 ## Installation
 

package/dist/index.d.ts

@@ -1,8 +1,8 @@
 import { Readable } from 'stream';
-import { ResourceId, JobId, UserId, EntityType, AnnotationId, components, GatheredContext, Logger, EventBus } from '@semiont/core';
+import * as _semiont_core from '@semiont/core';
+import { ResourceId, JobId, UserId, EntityType, AnnotationId, Annotation, GatheredContext, Logger, EventBus, components } from '@semiont/core';
 import { SemiontProject } from '@semiont/core/node';
 import { InferenceClient } from '@semiont/inference';
-import { WorkingTreeStore } from '@semiont/content';
 
 /**
  * Job Queue Type Definitions - Discriminated Union Design
@@ -17,7 +17,6 @@ import { WorkingTreeStore } from '@semiont/content';
  * - State machine is explicit and type-safe
  */
 
-type Annotation = components['schemas']['Annotation'];
 /**
  * Content fetcher - turns a ResourceId into a readable stream.
  * Workers use this to access resource content on demand.
@@ -275,6 +274,23 @@ interface JobQueryFilters {
     offset?: number;
 }
 
+interface JobQueue {
+    initialize(): Promise<void>;
+    destroy(): void;
+    createJob(job: AnyJob): Promise<void>;
+    getJob(jobId: JobId): Promise<AnyJob | null>;
+    updateJob(job: AnyJob, oldStatus?: JobStatus): Promise<void>;
+    pollNextPendingJob(predicate?: (job: AnyJob) => boolean): Promise<AnyJob | null>;
+    cancelJob(jobId: JobId): Promise<boolean>;
+    getStats(): Promise<{
+        pending: number;
+        running: number;
+        complete: number;
+        failed: number;
+        cancelled: number;
+    }>;
+}
+
 /**
  * Job Queue Manager
  *
@@ -282,7 +298,7 @@ interface JobQueryFilters {
  * Jobs are stored in directories by status for easy polling.
  */
 
-declare class JobQueue {
+declare class FsJobQueue implements JobQueue {
     private eventBus?;
     private jobsDir;
     private logger;
@@ -416,231 +432,32 @@ declare abstract class JobWorker {
     protected abstract executeJob(job: AnyJob): Promise<any>;
 }
 
+type Agent = components['schemas']['Agent'];
 /**
- * Reference Detection Worker
- *
- * Processes detection jobs: runs AI inference to find entities in resources
- * and creates annotations for each detected entity via the EventBus.
- *
- * This worker is INDEPENDENT of HTTP clients - it just processes jobs and
- * emits events on the EventBus for all writes.
- */
-
-type Agent$5 = components['schemas']['Agent'];
-interface DetectedAnnotation {
-    annotation: {
-        selector: {
-            start: number;
-            end: number;
-            exact: string;
-            prefix?: string;
-            suffix?: string;
-        };
-        entityTypes: string[];
-    };
-}
-declare class ReferenceAnnotationWorker extends JobWorker {
-    private inferenceClient;
-    private generator;
-    private eventBus;
-    private contentFetcher;
-    constructor(jobQueue: JobQueue, inferenceClient: InferenceClient, generator: Agent$5, eventBus: EventBus, contentFetcher: ContentFetcher, logger: Logger);
-    protected getWorkerName(): string;
-    protected canProcessJob(job: AnyJob): boolean;
-    protected executeJob(job: AnyJob): Promise<DetectionResult>;
-    /**
-     * Detect entity references in content using AI
-     *
-     * Public for testing charset handling - see entity-detection-charset.test.ts
-     */
-    detectReferences(content: string, entityTypes: string[], includeDescriptiveReferences?: boolean): Promise<DetectedAnnotation[]>;
-    private processDetectionJob;
-    /**
-     * Emit completion event with result data
-     * Override base class to emit on EventBus
-     */
-    protected emitCompletionEvent(job: RunningJob<DetectionParams, DetectionProgress>, result: DetectionResult): Promise<void>;
-    protected handleJobFailure(job: AnyJob, error: any): Promise<void>;
-    /**
-     * Update job progress and emit ephemeral events via EventBus
-     * Overrides base class to emit job lifecycle events and mark:progress events
-     */
-    protected updateJobProgress(job: AnyJob): Promise<void>;
-}
-
-/**
- * Generation Worker
- *
- * Processes generation jobs: runs AI inference to generate new resources
- * and emits events on the EventBus for all writes.
- *
- * This worker is INDEPENDENT of HTTP clients - it just processes jobs and
- * emits events on the EventBus for all writes.
- */
-
-type Agent$4 = components['schemas']['Agent'];
-declare class GenerationWorker extends JobWorker {
-    private inferenceClient;
-    private generator;
-    private eventBus;
-    private contentStore;
-    constructor(jobQueue: JobQueue, inferenceClient: InferenceClient, generator: Agent$4, eventBus: EventBus, contentStore: WorkingTreeStore, logger: Logger);
-    protected getWorkerName(): string;
-    protected canProcessJob(job: AnyJob): boolean;
-    protected executeJob(job: AnyJob): Promise<GenerationResult>;
-    private processGenerationJob;
-    /**
-     * Emit completion event with result data
-     * Override base class to emit on EventBus
-     */
-    protected emitCompletionEvent(job: RunningJob<GenerationParams, YieldProgress>, result: GenerationResult): Promise<void>;
-    protected handleJobFailure(job: AnyJob, error: any): Promise<void>;
-    /**
-     * Update job progress and emit ephemeral events via EventBus
-     * Overrides base class to emit job lifecycle events and yield:progress events
-     */
-    protected updateJobProgress(job: AnyJob): Promise<void>;
-}
-
-/**
- * Highlight Detection Worker
- *
- * Processes highlight-detection jobs: runs AI inference to find passages
- * that should be highlighted and creates highlight annotations.
- */
-
-type Agent$3 = components['schemas']['Agent'];
-declare class HighlightAnnotationWorker extends JobWorker {
-    private inferenceClient;
-    private generator;
-    private eventBus;
-    private contentFetcher;
-    private isFirstProgress;
-    constructor(jobQueue: JobQueue, inferenceClient: InferenceClient, generator: Agent$3, eventBus: EventBus, contentFetcher: ContentFetcher, logger: Logger);
-    protected getWorkerName(): string;
-    protected canProcessJob(job: AnyJob): boolean;
-    protected executeJob(job: AnyJob): Promise<HighlightDetectionResult>;
-    /**
-     * Emit completion event with result data
-     * Override base class to emit on EventBus
-     */
-    protected emitCompletionEvent(job: RunningJob<HighlightDetectionParams, HighlightDetectionProgress>, result: HighlightDetectionResult): Promise<void>;
-    /**
-     * Override updateJobProgress to emit events via EventBus
-     */
-    protected updateJobProgress(job: AnyJob): Promise<void>;
-    protected handleJobFailure(job: AnyJob, error: any): Promise<void>;
-    private processHighlightDetectionJob;
-    private createHighlightAnnotation;
-}
-
-/**
- * Assessment Detection Worker
- *
- * Processes assessment-detection jobs: runs AI inference to assess/evaluate
- * passages in the text and creates assessment annotations.
- */
-
-type Agent$2 = components['schemas']['Agent'];
-declare class AssessmentAnnotationWorker extends JobWorker {
-    private inferenceClient;
-    private generator;
-    private eventBus;
-    private contentFetcher;
-    private isFirstProgress;
-    constructor(jobQueue: JobQueue, inferenceClient: InferenceClient, generator: Agent$2, eventBus: EventBus, contentFetcher: ContentFetcher, logger: Logger);
-    protected getWorkerName(): string;
-    protected canProcessJob(job: AnyJob): boolean;
-    protected executeJob(job: AnyJob): Promise<AssessmentDetectionResult>;
-    /**
-     * Emit completion event with result data
-     * Override base class to emit on EventBus
-     */
-    protected emitCompletionEvent(job: RunningJob<AssessmentDetectionParams, AssessmentDetectionProgress>, result: AssessmentDetectionResult): Promise<void>;
-    /**
-     * Override updateJobProgress to emit events via EventBus
-     */
-    protected updateJobProgress(job: AnyJob): Promise<void>;
-    protected handleJobFailure(job: AnyJob, error: any): Promise<void>;
-    private processAssessmentDetectionJob;
-    private createAssessmentAnnotation;
-}
-
-/**
- * Comment Detection Worker
- *
- * Processes comment-detection jobs: runs AI inference to identify passages
- * that would benefit from explanatory comments and creates comment annotations.
- */
-
-type Agent$1 = components['schemas']['Agent'];
-declare class CommentAnnotationWorker extends JobWorker {
-    private inferenceClient;
-    private generator;
-    private eventBus;
-    private contentFetcher;
-    private isFirstProgress;
-    constructor(jobQueue: JobQueue, inferenceClient: InferenceClient, generator: Agent$1, eventBus: EventBus, contentFetcher: ContentFetcher, logger: Logger);
-    protected getWorkerName(): string;
-    protected canProcessJob(job: AnyJob): boolean;
-    protected executeJob(job: AnyJob): Promise<CommentDetectionResult>;
-    /**
-     * Emit completion event with result data
-     * Override base class to emit on EventBus
-     */
-    protected emitCompletionEvent(job: RunningJob<CommentDetectionParams, CommentDetectionProgress>, result: CommentDetectionResult): Promise<void>;
-    /**
-     * Override updateJobProgress to emit events via EventBus
-     */
-    protected updateJobProgress(job: AnyJob): Promise<void>;
-    protected handleJobFailure(job: AnyJob, error: any): Promise<void>;
-    private processCommentDetectionJob;
-    private createCommentAnnotation;
-}
-
-/**
- * Tag Detection Worker
- *
- * Processes tag-detection jobs: runs AI inference to identify passages
- * serving specific structural roles (IRAC, IMRAD, Toulmin, etc.) and
- * creates tag annotations with dual-body structure.
+ * Progress callback. The three positional args satisfy the minimum
+ * `JobProgress` required fields (`percentage`, `message`, `stage`).
+ * The fourth optional arg carries job-type-specific fields
+ * (`currentEntityType`, `completedEntityTypes`, `requestParams`, etc.)
+ * that the progress UI renders.
  */
-
-type Agent = components['schemas']['Agent'];
-declare class TagAnnotationWorker extends JobWorker {
-    private inferenceClient;
-    private generator;
-    private eventBus;
-    private contentFetcher;
-    private isFirstProgress;
-    constructor(jobQueue: JobQueue, inferenceClient: InferenceClient, generator: Agent, eventBus: EventBus, contentFetcher: ContentFetcher, logger: Logger);
-    protected getWorkerName(): string;
-    protected canProcessJob(job: AnyJob): boolean;
-    protected executeJob(job: AnyJob): Promise<TagDetectionResult>;
-    /**
-     * Emit completion event with result data
-     * Override base class to emit on EventBus
-     */
-    protected emitCompletionEvent(job: RunningJob<TagDetectionParams, TagDetectionProgress>, result: TagDetectionResult): Promise<void>;
-    /**
-     * Override updateJobProgress to emit events via EventBus
-     */
-    protected updateJobProgress(job: AnyJob): Promise<void>;
-    protected handleJobFailure(job: AnyJob, error: any): Promise<void>;
-    private processTagDetectionJob;
-    private createTagAnnotation;
+type OnProgress = (percentage: number, message: string, stage: string, extra?: Partial<JobProgress>) => void;
+type JobProgress = components['schemas']['JobProgress'];
+interface ProcessorResult<R> {
+    annotations: Record<string, unknown>[];
+    result: R;
 }
+declare function processHighlightJob(content: string, inferenceClient: InferenceClient, params: HighlightDetectionParams, userId: string, generator: Agent, onProgress: OnProgress): Promise<ProcessorResult<HighlightDetectionResult>>;
+declare function processCommentJob(content: string, inferenceClient: InferenceClient, params: CommentDetectionParams, userId: string, generator: Agent, onProgress: OnProgress): Promise<ProcessorResult<CommentDetectionResult>>;
+declare function processAssessmentJob(content: string, inferenceClient: InferenceClient, params: AssessmentDetectionParams, userId: string, generator: Agent, onProgress: OnProgress): Promise<ProcessorResult<AssessmentDetectionResult>>;
+declare function processReferenceJob(content: string, inferenceClient: InferenceClient, params: DetectionParams, userId: string, generator: Agent, onProgress: OnProgress, logger?: _semiont_core.Logger): Promise<ProcessorResult<DetectionResult>>;
+declare function processTagJob(content: string, inferenceClient: InferenceClient, params: TagDetectionParams, userId: string, generator: Agent, onProgress: OnProgress): Promise<ProcessorResult<TagDetectionResult>>;
+declare function processGenerationJob(inferenceClient: InferenceClient, params: GenerationParams, onProgress: OnProgress): Promise<{
+    content: string;
+    title: string;
+    format: string;
+    result: GenerationResult;
+}>;
 
-/**
- * Response parsers for annotation detection motivations
- *
- * Provides static methods to parse and validate AI responses for each motivation type.
- * Includes offset validation and correction logic.
- * Extracted from worker implementations to centralize parsing logic.
- *
- * NOTE: These are static utility methods without logger access.
- * Console statements kept for debugging - consider adding logger parameter in future.
- */
 /**
  * Represents a detected comment with validated position
  */
@@ -735,4 +552,4 @@ declare function generateResourceFromTopic(topic: string, entityTypes: string[],
     content: string;
 }>;
 
-export { AnnotationDetection, type AnyJob, AssessmentAnnotationWorker, type AssessmentDetectionJob, type AssessmentDetectionParams, type AssessmentDetectionProgress, type AssessmentDetectionResult, type CancelledJob, CommentAnnotationWorker, type CommentDetectionJob, type CommentDetectionParams, type CommentDetectionProgress, type CommentDetectionResult, type CompleteJob, type ContentFetcher, type DetectionJob, type DetectionParams, type DetectionProgress, type DetectionResult, type FailedJob, type GenerationJob, type GenerationParams, type GenerationResult, GenerationWorker, HighlightAnnotationWorker, type HighlightDetectionJob, type HighlightDetectionParams, type HighlightDetectionProgress, type HighlightDetectionResult, type JobMetadata, type JobQueryFilters, JobQueue, type JobStatus, type JobType, JobWorker, type PendingJob, ReferenceAnnotationWorker, type RunningJob, TagAnnotationWorker, type TagDetectionJob, type TagDetectionParams, type TagDetectionProgress, type TagDetectionResult, type YieldProgress, generateResourceFromTopic, isCancelledJob, isCompleteJob, isFailedJob, isPendingJob, isRunningJob };
+export { AnnotationDetection, type AnyJob, type AssessmentDetectionJob, type AssessmentDetectionParams, type AssessmentDetectionProgress, type AssessmentDetectionResult, type CancelledJob, type CommentDetectionJob, type CommentDetectionParams, type CommentDetectionProgress, type CommentDetectionResult, type CompleteJob, type ContentFetcher, type DetectionJob, type DetectionParams, type DetectionProgress, type DetectionResult, type FailedJob, FsJobQueue, type GenerationJob, type GenerationParams, type GenerationResult, type HighlightDetectionJob, type HighlightDetectionParams, type HighlightDetectionProgress, type HighlightDetectionResult, type JobMetadata, type JobQueryFilters, type JobQueue, type JobStatus, type JobType, JobWorker, type OnProgress, type PendingJob, type ProcessorResult, type RunningJob, type TagDetectionJob, type TagDetectionParams, type TagDetectionProgress, type TagDetectionResult, type YieldProgress, generateResourceFromTopic, isCancelledJob, isCompleteJob, isFailedJob, isPendingJob, isRunningJob, processAssessmentJob, processCommentJob, processGenerationJob, processHighlightJob, processReferenceJob, processTagJob };