@semiont/jobs 0.5.4 → 0.5.6

package/dist/index.d.ts

@@ -1,21 +1,579 @@
+import { Readable } from 'stream';
+import { ResourceId, JobId, UserId, EntityType, AnnotationId, Annotation, GatheredContext, TagSchema, Logger, EventBus, components } from '@semiont/core';
+import { SemiontProject } from '@semiont/core/node';
+import { InferenceClient } from '@semiont/inference';
+
 /**
- * @semiont/jobs
- *
- * Job queue and worker infrastructure
- *
- * Provides:
- * - JobQueue interface: backing-store-agnostic contract
- * - FsJobQueue: Filesystem-backed implementation
- * - JobWorker: Abstract base class for job workers
- * - Job processors: Extracted functions for each job type
- * - Job types: All job type definitions
- */
-export type { JobQueue } from './job-queue-interface';
-export { FsJobQueue } from './fs-job-queue';
-export { JobWorker } from './job-worker';
-export type { JobType, JobStatus, JobMetadata, DetectionJob, GenerationJob, HighlightDetectionJob, AssessmentDetectionJob, CommentDetectionJob, TagDetectionJob, AnyJob, PendingJob, RunningJob, CompleteJob, FailedJob, CancelledJob, JobQueryFilters, DetectionParams, GenerationParams, HighlightDetectionParams, AssessmentDetectionParams, CommentDetectionParams, TagDetectionParams, DetectionProgress, YieldProgress, HighlightDetectionProgress, AssessmentDetectionProgress, CommentDetectionProgress, TagDetectionProgress, DetectionResult, GenerationResult, HighlightDetectionResult, AssessmentDetectionResult, CommentDetectionResult, TagDetectionResult, ContentFetcher, } from './types';
-export { isPendingJob, isRunningJob, isCompleteJob, isFailedJob, isCancelledJob, } from './types';
-export { processHighlightJob, processCommentJob, processAssessmentJob, processReferenceJob, processTagJob, processGenerationJob, type OnProgress, type ProcessorResult, } from './processors';
-export { AnnotationDetection } from './workers/annotation-detection';
-export { generateResourceFromTopic } from './workers/generation/resource-generation';
-//# sourceMappingURL=index.d.ts.map
+ * Job Queue Type Definitions - Discriminated Union Design
+ *
+ * Jobs represent async work that can be queued, processed, and monitored.
+ * Uses TypeScript discriminated unions to enforce valid state transitions.
+ *
+ * Design principles:
+ * - Each job status has specific valid fields
+ * - Type narrowing works automatically via status discriminant
+ * - No optional fields that may or may not exist
+ * - State machine is explicit and type-safe
+ */
+
+/**
+ * Content fetcher - turns a ResourceId into a readable stream.
+ * Workers use this to access resource content on demand.
+ * The implementation is provided by the backend at startup.
+ */
+type ContentFetcher = (resourceId: ResourceId) => Promise<Readable | null>;
+type JobType = 'reference-annotation' | 'generation' | 'highlight-annotation' | 'assessment-annotation' | 'comment-annotation' | 'tag-annotation';
+type JobStatus = 'pending' | 'running' | 'complete' | 'failed' | 'cancelled';
+/**
+ * Job metadata - common to all states
+ */
+interface JobMetadata {
+    id: JobId;
+    type: JobType;
+    userId: UserId;
+    userName: string;
+    userEmail: string;
+    userDomain: string;
+    created: string;
+    retryCount: number;
+    maxRetries: number;
+}
+/**
+ * Locale conventions for detection/generation params.
+ *
+ * Two independent locales flow through these jobs:
+ *
+ *   - `language` — *annotation body* locale. The BCP-47 tag the LLM should
+ *     write generated body text in (comment text, assessment text, generated
+ *     resource content, tag category label). Sourced from the user's UI
+ *     locale. Stamped onto the W3C `TextualBody.language` field.
+ *
+ *   - `sourceLanguage` — *source resource* locale. The BCP-47 tag of the
+ *     content being analyzed. Sourced from `ResourceDescriptor` (carried as
+ *     `Representation.language` on the primary representation). Used in
+ *     prompts so the LLM analyzes non-English source correctly even when
+ *     the user's UI locale differs.
+ *
+ * Examples: a German user analyzing an English document → `language='de'`,
+ * `sourceLanguage='en'`. An English user detecting entities in a French
+ * document → `language='en'` (unused for entity references), `sourceLanguage='fr'`.
+ */
+/**
+ * Detection job parameters
+ */
+interface DetectionParams {
+    resourceId: ResourceId;
+    entityTypes: EntityType[];
+    includeDescriptiveReferences?: boolean;
+    /** Annotation body locale — see locale conventions above. */
+    language?: string;
+    /** Source-resource locale — see locale conventions above. */
+    sourceLanguage?: string;
+}
+/**
+ * Generation job parameters
+ */
+interface GenerationParams {
+    referenceId: AnnotationId;
+    sourceResourceId: ResourceId;
+    sourceResourceName: string;
+    annotation: Annotation;
+    prompt?: string;
+    title?: string;
+    entityTypes?: EntityType[];
+    /** Annotation body locale — language the *generated resource* is written in. */
+    language?: string;
+    /**
+     * Source-resource locale — language of the resource being referenced.
+     * Used in the prompt so the LLM understands the embedded source-context
+     * snippet correctly when source ≠ target language.
+     */
+    sourceLanguage?: string;
+    context?: GatheredContext;
+    temperature?: number;
+    maxTokens?: number;
+    storageUri?: string;
+}
+/**
+ * Highlight detection job parameters
+ */
+interface HighlightDetectionParams {
+    resourceId: ResourceId;
+    instructions?: string;
+    density?: number;
+    /** Source-resource locale — see locale conventions above. */
+    sourceLanguage?: string;
+}
+/**
+ * Assessment detection job parameters
+ */
+interface AssessmentDetectionParams {
+    resourceId: ResourceId;
+    instructions?: string;
+    tone?: 'analytical' | 'critical' | 'balanced' | 'constructive';
+    density?: number;
+    /** Annotation body locale — see locale conventions above. */
+    language?: string;
+    /** Source-resource locale — see locale conventions above. */
+    sourceLanguage?: string;
+}
+/**
+ * Comment detection job parameters
+ */
+interface CommentDetectionParams {
+    resourceId: ResourceId;
+    instructions?: string;
+    tone?: 'scholarly' | 'explanatory' | 'conversational' | 'technical';
+    density?: number;
+    /** Annotation body locale — see locale conventions above. */
+    language?: string;
+    /** Source-resource locale — see locale conventions above. */
+    sourceLanguage?: string;
+}
+/**
+ * Tag detection job parameters.
+ *
+ * Carries the *full* `TagSchema` (not just an id). The dispatcher resolves
+ * the caller-supplied `schemaId` against the per-KB tag-schema projection
+ * at job-creation time and embeds the resolved schema here, keeping the
+ * worker independent of the registry.
+ */
+interface TagDetectionParams {
+    resourceId: ResourceId;
+    schema: TagSchema;
+    categories: string[];
+    /** Annotation body locale — see locale conventions above. */
+    language?: string;
+    /** Source-resource locale — see locale conventions above. */
+    sourceLanguage?: string;
+}
+/**
+ * Detection job progress
+ */
+interface DetectionProgress {
+    totalEntityTypes: number;
+    processedEntityTypes: number;
+    currentEntityType?: string;
+    entitiesFound: number;
+    entitiesEmitted: number;
+}
+/**
+ * Detection job result
+ */
+interface DetectionResult {
+    totalFound: number;
+    totalEmitted: number;
+    errors: number;
+}
+/**
+ * Generation job progress
+ */
+interface YieldProgress {
+    stage: 'fetching' | 'generating' | 'creating' | 'linking';
+    percentage: number;
+    message?: string;
+}
+/**
+ * Generation job result
+ */
+interface GenerationResult {
+    resourceId: ResourceId;
+    resourceName: string;
+}
+/**
+ * Highlight detection job progress
+ */
+interface HighlightDetectionProgress {
+    stage: 'analyzing' | 'creating';
+    percentage: number;
+    message?: string;
+}
+/**
+ * Highlight detection job result
+ */
+interface HighlightDetectionResult {
+    highlightsFound: number;
+    highlightsCreated: number;
+}
+/**
+ * Assessment detection job progress
+ */
+interface AssessmentDetectionProgress {
+    stage: 'analyzing' | 'creating';
+    percentage: number;
+    message?: string;
+}
+/**
+ * Assessment detection job result
+ */
+interface AssessmentDetectionResult {
+    assessmentsFound: number;
+    assessmentsCreated: number;
+}
+/**
+ * Comment detection job progress
+ */
+interface CommentDetectionProgress {
+    stage: 'analyzing' | 'creating';
+    percentage: number;
+    message?: string;
+}
+/**
+ * Comment detection job result
+ */
+interface CommentDetectionResult {
+    commentsFound: number;
+    commentsCreated: number;
+}
+/**
+ * Tag detection job progress
+ */
+interface TagDetectionProgress {
+    stage: 'analyzing' | 'creating';
+    percentage: number;
+    currentCategory?: string;
+    processedCategories: number;
+    totalCategories: number;
+    message?: string;
+}
+/**
+ * Tag detection job result
+ */
+interface TagDetectionResult {
+    tagsFound: number;
+    tagsCreated: number;
+    byCategory: Record<string, number>;
+}
+/**
+ * Pending job - just created, waiting to be picked up
+ */
+interface PendingJob<P> {
+    status: 'pending';
+    metadata: JobMetadata;
+    params: P;
+}
+/**
+ * Running job - actively being processed
+ */
+interface RunningJob<P, PG> {
+    status: 'running';
+    metadata: JobMetadata;
+    params: P;
+    startedAt: string;
+    progress: PG;
+}
+/**
+ * Complete job - successfully finished
+ */
+interface CompleteJob<P, R> {
+    status: 'complete';
+    metadata: JobMetadata;
+    params: P;
+    startedAt: string;
+    completedAt: string;
+    result: R;
+}
+/**
+ * Failed job - encountered an error
+ */
+interface FailedJob<P> {
+    status: 'failed';
+    metadata: JobMetadata;
+    params: P;
+    startedAt?: string;
+    completedAt: string;
+    error: string;
+}
+/**
+ * Cancelled job - stopped by user
+ */
+interface CancelledJob<P> {
+    status: 'cancelled';
+    metadata: JobMetadata;
+    params: P;
+    startedAt?: string;
+    completedAt: string;
+}
+/**
+ * Generic job - discriminated union of all states
+ */
+type Job<P, PG, R> = PendingJob<P> | RunningJob<P, PG> | CompleteJob<P, R> | FailedJob<P> | CancelledJob<P>;
+type DetectionJob = Job<DetectionParams, DetectionProgress, DetectionResult>;
+type GenerationJob = Job<GenerationParams, YieldProgress, GenerationResult>;
+type HighlightDetectionJob = Job<HighlightDetectionParams, HighlightDetectionProgress, HighlightDetectionResult>;
+type AssessmentDetectionJob = Job<AssessmentDetectionParams, AssessmentDetectionProgress, AssessmentDetectionResult>;
+type CommentDetectionJob = Job<CommentDetectionParams, CommentDetectionProgress, CommentDetectionResult>;
+type TagDetectionJob = Job<TagDetectionParams, TagDetectionProgress, TagDetectionResult>;
+/**
+ * Discriminated union of all job types
+ */
+type AnyJob = DetectionJob | GenerationJob | HighlightDetectionJob | AssessmentDetectionJob | CommentDetectionJob | TagDetectionJob;
+declare function isPendingJob(job: AnyJob): job is PendingJob<any>;
+declare function isRunningJob(job: AnyJob): job is RunningJob<any, any>;
+declare function isCompleteJob(job: AnyJob): job is CompleteJob<any, any>;
+declare function isFailedJob(job: AnyJob): job is FailedJob<any>;
+declare function isCancelledJob(job: AnyJob): job is CancelledJob<any>;
+interface JobQueryFilters {
+    status?: JobStatus;
+    type?: JobType;
+    userId?: UserId;
+    limit?: number;
+    offset?: number;
+}
+
+interface JobQueue {
+    initialize(): Promise<void>;
+    destroy(): void;
+    createJob(job: AnyJob): Promise<void>;
+    getJob(jobId: JobId): Promise<AnyJob | null>;
+    updateJob(job: AnyJob, oldStatus?: JobStatus): Promise<void>;
+    pollNextPendingJob(predicate?: (job: AnyJob) => boolean): Promise<AnyJob | null>;
+    cancelJob(jobId: JobId): Promise<boolean>;
+    getStats(): Promise<{
+        pending: number;
+        running: number;
+        complete: number;
+        failed: number;
+        cancelled: number;
+    }>;
+}
+
+/**
+ * Job Queue Manager
+ *
+ * Filesystem-based job queue with atomic operations.
+ * Jobs are stored in directories by status for easy polling.
+ */
+
+declare class FsJobQueue implements JobQueue {
+    private eventBus?;
+    private jobsDir;
+    private logger;
+    private pendingQueue;
+    private watcher;
+    private loadDebounceTimer;
+    constructor(project: SemiontProject, logger: Logger, eventBus?: EventBus | undefined);
+    /**
+     * Initialize job queue directories, load pending jobs, and start fs.watch
+     */
+    initialize(): Promise<void>;
+    /**
+     * Clean up watcher
+     */
+    destroy(): void;
+    /**
+     * Load pending jobs from disk into in-memory queue
+     */
+    private loadPendingJobs;
+    /**
+     * Debounced version of loadPendingJobs — fs.watch can fire rapidly
+     */
+    private debouncedLoadPendingJobs;
+    /**
+     * Create a new job
+     */
+    createJob(job: AnyJob): Promise<void>;
+    /**
+     * Get a job by ID (searches all status directories)
+     */
+    getJob(jobId: JobId): Promise<AnyJob | null>;
+    /**
+     * Update a job (atomic: delete old, write new)
+     */
+    updateJob(job: AnyJob, oldStatus?: JobStatus): Promise<void>;
+    /**
+     * Poll for next pending job (FIFO) from in-memory queue.
+     * If a predicate is provided, returns the first matching job (skipping non-matching ones).
+     */
+    pollNextPendingJob(predicate?: (job: AnyJob) => boolean): Promise<AnyJob | null>;
+    /**
+     * List jobs with filters
+     */
+    listJobs(filters?: JobQueryFilters): Promise<AnyJob[]>;
+    /**
+     * Cancel a job
+     */
+    cancelJob(jobId: JobId): Promise<boolean>;
+    /**
+     * Clean up old completed/failed jobs (older than retention period)
+     */
+    cleanupOldJobs(retentionHours?: number): Promise<number>;
+    /**
+     * Get job file path
+     */
+    private getJobPath;
+    /**
+     * Get statistics about the queue
+     */
+    getStats(): Promise<{
+        pending: number;
+        running: number;
+        complete: number;
+        failed: number;
+        cancelled: number;
+    }>;
+}
+
+/**
+ * Job Processors
+ *
+ * Pure functions that take content + inference client + params,
+ * report progress via callback, and return annotations + results.
+ *
+ * No EventBus, no JobQueue, no side effects except calling inference.
+ * Driven by the remote worker process (worker-process.ts), which claims
+ * jobs over SSE and dispatches by jobType to these functions.
+ */
+
+type Agent = components['schemas']['Agent'];
+/**
+ * Progress callback. The three positional args satisfy the minimum
+ * `JobProgress` required fields (`percentage`, `message`, `stage`).
+ * The fourth optional arg carries job-type-specific fields
+ * (`currentEntityType`, `completedEntityTypes`, `requestParams`, etc.)
+ * that the progress UI renders.
+ */
+type OnProgress = (percentage: number, message: string, stage: string, extra?: Partial<JobProgress>) => void;
+type JobProgress = components['schemas']['JobProgress'];
+interface ProcessorResult<R> {
+    annotations: Record<string, unknown>[];
+    result: R;
+}
+declare function processHighlightJob(content: string, inferenceClient: InferenceClient, params: HighlightDetectionParams, userId: string, generator: Agent, onProgress: OnProgress): Promise<ProcessorResult<HighlightDetectionResult>>;
+declare function processCommentJob(content: string, inferenceClient: InferenceClient, params: CommentDetectionParams, userId: string, generator: Agent, onProgress: OnProgress): Promise<ProcessorResult<CommentDetectionResult>>;
+declare function processAssessmentJob(content: string, inferenceClient: InferenceClient, params: AssessmentDetectionParams, userId: string, generator: Agent, onProgress: OnProgress): Promise<ProcessorResult<AssessmentDetectionResult>>;
+declare function processReferenceJob(content: string, inferenceClient: InferenceClient, params: DetectionParams, userId: string, generator: Agent, onProgress: OnProgress, logger: Logger): Promise<ProcessorResult<DetectionResult>>;
+declare function processTagJob(content: string, inferenceClient: InferenceClient, params: TagDetectionParams, userId: string, generator: Agent, onProgress: OnProgress): Promise<ProcessorResult<TagDetectionResult>>;
+declare function processGenerationJob(inferenceClient: InferenceClient, params: GenerationParams, onProgress: OnProgress, logger: Logger): Promise<{
+    content: string;
+    title: string;
+    format: string;
+    result: GenerationResult;
+}>;
+
+/**
+ * Represents a detected comment with validated position
+ */
+interface CommentMatch {
+    exact: string;
+    start: number;
+    end: number;
+    prefix?: string;
+    suffix?: string;
+    comment: string;
+}
+/**
+ * Represents a detected highlight with validated position
+ */
+interface HighlightMatch {
+    exact: string;
+    start: number;
+    end: number;
+    prefix?: string;
+    suffix?: string;
+}
+/**
+ * Represents a detected assessment with validated position
+ */
+interface AssessmentMatch {
+    exact: string;
+    start: number;
+    end: number;
+    prefix?: string;
+    suffix?: string;
+    assessment: string;
+}
+/**
+ * Represents a detected tag with validated position
+ */
+interface TagMatch {
+    exact: string;
+    start: number;
+    end: number;
+    prefix?: string;
+    suffix?: string;
+    category: string;
+}
+
+/**
+ * Annotation Detection
+ *
+ * Orchestrates the full annotation detection pipeline:
+ * 1. Build AI prompts using MotivationPrompts
+ * 2. Call AI inference
+ * 3. Parse and validate results using MotivationParsers
+ *
+ * All methods take content as a string parameter.
+ * Workers are responsible for fetching content via ContentFetcher.
+ */
+
+declare class AnnotationDetection {
+    /**
+     * Fetch content from a ContentFetcher and read the stream to a string.
+     * Shared helper for all workers.
+     */
+    static fetchContent(contentFetcher: ContentFetcher, resourceId: ResourceId): Promise<string>;
+    /**
+     * Detect comments in content.
+     *
+     * `language` is the locale the LLM should write comment text in (annotation
+     * body locale). `sourceLanguage` is the locale of the content being analyzed
+     * (source-resource locale). See `types.ts` "Locale conventions" for the
+     * full discussion.
+     */
+    static detectComments(content: string, client: InferenceClient, instructions?: string, tone?: string, density?: number, language?: string, sourceLanguage?: string): Promise<CommentMatch[]>;
+    /**
+     * Detect highlights in content.
+     *
+     * Highlights have no body — only `sourceLanguage` (source-resource locale)
+     * applies, used in the prompt so the LLM analyzes non-English source
+     * correctly.
+     */
+    static detectHighlights(content: string, client: InferenceClient, instructions?: string, density?: number, sourceLanguage?: string): Promise<HighlightMatch[]>;
+    /**
+     * Detect assessments in content.
+     *
+     * `language` is the locale the LLM should write assessment text in
+     * (annotation body locale). `sourceLanguage` is the locale of the content
+     * being analyzed (source-resource locale).
+     */
+    static detectAssessments(content: string, client: InferenceClient, instructions?: string, tone?: string, density?: number, language?: string, sourceLanguage?: string): Promise<AssessmentMatch[]>;
+    /**
+     * Detect tags in content for a specific category.
+     *
+     * The full `TagSchema` is supplied by the dispatcher (resolved against
+     * the per-KB tag-schema projection at job-creation time) so the worker
+     * is independent of the registry.
+     *
+     * `sourceLanguage` is the locale of the content being analyzed. Body-locale
+     * (`language`) doesn't influence the tag prompt — categories are schema
+     * identifiers, not LLM-generated text — so it's consumed at the body-stamp
+     * site, not here.
+     */
+    static detectTags(content: string, client: InferenceClient, schema: TagSchema, category: string, sourceLanguage?: string): Promise<TagMatch[]>;
+}
+
+/**
+ * Resource Generation
+ *
+ * Generates markdown resources from topics using AI inference.
+ */
+
+/**
+ * Generate resource content using inference.
+ *
+ * Locale parameters: `locale` is the *body* locale — the language the
+ * generated resource should be written in (sourced from the user's UI
+ * locale). `sourceLanguage` is the *source* locale — the language of the
+ * referenced resource whose context (selected passage, surrounding text)
+ * is embedded into the prompt. They're independent: a German user can
+ * generate German content from an English source resource. See
+ * `types.ts` "Locale conventions" for the full discussion.
+ */
+declare function generateResourceFromTopic(topic: string, entityTypes: string[], client: InferenceClient, logger: Logger, userPrompt?: string, locale?: string, context?: GatheredContext, temperature?: number, maxTokens?: number, sourceLanguage?: string): Promise<{
+    title: string;
+    content: string;
+}>;
+
+export { AnnotationDetection, FsJobQueue, generateResourceFromTopic, isCancelledJob, isCompleteJob, isFailedJob, isPendingJob, isRunningJob, processAssessmentJob, processCommentJob, processGenerationJob, processHighlightJob, processReferenceJob, processTagJob };
+export type { AnyJob, AssessmentDetectionJob, AssessmentDetectionParams, AssessmentDetectionProgress, AssessmentDetectionResult, CancelledJob, CommentDetectionJob, CommentDetectionParams, CommentDetectionProgress, CommentDetectionResult, CompleteJob, ContentFetcher, DetectionJob, DetectionParams, DetectionProgress, DetectionResult, FailedJob, GenerationJob, GenerationParams, GenerationResult, HighlightDetectionJob, HighlightDetectionParams, HighlightDetectionProgress, HighlightDetectionResult, JobMetadata, JobQueryFilters, JobQueue, JobStatus, JobType, OnProgress, PendingJob, ProcessorResult, RunningJob, TagDetectionJob, TagDetectionParams, TagDetectionProgress, TagDetectionResult, YieldProgress };