@semiont/jobs 0.5.6 → 0.5.8

package/README.md

@@ -20,10 +20,11 @@ npm install @semiont/jobs
 
 **Dependencies:**
 - `@semiont/core` — Core types, `SemiontProject`, EventBus
-- `@semiont/sdk` — `SemiontSession`, `JobClaimAdapter` (worker process)
-- `@semiont/api-client` — HTTP transport, OpenAPI types
+- `@semiont/sdk` — `SemiontSession`, `WorkerBus` (worker process)
+- `@semiont/http-transport` — HTTP transport, OpenAPI types
 - `@semiont/inference` — InferenceClient for AI operations
 - `@semiont/content` — Content storage URI derivation
+- `@semiont/event-sourcing` — Annotation id generation
 - `@semiont/observability` — Spans and job-outcome metrics
 
 ## Quick Start
@@ -88,16 +89,16 @@ interface JobMetadata {
   id: JobId;
   type: JobType;
   userId: UserId;
-  userName: string;       // For building W3C Agent creator
-  userEmail: string;      // For building W3C Agent creator
-  userDomain: string;     // For building W3C Agent creator
+  userName: string;       // Audit-only snapshot of the requesting user
+  userEmail: string;      // Audit-only snapshot of the requesting user
+  userDomain: string;     // Audit-only snapshot of the requesting user
   created: string;
   retryCount: number;
   maxRetries: number;
 }
 ```
 
-The `userName`, `userEmail`, and `userDomain` fields are used by workers to build the W3C `Agent` for annotation `creator` attribution via `userToAgent()`.
+The `userName`, `userEmail`, and `userDomain` fields are an audit-only snapshot of the requesting user, persisted in the on-disk job file. Workers derive annotation `creator` attribution from `userId` via `didToAgent()`.
 
 ## Annotation Workers
 
@@ -112,9 +113,9 @@ The worker process (`worker-main.ts` → `startWorkerProcess` in `worker-process
 | `comment-annotation` | `processCommentJob` |
 | `tag-annotation` | `processTagJob` |
 
-Detection logic lives in the `AnnotationDetection` class (`src/workers/annotation-detection.ts`); generation synthesis in `generateResourceFromTopic()` (`src/workers/generation/resource-generation.ts`). Each processor fetches content via `session.client.browse.resourceContent(resourceId)`.
+Detection logic lives in the `AnnotationDetection` class (`src/workers/annotation-detection.ts`); generation synthesis in `generateResourceFromTopic()` (`src/workers/generation/resource-generation.ts`). Processors never fetch content themselves — the worker process fetches it via `session.client.browse.resourceContent(resourceId)` and passes it in.
 
-Workers emit bus events via `session.client.transport.emit('mark:create' | 'job:start' | 'job:report-progress' | 'job:complete' | 'job:fail', payload)` — the Stower actor in @semiont/make-meaning handles persistence.
+Workers emit bus events via `session.client.transport.emit('mark:create' | 'job:start' | 'job:report-progress' | 'job:complete' | 'job:fail', payload)` — the Stower actor in @semiont/make-meaning handles persistence to the event log, and the job command handlers mirror the same events into the queue files (completion, retry-on-failure with `maxRetries`, progress-as-heartbeat).
 
 ## Adding a Job Type
 
@@ -133,12 +134,12 @@ Jobs use TypeScript discriminated unions for type safety:
 ```typescript
 function handleJob(job: AnyJob) {
   if (job.status === 'running') {
-    console.log(job:progress);    // Available
+    console.log(job.progress);    // Available
     // console.log(job.result);   // Compile error
   }
   if (job.status === 'complete') {
     console.log(job.result);      // Available
-    // console.log(job:progress); // Compile error
+    // console.log(job.progress); // Compile error
   }
 }
 ```
@@ -148,7 +149,7 @@ function handleJob(job: AnyJob) {
 Jobs are stored as individual JSON files organized by status:
 
 ```
-data/jobs/
+{project.jobsDir}/
   pending/job-abc123.json
   running/job-def456.json
   complete/job-ghi789.json
@@ -172,7 +173,7 @@ Apache-2.0
 ## Related Packages
 
 - [`@semiont/core`](../core/) — Domain types, `SemiontProject`, EventBus
-- [`@semiont/sdk`](../sdk/) — `SemiontSession`, `JobClaimAdapter`
-- [`@semiont/api-client`](../api-client/) — HTTP transport, OpenAPI types
+- [`@semiont/sdk`](../sdk/) — `SemiontSession`, `WorkerBus`
+- [`@semiont/http-transport`](../http-transport/) — HTTP transport, OpenAPI types
 - [`@semiont/inference`](../inference/) — AI inference client
 - [`@semiont/make-meaning`](../make-meaning/) — Actor model, Knowledge Base, service orchestration

package/dist/index.d.ts

@@ -1,5 +1,4 @@
-import { Readable } from 'stream';
-import { ResourceId, JobId, UserId, EntityType, AnnotationId, Annotation, GatheredContext, TagSchema, Logger, EventBus, components } from '@semiont/core';
+import { JobId, UserId, ResourceId, EntityType, AnnotationId, Annotation, GatheredContext, TagSchema, Logger, EventBus, components, SupportedMediaType } from '@semiont/core';
 import { SemiontProject } from '@semiont/core/node';
 import { InferenceClient } from '@semiont/inference';
 
@@ -16,12 +15,6 @@ import { InferenceClient } from '@semiont/inference';
  * - State machine is explicit and type-safe
  */
 
-/**
- * Content fetcher - turns a ResourceId into a readable stream.
- * Workers use this to access resource content on demand.
- * The implementation is provided by the backend at startup.
- */
-type ContentFetcher = (resourceId: ResourceId) => Promise<Readable | null>;
 type JobType = 'reference-annotation' | 'generation' | 'highlight-annotation' | 'assessment-annotation' | 'comment-annotation' | 'tag-annotation';
 type JobStatus = 'pending' | 'running' | 'complete' | 'failed' | 'cancelled';
 /**
@@ -31,6 +24,13 @@ interface JobMetadata {
     id: JobId;
     type: JobType;
     userId: UserId;
+    /**
+     * Audit-only snapshot of the requesting user (with `userEmail` and
+     * `userDomain` below), stamped at job creation and persisted in the
+     * on-disk job file. No code path reads these back — annotation
+     * `creator` attribution is derived from `userId` via `didToAgent()`.
+     * Kept intentionally so job files are self-describing to a human.
+     */
     userName: string;
     userEmail: string;
     userDomain: string;
@@ -327,7 +327,22 @@ interface JobQueue {
     createJob(job: AnyJob): Promise<void>;
     getJob(jobId: JobId): Promise<AnyJob | null>;
     updateJob(job: AnyJob, oldStatus?: JobStatus): Promise<void>;
-    pollNextPendingJob(predicate?: (job: AnyJob) => boolean): Promise<AnyJob | null>;
+    /** Move a running job to `complete`. Returns false if the job isn't running. */
+    completeJob(jobId: JobId, result: Record<string, unknown>): Promise<boolean>;
+    /**
+     * Move a running job back to `pending` (retry, re-announced) while
+     * `retryCount < maxRetries`, else to `failed`. Returns what happened,
+     * or null if the job isn't running.
+     */
+    failJob(jobId: JobId, error: string): Promise<'retried' | 'failed' | null>;
+    /** Write progress into a running job's file (throttled, best-effort). */
+    recordProgress(jobId: JobId, progress: Record<string, unknown>): Promise<void>;
+    /**
+     * Cancel all pending jobs in a category — 'generation' is the
+     * `generation` type; 'annotation' is every `*-annotation` type.
+     * Running jobs are left to finish. Returns the number cancelled.
+     */
+    cancelPendingJobs(category: 'annotation' | 'generation'): Promise<number>;
     cancelJob(jobId: JobId): Promise<boolean>;
     getStats(): Promise<{
         pending: number;
@@ -342,33 +357,40 @@ interface JobQueue {
  * Job Queue Manager
  *
  * Filesystem-based job queue with atomic operations.
- * Jobs are stored in directories by status for easy polling.
+ * Jobs are stored in directories by status; status transitions are
+ * atomic delete + write across directories.
  */
 
 declare class FsJobQueue implements JobQueue {
     private eventBus?;
     private jobsDir;
     private logger;
-    private pendingQueue;
-    private watcher;
-    private loadDebounceTimer;
+    private reannounceTimer;
+    private cleanupTimer;
+    /** Per-job timestamp of the last progress write, for throttling. */
+    private lastProgressWrite;
     constructor(project: SemiontProject, logger: Logger, eventBus?: EventBus | undefined);
     /**
-     * Initialize job queue directories, load pending jobs, and start fs.watch
+     * Initialize job queue directories, announce any pending backlog,
+     * and start the re-announce interval. Idempotent.
      */
     initialize(): Promise<void>;
     /**
-     * Clean up watcher
+     * Stop the re-announce and retention intervals
      */
     destroy(): void;
     /**
-     * Load pending jobs from disk into in-memory queue
+     * Emit `job:queued` for a pending job, if an EventBus is wired and
+     * the job carries a `resourceId` (every current job type does).
      */
-    private loadPendingJobs;
+    private announce;
     /**
-     * Debounced version of loadPendingJobs — fs.watch can fire rapidly
+     * Announce every job currently in `pending/`. Files that vanish or
+     * fail to parse mid-scan (claimed, cancelled, partially written)
+     * are skipped — they're either gone for a good reason or picked up
+     * on the next tick.
      */
-    private debouncedLoadPendingJobs;
+    private announcePendingJobs;
     /**
      * Create a new job
      */
@@ -382,10 +404,25 @@ declare class FsJobQueue implements JobQueue {
      */
     updateJob(job: AnyJob, oldStatus?: JobStatus): Promise<void>;
     /**
-     * Poll for next pending job (FIFO) from in-memory queue.
-     * If a predicate is provided, returns the first matching job (skipping non-matching ones).
+     * Move a running job to `complete`. Returns false (and changes
+     * nothing) if the job is missing or not running — which also makes
+     * duplicate `job:complete` events harmless.
+     */
+    completeJob(jobId: JobId, result: Record<string, unknown>): Promise<boolean>;
+    /**
+     * Retry-or-fail a running job. While `retryCount < maxRetries` the
+     * job goes back to `pending` with the count bumped (and is
+     * re-announced); after that it lands in `failed` with the error.
+     * Returns null (and changes nothing) if the job isn't running.
+     */
+    failJob(jobId: JobId, error: string): Promise<'retried' | 'failed' | null>;
+    /**
+     * Write progress into a running job's file. Throttled per job, and
+     * a no-op for jobs that aren't running. Beyond surfacing live
+     * progress to `job:status-requested`, each write refreshes the
+     * file's mtime — the heartbeat `recoverStaleRunningJobs` watches.
      */
-    pollNextPendingJob(predicate?: (job: AnyJob) => boolean): Promise<AnyJob | null>;
+    recordProgress(jobId: JobId, progress: Record<string, unknown>): Promise<void>;
     /**
      * List jobs with filters
      */
@@ -394,6 +431,21 @@ declare class FsJobQueue implements JobQueue {
      * Cancel a job
      */
     cancelJob(jobId: JobId): Promise<boolean>;
+    /**
+     * Cancel all pending jobs in a category — the granularity of the
+     * `job:cancel-requested` UI signal. Running jobs are left to finish:
+     * interrupting a worker mid-inference would need a worker-side kill
+     * channel that doesn't exist.
+     */
+    cancelPendingJobs(category: 'annotation' | 'generation'): Promise<number>;
+    /**
+     * Recover running jobs orphaned by a dead worker: any `running/`
+     * file whose mtime is older than the stale window is fed through
+     * the same retry-or-fail path as `job:fail`. Progress writes
+     * refresh the mtime, so a live worker is never recovered out from
+     * under itself as long as it reports within the window.
+     */
+    recoverStaleRunningJobs(): Promise<number>;
     /**
      * Clean up old completed/failed jobs (older than retention period)
      */
@@ -447,7 +499,7 @@ declare function processTagJob(content: string, inferenceClient: InferenceClient
 declare function processGenerationJob(inferenceClient: InferenceClient, params: GenerationParams, onProgress: OnProgress, logger: Logger): Promise<{
     content: string;
     title: string;
-    format: string;
+    format: SupportedMediaType;
     result: GenerationResult;
 }>;
 
@@ -503,16 +555,11 @@ interface TagMatch {
  * 2. Call AI inference
  * 3. Parse and validate results using MotivationParsers
  *
- * All methods take content as a string parameter.
- * Workers are responsible for fetching content via ContentFetcher.
+ * All methods take content as a string parameter — the worker process
+ * fetches it and hands it in.
  */
 
 declare class AnnotationDetection {
-    /**
-     * Fetch content from a ContentFetcher and read the stream to a string.
-     * Shared helper for all workers.
-     */
-    static fetchContent(contentFetcher: ContentFetcher, resourceId: ResourceId): Promise<string>;
     /**
      * Detect comments in content.
      *
@@ -576,4 +623,4 @@ declare function generateResourceFromTopic(topic: string, entityTypes: string[],
 }>;
 
 export { AnnotationDetection, FsJobQueue, generateResourceFromTopic, isCancelledJob, isCompleteJob, isFailedJob, isPendingJob, isRunningJob, processAssessmentJob, processCommentJob, processGenerationJob, processHighlightJob, processReferenceJob, processTagJob };
-export type { AnyJob, AssessmentDetectionJob, AssessmentDetectionParams, AssessmentDetectionProgress, AssessmentDetectionResult, CancelledJob, CommentDetectionJob, CommentDetectionParams, CommentDetectionProgress, CommentDetectionResult, CompleteJob, ContentFetcher, DetectionJob, DetectionParams, DetectionProgress, DetectionResult, FailedJob, GenerationJob, GenerationParams, GenerationResult, HighlightDetectionJob, HighlightDetectionParams, HighlightDetectionProgress, HighlightDetectionResult, JobMetadata, JobQueryFilters, JobQueue, JobStatus, JobType, OnProgress, PendingJob, ProcessorResult, RunningJob, TagDetectionJob, TagDetectionParams, TagDetectionProgress, TagDetectionResult, YieldProgress };
+export type { AnyJob, AssessmentDetectionJob, AssessmentDetectionParams, AssessmentDetectionProgress, AssessmentDetectionResult, CancelledJob, CommentDetectionJob, CommentDetectionParams, CommentDetectionProgress, CommentDetectionResult, CompleteJob, DetectionJob, DetectionParams, DetectionProgress, DetectionResult, FailedJob, GenerationJob, GenerationParams, GenerationResult, HighlightDetectionJob, HighlightDetectionParams, HighlightDetectionProgress, HighlightDetectionResult, JobMetadata, JobQueryFilters, JobQueue, JobStatus, JobType, OnProgress, PendingJob, ProcessorResult, RunningJob, TagDetectionJob, TagDetectionParams, TagDetectionProgress, TagDetectionResult, YieldProgress };

package/dist/index.js

@@ -1,9 +1,14 @@
-import { promises, watch } from 'fs';
+import { promises } from 'fs';
 import * as path from 'path';
-import { reconcileSelector, getLocaleEnglishName, didToAgent } from '@semiont/core';
+import { jobId, reconcileSelector, getLocaleEnglishName, didToAgent } from '@semiont/core';
 import { generateAnnotationId } from '@semiont/event-sourcing';
 
 // src/fs-job-queue.ts
+var REANNOUNCE_INTERVAL_MS = 3e4;
+var STALE_RUNNING_MS = 30 * 6e4;
+var PROGRESS_WRITE_MIN_INTERVAL_MS = 5e3;
+var RETENTION_HOURS = 24;
+var CLEANUP_INTERVAL_MS = 36e5;
 var FsJobQueue = class {
   constructor(project, logger, eventBus) {
     this.eventBus = eventBus;
@@ -13,12 +18,13 @@ var FsJobQueue = class {
   eventBus;
   jobsDir;
   logger;
-  // In-memory pending queue: avoids fs.readdir() on every poll (6×/sec with 6 workers)
-  pendingQueue = [];
-  watcher = null;
-  loadDebounceTimer = null;
+  reannounceTimer = null;
+  cleanupTimer = null;
+  /** Per-job timestamp of the last progress write, for throttling. */
+  lastProgressWrite = /* @__PURE__ */ new Map();
   /**
-   * Initialize job queue directories, load pending jobs, and start fs.watch
+   * Initialize job queue directories, announce any pending backlog,
+   * and start the re-announce interval. Idempotent.
    */
   async initialize() {
     const statuses = ["pending", "running", "complete", "failed", "cancelled"];
@@ -26,62 +32,83 @@ var FsJobQueue = class {
       const dir = path.join(this.jobsDir, status);
       await promises.mkdir(dir, { recursive: true });
     }
-    await this.loadPendingJobs();
-    const pendingDir = path.join(this.jobsDir, "pending");
-    try {
-      this.watcher = watch(pendingDir, () => {
-        this.debouncedLoadPendingJobs();
-      });
-    } catch (error) {
-      this.logger.warn("Failed to watch pending directory", {
-        error: error instanceof Error ? error.message : String(error)
-      });
+    if (this.eventBus && !this.reannounceTimer) {
+      await this.announcePendingJobs();
+      this.reannounceTimer = setInterval(() => {
+        this.announcePendingJobs().catch((error) => {
+          this.logger.warn("Pending-job re-announce failed", {
+            error: error instanceof Error ? error.message : String(error)
+          });
+        });
+        this.recoverStaleRunningJobs().catch((error) => {
+          this.logger.warn("Stale-running recovery failed", {
+            error: error instanceof Error ? error.message : String(error)
+          });
+        });
+      }, REANNOUNCE_INTERVAL_MS);
+      this.reannounceTimer.unref?.();
+    }
+    if (!this.cleanupTimer) {
+      this.cleanupTimer = setInterval(() => {
+        this.cleanupOldJobs(RETENTION_HOURS).catch((error) => {
+          this.logger.warn("Job retention cleanup failed", {
+            error: error instanceof Error ? error.message : String(error)
+          });
+        });
+      }, CLEANUP_INTERVAL_MS);
+      this.cleanupTimer.unref?.();
     }
     this.logger.info("Job queue initialized");
   }
   /**
-   * Clean up watcher
+   * Stop the re-announce and retention intervals
    */
   destroy() {
-    if (this.watcher) {
-      this.watcher.close();
-      this.watcher = null;
+    if (this.reannounceTimer) {
+      clearInterval(this.reannounceTimer);
+      this.reannounceTimer = null;
     }
-    if (this.loadDebounceTimer) {
-      clearTimeout(this.loadDebounceTimer);
-      this.loadDebounceTimer = null;
+    if (this.cleanupTimer) {
+      clearInterval(this.cleanupTimer);
+      this.cleanupTimer = null;
     }
   }
   /**
-   * Load pending jobs from disk into in-memory queue
+   * Emit `job:queued` for a pending job, if an EventBus is wired and
+   * the job carries a `resourceId` (every current job type does).
    */
-  async loadPendingJobs() {
-    const pendingDir = path.join(this.jobsDir, "pending");
-    try {
-      const files = await promises.readdir(pendingDir);
-      files.sort();
-      const jobs = [];
-      for (const file of files) {
-        try {
-          const content = await promises.readFile(path.join(pendingDir, file), "utf-8");
-          jobs.push(JSON.parse(content));
-        } catch {
-        }
-      }
-      this.pendingQueue = jobs;
-    } catch {
-      this.pendingQueue = [];
+  announce(job) {
+    if (this.eventBus && "params" in job && "resourceId" in job.params) {
+      this.eventBus.get("job:queued").next({
+        jobId: job.metadata.id,
+        jobType: job.metadata.type,
+        resourceId: job.params.resourceId,
+        userId: job.metadata.userId
+      });
     }
   }
   /**
-   * Debounced version of loadPendingJobs — fs.watch can fire rapidly
+   * Announce every job currently in `pending/`. Files that vanish or
+   * fail to parse mid-scan (claimed, cancelled, partially written)
+   * are skipped — they're either gone for a good reason or picked up
+   * on the next tick.
    */
-  debouncedLoadPendingJobs() {
-    if (this.loadDebounceTimer) return;
-    this.loadDebounceTimer = setTimeout(async () => {
-      this.loadDebounceTimer = null;
-      await this.loadPendingJobs();
-    }, 100);
+  async announcePendingJobs() {
+    const pendingDir = path.join(this.jobsDir, "pending");
+    let files;
+    try {
+      files = await promises.readdir(pendingDir);
+    } catch {
+      return;
+    }
+    files.sort();
+    for (const file of files) {
+      try {
+        const content = await promises.readFile(path.join(pendingDir, file), "utf-8");
+        this.announce(JSON.parse(content));
+      } catch {
+      }
+    }
   }
   /**
    * Create a new job
@@ -91,16 +118,7 @@ var FsJobQueue = class {
     await promises.writeFile(jobPath, JSON.stringify(job, null, 2), "utf-8");
     this.logger.info("Job created", { jobId: job.metadata.id, status: job.status });
     if (job.status === "pending") {
-      this.pendingQueue.push(job);
-      this.pendingQueue.sort((a, b) => a.metadata.id.localeCompare(b.metadata.id));
-    }
-    if (this.eventBus && "params" in job && "resourceId" in job.params) {
-      this.eventBus.get("job:queued").next({
-        jobId: job.metadata.id,
-        jobType: job.metadata.type,
-        resourceId: job.params.resourceId,
-        userId: job.metadata.userId
-      });
+      this.announce(job);
     }
   }
   /**
@@ -129,34 +147,92 @@ var FsJobQueue = class {
         await promises.unlink(oldPath);
       } catch (error) {
       }
-      if (oldStatus === "pending") {
-        const idx = this.pendingQueue.findIndex((j) => j.metadata.id === job.metadata.id);
-        if (idx !== -1) this.pendingQueue.splice(idx, 1);
-      }
-      if (job.status === "pending") {
-        this.pendingQueue.push(job);
-        this.pendingQueue.sort((a, b) => a.metadata.id.localeCompare(b.metadata.id));
-      }
     }
     const newPath = this.getJobPath(job.metadata.id, job.status);
     await promises.writeFile(newPath, JSON.stringify(job, null, 2), "utf-8");
     if (oldStatus && oldStatus !== job.status) {
       this.logger.info("Job moved", { jobId: job.metadata.id, oldStatus, newStatus: job.status });
+      if (job.status === "pending") {
+        this.announce(job);
+      }
     } else {
       this.logger.info("Job updated", { jobId: job.metadata.id, status: job.status });
     }
   }
   /**
-   * Poll for next pending job (FIFO) from in-memory queue.
-   * If a predicate is provided, returns the first matching job (skipping non-matching ones).
+   * Move a running job to `complete`. Returns false (and changes
+   * nothing) if the job is missing or not running — which also makes
+   * duplicate `job:complete` events harmless.
    */
-  async pollNextPendingJob(predicate) {
-    if (!predicate) {
-      return this.pendingQueue.shift() ?? null;
+  async completeJob(jobId, result) {
+    const job = await this.getJob(jobId);
+    if (!job || job.status !== "running") {
+      return false;
     }
-    const index = this.pendingQueue.findIndex(predicate);
-    if (index === -1) return null;
-    return this.pendingQueue.splice(index, 1)[0] ?? null;
+    this.lastProgressWrite.delete(jobId);
+    const completed = {
+      status: "complete",
+      metadata: job.metadata,
+      params: job.params,
+      startedAt: job.startedAt,
+      completedAt: (/* @__PURE__ */ new Date()).toISOString(),
+      result
+    };
+    await this.updateJob(completed, "running");
+    return true;
+  }
+  /**
+   * Retry-or-fail a running job. While `retryCount < maxRetries` the
+   * job goes back to `pending` with the count bumped (and is
+   * re-announced); after that it lands in `failed` with the error.
+   * Returns null (and changes nothing) if the job isn't running.
+   */
+  async failJob(jobId, error) {
+    const job = await this.getJob(jobId);
+    if (!job || job.status !== "running") {
+      return null;
+    }
+    this.lastProgressWrite.delete(jobId);
+    if (job.metadata.retryCount < job.metadata.maxRetries) {
+      const retried = {
+        status: "pending",
+        metadata: { ...job.metadata, retryCount: job.metadata.retryCount + 1 },
+        params: job.params
+      };
+      await this.updateJob(retried, "running");
+      return "retried";
+    }
+    const failed = {
+      status: "failed",
+      metadata: job.metadata,
+      params: job.params,
+      startedAt: job.startedAt,
+      completedAt: (/* @__PURE__ */ new Date()).toISOString(),
+      error
+    };
+    await this.updateJob(failed, "running");
+    return "failed";
+  }
+  /**
+   * Write progress into a running job's file. Throttled per job, and
+   * a no-op for jobs that aren't running. Beyond surfacing live
+   * progress to `job:status-requested`, each write refreshes the
+   * file's mtime — the heartbeat `recoverStaleRunningJobs` watches.
+   */
+  async recordProgress(jobId, progress) {
+    const now = Date.now();
+    const lastWrite = this.lastProgressWrite.get(jobId) ?? 0;
+    if (now - lastWrite < PROGRESS_WRITE_MIN_INTERVAL_MS) {
+      return;
+    }
+    this.lastProgressWrite.set(jobId, now);
+    const job = await this.getJob(jobId);
+    if (!job || job.status !== "running") {
+      this.lastProgressWrite.delete(jobId);
+      return;
+    }
+    const updated = { ...job, progress };
+    await promises.writeFile(this.getJobPath(jobId, "running"), JSON.stringify(updated, null, 2), "utf-8");
   }
   /**
    * List jobs with filters
@@ -207,6 +283,63 @@ var FsJobQueue = class {
     await this.updateJob(cancelledJob, oldStatus);
     return true;
   }
+  /**
+   * Cancel all pending jobs in a category — the granularity of the
+   * `job:cancel-requested` UI signal. Running jobs are left to finish:
+   * interrupting a worker mid-inference would need a worker-side kill
+   * channel that doesn't exist.
+   */
+  async cancelPendingJobs(category) {
+    const matches = category === "generation" ? (type) => type === "generation" : (type) => type.endsWith("-annotation");
+    const pending = await this.listJobs({ status: "pending", limit: Number.MAX_SAFE_INTEGER });
+    let cancelled = 0;
+    for (const job of pending) {
+      if (!matches(job.metadata.type)) continue;
+      if (await this.cancelJob(job.metadata.id)) {
+        cancelled++;
+      }
+    }
+    if (cancelled > 0) {
+      this.logger.info("Cancelled pending jobs", { category, cancelled });
+    }
+    return cancelled;
+  }
+  /**
+   * Recover running jobs orphaned by a dead worker: any `running/`
+   * file whose mtime is older than the stale window is fed through
+   * the same retry-or-fail path as `job:fail`. Progress writes
+   * refresh the mtime, so a live worker is never recovered out from
+   * under itself as long as it reports within the window.
+   */
+  async recoverStaleRunningJobs() {
+    const runningDir = path.join(this.jobsDir, "running");
+    let files;
+    try {
+      files = await promises.readdir(runningDir);
+    } catch {
+      return 0;
+    }
+    const now = Date.now();
+    let recovered = 0;
+    for (const file of files) {
+      if (!file.endsWith(".json")) continue;
+      try {
+        const stat = await promises.stat(path.join(runningDir, file));
+        if (now - stat.mtimeMs < STALE_RUNNING_MS) continue;
+        const staleId = jobId(file.slice(0, -".json".length));
+        const outcome = await this.failJob(
+          staleId,
+          `worker presumed dead \u2014 no progress within ${STALE_RUNNING_MS / 6e4} minutes`
+        );
+        if (outcome) {
+          this.logger.warn("Recovered stale running job", { jobId: staleId, outcome });
+          recovered++;
+        }
+      } catch {
+      }
+    }
+    return recovered;
+  }
   /**
    * Clean up old completed/failed jobs (older than retention period)
    */
@@ -805,21 +938,6 @@ function logAnchorMethod(motivation, exact, anchorMethod) {
 
 // src/workers/annotation-detection.ts
 var AnnotationDetection = class {
-  /**
-   * Fetch content from a ContentFetcher and read the stream to a string.
-   * Shared helper for all workers.
-   */
-  static async fetchContent(contentFetcher, resourceId) {
-    const stream = await contentFetcher(resourceId);
-    if (!stream) {
-      throw new Error(`Could not load content for resource ${resourceId}`);
-    }
-    const chunks = [];
-    for await (const chunk of stream) {
-      chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
-    }
-    return Buffer.concat(chunks).toString("utf-8");
-  }
   /**
    * Detect comments in content.
    *
@@ -1065,10 +1183,19 @@ Knowledge graph context:
 ${parts.join("\n")}`;
     }
   }
+  let semanticContextSection = "";
+  const similar = context?.semanticContext?.similar ?? [];
+  if (similar.length > 0) {
+    const lines = [...similar].sort((a, b) => b.score - a.score).slice(0, 3).map((m) => `- (${m.score.toFixed(2)}) ${m.text.slice(0, 240)}`);
+    semanticContextSection = `
+
+Related passages from the knowledge base:
+${lines.join("\n")}`;
+  }
   const structureGuidance = finalMaxTokens >= 1e3 ? "organized into titled sections (## Section) with well-structured paragraphs" : "organized into well-structured paragraphs";
   const prompt = `Generate a concise, informative resource about "${topic}".
 ${entityTypes.length > 0 ? `Focus on these entity types: ${entityTypes.join(", ")}.` : ""}
-${userPrompt ? `Additional context: ${userPrompt}` : ""}${annotationSection}${contextSection}${graphContextSection}${sourceLanguageInstruction}${languageInstruction}
+${userPrompt ? `Additional context: ${userPrompt}` : ""}${annotationSection}${contextSection}${graphContextSection}${semanticContextSection}${sourceLanguageInstruction}${languageInstruction}
 
 Requirements:
 - Start with a clear heading (# Title)