@semiont/jobs 0.5.36 → 0.5.37

package/dist/index.js

@@ -2,6 +2,7 @@ import { promises, mkdtempSync, writeFileSync, readFileSync, rmSync } from 'fs';
 import * as path from 'path';
 import { join } from 'path';
 import { replyChannelsFor, jobId, deriveViews, GENERATABLE_MEDIA_TYPES, estimateTokens, isObject, isString, reconcileSelector, getLocaleEnglishName, cutChunk, chunkText } from '@semiont/core';
+import { connect, RetentionPolicy, DeliverPolicy, AckPolicy, nanos } from 'nats';
 import { withSpan, recordAnchorOutcome, recordDetectionCall } from '@semiont/observability';
 import { StructuredReadError } from '@semiont/inference';
 import { execFileSync } from 'child_process';
@@ -18,10 +19,7 @@ function willRetryAfter(metadata, failureClass) {
   return failureClass !== "deterministic" && metadata.retryCount < metadata.maxRetries;
 }
 
-// src/fs-job-queue.ts
-var REANNOUNCE_INTERVAL_MS = 3e4;
-var STALE_RUNNING_MS = 30 * 6e4;
-var PROGRESS_WRITE_MIN_INTERVAL_MS = 5e3;
+// src/checkpoint-merge.ts
 function mergeUnitCursors(existing, incoming, completed) {
   const done = new Set(completed);
   const merged = {};
@@ -35,6 +33,11 @@ function mergeUnitCursors(existing, incoming, completed) {
   }
   return merged;
 }
+
+// src/fs-job-queue.ts
+var REANNOUNCE_INTERVAL_MS = 3e4;
+var STALE_RUNNING_MS = 30 * 6e4;
+var PROGRESS_WRITE_MIN_INTERVAL_MS = 5e3;
 var RETENTION_HOURS = 24;
 var CLEANUP_INTERVAL_MS = 36e5;
 var FsJobQueue = class {
@@ -165,10 +168,43 @@ var FsJobQueue = class {
     }
     return null;
   }
+  /**
+   * Serializes claims. This driver's atomicity is single-process by design,
+   * but `claimJob`'s contract must hold even against interleaved awaits in
+   * that one process — a read-check-write with await points in it can admit
+   * two winners without this chain (the TOCTOU the operation exists to kill).
+   */
+  claimChain = Promise.resolve();
+  async claimNextJob(types) {
+    const claim = this.claimChain.then(() => this.doClaimNext(types));
+    this.claimChain = claim.catch(() => void 0);
+    return claim;
+  }
+  async doClaimNext(types) {
+    const pending = await this.listJobs({ status: "pending", limit: Number.MAX_SAFE_INTEGER });
+    const match = pending.find((j) => types.length === 0 || types.includes(j.metadata.type));
+    if (!match) return { declined: "none-available" };
+    const running = {
+      status: "running",
+      metadata: match.metadata,
+      params: match.params,
+      startedAt: (/* @__PURE__ */ new Date()).toISOString(),
+      progress: {}
+    };
+    await this.transition(running, "pending");
+    return { job: running };
+  }
   /**
    * Update a job (atomic: delete old, write new)
    */
-  async updateJob(job, oldStatus) {
+  /**
+   * Cross-status move (or in-place rewrite): delete old slot, write new.
+   * Private — the interface exposes named transitions only (JOB-QUEUE-DRIVER
+   * P0); a generic public patch is trivial on files and impossible on an
+   * in-flight message, so no driver may offer one. Re-entering `pending`
+   * announces, which is how failJob's retry reaches a listening worker.
+   */
+  async transition(job, oldStatus) {
     if (oldStatus && oldStatus !== job.status) {
       const oldPath = this.getJobPath(job.metadata.id, oldStatus);
       try {
@@ -206,7 +242,7 @@ var FsJobQueue = class {
       completedAt: (/* @__PURE__ */ new Date()).toISOString(),
       result
     };
-    await this.updateJob(completed, "running");
+    await this.transition(completed, "running");
     return true;
   }
   /**
@@ -235,7 +271,7 @@ var FsJobQueue = class {
         metadata: { ...metadata, retryCount: job.metadata.retryCount + 1 },
         params: job.params
       };
-      await this.updateJob(retried, "running");
+      await this.transition(retried, "running");
       return "retried";
     }
     if (failureClass === "deterministic" && job.metadata.retryCount < job.metadata.maxRetries) {
@@ -249,7 +285,7 @@ var FsJobQueue = class {
       completedAt: (/* @__PURE__ */ new Date()).toISOString(),
       error
     };
-    await this.updateJob(failed, "running");
+    await this.transition(failed, "running");
     return "failed";
   }
   /**
@@ -352,7 +388,7 @@ var FsJobQueue = class {
       startedAt: job.status === "running" ? job.startedAt : void 0,
       completedAt: (/* @__PURE__ */ new Date()).toISOString()
     };
-    await this.updateJob(cancelledJob, oldStatus);
+    await this.transition(cancelledJob, oldStatus);
     return true;
   }
   /**
@@ -474,6 +510,470 @@ var FsJobQueue = class {
     return stats;
   }
 };
+var STREAM = "JOBS";
+var CONSUMER = "gateway-claims";
+var BUCKET = "jobs";
+var PROGRESS_WRITE_MIN_INTERVAL_MS2 = 5e3;
+var MAX_CAS_ATTEMPTS = 20;
+var enc = new TextEncoder();
+var dec = new TextDecoder();
+function categoryOf(type) {
+  return type === "generation" ? "generation" : "annotation";
+}
+function subjectFor(job) {
+  return `jobs.${categoryOf(job.metadata.type)}.${job.metadata.type}`;
+}
+var JetStreamJobQueue = class {
+  constructor(options, logger, eventBus) {
+    this.options = options;
+    this.logger = logger;
+    this.eventBus = eventBus;
+    this.staleRunningMs = options.staleRunningMs ?? 30 * 6e4;
+    this.ackWaitMs = options.ackWaitMs ?? 3e4;
+    this.tickMs = options.tickMs ?? 3e4;
+  }
+  options;
+  logger;
+  eventBus;
+  nc;
+  js;
+  jsm;
+  kv;
+  iter = null;
+  /** Delivered messages this process holds — the lease, keyed by jobId,
+   *  with the job's type so a by-type claim can walk them without a read. */
+  held = /* @__PURE__ */ new Map();
+  heartbeat = null;
+  reconciling = false;
+  tick = null;
+  ticking = false;
+  lastProgressWrite = /* @__PURE__ */ new Map();
+  staleRunningMs;
+  ackWaitMs;
+  tickMs;
+  async initialize() {
+    this.nc = await connect({
+      servers: this.options.servers,
+      reconnect: this.options.reconnect ?? true,
+      timeout: 1e4
+    });
+    this.js = this.nc.jetstream();
+    this.jsm = await this.nc.jetstreamManager();
+    this.kv = await this.js.views.kv(BUCKET);
+    try {
+      await this.jsm.streams.info(STREAM);
+    } catch {
+      await this.jsm.streams.add({
+        name: STREAM,
+        subjects: ["jobs.>"],
+        retention: RetentionPolicy.Workqueue
+      });
+    }
+    try {
+      await this.jsm.consumers.info(STREAM, CONSUMER);
+    } catch {
+      await this.jsm.consumers.add(STREAM, {
+        durable_name: CONSUMER,
+        ack_policy: AckPolicy.Explicit,
+        deliver_policy: DeliverPolicy.All,
+        ack_wait: nanos(this.ackWaitMs),
+        // ONE retry authority: will-retry.ts decides; the stream never
+        // gives up a message on its own count.
+        max_deliver: -1
+      });
+    }
+    const consumer = await this.js.consumers.get(STREAM, CONSUMER);
+    this.iter = await consumer.consume({
+      callback: (m) => {
+        void this.onDelivery(m).catch((error) => {
+          this.logger.warn("Job delivery handling failed", {
+            error: error instanceof Error ? error.message : String(error)
+          });
+        });
+      }
+    });
+    this.heartbeat = setInterval(() => {
+      if (this.reconciling) return;
+      this.reconciling = true;
+      void this.reconcileHeld().catch((error) => {
+        this.logger.warn("Held-lease reconciliation failed", {
+          error: error instanceof Error ? error.message : String(error)
+        });
+      }).finally(() => {
+        this.reconciling = false;
+      });
+    }, Math.max(250, Math.floor(this.ackWaitMs / 4)));
+    this.heartbeat.unref?.();
+    this.tick = setInterval(() => {
+      if (this.ticking) return;
+      this.ticking = true;
+      void (async () => {
+        for (const [id, held] of [...this.held]) {
+          const envelope = await this.read(jobId(id));
+          if (envelope?.job.status === "pending") this.announce(envelope.job);
+        }
+        await this.recoverStaleRunningJobs();
+      })().catch((error) => {
+        this.logger.warn("Job-queue tick failed", {
+          error: error instanceof Error ? error.message : String(error)
+        });
+      }).finally(() => {
+        this.ticking = false;
+      });
+    }, this.tickMs);
+    this.tick.unref?.();
+  }
+  destroy() {
+    this.iter?.stop();
+    this.iter = null;
+    if (this.heartbeat) clearInterval(this.heartbeat);
+    this.heartbeat = null;
+    if (this.tick) clearInterval(this.tick);
+    this.tick = null;
+    this.held.clear();
+    void this.nc?.close();
+  }
+  /**
+   * A delivery is a job arriving at this gateway. What it means depends on
+   * the job's authoritative (KV) state:
+   *  - pending  → hold the lease, announce for a worker to claim;
+   *  - running  → hold silently (a claim raced ahead of the delivery, or a
+   *               redelivery reached a fresh instance after a gateway death);
+   *  - terminal → the work already concluded elsewhere; consume the message;
+   *  - unknown  → not ours to run; terminate it.
+   */
+  async onDelivery(m) {
+    let id;
+    try {
+      id = JSON.parse(dec.decode(m.data)).jobId;
+    } catch {
+      m.term();
+      return;
+    }
+    const envelope = await this.read(jobId(id));
+    if (!envelope) {
+      m.term();
+      return;
+    }
+    switch (envelope.job.status) {
+      case "pending":
+        this.held.set(id, { m, type: envelope.job.metadata.type });
+        this.announce(envelope.job);
+        break;
+      case "running":
+        this.held.set(id, { m, type: envelope.job.metadata.type });
+        break;
+      default:
+        m.ack();
+        this.held.delete(id);
+    }
+  }
+  /** Same wire shape as the fs driver: only jobs with a resourceId announce. */
+  announce(job) {
+    if (this.eventBus && "params" in job && "resourceId" in job.params) {
+      this.eventBus.get("job:queued").next({
+        jobId: job.metadata.id,
+        jobType: job.metadata.type,
+        resourceId: job.params.resourceId,
+        userId: job.metadata.userId
+      });
+    }
+  }
+  async read(jobIdArg) {
+    const entry = await this.kv.get(jobIdArg);
+    if (!entry || entry.operation !== "PUT") return null;
+    const envelope = JSON.parse(dec.decode(entry.value));
+    return { ...envelope, revision: entry.revision };
+  }
+  async write(jobIdArg, envelope, revision) {
+    const payload = enc.encode(JSON.stringify(envelope));
+    if (revision === void 0) return this.kv.put(jobIdArg, payload);
+    return this.kv.update(jobIdArg, payload, revision);
+  }
+  /**
+   * Read-transform-CAS with bounded retries: the transform sees the current
+   * job and returns the replacement (or null to stop). Every state
+   * transition goes through here, which is what makes each one atomic.
+   */
+  async cas(jobIdArg, transform, onMissing) {
+    for (let attempt = 0; attempt < MAX_CAS_ATTEMPTS; attempt++) {
+      const current = await this.read(jobIdArg);
+      if (!current) return onMissing;
+      const out = transform(current.job, current.lastProgressAt);
+      if (out === null) return onMissing;
+      if (!("envelope" in out)) return out.result;
+      try {
+        await this.write(jobIdArg, out.envelope, current.revision);
+        return out.result;
+      } catch {
+      }
+    }
+    throw new Error(`Job ${jobIdArg} transition failed after ${MAX_CAS_ATTEMPTS} CAS attempts`);
+  }
+  async createJob(job) {
+    const now = (/* @__PURE__ */ new Date()).toISOString();
+    await this.kv.create(job.metadata.id, enc.encode(JSON.stringify({ job, lastProgressAt: now })));
+    if (job.status === "pending") {
+      await this.js.publish(subjectFor(job), enc.encode(JSON.stringify({ jobId: job.metadata.id })));
+    }
+  }
+  async getJob(jobIdArg) {
+    const envelope = await this.read(jobIdArg);
+    return envelope?.job ?? null;
+  }
+  async claimNextJob(types) {
+    const matches = (t) => types.length === 0 || types.includes(t);
+    for (const [id, held] of [...this.held]) {
+      if (!matches(held.type)) continue;
+      const won = await this.tryClaim(jobId(id));
+      if (won) return won;
+    }
+    for (const key of await this.allKeys()) {
+      const envelope = await this.read(jobId(key));
+      if (!envelope || envelope.job.status !== "pending" || !matches(envelope.job.metadata.type)) continue;
+      const won = await this.tryClaim(jobId(key));
+      if (won) return won;
+    }
+    return { declined: "none-available" };
+  }
+  /** CAS one pending job to running; null when someone else won it. */
+  async tryClaim(jobIdArg) {
+    return this.cas(
+      jobIdArg,
+      (job) => {
+        if (job.status !== "pending") return { result: null };
+        const running = {
+          status: "running",
+          metadata: job.metadata,
+          params: job.params,
+          startedAt: (/* @__PURE__ */ new Date()).toISOString(),
+          progress: {}
+        };
+        return {
+          envelope: { job: running, lastProgressAt: (/* @__PURE__ */ new Date()).toISOString() },
+          result: { job: running }
+        };
+      },
+      null
+    );
+  }
+  async completeJob(jobIdArg, result) {
+    const moved = await this.cas(
+      jobIdArg,
+      (job) => {
+        if (job.status !== "running") return { result: false };
+        const completed = {
+          status: "complete",
+          metadata: job.metadata,
+          params: job.params,
+          startedAt: job.startedAt,
+          completedAt: (/* @__PURE__ */ new Date()).toISOString(),
+          result
+        };
+        return { envelope: { job: completed, lastProgressAt: (/* @__PURE__ */ new Date()).toISOString() }, result: true };
+      },
+      false
+    );
+    if (moved) this.settleLease(jobIdArg, "ack");
+    return moved;
+  }
+  async failJob(jobIdArg, error, completedUnits, failureClass, unitCursors) {
+    const outcome = await this.cas(
+      jobIdArg,
+      (job) => {
+        if (job.status !== "running") return { result: null };
+        const units = [.../* @__PURE__ */ new Set([...job.metadata.completedUnits ?? [], ...completedUnits ?? []])];
+        const cursors = mergeUnitCursors(job.metadata.unitCursors, unitCursors, units);
+        const metadata = {
+          ...job.metadata,
+          ...units.length > 0 ? { completedUnits: units } : {},
+          ...Object.keys(cursors).length > 0 ? { unitCursors: cursors } : {}
+        };
+        if ("unitCursors" in metadata && Object.keys(cursors).length === 0) delete metadata.unitCursors;
+        if (willRetryAfter(job.metadata, failureClass)) {
+          const retried = {
+            status: "pending",
+            metadata: { ...metadata, retryCount: job.metadata.retryCount + 1 },
+            params: job.params
+          };
+          return { envelope: { job: retried, lastProgressAt: (/* @__PURE__ */ new Date()).toISOString() }, result: "retried" };
+        }
+        const failed = {
+          status: "failed",
+          metadata,
+          params: job.params,
+          startedAt: job.startedAt,
+          completedAt: (/* @__PURE__ */ new Date()).toISOString(),
+          error
+        };
+        return { envelope: { job: failed, lastProgressAt: (/* @__PURE__ */ new Date()).toISOString() }, result: "failed" };
+      },
+      null
+    );
+    if (outcome === "retried") {
+      const held = this.held.get(jobIdArg);
+      if (held) {
+        this.held.delete(jobIdArg);
+        held.m.nak();
+      } else {
+        const envelope = await this.read(jobIdArg);
+        if (envelope) await this.js.publish(subjectFor(envelope.job), enc.encode(JSON.stringify({ jobId: jobIdArg })));
+      }
+    } else if (outcome === "failed") {
+      this.settleLease(jobIdArg, "term");
+    }
+    return outcome;
+  }
+  async checkpointUnits(jobIdArg, completedUnits, unitCursors) {
+    await this.cas(
+      jobIdArg,
+      (job) => {
+        if (job.status !== "running") return { result: void 0 };
+        const units = [.../* @__PURE__ */ new Set([...job.metadata.completedUnits ?? [], ...completedUnits])];
+        const cursors = mergeUnitCursors(job.metadata.unitCursors, unitCursors, units);
+        const metadata = { ...job.metadata, completedUnits: units };
+        if (Object.keys(cursors).length > 0) metadata.unitCursors = cursors;
+        else delete metadata.unitCursors;
+        const updated = { ...job, metadata };
+        return { envelope: { job: updated, lastProgressAt: (/* @__PURE__ */ new Date()).toISOString() }, result: void 0 };
+      },
+      void 0
+    );
+  }
+  async recordProgress(jobIdArg, progress) {
+    const last = this.lastProgressWrite.get(jobIdArg) ?? 0;
+    if (Date.now() - last < PROGRESS_WRITE_MIN_INTERVAL_MS2) return;
+    this.lastProgressWrite.set(jobIdArg, Date.now());
+    this.held.get(jobIdArg)?.m.working();
+    await this.cas(
+      jobIdArg,
+      (job) => {
+        if (job.status !== "running") return { result: void 0 };
+        const updated = {
+          status: "running",
+          metadata: job.metadata,
+          params: job.params,
+          startedAt: job.startedAt,
+          progress
+        };
+        return { envelope: { job: updated, lastProgressAt: (/* @__PURE__ */ new Date()).toISOString() }, result: void 0 };
+      },
+      void 0
+    );
+  }
+  async cancelPendingJobs(category) {
+    const matches = category === "generation" ? (type) => type === "generation" : (type) => type.endsWith("-annotation");
+    let cancelled = 0;
+    for (const key of await this.allKeys()) {
+      const envelope = await this.read(jobId(key));
+      if (!envelope || envelope.job.status !== "pending" || !matches(envelope.job.metadata.type)) continue;
+      const done = await this.cas(
+        jobId(key),
+        (job) => {
+          if (job.status !== "pending" || !matches(job.metadata.type)) return { result: false };
+          const record = {
+            status: "cancelled",
+            metadata: job.metadata,
+            params: job.params,
+            startedAt: void 0,
+            completedAt: (/* @__PURE__ */ new Date()).toISOString()
+          };
+          return { envelope: { job: record, lastProgressAt: (/* @__PURE__ */ new Date()).toISOString() }, result: true };
+        },
+        false
+      );
+      if (done) {
+        cancelled++;
+        this.settleLease(jobId(key), "term");
+      }
+    }
+    await this.jsm.streams.purge(STREAM, { filter: `jobs.${category}.>` });
+    return cancelled;
+  }
+  async cancelJob(jobIdArg) {
+    const done = await this.cas(
+      jobIdArg,
+      (job) => {
+        if (job.status !== "pending" && job.status !== "running") return { result: false };
+        const record = {
+          status: "cancelled",
+          metadata: job.metadata,
+          params: job.params,
+          startedAt: job.status === "running" ? job.startedAt : void 0,
+          completedAt: (/* @__PURE__ */ new Date()).toISOString()
+        };
+        return { envelope: { job: record, lastProgressAt: (/* @__PURE__ */ new Date()).toISOString() }, result: true };
+      },
+      false
+    );
+    if (done) this.settleLease(jobIdArg, "term");
+    return done;
+  }
+  async getStats() {
+    const stats = { pending: 0, running: 0, complete: 0, failed: 0, cancelled: 0 };
+    for (const key of await this.allKeys()) {
+      const envelope = await this.read(jobId(key));
+      if (envelope) stats[envelope.job.status]++;
+    }
+    return stats;
+  }
+  /**
+   * Worker-death recovery: the same contract the fs driver's mtime janitor
+   * implemented — a running job with no progress inside the stale window is
+   * retried or failed, checkpoint intact. Gateway-death recovery never comes
+   * through here; that is AckWait redelivery.
+   */
+  async recoverStaleRunningJobs() {
+    const now = Date.now();
+    let recovered = 0;
+    for (const key of await this.allKeys()) {
+      const envelope = await this.read(jobId(key));
+      if (!envelope || envelope.job.status !== "running") continue;
+      if (now - Date.parse(envelope.lastProgressAt) < this.staleRunningMs) continue;
+      const outcome = await this.failJob(
+        jobId(key),
+        `worker presumed dead \u2014 no progress within ${this.staleRunningMs / 6e4} minutes`
+      );
+      if (outcome) {
+        this.logger.warn("Recovered stale running job", { jobId: key, outcome });
+        recovered++;
+      }
+    }
+    return recovered;
+  }
+  /** The heartbeat body — extend live leases, settle concluded ones. */
+  async reconcileHeld() {
+    for (const [id, held] of [...this.held]) {
+      const envelope = await this.read(jobId(id));
+      const status = envelope?.job.status;
+      if (!envelope || status === "complete" || status === "failed" || status === "cancelled") {
+        this.held.delete(id);
+        held.m.ack();
+      } else {
+        held.m.working();
+      }
+    }
+  }
+  /**
+   * Materialize the KV key list BEFORE touching any entry: interleaving
+   * `get`/`update` with an open `keys()` iterator on the same connection
+   * makes the iterator drop entries (measured: the second pending job
+   * vanished from every sweep until this was split into two passes).
+   */
+  async allKeys() {
+    const keys = [];
+    for await (const key of await this.kv.keys()) keys.push(key);
+    return keys;
+  }
+  /** Conclude this process's lease on a job, if it holds one. */
+  settleLease(jobIdArg, how) {
+    const held = this.held.get(jobIdArg);
+    if (!held) return;
+    this.held.delete(jobIdArg);
+    if (how === "ack") held.m.ack();
+    else held.m.term();
+  }
+};
 
 // src/types.ts
 function isPendingJob(job) {
@@ -2435,6 +2935,6 @@ var WORKER_AWAITED_OPERATIONS = [
 ];
 replyChannelsFor(WORKER_AWAITED_OPERATIONS);
 
-export { AnnotationDetection, FsJobQueue, STALL_THRESHOLD_MS, generateResourceFromTopic, isCancelledJob, isCompleteJob, isFailedJob, isPendingJob, isRunningJob, processAssessmentJob, processCommentJob, processGenerationJob, processHighlightJob, processReferenceJob, processTagJob };
+export { AnnotationDetection, FsJobQueue, JetStreamJobQueue, STALL_THRESHOLD_MS, generateResourceFromTopic, isCancelledJob, isCompleteJob, isFailedJob, isPendingJob, isRunningJob, processAssessmentJob, processCommentJob, processGenerationJob, processHighlightJob, processReferenceJob, processTagJob };
 //# sourceMappingURL=index.js.map
 //# sourceMappingURL=index.js.map