@johpaz/hive-sdk 0.1.3 → 0.1.5

package/packages/core/src/{harness → gateway}/durable-queue.ts

@@ -1,18 +1,17 @@
 /**
- * DurableLaneQueue — persistent work queue backed by the harness_jobQueue
- * HiveDB collection. Ported from `hive`'s gateway/durable-queue.ts,
- * generalized for SDK consumers (job `type` is a plain string; the set of
- * "never auto-retry a logical failure" types is configurable instead of a
- * hardcoded `chat_turn` check — default matches `hive`'s convention).
+ * DurableLaneQueue — persistent work queue backed by the `jobQueue` HiveDB
+ * collection.
  *
  * Guarantees:
- * - FIFO + priority per lane
+ * - FIFO + priority per lane (same semantics as LaneQueue)
  * - 1 concurrent running job per lane
  * - maxGlobalConcurrency across all lanes (default 4)
  * - JobDoc persists every status transition → survives crashes
  * - Expired leases (crashed worker) are reclaimed or interrupted on next dispatch
- * - Logical failures ({ok:false, retryable:true}) retry with backoff+jitter
- *   via `failJobOrRetry`, except for `nonRetryableTypes` (default: chat_turn)
+ *
+ * The live LaneQueue instance is used as the in-memory dispatch layer:
+ * DurableLaneQueue wraps it so that the existing per-session serial behavior
+ * is preserved, while all queue state is mirrored to the DB.
  */
 
 import { logger } from "../utils/logger";
@@ -27,18 +26,21 @@ import {
   findPendingJobsByLane,
   findAllPendingJobs,
   findExpiredLeases,
+  getJob,
+  loadJobRetryPolicy,
   DEFAULT_JOB_RETRY_POLICY,
   type JobRetryPolicy,
 } from "./job-store";
-import type { JobDoc } from "./collections";
-import { getBootId } from "./boot-id";
+import type { JobDoc } from "../storage/collections";
+import { getBootId } from "../storage/boot-id";
 
-const log = logger.child("harness:durable-queue");
+const log = logger.child("durable-queue");
 
 const LEASE_CHECK_INTERVAL_MS = 10_000;
 const DEFAULT_MAX_GLOBAL_CONCURRENCY = 4;
 const DEFAULT_TASK_TIMEOUT_MS = 30 * 60 * 1000;
-const DEFAULT_NON_RETRYABLE_TYPES = ["chat_turn"];
+
+export type JobType = JobDoc["type"];
 
 export interface JobPayload {
   [key: string]: unknown;
@@ -57,6 +59,7 @@ export interface JobExecutorResult {
 export interface JobLiveCallbacks {
   onToken?: (token: string) => void;
   onStep?: (step: unknown) => Promise<void>;
+  /** Sends an already-serialized frame to the live socket (webchat turns). */
   sendRaw?: (payload: string) => void;
 }
 
@@ -66,52 +69,74 @@ export type JobExecutor = (
   callbacks?: JobLiveCallbacks
 ) => Promise<JobExecutorResult>;
 
-const executors = new Map<string, JobExecutor>();
+const executors = new Map<JobType, JobExecutor>();
 
-export function registerExecutor(type: string, executor: JobExecutor): void {
+export function registerExecutor(type: JobType, executor: JobExecutor): void {
   executors.set(type, executor);
   log.info(`[registerExecutor] Registered executor for type=${type}`);
 }
 
-export interface DurableLaneQueueOptions {
-  maxGlobalConcurrency?: number;
-  taskTimeoutMs?: number;
-  jobRetryPolicy?: JobRetryPolicy;
-  /** Job types that should never auto-retry a logical failure (e.g. user-facing turns). Default: ["chat_turn"]. */
-  nonRetryableTypes?: string[];
+export interface JobTerminalOutcome {
+  ok: boolean;
+  result?: unknown;
+  error?: string;
+}
+
+/** Fires exactly once when a job of `type` reaches a terminal state (completed, or failed with no retries left) — never on a retryable failure that gets requeued. */
+export type JobTerminalHook = (job: JobDoc, outcome: JobTerminalOutcome) => void | Promise<void>;
+
+const terminalHooks = new Map<JobType, JobTerminalHook>();
+
+/** Register a side-effect to run once a job type finishes for good. Kept decoupled from executors so this module doesn't need to import webchat-turn.ts (circular: webchat-turn.ts already imports durable-queue.ts for enqueueChatTurn). */
+export function registerTerminalHook(type: JobType, hook: JobTerminalHook): void {
+  terminalHooks.set(type, hook);
+}
+
+async function runTerminalHook(job: JobDoc, outcome: JobTerminalOutcome): Promise<void> {
+  const hook = terminalHooks.get(job.type);
+  if (!hook) return;
+  try {
+    await hook(job, outcome);
+  } catch (err) {
+    log.warn(`[runTerminalHook] Hook for type=${job.type} failed: ${(err as Error).message}`);
+  }
 }
 
 export class DurableLaneQueue {
   private maxGlobalConcurrency: number;
   private taskTimeoutMs: number;
   private jobRetryPolicy: JobRetryPolicy;
-  private nonRetryableTypes: Set<string>;
   private runningCount = 0;
   private runningAborts = new Map<string, { lane: string; controller: AbortController }>();
   private dispatchTimers = new Map<string, ReturnType<typeof setInterval>>();
   private leaseCheckTimer: ReturnType<typeof setInterval> | null = null;
   private bootId: string;
 
-  constructor(options: DurableLaneQueueOptions = {}) {
+  constructor(options: {
+    maxGlobalConcurrency?: number;
+    taskTimeoutMs?: number;
+    jobRetryPolicy?: JobRetryPolicy;
+  } = {}) {
     this.maxGlobalConcurrency = options.maxGlobalConcurrency ?? DEFAULT_MAX_GLOBAL_CONCURRENCY;
     this.taskTimeoutMs = options.taskTimeoutMs ?? DEFAULT_TASK_TIMEOUT_MS;
     this.jobRetryPolicy = options.jobRetryPolicy ?? DEFAULT_JOB_RETRY_POLICY;
-    this.nonRetryableTypes = new Set(options.nonRetryableTypes ?? DEFAULT_NON_RETRYABLE_TYPES);
     this.bootId = getBootId();
   }
 
   /**
-   * Enqueue a durable job. Returns the JobDoc. Callers should use `enqueue`
-   * instead of any in-memory queue for work that must survive crashes.
+   * Enqueue a durable job. Returns the JobDoc.
+   * Callers should use `enqueue` instead of the in-memory `laneQueue.enqueue`
+   * for any work that must survive crashes.
    */
   async enqueue(input: {
     lane: string;
-    type: string;
+    type: JobType;
     payload: JobPayload;
     run_id: string;
     priority?: number;
     max_attempts?: number;
     not_before?: number;
+    /** Client-supplied dedup key: a repeated key returns the existing job instead of creating a new one. */
     idempotency_key?: string | null;
     callbacks?: JobLiveCallbacks;
   }): Promise<JobDoc> {
@@ -126,6 +151,7 @@ export class DurableLaneQueue {
       idempotency_key: input.idempotency_key,
     });
 
+    // Stash live callbacks in memory (not serializable)
     if (input.callbacks) {
       liveCallbacks.set(job.id, input.callbacks);
     }
@@ -134,14 +160,19 @@ export class DurableLaneQueue {
     return job;
   }
 
-  /** Cancel a job by id (pending or running). A running job also gets its AbortSignal fired. */
+  /**
+   * Cancel a job by id (pending or running). A running job also gets its
+   * AbortSignal fired so the executor stops streaming/working.
+   */
   async cancel(jobId: string): Promise<boolean> {
     const running = this.runningAborts.get(jobId);
     if (running) running.controller.abort();
     return cancelJob(jobId);
   }
 
-  /** Cancel all pending and running jobs in a lane. */
+  /**
+   * Cancel all pending and running jobs in a lane.
+   */
   async cancelLane(lane: string): Promise<number> {
     let count = 0;
     for (const [jobId, entry] of this.runningAborts) {
@@ -158,13 +189,15 @@ export class DurableLaneQueue {
   }
 
   /**
-   * Start the maintenance tick (expired-lease reclaim + due-pending
-   * redispatch) and dispatch any jobs left pending by a previous boot.
+   * Start the lease expiry checker and dispatch any jobs left pending by a
+   * previous boot. Called once during boot.
    */
   start(): void {
     if (this.leaseCheckTimer) return;
     this.leaseCheckTimer = setInterval(() => this.runMaintenanceTick(), LEASE_CHECK_INTERVAL_MS);
     log.info(`[start] Maintenance tick running every ${LEASE_CHECK_INTERVAL_MS}ms`);
+    // Without this, jobs reclaimed to "pending" by reconcileOnBoot (or enqueued
+    // right before a crash) would sit until a new enqueue touched their lane.
     this.dispatchPendingLanes().catch((err) => {
       log.error(`[start] Failed to dispatch pending lanes: ${(err as Error).message}`);
     });
@@ -173,7 +206,9 @@ export class DurableLaneQueue {
   /**
    * Periodic tick: reclaim crashed jobs (expired leases) AND re-dispatch
    * lanes with jobs whose `not_before` (e.g. a backoff delay from
-   * `failJobOrRetry`) has now elapsed.
+   * `failJobOrRetry`) has now elapsed. `dispatchPendingLanes` is safe to call
+   * repeatedly — `scheduleDispatch` debounces per lane and a lane with
+   * nothing due is a cheap no-op.
    */
   private async runMaintenanceTick(): Promise<void> {
     await this.checkExpiredLeases();
@@ -200,6 +235,9 @@ export class DurableLaneQueue {
     this.dispatchTimers.clear();
   }
 
+  /**
+   * Schedule dispatch for a lane (debounced — coalesce rapid enqueues).
+   */
   private scheduleDispatch(lane: string): void {
     if (this.dispatchTimers.has(lane)) return;
     const timer = setTimeout(() => {
@@ -211,29 +249,36 @@ export class DurableLaneQueue {
     this.dispatchTimers.set(lane, timer);
   }
 
+  /**
+   * Dispatch pending jobs for a single lane, respecting global concurrency.
+   */
   private async dispatchLane(lane: string): Promise<void> {
     for (;;) {
       const pending = await findPendingJobsByLane(lane, 1);
       if (pending.length === 0) break;
 
       const job = pending[0];
-      // Interactive/user-facing types bypass the global cap so a busy batch
-      // of background jobs can't starve them — matches `hive`'s chat_turn
-      // convention: nonRetryableTypes doubles as the "must stay responsive" set.
-      if (!this.nonRetryableTypes.has(job.type) && this.runningCount >= this.maxGlobalConcurrency) break;
+      // Interactive chat turns bypass the global cap: a busy batch of
+      // workers/goals must not make the webchat stop responding.
+      if (job.type !== "chat_turn" && this.runningCount >= this.maxGlobalConcurrency) break;
 
       const claimed = await claimJob(job.id, this.bootId);
-      if (!claimed) continue;
+      if (!claimed) continue; // someone else won the claim — re-read the lane
 
       this.runningCount++;
       this.executeJob(claimed, lane).catch((err) => {
         log.error(`[dispatchLane] Unhandled error in job ${claimed.id}: ${(err as Error).message}`);
       });
 
+      // Only 1 running per lane — break after dispatching
       break;
     }
   }
 
+  /**
+   * Execute a claimed job: look up executor, run with timeout + abort,
+   * persist result.
+   */
   private async executeJob(job: JobDoc, lane: string): Promise<void> {
     const abortController = new AbortController();
     this.runningAborts.set(job.id, { lane, controller: abortController });
@@ -261,12 +306,17 @@ export class DurableLaneQueue {
 
       if (result.ok) {
         await completeJob(job.id, result.result ?? null, this.bootId);
+        await runTerminalHook(job, { ok: true, result: result.result });
       } else {
         const error = result.error ?? "Unknown error";
-        if (!this.nonRetryableTypes.has(job.type) && result.retryable !== false) {
-          await failJobOrRetry(job.id, error, this.bootId, this.jobRetryPolicy);
+        // chat_turn is user-facing — never auto-retry a logical failure, the
+        // user just sees the error and can re-ask.
+        if (job.type !== "chat_turn" && result.retryable !== false) {
+          const updated = await failJobOrRetry(job.id, error, this.bootId, this.jobRetryPolicy);
+          if (updated?.status === "failed") await runTerminalHook(job, { ok: false, error });
         } else {
           await failJob(job.id, error, this.bootId);
+          await runTerminalHook(job, { ok: false, error });
         }
       }
     } catch (err) {
@@ -274,10 +324,12 @@ export class DurableLaneQueue {
         await cancelJob(job.id);
       } else {
         const error = (err as Error).message;
-        if (!this.nonRetryableTypes.has(job.type)) {
-          await failJobOrRetry(job.id, error, this.bootId, this.jobRetryPolicy);
+        if (job.type !== "chat_turn") {
+          const updated = await failJobOrRetry(job.id, error, this.bootId, this.jobRetryPolicy);
+          if (updated?.status === "failed") await runTerminalHook(job, { ok: false, error });
         } else {
           await failJob(job.id, error, this.bootId);
+          await runTerminalHook(job, { ok: false, error });
         }
       }
     } finally {
@@ -286,10 +338,14 @@ export class DurableLaneQueue {
       leasedHere = false;
       this.runningAborts.delete(job.id);
       this.runningCount--;
+      // Try to dispatch next job in this lane
       this.scheduleDispatch(lane);
     }
   }
 
+  /**
+   * Periodically check for jobs with expired leases (crashed workers).
+   */
   private async checkExpiredLeases(): Promise<void> {
     try {
       const expired = await findExpiredLeases();
@@ -330,8 +386,12 @@ export function getDurableQueue(): DurableLaneQueue {
   return _durableQueue;
 }
 
-export function initDurableQueue(options?: DurableLaneQueueOptions): DurableLaneQueue {
+export function initDurableQueue(options?: {
+  maxGlobalConcurrency?: number;
+  taskTimeoutMs?: number;
+  jobRetryPolicy?: JobRetryPolicy;
+}): DurableLaneQueue {
   _durableQueue = new DurableLaneQueue(options);
   _durableQueue.start();
   return _durableQueue;
-}
+}

package/packages/core/src/{harness → gateway}/job-store.ts

@@ -1,27 +1,22 @@
 /**
- * job-store — durable persistence + lease/claim for the harness_jobQueue
- * collection. Ported from `hive`'s gateway/job-store.ts.
+ * job-store — durable persistence + lease/claim for the jobQueue collection.
  *
  * All claim transitions use OCC (expectedVersion). The "claim pending→running"
  * path guarantees only one process wins the race for the same job.
  */
 
-import { col, nextId, toIndexable } from "./db-helpers";
-import type { JobDoc } from "./collections";
-import { getBootId } from "./boot-id";
+import { col, nextId, updateDoc, toIndexable } from "../storage/hive";
+import type { JobDoc } from "../storage/collections";
+import { getBootId } from "../storage/boot-id";
 import { logger } from "../utils/logger";
+import { loadConfig } from "../config/loader";
 
-const log = logger.child("harness:job-store");
+const log = logger.child("job-store");
 
-const COLLECTION = "harness_jobQueue";
-const DEFAULT_LEASE_DURATION_MS = 30 * 60 * 1000;
 const MAX_RETRIES = 5;
 
-let leaseDurationMs = DEFAULT_LEASE_DURATION_MS;
-
-/** Override the job lease duration (default 30 minutes). Affects future claims/renewals. */
-export function setJobLeaseDurationMs(ms: number): void {
-  leaseDurationMs = ms;
+function jobLeaseDurationMs(): number {
+  return loadConfig().harness?.jobLeaseMs ?? 30 * 60 * 1000;
 }
 
 export interface JobRetryPolicy {
@@ -40,6 +35,17 @@ export const DEFAULT_JOB_RETRY_POLICY: JobRetryPolicy = {
   jitter: 0.2,
 };
 
+export function loadJobRetryPolicy(): JobRetryPolicy {
+  const cfg = loadConfig().harness?.jobRetry;
+  return {
+    maxRetries: cfg?.maxRetries ?? DEFAULT_JOB_RETRY_POLICY.maxRetries,
+    initialDelayMs: cfg?.initialDelayMs ?? DEFAULT_JOB_RETRY_POLICY.initialDelayMs,
+    backoffMultiplier: cfg?.backoffMultiplier ?? DEFAULT_JOB_RETRY_POLICY.backoffMultiplier,
+    maxDelayMs: cfg?.maxDelayMs ?? DEFAULT_JOB_RETRY_POLICY.maxDelayMs,
+    jitter: cfg?.jitter ?? DEFAULT_JOB_RETRY_POLICY.jitter,
+  };
+}
+
 /** Exponential backoff with full jitter, capped at policy.maxDelayMs. */
 export function computeBackoffDelay(retryCount: number, policy: JobRetryPolicy): number {
   const base = Math.min(policy.maxDelayMs, policy.initialDelayMs * Math.pow(policy.backoffMultiplier, retryCount));
@@ -54,14 +60,14 @@ function occRetryDelay(attempt: number): Promise<void> {
 }
 
 export async function findByIdempotencyKey(key: string): Promise<JobDoc | null> {
-  const c = await col<JobDoc>(COLLECTION);
+  const c = await col<JobDoc>("jobQueue");
   const entries = await c.findBy("idempotency_key", key);
   return entries.length > 0 ? entries[0].doc : null;
 }
 
 export async function createJob(input: {
   lane: string;
-  type: string;
+  type: JobDoc["type"];
   payload: unknown;
   run_id: string;
   priority?: number;
@@ -78,7 +84,7 @@ export async function createJob(input: {
     }
   }
 
-  const id = await nextId(COLLECTION);
+  const id = await nextId("jobQueue");
   const now = Date.now();
   const doc: JobDoc = {
     id,
@@ -102,14 +108,19 @@ export async function createJob(input: {
     last_error: null,
     idempotency_key: toIndexable(input.idempotency_key ?? null),
   };
-  const c = await col<JobDoc>(COLLECTION);
+  const c = await col<JobDoc>("jobQueue");
   await c.put(id, doc, { expectedVersion: 0 });
   log.info(`[createJob] Job ${id} created (lane=${input.lane} type=${input.type})`);
   return doc;
 }
 
+/**
+ * Atomically claim a pending job: transitions status pending→running only if
+ * the version hasn't changed since the read. Returns the claimed doc or null
+ * if another writer won the race.
+ */
 export async function claimJob(jobId: string, bootId: string = getBootId()): Promise<JobDoc | null> {
-  const c = await col<JobDoc>(COLLECTION);
+  const c = await col<JobDoc>("jobQueue");
   for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
     const entry = await c.get(jobId);
     if (!entry) return null;
@@ -123,7 +134,7 @@ export async function claimJob(jobId: string, bootId: string = getBootId()): Pro
       status: "running",
       attempts: doc.attempts + 1,
       boot_id: bootId,
-      lease_expires_at: now + leaseDurationMs,
+      lease_expires_at: now + jobLeaseDurationMs(),
       started_at: doc.started_at ?? now,
     };
     try {
@@ -131,6 +142,7 @@ export async function claimJob(jobId: string, bootId: string = getBootId()): Pro
       log.info(`[claimJob] Job ${jobId} claimed by boot ${bootId}`);
       return updated;
     } catch {
+      // OCC conflict — retry
       await occRetryDelay(attempt);
     }
   }
@@ -138,8 +150,12 @@ export async function claimJob(jobId: string, bootId: string = getBootId()): Pro
   return null;
 }
 
+/**
+ * Renew the lease on a running job to prevent it from being reclaimed by
+ * another process. Uses OCC.
+ */
 export async function renewLease(jobId: string, bootId: string = getBootId()): Promise<boolean> {
-  const c = await col<JobDoc>(COLLECTION);
+  const c = await col<JobDoc>("jobQueue");
   for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
     const entry = await c.get(jobId);
     if (!entry) return false;
@@ -147,11 +163,15 @@ export async function renewLease(jobId: string, bootId: string = getBootId()): P
     if (doc.status !== "running") return false;
     if (doc.boot_id !== bootId) return false;
 
-    const updated: JobDoc = { ...doc, lease_expires_at: Date.now() + leaseDurationMs };
+    const updated: JobDoc = {
+      ...doc,
+      lease_expires_at: Date.now() + jobLeaseDurationMs(),
+    };
     try {
       await c.put(jobId, updated, { expectedVersion: entry.version });
       return true;
     } catch {
+      // OCC conflict — retry
       await occRetryDelay(attempt);
     }
   }
@@ -160,7 +180,7 @@ export async function renewLease(jobId: string, bootId: string = getBootId()): P
 }
 
 export async function completeJob(jobId: string, result: unknown, bootId: string = getBootId()): Promise<void> {
-  const c = await col<JobDoc>(COLLECTION);
+  const c = await col<JobDoc>("jobQueue");
   for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
     const entry = await c.get(jobId);
     if (!entry) return;
@@ -182,6 +202,7 @@ export async function completeJob(jobId: string, result: unknown, bootId: string
       log.info(`[completeJob] Job ${jobId} completed`);
       return;
     } catch {
+      // OCC conflict — retry
       await occRetryDelay(attempt);
     }
   }
@@ -189,7 +210,7 @@ export async function completeJob(jobId: string, result: unknown, bootId: string
 }
 
 export async function failJob(jobId: string, error: string, bootId: string = getBootId()): Promise<void> {
-  const c = await col<JobDoc>(COLLECTION);
+  const c = await col<JobDoc>("jobQueue");
   for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
     const entry = await c.get(jobId);
     if (!entry) return;
@@ -211,6 +232,7 @@ export async function failJob(jobId: string, error: string, bootId: string = get
       log.info(`[failJob] Job ${jobId} failed: ${error}`);
       return;
     } catch {
+      // OCC conflict — retry
       await occRetryDelay(attempt);
     }
   }
@@ -221,7 +243,8 @@ export async function failJob(jobId: string, error: string, bootId: string = get
  * Fail a job that returned a LOGICAL failure ({ok:false}), retrying with
  * exponential backoff + jitter up to `policy.maxRetries` before giving up.
  * Distinct from `attempts`/`reclaimOrInterrupt`, which only handle crash /
- * lease-expiry recovery.
+ * lease-expiry recovery. `chat_turn` jobs must never be routed here (caller's
+ * responsibility) — a user-facing turn should not silently retry later.
  */
 export async function failJobOrRetry(
   jobId: string,
@@ -229,7 +252,7 @@ export async function failJobOrRetry(
   bootId: string = getBootId(),
   policy: JobRetryPolicy = DEFAULT_JOB_RETRY_POLICY
 ): Promise<JobDoc | null> {
-  const c = await col<JobDoc>(COLLECTION);
+  const c = await col<JobDoc>("jobQueue");
   for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
     const entry = await c.get(jobId);
     if (!entry) return null;
@@ -275,6 +298,7 @@ export async function failJobOrRetry(
       log.info(`[failJobOrRetry] Job ${jobId} scheduled for retry ${retryCount + 1}/${policy.maxRetries} in ${delay}ms: ${error}`);
       return updated;
     } catch {
+      // OCC conflict — retry
       await occRetryDelay(attempt);
     }
   }
@@ -287,11 +311,12 @@ export async function failJobOrRetry(
  * attempt count (already bumped at claim time). If attempts >= max_attempts,
  * marks it as interrupted instead.
  *
- * `force` skips the lease-expiry check — used at boot, where every "running"
- * row belongs to a dead process in a single-process HiveDB deployment.
+ * `force` skips the lease-expiry check: at boot every "running" row belongs to
+ * a dead process (HiveDB is single-process), so waiting out a 30-min lease
+ * would only delay recovery.
  */
 export async function reclaimOrInterrupt(jobId: string, opts?: { force?: boolean }): Promise<JobDoc | null> {
-  const c = await col<JobDoc>(COLLECTION);
+  const c = await col<JobDoc>("jobQueue");
   for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
     const entry = await c.get(jobId);
     if (!entry) return null;
@@ -319,12 +344,18 @@ export async function reclaimOrInterrupt(jobId: string, opts?: { force?: boolean
       }
     }
 
-    const updated: JobDoc = { ...doc, status: "pending", boot_id: null, lease_expires_at: null };
+    const updated: JobDoc = {
+      ...doc,
+      status: "pending",
+      boot_id: null,
+      lease_expires_at: null,
+    };
     try {
       await c.put(jobId, updated, { expectedVersion: entry.version });
       log.info(`[reclaimOrInterrupt] Job ${jobId} back to pending (attempt ${doc.attempts}/${doc.max_attempts})`);
       return updated;
     } catch {
+      // OCC conflict — retry
       await occRetryDelay(attempt);
     }
   }
@@ -333,28 +364,38 @@ export async function reclaimOrInterrupt(jobId: string, opts?: { force?: boolean
 }
 
 export async function cancelJob(jobId: string): Promise<boolean> {
-  const c = await col<JobDoc>(COLLECTION);
+  const c = await col<JobDoc>("jobQueue");
   for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
     const entry = await c.get(jobId);
     if (!entry) return false;
     const doc = entry.doc;
     if (doc.status === "completed" || doc.status === "failed" || doc.status === "cancelled") return false;
 
-    const updated: JobDoc = { ...doc, status: "cancelled", finished_at: Date.now(), boot_id: null, lease_expires_at: null };
+    const updated: JobDoc = {
+      ...doc,
+      status: "cancelled",
+      finished_at: Date.now(),
+      boot_id: null,
+      lease_expires_at: null,
+    };
     try {
       await c.put(jobId, updated, { expectedVersion: entry.version });
       log.info(`[cancelJob] Job ${jobId} cancelled`);
       return true;
     } catch {
+      // OCC conflict — retry
       await occRetryDelay(attempt);
     }
   }
   return false;
 }
 
-/** Find the next pending job for a given lane, ordered by priority then creation order. */
+/**
+ * Find the next pending job for a given lane, ordered by priority then
+ * creation order (id is a zero-padded autoincrement → lexical = FIFO).
+ */
 export async function findPendingJobsByLane(lane: string, limit = 10): Promise<JobDoc[]> {
-  const c = await col<JobDoc>(COLLECTION);
+  const c = await col<JobDoc>("jobQueue");
   const entries = await c.findBy("lane", lane);
   return entries
     .filter((e) => e.doc.status === "pending" && e.doc.not_before <= Date.now())
@@ -366,8 +407,12 @@ export async function findPendingJobsByLane(lane: string, limit = 10): Promise<J
     .map((e) => e.doc);
 }
 
+/**
+ * Scan for jobs whose lease has expired (running + lease_expires_at < now)
+ * across the entire collection.
+ */
 export async function findExpiredLeases(): Promise<JobDoc[]> {
-  const c = await col<JobDoc>(COLLECTION);
+  const c = await col<JobDoc>("jobQueue");
   const entries = await c.findBy("status", "running");
   const now = Date.now();
   return entries
@@ -375,25 +420,17 @@ export async function findExpiredLeases(): Promise<JobDoc[]> {
     .map((e) => e.doc);
 }
 
+/**
+ * Find all pending jobs across any lane.
+ */
 export async function findAllPendingJobs(): Promise<JobDoc[]> {
-  const c = await col<JobDoc>(COLLECTION);
+  const c = await col<JobDoc>("jobQueue");
   const entries = await c.findBy("status", "pending");
   return entries.filter((e) => e.doc.not_before <= Date.now()).map((e) => e.doc);
 }
 
 export async function getJob(jobId: string): Promise<JobDoc | null> {
-  const c = await col<JobDoc>(COLLECTION);
+  const c = await col<JobDoc>("jobQueue");
   const entry = await c.get(jobId);
   return entry ? entry.doc : null;
-}
-
-export async function ensureJobStoreIndexes(): Promise<void> {
-  const c = await col<JobDoc>(COLLECTION);
-  await c.createIndex("status");
-  await c.createIndex("lane");
-  await c.createIndex("type");
-  await c.createIndex("run_id");
-  await c.createIndex("idempotency_key");
-}
-
-export { COLLECTION as JOB_QUEUE_COLLECTION };
+}