@johpaz/hive-sdk 0.0.17 → 0.1.3

package/packages/core/src/harness/harness.test.ts

@@ -0,0 +1,236 @@
+/**
+ * Harness module tests — job-store retry/idempotency, durable-queue
+ * dispatch + retry wiring, run-store checkpoint/acceptance/epoch
+ * round-trip, proof-packet persistence, goal-verifier (deterministic
+ * check-tool + acceptance-criteria paths only — no network/LLM calls).
+ *
+ * Uses an isolated HIVE_HOME so this never touches a real dev database.
+ */
+
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import * as path from "node:path";
+
+import { describe, test, expect, beforeEach, afterEach } from "bun:test";
+import { closeHiveDB } from "../storage/HiveDBStorage.ts";
+import {
+  ensureHarnessIndexes,
+  createJob,
+  claimJob,
+  failJobOrRetry,
+  getJob,
+  computeBackoffDelay,
+  findByIdempotencyKey,
+  DurableLaneQueue,
+  registerExecutor,
+  createRun,
+  deserializeAcceptance,
+  deserializeEpoch,
+  buildRunEpoch,
+  buildProofPacket,
+  findProofPacketsByRun,
+  verifyGoal,
+  getBootId,
+  resetBootId,
+  col,
+  type JobRetryPolicy,
+  type JobDoc,
+} from "./index.ts";
+
+// Fresh HiveDB directory per test — hive-sdk's HiveDBStorage has no ":memory:"
+// mode, so isolation means pointing HIVE_HOME at a new temp dir each time and
+// letting the lazy singleton reopen there.
+let currentDir: string;
+
+beforeEach(async () => {
+  closeHiveDB();
+  resetBootId();
+  currentDir = mkdtempSync(path.join(tmpdir(), "hive-sdk-harness-test-"));
+  process.env.HIVE_HOME = currentDir;
+  await ensureHarnessIndexes();
+});
+
+afterEach(() => {
+  closeHiveDB();
+  rmSync(currentDir, { recursive: true, force: true });
+});
+
+const FAST_POLICY: JobRetryPolicy = {
+  maxRetries: 2,
+  initialDelayMs: 10,
+  backoffMultiplier: 2,
+  maxDelayMs: 1000,
+  jitter: 0,
+};
+
+async function waitFor(predicate: () => Promise<boolean>, timeoutMs = 3000): Promise<void> {
+  const start = Date.now();
+  while (Date.now() - start < timeoutMs) {
+    if (await predicate()) return;
+    await new Promise((r) => setTimeout(r, 10));
+  }
+  throw new Error("waitFor timed out");
+}
+
+describe("harness/job-store: idempotency", () => {
+  test("repeated idempotency_key returns the same job", async () => {
+    const first = await createJob({ lane: "s1", type: "worker_task", payload: { n: 1 }, run_id: "r1", idempotency_key: "dedupe-1" });
+    const second = await createJob({ lane: "s1", type: "worker_task", payload: { n: 2 }, run_id: "r1", idempotency_key: "dedupe-1" });
+    expect(second.id).toBe(first.id);
+    expect(JSON.parse(second.payload_json).n).toBe(1);
+  });
+
+  test("findByIdempotencyKey returns null for an unknown key", async () => {
+    expect(await findByIdempotencyKey("nope")).toBeNull();
+  });
+});
+
+describe("harness/job-store: retry/backoff", () => {
+  test("computeBackoffDelay grows exponentially and caps", () => {
+    const policy: JobRetryPolicy = { maxRetries: 10, initialDelayMs: 100, backoffMultiplier: 2, maxDelayMs: 500, jitter: 0 };
+    expect(computeBackoffDelay(0, policy)).toBe(100);
+    expect(computeBackoffDelay(1, policy)).toBe(200);
+    expect(computeBackoffDelay(3, policy)).toBe(500);
+  });
+
+  test("failJobOrRetry schedules a retry, then fails terminally after maxRetries", async () => {
+    const job = await createJob({ lane: "s2", type: "worker_task", payload: {}, run_id: "r2" });
+    const c = await col<JobDoc>("harness_jobQueue");
+    // Force not_before into the past after each retry so claimJob doesn't block on the backoff delay.
+    const forcePastNotBefore = async () => {
+      const entry = await c.get(job.id);
+      await c.put(job.id, { ...entry!.doc, not_before: Date.now() - 1 }, { expectedVersion: entry!.version });
+    };
+
+    await claimJob(job.id);
+    const r1 = await failJobOrRetry(job.id, "boom", getBootId(), FAST_POLICY);
+    expect(r1!.status).toBe("pending");
+    expect(r1!.retry_count).toBe(1);
+    await forcePastNotBefore();
+
+    await claimJob(job.id);
+    const r2 = await failJobOrRetry(job.id, "boom again", getBootId(), FAST_POLICY);
+    expect(r2!.status).toBe("pending");
+    expect(r2!.retry_count).toBe(2);
+    await forcePastNotBefore();
+
+    await claimJob(job.id);
+    const r3 = await failJobOrRetry(job.id, "final", getBootId(), FAST_POLICY);
+    expect(r3!.status).toBe("failed");
+  });
+});
+
+describe("harness/durable-queue", () => {
+  let queue: DurableLaneQueue | null = null;
+
+  afterEach(() => {
+    queue?.stop();
+    queue = null;
+  });
+
+  test("retries a retryable logical failure then completes", async () => {
+    let calls = 0;
+    registerExecutor("flaky_task", async () => {
+      calls++;
+      if (calls < 2) return { ok: false, error: "flaky", retryable: true };
+      return { ok: true, result: "recovered" };
+    });
+
+    queue = new DurableLaneQueue({ maxGlobalConcurrency: 2, jobRetryPolicy: FAST_POLICY });
+    const job = await queue.enqueue({ lane: "lane-flaky", type: "flaky_task", run_id: "r3", payload: {} });
+
+    await waitFor(async () => (await getJob(job.id))?.retry_count === 1);
+    await new Promise((r) => setTimeout(r, 30));
+    queue.start();
+
+    await waitFor(async () => (await getJob(job.id))?.status === "completed");
+    expect(calls).toBe(2);
+  });
+
+  test("nonRetryableTypes never auto-retries a logical failure", async () => {
+    let calls = 0;
+    registerExecutor("chat_turn", async () => {
+      calls++;
+      return { ok: false, error: "user-facing failure" };
+    });
+
+    queue = new DurableLaneQueue({ maxGlobalConcurrency: 2, jobRetryPolicy: FAST_POLICY });
+    const job = await queue.enqueue({ lane: "lane-chat", type: "chat_turn", run_id: "r4", payload: {} });
+
+    await waitFor(async () => (await getJob(job.id))?.status === "failed");
+    expect(calls).toBe(1);
+    expect((await getJob(job.id))!.retry_count).toBe(0);
+  });
+});
+
+describe("harness/run-store: acceptance + epoch round-trip", () => {
+  test("createRun persists and deserializes acceptance criteria and epoch", async () => {
+    const epoch = buildRunEpoch({ provider: "anthropic", model: "claude-x", appVersion: "1.2.3", toolNames: ["search", "exec"] });
+    const run = await createRun({
+      thread_id: "t1",
+      agent_id: "a1",
+      user_id: "u1",
+      channel: null,
+      kind: "goal",
+      max_iterations: 20,
+      acceptance: [
+        { id: "c1", description: "responds in Spanish" },
+        { id: "c2", description: "calls the search tool", checkTool: "search" },
+      ],
+      epoch,
+    });
+
+    const acceptance = deserializeAcceptance(run);
+    expect(acceptance).not.toBeNull();
+    expect(acceptance!.length).toBe(2);
+    expect(acceptance![1].checkTool).toBe("search");
+    expect(deserializeEpoch(run)).toEqual(epoch);
+  });
+});
+
+describe("harness/proof-packet", () => {
+  test("persists and retrieves a proof packet", async () => {
+    const packet = await buildProofPacket({
+      runId: "run-proof-1",
+      agentId: "a1",
+      intendedOutcome: "send the weekly report",
+      met: true,
+      checksRun: ["llm_verifier"],
+      evidence: ["report sent"],
+    });
+    expect(packet.met).toBe(true);
+    const found = await findProofPacketsByRun("run-proof-1");
+    expect(found.length).toBe(1);
+    expect(found[0].id).toBe(packet.id);
+  });
+});
+
+describe("harness/goal-verifier: deterministic check-tool path (no LLM)", () => {
+  test("a single boolean-returning check tool is interpreted directly", async () => {
+    const verdict = await verifyGoal({
+      goal: "workspace exists",
+      checkTool: "check_workspace",
+      messages: [],
+      providerCfg: { provider: "anthropic", model: "x", apiKey: "unused" },
+      runCheckTool: async () => true,
+    });
+    expect(verdict.met).toBe(true);
+  });
+
+  test("acceptance criteria aggregate as a conjunction across check tools", async () => {
+    const verdict = await verifyGoal({
+      goal: "onboard client",
+      messages: [],
+      providerCfg: { provider: "anthropic", model: "x", apiKey: "unused" },
+      runCheckTool: async (tool) => (tool === "check_a" ? true : { met: false, reason: "email not sent" }),
+      acceptance: [
+        { id: "c1", description: "workspace created", checkTool: "check_a" },
+        { id: "c2", description: "welcome email sent", checkTool: "check_b" },
+      ],
+    });
+    expect(verdict.met).toBe(false);
+    expect(verdict.acceptanceResults).toHaveLength(2);
+    expect(verdict.acceptanceResults![0].met).toBe(true);
+    expect(verdict.acceptanceResults![1].met).toBe(false);
+  });
+});

package/packages/core/src/harness/index.ts

@@ -0,0 +1,34 @@
+/**
+ * Hive Harness — durable task execution for the Hive Agent SDK.
+ *
+ * A generic (non-hive-app-specific) durable job queue + checkpointable run
+ * store, backed by HiveDB: retry/backoff on logical failure, lease-based
+ * crash recovery, idempotent job submission, goal verification, and proof
+ * packets. See docs/HIVE-HARNESS.md for the full write-up.
+ *
+ * This module does not wire itself into `AgentRunner` automatically — the
+ * host app decides what "durable" means for its own job types and registers
+ * executors accordingly via `registerExecutor()`.
+ */
+
+import { ensureJobStoreIndexes } from "./job-store";
+import { ensureRunStoreIndexes } from "./run-store";
+import { ensureProofPacketIndexes } from "./proof-packet";
+
+export * from "./collections";
+export * from "./job-store";
+export * from "./run-store";
+export * from "./durable-queue";
+export * from "./goal-verifier";
+export * from "./run-epoch";
+export * from "./proof-packet";
+export * from "./reconcile";
+export { getBootId, resetBootId } from "./boot-id";
+export { col, nextId, updateDoc, findByAny, toIndexable, fromIndexable, NO_PARENT } from "./db-helpers";
+
+/** Create the equality indexes the harness collections need. Idempotent — safe to call on every boot. */
+export async function ensureHarnessIndexes(): Promise<void> {
+  await ensureJobStoreIndexes();
+  await ensureRunStoreIndexes();
+  await ensureProofPacketIndexes();
+}

package/packages/core/src/harness/job-store.ts

@@ -0,0 +1,399 @@
+/**
+ * job-store — durable persistence + lease/claim for the harness_jobQueue
+ * collection. Ported from `hive`'s gateway/job-store.ts.
+ *
+ * All claim transitions use OCC (expectedVersion). The "claim pending→running"
+ * path guarantees only one process wins the race for the same job.
+ */
+
+import { col, nextId, toIndexable } from "./db-helpers";
+import type { JobDoc } from "./collections";
+import { getBootId } from "./boot-id";
+import { logger } from "../utils/logger";
+
+const log = logger.child("harness:job-store");
+
+const COLLECTION = "harness_jobQueue";
+const DEFAULT_LEASE_DURATION_MS = 30 * 60 * 1000;
+const MAX_RETRIES = 5;
+
+let leaseDurationMs = DEFAULT_LEASE_DURATION_MS;
+
+/** Override the job lease duration (default 30 minutes). Affects future claims/renewals. */
+export function setJobLeaseDurationMs(ms: number): void {
+  leaseDurationMs = ms;
+}
+
+export interface JobRetryPolicy {
+  maxRetries: number;
+  initialDelayMs: number;
+  backoffMultiplier: number;
+  maxDelayMs: number;
+  jitter: number;
+}
+
+export const DEFAULT_JOB_RETRY_POLICY: JobRetryPolicy = {
+  maxRetries: 3,
+  initialDelayMs: 1000,
+  backoffMultiplier: 2,
+  maxDelayMs: 5 * 60 * 1000,
+  jitter: 0.2,
+};
+
+/** Exponential backoff with full jitter, capped at policy.maxDelayMs. */
+export function computeBackoffDelay(retryCount: number, policy: JobRetryPolicy): number {
+  const base = Math.min(policy.maxDelayMs, policy.initialDelayMs * Math.pow(policy.backoffMultiplier, retryCount));
+  const jitterAmount = base * policy.jitter * Math.random();
+  return Math.round(base + jitterAmount);
+}
+
+/** Small randomized delay between OCC-conflict retries to reduce thundering-herd contention. */
+function occRetryDelay(attempt: number): Promise<void> {
+  const ms = 5 * (attempt + 1) + Math.random() * 10;
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+export async function findByIdempotencyKey(key: string): Promise<JobDoc | null> {
+  const c = await col<JobDoc>(COLLECTION);
+  const entries = await c.findBy("idempotency_key", key);
+  return entries.length > 0 ? entries[0].doc : null;
+}
+
+export async function createJob(input: {
+  lane: string;
+  type: string;
+  payload: unknown;
+  run_id: string;
+  priority?: number;
+  max_attempts?: number;
+  not_before?: number;
+  /** Client-supplied dedup key: a repeated key returns the existing job instead of creating a new one. */
+  idempotency_key?: string | null;
+}): Promise<JobDoc> {
+  if (input.idempotency_key) {
+    const existing = await findByIdempotencyKey(input.idempotency_key);
+    if (existing) {
+      log.info(`[createJob] Idempotent hit for key=${input.idempotency_key} → job ${existing.id}`);
+      return existing;
+    }
+  }
+
+  const id = await nextId(COLLECTION);
+  const now = Date.now();
+  const doc: JobDoc = {
+    id,
+    lane: input.lane,
+    type: input.type,
+    status: "pending",
+    priority: input.priority ?? 0,
+    payload_json: JSON.stringify(input.payload),
+    run_id: input.run_id,
+    attempts: 0,
+    max_attempts: input.max_attempts ?? 2,
+    not_before: input.not_before ?? now,
+    boot_id: null,
+    lease_expires_at: null,
+    result_json: null,
+    error: null,
+    created_at: now,
+    started_at: null,
+    finished_at: null,
+    retry_count: 0,
+    last_error: null,
+    idempotency_key: toIndexable(input.idempotency_key ?? null),
+  };
+  const c = await col<JobDoc>(COLLECTION);
+  await c.put(id, doc, { expectedVersion: 0 });
+  log.info(`[createJob] Job ${id} created (lane=${input.lane} type=${input.type})`);
+  return doc;
+}
+
+export async function claimJob(jobId: string, bootId: string = getBootId()): Promise<JobDoc | null> {
+  const c = await col<JobDoc>(COLLECTION);
+  for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
+    const entry = await c.get(jobId);
+    if (!entry) return null;
+    const doc = entry.doc;
+    if (doc.status !== "pending") return null;
+    if (doc.not_before > Date.now()) return null;
+
+    const now = Date.now();
+    const updated: JobDoc = {
+      ...doc,
+      status: "running",
+      attempts: doc.attempts + 1,
+      boot_id: bootId,
+      lease_expires_at: now + leaseDurationMs,
+      started_at: doc.started_at ?? now,
+    };
+    try {
+      await c.put(jobId, updated, { expectedVersion: entry.version });
+      log.info(`[claimJob] Job ${jobId} claimed by boot ${bootId}`);
+      return updated;
+    } catch {
+      await occRetryDelay(attempt);
+    }
+  }
+  log.warn(`[claimJob] Too much contention on job ${jobId}`);
+  return null;
+}
+
+export async function renewLease(jobId: string, bootId: string = getBootId()): Promise<boolean> {
+  const c = await col<JobDoc>(COLLECTION);
+  for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
+    const entry = await c.get(jobId);
+    if (!entry) return false;
+    const doc = entry.doc;
+    if (doc.status !== "running") return false;
+    if (doc.boot_id !== bootId) return false;
+
+    const updated: JobDoc = { ...doc, lease_expires_at: Date.now() + leaseDurationMs };
+    try {
+      await c.put(jobId, updated, { expectedVersion: entry.version });
+      return true;
+    } catch {
+      await occRetryDelay(attempt);
+    }
+  }
+  log.warn(`[renewLease] Too much contention on job ${jobId}`);
+  return false;
+}
+
+export async function completeJob(jobId: string, result: unknown, bootId: string = getBootId()): Promise<void> {
+  const c = await col<JobDoc>(COLLECTION);
+  for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
+    const entry = await c.get(jobId);
+    if (!entry) return;
+    const doc = entry.doc;
+    if (doc.status !== "running") return;
+    if (doc.boot_id !== bootId) return;
+
+    const updated: JobDoc = {
+      ...doc,
+      status: "completed",
+      result_json: JSON.stringify(result),
+      error: null,
+      finished_at: Date.now(),
+      boot_id: null,
+      lease_expires_at: null,
+    };
+    try {
+      await c.put(jobId, updated, { expectedVersion: entry.version });
+      log.info(`[completeJob] Job ${jobId} completed`);
+      return;
+    } catch {
+      await occRetryDelay(attempt);
+    }
+  }
+  log.warn(`[completeJob] Too much contention on job ${jobId}`);
+}
+
+export async function failJob(jobId: string, error: string, bootId: string = getBootId()): Promise<void> {
+  const c = await col<JobDoc>(COLLECTION);
+  for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
+    const entry = await c.get(jobId);
+    if (!entry) return;
+    const doc = entry.doc;
+    if (doc.status !== "running") return;
+    if (doc.boot_id !== bootId) return;
+
+    const updated: JobDoc = {
+      ...doc,
+      status: "failed",
+      error,
+      last_error: error,
+      finished_at: Date.now(),
+      boot_id: null,
+      lease_expires_at: null,
+    };
+    try {
+      await c.put(jobId, updated, { expectedVersion: entry.version });
+      log.info(`[failJob] Job ${jobId} failed: ${error}`);
+      return;
+    } catch {
+      await occRetryDelay(attempt);
+    }
+  }
+  log.warn(`[failJob] Too much contention on job ${jobId}`);
+}
+
+/**
+ * Fail a job that returned a LOGICAL failure ({ok:false}), retrying with
+ * exponential backoff + jitter up to `policy.maxRetries` before giving up.
+ * Distinct from `attempts`/`reclaimOrInterrupt`, which only handle crash /
+ * lease-expiry recovery.
+ */
+export async function failJobOrRetry(
+  jobId: string,
+  error: string,
+  bootId: string = getBootId(),
+  policy: JobRetryPolicy = DEFAULT_JOB_RETRY_POLICY
+): Promise<JobDoc | null> {
+  const c = await col<JobDoc>(COLLECTION);
+  for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
+    const entry = await c.get(jobId);
+    if (!entry) return null;
+    const doc = entry.doc;
+    if (doc.status !== "running") return null;
+    if (doc.boot_id !== bootId) return null;
+
+    const now = Date.now();
+    const retryCount = doc.retry_count ?? 0;
+
+    if (retryCount >= policy.maxRetries) {
+      const updated: JobDoc = {
+        ...doc,
+        status: "failed",
+        error,
+        last_error: error,
+        finished_at: now,
+        boot_id: null,
+        lease_expires_at: null,
+      };
+      try {
+        await c.put(jobId, updated, { expectedVersion: entry.version });
+        log.info(`[failJobOrRetry] Job ${jobId} failed terminally after ${retryCount} retr${retryCount === 1 ? "y" : "ies"}: ${error}`);
+        return updated;
+      } catch {
+        await occRetryDelay(attempt);
+        continue;
+      }
+    }
+
+    const delay = computeBackoffDelay(retryCount, policy);
+    const updated: JobDoc = {
+      ...doc,
+      status: "pending",
+      retry_count: retryCount + 1,
+      last_error: error,
+      not_before: now + delay,
+      boot_id: null,
+      lease_expires_at: null,
+    };
+    try {
+      await c.put(jobId, updated, { expectedVersion: entry.version });
+      log.info(`[failJobOrRetry] Job ${jobId} scheduled for retry ${retryCount + 1}/${policy.maxRetries} in ${delay}ms: ${error}`);
+      return updated;
+    } catch {
+      await occRetryDelay(attempt);
+    }
+  }
+  log.warn(`[failJobOrRetry] Too much contention on job ${jobId}`);
+  return null;
+}
+
+/**
+ * Re-enqueue a job whose lease has expired back to pending, bumping its
+ * attempt count (already bumped at claim time). If attempts >= max_attempts,
+ * marks it as interrupted instead.
+ *
+ * `force` skips the lease-expiry check — used at boot, where every "running"
+ * row belongs to a dead process in a single-process HiveDB deployment.
+ */
+export async function reclaimOrInterrupt(jobId: string, opts?: { force?: boolean }): Promise<JobDoc | null> {
+  const c = await col<JobDoc>(COLLECTION);
+  for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
+    const entry = await c.get(jobId);
+    if (!entry) return null;
+    const doc = entry.doc;
+    if (doc.status !== "running") return null;
+    if (!opts?.force && (doc.lease_expires_at === null || doc.lease_expires_at > Date.now())) return null;
+
+    const now = Date.now();
+    if (doc.attempts >= doc.max_attempts) {
+      const updated: JobDoc = {
+        ...doc,
+        status: "interrupted",
+        error: `Max attempts (${doc.max_attempts}) reached after lease expiry`,
+        finished_at: now,
+        boot_id: null,
+        lease_expires_at: null,
+      };
+      try {
+        await c.put(jobId, updated, { expectedVersion: entry.version });
+        log.warn(`[reclaimOrInterrupt] Job ${jobId} interrupted (attempts exhausted)`);
+        return updated;
+      } catch {
+        await occRetryDelay(attempt);
+        continue;
+      }
+    }
+
+    const updated: JobDoc = { ...doc, status: "pending", boot_id: null, lease_expires_at: null };
+    try {
+      await c.put(jobId, updated, { expectedVersion: entry.version });
+      log.info(`[reclaimOrInterrupt] Job ${jobId} back to pending (attempt ${doc.attempts}/${doc.max_attempts})`);
+      return updated;
+    } catch {
+      await occRetryDelay(attempt);
+    }
+  }
+  log.warn(`[reclaimOrInterrupt] Too much contention on job ${jobId}`);
+  return null;
+}
+
+export async function cancelJob(jobId: string): Promise<boolean> {
+  const c = await col<JobDoc>(COLLECTION);
+  for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
+    const entry = await c.get(jobId);
+    if (!entry) return false;
+    const doc = entry.doc;
+    if (doc.status === "completed" || doc.status === "failed" || doc.status === "cancelled") return false;
+
+    const updated: JobDoc = { ...doc, status: "cancelled", finished_at: Date.now(), boot_id: null, lease_expires_at: null };
+    try {
+      await c.put(jobId, updated, { expectedVersion: entry.version });
+      log.info(`[cancelJob] Job ${jobId} cancelled`);
+      return true;
+    } catch {
+      await occRetryDelay(attempt);
+    }
+  }
+  return false;
+}
+
+/** Find the next pending job for a given lane, ordered by priority then creation order. */
+export async function findPendingJobsByLane(lane: string, limit = 10): Promise<JobDoc[]> {
+  const c = await col<JobDoc>(COLLECTION);
+  const entries = await c.findBy("lane", lane);
+  return entries
+    .filter((e) => e.doc.status === "pending" && e.doc.not_before <= Date.now())
+    .sort((a, b) => {
+      if (b.doc.priority !== a.doc.priority) return b.doc.priority - a.doc.priority;
+      return a.doc.id.localeCompare(b.doc.id);
+    })
+    .slice(0, limit)
+    .map((e) => e.doc);
+}
+
+export async function findExpiredLeases(): Promise<JobDoc[]> {
+  const c = await col<JobDoc>(COLLECTION);
+  const entries = await c.findBy("status", "running");
+  const now = Date.now();
+  return entries
+    .filter((e) => e.doc.lease_expires_at !== null && (e.doc.lease_expires_at as number) < now)
+    .map((e) => e.doc);
+}
+
+export async function findAllPendingJobs(): Promise<JobDoc[]> {
+  const c = await col<JobDoc>(COLLECTION);
+  const entries = await c.findBy("status", "pending");
+  return entries.filter((e) => e.doc.not_before <= Date.now()).map((e) => e.doc);
+}
+
+export async function getJob(jobId: string): Promise<JobDoc | null> {
+  const c = await col<JobDoc>(COLLECTION);
+  const entry = await c.get(jobId);
+  return entry ? entry.doc : null;
+}
+
+export async function ensureJobStoreIndexes(): Promise<void> {
+  const c = await col<JobDoc>(COLLECTION);
+  await c.createIndex("status");
+  await c.createIndex("lane");
+  await c.createIndex("type");
+  await c.createIndex("run_id");
+  await c.createIndex("idempotency_key");
+}
+
+export { COLLECTION as JOB_QUEUE_COLLECTION };