@absurd-sqlite/bun-worker 0.1.0-alpha.0

package/test/sqlite.test.ts

@@ -0,0 +1,89 @@
+import { Database } from "bun:sqlite";
+import { describe, expect, it } from "bun:test";
+
+import { BunSqliteConnection } from "../src/sqlite";
+
+describe("BunSqliteConnection", () => {
+  it("rewrites postgres-style params and absurd schema names", async () => {
+    const db = new Database(":memory:");
+    const conn = new BunSqliteConnection(db);
+
+    await conn.exec("CREATE TABLE absurd_tasks (id, name)");
+    await conn.exec("INSERT INTO absurd.tasks (id, name) VALUES ($1, $2)", [
+      1,
+      "alpha",
+    ]);
+
+    const { rows } = await conn.query<{ id: number; name: string }>(
+      "SELECT id, name FROM absurd.tasks WHERE id = $1",
+      [1]
+    );
+
+    expect(rows).toEqual([{ id: 1, name: "alpha" }]);
+    db.close();
+  });
+
+  it("returns empty rows for non-reader statements", async () => {
+    const db = new Database(":memory:");
+    const conn = new BunSqliteConnection(db);
+
+    const { rows } = await conn.query("CREATE TABLE t (id)");
+    expect(rows).toEqual([]);
+
+    await conn.exec("INSERT INTO t (id) VALUES ($1)", [1]);
+    const { rows: inserted } = await conn.query<{ id: number }>(
+      "SELECT id FROM t"
+    );
+    expect(inserted).toEqual([{ id: 1 }]);
+    db.close();
+  });
+
+  it("decodes JSON from typeless columns", async () => {
+    const db = new Database(":memory:");
+    const conn = new BunSqliteConnection(db);
+
+    await conn.exec("CREATE TABLE t (payload)");
+    await conn.exec("INSERT INTO t (payload) VALUES ($1)", ['{"a":1}']);
+
+    const { rows } = await conn.query<{ payload: { a: number } }>(
+      "SELECT payload FROM t"
+    );
+
+    expect(rows[0]?.payload).toEqual({ a: 1 });
+    db.close();
+  });
+
+  it("decodes JSON from blob columns", async () => {
+    const db = new Database(":memory:");
+    const conn = new BunSqliteConnection(db);
+
+    await conn.exec("CREATE TABLE t_blob (payload BLOB)");
+    await conn.exec("INSERT INTO t_blob (payload) VALUES ($1)", [
+      Buffer.from(JSON.stringify({ b: 2 })),
+    ]);
+
+    const { rows } = await conn.query<{ payload: { b: number } }>(
+      "SELECT payload FROM t_blob"
+    );
+
+    expect(rows[0]?.payload).toEqual({ b: 2 });
+    db.close();
+  });
+
+  it("decodes datetime columns into Date objects", async () => {
+    const db = new Database(":memory:");
+    const conn = new BunSqliteConnection(db);
+    const now = Date.now();
+
+    await conn.exec("CREATE TABLE t_date (created_at DATETIME)");
+    await conn.exec("INSERT INTO t_date (created_at) VALUES ($1)", [now]);
+
+    const { rows } = await conn.query<{ created_at: Date }>(
+      "SELECT created_at FROM t_date"
+    );
+
+    expect(rows[0]?.created_at).toBeInstanceOf(Date);
+    expect(rows[0]?.created_at.getTime()).toBe(now);
+    db.close();
+  });
+});

package/test/step.test.ts

@@ -0,0 +1,259 @@
+import { describe, test, expect, beforeAll, afterEach, jest } from "bun:test";
+import type { Absurd } from "absurd-sdk";
+import { createTestAbsurd, randomName, type TestContext } from "./setup";
+
+describe("Step functionality", () => {
+  let ctx: TestContext;
+  let absurd: Absurd;
+
+  beforeAll(async () => {
+    ctx = await createTestAbsurd(randomName("step_queue"));
+    absurd = ctx.absurd;
+  });
+
+  afterEach(async () => {
+    await ctx.cleanupTasks();
+    await ctx.setFakeNow(null);
+    jest.useRealTimers();
+  });
+
+  test("step executes and returns value", async () => {
+    absurd.registerTask<{ value: number }, { result: string }>(
+      { name: "basic" },
+      async (params, ctx) => {
+        const result = await ctx.step("process", async () => {
+          return `processed-${params.value}`;
+        });
+        return { result };
+      }
+    );
+
+    const { taskID } = await absurd.spawn("basic", { value: 42 });
+    await absurd.workBatch(randomName("w"), 60, 1);
+
+    expect(await ctx.getTask(taskID)).toMatchObject({
+      state: "completed",
+      completed_payload: { result: "processed-42" },
+    });
+  });
+
+  test("step result is cached and not re-executed on retry", async () => {
+    let executionCount = 0;
+    let attemptCount = 0;
+
+    absurd.registerTask<void, { random: number; count: number }>(
+      { name: "cache", defaultMaxAttempts: 2 },
+      async (params, ctx) => {
+        attemptCount++;
+
+        const cached = await ctx.step("generate-random", async () => {
+          executionCount++;
+          return Math.random();
+        });
+
+        if (attemptCount === 1) {
+          throw new Error("Intentional failure");
+        }
+
+        return { random: cached, count: executionCount };
+      }
+    );
+
+    const { taskID } = await absurd.spawn("cache", undefined);
+
+    const workerID = randomName("w");
+    await absurd.workBatch(workerID, 60, 1);
+    expect(executionCount).toBe(1);
+
+    await absurd.workBatch(workerID, 60, 1);
+    expect(executionCount).toBe(1);
+    expect(attemptCount).toBe(2);
+
+    expect(await ctx.getTask(taskID)).toMatchObject({
+      state: "completed",
+      completed_payload: { count: 1 },
+      attempts: 2,
+    });
+  });
+
+  test("task with multiple steps only re-executes uncompleted steps on retry", async () => {
+    const executed: string[] = [];
+    let attemptCount = 0;
+
+    absurd.registerTask<void, { steps: string[]; attemptNum: number }>(
+      { name: "multistep-retry", defaultMaxAttempts: 2 },
+      async (params, ctx) => {
+        attemptCount++;
+
+        const step1 = await ctx.step("step1", async () => {
+          executed.push("step1");
+          return "result1";
+        });
+
+        const step2 = await ctx.step("step2", async () => {
+          executed.push("step2");
+          return "result2";
+        });
+
+        if (attemptCount === 1) {
+          throw new Error("Fail before step3");
+        }
+
+        const step3 = await ctx.step("step3", async () => {
+          executed.push("step3");
+          return "result3";
+        });
+
+        return { steps: [step1, step2, step3], attemptNum: attemptCount };
+      }
+    );
+
+    const { taskID } = await absurd.spawn("multistep-retry", undefined);
+
+    const workerID = randomName("w");
+    await absurd.workBatch(workerID, 60, 1);
+    expect(executed).toEqual(["step1", "step2"]);
+
+    await absurd.workBatch(workerID, 60, 1);
+    expect(executed).toEqual(["step1", "step2", "step3"]);
+
+    expect(await ctx.getTask(taskID)).toMatchObject({
+      state: "completed",
+      completed_payload: {
+        steps: ["result1", "result2", "result3"],
+        attemptNum: 2,
+      },
+      attempts: 2,
+    });
+  });
+
+  test("repeated step names work correctly", async () => {
+    absurd.registerTask<void, { results: number[] }>(
+      { name: "deduplicate" },
+      async (params, ctx) => {
+        const results: number[] = [];
+        for (let i = 0; i < 3; i++) {
+          const result = await ctx.step("loop-step", async () => {
+            return i * 10;
+          });
+          results.push(result);
+        }
+        return { results };
+      }
+    );
+
+    const { taskID } = await absurd.spawn("deduplicate", undefined);
+    await absurd.workBatch(randomName("w"), 60, 1);
+
+    expect(await ctx.getTask(taskID)).toMatchObject({
+      state: "completed",
+      completed_payload: { results: [0, 10, 20] },
+    });
+  });
+
+  test("failed step does not save checkpoint and re-executes on retry", async () => {
+    let attemptCount = 0;
+
+    absurd.registerTask<void, { result: string }>(
+      { name: "fail", defaultMaxAttempts: 2 },
+      async (_, ctx) => {
+        const result = await ctx.step("fail", async () => {
+          attemptCount++;
+          if (attemptCount === 1) {
+            throw new Error("Step fails on first attempt");
+          }
+          return "success";
+        });
+
+        return { result };
+      }
+    );
+
+    const { taskID } = await absurd.spawn("fail", undefined);
+
+    const workerID = randomName("w");
+    await absurd.workBatch(workerID, 60, 1);
+    expect(attemptCount).toBe(1);
+
+    await absurd.workBatch(workerID, 60, 1);
+    expect(attemptCount).toBe(2);
+
+    expect(await ctx.getTask(taskID)).toMatchObject({
+      state: "completed",
+      completed_payload: { result: "success" },
+      attempts: 2,
+    });
+  });
+
+  test("sleepFor suspends until duration elapses", async () => {
+    jest.useFakeTimers();
+    const base = new Date("2024-05-05T10:00:00Z");
+    jest.setSystemTime(base);
+    await ctx.setFakeNow(base);
+
+    const durationSeconds = 60;
+    absurd.registerTask({ name: "sleep-for" }, async (_params, ctx) => {
+      await ctx.sleepFor("wait-for", durationSeconds);
+      return { resumed: true };
+    });
+
+    const { taskID, runID } = await absurd.spawn("sleep-for", undefined);
+    await absurd.workBatch("worker-sleep", 120, 1);
+
+    const sleepingRun = await ctx.getRun(runID);
+    expect(sleepingRun).toMatchObject({
+      state: "sleeping",
+    });
+    const wakeTime = new Date(base.getTime() + durationSeconds * 1000);
+    expect(sleepingRun?.available_at?.getTime()).toBe(wakeTime.getTime());
+
+    const resumeTime = new Date(wakeTime.getTime() + 5 * 1000);
+    jest.setSystemTime(resumeTime);
+    await ctx.setFakeNow(resumeTime);
+    await absurd.workBatch("worker-sleep", 120, 1);
+
+    expect(await ctx.getTask(taskID)).toMatchObject({
+      state: "completed",
+      completed_payload: { resumed: true },
+    });
+  });
+
+  test("sleepUntil checkpoint prevents re-scheduling when wake time passed", async () => {
+    jest.useFakeTimers();
+    const base = new Date("2024-05-06T09:00:00Z");
+    jest.setSystemTime(base);
+    await ctx.setFakeNow(base);
+
+    const wakeTime = new Date(base.getTime() + 5 * 60 * 1000);
+    let executions = 0;
+
+    absurd.registerTask({ name: "sleep-until" }, async (_params, ctx) => {
+      executions++;
+      await ctx.sleepUntil("sleep-step", wakeTime);
+      return { executions };
+    });
+
+    const { taskID, runID } = await absurd.spawn("sleep-until", undefined);
+    await absurd.workBatch("worker-sleep", 120, 1);
+
+    const checkpointRow = await ctx.getCheckpoint(taskID, "sleep-step");
+    expect(checkpointRow).toMatchObject({
+      checkpoint_name: "sleep-step",
+      owner_run_id: runID,
+      state: wakeTime.toISOString(),
+    });
+
+    const sleepingRun = await ctx.getRun(runID);
+    expect(sleepingRun?.state).toBe("sleeping");
+
+    jest.setSystemTime(wakeTime);
+    await ctx.setFakeNow(wakeTime);
+    await absurd.workBatch("worker-sleep", 120, 1);
+
+    expect(await ctx.getTask(taskID)).toMatchObject({
+      state: "completed",
+      completed_payload: { executions: 2 },
+    });
+    expect(executions).toBe(2);
+  });
+});

package/test/wait-for.ts

@@ -0,0 +1,21 @@
+export async function waitFor(
+  fn: () => void | Promise<void>,
+  options?: { timeout?: number; interval?: number }
+): Promise<void> {
+  const timeout = options?.timeout ?? 1000;
+  const interval = options?.interval ?? 10;
+  const start = Date.now();
+
+  // Poll until the expectation stops throwing or we time out.
+  while (true) {
+    try {
+      await fn();
+      return;
+    } catch (err) {
+      if (Date.now() - start >= timeout) {
+        throw err;
+      }
+      await new Promise((resolve) => setTimeout(resolve, interval));
+    }
+  }
+}

package/test/worker.test.ts

@@ -0,0 +1,194 @@
+import { describe, it, expect, beforeAll, afterEach, jest } from "bun:test";
+import { EventEmitter, once } from "events";
+import type { TestContext } from "./setup";
+import { createTestAbsurd, randomName } from "./setup";
+import type { Absurd } from "absurd-sdk";
+import { waitFor } from "./wait-for";
+
+describe("Worker", () => {
+  let ctx: TestContext;
+  let absurd: Absurd;
+
+  beforeAll(async () => {
+    ctx = await createTestAbsurd(randomName("worker"));
+    absurd = ctx.absurd;
+  });
+
+  afterEach(async () => {
+    await ctx.cleanupTasks();
+    jest.restoreAllMocks();
+  });
+
+  it("respects concurrency limit and skips claims when at capacity", async () => {
+    const gate = new EventEmitter();
+    const atGate = new Set<number>();
+
+    gate.on("arrived", (id: number) => atGate.add(id));
+    gate.on("left", (id: number) => atGate.delete(id));
+
+    absurd.registerTask<{ id: number }, void>(
+      { name: "gated-task" },
+      async (params) => {
+        gate.emit("arrived", params.id);
+        await once(gate, "release");
+        gate.emit("left", params.id);
+      },
+    );
+
+    for (let i = 1; i <= 5; i++) {
+      await absurd.spawn("gated-task", { id: i });
+    }
+
+    const claimSpy = jest.spyOn(absurd, "claimTasks");
+
+    const concurrency = 2;
+    const worker = await absurd.startWorker({
+      concurrency,
+      pollInterval: 0.01,
+    });
+
+    // Wait until we are at capacity
+    await waitFor(() => expect(atGate.size).toBe(concurrency), {
+      timeout: 100,
+    });
+
+    expect(claimSpy).toHaveBeenCalledTimes(1);
+    expect(claimSpy).toHaveBeenCalledWith(
+      expect.objectContaining({ batchSize: concurrency }),
+    );
+
+    // Wait for ~5 poll cycles
+    await ctx.sleep(50);
+
+    // Things should still be the same, no more claims
+    expect(atGate.size).toBe(concurrency);
+    expect(claimSpy).toHaveBeenCalledTimes(1);
+
+    gate.emit("release");
+    await worker.close();
+    expect(atGate.size).toBe(0);
+  });
+
+  it("polls again immediately when capacity frees up", async () => {
+    const gate = new EventEmitter();
+    const atGate = new Set<number>();
+
+    gate.on("arrived", (id: number) => atGate.add(id));
+    gate.on("left", (id: number) => atGate.delete(id));
+
+    absurd.registerTask<{ id: number }, void>(
+      { name: "responsive-task" },
+      async (params) => {
+        gate.emit("arrived", params.id);
+        await once(gate, "release");
+        gate.emit("left", params.id);
+      },
+    );
+
+    for (let i = 1; i <= 3; i++) {
+      await absurd.spawn("responsive-task", { id: i });
+    }
+
+    const claimSpy = jest.spyOn(absurd, "claimTasks");
+
+    const worker = await absurd.startWorker({
+      concurrency: 2,
+      pollInterval: 3600, // once every hour
+    });
+
+    await waitFor(() => expect(atGate.size).toBe(2), {
+      timeout: 100,
+    });
+
+    expect(claimSpy).toHaveBeenCalledTimes(1);
+    const initialCallCount = claimSpy.mock.calls.length;
+
+    // Free current tasks so the worker can immediately claim again
+    gate.emit("release");
+
+    // Polls again without waiting for pollInterval
+    await waitFor(() => {
+      expect(claimSpy.mock.calls.length).toBeGreaterThan(initialCallCount);
+    });
+
+    // Release any remaining tasks so the worker can shut down
+    gate.emit("release");
+    await worker.close();
+  });
+
+  it("shuts down gracefully", async () => {
+    const gate = new EventEmitter();
+    const started: number[] = [];
+
+    gate.on("started", (id: number) => started.push(id));
+
+    absurd.registerTask<{ id: number }, void>(
+      { name: "shutdown-task" },
+      async (params) => {
+        gate.emit("started", params.id);
+        await once(gate, "release");
+      },
+    );
+
+    await absurd.spawn("shutdown-task", { id: 1 });
+    await absurd.spawn("shutdown-task", { id: 2 });
+
+    const worker = await absurd.startWorker({ concurrency: 1 });
+    await once(gate, "started");
+
+    const closePromise = worker.close();
+    const claimSpy = jest.spyOn(absurd, "claimTasks");
+
+    let resolved = false;
+    closePromise.then(() => {
+      resolved = true;
+    });
+
+    // Does not resolve until the task finishes
+    await ctx.sleep(50);
+    expect(resolved).toBe(false);
+
+    gate.emit("release");
+
+    await waitFor(() => expect(resolved).toBe(true), {
+      timeout: 200,
+    });
+
+    expect(started).toEqual([1]);
+    expect(claimSpy).not.toHaveBeenCalled();
+  });
+
+  it("calls onError for unexpected failures", async () => {
+    const errors: string[] = [];
+
+    absurd.registerTask<void, void>({ name: "test-task" }, async () => {});
+    await absurd.spawn("test-task", undefined);
+
+    jest.spyOn(absurd, "executeTask").mockRejectedValueOnce(
+      new Error("Execute failed, PG error"),
+    );
+
+    const worker = await absurd.startWorker({
+      pollInterval: 0.01,
+      onError: (err) => errors.push(err.message),
+    });
+
+    await waitFor(
+      () => expect(errors).toContain("Execute failed, PG error"),
+      {
+        timeout: 200,
+      },
+    );
+
+    jest.spyOn(absurd, "claimTasks").mockRejectedValue(new Error("Claim failed"));
+
+    await waitFor(() => expect(errors).toContain("Claim failed"), {
+      timeout: 200,
+    });
+
+    await worker.close();
+
+    expect(errors).toContain("Execute failed, PG error");
+    expect(errors).toContain("Claim failed");
+  });
+});

package/tsconfig.build.json

@@ -0,0 +1,12 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "noEmit": false,
+    "declaration": true,
+    "declarationMap": true,
+    "emitDeclarationOnly": true,
+    "outDir": "dist",
+    "rootDir": "src"
+  },
+  "include": ["src"]
+}

package/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "ESNext",
+    "moduleResolution": "Bundler",
+    "strict": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "noEmit": true,
+    "types": ["bun-types"]
+  },
+  "include": ["src", "test"]
+}