@absurd-sqlite/sdk 0.2.0-alpha.0

package/test/step.test.ts

@@ -0,0 +1,259 @@
+import { describe, test, expect, beforeAll, afterEach, vi } from "vitest";
+import { createTestAbsurd, randomName, type TestContext } from "./setup.js";
+import type { Absurd } from "../src/index.js";
+
+describe("Step functionality", () => {
+  let ctx: TestContext;
+  let absurd: Absurd;
+
+  beforeAll(async () => {
+    ctx = await createTestAbsurd(randomName("step_queue"));
+    absurd = ctx.absurd;
+  });
+
+  afterEach(async () => {
+    await ctx.cleanupTasks();
+    await ctx.setFakeNow(null);
+    vi.useRealTimers();
+  });
+
+  test("step executes and returns value", async () => {
+    absurd.registerTask<{ value: number }, { result: string }>(
+      { name: "basic" },
+      async (params, ctx) => {
+        const result = await ctx.step("process", async () => {
+          return `processed-${params.value}`;
+        });
+        return { result };
+      }
+    );
+
+    const { taskID } = await absurd.spawn("basic", { value: 42 });
+    await absurd.workBatch(randomName("w"), 60, 1);
+
+    expect(await ctx.getTask(taskID)).toMatchObject({
+      state: "completed",
+      completed_payload: { result: "processed-42" },
+    });
+  });
+
+  test("step result is cached and not re-executed on retry", async () => {
+    let executionCount = 0;
+    let attemptCount = 0;
+
+    absurd.registerTask<void, { random: number; count: number }>(
+      { name: "cache", defaultMaxAttempts: 2 },
+      async (params, ctx) => {
+        attemptCount++;
+
+        const cached = await ctx.step("generate-random", async () => {
+          executionCount++;
+          return Math.random();
+        });
+
+        if (attemptCount === 1) {
+          throw new Error("Intentional failure");
+        }
+
+        return { random: cached, count: executionCount };
+      }
+    );
+
+    const { taskID } = await absurd.spawn("cache", undefined);
+
+    const workerID = randomName("w");
+    await absurd.workBatch(workerID, 60, 1);
+    expect(executionCount).toBe(1);
+
+    await absurd.workBatch(workerID, 60, 1);
+    expect(executionCount).toBe(1);
+    expect(attemptCount).toBe(2);
+
+    expect(await ctx.getTask(taskID)).toMatchObject({
+      state: "completed",
+      completed_payload: { count: 1 },
+      attempts: 2,
+    });
+  });
+
+  test("task with multiple steps only re-executes uncompleted steps on retry", async () => {
+    const executed: string[] = [];
+    let attemptCount = 0;
+
+    absurd.registerTask<void, { steps: string[]; attemptNum: number }>(
+      { name: "multistep-retry", defaultMaxAttempts: 2 },
+      async (params, ctx) => {
+        attemptCount++;
+
+        const step1 = await ctx.step("step1", async () => {
+          executed.push("step1");
+          return "result1";
+        });
+
+        const step2 = await ctx.step("step2", async () => {
+          executed.push("step2");
+          return "result2";
+        });
+
+        if (attemptCount === 1) {
+          throw new Error("Fail before step3");
+        }
+
+        const step3 = await ctx.step("step3", async () => {
+          executed.push("step3");
+          return "result3";
+        });
+
+        return { steps: [step1, step2, step3], attemptNum: attemptCount };
+      }
+    );
+
+    const { taskID } = await absurd.spawn("multistep-retry", undefined);
+
+    const workerID = randomName("w");
+    await absurd.workBatch(workerID, 60, 1);
+    expect(executed).toEqual(["step1", "step2"]);
+
+    await absurd.workBatch(workerID, 60, 1);
+    expect(executed).toEqual(["step1", "step2", "step3"]);
+
+    expect(await ctx.getTask(taskID)).toMatchObject({
+      state: "completed",
+      completed_payload: {
+        steps: ["result1", "result2", "result3"],
+        attemptNum: 2,
+      },
+      attempts: 2,
+    });
+  });
+
+  test("repeated step names work correctly", async () => {
+    absurd.registerTask<void, { results: number[] }>(
+      { name: "deduplicate" },
+      async (params, ctx) => {
+        const results: number[] = [];
+        for (let i = 0; i < 3; i++) {
+          const result = await ctx.step("loop-step", async () => {
+            return i * 10;
+          });
+          results.push(result);
+        }
+        return { results };
+      }
+    );
+
+    const { taskID } = await absurd.spawn("deduplicate", undefined);
+    await absurd.workBatch(randomName("w"), 60, 1);
+
+    expect(await ctx.getTask(taskID)).toMatchObject({
+      state: "completed",
+      completed_payload: { results: [0, 10, 20] },
+    });
+  });
+
+  test("failed step does not save checkpoint and re-executes on retry", async () => {
+    let attemptCount = 0;
+
+    absurd.registerTask<void, { result: string }>(
+      { name: "fail", defaultMaxAttempts: 2 },
+      async (_, ctx) => {
+        const result = await ctx.step("fail", async () => {
+          attemptCount++;
+          if (attemptCount === 1) {
+            throw new Error("Step fails on first attempt");
+          }
+          return "success";
+        });
+
+        return { result };
+      }
+    );
+
+    const { taskID } = await absurd.spawn("fail", undefined);
+
+    const workerID = randomName("w");
+    await absurd.workBatch(workerID, 60, 1);
+    expect(attemptCount).toBe(1);
+
+    await absurd.workBatch(workerID, 60, 1);
+    expect(attemptCount).toBe(2);
+
+    expect(await ctx.getTask(taskID)).toMatchObject({
+      state: "completed",
+      completed_payload: { result: "success" },
+      attempts: 2,
+    });
+  });
+
+  test("sleepFor suspends until duration elapses", async () => {
+    vi.useFakeTimers();
+    const base = new Date("2024-05-05T10:00:00Z");
+    vi.setSystemTime(base);
+    await ctx.setFakeNow(base);
+
+    const durationSeconds = 60;
+    absurd.registerTask({ name: "sleep-for" }, async (_params, ctx) => {
+      await ctx.sleepFor("wait-for", durationSeconds);
+      return { resumed: true };
+    });
+
+    const { taskID, runID } = await absurd.spawn("sleep-for", undefined);
+    await absurd.workBatch("worker-sleep", 120, 1);
+
+    const sleepingRun = await ctx.getRun(runID);
+    expect(sleepingRun).toMatchObject({
+      state: "sleeping",
+    });
+    const wakeTime = new Date(base.getTime() + durationSeconds * 1000);
+    expect(sleepingRun?.available_at?.getTime()).toBe(wakeTime.getTime());
+
+    const resumeTime = new Date(wakeTime.getTime() + 5 * 1000);
+    vi.setSystemTime(resumeTime);
+    await ctx.setFakeNow(resumeTime);
+    await absurd.workBatch("worker-sleep", 120, 1);
+
+    expect(await ctx.getTask(taskID)).toMatchObject({
+      state: "completed",
+      completed_payload: { resumed: true },
+    });
+  });
+
+  test("sleepUntil checkpoint prevents re-scheduling when wake time passed", async () => {
+    vi.useFakeTimers();
+    const base = new Date("2024-05-06T09:00:00Z");
+    vi.setSystemTime(base);
+    await ctx.setFakeNow(base);
+
+    const wakeTime = new Date(base.getTime() + 5 * 60 * 1000);
+    let executions = 0;
+
+    absurd.registerTask({ name: "sleep-until" }, async (_params, ctx) => {
+      executions++;
+      await ctx.sleepUntil("sleep-step", wakeTime);
+      return { executions };
+    });
+
+    const { taskID, runID } = await absurd.spawn("sleep-until", undefined);
+    await absurd.workBatch("worker-sleep", 120, 1);
+
+    const checkpointRow = await ctx.getCheckpoint(taskID, "sleep-step");
+    expect(checkpointRow).toMatchObject({
+      checkpoint_name: "sleep-step",
+      owner_run_id: runID,
+      state: wakeTime.toISOString(),
+    });
+
+    const sleepingRun = await ctx.getRun(runID);
+    expect(sleepingRun?.state).toBe("sleeping");
+
+    vi.setSystemTime(wakeTime);
+    await ctx.setFakeNow(wakeTime);
+    await absurd.workBatch("worker-sleep", 120, 1);
+
+    expect(await ctx.getTask(taskID)).toMatchObject({
+      state: "completed",
+      completed_payload: { executions: 2 },
+    });
+    expect(executions).toBe(2);
+  });
+});

package/test/worker.test.ts

@@ -0,0 +1,193 @@
+import { describe, it, expect, beforeAll, afterEach, vi } from "vitest";
+import { EventEmitter, once } from "events";
+import type { TestContext } from "./setup";
+import { createTestAbsurd, randomName } from "./setup";
+import type { Absurd } from "../src/index";
+
+describe("Worker", () => {
+  let ctx: TestContext;
+  let absurd: Absurd;
+
+  beforeAll(async () => {
+    ctx = await createTestAbsurd(randomName("worker"));
+    absurd = ctx.absurd;
+  });
+
+  afterEach(async () => {
+    await ctx.cleanupTasks();
+    vi.restoreAllMocks();
+  });
+
+  it("respects concurrency limit and skips claims when at capacity", async () => {
+    const gate = new EventEmitter();
+    const atGate = new Set<number>();
+
+    gate.on("arrived", (id: number) => atGate.add(id));
+    gate.on("left", (id: number) => atGate.delete(id));
+
+    absurd.registerTask<{ id: number }, void>(
+      { name: "gated-task" },
+      async (params) => {
+        gate.emit("arrived", params.id);
+        await once(gate, "release");
+        gate.emit("left", params.id);
+      },
+    );
+
+    for (let i = 1; i <= 5; i++) {
+      await absurd.spawn("gated-task", { id: i });
+    }
+
+    const claimSpy = vi.spyOn(absurd, "claimTasks");
+
+    const concurrency = 2;
+    const worker = await absurd.startWorker({
+      concurrency,
+      pollInterval: 0.01,
+    });
+
+    // Wait until we are at capacity
+    await vi.waitFor(() => expect(atGate.size).toBe(concurrency), {
+      timeout: 100,
+    });
+
+    expect(claimSpy).toHaveBeenCalledTimes(1);
+    expect(claimSpy).toHaveBeenCalledWith(
+      expect.objectContaining({ batchSize: concurrency }),
+    );
+
+    // Wait for ~5 poll cycles
+    await ctx.sleep(50);
+
+    // Things should still be the same, no more claims
+    expect(atGate.size).toBe(concurrency);
+    expect(claimSpy).toHaveBeenCalledTimes(1);
+
+    gate.emit("release");
+    await worker.close();
+    expect(atGate.size).toBe(0);
+  });
+
+  it("polls again immediately when capacity frees up", async () => {
+    const gate = new EventEmitter();
+    const atGate = new Set<number>();
+
+    gate.on("arrived", (id: number) => atGate.add(id));
+    gate.on("left", (id: number) => atGate.delete(id));
+
+    absurd.registerTask<{ id: number }, void>(
+      { name: "responsive-task" },
+      async (params) => {
+        gate.emit("arrived", params.id);
+        await once(gate, "release");
+        gate.emit("left", params.id);
+      },
+    );
+
+    for (let i = 1; i <= 3; i++) {
+      await absurd.spawn("responsive-task", { id: i });
+    }
+
+    const claimSpy = vi.spyOn(absurd, "claimTasks");
+
+    const worker = await absurd.startWorker({
+      concurrency: 2,
+      pollInterval: 3600, // once every hour
+    });
+
+    await vi.waitFor(() => expect(atGate.size).toBe(2), {
+      timeout: 100,
+    });
+
+    expect(claimSpy).toHaveBeenCalledTimes(1);
+    const initialCallCount = claimSpy.mock.calls.length;
+
+    // Free current tasks so the worker can immediately claim again
+    gate.emit("release");
+
+    // Polls again without waiting for pollInterval
+    await vi.waitFor(() => {
+      expect(claimSpy.mock.calls.length).toBeGreaterThan(initialCallCount);
+    });
+
+    // Release any remaining tasks so the worker can shut down
+    gate.emit("release");
+    await worker.close();
+  });
+
+  it("shuts down gracefully", async () => {
+    const gate = new EventEmitter();
+    const started: number[] = [];
+
+    gate.on("started", (id: number) => started.push(id));
+
+    absurd.registerTask<{ id: number }, void>(
+      { name: "shutdown-task" },
+      async (params) => {
+        gate.emit("started", params.id);
+        await once(gate, "release");
+      },
+    );
+
+    await absurd.spawn("shutdown-task", { id: 1 });
+    await absurd.spawn("shutdown-task", { id: 2 });
+
+    const worker = await absurd.startWorker({ concurrency: 1 });
+    await once(gate, "started");
+
+    const closePromise = worker.close();
+    const claimSpy = vi.spyOn(absurd, "claimTasks");
+
+    let resolved = false;
+    closePromise.then(() => {
+      resolved = true;
+    });
+
+    // Does not resolve until the task finishes
+    await ctx.sleep(50);
+    expect(resolved).toBe(false);
+
+    gate.emit("release");
+
+    await vi.waitFor(() => expect(resolved).toBe(true), {
+      timeout: 200,
+    });
+
+    expect(started).toEqual([1]);
+    expect(claimSpy).not.toHaveBeenCalled();
+  });
+
+  it("calls onError for unexpected failures", async () => {
+    const errors: string[] = [];
+
+    absurd.registerTask<void, void>({ name: "test-task" }, async () => {});
+    await absurd.spawn("test-task", undefined);
+
+    vi.spyOn(absurd, "executeTask").mockRejectedValueOnce(
+      new Error("Execute failed, PG error"),
+    );
+
+    const worker = await absurd.startWorker({
+      pollInterval: 0.01,
+      onError: (err) => errors.push(err.message),
+    });
+
+    await vi.waitFor(
+      () => expect(errors).toContain("Execute failed, PG error"),
+      {
+        timeout: 200,
+      },
+    );
+
+    vi.spyOn(absurd, "claimTasks").mockRejectedValue(new Error("Claim failed"));
+
+    await vi.waitFor(() => expect(errors).toContain("Claim failed"), {
+      timeout: 200,
+    });
+
+    await worker.close();
+
+    expect(errors).toContain("Execute failed, PG error");
+    expect(errors).toContain("Claim failed");
+  });
+});

package/tsconfig.build.json

@@ -0,0 +1,9 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "rootDir": "./src",
+    "moduleResolution": "node"
+  },
+  "include": ["src/**/*.ts"],
+  "exclude": ["node_modules", "dist", "test/**/*.ts"]
+}

package/tsconfig.cjs.json

@@ -0,0 +1,11 @@
+{
+  "extends": "./tsconfig.build.json",
+  "compilerOptions": {
+    "module": "CommonJS",
+    "moduleResolution": "node",
+    "outDir": "./dist/cjs",
+    "declaration": false,
+    "declarationMap": false,
+    "sourceMap": false
+  }
+}

package/tsconfig.json

@@ -0,0 +1,20 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "ESNext",
+    "lib": ["ES2022"],
+    "moduleResolution": "bundler",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "resolveJsonModule": true,
+    "declaration": true,
+    "declarationMap": true,
+    "sourceMap": true,
+    "outDir": "./dist",
+    "types": ["node"]
+  },
+  "include": ["src/**/*.ts", "test/**/*.ts"],
+  "exclude": ["node_modules", "dist"]
+}

package/typedoc.json

@@ -0,0 +1,11 @@
+{
+  "entryPoints": ["src/index.ts"],
+  "out": "docs",
+  "plugin": [],
+  "exclude": ["**/*.test.ts", "**/__tests__/**"],
+  "excludePrivate": true,
+  "excludeProtected": false,
+  "skipErrorChecking": false,
+  "includeVersion": true,
+  "readme": "README.md"
+}

package/vitest.config.ts

@@ -0,0 +1,11 @@
+import { defineConfig } from "vitest/config";
+
+export default defineConfig({
+  test: {
+    environment: "node",
+    pool: "forks",
+    include: ["test/**/*.test.ts"],
+    testTimeout: 30000,
+    setupFiles: ["./test/setup.ts"],
+  },
+});