@terreno/jobs 57.4.0

package/src/tests/jobsWorkerLifecycle.test.ts

@@ -0,0 +1,509 @@
+import {beforeEach, describe, it} from "bun:test";
+import os from "node:os";
+import {TerrenoApp, type UserModel as UserModelType} from "@terreno/api";
+import {setupDb, UserModel} from "@terreno/api/testing";
+import {assert} from "chai";
+import {DateTime} from "luxon";
+
+import {JobsApp} from "../jobsApp";
+import {getJobsService} from "../jobsService";
+import {Job} from "../models/job";
+import {JobSchedule} from "../models/jobSchedule";
+import {getWorkerId, hasWorkerIdPrefix} from "../runners/mongoRunner";
+
+const typedUserModel = UserModel as unknown as UserModelType;
+
+const waitUntil = async (
+  predicate: () => Promise<boolean>,
+  options?: {intervalMs?: number; timeoutMs?: number}
+): Promise<void> => {
+  const intervalMs = options?.intervalMs ?? 25;
+  const timeoutMs = options?.timeoutMs ?? 5_000;
+  const deadline = DateTime.utc().plus({milliseconds: timeoutMs});
+
+  while (DateTime.utc() < deadline) {
+    if (await predicate()) {
+      return;
+    }
+    await Bun.sleep(intervalMs);
+  }
+
+  throw new Error("waitUntil timed out");
+};
+
+describe("jobs worker lifecycle", () => {
+  beforeEach(async (): Promise<void> => {
+    await setupDb();
+    await Job.deleteMany({});
+    await JobSchedule.deleteMany({});
+  });
+
+  it("does not poll until startWorker is called", async (): Promise<void> => {
+    const jobsApp = new JobsApp({pollIntervalMs: 50});
+    jobsApp.define("idle-check", {
+      handler: async () => {},
+    });
+
+    new TerrenoApp({
+      skipListen: true,
+      userModel: typedUserModel,
+    })
+      .register(jobsApp)
+      .build();
+
+    const enqueued = await getJobsService().enqueue({
+      name: "idle-check",
+      payload: {},
+    });
+
+    assert.equal(jobsApp.isWorkerActive(), false);
+
+    let pendingChecks = 0;
+    await waitUntil(async () => {
+      pendingChecks += 1;
+      const row = await Job.findExactlyOne({_id: enqueued._id});
+      return pendingChecks >= 3 && row.status === "pending";
+    });
+
+    const stillPending = await Job.findExactlyOne({_id: enqueued._id});
+    assert.equal(stillPending.status, "pending");
+  });
+
+  it("reclaims a running job when lockedAt exceeds lockTtlMs", async (): Promise<void> => {
+    let executionCount = 0;
+
+    const jobsApp = new JobsApp({lockTtlMs: 100, pollIntervalMs: 25});
+    jobsApp.define("reclaim-me", {
+      handler: async () => {
+        executionCount += 1;
+      },
+    });
+
+    new TerrenoApp({
+      skipListen: true,
+      userModel: typedUserModel,
+    })
+      .register(jobsApp)
+      .build();
+
+    const staleJob = await Job.create({
+      attemptCount: 0,
+      lockedAt: DateTime.utc().minus({milliseconds: 500}).toJSDate(),
+      lockedBy: "dead-worker:99999",
+      maxAttempts: 5,
+      name: "reclaim-me",
+      payload: {},
+      payloadRedacted: false,
+      runAt: DateTime.utc().minus({milliseconds: 500}).toJSDate(),
+      status: "running",
+    });
+
+    await jobsApp.startWorker();
+    await waitUntil(
+      async () => (await Job.findExactlyOne({_id: staleJob._id})).status === "completed"
+    );
+    await jobsApp.stopWorker();
+
+    assert.equal(executionCount, 1);
+    const reclaimed = await Job.findExactlyOne({_id: staleJob._id});
+    assert.equal(reclaimed.status, "completed");
+    assert.isTrue(hasWorkerIdPrefix(reclaimed.lockedBy));
+    assert.notEqual(reclaimed.lockedBy, getWorkerId());
+  });
+
+  it("executes a reclaimed job exactly once when two workers race", async (): Promise<void> => {
+    let executionCount = 0;
+
+    const workerA = new JobsApp({pollIntervalMs: 25});
+    const workerB = new JobsApp({pollIntervalMs: 25});
+    const handler = async (): Promise<void> => {
+      executionCount += 1;
+      await Bun.sleep(50);
+    };
+
+    workerA.define("race-once", {handler});
+    workerB.define("race-once", {handler});
+
+    new TerrenoApp({
+      skipListen: true,
+      userModel: typedUserModel,
+    })
+      .register(workerA)
+      .build();
+
+    const enqueued = await getJobsService().enqueue({
+      name: "race-once",
+      payload: {},
+    });
+
+    await Promise.all([workerA.startWorker(), workerB.startWorker()]);
+    await waitUntil(
+      async () => (await Job.findExactlyOne({_id: enqueued._id})).status === "completed"
+    );
+    await Promise.all([workerA.stopWorker(), workerB.stopWorker()]);
+
+    assert.equal(executionCount, 1);
+  });
+
+  it("records hostname and pid in lockedBy when claiming", async (): Promise<void> => {
+    const jobsApp = new JobsApp({pollIntervalMs: 25});
+    jobsApp.define("lock-identity", {
+      handler: async () => {},
+    });
+
+    new TerrenoApp({
+      skipListen: true,
+      userModel: typedUserModel,
+    })
+      .register(jobsApp)
+      .build();
+
+    const enqueued = await getJobsService().enqueue({
+      name: "lock-identity",
+      payload: {},
+    });
+
+    await jobsApp.startWorker();
+    await waitUntil(
+      async () => (await Job.findExactlyOne({_id: enqueued._id})).status === "completed"
+    );
+    await jobsApp.stopWorker();
+
+    const completed = await Job.findExactlyOne({_id: enqueued._id});
+    assert.isTrue(hasWorkerIdPrefix(completed.lockedBy));
+    assert.match(completed.lockedBy ?? "", new RegExp(`^${os.hostname()}:${process.pid}:`));
+  });
+
+  it("stopWorker aborts the poll loop and resolves cleanly", async (): Promise<void> => {
+    const jobsApp = new JobsApp({pollIntervalMs: 50});
+    jobsApp.define("stop-check", {
+      handler: async () => {},
+    });
+
+    new TerrenoApp({
+      skipListen: true,
+      userModel: typedUserModel,
+    })
+      .register(jobsApp)
+      .build();
+
+    await jobsApp.startWorker();
+    assert.isTrue(jobsApp.isWorkerActive());
+
+    await jobsApp.stopWorker();
+    assert.isFalse(jobsApp.isWorkerActive());
+
+    await jobsApp.stopWorker();
+    assert.isFalse(jobsApp.isWorkerActive());
+  });
+
+  it("stopWorker aborts an in-flight handler via ctx.signal", async (): Promise<void> => {
+    let sawAbort = false;
+
+    const jobsApp = new JobsApp({pollIntervalMs: 25});
+    jobsApp.define("slow-job", {
+      handler: async (_payload, ctx) => {
+        await waitUntil(() => Promise.resolve(ctx.signal.aborted), {
+          intervalMs: 25,
+          timeoutMs: 5_000,
+        });
+        sawAbort = ctx.signal.aborted;
+      },
+    });
+
+    new TerrenoApp({
+      skipListen: true,
+      userModel: typedUserModel,
+    })
+      .register(jobsApp)
+      .build();
+
+    const enqueued = await getJobsService().enqueue({
+      name: "slow-job",
+      payload: {},
+    });
+
+    await jobsApp.startWorker();
+    await waitUntil(
+      async () => (await Job.findExactlyOne({_id: enqueued._id})).status === "running"
+    );
+    await jobsApp.stopWorker();
+    await waitUntil(() => Promise.resolve(sawAbort));
+
+    assert.isTrue(sawAbort);
+    assert.isFalse(jobsApp.isWorkerActive());
+
+    const completed = await Job.findExactlyOne({_id: enqueued._id});
+    assert.equal(completed.status, "completed");
+  });
+
+  it("persists completed when the handler finishes after stopWorker aborts", async (): Promise<void> => {
+    let handlerStarted = false;
+    let handlerFinished = false;
+
+    const jobsApp = new JobsApp({pollIntervalMs: 25});
+    jobsApp.define("finish-after-abort", {
+      handler: async (_payload, ctx) => {
+        handlerStarted = true;
+        await waitUntil(() => Promise.resolve(ctx.signal.aborted));
+        handlerFinished = true;
+      },
+    });
+
+    new TerrenoApp({
+      skipListen: true,
+      userModel: typedUserModel,
+    })
+      .register(jobsApp)
+      .build();
+
+    const enqueued = await getJobsService().enqueue({
+      name: "finish-after-abort",
+      payload: {},
+    });
+
+    await jobsApp.startWorker();
+    await waitUntil(() => Promise.resolve(handlerStarted));
+    const stopPromise = jobsApp.stopWorker();
+    await waitUntil(() => Promise.resolve(handlerFinished));
+    await stopPromise;
+
+    const completed = await Job.findExactlyOne({_id: enqueued._id});
+    assert.equal(completed.status, "completed");
+    assert.isDefined(completed.lockedBy);
+  });
+
+  it("requeues and unlocks when abort throws in-flight work", async (): Promise<void> => {
+    const jobsApp = new JobsApp({pollIntervalMs: 25});
+    jobsApp.define("abort-throw", {
+      handler: async (_payload, ctx) => {
+        await waitUntil(() => Promise.resolve(ctx.signal.aborted));
+        const abortError = new Error("Aborted");
+        abortError.name = "AbortError";
+        throw abortError;
+      },
+    });
+
+    new TerrenoApp({
+      skipListen: true,
+      userModel: typedUserModel,
+    })
+      .register(jobsApp)
+      .build();
+
+    const enqueued = await getJobsService().enqueue({
+      name: "abort-throw",
+      payload: {},
+    });
+
+    await jobsApp.startWorker();
+    await waitUntil(
+      async () => (await Job.findExactlyOne({_id: enqueued._id})).status === "running"
+    );
+    await jobsApp.stopWorker();
+    await waitUntil(async () => {
+      const row = await Job.findExactlyOne({_id: enqueued._id});
+      return row.status === "pending" && row.lockedAt == null && row.lockedBy == null;
+    });
+  });
+
+  it("restarts a pending job on a later startWorker after stopWorker", async (): Promise<void> => {
+    let handlerRuns = 0;
+    let throwOnAbort = true;
+
+    const jobsApp = new JobsApp({pollIntervalMs: 25});
+    jobsApp.define("restart-me", {
+      handler: async (_payload, ctx) => {
+        handlerRuns += 1;
+        if (throwOnAbort) {
+          await waitUntil(() => Promise.resolve(ctx.signal.aborted));
+          const abortError = new Error("Aborted");
+          abortError.name = "AbortError";
+          throw abortError;
+        }
+      },
+    });
+
+    new TerrenoApp({
+      skipListen: true,
+      userModel: typedUserModel,
+    })
+      .register(jobsApp)
+      .build();
+
+    const enqueued = await getJobsService().enqueue({
+      name: "restart-me",
+      payload: {},
+    });
+
+    await jobsApp.startWorker();
+    await waitUntil(
+      async () => (await Job.findExactlyOne({_id: enqueued._id})).status === "running"
+    );
+    await jobsApp.stopWorker();
+    await waitUntil(async () => {
+      const row = await Job.findExactlyOne({_id: enqueued._id});
+      return row.status === "pending" && row.lockedAt == null && row.lockedBy == null;
+    });
+
+    throwOnAbort = false;
+    await jobsApp.startWorker();
+    await waitUntil(
+      async () => (await Job.findExactlyOne({_id: enqueued._id})).status === "completed"
+    );
+    await jobsApp.stopWorker();
+
+    assert.equal(handlerRuns, 2);
+  });
+
+  it("does not rerun a job that completed during stopWorker", async (): Promise<void> => {
+    let handlerRuns = 0;
+
+    const jobsApp = new JobsApp({pollIntervalMs: 25});
+    jobsApp.define("complete-once", {
+      handler: async (_payload, ctx) => {
+        handlerRuns += 1;
+        await waitUntil(() => Promise.resolve(ctx.signal.aborted));
+      },
+    });
+
+    new TerrenoApp({
+      skipListen: true,
+      userModel: typedUserModel,
+    })
+      .register(jobsApp)
+      .build();
+
+    const enqueued = await getJobsService().enqueue({
+      name: "complete-once",
+      payload: {},
+    });
+
+    await jobsApp.startWorker();
+    await waitUntil(
+      async () => (await Job.findExactlyOne({_id: enqueued._id})).status === "running"
+    );
+    await jobsApp.stopWorker();
+    await waitUntil(
+      async () => (await Job.findExactlyOne({_id: enqueued._id})).status === "completed"
+    );
+
+    await jobsApp.startWorker();
+    let idlePolls = 0;
+    await waitUntil(async () => {
+      idlePolls += 1;
+      return idlePolls >= 5;
+    });
+    await jobsApp.stopWorker();
+
+    assert.equal(handlerRuns, 1);
+  });
+
+  it("startWorker returns without awaiting the poll loop", async (): Promise<void> => {
+    let handlerStarted = false;
+    let releaseHandler!: () => void;
+    const handlerBlocked = new Promise<void>((resolve) => {
+      releaseHandler = resolve;
+    });
+
+    const jobsApp = new JobsApp({pollIntervalMs: 25});
+    jobsApp.define("async-start", {
+      handler: async () => {
+        handlerStarted = true;
+        await handlerBlocked;
+      },
+    });
+
+    new TerrenoApp({
+      skipListen: true,
+      userModel: typedUserModel,
+    })
+      .register(jobsApp)
+      .build();
+
+    const enqueued = await getJobsService().enqueue({
+      name: "async-start",
+      payload: {},
+    });
+
+    await jobsApp.startWorker();
+    assert.isTrue(jobsApp.isWorkerActive());
+    assert.isFalse(handlerStarted);
+
+    await waitUntil(() => Promise.resolve(handlerStarted));
+    releaseHandler();
+    await waitUntil(
+      async () => (await Job.findExactlyOne({_id: enqueued._id})).status === "completed"
+    );
+    await jobsApp.stopWorker();
+  });
+
+  it("startWorker rejects runners that do not implement start", async (): Promise<void> => {
+    const jobsApp = new JobsApp({
+      runner: {
+        enqueue: async () => {},
+        id: "noop",
+      },
+    });
+
+    try {
+      await jobsApp.startWorker();
+      assert.fail("expected startWorker to throw");
+    } catch (error: unknown) {
+      assert.match(String(error), /does not support start/);
+    }
+
+    assert.isFalse(jobsApp.isWorkerActive());
+  });
+
+  it("resets worker state when the runner start promise rejects", async (): Promise<void> => {
+    const jobsApp = new JobsApp({
+      runner: {
+        enqueue: async () => {},
+        id: "broken",
+        start: async () => {
+          throw new Error("start failed");
+        },
+      },
+    });
+
+    await jobsApp.startWorker();
+    await waitUntil(() => Promise.resolve(!jobsApp.isWorkerActive()));
+
+    assert.isFalse(jobsApp.isWorkerActive());
+  });
+
+  it("marks handler failures as pending retries without stopping the worker", async (): Promise<void> => {
+    const jobsApp = new JobsApp({pollIntervalMs: 25});
+    jobsApp.define("boom", {
+      handler: async () => {
+        throw new Error("handler exploded");
+      },
+      retry: {backoffMs: 500, maxAttempts: 5},
+    });
+
+    new TerrenoApp({
+      skipListen: true,
+      userModel: typedUserModel,
+    })
+      .register(jobsApp)
+      .build();
+
+    const enqueued = await getJobsService().enqueue({
+      name: "boom",
+      payload: {},
+    });
+
+    await jobsApp.startWorker();
+    await waitUntil(async () => {
+      const row = await Job.findExactlyOne({_id: enqueued._id});
+      return row.status === "pending" && row.attemptCount === 1;
+    });
+    assert.isTrue(jobsApp.isWorkerActive());
+    await jobsApp.stopWorker();
+
+    const pending = await Job.findExactlyOne({_id: enqueued._id});
+    assert.equal(pending.status, "pending");
+    assert.match(pending.lastError ?? "", /handler exploded/);
+  });
+});

package/src/tests/packageExports.test.ts

@@ -0,0 +1,24 @@
+import {describe, it} from "bun:test";
+import {assert} from "chai";
+
+import {
+  getJobsService,
+  Job,
+  JobSchedule,
+  JobsApp,
+  MongoJobRunner,
+  tryGetJobsService,
+  unregisterJobsService,
+} from "../index";
+
+describe("package public exports", () => {
+  it("re-exports JobsApp, models, MongoJobRunner, and getJobsService", (): void => {
+    assert.equal(JobsApp.name, "JobsApp");
+    assert.equal(MongoJobRunner.name, "MongoJobRunner");
+    assert.equal(Job.modelName, "Job");
+    assert.equal(JobSchedule.modelName, "JobSchedule");
+    assert.equal(typeof getJobsService, "function");
+    assert.equal(typeof tryGetJobsService, "function");
+    assert.equal(typeof unregisterJobsService, "function");
+  });
+});

package/src/tests/retryBackoff.test.ts

@@ -0,0 +1,76 @@
+import {describe, it} from "bun:test";
+import {assert} from "chai";
+import {DateTime} from "luxon";
+
+import {
+  computeRetryDelayMs,
+  computeRetryRunAt,
+  DEFAULT_BACKOFF_MS,
+  DEFAULT_MAX_BACKOFF_MS,
+} from "../retryBackoff";
+
+describe("retryBackoff", () => {
+  it("defaults backoffMs to 1000 and maxBackoffMs to 15 minutes", (): void => {
+    assert.equal(DEFAULT_BACKOFF_MS, 1_000);
+    assert.equal(DEFAULT_MAX_BACKOFF_MS, 15 * 60 * 1_000);
+  });
+
+  it("uses exponential backoff capped by maxBackoffMs with injected random", (): void => {
+    const retry = {backoffMs: 1_000, maxBackoffMs: 10_000};
+    const random = (): number => 0;
+
+    assert.equal(
+      computeRetryDelayMs({
+        attemptCount: 1,
+        backoffMs: retry.backoffMs,
+        maxBackoffMs: retry.maxBackoffMs,
+        random,
+      }),
+      500,
+      "first failure uses backoffMs * 2^0 with equal jitter floor"
+    );
+    assert.equal(
+      computeRetryDelayMs({
+        attemptCount: 2,
+        backoffMs: retry.backoffMs,
+        maxBackoffMs: retry.maxBackoffMs,
+        random,
+      }),
+      1_000,
+      "second failure uses backoffMs * 2^1 with equal jitter floor"
+    );
+    assert.equal(
+      computeRetryDelayMs({
+        attemptCount: 5,
+        backoffMs: retry.backoffMs,
+        maxBackoffMs: retry.maxBackoffMs,
+        random,
+      }),
+      5_000,
+      "attemptCount 5 applies backoffMs * 2^4 before cap"
+    );
+    assert.equal(
+      computeRetryDelayMs({
+        attemptCount: 10,
+        backoffMs: retry.backoffMs,
+        maxBackoffMs: retry.maxBackoffMs,
+        random,
+      }),
+      5_000,
+      "delay is capped at half of maxBackoffMs with zero random"
+    );
+  });
+
+  it("computes runAt from Luxon with deterministic jitter", (): void => {
+    const from = DateTime.utc(2026, 9, 10, 12, 0, 0);
+    const runAt = computeRetryRunAt({
+      attemptCount: 2,
+      backoffMs: 1_000,
+      from,
+      maxBackoffMs: 60_000,
+      random: () => 1,
+    });
+
+    assert.equal(runAt.toMillis(), from.plus({milliseconds: 2_000}).toMillis());
+  });
+});

package/src/tests/scheduleCron.test.ts

@@ -0,0 +1,87 @@
+import {describe, it} from "bun:test";
+import {isAPIError} from "@terreno/api";
+import {assert} from "chai";
+import {DateTime} from "luxon";
+
+import {
+  assertValidCronExpression,
+  assertValidIanaTimezone,
+  computeInitialNextRunAt,
+  computeNextRunAtAfter,
+  resolveScheduleTimezone,
+  validateScheduleDefinition,
+} from "../scheduleCron";
+
+describe("scheduleCron", () => {
+  it("computes different next runs for America/New_York vs UTC", (): void => {
+    const cron = "0 9 * * *";
+    const utcNext = computeInitialNextRunAt(cron, "UTC");
+    const newYorkNext = computeInitialNextRunAt(cron, "America/New_York");
+
+    assert.notEqual(utcNext.toISOString(), newYorkNext.toISOString());
+    assert.equal(
+      DateTime.fromJSDate(newYorkNext).setZone("America/New_York").hour,
+      9,
+      "New York schedule fires at 9am local"
+    );
+    assert.equal(
+      DateTime.fromJSDate(utcNext).setZone("UTC").hour,
+      9,
+      "UTC schedule fires at 9am UTC"
+    );
+  });
+
+  it("advances nextRunAt after a tick using the schedule timezone", (): void => {
+    const cron = "0 9 * * *";
+    const after = new Date("2026-09-10T14:00:00.000Z");
+    const utcNext = computeNextRunAtAfter(cron, "UTC", after);
+    const newYorkNext = computeNextRunAtAfter(cron, "America/New_York", after);
+
+    assert.equal(utcNext.toISOString(), "2026-09-11T09:00:00.000Z");
+    assert.equal(newYorkNext.toISOString(), "2026-09-11T13:00:00.000Z");
+  });
+
+  it("defaults schedule timezone resolution to JobsApp timezone then UTC", (): void => {
+    assert.equal(
+      resolveScheduleTimezone({defaultTimezone: "America/Chicago", scheduleTimezone: undefined}),
+      "America/Chicago"
+    );
+    assert.equal(resolveScheduleTimezone({defaultTimezone: "America/Chicago"}), "America/Chicago");
+    assert.equal(
+      resolveScheduleTimezone({
+        defaultTimezone: "America/Chicago",
+        scheduleTimezone: "America/New_York",
+      }),
+      "America/New_York"
+    );
+  });
+
+  it("rejects invalid cron expressions at define time", (): void => {
+    try {
+      assertValidCronExpression("not-a-cron", "UTC");
+      assert.fail("expected invalid cron to throw");
+    } catch (error: unknown) {
+      assert.isTrue(isAPIError(error));
+      assert.equal((error as {status: number}).status, 400);
+    }
+  });
+
+  it("rejects invalid IANA timezones at define time", (): void => {
+    try {
+      assertValidIanaTimezone("Not/A_Zone");
+      assert.fail("expected invalid timezone to throw");
+    } catch (error: unknown) {
+      assert.isTrue(isAPIError(error));
+      assert.equal((error as {status: number}).status, 400);
+    }
+  });
+
+  it("validateScheduleDefinition applies definition timezone over JobsApp default", (): void => {
+    const timezone = validateScheduleDefinition({
+      cron: "0 9 * * *",
+      defaultTimezone: "UTC",
+      scheduleTimezone: "America/New_York",
+    });
+    assert.equal(timezone, "America/New_York");
+  });
+});