@terreno/jobs 57.4.0

package/src/runners/vercelQueues.ts

@@ -0,0 +1,394 @@
+import {createRequire} from "node:module";
+
+import {DateTime} from "luxon";
+
+import {type ExecuteJobOutcome, executeJobById, type JobExecutionHost} from "../jobExecutor";
+import type {JobDocument} from "../modelTypes";
+import type {JobRunner} from "../types";
+
+/** Vercel Queues maximum delay and retention window (7 days). */
+export const VERCEL_QUEUE_MAX_DELAY_SECONDS = 604_800;
+export const VERCEL_QUEUE_DEFAULT_RETENTION_SECONDS = 86_400;
+export const VERCEL_QUEUE_MIN_RETENTION_SECONDS = 60;
+export const VERCEL_QUEUE_MIN_VISIBILITY_TIMEOUT_SECONDS = 30;
+export const VERCEL_QUEUE_MAX_VISIBILITY_TIMEOUT_SECONDS = 3_600;
+export const VERCEL_QUEUE_DEFAULT_VISIBILITY_TIMEOUT_SECONDS = 1_800;
+export const VERCEL_QUEUE_LOCKED_RETRY_SECONDS = 30;
+
+export interface VercelQueueSendOptions {
+  delaySeconds?: number;
+  headers?: Record<string, string>;
+  idempotencyKey?: string;
+  region?: string;
+  retentionSeconds?: number;
+}
+
+export interface VercelQueueSendResult {
+  messageId: string | null;
+}
+
+/** Narrow `send` seam — inject in tests; production loads `@vercel/queue`. */
+export type VercelQueueSendFn = (
+  topic: string,
+  payload: unknown,
+  options?: VercelQueueSendOptions
+) => Promise<VercelQueueSendResult>;
+
+/** `@vercel/queue` 0.5.1 message metadata (all fields required). */
+export interface VercelQueueMessageMetadata {
+  consumerGroup: string;
+  createdAt: Date;
+  deliveryCount: number;
+  expiresAt: Date;
+  messageId: string;
+  region: string;
+  topicName: string;
+}
+
+export type VercelQueueRetryDirective = {acknowledge: true} | {afterSeconds: number};
+
+export type VercelQueueMessageHandler = (
+  message: unknown,
+  metadata: VercelQueueMessageMetadata
+) => Promise<void>;
+
+export type VercelQueueRetryHandler = (
+  error: unknown,
+  metadata: VercelQueueMessageMetadata
+) => VercelQueueRetryDirective | undefined;
+
+export type VercelHandleCallbackFn = (
+  handler: VercelQueueMessageHandler,
+  options?: VercelHandleCallbackOptions
+) => VercelQueueRouteHandler;
+
+export interface VercelHandleCallbackOptions {
+  retry?: VercelQueueRetryHandler;
+  visibilityTimeoutSeconds?: number;
+}
+
+export type VercelQueueRouteRequest = Request | {request: Request};
+
+export type VercelQueueRouteHandler = (request: VercelQueueRouteRequest) => Promise<Response>;
+
+export interface VercelQueuesRunnerConfig {
+  /** Injected `send` for tests. Production wiring loads `@vercel/queue`. */
+  send?: VercelQueueSendFn;
+  region?: string;
+  retentionSeconds?: number;
+  /** Queue topic published by {@link VercelQueuesRunner.enqueue}. */
+  topic: string;
+}
+
+export interface VercelQueuesConsumerOptions {
+  handleCallback?: VercelHandleCallbackFn;
+  host: JobExecutionHost;
+  signal?: AbortSignal;
+  visibilityTimeoutSeconds?: number;
+}
+
+interface ResolvedVercelQueuesRunnerConfig {
+  region?: string;
+  retentionSeconds?: number;
+  topic: string;
+}
+
+const nodeRequire = createRequire(__filename);
+
+export class VercelQueueRunAtBeyondLimitError extends Error {
+  readonly maxDelaySeconds: number;
+  readonly runAt: Date;
+
+  constructor(runAt: Date, maxDelaySeconds: number) {
+    super(
+      `Job runAt ${runAt.toISOString()} exceeds Vercel Queue maximum delay of ${maxDelaySeconds} seconds`
+    );
+    this.name = "VercelQueueRunAtBeyondLimitError";
+    this.runAt = runAt;
+    this.maxDelaySeconds = maxDelaySeconds;
+  }
+}
+
+export class VercelQueueConsumerControl extends Error {
+  readonly directive: VercelQueueRetryDirective;
+
+  constructor(directive: VercelQueueRetryDirective) {
+    super("Vercel queue consumer control");
+    this.name = "VercelQueueConsumerControl";
+    this.directive = directive;
+  }
+}
+
+const requireNonEmpty = (value: string | undefined, label: string): string => {
+  const trimmed = value?.trim();
+  if (!trimmed) {
+    throw new Error(`VercelQueuesRunner requires ${label}`);
+  }
+  return trimmed;
+};
+
+const validateRetentionSeconds = (retentionSeconds: number | undefined): number | undefined => {
+  if (retentionSeconds === undefined) {
+    return undefined;
+  }
+
+  if (
+    !Number.isFinite(retentionSeconds) ||
+    retentionSeconds < VERCEL_QUEUE_MIN_RETENTION_SECONDS ||
+    retentionSeconds > VERCEL_QUEUE_MAX_DELAY_SECONDS
+  ) {
+    throw new Error(
+      `VercelQueuesRunner retentionSeconds must be between ${VERCEL_QUEUE_MIN_RETENTION_SECONDS} and ${VERCEL_QUEUE_MAX_DELAY_SECONDS} seconds`
+    );
+  }
+
+  return retentionSeconds;
+};
+
+const validateVisibilityTimeoutSeconds = (visibilityTimeoutSeconds: number | undefined): number => {
+  if (visibilityTimeoutSeconds === undefined) {
+    return VERCEL_QUEUE_DEFAULT_VISIBILITY_TIMEOUT_SECONDS;
+  }
+
+  if (
+    !Number.isFinite(visibilityTimeoutSeconds) ||
+    visibilityTimeoutSeconds < VERCEL_QUEUE_MIN_VISIBILITY_TIMEOUT_SECONDS ||
+    visibilityTimeoutSeconds > VERCEL_QUEUE_MAX_VISIBILITY_TIMEOUT_SECONDS
+  ) {
+    throw new Error(
+      `VercelQueuesConsumer visibilityTimeoutSeconds must be between ${VERCEL_QUEUE_MIN_VISIBILITY_TIMEOUT_SECONDS} and ${VERCEL_QUEUE_MAX_VISIBILITY_TIMEOUT_SECONDS} seconds`
+    );
+  }
+
+  return visibilityTimeoutSeconds;
+};
+
+const resolveConfig = (config: VercelQueuesRunnerConfig): ResolvedVercelQueuesRunnerConfig => ({
+  region: config.region?.trim() || undefined,
+  retentionSeconds: validateRetentionSeconds(config.retentionSeconds),
+  topic: requireNonEmpty(config.topic, "topic"),
+});
+
+const loadQueueSend = (): VercelQueueSendFn => {
+  try {
+    const module = nodeRequire("@vercel/queue") as {send: VercelQueueSendFn};
+    return module.send;
+  } catch {
+    throw new Error(
+      "VercelQueuesRunner requires optional peer dependency @vercel/queue. " +
+        "Install it with: bun add @vercel/queue"
+    );
+  }
+};
+
+const loadHandleCallback = (): VercelHandleCallbackFn => {
+  try {
+    const module = nodeRequire("@vercel/queue") as {handleCallback: VercelHandleCallbackFn};
+    return module.handleCallback;
+  } catch {
+    throw new Error(
+      "createVercelQueuesConsumer requires optional peer dependency @vercel/queue. " +
+        "Install it with: bun add @vercel/queue"
+    );
+  }
+};
+
+const readJobId = (message: unknown): string | undefined => {
+  if (!message || typeof message !== "object" || !("jobId" in message)) {
+    return undefined;
+  }
+
+  const {jobId} = message as {jobId?: unknown};
+  if (typeof jobId !== "string") {
+    return undefined;
+  }
+
+  const trimmed = jobId.trim();
+  return trimmed.length > 0 ? trimmed : undefined;
+};
+
+export const computeVercelQueueDelaySeconds = (
+  runAt: Date,
+  now = DateTime.utc()
+): number | undefined => {
+  const scheduled = DateTime.fromJSDate(runAt, {zone: "utc"});
+  if (scheduled <= now) {
+    return undefined;
+  }
+
+  const delaySeconds = Math.floor(scheduled.diff(now, "seconds").seconds);
+  if (delaySeconds <= 0) {
+    return undefined;
+  }
+
+  if (delaySeconds > VERCEL_QUEUE_MAX_DELAY_SECONDS) {
+    throw new VercelQueueRunAtBeyondLimitError(runAt, VERCEL_QUEUE_MAX_DELAY_SECONDS);
+  }
+
+  return delaySeconds;
+};
+
+export const resolveVercelQueueSendRetentionSeconds = (
+  delaySeconds: number | undefined,
+  configuredRetention?: number
+): number => {
+  const baseRetention = configuredRetention ?? VERCEL_QUEUE_DEFAULT_RETENTION_SECONDS;
+  if (delaySeconds === undefined) {
+    return baseRetention;
+  }
+
+  return Math.max(baseRetention, delaySeconds);
+};
+
+/** Clamp queue retry visibility changes to the vendor 0.5.1 range (30s–1h). */
+export const clampVercelQueueRetryAfterSeconds = (seconds: number): number =>
+  Math.max(
+    VERCEL_QUEUE_MIN_VISIBILITY_TIMEOUT_SECONDS,
+    Math.min(seconds, VERCEL_QUEUE_MAX_VISIBILITY_TIMEOUT_SECONDS)
+  );
+
+/**
+ * Seconds until {@link runAt}, clamped for `handleCallback` retry visibility (30–3600).
+ * not_due jobs more than 1h away retry at 3600s until due.
+ */
+export const computeVercelQueueRetryAfterSeconds = (runAt: Date, now = DateTime.utc()): number => {
+  const seconds = Math.ceil(DateTime.fromJSDate(runAt, {zone: "utc"}).diff(now, "seconds").seconds);
+  if (seconds <= 0) {
+    return VERCEL_QUEUE_MIN_VISIBILITY_TIMEOUT_SECONDS;
+  }
+
+  return clampVercelQueueRetryAfterSeconds(seconds);
+};
+
+export const buildVercelQueueSendOptions = (
+  job: JobDocument,
+  config: {region?: string; retentionSeconds?: number},
+  now = DateTime.utc()
+): VercelQueueSendOptions => {
+  const delaySeconds = computeVercelQueueDelaySeconds(job.runAt, now);
+  const retentionSeconds = resolveVercelQueueSendRetentionSeconds(
+    delaySeconds,
+    config.retentionSeconds
+  );
+
+  return {
+    idempotencyKey: job._id.toString(),
+    retentionSeconds,
+    ...(config.region ? {region: config.region} : {}),
+    ...(delaySeconds !== undefined ? {delaySeconds} : {}),
+  };
+};
+
+const normalizeRetryDirective = (
+  directive: VercelQueueRetryDirective
+): VercelQueueRetryDirective => {
+  if ("acknowledge" in directive) {
+    return directive;
+  }
+
+  return {afterSeconds: clampVercelQueueRetryAfterSeconds(directive.afterSeconds)};
+};
+
+export const resolveVercelQueueConsumerRetry = (
+  error: unknown
+): VercelQueueRetryDirective | undefined => {
+  if (error instanceof VercelQueueConsumerControl) {
+    return normalizeRetryDirective(error.directive);
+  }
+
+  return undefined;
+};
+
+export const applyVercelQueueExecutionOutcome = (outcome: ExecuteJobOutcome): void => {
+  if (outcome.kind === "not_found") {
+    throw new VercelQueueConsumerControl({acknowledge: true});
+  }
+
+  if (outcome.kind === "noop") {
+    return;
+  }
+
+  if (outcome.kind === "conflict") {
+    if (outcome.reason === "not_due") {
+      throw new VercelQueueConsumerControl({
+        afterSeconds: computeVercelQueueRetryAfterSeconds(outcome.runAt),
+      });
+    }
+
+    throw new VercelQueueConsumerControl({afterSeconds: VERCEL_QUEUE_LOCKED_RETRY_SECONDS});
+  }
+
+  if (outcome.job.status === "pending") {
+    const now = DateTime.utc();
+    const afterSeconds =
+      outcome.job.runAt > now.toJSDate()
+        ? computeVercelQueueRetryAfterSeconds(outcome.job.runAt, now)
+        : VERCEL_QUEUE_MIN_VISIBILITY_TIMEOUT_SECONDS;
+
+    throw new VercelQueueConsumerControl({afterSeconds});
+  }
+};
+
+/**
+ * Dispatch-only runner that publishes `{jobId}` to a Vercel Queue topic via `send`.
+ *
+ * Required `vercel.json` consumer (private route — platform auth only):
+ * `"experimentalTriggers": [{ "type": "queue/v2beta", "topic": "<topic>" }]`
+ */
+export class VercelQueuesRunner implements JobRunner {
+  readonly id = "vercel-queues";
+  /**
+   * Vercel push consumers are private `handleCallback` routes, not Express
+   * `POST /jobs/execute`. Use {@link createVercelQueuesConsumer} on the consumer route.
+   */
+  readonly requiresExecuteRoute = false;
+
+  private readonly config: ResolvedVercelQueuesRunnerConfig;
+  private readonly send: VercelQueueSendFn;
+
+  constructor(config: VercelQueuesRunnerConfig) {
+    this.config = resolveConfig(config);
+    this.send = config.send ?? loadQueueSend();
+  }
+
+  async enqueue(job: JobDocument): Promise<void> {
+    const payload = {jobId: job._id.toString()};
+    const options = buildVercelQueueSendOptions(job, this.config);
+
+    await this.send(this.config.topic, payload, options);
+  }
+}
+
+/**
+ * Wraps vendor `handleCallback` to execute a queued `{jobId}` through Terreno's
+ * Mongo-backed {@link executeJobById} seam. Platform auth stays in the vendor handler.
+ */
+export const createVercelQueuesConsumer = (
+  options: VercelQueuesConsumerOptions
+): VercelQueueRouteHandler => {
+  const handleCallback = options.handleCallback ?? loadHandleCallback();
+  const signal = options.signal ?? AbortSignal.timeout(30 * 60 * 1_000);
+  const visibilityTimeoutSeconds = validateVisibilityTimeoutSeconds(
+    options.visibilityTimeoutSeconds
+  );
+
+  return handleCallback(
+    async (message: unknown) => {
+      const jobId = readJobId(message);
+      if (!jobId) {
+        throw new VercelQueueConsumerControl({acknowledge: true});
+      }
+
+      const outcome = await executeJobById({
+        host: options.host,
+        jobId,
+        signal,
+      });
+      applyVercelQueueExecutionOutcome(outcome);
+    },
+    {
+      retry: resolveVercelQueueConsumerRetry,
+      visibilityTimeoutSeconds,
+    }
+  );
+};

package/src/scheduleCron.ts

@@ -0,0 +1,65 @@
+import {APIError} from "@terreno/api";
+import {CronTime} from "cron";
+import {DateTime} from "luxon";
+
+const INVALID_CRON_TITLE = "Invalid cron expression";
+const INVALID_TIMEZONE_TITLE = "Invalid IANA timezone";
+
+export const resolveScheduleTimezone = ({
+  defaultTimezone,
+  scheduleTimezone,
+}: {
+  defaultTimezone: string;
+  scheduleTimezone?: string;
+}): string => scheduleTimezone ?? defaultTimezone;
+
+/** @internal */
+export const assertValidIanaTimezone = (timezone: string): void => {
+  const zoneCheck = DateTime.now().setZone(timezone);
+  if (!zoneCheck.isValid) {
+    throw new APIError({
+      detail: zoneCheck.invalidReason ?? "Timezone is not a valid IANA identifier",
+      status: 400,
+      title: INVALID_TIMEZONE_TITLE,
+    });
+  }
+};
+
+/** @internal */
+export const assertValidCronExpression = (cron: string, timezone: string): void => {
+  try {
+    new CronTime(cron, timezone);
+  } catch (error: unknown) {
+    const detail = error instanceof Error ? error.message : String(error);
+    throw new APIError({
+      cause: error instanceof Error ? error : undefined,
+      detail,
+      status: 400,
+      title: INVALID_CRON_TITLE,
+    });
+  }
+};
+
+export const validateScheduleDefinition = ({
+  cron,
+  defaultTimezone,
+  scheduleTimezone,
+}: {
+  cron: string;
+  defaultTimezone: string;
+  scheduleTimezone?: string;
+}): string => {
+  const timezone = resolveScheduleTimezone({
+    defaultTimezone,
+    scheduleTimezone,
+  });
+  assertValidIanaTimezone(timezone);
+  assertValidCronExpression(cron, timezone);
+  return timezone;
+};
+
+export const computeInitialNextRunAt = (cron: string, timezone: string): Date =>
+  computeNextRunAtAfter(cron, timezone, DateTime.utc().toJSDate());
+
+export const computeNextRunAtAfter = (cron: string, timezone: string, after: Date): Date =>
+  new CronTime(cron, timezone).getNextDateFrom(after, timezone).toJSDate();

package/src/scheduler.ts

@@ -0,0 +1,112 @@
+import {createScopedLogger, logger} from "@terreno/api";
+import {DateTime} from "luxon";
+
+import {JobDispatchError} from "./dispatchError";
+import type {JobsService} from "./jobsService";
+import {Job} from "./models/job";
+import {JobSchedule} from "./models/jobSchedule";
+import type {JobScheduleDocument} from "./modelTypes";
+import {IN_FLIGHT_JOB_STATUSES} from "./modelTypes";
+import {computeNextRunAtAfter} from "./scheduleCron";
+
+/** @internal */
+export const buildScheduleRunIdempotencyKey = (scheduleId: string, dueRunAt: Date): string =>
+  `schedule:${scheduleId}:${dueRunAt.toISOString()}`;
+
+const tickOneSchedule = async ({
+  jobsService,
+  now,
+  schedule,
+}: {
+  jobsService: JobsService;
+  now: Date;
+  schedule: JobScheduleDocument;
+}): Promise<void> => {
+  const dueRunAt = schedule.nextRunAt;
+  const advancedNextRunAt = computeNextRunAtAfter(schedule.cron, schedule.timezone, now);
+
+  const claimed = await JobSchedule.findOneAndUpdate(
+    {
+      _id: schedule._id,
+      enabled: true,
+      nextRunAt: dueRunAt,
+    },
+    {
+      $set: {
+        nextRunAt: advancedNextRunAt,
+      },
+    },
+    {returnDocument: "before"}
+  );
+
+  if (!claimed) {
+    return;
+  }
+
+  const inFlight = await Job.findOneOrNone({
+    scheduleId: claimed._id,
+    status: {$in: IN_FLIGHT_JOB_STATUSES},
+  });
+
+  if (inFlight) {
+    return;
+  }
+
+  const idempotencyKey = buildScheduleRunIdempotencyKey(claimed._id.toString(), claimed.nextRunAt);
+
+  try {
+    await jobsService.enqueue({
+      idempotencyKey,
+      name: claimed.handlerName,
+      payload: {},
+      runAt: claimed.nextRunAt,
+      scheduleId: claimed._id.toString(),
+    });
+  } catch (error: unknown) {
+    const shouldRollbackSchedule =
+      !(error instanceof JobDispatchError) || error.compensationSucceeded;
+
+    if (shouldRollbackSchedule) {
+      await JobSchedule.updateOne(
+        {_id: claimed._id, nextRunAt: advancedNextRunAt},
+        {$set: {nextRunAt: dueRunAt}}
+      );
+    } else {
+      logger.error(
+        `[JobSchedule] Dispatch compensation lost race for "${claimed.name}" (${claimed._id.toString()}); schedule nextRunAt left advanced`
+      );
+    }
+
+    throw error;
+  }
+};
+
+export const tickDueSchedules = async ({
+  jobsService,
+  now = DateTime.utc().toJSDate(),
+}: {
+  defaultTimezone?: string;
+  jobsService: JobsService;
+  now?: Date;
+}): Promise<void> => {
+  const dueSchedules = await JobSchedule.find({
+    enabled: true,
+    nextRunAt: {$lte: now},
+  }).sort({nextRunAt: 1});
+
+  for (const schedule of dueSchedules) {
+    const scheduleLog = createScopedLogger({
+      labels: {scheduleName: schedule.name},
+      prefix: "[JobSchedule]",
+    });
+
+    try {
+      await tickOneSchedule({jobsService, now, schedule});
+    } catch (error: unknown) {
+      logger.error(
+        `[JobSchedule] Tick failed for "${schedule.name}": ${error instanceof Error ? error.message : String(error)}`
+      );
+      scheduleLog.error(`Tick failed: ${error instanceof Error ? error.message : String(error)}`);
+    }
+  }
+};

package/src/tests/bunSetup.ts

@@ -0,0 +1,9 @@
+import {registerSimpleMongoPreload} from "@terreno/test";
+
+process.env.TERRENO_TEST_USE_MEMORY_MONGO = "true";
+
+registerSimpleMongoPreload({
+  testEnv: {
+    tokenIssuer: "terreno-jobs.test",
+  },
+});