effect-mq 0.1.0 → 0.3.0

package/src/JobStore.ts

@@ -16,7 +16,7 @@
  *
  * @since 0.1.0
  */
-import { Brand, Context, Data, type Effect, type Option, Predicate } from "effect"
+import { Brand, Context, Cron, Data, Duration, type Effect, type Option, Predicate, Result } from "effect"
 
 /**
  * The identifier of an enqueued job. Produced by `enqueue` (either
@@ -47,6 +47,20 @@ export type QueueName = Brand.Branded<string, "effect-mq/QueueName">
  */
 export const QueueName: Brand.Constructor<QueueName> = Brand.nominal<QueueName>()
 
+/**
+ * The identifier of a repeatable-job schedule.
+ *
+ * @since 0.2.0
+ */
+export type ScheduleKey = Brand.Branded<string, "effect-mq/ScheduleKey">
+
+/**
+ * Brand a raw string as a `ScheduleKey`.
+ *
+ * @since 0.2.0
+ */
+export const ScheduleKey: Brand.Constructor<ScheduleKey> = Brand.nominal<ScheduleKey>()
+
 /**
  * The lifecycle states of a job.
  *
@@ -57,7 +71,13 @@ export const QueueName: Brand.Constructor<QueueName> = Brand.nominal<QueueName>(
  *
  * @since 0.1.0
  */
-export type JobState = "waiting" | "delayed" | "active" | "completed" | "failed"
+export type JobState =
+  | "waiting"
+  | "delayed"
+  | "active"
+  | "completed"
+  | "failed"
+  | "cancelled"
 
 /**
  * Retry backoff policy, persisted on the job record so any worker can route
@@ -75,19 +95,101 @@ export interface BackoffPolicy {
 }
 
 /**
- * Retention policy for terminal (completed/failed) jobs, persisted on the
- * record. Applied by the store after terminal acks, scoped to jobs with the
- * same name and state. Default (undefined) keeps records forever.
+ * Retention rule for one terminal state's records (per job name).
  *
- * @since 0.1.0
+ * @since 0.3.0
  */
-export interface KeepPolicy {
+export interface KeepStatePolicy {
   /** Keep at most this many terminal records (per name + state). */
   readonly count?: number | undefined
   /** Remove terminal records older than this many milliseconds. */
   readonly ageMs?: number | undefined
 }
 
+/**
+ * Retention policy persisted on the record, split by terminal state —
+ * completed jobs are usually noise, failed ones evidence. Applied by the
+ * store after terminal acks (scoped to jobs with the same name and state),
+ * and honoured by the store's periodic history sweep when one is configured.
+ * An absent state keeps its records until the store's `historyTtl` ceiling
+ * (forever without one). Producers accept a flat `{ count, age }` shorthand
+ * and normalize it to all three states before it reaches a driver.
+ *
+ * @since 0.1.0
+ */
+export interface KeepPolicy {
+  readonly completed?: KeepStatePolicy | undefined
+  readonly failed?: KeepStatePolicy | undefined
+  readonly cancelled?: KeepStatePolicy | undefined
+}
+
+/**
+ * The terminal states retention applies to.
+ *
+ * @since 0.3.0
+ */
+export type TerminalState = "completed" | "failed" | "cancelled"
+
+/**
+ * Store-level retention ceiling: one duration for all terminal states, or a
+ * per-state split (an absent state is never swept by the timer).
+ *
+ * @since 0.3.0
+ */
+export type HistoryTtlInput =
+  | Duration.Input
+  | {
+    readonly completed?: Duration.Input | undefined
+    readonly failed?: Duration.Input | undefined
+    readonly cancelled?: Duration.Input | undefined
+  }
+
+/**
+ * `HistoryTtlInput` normalized to milliseconds per terminal state.
+ *
+ * @since 0.3.0
+ */
+export interface HistoryTtlByState {
+  readonly completed?: number | undefined
+  readonly failed?: number | undefined
+  readonly cancelled?: number | undefined
+}
+
+/**
+ * Normalize a `HistoryTtlInput` to milliseconds per terminal state.
+ *
+ * @internal
+ */
+export const normalizeHistoryTtl = (
+  input: HistoryTtlInput
+): HistoryTtlByState => {
+  if (Predicate.isObject(input) && !Duration.isDuration(input)) {
+    if (!("completed" in input || "failed" in input || "cancelled" in input)) {
+      // {} or a DurationObject would silently normalize to a 0ms ceiling and
+      // wipe all history — refuse loudly; use "90 days"-style inputs.
+      throw new Error(
+        "effect-mq: historyTtl object must set at least one of completed/failed/cancelled"
+      )
+    }
+    // SAFETY: Duration inputs (numbers, strings, bigints, tuples, Duration
+    // instances, DurationObjects) never carry terminal-state keys, so this
+    // is the per-state split form.
+    const split = input as {
+      readonly completed?: Duration.Input | undefined
+      readonly failed?: Duration.Input | undefined
+      readonly cancelled?: Duration.Input | undefined
+    }
+    return {
+      completed: split.completed !== undefined ? Duration.toMillis(split.completed) : undefined,
+      failed: split.failed !== undefined ? Duration.toMillis(split.failed) : undefined,
+      cancelled: split.cancelled !== undefined ? Duration.toMillis(split.cancelled) : undefined
+    }
+  }
+  // SAFETY: not the split form (checked above), so a plain Duration input.
+  const ms = Duration.toMillis(input as Duration.Input)
+  return { completed: ms, failed: ms, cancelled: ms }
+}
+
 /**
  * One run of a job, persisted so the full run history is durable and
  * inspectable — the storage-level analogue of `tapError` before a rerun,
@@ -103,7 +205,7 @@ export interface AttemptRecord {
   readonly startedAt: number | undefined
   /** Ack/recovery time of this run (epoch millis). */
   readonly finishedAt: number
-  readonly outcome: "completed" | "retried" | "failed" | "stalled"
+  readonly outcome: "completed" | "retried" | "failed" | "stalled" | "cancelled"
   /** Schema-encoded `Exit`; undefined for `stalled`. */
   readonly exit: unknown
 }
@@ -131,6 +233,15 @@ export interface JobRecord {
   readonly stalledCount: number
   readonly backoff: BackoffPolicy | undefined
   readonly keep: KeepPolicy | undefined
+  /** Per-run execution time limit; the worker interrupts the handler past it. */
+  readonly timeoutMs: number | undefined
+  /**
+   * Set by `cancel` on an active job; the owning worker interrupts the
+   * handler on its next heartbeat and acks `Cancelled`.
+   */
+  readonly cancelRequested: boolean
+  /** The dedup key this job was enqueued under, if any (see `DedupePolicy`). */
+  readonly dedupeKey: string | undefined
   /** Epoch millis before which the job must not be claimed. */
   readonly runAt: number
   readonly enqueuedAt: number
@@ -142,6 +253,36 @@ export interface JobRecord {
   readonly failedReason: string | undefined
 }
 
+/**
+ * Deduplication policy carried on an enqueue request, scoped per job name.
+ * Dedup NEVER changes the job id — ids come from the caller, the idempotency
+ * key, or the store's id generator; the dedup key is a separate value with
+ * its own lifecycle:
+ *
+ * - `{ key }` (no ttl): dedupe while the keyed job is pending (waiting,
+ *   delayed, or active); a terminal keyed job frees the key.
+ * - `{ key, ttlMs }`: throttle — at most one job per key per window,
+ *   regardless of the keyed job's completion.
+ * - `extend` (requires `ttlMs`): each deduplicated enqueue pushes the window
+ *   out (debounce).
+ * - `replace`: while the keyed job is still *delayed*, the new enqueue's
+ *   payload/metadata/priority/attempts/backoff/keep/timeout/delay replace the
+ *   existing job's (id and ledger preserved), and a `ttlMs` window is
+ *   re-armed from the replace; in every other state normal dedup applies.
+ *
+ * A deduplicated (or replaced) enqueue returns the keyed job's id with
+ * `duplicate: true`.
+ *
+ * @since 0.3.0
+ */
+export interface DedupePolicy {
+  /** Non-empty; producers validate before the request reaches a driver. */
+  readonly key: string
+  readonly ttlMs: number | undefined
+  readonly extend: boolean
+  readonly replace: boolean
+}
+
 /**
  * @since 0.1.0
  */
@@ -160,9 +301,29 @@ export interface EnqueueRequest {
   readonly attemptsMax: number
   readonly backoff: BackoffPolicy | undefined
   readonly keep: KeepPolicy | undefined
+  readonly timeoutMs: number | undefined
+  /** Deduplicate against other enqueues sharing `dedupe.key` (same name). */
+  readonly dedupe: DedupePolicy | undefined
   readonly delayMs: number
 }
 
+/**
+ * Generator for store-assigned job ids, used when `EnqueueRequest.id` is
+ * undefined (custom ids and idempotency keys always win, and schedule tick
+ * ids stay slot-deterministic). May be effectful (e.g. draw from Effect's
+ * `Random`). The store retries a bounded number of times when a generated id
+ * collides with an existing job, then fails the enqueue with
+ * `JobStoreError` — generators must have enough entropy that collisions are
+ * pathological, not routine.
+ *
+ * @example `({ name }) => \`${name}_${ulid()}\``
+ *
+ * @since 0.2.0
+ */
+export type IdGenerator = (
+  request: EnqueueRequest
+) => string | Effect.Effect<string>
+
 /**
  * @since 0.1.0
  */
@@ -213,6 +374,7 @@ export type AckOutcome =
   | { readonly _tag: "Complete"; readonly exit: unknown }
   | { readonly _tag: "Retry"; readonly delayMs: number; readonly exit: unknown }
   | { readonly _tag: "Fail"; readonly exit: unknown }
+  | { readonly _tag: "Cancelled" }
 
 /**
  * Filters and pagination for `list`. Results are ordered newest-first
@@ -241,6 +403,45 @@ export interface ListResult {
   readonly cursor: string | undefined
 }
 
+/**
+ * A repeatable-job schedule as persisted by the store. Exactly one of `cron`
+ * (with optional IANA `tz`) or `everyMs` is set. The payload is stored
+ * schema-encoded, like job payloads. `nextRunAt` is maintained by the worker
+ * sweep via `advanceSchedule`; ticks enqueue with the deterministic id
+ * `sched/<key>/<slot>`, so concurrent sweepers dedup naturally.
+ *
+ * @since 0.2.0
+ */
+export interface ScheduleRecord {
+  readonly key: ScheduleKey
+  readonly jobName: string
+  readonly queue: QueueName
+  readonly cron: string | undefined
+  readonly tz: string | undefined
+  readonly everyMs: number | undefined
+  readonly payload: unknown
+  readonly metadata: Readonly<Record<string, string>>
+  readonly priority: number
+  readonly attemptsMax: number
+  readonly backoff: BackoffPolicy | undefined
+  readonly keep: KeepPolicy | undefined
+  readonly timeoutMs: number | undefined
+  /** Epoch millis of the next occurrence to enqueue. */
+  readonly nextRunAt: number
+}
+
+/**
+ * Result of a heartbeat: locks that could not be extended (lost to stall
+ * recovery or another worker) and active jobs with a pending cancel request
+ * (the worker must interrupt them and ack `Cancelled`).
+ *
+ * @since 0.2.0
+ */
+export interface ExtendLocksResult {
+  readonly lost: ReadonlyArray<JobId>
+  readonly cancelRequested: ReadonlyArray<JobId>
+}
+
 /**
  * A transient or fatal driver error (connection loss, serialization, etc.).
  *
@@ -260,6 +461,73 @@ export class JobStoreError extends Data.TaggedError("JobStoreError")<{
 export const isJobStoreError = (u: unknown): u is JobStoreError =>
   Predicate.hasProperty(u, "_tag") && u._tag === "JobStoreError"
 
+// Identity-based marking (WeakSet, works on frozen error instances) so the
+// error keeps its declared type and schema round-trip untouched.
+const unrecoverableRegistry = new WeakSet<object>()
+
+/**
+ * Mark an error value as unrecoverable: when a handler fails with it, the
+ * worker skips the remaining retry budget and fails the job immediately. The
+ * error itself is returned unchanged, so typed error channels and schemas
+ * are unaffected. Also exported as `Job.unrecoverable`.
+ *
+ * Marking is identity-based, so it only works for object errors — a
+ * primitive (string/number) failure is returned unmarked and retries
+ * normally; use the definition-level `retryable` predicate for those.
+ *
+ * @since 0.2.0
+ */
+export const unrecoverable = <E>(error: E): E => {
+  if (Object(error) === error) {
+    // SAFETY: `Object(x) === x` is true exactly for object values, which is
+    // what WeakSet membership requires.
+    unrecoverableRegistry.add(error as object)
+  }
+  return error
+}
+
+/**
+ * Whether `unrecoverable` marked this value.
+ *
+ * @internal
+ */
+// oxlint-disable-next-line anti-slop/no-unknown-parameters -- boundary classifier over arbitrary error values
+export const isMarkedUnrecoverable = (u: unknown): boolean =>
+  // SAFETY: `Object(u) === u` short-circuits to false for primitives, so the
+  // assertion only ever passes object values to the WeakSet.
+  Object(u) === u && unrecoverableRegistry.has(u as object)
+
+/**
+ * The next occurrence of a schedule strictly after `now`. `fromSlot` anchors
+ * `everyMs` schedules (occurrences stay on the `slot + k * every` grid).
+ * Returns undefined for an invalid cron expression.
+ *
+ * @internal
+ */
+export const nextOccurrence = (
+  schedule: Pick<ScheduleRecord, "cron" | "tz" | "everyMs">,
+  fromSlot: number,
+  now: number
+): number | undefined => {
+  if (schedule.cron !== undefined) {
+    const parsed = Cron.parse(schedule.cron, schedule.tz)
+    if (Result.isFailure(parsed)) return undefined
+    // Cron.next throws for parseable-but-unsatisfiable expressions (e.g.
+    // "0 0 30 2 *"); treat those as invalid rather than defecting the sweep.
+    try {
+      return Cron.next(parsed.success, new Date(Math.max(fromSlot, now))).getTime()
+    } catch {
+      return undefined
+    }
+  }
+  if (schedule.everyMs !== undefined && schedule.everyMs > 0) {
+    const behind = Math.max(0, now - fromSlot)
+    const steps = Math.floor(behind / schedule.everyMs) + 1
+    return fromSlot + steps * schedule.everyMs
+  }
+  return undefined
+}
+
 
 /**
  * The presented lock token no longer owns the job (it stalled and was
@@ -288,6 +556,35 @@ export class JobNotRetryableError extends Data.TaggedError("JobNotRetryableError
   readonly state: JobState
 }> {}
 
+/**
+ * `cancel` was called on a job that is already terminal.
+ *
+ * @since 0.2.0
+ */
+export class JobNotCancellableError extends Data.TaggedError("JobNotCancellableError")<{
+  readonly jobId: JobId
+  readonly state: JobState
+}> {}
+
+/**
+ * `promote` was called on a job that is not in the `delayed` state.
+ *
+ * @since 0.2.0
+ */
+export class JobNotPromotableError extends Data.TaggedError("JobNotPromotableError")<{
+  readonly jobId: JobId
+  readonly state: JobState
+}> {}
+
+/**
+ * Raised (as a defect) by `awaitResult` when the awaited job was cancelled.
+ *
+ * @since 0.2.0
+ */
+export class JobCancelledError extends Data.TaggedError("JobCancelledError")<{
+  readonly jobId: JobId
+}> {}
+
 /**
  * The service shape every store implements.
  *
@@ -339,7 +636,7 @@ export interface Service {
   readonly extendLocks: (
     locks: ReadonlyArray<{ readonly id: JobId; readonly token: string }>,
     durationMs: number
-  ) => Effect.Effect<ReadonlyArray<JobId>, JobStoreError>
+  ) => Effect.Effect<ExtendLocksResult, JobStoreError>
 
   /**
    * Sweep active jobs whose lock has expired. Each recovered job gets
@@ -391,6 +688,73 @@ export interface Service {
     JobStoreError | JobNotFoundError | JobNotRetryableError
   >
 
+  /**
+   * Cancel a job. Waiting/delayed jobs become terminal (`cancelled`)
+   * immediately; active jobs get `cancelRequested` set, and the owning
+   * worker interrupts the handler on its next heartbeat. Terminal jobs fail
+   * with `JobNotCancellableError`.
+   */
+  readonly cancel: (
+    id: JobId
+  ) => Effect.Effect<
+    void,
+    JobStoreError | JobNotFoundError | JobNotCancellableError
+  >
+
+  /** Move a delayed job to `waiting` now. */
+  readonly promote: (
+    id: JobId
+  ) => Effect.Effect<
+    void,
+    JobStoreError | JobNotFoundError | JobNotPromotableError
+  >
+
+  /**
+   * Durably pause a queue: claims return `Empty` until `resume` (delayed
+   * jobs still promote to `waiting`, they just aren't handed out). Affects
+   * every worker on the store.
+   */
+  readonly pause: (queue: QueueName) => Effect.Effect<void, JobStoreError>
+
+  /** Undo `pause` and wake idle workers. */
+  readonly resume: (queue: QueueName) => Effect.Effect<void, JobStoreError>
+
+  readonly pausedQueues: () => Effect.Effect<
+    ReadonlyArray<QueueName>,
+    JobStoreError
+  >
+
+  /** Create or replace a repeatable-job schedule (keyed by `schedule.key`). */
+  readonly upsertSchedule: (
+    schedule: ScheduleRecord
+  ) => Effect.Effect<void, JobStoreError>
+
+  /** Remove a schedule. Returns false when the key does not exist. */
+  readonly removeSchedule: (
+    key: ScheduleKey
+  ) => Effect.Effect<boolean, JobStoreError>
+
+  readonly listSchedules: (options?: {
+    readonly jobName?: string | undefined
+    readonly queue?: QueueName | undefined
+  }) => Effect.Effect<ReadonlyArray<ScheduleRecord>, JobStoreError>
+
+  /** Schedules whose `nextRunAt` is due (per the Effect `Clock`). */
+  readonly dueSchedules: () => Effect.Effect<
+    ReadonlyArray<ScheduleRecord>,
+    JobStoreError
+  >
+
+  /**
+   * Advance a schedule's `nextRunAt` from `expectedRunAt` to `nextRunAt`
+   * (conditional, so concurrent sweepers cannot regress it).
+   */
+  readonly advanceSchedule: (
+    key: ScheduleKey,
+    expectedRunAt: number,
+    nextRunAt: number
+  ) => Effect.Effect<void, JobStoreError>
+
   readonly counts: (
     queue?: QueueName
   ) => Effect.Effect<Record<JobState, number>, JobStoreError>