effect-mq 0.1.0

package/src/JobStore.ts

@@ -0,0 +1,446 @@
+/**
+ * The storage seam of effect-mq.
+ *
+ * `JobStore` is the minimal, storage-agnostic interface a queue backend must
+ * implement. Every method must be atomic within the driver. The reference
+ * implementation is `MemoryJobStore`; Postgres (via `effect-mq/drizzle`),
+ * Redis, etc. drivers implement the same service.
+ *
+ * The store works entirely on *encoded* (JSON-safe) payloads and exits —
+ * schema encoding/decoding happens in `Job` (producer side) and `Worker`
+ * (consumer side), so drivers stay dumb.
+ *
+ * Multiple stores can coexist in one application via named store keys (see
+ * `named`): each job definition binds to a store key, so business-critical
+ * jobs can live in Postgres while disposable ones live elsewhere.
+ *
+ * @since 0.1.0
+ */
+import { Brand, Context, Data, type Effect, type Option, Predicate } from "effect"
+
+/**
+ * The identifier of an enqueued job. Produced by `enqueue` (either
+ * store-assigned or derived from a custom id / idempotency key).
+ *
+ * @since 0.1.0
+ */
+export type JobId = Brand.Branded<string, "effect-mq/JobId">
+
+/**
+ * Brand a raw string as a `JobId`.
+ *
+ * @since 0.1.0
+ */
+export const JobId: Brand.Constructor<JobId> = Brand.nominal<JobId>()
+
+/**
+ * The name of a queue.
+ *
+ * @since 0.1.0
+ */
+export type QueueName = Brand.Branded<string, "effect-mq/QueueName">
+
+/**
+ * Brand a raw string as a `QueueName`.
+ *
+ * @since 0.1.0
+ */
+export const QueueName: Brand.Constructor<QueueName> = Brand.nominal<QueueName>()
+
+/**
+ * The lifecycle states of a job.
+ *
+ * - `waiting`: runnable now, ordered by (priority desc, enqueue order asc)
+ * - `delayed`: must not run before `runAt`
+ * - `active`: claimed by a worker holding a lock token
+ * - `completed` / `failed`: terminal, with an encoded `Exit` stored
+ *
+ * @since 0.1.0
+ */
+export type JobState = "waiting" | "delayed" | "active" | "completed" | "failed"
+
+/**
+ * Retry backoff policy, persisted on the job record so any worker can route
+ * retries consistently. Delay for attempt `n` (1-based):
+ *
+ * - `fixed`: `delayMs`
+ * - `exponential`: `delayMs * factor ** (n - 1)` (factor defaults to 2)
+ *
+ * @since 0.1.0
+ */
+export interface BackoffPolicy {
+  readonly _tag: "fixed" | "exponential"
+  readonly delayMs: number
+  readonly factor?: number | undefined
+}
+
+/**
+ * Retention policy for terminal (completed/failed) jobs, persisted on the
+ * record. Applied by the store after terminal acks, scoped to jobs with the
+ * same name and state. Default (undefined) keeps records forever.
+ *
+ * @since 0.1.0
+ */
+export interface KeepPolicy {
+  /** Keep at most this many terminal records (per name + state). */
+  readonly count?: number | undefined
+  /** Remove terminal records older than this many milliseconds. */
+  readonly ageMs?: number | undefined
+}
+
+/**
+ * One run of a job, persisted so the full run history is durable and
+ * inspectable — the storage-level analogue of `tapError` before a rerun,
+ * extended to successes. Attempt numbers are monotonic per job and survive
+ * `retry` (they are decoupled from the record's `attemptsMade` budget).
+ *
+ * @since 0.1.0
+ */
+export interface AttemptRecord {
+  /** 1-based, monotonic per job (= previous ledger length + 1). */
+  readonly attempt: number
+  /** Claim time of this run (epoch millis). */
+  readonly startedAt: number | undefined
+  /** Ack/recovery time of this run (epoch millis). */
+  readonly finishedAt: number
+  readonly outcome: "completed" | "retried" | "failed" | "stalled"
+  /** Schema-encoded `Exit`; undefined for `stalled`. */
+  readonly exit: unknown
+}
+
+/**
+ * A job as persisted by the store. `payload` and `exit` are schema-encoded
+ * (JSON-safe) values — the store never inspects them. The per-run ledger is
+ * fetched separately via `getAttempts` so listings stay cheap.
+ *
+ * @since 0.1.0
+ */
+export interface JobRecord {
+  readonly id: JobId
+  readonly name: string
+  readonly queue: QueueName
+  readonly payload: unknown
+  /** Flat, indexable projection of business context for querying/UIs. */
+  readonly metadata: Readonly<Record<string, string>>
+  readonly state: JobState
+  readonly priority: number
+  /** Total attempts allowed (including the first run). */
+  readonly attemptsMax: number
+  /** Attempts consumed in the current budget (reset by `retry`). */
+  readonly attemptsMade: number
+  readonly stalledCount: number
+  readonly backoff: BackoffPolicy | undefined
+  readonly keep: KeepPolicy | undefined
+  /** Epoch millis before which the job must not be claimed. */
+  readonly runAt: number
+  readonly enqueuedAt: number
+  readonly processedAt: number | undefined
+  readonly finishedAt: number | undefined
+  /** Schema-encoded `Exit`, present for completed/failed jobs. */
+  readonly exit: unknown
+  /** Set when the store itself failed the job (e.g. exceeded stall limit). */
+  readonly failedReason: string | undefined
+}
+
+/**
+ * @since 0.1.0
+ */
+export interface EnqueueRequest {
+  /**
+   * Custom/idempotency id. When a job with this id already exists (in any
+   * state), the request is a no-op and the result has `duplicate: true`.
+   * When `undefined` the store assigns a unique id.
+   */
+  readonly id: JobId | undefined
+  readonly name: string
+  readonly queue: QueueName
+  readonly payload: unknown
+  readonly metadata: Readonly<Record<string, string>>
+  readonly priority: number
+  readonly attemptsMax: number
+  readonly backoff: BackoffPolicy | undefined
+  readonly keep: KeepPolicy | undefined
+  readonly delayMs: number
+}
+
+/**
+ * @since 0.1.0
+ */
+export interface EnqueueResult {
+  readonly id: JobId
+  /** True when a job with this id already existed; nothing was modified. */
+  readonly duplicate: boolean
+}
+
+/**
+ * @since 0.1.0
+ */
+export interface ClaimOptions {
+  readonly queue: QueueName
+  /** Only jobs with these names may be claimed (the worker's registered handlers). */
+  readonly names: ReadonlyArray<string>
+  /** Worker-generated lock token; all subsequent acks must present it. */
+  readonly token: string
+  readonly lockDurationMs: number
+}
+
+/**
+ * Result of a claim attempt. `Empty.nextRunAt` is the earliest `runAt` among
+ * matching delayed jobs (so the worker knows how long to sleep), and
+ * `wakeToken` is an opaque cursor for `awaitWake` so wake-ups that happen
+ * between the claim and the wait are not lost.
+ *
+ * @since 0.1.0
+ */
+export type ClaimResult =
+  | { readonly _tag: "Claimed"; readonly job: JobRecord }
+  | {
+    readonly _tag: "Empty"
+    readonly nextRunAt: number | undefined
+    readonly wakeToken: number
+  }
+
+/**
+ * How a worker acknowledges a claimed job. Retry routing (backoff delay,
+ * attempts accounting) is computed by the worker; the store only applies it.
+ *
+ * Every outcome appends an `AttemptRecord` to the job's ledger (`Complete` →
+ * completed, `Retry` → retried, `Fail` → failed).
+ *
+ * @since 0.1.0
+ */
+export type AckOutcome =
+  | { readonly _tag: "Complete"; readonly exit: unknown }
+  | { readonly _tag: "Retry"; readonly delayMs: number; readonly exit: unknown }
+  | { readonly _tag: "Fail"; readonly exit: unknown }
+
+/**
+ * Filters and pagination for `list`. Results are ordered newest-first
+ * (`enqueuedAt` desc, then id desc); pass the returned `cursor` back to get
+ * the next page.
+ *
+ * @since 0.1.0
+ */
+export interface ListOptions {
+  readonly queue?: QueueName | undefined
+  readonly name?: string | undefined
+  readonly states?: ReadonlyArray<JobState> | undefined
+  /** Every entry must match the record's metadata exactly (AND semantics). */
+  readonly metadata?: Readonly<Record<string, string>> | undefined
+  readonly cursor?: string | undefined
+  /** Page size; default 50. */
+  readonly limit?: number | undefined
+}
+
+/**
+ * @since 0.1.0
+ */
+export interface ListResult {
+  readonly items: ReadonlyArray<JobRecord>
+  /** Present when more items may exist; pass back via `ListOptions.cursor`. */
+  readonly cursor: string | undefined
+}
+
+/**
+ * A transient or fatal driver error (connection loss, serialization, etc.).
+ *
+ * @since 0.1.0
+ */
+export class JobStoreError extends Data.TaggedError("JobStoreError")<{
+  readonly message: string
+  readonly cause?: unknown
+}> {}
+
+/**
+ * Tag-based guard (safe across duplicate module copies, unlike `instanceof`).
+ *
+ * @since 0.1.0
+ */
+// oxlint-disable-next-line anti-slop/no-unknown-parameters -- a type guard IS the boundary parser; `unknown` input is its purpose
+export const isJobStoreError = (u: unknown): u is JobStoreError =>
+  Predicate.hasProperty(u, "_tag") && u._tag === "JobStoreError"
+
+
+/**
+ * The presented lock token no longer owns the job (it stalled and was
+ * recovered, or another worker claimed it).
+ *
+ * @since 0.1.0
+ */
+export class LockLostError extends Data.TaggedError("LockLostError")<{
+  readonly jobId: JobId
+}> {}
+
+/**
+ * @since 0.1.0
+ */
+export class JobNotFoundError extends Data.TaggedError("JobNotFoundError")<{
+  readonly jobId: JobId
+}> {}
+
+/**
+ * `retry` was called on a job that is not in the `failed` state.
+ *
+ * @since 0.1.0
+ */
+export class JobNotRetryableError extends Data.TaggedError("JobNotRetryableError")<{
+  readonly jobId: JobId
+  readonly state: JobState
+}> {}
+
+/**
+ * The service shape every store implements.
+ *
+ * @since 0.1.0
+ */
+export interface Service {
+  /**
+   * Insert a job. Routing: `delayMs > 0` lands in `delayed`, otherwise
+   * `waiting`. Duplicate ids are a silent no-op (see `EnqueueRequest.id`).
+   */
+  readonly enqueue: (
+    request: EnqueueRequest
+  ) => Effect.Effect<EnqueueResult, JobStoreError>
+
+  /**
+   * Atomically: promote due delayed jobs, then claim the best runnable job
+   * matching `queue` + `names` (highest priority first, FIFO within a
+   * priority), locking it with `token` for `lockDurationMs`.
+   */
+  readonly claim: (
+    options: ClaimOptions
+  ) => Effect.Effect<ClaimResult, JobStoreError>
+
+  /**
+   * Acknowledge a claimed job. Verifies the lock token, releases the lock,
+   * increments `attemptsMade`, appends to the attempts ledger, then applies
+   * the outcome (`Complete`/`Fail` are terminal and apply the record's `keep`
+   * policy; `Retry` re-queues after `delayMs`).
+   */
+  readonly ack: (
+    id: JobId,
+    token: string,
+    outcome: AckOutcome
+  ) => Effect.Effect<void, JobStoreError | JobNotFoundError | LockLostError>
+
+  /**
+   * Return a claimed job to `waiting` without consuming an attempt or
+   * recording a ledger entry (used on worker shutdown).
+   */
+  readonly release: (
+    id: JobId,
+    token: string
+  ) => Effect.Effect<void, JobStoreError | JobNotFoundError | LockLostError>
+
+  /**
+   * Heartbeat: extend the given locks. Returns the ids whose lock could NOT
+   * be extended (lost to stall recovery or another worker).
+   */
+  readonly extendLocks: (
+    locks: ReadonlyArray<{ readonly id: JobId; readonly token: string }>,
+    durationMs: number
+  ) => Effect.Effect<ReadonlyArray<JobId>, JobStoreError>
+
+  /**
+   * Sweep active jobs whose lock has expired. Each recovered job gets
+   * `stalledCount + 1` and a `stalled` ledger entry; jobs exceeding
+   * `maxStalledCount` are failed (`failed: true` in the result), the rest
+   * return to `waiting`.
+   */
+  readonly recoverStalled: (options: {
+    readonly maxStalledCount: number
+  }) => Effect.Effect<
+    ReadonlyArray<{ readonly id: JobId; readonly failed: boolean }>,
+    JobStoreError
+  >
+
+  /**
+   * Resolve when new work *may* be runnable for the given queues since the
+   * `wakeToken` observed by a previous `claim`. Spurious wake-ups are fine;
+   * callers must combine with their own timeout. Must be interruptible.
+   * Polling-only drivers may never resolve.
+   */
+  readonly awaitWake: (
+    queues: ReadonlyArray<QueueName>,
+    wakeToken: number
+  ) => Effect.Effect<void, JobStoreError>
+
+  readonly getJob: (
+    id: JobId
+  ) => Effect.Effect<Option.Option<JobRecord>, JobStoreError>
+
+  /** The job's run ledger, oldest first. Empty for unknown ids. */
+  readonly getAttempts: (
+    id: JobId
+  ) => Effect.Effect<ReadonlyArray<AttemptRecord>, JobStoreError>
+
+  /** Query jobs (newest first) — the data layer for dashboards/UIs. */
+  readonly list: (
+    options: ListOptions
+  ) => Effect.Effect<ListResult, JobStoreError>
+
+  /**
+   * Re-run a failed job: back to `waiting` with a fresh attempt budget
+   * (`attemptsMade`/`stalledCount` reset, terminal fields cleared). The
+   * attempts ledger is preserved and keeps numbering monotonically.
+   */
+  readonly retry: (
+    id: JobId
+  ) => Effect.Effect<
+    void,
+    JobStoreError | JobNotFoundError | JobNotRetryableError
+  >
+
+  readonly counts: (
+    queue?: QueueName
+  ) => Effect.Effect<Record<JobState, number>, JobStoreError>
+
+  /** Remove a job (and its ledger). Refuses (returns false) when active. */
+  readonly remove: (id: JobId) => Effect.Effect<boolean, JobStoreError>
+}
+
+/**
+ * The default store key. Jobs without an explicit `store` binding use this.
+ *
+ * @since 0.1.0
+ */
+export class JobStore extends Context.Service<JobStore, Service>()(
+  "effect-mq/JobStore"
+) {}
+
+/**
+ * Phantom identifier for a named store — appears in `R` so the type system
+ * enforces that the right store layer is provided.
+ *
+ * @since 0.1.0
+ */
+export interface Named<in out Name extends string> {
+  readonly "~effect-mq/JobStore/Named": Name
+}
+
+/**
+ * Create a named store key. Jobs bound to it (via `Job.make`'s `store`
+ * option) require it in `R` instead of the default `JobStore`, letting
+ * different jobs run on different storage infrastructure:
+ *
+ * ```ts
+ * const Durable = JobStore.named("durable")     // -> Postgres in prod
+ * const Ephemeral = JobStore.named("ephemeral") // -> Redis in prod
+ * ```
+ *
+ * Keys are identified by their name string: two `named("durable")` calls are
+ * interchangeable.
+ *
+ * @since 0.1.0
+ */
+export const named = <const Name extends string>(
+  name: Name
+): Context.Key<Named<Name>, Service> =>
+  Context.Service<Named<Name>, Service>(`effect-mq/JobStore/${name}`)
+
+/**
+ * Any store key — the default `JobStore` or a `named` one.
+ *
+ * @since 0.1.0
+ */
+export type AnyKey = Context.Key<any, Service>