effect-mq 0.3.2 → 0.4.0

package/src/Job.ts

@@ -30,11 +30,12 @@
  *
  * @since 0.1.0
  */
-import { Clock, type Context, Duration, Effect, type Exit, Layer, Metric, Option, Predicate, Schedule, Schema } from "effect"
+import { Clock, type Context, DateTime, Duration, Effect, type Exit, Layer, Metric, Option, Predicate, Schedule, Schema } from "effect"
 import {
   type BackoffPolicy,
   type DedupePolicy,
   type KeepStatePolicy,
+  type TraceContext,
   JobCancelledError,
   JobId,
   type JobNotCancellableError,
@@ -130,6 +131,11 @@ export interface JobOptions {
   readonly priority?: number | undefined
   /** Total attempts including the first run. Default 1 (no retries). */
   readonly attempts?: number | undefined
+  /**
+   * Delay between retry attempts:
+   * `{ type: "fixed" | "exponential", delay, factor? }`. Default: retries
+   * are immediate.
+   */
   readonly backoff?: BackoffInput | undefined
   /** Retention for terminal records. Default: keep forever. */
   readonly keep?: KeepInput | undefined
@@ -158,9 +164,28 @@ export interface JobOptions {
  * @since 0.3.0
  */
 export type DedupeInput = string | {
+  /**
+   * The dedup key, scoped to this job's name (e.g. an employee id). Never
+   * changes the job id. Must be non-empty.
+   */
   readonly key: string
+  /**
+   * Throttle window: at most one job per key per window, even after the
+   * keyed job completes. Without `ttl`, dedup lasts while the keyed job is
+   * pending (waiting/delayed/active).
+   */
   readonly ttl?: Duration.Input | undefined
+  /**
+   * Debounce (requires `ttl`): every deduplicated enqueue pushes the window
+   * out again.
+   */
   readonly extend?: boolean | undefined
+  /**
+   * Latest-wins while the keyed job is still delayed: the new enqueue's
+   * payload/metadata/priority/attempts/backoff/keep/timeout/delay replace
+   * the existing job's (same id, ledger preserved). A landed replace
+   * re-arms the `ttl` window. In any other state, normal dedup applies.
+   */
   readonly replace?: boolean | undefined
 }
 
@@ -181,10 +206,51 @@ const normalizeDedupe = (input: DedupeInput | undefined): DedupePolicy | undefin
   }
 }
 
+/**
+ * When the job becomes runnable: a relative `delay` OR an absolute `at`
+ * (any `DateTime.Input` — a `DateTime` from `DateTime.makeZonedUnsafe` for
+ * wall-clock-in-timezone instants, a `Date`, ISO string, epoch millis, or
+ * date parts). Setting both is a compile error; an `at` in the past runs
+ * immediately.
+ *
+ * @since 0.4.0
+ */
+export type RunTimeInput =
+  | {
+    /**
+     * Run this long after enqueue (relative). Any `Duration.Input`:
+     * `"5 seconds"`, `Duration.minutes(10)`, millis, ...
+     *
+     * Mutually exclusive with `at` (setting both is a compile error).
+     */
+    readonly delay?: Duration.Input | undefined
+    /** Set `delay` for relative times, or `at` (alone) for absolute ones. */
+    readonly at?: undefined
+  }
+  | {
+    /**
+     * Run at an absolute instant (any `DateTime.Input`) — no duration math:
+     *
+     * ```ts
+     * at: DateTime.makeZonedUnsafe(
+     *   { year: 2026, month: 8, day: 24, hours: 9 },
+     *   { timeZone: "America/New_York", adjustForTimeZone: true }
+     * )
+     * // also: a Date, ISO string, epoch millis, or { year, month, ... } parts
+     * ```
+     *
+     * An `at` in the past runs immediately. Mutually exclusive with `delay`
+     * (setting both is a compile error).
+     */
+    readonly at?: DateTime.DateTime.Input | undefined
+    /** Set `at` for absolute times, or `delay` (alone) for relative ones. */
+    readonly delay?: undefined
+  }
+
 /**
  * @since 0.1.0
  */
-export interface EnqueueOptions extends JobOptions {
+export type EnqueueOptions = Omit<JobOptions, "delay"> & RunTimeInput & {
   /**
    * Explicit job id. Enqueueing an id that already exists is a no-op that
    * returns the existing id (idempotency). Overrides the definition's
@@ -199,6 +265,27 @@ export interface EnqueueOptions extends JobOptions {
   readonly dedupe?: DedupeInput | undefined
 }
 
+/**
+ * Options for `enqueueMany` — `EnqueueOptions` minus the per-job fields:
+ * `jobId` (a shared id would make every item after the first a duplicate)
+ * and `dedupe` (a shared key would collapse the batch into one job; per-item
+ * dedup still runs via the definition's `dedupe` callback).
+ *
+ * Built from parts rather than `Omit<EnqueueOptions, ...>`: a mapped type
+ * over the `delay`/`at` union would flatten it and lose their exclusivity.
+ *
+ * @since 0.4.0
+ */
+export type EnqueueManyOptions = Omit<JobOptions, "delay"> & RunTimeInput & {
+  /** Send every item to a different queue than the definition's. */
+  readonly queue?: string | undefined
+  /**
+   * Queryable business context, merged over each item's definition-derived
+   * `metadata` (the same overrides apply to every item).
+   */
+  readonly metadata?: Readonly<Record<string, string>> | undefined
+}
+
 interface ResolvedDefaults {
   readonly delayMs: number
   readonly priority: number
@@ -216,16 +303,32 @@ interface ResolvedDefaults {
  * @since 0.2.0
  */
 export interface ScheduleOptions<PayloadInput> {
+  /**
+   * Cron expression (5-field, e.g. `"0 9 * * 1"` = 9:00 every Monday).
+   * First fires at the next matching occurrence. Exactly one of `cron` or
+   * `every` must be set.
+   */
   readonly cron?: string | undefined
+  /** IANA time zone for `cron` (e.g. `"America/New_York"`). Default UTC. */
   readonly tz?: string | undefined
+  /**
+   * Fixed interval; first fires one interval from now and stays on that
+   * grid. Exactly one of `cron` or `every` must be set.
+   */
   readonly every?: Duration.Input | undefined
+  /** The payload every occurrence is enqueued with. */
   readonly payload: PayloadInput
   /** Queryable business context, merged over the definition's `metadata`. */
   readonly metadata?: Readonly<Record<string, string>> | undefined
+  /** Priority for each occurrence. Higher runs first; default 0. */
   readonly priority?: number | undefined
+  /** Attempt budget for each occurrence. Default 1. */
   readonly attempts?: number | undefined
+  /** Retry backoff for each occurrence. */
   readonly backoff?: BackoffInput | undefined
+  /** Retention for each occurrence's terminal record. */
   readonly keep?: KeepInput | undefined
+  /** Per-run execution time limit for each occurrence. */
   readonly timeout?: Duration.Input | undefined
 }
 
@@ -307,6 +410,26 @@ export interface Job<
     options?: EnqueueOptions | undefined
   ) => Effect.Effect<JobId, never, StoreId | Payload["EncodingServices"]>
 
+  /**
+   * Queue many jobs in bulk — one store round trip per chunk of plain
+   * items; items whose definition derives a `dedupe` key fall back to
+   * individual enqueues, in order. Returns ids aligned with the payloads.
+   * Per-item semantics match `enqueue`: `idempotencyKey`, `dedupe`, and
+   * `metadata` callbacks run for each payload, and duplicates are silent
+   * no-ops returning the existing id. Options apply to every item
+   * (`jobId`/`dedupe` are excluded — a shared id or dedup key would
+   * collapse the batch into one job).
+   *
+   * The batch is not transactional: a store failure mid-batch may leave a
+   * subset (not necessarily a prefix) enqueued. Safe under at-least-once —
+   * re-running the batch skips already-inserted items when ids are
+   * deterministic (`idempotencyKey`); store-assigned ids may re-insert.
+   */
+  readonly enqueueMany: (
+    payloads: ReadonlyArray<Payload["~type.make.in"]>,
+    options?: EnqueueManyOptions | undefined
+  ) => Effect.Effect<ReadonlyArray<JobId>, never, StoreId | Payload["EncodingServices"]>
+
   /** Read the current status of a previously enqueued job. */
   readonly poll: (
     jobId: JobId
@@ -368,18 +491,23 @@ export interface Job<
     jobId: JobId
   ) => Effect.Effect<void, JobNotFoundError | JobNotCancellableError, StoreId>
 
+  /**
+   * Cancel whatever pending job holds this dedup key (see `DedupeInput`).
+   * Idempotent: returns false when nothing pending holds the key.
+   */
+  readonly cancelByKey: (key: string) => Effect.Effect<boolean, never, StoreId>
+
   /** Run a delayed job now. */
   readonly promote: (
     jobId: JobId
   ) => Effect.Effect<void, JobNotFoundError | JobNotPromotableError, StoreId>
 
   /**
-   * Create or replace a durable repeatable schedule for this job. Ticks
-   * enqueue with a slot-deterministic id, so schedules are exactly-once per
-   * occurrence across all workers (assuming history retention windows
-   * comfortably exceed the sweep interval — a pruned tick job cannot dedup a
-   * pathologically stale sweeper). Missed occurrences (downtime) collapse
-   * into a single catch-up run.
+   * Create or replace a durable repeatable schedule for this job. Each
+   * occurrence is claimed and enqueued in one atomic store op, so ticks are
+   * exactly-once per occurrence across all workers — regardless of history
+   * retention. Missed occurrences (downtime) collapse into a single
+   * catch-up run.
    *
    * Re-registering with an *unchanged* cadence (same `cron`/`tz`/`every`) is
    * a no-op for the next occurrence — deploy-time re-registration neither
@@ -490,9 +618,33 @@ const Proto = {
         options?.dedupe ?? this.dedupe?.(payload)
       )
       const queue = options?.queue !== undefined ? QueueName(options.queue) : this.queue
-      return Schema.encodeEffect(this.payloadJsonSchema)(payload).pipe(
+      if (options?.at !== undefined && options.delay !== undefined) {
+        // Unrepresentable in TypeScript; guard untyped callers anyway.
+        throw new Error("effect-mq: set either `delay` or `at`, not both")
+      }
+      const delayMs = options?.at !== undefined
+        ? Effect.map(
+          Clock.currentTimeMillis,
+          (now) => Math.max(0, DateTime.toEpochMillis(DateTime.makeUnsafe(options.at ?? now)) - now)
+        )
+        : Effect.succeed(
+          options?.delay !== undefined
+            ? Duration.toMillis(options.delay)
+            : this.defaults.delayMs
+        )
+      // The enqueue span's context rides along on the record, so the
+      // handler's span joins the producing trace across processes.
+      const spanContext = Effect.currentSpan.pipe(
+        Effect.map((span) => ({
+          traceId: span.traceId,
+          spanId: span.spanId,
+          sampled: span.sampled
+        })),
+        Effect.catchTag("NoSuchElementError", () => Effect.succeed(undefined))
+      )
+      return Effect.all([Schema.encodeEffect(this.payloadJsonSchema)(payload), delayMs, spanContext]).pipe(
         Effect.orDie,
-        Effect.flatMap((encoded) =>
+        Effect.flatMap(([encoded, resolvedDelayMs, capturedSpan]) =>
           Effect.flatMap(this.store, (store) =>
             store.enqueue({
               id,
@@ -512,9 +664,12 @@ const Proto = {
                 ? Duration.toMillis(options.timeout)
                 : this.defaults.timeoutMs,
               dedupe,
-              delayMs: options?.delay !== undefined
-                ? Duration.toMillis(options.delay)
-                : this.defaults.delayMs
+              // `delayed` records scheduling INTENT (not queue backlog), so
+              // the worker's auto trace-linking stays deterministic.
+              trace: capturedSpan === undefined
+                ? undefined
+                : { ...capturedSpan, delayed: resolvedDelayMs > 0 } satisfies TraceContext,
+              delayMs: resolvedDelayMs
             }))
         ),
         Effect.orDie,
@@ -537,6 +692,91 @@ const Proto = {
     )
   },
 
+  enqueueMany(this: AnyWithProps, payloads: ReadonlyArray<any>, options?: EnqueueManyOptions) {
+    return Effect.suspend(() => {
+      const queue = options?.queue !== undefined ? QueueName(options.queue) : this.queue
+      if (options?.at !== undefined && options.delay !== undefined) {
+        // Unrepresentable in TypeScript; guard untyped callers anyway.
+        throw new Error("effect-mq: set either `delay` or `at`, not both")
+      }
+      const delayMs = options?.at !== undefined
+        ? Effect.map(
+          Clock.currentTimeMillis,
+          (now) => Math.max(0, DateTime.toEpochMillis(DateTime.makeUnsafe(options.at ?? now)) - now)
+        )
+        : Effect.succeed(
+          options?.delay !== undefined
+            ? Duration.toMillis(options.delay)
+            : this.defaults.delayMs
+        )
+      const spanContext = Effect.currentSpan.pipe(
+        Effect.map((span) => ({
+          traceId: span.traceId,
+          spanId: span.spanId,
+          sampled: span.sampled
+        })),
+        Effect.catchTag("NoSuchElementError", () => Effect.succeed(undefined))
+      )
+      const encodedAll = Effect.forEach(payloads, (fields) => {
+        const payload = this.payloadSchema.make(fields)
+        return Effect.map(
+          Schema.encodeEffect(this.payloadJsonSchema)(payload),
+          (encoded) => ({ payload, encoded })
+        )
+      })
+      return Effect.all([encodedAll, delayMs, spanContext]).pipe(
+        Effect.orDie,
+        Effect.flatMap(([items, resolvedDelayMs, capturedSpan]) =>
+          Effect.flatMap(this.store, (store) =>
+            store.enqueueMany(items.map(({ encoded, payload }) => ({
+              id: this.idempotencyKey !== undefined
+                ? JobId(`${this._tag}/${this.idempotencyKey(payload)}`)
+                : undefined,
+              name: this._tag,
+              queue,
+              payload: encoded,
+              metadata: { ...this.metadata?.(payload), ...options?.metadata },
+              priority: options?.priority ?? this.defaults.priority,
+              attemptsMax: Math.max(1, options?.attempts ?? this.defaults.attempts),
+              backoff: options?.backoff !== undefined
+                ? normalizeBackoff(options.backoff)
+                : this.defaults.backoff,
+              keep: options?.keep !== undefined
+                ? normalizeKeep(options.keep)
+                : this.defaults.keep,
+              timeoutMs: options?.timeout !== undefined
+                ? Duration.toMillis(options.timeout)
+                : this.defaults.timeoutMs,
+              dedupe: normalizeDedupe(this.dedupe?.(payload)),
+              trace: capturedSpan === undefined
+                ? undefined
+                : { ...capturedSpan, delayed: resolvedDelayMs > 0 } satisfies TraceContext,
+              delayMs: resolvedDelayMs
+            })))
+          )
+        ),
+        Effect.orDie,
+        Effect.tap((results) => {
+          const duplicates = results.filter((result) => result.duplicate).length
+          const update = (duplicate: "true" | "false", count: number) =>
+            count === 0 ? Effect.void : Metric.update(
+              Metrics.jobsEnqueued.pipe(
+                Metric.withAttributes({ name: this._tag, queue, duplicate })
+              ),
+              count
+            )
+          return Effect.andThen(
+            update("false", results.length - duplicates),
+            update("true", duplicates)
+          )
+        }),
+        Effect.map((results) => results.map((result) => result.id))
+      )
+    }).pipe(
+      Effect.withSpan(`${this._tag}.enqueueMany`, {}, { captureStackTrace: false })
+    )
+  },
+
   poll(this: AnyWithProps, jobId: JobId) {
     const self = this
     return Effect.flatMap(this.store, (store) =>
@@ -661,6 +901,15 @@ const Proto = {
       )
   },
 
+  cancelByKey(this: AnyWithProps, key: string) {
+    return Effect.flatMap(this.store, (store) =>
+      store.cancelByDedupe(this._tag, key).pipe(
+        Effect.catchTag("JobStoreError", (error) => Effect.die(error))
+      )).pipe(
+        Effect.withSpan(`${this._tag}.cancelByKey`, { attributes: { key } }, { captureStackTrace: false })
+      )
+  },
+
   promote(this: AnyWithProps, jobId: JobId) {
     return Effect.flatMap(this.store, (store) =>
       store.promote(jobId).pipe(
@@ -743,12 +992,14 @@ const Proto = {
 
 const boundMethods = [
   "enqueue",
+  "enqueueMany",
   "poll",
   "attempts",
   "awaitResult",
   "execute",
   "retry",
   "cancel",
+  "cancelByKey",
   "promote",
   "schedule",
   "unschedule",
@@ -805,8 +1056,11 @@ export const make = <
 >(
   name: Name,
   options: {
+    /** The payload schema: a `Schema.Struct` or its bare fields object. */
     readonly payload: Payload
+    /** Schema for the handler's success value (decodable via `awaitResult`/`attempts`). Default `Schema.Void`. */
     readonly success?: Success | undefined
+    /** Schema for the handler's typed failure (round-trips through storage). Default `Schema.Never`. */
     readonly error?: Error | undefined
     /**
      * Derive a stable job id from the payload. Enqueueing the same key twice
@@ -855,6 +1109,7 @@ export const make = <
      * Default: the default `JobStore`.
      */
     readonly store?: Context.Key<StoreId, StoreService> | undefined
+    /** Default enqueue options (`delay`, `priority`, `attempts`, `backoff`, `keep`, `timeout`); per-enqueue options override. */
     readonly defaults?: JobOptions | undefined
   }
 ): Job<

package/src/JobStore.ts

@@ -242,6 +242,8 @@ export interface JobRecord {
   readonly cancelRequested: boolean
   /** The dedup key this job was enqueued under, if any (see `DedupePolicy`). */
   readonly dedupeKey: string | undefined
+  /** The producer's span context, restored as the handler span's parent. */
+  readonly trace: TraceContext | undefined
   /** Epoch millis before which the job must not be claimed. */
   readonly runAt: number
   readonly enqueuedAt: number
@@ -283,6 +285,25 @@ export interface DedupePolicy {
   readonly replace: boolean
 }
 
+/**
+ * The producer's span context, persisted at enqueue so the handler's span
+ * can join the producing trace across processes (via `Tracer.externalSpan`).
+ *
+ * @since 0.4.0
+ */
+export interface TraceContext {
+  readonly traceId: string
+  readonly spanId: string
+  readonly sampled: boolean
+  /**
+   * Whether the enqueue explicitly scheduled the job for the future
+   * (`delay`/`at`). Drives the worker's `traceLinking: "auto"` policy:
+   * immediate work continues the producer trace, future work starts its own
+   * trace with a causal link.
+   */
+  readonly delayed: boolean
+}
+
 /**
  * @since 0.1.0
  */
@@ -304,6 +325,8 @@ export interface EnqueueRequest {
   readonly timeoutMs: number | undefined
   /** Deduplicate against other enqueues sharing `dedupe.key` (same name). */
   readonly dedupe: DedupePolicy | undefined
+  /** The producer's span context, for cross-process trace propagation. */
+  readonly trace: TraceContext | undefined
   readonly delayMs: number
 }
 
@@ -407,8 +430,9 @@ export interface ListResult {
  * A repeatable-job schedule as persisted by the store. Exactly one of `cron`
  * (with optional IANA `tz`) or `everyMs` is set. The payload is stored
  * schema-encoded, like job payloads. `nextRunAt` is maintained by the worker
- * sweep via `advanceSchedule`; ticks enqueue with the deterministic id
- * `sched/<key>/<slot>`, so concurrent sweepers dedup naturally.
+ * sweep via the atomic `tickSchedule` (CAS + insert + advance in one op),
+ * with tick jobs using the deterministic id `sched/<key>/<slot>`; concurrent
+ * sweepers lose the CAS, so each occurrence fires exactly once.
  *
  * @since 0.2.0
  */
@@ -599,6 +623,27 @@ export interface Service {
     request: EnqueueRequest
   ) => Effect.Effect<EnqueueResult, JobStoreError>
 
+  /**
+   * Insert many jobs in bulk: one store round trip per chunk of plain items
+   * (drivers chunk large batches); items carrying a dedup key run through
+   * the single-enqueue decision tree individually, in order. Results align
+   * positionally with the requests; each item carries full single-enqueue
+   * semantics (id dedup, dedup keys, delayed routing).
+   *
+   * The batch is NOT one transaction — items are independent, and a failure
+   * may leave a *subset* (not necessarily a prefix) applied. Safe under
+   * at-least-once: re-running a batch whose ids are deterministic skips
+   * what already landed, but items with store-assigned ids may re-insert.
+   * FIFO order within a priority holds except for an item whose
+   * auto/generated id collides with an existing row — it re-draws and lands
+   * after its batch-mates.
+   *
+   * @since 0.4.0
+   */
+  readonly enqueueMany: (
+    requests: ReadonlyArray<EnqueueRequest>
+  ) => Effect.Effect<ReadonlyArray<EnqueueResult>, JobStoreError>
+
   /**
    * Atomically: promote due delayed jobs, then claim the best runnable job
    * matching `queue` + `names` (highest priority first, FIFO within a
@@ -701,6 +746,20 @@ export interface Service {
     JobStoreError | JobNotFoundError | JobNotCancellableError
   >
 
+  /**
+   * Cancel whatever pending job is registered under a dedup key
+   * (name-scoped): pending states are cancelled exactly like `cancel`
+   * (waiting/delayed become terminal, active gets the heartbeat flag).
+   * Returns false when no pending job holds the key — idempotent by design,
+   * so "cancel it if anything is scheduled" needs no existence check.
+   *
+   * @since 0.4.0
+   */
+  readonly cancelByDedupe: (
+    name: string,
+    key: string
+  ) => Effect.Effect<boolean, JobStoreError>
+
   /** Move a delayed job to `waiting` now. */
   readonly promote: (
     id: JobId
@@ -745,6 +804,22 @@ export interface Service {
     JobStoreError
   >
 
+  /**
+   * Atomically claim one schedule occurrence: iff the schedule's `nextRunAt`
+   * still equals `expectedRunAt`, insert the tick job AND advance to
+   * `nextRunAt` in the same transaction, returning true. A stale sweeper's
+   * tick returns false without inserting — exactly-once per occurrence even
+   * when the previous slot's job row has been pruned by retention.
+   *
+   * @since 0.4.0
+   */
+  readonly tickSchedule: (
+    key: ScheduleKey,
+    expectedRunAt: number,
+    nextRunAt: number,
+    request: EnqueueRequest
+  ) => Effect.Effect<boolean, JobStoreError>
+
   /**
    * Advance a schedule's `nextRunAt` from `expectedRunAt` to `nextRunAt`
    * (conditional, so concurrent sweepers cannot regress it).