effect-mq 0.4.2 → 0.6.0

package/src/Job.ts

@@ -63,7 +63,7 @@ export {
    */
   unrecoverable
 }
-import { type JobContext, type RegisterOptions, Worker } from "./Worker.ts"
+import { type CurrentJob, type RegisterOptions, Worker } from "./Worker.ts"
 
 const TypeId = "~effect-mq/Job" as const
 
@@ -308,7 +308,12 @@ export type EnqueueManyOptions = Omit<JobOptions, "delay"> & RunTimeInput & {
   readonly metadata?: Readonly<Record<string, string>> | undefined
 }
 
-interface ResolvedDefaults {
+/**
+ * A definition's `defaults`, normalized to store units.
+ *
+ * @since 0.6.0
+ */
+export interface ResolvedDefaults {
   readonly delayMs: number
   readonly priority: number
   readonly attempts: number
@@ -352,6 +357,12 @@ export interface ScheduleOptions<PayloadInput> {
   readonly keep?: KeepInput | undefined
   /** Per-run execution time limit for each occurrence. */
   readonly timeout?: Duration.Input | undefined
+  /**
+   * Ownership label for declarative reconciliation: `JobSchedules.layer`
+   * prunes only schedules carrying its own group. Unlabeled schedules are
+   * never pruned. Usually set by `JobSchedules`, not by hand.
+   */
+  readonly group?: string | undefined
 }
 
 /**
@@ -378,8 +389,12 @@ export interface JobAttempt<A, E> {
   readonly attempt: number
   readonly startedAt: number | undefined
   readonly finishedAt: number
-  readonly outcome: "completed" | "retried" | "failed" | "stalled" | "cancelled"
-  /** Absent for `stalled` and `cancelled` entries. */
+  readonly outcome: "completed" | "retried" | "failed" | "stalled" | "cancelled" | "fanned-out"
+  /**
+   * Absent for `stalled`, `cancelled`, and `fanned-out` entries — and for
+   * `failed` ones settled store-side without a handler exit (e.g. a
+   * fail-fast flow settle).
+   */
   readonly exit: Option.Option<Exit.Exit<A, E>>
 }
 
@@ -548,19 +563,20 @@ export interface Job<
 
   /**
    * Attach the handler that processes this job, as a layer to provide on top
-   * of `Worker.layer` (bound to the same store).
+   * of `Worker.layer` (bound to the same store). The handler reads the
+   * running attempt from the `Worker.CurrentJob` service — the worker
+   * provides it per run, so it never appears in the layer's requirements.
    */
   readonly toLayer: <R>(
     handler: (
-      payload: Payload["Type"],
-      context: JobContext
+      payload: Payload["Type"]
     ) => Effect.Effect<Success["Type"], Error["Type"], R>,
     options?: RegisterOptions | undefined
   ) => Layer.Layer<
     never,
     never,
     | Worker
-    | R
+    | Exclude<R, CurrentJob>
     | Payload["DecodingServices"]
     | Success["EncodingServices"]
     | Error["EncodingServices"]
@@ -583,7 +599,13 @@ const defaultPollSchedule = Schedule.min([
   Schedule.spaced("1 second")
 ])
 
-const normalizeBackoff = (input: BackoffInput | undefined): BackoffPolicy | undefined =>
+/**
+ * Normalize a user-facing `BackoffInput` to the persisted policy. Shared
+ * with the flow runtime's child-spec builder.
+ *
+ * @internal
+ */
+export const normalizeBackoff = (input: BackoffInput | undefined): BackoffPolicy | undefined =>
   input === undefined ? undefined : {
     _tag: input.type,
     delayMs: Duration.toMillis(input.delay),
@@ -595,7 +617,13 @@ const normalizeKeepState = (input: KeepStateInput): KeepStatePolicy => ({
   ageMs: input.age !== undefined ? Duration.toMillis(input.age) : undefined
 })
 
-const normalizeKeep = (input: KeepInput | undefined): KeepPolicy | undefined => {
+/**
+ * Normalize a user-facing `KeepInput` to the persisted policy. Shared with
+ * the flow runtime's child-spec builder.
+ *
+ * @internal
+ */
+export const normalizeKeep = (input: KeepInput | undefined): KeepPolicy | undefined => {
   if (input === undefined) return undefined
   const split = "completed" in input || "failed" in input || "cancelled" in input
   const flat = "count" in input || "age" in input
@@ -691,6 +719,7 @@ const Proto = {
               trace: capturedSpan === undefined
                 ? undefined
                 : { ...capturedSpan, delayed: resolvedDelayMs > 0 } satisfies TraceContext,
+              parent: undefined,
               delayMs: resolvedDelayMs
             }))
         ),
@@ -773,6 +802,7 @@ const Proto = {
               trace: capturedSpan === undefined
                 ? undefined
                 : { ...capturedSpan, delayed: resolvedDelayMs > 0 } satisfies TraceContext,
+              parent: undefined,
               delayMs: resolvedDelayMs
             })))
           )
@@ -984,6 +1014,7 @@ const Proto = {
         timeoutMs: options.timeout !== undefined
           ? Duration.toMillis(options.timeout)
           : self.defaults.timeoutMs,
+        group: options.group,
         nextRunAt
       }
       const store = yield* self.store
@@ -1003,7 +1034,7 @@ const Proto = {
 
   toLayer(
     this: AnyWithProps,
-    handler: (payload: any, context: JobContext) => Effect.Effect<any, any, any>,
+    handler: (payload: any) => Effect.Effect<any, any, any>,
     options?: RegisterOptions
   ) {
     return Layer.effectDiscard(

package/src/JobSchedules.ts

@@ -0,0 +1,223 @@
+/**
+ * Declarative schedule reconciliation.
+ *
+ * `.schedule()` is imperative: it creates and updates, and a schedule whose
+ * call was deleted from code keeps firing forever (deletion drift). This
+ * module declares the FULL schedule set for a service as a layer; on
+ * startup it upserts everything declared (idempotent, cadence-preserving)
+ * and detects group members that are no longer declared:
+ *
+ * ```ts
+ * const SchedulesLive = JobSchedules.layer({
+ *   group: "billing-service",
+ *   schedules: [
+ *     JobSchedules.schedule(SendDigest, "daily", { cron: "0 9 * * *", payload: {} }),
+ *     JobSchedules.schedule(GenerateInvoice, "monthly", { cron: "0 0 1 * *", payload: {} })
+ *   ],
+ *   removal: "group",           // default "warn": log drift, prune nothing
+ *   removeAfter: "10 minutes"   // optional grace window for rolling deploys
+ * })
+ * ```
+ *
+ * Safety model: pruning is scoped to the ownership `group`. Schedules
+ * created by plain `.schedule()` calls carry no group and are NEVER pruned;
+ * other groups' schedules are never touched. The default `removal: "warn"`
+ * only logs — destructive pruning is an explicit opt-in.
+ *
+ * @since 0.5.0
+ */
+import { type Context, Duration, Effect, Layer } from "effect"
+import type { ScheduleOptions } from "./Job.ts"
+import type { ScheduleKey, Service as StoreService } from "./JobStore.ts"
+
+/**
+ * The structural view of a `Job.make` class that `schedule` needs: its tag,
+ * its store key, and its bound `schedule` verb.
+ *
+ * @since 0.5.0
+ */
+export interface SchedulableJob<PayloadInput, R> {
+  readonly _tag: string
+  readonly store: Context.Key<any, StoreService>
+  readonly schedule: (
+    key: string,
+    options: ScheduleOptions<PayloadInput>
+  ) => Effect.Effect<ScheduleKey, never, R>
+}
+
+/**
+ * One declared schedule: a job, its key, and its cadence/options. Built
+ * with `JobSchedules.schedule`; consumed by `JobSchedules.layer`.
+ *
+ * @since 0.5.0
+ */
+export interface ScheduleEntry<R> {
+  readonly jobName: string
+  readonly key: string
+  readonly store: Context.Key<any, StoreService>
+  readonly register: (group: string) => Effect.Effect<ScheduleKey, never, R>
+}
+
+/**
+ * Declare one schedule for the reconciled set. Identical semantics to
+ * `MyJob.schedule(key, options)` — including cadence preservation on
+ * unchanged `cron`/`tz`/`every` — plus the layer's ownership `group`.
+ *
+ * @since 0.5.0
+ */
+export const schedule = <PayloadInput, R>(
+  job: SchedulableJob<PayloadInput, R>,
+  key: string,
+  options: Omit<ScheduleOptions<PayloadInput>, "group">
+): ScheduleEntry<R> => ({
+  jobName: job._tag,
+  key,
+  store: job.store,
+  register: (group) => job.schedule(key, { ...options, group })
+})
+
+/**
+ * Options for `JobSchedules.layer`.
+ *
+ * @since 0.5.0
+ */
+export interface ReconcileOptions<
+  Entries extends ReadonlyArray<ScheduleEntry<any>>,
+  Stores extends ReadonlyArray<Context.Key<any, StoreService>>
+> {
+  /**
+   * Ownership label. Everything this layer declares is upserted with this
+   * group, and only schedules carrying this group are candidates for drift
+   * detection and pruning. Use one group per service/deployable.
+   */
+  readonly group: string
+  /** The full declared schedule set for this group. */
+  readonly schedules: Entries
+  /**
+   * What to do with group members that are no longer declared:
+   * - `"warn"` (default): log them at warning level, prune nothing.
+   * - `"group"`: remove them (never touches unlabeled or other-group rows).
+   */
+  readonly removal?: "warn" | "group" | undefined
+  /**
+   * Grace window before pruning (requires `removal: "group"`). During a
+   * rolling deploy, replicas on the previous release re-declare schedules
+   * the new release dropped; pruning immediately and re-adding would
+   * re-anchor `every` grids and double-fire crons. With a window, the prune
+   * runs this long after startup, re-checks the store, and removes only
+   * members still undeclared. Shutting down before the window fires skips
+   * the prune (the next startup re-evaluates).
+   */
+  readonly removeAfter?: Duration.Input | undefined
+  /**
+   * Extra store keys to reconcile even when no declared entry references
+   * them — needed when a release drops the LAST schedule a store had, since
+   * drift detection only reaches stores it can see.
+   */
+  readonly stores?: Stores | undefined
+}
+
+const reconcile = (options: {
+  readonly group: string
+  readonly schedules: ReadonlyArray<ScheduleEntry<any>>
+  readonly removal?: "warn" | "group" | undefined
+  readonly removeAfter?: Duration.Input | undefined
+  readonly stores?: ReadonlyArray<Context.Key<any, StoreService>> | undefined
+}) =>
+  Effect.gen(function*() {
+    const removal = options.removal ?? "warn"
+    if (options.removeAfter !== undefined && removal !== "group") {
+      return yield* Effect.die(
+        new Error(`effect-mq: \`removeAfter\` requires \`removal: "group"\``)
+      )
+    }
+    // Duplicate declarations are a config bug: the second would silently
+    // overwrite the first's cadence/payload on every startup.
+    const seen = new Set<string>()
+    for (const entry of options.schedules) {
+      const id = `${entry.jobName}/${entry.key}`
+      if (seen.has(id)) {
+        return yield* Effect.die(
+          new Error(`effect-mq: schedule "${id}" is declared twice in group "${options.group}"`)
+        )
+      }
+      seen.add(id)
+    }
+
+    // Upsert every declared schedule, collecting the declared key set per
+    // store (Context.Key identity groups entries onto their stores).
+    const byStore = new Map<Context.Key<any, StoreService>, Set<string>>()
+    for (const storeKey of options.stores ?? []) {
+      byStore.set(storeKey, new Set())
+    }
+    for (const entry of options.schedules) {
+      const registered = yield* entry.register(options.group)
+      const declared = byStore.get(entry.store) ?? new Set<string>()
+      declared.add(registered)
+      byStore.set(entry.store, declared)
+    }
+
+    // Drift detection per store: group members not in the declared set.
+    for (const [storeKey, declared] of byStore) {
+      const store = yield* storeKey
+      const prune = Effect.gen(function*() {
+        const members = yield* store.listSchedules({ group: options.group }).pipe(
+          Effect.retry({ times: 5 }),
+          Effect.orDie
+        )
+        const undeclared = members.filter((member) => !declared.has(member.key))
+        if (undeclared.length === 0) return
+        if (removal === "warn") {
+          return yield* Effect.logWarning(
+            `effect-mq: ${undeclared.length} schedule(s) in group "${options.group}" ` +
+              `are no longer declared and keep firing; \`removal: "group"\` would prune them`,
+            undeclared.map((member) => member.key)
+          )
+        }
+        for (const member of undeclared) {
+          yield* store.removeSchedule(member.key).pipe(Effect.retry({ times: 5 }), Effect.orDie)
+        }
+        yield* Effect.logInfo(
+          `effect-mq: pruned ${undeclared.length} undeclared schedule(s) from group "${options.group}"`,
+          undeclared.map((member) => member.key)
+        )
+      })
+      if (removal === "group" && options.removeAfter !== undefined) {
+        // Deferred prune, tied to the layer scope: it re-lists at fire time,
+        // so anything re-declared during the window survives.
+        yield* prune.pipe(
+          Effect.delay(Duration.toMillis(options.removeAfter)),
+          Effect.catchCause((cause) =>
+            Effect.logError(
+              `effect-mq: deferred schedule prune failed (group "${options.group}")`,
+              cause
+            )
+          ),
+          Effect.forkScoped
+        )
+      } else {
+        yield* prune
+      }
+    }
+  })
+
+/**
+ * Reconcile the declared schedule set on startup: upsert everything
+ * declared, then warn about (or, with `removal: "group"`, prune) group
+ * members that are no longer declared. See the module docs for the safety
+ * model.
+ *
+ * @since 0.5.0
+ */
+export const layer = <
+  const Entries extends ReadonlyArray<ScheduleEntry<any>>,
+  const Stores extends ReadonlyArray<Context.Key<any, StoreService>> = readonly []
+>(
+  options: ReconcileOptions<Entries, Stores>
+): Layer.Layer<never, never, EntryServices<Entries[number]> | StoreId<Stores[number]>> =>
+  Layer.effectDiscard(reconcile(options))
+
+// Naked-parameter conditionals so empty tuples distribute to `never` instead
+// of inferring `unknown` from nothing.
+type EntryServices<E> = E extends ScheduleEntry<infer R> ? R : never
+type StoreId<K> = K extends Context.Key<infer Id, StoreService> ? Id : never