effect-mq 0.4.2 → 0.5.0

package/src/JobSchedules.ts

@@ -0,0 +1,223 @@
+/**
+ * Declarative schedule reconciliation.
+ *
+ * `.schedule()` is imperative: it creates and updates, and a schedule whose
+ * call was deleted from code keeps firing forever (deletion drift). This
+ * module declares the FULL schedule set for a service as a layer; on
+ * startup it upserts everything declared (idempotent, cadence-preserving)
+ * and detects group members that are no longer declared:
+ *
+ * ```ts
+ * const SchedulesLive = JobSchedules.layer({
+ *   group: "billing-service",
+ *   schedules: [
+ *     JobSchedules.schedule(SendDigest, "daily", { cron: "0 9 * * *", payload: {} }),
+ *     JobSchedules.schedule(GenerateInvoice, "monthly", { cron: "0 0 1 * *", payload: {} })
+ *   ],
+ *   removal: "group",           // default "warn": log drift, prune nothing
+ *   removeAfter: "10 minutes"   // optional grace window for rolling deploys
+ * })
+ * ```
+ *
+ * Safety model: pruning is scoped to the ownership `group`. Schedules
+ * created by plain `.schedule()` calls carry no group and are NEVER pruned;
+ * other groups' schedules are never touched. The default `removal: "warn"`
+ * only logs — destructive pruning is an explicit opt-in.
+ *
+ * @since 0.5.0
+ */
+import { type Context, Duration, Effect, Layer } from "effect"
+import type { ScheduleOptions } from "./Job.ts"
+import type { ScheduleKey, Service as StoreService } from "./JobStore.ts"
+
+/**
+ * The structural view of a `Job.make` class that `schedule` needs: its tag,
+ * its store key, and its bound `schedule` verb.
+ *
+ * @since 0.5.0
+ */
+export interface SchedulableJob<PayloadInput, R> {
+  readonly _tag: string
+  readonly store: Context.Key<any, StoreService>
+  readonly schedule: (
+    key: string,
+    options: ScheduleOptions<PayloadInput>
+  ) => Effect.Effect<ScheduleKey, never, R>
+}
+
+/**
+ * One declared schedule: a job, its key, and its cadence/options. Built
+ * with `JobSchedules.schedule`; consumed by `JobSchedules.layer`.
+ *
+ * @since 0.5.0
+ */
+export interface ScheduleEntry<R> {
+  readonly jobName: string
+  readonly key: string
+  readonly store: Context.Key<any, StoreService>
+  readonly register: (group: string) => Effect.Effect<ScheduleKey, never, R>
+}
+
+/**
+ * Declare one schedule for the reconciled set. Identical semantics to
+ * `MyJob.schedule(key, options)` — including cadence preservation on
+ * unchanged `cron`/`tz`/`every` — plus the layer's ownership `group`.
+ *
+ * @since 0.5.0
+ */
+export const schedule = <PayloadInput, R>(
+  job: SchedulableJob<PayloadInput, R>,
+  key: string,
+  options: Omit<ScheduleOptions<PayloadInput>, "group">
+): ScheduleEntry<R> => ({
+  jobName: job._tag,
+  key,
+  store: job.store,
+  register: (group) => job.schedule(key, { ...options, group })
+})
+
+/**
+ * Options for `JobSchedules.layer`.
+ *
+ * @since 0.5.0
+ */
+export interface ReconcileOptions<
+  Entries extends ReadonlyArray<ScheduleEntry<any>>,
+  Stores extends ReadonlyArray<Context.Key<any, StoreService>>
+> {
+  /**
+   * Ownership label. Everything this layer declares is upserted with this
+   * group, and only schedules carrying this group are candidates for drift
+   * detection and pruning. Use one group per service/deployable.
+   */
+  readonly group: string
+  /** The full declared schedule set for this group. */
+  readonly schedules: Entries
+  /**
+   * What to do with group members that are no longer declared:
+   * - `"warn"` (default): log them at warning level, prune nothing.
+   * - `"group"`: remove them (never touches unlabeled or other-group rows).
+   */
+  readonly removal?: "warn" | "group" | undefined
+  /**
+   * Grace window before pruning (requires `removal: "group"`). During a
+   * rolling deploy, replicas on the previous release re-declare schedules
+   * the new release dropped; pruning immediately and re-adding would
+   * re-anchor `every` grids and double-fire crons. With a window, the prune
+   * runs this long after startup, re-checks the store, and removes only
+   * members still undeclared. Shutting down before the window fires skips
+   * the prune (the next startup re-evaluates).
+   */
+  readonly removeAfter?: Duration.Input | undefined
+  /**
+   * Extra store keys to reconcile even when no declared entry references
+   * them — needed when a release drops the LAST schedule a store had, since
+   * drift detection only reaches stores it can see.
+   */
+  readonly stores?: Stores | undefined
+}
+
+const reconcile = (options: {
+  readonly group: string
+  readonly schedules: ReadonlyArray<ScheduleEntry<any>>
+  readonly removal?: "warn" | "group" | undefined
+  readonly removeAfter?: Duration.Input | undefined
+  readonly stores?: ReadonlyArray<Context.Key<any, StoreService>> | undefined
+}) =>
+  Effect.gen(function*() {
+    const removal = options.removal ?? "warn"
+    if (options.removeAfter !== undefined && removal !== "group") {
+      return yield* Effect.die(
+        new Error(`effect-mq: \`removeAfter\` requires \`removal: "group"\``)
+      )
+    }
+    // Duplicate declarations are a config bug: the second would silently
+    // overwrite the first's cadence/payload on every startup.
+    const seen = new Set<string>()
+    for (const entry of options.schedules) {
+      const id = `${entry.jobName}/${entry.key}`
+      if (seen.has(id)) {
+        return yield* Effect.die(
+          new Error(`effect-mq: schedule "${id}" is declared twice in group "${options.group}"`)
+        )
+      }
+      seen.add(id)
+    }
+
+    // Upsert every declared schedule, collecting the declared key set per
+    // store (Context.Key identity groups entries onto their stores).
+    const byStore = new Map<Context.Key<any, StoreService>, Set<string>>()
+    for (const storeKey of options.stores ?? []) {
+      byStore.set(storeKey, new Set())
+    }
+    for (const entry of options.schedules) {
+      const registered = yield* entry.register(options.group)
+      const declared = byStore.get(entry.store) ?? new Set<string>()
+      declared.add(registered)
+      byStore.set(entry.store, declared)
+    }
+
+    // Drift detection per store: group members not in the declared set.
+    for (const [storeKey, declared] of byStore) {
+      const store = yield* storeKey
+      const prune = Effect.gen(function*() {
+        const members = yield* store.listSchedules({ group: options.group }).pipe(
+          Effect.retry({ times: 5 }),
+          Effect.orDie
+        )
+        const undeclared = members.filter((member) => !declared.has(member.key))
+        if (undeclared.length === 0) return
+        if (removal === "warn") {
+          return yield* Effect.logWarning(
+            `effect-mq: ${undeclared.length} schedule(s) in group "${options.group}" ` +
+              `are no longer declared and keep firing; \`removal: "group"\` would prune them`,
+            undeclared.map((member) => member.key)
+          )
+        }
+        for (const member of undeclared) {
+          yield* store.removeSchedule(member.key).pipe(Effect.retry({ times: 5 }), Effect.orDie)
+        }
+        yield* Effect.logInfo(
+          `effect-mq: pruned ${undeclared.length} undeclared schedule(s) from group "${options.group}"`,
+          undeclared.map((member) => member.key)
+        )
+      })
+      if (removal === "group" && options.removeAfter !== undefined) {
+        // Deferred prune, tied to the layer scope: it re-lists at fire time,
+        // so anything re-declared during the window survives.
+        yield* prune.pipe(
+          Effect.delay(Duration.toMillis(options.removeAfter)),
+          Effect.catchCause((cause) =>
+            Effect.logError(
+              `effect-mq: deferred schedule prune failed (group "${options.group}")`,
+              cause
+            )
+          ),
+          Effect.forkScoped
+        )
+      } else {
+        yield* prune
+      }
+    }
+  })
+
+/**
+ * Reconcile the declared schedule set on startup: upsert everything
+ * declared, then warn about (or, with `removal: "group"`, prune) group
+ * members that are no longer declared. See the module docs for the safety
+ * model.
+ *
+ * @since 0.5.0
+ */
+export const layer = <
+  const Entries extends ReadonlyArray<ScheduleEntry<any>>,
+  const Stores extends ReadonlyArray<Context.Key<any, StoreService>> = readonly []
+>(
+  options: ReconcileOptions<Entries, Stores>
+): Layer.Layer<never, never, EntryServices<Entries[number]> | StoreId<Stores[number]>> =>
+  Layer.effectDiscard(reconcile(options))
+
+// Naked-parameter conditionals so empty tuples distribute to `never` instead
+// of inferring `unknown` from nothing.
+type EntryServices<E> = E extends ScheduleEntry<infer R> ? R : never
+type StoreId<K> = K extends Context.Key<infer Id, StoreService> ? Id : never

package/src/JobStore.ts

@@ -450,6 +450,12 @@ export interface ScheduleRecord {
   readonly backoff: BackoffPolicy | undefined
   readonly keep: KeepPolicy | undefined
   readonly timeoutMs: number | undefined
+  /**
+   * Ownership label for declarative reconciliation (`JobSchedules.layer`):
+   * a reconciler only ever prunes schedules carrying ITS group. Unlabeled
+   * schedules (plain `.schedule()` calls) are never pruned.
+   */
+  readonly group: string | undefined
   /** Epoch millis of the next occurrence to enqueue. */
   readonly nextRunAt: number
 }
@@ -796,6 +802,8 @@ export interface Service {
   readonly listSchedules: (options?: {
     readonly jobName?: string | undefined
     readonly queue?: QueueName | undefined
+    /** Only schedules carrying this ownership group. */
+    readonly group?: string | undefined
   }) => Effect.Effect<ReadonlyArray<ScheduleRecord>, JobStoreError>
 
   /** Schedules whose `nextRunAt` is due (per the Effect `Clock`). */

package/src/MemoryJobStore.ts

@@ -753,7 +753,8 @@ const makeStoreUnsafe = (options?: MemoryJobStoreOptions | undefined): MemorySto
       Effect.sync(() =>
         Array.from(schedules.values()).filter((schedule) =>
           (options?.jobName === undefined || schedule.jobName === options.jobName) &&
-          (options?.queue === undefined || schedule.queue === options.queue)
+          (options?.queue === undefined || schedule.queue === options.queue) &&
+          (options?.group === undefined || schedule.group === options.group)
         )
       ),
 

package/src/drizzle-postgres/DrizzleJobStore.ts

@@ -136,6 +136,7 @@ type ScheduleRow = {
   readonly backoff: JobStore.BackoffPolicy | null
   readonly keep: JobStore.KeepPolicy | null
   readonly timeoutMs: number | string | null
+  readonly group: string | null
   readonly nextRunAt: Date
 }
 
@@ -153,6 +154,7 @@ const toSchedule = (row: ScheduleRow): JobStore.ScheduleRecord => ({
   backoff: row.backoff ?? undefined,
   keep: row.keep ?? undefined,
   timeoutMs: row.timeoutMs === null ? undefined : Number(row.timeoutMs),
+  group: row.group ?? undefined,
   nextRunAt: row.nextRunAt.getTime()
 })
 
@@ -1340,20 +1342,21 @@ export const make = (
       upsertSchedule: (schedule) =>
         db.execute(sql`
           INSERT INTO ${schedules} (key, job_name, queue, cron, tz, every_ms, payload, metadata,
-            priority, attempts_max, backoff, keep, timeout_ms, next_run_at)
+            priority, attempts_max, backoff, keep, timeout_ms, group_name, next_run_at)
           VALUES (${schedule.key}, ${schedule.jobName}, ${schedule.queue},
             ${schedule.cron ?? null}, ${schedule.tz ?? null}, ${schedule.everyMs ?? null},
             ${JSON.stringify(schedule.payload ?? null)}::jsonb, ${JSON.stringify(schedule.metadata)}::jsonb,
             ${schedule.priority}, ${schedule.attemptsMax},
             ${schedule.backoff === undefined ? null : JSON.stringify(schedule.backoff)}::jsonb,
             ${schedule.keep === undefined ? null : JSON.stringify(schedule.keep)}::jsonb,
-            ${schedule.timeoutMs ?? null}, ${new Date(schedule.nextRunAt)})
+            ${schedule.timeoutMs ?? null}, ${schedule.group ?? null}, ${new Date(schedule.nextRunAt)})
           ON CONFLICT (key) DO UPDATE SET
             job_name = EXCLUDED.job_name, queue = EXCLUDED.queue, cron = EXCLUDED.cron,
             tz = EXCLUDED.tz, every_ms = EXCLUDED.every_ms, payload = EXCLUDED.payload,
             metadata = EXCLUDED.metadata, priority = EXCLUDED.priority,
             attempts_max = EXCLUDED.attempts_max, backoff = EXCLUDED.backoff,
             keep = EXCLUDED.keep, timeout_ms = EXCLUDED.timeout_ms,
+            group_name = EXCLUDED.group_name,
             next_run_at = CASE
               WHEN ${schedules.cron} IS NOT DISTINCT FROM EXCLUDED.cron
                 AND ${schedules.tz} IS NOT DISTINCT FROM EXCLUDED.tz
@@ -1383,6 +1386,9 @@ export const make = (
           if (listOptions?.queue !== undefined) {
             conditions.push(sql`${schedules.queue} = ${listOptions.queue}`)
           }
+          if (listOptions?.group !== undefined) {
+            conditions.push(sql`${schedules.group} = ${listOptions.group}`)
+          }
           const rows = rowsOf(yield* db.execute<ScheduleRow>(sql`
             SELECT ${schedules.key} AS "key", ${schedules.jobName} AS "jobName",
               ${schedules.queue} AS "queue", ${schedules.cron} AS "cron", ${schedules.tz} AS "tz",
@@ -1390,7 +1396,7 @@ export const make = (
               ${schedules.metadata} AS "metadata", ${schedules.priority} AS "priority",
               ${schedules.attemptsMax} AS "attemptsMax", ${schedules.backoff} AS "backoff",
               ${schedules.keep} AS "keep", ${schedules.timeoutMs} AS "timeoutMs",
-              ${schedules.nextRunAt} AS "nextRunAt"
+              ${schedules.group} AS "group", ${schedules.nextRunAt} AS "nextRunAt"
             FROM ${schedules}
             WHERE ${sql.join(conditions, sql` AND `)}
             ORDER BY ${schedules.key}
@@ -1408,7 +1414,7 @@ export const make = (
               ${schedules.metadata} AS "metadata", ${schedules.priority} AS "priority",
               ${schedules.attemptsMax} AS "attemptsMax", ${schedules.backoff} AS "backoff",
               ${schedules.keep} AS "keep", ${schedules.timeoutMs} AS "timeoutMs",
-              ${schedules.nextRunAt} AS "nextRunAt"
+              ${schedules.group} AS "group", ${schedules.nextRunAt} AS "nextRunAt"
             FROM ${schedules}
             WHERE ${schedules.nextRunAt} <= ${now}
             ORDER BY ${schedules.nextRunAt} ASC

package/src/drizzle-postgres/schema.ts

@@ -201,6 +201,8 @@ const schedulesColumns = <JobName extends string, Queue extends string>() => ({
   backoff: jsonb("backoff").$type<BackoffPolicy>(),
   keep: jsonb("keep").$type<KeepPolicy>(),
   timeoutMs: bigint("timeout_ms", { mode: "number" }),
+  /** Ownership label for declarative reconciliation; NULL = never pruned. */
+  group: text("group_name"),
   nextRunAt: timestamp("next_run_at", { withTimezone: true, mode: "date" }).notNull()
 })
 

package/src/index.ts

@@ -11,6 +11,14 @@
  */
 export * as Job from "./Job.ts"
 
+/**
+ * Declarative schedule reconciliation: declare a service's full schedule
+ * set as a layer; startup upserts it and detects deletion drift.
+ *
+ * @since 0.5.0
+ */
+export * as JobSchedules from "./JobSchedules.ts"
+
 /**
  * The storage seam: the `JobStore` service, job records and typed errors.
  *

package/src/redis/RedisJobStore.ts

@@ -107,6 +107,7 @@ const toSchedule = (hash: ReadonlyMap<string, string>): JobStore.ScheduleRecord
   backoff: optionalJson<JobStore.BackoffPolicy>(hash.get("backoff")),
   keep: optionalJson<JobStore.KeepPolicy>(hash.get("keep")),
   timeoutMs: optionalNumber(hash.get("timeoutMs")),
+  group: optionalString(hash.get("group")),
   nextRunAt: Number(hash.get("nextRunAt") ?? 0)
 })
 
@@ -746,6 +747,7 @@ export const make = (
           schedule.backoff === undefined ? "" : JSON.stringify(schedule.backoff),
           schedule.keep === undefined ? "" : JSON.stringify(schedule.keep),
           schedule.timeoutMs === undefined ? "" : String(schedule.timeoutMs),
+          schedule.group ?? "",
           schedule.nextRunAt
         ).pipe(
           Effect.mapError(storeError("upsertSchedule failed")),

package/src/redis/scripts.ts

@@ -833,6 +833,7 @@ export const upsertSchedule = Redis.script(
     backoffJson: string,
     keepJson: string,
     timeoutMs: string,
+    group: string,
     nextRunAt: number
   ) => [
     prefix,
@@ -849,6 +850,7 @@ export const upsertSchedule = Redis.script(
     backoffJson,
     keepJson,
     timeoutMs,
+    group,
     nextRunAt
   ],
   {
@@ -856,7 +858,7 @@ export const upsertSchedule = Redis.script(
     lua: `${HELPERS}
 local key = ARGV[2]
 local sk = prefix .. ":schedule:" .. key
-local nextRunAt = tonumber(ARGV[15])
+local nextRunAt = tonumber(ARGV[16])
 local prevCron = redis.call("HGET", sk, "cron")
 if prevCron ~= false
   and prevCron == ARGV[5]
@@ -872,7 +874,7 @@ redis.call("HSET", sk,
   "payload", ARGV[8], "metadata", ARGV[9],
   "priority", ARGV[10], "attemptsMax", ARGV[11],
   "backoff", ARGV[12], "keep", ARGV[13], "timeoutMs", ARGV[14],
-  "nextRunAt", fmt(nextRunAt))
+  "group", ARGV[15], "nextRunAt", fmt(nextRunAt))
 redis.call("ZADD", prefix .. ":schedules", nextRunAt, key)
 return '{"ok":true}'
 `
@@ -908,6 +910,7 @@ for _, key in ipairs(keys) do
   local matches = true
   if filters.jobName ~= nil and byName.jobName ~= filters.jobName then matches = false end
   if matches and filters.queue ~= nil and byName.queue ~= filters.queue then matches = false end
+  if matches and filters.group ~= nil and byName.group ~= filters.group then matches = false end
   if matches then out[#out + 1] = record end
 end
 if #out == 0 then return "[]" end

package/src/testing/conformance.ts

@@ -805,6 +805,7 @@ export const jobStoreConformance = (
             backoff: { _tag: "fixed", delayMs: 1_000 },
             keep: undefined,
             timeoutMs: 5_000,
+            group: undefined,
             nextRunAt: 60_000
           }
           yield* store.upsertSchedule(schedule)
@@ -847,9 +848,49 @@ export const jobStoreConformance = (
       backoff: undefined,
       keep: undefined,
       timeoutMs: undefined,
+      group: undefined,
       nextRunAt: 60_000
     })
 
+    it.effect("schedule group labels persist and filter listSchedules", () =>
+      withStore((store) =>
+        Effect.gen(function*() {
+          const base = minutelySchedule()
+          yield* store.upsertSchedule({
+            ...base,
+            key: JobStore.ScheduleKey("TestJob/labeled"),
+            group: "svc-a"
+          })
+          yield* store.upsertSchedule({
+            ...base,
+            key: JobStore.ScheduleKey("TestJob/other"),
+            group: "svc-b"
+          })
+          yield* store.upsertSchedule({ ...base, key: JobStore.ScheduleKey("TestJob/plain") })
+
+          const labeled = yield* store.listSchedules({ group: "svc-a" })
+          expect(labeled.map((schedule) => schedule.key)).toEqual(["TestJob/labeled"])
+          expect(labeled[0]?.group).toBe("svc-a")
+
+          const all = yield* store.listSchedules()
+          expect(all).toHaveLength(3)
+          expect(all.find((schedule) => schedule.key === "TestJob/plain")?.group).toBeUndefined()
+
+          // Re-upsert relabels; an unchanged cadence still keeps its next
+          // occurrence (the reconciler re-registers on every startup).
+          yield* store.upsertSchedule({
+            ...base,
+            key: JobStore.ScheduleKey("TestJob/labeled"),
+            group: "svc-c",
+            nextRunAt: 999_999
+          })
+          const relabeled = yield* store.listSchedules({ group: "svc-c" })
+          expect(relabeled.map((schedule) => schedule.key)).toEqual(["TestJob/labeled"])
+          expect(relabeled[0]?.nextRunAt).toBe(60_000)
+          expect(yield* store.listSchedules({ group: "svc-a" })).toEqual([])
+        })
+      ))
+
     it.effect("tickSchedule fires a slot exactly once and advances atomically", () =>
       withStore((store) =>
         Effect.gen(function*() {
@@ -1226,6 +1267,7 @@ export const jobStoreConformance = (
             backoff: undefined,
             keep: undefined,
             timeoutMs: undefined,
+            group: undefined,
             nextRunAt: 60_000
           }
           yield* store.upsertSchedule(base)
@@ -1261,6 +1303,7 @@ export const jobStoreConformance = (
             backoff: undefined,
             keep: undefined,
             timeoutMs: undefined,
+            group: undefined,
             nextRunAt: 90_000
           }
           const cronSchedule: JobStore.ScheduleRecord = {
@@ -1352,6 +1395,7 @@ export const jobStoreConformance = (
             backoff: undefined,
             keep: undefined,
             timeoutMs: undefined,
+            group: undefined,
             nextRunAt: 60_000
           }
           yield* store.upsertSchedule(schedule)