effect-mq 0.2.0 → 0.3.0

package/src/MemoryJobStore.ts

@@ -16,6 +16,8 @@ import {
   type ClaimResult,
   type EnqueueRequest,
   type ExtendLocksResult,
+  type HistoryTtlByState,
+  type HistoryTtlInput,
   type IdGenerator,
   JobId,
   JobNotCancellableError,
@@ -24,10 +26,13 @@ import {
   JobNotRetryableError,
   type JobRecord,
   type JobState,
+  type KeepStatePolicy,
   JobStore,
   type ListResult,
   LockLostError,
   JobStoreError,
+  normalizeHistoryTtl,
+  type TerminalState,
   type QueueName,
   type ScheduleKey,
   type ScheduleRecord,
@@ -38,17 +43,18 @@ interface MemJob {
   readonly id: JobId
   readonly name: string
   readonly queue: QueueName
-  readonly payload: unknown
-  readonly metadata: Readonly<Record<string, string>>
+  payload: unknown
+  metadata: Readonly<Record<string, string>>
   state: JobState
-  readonly priority: number
-  readonly attemptsMax: number
+  priority: number
+  attemptsMax: number
   attemptsMade: number
   stalledCount: number
-  readonly backoff: JobRecord["backoff"]
-  readonly keep: JobRecord["keep"]
-  readonly timeoutMs: number | undefined
+  backoff: JobRecord["backoff"]
+  keep: JobRecord["keep"]
+  timeoutMs: number | undefined
   cancelRequested: boolean
+  readonly dedupeKey: string | undefined
   runAt: number
   readonly enqueuedAt: number
   processedAt: number | undefined
@@ -78,6 +84,7 @@ const snapshot = (job: MemJob): JobRecord => ({
   keep: job.keep,
   timeoutMs: job.timeoutMs,
   cancelRequested: job.cancelRequested,
+  dedupeKey: job.dedupeKey,
   runAt: job.runAt,
   enqueuedAt: job.enqueuedAt,
   processedAt: job.processedAt,
@@ -98,26 +105,53 @@ const metadataMatches = (
 
 interface MemoryStore {
   readonly service: Service
-  readonly sweepHistory: (now: number, ttlMs: number) => void
+  readonly sweepHistory: (now: number, ttlByState: HistoryTtlByState) => void
 }
 
 const makeStoreUnsafe = (options?: MemoryJobStoreOptions | undefined): MemoryStore => {
   const jobs = new Map<string, MemJob>()
   const schedules = new Map<ScheduleKey, ScheduleRecord>()
   const paused = new Set<QueueName>()
+  // Dedup registry: one entry per (name, key). `expiresAt` is set for
+  // ttl/throttle windows; pending-mode entries live as long as their job.
+  const dedupes = new Map<string, { jobId: JobId; expiresAt: number | undefined }>()
+  const dedupeMapKey = (name: string, key: string) => `${name}\u0000${key}`
   let seq = 0
   let idCounter = 0
   let wakeVersion = 0
-  let wake = Deferred.makeUnsafe<void>()
+  let lastBroadcast = 0
+  const lastWake = new Map<QueueName, number>()
+  interface Waiter {
+    readonly queues: ReadonlySet<QueueName>
+    readonly deferred: Deferred.Deferred<void>
+  }
+  const waiters = new Set<Waiter>()
+  const lastWakeFor = (queue: QueueName) => Math.max(lastWake.get(queue) ?? 0, lastBroadcast)
 
   // Synchronous on purpose: it is called inside the same synchronous block as
   // the state mutation, so no effect-op boundary (where an interrupt could
-  // land) can separate a mutation from its wake-up signal.
-  const signalWake = () => {
+  // land) can separate a mutation from its wake-up signal. A queue targets
+  // only waiters watching it; no queue broadcasts (rare maintenance verbs).
+  const signalWake = (queue?: QueueName) => {
     wakeVersion += 1
-    const current = wake
-    wake = Deferred.makeUnsafe<void>()
-    Deferred.doneUnsafe(current, Exit.succeed<void>(void 0))
+    if (queue === undefined) {
+      lastBroadcast = wakeVersion
+    } else {
+      lastWake.set(queue, wakeVersion)
+    }
+    // Snapshot-and-clear BEFORE resolving: doneUnsafe resumes waiting fibers
+    // synchronously, and a woken taker that re-parks registers a NEW waiter —
+    // resolving inside the live Set iteration would visit it and livelock.
+    const toWake: Array<Waiter> = []
+    for (const waiter of waiters) {
+      if (queue === undefined || waiter.queues.has(queue)) {
+        waiters.delete(waiter)
+        toWake.push(waiter)
+      }
+    }
+    for (const waiter of toWake) {
+      Deferred.doneUnsafe(waiter.deferred, Exit.succeed<void>(void 0))
+    }
   }
 
   const promoteDue = (now: number) => {
@@ -148,32 +182,71 @@ const makeStoreUnsafe = (options?: MemoryJobStoreOptions | undefined): MemorySto
     })
   }
 
+  // A job leaving the pending states frees its pending-mode dedup entry;
+  // live throttle windows deliberately outlast the job.
+  const releaseDedupe = (job: MemJob, now: number) => {
+    if (job.dedupeKey === undefined) return
+    const key = dedupeMapKey(job.name, job.dedupeKey)
+    const entry = dedupes.get(key)
+    if (
+      entry !== undefined && entry.jobId === job.id &&
+      (entry.expiresAt === undefined || entry.expiresAt <= now)
+    ) {
+      dedupes.delete(key)
+    }
+  }
+
   const markCancelled = (job: MemJob, now: number) => {
     clearLock(job)
     job.cancelRequested = false
     job.state = "cancelled"
     job.finishedAt = now
     recordAttempt(job, "cancelled", now, undefined)
+    releaseDedupe(job, now)
     applyKeep(job, now)
   }
 
+  const keepPolicyFor = (keep: JobRecord["keep"], state: JobState) => {
+    if (keep === undefined) return undefined
+    const policy = state === "completed"
+      ? keep.completed
+      : state === "failed"
+      ? keep.failed
+      : state === "cancelled"
+      ? keep.cancelled
+      : undefined
+    if (policy !== undefined) return policy
+    // Records persisted by 0.2.x carry the flat {count, ageMs} shape — honour
+    // it as an all-states policy so upgrades keep pruning.
+    if (
+      keep.completed === undefined && keep.failed === undefined && keep.cancelled === undefined &&
+      ("count" in keep || "ageMs" in keep)
+    ) {
+      // SAFETY: the flat legacy shape carries KeepStatePolicy fields.
+      return keep as KeepStatePolicy
+    }
+    return undefined
+  }
+
   // Terminal retention: keep at most `count` and drop older than `ageMs`
-  // among terminal jobs sharing this job's name + state.
+  // among terminal jobs sharing this job's name + state (policies are split
+  // per terminal state).
   const applyKeep = (job: MemJob, now: number) => {
-    if (job.keep === undefined) return
+    const policy = keepPolicyFor(job.keep, job.state)
+    if (policy === undefined) return
     const peers = Array.from(jobs.values())
       .filter((peer) => peer.name === job.name && peer.state === job.state)
       .toSorted((a, b) => ((b.finishedAt ?? 0) - (a.finishedAt ?? 0)) || (b.seq - a.seq))
     const remove = new Set<string>()
-    if (job.keep.ageMs !== undefined) {
+    if (policy.ageMs !== undefined) {
       for (const peer of peers) {
-        if (peer.finishedAt !== undefined && peer.finishedAt <= now - job.keep.ageMs) {
+        if (peer.finishedAt !== undefined && peer.finishedAt <= now - policy.ageMs) {
           remove.add(peer.id)
         }
       }
     }
-    if (job.keep.count !== undefined) {
-      for (const peer of peers.slice(Math.max(0, job.keep.count))) {
+    if (policy.count !== undefined) {
+      for (const peer of peers.slice(Math.max(0, policy.count))) {
         remove.add(peer.id)
       }
     }
@@ -182,22 +255,75 @@ const makeStoreUnsafe = (options?: MemoryJobStoreOptions | undefined): MemorySto
     }
   }
 
-  const sweepHistory = (now: number, ttlMs: number) => {
+  const sweepHistory = (now: number, ttlByState: HistoryTtlByState) => {
     for (const job of jobs.values()) {
-      if (
-        TERMINAL_STATES.has(job.state) &&
-        job.finishedAt !== undefined &&
-        job.finishedAt <= now - ttlMs
-      ) {
+      if (!TERMINAL_STATES.has(job.state) || job.finishedAt === undefined) continue
+      // SAFETY: TERMINAL_STATES membership was checked above.
+      const state = job.state as TerminalState
+      const ttl = ttlByState[state]
+      const keepAge = keepPolicyFor(job.keep, state)?.ageMs
+      // The sweep honours min(per-row keep age, store ceiling) — a quiet job
+      // name is pruned on the timer, not only when its group is acked.
+      const effective = keepAge !== undefined && (ttl === undefined || keepAge < ttl) ? keepAge : ttl
+      if (effective !== undefined && job.finishedAt <= now - effective) {
         jobs.delete(job.id)
       }
     }
+    // Dead dedup entries: expired window, or a pointer at a vanished job.
+    for (const [key, entry] of dedupes) {
+      const alive = entry.expiresAt !== undefined
+        ? entry.expiresAt > now
+        : jobs.has(entry.jobId) && !TERMINAL_STATES.has(jobs.get(entry.jobId)?.state ?? "completed")
+      if (!alive) dedupes.delete(key)
+    }
   }
 
   const service: Service = JobStore.of({
     enqueue: (request: EnqueueRequest) =>
       Effect.gen(function*() {
         const now = yield* Clock.currentTimeMillis
+        if (request.id !== undefined && jobs.has(request.id)) {
+          return { id: request.id, duplicate: true }
+        }
+        // The dedup decision tree runs BEFORE id generation, so a
+        // deduplicated enqueue never consults the id generator.
+        if (request.dedupe !== undefined) {
+          const mapKey = dedupeMapKey(request.name, request.dedupe.key)
+          const entry = dedupes.get(mapKey)
+          if (entry !== undefined) {
+            const keyed = jobs.get(entry.jobId)
+            const windowLive = entry.expiresAt !== undefined && now < entry.expiresAt
+            // Latest-wins while the keyed job is still delayed.
+            if (request.dedupe.replace && keyed !== undefined && keyed.state === "delayed") {
+              keyed.payload = request.payload
+              keyed.metadata = request.metadata
+              keyed.priority = request.priority
+              keyed.attemptsMax = request.attemptsMax
+              keyed.backoff = request.backoff
+              keyed.keep = request.keep
+              keyed.timeoutMs = request.timeoutMs
+              keyed.runAt = now + Math.max(0, request.delayMs)
+              // A landed replace re-arms the ttl window.
+              if (request.dedupe.ttlMs !== undefined) {
+                entry.expiresAt = now + request.dedupe.ttlMs
+              }
+              signalWake(keyed.queue)
+              return { id: keyed.id, duplicate: true }
+            }
+            if (windowLive) {
+              if (request.dedupe.extend && request.dedupe.ttlMs !== undefined) {
+                entry.expiresAt = now + request.dedupe.ttlMs
+              }
+              return { id: entry.jobId, duplicate: true }
+            }
+            const pending = keyed !== undefined && !TERMINAL_STATES.has(keyed.state)
+            if (entry.expiresAt === undefined && pending) {
+              return { id: entry.jobId, duplicate: true }
+            }
+            // Dead entry (expired window / finished job): fall through and
+            // let the new job take over the key below.
+          }
+        }
         let id = request.id
         if (id === undefined) {
           const generate = options?.idGenerator
@@ -222,8 +348,6 @@ const makeStoreUnsafe = (options?: MemoryJobStoreOptions | undefined): MemorySto
               })
             }
           }
-        } else if (jobs.has(id)) {
-          return { id, duplicate: true }
         }
         jobs.set(id, {
           id,
@@ -240,6 +364,7 @@ const makeStoreUnsafe = (options?: MemoryJobStoreOptions | undefined): MemorySto
           keep: request.keep,
           timeoutMs: request.timeoutMs,
           cancelRequested: false,
+          dedupeKey: request.dedupe?.key,
           runAt: now + Math.max(0, request.delayMs),
           enqueuedAt: now,
           processedAt: undefined,
@@ -251,7 +376,13 @@ const makeStoreUnsafe = (options?: MemoryJobStoreOptions | undefined): MemorySto
           lockToken: undefined,
           lockExpiresAt: undefined
         })
-        signalWake()
+        if (request.dedupe !== undefined) {
+          dedupes.set(dedupeMapKey(request.name, request.dedupe.key), {
+            jobId: id,
+            expiresAt: request.dedupe.ttlMs !== undefined ? now + request.dedupe.ttlMs : undefined
+          })
+        }
+        signalWake(request.queue)
         return { id, duplicate: false }
       }),
 
@@ -309,6 +440,7 @@ const makeStoreUnsafe = (options?: MemoryJobStoreOptions | undefined): MemorySto
             job.exit = outcome.exit
             job.finishedAt = now
             recordAttempt(job, "completed", now, outcome.exit)
+            releaseDedupe(job, now)
             applyKeep(job, now)
             break
           }
@@ -318,6 +450,7 @@ const makeStoreUnsafe = (options?: MemoryJobStoreOptions | undefined): MemorySto
             job.exit = outcome.exit
             job.finishedAt = now
             recordAttempt(job, "failed", now, outcome.exit)
+            releaseDedupe(job, now)
             applyKeep(job, now)
             break
           }
@@ -326,6 +459,7 @@ const makeStoreUnsafe = (options?: MemoryJobStoreOptions | undefined): MemorySto
             job.state = "cancelled"
             job.finishedAt = now
             recordAttempt(job, "cancelled", now, undefined)
+            releaseDedupe(job, now)
             applyKeep(job, now)
             break
           }
@@ -341,7 +475,7 @@ const makeStoreUnsafe = (options?: MemoryJobStoreOptions | undefined): MemorySto
             job.runAt = now + Math.max(0, outcome.delayMs)
             job.state = outcome.delayMs > 0 ? "delayed" : "waiting"
             job.seq = ++seq
-            signalWake()
+            signalWake(job.queue)
             break
           }
         }
@@ -365,7 +499,7 @@ const makeStoreUnsafe = (options?: MemoryJobStoreOptions | undefined): MemorySto
         }
         clearLock(job)
         job.state = "waiting"
-        signalWake()
+        signalWake(job.queue)
       }),
 
     extendLocks: (locks, durationMs) =>
@@ -415,6 +549,7 @@ const makeStoreUnsafe = (options?: MemoryJobStoreOptions | undefined): MemorySto
             job.state = "failed"
             job.finishedAt = now
             job.failedReason = "job stalled more than allowable limit"
+            releaseDedupe(job, now)
             recovered.push({ id: job.id, failed: true })
           } else {
             job.state = "waiting"
@@ -427,10 +562,14 @@ const makeStoreUnsafe = (options?: MemoryJobStoreOptions | undefined): MemorySto
         return recovered
       }),
 
-    awaitWake: (_queues, wakeToken) =>
+    awaitWake: (queues, wakeToken) =>
       Effect.suspend(() => {
-        if (wakeVersion > wakeToken) return Effect.void
-        return Deferred.await(wake)
+        if (queues.some((queue) => lastWakeFor(queue) > wakeToken)) return Effect.void
+        const waiter: Waiter = { queues: new Set(queues), deferred: Deferred.makeUnsafe<void>() }
+        waiters.add(waiter)
+        return Deferred.await(waiter.deferred).pipe(
+          Effect.ensuring(Effect.sync(() => waiters.delete(waiter)))
+        )
       }),
 
     getJob: (id) =>
@@ -510,7 +649,7 @@ const makeStoreUnsafe = (options?: MemoryJobStoreOptions | undefined): MemorySto
         job.processedAt = undefined
         job.runAt = now
         job.seq = ++seq
-        signalWake()
+        signalWake(job.queue)
       }),
 
     cancel: (id) =>
@@ -548,7 +687,7 @@ const makeStoreUnsafe = (options?: MemoryJobStoreOptions | undefined): MemorySto
         }
         job.state = "waiting"
         job.runAt = now
-        signalWake()
+        signalWake(job.queue)
       }),
 
     pause: (queue) =>
@@ -559,7 +698,7 @@ const makeStoreUnsafe = (options?: MemoryJobStoreOptions | undefined): MemorySto
     resume: (queue) =>
       Effect.sync(() => {
         if (paused.delete(queue)) {
-          signalWake()
+          signalWake(queue)
         }
       }),
 
@@ -579,7 +718,7 @@ const makeStoreUnsafe = (options?: MemoryJobStoreOptions | undefined): MemorySto
           schedule.key,
           sameCadence ? { ...schedule, nextRunAt: existing.nextRunAt } : schedule
         )
-        signalWake()
+        signalWake(schedule.queue)
       }),
 
     removeSchedule: (key) => Effect.sync(() => schedules.delete(key)),
@@ -642,11 +781,12 @@ const makeStoreUnsafe = (options?: MemoryJobStoreOptions | undefined): MemorySto
  */
 export interface MemoryJobStoreOptions {
   /**
-   * Store-level retention ceiling: terminal records (completed, failed,
-   * cancelled) older than this are removed by a periodic sweep. Per-job
-   * `keep` policies may only be stricter.
+   * Store-level retention ceiling: terminal records older than this are
+   * removed by a periodic sweep — one duration for all terminal states or a
+   * per-state split (`{ completed: "1 day", failed: "30 days" }`). The sweep
+   * also honours stricter per-job `keep.age` rules.
    */
-  readonly historyTtl?: Duration.Input | undefined
+  readonly historyTtl?: HistoryTtlInput | undefined
   /** Sweep cadence (default 1 minute). */
   readonly historySweepInterval?: Duration.Input | undefined
   /**
@@ -676,12 +816,12 @@ export const makeWith = (
   Effect.gen(function*() {
     const { service, sweepHistory } = makeStoreUnsafe(options)
     if (options?.historyTtl !== undefined) {
-      const ttlMs = Duration.toMillis(options.historyTtl)
+      const ttlByState = normalizeHistoryTtl(options.historyTtl)
       const intervalMs = Duration.toMillis(options.historySweepInterval ?? "1 minute")
       yield* Effect.gen(function*() {
         yield* Effect.sleep(intervalMs)
         const now = yield* Clock.currentTimeMillis
-        sweepHistory(now, ttlMs)
+        sweepHistory(now, ttlByState)
       }).pipe(Effect.forever, Effect.forkScoped)
     }
     return service

package/src/Worker.ts

@@ -534,6 +534,7 @@ export const make = <StoreId = JobStore>(
           backoff: schedule.backoff,
           keep: schedule.keep,
           timeoutMs: schedule.timeoutMs,
+          dedupe: undefined,
           delayMs: 0
         }))
         yield* retryStore(store.advanceSchedule(schedule.key, slot, next))