effect-mq 0.1.0 → 0.3.0

package/dist/MemoryJobStore.js

@@ -10,8 +10,9 @@
  *
  * @since 0.1.0
  */
-import { Clock, Deferred, Effect, Exit, Layer, Option } from "effect";
-import { JobId, JobNotFoundError, JobNotRetryableError, JobStore, LockLostError } from "./JobStore.js";
+import { Clock, Deferred, Duration, Effect, Exit, Layer, Option } from "effect";
+import { JobId, JobNotCancellableError, JobNotFoundError, JobNotPromotableError, JobNotRetryableError, JobStore, LockLostError, JobStoreError, normalizeHistoryTtl } from "./JobStore.js";
+const TERMINAL_STATES = new Set(["completed", "failed", "cancelled"]);
 const snapshot = (job) => ({
     id: job.id,
     name: job.name,
@@ -25,6 +26,9 @@ const snapshot = (job) => ({
     stalledCount: job.stalledCount,
     backoff: job.backoff,
     keep: job.keep,
+    timeoutMs: job.timeoutMs,
+    cancelRequested: job.cancelRequested,
+    dedupeKey: job.dedupeKey,
     runAt: job.runAt,
     enqueuedAt: job.enqueuedAt,
     processedAt: job.processedAt,
@@ -39,25 +43,46 @@ const metadataMatches = (record, filter) => {
     }
     return true;
 };
-/**
- * Build a fresh in-memory `JobStore` implementation.
- *
- * @since 0.1.0
- */
-export const make = Effect.sync(() => {
+const makeStoreUnsafe = (options) => {
     const jobs = new Map();
+    const schedules = new Map();
+    const paused = new Set();
+    // Dedup registry: one entry per (name, key). `expiresAt` is set for
+    // ttl/throttle windows; pending-mode entries live as long as their job.
+    const dedupes = new Map();
+    const dedupeMapKey = (name, key) => `${name}\u0000${key}`;
     let seq = 0;
     let idCounter = 0;
     let wakeVersion = 0;
-    let wake = Deferred.makeUnsafe();
+    let lastBroadcast = 0;
+    const lastWake = new Map();
+    const waiters = new Set();
+    const lastWakeFor = (queue) => Math.max(lastWake.get(queue) ?? 0, lastBroadcast);
     // Synchronous on purpose: it is called inside the same synchronous block as
     // the state mutation, so no effect-op boundary (where an interrupt could
-    // land) can separate a mutation from its wake-up signal.
-    const signalWake = () => {
+    // land) can separate a mutation from its wake-up signal. A queue targets
+    // only waiters watching it; no queue broadcasts (rare maintenance verbs).
+    const signalWake = (queue) => {
         wakeVersion += 1;
-        const current = wake;
-        wake = Deferred.makeUnsafe();
-        Deferred.doneUnsafe(current, Exit.succeed(void 0));
+        if (queue === undefined) {
+            lastBroadcast = wakeVersion;
+        }
+        else {
+            lastWake.set(queue, wakeVersion);
+        }
+        // Snapshot-and-clear BEFORE resolving: doneUnsafe resumes waiting fibers
+        // synchronously, and a woken taker that re-parks registers a NEW waiter —
+        // resolving inside the live Set iteration would visit it and livelock.
+        const toWake = [];
+        for (const waiter of waiters) {
+            if (queue === undefined || waiter.queues.has(queue)) {
+                waiters.delete(waiter);
+                toWake.push(waiter);
+            }
+        }
+        for (const waiter of toWake) {
+            Deferred.doneUnsafe(waiter.deferred, Exit.succeed(void 0));
+        }
     };
     const promoteDue = (now) => {
         for (const job of jobs.values()) {
@@ -79,24 +104,68 @@ export const make = Effect.sync(() => {
             exit
         });
     };
+    // A job leaving the pending states frees its pending-mode dedup entry;
+    // live throttle windows deliberately outlast the job.
+    const releaseDedupe = (job, now) => {
+        if (job.dedupeKey === undefined)
+            return;
+        const key = dedupeMapKey(job.name, job.dedupeKey);
+        const entry = dedupes.get(key);
+        if (entry !== undefined && entry.jobId === job.id &&
+            (entry.expiresAt === undefined || entry.expiresAt <= now)) {
+            dedupes.delete(key);
+        }
+    };
+    const markCancelled = (job, now) => {
+        clearLock(job);
+        job.cancelRequested = false;
+        job.state = "cancelled";
+        job.finishedAt = now;
+        recordAttempt(job, "cancelled", now, undefined);
+        releaseDedupe(job, now);
+        applyKeep(job, now);
+    };
+    const keepPolicyFor = (keep, state) => {
+        if (keep === undefined)
+            return undefined;
+        const policy = state === "completed"
+            ? keep.completed
+            : state === "failed"
+                ? keep.failed
+                : state === "cancelled"
+                    ? keep.cancelled
+                    : undefined;
+        if (policy !== undefined)
+            return policy;
+        // Records persisted by 0.2.x carry the flat {count, ageMs} shape — honour
+        // it as an all-states policy so upgrades keep pruning.
+        if (keep.completed === undefined && keep.failed === undefined && keep.cancelled === undefined &&
+            ("count" in keep || "ageMs" in keep)) {
+            // SAFETY: the flat legacy shape carries KeepStatePolicy fields.
+            return keep;
+        }
+        return undefined;
+    };
     // Terminal retention: keep at most `count` and drop older than `ageMs`
-    // among terminal jobs sharing this job's name + state.
+    // among terminal jobs sharing this job's name + state (policies are split
+    // per terminal state).
     const applyKeep = (job, now) => {
-        if (job.keep === undefined)
+        const policy = keepPolicyFor(job.keep, job.state);
+        if (policy === undefined)
             return;
         const peers = Array.from(jobs.values())
             .filter((peer) => peer.name === job.name && peer.state === job.state)
             .toSorted((a, b) => ((b.finishedAt ?? 0) - (a.finishedAt ?? 0)) || (b.seq - a.seq));
         const remove = new Set();
-        if (job.keep.ageMs !== undefined) {
+        if (policy.ageMs !== undefined) {
             for (const peer of peers) {
-                if (peer.finishedAt !== undefined && peer.finishedAt <= now - job.keep.ageMs) {
+                if (peer.finishedAt !== undefined && peer.finishedAt <= now - policy.ageMs) {
                     remove.add(peer.id);
                 }
             }
         }
-        if (job.keep.count !== undefined) {
-            for (const peer of peers.slice(Math.max(0, job.keep.count))) {
+        if (policy.count !== undefined) {
+            for (const peer of peers.slice(Math.max(0, policy.count))) {
                 remove.add(peer.id);
             }
         }
@@ -104,18 +173,100 @@ export const make = Effect.sync(() => {
             jobs.delete(id);
         }
     };
-    return JobStore.of({
+    const sweepHistory = (now, ttlByState) => {
+        for (const job of jobs.values()) {
+            if (!TERMINAL_STATES.has(job.state) || job.finishedAt === undefined)
+                continue;
+            // SAFETY: TERMINAL_STATES membership was checked above.
+            const state = job.state;
+            const ttl = ttlByState[state];
+            const keepAge = keepPolicyFor(job.keep, state)?.ageMs;
+            // The sweep honours min(per-row keep age, store ceiling) — a quiet job
+            // name is pruned on the timer, not only when its group is acked.
+            const effective = keepAge !== undefined && (ttl === undefined || keepAge < ttl) ? keepAge : ttl;
+            if (effective !== undefined && job.finishedAt <= now - effective) {
+                jobs.delete(job.id);
+            }
+        }
+        // Dead dedup entries: expired window, or a pointer at a vanished job.
+        for (const [key, entry] of dedupes) {
+            const alive = entry.expiresAt !== undefined
+                ? entry.expiresAt > now
+                : jobs.has(entry.jobId) && !TERMINAL_STATES.has(jobs.get(entry.jobId)?.state ?? "completed");
+            if (!alive)
+                dedupes.delete(key);
+        }
+    };
+    const service = JobStore.of({
         enqueue: (request) => Effect.gen(function* () {
             const now = yield* Clock.currentTimeMillis;
+            if (request.id !== undefined && jobs.has(request.id)) {
+                return { id: request.id, duplicate: true };
+            }
+            // The dedup decision tree runs BEFORE id generation, so a
+            // deduplicated enqueue never consults the id generator.
+            if (request.dedupe !== undefined) {
+                const mapKey = dedupeMapKey(request.name, request.dedupe.key);
+                const entry = dedupes.get(mapKey);
+                if (entry !== undefined) {
+                    const keyed = jobs.get(entry.jobId);
+                    const windowLive = entry.expiresAt !== undefined && now < entry.expiresAt;
+                    // Latest-wins while the keyed job is still delayed.
+                    if (request.dedupe.replace && keyed !== undefined && keyed.state === "delayed") {
+                        keyed.payload = request.payload;
+                        keyed.metadata = request.metadata;
+                        keyed.priority = request.priority;
+                        keyed.attemptsMax = request.attemptsMax;
+                        keyed.backoff = request.backoff;
+                        keyed.keep = request.keep;
+                        keyed.timeoutMs = request.timeoutMs;
+                        keyed.runAt = now + Math.max(0, request.delayMs);
+                        // A landed replace re-arms the ttl window.
+                        if (request.dedupe.ttlMs !== undefined) {
+                            entry.expiresAt = now + request.dedupe.ttlMs;
+                        }
+                        signalWake(keyed.queue);
+                        return { id: keyed.id, duplicate: true };
+                    }
+                    if (windowLive) {
+                        if (request.dedupe.extend && request.dedupe.ttlMs !== undefined) {
+                            entry.expiresAt = now + request.dedupe.ttlMs;
+                        }
+                        return { id: entry.jobId, duplicate: true };
+                    }
+                    const pending = keyed !== undefined && !TERMINAL_STATES.has(keyed.state);
+                    if (entry.expiresAt === undefined && pending) {
+                        return { id: entry.jobId, duplicate: true };
+                    }
+                    // Dead entry (expired window / finished job): fall through and
+                    // let the new job take over the key below.
+                }
+            }
             let id = request.id;
             if (id === undefined) {
-                // Store-assigned ids must never collide with user-supplied ones.
-                do {
-                    id = JobId(`j-${++idCounter}`);
-                } while (jobs.has(id));
-            }
-            else if (jobs.has(id)) {
-                return { id, duplicate: true };
+                const generate = options?.idGenerator;
+                if (generate === undefined) {
+                    // Store-assigned ids must never collide with user-supplied ones.
+                    do {
+                        id = JobId(`j-${++idCounter}`);
+                    } while (jobs.has(id));
+                }
+                else {
+                    // A user generator gets a bounded number of collision retries; a
+                    // healthy generator's entropy makes even one retry pathological.
+                    for (let i = 0; i < 5 && id === undefined; i++) {
+                        const raw = generate(request);
+                        const candidate = JobId(Effect.isEffect(raw) ? yield* raw : raw);
+                        if (!jobs.has(candidate)) {
+                            id = candidate;
+                        }
+                    }
+                    if (id === undefined) {
+                        return yield* new JobStoreError({
+                            message: "enqueue failed: could not generate a unique job id"
+                        });
+                    }
+                }
             }
             jobs.set(id, {
                 id,
@@ -130,6 +281,9 @@ export const make = Effect.sync(() => {
                 stalledCount: 0,
                 backoff: request.backoff,
                 keep: request.keep,
+                timeoutMs: request.timeoutMs,
+                cancelRequested: false,
+                dedupeKey: request.dedupe?.key,
                 runAt: now + Math.max(0, request.delayMs),
                 enqueuedAt: now,
                 processedAt: undefined,
@@ -141,7 +295,13 @@ export const make = Effect.sync(() => {
                 lockToken: undefined,
                 lockExpiresAt: undefined
             });
-            signalWake();
+            if (request.dedupe !== undefined) {
+                dedupes.set(dedupeMapKey(request.name, request.dedupe.key), {
+                    jobId: id,
+                    expiresAt: request.dedupe.ttlMs !== undefined ? now + request.dedupe.ttlMs : undefined
+                });
+            }
+            signalWake(request.queue);
             return { id, duplicate: false };
         }),
         claim: (options) => Effect.gen(function* () {
@@ -166,7 +326,7 @@ export const make = Effect.sync(() => {
                     }
                 }
             }
-            if (best === undefined) {
+            if (best === undefined || paused.has(options.queue)) {
                 const empty = { _tag: "Empty", nextRunAt, wakeToken: wakeVersion };
                 return empty;
             }
@@ -190,32 +350,53 @@ export const make = Effect.sync(() => {
             job.attemptsMade += 1;
             switch (outcome._tag) {
                 case "Complete": {
+                    job.cancelRequested = false;
                     job.state = "completed";
                     job.exit = outcome.exit;
                     job.finishedAt = now;
                     recordAttempt(job, "completed", now, outcome.exit);
+                    releaseDedupe(job, now);
                     applyKeep(job, now);
                     break;
                 }
                 case "Fail": {
+                    job.cancelRequested = false;
                     job.state = "failed";
                     job.exit = outcome.exit;
                     job.finishedAt = now;
                     recordAttempt(job, "failed", now, outcome.exit);
+                    releaseDedupe(job, now);
+                    applyKeep(job, now);
+                    break;
+                }
+                case "Cancelled": {
+                    job.cancelRequested = false;
+                    job.state = "cancelled";
+                    job.finishedAt = now;
+                    recordAttempt(job, "cancelled", now, undefined);
+                    releaseDedupe(job, now);
                     applyKeep(job, now);
                     break;
                 }
                 case "Retry": {
+                    if (job.cancelRequested) {
+                        // A cancel raced a natural failure before the heartbeat could
+                        // interrupt the run: cancellation wins over revival (mirrors
+                        // release/recoverStalled).
+                        markCancelled(job, now);
+                        break;
+                    }
                     recordAttempt(job, "retried", now, outcome.exit);
                     job.runAt = now + Math.max(0, outcome.delayMs);
                     job.state = outcome.delayMs > 0 ? "delayed" : "waiting";
                     job.seq = ++seq;
-                    signalWake();
+                    signalWake(job.queue);
                     break;
                 }
             }
         }),
         release: (id, token) => Effect.gen(function* () {
+            const now = yield* Clock.currentTimeMillis;
             const job = jobs.get(id);
             if (job === undefined) {
                 return yield* new JobNotFoundError({ jobId: id });
@@ -223,25 +404,36 @@ export const make = Effect.sync(() => {
             if (job.state !== "active" || job.lockToken !== token) {
                 return yield* new LockLostError({ jobId: id });
             }
+            if (job.cancelRequested) {
+                // A cancel arrived while the worker was shutting down: honour it
+                // instead of reviving the job.
+                markCancelled(job, now);
+                return;
+            }
             clearLock(job);
             job.state = "waiting";
-            signalWake();
+            signalWake(job.queue);
         }),
         extendLocks: (locks, durationMs) => Effect.gen(function* () {
             const now = yield* Clock.currentTimeMillis;
             const lost = [];
+            const cancelRequested = [];
             for (const lock of locks) {
                 const job = jobs.get(lock.id);
-                if (job !== undefined &&
-                    job.state === "active" &&
-                    job.lockToken === lock.token) {
-                    job.lockExpiresAt = now + durationMs;
+                if (job === undefined ||
+                    job.state !== "active" ||
+                    job.lockToken !== lock.token) {
+                    lost.push(lock.id);
+                }
+                else if (job.cancelRequested) {
+                    cancelRequested.push(lock.id);
                 }
                 else {
-                    lost.push(lock.id);
+                    job.lockExpiresAt = now + durationMs;
                 }
             }
-            return lost;
+            const result = { lost, cancelRequested };
+            return result;
         }),
         recoverStalled: (options) => Effect.gen(function* () {
             const now = yield* Clock.currentTimeMillis;
@@ -252,6 +444,11 @@ export const make = Effect.sync(() => {
                     job.lockExpiresAt > now) {
                     continue;
                 }
+                if (job.cancelRequested) {
+                    // The owning worker died before honouring the cancel: finish it.
+                    markCancelled(job, now);
+                    continue;
+                }
                 clearLock(job);
                 job.stalledCount += 1;
                 recordAttempt(job, "stalled", now, undefined);
@@ -259,6 +456,7 @@ export const make = Effect.sync(() => {
                     job.state = "failed";
                     job.finishedAt = now;
                     job.failedReason = "job stalled more than allowable limit";
+                    releaseDedupe(job, now);
                     recovered.push({ id: job.id, failed: true });
                 }
                 else {
@@ -271,10 +469,12 @@ export const make = Effect.sync(() => {
             }
             return recovered;
         }),
-        awaitWake: (_queues, wakeToken) => Effect.suspend(() => {
-            if (wakeVersion > wakeToken)
+        awaitWake: (queues, wakeToken) => Effect.suspend(() => {
+            if (queues.some((queue) => lastWakeFor(queue) > wakeToken))
                 return Effect.void;
-            return Deferred.await(wake);
+            const waiter = { queues: new Set(queues), deferred: Deferred.makeUnsafe() };
+            waiters.add(waiter);
+            return Deferred.await(waiter.deferred).pipe(Effect.ensuring(Effect.sync(() => waiters.delete(waiter))));
         }),
         getJob: (id) => Effect.sync(() => {
             const job = jobs.get(id);
@@ -333,13 +533,84 @@ export const make = Effect.sync(() => {
             job.state = "waiting";
             job.attemptsMade = 0;
             job.stalledCount = 0;
+            job.cancelRequested = false;
             job.exit = undefined;
             job.failedReason = undefined;
             job.finishedAt = undefined;
             job.processedAt = undefined;
             job.runAt = now;
             job.seq = ++seq;
-            signalWake();
+            signalWake(job.queue);
+        }),
+        cancel: (id) => Effect.gen(function* () {
+            const now = yield* Clock.currentTimeMillis;
+            const job = jobs.get(id);
+            if (job === undefined) {
+                return yield* new JobNotFoundError({ jobId: id });
+            }
+            switch (job.state) {
+                case "waiting":
+                case "delayed": {
+                    markCancelled(job, now);
+                    return;
+                }
+                case "active": {
+                    job.cancelRequested = true;
+                    return;
+                }
+                default: {
+                    return yield* new JobNotCancellableError({ jobId: id, state: job.state });
+                }
+            }
+        }),
+        promote: (id) => Effect.gen(function* () {
+            const now = yield* Clock.currentTimeMillis;
+            const job = jobs.get(id);
+            if (job === undefined) {
+                return yield* new JobNotFoundError({ jobId: id });
+            }
+            if (job.state !== "delayed") {
+                return yield* new JobNotPromotableError({ jobId: id, state: job.state });
+            }
+            job.state = "waiting";
+            job.runAt = now;
+            signalWake(job.queue);
+        }),
+        pause: (queue) => Effect.sync(() => {
+            paused.add(queue);
+        }),
+        resume: (queue) => Effect.sync(() => {
+            if (paused.delete(queue)) {
+                signalWake(queue);
+            }
+        }),
+        pausedQueues: () => Effect.sync(() => Array.from(paused)),
+        upsertSchedule: (schedule) => Effect.sync(() => {
+            // An unchanged cadence keeps its next occurrence: deploy-time
+            // re-registration must not re-anchor `every` grids or drop a pending
+            // catch-up run. A changed cadence takes the caller's fresh nextRunAt.
+            const existing = schedules.get(schedule.key);
+            const sameCadence = existing !== undefined &&
+                existing.cron === schedule.cron &&
+                existing.tz === schedule.tz &&
+                existing.everyMs === schedule.everyMs;
+            schedules.set(schedule.key, sameCadence ? { ...schedule, nextRunAt: existing.nextRunAt } : schedule);
+            signalWake(schedule.queue);
+        }),
+        removeSchedule: (key) => Effect.sync(() => schedules.delete(key)),
+        listSchedules: (options) => Effect.sync(() => Array.from(schedules.values()).filter((schedule) => (options?.jobName === undefined || schedule.jobName === options.jobName) &&
+            (options?.queue === undefined || schedule.queue === options.queue))),
+        dueSchedules: () => Effect.gen(function* () {
+            const now = yield* Clock.currentTimeMillis;
+            return Array.from(schedules.values())
+                .filter((schedule) => schedule.nextRunAt <= now)
+                .toSorted((a, b) => a.nextRunAt - b.nextRunAt);
+        }),
+        advanceSchedule: (key, expectedRunAt, nextRunAt) => Effect.sync(() => {
+            const schedule = schedules.get(key);
+            if (schedule !== undefined && schedule.nextRunAt === expectedRunAt) {
+                schedules.set(key, { ...schedule, nextRunAt });
+            }
         }),
         counts: (queue) => Effect.sync(() => {
             const counts = {
@@ -347,7 +618,8 @@ export const make = Effect.sync(() => {
                 delayed: 0,
                 active: 0,
                 completed: 0,
-                failed: 0
+                failed: 0,
+                cancelled: 0
             };
             for (const job of jobs.values()) {
                 if (queue !== undefined && job.queue !== queue)
@@ -364,18 +636,50 @@ export const make = Effect.sync(() => {
             return true;
         })
     });
+    return { service, sweepHistory };
+};
+/**
+ * Build a fresh in-memory `JobStore` implementation (no history sweeper —
+ * use `makeWith` for `historyTtl` support).
+ *
+ * @since 0.1.0
+ */
+export const make = Effect.sync(() => makeStoreUnsafe().service);
+/**
+ * Build a fresh in-memory `JobStore` with options; the history sweeper (when
+ * configured) lives in the surrounding `Scope`.
+ *
+ * @since 0.2.0
+ */
+export const makeWith = (options) => Effect.gen(function* () {
+    const { service, sweepHistory } = makeStoreUnsafe(options);
+    if (options?.historyTtl !== undefined) {
+        const ttlByState = normalizeHistoryTtl(options.historyTtl);
+        const intervalMs = Duration.toMillis(options.historySweepInterval ?? "1 minute");
+        yield* Effect.gen(function* () {
+            yield* Effect.sleep(intervalMs);
+            const now = yield* Clock.currentTimeMillis;
+            sweepHistory(now, ttlByState);
+        }).pipe(Effect.forever, Effect.forkScoped);
+    }
+    return service;
 });
 /**
- * A fresh in-memory `JobStore` layer. Pass a named store key to provide a
- * `JobStore.named(...)` slot instead of the default.
+ * A fresh in-memory `JobStore` layer.
  *
  * @since 0.1.0
  */
-export const layer = Layer.effect(JobStore, make);
+export const layer = Layer.effect(JobStore, makeWith());
+/**
+ * An in-memory layer with options (e.g. `historyTtl`).
+ *
+ * @since 0.2.0
+ */
+export const layerWith = (options) => Layer.effect(JobStore, makeWith(options));
 /**
  * An in-memory layer for a specific store key.
  *
  * @since 0.1.0
  */
-export const layerFor = (store) => Layer.effect(store, make);
+export const layerFor = (store, options) => Layer.effect(store, makeWith(options));
 //# sourceMappingURL=MemoryJobStore.js.map