effect-mq 0.3.0 → 0.3.2

package/src/Metrics.ts

@@ -0,0 +1,135 @@
+/**
+ * The metrics effect-mq emits, as Effect `Metric` instruments.
+ *
+ * Metrics are **process-local operational signal, not persisted state**:
+ * they live in the Effect metric registry of the emitting process and are
+ * exported by whatever exporter the application runs (the Otlp modules from
+ * `effect/unstable/observability`, `@effect/opentelemetry`, a Prometheus
+ * scraper, ...). Retention belongs to that metrics backend. The queue's
+ * *durable* analogues remain in the store — `store.counts()` for live depth
+ * and the attempt ledger for per-run history — and stay queryable forever.
+ *
+ * Producers emit `jobsEnqueued`; workers emit the rest from their loops.
+ * Everything is tagged with low-cardinality attributes only (job name,
+ * queue, outcome/result) — never ids or keys.
+ *
+ * This module is exported so applications can read the same instruments
+ * (e.g. `Metric.value(Metrics.jobRuns.pipe(Metric.withAttributes({...})))`)
+ * or reference the names when building dashboards.
+ *
+ * @since 0.3.1
+ */
+import { Metric } from "effect"
+
+/** Milliseconds, 5ms .. ~55 minutes. */
+const durationBoundaries = Metric.exponentialBoundaries({ start: 5, factor: 2, count: 20 })
+
+/**
+ * Producer-side enqueues. Tags: `name`, `queue`, `duplicate` ("true" when
+ * the enqueue deduplicated against an existing id or dedup key).
+ *
+ * @since 0.3.1
+ */
+export const jobsEnqueued = Metric.counter("effect_mq_jobs_enqueued", {
+  description: "Jobs submitted via Job.enqueue/execute, including deduplicated submissions"
+})
+
+/**
+ * Finished runs, one per handler attempt. Tags: `name`, `queue`, `outcome`
+ * (completed | retried | failed | cancelled | released).
+ *
+ * @since 0.3.1
+ */
+export const jobRuns = Metric.counter("effect_mq_job_runs", {
+  description: "Handler runs by outcome (released = handed back on worker shutdown)"
+})
+
+/**
+ * Handler execution time in milliseconds (claim to ack). Tags: `name`,
+ * `queue`, `outcome`.
+ *
+ * @since 0.3.1
+ */
+export const jobRunDuration = Metric.histogram("effect_mq_job_run_duration_ms", {
+  description: "Handler execution time in milliseconds, claim to ack",
+  boundaries: durationBoundaries
+})
+
+/**
+ * Queue latency in milliseconds: how long a job was runnable (past its
+ * `runAt`) before a worker claimed it. Tags: `name`, `queue`.
+ *
+ * @since 0.3.1
+ */
+export const jobWaitDuration = Metric.histogram("effect_mq_job_wait_duration_ms", {
+  description: "Time between a job becoming runnable and its claim, in milliseconds",
+  boundaries: durationBoundaries
+})
+
+/**
+ * Claim attempts by result. Tags: `queue`, `result` (claimed | empty).
+ * A high empty ratio means takers outnumber work.
+ *
+ * @since 0.3.1
+ */
+export const claims = Metric.counter("effect_mq_claims", {
+  description: "Store claim attempts by result"
+})
+
+/**
+ * Handlers currently executing. Tags: `queue`.
+ *
+ * @since 0.3.1
+ */
+export const jobsInFlight = Metric.gauge("effect_mq_jobs_in_flight", {
+  description: "Handlers currently executing"
+})
+
+/**
+ * Queue depth by state, sampled from `store.counts()` when
+ * `Worker.layer({ queueMetricsInterval })` is set. Tags: `queue`, `state`.
+ *
+ * @since 0.3.1
+ */
+export const queueDepth = Metric.gauge("effect_mq_queue_depth", {
+  description: "Jobs per state, sampled from store.counts() per registered queue"
+})
+
+/**
+ * Locks found gone at renewal, counted once per lost lock (the job may run
+ * twice). Tags: none.
+ *
+ * @since 0.3.1
+ */
+export const locksLost = Metric.counter("effect_mq_locks_lost", {
+  description: "Locks found lost at heartbeat renewal, once per lock"
+})
+
+/**
+ * Cross-process cancel requests delivered to running handlers by the
+ * heartbeat. Tags: none.
+ *
+ * @since 0.3.1
+ */
+export const cancelInterrupts = Metric.counter("effect_mq_cancel_interrupts", {
+  description: "Running handlers interrupted by a cancel request"
+})
+
+/**
+ * Jobs recovered by the stalled sweep. Tags: `outcome` (requeued | failed).
+ *
+ * @since 0.3.1
+ */
+export const stalledRecovered = Metric.counter("effect_mq_stalled_recovered", {
+  description: "Stalled jobs recovered (requeued) or failed past the stall limit"
+})
+
+/**
+ * Repeatable-schedule occurrences enqueued by this worker's sweep. Tags:
+ * `name` (the job name).
+ *
+ * @since 0.3.1
+ */
+export const scheduleTicks = Metric.counter("effect_mq_schedule_ticks", {
+  description: "Repeatable-schedule occurrences enqueued by the schedule sweep"
+})

package/src/Worker.ts

@@ -14,7 +14,7 @@
  *
  * @since 0.1.0
  */
-import { Cause, Clock, Context, Deferred, Duration, Effect, Exit, Fiber, FiberSet, Layer, Option, Result, Schedule, Schema, type Scope, Scope as Scope_, Tracer } from "effect"
+import { Cause, Clock, Context, Deferred, Duration, Effect, Exit, Fiber, FiberSet, Layer, Metric, Option, Result, Schedule, Schema, type Scope, Scope as Scope_, Tracer } from "effect"
 import {
   type AckOutcome,
   type BackoffPolicy,
@@ -31,6 +31,7 @@ import {
   type ScheduleRecord,
   type Service as StoreService
 } from "./JobStore.ts"
+import * as Metrics from "./Metrics.ts"
 
 /**
  * Information about the currently running attempt, passed to handlers as the
@@ -117,6 +118,12 @@ export interface WorkerOptions<StoreId = JobStore> {
   readonly pollInterval?: Duration.Input | undefined
   /** How often to tick due repeatable-job schedules (default 15s). */
   readonly scheduleSweepInterval?: Duration.Input | undefined
+  /**
+   * When set, sample `store.counts(queue)` for every registered queue into
+   * the `effect_mq_queue_depth` gauge at this cadence (default: off — depth
+   * sampling costs one store query per queue per tick).
+   */
+  readonly queueMetricsInterval?: Duration.Input | undefined
   /** Identifier used in lock tokens (default: random). */
   readonly id?: string | undefined
 }
@@ -311,6 +318,18 @@ export const make = <StoreId = JobStore>(
           attempt: record.attemptsMade + 1,
           attemptsMax: record.attemptsMax
         }
+        // One line per finished run: outcome counter + duration histogram
+        // (claim time to now, from the Effect Clock).
+        const recordRun = (outcome: string) =>
+          Effect.gen(function*() {
+            const finished = yield* Clock.currentTimeMillis
+            const attributes = { name: record.name, queue: record.queue, outcome }
+            yield* Metric.update(Metrics.jobRuns.pipe(Metric.withAttributes(attributes)), 1)
+            yield* Metric.update(
+              Metrics.jobRunDuration.pipe(Metric.withAttributes(attributes)),
+              Math.max(0, finished - (record.processedAt ?? finished))
+            )
+          })
         return Effect.gen(function*() {
           // The per-run time limit interrupts the handler internally; surface
           // it as a defect so it flows through normal retry accounting.
@@ -334,7 +353,10 @@ export const make = <StoreId = JobStore>(
           // keeps the child interruptible while the fork itself is masked.
           const fiber = yield* Effect.forkChild(restore(handlerEffect))
           inflight.set(record.id, { id: record.id, token, fiber })
+          const busy = Metrics.jobsInFlight.pipe(Metric.withAttributes({ queue: record.queue }))
+          yield* Metric.modify(busy, 1)
           const exit = yield* Effect.exit(restore(Fiber.join(fiber)))
+          yield* Metric.modify(busy, -1)
           inflight.delete(record.id)
           const wasCancelled = cancelling.delete(record.id)
 
@@ -342,7 +364,7 @@ export const make = <StoreId = JobStore>(
           // shutdown races it — cancellation wins, the job must not revive).
           if (wasCancelled && Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause)) {
             yield* ackSafely(store.ack(record.id, token, { _tag: "Cancelled" }), "ack")
-            return
+            return yield* recordRun("cancelled")
           }
 
           // Distinguish worker shutdown from a handler that interrupted
@@ -355,6 +377,7 @@ export const make = <StoreId = JobStore>(
             // without consuming an attempt.
             yield* Fiber.interrupt(fiber)
             yield* ackSafely(store.release(record.id, token), "release")
+            yield* recordRun("released")
             return yield* Effect.interrupt
           }
 
@@ -396,6 +419,15 @@ export const make = <StoreId = JobStore>(
             ? { _tag: "Fail", exit: exitValue }
             : routeFailure(record, exitValue)
           yield* ackSafely(store.ack(record.id, token, outcome), "ack")
+          yield* recordRun(
+            outcome._tag === "Complete"
+              ? "completed"
+              : outcome._tag === "Retry"
+              ? "retried"
+              : outcome._tag === "Cancelled"
+              ? "cancelled"
+              : "failed"
+          )
         })
       })
 
@@ -420,6 +452,10 @@ export const make = <StoreId = JobStore>(
             lockDurationMs
           }))
           if (result._tag === "Empty") {
+            yield* Metric.update(
+              Metrics.claims.pipe(Metric.withAttributes({ queue, result: "empty" })),
+              1
+            )
             const now = yield* Clock.currentTimeMillis
             const timeout = result.nextRunAt !== undefined
               ? Math.max(0, Math.min(result.nextRunAt - now, pollMs))
@@ -430,6 +466,17 @@ export const make = <StoreId = JobStore>(
               awaitPulse(observedPulse)
             ]))
           }
+          yield* Metric.update(
+            Metrics.claims.pipe(Metric.withAttributes({ queue, result: "claimed" })),
+            1
+          )
+          const claimedAt = yield* Clock.currentTimeMillis
+          yield* Metric.update(
+            Metrics.jobWaitDuration.pipe(
+              Metric.withAttributes({ name: result.job.name, queue })
+            ),
+            Math.max(0, claimedAt - result.job.runAt)
+          )
           yield* processJob(result.job, token, restore)
         })
       )
@@ -475,16 +522,28 @@ export const make = <StoreId = JobStore>(
         lockDurationMs
       ))
       if (result.lost.length > 0) {
+        yield* Metric.update(Metrics.locksLost, result.lost.length)
         yield* Effect.logWarning(
           "effect-mq: failed to renew locks; jobs may run twice",
           result.lost
         )
+        // A lost lock stays lost: drop the flight so we neither renew nor
+        // recount it every heartbeat (the run's eventual ack surfaces
+        // LockLostError on its own).
+        for (const id of result.lost) {
+          inflight.delete(id)
+        }
       }
       // Honour cross-process cancel requests: interrupt the handler fiber;
       // processJob acks the job as Cancelled.
       for (const id of result.cancelRequested) {
         const flight = inflight.get(id)
         if (flight !== undefined) {
+          // The store re-reports the request every heartbeat until the ack
+          // lands; count the first delivery only.
+          if (!cancelling.has(id)) {
+            yield* Metric.update(Metrics.cancelInterrupts, 1)
+          }
           cancelling.add(id)
           // Delivery only — awaiting the handler's exit here would park the
           // heartbeat behind arbitrary user finalizers and starve every other
@@ -502,6 +561,19 @@ export const make = <StoreId = JobStore>(
       yield* Effect.sleep(stalledMs)
       const recovered = yield* retryStore(store.recoverStalled({ maxStalledCount }))
       if (recovered.length > 0) {
+        const failed = recovered.filter((entry) => entry.failed).length
+        if (failed > 0) {
+          yield* Metric.update(
+            Metrics.stalledRecovered.pipe(Metric.withAttributes({ outcome: "failed" })),
+            failed
+          )
+        }
+        if (recovered.length - failed > 0) {
+          yield* Metric.update(
+            Metrics.stalledRecovered.pipe(Metric.withAttributes({ outcome: "requeued" })),
+            recovered.length - failed
+          )
+        }
         yield* Effect.logWarning("effect-mq: recovered stalled jobs", recovered)
       }
     }).pipe(
@@ -523,7 +595,7 @@ export const make = <StoreId = JobStore>(
             `effect-mq: schedule "${schedule.key}" has an invalid cron/every configuration; skipping`
           )
         }
-        yield* retryStore(store.enqueue({
+        const tick = yield* retryStore(store.enqueue({
           id: JobId(`sched/${schedule.key}/${slot}`),
           name: schedule.jobName,
           queue: schedule.queue,
@@ -537,6 +609,12 @@ export const make = <StoreId = JobStore>(
           dedupe: undefined,
           delayMs: 0
         }))
+        if (!tick.duplicate) {
+          yield* Metric.update(
+            Metrics.scheduleTicks.pipe(Metric.withAttributes({ name: schedule.jobName })),
+            1
+          )
+        }
         yield* retryStore(store.advanceSchedule(schedule.key, slot, next))
       })
 
@@ -561,6 +639,26 @@ export const make = <StoreId = JobStore>(
     yield* FiberSet.run(fibers, stalledLoop)
     yield* FiberSet.run(fibers, scheduleLoop)
 
+    if (options?.queueMetricsInterval !== undefined) {
+      const sampleMs = Duration.toMillis(options.queueMetricsInterval)
+      const depthLoop = Effect.gen(function*() {
+        yield* Effect.sleep(sampleMs)
+        for (const queue of startedQueues) {
+          const counts = yield* retryStore(store.counts(queue))
+          for (const [state, depth] of Object.entries(counts)) {
+            yield* Metric.update(
+              Metrics.queueDepth.pipe(Metric.withAttributes({ queue, state })),
+              depth
+            )
+          }
+        }
+      }).pipe(
+        Effect.catchCause((cause) => Effect.logWarning("effect-mq: queue depth sampling failed", cause)),
+        Effect.forever
+      )
+      yield* FiberSet.run(fibers, depthLoop)
+    }
+
     // SAFETY: the public `register` signature declares Scope, the handler's R
     // and the codec services as requirements; the implementation erases them
     // (the trailing assertion below) because the handler runs with the

package/src/index.ts

@@ -25,6 +25,14 @@ export * as JobStore from "./JobStore.ts"
  */
 export * as MemoryJobStore from "./MemoryJobStore.ts"
 
+/**
+ * The metrics effect-mq emits (Effect `Metric` instruments) — process-local,
+ * exported by your observability stack.
+ *
+ * @since 0.3.1
+ */
+export * as Metrics from "./Metrics.ts"
+
 /**
  * The worker runtime: `Worker.layer` and handler registration.
  *

package/src/testing/TestJobStore.ts

@@ -0,0 +1,143 @@
+/**
+ * A test harness for asserting what your services enqueue, without running
+ * a worker.
+ *
+ * Provide `TestJobStore.layer` in a unit test and jobs enqueued by the code
+ * under test simply accumulate in `waiting`/`delayed` (nothing claims them).
+ * `enqueuedOf` returns them with payloads **decoded through the job's own
+ * schema** — you assert against the typed values your service produced, not
+ * the encoded JSON the store persists:
+ *
+ * ```ts
+ * import { TestJobStore } from "effect-mq/testing"
+ *
+ * it.effect("signup enqueues a welcome email", () =>
+ *   Effect.gen(function*() {
+ *     yield* SignupService.register({ email: "ada@example.com" })
+ *
+ *     const emails = yield* TestJobStore.enqueuedOf(SendEmail)
+ *     expect(emails).toHaveLength(1)
+ *     expect(emails[0]?.payload.to).toBe("ada@example.com")
+ *     expect(emails[0]?.state).toBe("waiting")
+ *   }).pipe(Effect.provide(TestJobStore.layer)))
+ * ```
+ *
+ * Jobs bound to named stores use `TestJobStore.layerFor(Durable)` instead.
+ * The raw `JobStore` service is also exposed (as `.store`) for advanced
+ * scenarios — simulating claims/acks, reading `counts()`, and so on.
+ *
+ * @since 0.3.2
+ */
+import * as JobStore from "../JobStore.ts"
+import * as MemoryJobStore from "../MemoryJobStore.ts"
+import { Context, Effect, Layer, Schema } from "effect"
+
+/**
+ * The minimal structural view of a `Job.make` class that `enqueuedOf`
+ * needs: its tag and its JSON payload codec.
+ *
+ * @since 0.3.2
+ */
+export interface AnyJobDefinition {
+  readonly _tag: string
+  readonly payloadJsonSchema: Schema.Top & { readonly DecodingServices: never }
+}
+
+/**
+ * A stored job with its payload decoded back to the definition's payload
+ * type.
+ *
+ * @since 0.3.2
+ */
+export interface EnqueuedJob<Payload> extends Omit<JobStore.JobRecord, "payload"> {
+  readonly payload: Payload
+}
+
+const drainList = (store: JobStore.Service, name?: string) =>
+  Effect.gen(function*() {
+    const all: Array<JobStore.JobRecord> = []
+    let cursor: string | undefined = undefined
+    while (true) {
+      const page: JobStore.ListResult = yield* store.list({ name, cursor, limit: 200 }).pipe(Effect.orDie)
+      all.push(...page.items)
+      if (page.cursor === undefined) break
+      cursor = page.cursor
+    }
+    // list() is newest-first; flip to oldest-first for natural reading.
+    // Ties (same-instant enqueues) order by id, not submission order.
+    return all.toReversed()
+  })
+
+const makeApi = (store: JobStore.Service) => ({
+  store,
+  enqueued: (name?: string) => drainList(store, name),
+  enqueuedOf: <J extends AnyJobDefinition>(job: J) =>
+    drainList(store, job._tag).pipe(
+      Effect.flatMap(Effect.forEach((record) =>
+        Schema.decodeUnknownEffect(job.payloadJsonSchema)(record.payload).pipe(
+          Effect.orDie,
+          Effect.map((payload): EnqueuedJob<J["payloadJsonSchema"]["Type"]> => ({ ...record, payload }))
+        )
+      ))
+    )
+})
+
+/**
+ * Inspection API over the test store. `enqueued(name?)` returns raw records
+ * oldest-first; `enqueuedOf(JobClass)` additionally decodes payloads through
+ * the definition's schema.
+ *
+ * @since 0.3.2
+ */
+export class TestJobStore extends Context.Service<TestJobStore, ReturnType<typeof makeApi>>()(
+  "effect-mq/testing/TestJobStore"
+) {}
+
+/**
+ * A fresh in-memory store provided as BOTH the default `JobStore` (for the
+ * code under test) and the `TestJobStore` inspection service (for the
+ * assertions).
+ *
+ * @since 0.3.2
+ */
+export const layer: Layer.Layer<JobStore.JobStore | TestJobStore> = Layer.effectContext(
+  Effect.map(MemoryJobStore.makeWith(), (store) =>
+    Context.make(JobStore.JobStore, store).pipe(
+      Context.add(TestJobStore, TestJobStore.of(makeApi(store)))
+    ))
+)
+
+/**
+ * Like `layer`, for jobs bound to a `JobStore.named(...)` key.
+ *
+ * @since 0.3.2
+ */
+export const layerFor = <Id>(
+  store: Context.Key<Id, JobStore.Service>
+): Layer.Layer<Id | TestJobStore> =>
+  Layer.effectContext(
+    Effect.map(MemoryJobStore.makeWith(), (memory) =>
+      Context.make(store, memory).pipe(
+        Context.add(TestJobStore, TestJobStore.of(makeApi(memory)))
+      ))
+  )
+
+/**
+ * Convenience accessors so tests don't have to `yield* TestJobStore` first.
+ *
+ * @since 0.3.2
+ */
+export const enqueuedOf = <J extends AnyJobDefinition>(
+  job: J
+): Effect.Effect<Array<EnqueuedJob<J["payloadJsonSchema"]["Type"]>>, never, TestJobStore> =>
+  Effect.flatMap(TestJobStore, (api) => api.enqueuedOf(job))
+
+/**
+ * Raw records (optionally filtered by job name), oldest-first.
+ *
+ * @since 0.3.2
+ */
+export const enqueued = (
+  name?: string
+): Effect.Effect<Array<JobStore.JobRecord>, never, TestJobStore> =>
+  Effect.flatMap(TestJobStore, (api) => api.enqueued(name))

package/src/testing/index.ts

@@ -5,3 +5,10 @@
  * @since 0.1.0
  */
 export * from "./conformance.ts"
+
+/**
+ * Assert what services enqueue in unit tests, with typed payloads.
+ *
+ * @since 0.3.2
+ */
+export * as TestJobStore from "./TestJobStore.ts"