effect-mq 0.4.2 → 0.6.0

package/src/drizzle-postgres/schema.ts

@@ -36,6 +36,7 @@ import { sql } from "drizzle-orm"
 import {
   type AnyPgColumnBuilder,
   bigint,
+  bigserial,
   boolean,
   index,
   integer,
@@ -56,6 +57,10 @@ type BackoffPolicy = JobStore.BackoffPolicy
 type KeepPolicy = JobStore.KeepPolicy
 type AttemptOutcome = JobStore.AttemptRecord["outcome"]
 type TraceContext = JobStore.TraceContext
+type ParentEnvelope = JobStore.ParentEnvelope
+type EnqueueRequest = JobStore.EnqueueRequest
+type FlowChildStatus = JobStore.FlowChildRecord["status"]
+type FlowChildReport = JobStore.FlowChildReport
 
 /**
  * Table-factory options: `extraConfig` receives the table's columns (exactly
@@ -89,6 +94,14 @@ const jobsColumns = <JobName extends string, Queue extends string>() => ({
   cancelRequested: boolean("cancel_requested").notNull().default(false),
   dedupeKey: text("dedupe_key"),
   trace: jsonb("trace").$type<TraceContext>(),
+  /** Flow-parent link on child jobs (opaque envelope, like `trace`). */
+  parent: jsonb("parent").$type<ParentEnvelope>(),
+  /** Flow bookkeeping: NULL together until a FanOut ack lands the manifest. */
+  flowFailFast: boolean("flow_fail_fast"),
+  flowPending: integer("flow_pending"),
+  flowCompleted: integer("flow_completed"),
+  flowFailed: integer("flow_failed"),
+  flowCancelled: integer("flow_cancelled"),
   runAt: timestamp("run_at", { withTimezone: true, mode: "date" }).notNull(),
   enqueuedAt: timestamp("enqueued_at", { withTimezone: true, mode: "date" }).notNull(),
   processedAt: timestamp("processed_at", { withTimezone: true, mode: "date" }),
@@ -201,6 +214,8 @@ const schedulesColumns = <JobName extends string, Queue extends string>() => ({
   backoff: jsonb("backoff").$type<BackoffPolicy>(),
   keep: jsonb("keep").$type<KeepPolicy>(),
   timeoutMs: bigint("timeout_ms", { mode: "number" }),
+  /** Ownership label for declarative reconciliation; NULL = never pruned. */
+  group: text("group_name"),
   nextRunAt: timestamp("next_run_at", { withTimezone: true, mode: "date" }).notNull()
 })
 
@@ -243,6 +258,75 @@ export const mqDedupe = <JobName extends string = string>(
     ...options?.extraConfig?.(table) ?? []
   ])
 
+const flowChildrenColumns = () => ({
+  /** The parent (flow) job's id in this store. */
+  flowId: text("flow_id").notNull().$type<JobId>(),
+  /** Unique within the flow (the idempotency mechanism). */
+  childKey: text("child_key").notNull(),
+  /** The child job's name (projection of `spec.name` for dashboards). */
+  name: text("name").notNull(),
+  /** The CHILD store's context-key string (children may live elsewhere). */
+  storeKey: text("store_key").notNull(),
+  /** The FULL `EnqueueRequest`, so the sweeper can re-enqueue from storage. */
+  spec: jsonb("spec").notNull().$type<EnqueueRequest>(),
+  status: text("status").notNull().$type<FlowChildStatus>(),
+  /** The child's schema-encoded exit; NULL for store-side failures. */
+  exit: jsonb("exit"),
+  failedReason: text("failed_reason"),
+  /** True once no cancel needs delivering into the child's store. */
+  cascaded: boolean("cascaded").notNull(),
+  pendingSince: timestamp("pending_since", { withTimezone: true, mode: "date" }).notNull()
+})
+
+/**
+ * Flow dependency rows (one per fan-out child; see `JobStore.FlowChildRecord`).
+ * Lives next to the PARENT store's jobs table.
+ *
+ * @since 0.6.0
+ */
+export const mqFlowChildren = (
+  tableName = "effect_mq_flow_children",
+  options?: MqTableOptions<ReturnType<typeof flowChildrenColumns>>
+) =>
+  pgTable(tableName, flowChildrenColumns(), (table) => [
+    primaryKey({ columns: [table.flowId, table.childKey] }),
+    // flowSweepWork reconcile: pending rows older than the threshold.
+    index(`${tableName}_pending_idx`)
+      .on(table.pendingSince)
+      .where(sql`${table.status} = 'pending'`),
+    // flowSweepWork cascade: cancelled rows not yet delivered.
+    index(`${tableName}_cascade_idx`)
+      .on(table.flowId)
+      .where(sql`${table.status} = 'cancelled' AND NOT ${table.cascaded}`),
+    ...options?.extraConfig?.(table) ?? []
+  ])
+
+const flowOutboxColumns = () => ({
+  /** Store-assigned, oldest-first; exposed to callers as an opaque string. */
+  id: bigserial("id", { mode: "number" }).primaryKey(),
+  /** The flow definition's name, for relay routing. */
+  flowName: text("flow_name").notNull(),
+  /** The PARENT store's context-key string, for relay routing. */
+  parentStoreKey: text("parent_store_key").notNull(),
+  /** The full `FlowChildReport` to deliver into the parent store. */
+  report: jsonb("report").notNull().$type<FlowChildReport>()
+})
+
+/**
+ * Undelivered child-result reports (one per terminal transition of an
+ * envelope-carrying job; see `JobStore.OutboxEntry`). Lives next to the
+ * CHILD store's jobs table.
+ *
+ * @since 0.6.0
+ */
+export const mqFlowOutbox = (
+  tableName = "effect_mq_flow_outbox",
+  options?: MqTableOptions<ReturnType<typeof flowOutboxColumns>>
+) =>
+  pgTable(tableName, flowOutboxColumns(), (table) => [
+    ...options?.extraConfig?.(table) ?? []
+  ])
+
 const queueControlColumns = <Queue extends string>() => ({
   queue: text("queue").primaryKey().$type<Queue>(),
   paused: boolean("paused").notNull().default(false)
@@ -285,3 +369,13 @@ export type MqQueueControlTable = ReturnType<typeof mqQueueControl<any>>
  * @since 0.3.0
  */
 export type MqDedupeTable = ReturnType<typeof mqDedupe<any>>
+
+/**
+ * @since 0.6.0
+ */
+export type MqFlowChildrenTable = ReturnType<typeof mqFlowChildren>
+
+/**
+ * @since 0.6.0
+ */
+export type MqFlowOutboxTable = ReturnType<typeof mqFlowOutbox>

package/src/index.ts

@@ -4,6 +4,14 @@
  * @since 0.1.0
  */
 
+/**
+ * Cross-store parent-child flows: `Flow.make`, `Flow.children`, the
+ * two-phase `fanOut`/`collect` handler.
+ *
+ * @since 0.6.0
+ */
+export * as Flow from "./Flow.ts"
+
 /**
  * Schema-first job definitions: `Job.make`, `enqueue`, `toLayer`.
  *
@@ -11,6 +19,14 @@
  */
 export * as Job from "./Job.ts"
 
+/**
+ * Declarative schedule reconciliation: declare a service's full schedule
+ * set as a layer; startup upserts it and detects deletion drift.
+ *
+ * @since 0.5.0
+ */
+export * as JobSchedules from "./JobSchedules.ts"
+
 /**
  * The storage seam: the `JobStore` service, job records and typed errors.
  *

package/src/redis/RedisJobStore.ts

@@ -85,6 +85,18 @@ const toRecord = (hash: ReadonlyMap<string, string>): JobStore.JobRecord => ({
   cancelRequested: hash.get("cancelRequested") === "1",
   dedupeKey: optionalString(hash.get("dedupeKey")),
   trace: optionalJson<JobStore.TraceContext>(hash.get("trace")),
+  parent: optionalJson<JobStore.ParentEnvelope>(hash.get("parent")),
+  // Manifest presence IS the phase marker: hashes written before flows (or
+  // parents that never fanned out) simply lack the flow fields.
+  flow: optionalString(hash.get("flowPending")) === undefined
+    ? undefined
+    : {
+      failFast: hash.get("flowFailFast") === "1",
+      pending: Number(hash.get("flowPending")),
+      completed: Number(hash.get("flowCompleted") ?? 0),
+      failed: Number(hash.get("flowFailed") ?? 0),
+      cancelled: Number(hash.get("flowCancelled") ?? 0)
+    },
   runAt: Number(hash.get("runAt") ?? 0),
   enqueuedAt: Number(hash.get("enqueuedAt") ?? 0),
   processedAt: optionalNumber(hash.get("processedAt")),
@@ -107,6 +119,7 @@ const toSchedule = (hash: ReadonlyMap<string, string>): JobStore.ScheduleRecord
   backoff: optionalJson<JobStore.BackoffPolicy>(hash.get("backoff")),
   keep: optionalJson<JobStore.KeepPolicy>(hash.get("keep")),
   timeoutMs: optionalNumber(hash.get("timeoutMs")),
+  group: optionalString(hash.get("group")),
   nextRunAt: Number(hash.get("nextRunAt") ?? 0)
 })
 
@@ -114,15 +127,68 @@ const toSchedule = (hash: ReadonlyMap<string, string>): JobStore.ScheduleRecord
 const asArray = <A>(value: ReadonlyArray<A> | Record<string, never>): ReadonlyArray<A> =>
   Array.isArray(value) ? value : []
 
+/**
+ * Decode one outbox zset member: `<seq>\0<json>` where the json carries the
+ * verbatim parent envelope plus the terminal outcome. The full member string
+ * is the opaque entry id (deleteOutbox is then a plain ZREM).
+ */
+const toOutboxEntry = (member: string): JobStore.OutboxEntry => {
+  const sep = member.indexOf("\u0000")
+  const body: {
+    parent: JobStore.ParentEnvelope
+    outcome: JobStore.FlowChildReport["outcome"]
+    exit?: unknown
+    failedReason?: string
+  } = JSON.parse(member.slice(sep + 1))
+  return {
+    id: member,
+    flowName: body.parent.flowName,
+    parentStoreKey: body.parent.parentStoreKey,
+    report: {
+      flowId: body.parent.flowId,
+      childKey: body.parent.childKey,
+      outcome: body.outcome,
+      // The exit key is omitted entirely when absent (ledger convention),
+      // so a legitimate encoded null exit survives the round trip.
+      exit: Object.hasOwn(body, "exit") ? body.exit : undefined,
+      failedReason: body.failedReason
+    }
+  }
+}
+
 const JOB_STATES: ReadonlyArray<JobStore.JobState> = [
   "waiting",
   "delayed",
   "active",
+  "waiting-children",
   "completed",
   "failed",
   "cancelled"
 ]
 
+/**
+ * Fold one positional dependency-row tuple into a `FlowChildRecord`. The
+ * order must stay in lockstep with the HMGET field list in the
+ * `listChildResults` script:
+ * childKey, storeKey, childJobId, name, status, exit, failedReason, cascaded.
+ */
+const toChildRecord = (
+  flowId: JobStore.JobId,
+  row: ReadonlyArray<string>
+): JobStore.FlowChildRecord => ({
+  flowId,
+  childKey: row[0] ?? "",
+  storeKey: row[1] ?? "",
+  childJobId: JobStore.JobId(row[2] ?? ""),
+  name: row[3] ?? "",
+  // SAFETY: the status field is only ever written with FlowChildRecord
+  // status members ("pending" at insert, a report/settle outcome after).
+  status: (row[4] ?? "pending") as JobStore.FlowChildRecord["status"],
+  exit: optionalJson(row[5]),
+  failedReason: optionalString(row[6]),
+  cascaded: row[7] === "1"
+})
+
 /**
  * Build a `RedisJobStore` service. Needs the `Redis` service and a `Scope`
  * (the wake-up subscription and the optional history sweeper live in it).
@@ -159,6 +225,11 @@ export const make = (
     const evalEnqueueMany = redis.eval(scripts.enqueueMany)
     const evalSweepState = redis.eval(scripts.sweepState)
     const evalSweepDedupes = redis.eval(scripts.sweepDedupes)
+    const evalFanOut = redis.eval(scripts.fanOut)
+    const evalRecordChildResults = redis.eval(scripts.recordChildResults)
+    const evalListChildResults = redis.eval(scripts.listChildResults)
+    const evalFlowSweepWork = redis.eval(scripts.flowSweepWork)
+    const evalMarkChildrenCascaded = redis.eval(scripts.markChildrenCascaded)
 
     // Wake protocol: a queue-filtered waiter registry (same-process wake-ups
     // never depend on the pub/sub round trip), with the channel carrying
@@ -281,9 +352,71 @@ export const make = (
         request.dedupe?.ttlMs === undefined ? "" : String(request.dedupe.ttlMs),
         request.dedupe?.extend === true ? "1" : "0",
         request.dedupe?.replace === true ? "1" : "0",
-        request.trace === undefined ? "" : JSON.stringify(request.trace)
+        request.trace === undefined ? "" : JSON.stringify(request.trace),
+        request.parent === undefined ? "" : JSON.stringify(request.parent)
       )
 
+    // The FanOut ack. Large manifests chunk their dependency rows across
+    // several lock-token-guarded script calls (ARGV strides, like
+    // enqueueMany); only the FINAL chunk writes the manifest and flips the
+    // state, so a crash mid-staging leaves the job active and recoverable —
+    // the next attempt's first chunk clears the orphaned staged rows.
+    const fanOutAck = (
+      id: JobStore.JobId,
+      token: string,
+      failFast: boolean,
+      children: ReadonlyArray<JobStore.FlowChildSpec>
+    ) =>
+      Effect.gen(function*() {
+        if (children.some((child) => child.request.id === undefined)) {
+          // Validate BEFORE any script call, so a bad spec cannot leave the
+          // job half-acked (rows staged, ledger written, still active).
+          return yield* new JobStore.JobStoreError({
+            message: "FanOut child specs require an explicit request.id"
+          })
+        }
+        const now = yield* Clock.currentTimeMillis
+        const chunks: Array<ReadonlyArray<JobStore.FlowChildSpec>> = []
+        for (let start = 0; start < children.length; start += 500) {
+          chunks.push(children.slice(start, start + 500))
+        }
+        // An empty manifest still needs the final (state-flipping) call.
+        if (chunks.length === 0) chunks.push([])
+        for (let i = 0; i < chunks.length; i++) {
+          const chunk = chunks[i]
+          if (chunk === undefined) continue
+          const items: Array<string> = []
+          for (const child of chunk) {
+            items.push(
+              child.childKey,
+              child.storeKey,
+              child.request.id ?? "",
+              child.request.name,
+              JSON.stringify(child.request)
+            )
+          }
+          const reply: { error?: string; wake?: boolean; queue?: string } = JSON.parse(
+            yield* evalFanOut(
+              prefix,
+              id,
+              token,
+              i === chunks.length - 1 ? "1" : "0",
+              i === 0 ? "1" : "0",
+              failFast ? "1" : "0",
+              children.length,
+              now,
+              chunk.length,
+              items
+            ).pipe(Effect.mapError(storeError("ack failed")))
+          )
+          if (reply.error === "notfound") return yield* new JobStore.JobNotFoundError({ jobId: id })
+          if (reply.error === "locklost") return yield* new JobStore.LockLostError({ jobId: id })
+          if (reply.wake === true) {
+            yield* wakeUp(reply.queue !== undefined ? JobStore.QueueName(reply.queue) : undefined)
+          }
+        }
+      })
+
     // Shared by cancel and cancelByDedupe.
     const cancelJob = (id: JobStore.JobId) =>
       Effect.gen(function*() {
@@ -379,6 +512,7 @@ export const make = (
                 request.keep === undefined ? "" : JSON.stringify(request.keep),
                 request.timeoutMs === undefined ? "" : String(request.timeoutMs),
                 request.trace === undefined ? "" : JSON.stringify(request.trace),
+                request.parent === undefined ? "" : JSON.stringify(request.parent),
                 String(Math.max(0, request.delayMs))
               )
             }
@@ -503,15 +637,20 @@ export const make = (
           })
         }).pipe(Effect.mapError(storeError("claim failed"))),
 
-      ack: (id, token, outcome) =>
-        Effect.gen(function*() {
+      ack: (id, token, outcome) => {
+        if (outcome._tag === "FanOut") {
+          return fanOutAck(id, token, outcome.failFast, outcome.children)
+        }
+        // Narrowed binding: the closure below must see the FanOut-free union.
+        const settled = outcome
+        return Effect.gen(function*() {
           const now = yield* Clock.currentTimeMillis
-          const exitJson = outcome._tag === "Cancelled" || outcome.exit === undefined
+          const exitJson = settled._tag === "Cancelled" || settled.exit === undefined
             ? ""
-            : JSON.stringify(outcome.exit)
-          const delayMs = outcome._tag === "Retry" ? Math.max(0, outcome.delayMs) : 0
+            : JSON.stringify(settled.exit)
+          const delayMs = settled._tag === "Retry" ? Math.max(0, settled.delayMs) : 0
           const reply: { error?: string; wake?: boolean; queue?: string } = JSON.parse(
-            yield* evalAck(prefix, id, token, outcome._tag, exitJson, delayMs, now).pipe(
+            yield* evalAck(prefix, id, token, settled._tag, exitJson, delayMs, now).pipe(
               Effect.mapError(storeError("ack failed"))
             )
           )
@@ -520,7 +659,8 @@ export const make = (
           if (reply.wake === true) {
             yield* wakeUp(reply.queue !== undefined ? JobStore.QueueName(reply.queue) : undefined)
           }
-        }),
+        })
+      },
 
       release: (id, token) =>
         Effect.gen(function*() {
@@ -746,6 +886,7 @@ export const make = (
           schedule.backoff === undefined ? "" : JSON.stringify(schedule.backoff),
           schedule.keep === undefined ? "" : JSON.stringify(schedule.keep),
           schedule.timeoutMs === undefined ? "" : String(schedule.timeoutMs),
+          schedule.group ?? "",
           schedule.nextRunAt
         ).pipe(
           Effect.mapError(storeError("upsertSchedule failed")),
@@ -798,6 +939,7 @@ export const make = (
             request.keep === undefined ? "" : JSON.stringify(request.keep),
             request.timeoutMs === undefined ? "" : String(request.timeoutMs),
             request.trace === undefined ? "" : JSON.stringify(request.trace),
+            request.parent === undefined ? "" : JSON.stringify(request.parent),
             Math.max(0, request.delayMs),
             now
           )) === "1"
@@ -815,6 +957,147 @@ export const make = (
         evalAdvanceSchedule(prefix, key, expectedRunAt, nextRunAt).pipe(
           Effect.mapError(storeError("advanceSchedule failed")),
           Effect.asVoid
+        ),
+
+      recordChildResults: (reports) =>
+        Effect.gen(function*() {
+          if (reports.length === 0) {
+            const none: ReadonlyArray<{ applied: boolean; parentSettled: boolean }> = []
+            return none
+          }
+          const now = yield* Clock.currentTimeMillis
+          const all: Array<{ applied: boolean; parentSettled: boolean }> = []
+          // One atomic batch per chunk of 500 (ARGV headroom); the
+          // contract's per-batch settle semantics then apply per chunk.
+          for (let start = 0; start < reports.length; start += 500) {
+            const chunk = reports.slice(start, start + 500)
+            const items: Array<string> = []
+            for (const report of chunk) {
+              items.push(
+                report.flowId,
+                report.childKey,
+                report.outcome,
+                report.exit === undefined ? "" : JSON.stringify(report.exit),
+                report.failedReason ?? ""
+              )
+            }
+            const reply: {
+              results: ReadonlyArray<{ applied: boolean; parentSettled: boolean }>
+              wakes: ReadonlyArray<string> | Record<string, never>
+            } = JSON.parse(yield* evalRecordChildResults(prefix, now, chunk.length, items))
+            for (const queue of asArray(reply.wakes)) {
+              // A parent settled to runnable collect: wake its queue.
+              yield* wakeUp(JobStore.QueueName(queue))
+            }
+            for (const result of reply.results) {
+              all.push(result)
+            }
+          }
+          return all
+        }).pipe(Effect.mapError(storeError("recordChildResults failed"))),
+
+      peekOutbox: (peekOptions) =>
+        Effect.gen(function*() {
+          const limit = Math.floor(peekOptions.limit)
+          if (limit <= 0) {
+            const none: ReadonlyArray<JobStore.OutboxEntry> = []
+            return none
+          }
+          // The `after` cursor compares by the id's embedded score (the seq
+          // prefix before the NUL), so the walk moves past the named entry
+          // whether or not it still exists. Unparseable input reads as unset.
+          let afterSeq: number | undefined = undefined
+          if (peekOptions.after !== undefined) {
+            const nul = peekOptions.after.indexOf("\u0000")
+            const seq = nul > 0 ? Number(peekOptions.after.slice(0, nul)) : Number.NaN
+            if (Number.isFinite(seq)) afterSeq = seq
+          }
+          const raw = afterSeq === undefined
+            ? yield* redis.send("ZRANGE", `${prefix}:flowoutbox`, "0", String(limit - 1))
+            : yield* redis.send(
+              "ZRANGEBYSCORE",
+              `${prefix}:flowoutbox`,
+              `(${afterSeq}`,
+              "+inf",
+              "LIMIT",
+              "0",
+              String(limit)
+            )
+          // SAFETY: ZRANGE/ZRANGEBYSCORE always reply with arrays of bulk
+          // strings.
+          const members = raw as ReadonlyArray<string>
+          return members.map(toOutboxEntry)
+        }).pipe(Effect.mapError(storeError("peekOutbox failed"))),
+
+      deleteOutbox: (ids) =>
+        Effect.gen(function*() {
+          // Chunked ZREMs keep the variadic argument count bounded; each
+          // chunk is idempotent, so a partial failure just redelivers.
+          for (let start = 0; start < ids.length; start += 500) {
+            yield* redis.send("ZREM", `${prefix}:flowoutbox`, ...ids.slice(start, start + 500))
+          }
+        }).pipe(Effect.mapError(storeError("deleteOutbox failed"))),
+
+      listChildResults: (flowId, listOptions) =>
+        Effect.gen(function*() {
+          const limit = Math.max(1, listOptions?.limit ?? 1000)
+          const reply: { items: ReadonlyArray<ReadonlyArray<string>> | Record<string, never>; more: boolean } = JSON
+            .parse(
+              yield* evalListChildResults(prefix, flowId, listOptions?.cursor ?? "", limit)
+            )
+          const items = asArray(reply.items).map((row) => toChildRecord(flowId, row))
+          const last = items[items.length - 1]
+          return {
+            items,
+            cursor: reply.more && last !== undefined ? last.childKey : undefined
+          }
+        }).pipe(Effect.mapError(storeError("listChildResults failed"))),
+
+      flowSweepWork: (sweepOptions) =>
+        Effect.gen(function*() {
+          const now = yield* Clock.currentTimeMillis
+          const limit = Math.max(1, sweepOptions.limit ?? 1000)
+          const reply: {
+            reconcile:
+              | ReadonlyArray<{
+                flowId: string
+                children: ReadonlyArray<{ childKey: string; storeKey: string; spec: string }>
+              }>
+              | Record<string, never>
+            cascade:
+              | ReadonlyArray<{
+                flowId: string
+                children: ReadonlyArray<{ childKey: string; storeKey: string; childJobId: string }>
+              }>
+              | Record<string, never>
+          } = JSON.parse(yield* evalFlowSweepWork(prefix, sweepOptions.pendingAgeMs, limit, now))
+          const work: JobStore.FlowSweepWork = {
+            reconcile: asArray(reply.reconcile).map((group) => ({
+              flowId: JobStore.JobId(group.flowId),
+              children: group.children.map((child) => {
+                // The stored spec is the verbatim JSON this driver wrote at
+                // fan-out time (never routed through cjson), so it re-parses
+                // to the original EnqueueRequest.
+                const request: JobStore.EnqueueRequest = JSON.parse(child.spec)
+                return { childKey: child.childKey, storeKey: child.storeKey, request }
+              })
+            })),
+            cascade: asArray(reply.cascade).map((group) => ({
+              flowId: JobStore.JobId(group.flowId),
+              children: group.children.map((child) => ({
+                childKey: child.childKey,
+                storeKey: child.storeKey,
+                childJobId: JobStore.JobId(child.childJobId)
+              }))
+            }))
+          }
+          return work
+        }).pipe(Effect.mapError(storeError("flowSweepWork failed"))),
+
+      markChildrenCascaded: (flowId, childKeys) =>
+        evalMarkChildrenCascaded(prefix, flowId, JSON.stringify(childKeys)).pipe(
+          Effect.mapError(storeError("markChildrenCascaded failed")),
+          Effect.asVoid
         )
     }