@furystack/redis-task-runner 1.0.0

package/src/redis-task-runner.integration.spec.ts

@@ -0,0 +1,440 @@
+import { randomUUID } from 'node:crypto'
+import { BlobStore, InMemoryBlobStore } from '@furystack/blob-store'
+import { CrossNodeBus, defineInProcessCrossNodeBus } from '@furystack/cross-node-bus'
+import { createInjector, type Injector } from '@furystack/inject'
+import {
+  defineTaskHandler,
+  TaskRunner,
+  type AnyTaskHandlerDescriptor,
+  type TaskRunnerCapabilities,
+  type Worker,
+} from '@furystack/task-runner'
+import { runTaskToCompletion } from '@furystack/task-runner/testing'
+import { createClient, type RedisClientType } from 'redis'
+import { afterEach, beforeEach, describe, expect, it } from 'vitest'
+import { defineRedisTaskRunner } from './define-redis-task-runner.js'
+
+const REDIS_URL = process.env.REDIS_URL ?? 'redis://localhost:6379'
+
+type Harness = {
+  injector: Injector
+  runner: TaskRunner
+  workers: Worker[]
+  prefix: string
+  client: RedisClientType
+  [Symbol.asyncDispose]: () => Promise<void>
+}
+
+const buildHarness = async (options: {
+  prefix: string
+  workerCount: number
+  handlers: AnyTaskHandlerDescriptor[]
+  visibilityTimeoutMs?: number
+  schedulerIntervalMs?: number
+  concurrencyLimits?: Record<string, number>
+  workerConcurrency?: number
+}): Promise<Harness> => {
+  const client = createClient({ url: REDIS_URL })
+  await client.connect()
+
+  const injector = createInjector()
+  injector.bind(BlobStore, ({ onDispose }) => {
+    const store = new InMemoryBlobStore({ name: 'integration-blobs' })
+    // eslint-disable-next-line furystack/prefer-using-wrapper -- delegated to onDispose
+    onDispose(() => store[Symbol.dispose]())
+    return store
+  })
+  injector.bind(CrossNodeBus, defineInProcessCrossNodeBus())
+
+  injector.bind(
+    TaskRunner,
+    defineRedisTaskRunner({
+      client,
+      serviceName: 'integration',
+      topicPrefix: options.prefix,
+      visibilityTimeoutMs: options.visibilityTimeoutMs ?? 60_000,
+      blockTimeoutMs: 50,
+      retryBackoffMs: 50,
+      schedulerIntervalMs: options.schedulerIntervalMs ?? 50,
+      reconcilerIntervalMs: 200,
+      sweepIntervalMs: 100,
+      concurrencyLimits: options.concurrencyLimits,
+    }),
+  )
+
+  const runner = injector.get(TaskRunner)
+  const workers: Worker[] = []
+  for (let i = 0; i < options.workerCount; i++) {
+    workers.push(
+      runner.registerWorker({
+        name: `worker-${i}`,
+        handlers: options.handlers,
+        concurrency: options.workerConcurrency ?? 2,
+        tags: [],
+        compatibleVersions: {},
+      }),
+    )
+  }
+
+  return {
+    injector,
+    runner,
+    workers,
+    prefix: options.prefix,
+    client,
+    async [Symbol.asyncDispose]() {
+      for (const w of workers) w[Symbol.dispose]()
+      await injector[Symbol.asyncDispose]()
+      if (client.isOpen) await client.quit()
+    },
+  }
+}
+
+const cleanupStreams = async (prefix: string): Promise<void> => {
+  const client = createClient({ url: REDIS_URL })
+  try {
+    await client.connect()
+    const keys = await client.keys(`${prefix}*`)
+    if (keys.length > 0) await client.del(keys)
+  } catch {
+    // Redis unreachable; setup will fail loudly in a per-test path.
+  } finally {
+    if (client.isOpen) await client.quit()
+  }
+}
+
+const sleep = (ms: number): Promise<void> => new Promise((r) => setTimeout(r, ms))
+
+describe('RedisTaskRunner (integration)', () => {
+  let prefix: string
+
+  beforeEach(() => {
+    prefix = `rtr-${randomUUID().slice(0, 8)}/`
+  })
+
+  afterEach(async () => {
+    await cleanupStreams(prefix)
+  })
+
+  it('declares persistent + delayedDispatch=true capabilities', async () => {
+    await using harness = await buildHarness({ prefix, workerCount: 0, handlers: [] })
+    const caps: TaskRunnerCapabilities = harness.runner.capabilities
+    expect(caps.persistent).toBe(true)
+    expect(caps.delayedDispatch).toBe(true)
+    expect(caps.fleetCapEnforcement).toBe(true)
+  })
+
+  it('honors notBefore by parking the task in the scheduler ZSET until it is due', async () => {
+    const ranAt: number[] = []
+    const stamper = defineTaskHandler<Record<string, never>, void>({
+      type: 'stamp',
+      version: 1,
+      handler: async (ctx) => {
+        ranAt.push(ctx.now().getTime())
+      },
+    })
+
+    await using harness = await buildHarness({ prefix, workerCount: 1, handlers: [stamper] })
+    const submittedAt = Date.now()
+    const dueAt = submittedAt + 400
+    const task = await harness.runner.submit({
+      type: 'stamp',
+      payload: {},
+      handlerVersion: 1,
+      notBefore: new Date(dueAt),
+    })
+
+    // Task should still be 'pending' shortly after submit and the
+    // queue stream should be empty — payload is parked in the
+    // scheduler ZSET. Allow a small tolerance for clock drift.
+    await sleep(120)
+    const streamLen = await harness.client.xLen(`${prefix}tasks:queue:stamp:v1`).catch(() => 0)
+    expect(streamLen).toBe(0)
+    const schedSize = await harness.client.zCard(`${prefix}tasks:scheduler`).catch(() => 0)
+    expect(schedSize).toBe(1)
+
+    const completed = await runTaskToCompletion({ runner: harness.runner, taskId: task.id, timeoutMs: 5000 })
+    expect(completed.status).toBe('succeeded')
+    expect(ranAt).toHaveLength(1)
+    // First and only run should not have started before the due time.
+    expect(ranAt[0]).toBeGreaterThanOrEqual(dueAt - 50)
+  }, 15_000)
+
+  it('submit → claim → handler runs → terminal succeeded (happy path)', async () => {
+    const echo = defineTaskHandler<{ value: string }, { echoed: string }>({
+      type: 'echo',
+      version: 1,
+      handler: async (_ctx, payload) => ({ echoed: payload.value }),
+    })
+
+    await using harness = await buildHarness({ prefix, workerCount: 1, handlers: [echo] })
+    const task = await harness.runner.submit({
+      type: 'echo',
+      payload: { value: 'hello' },
+      handlerVersion: 1,
+    })
+
+    const completed = await runTaskToCompletion({ runner: harness.runner, taskId: task.id, timeoutMs: 5000 })
+    expect(completed.status).toBe('succeeded')
+    expect(completed.result).toEqual({ echoed: 'hello' })
+  }, 15_000)
+
+  it('two workers do not double-execute the same task across the fleet', async () => {
+    const runs = new Set<string>()
+    const counter = defineTaskHandler<{ n: number }, { ran: boolean }>({
+      type: 'count-once',
+      version: 1,
+      handler: async (ctx) => {
+        if (runs.has(ctx.taskId)) {
+          throw new Error(`Task ${ctx.taskId} ran twice`)
+        }
+        runs.add(ctx.taskId)
+        return { ran: true }
+      },
+    })
+
+    await using harness = await buildHarness({ prefix, workerCount: 2, handlers: [counter] })
+    const tasks = await Promise.all(
+      Array.from({ length: 8 }, (_, i) =>
+        harness.runner.submit({ type: 'count-once', payload: { n: i }, handlerVersion: 1 }),
+      ),
+    )
+
+    for (const task of tasks) {
+      const completed = await runTaskToCompletion({ runner: harness.runner, taskId: task.id, timeoutMs: 8000 })
+      expect(completed.status).toBe('succeeded')
+    }
+    expect(runs.size).toBe(8)
+  }, 30_000)
+
+  it('reclaims a stalled handler via XAUTOCLAIM and runs it on another consumer', async () => {
+    const seenAttempts: number[] = []
+
+    const flaky = defineTaskHandler<Record<string, never>, { ok: true }>({
+      type: 'flaky-handler',
+      version: 1,
+      visibilityTimeoutMs: 250,
+      retryPolicy: { maxAttempts: 1, backoff: 'none', baseDelayMs: 0, jitter: 0 },
+      handler: async (ctx) => {
+        seenAttempts.push(ctx.attempt)
+        if (ctx.attempt === 1) {
+          // Stall until our cancellation signal fires (the runner core
+          // aborts the prior AC when the broker-side reclaim hands the
+          // task to a fresh consumer). The throw lets the runner core
+          // unwind cleanly and recognise the AC was replaced.
+          await new Promise<void>((_, reject) => {
+            ctx.cancellationSignal.addEventListener('abort', () => reject(new Error('reclaimed')), { once: true })
+          })
+        }
+        return { ok: true }
+      },
+    })
+
+    await using harness = await buildHarness({
+      prefix,
+      workerCount: 2,
+      handlers: [flaky],
+      visibilityTimeoutMs: 250,
+    })
+
+    const task = await harness.runner.submit({
+      type: 'flaky-handler',
+      payload: {},
+      handlerVersion: 1,
+    })
+
+    const deadline = Date.now() + 10_000
+    while (Date.now() < deadline) {
+      const t = await harness.runner.get(task.id)
+      if (t?.status === 'succeeded') break
+      await sleep(100)
+    }
+    const final = await harness.runner.get(task.id)
+    expect(final?.status).toBe('succeeded')
+    expect(seenAttempts.length).toBeGreaterThanOrEqual(2)
+    expect(seenAttempts[0]).toBe(1)
+  }, 30_000)
+
+  it('draft does not enqueue; start releases to the queue', async () => {
+    let observed = 0
+    const observe = defineTaskHandler<Record<string, never>, void>({
+      type: 'observe',
+      version: 1,
+      handler: async () => {
+        observed += 1
+      },
+    })
+
+    await using harness = await buildHarness({ prefix, workerCount: 1, handlers: [observe] })
+
+    const drafted = await harness.runner.draft({ type: 'observe', payload: {}, handlerVersion: 1 })
+    expect(drafted.status).toBe('draft')
+
+    // Stream should not exist (or have zero entries) before start.
+    const streamKey = `${prefix}tasks:queue:observe:v1`
+    const lenBefore = await harness.client.xLen(streamKey).catch(() => 0)
+    expect(lenBefore).toBe(0)
+    expect(observed).toBe(0)
+
+    await harness.runner.start(drafted.id)
+    const completed = await runTaskToCompletion({ runner: harness.runner, taskId: drafted.id, timeoutMs: 5000 })
+    expect(completed.status).toBe('succeeded')
+    expect(observed).toBe(1)
+  }, 15_000)
+
+  it('cancel aborts a running handler via the bus broadcast', async () => {
+    let aborted = false
+    const long = defineTaskHandler<Record<string, never>, void>({
+      type: 'long-running',
+      version: 1,
+      handler: async (ctx) => {
+        try {
+          await ctx.sleep(60_000)
+        } catch {
+          aborted = true
+          throw ctx.cancellationSignal.reason instanceof Error ? ctx.cancellationSignal.reason : new Error('cancelled')
+        }
+      },
+    })
+
+    await using harness = await buildHarness({ prefix, workerCount: 1, handlers: [long] })
+    const task = await harness.runner.submit({ type: 'long-running', payload: {}, handlerVersion: 1 })
+
+    await sleep(300)
+    await harness.runner.cancel(task.id, 'integration test')
+
+    const deadline = Date.now() + 5000
+    while (Date.now() < deadline) {
+      const t = await harness.runner.get(task.id)
+      if (t?.status === 'cancelled') break
+      await sleep(50)
+    }
+    const final = await harness.runner.get(task.id)
+    expect(final?.status).toBe('cancelled')
+    expect(aborted).toBe(true)
+  }, 15_000)
+
+  it('enforces a fleet-wide concurrency cap across two workers', async () => {
+    let live = 0
+    let maxLive = 0
+    const capped = defineTaskHandler<Record<string, never>, void>({
+      type: 'capped',
+      version: 1,
+      handler: async (ctx) => {
+        live += 1
+        maxLive = Math.max(maxLive, live)
+        try {
+          await ctx.sleep(300)
+        } finally {
+          live -= 1
+        }
+      },
+    })
+
+    // 2 workers × concurrency 2 = 4 fleet slots, but the cap is 2 — so at
+    // most two `capped` handlers may run at once across the whole fleet.
+    await using harness = await buildHarness({
+      prefix,
+      workerCount: 2,
+      workerConcurrency: 2,
+      handlers: [capped],
+      concurrencyLimits: { capped: 2 },
+    })
+
+    const tasks = await Promise.all(
+      Array.from({ length: 6 }, () => harness.runner.submit({ type: 'capped', payload: {}, handlerVersion: 1 })),
+    )
+
+    for (const task of tasks) {
+      const completed = await runTaskToCompletion({ runner: harness.runner, taskId: task.id, timeoutMs: 15_000 })
+      expect(completed.status).toBe('succeeded')
+    }
+
+    // Cap holds (never exceeded) and is actually exercised (parallelism
+    // reached the cap rather than serializing).
+    expect(maxLive).toBeLessThanOrEqual(2)
+    expect(maxLive).toBe(2)
+  }, 30_000)
+
+  it('drains a worker: in-flight task finishes, no new tasks are claimed', async () => {
+    let started = 0
+    let finished = 0
+    let resolveStarted: () => void = () => {}
+    const startedSignal = new Promise<void>((resolve) => {
+      resolveStarted = resolve
+    })
+
+    const slow = defineTaskHandler<Record<string, never>, void>({
+      type: 'drainable',
+      version: 1,
+      handler: async (ctx) => {
+        started += 1
+        resolveStarted()
+        await ctx.sleep(600)
+        finished += 1
+      },
+    })
+
+    await using harness = await buildHarness({
+      prefix,
+      workerCount: 1,
+      workerConcurrency: 1,
+      handlers: [slow],
+    })
+
+    const inFlight = await harness.runner.submit({ type: 'drainable', payload: {}, handlerVersion: 1 })
+    await startedSignal
+
+    // Begin draining while the first task is mid-handler, then submit a
+    // second task that the draining worker must not pick up.
+    const drainPromise = harness.workers[0]?.drain({ timeoutMs: 5000 }) ?? Promise.resolve()
+    const queued = await harness.runner.submit({ type: 'drainable', payload: {}, handlerVersion: 1 })
+
+    await drainPromise
+
+    // The in-flight task completed despite the drain.
+    const inFlightFinal = await harness.runner.get(inFlight.id)
+    expect(inFlightFinal?.status).toBe('succeeded')
+    expect(finished).toBe(1)
+
+    // The second task was never claimed by the drained worker.
+    await sleep(400)
+    const queuedFinal = await harness.runner.get(queued.id)
+    expect(queuedFinal?.status).toBe('pending')
+    expect(started).toBe(1)
+  }, 20_000)
+
+  it('reaps a child spawned-without-await when the parent succeeds (orphan reaping)', async () => {
+    const blocker = defineTaskHandler<Record<string, never>, void>({
+      type: 'reap-blocker',
+      version: 1,
+      handler: async (ctx) => {
+        await ctx.sleep(60_000)
+      },
+    })
+    const spawnAndForget = defineTaskHandler<Record<string, never>, { done: true }>({
+      type: 'reap-parent',
+      version: 1,
+      handler: async (ctx) => {
+        await ctx.spawnChild<Record<string, never>, void>('reap-blocker', {})
+        return { done: true }
+      },
+    })
+
+    await using harness = await buildHarness({
+      prefix,
+      workerCount: 1,
+      workerConcurrency: 2,
+      handlers: [blocker, spawnAndForget],
+    })
+
+    const task = await harness.runner.submit({ type: 'reap-parent', payload: {}, handlerVersion: 1 })
+    const parent = await runTaskToCompletion({ runner: harness.runner, taskId: task.id, timeoutMs: 8000 })
+    expect(parent.status).toBe('succeeded')
+
+    const childId = parent.childTaskIds[0]
+    expect(childId).toBeTruthy()
+    const child = await runTaskToCompletion({ runner: harness.runner, taskId: childId, timeoutMs: 8000 })
+    expect(child.status).toBe('cancelled')
+  }, 30_000)
+})

package/src/redis-task-runner.ts

@@ -0,0 +1,67 @@
+import type { BlobStore } from '@furystack/blob-store'
+import type { CrossNodeBus } from '@furystack/cross-node-bus'
+import type { Injector } from '@furystack/inject'
+import type { DataSet } from '@furystack/repository'
+import type { Task, TaskReplayLogEntry } from '@furystack/task-runner'
+import { TaskRunnerCore, type TaskRunnerCoreOptions } from '@furystack/task-runner'
+import type { TaskRunnerTelemetry } from '@furystack/task-runner'
+import { RedisQueueAdapter, type RedisQueueAdapterOptions } from './redis-queue-adapter.js'
+
+export type RedisTaskRunnerOptions = TaskRunnerCoreOptions
+
+export type RedisTaskRunnerDeps = Omit<RedisQueueAdapterOptions, 'client' | 'serviceName'> & {
+  injector: Injector
+  bus: CrossNodeBus
+  blobStore: BlobStore
+  taskDs: DataSet<Task, 'id'>
+  replayDs: DataSet<TaskReplayLogEntry, 'id'>
+  telemetry: TaskRunnerTelemetry
+  client: RedisQueueAdapterOptions['client']
+  serviceName: string
+}
+
+/**
+ * Redis-Streams-backed {@link TaskRunnerCore}. Uses
+ * `@furystack/redis-task-runner` consumer-group claim plumbing while
+ * inheriting all task lifecycle, replay, retry, cancel-cascade, and
+ * telemetry behavior from the core class. Apps bind it via
+ * {@link defineRedisTaskRunner} so the adapter shares lifetime with
+ * the surrounding injector scope.
+ */
+export class RedisTaskRunner extends TaskRunnerCore {
+  readonly #queueAdapter: RedisQueueAdapter
+
+  constructor(deps: RedisTaskRunnerDeps, options?: RedisTaskRunnerOptions) {
+    const queueAdapter = new RedisQueueAdapter({
+      client: deps.client,
+      serviceName: deps.serviceName,
+      topicPrefix: deps.topicPrefix,
+      consumerGroup: deps.consumerGroup,
+      visibilityTimeoutMs: deps.visibilityTimeoutMs,
+      visibilityTimeoutByType: deps.visibilityTimeoutByType,
+      blockTimeoutMs: deps.blockTimeoutMs,
+      retryBackoffMs: deps.retryBackoffMs,
+      idempotencyTtlSec: deps.idempotencyTtlSec,
+      schedulerIntervalMs: deps.schedulerIntervalMs,
+      concurrencyLimits: deps.concurrencyLimits,
+    })
+    super(
+      {
+        injector: deps.injector,
+        bus: deps.bus,
+        blobStore: deps.blobStore,
+        taskDs: deps.taskDs,
+        replayDs: deps.replayDs,
+        telemetry: deps.telemetry,
+        queueAdapter,
+      },
+      options,
+    )
+    this.#queueAdapter = queueAdapter
+  }
+
+  public override [Symbol.dispose](): void {
+    super[Symbol.dispose]()
+    this.#queueAdapter[Symbol.dispose]()
+  }
+}

package/src/redis-task-runner.unit.spec.ts

@@ -0,0 +1,110 @@
+import { BlobStore, InMemoryBlobStore } from '@furystack/blob-store'
+import { CrossNodeBus, defineInProcessCrossNodeBus } from '@furystack/cross-node-bus'
+import { createInjector } from '@furystack/inject'
+import { TaskRunner } from '@furystack/task-runner'
+import { usingAsync } from '@furystack/utils'
+import { describe, expect, it, vi } from 'vitest'
+import { defineRedisTaskRunner } from './define-redis-task-runner.js'
+import { RedisTaskRunner } from './redis-task-runner.js'
+
+type RedisClientArg = Parameters<typeof defineRedisTaskRunner>[0]['client']
+
+/**
+ * Minimal stand-in for the `redis` client. The adapter only touches the
+ * client lazily (it `duplicate()`s + `connect()`s a read client on
+ * construction and issues stream commands on enqueue/subscribe), so a
+ * service-less unit run can exercise the runner end to end without a
+ * live broker. `duplicate()` returns the same mock so dispose assertions
+ * observe the read client teardown.
+ */
+const buildMockClient = () => {
+  const client = {
+    xAdd: vi.fn().mockResolvedValue('1-0'),
+    xGroupCreate: vi.fn().mockResolvedValue('OK'),
+    xReadGroup: vi.fn().mockResolvedValue(null),
+    xAutoClaim: vi.fn().mockResolvedValue({ nextId: '0-0', messages: [] }),
+    xAck: vi.fn().mockResolvedValue(1),
+    xClaimJustId: vi.fn().mockResolvedValue([]),
+    set: vi.fn().mockResolvedValue('OK'),
+    get: vi.fn().mockResolvedValue(null),
+    zAdd: vi.fn().mockResolvedValue(1),
+    zRem: vi.fn().mockResolvedValue(1),
+    eval: vi.fn().mockResolvedValue(0),
+    isOpen: true,
+    destroy: vi.fn(),
+    duplicate: vi.fn(),
+    connect: vi.fn(),
+  }
+  client.duplicate = vi.fn(() => client)
+  client.connect = vi.fn().mockResolvedValue(client)
+  return client
+}
+
+const bindRunner = (injector: ReturnType<typeof createInjector>, client: ReturnType<typeof buildMockClient>): void => {
+  injector.bind(BlobStore, ({ onDispose }) => {
+    const store = new InMemoryBlobStore({ name: 'unit-blobs' })
+    // eslint-disable-next-line furystack/prefer-using-wrapper -- delegated to onDispose
+    onDispose(() => store[Symbol.dispose]())
+    return store
+  })
+  injector.bind(CrossNodeBus, defineInProcessCrossNodeBus())
+  injector.bind(
+    TaskRunner,
+    defineRedisTaskRunner({
+      client: client as unknown as RedisClientArg,
+      serviceName: 'unit',
+      topicPrefix: 'unit/',
+      // Park the background timers so the unit run does not tick the
+      // reconciler/scheduler against the mock broker.
+      reconcilerIntervalMs: 1_000_000,
+      sweepIntervalMs: 1_000_000,
+      schedulerIntervalMs: 1_000_000,
+    }),
+  )
+}
+
+describe('@furystack/redis-task-runner', () => {
+  describe('defineRedisTaskRunner', () => {
+    it('resolves a RedisTaskRunner with Redis-backed capabilities', async () => {
+      await usingAsync(createInjector(), async (injector) => {
+        bindRunner(injector, buildMockClient())
+        const runner = injector.get(TaskRunner)
+
+        expect(runner).toBeInstanceOf(RedisTaskRunner)
+        expect(runner.capabilities.persistent).toBe(true)
+        expect(runner.capabilities.fleetCapEnforcement).toBe(true)
+        expect(runner.capabilities.delayedDispatch).toBe(true)
+        expect(runner.capabilities.maxPayloadBytes).toBe(Infinity)
+      })
+    })
+
+    it('tears down the duplicated read client when the owning injector is disposed', async () => {
+      const client = buildMockClient()
+      await usingAsync(createInjector(), async (injector) => {
+        bindRunner(injector, client)
+        injector.get(TaskRunner)
+      })
+
+      // duplicate() is called once for the dedicated read connection, which
+      // is force-closed via destroy() on dispose.
+      expect(client.duplicate).toHaveBeenCalledTimes(1)
+      expect(client.destroy).toHaveBeenCalledTimes(1)
+    })
+
+    it('routes submit() onto the per-(type,version) Redis stream', async () => {
+      const client = buildMockClient()
+      await usingAsync(createInjector(), async (injector) => {
+        bindRunner(injector, client)
+        const runner = injector.get(TaskRunner)
+
+        const task = await runner.submit({ type: 'echo', payload: { value: 'hi' }, handlerVersion: 1 })
+
+        expect(client.xAdd).toHaveBeenCalledWith('unit/tasks:queue:echo:v1', '*', {
+          taskId: task.id,
+          type: 'echo',
+          handlerVersion: '1',
+        })
+      })
+    })
+  })
+})