@prsm/queue 3.0.1 → 3.0.3

package/README.md

@@ -130,6 +130,27 @@ await queue.push({ action: 'sync' }, { group: 'tenant-456' })
 
 Each tenant processes independently. One slow tenant won't block others. Total concurrent tasks across all tenants is capped by `concurrency`. When the group is conditional, just omit the option - no branching needed.
 
+Groups are fully distributed - any instance can push to any group, and any instance with available concurrency will automatically discover and process tasks for that group. New groups are announced via Redis pub/sub, and existing groups are discovered at startup.
+
+## Push and Wait
+
+Push a task and wait for its result. Works across instances - instance A can push a task that instance B processes, and the result comes back to instance A via Redis pub/sub.
+
+```js
+const result = await queue.pushAndWait({ prompt: 'summarize this' })
+```
+
+With timeout and groups:
+
+```js
+const result = await queue.pushAndWait(
+  { prompt: 'summarize this' },
+  { group: 'tenant-123', timeout: '30s' }
+)
+```
+
+Throws if the task fails (after retries are exhausted) or if the timeout is reached. Retries are transparent - if the handler fails twice then succeeds on the third attempt, `pushAndWait` resolves with the successful result.
+
 ## Events
 
 ```js
@@ -176,11 +197,48 @@ app.post('/api/generate', async (req, res) => {
 
 Each tenant gets up to 2 concurrent LLM calls with a 50ms pause between them. Total concurrent calls across all tenants is capped at 20, protecting your server and API budget from any single tenant overwhelming the system.
 
-## WebSocket Integration with [mesh](https://github.com/nvms/mesh)
+## Fan-out with Groups
 
-Queue events are local-only - only the server that processes a task emits `complete`/`failed`. Use [@prsm/realtime](https://github.com/nvms/realtime) to push results to connected clients in real time.
+Use groups to fan out a single event to multiple independent handlers. Each group processes and retries independently.
 
-Send results to a specific client:
+```js
+const queue = new Queue({
+  concurrency: 10,
+  groups: { concurrency: 1 },
+})
+
+const handlers = {
+  "email": {
+    "user:created": (data) => sendWelcomeEmail(data.email),
+    "user:deleted": (data) => sendGoodbyeEmail(data.email),
+  },
+  "workspace": {
+    "user:created": (data) => createDefaultWorkspace(data.userId),
+  },
+  "slack": {
+    "user:created": (data) => notifySlack(`new user: ${data.email}`),
+  },
+}
+
+queue.process(async ({ event, data }, task) => {
+  await handlers[task.group]?.[event]?.(data)
+})
+
+await queue.ready()
+
+// emit to all groups
+await Promise.all(
+  Object.keys(handlers).map((group) =>
+    queue.push({ event: "user:created", data: { userId: "u1", email: "a@b.com" } }, { group })
+  )
+)
+```
+
+The `handlers` object is both the registry and the routing logic. Adding a new subscriber is adding a key.
+
+## WebSocket Integration
+
+Queue events are local-only - only the server that processes a task emits `complete`/`failed`. Use [@prsm/realtime](https://github.com/nvms/realtime) to push results to connected clients in real time.
 
 ```js
 import Queue from '@prsm/queue'
@@ -213,7 +271,7 @@ await queue.ready()
 await realtime.listen(8080)
 ```
 
-Both queue and realtime use the same Redis instance. No key conflicts (`queue:*` vs `mesh:*`).
+Both queue and realtime use the same Redis instance. No key conflicts (`queue:*` vs `rt:*`).
 
 ## Horizontal Scaling
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@prsm/queue",
-  "version": "3.0.1",
+  "version": "3.0.3",
   "description": "Redis-backed distributed task queue with grouped concurrency, retries, and rate limiting",
   "type": "module",
   "exports": {

package/src/queue.js

@@ -131,6 +131,8 @@ export default class Queue extends EventEmitter {
 
     this._redis = createClient(this._options.redisOptions)
     this._redis.on("error", () => {})
+    this._subClient = null
+    this._groupNotifyClient = null
     this._readyPromise = this._initialize()
   }
 
@@ -175,72 +177,109 @@ export default class Queue extends EventEmitter {
    * @param {{ group?: string, timeout?: number|string }} [options]
    * @returns {Promise<any>}
    */
-  pushAndWait(payload, { group, timeout = 0 } = {}) {
-    if (this._closed) return Promise.reject(new Error("Queue is closed"))
+  async pushAndWait(payload, { group, timeout = 0 } = {}) {
+    if (this._closed) throw new Error("Queue is closed")
     const task = group
       ? { uuid: randomUUID(), payload, createdAt: Date.now(), group, attempts: 0 }
       : { uuid: randomUUID(), payload, createdAt: Date.now(), attempts: 0 }
     this._pushed++
-    const result = this._awaitTask(task.uuid, timeout)
-    result.catch(() => {})
-    return this._enqueue(task, group).then(() => {
-      this.emit("new", { task })
-      return result
-    }, (err) => {
+    const { promise, ready } = this._awaitTask(task.uuid, timeout)
+    promise.catch(() => {})
+    await ready
+    try {
+      await this._enqueue(task, group)
+    } catch (err) {
       this._pushed--
       throw err
-    })
+    }
+    this.emit("new", { task })
+    return promise
   }
 
   /** @private */
   async _enqueue(task, group) {
     if (group) {
       await this._redis.lPush(`queue:groups:${group}`, JSON.stringify(task))
-      if (!this._groupWorkers.has(group)) {
-        this._groupWorkers.set(group, new Map())
-        this._groupInFlight.set(group, 0)
-        await this._startGroupWorkers(group)
-      }
+      await this._ensureGroupWorkers(group)
+      this._redis.publish("queue:group:notify", group).catch(() => {})
     } else {
       await this._redis.lPush("queue:tasks", JSON.stringify(task))
     }
   }
 
+  /** @private */
+  async _ensureGroupWorkers(group) {
+    if (this._groupWorkers.has(group) || this._closed || this._options.concurrency === 0) return
+    this._groupWorkers.set(group, new Map())
+    this._groupInFlight.set(group, 0)
+    await this._startGroupWorkers(group)
+  }
+
+  /** @private */
+  async _ensureSubClient() {
+    if (this._subClient) return this._subClient
+    this._subClient = this._redis.duplicate()
+    this._subClient.on("error", () => {})
+    await this._subClient.connect()
+    return this._subClient
+  }
+
   /** @private */
   _awaitTask(uuid, timeout = 0) {
     const ms_ = ms(timeout)
-    return new Promise((resolve, reject) => {
+    const channel = `queue:result:${uuid}`
+    let resolveReady
+
+    const ready = new Promise((r) => { resolveReady = r })
+
+    const promise = new Promise((resolve, reject) => {
       let timer
+      let settled = false
 
-      const onComplete = ({ task, result }) => {
-        if (task.uuid !== uuid) return
+      const settle = (fn, value) => {
+        if (settled) return
+        settled = true
         cleanup()
-        resolve(result)
+        fn(value)
       }
 
-      const onFailed = ({ task, error }) => {
+      const onLocal = (event) => ({ task, result, error }) => {
         if (task.uuid !== uuid) return
-        cleanup()
-        reject(error)
+        if (event === "complete") settle(resolve, result)
+        else settle(reject, error)
       }
 
+      const onComplete = onLocal("complete")
+      const onFailed = onLocal("failed")
+
       const cleanup = () => {
         if (timer) clearTimeout(timer)
         this.off("complete", onComplete)
         this.off("failed", onFailed)
+        this._subClient?.unsubscribe(channel).catch(() => {})
       }
 
       if (ms_ > 0) {
-        timer = setTimeout(() => {
-          cleanup()
-          reject(new Error("pushAndWait timed out"))
-        }, ms_)
+        timer = setTimeout(() => settle(reject, new Error("pushAndWait timed out")), ms_)
         timer.unref?.()
       }
 
       this.on("complete", onComplete)
       this.on("failed", onFailed)
+
+      this._ensureSubClient().then((sub) => {
+        if (settled) { resolveReady(); return }
+        sub.subscribe(channel, (message) => {
+          try {
+            const { status, result, error } = JSON.parse(message)
+            if (status === "complete") settle(resolve, result)
+            else settle(reject, error ? Object.assign(new Error(error.message), error) : new Error("Task failed"))
+          } catch {}
+        }).then(() => resolveReady()).catch(() => resolveReady())
+      }).catch(() => resolveReady())
     })
+
+    return { promise, ready }
   }
 
   /** @returns {Promise<void>} */
@@ -281,18 +320,44 @@ export default class Queue extends EventEmitter {
       if (client.isOpen) await client.disconnect()
     }
     this._workerClients = []
+    if (this._groupNotifyClient?.isOpen) await this._groupNotifyClient.unsubscribe().catch(() => {})
+    if (this._groupNotifyClient?.isOpen) await this._groupNotifyClient.disconnect().catch(() => {})
+    this._groupNotifyClient = null
+    if (this._subClient?.isOpen) await this._subClient.disconnect().catch(() => {})
+    this._subClient = null
     if (this._redis.isOpen) await this._redis.quit()
   }
 
   async _initialize() {
     await this._redis.connect()
     await this._startWorkers()
+    if (this._options.concurrency > 0) {
+      await this._subscribeToGroupNotifications()
+      await this._discoverExistingGroups()
+    }
     if (this._options.cleanupInterval > 0) {
       this._cleanupTimer = setInterval(() => this._periodicCleanup(), this._options.cleanupInterval)
       this._cleanupTimer.unref()
     }
   }
 
+  async _subscribeToGroupNotifications() {
+    this._groupNotifyClient = this._redis.duplicate()
+    this._groupNotifyClient.on("error", () => {})
+    await this._groupNotifyClient.connect()
+    await this._groupNotifyClient.subscribe("queue:group:notify", (group) => {
+      this._ensureGroupWorkers(group)
+    })
+  }
+
+  async _discoverExistingGroups() {
+    const keys = await this._redis.keys("queue:groups:*")
+    for (const key of keys) {
+      const group = key.slice("queue:groups:".length)
+      await this._ensureGroupWorkers(group)
+    }
+  }
+
   async _createWorkerClient() {
     const client = this._redis.duplicate()
     client.on("error", () => {})
@@ -467,6 +532,7 @@ export default class Queue extends EventEmitter {
 
     if (succeeded) {
       this._settle()
+      this._publishResult(task.uuid, { status: "complete", result })
       try { this.emit("complete", { task, result }) } finally { this._emitDrain() }
     } else if (task.attempts < opts.maxRetries && !this._closed) {
       let retried = false
@@ -479,14 +545,22 @@ export default class Queue extends EventEmitter {
         this.emit("retry", { task, error: handlerError, attempt: task.attempts })
       } else {
         this._settle()
+        this._publishResult(task.uuid, { status: "failed", error: { message: handlerError?.message, code: handlerError?.code, name: handlerError?.name } })
         try { this.emit("failed", { task, error: handlerError }) } finally { this._emitDrain() }
       }
     } else {
       this._settle()
+      this._publishResult(task.uuid, { status: "failed", error: { message: handlerError?.message, code: handlerError?.code, name: handlerError?.name } })
       try { this.emit("failed", { task, error: handlerError }) } finally { this._emitDrain() }
     }
   }
 
+  _publishResult(uuid, payload) {
+    if (!this._redis.isOpen) return
+    const channel = `queue:result:${uuid}`
+    this._redis.publish(channel, JSON.stringify(payload)).catch(() => {})
+  }
+
   _settle() {
     this._inFlight--
     this._totalSettled++
@@ -514,6 +588,7 @@ export default class Queue extends EventEmitter {
         }
         this._groupInFlight.delete(groupKey)
       }
+      this._workerClients = this._workerClients.filter((c) => c.isOpen)
     } catch {}
   }
 }