@prsm/queue 3.0.0 → 3.0.2

package/README.md

@@ -130,6 +130,27 @@ await queue.push({ action: 'sync' }, { group: 'tenant-456' })
 
 Each tenant processes independently. One slow tenant won't block others. Total concurrent tasks across all tenants is capped by `concurrency`. When the group is conditional, just omit the option - no branching needed.
 
+Groups are fully distributed - any instance can push to any group, and any instance with available concurrency will automatically discover and process tasks for that group. New groups are announced via Redis pub/sub, and existing groups are discovered at startup.
+
+## Push and Wait
+
+Push a task and wait for its result. Works across instances - instance A can push a task that instance B processes, and the result comes back to instance A via Redis pub/sub.
+
+```js
+const result = await queue.pushAndWait({ prompt: 'summarize this' })
+```
+
+With timeout and groups:
+
+```js
+const result = await queue.pushAndWait(
+  { prompt: 'summarize this' },
+  { group: 'tenant-123', timeout: '30s' }
+)
+```
+
+Throws if the task fails (after retries are exhausted) or if the timeout is reached. Retries are transparent - if the handler fails twice then succeeds on the third attempt, `pushAndWait` resolves with the successful result.
+
 ## Events
 
 ```js
@@ -147,7 +168,7 @@ queue.on('drain', () => {})
   uuid: string,
   payload: any,
   createdAt: number,
-  groupKey?: string,  // present when pushed with { group }
+  group?: string,     // present when pushed with { group }
   attempts: number
 }
 ```
@@ -176,17 +197,54 @@ app.post('/api/generate', async (req, res) => {
 
 Each tenant gets up to 2 concurrent LLM calls with a 50ms pause between them. Total concurrent calls across all tenants is capped at 20, protecting your server and API budget from any single tenant overwhelming the system.
 
-## WebSocket Integration with [mesh](https://github.com/nvms/mesh)
+## Fan-out with Groups
+
+Use groups to fan out a single event to multiple independent handlers. Each group processes and retries independently.
+
+```js
+const queue = new Queue({
+  concurrency: 10,
+  groups: { concurrency: 1 },
+})
+
+const handlers = {
+  "email": {
+    "user:created": (data) => sendWelcomeEmail(data.email),
+    "user:deleted": (data) => sendGoodbyeEmail(data.email),
+  },
+  "workspace": {
+    "user:created": (data) => createDefaultWorkspace(data.userId),
+  },
+  "slack": {
+    "user:created": (data) => notifySlack(`new user: ${data.email}`),
+  },
+}
+
+queue.process(async ({ event, data }, task) => {
+  await handlers[task.group]?.[event]?.(data)
+})
+
+await queue.ready()
+
+// emit to all groups
+await Promise.all(
+  Object.keys(handlers).map((group) =>
+    queue.push({ event: "user:created", data: { userId: "u1", email: "a@b.com" } }, { group })
+  )
+)
+```
+
+The `handlers` object is both the registry and the routing logic. Adding a new subscriber is adding a key.
 
-Queue events are local-only - only the server that processes a task emits `complete`/`failed`. Use [mesh](https://github.com/nvms/mesh) to push results to connected clients in real time.
+## WebSocket Integration
 
-Send results to a specific client:
+Queue events are local-only - only the server that processes a task emits `complete`/`failed`. Use [@prsm/realtime](https://github.com/nvms/realtime) to push results to connected clients in real time.
 
 ```js
 import Queue from '@prsm/queue'
-import { MeshServer } from '@mesh-kit/server'
+import { RealtimeServer } from '@prsm/realtime'
 
-const mesh = new MeshServer({ redis: { host: 'localhost', port: 6379 } })
+const realtime = new RealtimeServer({ redis: { host: 'localhost', port: 6379 } })
 const queue = new Queue({ concurrency: 5, groups: { concurrency: 1 } })
 
 queue.process(async (payload) => {
@@ -194,14 +252,14 @@ queue.process(async (payload) => {
 })
 
 queue.on('complete', ({ task, result }) => {
-  mesh.sendTo(task.payload.connectionId, 'job:complete', result)
+  realtime.sendTo(task.payload.connectionId, 'job:complete', result)
 })
 
 queue.on('failed', ({ task, error }) => {
-  mesh.sendTo(task.payload.connectionId, 'job:failed', { error: error.message })
+  realtime.sendTo(task.payload.connectionId, 'job:failed', { error: error.message })
 })
 
-mesh.exposeCommand('generate-report', async (ctx) => {
+realtime.exposeCommand('generate-report', async (ctx) => {
   const taskId = await queue.push({
     connectionId: ctx.connection.id,
     ...ctx.payload,
@@ -210,10 +268,10 @@ mesh.exposeCommand('generate-report', async (ctx) => {
 })
 
 await queue.ready()
-await mesh.listen(8080)
+await realtime.listen(8080)
 ```
 
-Both queue and mesh use the same Redis instance. No key conflicts (`queue:*` vs `mesh:*`).
+Both queue and realtime use the same Redis instance. No key conflicts (`queue:*` vs `rt:*`).
 
 ## Horizontal Scaling
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@prsm/queue",
-  "version": "3.0.0",
+  "version": "3.0.2",
   "description": "Redis-backed distributed task queue with grouped concurrency, retries, and rate limiting",
   "type": "module",
   "exports": {

package/src/queue.js

@@ -20,7 +20,7 @@ import ms from "@prsm/ms"
  * @property {string} uuid
  * @property {any} payload
  * @property {number} createdAt
- * @property {string} [groupKey]
+ * @property {string} [group]
  * @property {number} attempts
  */
 
@@ -131,6 +131,9 @@ export default class Queue extends EventEmitter {
 
     this._redis = createClient(this._options.redisOptions)
     this._redis.on("error", () => {})
+    this._subClient = null
+    this._groupNotifyClient = null
+    this._pendingWaits = new Map()
     this._readyPromise = this._initialize()
   }
 
@@ -157,7 +160,7 @@ export default class Queue extends EventEmitter {
   async push(payload, { group } = {}) {
     if (this._closed) throw new Error("Queue is closed")
     const task = group
-      ? { uuid: randomUUID(), payload, createdAt: Date.now(), groupKey: group, attempts: 0 }
+      ? { uuid: randomUUID(), payload, createdAt: Date.now(), group, attempts: 0 }
       : { uuid: randomUUID(), payload, createdAt: Date.now(), attempts: 0 }
     this._pushed++
     try {
@@ -175,72 +178,112 @@ export default class Queue extends EventEmitter {
    * @param {{ group?: string, timeout?: number|string }} [options]
    * @returns {Promise<any>}
    */
-  pushAndWait(payload, { group, timeout = 0 } = {}) {
-    if (this._closed) return Promise.reject(new Error("Queue is closed"))
+  async pushAndWait(payload, { group, timeout = 0 } = {}) {
+    if (this._closed) throw new Error("Queue is closed")
     const task = group
-      ? { uuid: randomUUID(), payload, createdAt: Date.now(), groupKey: group, attempts: 0 }
+      ? { uuid: randomUUID(), payload, createdAt: Date.now(), group, attempts: 0 }
       : { uuid: randomUUID(), payload, createdAt: Date.now(), attempts: 0 }
     this._pushed++
-    const result = this._awaitTask(task.uuid, timeout)
-    result.catch(() => {})
-    return this._enqueue(task, group).then(() => {
-      this.emit("new", { task })
-      return result
-    }, (err) => {
+    const { promise, ready } = this._awaitTask(task.uuid, timeout)
+    promise.catch(() => {})
+    await ready
+    try {
+      await this._enqueue(task, group)
+    } catch (err) {
       this._pushed--
       throw err
-    })
+    }
+    this.emit("new", { task })
+    return promise
   }
 
   /** @private */
   async _enqueue(task, group) {
     if (group) {
       await this._redis.lPush(`queue:groups:${group}`, JSON.stringify(task))
-      if (!this._groupWorkers.has(group)) {
-        this._groupWorkers.set(group, new Map())
-        this._groupInFlight.set(group, 0)
-        await this._startGroupWorkers(group)
-      }
+      await this._ensureGroupWorkers(group)
+      this._redis.publish("queue:group:notify", group).catch(() => {})
     } else {
       await this._redis.lPush("queue:tasks", JSON.stringify(task))
     }
   }
 
+  /** @private */
+  async _ensureGroupWorkers(group) {
+    if (this._groupWorkers.has(group) || this._closed || this._options.concurrency === 0) return
+    this._groupWorkers.set(group, new Map())
+    this._groupInFlight.set(group, 0)
+    await this._startGroupWorkers(group)
+  }
+
+  /** @private */
+  async _ensureSubClient() {
+    if (this._subClient) return this._subClient
+    this._subClient = this._redis.duplicate()
+    this._subClient.on("error", () => {})
+    await this._subClient.connect()
+    return this._subClient
+  }
+
   /** @private */
   _awaitTask(uuid, timeout = 0) {
     const ms_ = ms(timeout)
-    return new Promise((resolve, reject) => {
+    const channel = `queue:result:${uuid}`
+    let resolveReady
+
+    const ready = new Promise((r) => { resolveReady = r })
+
+    const promise = new Promise((resolve, reject) => {
       let timer
+      let settled = false
 
-      const onComplete = ({ task, result }) => {
-        if (task.uuid !== uuid) return
+      const settle = (fn, value) => {
+        if (settled) return
+        settled = true
         cleanup()
-        resolve(result)
+        fn(value)
       }
 
-      const onFailed = ({ task, error }) => {
+      const onLocal = (event) => ({ task, result, error }) => {
         if (task.uuid !== uuid) return
-        cleanup()
-        reject(error)
+        if (event === "complete") settle(resolve, result)
+        else settle(reject, error)
       }
 
+      const onComplete = onLocal("complete")
+      const onFailed = onLocal("failed")
+
       const cleanup = () => {
         if (timer) clearTimeout(timer)
         this.off("complete", onComplete)
         this.off("failed", onFailed)
+        this._pendingWaits.delete(uuid)
+        this._subClient?.unsubscribe(channel).catch(() => {})
       }
 
       if (ms_ > 0) {
-        timer = setTimeout(() => {
-          cleanup()
-          reject(new Error("pushAndWait timed out"))
-        }, ms_)
+        timer = setTimeout(() => settle(reject, new Error("pushAndWait timed out")), ms_)
         timer.unref?.()
       }
 
       this.on("complete", onComplete)
       this.on("failed", onFailed)
+
+      this._pendingWaits.set(uuid, true)
+
+      this._ensureSubClient().then((sub) => {
+        if (settled) { resolveReady(); return }
+        sub.subscribe(channel, (message) => {
+          try {
+            const { status, result, error } = JSON.parse(message)
+            if (status === "complete") settle(resolve, result)
+            else settle(reject, error ? Object.assign(new Error(error.message), error) : new Error("Task failed"))
+          } catch {}
+        }).then(() => resolveReady()).catch(() => resolveReady())
+      }).catch(() => resolveReady())
     })
+
+    return { promise, ready }
   }
 
   /** @returns {Promise<void>} */
@@ -281,18 +324,44 @@ export default class Queue extends EventEmitter {
       if (client.isOpen) await client.disconnect()
     }
     this._workerClients = []
+    if (this._groupNotifyClient?.isOpen) await this._groupNotifyClient.unsubscribe().catch(() => {})
+    if (this._groupNotifyClient?.isOpen) await this._groupNotifyClient.disconnect().catch(() => {})
+    this._groupNotifyClient = null
+    if (this._subClient?.isOpen) await this._subClient.disconnect().catch(() => {})
+    this._subClient = null
     if (this._redis.isOpen) await this._redis.quit()
   }
 
   async _initialize() {
     await this._redis.connect()
     await this._startWorkers()
+    if (this._options.concurrency > 0) {
+      await this._subscribeToGroupNotifications()
+      await this._discoverExistingGroups()
+    }
     if (this._options.cleanupInterval > 0) {
       this._cleanupTimer = setInterval(() => this._periodicCleanup(), this._options.cleanupInterval)
       this._cleanupTimer.unref()
     }
   }
 
+  async _subscribeToGroupNotifications() {
+    this._groupNotifyClient = this._redis.duplicate()
+    this._groupNotifyClient.on("error", () => {})
+    await this._groupNotifyClient.connect()
+    await this._groupNotifyClient.subscribe("queue:group:notify", (group) => {
+      this._ensureGroupWorkers(group)
+    })
+  }
+
+  async _discoverExistingGroups() {
+    const keys = await this._redis.keys("queue:groups:*")
+    for (const key of keys) {
+      const group = key.slice("queue:groups:".length)
+      await this._ensureGroupWorkers(group)
+    }
+  }
+
   async _createWorkerClient() {
     const client = this._redis.duplicate()
     client.on("error", () => {})
@@ -338,13 +407,13 @@ export default class Queue extends EventEmitter {
       timeout: this._options.groups.timeout,
       maxRetries: this._options.groups.maxRetries,
       retryKey: `queue:groups:${groupKey}`,
-      groupKey,
+      group: groupKey,
     }
     this._runWorkerLoop(workerId, client, `queue:groups:${groupKey}`, groupWorkers, opts)
   }
 
   async _runWorkerLoop(workerId, client, key, activeMap, opts) {
-    const delay = opts.groupKey ? this._options.groups.delay : this._options.delay
+    const delay = opts.group ? this._options.groups.delay : this._options.delay
 
     while (activeMap.get(workerId)) {
       try {
@@ -371,17 +440,17 @@ export default class Queue extends EventEmitter {
           }
 
           this._inFlight++
-          if (opts.groupKey) {
-            this._groupInFlight.set(opts.groupKey, (this._groupInFlight.get(opts.groupKey) || 0) + 1)
+          if (opts.group) {
+            this._groupInFlight.set(opts.group, (this._groupInFlight.get(opts.group) || 0) + 1)
           }
 
           try {
             await this._processTask(task, opts)
           } finally {
-            if (opts.groupKey) {
-              const count = (this._groupInFlight.get(opts.groupKey) || 1) - 1
-              if (count <= 0) this._groupInFlight.delete(opts.groupKey)
-              else this._groupInFlight.set(opts.groupKey, count)
+            if (opts.group) {
+              const count = (this._groupInFlight.get(opts.group) || 1) - 1
+              if (count <= 0) this._groupInFlight.delete(opts.group)
+              else this._groupInFlight.set(opts.group, count)
             }
             if (leaseId) await this._releaseGlobal(leaseId).catch(() => {})
           }
@@ -467,6 +536,7 @@ export default class Queue extends EventEmitter {
 
     if (succeeded) {
       this._settle()
+      this._publishResult(task.uuid, { status: "complete", result })
       try { this.emit("complete", { task, result }) } finally { this._emitDrain() }
     } else if (task.attempts < opts.maxRetries && !this._closed) {
       let retried = false
@@ -479,14 +549,22 @@ export default class Queue extends EventEmitter {
         this.emit("retry", { task, error: handlerError, attempt: task.attempts })
       } else {
         this._settle()
+        this._publishResult(task.uuid, { status: "failed", error: { message: handlerError?.message, code: handlerError?.code, name: handlerError?.name } })
         try { this.emit("failed", { task, error: handlerError }) } finally { this._emitDrain() }
       }
     } else {
       this._settle()
+      this._publishResult(task.uuid, { status: "failed", error: { message: handlerError?.message, code: handlerError?.code, name: handlerError?.name } })
       try { this.emit("failed", { task, error: handlerError }) } finally { this._emitDrain() }
     }
   }
 
+  _publishResult(uuid, payload) {
+    if (!this._redis.isOpen) return
+    const channel = `queue:result:${uuid}`
+    this._redis.publish(channel, JSON.stringify(payload)).catch(() => {})
+  }
+
   _settle() {
     this._inFlight--
     this._totalSettled++