@usermetrics/queuebit 1.0.0 → 1.0.1

package/README.md

@@ -1,15 +1,16 @@
 # QueueBit
 
-A high performance socket-based message queue server with guaranteed delivery, compatible with NATS queue patterns.
+A high performance socket-based message queue server with guaranteed delivery, compatible with NATS queue patterns.  
+Built in Load Balancer. (see examples).  
 
-It can run in-process in an existing nodejs app, separately as a nodejs server, and has clients 
-for the backend and frontend.
+It can run in-process in an existing nodejs app, separately as a nodejs server, or run clients 
+in the backend and/or frontend.
 
 ## Features
 
 - WebSocket-based message queue
 - Subject-based message routing
-- Queue groups for load-balanced message delivery
+- Load-balancer
 - Message expiry support
 - Remove after read (ephemeral messages)
 - Guaranteed delivery to all subscribers
@@ -82,7 +83,8 @@ Include Socket.IO and QueueBit client in your HTML:
 </script>
 ```
 
-See `examples/browser-example.html` for a complete browser example.
+See `examples/qpanel.html` for a complete browser example.
+Open it with Live Server in vscode to test.
 
 ### Server
 
@@ -185,89 +187,6 @@ Unsubscribe from messages.
 ##### `disconnect()`
 Disconnect from the server.
 
-## Publishing to NPM
-
-### First Time Setup
-
-1. Create an NPM account at https://www.npmjs.com/signup
-2. Run authentication setup:
-   ```cmd
-   setup-npm-auth.cmd
-   ```
-3. Update package.json with your username:
-   - Change `@yourusername/queuebit` to `@YOUR_NPM_USERNAME/queuebit`
-   - Or use an unscoped name like `queuebit-yourname` if available
-   - Update author field with your information
-
-### Publishing
-
-1. Update version number:
-   ```cmd
-   update-version.cmd
-   ```
-
-2. **Without 2FA:**
-   ```cmd
-   publish.cmd
-   ```
-
-3. **With 2FA enabled:**
-   ```cmd
-   publish-with-otp.cmd
-   ```
-
-### Troubleshooting
-
-- **403 Forbidden / 2FA Required**: Use `publish-with-otp.cmd`
-- **Package name taken**: Change name in package.json to something unique
-- **Not logged in**: Run `npm login` or `setup-npm-auth.cmd`
-- **Version already exists**: Increment version with `update-version.cmd`
-
-## Development
-
-### Install Dependencies
-```bash
-npm install
-```
-Or on Windows:
-```cmd
-install-deps.cmd
-```
-
-### Run Tests
-```bash
-npm test
-```
-
-### Publishing
-
-#### Update Version
-```cmd
-update-version.cmd
-```
-
-#### Dry Run (test without publishing)
-```cmd
-publish-dry-run.cmd
-```
-
-#### Publish to NPM
-```cmd
-publish.cmd
-```
-
-Or manually:
-```bash
-npm login
-npm test
-npm publish
-```
-
-## Testing
-
-```bash
-npm test
-```
 
 ## License
 

package/examples/inprocessserver.js

@@ -0,0 +1,60 @@
+/**
+ * In-process QueueBit demo.
+ * Starts the QueueBit server and an HTTP server in the same process.
+ * Use the qpanel.html dashboard to publish messages to the queue and see them received by the HTTP server.
+ * Run this example with /examples/start_node_inprocess.cmd
+ * No need to run a separate QueueBit server.
+ */
+const http = require('http');
+const { QueueBitServer } = require('../src/server');
+const { QueueBitClient } = require('../src/client-node');
+
+const webserverPORT = 3000;
+const queuebitPORT = 3333;
+
+// Start the QueueBit server in-process (constructor starts listening immediately)
+const queuebitServer = new QueueBitServer({ port: queuebitPORT });
+
+// Connect a client to the in-process QueueBit server
+// this is just for testing. typically you would connect from another process/frontend/server
+const messageQueue = new QueueBitClient(`http://localhost:${queuebitPORT}`);
+
+messageQueue.subscribe((msg) => {
+  console.log('Received message from queue:', msg);
+}, { subject: 'default' });
+
+// Create an HTTP server
+const server = http.createServer((req, res) => {
+  res.setHeader('Content-Type', 'application/json');
+
+  if (req.method === 'POST' && req.url === '/enqueue') {
+    let body = '';
+    req.on('data', chunk => { body += chunk.toString(); });
+    req.on('end', async () => {
+      try {
+        const message = JSON.parse(body);
+        const result = await messageQueue.publish(message);
+        res.writeHead(200);
+        res.end(JSON.stringify({ success: true, result }));
+      } catch (error) {
+        res.writeHead(400);
+        res.end(JSON.stringify({ error: error.message }));
+      }
+    });
+  } else if (req.method === 'GET' && req.url === '/messages') {
+    messageQueue.getMessages({ subject: 'default' }).then((result) => {
+      res.writeHead(200);
+      res.end(JSON.stringify(result));
+    });
+  } else {
+    res.writeHead(404);
+    res.end(JSON.stringify({ error: 'Not found' }));
+  }
+});
+
+server.listen(webserverPORT, () => {
+  console.log(`HTTP server running at http://localhost:${webserverPORT}`);
+  console.log('Endpoints:');
+  console.log('  POST /enqueue - Publish message to QueueBit');
+  console.log('  GET /messages - Get messages from QueueBit');
+});

package/examples/loadbalancer.js

@@ -0,0 +1,56 @@
+/**
+ * Load Balancer demo - load balanced message delivery.
+ * Messages are distributed round-robin across workers in the same load balancer.
+ * Only ONE worker receives each message (unlike regular subscribe where ALL receive).
+ * 
+ * Run this example with: node examples/queuegroup.js
+ */
+const { QueueBitServer } = require('../src/server');
+const { QueueBitClient } = require('../src/client-node');
+
+const PORT = 3333;
+
+// Start QueueBit server in-process
+new QueueBitServer({ port: PORT });
+
+async function sleep(ms) { return new Promise(r => setTimeout(r, ms)); }
+
+async function main() {
+  await sleep(500); // wait for server to start
+
+  // Create 3 worker clients - all in the same load balancer 'workers' on subject 'jobs'
+  const worker1 = new QueueBitClient(`http://localhost:${PORT}`);
+  const worker2 = new QueueBitClient(`http://localhost:${PORT}`);
+  const worker3 = new QueueBitClient(`http://localhost:${PORT}`);
+
+  await sleep(500); // wait for clients to connect
+
+  await worker1.subscribe((msg) => {
+    console.log(`Worker 1 received:`, msg.data);
+  }, { subject: 'jobs', queue: 'workers' });
+
+  await worker2.subscribe((msg) => {
+    console.log(`Worker 2 received:`, msg.data);
+  }, { subject: 'jobs', queue: 'workers' });
+
+  await worker3.subscribe((msg) => {
+    console.log(`Worker 3 received:`, msg.data);
+  }, { subject: 'jobs', queue: 'workers' });
+
+  // Publisher client
+  const publisher = new QueueBitClient(`http://localhost:${PORT}`);
+  await sleep(500);
+
+  console.log('\nPublishing 6 jobs - each worker should receive 2 (round-robin):\n');
+
+  for (let i = 1; i <= 6; i++) {
+    await publisher.publish({ job: `task-${i}`, payload: `data-${i}` }, { subject: 'jobs' });
+    await sleep(100);
+  }
+
+  await sleep(500);
+  console.log('\nDone. Each worker received ~2 messages.');
+  process.exit(0);
+}
+
+main();

package/examples/{browser-example.html → qpanel.html}

@@ -124,7 +124,7 @@
             <h2>Subscribe</h2>
             <input type="text" id="subscribeSubject" placeholder="Subject" class="small">
             <button onclick="subscribe()">Subscribe</button>
-            <button onclick="subscribeQueue()">Queue Group</button>
+            <button onclick="subscribeQueue()">Add Load Balancer</button>
         </div>
 
         <div class="col container">
@@ -162,6 +162,7 @@
         let lastUpdateTime = 0;
         let publishEndTime = 0;
         let isConnected = false;
+        let lbSubscriptionCount = 0; // track how many LB subscriptions added
 
         function connect() {
             const url = document.getElementById('serverUrl').value;
@@ -194,6 +195,7 @@
                 client.disconnect();
                 client = null;
                 isConnected = false;
+                lbSubscriptionCount = 0;
                 document.getElementById('status').textContent = 'Disconnected';
                 document.getElementById('status').style.color = 'black';
                 document.getElementById('serverVersion').textContent = '';
@@ -270,14 +272,25 @@
             }
 
             const subject = document.getElementById('subscribeSubject').value || 'default';
+            lbSubscriptionCount++;
+            const workerNum = lbSubscriptionCount;
+            const uniqueQueueName = `browser-worker-${workerNum}`; // unique name per worker
             
-            await client.subscribe((message) => {
+            const response = await client.subscribe((message) => {
                 if (!perfTestRunning) {
-                    addMessage(`[Queue: ${subject}] ${JSON.stringify(message.data)}`);
+                    console.log('LB message received:', JSON.stringify({ 
+                        queueName: message.queueName, 
+                        loadBalancerId: message.loadBalancerId,
+                        subject: message.subject
+                    }));
+                    const lbId = message.loadBalancerId !== undefined ? ` LB#${message.loadBalancerId}` : '';
+                    addMessage(`[Worker #${workerNum}${lbId}: ${subject}] ${JSON.stringify(message.data)}`);
                 }
-            }, { subject, queue: 'browser-workers' });
+            }, { subject, queue: uniqueQueueName });
 
-            console.log('Subscribed to queue group:', subject);
+            const lbId = response?.loadBalancerId !== undefined ? ` (LB#${response.loadBalancerId})` : '';
+            addMessage(`✓ Load Balancer Worker #${workerNum}${lbId} added for subject "${subject}"`);
+            console.log(`Worker #${workerNum} subscribed to load balancer: ${subject} (queue: ${uniqueQueueName})`);
         }
 
         function addMessage(text, isPerf = false) {
@@ -350,12 +363,6 @@
             const totalMessages = parseInt(document.getElementById('testMessageCount').value);
             console.log('totalMessages:', totalMessages);
             
-            const confirmed = confirm(`This will send ${totalMessages.toLocaleString()} messages to the queue. Continue?`);
-            if (!confirmed) {
-                console.log('Test cancelled by user');
-                return;
-            }
-
             console.log('Starting performance test...');
             perfTestRunning = true;
             receivedCount = 0;

package/examples/server2server.js

@@ -0,0 +1,56 @@
+/**
+ * Server to server queue.
+ * This adds queue functionality to a simple http server (the queue server is running elsewhere)
+ * Use the qpanel.html dashboard to publish messages to the queue and see them received by the HTTP server.
+ * run the server with /start_server.cmd and then run this example with /examples/start_node_client.cmd
+ */
+const http = require('http');
+const { QueueBitClient } = require('../src/client-node');
+
+const webserverPORT = 3000;
+const queuebitPORT = 3333;
+
+// Initialize a queuebit message queue. It will connect to the server running on port 3333
+const messageQueue = new QueueBitClient(`http://localhost:${queuebitPORT}`);
+
+// Create an HTTP server
+const server = http.createServer((req, res) => {
+  res.setHeader('Content-Type', 'application/json');
+
+  if (req.method === 'POST' && req.url === '/enqueue') {
+    let body = '';
+    req.on('data', chunk => {
+      body += chunk.toString();
+    });
+
+    req.on('end', () => {
+      try {
+        const message = JSON.parse(body);
+        messageQueue.enqueue(message);
+        res.writeHead(200);
+        res.end(JSON.stringify({ success: true, message: 'Message enqueued' }));
+      } catch (error) {
+        res.writeHead(400);
+        res.end(JSON.stringify({ error: 'Invalid JSON' }));
+      }
+    });
+  } else if (req.method === 'GET' && req.url === '/dequeue') {
+    const message = messageQueue.dequeue();
+    res.writeHead(200);
+    res.end(JSON.stringify({ message: message || 'Queue is empty' }));
+  } else if (req.method === 'GET' && req.url === '/size') {
+    res.writeHead(200);
+    res.end(JSON.stringify({ size: messageQueue.size() }));
+  } else {
+    res.writeHead(404);
+    res.end(JSON.stringify({ error: 'Not found' }));
+  }
+});
+
+server.listen(webserverPORT, () => {
+  console.log(`Server running at http://localhost:${webserverPORT}`);
+  console.log('Endpoints:');
+  console.log('  POST /enqueue - Add message to queue');
+  console.log('  GET /dequeue - Remove and return message from queue');
+  console.log('  GET /size - Get current queue size');
+});

package/examples/start_load_balancer.cmd

@@ -0,0 +1 @@
+node ./loadbalancer.js

package/examples/start_node_client.cmd

@@ -0,0 +1 @@
+node ./server2server.js

package/examples/start_node_inprocess.cmd

@@ -0,0 +1 @@
+call node inprocessserver.js

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@usermetrics/queuebit",
-  "version": "1.0.0",
+  "version": "1.0.1",
   "publishConfig": {
     "access": "public"
   },
@@ -9,8 +9,9 @@
   "browser": "src/client-browser.js",
   "scripts": {
     "test": "node test/test-harness.js",
-    "start": "node src/index.js",
+    "start": "node src/server-runner.js",
     "server": "node src/server-runner.js",
+    "auto": "nodemon src/server-runner.js",
     "prepublishOnly": "npm test"
   },
   "keywords": [

package/src/client-browser.js

@@ -79,12 +79,14 @@ class QueueBitClient {
 
   subscribe(callback, options = {}) {
     const subject = options.subject || 'default';
+    const queueName = options.queue || null;
+    const handlerKey = queueName ? `${subject}:${queueName}` : subject;
     
-    if (!this.messageHandlers.has(subject)) {
-      this.messageHandlers.set(subject, new Set());
+    if (!this.messageHandlers.has(handlerKey)) {
+      this.messageHandlers.set(handlerKey, new Set());
     }
     
-    this.messageHandlers.get(subject).add(callback);
+    this.messageHandlers.get(handlerKey).add(callback);
 
     return new Promise((resolve) => {
       this.socket.emit('subscribe', options, (response) => {
@@ -95,7 +97,9 @@ class QueueBitClient {
 
   unsubscribe(options = {}) {
     const subject = options.subject || 'default';
-    this.messageHandlers.delete(subject);
+    const queueName = options.queue || null;
+    const handlerKey = queueName ? `${subject}:${queueName}` : subject;
+    this.messageHandlers.delete(handlerKey);
 
     return new Promise((resolve) => {
       this.socket.emit('unsubscribe', options, (response) => {
@@ -114,8 +118,10 @@ class QueueBitClient {
 
   handleMessage(message) {
     const subject = message.subject || 'default';
-    const handlers = this.messageHandlers.get(subject);
-    
+    const queueName = message.queueName || null;
+    const handlerKey = queueName ? `${subject}:${queueName}` : subject;
+
+    const handlers = this.messageHandlers.get(handlerKey);
     if (handlers) {
       for (const handler of handlers) {
         handler(message);

package/src/client-node.js

@@ -1,7 +1,7 @@
 const { io } = require('socket.io-client');
 
 class QueueBitClient {
-  constructor(url = 'http://localhost:3000') {
+  constructor(url = 'http://localhost:3333') {
     this.socket = io(url, {
       transports: ['websocket'],
       upgrade: false,
@@ -60,12 +60,14 @@ class QueueBitClient {
 
   subscribe(callback, options = {}) {
     const subject = options.subject || 'default';
+    const queueName = options.queue || null;
+    const handlerKey = queueName ? `${subject}:${queueName}` : subject;
     
-    if (!this.messageHandlers.has(subject)) {
-      this.messageHandlers.set(subject, new Set());
+    if (!this.messageHandlers.has(handlerKey)) {
+      this.messageHandlers.set(handlerKey, new Set());
     }
     
-    this.messageHandlers.get(subject).add(callback);
+    this.messageHandlers.get(handlerKey).add(callback);
 
     return new Promise((resolve) => {
       this.socket.emit('subscribe', options, (response) => {
@@ -76,7 +78,9 @@ class QueueBitClient {
 
   unsubscribe(options = {}) {
     const subject = options.subject || 'default';
-    this.messageHandlers.delete(subject);
+    const queueName = options.queue || null;
+    const handlerKey = queueName ? `${subject}:${queueName}` : subject;
+    this.messageHandlers.delete(handlerKey);
 
     return new Promise((resolve) => {
       this.socket.emit('unsubscribe', options, (response) => {
@@ -95,8 +99,10 @@ class QueueBitClient {
 
   handleMessage(message) {
     const subject = message.subject || 'default';
-    const handlers = this.messageHandlers.get(subject);
-    
+    const queueName = message.queueName || null;
+    const handlerKey = queueName ? `${subject}:${queueName}` : subject;
+
+    const handlers = this.messageHandlers.get(handlerKey);
     if (handlers) {
       for (const handler of handlers) {
         handler(message);
@@ -109,4 +115,4 @@ class QueueBitClient {
   }
 }
 
-module.exports = { QueueBitClient };
+module.exports = { QueueBitClient, Queue: QueueBitClient };

package/src/server.js

@@ -10,7 +10,8 @@ class QueueBitServer {
     
     this.messages = new Map();
     this.subscribers = new Map();
-    this.queueGroups = new Map();
+    this.loadBalancers = new Map();
+    this.loadBalancerIdCounter = 0;
     this.deliveryQueue = [];
     this.deliveryBatchSize = 100;
     this.isDelivering = false;
@@ -124,36 +125,42 @@ class QueueBitServer {
 
   deliverMessage(message) {
     const subject = message.subject || 'default';
-    let delivered = false;
-
-    // Deliver to queue groups (load balanced)
-    const queueGroups = this.queueGroups.get(subject);
-    if (queueGroups) {
-      for (const [queueName, sockets] of queueGroups.entries()) {
-        if (sockets.length > 0) {
-          const socket = sockets[0];
-          sockets.push(sockets.shift());
-          
-          socket.emit('message', message);
-          delivered = true;
-          
-          if (message.removeAfterRead) {
-            this.removeMessage(message.id, subject);
-            return;
-          }
+
+    const loadBalancers = this.loadBalancers.get(subject);
+    if (loadBalancers && loadBalancers.size > 0) {
+      const activeLBs = [];
+      for (const [lbName, lb] of loadBalancers.entries()) {
+        if (lb.sockets.length > 0) {
+          activeLBs.push(lb);
         }
       }
+
+      if (activeLBs.length > 0) {
+        if (!this._lbRoundRobinIndex) this._lbRoundRobinIndex = {};
+        if (this._lbRoundRobinIndex[subject] === undefined) this._lbRoundRobinIndex[subject] = 0;
+
+        const idx = this._lbRoundRobinIndex[subject] % activeLBs.length;
+        this._lbRoundRobinIndex[subject]++;
+
+        const lb = activeLBs[idx];
+        const socket = lb.sockets[0];
+        lb.sockets.push(lb.sockets.shift()); // rotate within LB
+
+        socket.emit('message', { ...message, loadBalancerId: lb.id, queueName: lb.name });
+
+        // Always remove from store after LB delivery - LB messages are consumed, not persistent
+        this.removeMessage(message.id, subject);
+        return;
+      }
     }
 
-    // Deliver to regular subscribers (all receive)
+    // Only deliver to regular subscribers if no load balancer handled it
     const subscribers = this.subscribers.get(subject);
     if (subscribers) {
       for (const socket of subscribers) {
         socket.emit('message', message);
-        delivered = true;
       }
-      
-      if (message.removeAfterRead && !queueGroups) {
+      if (message.removeAfterRead) {
         this.removeMessage(message.id, subject);
       }
     }
@@ -161,20 +168,20 @@ class QueueBitServer {
 
   handleSubscribe(socket, options, callback) {
     const subject = options.subject || 'default';
-    const queueName = options.queue;
+    const lbName = options.queue;
 
-    if (queueName) {
-      // Queue group subscription (load balanced)
-      if (!this.queueGroups.has(subject)) {
-        this.queueGroups.set(subject, new Map());
+    if (lbName) {
+      if (!this.loadBalancers.has(subject)) {
+        this.loadBalancers.set(subject, new Map());
       }
       
-      const queues = this.queueGroups.get(subject);
-      if (!queues.has(queueName)) {
-        queues.set(queueName, []);
+      const lbs = this.loadBalancers.get(subject);
+      if (!lbs.has(lbName)) {
+        lbs.set(lbName, { id: ++this.loadBalancerIdCounter, name: lbName, sockets: [] });
       }
       
-      queues.get(queueName).push(socket);
+      lbs.get(lbName).sockets.push(socket);
+      // Do NOT replay existing messages to load balancer subscribers
     } else {
       // Regular subscription (all subscribers get messages)
       if (!this.subscribers.has(subject)) {
@@ -182,33 +189,38 @@ class QueueBitServer {
       }
       
       this.subscribers.get(subject).add(socket);
-    }
 
-    // Deliver any existing messages
-    const messages = this.messages.get(subject) || [];
-    for (const message of messages) {
-      if (!message.removeAfterRead) {
-        socket.emit('message', message);
+      // Replay existing messages only for regular subscribers
+      const messages = this.messages.get(subject) || [];
+      for (const message of messages) {
+        if (!message.removeAfterRead) {
+          socket.emit('message', message);
+        }
       }
     }
 
     if (callback) {
-      callback({ success: true, subject, queue: queueName });
+      callback({ 
+        success: true, 
+        subject, 
+        loadBalancer: lbName,
+        loadBalancerId: lbName ? this.loadBalancers.get(subject)?.get(lbName)?.id : undefined
+      });
     }
   }
 
   handleUnsubscribe(socket, options, callback) {
     const subject = options.subject || 'default';
-    const queueName = options.queue;
-
-    if (queueName) {
-      const queues = this.queueGroups.get(subject);
-      if (queues) {
-        const sockets = queues.get(queueName);
-        if (sockets) {
-          const index = sockets.indexOf(socket);
+    const lbName = options.queue;
+
+    if (lbName) {
+      const lbs = this.loadBalancers.get(subject);
+      if (lbs) {
+        const lb = lbs.get(lbName);
+        if (lb) {
+          const index = lb.sockets.indexOf(socket);
           if (index > -1) {
-            sockets.splice(index, 1);
+            lb.sockets.splice(index, 1);
           }
         }
       }
@@ -230,12 +242,12 @@ class QueueBitServer {
       subscribers.delete(socket);
     }
 
-    // Remove from all queue groups
-    for (const queues of this.queueGroups.values()) {
-      for (const sockets of queues.values()) {
-        const index = sockets.indexOf(socket);
+    // Remove from all load balancers
+    for (const lbs of this.loadBalancers.values()) {
+      for (const lb of lbs.values()) {
+        const index = lb.sockets.indexOf(socket);
         if (index > -1) {
-          sockets.splice(index, 1);
+          lb.sockets.splice(index, 1);
         }
       }
     }