snerdmq 0.3.1 → 0.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7b03b1fe8237e88a2fac994242af4955576569853c3794dbc49671644c506921
-  data.tar.gz: be2955f250acc99685168e2b5f9193faa5e9b8910fa5b7b209f3d027f96b7383
+  metadata.gz: 85e9f934b6976cdbd8c95718707762e2da87e93ea53448a51b3983b40d145f26
+  data.tar.gz: 82d99ccce4066f516da45f536e36f8539d569cdf47d1a74442b1557086830243
 SHA512:
-  metadata.gz: 27ddbdaa2acb42240c500878c509affb9d32e7acf9e4ffac3679b2a80cb1039582420670cfc11092bceb872cf347c7ffc9436bef2346aada6a0dcaada070da7d
-  data.tar.gz: ebbe4b904bdd68cd8750aaa339cc29cffa32437431686b36ca69547c95abf831b3faed39d1987544e614044fc729bbcd78c98fb66f3b869f9be8880eb12a69bd
+  metadata.gz: b18cee9ae796c2c612d44bea8c6aa0e81725e4a27f7f4001bb4c1d66acafc89ea2766c21e573dedebf45a8689b4b2a60abc885ad0d13a632de4da3d9934e2d5c
+  data.tar.gz: 815dc8ba1eb2977aba024af95fba389753a0428d3483c3db438e8ced86fada4666c0375d7d6d4b032a3705ac2f8c9fe30aea5c82d9967adc1260c17cfcf53d86

data/.github/workflows/ci.yml

@@ -0,0 +1,23 @@
+name: CI
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: '3.2'
+        bundler-cache: true
+
+    - name: Run tests
+      run: bundle exec rake test

data/README.md

@@ -1,22 +1,24 @@
 <div align="center">
   <img src="./assets/Designer-9.png" height="120" alt="SnerdMQ Ruby Logo" />
-  <h1>💎 SnerdMQ Ruby SDK v0.3.1</h1>
+  <h1>💎 SnerdMQ Ruby SDK v0.3.3</h1>
   <p>A zero-config, C-speed background job queue for Ruby. Ditch Redis and Sidekiq for lightweight, persistent background jobs.</p>
 
   [![Gem Version](https://badge.fury.io/rb/snerdmq.svg)](https://badge.fury.io/rb/snerdmq)
+  [![Docs](https://img.shields.io/badge/docs-speed--nerd.github.io-blue)](https://speed-nerd.github.io/docs/)
 </div>
 
 This is the official Ruby SDK wrapper for **SnerdMQ**. It handles all JSON-RPC communication and `IO.popen` orchestration so you can write lightning-fast background jobs without managing any external databases like Redis or Postgres.
 
-## ✨ v0.3.1 AI Features
+## ✨ v0.3.3 AI Features
 - **Smart API Rate-Limiting**: Natively tracks `rate_limit_group` execution velocity to prevent 429 "Too Many Requests" API errors.
 - **Payload-Hashing Deduplication**: Automatically computes cryptographic hashes to drop duplicate tasks instantly.
 - **Dynamic Float Prioritization**: A native Binary Max-Heap bypasses standard FIFO rules for high urgency tasks.
+- **Progress Streaming & Live Dashboard**: Handlers can stream progress updates to a built-in React UI dashboard served by the SDK.
 - **Ditch Sidekiq & Redis**: Gives your Ruby apps persistent state, automatic retries, and dead-letter queues right out of the box with zero external infrastructure.
 - **Zero Rust Required**: Our gem installation script automatically downloads the pre-compiled C-speed Rust binary for your OS.
 - **Thread Safe**: Uses native Ruby `Thread`s and `Mutex` locks to orchestrate I/O without blocking your main event loop.
 
-### ⚙️ Advanced Task Configuration (v0.3.1)
+### ⚙️ Advanced Task Configuration (v0.3.3)
 To power complex AI workflows, tasks can now be configured with advanced orchestration parameters:
 
 * **`auto_dedupe` (`true/false`)**: If set to `true`, the daemon computes a cryptographic hash of the `task_type` and `data`. If an identical payload is currently sitting in the queue pending execution, this new task is silently dropped. Excellent for preventing duplicate generative AI requests from trigger-happy users!
@@ -24,6 +26,7 @@ To power complex AI workflows, tasks can now be configured with advanced orchest
 * **`rate_limit_group` (`String`)**: A custom string (e.g. `"openai_api"` or `"db_writes"`) that groups tasks together for backpressure control.
 * **`max_per_minute` (`Integer`)**: Used in conjunction with `rate_limit_group`. If the queue processes more tasks in this group than the allowed limit within a 60-second rolling window, further tasks in this group are temporarily paused. This natively prevents 429 "Too Many Requests" errors when bursting third-party APIs.
 * **`execute_at` (`String` | `Time`)**: A timestamp of when the job should be executed in the future.
+* **`retry_after_hours` (`Float`)**: Backoff in **hours** before a failed job is retried (default `0.0`). See *Cron Jobs vs. Retryable Jobs* below.
 * **`cron` (`String`)**: A cron expression (e.g. `"0 * * * *"`) for recurring jobs. Shorthands like `"2h"` or `"10m"` are also supported.
 * **`webhook_url` (`String`)**: By providing a webhook URL, SnerdMQ will completely bypass your local Ruby blocks and dispatch the task payload via an HTTP POST request directly to the specified URL.
 * **`max_execution_seconds` (`Integer`)**: Optional hard timeout in seconds. If execution takes longer, it's marked as failed.
@@ -83,20 +86,28 @@ end
 queue.start_listening
 puts "SnerdMQ Ruby SDK is listening for jobs..."
 
-# 4. Enqueue a job from anywhere in your codebase (Now with v0.2.1 AI Features!)
+# 4. Enqueue a job from anywhere in your codebase
 queue.enqueue(
   task_id: "email-123",
   task_type: "send_email",
   data: { "to" => "john@wick.com", "subject" => "Continental Update" },
   max_retries: 3,
-  retry_after_hours: 0.0,
+  retry_after_hours: 0.5,      # Wait 30 minutes before retrying a failed job
   rate_limit_group: "email_api",
-  max_per_minute: 100,
-  auto_dedupe: true,
-  urgency_score: 0.99,
-  cron: "1h",
-  webhook_url: "https://api.example.com/webhook",
-  max_execution_seconds: 300
+  max_per_minute: 100
+)
+
+# 5. Need scheduling, deduplication, or serverless execution? All
+# orchestration options are opt-in — combine only what you need:
+queue.enqueue(
+  task_id: "email-digest-1",
+  task_type: "send_email",
+  data: { "to" => "john@wick.com", "subject" => "Daily Digest" },
+  cron: "0 8 * * *",           # Run every day at 08:00
+  auto_dedupe: true,           # Drop identical pending payloads
+  urgency_score: 0.99,         # Float to the front of the queue
+  webhook_url: "https://api.example.com/webhook", # Execute via HTTP instead of local blocks
+  max_execution_seconds: 300   # Hard timeout
 )
 
 # Keep main thread alive
@@ -105,7 +116,7 @@ sleep
 
 ### ☠️ Dead Letter Queue (Handling Permanent Failures)
 
-When a task fails repeatedly and exhausts its `maxRetries`, the SnerdMQ daemon permanently moves it to the Dead Letter Queue. You can hook into this event to alert your team, update your database, or send a Slack message by registering a Max Retry Handler.
+When a task fails repeatedly and exhausts its `max_retries`, the SnerdMQ daemon permanently moves it to the Dead Letter Queue. You can hook into this event to alert your team, update your database, or send a Slack message by registering a Max Retry Handler.
 
 ```ruby
 # 5. Catch tasks that have permanently failed (Dead Letter Queue)
@@ -116,18 +127,117 @@ end
 
 ---
 
-## 🌍 Advanced: Distributed Scaling
+## 📊 Live Dashboard
+
+SnerdMQ ships with a built-in **React UI dashboard** served directly by the SDK — no extra services or ports to manage in your infrastructure. It gives you a real-time window into your queue:
+
+- **Live stats**: total enqueued, processed, and failed jobs
+- **Recent Jobs table**: per-task status (`queued`, `active`, `completed`, `failed`, `dead_letter`), retry counts, and badges showing which features a task uses (cron / webhook / timeout)
+- **Real-time Progress Stream**: live output from `yield_progress` calls in your handlers
+
+```ruby
+queue = Snerdmq::SnerdQueue.new
+
+# Start the built-in dashboard on http://localhost:9090
+queue.start_dashboard(port: 9090)
+
+# ... register handlers, start listening, enqueue jobs ...
+```
+
+Then open **http://localhost:9090** in your browser. The dashboard UI automatically uses HTTP polling to stay up to date (progress events included), and the SDK also exposes a small JSON API (`/api/stats`, `/api/tasks`, `/api/progress`) if you want to build your own tooling on top. The dashboard assets ship inside the gem — nothing extra to deploy.
+
+> **Note:** `start_dashboard` only serves the UI — your jobs keep running whether or not the dashboard is open.
+
+---
+
+## 📡 Progress Reporting
+
+Long-running handlers can stream live updates to the Dashboard's Progress Stream (ideal for streaming LLM tokens or multi-step ETL work):
+
+```ruby
+queue.register_handler("generate_report") do |data|
+  (1..10).each do |step|
+    do_work(step)
+    queue.yield_progress("Step #{step}/10 complete")
+  end
+end
+```
+
+> `yield_progress` must be called **inside a task handler** — the SDK tracks which task is currently executing so each update lands on the right job in the dashboard.
+
+---
+
+## 🧩 Queue Topology: One Queue or Many?
+
+### ✅ Recommended: one queue, all job types (singleton)
+
+Each `Snerdmq::SnerdQueue` client spawns its own Rust daemon and **exclusively owns** its storage directory (`.snerdata` by default). The recommended pattern is **one client per application process**: register every job type on it and serve a single shared dashboard:
+
+```ruby
+require 'snerdmq'
+
+# ONE queue client for the whole app
+queue = Snerdmq::SnerdQueue.new
+
+# Job type #1: image processing
+queue.register_handler("process_image") do |data|
+  puts "Processing image: #{data['image_id']}"
+end
+
+# Job type #2: OTP emails — same queue, same daemon
+queue.register_handler("send_otp_email") do |data|
+  puts "Sending OTP to: #{data['to']}"
+end
+
+queue.start_listening
+
+# Both job types flow through the exact same queue
+queue.enqueue(task_id: "img-1", task_type: "process_image", data: { "image_id" => "abc123" }, max_retries: 3, retry_after_hours: 0.5)
+queue.enqueue(task_id: "otp-1", task_type: "send_otp_email", data: { "to" => "john@wick.com" }, max_retries: 3, retry_after_hours: 0.5)
+
+# ONE dashboard shows every job type
+queue.start_dashboard(port: 8080)
+```
+
+All job types share everything: the same persistent job log, retry/DLQ pipeline, rate-limit state, stats — and one dashboard at `http://localhost:8080` showing all of them.
+
+### 🚫 Same storage twice = fails fast
 
-By default, the SDK spins up the Rust daemon which writes the queue to a local file (`.snerdata/tasks/tasks.log`). 
+The daemon takes an **exclusive OS-level lock** on its storage directory at startup. A second client on the same storage fails instead of silently double-executing your jobs:
 
-If you have multiple Ruby microservices (or Rails instances) running behind a load balancer and want them to share the exact same queue, simply mount a **Shared Network Drive** (like AWS EFS or NFS) to all of your servers and pass the shared path:
+```ruby
+first = Snerdmq::SnerdQueue.new   # ✅ owns .snerdata
+second = Snerdmq::SnerdQueue.new  # ❌ daemon refuses to start:
+# "Another daemon is already running on storage '.snerdata'"
+```
+
+This applies across processes too — with **Puma/Unicorn clustered workers, every worker is a separate process** that spawns its own daemon, so each worker needs its own `storage_path` (or run a single dedicated worker process for jobs).
+
+### 🔀 Need multiple queues? Give each one its own storage
+
+```ruby
+images = Snerdmq::SnerdQueue.new(storage_path: ".snerdata-images")
+emails = Snerdmq::SnerdQueue.new(storage_path: ".snerdata-emails")
+
+images.start_dashboard(port: 8080) # separate dashboards, so separate ports
+emails.start_dashboard(port: 8081)
+```
+
+Now you have two fully independent engines: separate job logs, separate rate-limit state, separate dashboards. Only split when you actually need isolation (different teams, different retention, independent monitoring) — otherwise the singleton is simpler and recommended.
+
+---
+
+## 🌍 Advanced: Distributed Scaling
+
+Because the daemon exclusively locks its storage directory, scaling horizontally means **one queue per server**, each with its own storage. Your load balancer routes requests across servers, and every server processes the jobs it enqueued:
 
 ```ruby
 require 'snerdmq'
 
-# All of your Ruby servers point to the exact same shared file!
-# SnerdMQ's native OS file-locking guarantees zero data corruption.
-queue = Snerdmq::SnerdQueue.new(storage_path: "/mnt/aws-efs-shared-drive/snerd_tasks.log")
+# Each server runs its own daemon on its own storage dir (local disk works fine)
+queue = Snerdmq::SnerdQueue.new(storage_path: "/var/data/snerd") # per-server storage
 ```
 
+A shared network drive (AWS EFS or NFS) is still a good home for that storage when a single instance needs durable state — e.g. a container that restarts but must keep its queue. Native OS file locking (`flock`) keeps writes safe — no Redis required.
+
 *Built with ❤️ for John Wick tier engineering.*

data/bin/snerdmq-install

@@ -5,7 +5,7 @@ require 'uri'
 require 'fileutils'
 require 'rbconfig'
 
-REPO = "greyhands2/snerdmq"
+REPO = "speed-nerd/snerdmq"
 VERSION = "v0.1.1"
 
 # Determine OS and Architecture

data/lib/snerdmq/queue.rb

@@ -1,6 +1,7 @@
 require 'json'
 require 'thread'
 require 'timeout'
+require 'time'
 
 module Snerdmq
   class SnerdQueue
@@ -134,18 +135,6 @@ module Snerdmq
       @io.close if @io && !@io.closed?
     end
 
-    private
-
-    def send_message(msg)
-      @stdin_mutex.synchronize do
-        return if @shutting_down || @io.nil? || @io.closed?
-        @io.puts(msg.to_json)
-        @io.flush
-      end
-    rescue Errno::EPIPE
-      # Broken pipe if the daemon died unexpectedly
-    end
-
     def listen_to_stdout
       @io.each_line do |line|
         next if line.strip.empty?
@@ -178,6 +167,9 @@ module Snerdmq
               warn "[Snerd] Error from engine: #{msg['message']}"
             end
           elsif msg["action"] == "progress"
+            # Persist progress events so the dashboard (which falls back to
+            # HTTP polling in Ruby) can display them in the Progress Stream.
+            append_progress_event(msg)
             @ws_mutex.synchronize do
               @ws_clients.each do |ws|
                 ws.send(msg.to_json)
@@ -275,6 +267,7 @@ module Snerdmq
 
     def start_dashboard(port: 8080)
       require 'rack'
+      require 'puma'
       require 'faye/websocket'
       require 'json'
       
@@ -305,26 +298,43 @@ module Snerdmq
           else
             return [404, {}, ['Dashboard UI not found']]
           end
+        elsif req.get? && req.path == '/api/progress'
+          events = []
+          progress_path = File.join(@storage_path || './.snerdata', 'progress_events.log')
+          if File.exist?(progress_path)
+            File.readlines(progress_path).last(100).each do |line|
+              begin
+                ev = JSON.parse(line)
+                events << ev if ev.is_a?(Hash) && ev['ts']
+              rescue
+              end
+            end
+          end
+          return [200, { 'Content-Type' => 'application/json' }.merge(cors_headers), [events.to_json]]
         elsif req.get? && req.path == '/api/stats'
           stats = { enqueued: 0, processed: 0, failed: 0 }
+          tasks_map = {}
           tasks_path = File.join(@storage_path || './.snerdata', 'tasks', 'tasks.log')
           if File.exist?(tasks_path)
             File.readlines(tasks_path).each do |line|
               next if line.strip.empty?
               begin
                 t = JSON.parse(line)
-                stats[:enqueued] += 1
-                if t['deletedAt']
-                  if t['lastJobError']
-                    stats[:failed] += 1
-                  else
-                    stats[:processed] += 1
-                  end
-                end
+                tasks_map[t['taskId']] = t if t['taskId']
               rescue
               end
             end
           end
+          tasks_map.values.each do |t|
+            stats[:enqueued] += 1
+            if t['deletedAt']
+              if t['LastJobError']
+                stats[:failed] += 1
+              else
+                stats[:processed] += 1
+              end
+            end
+          end
           return [200, { 'Content-Type' => 'application/json' }.merge(cors_headers), [stats.to_json]]
         elsif req.get? && req.path == '/api/tasks'
           tasks_map = {}
@@ -343,9 +353,23 @@ module Snerdmq
           formatted = []
           tasks_map.values.each do |t|
             if t['deletedAt']
-              status = t['lastJobError'] ? 'failed' : 'completed'
+              if t['LastJobError'] && (t['retryCount'] || 0) >= (t['maxRetries'] || 3)
+                status = 'dead_letter'
+              elsif t['LastJobError']
+                status = 'failed'
+              else
+                status = 'completed'
+              end
+            elsif t['LastJobError']
+              status = 'failed'
             else
-              status = t['lastJobError'] ? 'failed' : 'queued'
+              status = 'queued'
+              if t['executeAt']
+                begin
+                  status = 'active' if Time.parse(t['executeAt']) <= Time.now
+                rescue
+                end
+              end
             end
             formatted << {
               id: t['taskId'],
@@ -354,7 +378,10 @@ module Snerdmq
               progress: 0,
               retryCount: t['retryCount'] || 0,
               maxRetries: t['maxRetries'] || 3,
-              retryAfterTime: t['retryAfterTime']
+              retryAfterTime: t['retryAfterTime'],
+              cronExpression: t['cronExpression'],
+              webhookUrl: t['webhookUrl'],
+              maxExecutionSeconds: t['maxExecutionSeconds']
             }
           end
           return [200, { 'Content-Type' => 'application/json' }.merge(cors_headers), [formatted.first(50).to_json]]
@@ -365,9 +392,39 @@ module Snerdmq
 
       Thread.new do
         puts "[Snerd] Dashboard running on http://localhost:#{port}"
-        Rack::Handler::Puma.run(app, Port: port, Silent: true)
+        server = Puma::Server.new(app)
+        server.add_tcp_listener('0.0.0.0', port)
+        server.run
+      end
+    end
+
+    private
+
+    def append_progress_event(msg)
+      dir = @storage_path || './.snerdata'
+      return unless File.directory?(dir)
+      path = File.join(dir, 'progress_events.log')
+
+      event = { ts: Time.now.to_f, task_id: msg['task_id'], data: msg['data'] }.to_json
+      File.open(path, 'a') { |f| f.puts(event) }
+
+      # Keep the file bounded: retain only the most recent events
+      if File.size(path) > 512 * 1024
+        lines = File.readlines(path).map(&:strip).reject(&:empty?)
+        File.write(path, lines.last(200).join("\n") + "\n")
       end
+    rescue
+      # Never break the listener loop over progress persistence
     end
 
+    def send_message(msg)
+      @stdin_mutex.synchronize do
+        return if @shutting_down || @io.nil? || @io.closed?
+        @io.puts(msg.to_json)
+        @io.flush
+      end
+    rescue Errno::EPIPE, Errno::EIO, IOError
+      # Broken pipe / IO error if the daemon died unexpectedly
+    end
   end
-end
+end

data/snerdmq.gemspec

@@ -3,13 +3,13 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "snerdmq"
-  spec.version       = "0.3.1"
+  spec.version       = "0.3.3"
   spec.authors       = ["Greyhands2"]
   spec.email         = ["developer@example.com"]
 
   spec.summary       = "A zero-config, persistent background job queue for Ruby."
   spec.description   = "The official Ruby SDK for the SnerdMQ Rust daemon. Execute robust, lightning-fast background jobs without Redis."
-  spec.homepage      = "https://github.com/greyhands2/snerdmq-ruby"
+  spec.homepage      = "https://github.com/speed-nerd/snerdmq-ruby"
   spec.license       = "MIT"
 
   spec.files         = Dir.chdir(File.expand_path(__dir__)) do