snerdmq 0.3.3 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 85e9f934b6976cdbd8c95718707762e2da87e93ea53448a51b3983b40d145f26
-  data.tar.gz: 82d99ccce4066f516da45f536e36f8539d569cdf47d1a74442b1557086830243
+  metadata.gz: 45cda2f70cd673e8e419afd54eb65adc3aee234eb91c61ccae9f12e535dbe247
+  data.tar.gz: e3720631cf201ab202b432c81e73d93933127faddfc07c40e27cde58bab199ae
 SHA512:
-  metadata.gz: b18cee9ae796c2c612d44bea8c6aa0e81725e4a27f7f4001bb4c1d66acafc89ea2766c21e573dedebf45a8689b4b2a60abc885ad0d13a632de4da3d9934e2d5c
-  data.tar.gz: 815dc8ba1eb2977aba024af95fba389753a0428d3483c3db438e8ced86fada4666c0375d7d6d4b032a3705ac2f8c9fe30aea5c82d9967adc1260c17cfcf53d86
+  metadata.gz: 35a3dde0f3b2febdf5c78787b2419f09e8af3d4581086e46c376c96486a72f9c8612d6f3a5d14be7188d342c74b1947450576e835b69bdb3f0d4aa4dc6eb8f0f
+  data.tar.gz: f77e67cf6d8e76b82a7cfeabbb881aaa781ae59214ef40d96538512b3110780055420cfc80c3eff874a59fe496ceb9d985de2aa6171b5865d75fb8acf8e22db0

data/.snerdata/membership.json

@@ -0,0 +1,6 @@
+{
+  "queue": "snerdmq-daemon",
+  "shards": 1,
+  "version": 2,
+  "claims": {}
+}

data/.snerdata/shard-0/.lock

@@ -0,0 +1 @@
+8643

data/.snerdata/shard-0/tasks/tasks.log

@@ -0,0 +1,4 @@
+{"taskId":"ruby-job-1","retryCount":0,"maxRetries":3,"retryAfterHours":0.0,"retryAfterTime":"2026-08-14T09:43:50.902732Z","taskData":"{\"user_id\":\"ruby_master\",\"message\":\"matz\"}","taskType":"test_ruby_job"}
+{"taskId":"ruby-job-1","retryCount":0,"maxRetries":3,"retryAfterHours":0.0,"retryAfterTime":"2026-08-16T10:42:24.239760Z","taskData":"{\"user_id\":\"ruby_master\",\"message\":\"matz\"}","taskType":"test_ruby_job","executeAt":"2026-08-16T10:42:24.239760Z"}
+{"taskId":"ruby-job-1","retryCount":0,"maxRetries":3,"retryAfterHours":0.0,"retryAfterTime":"2026-08-16T10:42:24.239760Z","taskData":"{\"user_id\":\"ruby_master\",\"message\":\"matz\"}","taskType":"test_ruby_job","deletedAt":"2026-09-25T10:43:04.524511Z","executeAt":"2026-08-16T10:42:24.239760Z"}
+{"taskId":"ruby-job-1","retryCount":0,"maxRetries":3,"retryAfterHours":0.0,"retryAfterTime":"2026-09-25T10:43:04.605292Z","taskData":"{\"user_id\":\"ruby_master\",\"message\":\"matz\"}","taskType":"test_ruby_job","executeAt":"2026-09-25T10:43:04.605292Z"}

data/README.md

@@ -1,24 +1,29 @@
 <div align="center">
   <img src="./assets/Designer-9.png" height="120" alt="SnerdMQ Ruby Logo" />
-  <h1>💎 SnerdMQ Ruby SDK v0.3.3</h1>
+  <h1>💎 SnerdMQ Ruby SDK v0.4.1</h1>
   <p>A zero-config, C-speed background job queue for Ruby. Ditch Redis and Sidekiq for lightweight, persistent background jobs.</p>
 
   [![Gem Version](https://badge.fury.io/rb/snerdmq.svg)](https://badge.fury.io/rb/snerdmq)
-  [![Docs](https://img.shields.io/badge/docs-speed--nerd.github.io-blue)](https://speed-nerd.github.io/docs/)
+  [![Docs](https://img.shields.io/badge/docs-speed--nerd.github.io-blue)](https://speed-nerd.github.io/docs/sdks/ruby/)
 </div>
 
 This is the official Ruby SDK wrapper for **SnerdMQ**. It handles all JSON-RPC communication and `IO.popen` orchestration so you can write lightning-fast background jobs without managing any external databases like Redis or Postgres.
 
-## ✨ v0.3.3 AI Features
+> 📚 **Full Documentation & Advanced Features:** Check out the [official Ruby SDK documentation](https://speed-nerd.github.io/docs/sdks/ruby/) on our docs site!
+
+## ✨ v0.4.1 AI Features
+- **Worker Pools**: Prevent slow generative AI tasks from starving fast DB tasks by dedicating workers to specific pools (e.g. `"urgent"`).
+- **Sharded Queues**: Distribute load across multiple queue nodes safely using file-backed lock sharding (`max_local_shards`).
 - **Smart API Rate-Limiting**: Natively tracks `rate_limit_group` execution velocity to prevent 429 "Too Many Requests" API errors.
 - **Payload-Hashing Deduplication**: Automatically computes cryptographic hashes to drop duplicate tasks instantly.
 - **Dynamic Float Prioritization**: A native Binary Max-Heap bypasses standard FIFO rules for high urgency tasks.
+- **Job Chaining (DAGs)**: Define complex workflow dependencies natively. Tasks wait in a blocked state until their parent tasks succeed.
 - **Progress Streaming & Live Dashboard**: Handlers can stream progress updates to a built-in React UI dashboard served by the SDK.
 - **Ditch Sidekiq & Redis**: Gives your Ruby apps persistent state, automatic retries, and dead-letter queues right out of the box with zero external infrastructure.
 - **Zero Rust Required**: Our gem installation script automatically downloads the pre-compiled C-speed Rust binary for your OS.
 - **Thread Safe**: Uses native Ruby `Thread`s and `Mutex` locks to orchestrate I/O without blocking your main event loop.
 
-### ⚙️ Advanced Task Configuration (v0.3.3)
+### ⚙️ Advanced Task Configuration (v0.4.1)
 To power complex AI workflows, tasks can now be configured with advanced orchestration parameters:
 
 * **`auto_dedupe` (`true/false`)**: If set to `true`, the daemon computes a cryptographic hash of the `task_type` and `data`. If an identical payload is currently sitting in the queue pending execution, this new task is silently dropped. Excellent for preventing duplicate generative AI requests from trigger-happy users!
@@ -30,6 +35,8 @@ To power complex AI workflows, tasks can now be configured with advanced orchest
 * **`cron` (`String`)**: A cron expression (e.g. `"0 * * * *"`) for recurring jobs. Shorthands like `"2h"` or `"10m"` are also supported.
 * **`webhook_url` (`String`)**: By providing a webhook URL, SnerdMQ will completely bypass your local Ruby blocks and dispatch the task payload via an HTTP POST request directly to the specified URL.
 * **`max_execution_seconds` (`Integer`)**: Optional hard timeout in seconds. If execution takes longer, it's marked as failed.
+* **`trigger_after_ids` (`Array` of `String`)**: A list of parent task IDs that must complete successfully before this task is allowed to dispatch. Enables complex DAG workflows natively within the queue.
+* **`pool` (`String`)**: Dedicate this task to a specific worker pool (e.g. `"urgent"`).
 
 ### Note on Hard Timeouts (`max_execution_seconds`)
 When `max_execution_seconds` is provided, the Ruby SDK wraps the execution of your handler in a `Timeout.timeout` block. If the task takes longer than the timeout, a `Timeout::Error` is raised and the execution will be marked as failed. The background Rust daemon also enforces this timeout at the IPC level.
@@ -107,7 +114,9 @@ queue.enqueue(
   auto_dedupe: true,           # Drop identical pending payloads
   urgency_score: 0.99,         # Float to the front of the queue
   webhook_url: "https://api.example.com/webhook", # Execute via HTTP instead of local blocks
-  max_execution_seconds: 300   # Hard timeout
+  max_execution_seconds: 300,  # Hard timeout
+  trigger_after_ids: ["parent-123"], # Wait for parent tasks to complete
+  pool: "urgent"               # Dedicate to a specific worker pool
 )
 
 # Keep main thread alive
@@ -211,7 +220,11 @@ second = Snerdmq::SnerdQueue.new  # ❌ daemon refuses to start:
 # "Another daemon is already running on storage '.snerdata'"
 ```
 
-This applies across processes too — with **Puma/Unicorn clustered workers, every worker is a separate process** that spawns its own daemon, so each worker needs its own `storage_path` (or run a single dedicated worker process for jobs).
+This applies across processes too — with **Puma/Unicorn clustered workers, every worker is a separate process**. To safely scale on the same disk without double-executing jobs, you must initialize the daemon with `max_local_shards`:
+```ruby
+# SnerdMQ will partition the .snerdata locks across shards
+queue = Snerdmq::SnerdQueue.new(max_local_shards: 4, max_workers: { "urgent" => 5 })
+```
 
 ### 🔀 Need multiple queues? Give each one its own storage
 
@@ -240,4 +253,120 @@ queue = Snerdmq::SnerdQueue.new(storage_path: "/var/data/snerd") # per-server st
 
 A shared network drive (AWS EFS or NFS) is still a good home for that storage when a single instance needs durable state — e.g. a container that restarts but must keep its queue. Native OS file locking (`flock`) keeps writes safe — no Redis required.
 
+
+---
+
+## 🚀 Advanced Orchestration
+
+### 🏊 Worker Pools
+
+SnerdMQ supports dedicating worker resources to specific tasks so that slow AI generation tasks don't starve fast database updates.
+
+In the SDK, simply assign a pool name when enqueueing the task using the `pool` parameter. When running the daemon, you can allocate concurrent workers per pool using the environment variable `SNERD_POOLS="default:100,urgent:50"`.
+
+### 🔗 Job Chaining (DAGs)
+
+You can define complex workflow dependencies natively. Tasks will wait in a blocked state until their parent tasks successfully complete.
+
+Simply pass an array of parent task IDs to the `trigger_after_ids` parameter when enqueueing. This easily unlocks Fan-In and Linear workflows natively within the queue.
+
+### 🍕 Sharded Queues (Scaling Out)
+
+SnerdMQ natively supports distributed execution across multiple servers while acting as a single logical queue. Just mount a shared storage drive (like AWS EFS) and boot multiple daemons. They will automatically lock and negotiate ownership of shards. No config required in the SDK for enqueueing! Just tell the daemon how many shards to claim on boot:
+
+```ruby
+# Boot a multi-tenant daemon that owns up to 4 shards locally
+queue = SnerdQueue.new(max_local_shards: 4)
+```
+
+```ruby
+# 1. Worker Pools: Route tasks to the 'urgent' pool
+queue.enqueue(
+  task_id: 'payment-job',
+  task_type: 'process_payment',
+  data: { amount: 100 },
+  pool: 'urgent'
+)
+
+# 2. Job Chaining: Block execution until parents succeed
+queue.enqueue(
+  task_id: 'final-job',
+  task_type: 'send_report',
+  data: { id: 1 },
+  trigger_after_ids: ['parent-job-1', 'parent-job-2']
+)
+```
+
+
+### 🕒 Cron & Scheduled Jobs
+```ruby
+# Run every day at 08:00
+queue.enqueue(
+  task_id: 'daily-digest',
+  task_type: 'send_email',
+  data: { template: 'daily' },
+  cron: '0 8 * * *'
+)
+```
+
+### 🛑 Hard Timeouts
+```ruby
+# Forcefully kill if running > 5 mins
+queue.enqueue(
+  task_id: 'risky-task',
+  task_type: 'process_data',
+  data: {},
+  max_execution_seconds: 300
+)
+```
+
+### 🌐 Webhook Callbacks
+```ruby
+# Execute via HTTP instead of local handlers
+queue.enqueue(
+  task_id: 'serverless-task',
+  task_type: 'resize_image',
+  data: { img: 'cat.jpg' },
+  webhook_url: 'https://api.example.com/webhooks/snerdmq'
+)
+```
+
 *Built with ❤️ for John Wick tier engineering.*
+
+
+## Architecture Best Practices
+
+When building production applications with SnerdMQ, it is recommended to initialize the queue as a Singleton, isolate your domain workers into separate files/functions, use Dead Letter Queues (DLQ) for failed tasks via `RegisterMaxRetryHandler`, and ensure manual graceful shutdown. The embedded Dashboard UI can also be easily served from the same instance.
+
+```ruby
+require 'snerdmq'
+
+queue = SnerdQueue.new(storage_path: "./.snerdata")
+
+def init_email_workers(queue)
+  queue.register_handler('send_email') do |data|
+    puts "Sending email to #{data['email']}..."
+  end
+
+  queue.register_max_retry_handler('send_email') do |data|
+    puts "Email to #{data['email']} failed permanently. Dead letter processing..."
+  end
+end
+
+def init_image_workers(queue)
+  queue.register_handler('process_image') do |data|
+    puts "Processing image #{data['imageId']}..."
+  end
+end
+
+init_email_workers(queue)
+init_image_workers(queue)
+
+queue.start_dashboard(8080)
+
+# Trap signals for graceful shutdown
+trap('INT') { queue.shutdown; exit }
+trap('TERM') { queue.shutdown; exit }
+
+queue.start_listening
+```

data/lib/snerdmq/queue.rb

@@ -5,9 +5,11 @@ require 'time'
 
 module Snerdmq
   class SnerdQueue
-    def initialize(binary_path: nil, storage_path: nil)
+    def initialize(binary_path: nil, storage_path: nil, max_local_shards: nil, max_workers: nil)
       @binary_path = binary_path
       @storage_path = storage_path
+      @max_local_shards = max_local_shards
+      @max_workers = max_workers
       
       if @binary_path.nil?
         ext = RbConfig::CONFIG['host_os'].match?(/mswin|msys|mingw|cygwin|bccwin|wince|emc/) ? '.exe' : ''
@@ -56,8 +58,12 @@ module Snerdmq
       args = []
       args << @storage_path if @storage_path
 
+      env = {}
+      env["SNERD_MAX_SHARDS"] = @max_local_shards.to_s if @max_local_shards
+      env["SNERD_MAX_WORKERS"] = @max_workers.to_s if @max_workers
+
       # Open a bidirectional pipe to the Rust daemon
-      @io = IO.popen([@binary_path] + args, "r+")
+      @io = IO.popen(env, [@binary_path] + args, "r+")
 
       # Re-register all existing handlers
       @handlers_mutex.synchronize do
@@ -74,7 +80,7 @@ module Snerdmq
       end
     end
 
-    def enqueue(task_id:, task_type:, data:, max_retries: 3, retry_after_hours: 0.0, rate_limit_group: nil, max_per_minute: nil, auto_dedupe: false, urgency_score: nil, execute_at: nil, cron: nil, webhook_url: nil, max_execution_seconds: nil)
+    def enqueue(task_id:, task_type:, data:, max_retries: 3, retry_after_hours: 0.0, rate_limit_group: nil, max_per_minute: nil, auto_dedupe: false, urgency_score: nil, execute_at: nil, cron: nil, webhook_url: nil, max_execution_seconds: nil, trigger_after_ids: nil, pool: nil)
       raise "[Snerd] Cannot enqueue task: Queue is not running. Call start_listening first." if @io.nil? || @shutting_down
       
       payload = {
@@ -97,6 +103,8 @@ module Snerdmq
       payload[:cron] = cron if cron
       payload[:webhook_url] = webhook_url if webhook_url
       payload[:max_execution_seconds] = max_execution_seconds if max_execution_seconds
+      payload[:trigger_after_ids] = trigger_after_ids if trigger_after_ids
+      payload[:pool] = pool if pool
 
       cond = ConditionVariable.new
       result = nil

data/snerdmq.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "snerdmq"
-  spec.version       = "0.3.3"
+  spec.version       = "0.4.1"
   spec.authors       = ["Greyhands2"]
   spec.email         = ["developer@example.com"]
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: snerdmq
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.4.1
 platform: ruby
 authors:
 - Greyhands2
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-08-19 00:00:00.000000000 Z
+date: 2026-09-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack
@@ -106,7 +106,10 @@ files:
 - ".github/PULL_REQUEST_TEMPLATE.md"
 - ".github/workflows/ci.yml"
 - ".gitignore"
-- ".snerdata/tasks/tasks.log"
+- ".snerdata/.membership.lock"
+- ".snerdata/membership.json"
+- ".snerdata/shard-0/.lock"
+- ".snerdata/shard-0/tasks/tasks.log"
 - CONTRIBUTING.md
 - Gemfile
 - Gemfile.lock

data/.snerdata/tasks/tasks.log

@@ -1,2 +0,0 @@
-{"taskId":"ruby-job-1","retryCount":0,"maxRetries":3,"retryAfterHours":0.0,"retryAfterTime":"2026-08-14T09:43:50.902732Z","taskData":"{\"user_id\":\"ruby_master\",\"message\":\"matz\"}","taskType":"test_ruby_job"}
-{"taskId":"ruby-job-1","retryCount":0,"maxRetries":3,"retryAfterHours":0.0,"retryAfterTime":"2026-08-16T10:42:24.239760Z","taskData":"{\"user_id\":\"ruby_master\",\"message\":\"matz\"}","taskType":"test_ruby_job","executeAt":"2026-08-16T10:42:24.239760Z"}