snerdmq 0.2.1 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/PULL_REQUEST_TEMPLATE.md +19 -0
- data/.github/workflows/ci.yml +23 -0
- data/.snerdata/tasks/tasks.log +1 -0
- data/CONTRIBUTING.md +34 -0
- data/Gemfile.lock +17 -1
- data/README.md +81 -9
- data/bin/snerdmq-install +1 -1
- data/lib/snerdmq/queue.rb +106 -27
- data/snerdmq.gemspec +2 -2
- data/static/index.html +6 -6
- metadata +6 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: e1d73035fba1f4d5bd42c382bc5c8798db8597ddbbe2829cf6fd9323747751e7
|
|
4
|
+
data.tar.gz: 42b2633c35e968f1640a603b4d4ab59de9cdf7257f9b045c53af7cd97ba68c21
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 11a0de6095c802f2a6aa85d0ca5ce4af45b5be5f8763212e10f74608b9ff3a79cdc71929e5937ee8961076f9094124f19321ac9bf9ae78b42133ba86ffbffcbc
|
|
7
|
+
data.tar.gz: ba85ccfeff123d455ea3694709ca2a7825ab535fc70b6963d29c6d6486a9126b38f0b2d868afd5d0e05964dc9d143fa7a8b9efb3499f7b79e0df383f3f945b0b
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
## Description
|
|
2
|
+
Please include a summary of the change and which issue is fixed (if any). Please also include relevant motivation and context.
|
|
3
|
+
|
|
4
|
+
## Type of Change
|
|
5
|
+
Please check the options that are relevant.
|
|
6
|
+
- [ ] Bug fix (non-breaking change which fixes an issue)
|
|
7
|
+
- [ ] New feature (non-breaking change which adds functionality)
|
|
8
|
+
- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected)
|
|
9
|
+
- [ ] Documentation update
|
|
10
|
+
- [ ] Performance improvement
|
|
11
|
+
|
|
12
|
+
## Checklist
|
|
13
|
+
- [ ] I have read the `CONTRIBUTING.md` document.
|
|
14
|
+
- [ ] My code follows the style guidelines of this project.
|
|
15
|
+
- [ ] I have performed a self-review of my own code.
|
|
16
|
+
- [ ] I have commented my code, particularly in hard-to-understand areas.
|
|
17
|
+
- [ ] I have made corresponding changes to the documentation.
|
|
18
|
+
- [ ] I have added tests that prove my fix is effective or that my feature works.
|
|
19
|
+
- [ ] New and existing unit tests pass locally with my changes.
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [ "main" ]
|
|
6
|
+
pull_request:
|
|
7
|
+
branches: [ "main" ]
|
|
8
|
+
|
|
9
|
+
jobs:
|
|
10
|
+
build:
|
|
11
|
+
runs-on: ubuntu-latest
|
|
12
|
+
|
|
13
|
+
steps:
|
|
14
|
+
- uses: actions/checkout@v4
|
|
15
|
+
|
|
16
|
+
- name: Set up Ruby
|
|
17
|
+
uses: ruby/setup-ruby@v1
|
|
18
|
+
with:
|
|
19
|
+
ruby-version: '3.2'
|
|
20
|
+
bundler-cache: true
|
|
21
|
+
|
|
22
|
+
- name: Run tests
|
|
23
|
+
run: bundle exec rake test
|
data/.snerdata/tasks/tasks.log
CHANGED
|
@@ -1 +1,2 @@
|
|
|
1
1
|
{"taskId":"ruby-job-1","retryCount":0,"maxRetries":3,"retryAfterHours":0.0,"retryAfterTime":"2026-08-14T09:43:50.902732Z","taskData":"{\"user_id\":\"ruby_master\",\"message\":\"matz\"}","taskType":"test_ruby_job"}
|
|
2
|
+
{"taskId":"ruby-job-1","retryCount":0,"maxRetries":3,"retryAfterHours":0.0,"retryAfterTime":"2026-08-16T10:42:24.239760Z","taskData":"{\"user_id\":\"ruby_master\",\"message\":\"matz\"}","taskType":"test_ruby_job","executeAt":"2026-08-16T10:42:24.239760Z"}
|
data/CONTRIBUTING.md
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# Contributing to SnerdMQ
|
|
2
|
+
|
|
3
|
+
First off, thank you for considering contributing to SnerdMQ! It's people like you that make open-source software such a great community to learn, inspire, and create.
|
|
4
|
+
|
|
5
|
+
## How to Contribute
|
|
6
|
+
|
|
7
|
+
### 1. Fork and Clone
|
|
8
|
+
We use the standard GitHub Fork and Pull Request workflow.
|
|
9
|
+
1. Click the "Fork" button in the top right corner of the repository.
|
|
10
|
+
2. Clone your fork locally: `git clone https://github.com/YOUR_USERNAME/snerdmq-ruby.git`
|
|
11
|
+
|
|
12
|
+
### 2. Branching Strategy
|
|
13
|
+
Create a branch for your feature or bug fix:
|
|
14
|
+
```bash
|
|
15
|
+
git checkout -b feat/your-feature-name
|
|
16
|
+
# or
|
|
17
|
+
git checkout -b fix/your-bugfix-name
|
|
18
|
+
```
|
|
19
|
+
Never commit directly to the `main` or `master` branch of your fork.
|
|
20
|
+
|
|
21
|
+
### 3. Making Changes
|
|
22
|
+
- Ensure your code follows the existing conventions and paradigms in the repository.
|
|
23
|
+
- Write clear, concise, and descriptive commit messages.
|
|
24
|
+
- If you're fixing a bug, please add a test that catches the bug to ensure it doesn't happen again.
|
|
25
|
+
- If you're adding a feature, please add appropriate tests and update the `README.md` if necessary.
|
|
26
|
+
|
|
27
|
+
### 4. Submitting a Pull Request
|
|
28
|
+
1. Push your branch to your fork: `git push origin your-branch-name`
|
|
29
|
+
2. Go to the original SnerdMQ repository and click "Compare & pull request".
|
|
30
|
+
3. Fill out the provided Pull Request template completely. Provide context, what you've changed, and how you tested it.
|
|
31
|
+
4. Wait for a maintainer to review your code. We may request some changes before it can be merged!
|
|
32
|
+
|
|
33
|
+
## Code of Conduct
|
|
34
|
+
By participating in this project, you are expected to uphold a welcoming, respectful, and inclusive environment for everyone.
|
data/Gemfile.lock
CHANGED
|
@@ -1,13 +1,29 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
snerdmq (
|
|
4
|
+
snerdmq (0.3.0)
|
|
5
|
+
faye-websocket
|
|
6
|
+
puma
|
|
7
|
+
rack
|
|
5
8
|
|
|
6
9
|
GEM
|
|
7
10
|
remote: https://rubygems.org/
|
|
8
11
|
specs:
|
|
12
|
+
base64 (0.1.1)
|
|
13
|
+
eventmachine (1.2.7)
|
|
14
|
+
faye-websocket (0.12.0)
|
|
15
|
+
eventmachine (>= 0.12.0)
|
|
16
|
+
websocket-driver (>= 0.8.0)
|
|
9
17
|
minitest (5.27.0)
|
|
18
|
+
nio4r (2.7.5)
|
|
19
|
+
puma (8.0.2)
|
|
20
|
+
nio4r (~> 2.0)
|
|
21
|
+
rack (3.2.7)
|
|
10
22
|
rake (13.4.2)
|
|
23
|
+
websocket-driver (0.8.2)
|
|
24
|
+
base64
|
|
25
|
+
websocket-extensions (>= 0.1.0)
|
|
26
|
+
websocket-extensions (0.1.5)
|
|
11
27
|
|
|
12
28
|
PLATFORMS
|
|
13
29
|
arm64-darwin-24
|
data/README.md
CHANGED
|
@@ -1,28 +1,49 @@
|
|
|
1
1
|
<div align="center">
|
|
2
2
|
<img src="./assets/Designer-9.png" height="120" alt="SnerdMQ Ruby Logo" />
|
|
3
|
-
<h1>💎 SnerdMQ Ruby SDK v0.2
|
|
3
|
+
<h1>💎 SnerdMQ Ruby SDK v0.3.2</h1>
|
|
4
4
|
<p>A zero-config, C-speed background job queue for Ruby. Ditch Redis and Sidekiq for lightweight, persistent background jobs.</p>
|
|
5
5
|
|
|
6
6
|
[](https://badge.fury.io/rb/snerdmq)
|
|
7
|
+
[](https://speed-nerd.github.io/docs/)
|
|
7
8
|
</div>
|
|
8
9
|
|
|
9
10
|
This is the official Ruby SDK wrapper for **SnerdMQ**. It handles all JSON-RPC communication and `IO.popen` orchestration so you can write lightning-fast background jobs without managing any external databases like Redis or Postgres.
|
|
10
11
|
|
|
11
|
-
## ✨ v0.2
|
|
12
|
+
## ✨ v0.3.2 AI Features
|
|
12
13
|
- **Smart API Rate-Limiting**: Natively tracks `rate_limit_group` execution velocity to prevent 429 "Too Many Requests" API errors.
|
|
13
14
|
- **Payload-Hashing Deduplication**: Automatically computes cryptographic hashes to drop duplicate tasks instantly.
|
|
14
15
|
- **Dynamic Float Prioritization**: A native Binary Max-Heap bypasses standard FIFO rules for high urgency tasks.
|
|
16
|
+
- **Progress Streaming & Live Dashboard**: Handlers can stream progress updates to a built-in React UI dashboard served by the SDK.
|
|
15
17
|
- **Ditch Sidekiq & Redis**: Gives your Ruby apps persistent state, automatic retries, and dead-letter queues right out of the box with zero external infrastructure.
|
|
16
18
|
- **Zero Rust Required**: Our gem installation script automatically downloads the pre-compiled C-speed Rust binary for your OS.
|
|
17
19
|
- **Thread Safe**: Uses native Ruby `Thread`s and `Mutex` locks to orchestrate I/O without blocking your main event loop.
|
|
18
20
|
|
|
19
|
-
### ⚙️ Advanced Task Configuration (v0.2
|
|
21
|
+
### ⚙️ Advanced Task Configuration (v0.3.2)
|
|
20
22
|
To power complex AI workflows, tasks can now be configured with advanced orchestration parameters:
|
|
21
23
|
|
|
22
24
|
* **`auto_dedupe` (`true/false`)**: If set to `true`, the daemon computes a cryptographic hash of the `task_type` and `data`. If an identical payload is currently sitting in the queue pending execution, this new task is silently dropped. Excellent for preventing duplicate generative AI requests from trigger-happy users!
|
|
23
25
|
* **`urgency_score` (`Float`)**: A value (e.g. `0.99`) used to bypass the standard FIFO queue. SnerdMQ uses a true Binary Max-Heap to continually float tasks with the highest urgency score to the very front of the execution line. Standard tasks default to `0.0`.
|
|
24
26
|
* **`rate_limit_group` (`String`)**: A custom string (e.g. `"openai_api"` or `"db_writes"`) that groups tasks together for backpressure control.
|
|
25
27
|
* **`max_per_minute` (`Integer`)**: Used in conjunction with `rate_limit_group`. If the queue processes more tasks in this group than the allowed limit within a 60-second rolling window, further tasks in this group are temporarily paused. This natively prevents 429 "Too Many Requests" errors when bursting third-party APIs.
|
|
28
|
+
* **`execute_at` (`String` | `Time`)**: A timestamp of when the job should be executed in the future.
|
|
29
|
+
* **`retry_after_hours` (`Float`)**: Backoff in **hours** before a failed job is retried (default `0.0`). See *Cron Jobs vs. Retryable Jobs* below.
|
|
30
|
+
* **`cron` (`String`)**: A cron expression (e.g. `"0 * * * *"`) for recurring jobs. Shorthands like `"2h"` or `"10m"` are also supported.
|
|
31
|
+
* **`webhook_url` (`String`)**: By providing a webhook URL, SnerdMQ will completely bypass your local Ruby blocks and dispatch the task payload via an HTTP POST request directly to the specified URL.
|
|
32
|
+
* **`max_execution_seconds` (`Integer`)**: Optional hard timeout in seconds. If execution takes longer, it's marked as failed.
|
|
33
|
+
|
|
34
|
+
### Note on Hard Timeouts (`max_execution_seconds`)
|
|
35
|
+
When `max_execution_seconds` is provided, the Ruby SDK wraps the execution of your handler in a `Timeout.timeout` block. If the task takes longer than the timeout, a `Timeout::Error` is raised and the execution will be marked as failed. The background Rust daemon also enforces this timeout at the IPC level.
|
|
36
|
+
|
|
37
|
+
### 🌐 HTTP Webhooks (Serverless Execution)
|
|
38
|
+
You can configure a task to execute externally via an HTTP POST request. By setting a `webhook_url`, the internal background processor will skip any registered handlers (`queue.register_handler`) and directly invoke the HTTP endpoint.
|
|
39
|
+
|
|
40
|
+
If the HTTP endpoint returns a non-200 status code, it triggers a retry. If it permanently fails (reaches `max_retries`), the Dead Letter Queue event is automatically fired via a final HTTP POST to the same `webhook_url` but with the header `X-SnerdMQ-Event: MaxRetriesReached`.
|
|
41
|
+
|
|
42
|
+
### 🕒 Cron Jobs vs. Retryable Jobs
|
|
43
|
+
When using the new scheduling features, it is important to understand the difference between Cron and Retry behaviors:
|
|
44
|
+
> - **A Cron Job** is a *Repeatable Job* that executes again **only after a success**, on a fixed schedule.
|
|
45
|
+
> - **A Retryable Job** is a *Recovery Job* that executes again **only after a failure**, attempting to recover using the `retry_after_hours` backoff.
|
|
46
|
+
> - **Combined:** If a Cron Job fails, it temporarily uses `retry_after_hours` to retry until it recovers. Once it succeeds, it goes back to ticking on its standard cron schedule!
|
|
26
47
|
|
|
27
48
|
## 📦 Installation
|
|
28
49
|
|
|
@@ -65,17 +86,28 @@ end
|
|
|
65
86
|
queue.start_listening
|
|
66
87
|
puts "SnerdMQ Ruby SDK is listening for jobs..."
|
|
67
88
|
|
|
68
|
-
# 4. Enqueue a job from anywhere in your codebase
|
|
89
|
+
# 4. Enqueue a job from anywhere in your codebase
|
|
69
90
|
queue.enqueue(
|
|
70
91
|
task_id: "email-123",
|
|
71
92
|
task_type: "send_email",
|
|
72
93
|
data: { "to" => "john@wick.com", "subject" => "Continental Update" },
|
|
73
94
|
max_retries: 3,
|
|
74
|
-
retry_after_hours: 0.
|
|
95
|
+
retry_after_hours: 0.5, # Wait 30 minutes before retrying a failed job
|
|
75
96
|
rate_limit_group: "email_api",
|
|
76
|
-
max_per_minute: 100
|
|
77
|
-
|
|
78
|
-
|
|
97
|
+
max_per_minute: 100
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
# 5. Need scheduling, deduplication, or serverless execution? All
|
|
101
|
+
# orchestration options are opt-in — combine only what you need:
|
|
102
|
+
queue.enqueue(
|
|
103
|
+
task_id: "email-digest-1",
|
|
104
|
+
task_type: "send_email",
|
|
105
|
+
data: { "to" => "john@wick.com", "subject" => "Daily Digest" },
|
|
106
|
+
cron: "0 8 * * *", # Run every day at 08:00
|
|
107
|
+
auto_dedupe: true, # Drop identical pending payloads
|
|
108
|
+
urgency_score: 0.99, # Float to the front of the queue
|
|
109
|
+
webhook_url: "https://api.example.com/webhook", # Execute via HTTP instead of local blocks
|
|
110
|
+
max_execution_seconds: 300 # Hard timeout
|
|
79
111
|
)
|
|
80
112
|
|
|
81
113
|
# Keep main thread alive
|
|
@@ -84,7 +116,7 @@ sleep
|
|
|
84
116
|
|
|
85
117
|
### ☠️ Dead Letter Queue (Handling Permanent Failures)
|
|
86
118
|
|
|
87
|
-
When a task fails repeatedly and exhausts its `
|
|
119
|
+
When a task fails repeatedly and exhausts its `max_retries`, the SnerdMQ daemon permanently moves it to the Dead Letter Queue. You can hook into this event to alert your team, update your database, or send a Slack message by registering a Max Retry Handler.
|
|
88
120
|
|
|
89
121
|
```ruby
|
|
90
122
|
# 5. Catch tasks that have permanently failed (Dead Letter Queue)
|
|
@@ -95,6 +127,46 @@ end
|
|
|
95
127
|
|
|
96
128
|
---
|
|
97
129
|
|
|
130
|
+
## 📊 Live Dashboard
|
|
131
|
+
|
|
132
|
+
SnerdMQ ships with a built-in **React UI dashboard** served directly by the SDK — no extra services or ports to manage in your infrastructure. It gives you a real-time window into your queue:
|
|
133
|
+
|
|
134
|
+
- **Live stats**: total enqueued, processed, and failed jobs
|
|
135
|
+
- **Recent Jobs table**: per-task status (`queued`, `active`, `completed`, `failed`, `dead_letter`), retry counts, and badges showing which features a task uses (cron / webhook / timeout)
|
|
136
|
+
- **Real-time Progress Stream**: live output from `yield_progress` calls in your handlers
|
|
137
|
+
|
|
138
|
+
```ruby
|
|
139
|
+
queue = Snerdmq::SnerdQueue.new
|
|
140
|
+
|
|
141
|
+
# Start the built-in dashboard on http://localhost:9090
|
|
142
|
+
queue.start_dashboard(port: 9090)
|
|
143
|
+
|
|
144
|
+
# ... register handlers, start listening, enqueue jobs ...
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
Then open **http://localhost:9090** in your browser. The dashboard UI automatically uses HTTP polling to stay up to date (progress events included), and the SDK also exposes a small JSON API (`/api/stats`, `/api/tasks`, `/api/progress`) if you want to build your own tooling on top. The dashboard assets ship inside the gem — nothing extra to deploy.
|
|
148
|
+
|
|
149
|
+
> **Note:** `start_dashboard` only serves the UI — your jobs keep running whether or not the dashboard is open.
|
|
150
|
+
|
|
151
|
+
---
|
|
152
|
+
|
|
153
|
+
## 📡 Progress Reporting
|
|
154
|
+
|
|
155
|
+
Long-running handlers can stream live updates to the Dashboard's Progress Stream (ideal for streaming LLM tokens or multi-step ETL work):
|
|
156
|
+
|
|
157
|
+
```ruby
|
|
158
|
+
queue.register_handler("generate_report") do |data|
|
|
159
|
+
(1..10).each do |step|
|
|
160
|
+
do_work(step)
|
|
161
|
+
queue.yield_progress("Step #{step}/10 complete")
|
|
162
|
+
end
|
|
163
|
+
end
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
> `yield_progress` must be called **inside a task handler** — the SDK tracks which task is currently executing so each update lands on the right job in the dashboard.
|
|
167
|
+
|
|
168
|
+
---
|
|
169
|
+
|
|
98
170
|
## 🌍 Advanced: Distributed Scaling
|
|
99
171
|
|
|
100
172
|
By default, the SDK spins up the Rust daemon which writes the queue to a local file (`.snerdata/tasks/tasks.log`).
|
data/bin/snerdmq-install
CHANGED
data/lib/snerdmq/queue.rb
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
require 'json'
|
|
2
2
|
require 'thread'
|
|
3
|
+
require 'timeout'
|
|
4
|
+
require 'time'
|
|
3
5
|
|
|
4
6
|
module Snerdmq
|
|
5
7
|
class SnerdQueue
|
|
@@ -72,7 +74,7 @@ module Snerdmq
|
|
|
72
74
|
end
|
|
73
75
|
end
|
|
74
76
|
|
|
75
|
-
def enqueue(task_id:, task_type:, data:, max_retries: 3, retry_after_hours: 0.0, rate_limit_group: nil, max_per_minute: nil, auto_dedupe: false, urgency_score: nil)
|
|
77
|
+
def enqueue(task_id:, task_type:, data:, max_retries: 3, retry_after_hours: 0.0, rate_limit_group: nil, max_per_minute: nil, auto_dedupe: false, urgency_score: nil, execute_at: nil, cron: nil, webhook_url: nil, max_execution_seconds: nil)
|
|
76
78
|
raise "[Snerd] Cannot enqueue task: Queue is not running. Call start_listening first." if @io.nil? || @shutting_down
|
|
77
79
|
|
|
78
80
|
payload = {
|
|
@@ -88,6 +90,13 @@ module Snerdmq
|
|
|
88
90
|
payload[:max_per_minute] = max_per_minute if max_per_minute
|
|
89
91
|
payload[:auto_dedupe] = auto_dedupe if auto_dedupe
|
|
90
92
|
payload[:urgency_score] = urgency_score if urgency_score
|
|
93
|
+
|
|
94
|
+
if execute_at
|
|
95
|
+
payload[:execute_at] = execute_at.respond_to?(:iso8601) ? execute_at.iso8601 : execute_at.to_s
|
|
96
|
+
end
|
|
97
|
+
payload[:cron] = cron if cron
|
|
98
|
+
payload[:webhook_url] = webhook_url if webhook_url
|
|
99
|
+
payload[:max_execution_seconds] = max_execution_seconds if max_execution_seconds
|
|
91
100
|
|
|
92
101
|
cond = ConditionVariable.new
|
|
93
102
|
result = nil
|
|
@@ -126,18 +135,6 @@ module Snerdmq
|
|
|
126
135
|
@io.close if @io && !@io.closed?
|
|
127
136
|
end
|
|
128
137
|
|
|
129
|
-
private
|
|
130
|
-
|
|
131
|
-
def send_message(msg)
|
|
132
|
-
@stdin_mutex.synchronize do
|
|
133
|
-
return if @shutting_down || @io.nil? || @io.closed?
|
|
134
|
-
@io.puts(msg.to_json)
|
|
135
|
-
@io.flush
|
|
136
|
-
end
|
|
137
|
-
rescue Errno::EPIPE
|
|
138
|
-
# Broken pipe if the daemon died unexpectedly
|
|
139
|
-
end
|
|
140
|
-
|
|
141
138
|
def listen_to_stdout
|
|
142
139
|
@io.each_line do |line|
|
|
143
140
|
next if line.strip.empty?
|
|
@@ -170,6 +167,9 @@ module Snerdmq
|
|
|
170
167
|
warn "[Snerd] Error from engine: #{msg['message']}"
|
|
171
168
|
end
|
|
172
169
|
elsif msg["action"] == "progress"
|
|
170
|
+
# Persist progress events so the dashboard (which falls back to
|
|
171
|
+
# HTTP polling in Ruby) can display them in the Progress Stream.
|
|
172
|
+
append_progress_event(msg)
|
|
173
173
|
@ws_mutex.synchronize do
|
|
174
174
|
@ws_clients.each do |ws|
|
|
175
175
|
ws.send(msg.to_json)
|
|
@@ -208,6 +208,7 @@ module Snerdmq
|
|
|
208
208
|
def handle_execute(msg)
|
|
209
209
|
task_id = msg["task_id"]
|
|
210
210
|
task_type = msg["task_type"]
|
|
211
|
+
max_execution_seconds = msg["max_execution_seconds"]
|
|
211
212
|
|
|
212
213
|
raw_data = msg["task_data"]
|
|
213
214
|
task_data = raw_data.is_a?(String) ? JSON.parse(raw_data) : raw_data
|
|
@@ -229,12 +230,25 @@ module Snerdmq
|
|
|
229
230
|
|
|
230
231
|
begin
|
|
231
232
|
Thread.current[:snerd_task_id] = task_id
|
|
232
|
-
|
|
233
|
+
if max_execution_seconds
|
|
234
|
+
Timeout.timeout(max_execution_seconds) do
|
|
235
|
+
handler.call(task_data)
|
|
236
|
+
end
|
|
237
|
+
else
|
|
238
|
+
handler.call(task_data)
|
|
239
|
+
end
|
|
233
240
|
send_message({
|
|
234
241
|
action: "result",
|
|
235
242
|
task_id: task_id,
|
|
236
243
|
status: "success"
|
|
237
244
|
})
|
|
245
|
+
rescue Timeout::Error
|
|
246
|
+
send_message({
|
|
247
|
+
action: "result",
|
|
248
|
+
task_id: task_id,
|
|
249
|
+
status: "error",
|
|
250
|
+
error_msg: "Task execution timed out after #{max_execution_seconds} seconds"
|
|
251
|
+
})
|
|
238
252
|
rescue => e
|
|
239
253
|
send_message({
|
|
240
254
|
action: "result",
|
|
@@ -253,6 +267,7 @@ module Snerdmq
|
|
|
253
267
|
|
|
254
268
|
def start_dashboard(port: 8080)
|
|
255
269
|
require 'rack'
|
|
270
|
+
require 'puma'
|
|
256
271
|
require 'faye/websocket'
|
|
257
272
|
require 'json'
|
|
258
273
|
|
|
@@ -283,26 +298,43 @@ module Snerdmq
|
|
|
283
298
|
else
|
|
284
299
|
return [404, {}, ['Dashboard UI not found']]
|
|
285
300
|
end
|
|
301
|
+
elsif req.get? && req.path == '/api/progress'
|
|
302
|
+
events = []
|
|
303
|
+
progress_path = File.join(@storage_path || './.snerdata', 'progress_events.log')
|
|
304
|
+
if File.exist?(progress_path)
|
|
305
|
+
File.readlines(progress_path).last(100).each do |line|
|
|
306
|
+
begin
|
|
307
|
+
ev = JSON.parse(line)
|
|
308
|
+
events << ev if ev.is_a?(Hash) && ev['ts']
|
|
309
|
+
rescue
|
|
310
|
+
end
|
|
311
|
+
end
|
|
312
|
+
end
|
|
313
|
+
return [200, { 'Content-Type' => 'application/json' }.merge(cors_headers), [events.to_json]]
|
|
286
314
|
elsif req.get? && req.path == '/api/stats'
|
|
287
315
|
stats = { enqueued: 0, processed: 0, failed: 0 }
|
|
316
|
+
tasks_map = {}
|
|
288
317
|
tasks_path = File.join(@storage_path || './.snerdata', 'tasks', 'tasks.log')
|
|
289
318
|
if File.exist?(tasks_path)
|
|
290
319
|
File.readlines(tasks_path).each do |line|
|
|
291
320
|
next if line.strip.empty?
|
|
292
321
|
begin
|
|
293
322
|
t = JSON.parse(line)
|
|
294
|
-
|
|
295
|
-
if t['deletedAt']
|
|
296
|
-
if t['lastJobError']
|
|
297
|
-
stats[:failed] += 1
|
|
298
|
-
else
|
|
299
|
-
stats[:processed] += 1
|
|
300
|
-
end
|
|
301
|
-
end
|
|
323
|
+
tasks_map[t['taskId']] = t if t['taskId']
|
|
302
324
|
rescue
|
|
303
325
|
end
|
|
304
326
|
end
|
|
305
327
|
end
|
|
328
|
+
tasks_map.values.each do |t|
|
|
329
|
+
stats[:enqueued] += 1
|
|
330
|
+
if t['deletedAt']
|
|
331
|
+
if t['LastJobError']
|
|
332
|
+
stats[:failed] += 1
|
|
333
|
+
else
|
|
334
|
+
stats[:processed] += 1
|
|
335
|
+
end
|
|
336
|
+
end
|
|
337
|
+
end
|
|
306
338
|
return [200, { 'Content-Type' => 'application/json' }.merge(cors_headers), [stats.to_json]]
|
|
307
339
|
elsif req.get? && req.path == '/api/tasks'
|
|
308
340
|
tasks_map = {}
|
|
@@ -321,9 +353,23 @@ module Snerdmq
|
|
|
321
353
|
formatted = []
|
|
322
354
|
tasks_map.values.each do |t|
|
|
323
355
|
if t['deletedAt']
|
|
324
|
-
|
|
356
|
+
if t['LastJobError'] && (t['retryCount'] || 0) >= (t['maxRetries'] || 3)
|
|
357
|
+
status = 'dead_letter'
|
|
358
|
+
elsif t['LastJobError']
|
|
359
|
+
status = 'failed'
|
|
360
|
+
else
|
|
361
|
+
status = 'completed'
|
|
362
|
+
end
|
|
363
|
+
elsif t['LastJobError']
|
|
364
|
+
status = 'failed'
|
|
325
365
|
else
|
|
326
|
-
status =
|
|
366
|
+
status = 'queued'
|
|
367
|
+
if t['executeAt']
|
|
368
|
+
begin
|
|
369
|
+
status = 'active' if Time.parse(t['executeAt']) <= Time.now
|
|
370
|
+
rescue
|
|
371
|
+
end
|
|
372
|
+
end
|
|
327
373
|
end
|
|
328
374
|
formatted << {
|
|
329
375
|
id: t['taskId'],
|
|
@@ -332,7 +378,10 @@ module Snerdmq
|
|
|
332
378
|
progress: 0,
|
|
333
379
|
retryCount: t['retryCount'] || 0,
|
|
334
380
|
maxRetries: t['maxRetries'] || 3,
|
|
335
|
-
retryAfterTime: t['retryAfterTime']
|
|
381
|
+
retryAfterTime: t['retryAfterTime'],
|
|
382
|
+
cronExpression: t['cronExpression'],
|
|
383
|
+
webhookUrl: t['webhookUrl'],
|
|
384
|
+
maxExecutionSeconds: t['maxExecutionSeconds']
|
|
336
385
|
}
|
|
337
386
|
end
|
|
338
387
|
return [200, { 'Content-Type' => 'application/json' }.merge(cors_headers), [formatted.first(50).to_json]]
|
|
@@ -343,9 +392,39 @@ module Snerdmq
|
|
|
343
392
|
|
|
344
393
|
Thread.new do
|
|
345
394
|
puts "[Snerd] Dashboard running on http://localhost:#{port}"
|
|
346
|
-
|
|
395
|
+
server = Puma::Server.new(app)
|
|
396
|
+
server.add_tcp_listener('0.0.0.0', port)
|
|
397
|
+
server.run
|
|
398
|
+
end
|
|
399
|
+
end
|
|
400
|
+
|
|
401
|
+
private
|
|
402
|
+
|
|
403
|
+
def append_progress_event(msg)
|
|
404
|
+
dir = @storage_path || './.snerdata'
|
|
405
|
+
return unless File.directory?(dir)
|
|
406
|
+
path = File.join(dir, 'progress_events.log')
|
|
407
|
+
|
|
408
|
+
event = { ts: Time.now.to_f, task_id: msg['task_id'], data: msg['data'] }.to_json
|
|
409
|
+
File.open(path, 'a') { |f| f.puts(event) }
|
|
410
|
+
|
|
411
|
+
# Keep the file bounded: retain only the most recent events
|
|
412
|
+
if File.size(path) > 512 * 1024
|
|
413
|
+
lines = File.readlines(path).map(&:strip).reject(&:empty?)
|
|
414
|
+
File.write(path, lines.last(200).join("\n") + "\n")
|
|
347
415
|
end
|
|
416
|
+
rescue
|
|
417
|
+
# Never break the listener loop over progress persistence
|
|
348
418
|
end
|
|
349
419
|
|
|
420
|
+
def send_message(msg)
|
|
421
|
+
@stdin_mutex.synchronize do
|
|
422
|
+
return if @shutting_down || @io.nil? || @io.closed?
|
|
423
|
+
@io.puts(msg.to_json)
|
|
424
|
+
@io.flush
|
|
425
|
+
end
|
|
426
|
+
rescue Errno::EPIPE, Errno::EIO, IOError
|
|
427
|
+
# Broken pipe / IO error if the daemon died unexpectedly
|
|
428
|
+
end
|
|
350
429
|
end
|
|
351
|
-
end
|
|
430
|
+
end
|
data/snerdmq.gemspec
CHANGED
|
@@ -3,13 +3,13 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
|
3
3
|
|
|
4
4
|
Gem::Specification.new do |spec|
|
|
5
5
|
spec.name = "snerdmq"
|
|
6
|
-
spec.version = "0.2
|
|
6
|
+
spec.version = "0.3.2"
|
|
7
7
|
spec.authors = ["Greyhands2"]
|
|
8
8
|
spec.email = ["developer@example.com"]
|
|
9
9
|
|
|
10
10
|
spec.summary = "A zero-config, persistent background job queue for Ruby."
|
|
11
11
|
spec.description = "The official Ruby SDK for the SnerdMQ Rust daemon. Execute robust, lightning-fast background jobs without Redis."
|
|
12
|
-
spec.homepage = "https://github.com/
|
|
12
|
+
spec.homepage = "https://github.com/speed-nerd/snerdmq-ruby"
|
|
13
13
|
spec.license = "MIT"
|
|
14
14
|
|
|
15
15
|
spec.files = Dir.chdir(File.expand_path(__dir__)) do
|