zizq 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/README.md +94 -0
- data/bin/profile-worker +145 -0
- data/bin/zizq-worker +174 -0
- data/lib/active_job/queue_adapters/zizq_adapter.rb +109 -0
- data/lib/zizq/ack_processor.rb +132 -0
- data/lib/zizq/active_job_config.rb +122 -0
- data/lib/zizq/backoff.rb +50 -0
- data/lib/zizq/bulk_enqueue.rb +87 -0
- data/lib/zizq/client.rb +982 -0
- data/lib/zizq/configuration.rb +164 -0
- data/lib/zizq/enqueue_request.rb +178 -0
- data/lib/zizq/enqueue_with.rb +109 -0
- data/lib/zizq/error.rb +43 -0
- data/lib/zizq/job.rb +188 -0
- data/lib/zizq/job_config.rb +244 -0
- data/lib/zizq/lifecycle.rb +58 -0
- data/lib/zizq/middleware.rb +79 -0
- data/lib/zizq/query.rb +566 -0
- data/lib/zizq/resources/error_enumerator.rb +241 -0
- data/lib/zizq/resources/error_page.rb +19 -0
- data/lib/zizq/resources/error_record.rb +19 -0
- data/lib/zizq/resources/job.rb +124 -0
- data/lib/zizq/resources/job_page.rb +57 -0
- data/lib/zizq/resources/page.rb +77 -0
- data/lib/zizq/resources/resource.rb +45 -0
- data/lib/zizq/resources.rb +16 -0
- data/lib/zizq/version.rb +9 -0
- data/lib/zizq/worker.rb +467 -0
- data/lib/zizq.rb +269 -0
- data/sig/generated/zizq/ack_processor.rbs +73 -0
- data/sig/generated/zizq/active_job_config.rbs +74 -0
- data/sig/generated/zizq/backoff.rbs +34 -0
- data/sig/generated/zizq/bulk_enqueue.rbs +72 -0
- data/sig/generated/zizq/client.rbs +419 -0
- data/sig/generated/zizq/configuration.rbs +95 -0
- data/sig/generated/zizq/enqueue_request.rbs +94 -0
- data/sig/generated/zizq/enqueue_with.rbs +88 -0
- data/sig/generated/zizq/error.rbs +41 -0
- data/sig/generated/zizq/job.rbs +136 -0
- data/sig/generated/zizq/job_config.rbs +150 -0
- data/sig/generated/zizq/lifecycle.rbs +34 -0
- data/sig/generated/zizq/middleware.rbs +50 -0
- data/sig/generated/zizq/query.rbs +327 -0
- data/sig/generated/zizq/resources/error_enumerator.rbs +148 -0
- data/sig/generated/zizq/resources/error_page.rbs +13 -0
- data/sig/generated/zizq/resources/error_record.rbs +20 -0
- data/sig/generated/zizq/resources/job.rbs +89 -0
- data/sig/generated/zizq/resources/job_page.rbs +33 -0
- data/sig/generated/zizq/resources/page.rbs +47 -0
- data/sig/generated/zizq/resources/resource.rbs +26 -0
- data/sig/generated/zizq/version.rbs +5 -0
- data/sig/generated/zizq/worker.rbs +152 -0
- data/sig/generated/zizq.rbs +180 -0
- data/sig/zizq.rbs +111 -0
- metadata +134 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: 77ab05fe491ab193d0c5c3b1106aac3e5effe7b1dcb7e0117b70e0e05652426b
|
|
4
|
+
data.tar.gz: 05f26bd94ca2127e7e859212348e063bc9354d866945ae500514dcfbcfe57828
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: f12216566271dbd68395eeada829ffd68bfd853e86c1a8e598dbac6c3c7f5d8af5008ee084a9a7ebaef99d74144c6d05995fe32e01f0349b47dac6beb082c702
|
|
7
|
+
data.tar.gz: 505804d9abd2c1cff8a749bd9cb6ad3d224d09545648a314b54a3094a96f2d18cb18257661833479bf1b59959d6d300730e2cbcbb70572afee5195bb538d8460
|
data/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Chris Corbyn <chris@zizq.io>
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
# Zizq — Official Ruby Client
|
|
2
|
+
|
|
3
|
+
Zizq is a simple, zero dependency, single binary job queue system that is both
|
|
4
|
+
fast and durable. It is designed to work in any stack through a simple HTTP
|
|
5
|
+
API.
|
|
6
|
+
|
|
7
|
+
This is the official Zizq client library for Ruby.
|
|
8
|
+
|
|
9
|
+
[](https://github.com/zizq-labs/zizq-ruby/actions/workflows/ci.yml)
|
|
10
|
+
|
|
11
|
+
## Features
|
|
12
|
+
|
|
13
|
+
* Multi-thread and/or multi-fiber concurrent worker (via [`async`](https://github.com/socketry/async))
|
|
14
|
+
* `Zizq::Job` based job classes, Active Job support, or completely custom
|
|
15
|
+
* Enqueue and process jobs from one language to another
|
|
16
|
+
* Arbitrary named queues
|
|
17
|
+
* Granular job priorities
|
|
18
|
+
* Scheduled jobs
|
|
19
|
+
* Configurable backoff policies
|
|
20
|
+
* Configurable job retention policies
|
|
21
|
+
* Job introspection and management APIs, with support for `jq` query filters
|
|
22
|
+
* Unique jobs
|
|
23
|
+
|
|
24
|
+
## Example
|
|
25
|
+
|
|
26
|
+
> [!TIP]
|
|
27
|
+
> The client is very flexible and supports being used in a range of different
|
|
28
|
+
> ways. Read the [full documentation](https://zizq.io/docs/clients/ruby/) on
|
|
29
|
+
> the website for more details.
|
|
30
|
+
|
|
31
|
+
Mixin-based job class.
|
|
32
|
+
|
|
33
|
+
```ruby
|
|
34
|
+
class SendEmailJob
|
|
35
|
+
include Zizq::Job
|
|
36
|
+
|
|
37
|
+
zizq_queue 'emails'
|
|
38
|
+
zizq_priority 100
|
|
39
|
+
|
|
40
|
+
def perform(user_id, template:)
|
|
41
|
+
# your application logic here
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
Enqueueing a job.
|
|
47
|
+
|
|
48
|
+
```ruby
|
|
49
|
+
Zizq.enqueue(SendEmailJob, 42, template: 'welcome')
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
> [!NOTE]
|
|
53
|
+
> Jobs can also be enqueued and processed without `Zizq::Job`, which is
|
|
54
|
+
> designed to support interoperability with any programming language.
|
|
55
|
+
|
|
56
|
+
Using the included `zizq-worker` executable.
|
|
57
|
+
|
|
58
|
+
```shell
|
|
59
|
+
$ zizq-worker --threads 5 --fibers 2 app.rb
|
|
60
|
+
I, [2026-03-24T15:25:57.738131 #1331422] INFO -- : Zizq worker starting: 5 threads, 2 fibers, prefetch=20
|
|
61
|
+
I, [2026-03-24T15:25:57.738222 #1331422] INFO -- : Queues: (all)
|
|
62
|
+
I, [2026-03-24T15:25:57.739861 #1331422] INFO -- : Worker 0:0 started
|
|
63
|
+
I, [2026-03-24T15:25:57.739962 #1331422] INFO -- : Worker 0:1 started
|
|
64
|
+
I, [2026-03-24T15:25:57.740131 #1331422] INFO -- : Worker 1:0 started
|
|
65
|
+
I, [2026-03-24T15:25:57.740211 #1331422] INFO -- : Worker 1:1 started
|
|
66
|
+
I, [2026-03-24T15:25:57.740352 #1331422] INFO -- : Worker 2:0 started
|
|
67
|
+
I, [2026-03-24T15:25:57.740408 #1331422] INFO -- : Worker 2:1 started
|
|
68
|
+
I, [2026-03-24T15:25:57.740532 #1331422] INFO -- : Worker 3:0 started
|
|
69
|
+
I, [2026-03-24T15:25:57.740590 #1331422] INFO -- : Worker 3:1 started
|
|
70
|
+
I, [2026-03-24T15:25:57.740722 #1331422] INFO -- : Worker 4:0 started
|
|
71
|
+
I, [2026-03-24T15:25:57.740776 #1331422] INFO -- : Worker 4:1 started
|
|
72
|
+
I, [2026-03-24T15:25:57.740844 #1331422] INFO -- : Zizq producer thread started
|
|
73
|
+
I, [2026-03-24T15:25:57.740878 #1331422] INFO -- : Connecting to http://localhost:7890...
|
|
74
|
+
I, [2026-03-24T15:25:57.792173 #1331422] INFO -- : Connected. Listening for jobs.
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
> [!NOTE]
|
|
78
|
+
> Workers can also be created directly in code. There is no requirement to use
|
|
79
|
+
> `zizq-worker`.
|
|
80
|
+
|
|
81
|
+
## Resources
|
|
82
|
+
|
|
83
|
+
* [Ruby Client Docs](https://zizq.io/docs/clients/ruby/)
|
|
84
|
+
* [Getting Started Docs](https://zizq.io/docs/getting-started/)
|
|
85
|
+
* [Zizq Command Reference](https://zizq.io/docs/cli/)
|
|
86
|
+
* [Zizq Node.js Client Source](https://github.com/zizq-labs/zizq-node)
|
|
87
|
+
* [Zizq Source](https://github.com/zizq-labs/zizq)
|
|
88
|
+
|
|
89
|
+
## Support & Feedback
|
|
90
|
+
|
|
91
|
+
If you need help using Zizq,
|
|
92
|
+
[create an issue](https://github.com/zizq-labs/zizq-ruby/issues) on the
|
|
93
|
+
[zizq-ruby](https://github.com/zizq-labs/zizq-ruby) repo. Feedback is very
|
|
94
|
+
welcome.
|
data/bin/profile-worker
ADDED
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
# Copyright (c) 2026 Chris Corbyn <chris@zizq.io>
|
|
3
|
+
# Licensed under the MIT License. See LICENSE file for details.
|
|
4
|
+
|
|
5
|
+
# frozen_string_literal: true
|
|
6
|
+
|
|
7
|
+
# Profile the worker using Vernier (wall-clock mode, all threads/fibers).
|
|
8
|
+
#
|
|
9
|
+
# All settings are controlled via environment variables — the same ones
|
|
10
|
+
# accepted by zizq-worker, plus:
|
|
11
|
+
#
|
|
12
|
+
# VERNIER_OUT Output file (default: profile.vernier.json)
|
|
13
|
+
# VERNIER_INTERVAL Sample interval in microseconds (default: 1000)
|
|
14
|
+
# VERNIER_TEXT Set to 1 to print a text summary to stderr
|
|
15
|
+
#
|
|
16
|
+
# Example:
|
|
17
|
+
#
|
|
18
|
+
# ZIZQ_THREADS=5 ZIZQ_FIBERS=5 bundle exec bin/profile-worker entrypoint.rb
|
|
19
|
+
#
|
|
20
|
+
|
|
21
|
+
require "vernier"
|
|
22
|
+
require "set"
|
|
23
|
+
require "zizq"
|
|
24
|
+
|
|
25
|
+
# --- Worker settings (same env vars as zizq-worker) ---
|
|
26
|
+
|
|
27
|
+
thread_count = Integer(ENV.fetch("ZIZQ_THREADS", Zizq::Worker::DEFAULT_THREADS))
|
|
28
|
+
fiber_count = Integer(ENV.fetch("ZIZQ_FIBERS", Zizq::Worker::DEFAULT_FIBERS))
|
|
29
|
+
prefetch = ENV.key?("ZIZQ_PREFETCH") ? Integer(ENV["ZIZQ_PREFETCH"]) : nil
|
|
30
|
+
retry_min_wait = Float(ENV.fetch("ZIZQ_RETRY_MIN_WAIT", Zizq::Worker::DEFAULT_RETRY_MIN_WAIT))
|
|
31
|
+
retry_max_wait = Float(ENV.fetch("ZIZQ_RETRY_MAX_WAIT", Zizq::Worker::DEFAULT_RETRY_MAX_WAIT))
|
|
32
|
+
retry_multiplier = Float(ENV.fetch("ZIZQ_RETRY_MULTIPLIER", Zizq::Worker::DEFAULT_RETRY_MULTIPLIER))
|
|
33
|
+
|
|
34
|
+
queues = if ENV.key?("ZIZQ_QUEUES")
|
|
35
|
+
ENV["ZIZQ_QUEUES"].split(",").map(&:strip).reject(&:empty?)
|
|
36
|
+
else
|
|
37
|
+
[]
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# --- Vernier settings ---
|
|
41
|
+
|
|
42
|
+
out = ENV.fetch("VERNIER_OUT", "profile.vernier.json")
|
|
43
|
+
interval = Integer(ENV.fetch("VERNIER_INTERVAL", 1000))
|
|
44
|
+
text_output = ENV["VERNIER_TEXT"] == "1"
|
|
45
|
+
|
|
46
|
+
# --- Load entrypoint ---
|
|
47
|
+
|
|
48
|
+
entrypoint = ARGV[0]
|
|
49
|
+
|
|
50
|
+
if entrypoint.nil?
|
|
51
|
+
warn "Usage: bundle exec bin/profile-worker <ENTRYPOINT>"
|
|
52
|
+
warn ""
|
|
53
|
+
warn "Example: ZIZQ_THREADS=5 bundle exec bin/profile-worker entrypoint.rb"
|
|
54
|
+
exit 1
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
unless File.file?(entrypoint)
|
|
58
|
+
warn "Error: entrypoint file not found: #{entrypoint}"
|
|
59
|
+
exit 1
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
require File.expand_path(entrypoint)
|
|
63
|
+
|
|
64
|
+
# --- Run under profiler ---
|
|
65
|
+
|
|
66
|
+
warn "Profiling worker (wall-clock, interval=#{interval}µs) → #{out}"
|
|
67
|
+
warn "Send SIGINT/SIGTERM to stop and write the profile."
|
|
68
|
+
|
|
69
|
+
Vernier.profile(out: out, interval: interval, allocation_sample_rate: 0) do
|
|
70
|
+
Zizq::Worker.run(
|
|
71
|
+
thread_count:,
|
|
72
|
+
fiber_count:,
|
|
73
|
+
prefetch:,
|
|
74
|
+
queues:,
|
|
75
|
+
retry_min_wait:,
|
|
76
|
+
retry_max_wait:,
|
|
77
|
+
retry_multiplier:,
|
|
78
|
+
)
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
warn "Profile written to #{out}"
|
|
82
|
+
warn "View with: https://profiler.firefox.com/ and load #{out}"
|
|
83
|
+
|
|
84
|
+
if text_output
|
|
85
|
+
require "json"
|
|
86
|
+
|
|
87
|
+
data = JSON.parse(File.read(out))
|
|
88
|
+
data["threads"].each do |thread|
|
|
89
|
+
samples = thread["samples"]
|
|
90
|
+
next unless samples && samples["length"] > 0
|
|
91
|
+
|
|
92
|
+
stack_table = thread["stackTable"]
|
|
93
|
+
frame_table = thread["frameTable"]
|
|
94
|
+
func_table = thread["funcTable"]
|
|
95
|
+
strings = thread["stringArray"]
|
|
96
|
+
|
|
97
|
+
# Count self-time: how often each function is at the top of the stack.
|
|
98
|
+
self_counts = Hash.new(0)
|
|
99
|
+
total_weight = 0.0
|
|
100
|
+
|
|
101
|
+
samples["length"].times do |i|
|
|
102
|
+
stack_idx = samples["stack"][i]
|
|
103
|
+
weight = samples["weight"] ? samples["weight"][i] : 1
|
|
104
|
+
total_weight += weight
|
|
105
|
+
|
|
106
|
+
frame_idx = stack_table["frame"][stack_idx]
|
|
107
|
+
func_idx = frame_table["func"][frame_idx]
|
|
108
|
+
name = strings[func_table["name"][func_idx]]
|
|
109
|
+
|
|
110
|
+
self_counts[name] += weight
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
# Count total-time: how often each function appears anywhere in the stack.
|
|
114
|
+
total_counts = Hash.new(0)
|
|
115
|
+
|
|
116
|
+
samples["length"].times do |i|
|
|
117
|
+
stack_idx = samples["stack"][i]
|
|
118
|
+
weight = samples["weight"] ? samples["weight"][i] : 1
|
|
119
|
+
seen = Set.new
|
|
120
|
+
|
|
121
|
+
while stack_idx
|
|
122
|
+
frame_idx = stack_table["frame"][stack_idx]
|
|
123
|
+
func_idx = frame_table["func"][frame_idx]
|
|
124
|
+
name = strings[func_table["name"][func_idx]]
|
|
125
|
+
|
|
126
|
+
total_counts[name] += weight unless seen.include?(name)
|
|
127
|
+
seen << name
|
|
128
|
+
|
|
129
|
+
stack_idx = stack_table["prefix"][stack_idx]
|
|
130
|
+
end
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
sorted = self_counts.sort_by { |_, v| -v }
|
|
134
|
+
|
|
135
|
+
warn ""
|
|
136
|
+
warn "=== Thread: #{thread["name"]} (#{samples["length"]} samples) ==="
|
|
137
|
+
warn "%8s %6s %8s %6s %s" % ["SELF", "(%)", "TOTAL", "(%)", "FUNCTION"]
|
|
138
|
+
sorted.first(40).each do |name, self_w|
|
|
139
|
+
total_w = total_counts[name]
|
|
140
|
+
self_pct = (self_w / total_weight * 100).round(1)
|
|
141
|
+
total_pct = (total_w / total_weight * 100).round(1)
|
|
142
|
+
warn "%8.1f %5.1f%% %8.1f %5.1f%% %s" % [self_w, self_pct, total_w, total_pct, name]
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
end
|
data/bin/zizq-worker
ADDED
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
# Copyright (c) 2026 Chris Corbyn <chris@zizq.io>
|
|
3
|
+
# Licensed under the MIT License. See LICENSE file for details.
|
|
4
|
+
|
|
5
|
+
# frozen_string_literal: true
|
|
6
|
+
|
|
7
|
+
require "optparse"
|
|
8
|
+
require "timeout"
|
|
9
|
+
require "zizq"
|
|
10
|
+
|
|
11
|
+
# Default deadline for a graceful `stop` before escalating to `kill`.
|
|
12
|
+
DEFAULT_SHUTDOWN_DEADLINE = 30.0
|
|
13
|
+
|
|
14
|
+
# --- Defaults from env vars, falling back to hardcoded defaults ---
|
|
15
|
+
|
|
16
|
+
thread_count = Integer(ENV.fetch("ZIZQ_THREADS", Zizq::Worker::DEFAULT_THREADS))
|
|
17
|
+
fiber_count = Integer(ENV.fetch("ZIZQ_FIBERS", Zizq::Worker::DEFAULT_FIBERS))
|
|
18
|
+
prefetch = ENV.key?("ZIZQ_PREFETCH") ? Integer(ENV["ZIZQ_PREFETCH"]) : nil
|
|
19
|
+
shutdown_deadline = Float(ENV.fetch("ZIZQ_SHUTDOWN_DEADLINE", DEFAULT_SHUTDOWN_DEADLINE))
|
|
20
|
+
retry_min_wait = Float(ENV.fetch("ZIZQ_RETRY_MIN_WAIT", Zizq::Worker::DEFAULT_RETRY_MIN_WAIT))
|
|
21
|
+
retry_max_wait = Float(ENV.fetch("ZIZQ_RETRY_MAX_WAIT", Zizq::Worker::DEFAULT_RETRY_MAX_WAIT))
|
|
22
|
+
retry_multiplier = Float(ENV.fetch("ZIZQ_RETRY_MULTIPLIER", Zizq::Worker::DEFAULT_RETRY_MULTIPLIER))
|
|
23
|
+
|
|
24
|
+
queues = if ENV.key?("ZIZQ_QUEUES")
|
|
25
|
+
ENV["ZIZQ_QUEUES"].split(",").map(&:strip).reject(&:empty?)
|
|
26
|
+
else
|
|
27
|
+
[]
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# --- CLI flag parsing (overrides env var defaults) ---
|
|
31
|
+
|
|
32
|
+
parser = OptionParser.new do |opts|
|
|
33
|
+
opts.banner = "Usage: zizq-worker [OPTIONS] <ENTRYPOINT>"
|
|
34
|
+
|
|
35
|
+
opts.separator ""
|
|
36
|
+
opts.separator "Start a Zizq worker process. The ENTRYPOINT is a Ruby file that loads your"
|
|
37
|
+
opts.separator "application (e.g. config/environment.rb for a Rails app)."
|
|
38
|
+
opts.separator ""
|
|
39
|
+
opts.separator "Client configuration (url, format, logger) should be set in the"
|
|
40
|
+
opts.separator "entrypoint via Zizq.configure."
|
|
41
|
+
opts.separator ""
|
|
42
|
+
opts.separator "Options:"
|
|
43
|
+
|
|
44
|
+
opts.on("-t", "--threads N", Integer, "Number of worker threads (default: #{Zizq::Worker::DEFAULT_THREADS}, env: ZIZQ_THREADS)") do |n|
|
|
45
|
+
thread_count = n
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
opts.on("-f", "--fibers N", Integer, "Number of fibers per thread (default: #{Zizq::Worker::DEFAULT_FIBERS}, env: ZIZQ_FIBERS)") do |n|
|
|
49
|
+
fiber_count = n
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
opts.on("-p", "--prefetch N", Integer, "Prefetch count (default: 2*threads*fibers, env: ZIZQ_PREFETCH)") do |n|
|
|
53
|
+
prefetch = n
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
queues_from_cli = false
|
|
57
|
+
opts.on("-q", "--queue QUEUE", "Queue to process (repeatable or comma-separated, env: ZIZQ_QUEUES)") do |q|
|
|
58
|
+
# First -q flag replaces the env var default entirely
|
|
59
|
+
unless queues_from_cli
|
|
60
|
+
queues = []
|
|
61
|
+
queues_from_cli = true
|
|
62
|
+
end
|
|
63
|
+
queues.concat(q.split(",").map(&:strip).reject(&:empty?))
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
opts.on("--shutdown-deadline N", Float, "Graceful shutdown deadline in seconds before escalating to kill (default: #{DEFAULT_SHUTDOWN_DEADLINE}, env: ZIZQ_SHUTDOWN_DEADLINE)") do |n|
|
|
67
|
+
shutdown_deadline = n
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
opts.on("--retry-min-wait N", Float, "Minimum retry wait in seconds (default: #{Zizq::Worker::DEFAULT_RETRY_MIN_WAIT}, env: ZIZQ_RETRY_MIN_WAIT)") do |n|
|
|
71
|
+
retry_min_wait = n
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
opts.on("--retry-max-wait N", Float, "Maximum retry wait in seconds (default: #{Zizq::Worker::DEFAULT_RETRY_MAX_WAIT}, env: ZIZQ_RETRY_MAX_WAIT)") do |n|
|
|
75
|
+
retry_max_wait = n
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
opts.on("--retry-multiplier N", Float, "Retry backoff multiplier (default: #{Zizq::Worker::DEFAULT_RETRY_MULTIPLIER}, env: ZIZQ_RETRY_MULTIPLIER)") do |n|
|
|
79
|
+
retry_multiplier = n
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
opts.on("-v", "--version", "Print version and exit") do
|
|
83
|
+
puts "zizq-worker #{Zizq::VERSION}"
|
|
84
|
+
exit
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
opts.on("-h", "--help", "Show this help message") do
|
|
88
|
+
puts opts
|
|
89
|
+
exit
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
begin
|
|
94
|
+
parser.parse!
|
|
95
|
+
rescue OptionParser::ParseError => e
|
|
96
|
+
warn e.message
|
|
97
|
+
warn "See 'zizq-worker --help' for usage information."
|
|
98
|
+
exit 1
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
# --- Validate options ---
|
|
102
|
+
|
|
103
|
+
if thread_count < 1
|
|
104
|
+
warn "Error: --threads must be at least 1 (got #{thread_count})"
|
|
105
|
+
exit 1
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
if fiber_count < 1
|
|
109
|
+
warn "Error: --fibers must be at least 1 (got #{fiber_count})"
|
|
110
|
+
exit 1
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
# --- Validate and load entrypoint ---
|
|
114
|
+
|
|
115
|
+
entrypoint = ARGV[0]
|
|
116
|
+
|
|
117
|
+
if entrypoint.nil?
|
|
118
|
+
warn "Error: missing required ENTRYPOINT argument."
|
|
119
|
+
warn ""
|
|
120
|
+
warn parser.help
|
|
121
|
+
exit 1
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
unless File.file?(entrypoint)
|
|
125
|
+
warn "Error: entrypoint file not found: #{entrypoint}"
|
|
126
|
+
exit 1
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
require File.expand_path(entrypoint)
|
|
130
|
+
|
|
131
|
+
# --- Start the worker ---
|
|
132
|
+
|
|
133
|
+
worker = Zizq::Worker.new(
|
|
134
|
+
thread_count:,
|
|
135
|
+
fiber_count:,
|
|
136
|
+
prefetch:,
|
|
137
|
+
queues:,
|
|
138
|
+
retry_min_wait:,
|
|
139
|
+
retry_max_wait:,
|
|
140
|
+
retry_multiplier:,
|
|
141
|
+
)
|
|
142
|
+
|
|
143
|
+
# `Zizq::Worker#stop` is patient (waits forever for in-flight jobs and
|
|
144
|
+
# acks to drain). We enforce the shutdown deadline at the CLI level: the
|
|
145
|
+
# signal handler calls `worker.stop` and spawns a shutdown thread that
|
|
146
|
+
# joins the worker with `Timeout::timeout`, escalating to `kill` on
|
|
147
|
+
# expiry.
|
|
148
|
+
worker_thread = Thread.new { worker.run }
|
|
149
|
+
|
|
150
|
+
# First INT/TERM: graceful stop with deadline watchdog.
|
|
151
|
+
# Second INT/TERM: hard process exit.
|
|
152
|
+
stopping = false
|
|
153
|
+
|
|
154
|
+
%w[INT TERM].each do |signal|
|
|
155
|
+
Signal.trap(signal) do
|
|
156
|
+
if stopping
|
|
157
|
+
exit(1)
|
|
158
|
+
else
|
|
159
|
+
worker.stop
|
|
160
|
+
stopping = true
|
|
161
|
+
Thread.new do
|
|
162
|
+
Timeout.timeout(shutdown_deadline) { worker_thread.join }
|
|
163
|
+
rescue Timeout::Error
|
|
164
|
+
worker.logger.warn do
|
|
165
|
+
"Worker did not stop within #{shutdown_deadline}s, killing..."
|
|
166
|
+
end
|
|
167
|
+
exit(1)
|
|
168
|
+
end
|
|
169
|
+
end
|
|
170
|
+
end
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
# Wait until the worker finishes cleanly.
|
|
174
|
+
worker_thread.join
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "zizq"
|
|
4
|
+
require "zizq/active_job_config"
|
|
5
|
+
|
|
6
|
+
module ActiveJob
|
|
7
|
+
module QueueAdapters
|
|
8
|
+
# ActiveJob adapter for Zizq jobs.
|
|
9
|
+
#
|
|
10
|
+
# To use, set the queue adapter in your Rails configuration:
|
|
11
|
+
#
|
|
12
|
+
# # config/application.rb
|
|
13
|
+
# config.active_job.queue_adapter = :zizq
|
|
14
|
+
#
|
|
15
|
+
# And configure the Zizq client to dispatch to ActiveJob:
|
|
16
|
+
#
|
|
17
|
+
# # config/initializers/zizq.rb
|
|
18
|
+
# Zizq.configure do |c|
|
|
19
|
+
# c.url = "http://localhost:7890"
|
|
20
|
+
# c.dispatcher = ActiveJob::QueueAdapters::ZizqAdapter::Dispatcher
|
|
21
|
+
# end
|
|
22
|
+
#
|
|
23
|
+
# To use Zizq features (unique jobs, backoff, retention) with ActiveJob
|
|
24
|
+
# classes, you can extend `Zizq::ActiveJobConfig` in your classes:
|
|
25
|
+
#
|
|
26
|
+
# class SendEmailJob < ApplicationJob
|
|
27
|
+
# extend Zizq::ActiveJobConfig
|
|
28
|
+
#
|
|
29
|
+
# zizq_unique true, scope: :active
|
|
30
|
+
# zizq_backoff exponent: 4.0, base: 15, jitter: 30
|
|
31
|
+
# end
|
|
32
|
+
#
|
|
33
|
+
class ZizqAdapter
|
|
34
|
+
# Enqueue a job for immediate execution.
|
|
35
|
+
def enqueue(job)
|
|
36
|
+
result = Zizq.enqueue_raw(**build_enqueue_request(job).to_enqueue_params)
|
|
37
|
+
job.provider_job_id = result.id
|
|
38
|
+
job.successfully_enqueued = true
|
|
39
|
+
result
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# Enqueue a job for execution at a specific time.
|
|
43
|
+
def enqueue_at(job, timestamp)
|
|
44
|
+
job.scheduled_at = timestamp
|
|
45
|
+
result = Zizq.enqueue_raw(**build_enqueue_request(job).to_enqueue_params)
|
|
46
|
+
job.provider_job_id = result.id
|
|
47
|
+
job.successfully_enqueued = true
|
|
48
|
+
result
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# Enqueue multiple jobs atomically in a single bulk request.
|
|
52
|
+
#
|
|
53
|
+
# Called by `ActiveJob.perform_all_later` (Rails 7.1+).
|
|
54
|
+
# Returns the number of successfully enqueued jobs.
|
|
55
|
+
def enqueue_all(jobs)
|
|
56
|
+
results = Zizq.enqueue_bulk do |b|
|
|
57
|
+
jobs.each do |job|
|
|
58
|
+
b.enqueue_raw(**build_enqueue_request(job).to_enqueue_params)
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
jobs.zip(results).each do |job, result|
|
|
63
|
+
job.provider_job_id = result.id
|
|
64
|
+
job.successfully_enqueued = true
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
jobs.size
|
|
68
|
+
rescue => e
|
|
69
|
+
jobs.each { |job| job.successfully_enqueued = false }
|
|
70
|
+
raise e
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# Dispatcher for Zizq workers that executes ActiveJob payloads.
|
|
74
|
+
#
|
|
75
|
+
# ActiveJob handles its own deserialization, callbacks, and error
|
|
76
|
+
# handling. We just pass the serialized payload to `Base.execute`.
|
|
77
|
+
module Dispatcher
|
|
78
|
+
def self.call(job)
|
|
79
|
+
ActiveJob::Base.execute(job.payload)
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
private
|
|
84
|
+
|
|
85
|
+
def build_enqueue_request(job)
|
|
86
|
+
klass = job.class
|
|
87
|
+
|
|
88
|
+
req = Zizq::EnqueueRequest.new(
|
|
89
|
+
queue: job.queue_name,
|
|
90
|
+
type: klass.name,
|
|
91
|
+
payload: job.serialize,
|
|
92
|
+
priority: job.priority,
|
|
93
|
+
ready_at: job.scheduled_at
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
if klass.respond_to?(:zizq_unique) && klass.zizq_unique
|
|
97
|
+
req.unique_key = klass.zizq_unique_key(*job.arguments)
|
|
98
|
+
req.unique_while = klass.zizq_unique_scope
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
req.retry_limit = klass.zizq_retry_limit if klass.respond_to?(:zizq_retry_limit) && klass.zizq_retry_limit
|
|
102
|
+
req.backoff = klass.zizq_backoff if klass.respond_to?(:zizq_backoff) && klass.zizq_backoff
|
|
103
|
+
req.retention = klass.zizq_retention if klass.respond_to?(:zizq_retention) && klass.zizq_retention
|
|
104
|
+
|
|
105
|
+
req
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
end
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
# Copyright (c) 2026 Chris Corbyn <chris@zizq.io>
|
|
2
|
+
# Licensed under the MIT License. See LICENSE file for details.
|
|
3
|
+
|
|
4
|
+
# rbs_inline: enabled
|
|
5
|
+
# frozen_string_literal: true
|
|
6
|
+
|
|
7
|
+
require "async"
|
|
8
|
+
require "async/barrier"
|
|
9
|
+
|
|
10
|
+
module Zizq
|
|
11
|
+
# Dedicated background thread that processes ack/nack HTTP requests on
|
|
12
|
+
# behalf of worker threads, decoupling job processing from network I/O.
|
|
13
|
+
#
|
|
14
|
+
# Workers push Ack/Nack items to a thread-safe queue. The processor runs
|
|
15
|
+
# an async event loop that spawns an independent fiber per ack/nack
|
|
16
|
+
# request, enabling true concurrent I/O over a single HTTP/2 connection.
|
|
17
|
+
# Each fiber handles its own retries with exponential backoff.
|
|
18
|
+
class AckProcessor
|
|
19
|
+
# Immutable value object representing a successful job completion.
|
|
20
|
+
Ack = Data.define(:job_id)
|
|
21
|
+
|
|
22
|
+
# Immutable value object representing a job failure.
|
|
23
|
+
Nack = Data.define(:job_id, :message, :error_type, :backtrace)
|
|
24
|
+
|
|
25
|
+
# @rbs client: Client
|
|
26
|
+
# @rbs capacity: Integer
|
|
27
|
+
# @rbs logger: Logger
|
|
28
|
+
# @rbs backoff: Backoff
|
|
29
|
+
# @rbs return: void
|
|
30
|
+
def initialize(client:, capacity:, logger:, backoff:)
|
|
31
|
+
@client = client
|
|
32
|
+
@logger = logger
|
|
33
|
+
@backoff = backoff
|
|
34
|
+
@queue = Thread::SizedQueue.new(capacity)
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# Push an Ack or Nack to the processing queue.
|
|
38
|
+
# Blocks if the queue is at capacity (backpressure).
|
|
39
|
+
#
|
|
40
|
+
# @rbs item: Ack | Nack
|
|
41
|
+
# @rbs return: void
|
|
42
|
+
def push(item)
|
|
43
|
+
@queue.push(item)
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# Start the background processor thread.
|
|
47
|
+
def start #: () -> Thread
|
|
48
|
+
@thread = Thread.new { run }
|
|
49
|
+
@thread.name = "zizq-ack-processor"
|
|
50
|
+
@thread
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# Close the queue and wait for the processor to drain. Waits indefinitely —
|
|
54
|
+
# callers who want a deadline should wrap the call in `Timeout::timeout`.
|
|
55
|
+
#
|
|
56
|
+
# @rbs return: void
|
|
57
|
+
def stop
|
|
58
|
+
@queue.close
|
|
59
|
+
@thread&.join
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
private
|
|
63
|
+
|
|
64
|
+
def run #: () -> void
|
|
65
|
+
Sync do
|
|
66
|
+
barrier = Async::Barrier.new
|
|
67
|
+
|
|
68
|
+
while (item = @queue.pop)
|
|
69
|
+
# Put the item into a batch.
|
|
70
|
+
batch = [item]
|
|
71
|
+
|
|
72
|
+
# Drain any additional ready items into the batch.
|
|
73
|
+
loop do
|
|
74
|
+
batch << @queue.pop(true) # non-blocking
|
|
75
|
+
rescue ThreadError
|
|
76
|
+
break
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# Partition: acks go bulk, nacks go individually.
|
|
80
|
+
acks, nacks = batch.partition { |i| i.is_a?(Ack) }
|
|
81
|
+
|
|
82
|
+
unless acks.empty?
|
|
83
|
+
barrier.async { process_ack_batch(acks) }
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
nacks.each do |nack|
|
|
87
|
+
barrier.async { process_nack(nack) }
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
barrier.wait
|
|
92
|
+
end
|
|
93
|
+
rescue => e
|
|
94
|
+
@logger.error { "Ack processor crashed: #{e.class}: #{e.message}" }
|
|
95
|
+
@logger.debug { e.backtrace&.join("\n") }
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def process_ack_batch(acks) #: (Array[Ack]) -> void
|
|
99
|
+
backoff = @backoff.fresh
|
|
100
|
+
ids = acks.map(&:job_id)
|
|
101
|
+
begin
|
|
102
|
+
@client.report_success_bulk(ids)
|
|
103
|
+
rescue ClientError => e
|
|
104
|
+
@logger.warn { "Bulk ack (#{ids.size} jobs) returned #{e.status} (dropping: #{e.message})" }
|
|
105
|
+
rescue => e
|
|
106
|
+
@logger.warn { "Retrying bulk ack (#{ids.size} jobs) in #{backoff.duration}s: #{e.message}" }
|
|
107
|
+
backoff.wait
|
|
108
|
+
retry
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
def process_nack(nack) #: (Nack) -> void
|
|
113
|
+
backoff = @backoff.fresh
|
|
114
|
+
begin
|
|
115
|
+
@client.report_failure(
|
|
116
|
+
nack.job_id,
|
|
117
|
+
message: nack.message,
|
|
118
|
+
error_type: nack.error_type,
|
|
119
|
+
backtrace: nack.backtrace
|
|
120
|
+
)
|
|
121
|
+
rescue NotFoundError
|
|
122
|
+
@logger.debug { "Nack for #{nack.job_id} returned 404 (already handled)" }
|
|
123
|
+
rescue ClientError => e
|
|
124
|
+
@logger.error { "Nack for #{nack.job_id} returned #{e.status} (dropping)" }
|
|
125
|
+
rescue => e
|
|
126
|
+
@logger.warn { "Retrying nack for #{nack.job_id} in #{backoff.duration}s: #{e.message}" }
|
|
127
|
+
backoff.wait
|
|
128
|
+
retry
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
end
|
|
132
|
+
end
|