tuber 0.0.1 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ruby.yml +68 -0
- data/.gitignore +17 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +112 -0
- data/Gemfile +16 -0
- data/LICENSE.txt +27 -0
- data/README.md +379 -8
- data/Rakefile +35 -0
- data/TODO +1 -0
- data/examples/demo.rb +97 -0
- data/lib/tuber/configuration.rb +31 -0
- data/lib/tuber/connection.rb +366 -0
- data/lib/tuber/errors.rb +78 -0
- data/lib/tuber/job/collection.rb +160 -0
- data/lib/tuber/job/record.rb +226 -0
- data/lib/tuber/job.rb +4 -0
- data/lib/tuber/stats/fast_struct.rb +98 -0
- data/lib/tuber/stats/stat_struct.rb +52 -0
- data/lib/tuber/stats.rb +81 -0
- data/lib/tuber/tube/collection.rb +253 -0
- data/lib/tuber/tube/record.rb +231 -0
- data/lib/tuber/tube.rb +4 -0
- data/lib/tuber/version.rb +6 -0
- data/lib/tuber.rb +104 -2
- data/test/connection_test.rb +291 -0
- data/test/errors_test.rb +35 -0
- data/test/job_test.rb +250 -0
- data/test/jobs_test.rb +152 -0
- data/test/prompt_regexp_test.rb +59 -0
- data/test/stat_struct_test.rb +58 -0
- data/test/stats_test.rb +45 -0
- data/test/test_helper.rb +74 -0
- data/test/tube_test.rb +299 -0
- data/test/tuber_test.rb +112 -0
- data/test/tubes_test.rb +329 -0
- data/tuber.gemspec +33 -0
- metadata +112 -6
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 5bf9725eaee223b9a587d41a70472161997c0792a7e3ad7917488e8e5dbacb62
|
|
4
|
+
data.tar.gz: 4759d84c9fae6eda87ed77f26817b6eae8d88fb2d80addf98723530c99551f7b
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 712f68857c4fcbedeaee298e94499abc9134337b1cd30de8f32c6a77fc8e640062bd6b1f0ba45122031d79a3c85cba7622c228f2d4447427e0d90c95668c6c47
|
|
7
|
+
data.tar.gz: 6bc153d12ffc33d3dcdd772ea31e8b2675a672779ace7812748bbd00c64346382bfea7c7d843ed0a24e610117f638f81445e0696dcfffa11633d93cc275cccd4
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
# CI runs the full suite against Tuber (the primary target) and the portable
|
|
2
|
+
# subset against stock beanstalkd (Tuber-only tests skip via require_tuber!).
|
|
3
|
+
|
|
4
|
+
name: Build Status
|
|
5
|
+
|
|
6
|
+
on:
|
|
7
|
+
push:
|
|
8
|
+
branches: [ "main" ]
|
|
9
|
+
pull_request:
|
|
10
|
+
branches: [ "main" ]
|
|
11
|
+
|
|
12
|
+
permissions:
|
|
13
|
+
contents: read
|
|
14
|
+
|
|
15
|
+
jobs:
|
|
16
|
+
test:
|
|
17
|
+
name: tuber (ruby ${{ matrix.ruby-version }})
|
|
18
|
+
runs-on: ubuntu-latest
|
|
19
|
+
strategy:
|
|
20
|
+
fail-fast: false
|
|
21
|
+
matrix:
|
|
22
|
+
ruby-version: ["2.7", "3.0", "3.1", "3.2", "3.3", "3.4", "4.0"]
|
|
23
|
+
|
|
24
|
+
steps:
|
|
25
|
+
- uses: actions/checkout@v4
|
|
26
|
+
# Not a service container: the image's entrypoint needs the `server`
|
|
27
|
+
# subcommand, and services can't pass a command.
|
|
28
|
+
- name: Start tuber
|
|
29
|
+
run: |
|
|
30
|
+
docker run -d --name tuber -p 11300:11300 ghcr.io/tuberq/tuber:latest server
|
|
31
|
+
for i in $(seq 1 30); do (echo > /dev/tcp/127.0.0.1/11300) 2>/dev/null && exit 0; sleep 1; done
|
|
32
|
+
docker logs tuber; exit 1
|
|
33
|
+
- name: Set up Ruby
|
|
34
|
+
uses: ruby/setup-ruby@v1
|
|
35
|
+
with:
|
|
36
|
+
ruby-version: ${{ matrix.ruby-version }}
|
|
37
|
+
bundler-cache: true
|
|
38
|
+
- name: Run tests
|
|
39
|
+
run: bundle exec rake test:full
|
|
40
|
+
env:
|
|
41
|
+
TUBER_TEST_ADDRESS: 127.0.0.1
|
|
42
|
+
|
|
43
|
+
beanstalkd-compat:
|
|
44
|
+
name: beanstalkd compat (ruby ${{ matrix.ruby-version }})
|
|
45
|
+
runs-on: ubuntu-latest
|
|
46
|
+
strategy:
|
|
47
|
+
fail-fast: false
|
|
48
|
+
matrix:
|
|
49
|
+
ruby-version: ["2.7", "4.0"]
|
|
50
|
+
|
|
51
|
+
steps:
|
|
52
|
+
- uses: actions/checkout@v4
|
|
53
|
+
- name: Download & Extract beanstalkd
|
|
54
|
+
run: curl -L https://github.com/beanstalkd/beanstalkd/archive/refs/tags/v1.13.tar.gz | tar xz
|
|
55
|
+
- name: Make beanstalkd
|
|
56
|
+
run: make
|
|
57
|
+
working-directory: beanstalkd-1.13
|
|
58
|
+
- name: Set up Ruby
|
|
59
|
+
uses: ruby/setup-ruby@v1
|
|
60
|
+
with:
|
|
61
|
+
ruby-version: ${{ matrix.ruby-version }}
|
|
62
|
+
bundler-cache: true
|
|
63
|
+
- name: Daemonize beanstalkd
|
|
64
|
+
run: ./beanstalkd-1.13/beanstalkd -l 127.0.0.1 &
|
|
65
|
+
- name: Run tests
|
|
66
|
+
run: bundle exec rake test:full
|
|
67
|
+
env:
|
|
68
|
+
TUBER_TEST_ADDRESS: 127.0.0.1
|
data/.gitignore
ADDED
data/.yardopts
ADDED
data/CHANGELOG.md
ADDED
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
# CHANGELOG for the tuber gem
|
|
2
|
+
|
|
3
|
+
The tuber gem is a fork of [beaneater](https://github.com/beanstalkd/beaneater), taken at
|
|
4
|
+
beaneater 1.1.4. Entries from 1.1.4 down are beaneater's history, kept for reference.
|
|
5
|
+
(0.0.1 was an empty placeholder published to claim the gem name.)
|
|
6
|
+
|
|
7
|
+
## 0.5.1 (August 13 2026)
|
|
8
|
+
|
|
9
|
+
* Fix empty keyword arguments forwarded through `Tube#transmit`, `Tubes#transmit` and `Jobs#transmit` arriving as an extra positional `{}` on Ruby 2.7
|
|
10
|
+
* CI: start the Tuber container with an explicit `server` subcommand, add Ruby 4.0 to the test matrix
|
|
11
|
+
|
|
12
|
+
## 0.5.0 (August 13 2026)
|
|
13
|
+
|
|
14
|
+
First functional release under the `tuber` name. Versioning restarts below 1.0
|
|
15
|
+
while the Tuber-first API settles; 1.0.0 will follow once it has. Entries below
|
|
16
|
+
1.1.4 with earlier version numbers are beaneater history.
|
|
17
|
+
|
|
18
|
+
* **Renamed from `beaneater` to `tuber`** (briefly `russet` while the gem name was being freed up, though never published under that name). The `Beaneater` class is now `Tuber`, and `require 'beaneater'` becomes `require 'tuber'`. Nothing else changed in the rename — see "Migrating from beaneater" in the README.
|
|
19
|
+
* Fix `_with_retry` retransmitting non-idempotent commands (`put`/`delete`/`release`/`bury`/`touch`/`kick` and batch variants) after a reconnect: the first send's fate is unknown, so a re-sent `put` could duplicate a job and a re-sent `delete` could raise `NOT_FOUND` for work that succeeded. These verbs now reconnect (and re-watch tubes) but re-raise the original connection error so the caller decides; idempotent commands keep the transparent retry.
|
|
20
|
+
* Add `Jobs#touch_all` (Tuber 0.12.0+): one-command heartbeat for every job the connection holds, returning how many were still held
|
|
21
|
+
* Add `Tube#flush_buried` (Tuber only): atomically deletes only the buried jobs in a tube, returning the count
|
|
22
|
+
* Add optional long-poll `timeout` to `reserve_batch` (Tuber only): `tubes.reserve_batch(count, timeout)` blocks up to `timeout` seconds for the first job, then drains what's ready
|
|
23
|
+
* Add `connect_timeout` (Ruby 3+), `resolv_timeout` (Ruby 3+), `read_timeout`, and `write_timeout` configuration options (@rveznaver)
|
|
24
|
+
* Fix fractional timeout handling for `read_timeout` and `write_timeout` socket options
|
|
25
|
+
* Add `TUBER_URL` environment variable and `config.tuber_url` as the primary way to point at a server; `BEANSTALKD_URL` and `config.beanstalkd_url` remain as compatible fallbacks/aliases
|
|
26
|
+
* Tests can target a specific server via `TUBER_TEST_ADDRESS`, and Tuber-only tests skip automatically against stock beanstalkd; CI runs the full suite against Tuber and the portable subset against beanstalkd
|
|
27
|
+
* Fix `StatStruct#[]` raising a `SyntaxError` when looking up hyphenated beanstalkd keys (e.g. `stats['current-jobs-ready']`); keys are now underscored before lookup to match how they are stored
|
|
28
|
+
|
|
29
|
+
## 1.1.4 (March 6 2026)
|
|
30
|
+
|
|
31
|
+
* Add frozen string literal comments for Ruby 3.4 compatibility (@nearapogee)
|
|
32
|
+
* Update tests for Ruby 3.4.x (@nearapogee)
|
|
33
|
+
* Add GitHub Actions CI workflow (@Ifiht)
|
|
34
|
+
* Update README.md (@Ifiht)
|
|
35
|
+
|
|
36
|
+
## 1.1.3 (Oct 14 2022)
|
|
37
|
+
|
|
38
|
+
* Fixes issue introduced in 1.1.2 re YML parsing (@pond)
|
|
39
|
+
* Fix job lookup test so it passes again
|
|
40
|
+
|
|
41
|
+
## 1.1.2 (Oct 10 2022)
|
|
42
|
+
|
|
43
|
+
* Fixes beaneater when used with Ruby 3.1 and YAML 4 (@pond)
|
|
44
|
+
|
|
45
|
+
## 1.1.1 (April 27th 2021)
|
|
46
|
+
|
|
47
|
+
* Support Ruby 3 keyword arguments (@yahonda)
|
|
48
|
+
* Add CI for Ruby 3 (@yahonda)
|
|
49
|
+
|
|
50
|
+
## 1.1.0 (April 25th 2021)
|
|
51
|
+
|
|
52
|
+
* 'clear' behavior was failing unexpectedly, swallow issues during delete as well (@bfolkens)
|
|
53
|
+
* Fix assigned but unused variables (@utilum)
|
|
54
|
+
* Fix last_used tube not stored (@albb0920)
|
|
55
|
+
* Fix deprecation warning in ruby 2.7 (@albb0920)
|
|
56
|
+
* Fix watched tubes not restored after reconnect (@albb0920)
|
|
57
|
+
* Fix keyword arguemnt warning (@albb0920)
|
|
58
|
+
|
|
59
|
+
## 1.0.0 (April 26th 2015)
|
|
60
|
+
|
|
61
|
+
* Beginning from version 1.0.0 the support for `Beaneater::Pool` has been dropped (@alup)
|
|
62
|
+
* `Jobs#find_all` method has been removed, since it is no longer necessary after removing pool (@alup)
|
|
63
|
+
* `Tubes` is now an enumerable allowing `tubes` to be handled as a collection (@Aethelflaed)
|
|
64
|
+
|
|
65
|
+
## 0.3.3 (August 16th 2014)
|
|
66
|
+
|
|
67
|
+
* Fix failure when job is not defined and fix exception handling for jobs (@nicholasorenrawlings)
|
|
68
|
+
* Add reserve_timeout option to job processing (@nicholasorenrawlings)
|
|
69
|
+
* Add travis-ci badge (@tdg5)
|
|
70
|
+
* Fix tests to run more reliably (@tdg5)
|
|
71
|
+
|
|
72
|
+
## 0.3.2 (Sept 15 2013)
|
|
73
|
+
|
|
74
|
+
* Fix #29 ExpectedCrlfError name and invocation (@tdg5)
|
|
75
|
+
|
|
76
|
+
## 0.3.1 (Jun 28 2013)
|
|
77
|
+
|
|
78
|
+
* Fixes issue with "chomp" nil exception when losing connection (Thanks @simao)
|
|
79
|
+
* Better handling of unknown or invalid commands during transmit
|
|
80
|
+
* Raise proper CRLF exception (Thanks @carlosmoutinho)
|
|
81
|
+
|
|
82
|
+
## 0.3.0 (Jan 23 2013)
|
|
83
|
+
|
|
84
|
+
* Replace Telnet with tcpsocket thanks @vidarh
|
|
85
|
+
|
|
86
|
+
## 0.2.2 (Dec 2 2012)
|
|
87
|
+
|
|
88
|
+
* Fixes status and ID parsing in a response (Thanks @justincase)
|
|
89
|
+
|
|
90
|
+
## 0.2.1 (Dec 1 2012)
|
|
91
|
+
|
|
92
|
+
* Convert command to ASCII_8Bit to avoid gsub issues
|
|
93
|
+
|
|
94
|
+
## 0.2.0 (Nov 12 2012)
|
|
95
|
+
* Fix 1.8.7 compatibility issues
|
|
96
|
+
* Add configuration block to beaneater and better job parsing
|
|
97
|
+
* BREAKING: json jobs now return as string by default not hashes
|
|
98
|
+
|
|
99
|
+
## 0.1.2 (Nov 7 2012)
|
|
100
|
+
|
|
101
|
+
* Add timeout: false to transmit to allow longer reserve
|
|
102
|
+
|
|
103
|
+
## 0.1.1 (Nov 4 2012)
|
|
104
|
+
|
|
105
|
+
* Add `Jobs#find_all` to fix #10
|
|
106
|
+
* Fixed issue with `tubes-list` by merging results
|
|
107
|
+
* Add `Job#ttr`, `Job#pri`, `Job#delay`
|
|
108
|
+
* Improved yardocs coverage and accuracy
|
|
109
|
+
|
|
110
|
+
## 0.1.0 (Nov 1 2012)
|
|
111
|
+
|
|
112
|
+
* Initial release!
|
data/Gemfile
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
source 'https://rubygems.org'
|
|
4
|
+
|
|
5
|
+
# Specify your gem's dependencies in tuber.gemspec
|
|
6
|
+
gemspec
|
|
7
|
+
|
|
8
|
+
group :development do
|
|
9
|
+
gem 'redcarpet', '~> 1'
|
|
10
|
+
gem 'github-markup'
|
|
11
|
+
gem 'yard'
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
group :development, :test do
|
|
15
|
+
gem 'coveralls', :require => false
|
|
16
|
+
end
|
data/LICENSE.txt
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
Copyright (c) 2012 Nico Taing
|
|
2
|
+
Copyright (c) 2026 Dan Milne
|
|
3
|
+
|
|
4
|
+
Tuber is a fork of beaneater (https://github.com/beanstalkd/beaneater),
|
|
5
|
+
originally written by Nico Taing and contributors, and is distributed under
|
|
6
|
+
the same MIT License.
|
|
7
|
+
|
|
8
|
+
MIT License
|
|
9
|
+
|
|
10
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
|
11
|
+
a copy of this software and associated documentation files (the
|
|
12
|
+
"Software"), to deal in the Software without restriction, including
|
|
13
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
|
14
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
|
15
|
+
permit persons to whom the Software is furnished to do so, subject to
|
|
16
|
+
the following conditions:
|
|
17
|
+
|
|
18
|
+
The above copyright notice and this permission notice shall be
|
|
19
|
+
included in all copies or substantial portions of the Software.
|
|
20
|
+
|
|
21
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
|
22
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
|
23
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
|
24
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
|
25
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
|
26
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
|
27
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
CHANGED
|
@@ -1,14 +1,385 @@
|
|
|
1
|
-
#
|
|
1
|
+
# Tuber
|
|
2
2
|
|
|
3
|
-
Ruby client for [Tuber](https://github.com/tuberq/tuber),
|
|
3
|
+
The Ruby client for [Tuber](https://github.com/tuberq/tuber), the simple, fast job queue server. One binary, zero dependencies — with unique/idempotent jobs, concurrency keys, job group pipelines, weighted tubes and batch operations built in.
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
Tuber (the server and this gem) is wire-compatible with [beanstalkd](https://github.com/beanstalkd/beanstalkd): point this client at a stock beanstalkd and everything except the Tuber-only extensions works unchanged. The gem is a fork of [beaneater](https://github.com/beanstalkd/beaneater), so it is also a drop-in beaneater replacement — see [Migrating from beaneater](#migrating-from-beaneater).
|
|
6
6
|
|
|
7
|
-
##
|
|
7
|
+
## Quick Start
|
|
8
8
|
|
|
9
|
-
|
|
10
|
-
|
|
9
|
+
```ruby
|
|
10
|
+
@tuber = Tuber.new('localhost:11300')
|
|
11
11
|
|
|
12
|
-
|
|
12
|
+
@tube = @tuber.tubes["my-tube"]
|
|
13
|
+
@tube.put '{"key": "foo"}', pri: 5
|
|
14
|
+
@tube.put '{"key": "bar"}', delay: 3
|
|
13
15
|
|
|
14
|
-
|
|
16
|
+
while @tube.peek(:ready)
|
|
17
|
+
job = @tube.reserve
|
|
18
|
+
puts job.body
|
|
19
|
+
job.delete
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
@tuber.close
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
## Installation
|
|
26
|
+
|
|
27
|
+
Install the [tuber server](https://github.com/tuberq/tuber) (a single binary; `brew install tuberq/tuber/tuber`, Docker image `ghcr.io/tuberq/tuber`, or a [release binary](https://github.com/tuberq/tuber/releases)) — or use an existing beanstalkd — then add the gem:
|
|
28
|
+
|
|
29
|
+
```ruby
|
|
30
|
+
# Gemfile
|
|
31
|
+
gem 'tuber'
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
## Migrating from beaneater
|
|
35
|
+
|
|
36
|
+
Tuber forked from beaneater 1.1.4. The rename is the only breaking change — every
|
|
37
|
+
method, option and return value is unchanged. Two things to update:
|
|
38
|
+
|
|
39
|
+
```ruby
|
|
40
|
+
require 'beaneater' # before
|
|
41
|
+
require 'tuber' # after
|
|
42
|
+
|
|
43
|
+
@beanstalk = Beaneater.new('localhost:11300') # before
|
|
44
|
+
@beanstalk = Tuber.new('localhost:11300') # after
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
That includes the nested constants (`Beaneater::Job` → `Tuber::Job`), the error
|
|
48
|
+
classes (`Beaneater::NotConnected` → `Tuber::NotConnected`) and the configuration
|
|
49
|
+
block (`Beaneater.configure` → `Tuber.configure`). In most codebases a
|
|
50
|
+
case-sensitive `Beaneater` → `Tuber` and `beaneater` → `tuber` replacement is the
|
|
51
|
+
whole migration.
|
|
52
|
+
|
|
53
|
+
The `BEANSTALKD_URL` environment variable is still honoured (it's a beanstalkd
|
|
54
|
+
convention, not a beaneater one), though `TUBER_URL` now takes precedence.
|
|
55
|
+
Likewise `config.beanstalkd_url` remains as an alias for `config.tuber_url`.
|
|
56
|
+
|
|
57
|
+
## Usage
|
|
58
|
+
|
|
59
|
+
### Configuration
|
|
60
|
+
|
|
61
|
+
To setup advanced options for tuber, you can pass configuration options using:
|
|
62
|
+
|
|
63
|
+
```ruby
|
|
64
|
+
Tuber.configure do |config|
|
|
65
|
+
# config.default_put_delay = 0
|
|
66
|
+
# config.default_put_pri = 65536
|
|
67
|
+
# config.default_put_ttr = 120
|
|
68
|
+
# config.job_parser = lambda { |body| body }
|
|
69
|
+
# config.job_serializer = lambda { |body| body }
|
|
70
|
+
# config.tuber_url = 'localhost:11300'
|
|
71
|
+
# config.connect_timeout = nil
|
|
72
|
+
# config.resolv_timeout = nil
|
|
73
|
+
# config.read_timeout = nil
|
|
74
|
+
# config.write_timeout = nil
|
|
75
|
+
end
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
The above options are all defaults, so only include a configuration block if you need to make changes.
|
|
79
|
+
|
|
80
|
+
`connect_timeout` and `resolv_timeout` are passed through to `TCPSocket.new` on Ruby 3.0 and newer (ignored on Ruby < 3.0 for compatibility). `read_timeout` and `write_timeout` apply socket read and write timeouts via `setsockopt`.
|
|
81
|
+
|
|
82
|
+
### Connection
|
|
83
|
+
|
|
84
|
+
```ruby
|
|
85
|
+
@tuber = Tuber.new('10.0.1.5:11300')
|
|
86
|
+
|
|
87
|
+
# Or use ENV['TUBER_URL'] (or ENV['BEANSTALKD_URL'])
|
|
88
|
+
@tuber = Tuber.new
|
|
89
|
+
|
|
90
|
+
@tuber.close
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
### Tubes
|
|
94
|
+
|
|
95
|
+
Tubes are named work queues. Jobs are `put` into the used tube and `reserve`d from watched tubes. Each tube has a _ready_, _delayed_, and _buried_ queue.
|
|
96
|
+
|
|
97
|
+
```ruby
|
|
98
|
+
@tube = @tuber.tubes.find("some-tube")
|
|
99
|
+
|
|
100
|
+
# Watch tubes for reserving jobs
|
|
101
|
+
@tuber.tubes.watch!('some-tube') # watch only these tubes
|
|
102
|
+
@tuber.tubes.watch('another-tube') # append to watch list
|
|
103
|
+
@tuber.tubes.ignore('some-tube') # stop watching
|
|
104
|
+
|
|
105
|
+
# List tubes
|
|
106
|
+
@tuber.tubes.all # => [<Tube name='foo'>, <Tube name='bar'>]
|
|
107
|
+
@tuber.tubes.used # => <Tube name='bar'>
|
|
108
|
+
@tuber.tubes.watched # => [<Tube name='foo'>]
|
|
109
|
+
|
|
110
|
+
# Manage tubes
|
|
111
|
+
@tube.pause(3) # pause for 3 seconds
|
|
112
|
+
@tube.clear # delete all jobs
|
|
113
|
+
@tube.flush # delete all jobs, returns count
|
|
114
|
+
@tube.flush_buried # delete only buried jobs (Tuber only), returns count
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
Each client manages two separate concerns: **use**/**using** controls where `put` places jobs, and **watch**/**watching** controls where `reserve` takes jobs from. These are fully orthogonal.
|
|
118
|
+
|
|
119
|
+
### Jobs
|
|
120
|
+
|
|
121
|
+
A job has a body (string) and metadata. The typical lifecycle:
|
|
122
|
+
|
|
123
|
+
```
|
|
124
|
+
put reserve delete
|
|
125
|
+
-----> [READY] ---------> [RESERVED] --------> *poof*
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
Jobs are in one of three states:
|
|
129
|
+
|
|
130
|
+
| State | Description |
|
|
131
|
+
| ------- | ----------- |
|
|
132
|
+
| ready | Waiting to be reserved and processed. |
|
|
133
|
+
| delayed | Waiting to become ready after a delay. |
|
|
134
|
+
| buried | Held aside after failure, waiting to be kicked. |
|
|
135
|
+
|
|
136
|
+
#### Inserting jobs
|
|
137
|
+
|
|
138
|
+
```ruby
|
|
139
|
+
@tube.put "job-data-here"
|
|
140
|
+
@tube.put({foo: 'bar'}.to_json)
|
|
141
|
+
@tube.put "job-data-here", pri: 1000, delay: 50, ttr: 200
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
- **pri** — integer < 2^32, lower values run first (default: 65536)
|
|
145
|
+
- **delay** — seconds to wait before the job becomes ready (default: 0)
|
|
146
|
+
- **ttr** — time to run, seconds a worker has to finish the job (default: 120)
|
|
147
|
+
|
|
148
|
+
#### Reserving and processing jobs
|
|
149
|
+
|
|
150
|
+
```ruby
|
|
151
|
+
job = @tuber.tubes.reserve # blocks until a job is available
|
|
152
|
+
job = @tuber.tubes.reserve(5) # wait up to 5 seconds
|
|
153
|
+
|
|
154
|
+
puts job.body
|
|
155
|
+
puts job.tube
|
|
156
|
+
puts job.stats.state # => 'reserved'
|
|
157
|
+
|
|
158
|
+
job.touch # extend ttr
|
|
159
|
+
job.delete # success
|
|
160
|
+
job.release delay: 5 # retry later
|
|
161
|
+
job.bury # set aside for inspection
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
#### Peeking and kicking
|
|
165
|
+
|
|
166
|
+
```ruby
|
|
167
|
+
@tuber.jobs.find(123) # peek at a specific job
|
|
168
|
+
@tube.peek(:ready) # peek at next ready job
|
|
169
|
+
@tube.peek(:buried)
|
|
170
|
+
@tube.peek(:delayed)
|
|
171
|
+
|
|
172
|
+
@tuber.tubes['some-tube'].kick(3) # kick 3 buried jobs back to ready
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
#### Automatic processing
|
|
176
|
+
|
|
177
|
+
Register handlers for tubes and let `process!` loop over incoming jobs:
|
|
178
|
+
|
|
179
|
+
```ruby
|
|
180
|
+
@tuber.jobs.register('some-tube', retry_on: [SomeError]) do |job|
|
|
181
|
+
do_something(job)
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
@tuber.jobs.register('other-tube') do |job|
|
|
185
|
+
do_something_else(job)
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
@tuber.jobs.process!
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
The loop reserves a job, calls the matching handler, then: deletes on success, releases on `retry_on` errors, and buries on other exceptions. Raise `AbortProcessingError` to stop the loop.
|
|
192
|
+
|
|
193
|
+
### Job Dependencies (Tuber only)
|
|
194
|
+
|
|
195
|
+
When using [Tuber](https://github.com/tuberq/tuber), you can group related jobs and chain dependent work using `group:` and `after:` options on `put`. After-jobs are held until every job in the group they depend on has been deleted:
|
|
196
|
+
|
|
197
|
+
```ruby
|
|
198
|
+
# Fan-out: enqueue grouped work
|
|
199
|
+
@tube.put "import-row-1", group: "import"
|
|
200
|
+
@tube.put "import-row-2", group: "import"
|
|
201
|
+
|
|
202
|
+
# Fan-in: this job waits until all "import" jobs are deleted
|
|
203
|
+
@tube.put "send-summary", after: "import"
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
Chain stages together by combining `after:` and `group:` on the same job to build a simple DAG pipeline:
|
|
207
|
+
|
|
208
|
+
```ruby
|
|
209
|
+
@tube.put "row-1", group: "extract"
|
|
210
|
+
@tube.put "row-2", group: "extract"
|
|
211
|
+
@tube.put "transform", after: "extract", group: "transform"
|
|
212
|
+
@tube.put "load", after: "transform"
|
|
213
|
+
```
|
|
214
|
+
|
|
215
|
+
Here `transform` waits for the extract group to finish, then becomes part of the `transform` group. `load` waits for `transform` to complete.
|
|
216
|
+
|
|
217
|
+
Buried jobs block group completion — kick them to let the group finish. Group names are global and can span multiple tubes.
|
|
218
|
+
|
|
219
|
+
### Unique Jobs / Idempotency (Tuber only)
|
|
220
|
+
|
|
221
|
+
Prevent duplicate jobs with the `idempotency:` option. If a job with the same key already exists in the tube, the original job is returned instead of creating a duplicate:
|
|
222
|
+
|
|
223
|
+
```ruby
|
|
224
|
+
@tube.put "send-report", idempotency: "daily-report"
|
|
225
|
+
# => <Tuber::Job id=1 body="send-report">
|
|
226
|
+
|
|
227
|
+
@tube.put "send-report", idempotency: "daily-report"
|
|
228
|
+
# => <Tuber::Job id=1 body="send-report"> (same job, no duplicate created)
|
|
229
|
+
```
|
|
230
|
+
|
|
231
|
+
The key is scoped to the tube and cleared when the job is deleted, so the same key can be reused afterwards.
|
|
232
|
+
|
|
233
|
+
Add a cooldown TTL to keep deduplicating for N seconds after deletion — useful for preventing rapid resubmission:
|
|
234
|
+
|
|
235
|
+
```ruby
|
|
236
|
+
@tube.put "send-report", idempotency: "daily-report", idempotency_ttl: 300
|
|
237
|
+
```
|
|
238
|
+
|
|
239
|
+
### Concurrency Keys (Tuber only)
|
|
240
|
+
|
|
241
|
+
Limit parallel processing of related jobs. When a job with a concurrency key is reserved, other ready jobs sharing the same key are hidden from `reserve` until the reservation ends:
|
|
242
|
+
|
|
243
|
+
```ruby
|
|
244
|
+
# Only one job per user can be processed at a time
|
|
245
|
+
@tube.put "process-user-42", concurrency: "user-42"
|
|
246
|
+
@tube.put "process-user-42-again", concurrency: "user-42"
|
|
247
|
+
```
|
|
248
|
+
|
|
249
|
+
The second job won't be reserved until the first is deleted, released, or buried. Set a higher limit to allow N concurrent reservations:
|
|
250
|
+
|
|
251
|
+
```ruby
|
|
252
|
+
# Allow up to 3 concurrent API jobs
|
|
253
|
+
@tube.put "api-call-1", concurrency: "api", concurrency_limit: 3
|
|
254
|
+
@tube.put "api-call-2", concurrency: "api", concurrency_limit: 3
|
|
255
|
+
```
|
|
256
|
+
|
|
257
|
+
### Weighted Tubes (Tuber only)
|
|
258
|
+
|
|
259
|
+
By default, `reserve` picks the highest-priority job across all watched tubes. Switch to weighted mode to select tubes randomly in proportion to their weight:
|
|
260
|
+
|
|
261
|
+
```ruby
|
|
262
|
+
@tuber.tubes.watch('email')
|
|
263
|
+
@tuber.tubes.watch('notifications', weight: 2)
|
|
264
|
+
@tuber.tubes.watch('batch-jobs', weight: 6)
|
|
265
|
+
|
|
266
|
+
@tuber.tubes.reserve_mode(:weighted)
|
|
267
|
+
|
|
268
|
+
job = @tuber.tubes.reserve # batch-jobs selected 6x as often as email
|
|
269
|
+
```
|
|
270
|
+
|
|
271
|
+
Tubes default to weight 1. Switch back with `reserve_mode(:fifo)`.
|
|
272
|
+
|
|
273
|
+
### Batch Reserve (Tuber only)
|
|
274
|
+
|
|
275
|
+
Reserve multiple jobs atomically in a single call:
|
|
276
|
+
|
|
277
|
+
```ruby
|
|
278
|
+
jobs = @tuber.tubes.reserve_batch(10) # up to 10 jobs
|
|
279
|
+
|
|
280
|
+
jobs.each do |job|
|
|
281
|
+
process(job)
|
|
282
|
+
job.delete
|
|
283
|
+
end
|
|
284
|
+
```
|
|
285
|
+
|
|
286
|
+
By default `reserve_batch` is non-blocking — it returns whatever is ready
|
|
287
|
+
immediately, possibly an empty array. Pass a timeout (in seconds) to long-poll
|
|
288
|
+
instead: the call blocks until the first job arrives, then drains everything
|
|
289
|
+
ready up to `count`, or returns an empty array when the timeout elapses. This
|
|
290
|
+
avoids hot-looping a worker on empty polls.
|
|
291
|
+
|
|
292
|
+
```ruby
|
|
293
|
+
jobs = @tuber.tubes.reserve_batch(10, 30) # block up to 30s for the first job
|
|
294
|
+
```
|
|
295
|
+
|
|
296
|
+
While blocked, a positive-timeout batch reserve may raise
|
|
297
|
+
`Tuber::DeadlineSoonError` if one of the connection's already-reserved jobs
|
|
298
|
+
is about to hit its TTR — service that job, then reserve again.
|
|
299
|
+
|
|
300
|
+
### Batch Touch (Tuber 0.12.0+)
|
|
301
|
+
|
|
302
|
+
A batch reserve starts the TTR clock on every job at the same instant, but a
|
|
303
|
+
worker processes them serially — so the tail of a large batch can expire and
|
|
304
|
+
return to the queue while the worker is still busy. `touch_all` extends the TTR
|
|
305
|
+
of every job the connection currently holds in a single command:
|
|
306
|
+
|
|
307
|
+
```ruby
|
|
308
|
+
jobs = @tuber.tubes.reserve_batch(10)
|
|
309
|
+
|
|
310
|
+
jobs.each do |job|
|
|
311
|
+
process(job)
|
|
312
|
+
job.delete
|
|
313
|
+
@tuber.jobs.touch_all # heartbeat whatever is still held
|
|
314
|
+
end
|
|
315
|
+
```
|
|
316
|
+
|
|
317
|
+
No ids are sent: the server tracks the reserved set per connection, so jobs
|
|
318
|
+
already deleted, released, buried or lost to a TTR timeout are simply absent.
|
|
319
|
+
Each job keeps its own TTR — deadlines are extended individually, not levelled
|
|
320
|
+
onto a common value.
|
|
321
|
+
|
|
322
|
+
The return value is how many jobs the connection *actually* still holds. A count
|
|
323
|
+
lower than expected means jobs hit their TTR and went back to the queue while the
|
|
324
|
+
worker was busy — otherwise invisible, since nothing notifies a worker that it
|
|
325
|
+
lost a job.
|
|
326
|
+
|
|
327
|
+
### Stats
|
|
328
|
+
|
|
329
|
+
```ruby
|
|
330
|
+
@tuber.stats # server-wide stats
|
|
331
|
+
@tuber.tubes['some-tube'].stats # tube stats
|
|
332
|
+
@tuber.jobs[some_job_id].stats # job stats
|
|
333
|
+
```
|
|
334
|
+
|
|
335
|
+
## Configuration
|
|
336
|
+
|
|
337
|
+
```ruby
|
|
338
|
+
Tuber.configure do |config|
|
|
339
|
+
config.default_put_delay = 0
|
|
340
|
+
config.default_put_pri = 65536
|
|
341
|
+
config.default_put_ttr = 120
|
|
342
|
+
config.job_parser = lambda { |body| body }
|
|
343
|
+
config.job_serializer = lambda { |body| body }
|
|
344
|
+
config.tuber_url = 'localhost:11300'
|
|
345
|
+
end
|
|
346
|
+
```
|
|
347
|
+
|
|
348
|
+
The `job_serializer` is applied to every `put` body — useful for automatic JSON encoding:
|
|
349
|
+
|
|
350
|
+
```ruby
|
|
351
|
+
Tuber.configure do |config|
|
|
352
|
+
config.job_serializer = lambda { |body| JSON.dump(body) }
|
|
353
|
+
end
|
|
354
|
+
```
|
|
355
|
+
|
|
356
|
+
## Error Handling
|
|
357
|
+
|
|
358
|
+
| Error | Description |
|
|
359
|
+
| ----------------------------- | ----------- |
|
|
360
|
+
| Tuber::NotConnected | Cannot connect to the server. |
|
|
361
|
+
| Tuber::InvalidTubeName | Tube name is not valid. |
|
|
362
|
+
| Tuber::NotFoundError | Job or tube not found. |
|
|
363
|
+
| Tuber::TimedOutError | Reserve timed out. |
|
|
364
|
+
| Tuber::JobNotReserved | Action requires a reserved job. |
|
|
365
|
+
|
|
366
|
+
See the [Tuber protocol](https://github.com/tuberq/tuber/blob/main/docs/protocol.md) (a superset of the [beanstalk protocol](https://github.com/beanstalkd/beanstalkd/blob/master/doc/protocol.txt)) for additional error types.
|
|
367
|
+
|
|
368
|
+
## Resources
|
|
369
|
+
|
|
370
|
+
* [Tuber](https://github.com/tuberq/tuber)
|
|
371
|
+
* [Tuber protocol](https://github.com/tuberq/tuber/blob/main/docs/protocol.md)
|
|
372
|
+
* [Tuber on RubyGems](https://rubygems.org/gems/tuber)
|
|
373
|
+
* [Beanstalkd](https://github.com/beanstalkd/beanstalkd) and the [beanstalk protocol](https://github.com/beanstalkd/beanstalkd/blob/master/doc/protocol.txt)
|
|
374
|
+
* [Backburner](https://github.com/nesquena/backburner) — Ruby job queue for Rails/Sinatra
|
|
375
|
+
|
|
376
|
+
## Contributors
|
|
377
|
+
|
|
378
|
+
Tuber is maintained by [Dan Milne](https://github.com/dkam), and builds on the work of
|
|
379
|
+
everyone who wrote beaneater, from which it is forked:
|
|
380
|
+
|
|
381
|
+
- [Nico Taing](https://github.com/Nico-Taing) - Creator and co-maintainer of beaneater
|
|
382
|
+
- [Nathan Esquenazi](https://github.com/nesquena) - Contributor and co-maintainer
|
|
383
|
+
- [Keith Rarick](https://github.com/kr) - Much code inspired and adapted from beanstalk-client
|
|
384
|
+
- [Vidar Hokstad](https://github.com/vidarh) - Replaced telnet with correct TCP socket handling
|
|
385
|
+
- [Andreas Loupasakis](https://github.com/alup) - Improve test coverage, improve job configuration
|