bushpig 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +9 -0
- data/Gemfile +3 -0
- data/Gemfile.lock +87 -0
- data/LICENSE.txt +21 -0
- data/README.md +37 -0
- data/Rakefile +2 -0
- data/bin/bushpig +4 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/bushpig.gemspec +49 -0
- data/default.nix +7 -0
- data/gemset.nix +941 -0
- data/lib/bushpig.rb +53 -0
- data/lib/bushpig/cli.rb +22 -0
- data/lib/bushpig/client.rb +27 -0
- data/lib/bushpig/job.rb +51 -0
- data/lib/bushpig/redis_pool.rb +31 -0
- data/lib/bushpig/server.rb +91 -0
- data/lib/bushpig/version.rb +5 -0
- data/shell.nix +4 -0
- metadata +197 -0
data/lib/bushpig.rb
ADDED
@@ -0,0 +1,53 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'bushpig/version'
|
4
|
+
|
5
|
+
require 'bushpig/redis_pool'
|
6
|
+
require 'bushpig/job'
|
7
|
+
require 'bushpig/client'
|
8
|
+
require 'bushpig/server'
|
9
|
+
|
10
|
+
module Bushpig
|
11
|
+
NAME = 'Bushpig'
|
12
|
+
LICENSE = 'See LICENSE and the MIT License for licensing details.'
|
13
|
+
|
14
|
+
def self.client
|
15
|
+
raise 'Bushpig client not configured' unless @client
|
16
|
+
|
17
|
+
@client
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.client=(client)
|
21
|
+
@client = client
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.server
|
25
|
+
raise 'Bushpig server not configured' unless @server
|
26
|
+
|
27
|
+
@server
|
28
|
+
end
|
29
|
+
|
30
|
+
def self.server=(server)
|
31
|
+
@server = server
|
32
|
+
end
|
33
|
+
|
34
|
+
def self.configure_server
|
35
|
+
yield self if server?
|
36
|
+
end
|
37
|
+
|
38
|
+
def self.configure_client
|
39
|
+
yield self unless server?
|
40
|
+
end
|
41
|
+
|
42
|
+
def self.server?
|
43
|
+
defined?(Bushpig::CLI)
|
44
|
+
end
|
45
|
+
|
46
|
+
def self.set_key(queue)
|
47
|
+
"set:#{queue}"
|
48
|
+
end
|
49
|
+
|
50
|
+
def self.job_key(job_id)
|
51
|
+
"job:#{job_id}"
|
52
|
+
end
|
53
|
+
end
|
data/lib/bushpig/cli.rb
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'thor'
|
4
|
+
|
5
|
+
module Bushpig
|
6
|
+
class CLI < Thor
|
7
|
+
package_name 'bushpig'
|
8
|
+
|
9
|
+
def self.exit_on_failure?
|
10
|
+
true
|
11
|
+
end
|
12
|
+
|
13
|
+
desc 'serve SET', 'Run the bushpig server'
|
14
|
+
def serve(queue)
|
15
|
+
require 'rails'
|
16
|
+
require File.expand_path('./config/environment.rb')
|
17
|
+
|
18
|
+
server = Bushpig.server
|
19
|
+
server.serve(queue)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Bushpig
|
4
|
+
class Client
|
5
|
+
attr_accessor :default_ttl
|
6
|
+
|
7
|
+
def initialize(pool)
|
8
|
+
@pool = pool
|
9
|
+
@default_ttl = nil
|
10
|
+
end
|
11
|
+
|
12
|
+
def redis_pool
|
13
|
+
@pool
|
14
|
+
end
|
15
|
+
|
16
|
+
def default_score
|
17
|
+
Time.now.to_f
|
18
|
+
end
|
19
|
+
|
20
|
+
def submit(queue, job, score: default_score, ttl: default_ttl)
|
21
|
+
redis_pool.with do |conn|
|
22
|
+
conn.set(Bushpig.job_key(job.job_id), job.job_payload, ex: ttl)
|
23
|
+
conn.zadd(Bushpig.set_key(queue), score, job.job_id)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
data/lib/bushpig/job.rb
ADDED
@@ -0,0 +1,51 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'digest'
|
4
|
+
require 'securerandom'
|
5
|
+
require 'json'
|
6
|
+
|
7
|
+
module Bushpig
|
8
|
+
module Job
|
9
|
+
def self.job(*args)
|
10
|
+
Struct.new(*args) do
|
11
|
+
def job_id
|
12
|
+
@job_id ||= SecureRandom.hex(32)
|
13
|
+
end
|
14
|
+
|
15
|
+
def job_id=(job_id)
|
16
|
+
@job_id = job_id
|
17
|
+
end
|
18
|
+
|
19
|
+
def job_payload
|
20
|
+
JSON.generate({ class: self.class.name, args: each.to_a })
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def self.keyed(unique_key, *args)
|
26
|
+
Struct.new(*args) do
|
27
|
+
@@unique_key = unique_key
|
28
|
+
|
29
|
+
def job_id
|
30
|
+
@@unique_key.inject(Digest::SHA256.new) do |digest, key|
|
31
|
+
digest.update(self[key].to_s)
|
32
|
+
end.hexdigest
|
33
|
+
end
|
34
|
+
|
35
|
+
def job_id=(job_id); end
|
36
|
+
|
37
|
+
def job_payload
|
38
|
+
JSON.generate({ class: self.class.name, args: each.to_a })
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def self.hydrate(job_id, job_payload)
|
44
|
+
h = JSON.parse(job_payload, symbolize_names: true)
|
45
|
+
klass = const_get(h[:class])
|
46
|
+
job = klass.new(*h[:args])
|
47
|
+
job.job_id = job_id
|
48
|
+
job
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'connection_pool'
|
4
|
+
require 'redis'
|
5
|
+
|
6
|
+
module Bushpig
|
7
|
+
class RedisPool
|
8
|
+
def initialize(size: 1, timeout: 5, redis_options: {})
|
9
|
+
@size = size
|
10
|
+
@timeout = timeout
|
11
|
+
@redis_options = redis_options
|
12
|
+
|
13
|
+
@pool = ConnectionPool.new(size: @size, timeout: @timeout) { Redis.new(@redis_options) }
|
14
|
+
end
|
15
|
+
|
16
|
+
def with(options = {}, &block)
|
17
|
+
raise ArgumentError, 'requires a block' unless block_given?
|
18
|
+
|
19
|
+
retries = 1
|
20
|
+
begin
|
21
|
+
@pool.with(options, &block)
|
22
|
+
rescue Redis::TimeoutError
|
23
|
+
if retries > 0
|
24
|
+
retries -= 1
|
25
|
+
retry
|
26
|
+
end
|
27
|
+
raise
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,91 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Bushpig
|
4
|
+
class Server
|
5
|
+
def initialize(pool, timeout: 2, &handler)
|
6
|
+
@pool = pool
|
7
|
+
@timeout = timeout.to_f
|
8
|
+
@handler = handler
|
9
|
+
@done = false
|
10
|
+
end
|
11
|
+
|
12
|
+
def redis_pool
|
13
|
+
@pool
|
14
|
+
end
|
15
|
+
|
16
|
+
def trap_signals
|
17
|
+
Signal.trap('INT') do
|
18
|
+
reset_signals
|
19
|
+
puts 'INT received, shutdown flagged'
|
20
|
+
@done = true
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def reset_signals
|
25
|
+
Signal.trap('INT', 'DEFAULT')
|
26
|
+
end
|
27
|
+
|
28
|
+
def serve(queue)
|
29
|
+
puts "Serving queue #{queue}"
|
30
|
+
trap_signals
|
31
|
+
|
32
|
+
until @done
|
33
|
+
job = fetch(queue)
|
34
|
+
next if job.nil?
|
35
|
+
|
36
|
+
handle(job)
|
37
|
+
complete(job)
|
38
|
+
end
|
39
|
+
|
40
|
+
reset_signals
|
41
|
+
puts "Stop serving queue #{queue}"
|
42
|
+
end
|
43
|
+
|
44
|
+
def handle(job)
|
45
|
+
puts "Job starting: jid-#{job.job_id} #{job}"
|
46
|
+
started = monotonic_time
|
47
|
+
@handler.call(job)
|
48
|
+
finished = monotonic_time
|
49
|
+
elapsed = finished - started
|
50
|
+
puts "Job completed: jid-#{job.job_id} #{elapsed} seconds"
|
51
|
+
rescue StandardError => e
|
52
|
+
# Job handler raised exception
|
53
|
+
finished = monotonic_time
|
54
|
+
elapsed = finished - started
|
55
|
+
puts "Job raised exception: jid-#{job.job_id} #{elapsed} seconds: #{e}"
|
56
|
+
# TODO: log exception to honeybadger
|
57
|
+
end
|
58
|
+
|
59
|
+
def monotonic_time
|
60
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
61
|
+
end
|
62
|
+
|
63
|
+
def fetch(queue)
|
64
|
+
redis_pool.with do |conn|
|
65
|
+
begin
|
66
|
+
res = conn.bzpopmin(Bushpig.set_key(queue), @timeout)
|
67
|
+
rescue Redis::TimeoutError
|
68
|
+
# TODO: warn user (once) that redis timeout set lower than pop timeout
|
69
|
+
conn.close
|
70
|
+
res = nil
|
71
|
+
end
|
72
|
+
return nil if res.nil?
|
73
|
+
|
74
|
+
(_set, jid, _score) = res
|
75
|
+
# conn.sadd('running', jid)
|
76
|
+
|
77
|
+
payload = conn.get(Bushpig.job_key(jid))
|
78
|
+
return nil if payload.nil? # most likely job expired
|
79
|
+
|
80
|
+
Bushpig::Job.hydrate(jid, payload)
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
def complete(job)
|
85
|
+
redis_pool.with do |conn|
|
86
|
+
# conn.srem('running', job.job_id)
|
87
|
+
conn.del(Bushpig.job_key(job.job_id))
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
data/shell.nix
ADDED
metadata
ADDED
@@ -0,0 +1,197 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: bushpig
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Shaun Sharples
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 1980-01-01 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '2.1'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '2.1'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: pry
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rake
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '10.0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '10.0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: rubocop
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: solargraph
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: connection_pool
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - "~>"
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '2.2'
|
90
|
+
type: :runtime
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - "~>"
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '2.2'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: json
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '1.8'
|
104
|
+
type: :runtime
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '1.8'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: redis
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - "~>"
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '3.3'
|
118
|
+
type: :runtime
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - "~>"
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '3.3'
|
125
|
+
- !ruby/object:Gem::Dependency
|
126
|
+
name: thor
|
127
|
+
requirement: !ruby/object:Gem::Requirement
|
128
|
+
requirements:
|
129
|
+
- - "~>"
|
130
|
+
- !ruby/object:Gem::Version
|
131
|
+
version: '1.0'
|
132
|
+
type: :runtime
|
133
|
+
prerelease: false
|
134
|
+
version_requirements: !ruby/object:Gem::Requirement
|
135
|
+
requirements:
|
136
|
+
- - "~>"
|
137
|
+
- !ruby/object:Gem::Version
|
138
|
+
version: '1.0'
|
139
|
+
description: Simple job system.
|
140
|
+
email:
|
141
|
+
- shaun.sharples@gmail.com
|
142
|
+
executables:
|
143
|
+
- bushpig
|
144
|
+
extensions: []
|
145
|
+
extra_rdoc_files: []
|
146
|
+
files:
|
147
|
+
- ".gitignore"
|
148
|
+
- Gemfile
|
149
|
+
- Gemfile.lock
|
150
|
+
- LICENSE.txt
|
151
|
+
- README.md
|
152
|
+
- Rakefile
|
153
|
+
- bin/bushpig
|
154
|
+
- bin/console
|
155
|
+
- bin/setup
|
156
|
+
- bushpig.gemspec
|
157
|
+
- default.nix
|
158
|
+
- gemset.nix
|
159
|
+
- lib/bushpig.rb
|
160
|
+
- lib/bushpig/cli.rb
|
161
|
+
- lib/bushpig/client.rb
|
162
|
+
- lib/bushpig/job.rb
|
163
|
+
- lib/bushpig/redis_pool.rb
|
164
|
+
- lib/bushpig/server.rb
|
165
|
+
- lib/bushpig/version.rb
|
166
|
+
- shell.nix
|
167
|
+
homepage: https://github.com/mpowered/bushpig
|
168
|
+
licenses:
|
169
|
+
- MIT
|
170
|
+
metadata:
|
171
|
+
allowed_push_host: https://rubygems.org
|
172
|
+
homepage_uri: https://github.com/mpowered/bushpig
|
173
|
+
source_code_uri: https://github.com/mpowered/bushpig
|
174
|
+
changelog_uri: https://github.com/mpowered/bushpig/blob/master/CHANGELOG.md
|
175
|
+
post_install_message:
|
176
|
+
rdoc_options: []
|
177
|
+
require_paths:
|
178
|
+
- lib
|
179
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
180
|
+
requirements:
|
181
|
+
- - ">="
|
182
|
+
- !ruby/object:Gem::Version
|
183
|
+
version: '2.3'
|
184
|
+
- - "<"
|
185
|
+
- !ruby/object:Gem::Version
|
186
|
+
version: '3'
|
187
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
188
|
+
requirements:
|
189
|
+
- - ">="
|
190
|
+
- !ruby/object:Gem::Version
|
191
|
+
version: '0'
|
192
|
+
requirements: []
|
193
|
+
rubygems_version: 3.1.3
|
194
|
+
signing_key:
|
195
|
+
specification_version: 4
|
196
|
+
summary: Simple job system.
|
197
|
+
test_files: []
|