bushpig 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/lib/bushpig.rb ADDED
@@ -0,0 +1,53 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'bushpig/version'
4
+
5
+ require 'bushpig/redis_pool'
6
+ require 'bushpig/job'
7
+ require 'bushpig/client'
8
+ require 'bushpig/server'
9
+
10
+ module Bushpig
11
+ NAME = 'Bushpig'
12
+ LICENSE = 'See LICENSE and the MIT License for licensing details.'
13
+
14
+ def self.client
15
+ raise 'Bushpig client not configured' unless @client
16
+
17
+ @client
18
+ end
19
+
20
+ def self.client=(client)
21
+ @client = client
22
+ end
23
+
24
+ def self.server
25
+ raise 'Bushpig server not configured' unless @server
26
+
27
+ @server
28
+ end
29
+
30
+ def self.server=(server)
31
+ @server = server
32
+ end
33
+
34
+ def self.configure_server
35
+ yield self if server?
36
+ end
37
+
38
+ def self.configure_client
39
+ yield self unless server?
40
+ end
41
+
42
+ def self.server?
43
+ defined?(Bushpig::CLI)
44
+ end
45
+
46
+ def self.set_key(queue)
47
+ "set:#{queue}"
48
+ end
49
+
50
+ def self.job_key(job_id)
51
+ "job:#{job_id}"
52
+ end
53
+ end
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'thor'
4
+
5
+ module Bushpig
6
+ class CLI < Thor
7
+ package_name 'bushpig'
8
+
9
+ def self.exit_on_failure?
10
+ true
11
+ end
12
+
13
+ desc 'serve SET', 'Run the bushpig server'
14
+ def serve(queue)
15
+ require 'rails'
16
+ require File.expand_path('./config/environment.rb')
17
+
18
+ server = Bushpig.server
19
+ server.serve(queue)
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Bushpig
4
+ class Client
5
+ attr_accessor :default_ttl
6
+
7
+ def initialize(pool)
8
+ @pool = pool
9
+ @default_ttl = nil
10
+ end
11
+
12
+ def redis_pool
13
+ @pool
14
+ end
15
+
16
+ def default_score
17
+ Time.now.to_f
18
+ end
19
+
20
+ def submit(queue, job, score: default_score, ttl: default_ttl)
21
+ redis_pool.with do |conn|
22
+ conn.set(Bushpig.job_key(job.job_id), job.job_payload, ex: ttl)
23
+ conn.zadd(Bushpig.set_key(queue), score, job.job_id)
24
+ end
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,51 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'digest'
4
+ require 'securerandom'
5
+ require 'json'
6
+
7
+ module Bushpig
8
+ module Job
9
+ def self.job(*args)
10
+ Struct.new(*args) do
11
+ def job_id
12
+ @job_id ||= SecureRandom.hex(32)
13
+ end
14
+
15
+ def job_id=(job_id)
16
+ @job_id = job_id
17
+ end
18
+
19
+ def job_payload
20
+ JSON.generate({ class: self.class.name, args: each.to_a })
21
+ end
22
+ end
23
+ end
24
+
25
+ def self.keyed(unique_key, *args)
26
+ Struct.new(*args) do
27
+ @@unique_key = unique_key
28
+
29
+ def job_id
30
+ @@unique_key.inject(Digest::SHA256.new) do |digest, key|
31
+ digest.update(self[key].to_s)
32
+ end.hexdigest
33
+ end
34
+
35
+ def job_id=(job_id); end
36
+
37
+ def job_payload
38
+ JSON.generate({ class: self.class.name, args: each.to_a })
39
+ end
40
+ end
41
+ end
42
+
43
+ def self.hydrate(job_id, job_payload)
44
+ h = JSON.parse(job_payload, symbolize_names: true)
45
+ klass = const_get(h[:class])
46
+ job = klass.new(*h[:args])
47
+ job.job_id = job_id
48
+ job
49
+ end
50
+ end
51
+ end
@@ -0,0 +1,31 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'connection_pool'
4
+ require 'redis'
5
+
6
+ module Bushpig
7
+ class RedisPool
8
+ def initialize(size: 1, timeout: 5, redis_options: {})
9
+ @size = size
10
+ @timeout = timeout
11
+ @redis_options = redis_options
12
+
13
+ @pool = ConnectionPool.new(size: @size, timeout: @timeout) { Redis.new(@redis_options) }
14
+ end
15
+
16
+ def with(options = {}, &block)
17
+ raise ArgumentError, 'requires a block' unless block_given?
18
+
19
+ retries = 1
20
+ begin
21
+ @pool.with(options, &block)
22
+ rescue Redis::TimeoutError
23
+ if retries > 0
24
+ retries -= 1
25
+ retry
26
+ end
27
+ raise
28
+ end
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,91 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Bushpig
4
+ class Server
5
+ def initialize(pool, timeout: 2, &handler)
6
+ @pool = pool
7
+ @timeout = timeout.to_f
8
+ @handler = handler
9
+ @done = false
10
+ end
11
+
12
+ def redis_pool
13
+ @pool
14
+ end
15
+
16
+ def trap_signals
17
+ Signal.trap('INT') do
18
+ reset_signals
19
+ puts 'INT received, shutdown flagged'
20
+ @done = true
21
+ end
22
+ end
23
+
24
+ def reset_signals
25
+ Signal.trap('INT', 'DEFAULT')
26
+ end
27
+
28
+ def serve(queue)
29
+ puts "Serving queue #{queue}"
30
+ trap_signals
31
+
32
+ until @done
33
+ job = fetch(queue)
34
+ next if job.nil?
35
+
36
+ handle(job)
37
+ complete(job)
38
+ end
39
+
40
+ reset_signals
41
+ puts "Stop serving queue #{queue}"
42
+ end
43
+
44
+ def handle(job)
45
+ puts "Job starting: jid-#{job.job_id} #{job}"
46
+ started = monotonic_time
47
+ @handler.call(job)
48
+ finished = monotonic_time
49
+ elapsed = finished - started
50
+ puts "Job completed: jid-#{job.job_id} #{elapsed} seconds"
51
+ rescue StandardError => e
52
+ # Job handler raised exception
53
+ finished = monotonic_time
54
+ elapsed = finished - started
55
+ puts "Job raised exception: jid-#{job.job_id} #{elapsed} seconds: #{e}"
56
+ # TODO: log exception to honeybadger
57
+ end
58
+
59
+ def monotonic_time
60
+ Process.clock_gettime(Process::CLOCK_MONOTONIC)
61
+ end
62
+
63
+ def fetch(queue)
64
+ redis_pool.with do |conn|
65
+ begin
66
+ res = conn.bzpopmin(Bushpig.set_key(queue), @timeout)
67
+ rescue Redis::TimeoutError
68
+ # TODO: warn user (once) that redis timeout set lower than pop timeout
69
+ conn.close
70
+ res = nil
71
+ end
72
+ return nil if res.nil?
73
+
74
+ (_set, jid, _score) = res
75
+ # conn.sadd('running', jid)
76
+
77
+ payload = conn.get(Bushpig.job_key(jid))
78
+ return nil if payload.nil? # most likely job expired
79
+
80
+ Bushpig::Job.hydrate(jid, payload)
81
+ end
82
+ end
83
+
84
+ def complete(job)
85
+ redis_pool.with do |conn|
86
+ # conn.srem('running', job.job_id)
87
+ conn.del(Bushpig.job_key(job.job_id))
88
+ end
89
+ end
90
+ end
91
+ end
@@ -0,0 +1,5 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Bushpig
4
+ VERSION = '0.1.0'
5
+ end
data/shell.nix ADDED
@@ -0,0 +1,4 @@
1
+ let
2
+ pkgs = import <nixpkgs> {};
3
+ in
4
+ pkgs.mkShell { buildInputs = with pkgs; [ ruby bundler bundix ]; }
metadata ADDED
@@ -0,0 +1,197 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: bushpig
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Shaun Sharples
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 1980-01-01 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: bundler
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '2.1'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '2.1'
27
+ - !ruby/object:Gem::Dependency
28
+ name: pry
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: '0'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: rake
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '10.0'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '10.0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: rubocop
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ">="
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ">="
67
+ - !ruby/object:Gem::Version
68
+ version: '0'
69
+ - !ruby/object:Gem::Dependency
70
+ name: solargraph
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - ">="
74
+ - !ruby/object:Gem::Version
75
+ version: '0'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - ">="
81
+ - !ruby/object:Gem::Version
82
+ version: '0'
83
+ - !ruby/object:Gem::Dependency
84
+ name: connection_pool
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - "~>"
88
+ - !ruby/object:Gem::Version
89
+ version: '2.2'
90
+ type: :runtime
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - "~>"
95
+ - !ruby/object:Gem::Version
96
+ version: '2.2'
97
+ - !ruby/object:Gem::Dependency
98
+ name: json
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - ">="
102
+ - !ruby/object:Gem::Version
103
+ version: '1.8'
104
+ type: :runtime
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - ">="
109
+ - !ruby/object:Gem::Version
110
+ version: '1.8'
111
+ - !ruby/object:Gem::Dependency
112
+ name: redis
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - "~>"
116
+ - !ruby/object:Gem::Version
117
+ version: '3.3'
118
+ type: :runtime
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - "~>"
123
+ - !ruby/object:Gem::Version
124
+ version: '3.3'
125
+ - !ruby/object:Gem::Dependency
126
+ name: thor
127
+ requirement: !ruby/object:Gem::Requirement
128
+ requirements:
129
+ - - "~>"
130
+ - !ruby/object:Gem::Version
131
+ version: '1.0'
132
+ type: :runtime
133
+ prerelease: false
134
+ version_requirements: !ruby/object:Gem::Requirement
135
+ requirements:
136
+ - - "~>"
137
+ - !ruby/object:Gem::Version
138
+ version: '1.0'
139
+ description: Simple job system.
140
+ email:
141
+ - shaun.sharples@gmail.com
142
+ executables:
143
+ - bushpig
144
+ extensions: []
145
+ extra_rdoc_files: []
146
+ files:
147
+ - ".gitignore"
148
+ - Gemfile
149
+ - Gemfile.lock
150
+ - LICENSE.txt
151
+ - README.md
152
+ - Rakefile
153
+ - bin/bushpig
154
+ - bin/console
155
+ - bin/setup
156
+ - bushpig.gemspec
157
+ - default.nix
158
+ - gemset.nix
159
+ - lib/bushpig.rb
160
+ - lib/bushpig/cli.rb
161
+ - lib/bushpig/client.rb
162
+ - lib/bushpig/job.rb
163
+ - lib/bushpig/redis_pool.rb
164
+ - lib/bushpig/server.rb
165
+ - lib/bushpig/version.rb
166
+ - shell.nix
167
+ homepage: https://github.com/mpowered/bushpig
168
+ licenses:
169
+ - MIT
170
+ metadata:
171
+ allowed_push_host: https://rubygems.org
172
+ homepage_uri: https://github.com/mpowered/bushpig
173
+ source_code_uri: https://github.com/mpowered/bushpig
174
+ changelog_uri: https://github.com/mpowered/bushpig/blob/master/CHANGELOG.md
175
+ post_install_message:
176
+ rdoc_options: []
177
+ require_paths:
178
+ - lib
179
+ required_ruby_version: !ruby/object:Gem::Requirement
180
+ requirements:
181
+ - - ">="
182
+ - !ruby/object:Gem::Version
183
+ version: '2.3'
184
+ - - "<"
185
+ - !ruby/object:Gem::Version
186
+ version: '3'
187
+ required_rubygems_version: !ruby/object:Gem::Requirement
188
+ requirements:
189
+ - - ">="
190
+ - !ruby/object:Gem::Version
191
+ version: '0'
192
+ requirements: []
193
+ rubygems_version: 3.1.3
194
+ signing_key:
195
+ specification_version: 4
196
+ summary: Simple job system.
197
+ test_files: []