subservient 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 5e6dc9fb826381c68f4d1f9f7f04f40e0dfb3a46
4
+ data.tar.gz: ced144a849ef9933b5b760c39f68c8c6d357ea48
5
+ SHA512:
6
+ metadata.gz: c85bf058fd785ad5ebf06c0001d7409910931478787ff35f2cceaf857a271ad9470bddfd4c7a1e1c4df55d8cd50295bb55756c7664caedbf7f7afea228a02a71
7
+ data.tar.gz: 650f9a4bebac9ace6a6e75fb55cfb06561948d8dab7b55bf32a30997638e7f512946bf32614842f9a98c3270e983ca0c75dba398dead29d01f9218c363204ae2
data/.gitignore ADDED
@@ -0,0 +1,3 @@
1
+ pkg
2
+ .DS_Store
3
+
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in subservient.gemspec
4
+ gemspec
data/Gemfile.lock ADDED
@@ -0,0 +1,19 @@
1
+ PATH
2
+ remote: .
3
+ specs:
4
+ subservient (0.0.1)
5
+ beaneater
6
+
7
+ GEM
8
+ remote: https://rubygems.org/
9
+ specs:
10
+ beaneater (0.3.2)
11
+ rake (10.3.2)
12
+
13
+ PLATFORMS
14
+ ruby
15
+
16
+ DEPENDENCIES
17
+ bundler (~> 1.6)
18
+ rake
19
+ subservient!
data/LICENSE.txt ADDED
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2014 Crane7
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,29 @@
1
+ # Subservient
2
+
3
+ Job queue processor using Beanstalkd
4
+
5
+ ## Installation
6
+
7
+ Add this line to your application's Gemfile:
8
+
9
+ gem 'subservient'
10
+
11
+ And then execute:
12
+
13
+ $ bundle
14
+
15
+ Or install it yourself as:
16
+
17
+ $ gem install subservient
18
+
19
+ ## Usage
20
+
21
+ TODO: Write usage instructions here
22
+
23
+ ## Contributing
24
+
25
+ 1. Fork it
26
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
27
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
28
+ 4. Push to the branch (`git push origin my-new-feature`)
29
+ 5. Create a new Pull Request
data/Rakefile ADDED
@@ -0,0 +1 @@
1
+ require "bundler/gem_tasks"
data/bin/subservient ADDED
@@ -0,0 +1,8 @@
1
+ #! /usr/bin/env ruby
2
+ require 'subservient'
3
+
4
+ # Try and load tasks from the working directory
5
+ Dir["**/*.rb"].each { |f| require File.expand_path(f) }
6
+
7
+ subservient = Subservient::Worker.new
8
+ subservient.go
data/example_task.rb ADDED
@@ -0,0 +1,41 @@
1
+ require 'subservient'
2
+ require 'json'
3
+
4
+ module Subservient
5
+ class Worker
6
+ ## Generate some example jobs
7
+ def generate_example_jobs n=20, clear: true
8
+ conn = Beaneater::Pool.new(Subservient::pool)
9
+ tube = conn.tubes[@tube]
10
+ tube.clear if clear
11
+
12
+ n.times do |i|
13
+ push_job tube, "Example#puts_input", data: { n: (i+1) }
14
+ push_job tube, "Example#puts_rand"
15
+ end
16
+ push_job tube, "Example#divide_by_zero"
17
+
18
+ conn.close
19
+ end
20
+
21
+ def push_job tube, task, input={}
22
+ tube.put ({ task: task }.merge(input).to_json)
23
+ end
24
+
25
+ end
26
+ end
27
+
28
+ class Example < Subservient::Task
29
+ def puts_input
30
+ puts @job.input.inspect
31
+ end
32
+
33
+ def puts_rand
34
+ puts SecureRandom.hex
35
+ end
36
+
37
+ def divide_by_zero
38
+ # This will raise ZeroDivisionError
39
+ puts 1/0
40
+ end
41
+ end
@@ -0,0 +1,9 @@
1
+ require "beaneater"
2
+
3
+ require "subservient/version"
4
+ require "subservient/errors"
5
+ require "subservient/config"
6
+
7
+ require "subservient/job"
8
+ require "subservient/task"
9
+ require "subservient/worker"
@@ -0,0 +1,18 @@
1
+ module Subservient
2
+ @@namespace = 'subservient'
3
+ def self.namespace
4
+ @@namespace
5
+ end
6
+ def self.namespace= value
7
+ @@namespace = value
8
+ end
9
+
10
+
11
+ @@pool_urls = ['localhost:11300']
12
+ def self.pool
13
+ @@pool_urls
14
+ end
15
+ def self.pool= value
16
+ @@pool_urls = value
17
+ end
18
+ end
@@ -0,0 +1,14 @@
1
+ module Subservient
2
+ class SubservientError < RuntimeError; end # Never raised directly
3
+
4
+ class WorkerError < SubservientError; end # Never raised directly
5
+ class AlreadyRunningError < WorkerError; end # Raised if worker running when Worker#start called
6
+
7
+ class JobError < SubservientError; end # Never raised directly
8
+ class MalformedJobError < JobError; end # Raised if job from BS is malformed
9
+ class MissingTaskError < JobError; end # Raised if task cannot be found
10
+
11
+ class TaskError < SubservientError; end # Never raised directly
12
+ class RetryJob < TaskError; end # Raised to retry a job
13
+ class BuryJob < TaskError; end # Raised to bury a job
14
+ end
@@ -0,0 +1,29 @@
1
+ require 'json'
2
+
3
+ module Subservient
4
+ class Job
5
+ attr_reader :bs_job, :input, :task, :task_method
6
+
7
+ def initialize bs_job
8
+ @bs_job = bs_job
9
+ @input = JSON.parse (@bs_job.body)
10
+
11
+ task_str = @input.fetch('task')
12
+ arr = task_str.rpartition('#').reject { |e| e.empty? or e == '#' }
13
+ @task_method = (arr[1] || :process).to_sym
14
+ task_class = Object.const_get(arr[0])
15
+ raise MissingTaskError unless task_class.ancestors.include? Subservient::Task
16
+ @task = task_class.new self
17
+ raise MissingTaskError unless @task.respond_to? @task_method
18
+ rescue JSON::JSONError, KeyError
19
+ raise MalformedJobError
20
+ rescue NameError
21
+ raise MissingTaskError
22
+ end
23
+
24
+ def execute
25
+ @task.execute_job
26
+ end
27
+
28
+ end
29
+ end
@@ -0,0 +1,49 @@
1
+ require 'json'
2
+
3
+ module Subservient
4
+ class Task
5
+ attr_reader :job
6
+ def initialize job
7
+ @job = job
8
+ end
9
+
10
+ def execute_job
11
+ self.send(@job.task_method)
12
+ @job.bs_job.delete
13
+ rescue NoMethodError
14
+ raise MissingTaskError
15
+ rescue RetryJob, *@@retry_on => e
16
+ raise BuryJob if @job.bs_job.stats.releases > @@max_retries
17
+ puts "#{e.inspect} - RELEASING JOB #{@job.bs_job.id}"
18
+ puts e.backtrace
19
+ @job.bs_job.release
20
+ rescue BuryJob, StandardError => e
21
+ puts "#{e.inspect} - BURYING JOB #{@job.bs_job.id}"
22
+ puts e.backtrace
23
+ @job.bs_job.bury
24
+ end
25
+
26
+ private
27
+ @@retry_on = []
28
+ def self.retry_on err
29
+ @@retry_on << err
30
+ end
31
+
32
+ @@max_retries = 3
33
+ def self.max_retries n
34
+ @@max_retries = n
35
+ end
36
+ end
37
+
38
+
39
+ @@tasks = []
40
+ def self.tasks
41
+ if @@tasks.empty?
42
+ ObjectSpace.each_object(Module) do |m|
43
+ @@tasks << m if m.ancestors.include? Subservient::Task
44
+ end
45
+ @@tasks -= [Subservient::Task]
46
+ end
47
+ return @@tasks
48
+ end
49
+ end
@@ -0,0 +1,3 @@
1
+ module Subservient
2
+ VERSION = '0.0.1'
3
+ end
@@ -0,0 +1,76 @@
1
+ module Subservient
2
+ class Worker
3
+ attr_reader :tube, :state
4
+ def initialize tube='default'
5
+ @tube = [Subservient::namespace, tube].join('.')
6
+ @state = :stopped
7
+ @thread_pool = []
8
+ end
9
+
10
+ def go threads: 5
11
+ start threads: threads
12
+
13
+ trap('SIGINT') do
14
+ exit
15
+ end
16
+ at_exit do
17
+ Thread.main.wakeup
18
+ stop
19
+ end
20
+ sleep
21
+ end
22
+ def start threads: 5
23
+ raise MissingTaskError if Subservient::tasks.empty?
24
+ raise AlreadyRunningError if @state != :stopped
25
+ @state = :started
26
+ threads.times do
27
+ start_worker_thread
28
+ end
29
+ end
30
+ def stop timeout: 30
31
+ @state = :shutdown # Tell worker threads not to accept any more jobs
32
+ @thread_pool.each do |thread|
33
+ thread.join(timeout) # Wait for all workers to exit gracefully...
34
+ thread.kill # but kill them if they take too long
35
+ end
36
+ @state = :stopped
37
+ end
38
+
39
+ private
40
+
41
+ def start_worker_thread
42
+ @thread_pool << Thread.new do
43
+ conn = Beaneater::Pool.new(Subservient::pool)
44
+ begin
45
+ work conn
46
+ ensure
47
+ conn.close
48
+ end
49
+ end
50
+ end
51
+
52
+ def work conn
53
+ puts "Worker started"
54
+ conn.tubes.watch!(@tube)
55
+ while @state == :started do
56
+ begin
57
+ # Increasing this will decrease load on beanstalkd, but will
58
+ # also increase the time it takes to shut down the worker
59
+ timeout = 2
60
+
61
+ bs_job = conn.tubes.reserve(timeout)
62
+ begin
63
+ job = Job.new bs_job
64
+ job.execute
65
+ rescue JobError => e
66
+ puts "#{e} - BURYING JOB #{bs_job.id}"
67
+ bs_job.bury
68
+ end
69
+ rescue Beaneater::TimedOutError
70
+ end
71
+ end
72
+ puts "Worker stopped"
73
+ end
74
+
75
+ end
76
+ end
@@ -0,0 +1,25 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'subservient/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "subservient"
8
+ spec.version = Subservient::VERSION
9
+ spec.authors = ["David Robertson"]
10
+ spec.email = ["david@davidr.me"]
11
+ spec.summary = "Job queue processor using Beanstalkd"
12
+ # spec.description = "TODO: Write a longer description. Optional."
13
+ spec.homepage = "http://crane7.co.uk"
14
+ spec.license = "MIT"
15
+
16
+ spec.files = `git ls-files -z`.split("\x0")
17
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
18
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
19
+ spec.require_paths = ["lib"]
20
+
21
+ spec.add_runtime_dependency "beaneater"
22
+
23
+ spec.add_development_dependency "bundler", "~> 1.6"
24
+ spec.add_development_dependency "rake"
25
+ end
metadata ADDED
@@ -0,0 +1,103 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: subservient
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - David Robertson
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2014-06-04 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: beaneater
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: '0'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ">="
25
+ - !ruby/object:Gem::Version
26
+ version: '0'
27
+ - !ruby/object:Gem::Dependency
28
+ name: bundler
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '1.6'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '1.6'
41
+ - !ruby/object:Gem::Dependency
42
+ name: rake
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ description:
56
+ email:
57
+ - david@davidr.me
58
+ executables:
59
+ - subservient
60
+ extensions: []
61
+ extra_rdoc_files: []
62
+ files:
63
+ - ".gitignore"
64
+ - Gemfile
65
+ - Gemfile.lock
66
+ - LICENSE.txt
67
+ - README.md
68
+ - Rakefile
69
+ - bin/subservient
70
+ - example_task.rb
71
+ - lib/subservient.rb
72
+ - lib/subservient/config.rb
73
+ - lib/subservient/errors.rb
74
+ - lib/subservient/job.rb
75
+ - lib/subservient/task.rb
76
+ - lib/subservient/version.rb
77
+ - lib/subservient/worker.rb
78
+ - subservient.gemspec
79
+ homepage: http://crane7.co.uk
80
+ licenses:
81
+ - MIT
82
+ metadata: {}
83
+ post_install_message:
84
+ rdoc_options: []
85
+ require_paths:
86
+ - lib
87
+ required_ruby_version: !ruby/object:Gem::Requirement
88
+ requirements:
89
+ - - ">="
90
+ - !ruby/object:Gem::Version
91
+ version: '0'
92
+ required_rubygems_version: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - ">="
95
+ - !ruby/object:Gem::Version
96
+ version: '0'
97
+ requirements: []
98
+ rubyforge_project:
99
+ rubygems_version: 2.2.2
100
+ signing_key:
101
+ specification_version: 4
102
+ summary: Job queue processor using Beanstalkd
103
+ test_files: []