beaneater 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +17 -0
- data/.yardopts +8 -0
- data/Gemfile +10 -0
- data/LICENSE.txt +22 -0
- data/README.md +399 -0
- data/REF +23 -0
- data/Rakefile +23 -0
- data/TODO +2 -0
- data/beaneater.gemspec +24 -0
- data/examples/demo.rb +96 -0
- data/lib/beaneater.rb +10 -0
- data/lib/beaneater/connection.rb +110 -0
- data/lib/beaneater/errors.rb +73 -0
- data/lib/beaneater/job.rb +2 -0
- data/lib/beaneater/job/collection.rb +91 -0
- data/lib/beaneater/job/record.rb +174 -0
- data/lib/beaneater/pool.rb +141 -0
- data/lib/beaneater/pool_command.rb +71 -0
- data/lib/beaneater/stats.rb +55 -0
- data/lib/beaneater/stats/fast_struct.rb +96 -0
- data/lib/beaneater/stats/stat_struct.rb +39 -0
- data/lib/beaneater/tube.rb +2 -0
- data/lib/beaneater/tube/collection.rb +134 -0
- data/lib/beaneater/tube/record.rb +158 -0
- data/lib/beaneater/version.rb +4 -0
- data/test/beaneater_test.rb +115 -0
- data/test/connection_test.rb +64 -0
- data/test/errors_test.rb +26 -0
- data/test/job_test.rb +213 -0
- data/test/jobs_test.rb +107 -0
- data/test/pool_command_test.rb +68 -0
- data/test/pool_test.rb +154 -0
- data/test/stat_struct_test.rb +41 -0
- data/test/stats_test.rb +42 -0
- data/test/test_helper.rb +21 -0
- data/test/tube_test.rb +164 -0
- data/test/tubes_test.rb +153 -0
- metadata +181 -0
data/REF
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
# Summary of what this does
|
2
|
+
#
|
3
|
+
# @!attribute [r] count
|
4
|
+
# @param [type] Name description
|
5
|
+
# @option name [Types] option_key (default_value) description
|
6
|
+
# @yield [a, b, c] Gives 3 random numbers to the block
|
7
|
+
# @return [type] description
|
8
|
+
# @raise [Types] description
|
9
|
+
# @example
|
10
|
+
# something.foo # => "test"
|
11
|
+
#
|
12
|
+
|
13
|
+
|
14
|
+
# Summary of what this does
|
15
|
+
#
|
16
|
+
# @param [type] Name description
|
17
|
+
# @option name [Types] option_key (default_value) description
|
18
|
+
# @yield [a, b, c] Gives 3 random numbers to the block
|
19
|
+
# @return [type] description
|
20
|
+
# @raise [Types] description
|
21
|
+
# @example
|
22
|
+
# something.foo # => "test"
|
23
|
+
#
|
data/Rakefile
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
require "bundler/gem_tasks"
|
2
|
+
require 'rake/testtask'
|
3
|
+
require 'yard'
|
4
|
+
require 'redcarpet'
|
5
|
+
|
6
|
+
Rake::TestTask.new do |t|
|
7
|
+
t.libs.push "lib"
|
8
|
+
t.test_files = FileList[File.expand_path('../test/**/*_test.rb', __FILE__)] -
|
9
|
+
FileList[File.expand_path('../test/**/beaneater_test.rb', __FILE__)]
|
10
|
+
t.verbose = true
|
11
|
+
end
|
12
|
+
|
13
|
+
# rake test:full
|
14
|
+
Rake::TestTask.new("test:full") do |t|
|
15
|
+
t.libs.push "lib"
|
16
|
+
t.test_files = FileList[File.expand_path('../test/**/*_test.rb', __FILE__)]
|
17
|
+
t.verbose = true
|
18
|
+
end
|
19
|
+
|
20
|
+
YARD::Rake::YardocTask.new do |t|
|
21
|
+
t.files = ['lib/beaneater/**/*.rb']
|
22
|
+
t.options = []
|
23
|
+
end
|
data/TODO
ADDED
data/beaneater.gemspec
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'beaneater/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |gem|
|
7
|
+
gem.name = "beaneater"
|
8
|
+
gem.version = Beaneater::VERSION
|
9
|
+
gem.authors = ["Nico Taing"]
|
10
|
+
gem.email = ["nico.taing@gmail.com"]
|
11
|
+
gem.description = %q{Simple beanstalkd client for ruby}
|
12
|
+
gem.summary = %q{Simple beanstalkd client for ruby.}
|
13
|
+
gem.homepage = ""
|
14
|
+
|
15
|
+
gem.files = `git ls-files`.split($/)
|
16
|
+
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
17
|
+
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
18
|
+
gem.require_paths = ["lib"]
|
19
|
+
gem.add_development_dependency 'minitest', "~> 4.1.0"
|
20
|
+
gem.add_development_dependency 'rake'
|
21
|
+
gem.add_development_dependency 'mocha'
|
22
|
+
gem.add_development_dependency 'fakeweb'
|
23
|
+
gem.add_development_dependency 'term-ansicolor'
|
24
|
+
end
|
data/examples/demo.rb
ADDED
@@ -0,0 +1,96 @@
|
|
1
|
+
require 'term/ansicolor'
|
2
|
+
class String; include Term::ANSIColor; end
|
3
|
+
def step(msg); "\n[STEP] #{msg}...".yellow; end
|
4
|
+
$:.unshift("../lib")
|
5
|
+
require 'beaneater'
|
6
|
+
|
7
|
+
# Establish a pool of beanstalks
|
8
|
+
puts step("Connecting to Beanstalk")
|
9
|
+
bc = Beaneater::Pool.new('localhost')
|
10
|
+
# bc = Beaneater::Pool.new(['localhost', 'localhost:11301', 'localhost:11302'])
|
11
|
+
puts bc
|
12
|
+
|
13
|
+
# Print out key stats
|
14
|
+
puts step("Print Stats")
|
15
|
+
p bc.stats.keys
|
16
|
+
p [bc.stats.total_connections, bc.stats[:total_connections], bc.stats['total_connections']]
|
17
|
+
|
18
|
+
# find tube
|
19
|
+
puts step("Find tube")
|
20
|
+
tube = bc.tubes.find('tube2')
|
21
|
+
puts tube
|
22
|
+
|
23
|
+
# Put job onto tube
|
24
|
+
puts step("Put job")
|
25
|
+
response = tube.put "foo bar", :pri => 1000, :ttr => 10, :delay => 0
|
26
|
+
puts response
|
27
|
+
|
28
|
+
# peek tube
|
29
|
+
puts step("Peek tube")
|
30
|
+
p tube.peek :ready
|
31
|
+
|
32
|
+
# watch tube
|
33
|
+
bc.tubes.watch!('tube2')
|
34
|
+
|
35
|
+
# Check tube stats
|
36
|
+
puts step("Get tube stats")
|
37
|
+
p tube.stats.keys
|
38
|
+
p tube.stats.name
|
39
|
+
p tube.stats.current_jobs_ready
|
40
|
+
|
41
|
+
# Reserve job from tube
|
42
|
+
puts step("Reserve job")
|
43
|
+
p job = bc.tubes.reserve
|
44
|
+
jid = job.id
|
45
|
+
|
46
|
+
# pause tube
|
47
|
+
puts step("Pause tube")
|
48
|
+
p tube.pause(1)
|
49
|
+
|
50
|
+
# Register jobs
|
51
|
+
puts step("Register jobs for tubes")
|
52
|
+
bc.jobs.register('tube_test', :retry_on => [Timeout::Error]) do |job|
|
53
|
+
p 'tube_test'
|
54
|
+
p job
|
55
|
+
raise Beaneater::AbortProcessingError
|
56
|
+
end
|
57
|
+
|
58
|
+
bc.jobs.register('tube_test2', :retry_on => [Timeout::Error]) do |job|
|
59
|
+
p 'tube_test2'
|
60
|
+
p job
|
61
|
+
raise Beaneater::AbortProcessingError
|
62
|
+
end
|
63
|
+
|
64
|
+
p bc.jobs.processors
|
65
|
+
|
66
|
+
response = bc.tubes.find('tube_test').put "foo register", :pri => 1000, :ttr => 10, :delay => 0
|
67
|
+
response = bc.tubes.find('tube_test2').put "foo baz", :pri => 1000, :ttr => 10, :delay => 0
|
68
|
+
|
69
|
+
# Process jobs
|
70
|
+
puts step("Process jobs")
|
71
|
+
2.times { bc.jobs.process! }
|
72
|
+
|
73
|
+
# Get job from id (peek job)
|
74
|
+
puts step("Get job from id")
|
75
|
+
p bc.jobs.find(jid)
|
76
|
+
p bc.jobs.peek(jid)
|
77
|
+
|
78
|
+
# Check job stats
|
79
|
+
puts step("Get job stats")
|
80
|
+
p job.stats.keys
|
81
|
+
p job.stats.tube
|
82
|
+
p job.stats.state
|
83
|
+
|
84
|
+
# bury job
|
85
|
+
puts step("Bury job")
|
86
|
+
p job.bury
|
87
|
+
|
88
|
+
# delete job
|
89
|
+
puts step("Delete job")
|
90
|
+
p job.delete
|
91
|
+
|
92
|
+
# list tubes
|
93
|
+
puts step("List tubes")
|
94
|
+
p bc.tubes.watched
|
95
|
+
p bc.tubes.used
|
96
|
+
p bc.tubes.all
|
data/lib/beaneater.rb
ADDED
@@ -0,0 +1,110 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
|
3
|
+
module Beaneater
|
4
|
+
# Represents a connection to beanstalkd server
|
5
|
+
class Connection
|
6
|
+
|
7
|
+
# @!attribute telnet_connection
|
8
|
+
# @return [Net::Telnet] returns Telnet connection object
|
9
|
+
# @!attribute address
|
10
|
+
# @return [String] returns Beanstalkd server address
|
11
|
+
# @!attribute host
|
12
|
+
# @return [String] returns Beanstalkd server host
|
13
|
+
# @!attribute port
|
14
|
+
# @return [Integer] returns Beanstalkd server port
|
15
|
+
attr_reader :telnet_connection, :address, :host, :port
|
16
|
+
|
17
|
+
# Default port value
|
18
|
+
DEFAULT_PORT = 11300
|
19
|
+
|
20
|
+
# Initialize new connection
|
21
|
+
#
|
22
|
+
# @param [String] address beanstalkd address
|
23
|
+
# @example
|
24
|
+
# Beaneater::Connection.new('localhost')
|
25
|
+
# Beaneater::Connection.new('localhost:11300')
|
26
|
+
def initialize(address)
|
27
|
+
@address = address
|
28
|
+
@telnet_connection = establish_connection
|
29
|
+
@mutex = Mutex.new
|
30
|
+
end
|
31
|
+
|
32
|
+
# Send commands to beanstalkd server via telnet_connection
|
33
|
+
#
|
34
|
+
# @param [String] command Beanstalkd command
|
35
|
+
# @param [Hash] options Settings for telnet
|
36
|
+
# @option options [Boolean] FailEOF raises EOF Exeception
|
37
|
+
#
|
38
|
+
# @example
|
39
|
+
# @beaneater_connection.transmit('bury 123')
|
40
|
+
def transmit(command, options={}, &block)
|
41
|
+
@mutex.lock
|
42
|
+
if telnet_connection
|
43
|
+
options.merge!("String" => command, "FailEOF" => true)
|
44
|
+
parse_response(command, telnet_connection.cmd(options, &block))
|
45
|
+
else # no telnet_connection
|
46
|
+
raise NotConnected, "Connection to beanstalk '#{@host}:#{@port}' is closed!" unless telnet_connection
|
47
|
+
end
|
48
|
+
ensure
|
49
|
+
@mutex.unlock
|
50
|
+
end
|
51
|
+
|
52
|
+
# Close connection with beanstalkd server
|
53
|
+
#
|
54
|
+
# @example
|
55
|
+
# @beaneater_connection.close
|
56
|
+
def close
|
57
|
+
@telnet_connection.close
|
58
|
+
@telnet_connection = nil
|
59
|
+
end
|
60
|
+
|
61
|
+
# Returns string representation of job
|
62
|
+
#
|
63
|
+
# @example
|
64
|
+
# @beaneater_connection.inspect
|
65
|
+
def to_s
|
66
|
+
"#<Beaneater::Connection host=#{host.inspect} port=#{port.inspect}>"
|
67
|
+
end
|
68
|
+
alias :inspect :to_s
|
69
|
+
|
70
|
+
protected
|
71
|
+
|
72
|
+
# Establish a telnet connection based on beanstalk address.
|
73
|
+
#
|
74
|
+
# @return [Net::Telnet] telnet connection for specified address.
|
75
|
+
# @raise [Beanstalk::NotConnected] Could not connect to specified beanstalkd instance.
|
76
|
+
# @example
|
77
|
+
# establish_connection('localhost:3005')
|
78
|
+
#
|
79
|
+
def establish_connection
|
80
|
+
@match = address.split(':')
|
81
|
+
@host, @port = @match[0], Integer(@match[1] || DEFAULT_PORT)
|
82
|
+
Net::Telnet.new('Host' => @host, "Port" => @port, "Prompt" => /\n/)
|
83
|
+
rescue Errno::ECONNREFUSED => e
|
84
|
+
raise NotConnected, "Could not connect to '#{@host}:#{@port}'"
|
85
|
+
rescue Exception => ex
|
86
|
+
raise NotConnected, "#{ex.class}: #{ex}"
|
87
|
+
end
|
88
|
+
|
89
|
+
# Parses the telnet response and returns the useful beanstalk response.
|
90
|
+
#
|
91
|
+
# @param [String] cmd Beanstalk command transmitted
|
92
|
+
# @param [String] res Telnet command response
|
93
|
+
# @return [Hash] Beanstalk command response with `status`, `id`, `body`, and `connection`
|
94
|
+
# @raise [Beaneater::UnexpectedResponse] Response from beanstalk command was an error status
|
95
|
+
# @example
|
96
|
+
# parse_response("delete 56", "DELETED 56\nFOO")
|
97
|
+
# # => { :body => "FOO", :status => "DELETED", :id => 56, :connection => <Connection> }
|
98
|
+
#
|
99
|
+
def parse_response(cmd, res)
|
100
|
+
res_lines = res.split(/\r?\n/)
|
101
|
+
status = res_lines.first
|
102
|
+
status, id = status.scan(/\w+/)
|
103
|
+
raise UnexpectedResponse.from_status(status, cmd) if UnexpectedResponse::ERROR_STATES.include?(status)
|
104
|
+
response = { :status => status, :body => YAML.load(res_lines[1..-1].join("\n")) }
|
105
|
+
response[:id] = id if id
|
106
|
+
response[:connection] = self
|
107
|
+
response
|
108
|
+
end
|
109
|
+
end # Connection
|
110
|
+
end # Beaneater
|
@@ -0,0 +1,73 @@
|
|
1
|
+
module Beaneater
|
2
|
+
# Raises when the beanstalkd instance cannot be accessed
|
3
|
+
class NotConnected < RuntimeError; end
|
4
|
+
# Raises when the tube name specified is invalid.
|
5
|
+
class InvalidTubeName < RuntimeError; end
|
6
|
+
# Raises when a job has not been reserved properly.
|
7
|
+
class JobNotReserved < RuntimeError; end
|
8
|
+
|
9
|
+
# Abstract class for errors that occur when a command does not complete successfully.
|
10
|
+
class UnexpectedResponse < RuntimeError
|
11
|
+
# Set of status states that are considered errors
|
12
|
+
ERROR_STATES = %w(OUT_OF_MEMORY INTERNAL_ERROR
|
13
|
+
BAD_FORMAT UNKNOWN_COMMAND JOB_TOO_BIG DRAINING
|
14
|
+
TIMED_OUT DEADLINE_SOON NOT_FOUND NOT_IGNORED EXPECTED_CRLF)
|
15
|
+
|
16
|
+
# @!attribute status
|
17
|
+
# @return [String] returns beanstalkd response status
|
18
|
+
# @!attribute cmd
|
19
|
+
# @return [String] returns beanstalkd request command
|
20
|
+
attr_reader :status, :cmd
|
21
|
+
|
22
|
+
# Initialize unexpected response error
|
23
|
+
#
|
24
|
+
# @param [UnexpectedResponse] status Unexpected response object
|
25
|
+
# @param [String] cmd Beanstalkd request command
|
26
|
+
#
|
27
|
+
# @example
|
28
|
+
# Beaneater::UnexpectedResponse.new(NotFoundError, 'bury 123')
|
29
|
+
#
|
30
|
+
def initialize(status, cmd)
|
31
|
+
@status, @cmd = status, cmd
|
32
|
+
super("Response failed with: #{status}")
|
33
|
+
end
|
34
|
+
|
35
|
+
# Translate beanstalkd error status to ruby Exeception
|
36
|
+
#
|
37
|
+
# @param [String] status Beanstalkd error status
|
38
|
+
# @param [String] cmd Beanstalkd request command
|
39
|
+
#
|
40
|
+
# @example
|
41
|
+
# Beaneater::UnexpectedResponse.new('NOT_FOUND', 'bury 123')
|
42
|
+
#
|
43
|
+
def self.from_status(status, cmd)
|
44
|
+
error_klazz_name = status.split('_').map { |w| w.capitalize }.join
|
45
|
+
error_klazz_name << "Error" unless error_klazz_name =~ /Error$/
|
46
|
+
error_klazz = Beaneater.const_get(error_klazz_name)
|
47
|
+
error_klazz.new(status, cmd)
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
# Raises when the beanstalkd instance runs out of memory
|
52
|
+
class OutOfMemoryError < UnexpectedResponse; end
|
53
|
+
# Raises when the beanstalkd instance is draining and new jobs cannot be inserted
|
54
|
+
class DrainingError < UnexpectedResponse; end
|
55
|
+
# Raises when the job or tube cannot be found
|
56
|
+
class NotFoundError < UnexpectedResponse; end
|
57
|
+
# Raises when the job reserved is going to be released within a second.
|
58
|
+
class DeadlineSoonError < UnexpectedResponse; end
|
59
|
+
# Raises when a beanstalkd has an internal error.
|
60
|
+
class InternalError < UnexpectedResponse; end
|
61
|
+
# Raises when a command was not properly formatted.
|
62
|
+
class BadFormatError < UnexpectedResponse; end
|
63
|
+
# Raises when a command was sent that is unknown.
|
64
|
+
class UnknownCommandError < UnexpectedResponse; end
|
65
|
+
# Raises when command does not have proper CRLF suffix.
|
66
|
+
class ExpectedCRLFError < UnexpectedResponse; end
|
67
|
+
# Raises when the body of a job was too large.
|
68
|
+
class JobTooBigError < UnexpectedResponse; end
|
69
|
+
# Raises when a job was attempted to be reserved but the timeout occured.
|
70
|
+
class TimedOutError < UnexpectedResponse; end
|
71
|
+
# Raises when a tube could not be ignored because it is the last watched tube.
|
72
|
+
class NotIgnoredError < UnexpectedResponse; end
|
73
|
+
end
|
@@ -0,0 +1,91 @@
|
|
1
|
+
module Beaneater
|
2
|
+
# Exception to stop processing jobs
|
3
|
+
class AbortProcessingError < RuntimeError; end
|
4
|
+
|
5
|
+
# Represents collection of jobs related commands.
|
6
|
+
class Jobs < PoolCommand
|
7
|
+
|
8
|
+
# @!attribute processors
|
9
|
+
# @return [Array<Proc>] returns Collection of proc to handle beanstalkd jobs
|
10
|
+
attr_reader :processors
|
11
|
+
|
12
|
+
# Number of retries to process a job
|
13
|
+
MAX_RETRIES = 3
|
14
|
+
|
15
|
+
# Delay in seconds before to make job ready again
|
16
|
+
RELEASE_DELAY = 1
|
17
|
+
|
18
|
+
# Peek (or find) a job across all beanstalkd servers from pool
|
19
|
+
#
|
20
|
+
# @param [Integer] id Job id
|
21
|
+
#
|
22
|
+
# @raise [Beaneater::NotFoundError] Job not found
|
23
|
+
#
|
24
|
+
# @example
|
25
|
+
# @beaneater_pool.jobs.find(123) # => <Beaneater::Job>
|
26
|
+
# @beaneater_pool.jobs.peek(123) # => <Beaneater::Job>
|
27
|
+
# @beaneater_pool.jobs.find[123] # => <Beaneater::Job>
|
28
|
+
#
|
29
|
+
# @api public
|
30
|
+
def find(id)
|
31
|
+
res = transmit_until_res("peek #{id}", :status => "FOUND")
|
32
|
+
Job.new(res)
|
33
|
+
rescue Beaneater::NotFoundError => ex
|
34
|
+
nil
|
35
|
+
end
|
36
|
+
alias_method :peek, :find
|
37
|
+
alias_method :[], :find
|
38
|
+
|
39
|
+
# Add processor to handle beanstalkd job
|
40
|
+
#
|
41
|
+
# @param [String] tube_name Tube name
|
42
|
+
# @param [Hash] options settings for processor
|
43
|
+
# @option options [Integer] max_retries Number of retries to process a job
|
44
|
+
# @option options [Array<RuntimeError>] retry_on Collection of errors to rescue and re-run processor
|
45
|
+
# @param [Proc] block Process beanstalkd job
|
46
|
+
#
|
47
|
+
# @example
|
48
|
+
# @beanstalk.jobs.register('some-tube', :retry_on => [SomeError]) do |job|
|
49
|
+
# do_something(job)
|
50
|
+
# end
|
51
|
+
#
|
52
|
+
# @beanstalk.jobs.register('other-tube') do |job|
|
53
|
+
# do_something_else(job)
|
54
|
+
# end
|
55
|
+
#
|
56
|
+
# @api public
|
57
|
+
def register(tube_name, options={}, &block)
|
58
|
+
@processors ||= {}
|
59
|
+
max_retries = options[:max_retries] || MAX_RETRIES
|
60
|
+
retry_on = Array(options[:retry_on])
|
61
|
+
@processors[tube_name.to_s] = { :block => block, :retry_on => retry_on, :max_retries => max_retries }
|
62
|
+
end
|
63
|
+
|
64
|
+
# Watch, reserve, process and delete or bury or release jobs
|
65
|
+
#
|
66
|
+
# @param [Hash] options Settings for processing
|
67
|
+
# @option options [Integer] Delay in seconds before to make job ready again
|
68
|
+
#
|
69
|
+
# @api public
|
70
|
+
def process!(options={})
|
71
|
+
release_delay = options.delete(:release_delay) || RELEASE_DELAY
|
72
|
+
tubes.watch!(*processors.keys)
|
73
|
+
loop do
|
74
|
+
job = tubes.reserve
|
75
|
+
processor = processors[job.tube]
|
76
|
+
begin
|
77
|
+
processor[:block].call(job)
|
78
|
+
job.delete
|
79
|
+
rescue AbortProcessingError
|
80
|
+
break
|
81
|
+
rescue *processor[:retry_on]
|
82
|
+
job.release(:delay => release_delay) if job.stats.releases < processor[:max_retries]
|
83
|
+
rescue StandardError => e # handles unspecified errors
|
84
|
+
job.bury
|
85
|
+
ensure # bury if still reserved
|
86
|
+
job.bury if job.exists? && job.reserved?
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end # process!
|
90
|
+
end # Jobs
|
91
|
+
end # Beaneater
|