beaneater 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/REF ADDED
@@ -0,0 +1,23 @@
1
+ # Summary of what this does
2
+ #
3
+ # @!attribute [r] count
4
+ # @param [type] Name description
5
+ # @option name [Types] option_key (default_value) description
6
+ # @yield [a, b, c] Gives 3 random numbers to the block
7
+ # @return [type] description
8
+ # @raise [Types] description
9
+ # @example
10
+ # something.foo # => "test"
11
+ #
12
+
13
+
14
+ # Summary of what this does
15
+ #
16
+ # @param [type] Name description
17
+ # @option name [Types] option_key (default_value) description
18
+ # @yield [a, b, c] Gives 3 random numbers to the block
19
+ # @return [type] description
20
+ # @raise [Types] description
21
+ # @example
22
+ # something.foo # => "test"
23
+ #
data/Rakefile ADDED
@@ -0,0 +1,23 @@
1
+ require "bundler/gem_tasks"
2
+ require 'rake/testtask'
3
+ require 'yard'
4
+ require 'redcarpet'
5
+
6
+ Rake::TestTask.new do |t|
7
+ t.libs.push "lib"
8
+ t.test_files = FileList[File.expand_path('../test/**/*_test.rb', __FILE__)] -
9
+ FileList[File.expand_path('../test/**/beaneater_test.rb', __FILE__)]
10
+ t.verbose = true
11
+ end
12
+
13
+ # rake test:full
14
+ Rake::TestTask.new("test:full") do |t|
15
+ t.libs.push "lib"
16
+ t.test_files = FileList[File.expand_path('../test/**/*_test.rb', __FILE__)]
17
+ t.verbose = true
18
+ end
19
+
20
+ YARD::Rake::YardocTask.new do |t|
21
+ t.files = ['lib/beaneater/**/*.rb']
22
+ t.options = []
23
+ end
data/TODO ADDED
@@ -0,0 +1,2 @@
1
+ - Add YARD docs for all classes (see REF file)
2
+ - Remove connection from pool if it's not responding and be able to add more connections and reattempt later
data/beaneater.gemspec ADDED
@@ -0,0 +1,24 @@
1
+ # -*- encoding: utf-8 -*-
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'beaneater/version'
5
+
6
+ Gem::Specification.new do |gem|
7
+ gem.name = "beaneater"
8
+ gem.version = Beaneater::VERSION
9
+ gem.authors = ["Nico Taing"]
10
+ gem.email = ["nico.taing@gmail.com"]
11
+ gem.description = %q{Simple beanstalkd client for ruby}
12
+ gem.summary = %q{Simple beanstalkd client for ruby.}
13
+ gem.homepage = ""
14
+
15
+ gem.files = `git ls-files`.split($/)
16
+ gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
17
+ gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
18
+ gem.require_paths = ["lib"]
19
+ gem.add_development_dependency 'minitest', "~> 4.1.0"
20
+ gem.add_development_dependency 'rake'
21
+ gem.add_development_dependency 'mocha'
22
+ gem.add_development_dependency 'fakeweb'
23
+ gem.add_development_dependency 'term-ansicolor'
24
+ end
data/examples/demo.rb ADDED
@@ -0,0 +1,96 @@
1
+ require 'term/ansicolor'
2
+ class String; include Term::ANSIColor; end
3
+ def step(msg); "\n[STEP] #{msg}...".yellow; end
4
+ $:.unshift("../lib")
5
+ require 'beaneater'
6
+
7
+ # Establish a pool of beanstalks
8
+ puts step("Connecting to Beanstalk")
9
+ bc = Beaneater::Pool.new('localhost')
10
+ # bc = Beaneater::Pool.new(['localhost', 'localhost:11301', 'localhost:11302'])
11
+ puts bc
12
+
13
+ # Print out key stats
14
+ puts step("Print Stats")
15
+ p bc.stats.keys
16
+ p [bc.stats.total_connections, bc.stats[:total_connections], bc.stats['total_connections']]
17
+
18
+ # find tube
19
+ puts step("Find tube")
20
+ tube = bc.tubes.find('tube2')
21
+ puts tube
22
+
23
+ # Put job onto tube
24
+ puts step("Put job")
25
+ response = tube.put "foo bar", :pri => 1000, :ttr => 10, :delay => 0
26
+ puts response
27
+
28
+ # peek tube
29
+ puts step("Peek tube")
30
+ p tube.peek :ready
31
+
32
+ # watch tube
33
+ bc.tubes.watch!('tube2')
34
+
35
+ # Check tube stats
36
+ puts step("Get tube stats")
37
+ p tube.stats.keys
38
+ p tube.stats.name
39
+ p tube.stats.current_jobs_ready
40
+
41
+ # Reserve job from tube
42
+ puts step("Reserve job")
43
+ p job = bc.tubes.reserve
44
+ jid = job.id
45
+
46
+ # pause tube
47
+ puts step("Pause tube")
48
+ p tube.pause(1)
49
+
50
+ # Register jobs
51
+ puts step("Register jobs for tubes")
52
+ bc.jobs.register('tube_test', :retry_on => [Timeout::Error]) do |job|
53
+ p 'tube_test'
54
+ p job
55
+ raise Beaneater::AbortProcessingError
56
+ end
57
+
58
+ bc.jobs.register('tube_test2', :retry_on => [Timeout::Error]) do |job|
59
+ p 'tube_test2'
60
+ p job
61
+ raise Beaneater::AbortProcessingError
62
+ end
63
+
64
+ p bc.jobs.processors
65
+
66
+ response = bc.tubes.find('tube_test').put "foo register", :pri => 1000, :ttr => 10, :delay => 0
67
+ response = bc.tubes.find('tube_test2').put "foo baz", :pri => 1000, :ttr => 10, :delay => 0
68
+
69
+ # Process jobs
70
+ puts step("Process jobs")
71
+ 2.times { bc.jobs.process! }
72
+
73
+ # Get job from id (peek job)
74
+ puts step("Get job from id")
75
+ p bc.jobs.find(jid)
76
+ p bc.jobs.peek(jid)
77
+
78
+ # Check job stats
79
+ puts step("Get job stats")
80
+ p job.stats.keys
81
+ p job.stats.tube
82
+ p job.stats.state
83
+
84
+ # bury job
85
+ puts step("Bury job")
86
+ p job.bury
87
+
88
+ # delete job
89
+ puts step("Delete job")
90
+ p job.delete
91
+
92
+ # list tubes
93
+ puts step("List tubes")
94
+ p bc.tubes.watched
95
+ p bc.tubes.used
96
+ p bc.tubes.all
data/lib/beaneater.rb ADDED
@@ -0,0 +1,10 @@
1
+ require 'net/telnet'
2
+
3
+ %w(version errors pool_command pool connection stats tube job).each do |f|
4
+ require "beaneater/#{f}"
5
+ end
6
+
7
+ # Simple ruby client for beanstalkd.
8
+ module Beaneater
9
+
10
+ end
@@ -0,0 +1,110 @@
1
+ require 'yaml'
2
+
3
+ module Beaneater
4
+ # Represents a connection to beanstalkd server
5
+ class Connection
6
+
7
+ # @!attribute telnet_connection
8
+ # @return [Net::Telnet] returns Telnet connection object
9
+ # @!attribute address
10
+ # @return [String] returns Beanstalkd server address
11
+ # @!attribute host
12
+ # @return [String] returns Beanstalkd server host
13
+ # @!attribute port
14
+ # @return [Integer] returns Beanstalkd server port
15
+ attr_reader :telnet_connection, :address, :host, :port
16
+
17
+ # Default port value
18
+ DEFAULT_PORT = 11300
19
+
20
+ # Initialize new connection
21
+ #
22
+ # @param [String] address beanstalkd address
23
+ # @example
24
+ # Beaneater::Connection.new('localhost')
25
+ # Beaneater::Connection.new('localhost:11300')
26
+ def initialize(address)
27
+ @address = address
28
+ @telnet_connection = establish_connection
29
+ @mutex = Mutex.new
30
+ end
31
+
32
+ # Send commands to beanstalkd server via telnet_connection
33
+ #
34
+ # @param [String] command Beanstalkd command
35
+ # @param [Hash] options Settings for telnet
36
+ # @option options [Boolean] FailEOF raises EOF Exeception
37
+ #
38
+ # @example
39
+ # @beaneater_connection.transmit('bury 123')
40
+ def transmit(command, options={}, &block)
41
+ @mutex.lock
42
+ if telnet_connection
43
+ options.merge!("String" => command, "FailEOF" => true)
44
+ parse_response(command, telnet_connection.cmd(options, &block))
45
+ else # no telnet_connection
46
+ raise NotConnected, "Connection to beanstalk '#{@host}:#{@port}' is closed!" unless telnet_connection
47
+ end
48
+ ensure
49
+ @mutex.unlock
50
+ end
51
+
52
+ # Close connection with beanstalkd server
53
+ #
54
+ # @example
55
+ # @beaneater_connection.close
56
+ def close
57
+ @telnet_connection.close
58
+ @telnet_connection = nil
59
+ end
60
+
61
+ # Returns string representation of job
62
+ #
63
+ # @example
64
+ # @beaneater_connection.inspect
65
+ def to_s
66
+ "#<Beaneater::Connection host=#{host.inspect} port=#{port.inspect}>"
67
+ end
68
+ alias :inspect :to_s
69
+
70
+ protected
71
+
72
+ # Establish a telnet connection based on beanstalk address.
73
+ #
74
+ # @return [Net::Telnet] telnet connection for specified address.
75
+ # @raise [Beanstalk::NotConnected] Could not connect to specified beanstalkd instance.
76
+ # @example
77
+ # establish_connection('localhost:3005')
78
+ #
79
+ def establish_connection
80
+ @match = address.split(':')
81
+ @host, @port = @match[0], Integer(@match[1] || DEFAULT_PORT)
82
+ Net::Telnet.new('Host' => @host, "Port" => @port, "Prompt" => /\n/)
83
+ rescue Errno::ECONNREFUSED => e
84
+ raise NotConnected, "Could not connect to '#{@host}:#{@port}'"
85
+ rescue Exception => ex
86
+ raise NotConnected, "#{ex.class}: #{ex}"
87
+ end
88
+
89
+ # Parses the telnet response and returns the useful beanstalk response.
90
+ #
91
+ # @param [String] cmd Beanstalk command transmitted
92
+ # @param [String] res Telnet command response
93
+ # @return [Hash] Beanstalk command response with `status`, `id`, `body`, and `connection`
94
+ # @raise [Beaneater::UnexpectedResponse] Response from beanstalk command was an error status
95
+ # @example
96
+ # parse_response("delete 56", "DELETED 56\nFOO")
97
+ # # => { :body => "FOO", :status => "DELETED", :id => 56, :connection => <Connection> }
98
+ #
99
+ def parse_response(cmd, res)
100
+ res_lines = res.split(/\r?\n/)
101
+ status = res_lines.first
102
+ status, id = status.scan(/\w+/)
103
+ raise UnexpectedResponse.from_status(status, cmd) if UnexpectedResponse::ERROR_STATES.include?(status)
104
+ response = { :status => status, :body => YAML.load(res_lines[1..-1].join("\n")) }
105
+ response[:id] = id if id
106
+ response[:connection] = self
107
+ response
108
+ end
109
+ end # Connection
110
+ end # Beaneater
@@ -0,0 +1,73 @@
1
+ module Beaneater
2
+ # Raises when the beanstalkd instance cannot be accessed
3
+ class NotConnected < RuntimeError; end
4
+ # Raises when the tube name specified is invalid.
5
+ class InvalidTubeName < RuntimeError; end
6
+ # Raises when a job has not been reserved properly.
7
+ class JobNotReserved < RuntimeError; end
8
+
9
+ # Abstract class for errors that occur when a command does not complete successfully.
10
+ class UnexpectedResponse < RuntimeError
11
+ # Set of status states that are considered errors
12
+ ERROR_STATES = %w(OUT_OF_MEMORY INTERNAL_ERROR
13
+ BAD_FORMAT UNKNOWN_COMMAND JOB_TOO_BIG DRAINING
14
+ TIMED_OUT DEADLINE_SOON NOT_FOUND NOT_IGNORED EXPECTED_CRLF)
15
+
16
+ # @!attribute status
17
+ # @return [String] returns beanstalkd response status
18
+ # @!attribute cmd
19
+ # @return [String] returns beanstalkd request command
20
+ attr_reader :status, :cmd
21
+
22
+ # Initialize unexpected response error
23
+ #
24
+ # @param [UnexpectedResponse] status Unexpected response object
25
+ # @param [String] cmd Beanstalkd request command
26
+ #
27
+ # @example
28
+ # Beaneater::UnexpectedResponse.new(NotFoundError, 'bury 123')
29
+ #
30
+ def initialize(status, cmd)
31
+ @status, @cmd = status, cmd
32
+ super("Response failed with: #{status}")
33
+ end
34
+
35
+ # Translate beanstalkd error status to ruby Exeception
36
+ #
37
+ # @param [String] status Beanstalkd error status
38
+ # @param [String] cmd Beanstalkd request command
39
+ #
40
+ # @example
41
+ # Beaneater::UnexpectedResponse.new('NOT_FOUND', 'bury 123')
42
+ #
43
+ def self.from_status(status, cmd)
44
+ error_klazz_name = status.split('_').map { |w| w.capitalize }.join
45
+ error_klazz_name << "Error" unless error_klazz_name =~ /Error$/
46
+ error_klazz = Beaneater.const_get(error_klazz_name)
47
+ error_klazz.new(status, cmd)
48
+ end
49
+ end
50
+
51
+ # Raises when the beanstalkd instance runs out of memory
52
+ class OutOfMemoryError < UnexpectedResponse; end
53
+ # Raises when the beanstalkd instance is draining and new jobs cannot be inserted
54
+ class DrainingError < UnexpectedResponse; end
55
+ # Raises when the job or tube cannot be found
56
+ class NotFoundError < UnexpectedResponse; end
57
+ # Raises when the job reserved is going to be released within a second.
58
+ class DeadlineSoonError < UnexpectedResponse; end
59
+ # Raises when a beanstalkd has an internal error.
60
+ class InternalError < UnexpectedResponse; end
61
+ # Raises when a command was not properly formatted.
62
+ class BadFormatError < UnexpectedResponse; end
63
+ # Raises when a command was sent that is unknown.
64
+ class UnknownCommandError < UnexpectedResponse; end
65
+ # Raises when command does not have proper CRLF suffix.
66
+ class ExpectedCRLFError < UnexpectedResponse; end
67
+ # Raises when the body of a job was too large.
68
+ class JobTooBigError < UnexpectedResponse; end
69
+ # Raises when a job was attempted to be reserved but the timeout occured.
70
+ class TimedOutError < UnexpectedResponse; end
71
+ # Raises when a tube could not be ignored because it is the last watched tube.
72
+ class NotIgnoredError < UnexpectedResponse; end
73
+ end
@@ -0,0 +1,2 @@
1
+ require 'beaneater/job/record'
2
+ require 'beaneater/job/collection'
@@ -0,0 +1,91 @@
1
+ module Beaneater
2
+ # Exception to stop processing jobs
3
+ class AbortProcessingError < RuntimeError; end
4
+
5
+ # Represents collection of jobs related commands.
6
+ class Jobs < PoolCommand
7
+
8
+ # @!attribute processors
9
+ # @return [Array<Proc>] returns Collection of proc to handle beanstalkd jobs
10
+ attr_reader :processors
11
+
12
+ # Number of retries to process a job
13
+ MAX_RETRIES = 3
14
+
15
+ # Delay in seconds before to make job ready again
16
+ RELEASE_DELAY = 1
17
+
18
+ # Peek (or find) a job across all beanstalkd servers from pool
19
+ #
20
+ # @param [Integer] id Job id
21
+ #
22
+ # @raise [Beaneater::NotFoundError] Job not found
23
+ #
24
+ # @example
25
+ # @beaneater_pool.jobs.find(123) # => <Beaneater::Job>
26
+ # @beaneater_pool.jobs.peek(123) # => <Beaneater::Job>
27
+ # @beaneater_pool.jobs.find[123] # => <Beaneater::Job>
28
+ #
29
+ # @api public
30
+ def find(id)
31
+ res = transmit_until_res("peek #{id}", :status => "FOUND")
32
+ Job.new(res)
33
+ rescue Beaneater::NotFoundError => ex
34
+ nil
35
+ end
36
+ alias_method :peek, :find
37
+ alias_method :[], :find
38
+
39
+ # Add processor to handle beanstalkd job
40
+ #
41
+ # @param [String] tube_name Tube name
42
+ # @param [Hash] options settings for processor
43
+ # @option options [Integer] max_retries Number of retries to process a job
44
+ # @option options [Array<RuntimeError>] retry_on Collection of errors to rescue and re-run processor
45
+ # @param [Proc] block Process beanstalkd job
46
+ #
47
+ # @example
48
+ # @beanstalk.jobs.register('some-tube', :retry_on => [SomeError]) do |job|
49
+ # do_something(job)
50
+ # end
51
+ #
52
+ # @beanstalk.jobs.register('other-tube') do |job|
53
+ # do_something_else(job)
54
+ # end
55
+ #
56
+ # @api public
57
+ def register(tube_name, options={}, &block)
58
+ @processors ||= {}
59
+ max_retries = options[:max_retries] || MAX_RETRIES
60
+ retry_on = Array(options[:retry_on])
61
+ @processors[tube_name.to_s] = { :block => block, :retry_on => retry_on, :max_retries => max_retries }
62
+ end
63
+
64
+ # Watch, reserve, process and delete or bury or release jobs
65
+ #
66
+ # @param [Hash] options Settings for processing
67
+ # @option options [Integer] Delay in seconds before to make job ready again
68
+ #
69
+ # @api public
70
+ def process!(options={})
71
+ release_delay = options.delete(:release_delay) || RELEASE_DELAY
72
+ tubes.watch!(*processors.keys)
73
+ loop do
74
+ job = tubes.reserve
75
+ processor = processors[job.tube]
76
+ begin
77
+ processor[:block].call(job)
78
+ job.delete
79
+ rescue AbortProcessingError
80
+ break
81
+ rescue *processor[:retry_on]
82
+ job.release(:delay => release_delay) if job.stats.releases < processor[:max_retries]
83
+ rescue StandardError => e # handles unspecified errors
84
+ job.bury
85
+ ensure # bury if still reserved
86
+ job.bury if job.exists? && job.reserved?
87
+ end
88
+ end
89
+ end # process!
90
+ end # Jobs
91
+ end # Beaneater