stalk_climber 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/.gitignore ADDED
@@ -0,0 +1,17 @@
1
+ *.gem
2
+ *.rbc
3
+ .bundle
4
+ .config
5
+ .yardoc
6
+ Gemfile.lock
7
+ InstalledFiles
8
+ _yardoc
9
+ coverage
10
+ doc/
11
+ lib/bundler/man
12
+ pkg
13
+ rdoc
14
+ spec/reports
15
+ test/tmp
16
+ test/version_tmp
17
+ tmp
data/.travis.yml ADDED
@@ -0,0 +1,19 @@
1
+ language: ruby
2
+
3
+ rvm:
4
+ - 1.9.2
5
+ - 1.9.3
6
+ - 2.0.0
7
+
8
+ before_install:
9
+ - sudo apt-get update -qq
10
+ - sudo apt-get install -qq beanstalkd
11
+ - sudo sed -i 's/#START=yes/START=yes/' /etc/default/beanstalkd
12
+ - sudo service beanstalkd start
13
+ - cat /etc/init.d/beanstalkd | sed 's/NAME=beanstalkd/NAME=beanstalkd2/' | sudo tee /etc/init.d/beanstalkd2 2>&1>/dev/null
14
+ - cat /etc/default/beanstalkd | sed 's/11300/11301/' | sudo tee /etc/default/beanstalkd2 2>&1>/dev/null
15
+ - sudo chmod +x /etc/init.d/beanstalkd2
16
+ - sudo service beanstalkd2 start
17
+
18
+ env:
19
+ - BEANSTALK_ADDRESSES='beanstalk://localhost:11300,beanstalk://localhost:11301'
data/Gemfile ADDED
@@ -0,0 +1,10 @@
1
+ source 'https://rubygems.org'
2
+
3
+ gemspec
4
+
5
+ gem 'debugger', :group => [:development, :test]
6
+
7
+ group :test do
8
+ gem 'coveralls', :require => false
9
+ gem 'mocha', :require => false
10
+ end
data/LICENSE ADDED
@@ -0,0 +1,20 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2013 Freewrite.org
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy of
6
+ this software and associated documentation files (the "Software"), to deal in
7
+ the Software without restriction, including without limitation the rights to
8
+ use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
9
+ the Software, and to permit persons to whom the Software is furnished to do so,
10
+ subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
17
+ FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
18
+ COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
19
+ IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
20
+ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,15 @@
1
+ # StalkClimber
2
+ [![Build Status](https://secure.travis-ci.org/freewrite/stalk_climber.png)](http://travis-ci.org/freewrite/stalk_climber)
3
+ [![Dependency Status](https://gemnasium.com/freewrite/stalk_climber.png)](https://gemnasium.com/freewrite/stalk_climber)
4
+ [![Coverage Status](https://coveralls.io/repos/freewrite/stalk_climber/badge.png?branch=master)](https://coveralls.io/r/freewrite/stalk_climber)
5
+ [![Code Climate](https://codeclimate.com/github/freewrite/stalk_climber.png)](https://codeclimate.com/github/freewrite/stalk_climber)
6
+
7
+ StalkClimber is a Ruby library allowing improved sequential access to Beanstalk via a job cache.
8
+
9
+ ## Contributing
10
+
11
+ 1. Fork it
12
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
13
+ 3. Commit your changes (`git commit -am 'Added some feature'`)
14
+ 4. Push to the branch (`git push origin my-new-feature`)
15
+ 5. Create new Pull Request
data/Rakefile ADDED
@@ -0,0 +1,9 @@
1
+ require 'bundler/gem_tasks'
2
+ require 'rake/testtask'
3
+
4
+ Rake::TestTask.new do |t|
5
+ t.libs << 'test'
6
+ t.pattern = 'test/**/*_test.rb'
7
+ end
8
+
9
+ task :default => :test
@@ -0,0 +1,38 @@
1
+ module StalkClimber
2
+ class Climber
3
+ include LazyEnumerable
4
+
5
+ attr_accessor :beanstalk_addresses, :test_tube
6
+ attr_reader :cache
7
+
8
+ # Returns or creates a ConnectionPool from beanstalk_addresses
9
+ def connection_pool
10
+ return @connection_pool unless @connection_pool.nil?
11
+ if self.beanstalk_addresses.nil?
12
+ raise RuntimeError, 'beanstalk_addresses must be set in order to establish a connection'
13
+ end
14
+ @connection_pool = ConnectionPool.new(self.beanstalk_addresses)
15
+ end
16
+
17
+
18
+ # Perform a threaded climb across all connections in the connection pool.
19
+ # An instance of Job is yielded to +block+
20
+ def climb(&block)
21
+ threads = []
22
+ self.connection_pool.connections.each do |connection|
23
+ threads << Thread.new { connection.each(&block) }
24
+ end
25
+ threads.each(&:join)
26
+ return
27
+ end
28
+ alias_method :each, :climb
29
+
30
+
31
+ # Creates a new Climber instance, optionally yielding the instance
32
+ # if a block is given
33
+ def initialize
34
+ yield(self) if block_given?
35
+ end
36
+
37
+ end
38
+ end
@@ -0,0 +1,143 @@
1
+ module StalkClimber
2
+ class Connection < Beaneater::Connection
3
+ include LazyEnumerable
4
+
5
+ DEFAULT_TUBE = 'stalk_climber'
6
+ PROBE_TRANSMISSION = "put 4294967295 0 300 2\r\n{}"
7
+
8
+ attr_accessor :test_tube
9
+ attr_reader :max_climbed_job_id, :min_climbed_job_id
10
+
11
+
12
+ # Returns or creates a Hash used for caching jobs by ID
13
+ def cache
14
+ return @cache ||= {}
15
+ end
16
+
17
+
18
+ # Resets the job cache and reinitializes the min and max climbed job ids
19
+ def clear_cache
20
+ @cache = nil
21
+ @min_climbed_job_id = Float::INFINITY
22
+ @max_climbed_job_id = 0
23
+ end
24
+
25
+
26
+ # Handles job enumeration in descending ID order, passing an instance of Job to +block+ for
27
+ # each existing job on the beanstalk server. Jobs are enumerated in three phases. Jobs between
28
+ # max_job_id and the max_climbed_job_id are pulled from beanstalk, cached, and given to
29
+ # +block+. Jobs that have already been cached are yielded to +block+ if they still exist,
30
+ # otherwise they are deleted from the cache. Finally, jobs between min_climbed_job_id and 1
31
+ # are pulled from beanstalk, cached, and given to +block+.
32
+ # Connection#each fulfills Enumberable contract, allowing connection to behave as an Enumerable.
33
+ def each(&block)
34
+ climb(&block)
35
+ end
36
+
37
+
38
+ # Initializes a new Connection to the beanstalk +address+ provided and
39
+ # configures the Connection to only use the configured test_tube for
40
+ # all transmissions.
41
+ # Optionally yields the instance if a block is given. The instance is yielded
42
+ # prior to test_tube configuration to allow the test_tube to be configured.
43
+ def initialize(address)
44
+ super
45
+ self.test_tube = DEFAULT_TUBE
46
+ clear_cache
47
+ yield(self) if block_given?
48
+ [
49
+ "use #{self.test_tube}",
50
+ "watch #{self.test_tube}",
51
+ 'ignore default',
52
+ ].each do |transmission|
53
+ transmit(transmission)
54
+ end
55
+ end
56
+
57
+
58
+ # Determintes the max job ID of the connection by inserting a job into the test tube
59
+ # and immediately deleting it. Before returning the max ID, the max ID is used to
60
+ # update the max_climbed_job_id (if sequentual) and possibly invalidate the cache.
61
+ # The cache will be invalidated if the max ID is less than any known IDs since
62
+ # new job IDs should always increment unless there's been a change in server state.
63
+ def max_job_id
64
+ job = Job.new(transmit(PROBE_TRANSMISSION))
65
+ job.delete
66
+ update_climbed_job_ids_from_max_id(job.id)
67
+ return job.id
68
+ end
69
+
70
+
71
+ # Safe form of with_job!, yields a Job instance to +block+ for the specified +job_id+.
72
+ # If the job does not exist, the error is caught and nil is passed to +block+ instead.
73
+ def with_job(job_id, &block)
74
+ begin
75
+ with_job!(job_id, &block)
76
+ rescue Beaneater::NotFoundError
77
+ block.call(nil)
78
+ end
79
+ end
80
+
81
+
82
+ # Yields a Job instance to +block+ for the specified +job_id+.
83
+ # If the job does not exist, a Beaneater::NotFoundError will bubble up from Beaneater.
84
+ def with_job!(job_id, &block)
85
+ job = Job.new(transmit("peek #{job_id}"))
86
+ block.call(job)
87
+ end
88
+
89
+
90
+ protected
91
+
92
+ # Helper method, similar to with_job, that retrieves the job identified by
93
+ # +job_id+, caches it, and updates counters before yielding the job.
94
+ # If the job does not exist, +block+ is not called and nothing is cached,
95
+ # however counters will be updated.
96
+ def cache_job_and_yield(job_id, &block)
97
+ with_job(job_id) do |job|
98
+ self.cache[job_id] = job unless job.nil?
99
+ @min_climbed_job_id = job_id if job_id < @min_climbed_job_id
100
+ @max_climbed_job_id = job_id if job_id > @max_climbed_job_id
101
+ yield(job) unless job.nil?
102
+ end
103
+ end
104
+
105
+
106
+ # Handles job enumeration. See Connection#each for more information.
107
+ def climb(&block)
108
+ max_id = max_job_id
109
+
110
+ initial_cached_jobs = cache.values_at(*cache.keys.sort.reverse)
111
+
112
+ max_id.downto(self.max_climbed_job_id + 1) do |job_id|
113
+ cache_job_and_yield(job_id, &block)
114
+ end
115
+
116
+ initial_cached_jobs.each do |job|
117
+ if job.exists?
118
+ yield job
119
+ else
120
+ self.cache.delete(job.id)
121
+ end
122
+ end
123
+
124
+ ([self.min_climbed_job_id - 1, max_id].min).downto(1) do |job_id|
125
+ cache_job_and_yield(job_id, &block)
126
+ end
127
+ return
128
+ end
129
+
130
+
131
+ # Uses +new_max_id+ to update the max_climbed_job_id (if sequentual) and possibly invalidate
132
+ # the cache. The cache will be invalidated if +new_max_id+ is less than any known IDs since
133
+ # new job IDs should always increment unless there's been a change in server state.
134
+ def update_climbed_job_ids_from_max_id(new_max_id)
135
+ if @max_climbed_job_id > 0 && @max_climbed_job_id == new_max_id - 1
136
+ @max_climbed_job_id = new_max_id
137
+ elsif new_max_id < @max_climbed_job_id
138
+ clear_cache
139
+ end
140
+ end
141
+
142
+ end
143
+ end
@@ -0,0 +1,30 @@
1
+ module StalkClimber
2
+ class ConnectionPool < Beaneater::Pool
3
+
4
+ class InvalidURIScheme < RuntimeError; end
5
+
6
+ attr_reader :addresses
7
+
8
+ # Constructs a Beaneater::Pool from a less strict URL
9
+ # +url+ can be a string i.e 'localhost:11300' or an array of addresses.
10
+ def initialize(addresses = nil)
11
+ @addresses = Array(parse_addresses(addresses) || host_from_env || Beaneater.configuration.beanstalkd_url)
12
+ @connections = @addresses.map { |address| Connection.new(address) }
13
+ end
14
+
15
+
16
+ protected
17
+
18
+ # Parses the given url into a collection of beanstalk addresses
19
+ def parse_addresses(addresses)
20
+ return if addresses.empty?
21
+ uris = addresses.is_a?(Array) ? addresses.dup : addresses.split(/[\s,]+/)
22
+ uris.map! do |uri_string|
23
+ uri = URI.parse(uri_string)
24
+ raise(InvalidURIScheme, "Invalid beanstalk URI: #{uri_string}") unless uri.scheme == 'beanstalk'
25
+ "#{uri.host}:#{uri.port || 11300}"
26
+ end
27
+ end
28
+
29
+ end
30
+ end
@@ -0,0 +1,179 @@
1
+ module StalkClimber
2
+ class Job
3
+
4
+ STATS_ATTRIBUTES = %w[age buries delay kicks pri releases reserves state time-left timeouts ttr tube]
5
+ attr_reader :id
6
+
7
+ STATS_ATTRIBUTES.each do |method_name|
8
+ define_method method_name do |force_refresh = false|
9
+ return stats(force_refresh)[method_name]
10
+ end
11
+ end
12
+
13
+
14
+ # Returns or fetches the body of the job obtained via the peek command
15
+ def body
16
+ return @body ||= connection.transmit("peek #{id}")[:body]
17
+ end
18
+
19
+
20
+ # Returns the connection provided by the job data given to the initialize method
21
+ def connection
22
+ return @connection
23
+ end
24
+
25
+
26
+ # Deletes the job from beanstalk. If the job is not found it is assumed that it
27
+ # has already been otherwise deleted.
28
+ def delete
29
+ begin
30
+ @connection.transmit("delete #{id}")
31
+ rescue Beaneater::NotFoundError
32
+ end
33
+ @status = 'DELETED'
34
+ @stats = nil
35
+ @body = nil
36
+ return true
37
+ end
38
+
39
+
40
+ # Determines if a job exists by retrieving stats for the job. If Beaneater can't find
41
+ # the jobm then it does not exist and false is returned. The stats command is used
42
+ # because it will return a response of a near constant size, whereas, depending on
43
+ # the job, the peek command could return a much larger response. Rather than waste
44
+ # the trip to the server, stats are updated each time the method is called.
45
+ def exists?
46
+ begin
47
+ stats(:force_refresh)
48
+ return true
49
+ rescue Beaneater::NotFoundError
50
+ return false
51
+ end
52
+ end
53
+
54
+
55
+ # Initializes a Job instance using +job_data+ which should be the Beaneater response to either
56
+ # a put, peek, or stats-job command. Other Beaneater responses are not supported.
57
+ #
58
+ # No single beanstalk command provides all the data an instance might need, so as more
59
+ # information is required, additional calls are made to beanstalk. For example, accessing both
60
+ # a job's tube and its body requires both a peek and stats-job call.
61
+ #
62
+ # Put provides only the ID of the job and as such yields the least informed instance. Both a
63
+ # peek and stats-job call may be required to retrieve anything but the ID of the instance
64
+ #
65
+ # Peek provides the ID and body of the job. A stats-job call may be required to access anything
66
+ # but the ID or body of the job.
67
+ #
68
+ # Stats-job provides the most information about the job, but lacks the crtical component of the
69
+ # job body. As such, a peek call would be required to access the body of the job.
70
+ def initialize(job_data)
71
+ case job_data[:status]
72
+ when 'INSERTED' # put
73
+ @id = job_data[:id].to_i
74
+ @body = @stats = nil
75
+ when 'FOUND' # peek
76
+ @id = job_data[:id].to_i
77
+ @body = job_data[:body]
78
+ @stats = nil
79
+ when 'OK' # stats-job
80
+ @body = nil
81
+ @stats = job_data.delete(:body)
82
+ @id = @stats.delete('id').to_i
83
+ else
84
+ raise RuntimeError, "Unexpected job status: #{job_data[:status]}"
85
+ end
86
+ @status = job_data[:status]
87
+ @connection = job_data[:connection]
88
+ end
89
+
90
+
91
+ # Returns or retrieves stats for the job. Optionally, a retrieve may be forced
92
+ # by passing a non-false value for +force_refresh+
93
+ def stats(force_refresh = false)
94
+ return @stats unless @stats.nil? || force_refresh
95
+ @stats = connection.transmit("stats-job #{id}")[:body]
96
+ @stats.delete('id')
97
+ return @stats
98
+ end
99
+
100
+
101
+ # :method: age
102
+ # Retrieves the age of the job from the job's stats. Passing a non-false value for
103
+ # +force_refresh+ will force retrieval of updated stats for the job
104
+ # :call-seq:
105
+ # age(force_refresh = false)
106
+
107
+ # :method: buries
108
+ # Retrieves the number of times the job has been buried from the job's stats. Passing
109
+ # a non-false value for +force_refresh+ will force retrieval of updated stats for the job
110
+ # :call-seq:
111
+ # buries(force_refresh = false)
112
+
113
+ # :method: delay
114
+ # Retrieves the the remaining delay before the job is ready from the job's stats. Passing
115
+ # a non-false value for +force_refresh+ will force retrieval of updated stats for the job
116
+ # :call-seq:
117
+ # delay(force_refresh = false)
118
+
119
+ # :method: kicks
120
+ # Retrieves the number of times the job has been kicked from the job's stats. Passing
121
+ # a non-false value for +force_refresh+ will force retrieval of updated stats for the job
122
+ # :call-seq:
123
+ # kicks(force_refresh = false)
124
+
125
+ # :method: pri
126
+ # Retrieves the priority of the job from the job's stats. Passing a non-false value for
127
+ # +force_refresh+ will force retrieval of updated stats for the job
128
+ # :call-seq:
129
+ # pri(force_refresh = false)
130
+
131
+ # :method: releases
132
+ # Retrieves the number of times the job has been released from the job's stats. Passing
133
+ # a non-false value for +force_refresh+ will force retrieval of updated stats for the job
134
+ # :call-seq:
135
+ # releases(force_refresh = false)
136
+
137
+ # :method: reserves
138
+ # Retrieves the number of times the job has been reserved from the job's stats. Passing
139
+ # a non-false value for +force_refresh+ will force retrieval of updated stats for the job
140
+ # :call-seq:
141
+ # reserves(force_refresh = false)
142
+
143
+ # :method: state
144
+ # Retrieves the state of the job from the job's stats. Value will be one of "ready",
145
+ # "delayed", "reserved", or "buried". Passing a non-false value for +force_refresh+
146
+ # will force retrieval of updated stats for the job
147
+ # :call-seq:
148
+ # timeouts(force_refresh = false)
149
+
150
+ # :method: time-left
151
+ # Retrieves the number of seconds left until the server puts this job into the ready
152
+ # queue. This number is only meaningful if the job is reserved or delayed. If the job
153
+ # is reserved and this amount of time elapses before its state changes, it is considered
154
+ # to have timed out. Passing a non-false value for +force_refresh+ will force retrieval
155
+ # of updated stats for the job
156
+ # :call-seq:
157
+ # timeouts(force_refresh = false)
158
+
159
+ # :method: timeouts
160
+ # Retrieves the number of times the job has timed out from the job's stats. Passing
161
+ # a non-false value for +force_refresh+ will force retrieval of updated stats for the job
162
+ # :call-seq:
163
+ # timeouts(force_refresh = false)
164
+
165
+ # :method: ttr
166
+ # Retrieves the time to run for the job, the number of seconds a worker is allowed to work
167
+ # to run the job. Passing a non-false value for +force_refresh+ will force retrieval of
168
+ # updated stats for the job
169
+ # :call-seq:
170
+ # timeouts(force_refresh = false)
171
+
172
+ # :method: tube
173
+ # Retrieves the name of the tube that contains job. Passing a non-false value for
174
+ # +force_refresh+ will force retrieval of updated stats for the job
175
+ # :call-seq:
176
+ # timeouts(force_refresh = false)
177
+
178
+ end
179
+ end
@@ -0,0 +1,15 @@
1
+ module StalkClimber
2
+ module LazyEnumerable
3
+ include Enumerable
4
+
5
+ def self.make_lazy(*methods)
6
+ methods.each do |method|
7
+ define_method method do |*args, &block|
8
+ lazy.public_send(method, *args, &block)
9
+ end
10
+ end
11
+ end
12
+
13
+ make_lazy(*(Enumerable.public_instance_methods - [:lazy]))
14
+ end
15
+ end
@@ -0,0 +1,3 @@
1
+ module StalkClimber
2
+ VERSION = '0.0.1'
3
+ end
@@ -0,0 +1,9 @@
1
+ module StalkClimber; end
2
+
3
+ require 'beaneater'
4
+ require 'stalk_climber/version'
5
+ require 'stalk_climber/lazy_enumerable'
6
+ require 'stalk_climber/connection'
7
+ require 'stalk_climber/connection_pool'
8
+ require 'stalk_climber/climber'
9
+ require 'stalk_climber/job'
@@ -0,0 +1,25 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'stalk_climber/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = 'stalk_climber'
8
+ spec.version = StalkClimber::VERSION
9
+ spec.authors = ['Freewrite.org']
10
+ spec.email = ['dev@freewrite.org']
11
+ spec.description = %q{Improved sequential access to Beanstalk}
12
+ spec.summary = %q{StalkClimber is a Ruby library allowing improved sequential access to Beanstalk via a job cache.}
13
+ spec.homepage = 'https://github.com/freewrite/stalk_climber'
14
+ spec.license = 'MIT'
15
+
16
+ spec.files = `git ls-files`.split($/)
17
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
18
+ spec.test_files = spec.files.grep(%r{^test/})
19
+ spec.require_paths = ['lib']
20
+
21
+ spec.add_development_dependency 'bundler', '~> 1.3'
22
+ spec.add_development_dependency 'rake'
23
+
24
+ spec.add_dependency 'beaneater'
25
+ end
@@ -0,0 +1,24 @@
1
+ require 'debugger'
2
+ require 'coveralls'
3
+ Coveralls.wear!
4
+
5
+ lib = File.expand_path('../../lib', __FILE__)
6
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
7
+
8
+ require 'test/unit'
9
+ require 'mocha/setup'
10
+
11
+ require 'stalk_climber'
12
+
13
+ BEANSTALK_ADDRESS = ENV['BEANSTALK_ADDRESS'] || 'beanstalk://localhost'
14
+ BEANSTALK_ADDRESSES = ENV['BEANSTALK_ADDRESSES'] || BEANSTALK_ADDRESS
15
+
16
+ class Test::Unit::TestCase
17
+
18
+ def seed_jobs(count = 5)
19
+ count.times.map do
20
+ StalkClimber::Job.new(@connection.transmit(StalkClimber::Connection::PROBE_TRANSMISSION))
21
+ end
22
+ end
23
+
24
+ end
@@ -0,0 +1,59 @@
1
+ require 'test_helper'
2
+
3
+ class ClimberTest < Test::Unit::TestCase
4
+
5
+ def test_climb_caches_jobs_for_later_use
6
+ climber = StalkClimber::Climber.new do |c|
7
+ c.beanstalk_addresses = BEANSTALK_ADDRESSES
8
+ end
9
+
10
+ test_jobs = {}
11
+ climber.connection_pool.connections.each do |connection|
12
+ test_jobs[connection.address] = []
13
+ 5.times.to_a.map! do
14
+ test_jobs[connection.address] << StalkClimber::Job.new(connection.transmit(StalkClimber::Connection::PROBE_TRANSMISSION))
15
+ end
16
+ end
17
+
18
+ jobs = {}
19
+ climber.each do |job|
20
+ jobs[job.connection.address] ||= {}
21
+ jobs[job.connection.address][job.id] = job
22
+ end
23
+
24
+ climber.expects(:with_job).never
25
+ climber.each do |job|
26
+ assert_equal jobs[job.connection.address][job.id], job
27
+ end
28
+
29
+ climber.connection_pool.connections.each do |connection|
30
+ test_jobs[connection.address].map(&:delete)
31
+ end
32
+ end
33
+
34
+
35
+ def test_connection_pool_creates_a_connection_pool
36
+ climber = StalkClimber::Climber.new do |c|
37
+ c.beanstalk_addresses = 'beanstalk://localhost'
38
+ end
39
+ assert_kind_of StalkClimber::ConnectionPool, climber.connection_pool
40
+ end
41
+
42
+
43
+ def test_connection_pool_raises_an_error_without_beanstalk_addresses
44
+ climber = StalkClimber::Climber.new
45
+ assert_raise RuntimeError do
46
+ climber.connection_pool
47
+ end
48
+ end
49
+
50
+
51
+ def test_each_is_an_alias_for_climb
52
+ assert_equal(
53
+ StalkClimber::Climber.instance_method(:climb),
54
+ StalkClimber::Climber.instance_method(:each),
55
+ 'Expected StalkClimber::Climber#each to be an alias for StalkClimber::Climber#climb'
56
+ )
57
+ end
58
+
59
+ end
@@ -0,0 +1,49 @@
1
+ # Tests adapted from Backburner::Connection tests
2
+ # https://github.com/nesquena/backburner/blob/master/test/connection_test.rb
3
+
4
+ require 'test_helper'
5
+
6
+ class ConnectionPoolTest < Test::Unit::TestCase
7
+
8
+ def test_for_bad_url_it_should_raise_a_bad_url_error
9
+ assert_raises(StalkClimber::ConnectionPool::InvalidURIScheme) do
10
+ StalkClimber::ConnectionPool.new('fake://foo')
11
+ end
12
+ end
13
+
14
+
15
+ def test_for_delegated_methods_it_should_delegate_methods_to_beanstalk_connection
16
+ connection = StalkClimber::ConnectionPool.new('beanstalk://localhost')
17
+ assert_equal 'localhost', connection.connections.first.host
18
+ end
19
+
20
+
21
+ def test_with_multiple_urls_it_should_support_array_of_connections
22
+ connection = StalkClimber::ConnectionPool.new(['beanstalk://localhost:11300','beanstalk://localhost'])
23
+ connections = connection.connections
24
+ assert_equal 2, connection.connections.size
25
+ assert_equal ['localhost:11300','localhost:11300'], connections.map(&:address)
26
+ end
27
+
28
+
29
+ def test_with_multiple_urls_it_should_support_single_string_with_commas
30
+ connection = StalkClimber::ConnectionPool.new('beanstalk://localhost:11300,beanstalk://localhost')
31
+ connections = connection.connections
32
+ assert_equal 2, connections.size
33
+ assert_equal ['localhost:11300','localhost:11300'], connections.map(&:address)
34
+ end
35
+
36
+
37
+ def test_with_single_url_it_should_set_up_connection_pool
38
+ connection = StalkClimber::ConnectionPool.new('beanstalk://localhost')
39
+ assert_kind_of StalkClimber::ConnectionPool, connection
40
+ assert_kind_of Beaneater::Pool, connection
41
+ end
42
+
43
+
44
+ def test_with_a_single_url_it_should_convert_url_to_address_array
45
+ connection = StalkClimber::ConnectionPool.new('beanstalk://localhost')
46
+ assert_equal ['localhost:11300'], connection.addresses
47
+ end
48
+
49
+ end
@@ -0,0 +1,192 @@
1
+ require 'test_helper'
2
+
3
+ class ConnectionTest < Test::Unit::TestCase
4
+
5
+ def setup
6
+ @connection = StalkClimber::Connection.new('localhost:11300')
7
+ end
8
+
9
+
10
+ def test_cache_creates_and_returns_hash_instance_variable
11
+ refute @connection.instance_variable_get(:@cache)
12
+ assert_equal({}, @connection.cache)
13
+ assert_equal({}, @connection.instance_variable_get(:@cache))
14
+ end
15
+
16
+
17
+ def test_cache_is_reset_if_max_job_id_lower_than_max_climbed_job_id
18
+ seeds = seed_jobs
19
+ @connection.each {}
20
+ assert_not_equal({}, @connection.cache)
21
+ assert_not_equal(Float::INFINITY, @connection.min_climbed_job_id)
22
+ assert_not_equal(0, @connection.max_climbed_job_id)
23
+
24
+ @connection.send(:update_climbed_job_ids_from_max_id, 1)
25
+ assert_equal({}, @connection.cache)
26
+ assert_equal(Float::INFINITY, @connection.min_climbed_job_id)
27
+ assert_equal(0, @connection.max_climbed_job_id)
28
+ seeds.map(&:delete)
29
+ end
30
+
31
+
32
+ def test_connection_is_some_kind_of_enumerable
33
+ assert @connection.kind_of?(Enumerable)
34
+ end
35
+
36
+
37
+ def test_each_caches_jobs_for_later_use
38
+ seeds = seed_jobs
39
+
40
+ jobs = {}
41
+ @connection.each do |job|
42
+ jobs[job.id] = job
43
+ end
44
+
45
+ @connection.expects(:with_job).never
46
+ @connection.each do |job|
47
+ assert_equal jobs[job.id], job
48
+ end
49
+
50
+ seeds.map(&:delete)
51
+ end
52
+
53
+
54
+ def test_each_deletes_cached_jobs_that_no_longer_exist
55
+ seeds = seed_jobs
56
+
57
+ jobs = {}
58
+ @connection.each do |job|
59
+ jobs[job.id] = job
60
+ end
61
+
62
+ deleted_job = jobs[jobs.keys[2]]
63
+ deleted_job.delete
64
+
65
+ @connection.expects(:with_job).never
66
+ @connection.each do |job|
67
+ assert_equal jobs.delete(job.id), job
68
+ end
69
+
70
+ assert_equal 1, jobs.length
71
+ assert_equal deleted_job.id, jobs.values.first.id
72
+
73
+ seeds.map(&:delete)
74
+ end
75
+
76
+
77
+ def test_each_hits_jobs_below_climbed_range_that_have_not_been_hit
78
+ seeds = seed_jobs(10)
79
+
80
+ count = 0
81
+ @connection.each do |job|
82
+ count += 1
83
+ break if count == 5
84
+ end
85
+
86
+ initial_min_climbed_job_id = @connection.min_climbed_job_id
87
+
88
+ all_jobs = {}
89
+ @connection.each do |job|
90
+ all_jobs[job.id] = job
91
+ end
92
+
93
+ seeds.each do |job|
94
+ assert_equal all_jobs[job.id].body, job.body
95
+ assert_equal all_jobs[job.id].id, job.id
96
+ job.delete
97
+ end
98
+
99
+ assert @connection.min_climbed_job_id < initial_min_climbed_job_id
100
+ end
101
+
102
+
103
+ def test_each_only_hits_each_job_once
104
+ seeds = seed_jobs
105
+
106
+ jobs = {}
107
+ @connection.each do |job|
108
+ refute jobs[job.id]
109
+ jobs[job.id] = job
110
+ end
111
+
112
+ seeds.map(&:delete)
113
+ end
114
+
115
+
116
+ def test_each_calls_climb
117
+ @connection.expects(:climb)
118
+ @connection.each {}
119
+ end
120
+
121
+
122
+ def test_max_job_id_returns_expected_max_job_id
123
+ initial_max = @connection.max_job_id
124
+ seed_jobs(3).map(&:delete)
125
+ # 3 new jobs, +1 for the probe job
126
+ assert_equal initial_max + 4, @connection.max_job_id
127
+ end
128
+
129
+
130
+ def test_max_job_id_should_increment_max_climbed_id_if_successor
131
+ @connection.each {}
132
+ max = @connection.max_climbed_job_id
133
+ @connection.max_job_id
134
+ assert_equal max + 1, @connection.max_climbed_job_id
135
+ end
136
+
137
+
138
+ def test_max_job_id_should_not_increment_max_climbed_id_unless_successor
139
+ @connection.each {}
140
+ max = @connection.max_climbed_job_id
141
+ seed_jobs(1).map(&:delete)
142
+ @connection.max_job_id
143
+ assert_equal max, @connection.max_climbed_job_id
144
+ end
145
+
146
+
147
+ def test_each_sets_min_and_max_climbed_job_ids_appropriately
148
+ assert_equal Float::INFINITY, @connection.min_climbed_job_id
149
+ assert_equal 0, @connection.max_climbed_job_id
150
+
151
+ max_id = @connection.max_job_id
152
+ @connection.expects(:max_job_id).once.returns(max_id)
153
+ @connection.each {}
154
+
155
+ assert_equal 1, @connection.min_climbed_job_id
156
+ assert_equal max_id, @connection.max_climbed_job_id
157
+ end
158
+
159
+
160
+ def test_test_tube_is_initialized_but_configurable
161
+ assert_equal StalkClimber::Connection::DEFAULT_TUBE, @connection.test_tube
162
+ tube_name = 'test_tube'
163
+ @connection.test_tube = tube_name
164
+ assert_equal tube_name, @connection.test_tube
165
+ end
166
+
167
+
168
+ def test_with_job_yields_nil_or_the_requested_job_if_it_exists
169
+ assert_nothing_raised do
170
+ @connection.with_job(@connection.max_job_id) do |job|
171
+ assert_equal nil, job
172
+ end
173
+ end
174
+
175
+ probe = seed_jobs(1).first
176
+ @connection.with_job(probe.id) do |job|
177
+ assert_equal probe.id, job.id
178
+ end
179
+ probe.delete
180
+ end
181
+
182
+
183
+ def test_with_job_bang_does_not_execute_block_and_raises_error_if_job_does_not_exist
184
+ block = lambda {}
185
+ block.expects(:call).never
186
+ assert_raise Beaneater::NotFoundError do
187
+ Object.any_instance.expects(:yield).never
188
+ @connection.with_job!(@connection.max_job_id, &block)
189
+ end
190
+ end
191
+
192
+ end
@@ -0,0 +1,166 @@
1
+ require 'test_helper'
2
+
3
+ class Job < Test::Unit::TestCase
4
+
5
+ def setup
6
+ @connection = StalkClimber::Connection.new('localhost:11300')
7
+ @job = seed_jobs(1).first
8
+ end
9
+
10
+
11
+ def test_body_retrives_performs_peek
12
+ body = {:test => true}
13
+
14
+ @job.connection.expects(:transmit).returns({
15
+ :body => body,
16
+ })
17
+ assert_equal body, @job.body
18
+
19
+ @job.connection.expects(:transmit).never
20
+ assert_equal body, @job.body
21
+ end
22
+
23
+
24
+ def test_connection_stored_with_instance
25
+ assert @job.instance_variable_get(:@connection)
26
+ assert @job.connection
27
+ end
28
+
29
+
30
+ def test_delete
31
+ assert @job.delete
32
+ assert_raise Beaneater::NotFoundError do
33
+ @connection.transmit("peek #{@job.id}")
34
+ end
35
+ refute @job.instance_variable_get(:@body)
36
+ refute @job.instance_variable_get(:@stats)
37
+ assert_equal 'DELETED', @job.instance_variable_get(:@status)
38
+ end
39
+
40
+
41
+ def test_exists?
42
+ assert @job.exists?
43
+
44
+ @job.delete
45
+ refute @job.exists?
46
+ end
47
+
48
+
49
+ def test_initialize_with_peek_response
50
+ job = StalkClimber::Job.new(@connection.transmit("peek #{@job.id}"))
51
+ @connection.expects(:transmit).never
52
+ assert_equal @connection, job.connection
53
+ assert_equal @job.id, job.id
54
+ assert_equal 'FOUND', job.instance_variable_get(:@status)
55
+ assert_equal '{}', job.body
56
+ refute job.instance_variable_get(:@stats)
57
+ end
58
+
59
+
60
+ def test_initialize_with_put_response
61
+ @connection.expects(:transmit).never
62
+ assert @job.id
63
+ assert_equal @connection, @job.connection
64
+ assert_equal 'INSERTED', @job.instance_variable_get(:@status)
65
+ refute @job.instance_variable_get(:@body)
66
+ refute @job.instance_variable_get(:@stats)
67
+ end
68
+
69
+
70
+ def test_initialize_with_stats_response
71
+ job = StalkClimber::Job.new(@connection.transmit("stats-job #{@job.id}"))
72
+ @connection.expects(:transmit).never
73
+ assert_equal @job.id, job.id
74
+ assert_equal @connection, job.connection
75
+ assert_equal 'OK', job.instance_variable_get(:@status)
76
+ assert job.stats
77
+ assert job.instance_variable_get(:@stats)
78
+ refute job.instance_variable_get(:@body)
79
+ StalkClimber::Job::STATS_ATTRIBUTES.each do |method_name|
80
+ assert job.send(method_name)
81
+ end
82
+ end
83
+
84
+
85
+ def test_initialize_with_stats_attribute_methods_return_correct_values
86
+ body = {
87
+ 'age'=>3,
88
+ 'buries'=>0,
89
+ 'delay'=>0,
90
+ 'id' => 4412,
91
+ 'kicks'=>0,
92
+ 'pri'=>4294967295,
93
+ 'releases'=>0,
94
+ 'reserves'=>0,
95
+ 'state'=>'ready',
96
+ 'time-left'=>0,
97
+ 'timeouts'=>0,
98
+ 'ttr'=>300,
99
+ 'tube'=>'default',
100
+ }
101
+ stats = {
102
+ :body => body,
103
+ :connection => @connection,
104
+ :id => 149,
105
+ :status => 'OK',
106
+ }
107
+ @connection.expects(:transmit).returns(stats)
108
+ job = StalkClimber::Job.new(@connection.transmit("stats-job #{@job.id}"))
109
+ @connection.expects(:transmit).never
110
+ StalkClimber::Job::STATS_ATTRIBUTES.each do |method_name|
111
+ assert_equal body[method_name], job.send(method_name), "Expected #{body[method_name.to_sym]} for #{method_name}, got #{job.send(method_name)}"
112
+ end
113
+ end
114
+
115
+
116
+ def test_initialize_with_unknown_status_raises_error
117
+ assert_raise RuntimeError do
118
+ StalkClimber::Job.new({:status => 'DELETED'})
119
+ end
120
+ end
121
+
122
+
123
+ def test_stats_attribute_method_can_force_refresh
124
+ initial_value = @job.age
125
+ @connection.expects(:transmit).returns({:body => {'age' => initial_value + 100}})
126
+ assert_equal initial_value + 100, @job.age(true)
127
+ end
128
+
129
+
130
+ def test_stats_can_force_a_refresh
131
+ body = {
132
+ 'age'=>3,
133
+ 'buries'=>0,
134
+ 'delay'=>0,
135
+ 'kicks'=>0,
136
+ 'id' => 4412,
137
+ 'pri'=>4294967295,
138
+ 'releases'=>0,
139
+ 'reserves'=>0,
140
+ 'state'=>'ready',
141
+ 'time-left'=>0,
142
+ 'timeouts'=>0,
143
+ 'ttr'=>300,
144
+ 'tube'=>'default',
145
+ }
146
+ stats_1 = {
147
+ :body => {},
148
+ :connection => @connection,
149
+ :id => 149,
150
+ :status => 'OK',
151
+ }
152
+ stats_2 = {
153
+ :body => body,
154
+ :connection => @connection,
155
+ :id => 149,
156
+ :status => 'OK',
157
+ }
158
+ @connection.expects(:transmit).twice.returns(stats_1, stats_2)
159
+ job = StalkClimber::Job.new(@connection.transmit("stats-job #{@job.id}"))
160
+ job.stats(:force_refresh)
161
+ StalkClimber::Job::STATS_ATTRIBUTES.each do |method_name|
162
+ assert_equal body[method_name], job.send(method_name), "Expected #{body[method_name.to_sym]} for #{method_name}, got #{job.send(method_name)}"
163
+ end
164
+ end
165
+
166
+ end
metadata ADDED
@@ -0,0 +1,119 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: stalk_climber
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ prerelease:
6
+ platform: ruby
7
+ authors:
8
+ - Freewrite.org
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2013-09-03 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: bundler
16
+ requirement: !ruby/object:Gem::Requirement
17
+ none: false
18
+ requirements:
19
+ - - ~>
20
+ - !ruby/object:Gem::Version
21
+ version: '1.3'
22
+ type: :development
23
+ prerelease: false
24
+ version_requirements: !ruby/object:Gem::Requirement
25
+ none: false
26
+ requirements:
27
+ - - ~>
28
+ - !ruby/object:Gem::Version
29
+ version: '1.3'
30
+ - !ruby/object:Gem::Dependency
31
+ name: rake
32
+ requirement: !ruby/object:Gem::Requirement
33
+ none: false
34
+ requirements:
35
+ - - ! '>='
36
+ - !ruby/object:Gem::Version
37
+ version: '0'
38
+ type: :development
39
+ prerelease: false
40
+ version_requirements: !ruby/object:Gem::Requirement
41
+ none: false
42
+ requirements:
43
+ - - ! '>='
44
+ - !ruby/object:Gem::Version
45
+ version: '0'
46
+ - !ruby/object:Gem::Dependency
47
+ name: beaneater
48
+ requirement: !ruby/object:Gem::Requirement
49
+ none: false
50
+ requirements:
51
+ - - ! '>='
52
+ - !ruby/object:Gem::Version
53
+ version: '0'
54
+ type: :runtime
55
+ prerelease: false
56
+ version_requirements: !ruby/object:Gem::Requirement
57
+ none: false
58
+ requirements:
59
+ - - ! '>='
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ description: Improved sequential access to Beanstalk
63
+ email:
64
+ - dev@freewrite.org
65
+ executables: []
66
+ extensions: []
67
+ extra_rdoc_files: []
68
+ files:
69
+ - .gitignore
70
+ - .travis.yml
71
+ - Gemfile
72
+ - LICENSE
73
+ - README.md
74
+ - Rakefile
75
+ - lib/stalk_climber.rb
76
+ - lib/stalk_climber/climber.rb
77
+ - lib/stalk_climber/connection.rb
78
+ - lib/stalk_climber/connection_pool.rb
79
+ - lib/stalk_climber/job.rb
80
+ - lib/stalk_climber/lazy_enumerable.rb
81
+ - lib/stalk_climber/version.rb
82
+ - stalk_climber.gemspec
83
+ - test/test_helper.rb
84
+ - test/unit/climber_test.rb
85
+ - test/unit/connection_pool_test.rb
86
+ - test/unit/connection_test.rb
87
+ - test/unit/job_test.rb
88
+ homepage: https://github.com/freewrite/stalk_climber
89
+ licenses:
90
+ - MIT
91
+ post_install_message:
92
+ rdoc_options: []
93
+ require_paths:
94
+ - lib
95
+ required_ruby_version: !ruby/object:Gem::Requirement
96
+ none: false
97
+ requirements:
98
+ - - ! '>='
99
+ - !ruby/object:Gem::Version
100
+ version: '0'
101
+ required_rubygems_version: !ruby/object:Gem::Requirement
102
+ none: false
103
+ requirements:
104
+ - - ! '>='
105
+ - !ruby/object:Gem::Version
106
+ version: '0'
107
+ requirements: []
108
+ rubyforge_project:
109
+ rubygems_version: 1.8.25
110
+ signing_key:
111
+ specification_version: 3
112
+ summary: StalkClimber is a Ruby library allowing improved sequential access to Beanstalk
113
+ via a job cache.
114
+ test_files:
115
+ - test/test_helper.rb
116
+ - test/unit/climber_test.rb
117
+ - test/unit/connection_pool_test.rb
118
+ - test/unit/connection_test.rb
119
+ - test/unit/job_test.rb