sneakers 0.0.5 → 0.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- sneakers (0.0.4)
4
+ sneakers (0.0.5)
5
5
  bunny (>= 0.10.0)
6
6
  serverengine
7
7
  thor
@@ -10,28 +10,30 @@ PATH
10
10
  GEM
11
11
  remote: https://rubygems.org/
12
12
  specs:
13
- activesupport (3.2.14)
13
+ activesupport (3.2.15)
14
14
  i18n (~> 0.6, >= 0.6.4)
15
15
  multi_json (~> 1.0)
16
16
  amq-protocol (1.8.0)
17
17
  arrayfields (4.9.0)
18
- awesome_print (1.1.0)
18
+ awesome_print (1.2.0)
19
19
  bluff (0.1.0)
20
20
  bunny (0.10.8)
21
21
  amq-protocol (>= 1.6.0)
22
22
  cane (2.6.0)
23
23
  parallel
24
+ celluloid (0.15.2)
25
+ timers (~> 1.1.0)
24
26
  chronic (0.10.2)
25
- churn (0.0.33)
27
+ churn (0.0.34)
26
28
  chronic (>= 0.2.3)
27
29
  hirb
28
30
  json_pure
29
31
  main
30
32
  ruby_parser (~> 3.0)
31
33
  sexp_processor (~> 4.1)
32
- code_analyzer (0.4.2)
34
+ code_analyzer (0.4.3)
33
35
  sexp_processor
34
- code_metrics (0.1.0)
36
+ code_metrics (0.1.1)
35
37
  coderay (1.0.9)
36
38
  colored (1.2)
37
39
  erubis (2.7.0)
@@ -44,22 +46,22 @@ GEM
44
46
  ruby_parser (~> 3.1, > 3.1.0)
45
47
  sexp_processor (~> 4.0)
46
48
  formatador (0.2.4)
47
- guard (1.8.2)
49
+ guard (2.1.0)
48
50
  formatador (>= 0.2.4)
49
- listen (>= 1.0.0)
50
- lumberjack (>= 1.0.2)
51
- pry (>= 0.9.10)
52
- thor (>= 0.14.6)
53
- guard-minitest (1.3.0)
54
- guard (>= 1.8)
55
- minitest (>= 2.1)
51
+ listen (~> 2.1)
52
+ lumberjack (~> 1.0)
53
+ pry (>= 0.9.12)
54
+ thor (>= 0.18.1)
55
+ guard-minitest (2.0.0)
56
+ guard (~> 2.0)
57
+ minitest (>= 3.0)
56
58
  hirb (0.7.1)
57
59
  i18n (0.6.5)
58
60
  json_pure (1.8.0)
59
- listen (1.3.1)
61
+ listen (2.1.1)
62
+ celluloid (>= 0.15.2)
60
63
  rb-fsevent (>= 0.9.3)
61
64
  rb-inotify (>= 0.9)
62
- rb-kqueue (>= 0.2)
63
65
  lumberjack (1.0.4)
64
66
  main (5.2.0)
65
67
  arrayfields (>= 4.7.4)
@@ -68,7 +70,7 @@ GEM
68
70
  map (>= 5.1.0)
69
71
  map (6.5.1)
70
72
  method_source (0.8.2)
71
- metric_fu (4.4.1)
73
+ metric_fu (4.4.4)
72
74
  bluff
73
75
  cane (~> 2.5, >= 2.5.2)
74
76
  churn (~> 0.0.28)
@@ -84,19 +86,19 @@ GEM
84
86
  roodi (~> 3.1)
85
87
  metric_fu-Saikuro (1.1.1.0)
86
88
  mini_portile (0.5.1)
87
- minitest (5.0.7)
88
- multi_json (1.8.0)
89
+ minitest (5.0.8)
90
+ multi_json (1.8.2)
89
91
  nokogiri (1.6.0)
90
92
  mini_portile (~> 0.5.0)
91
- parallel (0.8.3)
93
+ parallel (0.9.0)
92
94
  pry (0.9.12.2)
93
95
  coderay (~> 1.0.5)
94
96
  method_source (~> 0.8)
95
97
  slop (~> 3.4)
96
- rails_best_practices (1.14.3)
98
+ rails_best_practices (1.14.4)
97
99
  activesupport
98
100
  awesome_print
99
- code_analyzer (>= 0.4.2)
101
+ code_analyzer (>= 0.4.3)
100
102
  colored
101
103
  erubis
102
104
  i18n
@@ -106,17 +108,15 @@ GEM
106
108
  rb-fsevent (0.9.3)
107
109
  rb-inotify (0.9.2)
108
110
  ffi (>= 0.5.0)
109
- rb-kqueue (0.2.0)
110
- ffi (>= 0.5.0)
111
111
  redcard (1.1.0)
112
- reek (1.3.3)
112
+ reek (1.3.4)
113
113
  ruby2ruby (~> 2.0.2)
114
- ruby_parser (~> 3.1, >= 3.1.1)
114
+ ruby_parser (~> 3.2)
115
115
  sexp_processor
116
116
  require_all (1.3.1)
117
117
  roodi (3.1.1)
118
118
  ruby_parser (~> 3.2, >= 3.2.2)
119
- rr (1.1.1)
119
+ rr (1.1.2)
120
120
  ruby-prof (0.13.0)
121
121
  ruby-progressbar (1.2.0)
122
122
  ruby2ruby (2.0.6)
@@ -136,6 +136,7 @@ GEM
136
136
  slop (3.4.6)
137
137
  thor (0.18.1)
138
138
  thread (0.1.1)
139
+ timers (1.1.0)
139
140
 
140
141
  PLATFORMS
141
142
  ruby
@@ -11,7 +11,7 @@ class BenchmarkWorker
11
11
  :prefetch => 50,
12
12
  :timeout_job_after => 1,
13
13
  :exchange => 'dummy',
14
- :heartbeat_interval => 5
14
+ :heartbeat => 5
15
15
  def work(msg)
16
16
  ack!
17
17
  end
@@ -20,7 +20,7 @@ class ProfilingWorker
20
20
  :prefetch => 50,
21
21
  :timeout_job_after => 1,
22
22
  :exchange => 'dummy',
23
- :heartbeat_interval => 5
23
+ :heartbeat => 5
24
24
  def work(msg)
25
25
  ack!
26
26
  end
@@ -1,8 +1,11 @@
1
- $: << File.expand_path('../lib', File.dirname(__FILE__))
2
- require 'sneakers'
1
+ require "sneakers"
3
2
  require 'open-uri'
4
3
  require 'nokogiri'
5
4
 
5
+ require 'logger'
6
+
7
+ Sneakers.configure :log => STDOUT
8
+ Sneakers.logger.level = Logger::INFO
6
9
 
7
10
  class TitleScraper
8
11
  include Sneakers::Worker
@@ -11,7 +11,7 @@ class WorkflowWorker
11
11
  :prefetch => 50,
12
12
  :timeout_job_after => 1,
13
13
  :exchange => 'dummy',
14
- :heartbeat_interval => 5
14
+ :heartbeat => 5
15
15
 
16
16
  def work(msg)
17
17
  logger.info("Seriously, i'm DONE.")
data/lib/sneakers.rb CHANGED
@@ -1,6 +1,7 @@
1
1
  require "sneakers/version"
2
2
  require 'thread/pool'
3
3
  require 'bunny'
4
+ require 'logger'
4
5
 
5
6
 
6
7
  module Sneakers
@@ -28,7 +29,6 @@ module Sneakers
28
29
  :workers => 4,
29
30
  :log => 'sneakers.log',
30
31
  :pid_path => 'sneakers.pid',
31
- :amqp => 'amqp://guest:guest@localhost:55672',
32
32
 
33
33
  #workers
34
34
  :timeout_job_after => 5,
@@ -37,8 +37,11 @@ module Sneakers
37
37
  :env => ENV['RACK_ENV'],
38
38
  :durable => true,
39
39
  :ack => true,
40
- :heartbeat_interval => 2,
40
+ :heartbeat => 2,
41
+ :amqp => 'amqp://guest:guest@localhost:5672',
42
+ :vhost => '/',
41
43
  :exchange => 'sneakers',
44
+ :exchange_type => :direct,
42
45
  :hooks => {}
43
46
  }
44
47
 
data/lib/sneakers/cli.rb CHANGED
@@ -26,10 +26,19 @@ module Sneakers
26
26
  method_option :debug
27
27
  method_option :front
28
28
  method_option :require
29
- method_option :amqp
30
29
 
31
30
  desc "work FirstWorker,SecondWorker ... ,NthWorker", "Run workers"
32
31
  def work(workers)
32
+ opts = {
33
+ :daemonize => !options[:front]
34
+ }
35
+ unless opts[:daemonize]
36
+ opts[:log] = STDOUT
37
+ end
38
+
39
+ Sneakers.configure(opts)
40
+ puts Sneakers::Config
41
+
33
42
  require_boot File.expand_path(options[:require]) if options[:require]
34
43
 
35
44
  workers, missing_workers = Sneakers::Utils.parse_workers(workers)
@@ -52,13 +61,6 @@ module Sneakers
52
61
  return
53
62
  end
54
63
 
55
- opts = {
56
- :daemonize => !options[:front],
57
- :log => options[:front] ? STDOUT : Sneakers::Config[:log],
58
- :amqp => (options[:amqp] || Sneakers::Config[:amqp])
59
- }
60
-
61
- Sneakers.configure(opts)
62
64
  r = Sneakers::Runner.new(workers)
63
65
 
64
66
  pid = Sneakers::Config[:pid_path]
@@ -9,8 +9,8 @@ module Sneakers
9
9
  @channel.acknowledge(tag, false)
10
10
  end
11
11
 
12
- def reject(tag)
13
- @channel.reject(tag, false)
12
+ def reject(tag, requeue=false)
13
+ @channel.reject(tag, requeue)
14
14
  end
15
15
 
16
16
  def error(tag, err)
@@ -20,6 +20,10 @@ module Sneakers
20
20
  def timeout(tag)
21
21
  reject(tag)
22
22
  end
23
+
24
+ def noop(tag)
25
+
26
+ end
23
27
  end
24
28
  end
25
29
  end
@@ -21,7 +21,7 @@ module Sneakers
21
21
  private
22
22
 
23
23
  def ensure_connection!
24
- @bunny = Bunny.new(:heartbeat_interval => @opts[:heartbeat_interval])
24
+ @bunny = Bunny.new(:heartbeat => @opts[:heartbeat])
25
25
  @bunny.start
26
26
  @channel = @bunny.create_channel
27
27
  @exchange = @channel.exchange(@opts[:exchange], :type => :direct, :durable => @opts[:durable])
@@ -16,13 +16,16 @@ class Sneakers::Queue
16
16
  # :ack
17
17
  #
18
18
  def subscribe(worker)
19
- @bunny = Bunny.new(@opts[:amqp], :heartbeat_interval => @opts[:heartbeat_interval])
19
+ @bunny = Bunny.new(@opts[:amqp], :vhost => @opts[:vhost], :heartbeat => @opts[:heartbeat])
20
20
  @bunny.start
21
21
 
22
22
  @channel = @bunny.create_channel
23
23
  @channel.prefetch(@opts[:prefetch])
24
24
 
25
- @exchange = @channel.exchange(@opts[:exchange], :type => :direct, :durable => @opts[:durable])
25
+ @exchange = @channel.exchange(@opts[:exchange],
26
+ :type => @opts[:exchange_type],
27
+ :durable => @opts[:durable])
28
+
26
29
  handler = @handler_klass.new(@channel)
27
30
 
28
31
  queue = @channel.queue(@name, :durable => @opts[:durable])
@@ -1,3 +1,3 @@
1
1
  module Sneakers
2
- VERSION = "0.0.5"
2
+ VERSION = "0.0.6"
3
3
  end
@@ -17,6 +17,7 @@ module Sneakers
17
17
  queue_name = self.class.queue_name
18
18
 
19
19
  opts = Sneakers::Config.merge(opts)
20
+
20
21
  queue_name = Support::QueueName.new(queue_name, opts).to_s
21
22
 
22
23
  @should_ack = opts[:ack]
@@ -25,11 +26,7 @@ module Sneakers
25
26
 
26
27
  @queue = queue || Sneakers::Queue.new(
27
28
  queue_name,
28
- :prefetch => opts[:prefetch],
29
- :durable => opts[:durable],
30
- :ack => @should_ack,
31
- :heartbeat_interval => opts[:heartbeat_interval],
32
- :exchange => opts[:exchange]
29
+ opts
33
30
  )
34
31
 
35
32
  @opts = opts
@@ -37,8 +34,8 @@ module Sneakers
37
34
  end
38
35
 
39
36
  def ack!; :ack end
40
- def nack!; :nack end
41
37
  def reject!; :reject; end
38
+ def requeue!; :requeue; end
42
39
 
43
40
  def publish(msg, routing)
44
41
  return unless routing[:to_queue]
@@ -76,8 +73,12 @@ module Sneakers
76
73
  handler.timeout(hdr.delivery_tag)
77
74
  elsif res == :error
78
75
  handler.error(hdr.delivery_tag, error)
79
- else
76
+ elsif res == :reject
80
77
  handler.reject(hdr.delivery_tag)
78
+ elsif res == :requeue
79
+ handler.reject(hdr.delivery_tag, true)
80
+ else
81
+ handler.noop(hdr.delivery_tag)
81
82
  end
82
83
  metrics.increment("work.#{self.class.name}.handled.#{res || 'reject'}")
83
84
  end
@@ -26,7 +26,7 @@ module Sneakers
26
26
  worker.run
27
27
  end
28
28
  # end per worker
29
-
29
+ #
30
30
  until @stop_flag.wait_for_set(10.0)
31
31
  Sneakers.logger.info("Heartbeat: running threads [#{Thread.list.count}]")
32
32
  # report aggregated stats?
@@ -14,8 +14,10 @@ describe Sneakers::Queue do
14
14
  :prefetch => 25,
15
15
  :durable => true,
16
16
  :ack => true,
17
- :heartbeat_interval => 2,
18
- :exchange => "sneakers"
17
+ :heartbeat => 2,
18
+ :vhost => '/',
19
+ :exchange => "sneakers",
20
+ :exchange_type => :direct
19
21
  )
20
22
  mkbunny = Object.new
21
23
  mkchan = Object.new
@@ -24,7 +26,7 @@ describe Sneakers::Queue do
24
26
 
25
27
  mock(mkbunny).start {}
26
28
  mock(mkbunny).create_channel{ mkchan }
27
- mock(Bunny).new(anything, :heartbeat_interval => 2){ mkbunny }
29
+ mock(Bunny).new(anything, :vhost => '/', :heartbeat => 2){ mkbunny }
28
30
 
29
31
  mock(mkchan).prefetch(25)
30
32
  mock(mkchan).exchange("sneakers", :type => :direct, :durable => true){ mkex }
@@ -13,7 +13,7 @@ class DummyWorker
13
13
  :prefetch => 40,
14
14
  :timeout_job_after => 1,
15
15
  :exchange => 'dummy',
16
- :heartbeat_interval => 5
16
+ :heartbeat => 5
17
17
 
18
18
  def work(msg)
19
19
  end
@@ -127,7 +127,7 @@ describe Sneakers::Worker do
127
127
  stub(@queue).name { 'test-queue' }
128
128
  stub(@queue).opts { {} }
129
129
 
130
- Sneakers.configure(:env => 'test')
130
+ Sneakers.configure(:env => 'test', :daemonize => true, :log => 'sneakers.log')
131
131
  Sneakers::Worker.configure_logger(Logger.new('/dev/null'))
132
132
  Sneakers::Worker.configure_metrics
133
133
  end
@@ -142,22 +142,14 @@ describe Sneakers::Worker do
142
142
  it "should build a queue with correct configuration given defaults" do
143
143
  @defaults_q.name.must_equal('defaults_test')
144
144
  @defaults_q.opts.must_equal(
145
- :durable => true,
146
- :ack => true,
147
- :prefetch => 10,
148
- :heartbeat_interval => 2,
149
- :exchange => 'sneakers'
145
+ {:runner_config_file=>nil, :metrics=>nil, :daemonize=>true, :start_worker_delay=>0.2, :workers=>4, :log=>"sneakers.log", :pid_path=>"sneakers.pid", :timeout_job_after=>5, :prefetch=>10, :threads=>10, :env=>"test", :durable=>true, :ack=>true, :amqp=>"amqp://guest:guest@localhost:5672", :vhost=>"/", :exchange=>"sneakers", :exchange_type=>:direct, :hooks=>{}, :handler=>Sneakers::Handlers::Oneshot, :heartbeat => 2}
150
146
  )
151
147
  end
152
148
 
153
149
  it "should build a queue with given configuration" do
154
150
  @dummy_q.name.must_equal('downloads_test')
155
151
  @dummy_q.opts.must_equal(
156
- :durable => false,
157
- :ack => false,
158
- :prefetch => 40,
159
- :heartbeat_interval => 5,
160
- :exchange => 'dummy'
152
+ {:runner_config_file=>nil, :metrics=>nil, :daemonize=>true, :start_worker_delay=>0.2, :workers=>4, :log=>"sneakers.log", :pid_path=>"sneakers.pid", :timeout_job_after=>1, :prefetch=>40, :threads=>50, :env=>"test", :durable=>false, :ack=>false, :amqp=>"amqp://guest:guest@localhost:5672", :vhost=>"/", :exchange=>"dummy", :exchange_type=>:direct, :hooks=>{}, :handler=>Sneakers::Handlers::Oneshot, :heartbeat =>5}
161
153
  )
162
154
  end
163
155
  end
@@ -238,18 +230,18 @@ describe Sneakers::Worker do
238
230
  @worker.do_work(@header, nil, :ack, handler)
239
231
  end
240
232
 
241
- it "should work and handle nacks" do
233
+ it "should work and handle rejects" do
242
234
  handler = Object.new
243
235
  mock(handler).reject("tag")
244
236
 
245
- @worker.do_work(@header, nil, :nack, handler)
237
+ @worker.do_work(@header, nil, :reject, handler)
246
238
  end
247
239
 
248
- it "should work and handle rejects" do
240
+ it "should work and handle requeues" do
249
241
  handler = Object.new
250
- mock(handler).reject("tag")
242
+ mock(handler).reject("tag", true)
251
243
 
252
- @worker.do_work(@header, nil, :reject, handler)
244
+ @worker.do_work(@header, nil, :requeue, handler)
253
245
  end
254
246
 
255
247
  it "should work and handle user-land timeouts" do
@@ -310,6 +302,7 @@ describe Sneakers::Worker do
310
302
  stub(@handler).reject("tag")
311
303
  stub(@handler).timeout("tag")
312
304
  stub(@handler).error("tag", anything)
305
+ stub(@handler).noop("tag")
313
306
 
314
307
  @header = Object.new
315
308
  stub(@header).delivery_tag { "tag" }
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: sneakers
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.5
4
+ version: 0.0.6
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2013-10-16 00:00:00.000000000 Z
12
+ date: 2013-10-20 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: serverengine