sneakers 0.0.1 → 0.0.2

data/.gitignore

@@ -1,17 +1,6 @@
-*.gem
-*.rbc
-.bundle
-.config
-.yardoc
-Gemfile.lock
-InstalledFiles
-_yardoc
-coverage
-doc/
-lib/bundler/man
-pkg
-rdoc
-spec/reports
-test/tmp
-test/version_tmp
-tmp
+sneakers.yaml
+sneakers.log
+sneakers.pid
+pkg/
+coverage/
+tmp/

data/Gemfile

@@ -1,4 +1,3 @@
 source 'https://rubygems.org'
 
-# Specify your gem's dependencies in sneakers.gemspec
 gemspec

data/Gemfile.lock

@@ -0,0 +1,152 @@
+PATH
+  remote: .
+  specs:
+    sneakers (0.0.2)
+      bunny (>= 0.9.0.rc2)
+      serverengine
+      thor
+      thread
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (3.2.14)
+      i18n (~> 0.6, >= 0.6.4)
+      multi_json (~> 1.0)
+    amq-protocol (1.7.0)
+    arrayfields (4.9.0)
+    awesome_print (1.1.0)
+    bluff (0.1.0)
+    bunny (1.0.0.pre6)
+      amq-protocol (>= 1.7.0)
+    cane (2.6.0)
+      parallel
+    chronic (0.10.2)
+    churn (0.0.33)
+      chronic (>= 0.2.3)
+      hirb
+      json_pure
+      main
+      ruby_parser (~> 3.0)
+      sexp_processor (~> 4.1)
+    code_analyzer (0.4.2)
+      sexp_processor
+    code_metrics (0.1.0)
+    coderay (1.0.9)
+    colored (1.2)
+    erubis (2.7.0)
+    fattr (2.2.1)
+    ffi (1.9.0)
+    flay (2.4.0)
+      ruby_parser (~> 3.0)
+      sexp_processor (~> 4.0)
+    flog (4.1.2)
+      ruby_parser (~> 3.1, > 3.1.0)
+      sexp_processor (~> 4.0)
+    formatador (0.2.4)
+    guard (1.8.2)
+      formatador (>= 0.2.4)
+      listen (>= 1.0.0)
+      lumberjack (>= 1.0.2)
+      pry (>= 0.9.10)
+      thor (>= 0.14.6)
+    guard-minitest (1.3.0)
+      guard (>= 1.8)
+      minitest (>= 2.1)
+    hirb (0.7.1)
+    i18n (0.6.5)
+    json_pure (1.8.0)
+    listen (1.3.1)
+      rb-fsevent (>= 0.9.3)
+      rb-inotify (>= 0.9)
+      rb-kqueue (>= 0.2)
+    lumberjack (1.0.4)
+    main (5.2.0)
+      arrayfields (>= 4.7.4)
+      chronic (>= 0.6.2)
+      fattr (>= 2.2.0)
+      map (>= 5.1.0)
+    map (6.5.1)
+    method_source (0.8.2)
+    metric_fu (4.4.1)
+      bluff
+      cane (~> 2.5, >= 2.5.2)
+      churn (~> 0.0.28)
+      code_metrics (~> 0.1)
+      coderay
+      flay (~> 2.1, >= 2.0.1)
+      flog (~> 4.1, >= 4.1.1)
+      metric_fu-Saikuro (>= 1.1.1.0)
+      multi_json
+      rails_best_practices (~> 1.14, >= 1.14.3)
+      redcard
+      reek (~> 1.3, >= 1.3.3)
+      roodi (~> 3.1)
+    metric_fu-Saikuro (1.1.1.0)
+    mini_portile (0.5.1)
+    minitest (5.0.7)
+    multi_json (1.8.0)
+    nokogiri (1.6.0)
+      mini_portile (~> 0.5.0)
+    parallel (0.8.3)
+    pry (0.9.12.2)
+      coderay (~> 1.0.5)
+      method_source (~> 0.8)
+      slop (~> 3.4)
+    rails_best_practices (1.14.3)
+      activesupport
+      awesome_print
+      code_analyzer (>= 0.4.2)
+      colored
+      erubis
+      i18n
+      require_all
+      ruby-progressbar
+    rake (10.1.0)
+    rb-fsevent (0.9.3)
+    rb-inotify (0.9.2)
+      ffi (>= 0.5.0)
+    rb-kqueue (0.2.0)
+      ffi (>= 0.5.0)
+    redcard (1.1.0)
+    reek (1.3.3)
+      ruby2ruby (~> 2.0.2)
+      ruby_parser (~> 3.1, >= 3.1.1)
+      sexp_processor
+    require_all (1.3.1)
+    roodi (3.1.1)
+      ruby_parser (~> 3.2, >= 3.2.2)
+    rr (1.1.1)
+    ruby-prof (0.13.0)
+    ruby-progressbar (1.2.0)
+    ruby2ruby (2.0.6)
+      ruby_parser (~> 3.1)
+      sexp_processor (~> 4.0)
+    ruby_parser (3.2.2)
+      sexp_processor (~> 4.1)
+    serverengine (1.5.4)
+      sigdump (~> 0.2.2)
+    sexp_processor (4.3.0)
+    sigdump (0.2.2)
+    simplecov (0.7.1)
+      multi_json (~> 1.0)
+      simplecov-html (~> 0.7.1)
+    simplecov-html (0.7.1)
+    simplecov-rcov-text (0.0.2)
+    slop (3.4.6)
+    thor (0.18.1)
+    thread (0.1.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  guard-minitest
+  metric_fu
+  nokogiri
+  rake
+  rr
+  ruby-prof
+  simplecov
+  simplecov-rcov-text
+  sneakers!

data/Guardfile

@@ -0,0 +1,8 @@
+
+guard :minitest do
+  watch(%r{^spec/(.*)_spec\.rb})
+  watch(%r{^lib/(.+)\.rb})         { |m| "spec/#{m[1]}_spec.rb" }
+  watch(%r{^spec/spec_helper\.rb}) { 'spec' }
+end
+
+

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2013 jondot
+Copyright (c) 2013 Dotan Nahum
 
 MIT License
 
@@ -19,4 +19,4 @@ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
 LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -1,6 +1,20 @@
 # Sneakers
 
-TODO: Write a gem description
+
+```
+      __
+  ,--'  >  
+  `=====   
+
+```
+
+
+A high-performance RabbitMQ background processing framework for
+Ruby.
+
+Visit the [[wiki]]](https://github.com/jondot/sneakers/wiki) for
+complete docs.
+
 
 ## Installation
 
@@ -16,14 +30,131 @@ Or install it yourself as:
 
     $ gem install sneakers
 
-## Usage
 
-TODO: Write usage instructions here
+## Quick start 
+
+Set up a Gemfile
+
+```ruby
+source 'https://rubygems.org'
+gem 'sneakers'
+gem 'json'
+gem 'redis'
+```
+
+And a worker
+
+```ruby
+require 'sneakers'
+require 'redis'
+require 'json'
+
+$redis = Redis.new
+
+class Processor
+  include Sneakers::Worker
+  from_queue :logs
+
+
+  def work(msg)
+    err = JSON.parse(msg)
+    if err["type"] == "error"
+      $redis.incr "processor:#{err["error"]}"
+    end
+
+    ack!
+  end
+end
+```
+
+
+As an example, make a message look like this:
+We'll count errors and error types with Redis. Specifically for an error that looks like this:
+
+```javascript
+{
+   "type": "error",
+   "message": "HALP!",
+   "error": "CODE001"
+}
+```
+
+
+Let's test it out quickly from the command line:
+
+
+```bash
+sneakers work Processor --require boot.rb --front
+```
+
+We just told Sneakers to spawn a worker named `Processor`, but first `--require` a file that we dedicate to setting up environment, including workers and what-not.
+
+For simplicity, we also told Sneakers to *not* daemonize and work at the `--front`.
+
+If you go to your RabbitMQ admin now, you'll see a new queue named `logs` was created. Push a couple messages, and this is the output you should see at your terminal.
+
+
+```
+2013-10-11T19:26:36Z p-4718 t-ovqgyb31o DEBUG: [worker-logs:1:213mmy][#<Thread:0x007fae6b05cc58>][logs][{:prefetch=>10, :durable=>true, :ack=>true, :heartbeat_interval=>2, :exchange=>"sneakers"}] Working off: log log
+2013-10-11T19:26:36Z p-4718 t-ovqgyrxu4 INFO: log log
+2013-10-11T19:26:40Z p-4719 t-ovqgyb364 DEBUG: [worker-logs:1:h23iit][#<Thread:0x007fae6b05cd98>][logs][{:prefetch=>10, :durable=>true, :ack=>true, :heartbeat_interval=>2, :exchange=>"sneakers"}] Working off: log log
+2013-10-11T19:26:40Z p-4719 t-ovqgyrx8g INFO: log log
+```
+
+And redis will show this: 
+
+
+```
+➜  ~  redis-cli monitor
+1381520329.888581 [0 127.0.0.1:49182] "incr" "processor:CODE001"
+```
+
+
+We're basically done with the ceremonies and all is left is to do some real work.
+
+
+
+### Looking at metrics
+
+Let's use the `logging_metrics` provider just for the sake of fun of seeing the metrics as they happen.
+
+```ruby
+# boot.rb
+require 'sneakers'
+require 'redis'
+require 'json'
+require 'sneakers/metrics/logging_metrics'
+Sneakers.configure :metrics => Sneakers::Metrics::LoggingMetrics.new
+
+# ... rest of code
+```
+
+Now push a message again and you'll see:
+
+```
+2013-10-11T19:44:37Z p-9219 t-oxh8owywg INFO: INC: work.Processor.started
+2013-10-11T19:44:37Z p-9219 t-oxh8owywg INFO: TIME: work.Processor.time 0.00242
+2013-10-11T19:44:37Z p-9219 t-oxh8owywg INFO: INC: work.Processor.handled.ack
+```
+
+Which increments start + end, and times the work unit.
+
+
+
+From here, you can continue over to the
+[[Wiki]](https://github.com/jondot/sneakers/wiki)
+
+# Contributing
+
+Fork, implement, add tests, pull request, get my everlasting thanks and a respectable place here :).
+
+# Copyright
+
+Copyright (c) 2013 [Dotan Nahum](http://gplus.to/dotan) [@jondot](http://twitter.com/jondot). See MIT-LICENSE for further details.
+
+
+
+
+
 
-## Contributing
 
-1. Fork it
-2. Create your feature branch (`git checkout -b my-new-feature`)
-3. Commit your changes (`git commit -am 'Add some feature'`)
-4. Push to the branch (`git push origin my-new-feature`)
-5. Create new Pull Request

data/Rakefile

@@ -1 +1,10 @@
 require "bundler/gem_tasks"
+require 'metric_fu'
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << "spec"
+  t.test_files = FileList['spec/**/*_spec.rb']
+end
+
+

data/bin/sneakers

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+require 'sneakers'
+require 'sneakers/cli'
+
+Sneakers::CLI.start

data/examples/benchmark_worker.rb

@@ -0,0 +1,21 @@
+$: << File.expand_path('../lib', File.dirname(__FILE__))
+require 'sneakers'
+
+class BenchmarkWorker
+  include Sneakers::Worker
+  from_queue 'downloads',
+             :env => 'test',
+             :durable => false,
+             :ack => true,
+             :threads => 50,
+             :prefetch => 50,
+             :timeout_job_after => 1,
+             :exchange => 'dummy',
+             :heartbeat_interval => 5
+  def work(msg)
+    ack!
+  end
+end
+
+
+

data/examples/metrics_worker.rb

@@ -0,0 +1,28 @@
+$: << File.expand_path('../lib', File.dirname(__FILE__))
+require 'sneakers'
+require 'sneakers/runner'
+require 'sneakers/metrics/logging_metrics'
+require 'open-uri'
+require 'nokogiri'
+
+
+class MetricsWorker
+  include Sneakers::Worker
+
+  from_queue 'downloads'
+
+  def work(msg)
+    doc = Nokogiri::HTML(open(msg))
+    logger.info "FOUND <#{doc.css('title').text}>"
+    ack!
+  end
+end
+
+
+Sneakers.configure( :daemonize => false, :log => STDOUT, :metrics => Sneakers::Metrics::LoggingMetrics.new)
+r = Sneakers::Runner.new([ MetricsWorker ])
+r.run
+
+
+
+

data/examples/profiling_worker.rb

@@ -0,0 +1,55 @@
+$: << File.expand_path('../lib', File.dirname(__FILE__))
+require 'sneakers'
+require 'sneakers/runner'
+require 'ruby-prof'
+
+
+puts "feeding messages in"
+1000.times {
+  Sneakers.publish("{}", :to_queue => 'downloads')
+puts "done"
+
+
+class ProfilingWorker
+  include Sneakers::Worker
+  from_queue 'downloads',
+             :env => '',
+             :durable => false,
+             :ack => true,
+             :threads => 50,
+             :prefetch => 50,
+             :timeout_job_after => 1,
+             :exchange => 'dummy',
+             :heartbeat_interval => 5
+  def work(msg)
+    ack!
+  end
+end
+
+
+
+r = Sneakers::Runner.new
+Sneakers::Worker.configure_logger(Logger.new('/dev/null'))
+
+# ctrl-c and Ruby 2.0 breaks signal handling
+# Sidekiq has same issues
+# https://github.com/mperham/sidekiq/issues/728
+#
+# so we use a timeout and a thread that kills profiling
+puts "profiling start"
+RubyProf.start
+
+Thread.new do
+  sleep 10
+  puts "stopping profiler"
+  result = RubyProf.stop
+
+  # Print a flat profile to text
+  printer = RubyProf::FlatPrinter.new(result)
+  printer.print(STDOUT)
+  r.stop
+  exit(0)
+end
+
+r.run([ ProfilingWorker ])
+