boardintel_frenzy_bunnies 0.0.9

data/lib/frenzy_bunnies/web.rb

@@ -0,0 +1,51 @@
+require 'sinatra/base'
+require 'json'
+
+class FrenzyBunnies::Web < Sinatra::Base
+  configure do
+    # disable logging
+    set :public_folder, File.expand_path('web/public', File.dirname(__FILE__))
+  end
+
+  before do
+    content_type 'application/json'
+  end
+
+  not_found do
+    'Cant find that, sorry.'
+  end
+
+  error do
+  'Oops. There was an error - ' + env['sinatra.error'].name
+  end
+
+  get '/ping' do
+    'ok'
+  end
+
+  get '/health' do
+    settings.health_collector.collect.to_json
+  end
+
+  get '/stats' do
+    jobs.map do |klass|
+      { :name => klass.name,
+        :stats => klass.jobs_stats }
+    end.to_json
+  end
+
+  get '/' do
+    redirect '/index.html'
+  end
+
+  def self.run_with(jobs, opts={})
+    set :jobs, (jobs || [])
+    set :health_collector, FrenzyBunnies::Health::Collector.new({:jvm => {:threadfilter => opts[:threadfilter]}})
+    @logger = opts[:logger]
+    @logger.info "* running web dashboard bound to #{opts[:host]} on port #{opts[:port]}."
+    Rack::Handler::WEBrick.run self, :Host => opts[:host], :Port => opts[:port],  :Logger => WEBrick::Log.new("/dev/null"), :AccessLog => [nil, nil]
+  end
+  def jobs
+    settings.jobs
+  end
+end

data/lib/frenzy_bunnies/worker.rb

@@ -0,0 +1,102 @@
+require 'atomic'
+
+module FrenzyBunnies::Worker
+  def ack!
+    true
+  end
+
+  def work
+  end
+
+  def self.included(base)
+    base.extend ClassMethods
+  end
+
+  module ClassMethods
+
+    def from_queue(q, opts={})
+      @queue_name = q
+      @queue_opts = opts
+    end
+
+    def start(context)
+      @jobs_stats = { :failed => Atomic.new(0), :passed => Atomic.new(0) }
+      @working_since = Time.now
+
+      @logger = context.logger
+
+      queue_name = "#{@queue_name}_#{context.env}"
+
+      @queue_opts[:prefetch] ||= 10
+      @queue_opts[:durable] ||= false
+      @queue_opts[:timeout_job_after] = 5 if @queue_opts[:timeout_job_after].nil?
+      @queue_opts[:handler] ||= FrenzyBunnies::Handlers::Oneshot
+
+      if @queue_opts[:threads]
+        @thread_pool = MarchHare::ThreadPools.fixed_of_size(@queue_opts[:threads])
+      else
+        @thread_pool = MarchHare::ThreadPools.dynamically_growing
+      end
+
+      q = context.queue_factory.build_queue(queue_name, @queue_opts)
+
+      say "#{@queue_opts[:threads] ? "#{@queue_opts[:threads]} threads " : ''}with #{@queue_opts[:prefetch]} prefetch on <#{queue_name}>."
+
+      q.subscribe(:ack => true, :blocking => false, :executor => @thread_pool) do |headers, msg|
+        worker = new
+        handler = @queue_opts[:handler].new(headers.channel, q, @logger, { queue_options: @queue_opts })
+        begin
+          Timeout::timeout(@queue_opts[:timeout_job_after]) do
+            if worker.work(msg)
+              handler.acknowledge(headers, msg)
+              incr! :passed
+            else
+              handler.reject(headers, msg)
+              incr! :failed
+              error "REJECTED", msg
+            end
+          end
+        rescue Timeout::Error
+          handler.timeout(headers, msg)
+          incr! :failed
+          error "TIMEOUT #{@queue_opts[:timeout_job_after]}s", msg
+        rescue
+          handler.reject(headers, msg)
+          incr! :failed
+          error "ERROR #{$!}", msg
+        end
+      end
+
+      say "workers up."
+    end
+
+    def stop
+      say "stopping"
+      @thread_pool.shutdown_now
+      say "pool shutdown"
+      # @s.cancel  #for some reason when the channel socket is broken, this is holding the process up and we're zombie.
+      say "stopped"
+    end
+
+    def queue_opts
+      @queue_opts
+    end
+
+    def jobs_stats
+      Hash[ @jobs_stats.map{ |k,v| [k, v.value] } ].merge({ :since => @working_since.to_i })
+    end
+  private
+    def say(text)
+      @logger.info "[#{self.name}] #{text}"
+    end
+
+    def error(text, msg)
+      @logger.error "[#{self.name}] #{text} <#{msg}>"
+    end
+
+    def incr!(what)
+      @jobs_stats[what].update { |v| v + 1 }
+    end
+  end
+end
+

data/lib/frenzy_bunnies.rb

@@ -0,0 +1,15 @@
+require 'march_hare'
+require 'timeout'
+
+module FrenzyBunnies
+end
+
+
+require "frenzy_bunnies/version"
+require 'frenzy_bunnies/health'
+require 'frenzy_bunnies/queue_factory'
+require 'frenzy_bunnies/context'
+require 'frenzy_bunnies/handlers/oneshot'
+require 'frenzy_bunnies/worker'
+require 'frenzy_bunnies/web'
+

data/spec/frenzy_bunnies/worker_spec.rb

@@ -0,0 +1,117 @@
+require 'spec_helper'
+require 'frenzy_bunnies'
+
+class DummyWorker
+  include FrenzyBunnies::Worker
+  from_queue 'new.feeds'
+
+  def work(msg)
+  end 
+end
+
+class CustomWorker
+  include FrenzyBunnies::Worker
+  from_queue 'new.feeds', :prefetch => 20, :durable => true, :timeout_job_after => 13, :threads => 25
+
+  def work(msg)
+  end 
+end
+
+def with_test_queuefactory(ctx, ack=true, msg=nil, nowork=false)
+  qf = Object.new
+  q = Object.new
+  s = Object.new
+  hdr = Object.new
+  mock(qf).build_queue(anything, anything) { q }
+  mock(q).subscribe(anything){ s }
+
+  mock(s).each(anything) { |h,b| b.call(hdr, msg) unless nowork }
+  mock(hdr).ack{true} if !nowork && ack
+  mock(hdr).reject{true} if !nowork && !ack
+
+  mock(ctx).queue_factory { qf } # should return our own
+end
+
+describe FrenzyBunnies::Worker do
+  it "should start with a clean slate" do
+    # check stats, default configuration
+    ctx = FrenzyBunnies::Context.new(:logger=> Logger.new(nil))
+    with_test_queuefactory(ctx, nil, nil, true)
+
+
+    DummyWorker.start(ctx)
+    DummyWorker.jobs_stats[:failed].must_equal 0
+    DummyWorker.jobs_stats[:passed].must_equal 0
+    q = DummyWorker.queue_opts
+    q.must_equal({:prefetch=>10, :durable=>false, :timeout_job_after=>5})
+    
+  end
+  it "should respond to configuration tweaks" do
+    # check that all params are changed
+    ctx = FrenzyBunnies::Context.new(:logger=> Logger.new(nil))
+    with_test_queuefactory(ctx, nil, nil, true)
+
+    CustomWorker.start(ctx)
+    CustomWorker.jobs_stats[:failed].must_equal 0
+    CustomWorker.jobs_stats[:passed].must_equal 0
+    q = CustomWorker.queue_opts
+    q.must_equal({:prefetch=>20, :durable=>true, :timeout_job_after=>13, :threads=>25})
+  end
+  it "should stop when asked to" do
+    # validate that a worker stops
+    # check stats, default configuration
+    ctx = FrenzyBunnies::Context.new(:logger=> Logger.new(nil))
+    with_test_queuefactory(ctx, nil, nil, true)
+
+
+    DummyWorker.start(ctx)
+    DummyWorker.stop
+  end
+  it "should be passed a message to work on" do
+    ctx = FrenzyBunnies::Context.new(:logger=> Logger.new(nil))
+    with_test_queuefactory(ctx, true, "work!")
+
+    any_instance_of(DummyWorker){ |w| mock(w).work("work!"){ true } }
+    DummyWorker.start(ctx)
+  end
+  it "should acknowledge a unit of work when worker succeeds" do
+    ctx = FrenzyBunnies::Context.new(:logger=> Logger.new(nil))
+    with_test_queuefactory(ctx)
+
+    any_instance_of(DummyWorker){ |w| mock(w).work(anything){ true } }
+    DummyWorker.start(ctx)
+    DummyWorker.jobs_stats[:passed].must_equal 1
+    DummyWorker.jobs_stats[:failed].must_equal 0
+  end
+  it "should reject a unit of work when worker fails" do
+    ctx = FrenzyBunnies::Context.new(:logger=> Logger.new(nil))
+    with_test_queuefactory(ctx,false)
+
+    any_instance_of(DummyWorker){ |w| mock(w).work(anything){ false } }
+    mock(DummyWorker).error(anything, anything){ |text, _| text.must_match(/^REJECTED/) }
+    DummyWorker.start(ctx)
+    DummyWorker.jobs_stats[:failed].must_equal 1
+    DummyWorker.jobs_stats[:passed].must_equal 0
+  end
+  it "should reject a unit of work when worker times out" do
+    ctx = FrenzyBunnies::Context.new(:logger=> Logger.new(nil))
+    with_test_queuefactory(ctx,false)
+    DummyWorker.queue_opts[:timeout_job_after] = 1
+    any_instance_of(DummyWorker){ |w| mock(w).work(anything){ sleep(2) }}
+    mock(DummyWorker).error(anything, anything){ |text, _| text.must_match(/^TIMEOUT/) }
+    DummyWorker.start(ctx)
+    DummyWorker.jobs_stats[:failed].must_equal 1
+    DummyWorker.jobs_stats[:passed].must_equal 0
+    DummyWorker.queue_opts[:timeout_job_after] = 5
+  end
+  it "should reject a unit of work when worker fails exceptionally" do
+    ctx = FrenzyBunnies::Context.new(:logger=> Logger.new(nil))
+    with_test_queuefactory(ctx,false)
+
+    any_instance_of(DummyWorker){ |w| mock(w).work(anything){ throw :error } }
+    mock(DummyWorker).error(anything, anything){ |text, _| text.must_match(/^ERROR/) }
+    DummyWorker.start(ctx)
+    DummyWorker.jobs_stats[:failed].must_equal 1
+    DummyWorker.jobs_stats[:passed].must_equal 0
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,35 @@
+#require 'simplecov'
+#SimpleCov.start if ENV["COVERAGE"]
+
+require 'minitest/autorun'
+
+
+
+require 'rr'
+
+
+class MiniTest::Unit::TestCase
+  include RR::Adapters::MiniTest
+end
+
+
+require 'thor'
+# This is to silence the 'task' warning for the mocks.
+#
+class Thor
+  class << self
+    def create_task(meth) #:nodoc:
+      if @usage && @desc
+        base_class = @hide ? Thor::HiddenTask : Thor::Task
+        tasks[meth] = base_class.new(meth, @desc, @long_desc, @usage, method_options)
+        @usage, @desc, @long_desc, @method_options, @hide = nil
+        true
+      elsif self.all_tasks[meth] || meth == "method_missing"
+        true
+      else
+        false
+      end
+    end
+  end
+end
+

metadata

@@ -0,0 +1,197 @@
+--- !ruby/object:Gem::Specification
+name: boardintel_frenzy_bunnies
+version: !ruby/object:Gem::Version
+  version: 0.0.9
+platform: ruby
+authors:
+- Dotan Nahum
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-01-05 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: march_hare
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.3'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sinatra
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: atomic
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-coffeescript
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: RabbitMQ JRuby based workers on top of march_hare
+email:
+- jondotan@gmail.com
+executables:
+- frenzy_bunnies
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- Guardfile
+- LICENSE
+- README.md
+- Rakefile
+- bin/frenzy_bunnies
+- examples/feed.rb
+- examples/feed_worker.rb
+- examples/feed_workers_bin.rb
+- fb-cap.png
+- frenzy_bunnies.gemspec
+- lib/frenzy_bunnies.rb
+- lib/frenzy_bunnies/cli.rb
+- lib/frenzy_bunnies/context.rb
+- lib/frenzy_bunnies/handlers/maxretry.rb
+- lib/frenzy_bunnies/handlers/oneshot.rb
+- lib/frenzy_bunnies/health.rb
+- lib/frenzy_bunnies/health/collector.rb
+- lib/frenzy_bunnies/health/providers/jvm.rb
+- lib/frenzy_bunnies/queue_factory.rb
+- lib/frenzy_bunnies/version.rb
+- lib/frenzy_bunnies/web.rb
+- lib/frenzy_bunnies/web/public/css/bootstrap.min.css
+- lib/frenzy_bunnies/web/public/img/bunny16.png
+- lib/frenzy_bunnies/web/public/img/bunny32.png
+- lib/frenzy_bunnies/web/public/index.html
+- lib/frenzy_bunnies/web/public/js/app.coffee
+- lib/frenzy_bunnies/web/public/js/app.js
+- lib/frenzy_bunnies/web/public/js/backbone-min.js
+- lib/frenzy_bunnies/web/public/js/bootstrap.js
+- lib/frenzy_bunnies/web/public/js/bootstrap.min.js
+- lib/frenzy_bunnies/web/public/js/jquery-1.8.0.min.js
+- lib/frenzy_bunnies/web/public/js/jquery.filesize.js
+- lib/frenzy_bunnies/web/public/js/jquery.timeago.js
+- lib/frenzy_bunnies/web/public/js/underscore-min.js
+- lib/frenzy_bunnies/worker.rb
+- spec/frenzy_bunnies/worker_spec.rb
+- spec/spec_helper.rb
+homepage: ''
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: RabbitMQ JRuby based workers on top of march_hare
+test_files:
+- spec/frenzy_bunnies/worker_spec.rb
+- spec/spec_helper.rb