mlanett-hive 0.3.0

data/lib/hive/utilities/storage_base.rb

@@ -0,0 +1,41 @@
+# -*- encoding: utf-8 -*-
+
+class Hive::Utilities::StorageBase
+
+  module Resolver
+    # resolution is as so:
+    # nil, :mock => :mock
+    # :redis     => redis://127.0.0.1:6379/1
+    # string     => CLASS.new
+    # CLASS      => CLASS.new
+    # PROC       => yield (recursive)
+    # [ ARRAY ]  => first, *args (recursive)
+    def resolve( storage, *args )
+      storage ||= :mock
+      case
+      when storage.respond_to?(:call)
+        resolve(storage.call(*args))
+      else
+        case storage
+        when :mock
+          resolve( Hive::Mocks::Storage, *args )
+        when :redis
+          resolve( Hive::Redis::Storage, *args )
+        when Class
+          storage.new(*args)
+        when String
+          resolve( Hive::Utilities::Resolver.resolve_class(storage), *args )
+        when Array
+          args    = storage.dup + args
+          storage = args.shift
+          resolve( storage, *args )
+        else
+          return storage
+        end
+      end
+    end # resolve
+  end # Resolver
+
+  extend Resolver
+
+end # Hive::Utilities::StorageBase

data/lib/hive/version.rb

@@ -0,0 +1,3 @@
+module Hive
+  VERSION = "0.3.0"
+end

data/lib/hive/worker.rb

@@ -0,0 +1,162 @@
+# -*- encoding: utf-8 -*-
+
+=begin
+
+  A Worker is a forked process which runs jobs.
+  Jobs are short lived and run repeatedly.
+
+=end
+
+class Hive::Worker
+
+  include Hive::Utilities::Observeable
+
+  # forks a new process
+  # creates a new instance of the job class
+  # runs a loop which calls the job
+  def self.spawn( prototype_job, options = {} )
+    policy   = options[:policy] || Hive::Policy.resolve
+    name     = options[:name] || policy.name || prototype_job.to_s
+    storage  = policy.storage
+    registry = options[:registry] || Hive::Registry.new( name, storage )
+
+    foptions = { stdout: "/tmp/debug.log" }
+    policy.before_fork
+    Hive::Utilities::Process.fork_and_detach( foptions ) do
+      policy.after_fork
+      # $0 = "$0 #{name}"
+      worker = new( prototype_job, options )
+      trap("TERM") { worker.quit! }
+      worker.run
+    end
+  end
+
+  attr :job
+  attr :name
+  attr :policy
+  attr :registry
+  attr :state
+  attr :storage
+  attr :worker_expire
+  attr :worker_jobs
+
+  # @param options[:name] is optional
+  # @param options[:policy] is optional
+  # @param options[:registry] is optional
+  def initialize( prototype_job, options = {} )
+    @policy   = options[:policy] || Hive::Policy.resolve
+    @name     = options[:name] || policy.name || prototype_job.to_s
+    @storage  = policy.storage
+    @registry = options[:registry] || Hive::Registry.new( name, storage )
+    @job      = Hive::Idler.new( resolve_job( prototype_job ), min_sleep: policy.worker_idle_min_sleep, max_sleep: policy.worker_idle_max_sleep )
+
+    # type checks
+    policy.pool_min_workers
+    registry.workers
+
+    # post-fork processing
+    storage.reconnect_after_fork
+    registry.reconnect_after_fork
+
+    # set up observers
+    policy.observers.each do |observer|
+      o = Hive::Utilities::ObserverBase.resolve(observer)
+      add_observer(o)
+    end
+
+    # manage the registry via an observer
+    add_observer( Hive::LifecycleObserver.new( key, registry ) )
+  end
+
+  def run()
+    @state         = :running
+    @worker_jobs   = 0
+    @worker_expire = Time.now + policy.worker_max_lifetime
+
+    context = { worker: self }
+    with_start_and_stop do
+      while running? do
+        with_quitting_checks do
+          with_heartbeat do
+            job.call(context)
+          end
+        end
+      end
+    end
+  end
+
+  def quit!()
+    @state = :quitting
+  end
+
+  def running?
+    state == :running
+  end
+
+  def to_s
+    %Q[Worker(#{key})]
+  end
+
+  # the key is a constant string which uniquely identifies this worker
+  # WARNING this would be invalidated if we forked or set this before forking
+  def key
+    @key ||= Hive::Key.new( name, Process.pid )
+  end
+
+  def mq
+    @mq ||= Hive::Messager.new( storage, my_address: key )
+  end
+
+  # ----------------------------------------------------------------------------
+  protected
+  # ----------------------------------------------------------------------------
+
+  def with_start_and_stop(&block)
+    notify :worker_started
+    begin
+      yield
+    ensure
+      notify :worker_stopped
+    end
+  end
+
+  def with_quitting_checks(&block)
+    yield
+  ensure
+    @worker_jobs += 1
+    quit! if policy.worker_max_jobs <= worker_jobs
+    quit! if worker_expire <= Time.now
+  end
+
+  def with_heartbeat(&block)
+    begin
+      yield
+    rescue => x
+      notify :job_error, x
+    ensure
+      notify :worker_heartbeat
+    end
+  end
+
+  def resolve_job( job_factory )
+    raise Hive::ConfigurationError if ! job_factory
+
+    case
+    when job_factory.respond_to?(:call)
+      # A job factory can not be a proc, because a job itself is a proc; we would call it.
+      # Once we've found a proc, we have the result.
+      job_factory
+    when job_factory.respond_to?(:new)
+      context = { worker: self }
+      resolve_job(job_factory.new(context))
+    else
+      case job_factory
+      when String, Symbol
+        resolve_job(Hive::Utilities::Resolver.resolve_class(job_factory.to_s))
+      else
+        raise Hive::ConfigurationError, "Unknown kind of job #{job_factory.inspect}"
+      end
+    end
+  end
+
+end # Hive::Worker

data/spec/checker_spec.rb

@@ -0,0 +1,20 @@
+# -*- encoding: utf-8 -*-
+
+require "helper"
+require "hive/checker"
+
+describe Checker do
+
+  it "does something" do
+    activity_count = 192
+    last_time      = Time.now - 3600
+    c = Checker.new activity_count, last_time
+
+    c.check?.should_not be_false
+    c.estimated_delay.should be >= 1.0
+    c.estimated_delay.should be < 8.0
+
+    c.checked 12
+  end
+
+end

data/spec/configuration_spec.rb

@@ -0,0 +1,50 @@
+# -*- encoding: utf-8 -*-
+
+require "helper"
+
+describe Hive::Configuration do
+
+  it "should parse command-line switches" do
+    c = Hive::Configuration.parse %w(--dry-run --env the_env --name a_name --chdir .)
+    c.env.should eq("the_env")
+    c.name.should eq("a_name")
+  end
+
+  it "should parse the DSL" do
+    script = <<-EOT.gsub(/^ +/,'')
+      set_env  "the_env"
+      set_name "a_name"
+      chdir    "."
+      before_fork() { true }
+      after_fork() { true }
+    EOT
+    c = Hive::Configuration.parse ["--dry-run", "--script", script]
+    c.env.should eq("the_env")
+    c.name.should eq("a_name")
+    c.before_forks.size.should eq(1)
+    c.after_forks.size.should eq(1)
+  end
+
+  it "enumerates pool names and policies" do
+    script = <<-EOT.gsub(/^ +/,'')
+      add_pool "Test", pool_max_workers: 1
+    EOT
+    c = Hive::Configuration.parse ["--dry-run", "--script", script]
+    c.policies.first.first.should eq("Test")
+    c.policies.first.last.pool_max_workers.should eq(1)
+  end
+
+  it "adds blocks to pools" do
+    script = <<-EOT.gsub(/^ +/,'')
+      before_fork() { true }
+      after_fork() { false }
+      add_pool "Test"
+    EOT
+    c = Hive::Configuration.parse ["--dry-run", "--script", script]
+    c.policies.to_a.size.should eq(1)
+    pool = c.policies.to_a.first.last
+    pool.before_forks.first.call.should eq(true)
+    pool.after_forks.first.call.should eq(false)
+  end
+
+end

data/spec/helper.rb

@@ -0,0 +1,33 @@
+# -*- encoding: utf-8 -*-
+
+=begin
+@see https://www.relishapp.com/rspec/rspec-core/docs/hooks/around-hooks
+@see https://www.relishapp.com/rspec/rspec-expectations/docs/built-in-matchers/be-matchers
+@see https://www.relishapp.com/rspec/rspec-expectations/docs/built-in-matchers/equality-matchers
+@see https://www.relishapp.com/rspec/rspec-expectations/docs/built-in-matchers/operator-matchers
+@see https://www.relishapp.com/rspec/rspec-expectations/docs/built-in-matchers/predicate-matchers
+=end
+
+require "bundler/setup"       # set up gem paths
+#require "ruby-debug"          # because sometimes you need it
+
+#require "simplecov"           # code coverage
+#SimpleCov.start               # must be loaded before our own code
+
+require "hive"                # load this gem
+require "support/jobs"        # simple helpers for testing
+require "support/redis"       # simple helpers for testing
+require "support/timing"      # simple helpers for testing
+
+RSpec.configure do |spec|
+  # @see https://www.relishapp.com/rspec/rspec-core/docs/helper-methods/define-helper-methods-in-a-module
+  spec.include RedisClient, redis: true
+  spec.include Timing, time: true
+  spec.include Hive::Idler::Utilities
+
+  # nuke the Redis database around each run
+  # @see https://www.relishapp.com/rspec/rspec-core/docs/hooks/around-hooks
+  spec.around( :each, redis: true ) do |example|
+    with_clean_redis { example.run }
+  end
+end

data/spec/idler_spec.rb

@@ -0,0 +1,58 @@
+# -*- encoding: utf-8 -*-
+
+require "helper"
+
+describe Hive::Idler do
+
+  it "should accept a proc" do
+    idler = Hive::Idler.new() { true }
+    expect { idler.call }.to_not raise_error
+  end
+
+  it "should accept a lambda" do
+    job = ->() { true }
+    idler = Hive::Idler.new(job)
+    expect { idler.call }.to_not raise_error
+  end
+
+  it "should accept an object with an interface" do
+    idler = Hive::Idler.new( TrueJob.new )
+    expect { idler.call }.to_not raise_error
+  end
+
+  it "should refuse non-callable jobs" do
+    fake_job = Object.new
+    expect { Hive::Idler.new(fake_job) }.to raise_error
+  end
+
+  describe "when dealing with sleep times", time: true do
+
+    it "should not run idle tasks too much" do
+      count  = 0
+      Hive::Idler.wait_until { count += 1; false }
+      count.should be <= 10
+    end
+
+    it "should sleep after a failure" do
+      job = ->() { false }
+      idler = Hive::Idler.new( job, min_sleep: 0.125 )
+      time { idler.call }.should be >= 0.125
+    end
+
+    it "should not sleep after a success" do
+      result = false
+      job    = ->() { result }
+      idler  = Hive::Idler.new( job, min_sleep: 0.125 )
+      idler.call
+      time { result = true; idler.call }.should be < 0.1
+    end
+
+    it "should not sleep too long" do
+      job    = ->() { false }
+      idler  = Hive::Idler.new( job, min_sleep: 0.125 )
+      time { idler.call }.should be < 0.25
+    end
+
+  end
+
+end

data/spec/key_spec.rb

@@ -0,0 +1,41 @@
+# -*- encoding: utf-8 -*-
+
+require "helper"
+
+describe Hive::Key do
+
+  it "can make keys" do
+    name = "processor"
+    pid  = 1234
+    host = "foo.example.com"
+    Hive::Key.new( name, pid, host ).to_s.should eq("processor-1234@foo.example.com")
+  end
+
+  it "can parse keys" do
+    Hive::Key.parse("processor-1234@foo.example.com").should eq(Hive::Key.new( "processor", "1234", "foo.example.com" ))
+    Hive::Key.parse("test-job-1234@foo.example.com").should eq(Hive::Key.new( "test-job", "1234", "foo.example.com" ))
+  end
+
+  it "can compare to another key" do
+    key1 = Hive::Key.new "Alpha", 1234, "example.com"
+    key2 = Hive::Key.new "Alpha", 1234, "example.com"
+    key1.should eq(key2)
+
+    key1 = Hive::Key.new "Alpha", 1234, "example.com"
+    key2 = Hive::Key.new "Alpha", "1234", "example.com"
+    key1.should eq(key2)
+
+    key1 = Hive::Key.new "Alpha", 1234, "example.com"
+    key2 = Hive::Key.new "Beta", 1234, "example.com"
+    key1.should_not eq(key2)
+
+    key1 = Hive::Key.new "Alpha", 1234, "example.com"
+    key2 = Hive::Key.new "Alpha", 2345, "example.com"
+    key1.should_not eq(key2)
+
+    key1 = Hive::Key.new "Alpha", 1234, "example.com"
+    key2 = Hive::Key.new "Alpha", 1234, "example.org"
+    key1.should_not eq(key2)
+  end
+
+end

data/spec/messager_spec.rb

@@ -0,0 +1,131 @@
+# -*- encoding: utf-8 -*-
+
+require "helper"
+
+describe Hive::Messager, redis: true do
+
+  before do
+    @a = "me@example.com"
+    @b = "you@example.com"
+  end
+
+  it "can take to_address initially or per message" do
+    storage = Hive::Mocks::Storage.new
+
+    a = Hive::Messager.new( storage, my_address: @a )
+    expect { a.send "Hello" }.to raise_exception
+    expect { a.send "Hello", to: @b }.to_not raise_exception
+
+    b = Hive::Messager.new( storage, my_address: @a, to_address: @b )
+    expect { b.send "Hello" }.to_not raise_exception
+  end
+
+  it "the message id varies with the source, content and timestamp" do
+    storage = Hive::Mocks::Storage.new
+    a       = Hive::Messager.new( storage, my_address: @a, to_address: @b )
+    b       = Hive::Messager.new( storage, my_address: @b, to_address: @a )
+    now     = 1234567890
+
+    id1     = a.send "Hello", at: now
+    id3     = a.send "Hello", at: now+1
+    id4     = a.send "Goodbye", at: now
+    id5     = b.send "Hello", at: now
+
+    id3.should_not eq(id1)
+    id4.should_not eq(id1)
+    id5.should_not eq(id1)
+  end
+
+  describe "sending and receiving messages" do
+
+    it "can match against a string" do
+      storage = Hive::Mocks::Storage.new
+      a = Hive::Messager.new( storage, my_address: @a )
+      b = Hive::Messager.new( storage, my_address: @b )
+      b.expect("Hello") { |message| false }
+      a.send "Hello", to: @b
+      expect { b.receive }.to_not raise_exception
+    end
+
+    it "can match against a regexp" do
+      storage = Hive::Mocks::Storage.new
+      a = Hive::Messager.new( storage, my_address: @a )
+      b = Hive::Messager.new( storage, my_address: @b )
+      b.expect(/ello/) { |message| false }
+      a.send "Hello", to: @b
+      expect { b.receive }.to_not raise_exception
+    end
+
+    it "returns true if it got a message" do
+      storage = Hive::Mocks::Storage.new
+      a = Hive::Messager.new( storage, my_address: @a )
+      b = Hive::Messager.new( storage, my_address: @b )
+      b.expect(//) { |message| false }
+      a.receive.should eq(false)
+      a.send "Hello", to: @b
+      b.receive.should eq(true)
+      b.receive.should eq(false)
+    end
+
+    it "can send and receive messages" do
+      storage = Hive::Mocks::Storage.new
+      a = Hive::Messager.new( storage, my_address: @a )
+      b = Hive::Messager.new( storage, my_address: @b )
+
+      callback = double("callback")
+      callback.should_receive(:call).with(anything)
+
+      reply_to_id = nil
+
+      b.expect("Hello") { |message|
+        message.body.should eq("Hello")
+        message.from.should eq(@a)
+        reply_to_id = message.id
+        b.reply( "Goodbye", to: message )
+      }
+
+      a.expect("Goodbye") { |message|
+        message.body.should eq("Goodbye")
+        message.from.should eq(@b)
+        message.id.should_not be_nil
+        message.reply_to_id.should eq(reply_to_id)
+        callback.call(message)
+      }
+
+      a.send "Hello", to: @b
+      b.receive
+      a.receive
+    end
+
+  end
+
+  describe "when working with multiple processes", redis: true do
+
+    it "can send a message between processes" do
+      storage = Hive::Redis::Storage.new(redis)
+      me = Hive::Messager.new( storage, my_address: @a )
+
+      ok = false
+      me.expect("Goodbye") do |message|
+        ok = true
+      end
+
+      Hive::Utilities::Process.fork_and_detach do
+        redis.client.disconnect
+        me = Hive::Messager.new( storage, my_address: @b )
+        ok = false
+        me.expect("Hello") do |message|
+          me.reply "Goodbye", to: message
+          ok = true
+        end
+        wait_until { me.receive; ok }
+      end
+
+      me.send "Hello", to: @b
+      wait_until { me.receive }
+      ok.should be_true
+    end
+
+  end
+
+end