collective 0.2.0

data/lib/collective/utilities/storage_base.rb

@@ -0,0 +1,41 @@
+# -*- encoding: utf-8 -*-
+
+class Collective::Utilities::StorageBase
+
+  module Resolver
+    # resolution is as so:
+    # nil, :mock => :mock
+    # :redis     => redis://127.0.0.1:6379/1
+    # string     => CLASS.new
+    # CLASS      => CLASS.new
+    # PROC       => yield (recursive)
+    # [ ARRAY ]  => first, *args (recursive)
+    def resolve( storage, *args )
+      storage ||= :mock
+      case
+      when storage.respond_to?(:call)
+        resolve(storage.call(*args))
+      else
+        case storage
+        when :mock
+          resolve( Collective::Mocks::Storage, *args )
+        when :redis
+          resolve( Collective::Redis::Storage, *args )
+        when Class
+          storage.new(*args)
+        when String
+          resolve( Collective.resolve_class(storage), *args )
+        when Array
+          args    = storage.dup + args
+          storage = args.shift
+          resolve( storage, *args )
+        else
+          return storage
+        end
+      end
+    end # resolve
+  end # Resolver
+
+  extend Resolver
+
+end # Collective::Utilities::StorageBase

data/lib/collective/version.rb

@@ -0,0 +1,3 @@
+module Collective
+  VERSION = "0.2.0"
+end

data/lib/collective/worker.rb

@@ -0,0 +1,161 @@
+# -*- encoding: utf-8 -*-
+
+=begin
+
+  A Worker is a forked process which runs jobs.
+  Jobs are short lived and run repeatedly.
+
+=end
+
+class Collective::Worker
+
+  include Collective::Utilities::Observeable
+
+  # forks a new process
+  # creates a new instance of the job class
+  # runs a loop which calls the job
+  def self.spawn( prototype_job, options = {} )
+    policy   = options[:policy] || Collective::Policy.resolve
+    name     = options[:name] || policy.name || prototype_job.to_s
+    storage  = policy.storage
+    registry = options[:registry] || Collective::Registry.new( name, storage )
+
+    foptions = { stdout: "/tmp/debug.log" }
+    Collective::Utilities::Process.fork_and_detach( foptions ) do
+      if after_forks = policy.after_forks then
+        after_forks.each { |af| af.call }
+      end
+      # $0 = "$0 #{name}"
+      worker = new( prototype_job, options )
+      trap("TERM") { worker.quit! }
+      worker.run
+    end
+  end
+
+  attr :job
+  attr :name
+  attr :policy
+  attr :registry
+  attr :state
+  attr :storage
+  attr :worker_expire
+  attr :worker_jobs
+
+  # @param options[:name] is optional
+  # @param options[:policy] is optional
+  # @param options[:registry] is optional
+  def initialize( prototype_job, options = {} )
+    @policy   = options[:policy] || Collective::Policy.resolve
+    @name     = options[:name] || policy.name || prototype_job.to_s
+    @storage  = policy.storage
+    @registry = options[:registry] || Collective::Registry.new( name, storage )
+    @job      = Collective::Idler.new( resolve_job( prototype_job ), min_sleep: policy.worker_idle_min_sleep, max_sleep: policy.worker_idle_max_sleep )
+
+    # type checks
+    policy.pool_min_workers
+    registry.workers
+
+    # post-fork processing
+    storage.reconnect_after_fork
+    registry.reconnect_after_fork
+
+    # set up observers
+    policy.observers.each do |observer|
+      o = Collective::Utilities::ObserverBase.resolve(observer)
+      add_observer(o)
+    end
+
+    # manage the registry via an observer
+    add_observer( Collective::LifecycleObserver.new( key, registry ) )
+  end
+
+  def run()
+    @state         = :running
+    @worker_jobs   = 0
+    @worker_expire = Time.now + policy.worker_max_lifetime
+
+    context = { worker: self }
+    with_start_and_stop do
+      while running? do
+        with_quitting_checks do
+          with_heartbeat do
+            job.call(context)
+          end
+        end
+      end
+    end
+  end
+
+  def quit!()
+    @state = :quitting
+  end
+
+  def running?
+    state == :running
+  end
+
+  def to_s
+    %Q[Worker(#{key})]
+  end
+
+  # the key is a constant string which uniquely identifies this worker
+  # WARNING this would be invalidated if we forked or set this before forking
+  def key
+    @key ||= Collective::Key.new( name, Process.pid )
+  end
+
+  def mq
+    @mq ||= Collective::Messager.new( storage, my_address: key )
+  end
+
+  # ----------------------------------------------------------------------------
+  protected
+  # ----------------------------------------------------------------------------
+
+  def with_start_and_stop(&block)
+    notify :worker_started
+    begin
+      yield
+    ensure
+      notify :worker_stopped
+    end
+  end
+
+  def with_quitting_checks(&block)
+    yield
+  ensure
+    @worker_jobs += 1
+    quit! if policy.worker_max_jobs <= worker_jobs
+    quit! if worker_expire <= Time.now
+  end
+
+  def with_heartbeat(&block)
+    begin
+      yield
+    rescue => x
+      notify :job_error, x
+    ensure
+      notify :worker_heartbeat
+    end
+  end
+
+  def resolve_job( job_factory )
+    raise Collective::ConfigurationError if ! job_factory
+
+    case
+    when job_factory.respond_to?(:call)
+      job_factory
+    when job_factory.respond_to?(:new)
+      context = { worker: self }
+      resolve_job(job_factory.new(context))
+    else
+      case job_factory
+      when String, Symbol
+        resolve_job(Collective.resolve_class(job_factory.to_s))
+      else
+        raise Collective::ConfigurationError, "Unknown kind of job #{job_factory.inspect}"
+      end
+    end
+  end
+
+end # Collective::Worker

data/spec/checker_spec.rb

@@ -0,0 +1,20 @@
+# -*- encoding: utf-8 -*-
+
+require "helper"
+require "collective/checker"
+
+describe Checker do
+
+  it "does something" do
+    activity_count = 192
+    last_time      = Time.now - 3600
+    c = Checker.new activity_count, last_time
+
+    c.check?.should_not be_false
+    c.estimated_delay.should be >= 1.0
+    c.estimated_delay.should be < 8.0
+
+    c.checked 12
+  end
+
+end

data/spec/configuration_spec.rb

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+
+require "helper"
+
+describe Collective::Configuration do
+
+  it "should parse command-line switches" do
+    c = Collective::Configuration.parse %w(--dry-run --env the_env --name a_name --chdir .)
+    c.env.should eq("the_env")
+    c.name.should eq("a_name")
+  end
+
+  it "should parse the DSL" do
+    script = <<-EOT.gsub(/^ +/,'')
+      set_env  "the_env"
+      set_name "a_name"
+      chdir    "."
+    EOT
+    c = Collective::Configuration.parse ["--dry-run", "--script", script]
+    c.env.should eq("the_env")
+    c.name.should eq("a_name")
+  end
+
+end

data/spec/helper.rb

@@ -0,0 +1,33 @@
+# -*- encoding: utf-8 -*-
+
+=begin
+@see https://www.relishapp.com/rspec/rspec-core/docs/hooks/around-hooks
+@see https://www.relishapp.com/rspec/rspec-expectations/docs/built-in-matchers/be-matchers
+@see https://www.relishapp.com/rspec/rspec-expectations/docs/built-in-matchers/equality-matchers
+@see https://www.relishapp.com/rspec/rspec-expectations/docs/built-in-matchers/operator-matchers
+@see https://www.relishapp.com/rspec/rspec-expectations/docs/built-in-matchers/predicate-matchers
+=end
+
+require "bundler/setup"       # set up gem paths
+require "ruby-debug"          # because sometimes you need it
+
+#require "simplecov"           # code coverage
+#SimpleCov.start               # must be loaded before our own code
+
+require "collective"                # load this gem
+require "support/jobs"        # simple helpers for testing
+require "support/redis"       # simple helpers for testing
+require "support/timing"      # simple helpers for testing
+
+RSpec.configure do |spec|
+  # @see https://www.relishapp.com/rspec/rspec-core/docs/helper-methods/define-helper-methods-in-a-module
+  spec.include RedisClient, redis: true
+  spec.include Timing, time: true
+  spec.include Collective::Idler::Utilities
+
+  # nuke the Redis database around each run
+  # @see https://www.relishapp.com/rspec/rspec-core/docs/hooks/around-hooks
+  spec.around( :each, redis: true ) do |example|
+    with_clean_redis { example.run }
+  end
+end

data/spec/idler_spec.rb

@@ -0,0 +1,58 @@
+# -*- encoding: utf-8 -*-
+
+require "helper"
+
+describe Collective::Idler do
+
+  it "should accept a proc" do
+    idler = Collective::Idler.new() { true }
+    expect { idler.call }.to_not raise_error
+  end
+
+  it "should accept a lambda" do
+    job = ->() { true }
+    idler = Collective::Idler.new(job)
+    expect { idler.call }.to_not raise_error
+  end
+
+  it "should accept an object with an interface" do
+    idler = Collective::Idler.new( TrueJob.new )
+    expect { idler.call }.to_not raise_error
+  end
+
+  it "should refuse non-callable jobs" do
+    fake_job = Object.new
+    expect { Collective::Idler.new(fake_job) }.to raise_error
+  end
+
+  describe "when dealing with sleep times", time: true do
+
+    it "should not run idle tasks too much" do
+      count  = 0
+      Collective::Idler.wait_until { count += 1; false }
+      count.should be <= 10
+    end
+
+    it "should sleep after a failure" do
+      job = ->() { false }
+      idler = Collective::Idler.new( job, min_sleep: 0.125 )
+      time { idler.call }.should be >= 0.125
+    end
+
+    it "should not sleep after a success" do
+      result = false
+      job    = ->() { result }
+      idler  = Collective::Idler.new( job, min_sleep: 0.125 )
+      idler.call
+      time { result = true; idler.call }.should be < 0.1
+    end
+
+    it "should not sleep too long" do
+      job    = ->() { false }
+      idler  = Collective::Idler.new( job, min_sleep: 0.125 )
+      time { idler.call }.should be < 0.25
+    end
+
+  end
+
+end

data/spec/key_spec.rb

@@ -0,0 +1,41 @@
+# -*- encoding: utf-8 -*-
+
+require "helper"
+
+describe Collective::Key do
+
+  it "can make keys" do
+    name = "processor"
+    pid  = 1234
+    host = "foo.example.com"
+    Collective::Key.new( name, pid, host ).to_s.should eq("processor-1234@foo.example.com")
+  end
+
+  it "can parse keys" do
+    Collective::Key.parse("processor-1234@foo.example.com").should eq(Collective::Key.new( "processor", "1234", "foo.example.com" ))
+    Collective::Key.parse("test-job-1234@foo.example.com").should eq(Collective::Key.new( "test-job", "1234", "foo.example.com" ))
+  end
+
+  it "can compare to another key" do
+    key1 = Collective::Key.new "Alpha", 1234, "example.com"
+    key2 = Collective::Key.new "Alpha", 1234, "example.com"
+    key1.should eq(key2)
+
+    key1 = Collective::Key.new "Alpha", 1234, "example.com"
+    key2 = Collective::Key.new "Alpha", "1234", "example.com"
+    key1.should eq(key2)
+
+    key1 = Collective::Key.new "Alpha", 1234, "example.com"
+    key2 = Collective::Key.new "Beta", 1234, "example.com"
+    key1.should_not eq(key2)
+
+    key1 = Collective::Key.new "Alpha", 1234, "example.com"
+    key2 = Collective::Key.new "Alpha", 2345, "example.com"
+    key1.should_not eq(key2)
+
+    key1 = Collective::Key.new "Alpha", 1234, "example.com"
+    key2 = Collective::Key.new "Alpha", 1234, "example.org"
+    key1.should_not eq(key2)
+  end
+
+end

data/spec/messager_spec.rb

@@ -0,0 +1,131 @@
+# -*- encoding: utf-8 -*-
+
+require "helper"
+
+describe Collective::Messager, redis: true do
+
+  before do
+    @a = "me@example.com"
+    @b = "you@example.com"
+  end
+
+  it "can take to_address initially or per message" do
+    storage = Collective::Mocks::Storage.new
+
+    a = Collective::Messager.new( storage, my_address: @a )
+    expect { a.send "Hello" }.to raise_exception
+    expect { a.send "Hello", to: @b }.to_not raise_exception
+
+    b = Collective::Messager.new( storage, my_address: @a, to_address: @b )
+    expect { b.send "Hello" }.to_not raise_exception
+  end
+
+  it "the message id varies with the source, content and timestamp" do
+    storage = Collective::Mocks::Storage.new
+    a       = Collective::Messager.new( storage, my_address: @a, to_address: @b )
+    b       = Collective::Messager.new( storage, my_address: @b, to_address: @a )
+    now     = 1234567890
+
+    id1     = a.send "Hello", at: now
+    id3     = a.send "Hello", at: now+1
+    id4     = a.send "Goodbye", at: now
+    id5     = b.send "Hello", at: now
+
+    id3.should_not eq(id1)
+    id4.should_not eq(id1)
+    id5.should_not eq(id1)
+  end
+
+  describe "sending and receiving messages" do
+
+    it "can match against a string" do
+      storage = Collective::Mocks::Storage.new
+      a = Collective::Messager.new( storage, my_address: @a )
+      b = Collective::Messager.new( storage, my_address: @b )
+      b.expect("Hello") { |message| false }
+      a.send "Hello", to: @b
+      expect { b.receive }.to_not raise_exception
+    end
+
+    it "can match against a regexp" do
+      storage = Collective::Mocks::Storage.new
+      a = Collective::Messager.new( storage, my_address: @a )
+      b = Collective::Messager.new( storage, my_address: @b )
+      b.expect(/ello/) { |message| false }
+      a.send "Hello", to: @b
+      expect { b.receive }.to_not raise_exception
+    end
+
+    it "returns true if it got a message" do
+      storage = Collective::Mocks::Storage.new
+      a = Collective::Messager.new( storage, my_address: @a )
+      b = Collective::Messager.new( storage, my_address: @b )
+      b.expect(//) { |message| false }
+      a.receive.should eq(false)
+      a.send "Hello", to: @b
+      b.receive.should eq(true)
+      b.receive.should eq(false)
+    end
+
+    it "can send and receive messages" do
+      storage = Collective::Mocks::Storage.new
+      a = Collective::Messager.new( storage, my_address: @a )
+      b = Collective::Messager.new( storage, my_address: @b )
+
+      callback = double("callback")
+      callback.should_receive(:call).with(anything)
+
+      reply_to_id = nil
+
+      b.expect("Hello") { |message|
+        message.body.should eq("Hello")
+        message.from.should eq(@a)
+        reply_to_id = message.id
+        b.reply( "Goodbye", to: message )
+      }
+
+      a.expect("Goodbye") { |message|
+        message.body.should eq("Goodbye")
+        message.from.should eq(@b)
+        message.id.should_not be_nil
+        message.reply_to_id.should eq(reply_to_id)
+        callback.call(message)
+      }
+
+      a.send "Hello", to: @b
+      b.receive
+      a.receive
+    end
+
+  end
+
+  describe "when working with multiple processes", redis: true do
+
+    it "can send a message between processes" do
+      storage = Collective::Redis::Storage.new(redis)
+      me = Collective::Messager.new( storage, my_address: @a )
+
+      ok = false
+      me.expect("Goodbye") do |message|
+        ok = true
+      end
+
+      Collective::Utilities::Process.fork_and_detach do
+        redis.client.disconnect
+        me = Collective::Messager.new( storage, my_address: @b )
+        ok = false
+        me.expect("Hello") do |message|
+          me.reply "Goodbye", to: message
+          ok = true
+        end
+        wait_until { me.receive; ok }
+      end
+
+      me.send "Hello", to: @b
+      wait_until { me.receive }
+      ok.should be_true
+    end
+
+  end
+
+end