collective 0.2.0

data/spec/support/jobs.rb

@@ -0,0 +1,58 @@
+# -*- encoding: utf-8 -*-
+
+require "support/redis"
+
+class QuitJob
+  def call( context = {} )
+    context[:worker].quit!
+  end
+end
+
+class TrueJob
+  def call
+    true
+  end
+end
+
+class QuitJobWithSet
+  include RedisClient
+  def call( context = {} )
+    redis.set("QuitJobWithSet",Process.pid)
+    context[:worker].quit!
+  end
+end
+
+class ForeverJobWithSet
+  include RedisClient
+  def call( context = {} )
+    redis.set("ForeverJobWithSet",Process.pid)
+    false
+  end
+end
+
+class ForeverUntilQuitJob
+  include RedisClient
+  def call( context = {} )
+    if redis.get("ForeverUntilQuitJob") then
+      context[:worker].quit!
+    else
+      false
+    end
+  end
+end
+
+class ListenerJob
+  include RedisClient
+
+  def initialize( context = {} )
+    storage = Collective::Redis::Storage.new(redis)
+    @mq = Collective::Messager.new( storage, my_address: context[:worker].key )
+    @mq.expect("Quit")   { |message| context[:worker].quit! }
+    @mq.expect("Exit!")  { |message| Kernel.exit! }
+    @mq.expect("State?") { |message| @mq.reply "State: #{context[:worker].state}", to: message }
+  end
+
+  def call( context = {} )
+    @mq.receive
+  end
+end

data/spec/support/redis.rb

@@ -0,0 +1,22 @@
+# -*- encoding: utf-8 -*-
+
+require "redis"
+
+module RedisClient
+
+  TEST_REDIS = { url: "redis://127.0.0.1:6379/1" }
+
+  def redis
+    @redis ||= ::Redis.connect(TEST_REDIS)
+  end
+
+  def with_clean_redis(&block)
+    redis.client.disconnect # auto connect after fork
+    redis.flushall
+    yield
+  ensure
+    redis.flushall
+    redis.quit              # quit (close) connection, not server
+  end
+
+end # RedisClient

data/spec/support/timing.rb

@@ -0,0 +1,32 @@
+# -*- encoding: utf-8 -*-
+
+module Timing
+
+  def time(&block)
+    _time(&block)
+    elapsed
+  end
+
+  protected
+
+  def start
+    @start = Time.now.to_f
+  end
+
+  def finish
+    @finish = Time.now.to_f
+  end
+
+  def elapsed
+    @finish - @start
+  end
+
+  def _time(&block)
+    # elapsed time should be known whether or not it raises an error
+    start
+    yield
+  ensure
+    finish
+  end
+
+end # Timing

data/spec/utilities/observer_base_spec.rb

@@ -0,0 +1,50 @@
+# -*- encoding: utf-8 -*-
+
+require "helper"
+require "collective"
+
+class TestObserver < Collective::Utilities::ObserverBase
+  attr :alpha
+  attr :beta
+  def initialize( alpha = 1, beta = 2 )
+    @alpha = alpha
+    @beta = beta
+  end
+end
+
+describe Collective::Utilities::ObserverBase do
+
+  it "can instaniate from a class name" do
+    o = Collective::Utilities::ObserverBase.resolve TestObserver
+    o.should be_instance_of TestObserver
+    o.alpha.should eq(1)
+    o.beta.should eq(2)
+  end
+
+  it "can instaniate from a string" do
+    o = Collective::Utilities::ObserverBase.resolve "TestObserver"
+    o.should be_instance_of TestObserver
+    o.alpha.should eq(1)
+    o.beta.should eq(2)
+  end
+
+  it "can instaniate from a symbol" do
+    o = Collective::Utilities::ObserverBase.resolve :log
+    o.should be_instance_of Collective::Utilities::LogObserver
+  end
+
+  it "can instaniate from a block" do
+    o = Collective::Utilities::ObserverBase.resolve (->() { TestObserver.new })
+    o.should be_instance_of TestObserver
+    o.alpha.should eq(1)
+    o.beta.should eq(2)
+  end
+
+  it "can instaniate from an array" do
+    o = Collective::Utilities::ObserverBase.resolve [ TestObserver, 2, 4 ]
+    o.should be_instance_of TestObserver
+    o.alpha.should eq(2)
+    o.beta.should eq(4)
+  end
+  
+end

data/spec/utilities/process_spec.rb

@@ -0,0 +1,17 @@
+# -*- encoding: utf-8 -*-
+
+require "helper"
+require "collective"
+
+describe Collective::Utilities::Process do
+  it "does not fail like Process.wait2" do
+    system("false")
+    pid = $?.pid
+    expect { ::Process.wait2( pid ) }.to raise_exception
+  end
+  it "can handle dead processes" do
+    system("false")
+    pid = $?.pid
+    expect { Collective::Utilities::Process.wait_and_terminate(pid) }.to_not raise_exception
+  end
+end

data/spec/worker_spec.rb

@@ -0,0 +1,121 @@
+# -*- encoding: utf-8 -*-
+
+require "helper"
+
+describe Collective::Worker do
+
+  it "should run once" do
+    count  = 0
+    worker = nil
+    job    = ->(context={}) { count += 1; worker.quit! }
+    worker = Collective::Worker.new( job )
+    worker.run
+    count.should eq 1
+  end
+
+  it "should run with a classname" do
+    worker = Collective::Worker.new("QuitJob")
+    expect { worker.run }.should_not raise_error
+  end
+
+  it "should run with a class" do
+    worker = Collective::Worker.new(QuitJob)
+    expect { worker.run }.should_not raise_error
+  end
+
+  it "should run with a lambda" do
+    job    = ->(context) { context[:worker].quit! }
+    worker = Collective::Worker.new( job )
+    expect { worker.run }.should_not raise_error
+  end
+
+  it "should pass a context with a worker" do
+    ok     = false
+    worker = nil
+    job    = ->(context) { worker.should eq context[:worker]; worker.quit! }
+    worker = Collective::Worker.new( job )
+    worker.run
+  end
+
+  it "should use observers" do
+    obsrvr  = Collective::Utilities::ObserverBase.new
+    obsrvr.should_receive(:notify).with(anything,:worker_started).ordered
+    obsrvr.should_receive(:notify).with(anything,:worker_heartbeat).ordered
+    obsrvr.should_receive(:notify).with(anything,:worker_stopped).ordered
+
+    job     = ->(context) { context[:worker].quit! }
+    policy  = Collective::Policy.resolve({ observers: [ obsrvr ] })
+    worker  = Collective::Worker.new job, policy: policy
+
+    worker.run
+  end
+
+  it "should exit when the policy says to run out (of jobs)" do
+    count  = 0
+    job    = ->(context) { count += 1; true }
+    policy = Collective::Policy.resolve({ worker_max_jobs: 5 })
+    worker = Collective::Worker.new job, policy: policy
+    worker.run
+    count.should be <= 5
+  end
+
+  it "should execute after_fork blocks"
+
+  describe "when testing lifetime", time: true do
+    it "should exit when the policy says to run out (of time)" do
+      overhead = 1
+      worker_max_lifetime = 2
+      count    = 0
+      job      = ->(context) { count += 1; true }
+      policy   = Collective::Policy.resolve worker_max_lifetime: worker_max_lifetime, worker_max_jobs: 1e9
+      worker   = Collective::Worker.new job, policy: policy
+      time { worker.run }
+      elapsed.should be <= worker_max_lifetime + overhead
+    end
+  end
+
+  describe "when spawning a process", redis: true do
+
+    before do
+      @policy = Collective::Policy.resolve worker_max_lifetime: 4, worker_max_jobs: 100, storage: :redis
+    end
+
+    it "should spawn a new process" do
+      Collective::Worker.spawn( QuitJobWithSet )
+      wait_until { redis.get("QuitJobWithSet").to_i > 0 }
+      redis.get("QuitJobWithSet").to_i.should be > 0
+    end
+
+    it "should respond to TERM" do
+      Collective::Worker.spawn( ForeverJobWithSet )
+
+      wait_until { redis.get("ForeverJobWithSet").to_i != 0 }
+      pid = redis.get("ForeverJobWithSet").to_i
+      Collective::Utilities::Process.alive?(pid).should be_true
+
+      Process.kill( "TERM", pid )
+      wait_until { ! Collective::Utilities::Process.alive?(pid) }
+      Collective::Utilities::Process.alive?(pid).should be_false
+    end
+
+    it "uses the registry" do
+      job       = ForeverUntilQuitJob
+      registry  = Collective::Registry.new( job.to_s, @policy.storage )
+      registry.workers.size.should eq(0)
+
+      Collective::Worker.spawn ForeverUntilQuitJob, registry: registry, policy: @policy
+
+      wait_until { registry.workers.size > 0 }
+      registry.workers.size.should eq(1)
+
+      key = registry.workers.first
+      key.name.should eq(job.to_s)
+
+      redis.set("ForeverUntilQuitJob",true)
+      wait_until { registry.workers.size == 0 }
+      registry.workers.size.should eq(0)
+    end
+
+  end
+
+end

data/unused/times.rb

@@ -0,0 +1,45 @@
+start, time
+stop,  time
+start, time
+stop,  time
+start, time
+stop,  time
+
+def generate
+  now = Time.now.to_i
+  10.times do
+    puts -now
+    now += rand(7)
+    puts now
+    now += rand(3)
+  end
+end
+
+times = [-1318798764,1318798766,-1318798768,1318798768,-1318798768,1318798772,-1318798773,1318798776,-1318798777,1318798781,-1318798781,1318798781,-1318798783,1318798784,-1318798786,1318798792,-1318798793,1318798796,-1318798796,1318798800]
+
+def analyze( start, times )
+  whats = []
+  whens = []
+  last  = nil
+  ( times + [Time.now.to_i] ).each do |time|
+    if time < 0 then
+      if last && whats[last] == :start then
+        whats << :stop
+        whens << -time
+      end
+      whats << :start
+      whens << -time
+    else
+      if last && whats[last] == :stop then
+        whats << :start
+        whens << time
+      end
+      whats << :stop
+      whens << time
+    end
+    last = (last || -1) + 1
+  end
+  [ whats, whens ]
+end
+
+analyze(times)

metadata

@@ -0,0 +1,148 @@
+--- !ruby/object:Gem::Specification
+name: collective
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+  prerelease: !!null 
+platform: ruby
+authors:
+- Mark Lanett
+autorequire: !!null 
+bindir: bin
+cert_chain: []
+date: 2011-12-26 00:00:00.000000000 -08:00
+default_executable: !!null 
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: &70125036305720 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70125036305720
+- !ruby/object:Gem::Dependency
+  name: redis-namespace
+  requirement: &70125036305300 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70125036305300
+description: !!null 
+email:
+- mark.lanett@gmail.com
+executables:
+- collective
+extensions: []
+extra_rdoc_files: []
+files:
+- .autotest
+- .gitignore
+- .rspec
+- Gemfile
+- Guardfile
+- README
+- Rakefile
+- bin/collective
+- collective.gemspec
+- demo/demo
+- demo/demo.rb
+- demo/demo3
+- demo/job1.rb
+- demo/job2.rb
+- demo/job3.rb
+- demo/populate.rb
+- lib/collective.rb
+- lib/collective/checker.rb
+- lib/collective/configuration.rb
+- lib/collective/idler.rb
+- lib/collective/key.rb
+- lib/collective/lifecycle_observer.rb
+- lib/collective/log.rb
+- lib/collective/messager.rb
+- lib/collective/mocks/storage.rb
+- lib/collective/monitor.rb
+- lib/collective/policy.rb
+- lib/collective/pool.rb
+- lib/collective/redis/storage.rb
+- lib/collective/registry.rb
+- lib/collective/squiggly.rb
+- lib/collective/utilities/airbrake_observer.rb
+- lib/collective/utilities/hoptoad_observer.rb
+- lib/collective/utilities/log_observer.rb
+- lib/collective/utilities/observeable.rb
+- lib/collective/utilities/observer_base.rb
+- lib/collective/utilities/process.rb
+- lib/collective/utilities/signal_hook.rb
+- lib/collective/utilities/storage_base.rb
+- lib/collective/version.rb
+- lib/collective/worker.rb
+- spec/checker_spec.rb
+- spec/configuration_spec.rb
+- spec/helper.rb
+- spec/idler_spec.rb
+- spec/key_spec.rb
+- spec/messager_spec.rb
+- spec/mocks/storage_spec.rb
+- spec/monitor_spec.rb
+- spec/policy_spec.rb
+- spec/pool_spec.rb
+- spec/redis/storage_spec.rb
+- spec/registry_spec.rb
+- spec/support/jobs.rb
+- spec/support/redis.rb
+- spec/support/timing.rb
+- spec/utilities/observer_base_spec.rb
+- spec/utilities/process_spec.rb
+- spec/worker_spec.rb
+- unused/times.rb
+has_rdoc: true
+homepage: ''
+licenses: []
+post_install_message: !!null 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: collective
+rubygems_version: 1.5.0
+signing_key: !!null 
+specification_version: 3
+summary: Manage a collection of worker processes
+test_files:
+- spec/checker_spec.rb
+- spec/configuration_spec.rb
+- spec/helper.rb
+- spec/idler_spec.rb
+- spec/key_spec.rb
+- spec/messager_spec.rb
+- spec/mocks/storage_spec.rb
+- spec/monitor_spec.rb
+- spec/policy_spec.rb
+- spec/pool_spec.rb
+- spec/redis/storage_spec.rb
+- spec/registry_spec.rb
+- spec/support/jobs.rb
+- spec/support/redis.rb
+- spec/support/timing.rb
+- spec/utilities/observer_base_spec.rb
+- spec/utilities/process_spec.rb
+- spec/worker_spec.rb