collective 0.2.0

data/spec/mocks/storage_spec.rb

@@ -0,0 +1,108 @@
+# -*- encoding: utf-8 -*-
+
+require "helper"
+
+describe Collective::Mocks::Storage do
+  
+  before do
+    @it = Collective::Mocks::Storage.new
+  end
+  
+  it "should be concrete" do
+    @it.should_not be_nil
+  end
+  
+  describe "simple values" do
+    
+    before do
+      @it.put("foo","bar")
+    end
+    
+    it "should be settable" do
+      @it.get("foo").should eq("bar")
+    end
+    
+    it "should be replaceable" do
+      @it.put("foo","goo")
+      @it.get("foo").should eq("goo")
+    end
+    
+    it "should be deleteable" do
+      @it.get("foo").should_not be_nil
+      @it.del("foo")
+      @it.get("foo").should be_nil
+    end
+    
+  end # simple values
+  
+  describe "lists" do
+    
+    before do
+      @it.set_add("foos","A")
+    end
+    
+    it "should not add to a list twice" do
+      @it.set_add("foos","A")
+      @it.set_size("foos").should eq 1
+    end
+    
+    it "should be able to add to, enumerate, and remove from a list" do
+      @it.set_add("foos","B")
+      @it.set_size("foos").should eq 2
+      @it.set_get_all("foos").should be_include("B")
+      @it.set_remove("foos","A")
+      @it.set_size("foos").should eq 1
+    end
+
+    it "can detect membership in a list" do
+      @it.set_add("foos","A")
+      @it.set_member?("foos","A").should be_true
+    end
+
+  end # lists
+  
+  describe "maps" do
+    
+    before do
+      @it.map_set("good","A","E")
+      @it.map_set("food","A","B")
+      @it.map_set("food","C","D")
+    end
+    
+    it "should count a set in a key as one item" do
+      @it.map_size("food").should eq 2
+      @it.map_size("good").should eq 1
+    end
+    
+    it "should retain the set" do
+      @it.map_get("food","A").should eq "B"
+      @it.map_get("food","C").should eq "D"
+      @it.map_get("good","A").should eq "E"
+    end
+    
+    it "should add to the set" do
+      @it.map_get_all_keys("food").size.should eq 2
+    end
+    
+    it "should delete the set" do
+      @it.del("food")
+      @it.map_size("food").should eq 0
+    end
+    
+  end # maps
+  
+  describe "priority queues" do
+
+    it "can add items and remove them in order" do
+      @it.queue_add "foo", "A", 1
+      @it.queue_add "foo", "C", 3
+      @it.queue_add "foo", "B", 2
+      @it.queue_pop("foo",0).should eq(nil)
+      @it.queue_pop("foo",9).should eq("A")
+      @it.queue_pop("foo",9).should eq("B")
+      @it.queue_pop("foo",9).should eq("C")
+    end
+
+  end
+
+end

data/spec/monitor_spec.rb

@@ -0,0 +1,15 @@
+# -*- encoding: utf-8 -*-
+
+require "helper"
+
+describe Collective::Monitor do
+
+  it "can monitor all pools continuously"
+
+  it "can start all workers in all pools"
+
+  it "can stop all workers in all pools"
+
+  it "can restart all workers in all pools"
+
+end

data/spec/policy_spec.rb

@@ -0,0 +1,43 @@
+# -*- encoding: utf-8 -*-
+
+require "helper"
+
+describe Collective::Policy do
+  
+  it "should have defaults" do
+    p = Collective::Policy.resolve
+    p.worker_max_jobs.should eq(100)
+  end
+
+  it "should be changeable" do
+    p = Collective::Policy.resolve worker_max_jobs: 5
+    p.worker_max_jobs.should eq(5)
+  end
+
+  it "should work with symbols" do
+    p = Collective::Policy.resolve worker_max_jobs: 5
+    p.worker_max_jobs.should eq(5)
+  end
+
+  it "should support observers" do
+    o = Collective::Utilities::ObserverBase.new
+    p = Collective::Policy.resolve observers: [o]
+    p.observers.should eq([o])
+  end
+
+  it "is copied from another policy" do
+    p1 = Collective::Policy.resolve worker_max_jobs: 7, worker_max_lifetime: 999
+    p2 = Collective::Policy.resolve policy: p1, worker_max_jobs: 12
+
+    p2.worker_max_lifetime.should eq(p1.worker_max_lifetime)
+    p1.worker_max_jobs.should eq(7)
+    p2.worker_max_jobs.should eq(12)
+  end
+
+  it "can resolve storage" do
+    p = Collective::Policy.resolve storage: nil
+    s = p.storage
+    s.should be_instance_of(Collective::Mocks::Storage)
+  end
+
+end

data/spec/pool_spec.rb

@@ -0,0 +1,119 @@
+# -*- encoding: utf-8 -*-
+
+require "helper"
+require "redis"
+
+describe Collective::Pool do
+
+  before do
+    @name = "#{ described_class || 'Test' }::#{example.description}"
+  end
+
+  describe "name" do
+
+    it "needs to be specified for a proc" do
+      job  = ->(context) { false }
+      expect { pool = Collective::Pool.new( job ) }.to raise_error
+      expect { pool = Collective::Pool.new( job, name: @name ) }.to_not raise_error
+    end
+
+    it "does not need to be specified for a class" do
+      job = TrueJob
+      expect { pool = Collective::Pool.new( job ) }.to_not raise_error
+    end
+
+  end
+
+  describe "when spawning proceses", redis: true do
+
+    def make_policy( options = {} )
+      options = { name: @name, worker_max_lifetime: 10, storage: :redis, observers: [ [ :log, "/tmp/debug.log" ] ] }.merge(options)
+      Collective::Policy.resolve(options)
+    end
+
+    it "should spawn a worker" do
+      job     = ->(context) {}
+      pool    = Collective::Pool.new( job, make_policy )
+
+      pool.stub(:spawn) {} # must be called at least once
+
+      pool.synchronize
+    end
+
+    it "spins up an actual worker" do
+      pool    = Collective::Pool.new( ListenerJob, make_policy )
+
+      pool.registry.workers.size.should be == 0
+
+      pool.synchronize
+      wait_until { pool.registry.workers.size > 0 }
+      pool.registry.workers.size.should be > 0
+      other = pool.registry.workers.first
+
+      pool.mq.expect(/State/) { |message| puts message.body }
+      pool.mq.send "State?", to: other
+      pool.mq.receive
+      pool.mq.send "Quit", to: other
+    end
+
+    it "spins up a worker only once" do
+      policy   = make_policy pool_max_workers: 1
+      pool     = Collective::Pool.new( ListenerJob, policy )
+      registry = pool.registry
+
+      registry.checked_workers( policy ).live.size.should eq(0)
+
+      pool.synchronize
+      registry.checked_workers( policy ).live.size.should eq(1)
+
+      pool.synchronize
+      registry.checked_workers( policy ).live.size.should eq(1)
+    end
+
+    it "should spin up new workers as necessary" do
+      policy   = make_policy pool_min_workers: 2, pool_max_workers: 2
+      pool     = Collective::Pool.new( ListenerJob, policy )
+      registry = pool.registry
+
+      registry.checked_workers( policy ).live.size.should eq(0)
+
+      pool.synchronize
+      registry.checked_workers( policy ).live.size.should eq(2)
+
+      first = registry.checked_workers( policy ).live.first
+      pool.mq.send "Quit", to: first
+      wait_until { registry.checked_workers( policy ).live.size == 1 }
+      registry.checked_workers( policy ).live.size.should eq(1)
+
+      pool.synchronize
+      registry.checked_workers( policy ).live.size.should eq(2)
+    end
+
+    it "spins down workers when there are too many" do
+      policy   = make_policy pool_min_workers: 2, pool_max_workers: 2
+      pool     = Collective::Pool.new( ListenerJob, policy )
+      registry = pool.registry
+
+      registry.checked_workers( policy ).live.size.should eq(0)
+
+      pool.synchronize
+      registry.checked_workers( policy ).live.size.should eq(2)
+
+      # create a second pool with its own policy to force quitting
+      policy2   = make_policy pool_min_workers: 1, pool_max_workers: 1
+      pool2     = Collective::Pool.new( ListenerJob, policy2 )
+      pool2.synchronize
+      registry.checked_workers( policy ).live.size.should eq(1)
+    end
+
+  end
+
+  describe "when tracking how long workers run", time: true do
+
+    it "should warn when a worker is running late"
+
+    it "should kill when a worker is running too late"
+
+  end
+
+end

data/spec/redis/storage_spec.rb

@@ -0,0 +1,133 @@
+# -*- encoding: utf-8 -*-
+
+require "helper"
+
+describe Collective::Redis::Storage, redis: true do
+  
+  before do
+    @it = Collective::Redis::Storage.new(redis)
+  end
+  
+  it "should be concrete" do
+    @it.should_not be_nil
+  end
+  
+  describe "simple values" do
+
+    before do
+      @it.put("foo","bar")
+    end
+
+    it "should be settable" do
+      @it.get("foo").should eq("bar")
+    end
+    
+    it "should be replaceable" do
+      @it.put("foo","goo")
+      @it.get("foo").should eq("goo")
+    end
+    
+    it "should be deleteable" do
+      @it.get("foo").should_not be_nil
+      @it.del("foo")
+      @it.get("foo").should be_nil
+    end
+    
+  end # simple values
+  
+  describe "lists" do
+    
+    before do
+      @it.del("foos")
+      @it.set_add("foos","A")
+    end
+    
+    it "should not add to a list twice" do
+      @it.set_add("foos","A")
+      @it.set_size("foos").should eq 1
+    end
+    
+    it "should be able to add to, enumerate, and remove from a list" do
+      @it.set_add("foos","B")
+      @it.set_size("foos").should eq 2
+      @it.set_get_all("foos").should be_include("B")
+      @it.set_remove("foos","A")
+      @it.set_size("foos").should eq 1
+    end
+
+    it "can detect membership in a list" do
+      @it.set_add("foos","A")
+      @it.set_member?("foos","A").should be_true
+    end
+
+  end # lists
+  
+  describe "maps" do
+    
+    before do
+      @it.map_set("good","A","E")
+      @it.map_set("food","A","B")
+      @it.map_set("food","C","D")
+    end
+    
+    it "should count a set in a key as one item" do
+      @it.map_size("food").should eq 2
+      @it.map_size("good").should eq 1
+    end
+    
+    it "should retain the set" do
+      @it.map_get("food","A").should eq "B"
+      @it.map_get("food","C").should eq "D"
+      @it.map_get("good","A").should eq "E"
+    end
+    
+    it "should add to the set" do
+      @it.map_get_all_keys("food").size.should eq 2
+    end
+    
+    it "should delete the set" do
+      @it.del("food")
+      @it.map_size("food").should eq 0
+    end
+    
+  end # maps
+
+  describe "priority queues" do
+
+    it "can add items and remove them in order" do
+      @it.queue_add "foo", "A", 1
+      @it.queue_add "foo", "C", 3
+      @it.queue_add "foo", "B", 2
+      @it.queue_pop("foo",0).should eq(nil)
+      @it.queue_pop("foo",9).should eq("A")
+      @it.queue_pop("foo",9).should eq("B")
+      @it.queue_pop("foo",9).should eq("C")
+    end
+
+    it "can handle some load" do
+      # expect 1..1000 in the queue
+      # write our 1..1000 in two separate processes
+      # plus some extra just to mess us up
+
+      Collective::Utilities::Process.fork_and_detach do
+        redis.client.reconnect
+        q = Collective::Redis::Storage.new(redis)
+        (1..2000).each { |i| q.queue_add( "foo", i.to_s, i ) if i % 2 == 0 }
+      end
+
+      Collective::Utilities::Process.fork_and_detach do
+        redis.client.reconnect
+        q = Collective::Redis::Storage.new(redis)
+        (1..2000).each { |i| q.queue_add( "foo", i.to_s, i ) if i % 2 == 1 }
+      end
+
+      (1..1000).each do |i|
+        i2 = @it.queue_pop_sync( "foo", i, timeout: 10 )
+        i2.should eq(i.to_s)
+      end
+
+    end
+
+  end
+
+end

data/spec/registry_spec.rb

@@ -0,0 +1,52 @@
+# -*- encoding: utf-8 -*-
+
+require "helper"
+
+describe Collective::Registry, redis: true do
+
+  before do
+    @policy = Collective::Policy.resolve
+  end
+
+  it "can register a worker" do
+    registry = Collective::Registry.new( "Test", @policy.storage )
+    worker   = Collective::Worker.new( TrueJob, registry: registry )
+
+    registry.register( worker.key )
+    registry.workers.should be_include( worker.key )
+  end
+
+  it "can unregister a worker" do
+    registry = Collective::Registry.new( "Test", @policy.storage )
+    worker   = Collective::Worker.new( TrueJob, registry: registry )
+
+    registry.register( worker.key )
+    registry.workers.should be_include( worker.key )
+
+    registry.unregister( worker.key )
+    registry.workers.should_not be_include( worker.key )
+  end
+
+  it "can find live workers" do
+    registry  = Collective::Registry.new( "Test", @policy.storage )
+    heartbeat = @policy.worker_late / 2
+    key       = Collective::Key.new("Test",1234)
+
+    registry.register(key)
+    checked = registry.checked_workers(@policy)
+    checked.live.should eq([key])
+  end
+
+  it "can find late workers" do
+    registry  = Collective::Registry.new( "Test", @policy.storage )
+    heartbeat = @policy.worker_late
+    key       = Collective::Key.new("Test",1234)
+
+    now       = Time.now.to_i
+    registry.register(key) # should register with heartbeat = now or now+1
+    registry.stub(:now) { now + @policy.worker_late + 2 }
+    checked = registry.checked_workers(@policy)
+    checked.late.should eq([key])
+  end
+
+end