rplex 0.0.3 → 0.0.4

data/Gemfile

@@ -1,7 +1,9 @@
 source :rubygems
-gem "sinatra","~>1.3.1"
+gem "sinatra","~>1.3.2"
 gem 'rest-client',"~>1.6.7"
-gem "hoe","~>2.12.4"
-gem "rack-test","~>0.6.1"
-gem 'rcov',"~>0.9.11"
-gem 'rdoc',"~>3.11"
+group :development do
+  gem "rake","~>0.9.2.2"
+  gem "hoe","~>3.0.6"
+  gem "rack-test","~>0.6.1"
+  gem 'rdoc',"~>3.12"
+end

data/History.txt

@@ -1,3 +1,6 @@
+=== 0.0.4 / 2012-05-31
+* Added ability to limit queue sizes and a general configuration mechanism for the API
+* Queues can be removed
 === 0.0.3 / 2012-02-24
 * Added reset queue functionality
 === 0.0.2 / 2012-01-04

data/README.txt

@@ -22,6 +22,9 @@ Each worker polls the rplex queue, and then uses the information to grab the bui
 === Is that all?
 Queues are created automatically when a worker tries to get data from the queue, data can be posted to a subset of the workers if necessary and that as they say is it.
 
+=== Really?
+Well, it used to be all of it. Now, you can also limit the size of the queues and remove queues using the client (and the API)
+
 == Example
 Start the rplex service.
 

data/Rakefile

@@ -1,17 +1,18 @@
-# -*- ruby -*-
+# -*- coding: utf-8 -*-
+# encoding: utf-8
 $:.unshift File.join(File.dirname(__FILE__),"lib")
+require 'rplex'
 require 'rubygems'
 require 'hoe'
-require 'rplex'
 
 Hoe.spec 'rplex' do |prj|
   developer('Vassilis Rizopoulos', 'var@zuehlke.com')
   prj.version=Rplex::Version::STRING
-  prj.summary = 'rplex multiplexes jobs across multiple workers'
-  prj.description = prj.paragraphs_of('README.txt', 1..4).join("\n\n")
-  prj.url = "http://github.com/damphyr/rplex"
+  prj.summary = 'rplex demultiplexes jobs across multiple workers'
+  prj.description = prj.paragraphs_of('README.txt', 1..5).join("\n\n")
+  prj.urls = ["http://github.com/damphyr/rplex"]
   prj.changes = prj.paragraphs_of('History.txt', 0..1).join("\n\n")
-  prj.extra_deps<<["sinatra", "~>1.3.1"]
+  prj.extra_deps<<["sinatra", "~>1.3.2"]
   prj.extra_deps<<['rest-client',"~>1.6.7"]
 end
 

data/lib/rplex/client.rb

@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+# encoding: utf-8
 require 'rest_client'
 require 'json'
 module Rplex
@@ -52,6 +54,32 @@ module Rplex
       raise ClientError, $!.message
     end
     
+    def configuration
+      srv="#{@service}/configuration/#{@name}"
+      response=RestClient.get(srv,:accept => :json)
+      unless response.empty?
+        return JSON.parse(response)
+      else
+        return {}
+      end
+    rescue
+      raise ClientError, $!.message
+    end
+    
+    def configure max_size
+      response=RestClient.post("#{@service}/configuration", {"worker"=>@name, "maximum_size" => max_size}, :content_type => :json, :accept => :json)
+      return response
+    rescue
+      raise ClientError, $!.message
+    end
+    
+    def remove
+      response=RestClient.delete("#{@service}/configuration/#{worker}")
+      return response
+    rescue
+      raise ClientError, $!.message
+    end
+    
     def to_s
       "#{@name} working with #{@service}"
     end

data/lib/rplex/jobs.rb

@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+# encoding: utf-8
 require 'thread'
 
 module Rplex
@@ -22,6 +24,10 @@ module Rplex
       if valid?(job_data)
         @queues.each do |w,q|
           if workers.include?(w)
+            #this handles a SizedQueue without blocking
+            if q.respond_to?(:max) && q.size == q.max
+              q.pop
+            end
             q.push(job_data)
             queued_in+=1  
           end
@@ -56,5 +62,34 @@ module Rplex
     def reset workers
       workers.each{|worker| @queues[worker].clear if @queues[worker]}
     end
+    #Configures the named worker
+    #
+    #worker_config is a Hash with possible keys:
+    # "maximum_size" - when 0 then it's unlimited
+    #
+    #Will create a queue for the worker if it doesn't exist
+    #
+    #Configuring a worker will reset it's queue
+    def configure worker,worker_config
+      if worker_config["maximum_size"]>0
+        @queues[worker]=SizedQueue.new(worker_config["maximum_size"])
+      else
+        @queues[worker]=Queue.new
+      end
+      configuration(worker)
+    end
+    #Returns the worker's configuration
+    def configuration worker
+      if @queues[worker]  
+        @queues[worker].respond_to?(:max) ? max_size=@queues[worker].max : max_size=0
+        {'worker'=>worker,'maximum_size'=>max_size}
+      else
+        raise InvalidData,"non existent queue"
+      end
+    end
+    #Removes a queue
+    def remove worker
+      @queues.delete(worker)
+    end
   end
 end

data/lib/rplex/server.rb

@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+# encoding: utf-8
 require 'sinatra/base'
 require 'rplex'
 require 'rplex/jobs'
@@ -9,6 +11,24 @@ module Rplex
       super
       @overseer = Rplex::Overseer.new
     end
+    get '/' do
+      [200,{'Content-Type' => 'application/json'},{"version"=>Rplex::Version::STRING}.to_json]
+    end
+    get '/backlog' do  
+      [200,{'Content-Type' => 'application/json'},@overseer.backlog.to_json]
+    end
+    post '/reset' do
+      begin
+        reply={}
+        workers= params["workers"] ? params["workers"] : @overseer.workers
+        @overseer.reset(workers)
+        [200,{'Content-Type' => 'application/json'},@overseer.backlog.to_json]
+      rescue
+        status 500
+      end
+    end
+    ###########
+    # /job
     post '/job' do
       begin
         reply={}
@@ -25,27 +45,42 @@ module Rplex
       if reply
         [200,{'Content-Type' => 'application/json'},reply.to_json]
       else
+        #No jobs. No change
+        #if the worker was not there before it has now been created, so no 404
         status 204
       end
     end
-    get '/' do
-      [200,{'Content-Type' => 'application/json'},{"version"=>Rplex::Version::STRING}.to_json]
-    end
     
-    get '/backlog' do  
-      [200,{'Content-Type' => 'application/json'},@overseer.backlog.to_json]
+    ###########
+    # /configuration
+    get '/configuration' do 
+      config=@overseer.workers.map{|worker| @overseer.configuration(worker)}
+      [200,{'Content-Type' => 'application/json'},config.to_json]
     end
-    
-    post '/reset' do
+    get '/configuration/:worker' do |worker|
       begin
-        reply={}
-        workers= params["workers"] ? params["workers"] : @overseer.workers
-        @overseer.reset(workers)
-        [200,{'Content-Type' => 'application/json'},@overseer.backlog.to_json]
+        [200,{'Content-Type' => 'application/json'},@overseer.configuration(worker).to_json]
+      rescue InvalidData
+        [404,{'Content-Type' => 'application/json'},$!.message]
+      end
+    end
+    post '/configuration' do
+      begin
+        worker=params['worker']
+        if params['maximum_size']
+          max_size=params['maximum_size'].to_i
+          @overseer.configure(worker,{'maximum_size'=>max_size})
+          [200,{'Content-Type' => 'application/json'},@overseer.configuration(worker).to_json]
+        else
+          status 500
+        end
       rescue
         status 500
       end
     end
+    delete '/configuration/:worker' do |worker|
+      @overseer.remove(worker)
+    end
     
     def self.define_settings cfg={}
       cfg||={}

data/lib/rplex.rb

@@ -1,8 +1,10 @@
+# -*- coding: utf-8 -*-
+# encoding: utf-8
 module Rplex
   module Version
     MAJOR = 0
     MINOR = 0 
-    TINY = 3
+    TINY = 4
     STRING = "#{MAJOR}.#{MINOR}.#{TINY}"
   end
 end

data/test/test_jobs.rb

@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+# encoding: utf-8
 $:.unshift File.join(File.dirname(__FILE__),"..","lib")
 require "test/unit"
 require "rplex/jobs"
@@ -32,10 +34,12 @@ class TestOverseer < Test::Unit::TestCase
     ov['worker1']
     ov['worker2']
     ov<<job_data
-    assert_equal([['worker1',1],['worker2',1]], ov.backlog)
+    assert_equal(2, ov.backlog.size)
+    assert(ov.backlog.include?(['worker1',1]), "worker1 not present")
+    assert(ov.backlog.include?(['worker2',1]), "worker2 not present")
   end
   
-  def test_reset
+  def test_reset_remove
     ov=Rplex::Overseer.new
     assert(ov.backlog.empty?)
     jd1={"identifier"=>"8888","data"=>{:foo=>"bar",:bar=>"foo"}}
@@ -44,10 +48,51 @@ class TestOverseer < Test::Unit::TestCase
     ov['worker2']
     ov<<jd1
     ov<<jd2
-    assert_equal([['worker1',2],['worker2',2]], ov.backlog)
+    assert_equal(2, ov.backlog.size)
+    assert(ov.backlog.include?(['worker1',2]), "worker1 not present")
+    assert(ov.backlog.include?(['worker2',2]), "worker2 not present")
     ov.reset(['worker2'])
-    assert_equal([['worker1',2],['worker2',0]], ov.backlog)
+    assert(ov.backlog.include?(['worker2',0]), "worker2 not 0ed")
     ov.reset(['worker1'])
-    assert_equal([['worker1',0],['worker2',0]], ov.backlog)
+    assert(ov.backlog.include?(['worker1',0]), "worker1 not 0ed")
+    ov.remove('worker1')
+    assert_equal([['worker2',0]], ov.backlog)
+  end
+  
+  def test_configure
+    ret=nil
+    ov=Rplex::Overseer.new
+    assert_nothing_raised() { ret=ov.configure('worker1',{'maximum_size'=>2}) }
+    assert_equal({'worker'=>'worker1','maximum_size'=>2}, ret)
+    assert_equal([['worker1',0]], ov.backlog)
+    ov<<{"identifier"=>"8888","data"=>{:foo=>"bar",:bar=>"foo"}}
+    assert_equal([['worker1',1]], ov.backlog)
+    ov<<{"identifier"=>"8889","data"=>{:foo=>"bar",:bar=>"foo"}}
+    assert_equal([['worker1',2]], ov.backlog)
+    ov<<{"identifier"=>"8899","data"=>{:foo=>"bar",:bar=>"foo"}}
+    assert_equal([['worker1',2]], ov.backlog)
+    assert_nothing_raised() { ret=ov.configure('worker1',{'maximum_size'=>0}) }
+    assert_equal({'worker'=>'worker1','maximum_size'=>0}, ret)
+    assert_equal([['worker1',0]], ov.backlog)
+    #and now the ugly stuff
+    assert_raise(ArgumentError) { ov.configure('worker1','maximum_size'=>"foo") }
+  end
+  
+  def test_configuration
+    assert_raise(Rplex::InvalidData) {  Rplex::Overseer.new.configuration('worker1') }
+  end
+  def test_full_sized_queue
+    ov=Rplex::Overseer.new
+    jd1={"identifier"=>"8888","data"=>{:foo=>"bar",:bar=>"foo"}}
+    jd2={"identifier"=>"9999","data"=>{:foo=>"bar",:bar=>"foo"}}
+    jd3={"identifier"=>"7777","data"=>{:foo=>"bar",:bar=>"foo"}}
+    ov.configure('worker1',{'maximum_size'=>2})
+    ov<<jd1
+    ov<<jd2
+    ov<<jd3
+    assert_equal([['worker1',2]], ov.backlog)
+    assert_equal(jd2,ov['worker1'])
+    assert_equal(jd3,ov['worker1'])
+    assert_nil(ov['worker1'])
   end
 end

data/test/test_rplex.rb

@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+# encoding: utf-8
 $:.unshift File.join(File.dirname(__FILE__),"..","lib")
 require "test/unit"
 require "rplex"

data/test/test_server.rb

@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+# encoding: utf-8
 $:.unshift File.join(File.dirname(__FILE__),"..","lib")
 require 'test/unit'
 require 'rubygems'
@@ -40,8 +42,9 @@ class ServerTest < Test::Unit::TestCase
     assert_equal(200, last_response.status)
   end
   
-  def test_reset
-    get '/job/worker'
+  def test_reset_remove
+    #do this because tests are otherwise exec sequence dependent
+    post '/configuration', {'worker'=>'worker','maximum_size'=>0}
     payload={"identifier"=>"8888","data"=>{"url"=>"http://foo.bar/drops","revision"=>"8888"}}
     post '/job', payload
     post '/job', payload
@@ -50,5 +53,19 @@ class ServerTest < Test::Unit::TestCase
     post '/reset',{"workers"=>['worker']}
     get '/backlog'
     assert_equal([["worker",0]], JSON.parse(last_response.body))
+    delete '/configuration/worker'
+    get '/backlog'
+    assert_equal([], JSON.parse(last_response.body))
+  end
+  
+  def test_configuration
+    get '/configuration'
+    assert_equal([], JSON.parse(last_response.body))
+    get '/job/worker'
+    get '/configuration'
+    assert_equal([{'worker'=>'worker','maximum_size'=>0}], JSON.parse(last_response.body))
+    post '/configuration', {'worker'=>'worker','maximum_size'=>1}
+    get '/configuration/worker'
+    assert_equal({'worker'=>'worker','maximum_size'=>1}, JSON.parse(last_response.body))
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rplex
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
   prerelease: 
 platform: ruby
 authors:
@@ -9,22 +9,22 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-04-20 00:00:00.000000000Z
+date: 2012-06-04 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &2153566880 !ruby/object:Gem::Requirement
+  requirement: &2156779420 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.3.1
+        version: 1.3.2
   type: :runtime
   prerelease: false
-  version_requirements: *2153566880
+  version_requirements: *2156779420
 - !ruby/object:Gem::Dependency
   name: rest-client
-  requirement: &2153565980 !ruby/object:Gem::Requirement
+  requirement: &2156778600 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: 1.6.7
   type: :runtime
   prerelease: false
-  version_requirements: *2153565980
+  version_requirements: *2156778600
 - !ruby/object:Gem::Dependency
   name: rdoc
-  requirement: &2153565160 !ruby/object:Gem::Requirement
+  requirement: &2156777860 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,18 +43,18 @@ dependencies:
         version: '3.10'
   type: :development
   prerelease: false
-  version_requirements: *2153565160
+  version_requirements: *2156777860
 - !ruby/object:Gem::Dependency
   name: hoe
-  requirement: &2153564380 !ruby/object:Gem::Requirement
+  requirement: &2156776880 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '2.12'
+        version: '3.0'
   type: :development
   prerelease: false
-  version_requirements: *2153564380
+  version_requirements: *2156776880
 description: ! 'https://github.com/damphyr/rplex
 
 
@@ -64,7 +64,10 @@ description: ! 'https://github.com/damphyr/rplex
   Simple asynchronous data based job management
 
 
-  == INSTALL:'
+  == INSTALL:
+
+
+  * sudo gem install rplex'
 email:
 - var@zuehlke.com
 executables: []
@@ -112,7 +115,7 @@ rubyforge_project: rplex
 rubygems_version: 1.8.12
 signing_key: 
 specification_version: 3
-summary: rplex multiplexes jobs across multiple workers
+summary: rplex demultiplexes jobs across multiple workers
 test_files:
 - test/test_jobs.rb
 - test/test_rplex.rb