background_queue 0.2.0

data/spec/spec_helper.rb

@@ -0,0 +1,15 @@
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'rspec'
+
+
+# Requires supporting files with custom matchers and macros, etc,
+# in ./support/ and its subdirectories.
+Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}
+Dir["#{File.dirname(__FILE__)}/shared/**/*.rb"].each {|f| require f}
+
+RSpec.configure do |config|
+  
+end
+
+

data/spec/support/default_task.rb

@@ -0,0 +1,9 @@
+require File.expand_path(File.dirname(__FILE__) + '/simple_task')
+class DefaultTask < SimpleTask
+
+  def initialize
+    super(:owner_id, :job_id, :id, :priority)
+  end
+  
+  
+end

data/spec/support/private.rb

@@ -0,0 +1,23 @@
+#i want to be able to test private methods
+#this will let me prefix the call with __prv__ and it will then use send to route to the private call
+#this assumes the class getting tested does not use method_missing
+class Object
+  
+  alias_method :_original_method_missing, :method_missing
+  
+  def method_missing(sym, *args, &block)
+    if sym.to_s[0, 7] == "__prv__"
+      #we are explicily saying this should be a private method.. lets make sure...
+      
+      private_method = sym.to_s[7, sym.to_s.length - 7].intern
+      
+      raise "Method #{private_method} is public when it was expected to be private" if respond_to?(private_method)
+      
+      send private_method, *args, &block
+    else
+      _original_method_missing(sym, *args, &block)
+    end
+  end
+  
+  
+end

data/spec/support/simple_server.rb

@@ -0,0 +1,28 @@
+class SimpleServer
+  attr_accessor :config
+  attr_accessor :thread_manager
+  attr_accessor :task_queue
+  attr_accessor :event_server
+  attr_accessor :workers
+  attr_accessor :jobs
+  attr_accessor :logger
+  
+  def initialize(options={})
+    @logger = Logger.new("/dev/null")
+    @config = options[:config]
+    @thread_manager = options[:thread_manager]
+    @task_queue = options[:task_queue]
+    @event_server = options[:event_server]
+    @workers = options[:workers]
+    @jobs = options[:jobs]
+  end
+  
+  def running?
+    true
+  end
+  
+  def change_stat(stat, delta)
+    
+  end
+ 
+end

data/spec/support/simple_task.rb

@@ -0,0 +1,58 @@
+class SimpleTask
+  attr_accessor :id
+  attr_accessor :priority
+  
+  attr_accessor :owner_id
+  attr_accessor :job_id
+  attr_accessor :worker
+  
+  attr_accessor :running
+  
+  attr_accessor :options
+  
+  def initialize(owner_id, job_id, id, priority, options={})
+    @owner_id = owner_id
+    @job_id = job_id
+    @id = id
+    @priority = priority
+    @worker = :worker
+    @running = false
+    @options = options
+  end
+  
+  def running?
+    @running
+  end
+  
+  def synchronous?
+    @options[:synchronous] == true
+  end
+  
+  def domain
+    @options[:domain]
+  end
+  
+  def set_job(job)
+    @job = job
+  end
+  
+  def get_job
+    @job
+  end
+  
+  def is_excluded_from_count?
+    @options[:exclude] == true
+  end
+  
+  def weighted?
+    @options[:percent] && @options[:percent] > 0
+  end
+  
+  def weighted_percent
+    @options[:percent]
+  end
+  
+  def initial_progress_caption
+    @options[:initial_progress_caption]
+  end
+end

data/spec/support/test_worker_server.rb

@@ -0,0 +1,205 @@
+require 'webrick'
+
+class TestWorkerServer
+
+  
+  attr_reader :control_calling
+  
+  def initialize(port, control_calling = false)
+    @port = port
+    @control_calling = control_calling
+  end
+  
+  def start(proc)
+    @server = WEBrick::HTTPServer.new(:BindAddress=>"127.0.0.1", :Port => @port, :Logger=>WEBrick::BasicLog.new([], -1000), :AccessLog=>[]) 
+    @server.mount "/background_queue", TestWorkerServer::ProcServlet, proc, self
+    @mutex = Mutex.new
+    @condvar = ConditionVariable.new
+    @called = false
+    if @control_calling
+      @cmutex = Mutex.new
+      @ccondvar = ConditionVariable.new
+      @ccalled = false
+    end
+    
+    @thread = Thread.new {
+      @server.start
+    }
+  end
+  
+  
+  
+  def stop
+    @server.shutdown
+    Thread.kill(@thread) #brutal.... but quick...
+    #@thread.join
+  end
+  
+  def wait_to_be_called
+    was_called = false
+    @mutex.synchronize {
+      unless @called
+        @condvar.wait(@mutex, 5)
+      end
+      was_called = @called
+      @called = false
+      #puts "called=#{was_called}"
+      was_called
+    }
+  end
+  
+  def mark_as_called
+    @mutex.synchronize {
+      @called = true
+      @condvar.signal
+    }
+  end
+  
+  def can_to_be_called?
+    was_called = false
+    @cmutex.synchronize {
+      unless @ccalled
+        @ccondvar.wait(@cmutex, 5)
+      end
+      was_called = @ccalled
+      @ccalled = false
+      was_called
+    }
+  end
+  
+  def allow_to_be_called
+    @cmutex.synchronize {
+      @ccalled = true
+      @ccondvar.signal
+    }
+  end
+  
+  
+
+  class ProcServlet < WEBrick::HTTPServlet::AbstractServlet
+
+    def initialize(server, proc, test_server)
+      super(server)
+      @proc = proc
+      @test_server = test_server
+    end
+
+  
+    def do_POST(request, response)
+      begin
+        response.test_server = @test_server
+        @test_server.can_to_be_called? if @test_server.control_calling
+        @proc.call(TestWorkerServer::Contoller.new(request, response))
+      rescue Exception=>e
+        puts e.message
+        puts e.backtrace.join("\n")
+      end
+    end
+  
+  end
+  
+  class Contoller
+    
+    attr_accessor :request
+    attr_accessor :response
+    attr_accessor :params
+      
+    def initialize(request, response)
+      @request = request
+      @response = response
+      @logger = WEBrick::BasicLog.new("/tmp/bq-controller.log", 5)
+      @params = BackgroundQueue::Utils::AnyKeyHash.new(@request.query)
+    end
+    
+    def headers
+      @response.header
+    end
+    
+    def logger
+      @logger
+    end
+    
+    def render(opts)
+      @response.status = opts[:status] if opts[:status]
+      @response.content_type = opts[:type] if opts[:type]
+       
+       
+      if opts[:text].instance_of?(String)
+        @response.body = opts[:text]
+      elsif opts[:text].instance_of?(Proc)
+        @response.chunked = true
+        @response.body = opts[:text]
+      end
+    end
+    
+  end
+
+end
+
+module WEBrick
+  class HTTPResponse
+    
+    attr_accessor :test_server
+    
+    alias_method :old_send_response, :send_response
+    
+    def send_response(socket)
+      old_send_response(socket)
+      socket.extend SocketExtension
+      socket.set_test_server(@test_server)
+    end
+    
+    alias_method :old_send_body, :send_body
+   
+    def send_body(socket)
+      if @body.instance_of?(Proc)
+        begin
+          @body.call(self, ChunkedOutput.new(socket, self))
+          _write_data(socket, "0#{CRLF}#{CRLF}")
+        rescue Exception=>e
+          puts e.message
+          puts e.backtrace.join("\n")
+        end
+      else
+        old_send_body(socket)
+      end
+    end
+    
+    def send_chunked_data(socket, buf)
+      data = ""
+      data << format("%x", buf.size) << CRLF
+      data << buf << CRLF
+      _write_data(socket, data)
+      @sent_size += buf.size
+    end
+  end
+end
+
+class ChunkedOutput
+  
+  def initialize(socket, response)
+    @socket = socket
+    @response = response
+  end
+  
+  def write(buf)
+    @response.send_chunked_data(@socket, buf)
+  end
+  
+  def flush
+    @socket.flush
+  end
+end
+
+module SocketExtension
+  
+  def set_test_server(ts)
+    @test_server = ts
+  end
+  
+  def close
+    super
+    @test_server.mark_as_called unless @test_server.nil?
+  end
+end
+  

metadata

@@ -0,0 +1,254 @@
+--- !ruby/object:Gem::Specification
+name: background_queue
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+  prerelease: 
+platform: ruby
+authors:
+- MarkPent
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-10-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: &21144380 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *21144380
+- !ruby/object:Gem::Dependency
+  name: rufus-scheduler
+  requirement: &21156100 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *21156100
+- !ruby/object:Gem::Dependency
+  name: eventmachine
+  requirement: &21161240 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.12.10
+  type: :runtime
+  prerelease: false
+  version_requirements: *21161240
+- !ruby/object:Gem::Dependency
+  name: ipaddress
+  requirement: &21156960 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+  type: :runtime
+  prerelease: false
+  version_requirements: *21156960
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &21170000 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.9.0
+  type: :development
+  prerelease: false
+  version_requirements: *21170000
+- !ruby/object:Gem::Dependency
+  name: jeweler
+  requirement: &21166880 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.8.3
+  type: :development
+  prerelease: false
+  version_requirements: *21166880
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: &21166160 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.7'
+  type: :development
+  prerelease: false
+  version_requirements: *21166160
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: &21164920 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.12'
+  type: :development
+  prerelease: false
+  version_requirements: *21164920
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: &21180500 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :development
+  prerelease: false
+  version_requirements: *21180500
+- !ruby/object:Gem::Dependency
+  name: redcarpet
+  requirement: &21179640 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.1.1
+  type: :development
+  prerelease: false
+  version_requirements: *21179640
+description: Organise background tasks so they will not overload the machine(s) running
+  the tasks, while still giving a fair, balanced allocation of running time to members
+  in the queue
+email: mark.pent@gmail.com
+executables:
+- bg_queue
+extensions: []
+extra_rdoc_files:
+- LICENSE.txt
+- README.md
+- TODO
+files:
+- .document
+- .rspec
+- .rvmrc
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- TODO
+- VERSION
+- background_queue.gemspec
+- bin/bg_queue
+- lib/background_queue.rb
+- lib/background_queue/client.rb
+- lib/background_queue/client_lib/command.rb
+- lib/background_queue/client_lib/config.rb
+- lib/background_queue/client_lib/connection.rb
+- lib/background_queue/client_lib/job_handle.rb
+- lib/background_queue/command.rb
+- lib/background_queue/config.rb
+- lib/background_queue/server_lib/balanced_queue.rb
+- lib/background_queue/server_lib/config.rb
+- lib/background_queue/server_lib/event_connection.rb
+- lib/background_queue/server_lib/event_server.rb
+- lib/background_queue/server_lib/job.rb
+- lib/background_queue/server_lib/job_registry.rb
+- lib/background_queue/server_lib/lru.rb
+- lib/background_queue/server_lib/owner.rb
+- lib/background_queue/server_lib/priority_queue.rb
+- lib/background_queue/server_lib/queue_registry.rb
+- lib/background_queue/server_lib/server.rb
+- lib/background_queue/server_lib/sorted_workers.rb
+- lib/background_queue/server_lib/task.rb
+- lib/background_queue/server_lib/task_registry.rb
+- lib/background_queue/server_lib/thread_manager.rb
+- lib/background_queue/server_lib/worker.rb
+- lib/background_queue/server_lib/worker_balancer.rb
+- lib/background_queue/server_lib/worker_client.rb
+- lib/background_queue/server_lib/worker_thread.rb
+- lib/background_queue/utils.rb
+- lib/background_queue/worker/base.rb
+- lib/background_queue/worker/calling.rb
+- lib/background_queue/worker/config.rb
+- lib/background_queue/worker/environment.rb
+- lib/background_queue/worker/worker_loader.rb
+- lib/background_queue_server.rb
+- lib/background_queue_worker.rb
+- spec/background_queue/client_lib/command_spec.rb
+- spec/background_queue/client_lib/config_spec.rb
+- spec/background_queue/client_lib/connection_spec.rb
+- spec/background_queue/client_spec.rb
+- spec/background_queue/command_spec.rb
+- spec/background_queue/config_spec.rb
+- spec/background_queue/server_lib/balanced_queue_spec.rb
+- spec/background_queue/server_lib/config_spec.rb
+- spec/background_queue/server_lib/event_connection_spec.rb
+- spec/background_queue/server_lib/event_server_spec.rb
+- spec/background_queue/server_lib/integration/full_test_spec.rb
+- spec/background_queue/server_lib/integration/queue_integration_spec.rb
+- spec/background_queue/server_lib/integration/serialize_spec.rb
+- spec/background_queue/server_lib/job_registry_spec.rb
+- spec/background_queue/server_lib/job_spec.rb
+- spec/background_queue/server_lib/owner_spec.rb
+- spec/background_queue/server_lib/priority_queue_spec.rb
+- spec/background_queue/server_lib/server_spec.rb
+- spec/background_queue/server_lib/sorted_workers_spec.rb
+- spec/background_queue/server_lib/task_registry_spec.rb
+- spec/background_queue/server_lib/task_spec.rb
+- spec/background_queue/server_lib/thread_manager_spec.rb
+- spec/background_queue/server_lib/worker_balancer_spec.rb
+- spec/background_queue/server_lib/worker_client_spec.rb
+- spec/background_queue/server_lib/worker_thread_spec.rb
+- spec/background_queue/utils_spec.rb
+- spec/background_queue/worker/base_spec.rb
+- spec/background_queue/worker/calling_spec.rb
+- spec/background_queue/worker/environment_spec.rb
+- spec/background_queue/worker/worker_loader_spec.rb
+- spec/background_queue_spec.rb
+- spec/resources/config-client.yml
+- spec/resources/config-serialize.yml
+- spec/resources/config.yml
+- spec/resources/example_worker.rb
+- spec/resources/example_worker_with_error.rb
+- spec/resources/test_worker.rb
+- spec/shared/queue_registry_shared.rb
+- spec/spec_helper.rb
+- spec/support/default_task.rb
+- spec/support/private.rb
+- spec/support/simple_server.rb
+- spec/support/simple_task.rb
+- spec/support/test_worker_server.rb
+homepage: http://github.com/markpent/background_queue
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 2926344861987835839
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.17
+signing_key: 
+specification_version: 3
+summary: Background processing
+test_files: []