blat 0.1.0a → 0.1.0b

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 878ccb8c27eeeeea95bf2771feb5164f0ea9f33b
-  data.tar.gz: 6e06378de9ddd8947ceb87ecced8c1418477a0c7
+  metadata.gz: a18d2f9b96d5a2d9b6e7e8ae085e1bfed1855d32
+  data.tar.gz: df5cc403ee46326005387f73432d73edb0fb69bf
 SHA512:
-  metadata.gz: b9a812fa61e9627a5167e6595dcfde2c616349cac35b67428ba4bdebecbd28c524cb9888cabb99f639f0f0edc43ec371b1d8da9a6b776079f4ccd3e5bf844869
-  data.tar.gz: 5f8d35878a6993d6ace44d9a642e590054fff062d5f631893dd42f996ccb88011d34b3b52f0e457af592e680509cedc550bef3297d12186ce3f026bacf5770a0
+  metadata.gz: f324fa27a4c93dd576a2663b87a047c3f0c74cf2387ad8fdf362eb4cf884b156a09232421101a7ca6cd947c03ad3d7fe5870ff6a1d7a95897a107fd31ca66fa7
+  data.tar.gz: cf89d6d6a43935fbd03dd811a770ff53ba091318cb89ee88037b9017efaed91e178dd1dcd8387624383b0d476b35f9bf9f5dc22d8fe3046876b7e07362d4c51b

data/lib/blat.rb

@@ -1,16 +1,11 @@
 
-require 'blat/pool'
 require 'blat/batch'
-require 'blat/formats'
+require 'blat/queue'
 
-# Blat is a hugely parallel wrapper for cURL designed to download data as
+# Blat is a simple wrapper for cURL::Multi designed to download data as
 # aggressively as possible.
-#
-# Blat makes use of many threads at once in a producer-consumer pattern, and
-# accepts tasks in the form of Blat::Jobs, which contain configuration and
-# results from each request.
 module Blat
 
-  VERSION = '0.1.0a'
+  VERSION = '0.1.0b'
 
 end

data/lib/blat/batch.rb

@@ -1,121 +1,44 @@
 
 
-require 'blat/pool'
+require 'curl'
 
 module Blat
 
-  # The batch downloader is a simpler wrapper around Pool that runs in a
-  # blocking manner.  The idea of this is that you can put a list of URLs in,
-  # run #run and then retrieve the results easily and quickly.
-  #
-  # example:
-  #
-  #   urls = File.read('url.list').lines
-  #   b = Blat::Batch.new( list )
-  #   b.run(10)
-  #   puts "Results: #{b.results}"
-  #
-  class Batch
+  module Blat::Batch
 
-    # Create a new batch downloader for a given list of URLS, and a given set
-    # of configuration options.
+    # Blat::Batch::run takes a list of links and downloads them all before
+    # returning.  It is a very simple interface to Curl::Multi for smallish
+    # tasks.
     #
-    # [:urls] An array of URLs to download.
-    # [:config] (optional) configuration to pass to the Jobs.  See Blat::Job
-    # for more information.
-    def initialize(urls, config = {})
-
-      # Config for each object
-      @config       = config
-
-      # URLS in as a string
-      @urls         = urls
-      @urls_mx      = Mutex.new
-
-      # Stores results as Job objects
-      @results      = []
-      @results_mx   = Mutex.new
-
-      # Keep this to see if we have finished
-      @url_count    = urls.length
-    end
-
-    # Run a batch with a given number of workers.
+    # [max_connections] Defines how many parallel connections to use
+    # [links] Is the list of strings or Curl::Easy objects to download.  The list object must support #map and #each
+    # [pipeline] Indicates if Curl::Multi should pipeline its HTTP requests
+    # [&block] If given, this block is called to configure each Curl::Easy object prior to it being pushed into the queue.
     #
-    # If a block is provided, it is called with the curl object just before
-    # requests are made.  This is to allow setting of various parameters, e.g.:
-    #
-    #  batch.run(10){ |c|
-    #     c.follow_location = true
-    #  }
-    #
-    def run(workers, &block)
-
-      # Figure out if people have overestimated the workers needed
-      workers = [workers, @urls.length].min
-
-      # Construct a pool
-      x = Blat::Pool.new(workers) do |job|
-        @results_mx.synchronize { @results << job }
-      end
+    def self.run(max_connections, links, pipeline = true, &block)
+      multi = Curl::Multi.new
 
-      # Set work to do
-      x.work do
+      # Set options
+      multi.max_connects  = max_connections.to_i
+      multi.pipeline      = (pipeline == true)
 
-        # Get the URL from the list
-        url = @urls_mx.synchronize { @urls.pop }
-
-        # If it's set, configure and return a job
-        if url
-          Blat::Job.new(@config) do |c|
-
-            # Configure with block if appropriate
-            yield(c) if block_given?
-
-            c.url= url
-          end
-        else
-          # If not, return nil to set the worker to idle
-          nil
-        end
+      curls = links.map do |l|
+        c = l
+        c = Curl::Easy.new(l) unless l.is_a?(Curl::Easy)
+        c
       end
 
-      # Wait until workers are idle
-      x.wait_until_idle
-
-      # Close them all.
-      x.close
-    end
-
-    # Is the batch complete?
-    def complete?
-      @results_mx.synchronize do
-        @results.length == @url_count
+      # Pump links in
+      curls.each do |c|
+        yield(c) if block_given?
+        multi.add(c)
       end
-    end
 
-    # Report progress with three vars
-    #
-    # remaining (yet to do)
-    # complete (completed)
-    # in_progress (currently running)
-    # total (remaining + complete + in progress)
-    def progress
-      remaining = @urls_mx.synchronize { @urls.length }
-      complete  = @results_mx.synchronize { @results.length }
-      return remaining,
-             complete,
-             (@url_count - complete - remaining),
-             @url_count
-    end
+      # Wait
+      multi.perform
 
-    # Get results as a list
-    def results
-      @results_mx.synchronize do
-        return @results
-      end
+      return curls
     end
-
   end
 
 end

data/lib/blat/queue.rb

@@ -0,0 +1,152 @@
+
+require 'curl'
+
+module Blat
+
+  # The Blat::Queue class represents a download queue that handles requests
+  # using Curl::Multi.  It, and its descendants, accept a large number of
+  # Curl::Easy objects and download them in parallel.
+  #
+  # In order to know when each request has completed, use
+  # Curl::Easy::on_complete.  This is made simpler by Queue#add, which will
+  # yield to a block on completion of each download.
+  #
+  class Queue
+
+    attr_reader :max_connections, :pipeline
+
+    # Create a new Blat::Queue with a given number of maximum connections.
+    #
+    # The 'pipeline' options controls Curl::Multi's pipelining feature, which
+    # tries to use the same http connection for many requests to the same server.
+    def initialize(max_connections, pipeline = true)
+      @multi = Curl::Multi.new
+
+      # Set properties
+      @max_connects         = max_connections.to_i
+      @pipeline             = (pipeline == true)
+      @multi.max_connects   = @max_connects
+      @multi.pipeline       = @pipeline
+    end
+
+    # Add a URL or a Curl::Easy object to the queue.
+    #
+    # Optionally, provide a callback for calling when requests are complete,
+    # e.g.:
+    #
+    #  q.add('http://google.com') do |c|
+    #    puts "Complete request: #{r}"
+    #  end
+    #
+    def add(curl_or_link, &block)
+      # Convert to curl if necessary
+      curl = curl_or_link.is_a?(Curl::Easy) ? curl_or_link : Curl::Easy.new(curl_or_link)
+      curl.on_complete { |c| block.yield(c) } if block_given?
+
+      # Add
+      @multi.add(curl)
+
+      # Return
+      return curl
+    end
+
+    # Returns the number of active requests
+    def request_count
+      requests.length
+    end
+
+    # Returns a list of active requests
+    def requests
+      @multi.requests
+    end
+
+    # Remove a request from the queue.
+    #
+    # This needn't be called if a request has completed.
+    def remove(curl)
+      @multi.remove(curl)
+    end
+
+    # Wait for all requests to finish (blocking).
+    #
+    # If a block is given it is executed repeatedly whilst waiting.
+    def wait(&block)
+      @multi.perform do
+        yield if block_given?
+      end
+    end
+
+    alias_method :perform, :wait
+
+    # Is the queue idle?
+    def idle?
+      @multi.idle?
+    end
+
+  end
+
+  # Similar to a queue, except that it explicitly calls a block in order to
+  # acquire new URLs.  
+  #
+  # This makes it suitable for use in producer/consumer patterns.
+  class ConsumingQueue < Queue 
+
+    # Executes the given block in order to keep the curl pool working at its
+    # maximum capacity.
+    #
+    # consume blocks as long as links are being downloaded, as it relies on
+    # Curl::Multi#perform
+    #
+    # Note that blocks providing links must also perform their own
+    # configuration, e.g.:
+    #
+    #  q.consume do
+    #    url = get_url
+    #    if(url)
+    #      c = Curl::Easy.new(url)
+    #      c.follow_location = true
+    #      c.on_complete{ |c| puts "Retrieved: #{c.body_str}" }
+    #      c
+    #    else
+    #      nil
+    #    end
+    #  end
+    #
+    def consume(connections = @max_connects, &block)
+      @multi.perform do
+        while request_count < connections && new_link = yield
+          add(new_link) if new_link
+        end
+      end
+    end
+
+  end
+
+  # The ListConsumingQueue is similar to the ConsumingQueue except that
+  # it takes its argument in the form of an Enumerable object.
+  class ListConsumingQueue < ConsumingQueue
+
+    # Download all of the URLs or Curl::Easy objects in the given list, and
+    # optionally execute the given block on completion for each
+    def consume(list, connections = @max_connects)
+      item = 0            # Start at item 0
+      list = list.to_a    # Ensure we can address with []
+
+      @multi.perform do
+        while request_count < connections && new_link = list[item]
+
+          item += 1
+
+          # Add with config block if appropriate
+          if block_given?
+            add(new_link) { |req| yield(req) }
+          else
+            add(new_link)
+          end
+
+        end
+      end
+    end
+  end
+
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: blat
 version: !ruby/object:Gem::Version
-  version: 0.1.0a
+  version: 0.1.0b
 platform: ruby
 authors:
 - Stephen Wattam
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-06-02 00:00:00.000000000 Z
+date: 2013-06-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: curb
@@ -24,15 +24,15 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '0.8'
-description: A very parallel cURL wrapper for ongoing download tasks
+description: Curl::Multi wrapper for high-performance and/or long-running download
+  tasks
 email: stephenwattam@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- lib/blat/pool.rb
-- lib/blat/formats.rb
 - lib/blat/batch.rb
+- lib/blat/queue.rb
 - ./lib/blat.rb
 homepage: http://stephenwattam.com/projects/blat
 licenses:

data/lib/blat/formats.rb

@@ -1,118 +0,0 @@
-
-
-module Blat
-
-  # Blat::Job represents a single download task, both as a request and response.
-  #
-  # Jobs are provided to workers in a pool by a dispatcher block.  Each job
-  # contains:
-  #
-  # * Configuration for the worker.  Current configuration supported is
-  # detailed below and in the Pool documentation
-  # * A way of configuring a curl request (in order to set the url and other
-  # parameters)
-  # * Data returned by the download.  This is stored as a hash in the #data
-  # parameter.
-  #
-  # == Worker Configuration
-  #
-  # Workers are configured by setting values in a hash.  This hash is sent to
-  # the worker from the Job class, and contains options that affect the process
-  # of downloading.  This is in addition to configuration on the curl object
-  # performed through Blat::Job.configure()
-  #
-  # Workers currently support the following configuration options:
-  #
-  # [:max_body_size] If set, downloads will cease after this many bytes have
-  # been downloaded.  If truncated, data[:response_properties][:truncated] will
-  # be set to true.
-  #
-  # == Returned Values
-  #
-  # When a job has been finalised, its #data property will be set to a hash
-  # left by the worker.  This is currently specified as:
-  #
-  # [:head] The head string returned from the server (response.header_str)
-  # [:body] The body string returned from the server (response.body)
-  # [:response_properties] A hash with metadata in.  Partially specified by the
-  # worker configuration, this contains things such as the number of bytes
-  # downloaded and duration of the request.
-  # [:response] The raw response from curl
-  # [:error] Any errors encountered during download, such as network errors.
-  # If this is nil the request was successful.
-  # 
-  # Response properties are currently set to:
-  #
-  #  response_properties = {
-  #    round_trip_time:    res.total_time,
-  #    redirect_time:      res.redirect_time,
-  #    dns_lookup_time:    res.name_lookup_time,
-  #    effective_uri:      res.last_effective_url,
-  #    code:               res.response_code,
-  #    download_speed:     res.download_speed,
-  #    downloaded_bytes:   res.downloaded_bytes || 0,
-  #    truncated:          ignore == true
-  #  }
-  #
-  class Job
-
-    attr_reader :config, :data 
-
-    # Construct a new Job with a block for configuring curl options.
-    def initialize(config = {}, &block)
-      raise 'No curl configuration block given' unless block_given?
-
-      @curl_config_block  = block
-      @config             = config
-      @finalised          = false
-    end
-
-    # Configure a curl object to make the request
-    def configure(curl)
-      @curl_config_block.yield(curl)
-    end
-
-    # Has this job been completed?
-    def finalised?
-      @finalise
-    end
-
-    # Allow people to use closed? instead.
-    alias :closed? :finalised?
-
-    # Write result and prevent further editing
-    def finalise!(data = {})
-      raise 'Job is already finalised.' if finalised?
-      @data = data
-      @finalised = true
-    end
-  end
-
-  # --------------------------------------------------------------------------
-  
-
-  # SimpleJob is a quick and easy way of wrapping a URL to create a job.
-  #
-  # It accepts:
-  #
-  # [:url] The URL to download
-  # [:curl_config] A hash of properties to set on the curl object, for example: {'follow_location' => true}
-  # [:config] The worker configuration properties.
-  class SimpleJob < Job
-    def initialize(url, curl_config = {}, config = {})
-      curl_config.merge!({url: url})
-
-      super(config){ |c|
-        curl_config.each do |k,v|
-          if v.is_a?(Array)
-            curl.send(k.to_s + '=', *v) 
-          else
-            curl.send(k.to_s + '=', v) 
-          end
-        end
-      }
-    end
-  end
-
-
-end

data/lib/blat/pool.rb

@@ -1,432 +0,0 @@
-
-require 'thread'
-require 'curl'
-
-require 'blat/formats'
-
-module Blat 
-
-  # The Blat::Pool class controls a number of workers as they go about running
-  # curl Jobs.  This is the main class of Blat, and is the most flexible way of
-  # using the gem (Batch is simpler but less full-featured).
-  #
-  # == Workflow
-  #
-  # The pool is created with a size and a callback to present results to.
-  # This callback may be presented as a proc object or as a block, and is
-  # called with a finalised Blat::Job object upon completion of each request.
-  #
-  #  x = Blat::Pool.new(100){ |job|
-  #         puts "#{job.data[:body]}"
-  #      }
-  #
-  # Once a pool is configured, it may be commanded to start downloading by
-  # presenting it with a dispatcher.  This is a procedure that returns either a
-  # Blat::Job object or nil---workers will call this block in order to acquire
-  # work, and will enter an idle state when nil is returned.
-  #
-  #  job_list = File.read('urls').lines.map{ |l| Blat::SimpleJob.new(l) }
-  #  
-  #  x.work{
-  #   job_list.pop
-  #  }
-  #
-  # Downloading can be waited upon any number of ways.  The status of the pool
-  # may be requested with #count_idle and #all_idle? , and it's possible to
-  # wait until idle using #wait_until_idle :
-  #
-  #  x.wait_until_idle
-  #  x.close
-  #
-  # == Worker Configuration
-  #
-  # Workers are configured by setting values in a hash.  This hash is sent to
-  # the worker from the Job class, and contains options that affect the process
-  # of downloading.  This is in addition to configuration on the curl object
-  # performed through Blat::Job.configure()
-  #
-  # Workers currently support the following configuration options:
-  #
-  # [:max_body_size] If set, downloads will cease after this many bytes have
-  # been downloaded.  If truncated, data[:response_properties][:truncated] will
-  # be set to true.
-  #
-  # == Returned Values
-  #
-  # When a job has been finalised, its #data property will be set to a hash
-  # left by the worker.  This is currently specified as:
-  #
-  # [:head] The head string returned from the server (response.header_str)
-  # [:body] The body string returned from the server (response.body)
-  # [:response_properties] A hash with metadata in.  Partially specified by the
-  # worker configuration, this contains things such as the number of bytes
-  # downloaded and duration of the request.
-  # [:response] The raw response from curl
-  # [:error] Any errors encountered during download, such as network errors.
-  # If this is nil the request was successful.
-  # 
-  # Response properties are currently set to:
-  #
-  #  response_properties = {
-  #    round_trip_time:    res.total_time,
-  #    redirect_time:      res.redirect_time,
-  #    dns_lookup_time:    res.name_lookup_time,
-  #    effective_uri:      res.last_effective_url,
-  #    code:               res.response_code,
-  #    download_speed:     res.download_speed,
-  #    downloaded_bytes:   res.downloaded_bytes || 0,
-  #    truncated:          ignore == true
-  #  }
-  #
-  class Pool
-
-    # Construct a new pool with a given size and a callback used to output
-    # data.
-    #
-    #  x = Blat::Pool.new(100){ |job|
-    #         puts "Job complete: #{job}"
-    #      }
-    #
-    def initialize(size, finalise_callback = nil, &block)
-
-      @m              = Mutex.new # Data mutex for "producer" status
-      @t              = {} # threads
-      @w              = [] # workers
-      @idle           = []
-      @idle_mutex     = Mutex.new
-      @size           = size.to_i # number of simultaneous workers
-
-      # Pass a block for handling returns
-      if block
-        @finalise_callback = block
-      elsif finalise_callback && finalise_callback.is_a?(Proc)
-        @finalise_callback = finalise_callback
-      else
-        raise 'No callback given for final data'
-      end
-
-    end
-
-    # ------------------------------------------------------------------------
-    # Workers call these to report status
-    #
-
-    # Workers can register as active by calling this
-    def worker_active(worker_id)
-      @idle_mutex.synchronize{ @idle[worker_id] = false }
-    end
-
-    # Workers can register as idle by calling this
-    def worker_idle(worker_id)
-      @idle_mutex.synchronize{ @idle[worker_id] = true }
-    end
-
-    # Workers can register that they have completed
-    # a job by calling this.
-    def work_complete(job)
-      @finalise_callback.call(job) 
-    end
-
-    # ------------------------------------------------------------------------
-    # Worker status
-    #
-
-    # check to see if all workers are idle
-    def all_idle?
-      @idle_mutex.synchronize{ @idle.inject(true) { |m, o| m && o} }
-    end
-
-    # Return the number of idle workers
-    def count_idle
-      @idle_mutex.synchronize{ @idle.count(true) }
-    end
-
-    # ------------------------------------------------------------------------
-    # Set work and initialise workers
-    #
-    
-    # Create workers without running them.
-    #
-    # This is usually not very useful to call on its own, and is called by
-    # #work when creating threads.
-    def init_workers
-      #$log.debug "Maintaining #{@size} worker object[s] (#{@w.length} currently active)."
-      @w = []
-      (@size - @w.length).times do |s|
-        @w << Worker.new(s, self)
-        @idle[s] = true
-      end
-      #$log.info "#{@w.length} worker[s] created."
-    end
-
-    # Run a worker over every point competitively.
-    # Will create @size workers if they do not already exist (there is no need
-    # to also call init_workers)
-    def work(dispatcher = nil, &block)
-
-      raise "No dispatcher provided" unless block_given? || (dispatcher && dispatcher.is_?(Proc))
-
-      init_workers
-
-      # Make things do the work
-      #$log.debug "Starting threads..."
-      @start_time = Time.now
-      @w.each do |w|
-        # Give each worker a handle back to the dispatcher to get data.
-        @t[w] = Thread.new(dispatcher || block) do |d|
-          begin
-            w.work(d)
-          rescue SignalException => e
-            #$log.fatal "Signal caught: #{e.message}"
-            #$log.fatal "Since I'm sampling right now, I will kill workers before shutdown."
-            kill_workers
-            raise e
-          end
-        end
-
-        # Pass exceptions up
-        @t[w].abort_on_exception = true
-      end
-      #$log.info "#{@t.length} download thread[s] started."
-    end
-
-    # ------------------------------------------------------------------------
-    # Wait on conditions and close the pool
-    #
-
-    # Block until all workers are idle, checking every poll_rate seconds.
-    def wait_until_idle(poll_rate = 0.5)
-      #$log.debug "Waiting until idle, polling every #{poll_rate}s..."
-      sleep(poll_rate)
-      sleep(poll_rate) until all_idle?
-    end
-
-    # Wait for threads to complete.
-    def wait_until_closed
-      #$log.debug "Waiting for #{@t.length} worker[s] to close."
-      @t.each { |w, t| t.join }
-      #$log.info "Workers all terminated naturally."
-    end
-
-    # Tell workers to die forcibly
-    def kill_workers
-      #$log.debug "Forcing #{@t.length} worker threads to die..."
-      @t.each { |t| t.kill }
-      #$log.info "Worker threads killed."
-    end
-
-    # Close all workers' connections to the servers cleanly,
-    #
-    # This is non-blocking.  Call #close or #wait to block:
-    #
-    #  pool.close_nonblock
-    #  pool.wait_until_closed
-    #
-    def close_nonblock
-      #$log.debug "Requesting closure of #{@w.length} worker[s]..."
-      @w.each { |w| w.close }
-    end
-
-    # Cleanly close the pool, waiting for workers to end their
-    # current request.  Blocks, unlike #close.
-    def close
-      close_nonblock
-      wait_until_closed
-    end
-
-  private
-
-    # Workers are instantiated and maintained by a Blat::Pool and continually
-    # poll for available work, passing it off for integration with the final
-    # results set.
-    #
-    # Though it is possible to create your own, I would recommend instead using
-    # a pool.
-    #
-    # == Worker Configuration
-    #
-    # Workers are configured by setting values in a hash.  This hash is sent to
-    # the worker from the Job class, and contains options that affect the process
-    # of downloading.  This is in addition to configuration on the curl object
-    # performed through Blat::Job.configure()
-    #
-    # Workers currently support the following configuration options:
-    #
-    # [:max_body_size] If set, downloads will cease after this many bytes have
-    # been downloaded.  If truncated, data[:response_properties][:truncated] will
-    # be set to true.
-    #
-    # == Returned Values
-    #
-    # When a job has been finalised, its #data property will be set to a hash
-    # left by the worker.  This is currently specified as:
-    #
-    # [:head] The head string returned from the server (response.header_str)
-    # [:body] The body string returned from the server (response.body)
-    # [:response_properties] A hash with metadata in.  Partially specified by the
-    # worker configuration, this contains things such as the number of bytes
-    # downloaded and duration of the request.
-    # [:response] The raw response from curl
-    # [:error] Any errors encountered during download, such as network errors.
-    # If this is nil the request was successful.
-    # 
-    # Response properties are currently set to:
-    #
-    #  response_properties = {
-    #    round_trip_time:    res.total_time,
-    #    redirect_time:      res.redirect_time,
-    #    dns_lookup_time:    res.name_lookup_time,
-    #    effective_uri:      res.last_effective_url,
-    #    code:               res.response_code,
-    #    download_speed:     res.download_speed,
-    #    downloaded_bytes:   res.downloaded_bytes || 0,
-    #    truncated:          ignore == true
-    #  }
-    #
-    class Worker
-
-      # Construct a new worker with a given ID and linked to a given pool.
-      #
-      # The pool will be called to report idle/working states.
-      def initialize(id, pool)
-        @id       = id
-        @pool     = pool
-        @abort    = false
-      end
-
-      # Should be run in a thread.  Performs work until the dispatcher runs 
-      # out of data.
-      def work(dispatcher)
-        # start idle
-        last_idle_state = true
-
-        loop do
-          while (job = dispatcher.call).is_a?(Job) do
-
-            # If we were idle last, tell the pool
-            @pool.worker_active(@id) if last_idle_state == true
-
-            # tell people
-            #$log.debug "W#{@id}: Downloading job #{job}"
-
-            # Make the request
-            complete_request(job, new_curl(job), job.config)
-
-            return if @abort 
-          end
-          return if @abort
-
-          # TODO: configurable
-          @pool.worker_idle(@id)
-          last_idle_state = true
-          sleep(1)
-        end
-
-      # rescue StandardError => e
-        #$log.warn "W#{@id}: Error: #{e}"
-        #$log.debug "#{e.backtrace.join("\n")}"
-      end
-
-      # Closes the connection to the server
-      def close
-        @abort = true
-      end
-
-    private
-
-      # Datapoint is complete, run callback
-      def finalise(job, head, body, response_properties, response, error)
-        job.finalise!( 
-                      head: head,
-                      body: body,
-                      response_properties: response_properties,
-                      response: response,
-                      error: error
-                     )
-
-        @pool.work_complete(job)
-      end
-
-      # ---------- called by workers below this line
-
-      # Submit a complete dp to the pool
-      def complete_request(job, res, config)
-
-        # Somewhere to store the body in a size-aware way
-        body    = ''
-
-        # If limiting body size, use a callback to handle incoming data
-        if config[:max_body_size]
-          ignore  = false
-
-          res.on_body do |str|
-            # Read up to the limit of bytes
-            if !ignore && config[:max_body_size] && (body.length + str.length) > config[:max_body_size]
-              body += str[0..(body.length + str.length) - config[:max_body_size]]
-              #$log.warn "W#{@id}: Job #{job} exceeded byte limit (#{config[:max_body_size]}b)"
-              ignore = true
-            elsif not ignore
-              body += str
-            else
-              # ignore data
-            end
-
-            # Have to return number of bytes to curb
-            str.length
-          end
-        end
-
-        # Perform a request prepared elsewhere,
-        # can run alongside other requests
-        res.perform
-
-        # Load body directly from response if not using the system above
-        body = res.body_str unless config[:max_body_size]
-
-        # Load stuff out of response object.
-        response_properties = {
-          round_trip_time:    res.total_time,
-          redirect_time:      res.redirect_time,
-          dns_lookup_time:    res.name_lookup_time,
-          effective_uri:      res.last_effective_url,
-          code:               res.response_code,
-          download_speed:     res.download_speed,
-          downloaded_bytes:   res.downloaded_bytes || 0,
-          truncated:          ignore == true
-        }
-
-        # write to datapoint list
-        finalise(job, res.header_str, body, response_properties, res, nil)
-
-      rescue SignalException => e
-        raise e
-      rescue StandardError => e
-        # if e.class.to_s =~ /^Curl::Err::/ then
-        #   #$log.debug "W#{@id}: Job #{job}: #{e.to_s[11..-1]}"
-        # else
-        #   #$log.error "W#{@id}: Exception retrieving #{job}: #{e.to_s}."
-        #   #$log.debug "#{e.backtrace.join("\n")}"
-        # end
-
-        # write to datapoint list
-        finalise(job, res.header_str, body, response_properties, res, e)
-      end
-
-      # Returns a new curl object to use downloading things.
-      def new_curl(job)
-        # Set up curl
-        c = Curl::Easy.new
-
-        # Configure the curl object
-        job.configure(c)
-
-        # Return it for work
-        return c
-      end
-
-    end
-
-  end
-
-end
-