extraloop 0.0.2 → 0.0.3

data/lib/extraloop/json_extractor.rb

@@ -1,36 +1,37 @@
-require 'pry'
-class JsonExtractor < ExtractorBase
+module ExtraLoop
+  class JsonExtractor < ExtractorBase
 
-  def initialize(*args)
-    @path = args[2] && args[2].is_a?(Array) ? args[2] : nil
-    super(*args)
-  end
+    def initialize(*args)
+      @path = args[2] && args[2].is_a?(Array) ? args[2] : nil
+      super(*args)
+    end
 
-  def extract_field(node, record=nil)
-    output = node = node.is_a?(String) ? parse(node) : node
-    output = node.get_in(@path) if @path
-    output = node[@attribute.to_s] if @attribute
-    output = @environment.run(output, record, &@callback) if @callback
+    def extract_field(node, record=nil)
+      output = node = node.is_a?(String) ? parse(node) : node
+      output = node.get_in(@path) if @path
+      output = node[@attribute.to_s] if @attribute
+      output = @environment.run(output, record, &@callback) if @callback
 
-    # when no attribute and no callback is provided, try fetching by field name
-    if !@attribute && !@callback  
-     output = node[@field_name.to_s] if node[@field_name.to_s]
+      # when no attribute and no callback is provided, try fetching by field name
+      if !@attribute && !@callback  
+       output = node[@field_name.to_s] if node[@field_name.to_s]
+      end
+      output
     end
-    output
-  end
 
-  def extract_list(input)
-    #TODO: implement more clever stuff here after looking 
-    # into possible hash traversal techniques
+    def extract_list(input)
+      #TODO: implement more clever stuff here after looking 
+      # into possible hash traversal techniques
 
-    input = input.is_a?(String) ? parse(input) : input
-    input = input.get_in(@path) if @path
+      input = input.is_a?(String) ? parse(input) : input
+      input = input.get_in(@path) if @path
 
-    @callback && Array(@environment.run(input, &@callback)) || input
-  end
+      @callback && Array(@environment.run(input, &@callback)) || input
+    end
 
-  def parse(input)
-    super(input)
-    @environment.document = (Yajl::Parser.new).parse(input).extend(Utils::DeepFetchable)
+    def parse(input)
+      super(input)
+      @environment.document = (Yajl::Parser.new).parse(input).extend(Utils::DeepFetchable)
+    end
   end
 end

data/lib/extraloop/loggable.rb

@@ -1,64 +1,65 @@
 autoload :Logging,  "logging"
 
-# Decorates a class with an instance of Logging.logger and a convenient 
-# helper method to log messages.
+module ExtraLoop
+  # Decorates a class with an instance of Logging.logger and a convenient 
+  # helper method to log messages.
 
-module Loggable
-  protected
+  module Loggable
+    protected
 
-  #
-  # Initializes the incorporated logger object.
-  #
-  # Returns nothing.
-  #
+    #
+    # Initializes the incorporated logger object.
+    #
+    # Returns nothing.
+    #
 
-  def init_log!
-    return unless @options[:log]
+    def init_log!
+      return unless @options[:log]
 
-    @options[:log] ||= {
-      :appenders => [ Logging.appenders.stderr ],
-      :log_level => :info
-    }
+      @options[:log] ||= {
+        :appenders => [ Logging.appenders.stderr ],
+        :log_level => :info
+      }
 
-    if @options[:log] && @options[:log][:appenders] && @options[:log][:appenders].any?
-      @log = Logging.logger["#{self}"]
-      @log.add_appenders(@options[:log][:appenders])
-      @log.level = @options[:log] && @options[:log][:log_level] || :info
+      if @options[:log] && @options[:log][:appenders] && @options[:log][:appenders].any?
+        @log = Logging.logger["#{self}"]
+        @log.add_appenders(@options[:log][:appenders])
+        @log.level = @options[:log] && @options[:log][:log_level] || :info
+      end
     end
-  end
 
-  # 
-  # Convenience method for logging messages.
-  #
-  # messages  - the message content
-  # log_level - the message's log level (can be either :info, :debug, :error, :warning; defaults to :info)
-  #
-  # Returns nothing.
-  #
+    # 
+    # Convenience method for logging messages.
+    #
+    # messages  - the message content
+    # log_level - the message's log level (can be either :info, :debug, :error, :warning; defaults to :info)
+    #
+    # Returns nothing.
+    #
 
-  def log(message, log_level = :info)
-    @log.send(log_level, message) if @log
+    def log(message, log_level = :info)
+      @log.send(log_level, message) if @log
+    end
   end
-end
-
-
-# 
-#  Monkey patches ScraperBase.
-#
-class ScraperBase
-  include Loggable
-  alias_method :base_initialize, :initialize
 
+  # 
+  #  Monkey patches ScraperBase.
   #
-  # Wrapp ScraperBase#initialize method into Loggable#initialize
-  #
-  # args - The arguments to be passed over to the ScraperBase#initialize method.
-  #
-  # Returns itself.
-  #
-  def initialize(*args)
-    base_initialize(*args)
-    init_log!
-    self
+  class ScraperBase
+    include Loggable
+    alias_method :base_initialize, :initialize
+
+    #
+    # Wrapp ScraperBase#initialize method into Loggable#initialize
+    #
+    # args - The arguments to be passed over to the ScraperBase#initialize method.
+    #
+    # Returns itself.
+    #
+    def initialize(*args)
+      base_initialize(*args)
+      init_log!
+      self
+    end
   end
 end

data/lib/extraloop/scraper_base.rb

@@ -1,166 +1,169 @@
-class ScraperBase
-  include Hookable
-  include Utils::Support
-
-  attr_reader :results, :options
-
-  #
-  # Public: Initalizes a web scraper.
-  #
-  # urls      - One or several urls.
-  # options   - Hash of scraper options
-  #   async        : Whether the scraper should issue HTTP requests in series or in parallel (set to false to suppress logging completely).
-  #   log          : logging options (defaults to standard error).
-  #     appenders    : specifies where the log messages should be appended to (defaults to standard error).
-  #     log_level    : specifies the log level (defaults to info).
-  # arguments - Hash of arguments to be passed to the Typhoeus HTTP client (optional).
-  #
-  #
-  #
-  # Returns itself.
-  #
-
-  def initialize(urls, options = {}, arguments = {})
-    @urls = Array(urls)
-    @loop_extractor_args = nil
-    @extractor_args = []
-    @loop = nil
-
-    @request_arguments = arguments
-
-    @options = {
-      :async  => false
-    }.merge(options)
-
-
-    @response_count = 0
-    @queued_count = 0
-
-    @hooks = {}
-    @failed_requests = []
-
-    hydra_options = @options[:hydra] && @options[:hydra][:max_concurrency] || {:max_concurrency => 10}
-    @hydra = Typhoeus::Hydra.new hydra_options
-    self
-  end
+module ExtraLoop
+  class ScraperBase
+    include Hookable
+    include Utils::Support
+
+    attr_reader :results, :options
+
+    #
+    # Public: Initalizes a web scraper.
+    #
+    # urls      - One or several urls.
+    # options   - Hash of scraper options
+    #   async        : Whether the scraper should issue HTTP requests in series or in parallel (set to false to suppress logging completely).
+    #   log          : logging options (defaults to standard error).
+    #     appenders    : specifies where the log messages should be appended to (defaults to standard error).
+    #     log_level    : specifies the log level (defaults to info).
+    # arguments - Hash of arguments to be passed to the Typhoeus HTTP client (optional).
+    #
+    #
+    #
+    # Returns itself.
+    #
+
+    def initialize(urls, options = {}, arguments = {})
+      @urls = Array(urls)
+      @loop_extractor_args = nil
+      @extractor_args = []
+      @loop = nil
+
+      @request_arguments = arguments
+
+      @options = {
+        :async  => false
+      }.merge(options)
+
+
+      @response_count = 0
+      @queued_count = 0
+
+      @hooks = {}
+      @failed_requests = []
+
+      hydra_options = @options[:hydra] && @options[:hydra][:max_concurrency] || {:max_concurrency => 10}
+      @hydra = Typhoeus::Hydra.new hydra_options
+      self
+    end
 
 
-  # Public: Sets the scraper extraction loop.
-  #
-  # Delegates to Extractor, will raise an exception if neither a selector, a block, or an attribute name is provided.
-  #
-  #
-  # selector  - The CSS3 selector identifying the node list over which iterate (optional).
-  # callback  - A block of code (optional).
-  # attribute - An attribute name (optional).
-  #
-  # Returns itself.
-  #
-
-  def loop_on(*args)
-    @loop_extractor_args = args.insert(0, nil, ExtractionEnvironment.new(self))
-    self
-  end
+    # Public: Sets the scraper extraction loop.
+    #
+    # Delegates to Extractor, will raise an exception if neither a selector, a block, or an attribute name is provided.
+    #
+    #
+    # selector  - The CSS3 selector identifying the node list over which iterate (optional).
+    # callback  - A block of code (optional).
+    # attribute - An attribute name (optional).
+    #
+    # Returns itself.
+    #
+
+    def loop_on(*args)
+      @loop_extractor_args = args.insert(0, nil, ExtractionEnvironment.new(self))
+      self
+    end
 
-  # Public: Registers a new extractor to be added to the loop.
-  #
-  # Delegates to Extractor, will raise an exception if neither a selector, a block, or an attribute name is provided.
-  #
-  # selector  - The CSS3 selector identifying the node list over which iterate (optional).
-  # callback  - A block of code (optional).
-  # attribute - An attribute name (optional).
-  #
-  # Returns itself.
-  #
-  #
-
-  def extract(*args)
-    @extractor_args << args.insert(1, ExtractionEnvironment.new(self))
-    self
-  end
+    # Public: Registers a new extractor to be added to the loop.
+    #
+    # Delegates to Extractor, will raise an exception if neither a selector, a block, or an attribute name is provided.
+    #
+    # selector  - The CSS3 selector identifying the node list over which iterate (optional).
+    # callback  - A block of code (optional).
+    # attribute - An attribute name (optional).
+    #
+    # Returns itself.
+    #
+    #
+
+    def extract(*args)
+      @extractor_args << args.insert(1, ExtractionEnvironment.new(self))
+      self
+    end
 
-  #
-  # Public: Runs the main scraping loop.
-  #
-  # Returns nothing
-  #
-  def run
-    @urls.each do |url|
-      issue_request(url)
-
-      # if the scraper is asynchronous start processing the Hydra HTTP queue 
-      # only after that the last url has been appended to the queue (see #issue_request).
-      #
-      if @options[:async]
-        if url == @urls.last
+    #
+    # Public: Runs the main scraping loop.
+    #
+    # Returns nothing
+    #
+    def run
+      @urls.each do |url|
+        issue_request(url)
+
+        # if the scraper is asynchronous start processing the Hydra HTTP queue 
+        # only after that the last url has been appended to the queue (see #issue_request).
+        #
+        if @options[:async]
+          if url == @urls.last
+            @hydra.run
+          end
+        else
           @hydra.run
         end
-      else
-        @hydra.run
       end
+      self
     end
-    self
-  end
 
-  protected
+    protected
 
-  def issue_request(url)
+    def issue_request(url)
 
-    @request_arguments[:params] = merge_request_parameters(url)
-    url_without_params = url.gsub(/\?.*/,"")
+      @request_arguments[:params] = merge_request_parameters(url)
+      url_without_params = url.gsub(/\?.*/,"")
 
-    arguments = {
-      'headers' => [
-        'User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64; rv:6.0a2) Gecko/20110613 Firefox/6.0a2',
-        'accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
-      ].join("\n")
-    }
+      arguments = {
+        'headers' => [
+          'User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64; rv:6.0a2) Gecko/20110613 Firefox/6.0a2',
+          'accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
+        ].join("\n")
+      }
 
-    arguments.merge!(@request_arguments)
-    request = Typhoeus::Request.new(*[url_without_params, arguments])
+      arguments.merge!(@request_arguments)
+      request = Typhoeus::Request.new(*[url_without_params, arguments])
 
-    request.on_complete do |response|
-      handle_response(response)
-    end
+      request.on_complete do |response|
+        handle_response(response)
+      end
 
-    log("queueing url: #{url}, params #{arguments[:params]}", :debug)
-    @queued_count += 1
-    @hydra.queue(request)
-  end
+      log("queueing url: #{url}, params #{arguments[:params]}", :debug)
+      @queued_count += 1
+      @hydra.queue(request)
+    end
 
-  def merge_request_parameters(url)
-    url_params = URI::parse(url).extend(Utils::URIAddition).query_hash
-    return @request_arguments[:params] || {} unless url_params && url_params.respond_to?(:merge)
+    def merge_request_parameters(url)
+      url_params = URI::parse(url).extend(Utils::URIAddition).query_hash
+      return @request_arguments[:params] || {} unless url_params && url_params.respond_to?(:merge)
 
-    params = symbolize_keys(@request_arguments[:params] ||= {})
-    url_params.merge(params)
-  end
+      params = symbolize_keys(@request_arguments[:params] ||= {})
+      url_params.merge(params)
+    end
 
-  def handle_response(response)
-    @response_count += 1
-    @loop = prepare_loop(response)
-    log("response ##{@response_count} of #{@queued_count}, status code: [#{response.code}], URL fragment: ...#{response.effective_url.split('/').last if response.effective_url}")
-    @loop.run
+    def handle_response(response)
+      @response_count += 1
+      @loop = prepare_loop(response)
+      log("response ##{@response_count} of #{@queued_count}, status code: [#{response.code}], URL fragment: ...#{response.effective_url.split('/').last if response.effective_url}")
+      @loop.run
 
-    @environment = @loop.environment
-    run_hook(:data, [@loop.records, response])
-  end
+      @environment = @loop.environment
+      run_hook(:data, [@loop.records, response])
+    end
 
-  def prepare_loop(response)
-    format = @options[:format] || detect_format(response.headers_hash.fetch('Content-Type', nil))
-    extractor_class = format == :json ? JsonExtractor : DomExtractor
-    loop_extractor = extractor_class.new(*@loop_extractor_args)
-    extractors = @extractor_args.map { |args|  extractor_class.new(*args) }
-    ExtractionLoop.new(loop_extractor, extractors, response.body, @hooks, self)
-  end
+    def prepare_loop(response)
+      format = @options[:format] || detect_format(response.headers_hash.fetch('Content-Type', nil))
+      extractor_class = format == :json ? JsonExtractor : DomExtractor
+      loop_extractor = extractor_class.new(*@loop_extractor_args)
+      extractors = @extractor_args.map { |args|  extractor_class.new(*args) }
+      ExtractionLoop.new(loop_extractor, extractors, response.body, @hooks, self)
+    end
 
-  def detect_format(content_type)
-    #TODO: add support for xml/rdf documents
-    if content_type && content_type =~ /json$/
-      :json
-    else
-      :html
+    def detect_format(content_type)
+      #TODO: add support for xml/rdf documents
+      if content_type && content_type =~ /json$/
+        :json
+      else
+        :html
+      end
     end
+
   end
 
 end