scruber 0.1.4 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8deee66960a3768ace0af72a5cb1eced62c90329
-  data.tar.gz: a0d3f330d8b838aee078f2d752226a1e5432b311
+  metadata.gz: 4661c429c6b33a12841569c62835cc587e7f9464
+  data.tar.gz: 9cb87a48248746b30d2ece9db71136560629bdcb
 SHA512:
-  metadata.gz: 30df32ccd86afde913d47483e9f327b94869c52a21f7c1a43f442ef8a1f138a1500d0746d4641a4542937aa2dbba7e53e4697c95b234cfdc3d07eeb8ab3d13ed
-  data.tar.gz: 4e57023647a62f7f312a8a77b89097920e7ab6750c1fd6562d1b6b6b4b3ff239b484f08aed6920a259365100ba4efe9d3f168b1a873183a377c877261d49ba15
+  metadata.gz: c86674726ae45109383e8d0712612895136384eba1414e061c7d1de3dd9b699af6ab4022de037a537b1377a95b46be2aba10b4e9cea1664449ec880ae1b8189d
+  data.tar.gz: 738d34f37fdc629eb3f4755fffeb70e640e53bd7e33333357af0f9af286d648dd92846220a7ef0f9d06918e412fdf8c801a0b25e15cf74b912b0e0459e04009d

data/lib/core_ext/const_missing.rb

@@ -0,0 +1,9 @@
+class Object
+  def self.const_missing(name)
+    if Scruber::Helpers.const_defined?(name)
+      Scruber::Helpers.const_get(name)
+    else
+      super
+    end
+  end
+end

data/lib/scruber/cli/project_generator.rb

@@ -19,6 +19,8 @@ module Scruber
         raise ::Thor::Error, "ERROR: #{path} already exists." if File.exist?(path)
         say "Creating scruber project at #{path}"
         FileUtils.mkdir_p(path)
+        FileUtils.mkdir_p(path+'/lib')
+        FileUtils.mkdir_p(path+'/log')
       end
 
       def create_files

data/lib/scruber/cli/templates/application.tt

@@ -15,4 +15,5 @@ Scruber.configure do |config|
   config.fetcher_agent_options = {}
   config.queue_adapter = :<%= options[:queue] %>
   config.queue_options = {}
+  config.autoload_paths << Scruber.root.join('lib')
 end

data/lib/scruber/cli.rb

@@ -16,6 +16,7 @@ module Scruber
       register Generators, 'generate', 'generate [GENERATOR]', 'Generate something'
 
       desc 'start', 'Run scraper'
+      method_option :silent, :type => :boolean, :aliases => '-s', default: false
       def start(name)
         if defined?(APP_PATH)
           scraper_path = Scruber::AppSearcher.find_scraper(name, APP_PATH)
@@ -28,6 +29,7 @@ module Scruber
           ENV['SCRUBER_SCRAPER_NAME'] = File.basename(scraper_path).gsub(/\.rb\Z/, '').underscore
           say "starting #{ENV['SCRUBER_SCRAPER_NAME']}"
           
+          Scruber.configuration.silent = options[:silent]
           require scraper_path
         else
           raise ::Thor::Error, "ERROR: Scruber project not found."

data/lib/scruber/core/configuration.rb

@@ -1,12 +1,26 @@
 module Scruber
   module Core
+    # 
+    # Configuration class
+    # 
+    # @author Ivan Goncharov
+    # @attr [Symbol] fetcher_adapter Fetcher adapter name
+    # @attr [Hash] fetcher_options Fetcher options, see {Scruber::FetcherAdapters::AbstractAdapter} options
+    # @attr [Symbol] fetcher_agent_adapter Fetcher agent adapter name
+    # @attr [Hash] fetcher_agent_options Fetcher agent options, see {Scruber::Helpers::FetcherAgentAdapters::AbstractAdapter}
+    # @attr [Symbol] queue_adapter Queue adapter name
+    # @attr [Hash] queue_options Queue options, see {Scruber::QueueAdapters::AbstractAdapter}
+    # @attr [Array<String>] autoload_paths Array with paths for autoloading classes
+    # @attr [Boolean] silent Don't output anything if true
     class Configuration
       attr_accessor :fetcher_adapter,
                     :fetcher_options,
                     :fetcher_agent_adapter,
                     :fetcher_agent_options,
                     :queue_adapter,
-                    :queue_options
+                    :queue_options,
+                    :autoload_paths,
+                    :silent
 
       def initialize
         @fetcher_adapter = :typhoeus_fetcher
@@ -15,8 +29,15 @@ module Scruber
         @fetcher_agent_options = {}
         @queue_adapter = :memory
         @queue_options = {}
+        @autoload_paths = []
+        @silent = false
       end
 
+      # 
+      # Merge options from hash
+      # @param options [Hash] options
+      # 
+      # @return [void]
       def merge_options(options)
         @fetcher_adapter = options.fetch(:fetcher_adapter){ @fetcher_adapter }
         @fetcher_options.merge! options.fetch(:fetcher_options){ {} }
@@ -24,6 +45,8 @@ module Scruber
         @fetcher_agent_options.merge! options.fetch(:fetcher_agent_options){ {} }
         @queue_adapter = options.fetch(:queue_adapter){ @queue_adapter }
         @queue_options.merge! options.fetch(:queue_options){ {} }
+        @autoload_paths += options.fetch(:autoload_paths){ [] }
+        @silent = options.fetch(:silent){ false }
       end
     end
   end

data/lib/scruber/core/crawler.rb

@@ -1,8 +1,33 @@
 module Scruber
   module Core
+    # 
+    # Crawler class
+    # 
+    # Main class-runner for scrapers.
+    # 
+    # @example Simple scraper
+    #   Scruber::Core::Crawler.new(:simple) do
+    #     get 'http://example.com'
+    #     parse :html do |page,html|
+    #       puts html.at('title').text
+    #     end
+    #   end
+    # 
+    # @author Ivan Goncharov
+    #     
     class Crawler
       attr_reader :queue, :fetcher, :scraper_name
 
+      # 
+      # Initialize crawler with scraper name and/or with options
+      #     
+      #   Crawler.new(:sample, fetcher_adapter: :custom)
+      #   Crawler.new(:sample)
+      #   Crawler.new(fetcher_adapter: :custom)
+      # 
+      # @param args [Array] if first arg is a Symbol, it will be used as scraper_name, hash will me used as configuration options (see {Scruber::Core::Configuration})
+      # 
+      # @return [Scruber::Core::Crawler] [description]
       def initialize(*args)
         if args.first.is_a?(Hash)
           scraper_name = nil
@@ -14,23 +39,29 @@ module Scruber
         @scraper_name = scraper_name.present? ? scraper_name : ENV['SCRUBER_SCRAPER_NAME']
         raise Scruber::ArgumentError.new("Scraper name is empty. Pass it to `Scruber.run :name do` or through ENV['SCRUBER_SCRAPER_NAME']") if @scraper_name.blank?
         @scraper_name = @scraper_name.to_sym
-        Scruber.configuration.merge_options(options)
         @callbacks_options = {}
         @callbacks = {}
+        @on_page_error_callback = nil
         @on_complete_callbacks = []
-        @queue = Scruber::Queue.new(scraper_name: scraper_name)
+
+        Scruber.configuration.merge_options(options)
+        ActiveSupport::Dependencies.autoload_paths = Scruber.configuration.autoload_paths
+
+        @queue = Scruber::Queue.new(scraper_name: @scraper_name)
         @fetcher = Scruber::Fetcher.new
+        initialize_progressbar
         load_extenstions
       end
 
       # 
-      # Run crawling.
+      # Crawling engine
       # 
       # @param block [Proc] crawler body
       def run(&block)
         instance_eval &block
         while @queue.has_work? do
           @fetcher.run @queue
+          show_progress
           while page = @queue.fetch_downloaded do
             if @callbacks[page.page_type.to_sym]
               processed_page = process_page(page, page.page_type.to_sym)
@@ -38,16 +69,40 @@ module Scruber
               page.processed! unless page.sent_to_redownload?
             end
           end
+          if @on_page_error_callback
+            while page = @queue.fetch_error do
+              instance_exec page, &(@on_page_error_callback)
+            end
+          end
         end
         @on_complete_callbacks.sort_by{|c| -c[0] }.each do |(_,callback)|
           instance_exec &(callback)
         end
       end
 
+      # 
+      # Register parser
+      # 
+      # @param page_type [Symbol] type of page
+      # @param options [Hash] options for parser
+      # @option options [Symbol] :format format of page. Scruber automatically process 
+      #                                  page body depends on this format. For example :json or :html
+      # @param block [Proc] body of parser
+      # 
+      # @return [void]
       def parser(page_type, options={}, &block)
         register_callback(page_type, options, &block)
       end
 
+      # 
+      # Method missing callback. Scruber allows to register
+      # regexp and proc body to process calls
+      # 
+      # @param method_sym [Symbol] missing method name
+      # @param arguments [Array] arguments
+      # @param block [Proc] block (if passed)
+      # 
+      # @return [type] [description]
       def method_missing(method_sym, *arguments, &block)
         Scruber::Core::Crawler._registered_method_missings.each do |(pattern, func)|
           if (scan_results = method_sym.to_s.scan(pattern)).present?
@@ -68,34 +123,137 @@ module Scruber
       end
 
       class << self
+
+        # 
+        # Register method missing callback
+        # 
+        # @param pattern [Regexp] Regexp to match missing name
+        # @param block [Proc] Body to process missing method
+        # 
+        # @return [void]
         def register_method_missing(pattern, &block)
           _registered_method_missings[pattern] = block
         end
 
+        # 
+        # Registered method missing callbacks dictionary
+        # 
+        # @return [Hash] callbacks
         def _registered_method_missings
           @registered_method_missings ||= {}
         end
       end
 
+      # 
+      # Register callback which will be executed when
+      # downloading and parsing will be completed.
+      # For example when you need to write results to file,
+      # or to close files.
+      # @example Close file descriptors
+      #     on_complete -1 do
+      #       Scruber::Core::Extensions::CsvOutput.close_all
+      #     end
+      # 
+      # @param priority [Integer] priority of this callback
+      # @param block [Proc] body of callback
+      # 
+      # @return [void]
+      def on_complete(priority=1, &block)
+        @on_complete_callbacks.push [priority,block]
+      end
+
+      # 
+      # Register callback which will be executed for
+      # error pages, like 404 or 500
+      # Attention! You should call one of these methods for page
+      # to prevent infinite loop: page.processed!, page.delete, page.redownload!(0)
+      # @example Processing error page
+      #     on_page_error do |page|
+      #       if page.response_body =~ /distil/
+      #         page.page.redownload!(0)
+      #       elsif page.response_code == /404/
+      #         get page.at('a.moved_to').attr('href')
+      #         page.processed!
+      #       else
+      #         page.delete
+      #       end
+      #     end
+      # 
+      # @param block [Proc] body of callback
+      # 
+      # @return [void]
+      def on_page_error(&block)
+        @on_page_error_callback = block
+      end
+
       private
 
+        # 
+        # Register parser
+        # 
+        # @param page_type [Symbol] type of page
+        # @param options [Hash] options for parser
+        # @option options [Symbol] :format format of page. Scruber automatically process 
+        #                                  page body depends on this format. For example :json or :html
+        # @param block [Proc] body of parser
+        # 
+        # @return [void]
         def register_callback(page_type, options, &block)
           @callbacks_options[page_type.to_sym] = options || {}
           @callbacks[page_type.to_sym] = block
         end
 
-        def on_complete(priority=1, &block)
-          @on_complete_callbacks.push [priority,block]
-        end
-
+        # 
+        # Process page body depends on format of this page
+        # For example, if page_format = :html, then
+        # it will return Nokogiri::HTML(page.response_body)
+        # 
+        # @param page [Page] page from queue
+        # @param page_type [Symbol] name of parser
+        # 
+        # @return [Object] depends on page_type it will return different objects
         def process_page(page, page_type)
           page_format = @callbacks_options[page_type].fetch(:format){ nil }
           Scruber::Core::PageFormat.process(page, page_format)
         end
 
+        # 
+        # Loads all extensions
+        # 
+        # @return [void]
         def load_extenstions
           Scruber::Core::Extensions::Base.descendants.each(&:register)
         end
+
+        # 
+        # Initialize progressbar, that shows progress in console
+        # 
+        # @return [void]
+        def initialize_progressbar
+          unless Scruber.configuration.silent
+            @progressbar = PowerBar.new
+            @progressbar.settings.tty.finite.template.main = "${<msg>} ${<bar> }\e[0m \e[33;1m${<percent>%} (${<done>/<total>})"
+            @progressbar.settings.tty.finite.template.padchar = "\e[30;1m#{@progressbar.settings.tty.finite.template.padchar}"
+            @progressbar.settings.tty.finite.template.barchar = "\e[34;1m#{@progressbar.settings.tty.finite.template.barchar}"
+            @progressbar.settings.tty.finite.template.exit = "\e[?25h\e[0m"  # clean up after us
+            @progressbar.settings.tty.finite.template.close = "\e[?25h\e[0m\n" # clean up after us
+            @progressbar.settings.tty.finite.output = Proc.new{ |s|
+              $stderr.print s
+            }
+          end
+        end
+
+        # 
+        # Out progress to console
+        # 
+        # @return [void]
+        def show_progress
+          if @progressbar
+            s = queue.size
+            @progressbar.show({:msg => @proggress_status, :done => queue.downloaded_count, :total => s}) unless s.zero?
+          end
+        end
     end
+
   end
 end

data/lib/scruber/core/extensions/base.rb

@@ -1,12 +1,22 @@
 module Scruber
   module Core
     module Extensions
+      # 
+      # Base class for extensions
+      # @abstract
+      # 
+      # @author [revis0r]
+      # 
       class Base
         module CoreMethods
 
         end
 
         class << self
+          # 
+          # Register extension in crawler core
+          # 
+          # @return [void]
           def register
             Scruber::Core::Crawler.include self.const_get(:CoreMethods)
           end

data/lib/scruber/core/extensions/csv_output.rb

@@ -1,6 +1,20 @@
 module Scruber
   module Core
     module Extensions
+      # 
+      # Helper to write csv files
+      # @example Writing log and products data
+      #   Scruber.run :simple do
+      #     csv_file Scruber.root.join('log.csv'), col_sep: ';'
+      #     csv_products_file Scruber.root.join('products.csv'), col_sep: ';'
+      # 
+      #     csv_out [Time.now.to_i, 'sample log record']
+      #     csv_product_out ['ID', 'Title']
+      #     csv_product_out ['1', 'Soap']
+      #   end
+      # 
+      # @author Ivan Goncharov
+      # 
       class CsvOutput < Base
         module CoreMethods
           def csv_file(path, options={})
@@ -16,6 +30,13 @@ module Scruber
             Scruber::Core::Extensions::CsvOutput.csv_out :default, fields
           end
 
+          # 
+          # Registering method missing callbacks on including
+          # to crawling class
+          # 
+          # @param base [Class] class where module was included
+          # 
+          # @return [void]
           def self.included(base)
             Scruber::Core::Crawler.register_method_missing /\Acsv_(\w+)_file\Z/ do |meth, scan_results, args|
               file_id = scan_results.first.first.to_sym

data/lib/scruber/core/extensions/log.rb

@@ -0,0 +1,39 @@
+module Scruber
+  module Core
+    module Extensions
+      # 
+      # Logging class
+      # Allows to write logs to file and console, depends on configuration
+      # 
+      # @author Ivan Goncharov
+      # 
+      class Log < Base
+        module CoreMethods
+          # 
+          # Writing log
+          # 
+          # @param text [String] text
+          # @param color [Symbol] color of text to write
+          # 
+          # @return [void]
+          def log(text, color=:white)
+            Scruber.logger.info(scraper_name){ text } rescue nil
+            if @progressbar
+              @progressbar.print "#{Paint[text, color]}\n"
+            end
+          end
+
+          # 
+          # Setting status for console progressbar
+          # 
+          # @param text [String] text
+          # 
+          # @return [void]
+          def set_status(text)
+            @proggress_status = text
+          end
+        end
+      end
+    end
+  end
+end

data/lib/scruber/core/extensions/loop.rb

@@ -1,13 +1,47 @@
 module Scruber
   module Core
     module Extensions
+      # 
+      # Helper for reading dictionaries.
+      # @example Adding dictionary and reading it
+      #     Scruber.run :sample do
+      #       add_dictionary :zip_codes_usa, Scruber.root.join('dict', 'zip_codes_usa.csv'), :csv
+      #       seed do
+      #         loop :zip_codes_usa, state: 'NY' do |row|
+      #           get 'https://example.com/by_zip/'+row['zip'].to_s
+      #         end
+      #       end
+      #     end
+      # 
+      # @author Ivan Goncharov
+      # 
       class Loop < Base
         module CoreMethods
+          # 
+          # Iterate records from dictionary
+          # 
+          # @param dictionary [Symbol] name of dictionary
+          # @param options [Hash] search conditions
+          # @param block [Proc] body, yields row of dictionary
+          # 
+          # @return [void]
           def loop(dictionary, options={}, &block)
             Scruber::Core::Extensions::Loop.loop dictionary, options do |*args|
               instance_exec *args, &block
             end
           end
+
+          # 
+          # Registering dictionary in system
+          # 
+          # @param name [Symbol] name of dictionary
+          # @param file_path [String] path to file
+          # @param file_type [Symbol] type of file, :xml, :csv, etc..
+          # 
+          # @return [void]
+          def add_dictionary(name, file_path, file_type)
+            Scruber::Core::Extensions::Loop.add_dictionary(name, file_path, file_type)
+          end
         end
 
         class << self

data/lib/scruber/core/extensions/parser_aliases.rb

@@ -1,6 +1,30 @@
 module Scruber
   module Core
     module Extensions
+      # 
+      # DSL for registering parsers.
+      # @example Sample of DSL
+      #   Scruber.run :sample do
+      #     get 'https://example.com'
+      #     get_product 'https://example.com/product1.html'
+      #   
+      #     # Parsing https://example.com
+      #     parse :html do |page,doc|
+      #       log doc.at('title').text
+      #     end
+      #   
+      #     # Parsing https://example.com/product1.html
+      #     parse_product :html do |page,doc|
+      #       log doc.at('title').text
+      #     end
+      #     # Alias to
+      #     # parser :product, format: :html do |page,doc|
+      #     #   log doc.at('title').text
+      #     # end
+      #   end
+      # 
+      # @author Ivan Gocharov
+      # 
       class ParserAliases < Base
         module CoreMethods
           def parse(*args, &block)

data/lib/scruber/core/extensions/queue_aliases.rb

@@ -1,6 +1,21 @@
 module Scruber
   module Core
     module Extensions
+      # 
+      # DSL for adding pages to queue
+      # @example Sample of DSL
+      #   Scruber.run :sample do
+      #     get_product 'https://example.com/product1.html'
+      #     # Alias to
+      #     # queue.add 'https://example.com/product1.html', page_type: :product
+      # 
+      #     post_subscribe 'https://example.com/subscribe', body: { email: 'sample@example.com' }
+      #     # Alias to
+      #     # queue.add 'https://example.com/product1.html', method: :post, page_type: :subscribe, body: { email: 'sample@example.com' }
+      #   end
+      # 
+      # @author Ivan Gocharov
+      # 
       class QueueAliases < Base
         module CoreMethods
           %w(get post head).each do |meth|

data/lib/scruber/core/extensions/seed.rb

@@ -0,0 +1,23 @@
+module Scruber
+  module Core
+    module Extensions
+
+      # 
+      # Seed DSL
+      # Seed block executes only when queue was not initialized yet
+      # (queue has no any page, processed or pending)
+      # 
+      # @author Ivan Goncharov
+      # 
+      class Seed < Base
+        module CoreMethods
+          def seed(&block)
+            unless queue.initialized?
+              instance_exec &block
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/scruber/fetcher_adapters/abstract_adapter.rb

@@ -32,8 +32,18 @@ module Scruber
           if page.max_retry_times.nil?
             page.max_retry_times = @max_retry_times
           end
+          if page.max_retry_times && page.retry_count >= page.max_retry_times.to_i
+            page.retry_at = 1.year.from_now.to_i
+          end
         else
-          page.fetched_at = Time.now.to_i
+          # Monkey patch to prevent redownloading of 404 pages
+          # and processing 404 pages by regular parsers
+          if page.response_code == 404
+            page.retry_count = 1 if page.retry_count.nil? || page.retry_count.zero?
+            page.max_retry_times = page.retry_count
+          else
+            page.fetched_at = Time.now.to_i
+          end
         end
         page
       end
@@ -89,26 +99,16 @@ module Scruber
 
       def bad_response?(page)
         case page.response_code
-        when 0
-          true
-        when 1
-          true
-        when 100..199
+        when 0..1
           true
-        when 200
-          false
-        when 201..299
+        when 200..299
           false
         when 300..399
           @options.fetch(:followlocation) { false }
         when 404
           false
         when 407
-          raise "RejectedByProxyError"
-        when 400..499
-          true
-        when 500..599
-          true
+          raise "RejectedByProxy"
         else
           true
         end

data/lib/scruber/fetcher_adapters/typhoeus_fetcher.rb

@@ -64,7 +64,7 @@ module Scruber
         page.response_total_time = response.total_time
         
         if response.timed_out?
-          page[:response_code] = 1
+          page.response_code = 1
         end
 
         page = after_request_callback(page)