scruber 0.1.4 → 0.1.5

data/lib/scruber/queue_adapters/abstract_adapter.rb

@@ -1,12 +1,44 @@
 module Scruber
   module QueueAdapters
+    # 
+    # Abstract Queue Adapter
+    # 
+    # @abstract
+    # @author Ivan Goncharov
+    # 
     class AbstractAdapter
-      
+      # 
+      # Queue page wrapper
+      # 
+      # @author Ivan Goncharov
+      # 
+      # @attr [Object] id ID of page. Will be autogenerated if not passed
+      # @attr [String] url URL of page
+      # @attr [String] method Request method, post, get, head
+      # @attr [String] user_agent Fixed User-Agent for requesting this page
+      # @attr [Hash] headers Headers for requesting this page
+      # @attr [Object] fetcher_agent_id ID of FetcherAgent, assigned to this page
+      # @attr [Object] proxy_id ID of proxy, assigned to this page
+      # @attr [String] response_body Response body
+      # @attr [Integer] response_code Response code
+      # @attr [Hash] response_headers Response headers
+      # @attr [Float] response_total_time Response total time
+      # @attr [Integer] retry_at Minimal timestamp of next retry
+      # @attr [Integer] fetched_at Download completion timestamp
+      # @attr [Integer] retry_count Number of download attempts
+      # @attr [Integer] max_retry_times Max number of download attempts
+      # @attr [Integer] enqueued_at Timestamp added to the queue
+      # @attr [String] page_type Page type
+      # @attr [Scruber::QueueAdapters::AbstractAdapter::Page] queue Queue object
+      # @attr [Integer] priority Priority of page in queue for fetcher
+      # @attr [Integer] processed_at Processed by parser timestamp
+      # @attr [Hash] options All options
       class Page
-        attr_accessor :url,
+        attr_accessor :id,
+                      :url,
                       :method,
                       :user_agent,
-                      :post_body,
+                      :body,
                       :headers,
                       :fetcher_agent_id,
                       :proxy_id,
@@ -25,14 +57,16 @@ module Scruber
                       :processed_at,
                       :options
 
-        def initialize(queue, url, options={})
+        def initialize(queue, options={})
           @queue = queue
-          @url = url
 
           options = options.with_indifferent_access
+          @options = options
+          @id = options.fetch(:id) { generate_page_id }
+          @url = options.fetch(:url) { raise "URL not provided" }
           @method = options.fetch(:method) { :get }
           @user_agent = options.fetch(:user_agent) { nil }
-          @post_body = options.fetch(:post_body) { nil }
+          @body = options.fetch(:body) { nil }
           @headers = options.fetch(:headers) { {} }
           @fetcher_agent_id = options.fetch(:fetcher_agent_id) { nil }
           @proxy_id = options.fetch(:proxy_id) { nil }
@@ -49,13 +83,16 @@ module Scruber
           # @queue = options.fetch(:queue) { 'default' }
           @priority = options.fetch(:priority) { 0 }
           @processed_at = options.fetch(:processed_at) { 0 }
-          @options = options
 
           @_fetcher_agent = false
           @_proxy = false
           @_redownload = false
         end
 
+        # 
+        # Returns assigned to this page FetcherAgent
+        # 
+        # @return [Scruber::Helpers::FetcherAgent] Agent object
         def fetcher_agent
           if @_fetcher_agent == false
             @_fetcher_agent = (@fetcher_agent_id ? Scruber::Helpers::FetcherAgent.find(@fetcher_agent_id) : nil)
@@ -64,6 +101,10 @@ module Scruber
           end
         end
 
+        # 
+        # Returns assigned to this page proxy
+        # 
+        # @return [Proxy] proxy object
         def proxy
           if @_proxy == false
             @_proxy = (@proxy_id ? Scruber::Helpers::ProxyRotator.find(@proxy_id) : nil)
@@ -72,6 +113,10 @@ module Scruber
           end
         end
 
+        # 
+        # Returns cookies from response headers
+        # 
+        # @return [Array] array of cookies
         def response_cookies
           cookies = self.response_headers['Set-Cookie']
           if cookies.blank?
@@ -93,50 +138,144 @@ module Scruber
           instance_variable_get("@#{k.to_s}")
         end
 
+        # 
+        # Delete page from queue
+        # 
+        # @return [void]
         def delete
           raise NotImplementedError
         end
 
+        # 
+        # Mark page as processed by parser and save it
+        # 
+        # @return [void]
         def processed!
           @processed_at = Time.now.to_i
           @_redownload = false
           save
         end
 
-        def redownload!
+        # 
+        # Mark page as pending and return to queue
+        # 
+        # @param new_retry_count [Integer] new count of reties. Allows to reset retries count
+        # 
+        # @return [void]
+        def redownload!(new_retry_count=nil)
           @_redownload = true
 
-          @processed_at = nil
-          @retry_count += 1
+          @processed_at = 0
+          if new_retry_count
+            @retry_count = new_retry_count
+          else
+            @retry_count += 1
+          end
           @fetched_at = 0
           @response_body = nil
           save
         end
 
+        # 
+        # Marked as page for redownloading
+        # 
+        # @return [Boolean] true if need to redownload
         def sent_to_redownload?
           @_redownload
         end
+
+        private
+
+          def generate_page_id
+            Digest::MD5.hexdigest @options.slice(:method, :url, :headers, :body).to_json
+          end
       end
 
       def initialize(options={})
         @options = options
       end
 
+      # 
+      # Add page to queue
+      # @param url [String] URL of page
+      # @param options [Hash] Other options, see {Scruber::QueueAdapters::AbstractAdapter::Page}
+      # 
+      # @return [void]
       def add(url, options={})
         raise NotImplementedError
       end
 
+      # 
+      # Search page by id
+      # @param id [Object] id of page
+      # 
+      # @return [Page] page object
+      def find(id)
+        raise NotImplementedError
+      end
+
+      # 
+      # Size of queue
+      # 
+      # @return [Integer] count of pages in queue
+      def size
+        raise NotImplementedError
+      end
+
+      # 
+      # Fetch pending page for fetching
+      # @param count [Integer] count of pages to fetch
+      # 
+      # @return [Scruber::QueueAdapters::AbstractAdapter::Page|Array<Scruber::QueueAdapters::AbstractAdapter::Page>] page of count = nil, or array of pages of count > 0
       def fetch_pending(count=nil)
         raise NotImplementedError
       end
 
+      # 
+      # Fetch downloaded and not processed pages for feching
+      # @param count [Integer] count of pages to fetch
+      # 
+      # @return [Scruber::QueueAdapters::AbstractAdapter::Page|Array<Scruber::QueueAdapters::AbstractAdapter::Page>] page of count = nil, or array of pages of count > 0
       def fetch_downloaded(count=nil)
         raise NotImplementedError
       end
 
+      # 
+      # Fetch error page
+      # @param count [Integer] count of pages to fetch
+      # 
+      # @return [Scruber::QueueAdapters::AbstractAdapter::Page|Array<Scruber::QueueAdapters::AbstractAdapter::Page>] page of count = nil, or array of pages of count > 0
+      def fetch_error(count=nil)
+        raise NotImplementedError
+      end
+
+      # 
+      # Count of downloaded pages
+      # Using to show downloading progress.
+      # 
+      # @return [Integer] count of downloaded pages
+      def downloaded_count
+        raise NotImplementedError
+      end
+
+      # 
+      # Check if queue was initialized.
+      # Using for `seed` method. If queue was initialized,
+      # then no need to run seed block.
+      # 
+      # @return [Boolean] true if queue already was initialized
       def initialized?
         raise NotImplementedError
       end
+
+      # 
+      # Used by Core. It checks for pages that are
+      # not downloaded or not parsed yet.
+      # 
+      # @return [Boolean] true if queue still has work for scraper
+      def has_work?
+        raise NotImplementedError
+      end
     end
   end
 end

data/lib/scruber/queue_adapters/memory.rb

@@ -1,46 +1,118 @@
 module Scruber
   module QueueAdapters
+    # 
+    # Memory Queue Adapter
+    # 
+    # Simple queue adapted which stores pages in memory.
+    # Nice solution for small scrapes.
+    # Easy to use. No need to setup any database, but
+    # no ability to reparse pages if something went wrong.
+    # 
+    # @author Ivan Goncharov
+    # 
     class Memory < AbstractAdapter
       attr_reader :error_pages
 
+      # 
+      # Queue item class
+      # 
+      # @author Ivan Goncharov
+      # 
+      # @attr (see Scruber::QueueAdapters::AbstractAdapter::Page)
+      # 
       class Page < Scruber::QueueAdapters::AbstractAdapter::Page
+
+        # 
+        # Save page
+        # 
+        # Depends on page attributes it push page
+        # to pending, downloaded or error queue.
+        # 
+        # @return [void]
         def save
           if self.processed_at.to_i > 0
-            nil
+            @queue.add_processed_page self
           elsif self.fetched_at > 0
             @queue.add_downloaded self
           elsif self.max_retry_times && self.retry_count >= self.max_retry_times.to_i
             @queue.add_error_page self
           else
-            @queue.push self
+            @queue.add self
           end
         end
 
+        # 
+        # Delete page from all queues
+        # 
+        # @return [void]
         def delete
           @queue.delete self
         end
       end
 
+      # 
+      # Queue initializer
+      # @param options [Hash] See {Scruber::QueueAdapters::AbstractAdapter#initializer}
+      # 
+      # @return [Scruber::QueueAdapters::Memory] class instance
       def initialize(options={})
         super(options)
+        @processed_ids = []
         @queue = []
         @downloaded_pages = []
         @error_pages = []
       end
 
-      def push(url_or_page, options={})
-        if url_or_page.is_a?(Page)
-          @queue.push url_or_page
-        else
-          @queue.push Page.new(self, url_or_page, options)
+      # 
+      # Add page to queue
+      # @param url_or_page [String|Page] URL of page or Page object
+      # @param options [Hash] Other options, see {Scruber::QueueAdapters::AbstractAdapter::Page}
+      # 
+      # @return [void]
+      def add(url_or_page, options={})
+        unless url_or_page.is_a?(Page)
+          url_or_page = Page.new(self, options.merge(url: url_or_page))
+        end
+        @queue.push(url_or_page) unless @processed_ids.include?(url_or_page.id) || find(url_or_page.id)
+      end
+      alias_method :push, :add
+
+      # 
+      # Search page by id
+      # @param id [Object] id of page
+      # 
+      # @return [Page] page
+      def find(id)
+        [@queue, @downloaded_pages, @error_pages].each do |q|
+          q.each do |i|
+            return i if i.id == id
+          end
         end
+        nil
       end
-      alias_method :add, :push
 
-      def queue_size
+      # 
+      # Size of queue
+      # 
+      # @return [Integer] count of pages in queue
+      def size
         @queue.count
       end
 
+      # 
+      # Count of downloaded pages
+      # Using to show downloading progress.
+      # 
+      # @return [Integer] count of downloaded pages
+      def downloaded_count
+        @downloaded_pages.count
+      end
+
+      # 
+      # Fetch downloaded and not processed pages for feching
+      # @param count [Integer] count of pages to fetch
+      # 
+      # @return [Scruber::QueueAdapters::AbstractAdapter::Page|Array<Scruber::QueueAdapters::AbstractAdapter::Page>] page of count = nil, or array of pages of count > 0
       def fetch_downloaded(count=nil)
         if count.nil?
           @downloaded_pages.shift
@@ -49,6 +121,24 @@ module Scruber
         end
       end
 
+      # 
+      # Fetch error page
+      # @param count [Integer] count of pages to fetch
+      # 
+      # @return [Scruber::QueueAdapters::AbstractAdapter::Page|Array<Scruber::QueueAdapters::AbstractAdapter::Page>] page of count = nil, or array of pages of count > 0
+      def fetch_error(count=nil)
+        if count.nil?
+          @error_pages.shift
+        else
+          @error_pages.shift(count)
+        end
+      end
+
+      # 
+      # Fetch pending page for fetching
+      # @param count [Integer] count of pages to fetch
+      # 
+      # @return [Scruber::QueueAdapters::AbstractAdapter::Page|Array<Scruber::QueueAdapters::AbstractAdapter::Page>] page of count = nil, or array of pages of count > 0
       def fetch_pending(count=nil)
         if count.nil?
           @queue.shift
@@ -57,24 +147,64 @@ module Scruber
         end
       end
 
+      # 
+      # Internal method to add page to downloaded queue
+      # 
+      # @param page [Scruber::QueueAdapters::Memory::Page] page
+      # 
+      # @return [void]
       def add_downloaded(page)
         @downloaded_pages.push page
       end
 
+      # 
+      # Internal method to add page to error queue
+      # 
+      # @param page [Scruber::QueueAdapters::Memory::Page] page
+      # 
+      # @return [void]
       def add_error_page(page)
         @error_pages.push page
       end
 
+      # 
+      # Saving processed page id to prevent
+      # adding identical pages to queue
+      # 
+      # @param page [Page] page
+      # 
+      # @return [void]
+      def add_processed_page(page)
+        @processed_ids.push page.id
+      end
+
+      # 
+      # Used by Core. It checks for pages that are
+      # not downloaded or not parsed yet.
+      # 
+      # @return [Boolean] true if queue still has work for scraper
       def has_work?
         @queue.count > 0 || @downloaded_pages.count > 0
       end
 
+      # 
+      # Delete page from all internal queues
+      # 
+      # @param page [Scruber::QueueAdapters::Memory::Page] page
+      # 
+      # @return [void]
       def delete(page)
         @queue -= [page]
         @downloaded_pages -= [page]
         @error_pages -= [page]
       end
 
+      # 
+      # Check if queue was initialized.
+      # Using for `seed` method. If queue was initialized,
+      # then no need to run seed block.
+      # 
+      # @return [Boolean] true if queue already was initialized
       def initialized?
         @queue.present? || @downloaded_pages.present? || @error_pages.present?
       end

data/lib/scruber/version.rb

@@ -1,3 +1,3 @@
 module Scruber
-  VERSION = "0.1.4"
+  VERSION = "0.1.5"
 end

data/lib/scruber.rb

@@ -3,7 +3,11 @@ require 'nokogiri'
 require 'http-cookie'
 require 'pickup'
 require 'csv'
+require 'paint'
+require 'powerbar'
+require 'core_ext/const_missing'
 require 'active_support'
+require 'active_support/dependencies'
 require 'active_support/core_ext/object'
 require 'active_support/core_ext/hash'
 
@@ -22,6 +26,8 @@ require "scruber/core/page_format/html"
 
 require "scruber/core/extensions/base"
 require "scruber/core/extensions/loop"
+require "scruber/core/extensions/log"
+require "scruber/core/extensions/seed"
 require "scruber/core/extensions/csv_output"
 require "scruber/core/extensions/queue_aliases"
 require "scruber/core/extensions/parser_aliases"
@@ -30,16 +36,13 @@ require "scruber/helpers/dictionary_reader"
 require "scruber/helpers/dictionary_reader/xml"
 require "scruber/helpers/dictionary_reader/csv"
 
-# require "scruber/core/configuration"
-# require "scruber/core/configuration"
-
 module Scruber
   class ArgumentError < ::ArgumentError; end
+
   module Core
-    autoload :Configuration, "scruber/core/configuration"
-    autoload :Crawler,       "scruber/core/crawler"
+    autoload :Configuration,    "scruber/core/configuration"
+    autoload :Crawler,          "scruber/core/crawler"
   end
-
   module Helpers
     autoload :UserAgentRotator,   "scruber/helpers/user_agent_rotator"
     autoload :ProxyRotator,       "scruber/helpers/proxy_rotator"
@@ -51,11 +54,11 @@ module Scruber
   end
 
   class << self
-    attr_writer :configuration
+    attr_writer :configuration, :logger
 
     def run(*args, &block)
       raise "You need a block to build!" unless block_given?
-
+      
       Core::Crawler.new(*args).run(&block)
     end
 
@@ -63,8 +66,20 @@ module Scruber
       @configuration ||= Core::Configuration.new
     end
 
+    def logger
+      @logger ||= Scruber.root.nil? ? nil : Logger.new(Scruber.root.join('log', 'crawler.log'))
+    end
+
     def configure(&block)
       yield configuration
     end
+
+    def root
+      if defined?(APP_PATH)
+        Pathname.new(File.expand_path('../../', APP_PATH))
+      else
+        nil
+      end
+    end
   end
 end

data/scruber.gemspec

@@ -30,11 +30,13 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "typhoeus", "1.1.2"
-  spec.add_dependency "pickup", "0.0.11"
-  spec.add_dependency "nokogiri", "1.8.2"
+  spec.add_dependency "typhoeus", '~> 1.1', '>= 1.1.2'
+  spec.add_dependency "pickup", "~> 0.0.11"
+  spec.add_dependency "nokogiri", '~> 1.8', '>= 1.8.2'
   spec.add_dependency "http-cookie", "1.0.3"
-  spec.add_dependency "activesupport", "5.1.5"
+  spec.add_dependency "activesupport", '~> 5.1', '>= 5.1.5'
+  spec.add_dependency "powerbar", '~> 2.0', '>= 2.0.1'
+  spec.add_dependency "paint", '~> 2.0', '>= 2.0.1'
   spec.add_runtime_dependency "thor", "0.20.0"
   spec.add_development_dependency "bundler", "~> 1.15"
   spec.add_development_dependency "rake", "~> 10.0"