scruber 0.1.1

data/lib/scruber/helpers/user_agent_rotator.rb

@@ -0,0 +1,91 @@
+module Scruber
+  module Helpers
+    class UserAgentRotator
+
+      class UserAgent
+        attr_accessor :name, :tags
+
+        def initialize(name, options={})
+          @name = name
+          raise Scruber::ArgumentError.new("You need to specify name") if @name.blank?
+          @tags = options.fetch(:tags){ [] } || []
+          if !@tags.is_a?(Array)
+            @tags = [@tags]
+          end
+          @tags = @tags.compact.map(&:to_sym)
+        end
+
+        def id
+          @name
+        end
+      end
+
+      class Configuration
+        include Scruber::Core::Extensions::Loop::CoreMethods
+
+        attr_reader :user_agents, :tags
+
+        def initialize
+          @tags = :all
+          @user_agents = []
+        end
+
+        def configure(&block)
+          instance_eval &block
+        end
+
+        def clean
+          @user_agents = []
+        end
+
+        def add(name, options={})
+          ua = UserAgent.new(name, options)
+          @user_agents.push ua
+        end
+
+        def set_filter(tags)
+          @tags = tags
+        end
+      end
+
+      class << self
+        attr_writer :configuration
+        attr_accessor :cursor
+
+        def configuration
+          @configuration ||= Configuration.new
+        end
+
+        def configure(&block)
+          configuration.configure(&block)
+        end
+
+        def next(tags=nil)
+          raise Scruber::ArgumentError.new("UserAgent rotator not configured") if @configuration.nil?
+          tags = @configuration.tags if tags.blank?
+          user_agents = get_collection(tags)
+          if @cursor.nil? || @cursor >= user_agents.count-1
+            @cursor = 0
+          else
+            @cursor += 1
+          end
+          user_agents[@cursor].name
+        end
+
+        private
+
+          def get_collection(tags)
+            if tags == :all
+              @configuration.user_agents
+            else
+              if !tags.is_a?(Array)
+                tags = [tags]
+              end
+              tags = tags.compact.map(&:to_sym)
+              @configuration.user_agents.select{|ua| tags.all?{|t| ua.tags.include?(t) } }
+            end
+          end
+      end
+    end
+  end
+end

data/lib/scruber/queue.rb

@@ -0,0 +1,34 @@
+module Scruber
+  module Queue
+    class << self
+      attr_writer :adapter
+
+      def new(options={})
+        adapter.new(::Scruber.configuration.queue_options.merge(options))
+      end
+
+      def adapter
+        unless @adapter
+          @adapter = ::Scruber.configuration.queue_adapter || _adapters.keys.first
+        end
+        raise Scruber::ArgumentError.new("Adapter not found") unless @adapter
+        _adapters[@adapter]
+      end
+
+      def add_adapter(label, claz)
+        unless claz.method_defined?(:add)
+          raise NoMethodError, "add is not declared in the #{label.inspect}"
+        end
+        _adapters[label] = claz
+      end
+
+      def [](label)
+        _adapters[label]
+      end
+
+      def _adapters
+        @_adapters ||= {}
+      end
+    end
+  end
+end

data/lib/scruber/queue_adapters/abstract_adapter.rb

@@ -0,0 +1,112 @@
+module Scruber
+  module QueueAdapters
+    class AbstractAdapter
+      
+      class Page
+        attr_accessor :url,
+                      :method,
+                      :user_agent,
+                      :post_body,
+                      :headers,
+                      :fetcher_agent_id,
+                      :proxy_id,
+                      :response_body,
+                      :response_code,
+                      :response_headers,
+                      :response_total_time,
+                      :retry_at,
+                      :fetched_at,
+                      :retry_count,
+                      :max_retry_times,
+                      :enqueued_at,
+                      :page_type,
+                      :queue,
+                      :priority,
+                      :processed_at,
+                      :options
+
+        def initialize(queue, url, options={})
+          @queue = queue
+          @url = url
+          @method = options.fetch(:method) { :get }
+          @user_agent = options.fetch(:user_agent) { nil }
+          @post_body = options.fetch(:post_body) { nil }
+          @headers = options.fetch(:headers) { {} }
+          @fetcher_agent_id = options.fetch(:fetcher_agent_id) { nil }
+          @proxy_id = options.fetch(:proxy_id) { nil }
+          @response_body = options.fetch(:response_body) { nil }
+          @response_code = options.fetch(:response_code) { nil }
+          @response_headers = options.fetch(:response_headers) { {} }
+          @response_total_time = options.fetch(:response_total_time) { nil }
+          @retry_at = options.fetch(:retry_at) { 0 }
+          @fetched_at = options.fetch(:fetched_at) { 0 }
+          @retry_count = options.fetch(:retry_count) { 0 }
+          @max_retry_times = options.fetch(:max_retry_times) { nil }
+          @enqueued_at = options.fetch(:enqueued_at) { 0 }
+          @page_type = options.fetch(:page_type) { :seed }
+          # @queue = options.fetch(:queue) { 'default' }
+          @priority = options.fetch(:priority) { 0 }
+          @processed_at = options.fetch(:processed_at) { 0 }
+          @options = options
+
+          @_fetcher_agent = false
+          @_proxy = false
+        end
+
+        def fetcher_agent
+          if @_fetcher_agent == false
+            @_fetcher_agent = (@fetcher_agent_id ? Scruber::Helpers::FetcherAgent.find(@fetcher_agent_id) : nil)
+          else
+            @_fetcher_agent
+          end
+        end
+
+        def proxy
+          if @_proxy == false
+            @_proxy = (@proxy_id ? Scruber::Helpers::ProxyRotator.find(@proxy_id) : nil)
+          else
+            @_proxy
+          end
+        end
+
+        def response_cookies
+          cookies = self.response_headers['Set-Cookie']
+          if cookies.blank?
+            []
+          else
+            if cookies.is_a?(Array)
+              cookies
+            else
+              [cookies]
+            end
+          end
+        end
+
+        def save
+          raise NotImplementedError
+        end
+
+        def [](k)
+          instance_variable_get("@#{k.to_s}")
+        end
+
+      end
+
+      def initialize(options={})
+        @options = options
+      end
+
+      def add(url, options={})
+        raise NotImplementedError
+      end
+
+      def fetch_pending(count=nil)
+        raise NotImplementedError
+      end
+
+      def fetch_downloaded(count=nil)
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/scruber/queue_adapters/memory.rb

@@ -0,0 +1,70 @@
+module Scruber
+  module QueueAdapters
+    class Memory < AbstractAdapter
+      attr_reader :error_pages
+
+      class Page < Scruber::QueueAdapters::AbstractAdapter::Page
+        def save
+          if self.fetched_at > 0
+            @queue.add_downloaded self
+          elsif self.retry_count >= self.max_retry_times.to_i
+            @queue.add_error_page self
+          else
+            @queue.push self
+          end
+        end
+      end
+
+      def initialize(options={})
+        super(options)
+        @queue = []
+        @downloaded_pages = []
+        @error_pages = []
+      end
+
+      def push(url_or_page, options={})
+        if url_or_page.is_a?(Page)
+          @queue.push url_or_page
+        else
+          @queue.push Page.new(self, url_or_page, options)
+        end
+      end
+      alias_method :add, :push
+
+      def queue_size
+        @queue.count
+      end
+
+      def fetch_downloaded(count=nil)
+        if count.nil?
+          @downloaded_pages.shift
+        else
+          @downloaded_pages.shift(count)
+        end
+      end
+
+      def fetch_pending(count=nil)
+        if count.nil?
+          @queue.shift
+        else
+          @queue.shift(count)
+        end
+      end
+
+      def add_downloaded(page)
+        @downloaded_pages.push page
+      end
+
+      def add_error_page(page)
+        @error_pages.push page
+      end
+
+      def has_work?
+        @queue.count > 0 || @downloaded_pages.count > 0
+      end
+
+    end
+  end
+end
+
+Scruber::Queue.add_adapter(:memory, Scruber::QueueAdapters::Memory)

data/lib/scruber/version.rb

@@ -0,0 +1,3 @@
+module Scruber
+  VERSION = "0.1.1"
+end

data/lib/scruber.rb

@@ -0,0 +1,69 @@
+require "scruber/version"
+require 'nokogiri'
+require 'http-cookie'
+require 'pickup'
+require 'csv'
+require 'active_support'
+require 'active_support/core_ext/object'
+require 'active_support/core_ext/hash'
+
+require "scruber/fetcher"
+require "scruber/fetcher_adapters/abstract_adapter"
+require "scruber/fetcher_adapters/typhoeus_fetcher"
+
+require "scruber/queue"
+require "scruber/queue_adapters/abstract_adapter"
+require "scruber/queue_adapters/memory"
+
+require "scruber/core/page_format"
+require "scruber/core/page_format/base"
+require "scruber/core/page_format/xml"
+require "scruber/core/page_format/html"
+
+require "scruber/core/extensions/base"
+require "scruber/core/extensions/loop"
+require "scruber/core/extensions/csv_output"
+
+# require "scruber/core/configuration"
+# require "scruber/core/configuration"
+
+module Scruber
+  class ArgumentError < ::ArgumentError; end
+  module Core
+    autoload :Configuration, "scruber/core/configuration"
+    autoload :Crawler,       "scruber/core/crawler"
+  end
+
+  module Helpers
+    autoload :UserAgentRotator,   "scruber/helpers/user_agent_rotator"
+    autoload :ProxyRotator,       "scruber/helpers/proxy_rotator"
+    autoload :FetcherAgent,       "scruber/helpers/fetcher_agent"
+    module FetcherAgentAdapters
+      autoload :AbstractAdapter,  "scruber/helpers/fetcher_agent_adapters/abstract_adapter"
+      autoload :Memory,           "scruber/helpers/fetcher_agent_adapters/memory"
+    end
+    autoload :DictionaryReader,   "scruber/helpers/dictionary_reader"
+    module DictionaryReader
+      autoload :Xml,              "scruber/helpers/dictionary_reader/xml"
+      autoload :Csv,              "scruber/helpers/dictionary_reader/csv"
+    end
+  end
+
+  class << self
+    attr_writer :configuration
+
+    def run(options={}, &block)
+      raise "You need a block to build!" unless block_given?
+
+      Core::Crawler.new(options).run(&block)
+    end
+
+    def configuration
+      @configuration ||= Core::Configuration.new
+    end
+
+    def configure(&block)
+      yield configuration
+    end
+  end
+end

data/scruber.gemspec

@@ -0,0 +1,43 @@
+# coding: utf-8
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "scruber/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "scruber"
+  spec.version       = Scruber::VERSION
+  spec.authors       = ["Ivan Goncharov"]
+  spec.email         = ["revis0r.mob@gmail.com"]
+
+  spec.summary       = %q{Crawling framework}
+  spec.description   = %q{Crawling framework}
+  spec.homepage      = "https://github.com/scruber/scruber"
+  spec.license       = "MIT"
+
+  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
+  # to allow pushing to a single host or delete this section to allow pushing to any host.
+  if spec.respond_to?(:metadata)
+    spec.metadata["allowed_push_host"] = "https://rubygems.org"
+  else
+    raise "RubyGems 2.0 or newer is required to protect against " \
+      "public gem pushes."
+  end
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "typhoeus", "1.1.2"
+  spec.add_dependency "pickup", "0.0.11"
+  spec.add_dependency "nokogiri", "1.8.2"
+  spec.add_dependency "http-cookie", "1.0.3"
+  spec.add_dependency "activesupport", "5.1.5"
+  spec.add_runtime_dependency "thor", "0.20.0"
+  spec.add_development_dependency "bundler", "~> 1.15"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "webmock", "3.0.1"
+end

metadata

@@ -0,0 +1,233 @@
+--- !ruby/object:Gem::Specification
+name: scruber
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Ivan Goncharov
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2018-03-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: typhoeus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.1.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.1.2
+- !ruby/object:Gem::Dependency
+  name: pickup
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.0.11
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.0.11
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.8.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.8.2
+- !ruby/object:Gem::Dependency
+  name: http-cookie
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.3
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 5.1.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 5.1.5
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.20.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.20.0
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.0.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.0.1
+description: Crawling framework
+email:
+- revis0r.mob@gmail.com
+executables:
+- scruber
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- exe/scruber
+- lib/scruber.rb
+- lib/scruber/app_searcher.rb
+- lib/scruber/cli.rb
+- lib/scruber/cli/project_generator.rb
+- lib/scruber/cli/templates/Gemfile.tt
+- lib/scruber/cli/templates/application.tt
+- lib/scruber/cli/templates/bin/scruber.tt
+- lib/scruber/cli/templates/boot.tt
+- lib/scruber/cli/templates/gitignore.tt
+- lib/scruber/cli/templates/initializers/proxies.tt
+- lib/scruber/cli/templates/initializers/user_agents.tt
+- lib/scruber/cli/templates/scrapers/sample.tt
+- lib/scruber/core/configuration.rb
+- lib/scruber/core/crawler.rb
+- lib/scruber/core/extensions/base.rb
+- lib/scruber/core/extensions/csv_output.rb
+- lib/scruber/core/extensions/loop.rb
+- lib/scruber/core/page_format.rb
+- lib/scruber/core/page_format/base.rb
+- lib/scruber/core/page_format/html.rb
+- lib/scruber/core/page_format/xml.rb
+- lib/scruber/fetcher.rb
+- lib/scruber/fetcher_adapters/abstract_adapter.rb
+- lib/scruber/fetcher_adapters/typhoeus_fetcher.rb
+- lib/scruber/helpers/dictionary_reader.rb
+- lib/scruber/helpers/dictionary_reader/csv.rb
+- lib/scruber/helpers/dictionary_reader/xml.rb
+- lib/scruber/helpers/fetcher_agent.rb
+- lib/scruber/helpers/fetcher_agent_adapters/abstract_adapter.rb
+- lib/scruber/helpers/fetcher_agent_adapters/memory.rb
+- lib/scruber/helpers/proxy_rotator.rb
+- lib/scruber/helpers/user_agent_rotator.rb
+- lib/scruber/queue.rb
+- lib/scruber/queue_adapters/abstract_adapter.rb
+- lib/scruber/queue_adapters/memory.rb
+- lib/scruber/version.rb
+- scruber.gemspec
+homepage: https://github.com/scruber/scruber
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.14
+signing_key: 
+specification_version: 4
+summary: Crawling framework
+test_files: []