scruber 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 6aecb299dd5c70bfe864f65d2faed704cdc3cde0
+  data.tar.gz: f0697ca03b0f1552e03fb1ee70d0265d5b8bed04
+SHA512:
+  metadata.gz: 2f52170ba8af4152b88e694537fa9e55156fa6d082c1e9ec409ae46808dc9ca28ff47ed241e8072c0edbba13b3e0be9f00504aef4542da52c3a280c282d8df71
+  data.tar.gz: 580d4c3840526dba3e62826433f8dddd09cdc42f569a2db05beb69b3428bfbd0fea92dc2faa0367175870327605f27f960e04be3d65d5c2dbd3c44c61719252d

data/.gitignore

@@ -0,0 +1,15 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+.ruby-version
+.ruby-gemset
+todo
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.3.1
+before_install: gem install bundler -v 1.15.3

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in scruber.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 Ivan Goncharov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,39 @@
+# Scruber
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/scruber`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'scruber'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install scruber
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/scruber.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "scruber"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/scruber

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require "scruber/cli"
+
+Scruber::CLI::Root.start(ARGV)

data/lib/scruber/app_searcher.rb

@@ -0,0 +1,31 @@
+module Scruber
+  module AppSearcher
+
+    extend self
+
+    RUBY = Gem.ruby
+    EXECUTABLES = ["bin/scruber"]
+
+    def exec_app(name)
+      original_cwd = Dir.pwd
+
+      loop do
+        if exe = find_executable
+          exec RUBY, exe, *ARGV
+          break # non reachable, hack to be able to stub exec in the test suite
+        end
+
+        # If we exhaust the search there is no executable, this could be a
+        # call to generate a new application, so restore the original cwd.
+        Dir.chdir(original_cwd) && return if Pathname.new(Dir.pwd).root?
+
+        # Otherwise keep moving upwards in search of an executable.
+        Dir.chdir("..")
+      end
+    end
+
+    def find_executable
+      EXECUTABLES.find { |exe| File.file?(exe) }
+    end
+  end
+end

data/lib/scruber/cli/project_generator.rb

@@ -0,0 +1,47 @@
+require "thor"
+require 'thor/group'
+require 'fileutils'
+
+module Scruber
+  module CLI
+    class ProjectGenerator < Thor::Group
+      include Thor::Actions
+
+      argument :path
+      class_option :queue, :default => 'memory', :aliases => '-q'
+      class_option :fetcher_agent, :default => 'memory', :aliases => '-fa'
+
+      def self.source_root
+        File.dirname(__FILE__) + '/templates'
+      end
+
+      def create_directories
+        raise ::Thor::Error, "ERROR: #{path} already exists." if File.exist?(path)
+        say "Creating scruber project at #{path}"
+        FileUtils.mkdir_p(path)
+      end
+
+      def create_files
+        template 'Gemfile.tt', path+'/Gemfile'
+        template 'gitignore.tt', path+'/.gitignore'
+        template 'bin/scruber.tt', path+'/bin/scruber'
+        template 'application.tt', path+'/config/application.rb'
+        template 'boot.tt', path+'/config/boot.rb'
+        template 'boot.tt', path+'/config/boot.rb'
+        template 'initializers/proxies.tt', path+'/config/initializers/proxies.rb'
+        template 'initializers/user_agents.tt', path+'/config/initializers/user_agents.rb'
+        template 'scrapers/sample.tt', path+'/scrapers/sample.rb'
+      end
+
+      def init_project
+        inside path do
+          run "bundle"
+        end
+      end
+
+      def print_instructions
+        say "Run `scruber start sample` to run sample scraper"
+      end
+    end
+  end
+end

data/lib/scruber/cli/templates/Gemfile.tt

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+gem 'scruber'
+<% if @queue == 'mongo' %>
+gem 'scruber-mongo'
+<% end %>

data/lib/scruber/cli/templates/application.tt

@@ -0,0 +1,18 @@
+require File.expand_path('../boot', __FILE__)
+
+Bundler.require(:default)
+
+Scruber.configure do |config|
+  config.fetcher_adapter = :typhoeus_fetcher
+  config.fetcher_options = {
+    max_concurrency: 1,
+    max_retry_times: 5,
+    retry_delays: [1,2,2,4,4],
+    followlocation: false,
+    request_timeout: 15,
+  }
+  config.fetcher_agent_adapter = :<%= options[:fetcher_agent] %>
+  config.fetcher_agent_options = {}
+  config.queue_adapter = :<%= options[:queue] %>
+  config.queue_options = {}
+end

data/lib/scruber/cli/templates/bin/scruber.tt

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+APP_PATH = File.expand_path('../config/application', __dir__)
+require "scruber/cli"
+require_relative '../config/application'
+
+Scruber::CLI::Root.start(ARGV)

data/lib/scruber/cli/templates/boot.tt

@@ -0,0 +1,3 @@
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__)
+
+require 'bundler/setup' # Set up gems listed in the Gemfile.

data/lib/scruber/cli/templates/gitignore.tt

@@ -0,0 +1,12 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/lib/scruber/cli/templates/initializers/proxies.tt

@@ -0,0 +1,10 @@
+# Scruber::Core::Extensions::Loop.add_dictionary(:proxy_list, File.expand_path(File.dirname(__FILE__))+'/proxies.xml', :xml)
+
+# Scruber::Helpers::ProxyRotator.configure do
+#   clean
+#   set_mode :round_robin
+
+#   loop :proxy_list do |ua|
+#     add ua['name'], tags: ua['tags'].split(',').map(&:strip)
+#   end
+# end

data/lib/scruber/cli/templates/initializers/user_agents.tt

@@ -0,0 +1,14 @@
+# Scruber::Core::Extensions::Loop.add_dictionary(:user_agents, File.expand_path(File.dirname(__FILE__))+'/user_agents.xml', :xml)
+
+Scruber::Helpers::UserAgentRotator.configure do
+  clean
+  set_filter :all
+  
+  add "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36", tags: [:desktop, :modern, :chrome]
+
+  # How to access proxy_list dictionary
+  # 
+  # loop :user_agents do |ua|
+  #   add ua['name'], tags: ua['tags'].split(',').map(&:strip)
+  # end
+end

data/lib/scruber/cli/templates/scrapers/sample.tt

@@ -0,0 +1,7 @@
+Scruber.run do
+  queue.add "http://example.com"
+  
+  parser :seed do |page|
+    puts page.response_body
+  end
+end

data/lib/scruber/cli.rb

@@ -0,0 +1,40 @@
+require "thor"
+require "scruber"
+require "scruber/cli/project_generator"
+require "scruber/app_searcher"
+
+module Scruber
+  module CLI
+
+    class Root < Thor
+     def self.exit_on_failure?
+        true
+      end
+
+      register(ProjectGenerator, "new", "new PATH", "Create new project")
+
+      desc 'start', 'Run scraper'
+      def start(name)
+        if defined?(APP_PATH)
+          # raise ::Thor::Error, "ERROR: Scruber project not found." unless File.exist?(File.expand_path('config/application', Dir.pwd))
+          raise ::Thor::Error, "ERROR: Scraper not found." unless File.exist?(File.expand_path('../../scrapers/'+name+'.rb', APP_PATH))
+          say "booting..."
+          require APP_PATH
+          Dir[File.expand_path('../initializers/*.rb', APP_PATH)].sort.each do |i|
+            require i
+          end
+          say "starting #{name}"
+          require File.expand_path('../../scrapers/'+name+'.rb', APP_PATH)
+        else
+          Scruber::AppSearcher.exec_app(name)
+        end
+      end
+
+      desc 'version', 'Display version'
+      map %w[-v --version] => :version
+      def version
+        say "Scruber #{VERSION}"
+      end
+    end
+  end
+end

data/lib/scruber/core/configuration.rb

@@ -0,0 +1,30 @@
+module Scruber
+  module Core
+    class Configuration
+      attr_accessor :fetcher_adapter,
+                    :fetcher_options,
+                    :fetcher_agent_adapter,
+                    :fetcher_agent_options,
+                    :queue_adapter,
+                    :queue_options
+
+      def initialize
+        @fetcher_adapter = :typhoeus_fetcher
+        @fetcher_options = {}
+        @fetcher_agent_adapter = :memory
+        @fetcher_agent_options = {}
+        @queue_adapter = :memory
+        @queue_options = {}
+      end
+
+      def merge_options(options)
+        @fetcher_adapter = options.fetch(:fetcher_adapter){ @fetcher_adapter }
+        @fetcher_options.merge! options.fetch(:fetcher_options){ {} }
+        @fetcher_agent_adapter = options.fetch(:fetcher_agent_adapter){ @fetcher_agent_adapter }
+        @fetcher_agent_options.merge! options.fetch(:fetcher_agent_options){ {} }
+        @queue_adapter = options.fetch(:queue_adapter){ @queue_adapter }
+        @queue_options.merge! options.fetch(:queue_options){ {} }
+      end
+    end
+  end
+end

data/lib/scruber/core/crawler.rb

@@ -0,0 +1,92 @@
+module Scruber
+  module Core
+    class Crawler
+      attr_reader :queue, :fetcher
+
+      def initialize(options={})
+        Scruber.configuration.merge_options(options)
+        @callbacks_options = {}
+        @callbacks = {}
+        @on_complete_callbacks = {}
+        @queue = Scruber::Queue.new
+        @fetcher = Scruber::Fetcher.new
+        load_extenstions
+      end
+
+      # 
+      # Run crawling.
+      # 
+      # @param block [Proc] crawler body
+      def run(&block)
+        instance_eval &block
+        while @queue.has_work? do
+          @fetcher.run @queue
+          while page = @queue.fetch_downloaded do
+            if @callbacks[page.page_type.to_sym]
+              processed_page = process_page(page, page.page_type.to_sym)
+              instance_exec page, processed_page, &(@callbacks[page.page_type.to_sym])
+            end
+          end
+        end
+        @on_complete_callbacks.each do |_,callback|
+          instance_exec &(callback)
+        end
+      end
+
+      def parser(page_type, options={}, &block)
+        register_callback(page_type, options, &block)
+      end
+
+      def method_missing(method_sym, *arguments, &block)
+        Scruber::Core::Crawler._registered_method_missings.find do |(pattern, func)|
+          if method_sym.to_s =~ pattern
+            instance_exec method_sym, arguments, &(func)
+            true
+          else
+            false
+          end
+        end || super
+      end
+
+      def respond_to?(method_sym, include_private = false)
+        !Scruber::Core::Crawler._registered_method_missings.find do |(pattern, block)|
+          if method_sym.to_s =~ pattern
+            true
+          else
+            false
+          end
+        end.nil? || super(method_sym, include_private)
+      end
+
+      class << self
+        def register_method_missing(pattern, &block)
+          _registered_method_missings[pattern] = block
+        end
+
+        def _registered_method_missings
+          @registered_method_missings ||= {}
+        end
+      end
+
+      private
+
+        def register_callback(page_type, options, &block)
+          @callbacks_options[page_type.to_sym] = options || {}
+          @callbacks[page_type.to_sym] = block
+        end
+
+        def on_complete_callback(name, &block)
+          @on_complete_callbacks[name] = block
+        end
+
+        def process_page(page, page_type)
+          page_format = @callbacks_options[page_type].fetch(:page_format){ nil }
+          Scruber::Core::PageFormat.process(page, page_format)
+        end
+
+        def load_extenstions
+          Scruber::Core::Extensions::Base.descendants.each(&:register)
+        end
+    end
+  end
+end

data/lib/scruber/core/extensions/base.rb

@@ -0,0 +1,26 @@
+module Scruber
+  module Core
+    module Extensions
+      class Base
+        module CoreMethods
+
+        end
+
+        class << self
+          def register
+            Scruber::Core::Crawler.include self.const_get(:CoreMethods)
+          end
+
+          def inherited(subclass)
+            @descendants ||= []
+            @descendants << subclass
+          end
+
+          def descendants
+            @descendants 
+          end
+        end
+      end
+    end
+  end
+end

data/lib/scruber/core/extensions/csv_output.rb

@@ -0,0 +1,62 @@
+module Scruber
+  module Core
+    module Extensions
+      class CsvOutput < Base
+        module CoreMethods
+          def csv_file(path, options={})
+            file_id = options.fetch(:file_id) { :default }.to_sym
+            options.delete(:file_id)
+            Scruber::Core::Extensions::CsvOutput.register_csv file_id, path, options
+            on_complete_callback :close_csv_files do
+              Scruber::Core::Extensions::CsvOutput.close_all
+            end
+          end
+
+          def csv_out(fields)
+            Scruber::Core::Extensions::CsvOutput.csv_out :default, fields
+          end
+
+          def self.included(base)
+            Scruber::Core::Crawler.register_method_missing /\Acsv_(\w+)_file\Z/ do |meth, args|
+              file_id = meth.to_s.scan(/\Acsv_(\w+)_file\Z/).first.first.to_sym
+              path, options = args
+              options = {} if options.nil?
+              csv_file path, options.merge({file_id: file_id})
+              Scruber::Core::Crawler.class_eval do
+                define_method "csv_#{file_id}_out".to_sym do |fields|
+                  Scruber::Core::Extensions::CsvOutput.csv_out(file_id, fields)
+                end
+              end
+            end
+          end
+        end
+
+        class << self
+          def csv_out(file_id, fields)
+            if _registered_files.keys.include?(file_id.to_sym)
+              _registered_files[file_id.to_sym] << fields
+            else
+              raise ArgumentError, "file #{file_id.inspect} not registered"
+            end
+          end
+
+          def register_csv(file_id, path, options)
+            mode = options.fetch(:mode){ 'wb' }
+            options.delete(:mode)
+            _registered_files[file_id] = CSV.open(path, mode, options)
+          end
+
+          def _registered_files
+            @registered_files ||= {}
+          end
+
+          def close_all
+            _registered_files.each{ |_,f| f.close }
+            @registered_files = {}
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/scruber/core/extensions/loop.rb

@@ -0,0 +1,39 @@
+module Scruber
+  module Core
+    module Extensions
+      class Loop < Base
+        module CoreMethods
+          def loop(dictionary, options={}, &block)
+            Scruber::Core::Extensions::Loop.loop dictionary, options do |*args|
+              instance_exec *args, &block
+            end
+          end
+        end
+
+        class << self
+          def loop(dictionary, options={})
+            if _registered_dictionaries.keys.include?(dictionary.to_sym)
+              Scruber::Helpers::DictionaryReader.read(_registered_dictionaries[dictionary.to_sym][:file_path], _registered_dictionaries[dictionary.to_sym][:file_type], options) do |obj|
+                yield obj
+              end
+            else
+              raise ArgumentError, "dictionary not registered, available dictionaries #{_registered_dictionaries.keys.inspect}"
+            end
+          end
+
+          def add_dictionary(name, file_path, file_type)
+            _registered_dictionaries[name.to_sym] = {
+              file_path: file_path,
+              file_type: file_type
+            }
+          end
+
+          def _registered_dictionaries
+            @registered_dictionaries ||= {}
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/scruber/core/page_format/base.rb

@@ -0,0 +1,11 @@
+module Scruber
+  module Core
+    module PageFormat
+      class Base
+        def self.process(page)
+          raise NotImplementedError
+        end
+      end
+    end
+  end
+end

data/lib/scruber/core/page_format/html.rb

@@ -0,0 +1,13 @@
+module Scruber
+  module Core
+    module PageFormat
+      class HTML < Base
+        def self.process(page)
+          Nokogiri::HTML(page.response_body) rescue nil
+        end
+      end
+    end
+  end
+end
+
+Scruber::Core::PageFormat.add(:html, Scruber::Core::PageFormat::HTML)

data/lib/scruber/core/page_format/xml.rb

@@ -0,0 +1,13 @@
+module Scruber
+  module Core
+    module PageFormat
+      class XML < Base
+        def self.process(page)
+          Nokogiri.parse(page.response_body) rescue nil
+        end
+      end
+    end
+  end
+end
+
+Scruber::Core::PageFormat.add(:xml, Scruber::Core::PageFormat::XML)