web_crawler 0.2.0

data/.gitignore

@@ -0,0 +1,5 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+.idea

data/.rspec

@@ -0,0 +1 @@
+--color

data/Gemfile

@@ -0,0 +1,11 @@
+source :gemcutter
+
+# Specify your gem's dependencies in web_crawler.gemspec
+gemspec
+
+group :development, :test do
+  gem "rspec", ">=2.6"
+  gem "autotest"
+  gem "autotest-growl"
+  gem "fakeweb"
+end

data/README

@@ -0,0 +1 @@
+Web crawler help you with parse and collect data from the web

data/Rakefile

@@ -0,0 +1,2 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks

data/bin/wcrawler

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+
+$:.unshift File.expand_path("../../lib", __FILE__)
+
+# Check if an older version of bundler is installed
+require 'web_crawler'
+require 'web_crawler/application'
+
+begin
+  WebCrawler::Application.start
+end
+
+

data/lib/ext/array.rb

@@ -0,0 +1,100 @@
+require 'enumerator'
+
+class Array
+  # Splits or iterates over the array in groups of size +number+,
+  # padding any remaining slots with +fill_with+ unless it is +false+.
+  #
+  #   %w(1 2 3 4 5 6 7).in_groups_of(3) {|group| p group}
+  #   ["1", "2", "3"]
+  #   ["4", "5", "6"]
+  #   ["7", nil, nil]
+  #
+  #   %w(1 2 3).in_groups_of(2, ' ') {|group| p group}
+  #   ["1", "2"]
+  #   ["3", " "]
+  #
+  #   %w(1 2 3).in_groups_of(2, false) {|group| p group}
+  #   ["1", "2"]
+  #   ["3"]
+  def in_groups_of(number, fill_with = nil)
+    if fill_with == false
+      collection = self
+    else
+      # size % number gives how many extra we have;
+      # subtracting from number gives how many to add;
+      # modulo number ensures we don't add group of just fill.
+      padding = (number - size % number) % number
+      collection = dup.concat([fill_with] * padding)
+    end
+
+    if block_given?
+      collection.each_slice(number) { |slice| yield(slice) }
+    else
+      groups = []
+      collection.each_slice(number) { |group| groups << group }
+      groups
+    end
+  end
+
+  # Splits or iterates over the array in +number+ of groups, padding any
+  # remaining slots with +fill_with+ unless it is +false+.
+  #
+  #   %w(1 2 3 4 5 6 7 8 9 10).in_groups(3) {|group| p group}
+  #   ["1", "2", "3", "4"]
+  #   ["5", "6", "7", nil]
+  #   ["8", "9", "10", nil]
+  #
+  #   %w(1 2 3 4 5 6 7).in_groups(3, '&nbsp;') {|group| p group}
+  #   ["1", "2", "3"]
+  #   ["4", "5", "&nbsp;"]
+  #   ["6", "7", "&nbsp;"]
+  #
+  #   %w(1 2 3 4 5 6 7).in_groups(3, false) {|group| p group}
+  #   ["1", "2", "3"]
+  #   ["4", "5"]
+  #   ["6", "7"]
+  def in_groups(number, fill_with = nil)
+    # size / number gives minor group size;
+    # size % number gives how many objects need extra accommodation;
+    # each group hold either division or division + 1 items.
+    division = size / number
+    modulo = size % number
+
+    # create a new array avoiding dup
+    groups = []
+    start = 0
+
+    number.times do |index|
+      length = division + (modulo > 0 && modulo > index ? 1 : 0)
+      padding = fill_with != false &&
+        modulo > 0 && length == division ? 1 : 0
+      groups << slice(start, length).concat([fill_with] * padding)
+      start += length
+    end
+
+    if block_given?
+      groups.each { |g| yield(g) }
+    else
+      groups
+    end
+  end
+
+  # Divides the array into one or more subarrays based on a delimiting +value+
+  # or the result of an optional block.
+  #
+  #   [1, 2, 3, 4, 5].split(3)                # => [[1, 2], [4, 5]]
+  #   (1..10).to_a.split { |i| i % 3 == 0 }   # => [[1, 2], [4, 5], [7, 8], [10]]
+  def split(value = nil)
+    using_block = block_given?
+
+    inject([[]]) do |results, element|
+      if (using_block && yield(element)) || (value == element)
+        results << []
+      else
+        results.last << element
+      end
+
+      results
+    end
+  end
+end

data/lib/ext/hash.rb

@@ -0,0 +1,45 @@
+class Hash
+  # Return a new hash with all keys converted to strings.
+  def stringify_keys
+    dup.stringify_keys!
+  end
+
+  # Destructively convert all keys to strings.
+  def stringify_keys!
+    keys.each do |key|
+      self[key.to_s] = delete(key)
+    end
+    self
+  end
+
+  # Return a new hash with all keys converted to symbols, as long as
+  # they respond to +to_sym+.
+  def symbolize_keys
+    dup.symbolize_keys!
+  end
+
+  # Destructively convert all keys to symbols, as long as they respond
+  # to +to_sym+.
+  def symbolize_keys!
+    keys.each do |key|
+      self[(key.to_sym rescue key) || key] = delete(key)
+    end
+    self
+  end
+
+  alias_method :to_options,  :symbolize_keys
+  alias_method :to_options!, :symbolize_keys!
+
+  # Validate all keys in a hash match *valid keys, raising ArgumentError on a mismatch.
+  # Note that keys are NOT treated indifferently, meaning if you use strings for keys but assert symbols
+  # as keys, this will fail.
+  #
+  # ==== Examples
+  #   { :name => "Rob", :years => "28" }.assert_valid_keys(:name, :age) # => raises "ArgumentError: Unknown key(s): years"
+  #   { :name => "Rob", :age => "28" }.assert_valid_keys("name", "age") # => raises "ArgumentError: Unknown key(s): name, age"
+  #   { :name => "Rob", :age => "28" }.assert_valid_keys(:name, :age) # => passes, raises nothing
+  def assert_valid_keys(*valid_keys)
+    unknown_keys = keys - [valid_keys].flatten
+    raise(ArgumentError, "Unknown key(s): #{unknown_keys.join(", ")}") unless unknown_keys.empty?
+  end
+end

data/lib/ext/http_response.rb

@@ -0,0 +1,19 @@
+module RedirectPath
+
+  def redirect_path=(path)
+    @redirect_path = path.flatten.compact.map(&:to_s).reject(&:empty?)
+  end
+
+  def redirect_path
+    @redirect_path
+  end
+
+  def redirect?
+    !!redirect_path
+  end
+
+end
+
+class Net::HTTPResponse
+  include RedirectPath
+end

data/lib/web_crawler/application.rb

@@ -0,0 +1,49 @@
+module WebCrawler
+  class Application < CLI
+
+    desc "test", "Test task"
+    def test
+    end
+
+    desc "get <URL...>", "Get pages from passed urls"
+    method_option :parser, type: :array, desc: "first item is a parser class, second item is a path to parser file"
+    method_option 'same-host', type: :boolean, desc: "find urls with same host only"
+
+    def get(url, *urls)
+      urls.unshift url
+
+      batch = BatchRequest.new(*urls, symbolized_options)
+      batch.process
+    end
+
+    map 'show-urls' => :show_urls
+    desc "show-urls <URL...>", "Get pages from passed urls"
+    method_option 'same-host', type: :boolean, desc: "find urls with same host only"
+    method_option 'cols', type: :numeric, desc: "output columns size"
+
+    def show_urls(url, *urls)
+      urls.unshift url
+      batch          = BatchRequest.new(*urls, symbolized_options)
+      options[:cols] ||= 1
+      Follower.new(batch.process, same_host: options['same-host']).collect.first.in_groups_of(options[:cols], "")
+    end
+
+    desc "factory URL_PATTERN [params,...]", "Generate urls and run get action"
+    inherited_method_options :get
+    method_option :output, type: :boolean, desc: "show output and exit"
+    method_option :list, type: :boolean, desc: "show output like a list and exit"
+
+    def factory(pattern, *params)
+      params.map! { |param| eval(param) }
+      urls = FactoryUrl.new(pattern, params)
+      puts options.inspect
+      sep = options[:list] ? "\n" : ' '
+      if options[:output] || options[:list]
+        puts urls.factory.map { |u| u.inspect }.join(sep).gsub('"', "'")
+      else
+        get *urls.factory
+      end
+    end
+
+  end
+end

data/lib/web_crawler/batch_request.rb

@@ -0,0 +1,63 @@
+module WebCrawler
+
+  class BatchRequest
+
+    attr_reader :urls, :responses, :requests
+    attr_writer :requests
+    
+    include Enumerable
+
+    def initialize(*urls)
+      @options = urls.last.is_a?(Hash) ? urls.pop : { }
+      set_handler
+
+      @urls, @requests = urls.flatten, []
+      init_requests!
+    end
+
+    def process
+      if @handler
+        block_given? ? yield(@handler.process) : @handler.process
+      else
+        @responses ||= requests.map do |req|
+          block_given? ? yield(req.process) : req.process
+        end
+      end
+    end
+
+    def each &block
+      @responses = []
+      requests.each do |req|
+        @responses << req.process
+        block.call(@responses.last)
+      end
+    end
+
+    def responses=(value)
+      @responses += value.flatten
+    end
+
+    def response
+      responses.first
+    end
+
+    def build_request(url)
+      request_class.new(url)
+    end
+
+    protected
+
+    def set_handler
+      @handler = WebCrawler::HandlerParser.new(@options[:parser], self) if @options[:parser]
+    end
+
+    def init_requests!
+      @requests = @urls.map { |url| build_request(url) }
+    end
+
+    def request_class
+      @options[:cached] ? CachedRequest : Request
+    end
+  end
+
+end

data/lib/web_crawler/cache_adapter/base.rb

@@ -0,0 +1,33 @@
+class WebCrawler::CacheAdapter::Base
+
+  def expired?(response, &block)
+    block_result = block_given? ? block.call : false
+    (response.foul? && response.date < expire_within) || block_result
+  end
+
+  def expire_within(seconds = nil)
+    Time.now - (seconds || WebCrawler.config.cache.expire_within)
+  end
+
+  def prepare_response(response)
+    response.set_cached_flag
+    response
+  end
+
+  def put response
+    prepare_response(response.dup)
+  end
+
+  def set response
+    put response
+  end
+
+  def get uri
+    raise NotImplementedError
+  end
+
+  def exist? uri
+    raise NotImplementedError
+  end
+
+end

data/lib/web_crawler/cache_adapter/file.rb

@@ -0,0 +1,52 @@
+require "pathname"
+
+module WebCrawler::CacheAdapter
+
+  class File < Base
+
+    attr_reader :dir
+
+    def initialize(dir)
+      @dir = Pathname.new dir
+    end
+
+    def put response
+      response.tap { write(super) }
+    end
+
+    def get uri
+      response = read(uri)
+      expire!(response) if expired?(response)
+      response
+    end
+
+    def exist? uri
+      file(uri).exist?
+    end
+
+    def file(response_or_url)
+      url = response_or_url.url rescue response_or_url
+      dir.join(uri_to_filename(url))
+    end
+
+    def expire!(response)
+      file(response).delete
+    end
+
+    protected
+
+    def read(uri)
+      Marshal.load(file(uri).read)
+    end
+
+    def write(response)
+      file(response).open('w+') { |f| f << Marshal.dump(response) }
+    end
+
+    def uri_to_filename(uri)
+      uri.to_s.gsub(/\W/, '_').gsub(/_+/, '_')
+    end
+
+  end
+
+end

data/lib/web_crawler/cache_adapter/memory.rb

@@ -0,0 +1,23 @@
+module WebCrawler::CacheAdapter
+
+  class Memory < Base
+    class << self
+      attr_accessor :cache
+    end
+
+    self.cache = {}
+    
+    def put response
+      response.tap { self.class.cache[response.url.to_s] = super }
+    end
+
+    def get uri
+      self.class.cache[uri.to_s]
+    end
+
+    def exist? uri
+      self.class.cache.key? uri.to_s
+    end
+  end
+
+end

data/lib/web_crawler/cache_adapter.rb

@@ -0,0 +1,11 @@
+module WebCrawler
+
+  module CacheAdapter
+
+    autoload :Base, 'web_crawler/cache_adapter/base'
+    autoload :Memory, 'web_crawler/cache_adapter/memory'
+    autoload :File, 'web_crawler/cache_adapter/file'
+
+  end
+
+end

data/lib/web_crawler/cached_request.rb

@@ -0,0 +1,30 @@
+module WebCrawler
+
+  class CachedRequest < Request
+    extend ::Forwardable
+
+    def initialize(url, options = { })
+      super(url)
+      @cache = options[:cache] || WebCrawler.config.cache_adapter
+    end
+
+    def process
+      cached do
+        Response.new *fetch(url)
+      end
+    end
+
+    protected
+
+    def cached
+      @response = if @cache.exist? url
+                    @cache.get url
+                  else
+                    @cache.put yield
+                  end
+      @response
+    end
+
+  end
+
+end

data/lib/web_crawler/cli/thor_hooks.rb

@@ -0,0 +1,94 @@
+class Thor
+  module Hooks
+
+    def self.included(base)
+      base.send :include, InstanceMethods
+      base.send :extend, ClassMethods
+    end
+
+    module InstanceMethods
+      attr_reader :response
+
+      def before_hooks
+        self.class.before_hooks
+      end
+
+      def after_hooks
+        self.class.after_hooks
+      end
+
+      # Invoke the given task if the given args.
+      def invoke_task(task, *args) #:nodoc:
+        self.class.run_hooks :before, self, task
+        @task_result = super(task, *args)
+        @task_result.tap do
+          self.class.run_hooks :after, self, task
+        end
+      end
+    end
+
+    module ClassMethods
+      def hooks
+        @@hooks ||= { before: [], after: [] }
+      end
+
+      def before_hooks
+        hooks[:before]
+      end
+
+      def after_hooks
+        hooks[:after]
+      end
+
+      def before_action(*args, &block)
+        options = args.last.is_a?(Hash) ? args.pop : { }
+        check_hooks_options! options
+        add_hook :before, args, options, &block
+      end
+
+      def after_action(*args, &block)
+        options = args.last.is_a?(Hash) ? args.pop : { }
+        check_hooks_options! options
+        add_hook :after, args, options, &block
+      end
+
+      def render(*args, &block)
+        after_action(*args) do
+          block.call @task_result, @options
+        end
+      end
+
+      def run_hooks(place, instance, task)
+        hooks[place].each { |hook| self.run_hook(instance, task, hook) }
+      end
+
+
+      protected
+
+      def check_hooks_options!(options)
+        raise ArgumentError, <<-M.gsub(/^\s+/, '') if options.keys.include?(:only) && options.keys.include?(:except)
+        both ":only" and ":except" given. You should use alone option ":only" or ":except"
+        M
+      end
+
+      def add_hook(place, args, options, &block)
+        options[:only]   ||= []
+        options[:except] ||= []
+        options[:only]   = [*options[:only]]
+        options[:except] = [*options[:except]]
+        hooks[place] << { block: block, options: options, args: args }
+      end
+
+      def run_hook(instance, task, hook)
+        instance.instance_eval(&hook[:block]) if runnable?(task, hook)
+      end
+
+      def runnable?(task, hook)
+        with_only   = hook[:options][:only].empty? || hook[:options][:only].include?(task.name.to_sym)
+        with_except = !hook[:options][:except].include?(task.name.to_sym)
+        with_only && with_except
+      end
+    end
+
+  end
+end

data/lib/web_crawler/cli/thor_inherited_options.rb

@@ -0,0 +1,26 @@
+class Thor
+  module InheritedOptions
+
+    def self.included(base)
+      base.send :extend, ClassMethods
+    end
+
+    module ClassMethods
+      def option_to_hash(option)
+        values = option.instance_variables.map { |v| option.instance_variable_get v }
+        keys   = option.instance_variables.map { |sym| sym.to_s.sub('@', '') }
+        Hash[keys.zip values]
+      end
+
+      def inherited_method_options(from_action, for_action = nil)
+        tasks[from_action.to_s].options.each do |name, option|
+          option_hash = option_to_hash(option).symbolize_keys
+          option_hash.merge! for: for_action.to_s if for_action
+          option_hash[:desc] = option_hash[:description]
+          method_option name, option_hash
+        end
+      end
+    end
+
+  end
+end