web_crawler 0.3.1 → 0.5.0

data/Gemfile

@@ -2,6 +2,8 @@ source :gemcutter
 
 gem 'thor', '>=0.14.6'
 gem 'mime-types', '>=1.16'
+gem 'parallel', '>=0.5.5'
+gem 'activesupport'
 
 # Specify your gem's dependencies in web_crawler.gemspec
 gemspec

data/README

@@ -1 +1,22 @@
-Web crawler help you with parse and collect data from the web
+Web crawler help you with parse and collect data from the web
+
+#TODO
+
+Base web crawler class for API present
+Its showld work like this:
+
+class MyCrawler < WebCrawler::Base
+
+    target "www.example.com"
+    target "www.example.com/page2"
+    target %[www.example.com/contacts www.example.com/about]
+    target "www.example.com/category_:category/page:page/", :categories => [1,2,3,4], :page => 1..100
+
+    target { call_advanced_logic_for_url_generating }
+
+    logger "path/to/log/file" # or Logger.new(...)
+
+
+    
+
+end

data/lib/web_crawler.rb

@@ -6,6 +6,7 @@ require 'forwardable'
 require "ext/hash"
 require "ext/array"
 require "ext/http_response"
+require "active_support/core_ext"
 
 module WebCrawler
   autoload :Request, 'web_crawler/request'
@@ -26,6 +27,7 @@ module WebCrawler
   autoload :View, 'web_crawler/view'
   autoload :CLI, 'web_crawler/cli'
   autoload :Application, 'web_crawler/application'
+  autoload :Base, 'web_crawler/base'
 
   include Configurable
   extend Utility

data/lib/web_crawler/application.rb

@@ -2,7 +2,35 @@ module WebCrawler
   class Application < CLI
 
     desc "test", "Test task"
+
     def test
+      urls = FactoryUrl.new('http://www.superjob.ru/rabota/554/veb-programmist/?from=$1', [[140]]).factory
+
+      logger.info "start requests with #{urls.join(' ')} in 4 processes"
+
+      targets = BatchRequest.new(urls).process
+
+      logger.info "#{targets.size} targets collected"
+
+      urls = Follower.new(targets, same_host: false).collect { |url| url =~ /vacancy\/\?id=\d+/ }
+
+      logger.info "#{urls.size} urls collected"
+      logger.info "start requests with in 4 processes"
+
+      puts BatchRequest.new(urls).process.inspect
+
+      ""
+    end
+
+    desc "runner CLASS", "Run crawler class"
+    method_option :lib, type: :array, desc: "lib directories"
+    def runner(name)
+      $:.unshift './'
+      Array.wrap(@options[:lib]).each { |l| $:.unshift l }
+      require name.underscore
+
+      klass = name.classify.constantize
+      klass.run allow_format(:json, :yaml)
     end
 
     desc "get <URL...>", "Get pages from passed urls"
@@ -36,8 +64,7 @@ module WebCrawler
     def factory(pattern, *params)
       params.map! { |param| eval(param) }
       urls = FactoryUrl.new(pattern, params)
-      puts options.inspect
-      sep = options[:list] ? "\n" : ' '
+      sep  = options[:list] ? "\n" : ' '
       if options[:output] || options[:list]
         puts urls.factory.map { |u| u.inspect }.join(sep).gsub('"', "'")
       else
@@ -45,5 +72,9 @@ module WebCrawler
       end
     end
 
+    protected
+    def allow_format(*allow)
+      allow.flatten.select { |f| f == @options[:format] }.first
+    end
   end
 end

data/lib/web_crawler/base.rb

@@ -0,0 +1,113 @@
+require "set"
+
+module WebCrawler
+  class Base
+    class_attribute :targets, :logger, :mappers, :cache, :follower, :responses
+
+    self.targets, self.logger, self.mappers = Set.new, WebCrawler.config.logger, []
+
+    delegate :run, :to => :'self.class'
+
+    class << self
+
+      include ActiveSupport::Callbacks
+
+      def run(format=nil, format_options={ })
+        compile_targets
+        self.responses = WebCrawler::BatchRequest.new(targets.to_a).process
+        if format
+          formated(process(responses), format, format_options)
+        else
+          process(responses)
+        end
+      end
+
+      protected
+
+      def after(&block)
+        @after_callback = block
+      end
+
+      def compile_targets
+        following    = targets.select { |target| target.is_a?(Array) && target.first.is_a?(Proc) }
+        self.targets = targets - following
+
+        following.each do |target|
+          target.first.call(target.last)
+        end
+      end
+
+      def log_to(logger_or_path)
+        case logger_or_path
+          when Logger
+            WebCrawler.config.logger = self.logger = logger_or_path
+          when nil
+            WebCrawler.config.logger = self.logger = Logger.new('/dev/null')
+          else
+            WebCrawler.config.logger = self.logger = Logger.new(logger_or_path)
+        end
+      end
+
+      def cache_to(path_or_cache_adapter)
+        adapter = nil
+        adapter = path_or_cache_adapter if path_or_cache_adapter.is_a? WebCrawler::CacheAdapter::Base
+        adapter = WebCrawler::CacheAdapter::File.new(path_or_cache_adapter) if File.directory? path_or_cache_adapter
+
+        WebCrawler.configure do
+          config.cache.adapter = adapter
+        end if adapter
+      end
+
+      def follow(*targets)
+        options   = targets.extract_options!
+        responses = WebCrawler::BatchRequest.new(targets).process
+        self.target WebCrawler::Follower.new(responses, options).collect
+      end
+
+      def context(selector, name=selector, &block)
+        mapper = WebCrawler::Parsers::Mapper.new(name, self, selector)
+        if block.arity.zero?
+          mapper.instance_exec(&block)
+        else
+          mapper.callback(&block)
+        end
+        self.mappers += [mapper]
+      end
+
+      def target(*targets, &block)
+        options = targets.extract_options!
+        unless options.empty?
+          raise ArgumentError, 'target accept only one pattern if options given' if targets.size > 1
+          targets = generate_urls(targets.first, options)
+        end
+        if block_given?
+          self.targets << [block, targets]
+        else
+          self.targets += targets.flatten
+        end
+      end
+
+      def generate_urls(pattern, options)
+        WebCrawler::FactoryUrl.new(pattern, options).factory
+      end
+
+      def formated(data, format, options)
+        require "active_support/core_ext/string"
+        WebCrawler::View.factory(format, data, options).render
+      end
+
+      def process(responses)
+        return responses.map(&:to_s) if mappers.empty?
+        
+        { }.tap do |results|
+          mappers.each do |mapper|
+            results[mapper.name] = responses.map do |response|
+              mapper.collect(response)
+            end.flatten
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/web_crawler/batch_request.rb

@@ -1,10 +1,15 @@
+require "parallel"
+
 module WebCrawler
 
+  # Usage:
+  #  BatchRequest.new(urls).process #=> array of Responses
+  #
   class BatchRequest
 
     attr_reader :urls, :responses, :requests
     attr_writer :requests
-    
+
     include Enumerable
 
     def initialize(*urls)
@@ -19,10 +24,11 @@ module WebCrawler
       if @handler
         block_given? ? yield(@handler.process) : @handler.process
       else
-        @responses ||= requests.map do |req|
+        ready = requests.select{|r| r.ready? }
+        @responses ||= Parallel.map(requests - ready) do |req|
           WebCrawler.logger.info "start request to #{req.url.to_s}"
           block_given? ? yield(req.process) : req.process
-        end
+        end.compact + ready.map(&:process)
       end
     end
 
@@ -57,7 +63,7 @@ module WebCrawler
     end
 
     def request_class
-      @options[:cached] ? CachedRequest : Request
+      !@options[:no_cached] && WebCrawler.config.cache.adapter.is_a?(WebCrawler::CacheAdapter::Base) ? CachedRequest : Request
     end
   end
 

data/lib/web_crawler/cached_request.rb

@@ -5,23 +5,32 @@ module WebCrawler
 
     def initialize(url, options = { })
       super(url)
-      @cache = options[:cache] || WebCrawler.config.cache_adapter
+      @cache = options[:cache] || WebCrawler.config.cache.adapter
+      @ready = true if @cache.exist? url
     end
 
     def process
-      cached do
+      @response || cached do
         Response.new *fetch(url)
       end
     end
 
     protected
 
+    def load_response
+      @response = @cache.get url
+    end
+
+    def put_response(response)
+      @response = @cache.put(response)
+    end
+
     def cached
-      @response = if @cache.exist? url
-                    @cache.get url
-                  else
-                    @cache.put yield
-                  end
+      if @cache.exist? url
+        load_response
+      else
+        put_response(yield)
+      end
       @response
     end
 

data/lib/web_crawler/configuration.rb

@@ -3,8 +3,8 @@ require "logger"
 module WebCrawler
   class BaseConfiguration
 
-    def initialize(options = {})
-      @@options ||= {}
+    def initialize(options = { })
+      @@options ||= { }
       @@options.merge! options
     end
 
@@ -15,7 +15,7 @@ module WebCrawler
     def config
       self
     end
-    
+
     private
 
     def method_missing(name, *args, &blk)
@@ -48,7 +48,7 @@ module WebCrawler
     end
 
     def cache(&block)
-      @cache ||= BaseConfiguration.new expire_within: 60
+      @cache ||= BaseConfiguration.new(expire_within: 60, adapter: self.cache_adapter)
       if block_given?
         @cache.instance_eval(block)
       else
@@ -58,7 +58,7 @@ module WebCrawler
 
     def logger
       @logger ||= Logger.new(STDOUT).tap do |log|
-       log.level = Logger.const_get log_level.to_s.upcase
+        log.level = Logger.const_get log_level.to_s.upcase
       end
     end
 

data/lib/web_crawler/factory_url.rb

@@ -1,4 +1,13 @@
 module WebCrawler
+  #
+  # p = FactoryUrl.new "http://www.somehost.com/:second/:first/", :first => 0..10, :second => "a".."z"
+  # p.urls #=> ["http://www.somehost.com/a/1",
+  #        #    "http://www.somehost.com/b/1",
+  #        #    "http://www.somehost.com/c/1",
+  #        #    ...
+  #        #    "http://www.somehost.com/x/10",
+  #        #    "http://www.somehost.com/y/10",
+  #        #    "http://www.somehost.com/z/10/"]
   #
   # p = FactoryUrl.new "http://www.somehost.com/$1/$2?param=$3", 0..10, "a".."z", [3,7,34,876,92]
   # p.urls #=> ["http://www.somehost.com/1/a?param=3",
@@ -8,6 +17,7 @@ module WebCrawler
   #        #    "http://www.somehost.com/10/x?param=34",
   #        #    "http://www.somehost.com/10/y?param=876",
   #        #    "http://www.somehost.com/10/z?param=92"]
+  #
   # p = FactoryUrl.new 0..10, "a".."z", [3,7,34,876,92] do |first, second, third|
   #   "http://www.somehost.com/#{first}/#{second}?param=#{third}"
   # end
@@ -18,21 +28,31 @@ module WebCrawler
     attr_reader :urls, :params, :pattern
 
     def initialize(*args, &block)
+      @options = args.extract_options!
       if block_given?
         @block = block
       else
         @pattern = args.shift
         raise ArgumentError, "first argument must be an url pattern(String)" unless pattern.is_a? String
       end
-      @params = normalize_arguments(args)
+      
+      if @options.empty?
+        @params = normalize_arguments(args)
+      else
+        values, keys = @options.values.map(&:to_a), @options.keys
+        values = values.shift.product(*values)
+        @params = values.map{|a| Hash[keys.zip(a)]}
+      end
     end
 
     def factory
-      if pattern
-        @urls ||= params.map { |opts| pattern.gsub(/\$(\d+)/) { opts[$1.to_i - 1] } }
-      else
-        @urls ||= params.map { |opts| @block.call *opts }
-      end
+      @urls ||= if pattern && params.first.is_a?(Hash)
+                  params.map { |opts| pattern.gsub(/:([a-z_]+)/) { opts[$1.to_sym] } }
+                elsif pattern
+                  params.map { |opts| pattern.gsub(/\$(\d+)/) { opts[$1.to_i - 1] } }
+                else
+                  params.map { |opts| @block.call *opts }
+                end
     end
 
     def each
@@ -46,7 +66,7 @@ module WebCrawler
     def normalize_arguments(args)
       args = args.first if args.size == 1 && args.first.is_a?(Enumerable)
       args.shift if args.first.is_a? String
-      params = args.map { |arg| convert_to_a(arg) }
+      params  = args.map { |arg| convert_to_a(arg) }
       @params = params.shift.product(*params)
     end
 

data/lib/web_crawler/follower.rb

@@ -1,26 +1,28 @@
 class WebCrawler::Follower
 
+  attr_reader :options
+
   def initialize(*responses)
-    @options   = responses.last.is_a?(Hash) ? responses.pop : {}
+    @options   = responses.extract_options!
     @responses = responses.flatten
   end
 
-  def process(options = {})
+  def process(options = { })
     WebCrawler::BatchRequest.new(collect, options).process
   end
 
   def follow(response)
-    @responses << response
+    @responses += Array.wrap(response)
     self
   end
 
-  def collect
-    @responses.map do |response|
+  def collect(&block)
+    urls = @responses.map do |response|
       parser = WebCrawler::Parsers::Url.new(response.url.host, url: response.url.request_uri, same_host: @options[:same_host])
-      parser.parse(response.body) do |url|
-        url
-      end
-    end
+      parser.parse(response.body, &block)
+    end.flatten
+    urls = urls.select { |url| url =~ @options[:only] } if @options[:only]
+    urls
   end
 
 end

data/lib/web_crawler/parsers.rb

@@ -2,4 +2,5 @@ require "hpricot"
 
 module WebCrawler::Parsers
   autoload :Url, 'web_crawler/parsers/url'
+  autoload :Mapper, 'web_crawler/parsers/mapper'
 end