kudzu 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ab6c69046e9faa3788ead18864ee6d13ddbe2980
-  data.tar.gz: c9868fabe9542d877d6519e0f5297419c882a8e5
+  metadata.gz: 0ba76e95628d94560421358aa7982bdc429971e4
+  data.tar.gz: e1875f5760573a021fcf129018aaba5f6213ad23
 SHA512:
-  metadata.gz: 7018e08e6744a9e74e601bad26a88df3d60140ddaa055fc194c88263cff37137402de967203999d9ba9c2bda199228215f380e207a4b12b6c2e50c5774827e16
-  data.tar.gz: b03edd059ea5b5cb0f50fd0bc660c02e8727c1bb7f8f735fe86ba6f08b2a8bf743a04570eb171dc99f3ffcacba26c1df8a99cc2a1187e2e2fb481c01adb59e6c
+  metadata.gz: 4f17f799c2ad67722860bbad00e9e220db8265221d598aebbbd181fea41546454e954fc67db18abba2b8b68797fe44da3887ec4ea3a5d2486fd1afd61584c152
+  data.tar.gz: 65d3bf42fafbcf835740ebe5534a52163f09ef313795152fafadc87aeaa335d540b037180a4f03208471361a9284635ff15a85471f3def22ba43e341ee1eb724

data/lib/kudzu/adapter/memory/frontier.rb

@@ -9,7 +9,7 @@ module Kudzu
           @queued = {}
         end
 
-        def enqueue(links, depth: 1)
+        def enqueue(links)
           @monitor.synchronize do
             Array(links).each do |link|
               next if @queued.key?(link.url)

data/lib/kudzu/adapter/memory/model/link.rb

@@ -1,14 +1,10 @@
 module Kudzu
   module Adapter
     module Memory
-      class Link
-        include Kudzu::Adapter::Base::Link
+      class Link < Kudzu::Model::Base
+        include Kudzu::Model::Link
 
         attr_accessor :uuid, :url, :title, :state, :depth
-
-        def initialize(attr = {})
-          attr.each { |k, v| public_send("#{k}=", v) if respond_to?("#{k}=") }
-        end
       end
     end
   end

data/lib/kudzu/adapter/memory/model/page.rb

@@ -1,16 +1,11 @@
 module Kudzu
   module Adapter
     module Memory
-      class Page
-        include Kudzu::Adapter::Base::Page
+      class Page < Kudzu::Model::Base
+        include Kudzu::Model::Page
 
         attr_accessor :url, :title, :status, :mime_type, :size, :charset, :digest,
-                      :response_header, :response_time, :redirect_from, :fetched_at, :revised_at,
-                      :revisit_interval, :revisit_at
-
-        def initialize(attr = {})
-          attr.each { |k, v| public_send("#{k}=", v) if respond_to?("#{k}=") }
-        end
+                      :response_header, :response_time, :redirect_from, :fetched_at, :revised_at
       end
     end
   end

data/lib/kudzu/adapter/memory/repository.rb

@@ -6,7 +6,6 @@ module Kudzu
 
         def initialize
           @page = {}
-          @digest = {}
         end
 
         def find_by_url(url)
@@ -15,7 +14,6 @@ module Kudzu
 
         def register(page)
           @page[page.url] = page
-          @digest[page.digest] = true
         end
 
         def delete(page)

data/lib/kudzu/adapter/memory.rb

@@ -1,4 +1,3 @@
-require_relative 'base/all'
-require_relative 'memory/all'
-
-Kudzu.adapter = Kudzu::Adapter::Memory
+Dir[File.join(__dir__, 'memory/**/*.rb')].each do |file|
+  require_relative file
+end

data/lib/kudzu/agent/all.rb

@@ -1,3 +1,3 @@
-Dir[File.join(__dir__, '*.rb')].each do |file|
+Dir[File.join(__dir__, '**/*.rb')].each do |file|
   require_relative file
 end

data/lib/kudzu/agent/fetcher.rb

@@ -1,67 +1,65 @@
-require 'net/http'
-require 'http-cookie'
-
 module Kudzu
   class Agent
     class Fetcher
-      class Response
-        attr_accessor :url, :status, :header, :body, :time, :redirected
-  
-        def initialize(attr = {})
-          attr.each { |k, v| public_send("#{k}=", v) }
-        end
-
-        def redirected?
-          redirected
-        end
-      end
-
       attr_reader :pool
 
       def initialize(config, robots = nil)
         @config = config
-        @pool = Kudzu::Util::ConnectionPool.new(@config.max_connection || 100)
-        @sleeper = Kudzu::Agent::Sleeper.new(@config, robots)
+        @pool = Http::ConnectionPool.new(@config.max_connection || 100)
+        @sleeper = Sleeper.new(@config, robots)
+        @filterer = PageFilterer.new(@config)
         @jar = HTTP::CookieJar.new
       end
 
-      def fetch(url, request_header: {}, redirect: max_redirect, method: :get)
+      def fetch(url, request_header: {}, method: :get, redirect: @config.max_redirect, redirect_from: nil)
         uri = Addressable::URI.parse(url)
-        http = @pool.checkout(pool_name(uri)) { build_http(uri) }
         request = build_request(uri, request_header: request_header, method: method)
-
-        append_cookie(url, request) if @config.handle_cookie
-
-        @sleeper.politeness_delay(url)
-
-        response = nil
-        response_time = Benchmark.realtime { response = http.request(request) }
-
-        parse_cookie(url, response) if @config.handle_cookie
+        response, response_time = send_request(uri, request)
 
         if redirection?(response.code) && response['location'] && redirect > 0
-          fetch(uri.join(response['location']).to_s, request_header: request_header, redirect: redirect - 1)
+          fetch(uri.join(response['location']).to_s, request_header: request_header,
+                                                     redirect: redirect - 1,
+                                                     redirect_from: redirect_from || url)
         else
-          res = build_response(url, response, response_time)
-          res.redirected = (redirect != max_redirect)
-          res
+          build_response(url, response, response_time, redirect_from)
         end
       end
 
       private
 
-      def max_redirect
-        @config.max_redirect || 5
-      end
-
       def pool_name(uri)
         "#{uri.scheme}_#{uri.host}_#{uri.port || uri.default_port}"
       end
 
+      def send_request(uri, request)
+        start_http(uri, request) do |http|
+          http.request(request) do |response|
+            unless @filterer.allowed_response_header?(uri.to_s, response)
+              http.finish
+              break response
+            end
+          end
+        end
+      end
+
+      def start_http(uri, request)
+        http = @pool.checkout(pool_name(uri)) { build_http(uri) }
+        append_cookie(uri, request) if @config.handle_cookie
+        @sleeper.politeness_delay(uri)
+
+        start = Time.now.to_f
+        response = yield http
+        response_time = Time.now.to_f - start
+
+        parse_cookie(uri, response) if @config.handle_cookie
+        return response, response_time
+      end
+
       def build_http(uri)
         http = Net::HTTP.new(uri.host, uri.port || uri.default_port)
         http.open_timeout = @config.open_timeout if @config.open_timeout
         http.read_timeout = @config.read_timeout if @config.read_timeout
+        http.keep_alive_timeout = @config.keep_alive if @config.keep_alive
         if uri.scheme == 'https'
           http.use_ssl = true
           http.verify_mode = OpenSSL::SSL::VERIFY_NONE
@@ -70,7 +68,7 @@ module Kudzu
       end
 
       def build_request(uri, request_header:, method:)
-        request = request_klass_for(method).new(uri.request_uri)
+        request = Object.const_get("Net::HTTP::#{method.capitalize}").new(uri.request_uri)
         request.basic_auth uri.user, uri.password if uri.user && uri.password
 
         request['User-Agent'] = @config.user_agent
@@ -80,16 +78,15 @@ module Kudzu
         request
       end
 
-      def request_klass_for(method)
-        Object.const_get("Net::HTTP::#{method.capitalize}")
-      end
-
-      def build_response(url, response, response_time)
+      def build_response(url, response, response_time, redirect_from)
+        fetched = response.instance_variable_get("@read")
         Response.new(url: url,
                      status: response.code.to_i,
-                     header: Hash[response.each.to_a],
-                     body: response.body.to_s,
-                     time: response_time)
+                     body: fetched ? response.body.to_s : nil,
+                     response_header: Hash[response.each.to_a],
+                     response_time: response_time,
+                     redirect_from: redirect_from,
+                     fetched: fetched)
       end
 
       def redirection?(code)
@@ -97,12 +94,12 @@ module Kudzu
         300 <= code && code <= 399
       end
 
-      def parse_cookie(url, response)
-        @jar.parse(response['set-cookie'], url) if response['set-cookie']
+      def parse_cookie(uri, response)
+        @jar.parse(response['set-cookie'], uri.to_s) if response['set-cookie']
       end
 
-      def append_cookie(url, request)
-        cookies = @jar.cookies(url)
+      def append_cookie(uri, request)
+        cookies = @jar.cookies(uri.to_s)
         unless cookies.empty?
           if request['Cookie']
             request['Cookie'] += '; ' + cookies.join('; ')

data/lib/kudzu/agent/http/connection.rb

@@ -0,0 +1,9 @@
+module Kudzu
+  class Agent
+    class Http
+      class Connection < Kudzu::Model::Base
+        attr_accessor :name, :http, :last_used
+      end
+    end
+  end
+end

data/lib/kudzu/agent/http/connection_pool.rb

@@ -0,0 +1,50 @@
+module Kudzu
+  class Agent
+    class Http
+      class ConnectionPool
+        def initialize(max_size = 10)
+          @max_size = max_size
+        end
+
+        def checkout(name)
+          pool[name] ||= Connection.new(name: name, http: yield)
+
+          conn = pool[name]
+          conn.last_used = Time.now
+
+          if pool.size > @max_size
+            reduce
+          end
+
+          conn.http
+        end
+
+        def close
+          pool.values.each do |conn|
+            finish_http(conn.http)
+          end
+          Thread.current[:kudzu_connection] = nil
+        end
+
+        private
+
+        def pool
+          Thread.current[:kudzu_connection] ||= {}
+          Thread.current[:kudzu_connection]
+        end
+
+        def reduce
+          conns = pool.values.sort_by { |conn| conn.last_used }
+          conns.first(pool.size - @max_size).each do |conn|
+            finish_http(conn.http)
+            pool.delete(conn.name)
+          end
+        end
+
+        def finish_http(http)
+          http.finish if http && http.started?
+        end
+      end
+    end
+  end
+end

data/lib/kudzu/agent/page_filterer.rb

@@ -0,0 +1,58 @@
+module Kudzu
+  class Agent
+    class PageFilterer
+      def initialize(config)
+        @config = config
+      end
+
+      def allowed?(response)
+        filter = @config.find_filter(response.url)
+
+        if filter.nil? || (allowed_mime_type?(response.mime_type, filter) &&
+                           allowed_size?(response.size, filter) &&
+                           allowed_index?(response))
+          Kudzu.log :info, "passed page: #{response.url}"
+          true
+        else
+          Kudzu.log :info, "dropped page: #{response.url}"
+          false
+        end
+      end
+
+      def allowed_response_header?(url, response_header)
+        filter = @config.find_filter(url)
+
+        if response_header['content-type']
+          mime_type = Util::ContentTypeParser.parse(response_header['content-type']).first
+        end
+        if response_header['content-length']
+          size = response_header['content-length'].to_i
+        end
+
+        filter.nil? || (allowed_mime_type?(mime_type, filter) &&
+                        allowed_size?(size, filter))
+      end
+
+      private
+
+      def allowed_mime_type?(mime_type, filter)
+        return true if mime_type.nil?
+        Util::Matcher.match?(mime_type, allows: filter.allow_mime_type, denies: filter.deny_mime_type)
+      end
+
+      def allowed_size?(size, filter)
+        return true if filter.max_size.nil? || size.nil?
+        size.to_i < filter.max_size.to_i
+      end
+
+      def allowed_index?(response)
+        return true if response.body.nil? || !response.html?
+        return true unless @config.respect_noindex
+
+        doc = response.parsed_doc
+        doc.xpath('html/head/meta[@name]')
+           .all? { |meta| meta[:name] !~ /^robots$/i || meta[:content] !~ /noindex/i }
+      end
+    end
+  end
+end

data/lib/kudzu/agent/reference.rb

@@ -0,0 +1,9 @@
+module Kudzu
+  class Agent
+    class Reference < Kudzu::Model::Base
+      include Kudzu::Model::Link
+
+      attr_accessor :url, :title
+    end
+  end
+end

data/lib/kudzu/agent/response.rb

@@ -0,0 +1,14 @@
+module Kudzu
+  class Agent
+    class Response < Kudzu::Model::Base
+      include Kudzu::Model::Page
+
+      attr_accessor :url, :status, :body, :response_header, :response_time, :redirect_from, :fetched,
+                    :size, :digest, :mime_type, :charset, :title
+
+      def fetched?
+        fetched
+      end
+    end
+  end
+end

data/lib/kudzu/agent/robots/parser.rb

@@ -0,0 +1,91 @@
+module Kudzu
+  class Agent
+    class Robots
+      class Parser
+        UNMATCH_REGEXP = /^$/
+
+        class << self
+          def parse(body)
+            txt = Txt.new
+            sets = []
+            prev_key = nil
+
+            parse_body(body).each do |key, value|
+              case key
+              when 'user-agent'
+                new_set = RuleSet.new(user_agent: ua_regexp(value))
+                txt.sets << new_set
+                if prev_key == 'user-agent'
+                  sets << new_set
+                else
+                  sets = [new_set]
+                end
+              when 'allow'
+                re = path_regexp(value)
+                sets.each { |set| set.rules << Rule.new(path: re, allow: true) }
+              when 'disallow'
+                re = path_regexp(value)
+                sets.each { |set| set.rules << Rule.new(path: re, allow: false) }
+              when 'crawl-delay'
+                sets.each { |set| set.crawl_delay = value.to_i }
+              when 'sitemap'
+                txt.sitemaps << value
+              end
+
+              prev_key = key
+            end
+
+            sort(txt)
+          end
+
+          private
+
+          def parse_body(body)
+            lines = body.to_s.split(/\r|\n|\r\n/)
+            lines.map { |line| parse_line(line) }.compact
+          end
+
+          def parse_line(line)
+            line.strip!
+            if line.empty? || line.start_with?('#')
+              nil
+            else
+              split_line(line)
+            end
+          end
+
+          def split_line(line)
+            key, value = line.split(':', 2)
+            key = key.to_s.strip.downcase
+            value = value.to_s.sub(/#.*$/, '').strip
+            if key.empty? || value.empty?
+              nil
+            else
+              [key, value]
+            end
+          end
+
+          def ua_regexp(value)
+            Regexp.new(Regexp.escape(value).gsub('\*', '.*'))
+          rescue RegexpError
+            UNMATCH_REGEXP
+          end
+
+          def path_regexp(value)
+            Regexp.new('^' + Regexp.escape(value).gsub('\*', '.*').gsub('\$', '$'))
+          rescue RegexpError
+            UNMATCH_REGEXP
+          end
+
+          def sort(txt)
+            txt.sets.sort_by! { |rule| [-rule.user_agent.to_s.count('*'), rule.user_agent.to_s.length] }.reverse!
+            txt.sets.each do |set|
+              set.rules.sort_by! { |rule| rule.path.to_s.length }.reverse!
+            end
+            txt
+          end
+        end
+      end
+    end
+  end
+end

data/lib/kudzu/agent/robots/txt.rb

@@ -0,0 +1,34 @@
+module Kudzu
+  class Agent
+    class Robots
+      class Txt < Kudzu::Model::Base
+        attr_accessor :sets, :sitemaps
+
+        def initialize
+          self.sets = []
+          self.sitemaps = []
+        end
+      end
+
+      class RuleSet < Kudzu::Model::Base
+        attr_accessor :user_agent, :rules, :crawl_delay
+
+        def initialize(attr = {})
+          self.rules = []
+          super
+        end
+
+        def allowed_path?(uri)
+          rules.each do |rule|
+            return rule.allow if uri.path =~ rule.path
+          end
+          return true
+        end
+      end
+
+      class Rule < Kudzu::Model::Base
+        attr_accessor :path, :allow
+      end
+    end
+  end
+end

data/lib/kudzu/agent/robots.rb

@@ -3,7 +3,6 @@ module Kudzu
     class Robots
       def initialize(config)
         @user_agent = config.user_agent
-        @page_fetcher = Kudzu::Agent::Fetcher.new(config)
         @monitor = Monitor.new
         @txt = {}
       end
@@ -49,11 +48,11 @@ module Kudzu
 
       def fetch_and_parse(uri)
         response = fetch(uri)
-        if response && response.status == 200
+        if response && response.code.to_i == 200
           body = response.body.force_encoding('utf-8').encode('utf-8', undef: :replace, invalid: :replace)
-          Parser.new.parse(body)
+          Parser.parse(body)
         else
-          Parser.new.parse('')
+          Parser.parse('')
         end
       end
 
@@ -62,127 +61,17 @@ module Kudzu
         uri.path = 'robots.txt'
         uri.fragment = uri.query = nil
 
-        begin
-          @page_fetcher.fetch(uri.to_s)
-        rescue
-          nil
-        end
-      end
-
-      class Txt
-        attr_accessor :sets, :sitemaps
-
-        def initialize
-          self.sets = []
-          self.sitemaps = []
-        end
-      end
-
-      class RuleSet
-        attr_accessor :user_agent, :rules, :crawl_delay
-
-        def initialize(attr = {})
-          self.rules = []
-          attr.each { |k, v| public_send("#{k}=", v) }
-        end
-
-        def allowed_path?(uri)
-          rules.each do |rule|
-            return rule.allow if uri.path =~ rule.path
-          end
-          return true
-        end
-      end
-
-      class Rule
-        attr_accessor :path, :allow
-
-        def initialize(attr = {})
-          attr.each { |k, v| public_send("#{k}=", v) }
-        end
-      end
-
-      class Parser
-        UNMATCH_REGEXP = /^$/
-
-        def parse(body)
-          txt = Txt.new
-          sets = []
-          prev_key = nil
-
-          parse_body(body).each do |key, value|
-            case key
-            when 'user-agent'
-              new_set = RuleSet.new(user_agent: ua_regexp(value))
-              txt.sets << new_set
-              if prev_key == 'user-agent'
-                sets << new_set
-              else
-                sets = [new_set]
-              end
-            when 'allow'
-              re = path_regexp(value)
-              sets.each { |set| set.rules << Rule.new(path: re, allow: true) }
-            when 'disallow'
-              re = path_regexp(value)
-              sets.each { |set| set.rules << Rule.new(path: re, allow: false) }
-            when 'crawl-delay'
-              sets.each { |set| set.crawl_delay = value.to_i }
-            when 'sitemap'
-              txt.sitemaps << value
-            end
-
-            prev_key = key
-          end
-
-          sort(txt)
-        end
-
-        private
-
-        def parse_body(body)
-          lines = body.to_s.split(/\r|\n|\r\n/)
-          lines.map { |line| parse_line(line) }.compact
-        end
-
-        def parse_line(line)
-          line.strip!
-          if line.empty? || line.start_with?('#')
-            nil
-          else
-            split_line(line)
-          end
-        end
-
-        def split_line(line)
-          key, value = line.split(':', 2)
-          key = key.to_s.strip.downcase
-          value = value.to_s.sub(/#.*$/, '').strip
-          if key.empty? || value.empty?
-            nil
-          else
-            [key, value]
-          end
-        end
-
-        def ua_regexp(value)
-          Regexp.new(Regexp.escape(value).gsub('\*', '.*'))
-        rescue RegexpError
-          UNMATCH_REGEXP
+        http = Net::HTTP.new(uri.host, uri.port || uri.default_port)
+        if uri.scheme == 'https'
+          http.use_ssl = true
+          http.verify_mode = OpenSSL::SSL::VERIFY_NONE
         end
 
-        def path_regexp(value)
-          Regexp.new('^' + Regexp.escape(value).gsub('\*', '.*').gsub('\$', '$'))
-        rescue RegexpError
-          UNMATCH_REGEXP
-        end
-
-        def sort(txt)
-          txt.sets.sort_by! { |rule| [-rule.user_agent.to_s.count('*'), rule.user_agent.to_s.length] }.reverse!
-          txt.sets.each do |set|
-            set.rules.sort_by! { |rule| rule.path.to_s.length }.reverse!
-          end
-          txt
+        begin
+          http.get(uri.to_s)
+        rescue => e
+          Kudzu.log :error, "failed to fetch robots.txt: #{uri}", error: e
+          nil
         end
       end
     end

data/lib/kudzu/agent/sleeper.rb

@@ -8,8 +8,8 @@ module Kudzu
         @last_accessed = {}
       end
 
-      def politeness_delay(url)
-        uri = Addressable::URI.parse(url)
+      def politeness_delay(uri)
+        uri = Addressable::URI.parse(uri) if uri.is_a?(String)
         delay_sec = delay_second(uri)
         return unless delay_sec