kudzu 1.0.0 → 1.1.0

data/lib/kudzu/agent/url_extractor.rb

@@ -1,5 +1,3 @@
-require 'nokogiri'
-
 module Kudzu
   class Agent
     class UrlExtractor
@@ -7,25 +5,32 @@ module Kudzu
         @config = config
       end
 
-      def extract(page, base_url)
-        hrefs = if page.html?
-                  FromHTML.new(@config).extract(page)
-                elsif page.xml?
-                  FromXML.new(@config).extract(page)
-                else
-                  []
-                end
-
-        hrefs.select do |href|
-          href[:url] = normalize(href[:url], base_url)
+      def extract(response)
+        refs = if response.html?
+                 ForHTML.new(@config).extract(response)
+               elsif response.xml?
+                 ForXML.new(@config).extract(response)
+               else
+                 []
+               end
+
+        refs.each do |ref|
+          ref.url = sanitize(ref.url)
+          ref.url = normalize(ref.url, response.url)
         end
+        refs.reject { |ref| ref.url.nil? }.uniq
       end
 
       private
 
+      def sanitize(url)
+        url.gsub(/^( |　|%20)+/, '')
+      end
+
       def normalize(url, base_url)
-        uri = Addressable::URI.parse(base_url.to_s).join(url.to_s).normalize
+        uri = Addressable::URI.parse(base_url).join(url).normalize
         uri.path = '/' unless uri.path
+        uri.path = uri.path.gsub(%r|/{2,}|, '/')
         uri.fragment = nil
 
         if uri.scheme.in?(%w(http https))
@@ -33,35 +38,37 @@ module Kudzu
         else
           nil
         end
+      rescue => e
+        Kudzu.log :warn, "failed to normalize url: #{url}", error: e
+        nil
       end
 
-      class FromHTML < UrlExtractor
+      class ForHTML
         def initialize(config)
-          super
-          @content_type_parser = Kudzu::Util::ContentTypeParser.new
+          @config = config
         end
 
-        def extract(page)
-          doc = Nokogiri::HTML(page.decoded_body)
-          return [] if nofollow?(doc)
+        def extract(response)
+          doc = response.parsed_doc
+          return [] if @config.respect_nofollow && nofollow?(doc)
 
-          if (filter = @config.find_filter(page.url))
+          if (filter = @config.find_filter(response.url))
             if filter.allow_element
               doc = doc.search(*Array(filter.allow_element))
             end
             if filter.deny_element
+              doc = doc.dup
               doc.search(*Array(filter.deny_element)).remove
             end
           end
 
-          hrefs = from_html(doc) + from_html_in_meta(doc)
-          hrefs.reject { |href| href[:url].empty? }.uniq
+          refs = from_html(doc) + from_meta(doc)
+          refs.reject { |ref| ref.url.nil? || ref.url.empty? }
         end
 
         private
 
         def nofollow?(doc)
-          return false unless @config.respect_nofollow
           nodes = doc.xpath('//meta[@name]')
           nodes.any? { |node| node[:name] =~ /^robots$/i && node[:content] =~ /nofollow/i }
         end
@@ -73,10 +80,10 @@ module Kudzu
             nodes.reject! { |url| url[:rel] =~ /nofollow/i }
           end
 
-          nodes.map { |node|
-            { url: (node[:href] || node[:src]).to_s.strip,
-              title: node_to_title(node) }
-          }
+          nodes.map do |node|
+            Reference.new(url: (node[:href] || node[:src]).to_s,
+                          title: node_to_title(node))
+          end
         end
 
         def node_to_title(node)
@@ -87,35 +94,42 @@ module Kudzu
           end
         end
 
-        def from_html_in_meta(doc)
+        def from_meta(doc)
           nodes = doc.xpath('.//meta[@http-equiv]').select { |node| node[:'http-equiv'] =~ /^refresh$/i }
-          urls = nodes.map { |node| @content_type_parser.parse(node[:content]).last[:url] }.compact
-          urls.map { |url| { url: url.to_s.strip } }
+          urls = nodes.map { |node| Util::ContentTypeParser.parse(node[:content]).last[:url] }.compact
+          urls.map do |url|
+            Reference.new(url: url.to_s)
+          end
         end
       end
 
-      class FromXML < UrlExtractor
-        def extract(page)
-          doc = Nokogiri::XML(page.decoded_body)
+      class ForXML
+        def initialize(config)
+          @config = config
+        end
+
+        def extract(response)
+          doc = response.parsed_doc.dup
           doc.remove_namespaces!
-          hrefs = from_xml_rss(doc) + from_xml_atom(doc)
-          hrefs.reject { |href| href[:url].empty? }.uniq
+
+          refs = from_rss(doc) + from_atom(doc)
+          refs.reject { |ref| ref.url.nil? || ref.url.empty? }
         end
 
         private
 
-        def from_xml_rss(doc)
-          doc.xpath('rss/channel').map { |node|
-            { url: node.xpath('./item/link').inner_text.strip,
-              title: node.xpath('./item/title').inner_text }
-          }
+        def from_rss(doc)
+          doc.xpath('rss/channel').map do |node|
+            Reference.new(url: node.xpath('./item/link').inner_text,
+                          title: node.xpath('./item/title').inner_text)
+          end
         end
 
-        def from_xml_atom(doc)
-          doc.xpath('feed/entry').map { |node|
-            { url: node.xpath('./link[@href]/@href').to_s.strip,
-              title: node.xpath('./title').inner_text }
-          }
+        def from_atom(doc)
+          doc.xpath('feed/entry').map do |node|
+            Reference.new(url: node.xpath('./link[@href]/@href').to_s,
+                          title: node.xpath('./title').inner_text)
+          end
         end
       end
     end

data/lib/kudzu/agent/{url_filter.rb → url_filterer.rb}

@@ -1,22 +1,28 @@
 module Kudzu
   class Agent
-    class UrlFilter
-      def initialize(config)
+    class UrlFilterer
+      def initialize(config, robots = nil)
         @config = config
-        @matcher = Kudzu::Util::Matcher.new
+        @robots = robots
       end
 
-      def filter(hrefs, base_url)
+      def filter(refs, base_url)
         base_uri = Addressable::URI.parse(base_url)
         filter = @config.find_filter(base_uri)
 
-        hrefs.partition do |href|
-          allowed?(href[:url], base_uri, filter: filter)
+        refs.select do |ref|
+          if allowed?(ref.uri, base_uri, filter: filter)
+            Kudzu.log :debug, "passed url: #{ref.url}"
+            true
+          else
+            Kudzu.log :debug, "dropped url: #{ref.url}"
+            false
+          end
         end
       end
 
-      def allowed?(url, base_uri, filter: nil)
-        uri = Addressable::URI.parse(url)
+      def allowed?(uri, base_uri, filter: nil)
+        uri = Addressable::URI.parse(uri) if uri.is_a?(String)
         base_uri = Addressable::URI.parse(base_uri) if base_uri.is_a?(String)
         filter ||= @config.find_filter(base_uri)
         return true unless filter
@@ -26,7 +32,8 @@ module Kudzu
           allowed_url?(uri, filter) &&
           allowed_host?(uri, filter) &&
           allowed_path?(uri, filter) &&
-          allowed_ext?(uri, filter)
+          allowed_ext?(uri, filter) &&
+          allowed_by_robots?(uri)
       end
 
       private
@@ -44,21 +51,27 @@ module Kudzu
       end
 
       def allowed_url?(uri, filter)
-        @matcher.match?(uri.to_s, allows: filter.allow_url, denies: filter.deny_url)
+        Util::Matcher.match?(uri.to_s, allows: filter.allow_url, denies: filter.deny_url)
       end
 
       def allowed_host?(uri, filter)
-        @matcher.match?(uri.host, allows: filter.allow_host, denies: filter.deny_host)
+        Util::Matcher.match?(uri.host, allows: filter.allow_host, denies: filter.deny_host)
       end
 
       def allowed_path?(uri, filter)
-        @matcher.match?(uri.path, allows: filter.allow_path, denies: filter.deny_path)
+        Util::Matcher.match?(uri.path, allows: filter.allow_path, denies: filter.deny_path)
       end
 
       def allowed_ext?(uri, filter)
         ext = uri.extname.to_s.sub(/^\./, '')
         return true if ext.empty?
-        @matcher.match?(ext, allows: filter.allow_ext, denies: filter.deny_ext)
+        Util::Matcher.match?(ext, allows: filter.allow_ext, denies: filter.deny_ext)
+      end
+
+      def allowed_by_robots?(uri)
+        return true unless @robots
+        return true unless @config.respect_robots_txt
+        @robots.allowed?(uri)
       end
     end
   end

data/lib/kudzu/agent/util/charset_detector.rb

@@ -0,0 +1,84 @@
+module Kudzu
+  class Agent
+    class Util
+      class CharsetDetector
+        CORRECTION = {
+          'utf_8' => 'utf-8',
+          'shift-jis' => 'shift_jis',
+          'x-sjis' => 'shift_jis',
+          'euc_jp' => 'euc-jp'
+        }
+
+        class << self
+          def detect(response)
+            if response.html?
+              from_html(response.body) || from_text(response.body)
+            elsif response.xml?
+              from_xml(response.body) || from_text(response.body)
+            elsif response.text?
+              from_text(response.body)
+            end
+          rescue => e
+            Kudzu.log :warn, "failed to detect charset: #{response.url}", error: e
+            nil
+          end
+
+          private
+
+          def from_html(body)
+            doc = Nokogiri::HTML(body.encode('utf-8', undef: :replace, invalid: :replace))
+
+            if (node = doc.xpath('//meta/@charset').first)
+              charset = correct(node.to_s)
+              return charset if charset
+            end
+
+            doc.xpath('//meta[@http-equiv]').each do |meta|
+              if meta['http-equiv'] =~ /content-type/i
+                charset = ContentTypeParser.parse(meta[:content].to_s)[1][:charset]
+                charset = correct(node.to_s)
+                return charset if charset
+              end
+            end
+
+            return nil
+          end
+
+          def from_xml(body)
+            doc = Nokogiri::XML(body.encode('utf-8', undef: :replace, invalid: :replace))
+            if doc.encoding
+              correct(doc.encoding)
+            else
+              nil
+            end
+          end
+
+          def from_text(text)
+            if text.ascii_only?
+              'ascii'
+            else
+              detection = CharlockHolmes::EncodingDetector.detect(text)
+              if detection && detection.key?(:encoding)
+                detection[:encoding].downcase
+              else
+                nil
+              end
+            end
+          end
+
+          def correct(charset)
+            charset = charset.downcase
+            charset = CORRECTION[charset] if CORRECTION.key?(charset)
+
+            begin
+              Encoding.find(charset)
+            rescue
+              charset = nil
+            end
+            charset
+          end
+        end
+      end
+    end
+  end
+end

data/lib/kudzu/agent/util/content_type_parser.rb

@@ -0,0 +1,28 @@
+module Kudzu
+  class Agent
+    class Util
+      class ContentTypeParser
+        class << self
+          def parse(content_type)
+            mime, *kvs = content_type.to_s.split(';').map { |str| str.strip.downcase }
+            params = kvs.each_with_object({}) do |kv, hash|
+                       k, v = kv.to_s.split('=').map { |str| str.strip }
+                       hash[k.to_sym] = unquote(v) if k && v
+                     end
+            return mime, params
+          end
+
+          private
+
+          def unquote(str)
+            if str =~ /^"(.*?)"$/
+              $1.gsub(/\\(.)/, '\1')
+            else
+              str
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/kudzu/agent/util/matcher.rb

@@ -0,0 +1,25 @@
+module Kudzu
+  class Agent
+    class Util
+      class Matcher
+        class << self
+          def match?(text, allows: nil, denies: nil)
+            match_to_allows?(text, allows) && !match_to_denies?(text, denies)
+          end
+
+          private
+
+          def match_to_allows?(text, allows)
+            allows = Array(allows)
+            allows.empty? || allows.any? { |allow| Kudzu::Common.match?(text, allow) }
+          end
+
+          def match_to_denies?(text, denies)
+            denies = Array(denies)
+            !denies.empty? && denies.any? { |deny| Kudzu::Common.match?(text, deny) }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/kudzu/agent/util/mime_type_detector.rb

@@ -0,0 +1,38 @@
+module Kudzu
+  class Agent
+    class Util
+      class MimeTypeDetector
+        DEFALUT_MIME_TYPE = 'application/octet-stream'
+
+        class << self
+          def detect(response)
+            from_header(response.response_header) ||
+              from_body(response.body) ||
+              from_url(response.url) ||
+              DEFALUT_MIME_TYPE
+          rescue => e
+            Kudzu.log :warn, "failed to detect mime: #{response.url}", error: e
+            nil
+          end
+
+          private
+
+          def from_header(header)
+            ContentTypeParser.parse(header['content-type']).first
+          end
+
+          def from_body(body)
+            mime = MIME.check_magics(StringIO.new(body))
+            mime.to_s if mime
+          end
+
+          def from_url(url)
+            uri = Addressable::URI.parse(url)
+            mime = MIME.check_globs(uri.basename)
+            mime.to_s if mime
+          end
+        end
+      end
+    end
+  end
+end

data/lib/kudzu/agent/util/title_parser.rb

@@ -0,0 +1,30 @@
+module Kudzu
+  class Agent
+    class Util
+      class TitleParser
+        class << self
+          def parse(response)
+            if response.html?
+              from_html(response.parsed_doc)
+            else
+              Addressable::URI.parse(response.url).basename
+            end
+          rescue => e
+            Kudzu.log :warn, "failed to parse title: #{response.url}", error: e
+            nil
+          end
+
+          private
+
+          def from_html(doc)
+            if (node = doc.xpath('//head/title').first)
+              node.inner_text.to_s
+            else
+              ''
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/kudzu/agent.rb

@@ -0,0 +1,42 @@
+require_relative 'agent/all'
+
+module Kudzu
+  class Agent
+    def initialize(config, &block)
+      @config = config
+
+      @robots = Robots.new(@config)
+      @fetcher = Fetcher.new(@config, @robots)
+      @url_extractor = UrlExtractor.new(@config)
+      @url_filterer = UrlFilterer.new(@config, @robots)
+      @page_filterer = PageFilterer.new(@config)
+    end
+
+    def start
+      yield
+      @fetcher.pool.close
+   end
+
+    def fetch(url, request_header = {})
+      response = @fetcher.fetch(url, request_header: request_header)
+      return response unless response.fetched?
+
+      response.size = response.body.size
+      response.digest = Digest::MD5.hexdigest(response.body)
+      response.mime_type = Util::MimeTypeDetector.detect(response)
+      response.charset = Util::CharsetDetector.detect(response) if response.text?
+      response.title = Util::TitleParser.parse(response)
+      response
+    end
+
+    def extract_refs(response)
+      refs = @url_extractor.extract(response)
+      @url_filterer.filter(refs, response.url)
+    end
+
+    def filter_response?(response)
+      return false if response.redirect_from && !@url_filterer.allowed?(response.url, response.redirect_from)
+      !@page_filterer.allowed?(response)
+    end
+  end
+end

data/lib/kudzu/callback.rb

@@ -6,12 +6,14 @@ module Kudzu
                  :on_server_error, # 5xx
                  :on_filter,       # 2xx, filtered
                  :on_failure,      # Exception
+                 :before_enqueue, 
+                 :after_enqueue,
+                 :before_fetch, 
+                 :after_fetch,
                  :before_register,
                  :after_register,
                  :before_delete,
                  :after_delete,
-                 :before_enqueue, 
-                 :after_enqueue,
                  ]
 
     def initialize(&block)

data/lib/kudzu/config/filter.rb

@@ -13,27 +13,27 @@ module Kudzu
       attr_accessor :path
       attr_accessor *SIMPLE_CONFIGS
 
+      def initialize(path, config = {}, &block)
+        @path = path
+        DEFAULT_CONFIG.merge(config).each do |key, value|
+          send("#{key}=", value)
+        end
+        if block
+          Delegator.new(self).instance_eval(&block)
+        end
+      end
+
       class Delegator
         def initialize(filter)
           @filter = filter
         end
 
-        Kudzu::Config::Filter::SIMPLE_CONFIGS.each do |key|
+        SIMPLE_CONFIGS.each do |key|
           define_method(key) do |value|
             @filter.send("#{key}=", value)
           end
         end
       end
-
-      def initialize(path, config = {}, &block)
-        @path = path
-        DEFAULT_CONFIG.merge(config).each do |key, value|
-          send("#{key}=", value)
-        end
-        if block
-          Kudzu::Config::Filter::Delegator.new(self).instance_eval(&block)
-        end
-      end
     end
   end
 end

data/lib/kudzu/config.rb

@@ -4,16 +4,15 @@ require_relative 'config/filter'
 module Kudzu
   class Config
     SIMPLE_CONFIGS   = [:config_file,
-                        :user_agent, :thread_num, :open_timeout, :read_timeout,
+                        :user_agent, :thread_num, :open_timeout, :read_timeout, :keep_alive,
                         :max_connection, :max_redirect, :max_depth, :default_request_header,
                         :politeness_delay, :handle_cookie,
                         :respect_robots_txt, :respect_nofollow, :respect_noindex,
-                        :log_file, :log_level,
-                        :revisit_mode, :revisit_min_interval, :revisit_max_interval, :revisit_default_interval,
                         :filters]
     DEFAULT_CONFIG   = { user_agent: "Kudzu/#{Kudzu::VERSION}",
                          open_timeout: 10,
                          read_timeout: 10,
+                         keep_alive: 5,
                          thread_num: 1,
                          max_connection: 10,
                          max_redirect: 3,
@@ -21,37 +20,17 @@ module Kudzu
                          handle_cookie: true,
                          respect_robots_txt: true,
                          respect_nofollow: true,
-                         respect_noindex: true,
-                         revisit_mode: false,
-                         revisit_min_interval: 1,
-                         revisit_max_interval: 10,
-                         revisit_default_interval: 5 }
+                         respect_noindex: true }
 
     attr_accessor *SIMPLE_CONFIGS
 
-    class Delegator
-      def initialize(config)
-        @config = config
-      end
-
-      Kudzu::Config::SIMPLE_CONFIGS.each do |key|
-        define_method(key) do |value|
-          @config.send("#{key}=", value)
-        end
-      end
-
-      def add_filter(base_url = nil, config = {}, &block)
-        @config.add_filter(base_url, config, &block)
-      end
-    end
-
     def initialize(config = {}, &block)
       self.filters = {}
       DEFAULT_CONFIG.merge(config).each do |key, value|
         send("#{key}=", value)
       end
       if config_file || block
-        delegator = Kudzu::Config::Delegator.new(self)
+        delegator = Delegator.new(self)
         delegator.instance_eval(File.read(config_file)) if config_file
         delegator.instance_eval(&block) if block
       end
@@ -75,5 +54,21 @@ module Kudzu
       end
       nil
     end
+
+    class Delegator
+      def initialize(config)
+        @config = config
+      end
+
+      SIMPLE_CONFIGS.each do |key|
+        define_method(key) do |value|
+          @config.send("#{key}=", value)
+        end
+      end
+
+      def add_filter(base_url = nil, config = {}, &block)
+        @config.add_filter(base_url, config, &block)
+      end
+    end
   end
 end