kudzu 1.0.0

data/lib/kudzu/agent/filter.rb

@@ -0,0 +1,40 @@
+require 'nokogiri'
+
+module Kudzu
+  class Agent
+    class Filter
+      def initialize(config)
+        @config = config
+        @matcher = Kudzu::Util::Matcher.new
+      end
+
+      def allowed?(page)
+        filter = @config.find_filter(page.url)
+        return true unless filter
+
+        allowed_mime_type?(page.mime_type, filter) && allowed_size?(page.size, filter) && allowed_index?(page)
+      end
+
+      private
+
+      def allowed_mime_type?(mime_type, filter)
+        return true if mime_type.nil?
+        @matcher.match?(mime_type, allows: filter.allow_mime_type, denies: filter.deny_mime_type)
+      end
+
+      def allowed_size?(size, filter)
+        return true if filter.max_size.nil? || size.nil?
+        size.to_i < filter.max_size.to_i
+      end
+
+      def allowed_index?(page)
+        return true unless page.html?
+        return true unless @config.respect_noindex
+
+        doc = Nokogiri::HTML(page.body.encode('ascii', undef: :replace, invalid: :replace))
+        doc.xpath('html/head/meta[@name]')
+           .all? { |meta| meta[:name] !~ /^robots$/i || meta[:content] !~ /noindex/i }
+      end
+    end
+  end
+end

data/lib/kudzu/agent/mime_type_detector.rb

@@ -0,0 +1,34 @@
+require 'shared-mime-info'
+
+module Kudzu
+  class Agent
+    class MimeTypeDetector
+      DEFALUT_MIME_TYPE = 'application/octet-stream'
+
+      def initialize
+        @content_type_parser = Kudzu::Util::ContentTypeParser.new
+      end
+
+      def detect(page)
+        from_header(page.response_header) || from_body(page.body) || from_url(page.url) || DEFALUT_MIME_TYPE
+      end
+
+      private
+
+      def from_header(header)
+        @content_type_parser.parse(header['content-type']).first
+      end
+
+      def from_body(body)
+        mime = MIME.check_magics(StringIO.new(body))
+        mime.to_s if mime
+      end
+
+      def from_url(url)
+        uri = Addressable::URI.parse(url)
+        mime = MIME.check_globs(uri.basename)
+        mime.to_s if mime
+      end
+    end
+  end
+end

data/lib/kudzu/agent/robots.rb

@@ -0,0 +1,190 @@
+module Kudzu
+  class Agent
+    class Robots
+      def initialize(config)
+        @user_agent = config.user_agent
+        @page_fetcher = Kudzu::Agent::Fetcher.new(config)
+        @monitor = Monitor.new
+        @txt = {}
+      end
+
+      def allowed?(uri)
+        uri = Addressable::URI.parse(uri) if uri.is_a?(String)
+        set = find_set(uri)
+        return true unless set
+        set.allowed_path?(uri)
+      end
+
+      def crawl_delay(uri)
+        uri = Addressable::URI.parse(uri) if uri.is_a?(String)
+        set = find_set(uri)
+        return nil unless set
+        set.crawl_delay
+      end
+
+      def sitemaps(uri)
+        uri = Addressable::URI.parse(uri) if uri.is_a?(String)
+        txt = find_txt(uri)
+        return [] unless txt
+        txt.sitemaps
+      end
+
+      private
+
+      def find_txt(uri)
+        @monitor.synchronize do
+          @txt[uri.host] ||= fetch_and_parse(uri)
+        end
+      end
+
+      def find_set(uri)
+        txt = find_txt(uri)
+        return unless txt
+
+        txt.sets.each do |set|
+          return set if @user_agent =~ set.user_agent
+        end
+        return nil
+      end
+
+      def fetch_and_parse(uri)
+        response = fetch(uri)
+        if response && response.status == 200
+          body = response.body.force_encoding('utf-8').encode('utf-8', undef: :replace, invalid: :replace)
+          Parser.new.parse(body)
+        else
+          Parser.new.parse('')
+        end
+      end
+
+      def fetch(base_uri)
+        uri = base_uri.dup
+        uri.path = 'robots.txt'
+        uri.fragment = uri.query = nil
+
+        begin
+          @page_fetcher.fetch(uri.to_s)
+        rescue
+          nil
+        end
+      end
+
+      class Txt
+        attr_accessor :sets, :sitemaps
+
+        def initialize
+          self.sets = []
+          self.sitemaps = []
+        end
+      end
+
+      class RuleSet
+        attr_accessor :user_agent, :rules, :crawl_delay
+
+        def initialize(attr = {})
+          self.rules = []
+          attr.each { |k, v| public_send("#{k}=", v) }
+        end
+
+        def allowed_path?(uri)
+          rules.each do |rule|
+            return rule.allow if uri.path =~ rule.path
+          end
+          return true
+        end
+      end
+
+      class Rule
+        attr_accessor :path, :allow
+
+        def initialize(attr = {})
+          attr.each { |k, v| public_send("#{k}=", v) }
+        end
+      end
+
+      class Parser
+        UNMATCH_REGEXP = /^$/
+
+        def parse(body)
+          txt = Txt.new
+          sets = []
+          prev_key = nil
+
+          parse_body(body).each do |key, value|
+            case key
+            when 'user-agent'
+              new_set = RuleSet.new(user_agent: ua_regexp(value))
+              txt.sets << new_set
+              if prev_key == 'user-agent'
+                sets << new_set
+              else
+                sets = [new_set]
+              end
+            when 'allow'
+              re = path_regexp(value)
+              sets.each { |set| set.rules << Rule.new(path: re, allow: true) }
+            when 'disallow'
+              re = path_regexp(value)
+              sets.each { |set| set.rules << Rule.new(path: re, allow: false) }
+            when 'crawl-delay'
+              sets.each { |set| set.crawl_delay = value.to_i }
+            when 'sitemap'
+              txt.sitemaps << value
+            end
+
+            prev_key = key
+          end
+
+          sort(txt)
+        end
+
+        private
+
+        def parse_body(body)
+          lines = body.to_s.split(/\r|\n|\r\n/)
+          lines.map { |line| parse_line(line) }.compact
+        end
+
+        def parse_line(line)
+          line.strip!
+          if line.empty? || line.start_with?('#')
+            nil
+          else
+            split_line(line)
+          end
+        end
+
+        def split_line(line)
+          key, value = line.split(':', 2)
+          key = key.to_s.strip.downcase
+          value = value.to_s.sub(/#.*$/, '').strip
+          if key.empty? || value.empty?
+            nil
+          else
+            [key, value]
+          end
+        end
+
+        def ua_regexp(value)
+          Regexp.new(Regexp.escape(value).gsub('\*', '.*'))
+        rescue RegexpError
+          UNMATCH_REGEXP
+        end
+
+        def path_regexp(value)
+          Regexp.new('^' + Regexp.escape(value).gsub('\*', '.*').gsub('\$', '$'))
+        rescue RegexpError
+          UNMATCH_REGEXP
+        end
+
+        def sort(txt)
+          txt.sets.sort_by! { |rule| [-rule.user_agent.to_s.count('*'), rule.user_agent.to_s.length] }.reverse!
+          txt.sets.each do |set|
+            set.rules.sort_by! { |rule| rule.path.to_s.length }.reverse!
+          end
+          txt
+        end
+      end
+    end
+  end
+end

data/lib/kudzu/agent/sleeper.rb

@@ -0,0 +1,44 @@
+module Kudzu
+  class Agent
+    class Sleeper
+      def initialize(config, robots = nil)
+        @config = config
+        @robots = robots
+        @monitor = Monitor.new
+        @last_accessed = {}
+      end
+
+      def politeness_delay(url)
+        uri = Addressable::URI.parse(url)
+        delay_sec = delay_second(uri)
+        return unless delay_sec
+
+        sleep_sec = sleep_second(uri, delay_sec)
+        sleep sleep_sec if sleep_sec > 0
+      end
+
+      private
+
+      def delay_second(uri)
+        if @config.respect_robots_txt && @robots && (crawl_delay = @robots.crawl_delay(uri))
+          crawl_delay.to_f
+        elsif @config.politeness_delay
+          @config.politeness_delay.to_f
+        end
+      end
+
+      def sleep_second(uri, delay_sec)
+        @monitor.synchronize do
+          now = Time.now.to_f
+          value = if @last_accessed[uri.host]
+                    (@last_accessed[uri.host] + delay_sec) - now
+                  else
+                    0
+                  end
+          @last_accessed[uri.host] = now + value
+          value
+        end
+      end
+    end
+  end
+end

data/lib/kudzu/agent/title_parser.rb

@@ -0,0 +1,16 @@
+require 'nokogiri'
+
+module Kudzu
+  class Agent
+    class TitleParser
+      def parse(page)
+        doc = Nokogiri::HTML(page.decoded_body)
+        if (node = doc.xpath('//head/title').first)
+          node.inner_text.to_s
+        else
+          ''
+        end
+      end
+    end
+  end
+end

data/lib/kudzu/agent/url_extractor.rb

@@ -0,0 +1,123 @@
+require 'nokogiri'
+
+module Kudzu
+  class Agent
+    class UrlExtractor
+      def initialize(config)
+        @config = config
+      end
+
+      def extract(page, base_url)
+        hrefs = if page.html?
+                  FromHTML.new(@config).extract(page)
+                elsif page.xml?
+                  FromXML.new(@config).extract(page)
+                else
+                  []
+                end
+
+        hrefs.select do |href|
+          href[:url] = normalize(href[:url], base_url)
+        end
+      end
+
+      private
+
+      def normalize(url, base_url)
+        uri = Addressable::URI.parse(base_url.to_s).join(url.to_s).normalize
+        uri.path = '/' unless uri.path
+        uri.fragment = nil
+
+        if uri.scheme.in?(%w(http https))
+          uri.to_s
+        else
+          nil
+        end
+      end
+
+      class FromHTML < UrlExtractor
+        def initialize(config)
+          super
+          @content_type_parser = Kudzu::Util::ContentTypeParser.new
+        end
+
+        def extract(page)
+          doc = Nokogiri::HTML(page.decoded_body)
+          return [] if nofollow?(doc)
+
+          if (filter = @config.find_filter(page.url))
+            if filter.allow_element
+              doc = doc.search(*Array(filter.allow_element))
+            end
+            if filter.deny_element
+              doc.search(*Array(filter.deny_element)).remove
+            end
+          end
+
+          hrefs = from_html(doc) + from_html_in_meta(doc)
+          hrefs.reject { |href| href[:url].empty? }.uniq
+        end
+
+        private
+
+        def nofollow?(doc)
+          return false unless @config.respect_nofollow
+          nodes = doc.xpath('//meta[@name]')
+          nodes.any? { |node| node[:name] =~ /^robots$/i && node[:content] =~ /nofollow/i }
+        end
+
+        def from_html(doc)
+          nodes = doc.xpath('.//*[@href or @src]').to_a
+
+          if @config.respect_nofollow
+            nodes.reject! { |url| url[:rel] =~ /nofollow/i }
+          end
+
+          nodes.map { |node|
+            { url: (node[:href] || node[:src]).to_s.strip,
+              title: node_to_title(node) }
+          }
+        end
+
+        def node_to_title(node)
+          unless node.inner_text.empty?
+            node.inner_text
+          else
+            (node[:title] || node[:alt]).to_s
+          end
+        end
+
+        def from_html_in_meta(doc)
+          nodes = doc.xpath('.//meta[@http-equiv]').select { |node| node[:'http-equiv'] =~ /^refresh$/i }
+          urls = nodes.map { |node| @content_type_parser.parse(node[:content]).last[:url] }.compact
+          urls.map { |url| { url: url.to_s.strip } }
+        end
+      end
+
+      class FromXML < UrlExtractor
+        def extract(page)
+          doc = Nokogiri::XML(page.decoded_body)
+          doc.remove_namespaces!
+          hrefs = from_xml_rss(doc) + from_xml_atom(doc)
+          hrefs.reject { |href| href[:url].empty? }.uniq
+        end
+
+        private
+
+        def from_xml_rss(doc)
+          doc.xpath('rss/channel').map { |node|
+            { url: node.xpath('./item/link').inner_text.strip,
+              title: node.xpath('./item/title').inner_text }
+          }
+        end
+
+        def from_xml_atom(doc)
+          doc.xpath('feed/entry').map { |node|
+            { url: node.xpath('./link[@href]/@href').to_s.strip,
+              title: node.xpath('./title').inner_text }
+          }
+        end
+      end
+    end
+  end
+end