kudzu 1.0.0 → 1.1.0

data/lib/kudzu/agent/charset_detector.rb

@@ -1,84 +0,0 @@
-require 'nokogiri'
-require 'charlock_holmes'
-
-module Kudzu
-  class Agent
-    class CharsetDetector
-      CORRECTION = {
-        'utf_8' => 'utf-8',
-        'shift-jis' => 'shift_jis',
-        'x-sjis' => 'shift_jis',
-        'euc_jp' => 'euc-jp'
-      }
-
-      def initialize
-        @parser = Kudzu::Util::ContentTypeParser.new
-      end
-
-      def detect(page)
-        if page.html?
-          from_html(page.body) || from_text(page.body)
-        elsif page.xml?
-          from_xml(page.body) || from_text(page.body)
-        elsif page.text?
-          from_text(page.body)
-        end
-      end
-
-      private
-
-      def from_html(body)
-        doc = Nokogiri::HTML(body.encode('ascii', undef: :replace, invalid: :replace))
-
-        if (node = doc.xpath('//meta/@charset').first)
-          charset = correct(node.to_s)
-          return charset if charset
-        end
-
-        doc.xpath('//meta[@http-equiv]').each do |meta|
-          if meta['http-equiv'] =~ /content-type/i
-            charset = @parser.parse(meta[:content].to_s)[1][:charset]
-            charset = correct(node.to_s)
-            return charset if charset
-          end
-        end
-
-        return nil
-      end
-
-      def from_xml(body)
-        doc = Nokogiri::XML(body.encode('ascii', undef: :replace, invalid: :replace))
-        if doc.encoding
-          correct(doc.encoding)
-        else
-          nil
-        end
-      end
-
-      def from_text(text)
-        if text.ascii_only?
-          'ascii'
-        else
-          detection = CharlockHolmes::EncodingDetector.detect(text)
-          if detection && detection.key?(:encoding)
-            detection[:encoding].downcase
-          else
-            nil
-          end
-        end
-      end
-
-      def correct(charset)
-        charset = charset.downcase
-        charset = CORRECTION[charset] if CORRECTION.key?(charset)
-
-        begin
-          Encoding.find(charset)
-        rescue
-          charset = nil
-        end
-        charset
-      end
-    end
-  end
-end

data/lib/kudzu/agent/filter.rb

@@ -1,40 +0,0 @@
-require 'nokogiri'
-
-module Kudzu
-  class Agent
-    class Filter
-      def initialize(config)
-        @config = config
-        @matcher = Kudzu::Util::Matcher.new
-      end
-
-      def allowed?(page)
-        filter = @config.find_filter(page.url)
-        return true unless filter
-
-        allowed_mime_type?(page.mime_type, filter) && allowed_size?(page.size, filter) && allowed_index?(page)
-      end
-
-      private
-
-      def allowed_mime_type?(mime_type, filter)
-        return true if mime_type.nil?
-        @matcher.match?(mime_type, allows: filter.allow_mime_type, denies: filter.deny_mime_type)
-      end
-
-      def allowed_size?(size, filter)
-        return true if filter.max_size.nil? || size.nil?
-        size.to_i < filter.max_size.to_i
-      end
-
-      def allowed_index?(page)
-        return true unless page.html?
-        return true unless @config.respect_noindex
-
-        doc = Nokogiri::HTML(page.body.encode('ascii', undef: :replace, invalid: :replace))
-        doc.xpath('html/head/meta[@name]')
-           .all? { |meta| meta[:name] !~ /^robots$/i || meta[:content] !~ /noindex/i }
-      end
-    end
-  end
-end

data/lib/kudzu/agent/mime_type_detector.rb

@@ -1,34 +0,0 @@
-require 'shared-mime-info'
-
-module Kudzu
-  class Agent
-    class MimeTypeDetector
-      DEFALUT_MIME_TYPE = 'application/octet-stream'
-
-      def initialize
-        @content_type_parser = Kudzu::Util::ContentTypeParser.new
-      end
-
-      def detect(page)
-        from_header(page.response_header) || from_body(page.body) || from_url(page.url) || DEFALUT_MIME_TYPE
-      end
-
-      private
-
-      def from_header(header)
-        @content_type_parser.parse(header['content-type']).first
-      end
-
-      def from_body(body)
-        mime = MIME.check_magics(StringIO.new(body))
-        mime.to_s if mime
-      end
-
-      def from_url(url)
-        uri = Addressable::URI.parse(url)
-        mime = MIME.check_globs(uri.basename)
-        mime.to_s if mime
-      end
-    end
-  end
-end

data/lib/kudzu/agent/title_parser.rb

@@ -1,16 +0,0 @@
-require 'nokogiri'
-
-module Kudzu
-  class Agent
-    class TitleParser
-      def parse(page)
-        doc = Nokogiri::HTML(page.decoded_body)
-        if (node = doc.xpath('//head/title').first)
-          node.inner_text.to_s
-        else
-          ''
-        end
-      end
-    end
-  end
-end

data/lib/kudzu/logger.rb

@@ -1,20 +0,0 @@
-module Kudzu
-  class Logger
-    def initialize(file, level)
-      if file.is_a?(::Logger)
-        @logger = file
-      elsif file
-        @logger = ::Logger.new(file)
-        @logger.level = level
-      else
-        @logger = nil
-      end
-    end
-
-    def log(level, message, error: nil)
-      return unless @logger
-      message += " #{error.class} #{error.message} #{error.backtrace.join("\n")}" if error
-      @logger.send(level, message)
-    end
-  end
-end

data/lib/kudzu/revisit/all.rb

@@ -1,3 +0,0 @@
-Dir[File.join(__dir__, '*.rb')].each do |file|
-  require_relative file
-end

data/lib/kudzu/revisit/scheduler.rb

@@ -1,28 +0,0 @@
-module Kudzu
-  class Revisit
-    class Scheduler
-      def initialize(config)
-        @config = config
-      end
-
-      def schedule(page, modified: true)
-        page.revisit_interval = next_interval(page.revisit_interval, modified)
-        page.revisit_at = page.fetched_at + page.revisit_interval * 86400
-      end
-
-      private
-
-      def next_interval(curr_interval, modified)
-        if curr_interval
-          if modified
-            [curr_interval - 1, @config.revisit_min_interval].max
-          else
-            [curr_interval + 1, @config.revisit_max_interval].min
-          end
-        else
-          @config.revisit_default_interval
-        end
-      end
-    end
-  end
-end

data/lib/kudzu/util/all.rb

@@ -1,3 +0,0 @@
-Dir[File.join(__dir__, '*.rb')].each do |file|
-  require_relative file
-end

data/lib/kudzu/util/connection_pool.rb

@@ -1,56 +0,0 @@
-module Kudzu
-  class Util
-    class ConnectionPool
-      class Connection
-        attr_accessor :name, :http, :last_used
-
-        def initialize(attr = {})
-          attr.each { |k, v| public_send("#{k}=", v) }
-        end
-      end
-
-      def initialize(max_size = 10)
-        @max_size = max_size
-      end
-
-      def checkout(name)
-        pool[name] ||= Connection.new(name: name, http: yield)
-
-        conn = pool[name]
-        conn.last_used = Time.now
-
-        if pool.size > @max_size
-          reduce
-        end
-
-        conn.http
-      end
-
-      def close
-        pool.values.each do |conn|
-          finish_http(conn.http)
-        end
-        Thread.current[:kudzu_connection] = nil
-      end
-
-      private
-
-      def pool
-        Thread.current[:kudzu_connection] ||= {}
-        Thread.current[:kudzu_connection]
-      end
-
-      def reduce
-        conns = pool.values.sort_by { |conn| conn.last_used }
-        conns.first(pool.size - @max_size).each do |conn|
-          finish_http(conn.http)
-          pool.delete(conn.name)
-        end
-      end
-
-      def finish_http(http)
-        http.finish if http && http.started?
-      end
-    end
-  end
-end

data/lib/kudzu/util/content_type_parser.rb

@@ -1,24 +0,0 @@
-module Kudzu
-  class Util
-    class ContentTypeParser
-      def parse(content_type)
-        mime, *kvs = content_type.to_s.split(';').map { |str| str.strip.downcase }
-        params = kvs.each_with_object({}) do |kv, hash|
-                   k, v = kv.to_s.split('=').map { |str| str.strip }
-                   hash[k.to_sym] = unquote(v) if k && v
-                 end
-        return mime, params
-      end
-
-      private
-
-      def unquote(str)
-        if str =~ /^"(.*?)"$/
-          $1.gsub(/\\(.)/, '\1')
-        else
-          str
-        end
-      end
-    end
-  end
-end

data/lib/kudzu/util/matcher.rb

@@ -1,21 +0,0 @@
-module Kudzu
-  class Util
-    class Matcher
-      def match?(text, allows: nil, denies: nil)
-        match_to_allows?(text, allows) && !match_to_denies?(text, denies)
-      end
-
-      private
-
-      def match_to_allows?(text, allows)
-        allows = Array(allows)
-        allows.empty? || allows.any? { |allow| Kudzu::Common.match?(text, allow) }
-      end
-
-      def match_to_denies?(text, denies)
-        denies = Array(denies)
-        !denies.empty? && denies.any? { |deny| Kudzu::Common.match?(text, deny) }
-      end
-    end
-  end
-end

data/lib/kudzu/util/thread_pool.rb

@@ -1,38 +0,0 @@
-module Kudzu
-  class Util
-    class ThreadPool
-      def initialize(size)
-        @size = size
-        @queue = Queue.new
-        @threads = []
-      end
-
-      def start(&block)
-        @threads = 1.upto(@size).map { create_thread(&block) }
-      end
-
-      def wait
-        until @queue.num_waiting == @threads.select { |t| t.alive? }.size
-          Thread.pass
-          sleep 1
-        end
-      end
-
-      def shutdown
-        @threads.each { |t| t.kill }
-        @threads = []
-      end
-
-      private
-
-      def create_thread(&block)
-        Thread.start do
-          loop do
-            ret = block.call(@queue)
-            break if ret == :end
-          end
-        end
-      end
-    end
-  end
-end