kudzu 1.0.0 → 1.1.0

data/lib/kudzu/crawler.rb

@@ -1,57 +1,39 @@
-require 'addressable'
-require 'nokogiri'
+require_relative 'model/all'
+require_relative 'adapter/memory'
+require_relative 'agent'
+require_relative 'callback'
 require_relative 'common'
 require_relative 'config'
-require_relative 'callback'
-require_relative 'logger'
-require_relative 'adapter/memory'
-require_relative 'util/all'
-require_relative 'agent/all'
-require_relative 'revisit/all'
+require_relative 'thread_pool'
 
 module Kudzu
   class Crawler
     attr_reader :uuid, :config
-    attr_reader :frontier, :repository
+    attr_reader :frontier, :repository, :agent
 
     def initialize(options = {}, &block)
       @uuid = options[:uuid] || SecureRandom.uuid
       @config = Kudzu::Config.new(options, &block)
-    end
-
-    def prepare(&block)
-      @logger = Kudzu::Logger.new(@config.log_file, @config.log_level)
-      @callback = Kudzu::Callback.new(&block)
 
       @frontier = Kudzu.adapter::Frontier.new(@uuid)
       @repository = Kudzu.adapter::Repository.new
-
-      @robots = Kudzu::Agent::Robots.new(@config)
-      @page_fetcher = Kudzu::Agent::Fetcher.new(@config, @robots)
-      @page_filter = Kudzu::Agent::Filter.new(@config)
-      @charset_detector = Kudzu::Agent::CharsetDetector.new
-      @mime_type_detector = Kudzu::Agent::MimeTypeDetector.new
-      @title_parser = Kudzu::Agent::TitleParser.new
-
-      @url_extractor = Kudzu::Agent::UrlExtractor.new(@config)
-      @url_filter = Kudzu::Agent::UrlFilter.new(@config)
-
-      @revisit_scheduler = Kudzu::Revisit::Scheduler.new(@config)
+      @agent = Kudzu.agent.new(@config)
     end
 
     def run(seed_url, &block)
-      prepare(&block)
+      @callback = Kudzu::Callback.new(&block)
 
-      seeds = Array(seed_url).map { |url| { url: url } }
-      enqueue_hrefs(seeds, 1)
+      seed_refs = Array(seed_url).map { |url| Kudzu::Agent::Reference.new(url: url) }
+      enqueue_links(refs_to_links(seed_refs, 1))
 
-      if @config.thread_num.to_i <= 1
-        single_thread
-      else
-        multi_thread(@config.thread_num)
+      @agent.start do
+        if @config.thread_num.to_i <= 1
+          single_thread
+        else
+          multi_thread(@config.thread_num)
+        end
       end
 
-      @page_fetcher.pool.close
       @frontier.clear
     end
 
@@ -66,7 +48,7 @@ module Kudzu
     end
 
     def multi_thread(thread_num)
-      @thread_pool = Kudzu::Util::ThreadPool.new(thread_num)
+      @thread_pool = Kudzu::ThreadPool.new(thread_num)
 
       @thread_pool.start do |queue|
         limit_num = [thread_num - queue.size, 0].max
@@ -82,22 +64,25 @@ module Kudzu
     end
 
     def visit_link(link)
-      page = @repository.find_by_url(link.url)
-      response = fetch_link(link, build_request_header(page))
+      response = fetch(link, @config.default_request_header.to_h)
       return unless response
 
-      page = @repository.find_by_url(response.url) if response.redirected?
+      page = @repository.find_by_url(response.url)
       page.url = response.url
       page.status = response.status
-      page.response_time = response.time
+      page.response_time = response.response_time
       page.fetched_at = Time.now
 
-      if page.status_success?
-        handle_success(page, link, response)
-      elsif page.status_not_modified?
-        @revisit_scheduler.schedule(page, modified: false)
-        register_page(page)
-      elsif page.status_not_found? || page.status_gone?
+      if response.fetched?
+        if page.status_success?
+          handle_success(page, link, response)
+        elsif page.status_not_modified?
+          register_page(page)
+        elsif page.status_not_found? || page.status_gone?
+          delete_page(page)
+        end
+      else
+        page.filtered = true
         delete_page(page)
       end
 
@@ -120,113 +105,44 @@ module Kudzu
       end
     end
 
-    def build_request_header(page)
-      header = @config.default_request_header.to_h
-      if @config.revisit_mode
-        header['If-Modified-Since'] = page.last_modified.httpdate if page.last_modified
-        header['If-None-Match'] = page.etag if page.etag
+    def fetch(link, request_header)
+      response = nil
+      @callback.around(:fetch, link, request_header, response) do
+        response = @agent.fetch(link.url, request_header)
+      end
+      if response.fetched?
+        Kudzu.log :info, "fetched page: #{response.status} #{response.url}"
+      else
+        Kudzu.log :info, "skipped page: #{response.status} #{response.url}"
       end
-      header
-    end
-
-    def fetch_link(link, request_header)
-      response = @page_fetcher.fetch(link.url, request_header: request_header)
-      @logger.log :info, "page fetched: #{response.status} #{response.url}"
       response
     rescue Exception => e
-      @logger.log :warn, "couldn't fetch page: #{link.url}", error: e
+      Kudzu.log :warn, "failed to fetch page: #{link.url}", error: e
       @callback.on(:failure, link, e)
       nil
     end
 
     def handle_success(page, link, response)
-      digest = Digest::MD5.hexdigest(response.body)
-      @revisit_scheduler.schedule(page, modified: page.digest != digest)
-
-      page.response_header = response.header
+      page.response_header = response.response_header
       page.body = response.body
-      page.size = response.body.size
-      page.mime_type = detect_mime_type(page)
-      page.charset = detect_charset(page)
-      page.title = parse_title(page)
-      page.redirect_from = link.url if response.redirected?
-      page.revised_at = Time.now if page.digest != digest
-      page.digest = digest
+      page.size = response.size
+      page.mime_type = response.mime_type
+      page.charset = response.charset
+      page.title = response.title
+      page.redirect_from = response.redirect_from
+      page.revised_at = Time.now if page.digest != response.digest
+      page.digest = response.digest
 
-      if follow_hrefs_from?(page, link)
-        hrefs = extract_hrefs(page, page.url)
-        enqueue_hrefs(hrefs, link.depth + 1) unless hrefs.empty?
+      if @config.max_depth.nil? || link.depth < @config.max_depth.to_i
+        refs = @agent.extract_refs(response)
+        enqueue_links(refs_to_links(refs, link.depth + 1)) unless refs.empty?
       end
 
-      if allowed_page?(page)
-        register_page(page)
-      else
+      if @agent.filter_response?(response)
         page.filtered = true
         delete_page(page)
-      end
-    end
-
-    def detect_mime_type(page)
-      @mime_type_detector.detect(page)
-    rescue => e
-      @logger.log :warn, "couldn't detect mime type for #{page.url}", error: e
-      nil
-    end
-
-    def detect_charset(page)
-      if page.text?
-        @charset_detector.detect(page)
-      else
-        nil
-      end
-    rescue => e
-      @logger.log :warn, "couldn't detect charset for #{page.url}", error: e
-      nil
-    end
-
-    def parse_title(page)
-      if page.html?
-        @title_parser.parse(page)
       else
-        Addressable::URI.parse(page.url).basename
-      end
-    rescue => e
-      @logger.log :warn, "couldn't parse title for #{page.url}", error: e
-      nil
-    end
-
-    def follow_hrefs_from?(page, link)
-      (page.html? || page.xml?) && (@config.max_depth.nil? || link.depth < @config.max_depth.to_i)
-    end
-
-    def extract_hrefs(page, base_url)
-      hrefs = @url_extractor.extract(page, base_url)
-      passed, dropped = @url_filter.filter(hrefs, base_url)
-
-      if @config.respect_robots_txt
-        passed, dropped_by_robots = passed.partition { |href| @robots.allowed?(href[:url]) }
-        dropped += dropped_by_robots
-      end
-
-      if @config.log_level == :debug
-        passed.each { |href| @logger.log :debug, "url passed: #{href[:url]}" }
-        dropped.each { |href| @logger.log :debug, "url dropped: #{href[:url]}" }
-      end
-
-      passed
-    rescue => e
-      @logger.log :warn, "couldn't extract links from #{page.url}", error: e
-      []
-    end
-
-    def allowed_page?(page)
-      if @page_filter.allowed?(page) &&
-         (!page.redirect_from || @url_filter.allowed?(page.url, page.redirect_from)) 
-        @logger.log :info, "page passed: #{page.url}"
-        true
-      else
-        @logger.log :info, "page dropped: #{page.url}"
-        false
+        register_page(page)
       end
     end
 
@@ -242,16 +158,19 @@ module Kudzu
       end
     end
 
-    def enqueue_hrefs(hrefs, depth)
-      links = hrefs.map do |href|
-                Kudzu.adapter::Link.new(uuid: @uuid,
-                                       url: href[:url],
-                                       title: href[:title],
-                                       state: 0,
-                                       depth: depth)
-              end
+    def refs_to_links(refs, depth)
+      refs.map do |ref|
+        Kudzu.adapter::Link.new(uuid: @uuid,
+                                url: ref.url,
+                                title: ref.title,
+                                state: 0,
+                                depth: depth)
+      end
+    end
+
+    def enqueue_links(links)
       @callback.around(:enqueue, links) do
-        @frontier.enqueue(links, depth: depth)
+        @frontier.enqueue(links)
       end
     end
   end

data/lib/kudzu/model/base.rb

@@ -0,0 +1,9 @@
+module Kudzu
+  module Model
+    class Base
+      def initialize(attr = {})
+        attr.each { |k, v| public_send("#{k}=", v) if respond_to?("#{k}=") }
+      end
+    end
+  end
+end

data/lib/kudzu/model/link.rb

@@ -0,0 +1,9 @@
+module Kudzu
+  module Model
+    module Link
+      def uri
+        Addressable::URI.parse(url)
+      end
+    end
+  end
+end

data/lib/kudzu/model/page.rb

@@ -0,0 +1,112 @@
+module Kudzu
+  module Model
+    module Page
+      def last_modified
+        last_modified = response_header['last-modified']
+        Time.parse(last_modified).localtime if last_modified
+      rescue
+        nil
+      end
+
+      def etag
+        response_header['etag']
+      end
+
+      def html?
+        !mime_type.to_s.match(%r{text/html|application/xhtml\+xml}).nil?
+      end
+
+      def xml?
+        !mime_type.to_s.match(%r{text/xml|application/xml|application/rss\+xml|application/atom\+xml}).nil?
+      end
+
+      def css?
+        !mime_type.to_s.match(%r{text/css}).nil?
+      end
+
+      def js?
+        !mime_type.to_s.match(%r{text/javascript|application/javascript|application/x-javascript}).nil?
+      end
+
+      def text?
+        html? || xml? || !mime_type.to_s.match(%r{text/}).nil?
+      end
+
+      def status_success?
+        200 <= status && status <= 299
+      end
+
+      def status_redirection?
+        300 <= status && status <= 399
+      end
+
+      def status_client_error?
+        400 <= status && status <= 499
+      end
+
+      def status_server_error?
+        500 <= status && status <= 599
+      end
+
+      def status_not_modified?
+        status == 304
+      end
+
+      def status_not_found?
+        status == 404
+      end
+
+      def status_gone?
+        status == 410
+      end
+
+      def body
+        @body
+      end
+
+      def body=(body)
+        @body = body
+      end
+
+      def filtered
+        @filtered
+      end
+
+      def filtered=(filtered)
+        @filtered = filtered
+      end
+
+      def decoded_body
+        @decoded_body ||= decode_body(body)
+      end
+
+      def parsed_doc
+        @parsed_doc ||= if html?
+                          Nokogiri::HTML(decoded_body)
+                        elsif xml?
+                          Nokogiri::XML(decoded_body)
+                        end
+      end
+
+      private
+
+      def decode_body(body)
+        if text?
+          if find_encoding
+            body.force_encoding(charset).encode('utf-8', invalid: :replace, undef: :replace)
+          else
+            body.encode('utf-8', invalid: :replace, undef: :replace)
+          end
+        else
+          body
+        end
+      end
+
+      def find_encoding
+        Encoding.find(charset)
+      rescue
+        nil
+      end
+    end
+  end
+end

data/lib/kudzu/thread_pool.rb

@@ -0,0 +1,36 @@
+module Kudzu
+  class ThreadPool
+    def initialize(size)
+      @size = size
+      @queue = Queue.new
+      @threads = []
+    end
+
+    def start(&block)
+      @threads = 1.upto(@size).map { create_thread(&block) }
+    end
+
+    def wait
+      until @queue.num_waiting == @threads.select { |t| t.alive? }.size
+        Thread.pass
+        sleep 1
+      end
+    end
+
+    def shutdown
+      @threads.each { |t| t.kill }
+      @threads = []
+    end
+
+    private
+
+    def create_thread(&block)
+      Thread.start do
+        loop do
+          ret = block.call(@queue)
+          break if ret == :end
+        end
+      end
+    end
+  end
+end

data/lib/kudzu/version.rb

@@ -1,3 +1,3 @@
 module Kudzu
-  VERSION = '1.0.0'
+  VERSION = '1.1.0'
 end

data/lib/kudzu.rb

@@ -1,8 +1,26 @@
+require 'net/http'
+require 'http-cookie'
+require 'addressable'
+require 'nokogiri'
+require 'shared-mime-info'
+require 'charlock_holmes'
+
+require 'kudzu/version'
+require 'kudzu/crawler'
+
 module Kudzu
   class << self
-    attr_accessor :adapter
+    attr_accessor :adapter, :agent, :logger
+
+    def log(level, message, error: nil)
+      return unless @logger
+      if error
+        message += " - #{error.class}: #{error.message} at #{error.backtrace.take(5).join("\n")}"
+      end
+      @logger.send(level, message)
+    end
   end
 end
 
-require 'kudzu/version'
-require 'kudzu/crawler'
+Kudzu.adapter = Kudzu::Adapter::Memory
+Kudzu.agent = Kudzu::Agent

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kudzu
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Yoshikazu Kaneta
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-12-20 00:00:00.000000000 Z
+date: 2018-01-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -174,38 +174,40 @@ files:
 - README.md
 - Rakefile
 - lib/kudzu.rb
-- lib/kudzu/adapter/base/all.rb
-- lib/kudzu/adapter/base/link.rb
-- lib/kudzu/adapter/base/page.rb
 - lib/kudzu/adapter/memory.rb
-- lib/kudzu/adapter/memory/all.rb
 - lib/kudzu/adapter/memory/frontier.rb
 - lib/kudzu/adapter/memory/model/link.rb
 - lib/kudzu/adapter/memory/model/page.rb
 - lib/kudzu/adapter/memory/repository.rb
+- lib/kudzu/agent.rb
 - lib/kudzu/agent/all.rb
-- lib/kudzu/agent/charset_detector.rb
 - lib/kudzu/agent/fetcher.rb
-- lib/kudzu/agent/filter.rb
-- lib/kudzu/agent/mime_type_detector.rb
+- lib/kudzu/agent/http/connection.rb
+- lib/kudzu/agent/http/connection_pool.rb
+- lib/kudzu/agent/page_filterer.rb
+- lib/kudzu/agent/reference.rb
+- lib/kudzu/agent/response.rb
 - lib/kudzu/agent/robots.rb
+- lib/kudzu/agent/robots/parser.rb
+- lib/kudzu/agent/robots/txt.rb
 - lib/kudzu/agent/sleeper.rb
-- lib/kudzu/agent/title_parser.rb
 - lib/kudzu/agent/url_extractor.rb
-- lib/kudzu/agent/url_filter.rb
+- lib/kudzu/agent/url_filterer.rb
+- lib/kudzu/agent/util/charset_detector.rb
+- lib/kudzu/agent/util/content_type_parser.rb
+- lib/kudzu/agent/util/matcher.rb
+- lib/kudzu/agent/util/mime_type_detector.rb
+- lib/kudzu/agent/util/title_parser.rb
 - lib/kudzu/callback.rb
 - lib/kudzu/common.rb
 - lib/kudzu/config.rb
 - lib/kudzu/config/filter.rb
 - lib/kudzu/crawler.rb
-- lib/kudzu/logger.rb
-- lib/kudzu/revisit/all.rb
-- lib/kudzu/revisit/scheduler.rb
-- lib/kudzu/util/all.rb
-- lib/kudzu/util/connection_pool.rb
-- lib/kudzu/util/content_type_parser.rb
-- lib/kudzu/util/matcher.rb
-- lib/kudzu/util/thread_pool.rb
+- lib/kudzu/model/all.rb
+- lib/kudzu/model/base.rb
+- lib/kudzu/model/link.rb
+- lib/kudzu/model/page.rb
+- lib/kudzu/thread_pool.rb
 - lib/kudzu/version.rb
 homepage: https://github.com/kanety/kudzu
 licenses:

data/lib/kudzu/adapter/base/link.rb

@@ -1,8 +0,0 @@
-module Kudzu
-  module Adapter
-    module Base
-      module Link
-      end
-    end
-  end
-end

data/lib/kudzu/adapter/base/page.rb

@@ -1,106 +0,0 @@
-module Kudzu
-  module Adapter
-    module Base
-      module Page
-        def last_modified
-          last_modified = response_header['last-modified']
-          Time.parse(last_modified).localtime if last_modified
-        rescue
-          nil
-        end
-
-        def etag
-          response_header['etag']
-        end
-
-        def html?
-          !mime_type.to_s.match(%r{text/html|application/xhtml\+xml}).nil?
-        end
-
-        def xml?
-          !mime_type.to_s.match(%r{text/xml|application/xml|application/rss\+xml|application/atom\+xml}).nil?
-        end
-
-        def css?
-          !mime_type.to_s.match(%r{text/css}).nil?
-        end
-
-        def js?
-          !mime_type.to_s.match(%r{text/javascript|application/javascript|application/x-javascript}).nil?
-        end
-
-        def text?
-          html? || xml? || !mime_type.to_s.match(%r{text/}).nil?
-        end
-
-        def status_success?
-          200 <= status && status <= 299
-        end
-
-        def status_redirection?
-          300 <= status && status <= 399
-        end
-
-        def status_client_error?
-          400 <= status && status <= 499
-        end
-
-        def status_server_error?
-          500 <= status && status <= 599
-        end
-
-        def status_not_modified?
-          status == 304
-        end
-
-        def status_not_found?
-          status == 404
-        end
-
-        def status_gone?
-          status == 410
-        end
-
-        def body
-          @body
-        end
-
-        def body=(body)
-          @body = body
-        end
-
-        def filtered
-          @filtered
-        end
-
-        def filtered=(filtered)
-          @filtered = filtered
-        end
-
-        def decoded_body
-          @decoded_body ||= decode_body(body)
-        end
-
-        private
-
-        def decode_body(body)
-          if text?
-            if find_encoding
-              body.force_encoding(charset).encode('utf-8', invalid: :replace, undef: :replace)
-            else
-              body.encode('utf-8', invalid: :replace, undef: :replace)
-            end
-          else
-            body
-          end
-        end
-
-        def find_encoding
-          Encoding.find(charset)
-        rescue
-          nil
-        end
-      end
-    end
-  end
-end

data/lib/kudzu/adapter/memory/all.rb

@@ -1,3 +0,0 @@
-Dir[File.join(__dir__, '**/*.rb')].each do |file|
-  require_relative file
-end