kudzu 1.0.0

data/lib/kudzu/agent/url_filter.rb

@@ -0,0 +1,65 @@
+module Kudzu
+  class Agent
+    class UrlFilter
+      def initialize(config)
+        @config = config
+        @matcher = Kudzu::Util::Matcher.new
+      end
+
+      def filter(hrefs, base_url)
+        base_uri = Addressable::URI.parse(base_url)
+        filter = @config.find_filter(base_uri)
+
+        hrefs.partition do |href|
+          allowed?(href[:url], base_uri, filter: filter)
+        end
+      end
+
+      def allowed?(url, base_uri, filter: nil)
+        uri = Addressable::URI.parse(url)
+        base_uri = Addressable::URI.parse(base_uri) if base_uri.is_a?(String)
+        filter ||= @config.find_filter(base_uri)
+        return true unless filter
+
+        focused_host?(uri, base_uri, filter) &&
+          focused_descendants?(uri, base_uri, filter) &&
+          allowed_url?(uri, filter) &&
+          allowed_host?(uri, filter) &&
+          allowed_path?(uri, filter) &&
+          allowed_ext?(uri, filter)
+      end
+
+      private
+
+      def focused_host?(uri, base_uri, filter)
+        return true unless filter.focus_host
+        uri.host == base_uri.host
+      end
+
+      def focused_descendants?(uri, base_uri, filter)
+        return true unless filter.focus_descendants
+        dir = Kudzu::Common.path_to_dir(uri.path)
+        base_dir = Kudzu::Common.path_to_dir(base_uri.path)
+        uri.host == base_uri.host && dir =~ /^#{Regexp.escape(base_dir)}/i
+      end
+
+      def allowed_url?(uri, filter)
+        @matcher.match?(uri.to_s, allows: filter.allow_url, denies: filter.deny_url)
+      end
+
+      def allowed_host?(uri, filter)
+        @matcher.match?(uri.host, allows: filter.allow_host, denies: filter.deny_host)
+      end
+
+      def allowed_path?(uri, filter)
+        @matcher.match?(uri.path, allows: filter.allow_path, denies: filter.deny_path)
+      end
+
+      def allowed_ext?(uri, filter)
+        ext = uri.extname.to_s.sub(/^\./, '')
+        return true if ext.empty?
+        @matcher.match?(ext, allows: filter.allow_ext, denies: filter.deny_ext)
+      end
+    end
+  end
+end

data/lib/kudzu/callback.rb

@@ -0,0 +1,41 @@
+module Kudzu
+  class Callback
+    CALLBACKS = [:on_success,      # 2xx
+                 :on_redirection,  # 3xx
+                 :on_client_error, # 4xx
+                 :on_server_error, # 5xx
+                 :on_filter,       # 2xx, filtered
+                 :on_failure,      # Exception
+                 :before_register,
+                 :after_register,
+                 :before_delete,
+                 :after_delete,
+                 :before_enqueue, 
+                 :after_enqueue,
+                 ]
+
+    def initialize(&block)
+      @callback = {}
+      instance_eval(&block) if block
+    end
+
+    CALLBACKS.each do |key|
+      define_method(key) do |&block|
+        @callback[key] = block
+      end
+    end
+
+    def on(name, *args)
+      on_name = "on_#{name}".to_sym
+      @callback[on_name].call(*args) if @callback.key?(on_name)
+    end
+
+    def around(name, *args)
+      before_name = "before_#{name}".to_sym
+      after_name = "after_#{name}".to_sym
+      @callback[before_name].call(*args) if @callback.key?(before_name)
+      yield
+      @callback[after_name].call(*args) if @callback.key?(after_name)
+    end
+  end
+end

data/lib/kudzu/common.rb

@@ -0,0 +1,23 @@
+module Kudzu
+  class Common
+    class << self
+      def match?(text, pattern)
+        if pattern.is_a?(String)
+          File.fnmatch(pattern, text)
+        elsif pattern.is_a?(Regexp)
+          text =~ pattern
+        else
+          false
+        end
+      end
+
+      def path_to_dir(path)
+        if path.end_with?('/')
+          path
+        else
+          File.dirname(path) + '/'
+        end
+      end
+    end
+  end
+end

data/lib/kudzu/config.rb

@@ -0,0 +1,79 @@
+require 'ostruct'
+require_relative 'config/filter'
+
+module Kudzu
+  class Config
+    SIMPLE_CONFIGS   = [:config_file,
+                        :user_agent, :thread_num, :open_timeout, :read_timeout,
+                        :max_connection, :max_redirect, :max_depth, :default_request_header,
+                        :politeness_delay, :handle_cookie,
+                        :respect_robots_txt, :respect_nofollow, :respect_noindex,
+                        :log_file, :log_level,
+                        :revisit_mode, :revisit_min_interval, :revisit_max_interval, :revisit_default_interval,
+                        :filters]
+    DEFAULT_CONFIG   = { user_agent: "Kudzu/#{Kudzu::VERSION}",
+                         open_timeout: 10,
+                         read_timeout: 10,
+                         thread_num: 1,
+                         max_connection: 10,
+                         max_redirect: 3,
+                         politeness_delay: 0.5,
+                         handle_cookie: true,
+                         respect_robots_txt: true,
+                         respect_nofollow: true,
+                         respect_noindex: true,
+                         revisit_mode: false,
+                         revisit_min_interval: 1,
+                         revisit_max_interval: 10,
+                         revisit_default_interval: 5 }
+
+    attr_accessor *SIMPLE_CONFIGS
+
+    class Delegator
+      def initialize(config)
+        @config = config
+      end
+
+      Kudzu::Config::SIMPLE_CONFIGS.each do |key|
+        define_method(key) do |value|
+          @config.send("#{key}=", value)
+        end
+      end
+
+      def add_filter(base_url = nil, config = {}, &block)
+        @config.add_filter(base_url, config, &block)
+      end
+    end
+
+    def initialize(config = {}, &block)
+      self.filters = {}
+      DEFAULT_CONFIG.merge(config).each do |key, value|
+        send("#{key}=", value)
+      end
+      if config_file || block
+        delegator = Kudzu::Config::Delegator.new(self)
+        delegator.instance_eval(File.read(config_file)) if config_file
+        delegator.instance_eval(&block) if block
+      end
+    end
+
+    def add_filter(base_url = nil, config = {}, &block)
+      base_uri = Addressable::URI.parse(base_url || '*')
+      host = base_uri.host.presence || '*'
+      path = base_uri.path.presence || '*'
+      filters[host] ||= []
+      filters[host] << Filter.new(path, config, &block)
+    end
+
+    def find_filter(uri)
+      uri = Addressable::URI.parse(uri) if uri.is_a?(String)
+      filters.each do |host, filters|
+        next unless Kudzu::Common.match?(uri.host, host)
+        filters.each do |filter|
+          return filter if Kudzu::Common.match?(uri.path, filter.path)
+        end
+      end
+      nil
+    end
+  end
+end

data/lib/kudzu/config/filter.rb

@@ -0,0 +1,39 @@
+module Kudzu
+  class Config
+    class Filter
+      SIMPLE_CONFIGS = [# url filter
+                        :focus_host, :focus_descendants, :allow_element, :deny_element,
+                        :allow_url, :deny_url, :allow_host, :deny_host, :allow_path, :deny_path,
+                        :allow_ext, :deny_ext,
+                        # page filter
+                        :allow_mime_type, :deny_mime_type, :max_size]
+      DEFAULT_CONFIG = { focus_host: false,
+                         focus_descendants: false }
+
+      attr_accessor :path
+      attr_accessor *SIMPLE_CONFIGS
+
+      class Delegator
+        def initialize(filter)
+          @filter = filter
+        end
+
+        Kudzu::Config::Filter::SIMPLE_CONFIGS.each do |key|
+          define_method(key) do |value|
+            @filter.send("#{key}=", value)
+          end
+        end
+      end
+
+      def initialize(path, config = {}, &block)
+        @path = path
+        DEFAULT_CONFIG.merge(config).each do |key, value|
+          send("#{key}=", value)
+        end
+        if block
+          Kudzu::Config::Filter::Delegator.new(self).instance_eval(&block)
+        end
+      end
+    end
+  end
+end

data/lib/kudzu/crawler.rb

@@ -0,0 +1,258 @@
+require 'addressable'
+require 'nokogiri'
+require_relative 'common'
+require_relative 'config'
+require_relative 'callback'
+require_relative 'logger'
+require_relative 'adapter/memory'
+require_relative 'util/all'
+require_relative 'agent/all'
+require_relative 'revisit/all'
+
+module Kudzu
+  class Crawler
+    attr_reader :uuid, :config
+    attr_reader :frontier, :repository
+
+    def initialize(options = {}, &block)
+      @uuid = options[:uuid] || SecureRandom.uuid
+      @config = Kudzu::Config.new(options, &block)
+    end
+
+    def prepare(&block)
+      @logger = Kudzu::Logger.new(@config.log_file, @config.log_level)
+      @callback = Kudzu::Callback.new(&block)
+
+      @frontier = Kudzu.adapter::Frontier.new(@uuid)
+      @repository = Kudzu.adapter::Repository.new
+
+      @robots = Kudzu::Agent::Robots.new(@config)
+      @page_fetcher = Kudzu::Agent::Fetcher.new(@config, @robots)
+      @page_filter = Kudzu::Agent::Filter.new(@config)
+      @charset_detector = Kudzu::Agent::CharsetDetector.new
+      @mime_type_detector = Kudzu::Agent::MimeTypeDetector.new
+      @title_parser = Kudzu::Agent::TitleParser.new
+
+      @url_extractor = Kudzu::Agent::UrlExtractor.new(@config)
+      @url_filter = Kudzu::Agent::UrlFilter.new(@config)
+
+      @revisit_scheduler = Kudzu::Revisit::Scheduler.new(@config)
+    end
+
+    def run(seed_url, &block)
+      prepare(&block)
+
+      seeds = Array(seed_url).map { |url| { url: url } }
+      enqueue_hrefs(seeds, 1)
+
+      if @config.thread_num.to_i <= 1
+        single_thread
+      else
+        multi_thread(@config.thread_num)
+      end
+
+      @page_fetcher.pool.close
+      @frontier.clear
+    end
+
+    private
+
+    def single_thread
+      loop do
+        link = @frontier.dequeue.first
+        break unless link
+        visit_link(link)
+      end
+    end
+
+    def multi_thread(thread_num)
+      @thread_pool = Kudzu::Util::ThreadPool.new(thread_num)
+
+      @thread_pool.start do |queue|
+        limit_num = [thread_num - queue.size, 0].max
+        @frontier.dequeue(limit: limit_num).each do |link|
+          queue.push(link)
+        end
+        link = queue.pop
+        visit_link(link)
+      end
+
+      @thread_pool.wait
+      @thread_pool.shutdown
+    end
+
+    def visit_link(link)
+      page = @repository.find_by_url(link.url)
+      response = fetch_link(link, build_request_header(page))
+      return unless response
+
+      page = @repository.find_by_url(response.url) if response.redirected?
+      page.url = response.url
+      page.status = response.status
+      page.response_time = response.time
+      page.fetched_at = Time.now
+
+      if page.status_success?
+        handle_success(page, link, response)
+      elsif page.status_not_modified?
+        @revisit_scheduler.schedule(page, modified: false)
+        register_page(page)
+      elsif page.status_not_found? || page.status_gone?
+        delete_page(page)
+      end
+
+      run_callback(page, link)
+    end
+
+    def run_callback(page, link)
+      if page.status_success?
+        if page.filtered
+          @callback.on(:filter, page, link)
+        else
+          @callback.on(:success, page, link)
+        end
+      elsif page.status_redirection?
+        @callback.on(:redirection, page, link)
+      elsif page.status_client_error?
+        @callback.on(:client_error, page, link)
+      elsif page.status_server_error?
+        @callback.on(:server_error, page, link)
+      end
+    end
+
+    def build_request_header(page)
+      header = @config.default_request_header.to_h
+      if @config.revisit_mode
+        header['If-Modified-Since'] = page.last_modified.httpdate if page.last_modified
+        header['If-None-Match'] = page.etag if page.etag
+      end
+      header
+    end
+
+    def fetch_link(link, request_header)
+      response = @page_fetcher.fetch(link.url, request_header: request_header)
+      @logger.log :info, "page fetched: #{response.status} #{response.url}"
+      response
+    rescue Exception => e
+      @logger.log :warn, "couldn't fetch page: #{link.url}", error: e
+      @callback.on(:failure, link, e)
+      nil
+    end
+
+    def handle_success(page, link, response)
+      digest = Digest::MD5.hexdigest(response.body)
+      @revisit_scheduler.schedule(page, modified: page.digest != digest)
+
+      page.response_header = response.header
+      page.body = response.body
+      page.size = response.body.size
+      page.mime_type = detect_mime_type(page)
+      page.charset = detect_charset(page)
+      page.title = parse_title(page)
+      page.redirect_from = link.url if response.redirected?
+      page.revised_at = Time.now if page.digest != digest
+      page.digest = digest
+
+      if follow_hrefs_from?(page, link)
+        hrefs = extract_hrefs(page, page.url)
+        enqueue_hrefs(hrefs, link.depth + 1) unless hrefs.empty?
+      end
+
+      if allowed_page?(page)
+        register_page(page)
+      else
+        page.filtered = true
+        delete_page(page)
+      end
+    end
+
+    def detect_mime_type(page)
+      @mime_type_detector.detect(page)
+    rescue => e
+      @logger.log :warn, "couldn't detect mime type for #{page.url}", error: e
+      nil
+    end
+
+    def detect_charset(page)
+      if page.text?
+        @charset_detector.detect(page)
+      else
+        nil
+      end
+    rescue => e
+      @logger.log :warn, "couldn't detect charset for #{page.url}", error: e
+      nil
+    end
+
+    def parse_title(page)
+      if page.html?
+        @title_parser.parse(page)
+      else
+        Addressable::URI.parse(page.url).basename
+      end
+    rescue => e
+      @logger.log :warn, "couldn't parse title for #{page.url}", error: e
+      nil
+    end
+
+    def follow_hrefs_from?(page, link)
+      (page.html? || page.xml?) && (@config.max_depth.nil? || link.depth < @config.max_depth.to_i)
+    end
+
+    def extract_hrefs(page, base_url)
+      hrefs = @url_extractor.extract(page, base_url)
+      passed, dropped = @url_filter.filter(hrefs, base_url)
+
+      if @config.respect_robots_txt
+        passed, dropped_by_robots = passed.partition { |href| @robots.allowed?(href[:url]) }
+        dropped += dropped_by_robots
+      end
+
+      if @config.log_level == :debug
+        passed.each { |href| @logger.log :debug, "url passed: #{href[:url]}" }
+        dropped.each { |href| @logger.log :debug, "url dropped: #{href[:url]}" }
+      end
+
+      passed
+    rescue => e
+      @logger.log :warn, "couldn't extract links from #{page.url}", error: e
+      []
+    end
+
+    def allowed_page?(page)
+      if @page_filter.allowed?(page) &&
+         (!page.redirect_from || @url_filter.allowed?(page.url, page.redirect_from)) 
+        @logger.log :info, "page passed: #{page.url}"
+        true
+      else
+        @logger.log :info, "page dropped: #{page.url}"
+        false
+      end
+    end
+
+    def register_page(page)
+      @callback.around(:register, page) do
+        @repository.register(page)
+      end
+    end
+
+    def delete_page(page)
+      @callback.around(:delete, page) do
+        @repository.delete(page)
+      end
+    end
+
+    def enqueue_hrefs(hrefs, depth)
+      links = hrefs.map do |href|
+                Kudzu.adapter::Link.new(uuid: @uuid,
+                                       url: href[:url],
+                                       title: href[:title],
+                                       state: 0,
+                                       depth: depth)
+              end
+      @callback.around(:enqueue, links) do
+        @frontier.enqueue(links, depth: depth)
+      end
+    end
+  end
+end