kudzu 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ab6c69046e9faa3788ead18864ee6d13ddbe2980
+  data.tar.gz: c9868fabe9542d877d6519e0f5297419c882a8e5
+SHA512:
+  metadata.gz: 7018e08e6744a9e74e601bad26a88df3d60140ddaa055fc194c88263cff37137402de967203999d9ba9c2bda199228215f380e207a4b12b6c2e50c5774827e16
+  data.tar.gz: b03edd059ea5b5cb0f50fd0bc660c02e8727c1bb7f8f735fe86ba6f08b2a8bf743a04570eb171dc99f3ffcacba26c1df8a99cc2a1187e2e2fb481c01adb59e6c

data/README.md

@@ -0,0 +1,61 @@
+# Kudzu
+
+A simple web crawler for ruby.
+
+## Features
+
+* Run single-thread or multi-thread.
+* Pool HTTP connection.
+* Restrict links by url-based patterns.
+* Respect robots.txt.
+* Store page contents via adapter.
+
+## Dependencies
+
+* ruby 2.3+
+* libicu
+
+## Installation
+
+Add to your application's Gemfile:
+
+```ruby
+gem 'kudzu'
+```
+
+Then run:
+
+    $ bundle install
+
+## Usage
+
+Crawl html files in `example.com`:
+
+```ruby
+crawler = Kudzu::Crawler.new do
+  user_agent 'YOUR_AWESOME_APP'
+  add_filter do
+    focus_host true
+    allow_mime_type %w(text/html)
+  end
+end
+crawler.run('http://example.com/') do
+  on_success do |page, link|
+    puts page.url
+  end
+end
+```
+
+## Adapters
+
+This gem supports only in-memory crawling by default. Use following adapter to save page contents persistently:
+
+* [kudzu-adapter-active_record](https://github.com/kanety/kudzu-adapter-active_record)
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/kanety/kudzu. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/lib/kudzu.rb

@@ -0,0 +1,8 @@
+module Kudzu
+  class << self
+    attr_accessor :adapter
+  end
+end
+
+require 'kudzu/version'
+require 'kudzu/crawler'

data/lib/kudzu/adapter/base/all.rb

@@ -0,0 +1,3 @@
+Dir[File.join(__dir__, '*.rb')].each do |file|
+  require_relative file
+end

data/lib/kudzu/adapter/base/link.rb

@@ -0,0 +1,8 @@
+module Kudzu
+  module Adapter
+    module Base
+      module Link
+      end
+    end
+  end
+end

data/lib/kudzu/adapter/base/page.rb

@@ -0,0 +1,106 @@
+module Kudzu
+  module Adapter
+    module Base
+      module Page
+        def last_modified
+          last_modified = response_header['last-modified']
+          Time.parse(last_modified).localtime if last_modified
+        rescue
+          nil
+        end
+
+        def etag
+          response_header['etag']
+        end
+
+        def html?
+          !mime_type.to_s.match(%r{text/html|application/xhtml\+xml}).nil?
+        end
+
+        def xml?
+          !mime_type.to_s.match(%r{text/xml|application/xml|application/rss\+xml|application/atom\+xml}).nil?
+        end
+
+        def css?
+          !mime_type.to_s.match(%r{text/css}).nil?
+        end
+
+        def js?
+          !mime_type.to_s.match(%r{text/javascript|application/javascript|application/x-javascript}).nil?
+        end
+
+        def text?
+          html? || xml? || !mime_type.to_s.match(%r{text/}).nil?
+        end
+
+        def status_success?
+          200 <= status && status <= 299
+        end
+
+        def status_redirection?
+          300 <= status && status <= 399
+        end
+
+        def status_client_error?
+          400 <= status && status <= 499
+        end
+
+        def status_server_error?
+          500 <= status && status <= 599
+        end
+
+        def status_not_modified?
+          status == 304
+        end
+
+        def status_not_found?
+          status == 404
+        end
+
+        def status_gone?
+          status == 410
+        end
+
+        def body
+          @body
+        end
+
+        def body=(body)
+          @body = body
+        end
+
+        def filtered
+          @filtered
+        end
+
+        def filtered=(filtered)
+          @filtered = filtered
+        end
+
+        def decoded_body
+          @decoded_body ||= decode_body(body)
+        end
+
+        private
+
+        def decode_body(body)
+          if text?
+            if find_encoding
+              body.force_encoding(charset).encode('utf-8', invalid: :replace, undef: :replace)
+            else
+              body.encode('utf-8', invalid: :replace, undef: :replace)
+            end
+          else
+            body
+          end
+        end
+
+        def find_encoding
+          Encoding.find(charset)
+        rescue
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/kudzu/adapter/memory.rb

@@ -0,0 +1,4 @@
+require_relative 'base/all'
+require_relative 'memory/all'
+
+Kudzu.adapter = Kudzu::Adapter::Memory

data/lib/kudzu/adapter/memory/all.rb

@@ -0,0 +1,3 @@
+Dir[File.join(__dir__, '**/*.rb')].each do |file|
+  require_relative file
+end

data/lib/kudzu/adapter/memory/frontier.rb

@@ -0,0 +1,38 @@
+module Kudzu
+  module Adapter
+    module Memory
+      class Frontier
+        def initialize(uuid)
+          @uuid = uuid
+          @monitor = Monitor.new
+          @queue = []
+          @queued = {}
+        end
+
+        def enqueue(links, depth: 1)
+          @monitor.synchronize do
+            Array(links).each do |link|
+              next if @queued.key?(link.url)
+              @queued[link.url] = true
+              @queue << link
+            end
+          end
+        end
+
+        def dequeue(limit: 1)
+          @monitor.synchronize do
+            links = @queue.shift(limit)
+            links.each do |link|
+              link.state = 1
+            end
+          end
+        end
+
+        def clear
+          @queue.clear
+          @queued.clear
+        end
+      end
+    end
+  end
+end

data/lib/kudzu/adapter/memory/model/link.rb

@@ -0,0 +1,15 @@
+module Kudzu
+  module Adapter
+    module Memory
+      class Link
+        include Kudzu::Adapter::Base::Link
+
+        attr_accessor :uuid, :url, :title, :state, :depth
+
+        def initialize(attr = {})
+          attr.each { |k, v| public_send("#{k}=", v) if respond_to?("#{k}=") }
+        end
+      end
+    end
+  end
+end

data/lib/kudzu/adapter/memory/model/page.rb

@@ -0,0 +1,17 @@
+module Kudzu
+  module Adapter
+    module Memory
+      class Page
+        include Kudzu::Adapter::Base::Page
+
+        attr_accessor :url, :title, :status, :mime_type, :size, :charset, :digest,
+                      :response_header, :response_time, :redirect_from, :fetched_at, :revised_at,
+                      :revisit_interval, :revisit_at
+
+        def initialize(attr = {})
+          attr.each { |k, v| public_send("#{k}=", v) if respond_to?("#{k}=") }
+        end
+      end
+    end
+  end
+end

data/lib/kudzu/adapter/memory/repository.rb

@@ -0,0 +1,27 @@
+module Kudzu
+  module Adapter
+    module Memory
+      class Repository
+        attr_reader :page
+
+        def initialize
+          @page = {}
+          @digest = {}
+        end
+
+        def find_by_url(url)
+          @page[url] || Page.new(url: url)
+        end
+
+        def register(page)
+          @page[page.url] = page
+          @digest[page.digest] = true
+        end
+
+        def delete(page)
+          @page.delete(page.url)
+        end
+      end
+    end
+  end
+end

data/lib/kudzu/agent/all.rb

@@ -0,0 +1,3 @@
+Dir[File.join(__dir__, '*.rb')].each do |file|
+  require_relative file
+end

data/lib/kudzu/agent/charset_detector.rb

@@ -0,0 +1,84 @@
+require 'nokogiri'
+require 'charlock_holmes'
+
+module Kudzu
+  class Agent
+    class CharsetDetector
+      CORRECTION = {
+        'utf_8' => 'utf-8',
+        'shift-jis' => 'shift_jis',
+        'x-sjis' => 'shift_jis',
+        'euc_jp' => 'euc-jp'
+      }
+
+      def initialize
+        @parser = Kudzu::Util::ContentTypeParser.new
+      end
+
+      def detect(page)
+        if page.html?
+          from_html(page.body) || from_text(page.body)
+        elsif page.xml?
+          from_xml(page.body) || from_text(page.body)
+        elsif page.text?
+          from_text(page.body)
+        end
+      end
+
+      private
+
+      def from_html(body)
+        doc = Nokogiri::HTML(body.encode('ascii', undef: :replace, invalid: :replace))
+
+        if (node = doc.xpath('//meta/@charset').first)
+          charset = correct(node.to_s)
+          return charset if charset
+        end
+
+        doc.xpath('//meta[@http-equiv]').each do |meta|
+          if meta['http-equiv'] =~ /content-type/i
+            charset = @parser.parse(meta[:content].to_s)[1][:charset]
+            charset = correct(node.to_s)
+            return charset if charset
+          end
+        end
+
+        return nil
+      end
+
+      def from_xml(body)
+        doc = Nokogiri::XML(body.encode('ascii', undef: :replace, invalid: :replace))
+        if doc.encoding
+          correct(doc.encoding)
+        else
+          nil
+        end
+      end
+
+      def from_text(text)
+        if text.ascii_only?
+          'ascii'
+        else
+          detection = CharlockHolmes::EncodingDetector.detect(text)
+          if detection && detection.key?(:encoding)
+            detection[:encoding].downcase
+          else
+            nil
+          end
+        end
+      end
+
+      def correct(charset)
+        charset = charset.downcase
+        charset = CORRECTION[charset] if CORRECTION.key?(charset)
+
+        begin
+          Encoding.find(charset)
+        rescue
+          charset = nil
+        end
+        charset
+      end
+    end
+  end
+end

data/lib/kudzu/agent/fetcher.rb

@@ -0,0 +1,116 @@
+require 'net/http'
+require 'http-cookie'
+
+module Kudzu
+  class Agent
+    class Fetcher
+      class Response
+        attr_accessor :url, :status, :header, :body, :time, :redirected
+  
+        def initialize(attr = {})
+          attr.each { |k, v| public_send("#{k}=", v) }
+        end
+
+        def redirected?
+          redirected
+        end
+      end
+
+      attr_reader :pool
+
+      def initialize(config, robots = nil)
+        @config = config
+        @pool = Kudzu::Util::ConnectionPool.new(@config.max_connection || 100)
+        @sleeper = Kudzu::Agent::Sleeper.new(@config, robots)
+        @jar = HTTP::CookieJar.new
+      end
+
+      def fetch(url, request_header: {}, redirect: max_redirect, method: :get)
+        uri = Addressable::URI.parse(url)
+        http = @pool.checkout(pool_name(uri)) { build_http(uri) }
+        request = build_request(uri, request_header: request_header, method: method)
+
+        append_cookie(url, request) if @config.handle_cookie
+
+        @sleeper.politeness_delay(url)
+
+        response = nil
+        response_time = Benchmark.realtime { response = http.request(request) }
+
+        parse_cookie(url, response) if @config.handle_cookie
+
+        if redirection?(response.code) && response['location'] && redirect > 0
+          fetch(uri.join(response['location']).to_s, request_header: request_header, redirect: redirect - 1)
+        else
+          res = build_response(url, response, response_time)
+          res.redirected = (redirect != max_redirect)
+          res
+        end
+      end
+
+      private
+
+      def max_redirect
+        @config.max_redirect || 5
+      end
+
+      def pool_name(uri)
+        "#{uri.scheme}_#{uri.host}_#{uri.port || uri.default_port}"
+      end
+
+      def build_http(uri)
+        http = Net::HTTP.new(uri.host, uri.port || uri.default_port)
+        http.open_timeout = @config.open_timeout if @config.open_timeout
+        http.read_timeout = @config.read_timeout if @config.read_timeout
+        if uri.scheme == 'https'
+          http.use_ssl = true
+          http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+        end
+        http.start
+      end
+
+      def build_request(uri, request_header:, method:)
+        request = request_klass_for(method).new(uri.request_uri)
+        request.basic_auth uri.user, uri.password if uri.user && uri.password
+
+        request['User-Agent'] = @config.user_agent
+        request_header.each do |key, value|
+          request[key] = value
+        end
+        request
+      end
+
+      def request_klass_for(method)
+        Object.const_get("Net::HTTP::#{method.capitalize}")
+      end
+
+      def build_response(url, response, response_time)
+        Response.new(url: url,
+                     status: response.code.to_i,
+                     header: Hash[response.each.to_a],
+                     body: response.body.to_s,
+                     time: response_time)
+      end
+
+      def redirection?(code)
+        code = code.to_i
+        300 <= code && code <= 399
+      end
+
+      def parse_cookie(url, response)
+        @jar.parse(response['set-cookie'], url) if response['set-cookie']
+      end
+
+      def append_cookie(url, request)
+        cookies = @jar.cookies(url)
+        unless cookies.empty?
+          if request['Cookie']
+            request['Cookie'] += '; ' + cookies.join('; ')
+          else
+            request['Cookie'] = cookies.join('; ')
+          end
+        end
+      end
+    end
+  end
+end