kudzu 1.0.0

data/lib/kudzu/logger.rb

@@ -0,0 +1,20 @@
+module Kudzu
+  class Logger
+    def initialize(file, level)
+      if file.is_a?(::Logger)
+        @logger = file
+      elsif file
+        @logger = ::Logger.new(file)
+        @logger.level = level
+      else
+        @logger = nil
+      end
+    end
+
+    def log(level, message, error: nil)
+      return unless @logger
+      message += " #{error.class} #{error.message} #{error.backtrace.join("\n")}" if error
+      @logger.send(level, message)
+    end
+  end
+end

data/lib/kudzu/revisit/all.rb

@@ -0,0 +1,3 @@
+Dir[File.join(__dir__, '*.rb')].each do |file|
+  require_relative file
+end

data/lib/kudzu/revisit/scheduler.rb

@@ -0,0 +1,28 @@
+module Kudzu
+  class Revisit
+    class Scheduler
+      def initialize(config)
+        @config = config
+      end
+
+      def schedule(page, modified: true)
+        page.revisit_interval = next_interval(page.revisit_interval, modified)
+        page.revisit_at = page.fetched_at + page.revisit_interval * 86400
+      end
+
+      private
+
+      def next_interval(curr_interval, modified)
+        if curr_interval
+          if modified
+            [curr_interval - 1, @config.revisit_min_interval].max
+          else
+            [curr_interval + 1, @config.revisit_max_interval].min
+          end
+        else
+          @config.revisit_default_interval
+        end
+      end
+    end
+  end
+end

data/lib/kudzu/util/all.rb

@@ -0,0 +1,3 @@
+Dir[File.join(__dir__, '*.rb')].each do |file|
+  require_relative file
+end

data/lib/kudzu/util/connection_pool.rb

@@ -0,0 +1,56 @@
+module Kudzu
+  class Util
+    class ConnectionPool
+      class Connection
+        attr_accessor :name, :http, :last_used
+
+        def initialize(attr = {})
+          attr.each { |k, v| public_send("#{k}=", v) }
+        end
+      end
+
+      def initialize(max_size = 10)
+        @max_size = max_size
+      end
+
+      def checkout(name)
+        pool[name] ||= Connection.new(name: name, http: yield)
+
+        conn = pool[name]
+        conn.last_used = Time.now
+
+        if pool.size > @max_size
+          reduce
+        end
+
+        conn.http
+      end
+
+      def close
+        pool.values.each do |conn|
+          finish_http(conn.http)
+        end
+        Thread.current[:kudzu_connection] = nil
+      end
+
+      private
+
+      def pool
+        Thread.current[:kudzu_connection] ||= {}
+        Thread.current[:kudzu_connection]
+      end
+
+      def reduce
+        conns = pool.values.sort_by { |conn| conn.last_used }
+        conns.first(pool.size - @max_size).each do |conn|
+          finish_http(conn.http)
+          pool.delete(conn.name)
+        end
+      end
+
+      def finish_http(http)
+        http.finish if http && http.started?
+      end
+    end
+  end
+end

data/lib/kudzu/util/content_type_parser.rb

@@ -0,0 +1,24 @@
+module Kudzu
+  class Util
+    class ContentTypeParser
+      def parse(content_type)
+        mime, *kvs = content_type.to_s.split(';').map { |str| str.strip.downcase }
+        params = kvs.each_with_object({}) do |kv, hash|
+                   k, v = kv.to_s.split('=').map { |str| str.strip }
+                   hash[k.to_sym] = unquote(v) if k && v
+                 end
+        return mime, params
+      end
+
+      private
+
+      def unquote(str)
+        if str =~ /^"(.*?)"$/
+          $1.gsub(/\\(.)/, '\1')
+        else
+          str
+        end
+      end
+    end
+  end
+end

data/lib/kudzu/util/matcher.rb

@@ -0,0 +1,21 @@
+module Kudzu
+  class Util
+    class Matcher
+      def match?(text, allows: nil, denies: nil)
+        match_to_allows?(text, allows) && !match_to_denies?(text, denies)
+      end
+
+      private
+
+      def match_to_allows?(text, allows)
+        allows = Array(allows)
+        allows.empty? || allows.any? { |allow| Kudzu::Common.match?(text, allow) }
+      end
+
+      def match_to_denies?(text, denies)
+        denies = Array(denies)
+        !denies.empty? && denies.any? { |deny| Kudzu::Common.match?(text, deny) }
+      end
+    end
+  end
+end

data/lib/kudzu/util/thread_pool.rb

@@ -0,0 +1,38 @@
+module Kudzu
+  class Util
+    class ThreadPool
+      def initialize(size)
+        @size = size
+        @queue = Queue.new
+        @threads = []
+      end
+
+      def start(&block)
+        @threads = 1.upto(@size).map { create_thread(&block) }
+      end
+
+      def wait
+        until @queue.num_waiting == @threads.select { |t| t.alive? }.size
+          Thread.pass
+          sleep 1
+        end
+      end
+
+      def shutdown
+        @threads.each { |t| t.kill }
+        @threads = []
+      end
+
+      private
+
+      def create_thread(&block)
+        Thread.start do
+          loop do
+            ret = block.call(@queue)
+            break if ret == :end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/kudzu/version.rb

@@ -0,0 +1,3 @@
+module Kudzu
+  VERSION = '1.0.0'
+end

metadata

@@ -0,0 +1,234 @@
+--- !ruby/object:Gem::Specification
+name: kudzu
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Yoshikazu Kaneta
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-12-20 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: charlock_holmes
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: shared-mime-info
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mime-types
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: http-cookie
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A simple web crawler for ruby
+email:
+- kaneta@sitebridge.co.jp
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- Rakefile
+- lib/kudzu.rb
+- lib/kudzu/adapter/base/all.rb
+- lib/kudzu/adapter/base/link.rb
+- lib/kudzu/adapter/base/page.rb
+- lib/kudzu/adapter/memory.rb
+- lib/kudzu/adapter/memory/all.rb
+- lib/kudzu/adapter/memory/frontier.rb
+- lib/kudzu/adapter/memory/model/link.rb
+- lib/kudzu/adapter/memory/model/page.rb
+- lib/kudzu/adapter/memory/repository.rb
+- lib/kudzu/agent/all.rb
+- lib/kudzu/agent/charset_detector.rb
+- lib/kudzu/agent/fetcher.rb
+- lib/kudzu/agent/filter.rb
+- lib/kudzu/agent/mime_type_detector.rb
+- lib/kudzu/agent/robots.rb
+- lib/kudzu/agent/sleeper.rb
+- lib/kudzu/agent/title_parser.rb
+- lib/kudzu/agent/url_extractor.rb
+- lib/kudzu/agent/url_filter.rb
+- lib/kudzu/callback.rb
+- lib/kudzu/common.rb
+- lib/kudzu/config.rb
+- lib/kudzu/config/filter.rb
+- lib/kudzu/crawler.rb
+- lib/kudzu/logger.rb
+- lib/kudzu/revisit/all.rb
+- lib/kudzu/revisit/scheduler.rb
+- lib/kudzu/util/all.rb
+- lib/kudzu/util/connection_pool.rb
+- lib/kudzu/util/content_type_parser.rb
+- lib/kudzu/util/matcher.rb
+- lib/kudzu/util/thread_pool.rb
+- lib/kudzu/version.rb
+homepage: https://github.com/kanety/kudzu
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2.2
+signing_key: 
+specification_version: 4
+summary: A simple web crawler for ruby
+test_files: []