daimon_skycrawlers 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e9ebc710f18b92107a91a3bb5a4c8972051ede8f
-  data.tar.gz: 61db84bcdc73557ae9b28cc7894c04a1055dda6d
+  metadata.gz: f03168cc7d465dd69da00eabc00c5bd1d4654455
+  data.tar.gz: e55f902ab4f78340ee5df80499f581cefb2bad96
 SHA512:
-  metadata.gz: 76d7b629eefc04a89d5cdce8939a9e83f35c68a9b8d1be3599e3c7d59b815f506eb1c295d1e3095f18ca7ec12169fa5e365cda981561804bbb4d2d070eef3051
-  data.tar.gz: 04b326f7a8531d364c3d41d12f99328831b2eda1acefa72869f588750e065abc0a044fd6d87b92f20a6359983dba8b00c42a4ab40d75979f92c6a553ec16f9e9
+  metadata.gz: 0bd1e1e832766d27779e91bba3ae70c70d44863373cd5172d2c67e40ad6ded2dd1329a68f9652aa58fd00bac7e38fd75e3c43092fbeaec4bca42fada779b2ffd
+  data.tar.gz: 99dc774a495bfec4e8b0b0693333340fc57be9321ef9fb5d9b2d64f61856ab6defa10c1e8292dc13d90bcca4a69e80e5f2a4aeec033c7c1a345e64d2c965eaec

data/.travis.yml

@@ -2,8 +2,8 @@ language: ruby
 sudo: false
 cache: bundler
 rvm:
-  - 2.2.4
-  - 2.3.0
+  - 2.2.5
+  - 2.3.1
 
 before_install: gem install bundler -v 1.11.2
 

data/Gemfile

@@ -1,3 +1,3 @@
-source 'https://rubygems.org'
+source "https://rubygems.org"
 
 gemspec

data/README.md

@@ -1,8 +1,8 @@
-# DaimonSkycrawlers
+## Caution!! This product is NOT production-ready.
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/daimon_skycrawlers`. To experiment with that code, run `bin/console` for an interactive prompt.
+# DaimonSkycrawlers
 
-TODO: Delete this and the text above, and describe your gem
+DaimonSkyCrawlers is a crawler framework.
 
 ## Requirements
 
@@ -33,32 +33,50 @@ Or install it yourself as:
 
 1. Create project
 
-        $ bundle exec daimon-skycrawlers new mycrawlers
-        $ cd mycrawlers
+```
+$ bundle exec daimon-skycrawlers new mycrawlers
+$ cd mycrawlers
+```
 
 2. Install dependencies
 
-        $ bundle install
+```
+$ bundle install
+```
 
 3. Create database
 
-        $ bundle exec rake db:create
-        $ bundle exec rake db:migrate
+```
+$ bundle exec rake db:create
+$ bundle exec rake db:migrate
+```
 
 4. Open new terminal and run crawler/processor
 
-        $ bundle exec ruby crawler.rb # on new terminal
-        $ bundle exec ruby processor.rb # on new terminal
+```
+$ bundle exec ruby crawler.rb # on new terminal
+$ bundle exec ruby processor.rb # on new terminal
+```
 
 5. Enqueue task
 
-        $ bundle exec ruby enqueue.rb http://example.com/
+```
+$ bundle exec ruby enqueue.rb url http://example.com/
+```
 
 6. You'll see `It works with 'http://example.com'` on your terminal which runs your processor!
 
+7. You can re-enqueue task for processor
+
+```
+$ bundle exec ruby enqueue.rb response http://example.com/
+```
+
+Display `It works with 'http://example.com'` again on your terminal which runs your processor.
+
 ## Development
 
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+After checking out the repo, run `bundle install` to install dependencies. Then, run `bundle exec rake test` to run the tests. You can also run `bundle console` for an interactive prompt that will allow you to experiment.
 
 To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 

data/Rakefile

@@ -1,17 +1,12 @@
 require "bundler/setup"
 require "bundler/gem_tasks"
 require "rake/testtask"
+require "daimon_skycrawlers/tasks"
 
 Rake::TestTask.new(:test) do |t|
   t.libs << "test"
   t.libs << "lib"
-  t.test_files = FileList['test/**/*_test.rb']
+  t.test_files = FileList["test/**/*_test.rb"]
 end
 
-require "cucumber/rake/task"
-
-Cucumber::Rake::Task.new(:features) do |t|
-  t.cucumber_opts = "features --format pretty"
-end
-
-task :default => [:test, :features]
+task :default => [:test]

data/bin/daimon-skycrawlers

@@ -1,9 +1,9 @@
 #!/usr/bin/env ruby
 
-if File.exist?(File.expand_path('../.git', __dir__))
-  $LOAD_PATH << File.expand_path('../lib', __dir__)
+if File.exist?(File.expand_path("../.git", __dir__))
+  $LOAD_PATH << File.expand_path("../lib", __dir__)
 end
 
-require 'daimon_skycrawlers/cli'
+require "daimon_skycrawlers/cli"
 
 DaimonSkycrawlers::CLI.start

data/daimon_skycrawlers.gemspec

@@ -1,7 +1,7 @@
 # coding: utf-8
-lib = File.expand_path('../lib', __FILE__)
+lib = File.expand_path("../lib", __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 'daimon_skycrawlers/version'
+require "daimon_skycrawlers/version"
 
 Gem::Specification.new do |spec|
   spec.name          = "daimon_skycrawlers"
@@ -25,13 +25,14 @@ Gem::Specification.new do |spec|
   spec.add_dependency "nokogiri"
   spec.add_dependency "activerecord"
   spec.add_dependency "pg"
+  spec.add_dependency "timers"
 
   spec.add_development_dependency "bundler", "~> 1.11"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "test-unit"
   spec.add_development_dependency "test-unit-rr"
   spec.add_development_dependency "test-unit-notify"
-  spec.add_development_dependency "cucumber"
   spec.add_development_dependency "pry"
   spec.add_development_dependency "tapp"
+  spec.add_development_dependency "sqlite3"
 end

data/lib/daimon_skycrawlers/cli.rb

@@ -1,6 +1,6 @@
-require 'thor'
-require 'daimon_skycrawlers/generator/new'
-require 'daimon_skycrawlers/version'
+require "thor"
+require "daimon_skycrawlers/generator/new"
+require "daimon_skycrawlers/version"
 
 module DaimonSkycrawlers
   class CLI < Thor

data/lib/daimon_skycrawlers/config.rb

@@ -0,0 +1,8 @@
+module DaimonSkycrawlers
+  module ConfigMixin
+    def initialize
+      super
+      @log = DaimonSkycrawlers.configuration.logger
+    end
+  end
+end

data/lib/daimon_skycrawlers/consumer/base.rb

@@ -0,0 +1,16 @@
+require "songkick_queue"
+require "daimon_skycrawlers/logger"
+require "daimon_skycrawlers/config"
+
+module DaimonSkycrawlers
+  module Consumer
+    class Base
+      include DaimonSkycrawlers::LoggerMixin
+      include DaimonSkycrawlers::ConfigMixin
+
+      def process(message)
+        raise NotImplementedError, "Must implement in subclass"
+      end
+    end
+  end
+end

data/lib/daimon_skycrawlers/consumer/http_response.rb

@@ -0,0 +1,47 @@
+require "songkick_queue"
+require "daimon_skycrawlers"
+require "daimon_skycrawlers/consumer/base"
+require "daimon_skycrawlers/processor/default"
+
+module DaimonSkycrawlers
+  module Consumer
+    class HTTPResponse < Base
+      include SongkickQueue::Consumer
+
+      class << self
+        def register(processor = nil, &block)
+          if block_given?
+            processors << block
+          else
+            processors << processor
+          end
+        end
+
+        def processors
+          @processors ||= []
+        end
+
+        def default_processor
+          DaimonSkycrawlers::Processor::Default.new
+        end
+
+        def queue_name
+          "#{DaimonSkycrawlers.configuration.queue_name_prefix}.http-response"
+        end
+      end
+
+      consume_from_queue queue_name
+
+      def process(message)
+        if self.class.processors.empty?
+          processors = [self.class.default_processor]
+        else
+          processors = self.class.processors
+        end
+        processors.each do |processor|
+          processor.call(message)
+        end
+      end
+    end
+  end
+end

data/lib/daimon_skycrawlers/consumer/url.rb

@@ -0,0 +1,44 @@
+require "songkick_queue"
+require "daimon_skycrawlers"
+require "daimon_skycrawlers/consumer/base"
+
+module DaimonSkycrawlers
+  module Consumer
+    class URL < Base
+      include SongkickQueue::Consumer
+
+      class << self
+        def register(crawler)
+          crawlers << crawler
+        end
+
+        def crawlers
+          @crawlers ||= []
+        end
+
+        def queue_name
+          "#{DaimonSkycrawlers.configuration.queue_name_prefix}.url"
+        end
+      end
+
+      consume_from_queue queue_name
+
+      def process(message)
+        url = message[:url]
+        depth = Integer(message[:depth] || 0)
+
+        crawler_interval = DaimonSkycrawlers.configuration.crawler_interval
+
+        # XXX When several crawlers are registered, how should they behave?
+        self.class.crawlers.each do |crawler|
+          crawler.fetch(url, depth: depth)
+          if crawler.skipped?
+            sleep(crawler_interval) if crawler.n_processed_urls % 50 == 0
+          else
+            sleep(crawler_interval)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/daimon_skycrawlers/consumer.rb

@@ -0,0 +1,4 @@
+module DaimonSkycrawlers
+  module Consumer
+  end
+end

data/lib/daimon_skycrawlers/crawler/base.rb

@@ -0,0 +1,75 @@
+require "uri"
+require "faraday"
+
+require "daimon_skycrawlers/logger"
+require "daimon_skycrawlers/config"
+require "daimon_skycrawlers/storage"
+require "daimon_skycrawlers/processor"
+
+module DaimonSkycrawlers
+  module Crawler
+    class Base
+      include DaimonSkycrawlers::LoggerMixin
+      include DaimonSkycrawlers::ConfigMixin
+
+      attr_writer :storage
+
+      def initialize(base_url = nil, options = {})
+        super()
+        @base_url = base_url
+        @options = options
+        @prepare = ->(connection) {}
+        @skipped = false
+        @n_processed_urls = 0
+      end
+
+      def setup_connection(options = {})
+        @connection = Faraday.new(@base_url, @options.merge(options)) do |faraday|
+          yield faraday
+        end
+      end
+
+      #
+      # Call this method before DaimonSkycrawlers.register_crawler
+      # For example, you can login before fetch URL
+      #
+      def prepare(&block)
+        @prepare = block
+      end
+
+      def storage
+        @storage ||= Storage::RDB.new
+      end
+
+      def skipped?
+        @skipped
+      end
+
+      def connection
+        @connection ||= Faraday.new(@base_url, @options)
+      end
+
+      def fetch(path, params = {}, **kw)
+        raise NotImplementedError, "Must implement this method in subclass"
+      end
+
+      def get(path, params = {})
+        @connection.get(path, params)
+      end
+
+      def post(path, params = {})
+        @connection.post(path, params)
+      end
+
+      def n_processed_urls
+        @n_processed_urls
+      end
+
+      private
+
+      def schedule_to_process(url, message = {})
+        DaimonSkycrawlers::Processor.enqueue_http_response(url, message)
+      end
+    end
+  end
+end

data/lib/daimon_skycrawlers/crawler/default.rb

@@ -0,0 +1,33 @@
+require "daimon_skycrawlers/crawler/base"
+require "daimon_skycrawlers/filter/update_checker"
+
+module DaimonSkycrawlers
+  module Crawler
+    class Default < Base
+      def fetch(path, depth: 3, **kw)
+        @n_processed_urls += 1
+        @skipped = false
+        url = connection.url_prefix + path
+        update_checker = DaimonSkycrawlers::Filter::UpdateChecker.new(storage: storage)
+        unless update_checker.call(url.to_s, connection: connection)
+          log.info("Skip #{url}")
+          @skipped = true
+          schedule_to_process(url.to_s, heartbeat: true)
+          return
+        end
+        @prepare.call(connection)
+        response = get(path)
+        data = [url.to_s, response.headers, response.body]
+
+        yield(*data) if block_given?
+
+        storage.save(*data)
+        message = {
+          depth: depth
+        }
+        message = message.merge(kw)
+        schedule_to_process(url.to_s, message)
+      end
+    end
+  end
+end

data/lib/daimon_skycrawlers/crawler.rb

@@ -1,89 +1,31 @@
-require 'uri'
-
-require 'daimon_skycrawlers'
-require 'daimon_skycrawlers/version'
-require 'daimon_skycrawlers/configure_songkick_queue'
-require 'daimon_skycrawlers/url_consumer'
-require 'daimon_skycrawlers/storage'
-require 'daimon_skycrawlers/parser'
-
-require 'faraday'
+require "daimon_skycrawlers"
+require "daimon_skycrawlers/queue"
+require "daimon_skycrawlers/timer"
+require "daimon_skycrawlers/consumer/url"
 
 module DaimonSkycrawlers
-  class Crawler
+  module Crawler
     class << self
-      def run(process_name: 'daimon-skycrawler:url')
-        SongkickQueue::Worker.new(process_name, [URLConsumer]).run
+      def run(process_name: default_process_name)
+        DaimonSkycrawlers::Timer.setup_shutdown_timer(config.queue_name_prefix, interval: config.shutdown_interval)
+        SongkickQueue::Worker.new(process_name, [DaimonSkycrawlers::Consumer::URL]).run
       end
 
-      def enqueue_url(url, depth: 3, interval: 1)
-        SongkickQueue.publish('daimon-skycrawler.url', url: url, depth: depth, interval: interval)
+      def enqueue_url(url, message = {})
+        message[:url] = url
+        SongkickQueue.publish(queue_name, message)
       end
-    end
-
-    attr_writer :storage
-    attr_writer :parser
 
-    def initialize(base_url, options = {})
-      @base_url = base_url
-      @options = options
-    end
-
-    def setup_connection(options = {})
-      @connection = Faraday.new(@base_url, options) do |faraday|
-        yield faraday
+      def config
+        DaimonSkycrawlers.configuration
       end
-    end
-
-    def storage
-      @storage ||= Storage::RDB.new
-    end
-
-    def parser
-      @parser ||= Parser::Default.new
-    end
-
-    # TODO Support POST when we need
-    # TODO `params` should be a part of `path`. such as `path == "/hoi?hi=yoyo"`.
-    def fetch(path, params = {}, depth: 3)
-      @connection ||= Faraday.new(@base_url)
-      response = get(path)
-
-      url = @connection.url_prefix + path
-
-      data = [url.to_s, response.headers, response.body]
 
-      yield(*data) if block_given?
-
-      storage.save(*data)
-
-      schedule_to_process(url.to_s)
-
-      parser.parse(response.body)
-      urls = parser.links
-
-      enqueue_next_urls(urls, depth: depth - 1, interval: 1)
-    end
-
-    def get(path, params = {})
-      @connection.get(path, params)
-    end
-
-    def post(path, params = {})
-      @connection.post(path, params)
-    end
-
-    private
-
-    def schedule_to_process(url)
-      DaimonSkycrawlers::Processor.enqueue_http_response(url)
-    end
-
-    def enqueue_next_urls(urls, depth: 3, interval: 1)
-      return if depth <= 0
+      def queue_name
+        "#{config.queue_name_prefix}.url"
+      end
 
-      urls.each do |url|
-        self.class.enqueue_url(url, depth: depth, interval: interval)
+      def default_process_name
+        "#{config.queue_name_prefix}:url"
       end
     end
   end

data/lib/daimon_skycrawlers/filter/base.rb

@@ -0,0 +1,24 @@
+require "daimon_skycrawlers/logger"
+require "daimon_skycrawlers/config"
+
+module DaimonSkycrawlers
+  module Filter
+    class Base
+      include DaimonSkycrawlers::LoggerMixin
+      include DaimonSkycrawlers::ConfigMixin
+
+      def initialize(storage: nil)
+        super()
+        @storage = storage
+      end
+
+      def storage
+        @storage ||= DaimonSkycrawlers::Storage::RDB.new
+      end
+
+      def call(url)
+        raise NotImplementedError, "Must implement this method in subclass"
+      end
+    end
+  end
+end

data/lib/daimon_skycrawlers/filter/duplicate_checker.rb

@@ -0,0 +1,23 @@
+require "set"
+require "daimon_skycrawlers/filter/base"
+
+module DaimonSkycrawlers
+  module Filter
+    class DuplicateChecker < Base
+      def initialize(base_url: nil)
+        @base_url = nil
+        @base_url = URI(base_url) if base_url
+        @urls = Set.new
+      end
+
+      def call(url)
+        unless URI(url).absolute?
+          url = (@base_url + url).to_s
+        end
+        return false if @urls.include?(url)
+        @urls << url
+        true
+      end
+    end
+  end
+end

data/lib/daimon_skycrawlers/filter/update_checker.rb

@@ -0,0 +1,31 @@
+require "faraday"
+require "daimon_skycrawlers/filter/base"
+
+module DaimonSkycrawlers
+  module Filter
+    class UpdateChecker < Base
+      def initialize(storage: nil, base_url: nil)
+        super(storage: storage)
+        @base_url = nil
+        @base_url = URI(base_url) if base_url
+      end
+
+      def call(url, connection: nil)
+        unless URI(url).absolute?
+          url = (@base_url + url).to_s
+        end
+        page = storage.find(url)
+        return true unless page
+        if connection
+          headers = connection.head(url)
+        else
+          headers = Faraday.head(url)
+        end
+        return false if headers["etag"] && page.etag && headers["etag"] == page.etag
+        return false if headers["last-modified"].nil? && page.last_modified_at.nil?
+        return false if headers["last-modified"] <= page.last_modified_at
+        true
+      end
+    end
+  end
+end

data/lib/daimon_skycrawlers/filter.rb

@@ -0,0 +1,4 @@
+module DaimonSkycrawlers
+  module Filter
+  end
+end

data/lib/daimon_skycrawlers/generator/new.rb

@@ -1,4 +1,4 @@
-require 'thor'
+require "thor"
 
 module DaimonSkycrawlers
   module Generator
@@ -8,7 +8,7 @@ module DaimonSkycrawlers
       argument :name
 
       def self.source_root
-        File.join(File.dirname(__FILE__), "templates", "new")
+        File.join(__dir__, "templates", "new")
       end
 
       def create_files
@@ -26,6 +26,7 @@ module DaimonSkycrawlers
           "Rakefile",
           "crawler.rb",
           "enqueue.rb",
+          "init.rb",
           "processor.rb",
         ].each do |path|
           copy_file(path, "#{name}/#{path}")

data/lib/daimon_skycrawlers/generator/templates/new/Gemfile

@@ -1,4 +1,4 @@
-source 'https://rubygems.org'
+source "https://rubygems.org"
 
-gem 'rake'
-gem 'daimon_skycrawlers'
+gem "rake"
+gem "daimon_skycrawlers"