daimon_skycrawlers 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 411d6b35b2c909712f9f79cd5e45440226ebba28
-  data.tar.gz: afa7bdc3cc98a28742c64ba46d7237fcce981b92
+  metadata.gz: 4e4bd1308a554f55bce802a5d0a038cbb0f0470e
+  data.tar.gz: a484737c74cc9ff3304a9fda0714282f9f6b0e61
 SHA512:
-  metadata.gz: f8844e9ebd88e7be7344a607866c15ea74be64a8a405f05f196d54f0125bf66be6a81c4ff27ea3ee4480a82989abbedd79cd934c122c7ac1bee5d83bbfa1b6d3
-  data.tar.gz: 37fe8376a4165aef7f29aa29af51bb637402c48d9d5327c44ea05c37dac2418c39b1f5e55303ec8c66e9cfe8eb3e77f0dc264648caa262abc551c72f06944b55
+  metadata.gz: c9d4b01cb37808ce43a9786324e6e0d842d67bc44269c39964e4570c5dbab1f0b818449ac6a67864d6834e2b67a1a9f857252fd0d3cc91e87961ad252eccc785
+  data.tar.gz: c1babe4300e744678672482e3a82c8bc5fa37a5e52ebe8e634b3ae0cdaea17adc75c5132a5a8918c2b3dce76b47c783fedea8e911b18f2e6f0bd7adee0d0525b

data/README.md

@@ -33,52 +33,60 @@ Or install it yourself as:
 
 1. Create project
 
-     ```
-     $ bundle exec daimon-skycrawlers new mycrawlers
-     $ cd mycrawlers
-     ```
-     or
-     ```
-     $ daimon-skycrawlers new mycrawlers
-     $ cd mycrawlers
-     ```
+    ```
+    $ bundle exec daimon-skycrawlers new mycrawlers
+    $ cd mycrawlers
+    ```
+    or
+    ```
+    $ daimon-skycrawlers new mycrawlers
+    $ cd mycrawlers
+    ```
 
 2. Install dependencies
 
-     ```
-     $ bundle install
-     ```
+    ```
+    $ bundle install
+    ```
 
 3. Create database
 
-     ```
-     $ bundle exec rake db:create
-     $ bundle exec rake db:migrate
-     ```
+    ```
+    $ bundle exec rake db:create
+    $ bundle exec rake db:migrate
+    ```
 
 4. Open new terminal and run crawler/processor
 
-     ```
-     $ bin/crawler   # on new terminal
-     $ bin/processor # on new terminal
-     ```
+    ```
+    $ daimon-skycrawlers exec crawler   # on new terminal
+    $ daimon-skycrawlers exec processor # on new terminal
+    ```
 
     NOTE: Execute step 5 as soon as possible. Because bin/crawler and
     bin/processor will stop after 10 seconds by default if their
     queues are empty.
 
+    NOTE: You can change `shutdown_interval` using following code in config/init.rb:
+
+    ```ruby
+    DaimonSkycrawlers.configure do |config|
+      config.shutdown_interval = 30
+    end
+    ```
+
 5. Enqueue task
 
-     ```
-     $ bin/enqueue url http://example.com/
-     ```
+    ```
+    $ daimon-skycrawlers enqueue url http://example.com/
+    ```
 
 6. You'll see `It works with 'http://example.com'` on your terminal which runs your processor!
 7. You can re-enqueue task for processor
 
-     ```
-     $ bin/enqueue response http://example.com/
-     ```
+    ```
+    $ daimon-skycrawlers enqueue response http://example.com/
+    ```
 
 Display `It works with 'http://example.com'` again on your terminal which runs your processor.
 

data/daimon_skycrawlers.gemspec

@@ -28,6 +28,8 @@ Gem::Specification.new do |spec|
   spec.add_dependency "railties"
   spec.add_dependency "pg"
   spec.add_dependency "timers"
+  spec.add_dependency "sitemap-parser"
+  spec.add_dependency "webrobots"
 
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "test-unit"

data/lib/daimon_skycrawlers/cli.rb

@@ -1,10 +1,14 @@
 require "thor"
 require "daimon_skycrawlers/generator/new"
+require "daimon_skycrawlers/commands/enqueue"
+require "daimon_skycrawlers/commands/runner"
 require "daimon_skycrawlers/version"
 
 module DaimonSkycrawlers
   class CLI < Thor
     register(Generator::New, "new", "new NAME", "Create new project")
+    register(Commands::Runner, "exec", "exec [COMMAND]", "Execute crawler/processor")
+    register(Commands::Enqueue, "enqueue", "enqueue [TYPE] URL [messages...]", "Enqueue URL")
 
     desc "version", "Show version"
     def version

data/lib/daimon_skycrawlers/commands/enqueue.rb

@@ -0,0 +1,58 @@
+require "daimon_skycrawlers"
+require "daimon_skycrawlers/crawler"
+require "daimon_skycrawlers/processor"
+require "daimon_skycrawlers/version"
+require "sitemap-parser"
+require "webrobots"
+
+module DaimonSkycrawlers
+  module Commands
+    class Enqueue < Thor
+      desc "url URL [key1:value1 key2:value2...]", "Enqueue URL for URL consumer"
+      def url(url, *rest)
+        load_init
+        message = rest.map {|arg| arg.split(":") }.to_h
+        log.debug("Enqueue URL for crawler: #{url} : #{message}")
+        DaimonSkycrawlers::Crawler.enqueue_url(url, message)
+      end
+
+      desc "response URL [key1:value1 key2:value2...]", "Enqueue URL for HTTP response consumer"
+      def response(url, *rest)
+        load_init
+        message = rest.map {|arg| arg.split(":") }.to_h
+        log.debug("Enqueue URL for processor: #{url} : #{message}")
+        DaimonSkycrawlers::Processor.enqueue_http_response(url, message)
+      end
+
+      desc "sitemap [OPTIONS] URL", "Enqueue URLs from simtemap.xml"
+      method_option("robots-txt", aliases: ["-r"], type: :boolean,
+                    desc: "URL for robots.txt. Detect robots.txt automatically if URL is not robots.txt")
+      def sitemap(url)
+        load_init
+        if options["robots-txt"]
+          webrobots = WebRobots.new("DaimonSkycrawlers/#{DaimonSkycrawlers::VERSION}")
+          sitemaps = webrobots.sitemaps(url).uniq
+        else
+          sitemaps = [url]
+        end
+        urls = sitemaps.flat_map do |sitemap|
+          sitemap_parser = SitemapParser.new(sitemap)
+          sitemap_parser.to_a
+        end
+        urls.each do |_url|
+          DaimonSkycrawlers::Crawler.enqueue_url(_url)
+        end
+      end
+
+      private
+
+      def load_init
+        DaimonSkycrawlers.load_init
+      end
+
+      def log
+        DaimonSkycrawlers.configuration.logger
+      end
+    end
+  end
+end

data/lib/daimon_skycrawlers/commands/runner.rb

@@ -0,0 +1,47 @@
+require "thor"
+require "daimon_skycrawlers"
+require "daimon_skycrawlers/crawler"
+
+module DaimonSkycrawlers
+  module Commands
+    class Runner < Thor
+      namespace "exec"
+
+      desc "crawler", "Execute crawler"
+      def crawler
+        load_init
+        Dir.glob("app/crawlers/**/*.rb") do |path|
+          require(File.expand_path(path, Dir.pwd))
+          log.info("Loaded crawler: #{path}")
+        end
+        DaimonSkycrawlers::Crawler.run
+      rescue => ex
+        puts ex.message
+        exit(false)
+      end
+
+      desc "processor", "Execute processor"
+      def processor
+        load_init
+        Dir.glob("app/processors/**/*.rb") do |path|
+          require(File.expand_path(path, Dir.pwd))
+          log.info("Loaded processor: #{path}")
+        end
+        DaimonSkycrawlers::Processor.run
+      rescue => ex
+        puts ex.message
+        exit(false)
+      end
+
+      private
+
+      def load_init
+        DaimonSkycrawlers.load_init
+      end
+
+      def log
+        DaimonSkycrawlers.configuration.logger
+      end
+    end
+  end
+end

data/lib/daimon_skycrawlers/crawler/default.rb

@@ -1,5 +1,6 @@
 require "daimon_skycrawlers/crawler/base"
 require "daimon_skycrawlers/filter/update_checker"
+require "daimon_skycrawlers/filter/robots_txt_checker"
 
 module DaimonSkycrawlers
   module Crawler
@@ -13,11 +14,16 @@ module DaimonSkycrawlers
         @n_processed_urls += 1
         @skipped = false
         url = connection.url_prefix + path
+        if @options[:obey_robots_txt]
+          robots_txt_checker = DaimonSkycrawlers::Filter::RobotsTxtChecker.new(base_url: @base_url)
+          unless robots_txt_checker.call(url)
+            skip(url)
+            return
+          end
+        end
         update_checker = DaimonSkycrawlers::Filter::UpdateChecker.new(storage: storage)
         unless update_checker.call(url.to_s, connection: connection)
-          log.info("Skip #{url}")
-          @skipped = true
-          schedule_to_process(url.to_s, heartbeat: true)
+          skip(url)
           return
         end
         @prepare.call(connection)
@@ -33,6 +39,14 @@ module DaimonSkycrawlers
         message = message.merge(kw)
         schedule_to_process(url.to_s, message)
       end
+
+      private
+
+      def skip(url)
+        log.info("Skip #{url}")
+        @skipped = true
+        schedule_to_process(url.to_s, heartbeat: true)
+      end
     end
   end
 end

data/lib/daimon_skycrawlers/filter/base.rb

@@ -3,6 +3,14 @@ require "daimon_skycrawlers/config"
 
 module DaimonSkycrawlers
   module Filter
+    #
+    # Base class of filters.
+    #
+    # You must implement `#call` in your filter and it must return
+    # true or false. If your filter returns true, processors can
+    # process given URL after your filter. Otherwise framework skips
+    # given URL to skip processors.
+    #
     class Base
       include DaimonSkycrawlers::LoggerMixin
       include DaimonSkycrawlers::ConfigMixin

data/lib/daimon_skycrawlers/filter/duplicate_checker.rb

@@ -6,6 +6,8 @@ module DaimonSkycrawlers
     #
     # This filter provides duplication checker for given URL.
     #
+    # Skip processing duplicated URLs.
+    #
     class DuplicateChecker < Base
       def initialize(base_url: nil)
         @base_url = nil

data/lib/daimon_skycrawlers/filter/robots_txt_checker.rb

@@ -0,0 +1,29 @@
+require "webrobots"
+require "daimon_skycrawlers/filter/base"
+require "daimon_skycrawlers/version"
+
+module DaimonSkycrawlers
+  module Filter
+    #
+    # This filter provides robots.txt checker for given URL.
+    # We want to obey robots.txt provided by a web site.
+    #
+    class RobotsTxtChecker < Base
+      def initialize(base_url: nil, user_agent: "DaimonSkycrawlers/#{DaimonSkycrawlers::VERSION}")
+        super()
+        @webrobots = WebRobots.new(user_agent)
+      end
+
+      #
+      # @param [String] url
+      # @return [true|false] Return true when web site allows to fetch the URL, otherwise return false
+      #
+      def call(url)
+        unless URI(url).absolute?
+          url = (@base_url + url).to_s
+        end
+        @webrobots.allowed?(url)
+      end
+    end
+  end
+end

data/lib/daimon_skycrawlers/filter/update_checker.rb

@@ -6,6 +6,9 @@ module DaimonSkycrawlers
     #
     # This filter provides update checker for given URL.
     #
+    # Skip processing URLs that is latest (not updated since previous
+    # access).
+    #
     class UpdateChecker < Base
       def initialize(storage: nil, base_url: nil)
         super(storage: storage)

data/lib/daimon_skycrawlers/generator/new.rb

@@ -40,9 +40,6 @@ module DaimonSkycrawlers
           "Rakefile",
           "app/crawlers/sample_crawler.rb",
           "app/processors/sample_processor.rb",
-          "bin/crawler",
-          "bin/enqueue",
-          "bin/processor",
           "config/init.rb",
         ].each do |path|
           copy_file(path, "#{name}/#{path}", mode: :preserve)

data/lib/daimon_skycrawlers/version.rb

@@ -1,3 +1,3 @@
 module DaimonSkycrawlers
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

data/lib/daimon_skycrawlers.rb

@@ -62,5 +62,17 @@ module DaimonSkycrawlers
     def configure
       yield configuration
     end
+
+    #
+    # Load "config/init.rb"
+    #
+    # @return [void]
+    #
+    def load_init
+      require(File.expand_path("config/init.rb", Dir.pwd))
+    rescue LoadError => ex
+      puts ex.message
+      exit(false)
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: daimon_skycrawlers
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Ryunosuke SATO
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-09-29 00:00:00.000000000 Z
+date: 2016-10-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -150,6 +150,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: sitemap-parser
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webrobots
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -281,6 +309,8 @@ files:
 - db/schema.rb
 - lib/daimon_skycrawlers.rb
 - lib/daimon_skycrawlers/cli.rb
+- lib/daimon_skycrawlers/commands/enqueue.rb
+- lib/daimon_skycrawlers/commands/runner.rb
 - lib/daimon_skycrawlers/config.rb
 - lib/daimon_skycrawlers/consumer.rb
 - lib/daimon_skycrawlers/consumer/base.rb
@@ -292,6 +322,7 @@ files:
 - lib/daimon_skycrawlers/filter.rb
 - lib/daimon_skycrawlers/filter/base.rb
 - lib/daimon_skycrawlers/filter/duplicate_checker.rb
+- lib/daimon_skycrawlers/filter/robots_txt_checker.rb
 - lib/daimon_skycrawlers/filter/update_checker.rb
 - lib/daimon_skycrawlers/generator/new.rb
 - lib/daimon_skycrawlers/generator/templates/new/Gemfile
@@ -299,9 +330,6 @@ files:
 - lib/daimon_skycrawlers/generator/templates/new/Rakefile
 - lib/daimon_skycrawlers/generator/templates/new/app/crawlers/sample_crawler.rb
 - lib/daimon_skycrawlers/generator/templates/new/app/processors/sample_processor.rb
-- lib/daimon_skycrawlers/generator/templates/new/bin/crawler
-- lib/daimon_skycrawlers/generator/templates/new/bin/enqueue
-- lib/daimon_skycrawlers/generator/templates/new/bin/processor
 - lib/daimon_skycrawlers/generator/templates/new/config/database.yml.erb
 - lib/daimon_skycrawlers/generator/templates/new/config/init.rb
 - lib/daimon_skycrawlers/logger.rb
@@ -324,9 +352,6 @@ files:
 - sample/spider/Rakefile
 - sample/spider/app/crawlers/blog_crawler.rb
 - sample/spider/app/processors/blog_spider.rb
-- sample/spider/bin/crawler
-- sample/spider/bin/enqueue
-- sample/spider/bin/processor
 - sample/spider/config/database.yml
 - sample/spider/config/init.rb
 - sample/spider/db/migrate/20160830155803_create_pages.rb

data/lib/daimon_skycrawlers/generator/templates/new/bin/crawler

@@ -1,10 +0,0 @@
-#!/usr/bin/env ruby
-
-require_relative "../config/init"
-require "daimon_skycrawlers/crawler"
-
-Dir.glob("app/crawlers/**/*.rb") do |path|
-  require(File.expand_path(path, File.dirname(__dir__)))
-end
-
-DaimonSkycrawlers::Crawler.run

data/lib/daimon_skycrawlers/generator/templates/new/bin/enqueue

@@ -1,23 +0,0 @@
-#!/usr/bin/env ruby
-
-require "thor"
-
-require_relative "../config/init"
-require "daimon_skycrawlers/crawler"
-require "daimon_skycrawlers/processor"
-
-class Enqueue < Thor
-  desc "url URL [key1:value1 key2:value2...]", "Enqueue URL for URL consumer"
-  def url(url, *rest)
-    message = rest.map {|arg| arg.split(":") }.to_h
-    DaimonSkycrawlers::Crawler.enqueue_url(url, message)
-  end
-
-  desc "response URL [key1:value1 key2:value2...]", "Enqueue URL for HTTP response consumer"
-  def response(url, *rest)
-    message = rest.map {|arg| arg.split(":") }.to_h
-    DaimonSkycrawlers::Processor.enqueue_http_response(url, message)
-  end
-end
-
-Enqueue.start(ARGV)

data/lib/daimon_skycrawlers/generator/templates/new/bin/processor

@@ -1,10 +0,0 @@
-#!/usr/bin/env ruby
-
-require_relative "../config/init"
-require "daimon_skycrawlers/processor"
-
-Dir.glob("app/processors/**/*.rb") do |path|
-  require(File.expand_path(path, File.dirname(__dir__)))
-end
-
-DaimonSkycrawlers::Processor.run

data/sample/spider/bin/crawler

@@ -1,10 +0,0 @@
-#!/usr/bin/env ruby
-
-require_relative "../config/init"
-require "daimon_skycrawlers/crawler"
-
-Dir.glob("app/crawlers/**/*.rb") do |path|
-  require(File.expand_path(path, File.dirname(__dir__)))
-end
-
-DaimonSkycrawlers::Crawler.run

data/sample/spider/bin/enqueue

@@ -1,23 +0,0 @@
-#!/usr/bin/env ruby
-
-require "thor"
-
-require_relative "../config/init"
-require "daimon_skycrawlers/crawler"
-require "daimon_skycrawlers/processor"
-
-class Enqueue < Thor
-  desc "url URL [key1:value1 key2:value2...]", "Enqueue URL for URL consumer"
-  def url(url, *rest)
-    message = rest.map {|arg| arg.split(":") }.to_h
-    DaimonSkycrawlers::Crawler.enqueue_url(url, message)
-  end
-
-  desc "response URL [key1:value1 key2:value2...]", "Enqueue URL for HTTP response consumer"
-  def response(url, *rest)
-    message = rest.map {|arg| arg.split(":") }.to_h
-    DaimonSkycrawlers::Processor.enqueue_http_response(url, message)
-  end
-end
-
-Enqueue.start(ARGV)

data/sample/spider/bin/processor

@@ -1,11 +0,0 @@
-#!/usr/bin/env ruby
-
-require_relative "../config/init"
-
-require "daimon_skycrawlers/processor"
-
-Dir.glob("app/processors/**/*.rb") do |path|
-  require(File.expand_path(path, File.dirname(__dir__)))
-end
-
-DaimonSkycrawlers::Processor.run