daimon_skycrawlers 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4e4bd1308a554f55bce802a5d0a038cbb0f0470e
-  data.tar.gz: a484737c74cc9ff3304a9fda0714282f9f6b0e61
+  metadata.gz: cbbd476464b9b44d5bc5f71bc999820126379e2d
+  data.tar.gz: 473ac2bd6f9c63a7307357aa2fae7da2be6efde7
 SHA512:
-  metadata.gz: c9d4b01cb37808ce43a9786324e6e0d842d67bc44269c39964e4570c5dbab1f0b818449ac6a67864d6834e2b67a1a9f857252fd0d3cc91e87961ad252eccc785
-  data.tar.gz: c1babe4300e744678672482e3a82c8bc5fa37a5e52ebe8e634b3ae0cdaea17adc75c5132a5a8918c2b3dce76b47c783fedea8e911b18f2e6f0bd7adee0d0525b
+  metadata.gz: 788e296ef3fbd73c39db3ca0f6e6507e9c2074893c4895598c3e418849cf70a8951cc65c5fbff88d1a832c721edd53bd7d4078699090b6da36efaf460027a944
+  data.tar.gz: 42ba6b8ad282060811d2817d85c8b276ccc0890c4a15c546571f22594cb1a552be2838c166ced8fa609807dc0399b910acc53ba7f56e927222955af2008a831a

data/lib/daimon_skycrawlers/cli.rb

@@ -1,5 +1,6 @@
 require "thor"
 require "daimon_skycrawlers/generator/new"
+require "daimon_skycrawlers/generator/generate"
 require "daimon_skycrawlers/commands/enqueue"
 require "daimon_skycrawlers/commands/runner"
 require "daimon_skycrawlers/version"
@@ -7,6 +8,7 @@ require "daimon_skycrawlers/version"
 module DaimonSkycrawlers
   class CLI < Thor
     register(Generator::New, "new", "new NAME", "Create new project")
+    register(Generator::Generate, "generate", "generate COMMAND", "Generate new code")
     register(Commands::Runner, "exec", "exec [COMMAND]", "Execute crawler/processor")
     register(Commands::Enqueue, "enqueue", "enqueue [TYPE] URL [messages...]", "Enqueue URL")
 

data/lib/daimon_skycrawlers/consumer/http_response.rb

@@ -67,7 +67,7 @@ module DaimonSkycrawlers
           processors = self.class.processors
         end
         processors.each do |processor|
-          processor.call(message)
+          processor.process(message)
         end
       end
     end

data/lib/daimon_skycrawlers/consumer/url.rb

@@ -44,14 +44,11 @@ module DaimonSkycrawlers
       # @private
       #
       def process(message)
-        url = message[:url]
-        depth = Integer(message[:depth] || 0)
-
         crawler_interval = DaimonSkycrawlers.configuration.crawler_interval
 
         # XXX When several crawlers are registered, how should they behave?
         self.class.crawlers.each do |crawler|
-          crawler.fetch(url, depth: depth)
+          crawler.process(message)
           if crawler.skipped?
             sleep(crawler_interval) if crawler.n_processed_urls % 50 == 0
           else

data/lib/daimon_skycrawlers/crawler/base.rb

@@ -20,13 +20,19 @@ module DaimonSkycrawlers
       #   @return [void]
       attr_writer :storage
 
+      # @!attribute [r] n_processed_urls
+      #   The number of processed URLs.
+      #   @return [Integer]
+      attr_reader :n_processed_urls
+
       #
       # @param [String] Base URL for crawler
       # @param [Hash] options for Faraday
       #
-      def initialize(base_url = nil, options = {})
+      def initialize(base_url = nil, faraday_options: {}, options: {})
         super()
         @base_url = base_url
+        @faraday_options = faraday_options
         @options = options
         @prepare = ->(connection) {}
         @skipped = false
@@ -41,7 +47,9 @@ module DaimonSkycrawlers
       # @yieldparam faraday [Faraday]
       #
       def setup_connection(options = {})
-        @connection = Faraday.new(@base_url, @options.merge(options)) do |faraday|
+        merged_options = @faraday_options.merge(options)
+        faraday_options = merged_options.empty? ? nil : merged_options
+        @connection = Faraday.new(@base_url, faraday_options) do |faraday|
           yield faraday
         end
       end
@@ -66,10 +74,26 @@ module DaimonSkycrawlers
       end
 
       def connection
-        @connection ||= Faraday.new(@base_url, @options)
+        @connection ||= Faraday.new(@base_url, @faraday_options)
+      end
+
+      def process(message, &block)
+        url = message.delete(:url)
+
+        @skipped = false
+        @n_processed_urls += 1
+        # url can be a path
+        url = connection.url_prefix + url
+
+        apply_filters(url)
+
+        unless skipped?
+          @prepare.call(connection)
+          fetch(url, message, &block)
+        end
       end
 
-      def fetch(path, params = {}, **kw)
+      def fetch(path, message = {})
         raise NotImplementedError, "Must implement this method in subclass"
       end
 
@@ -81,11 +105,28 @@ module DaimonSkycrawlers
         @connection.post(path, params)
       end
 
-      def n_processed_urls
-        @n_processed_urls
+      private
+
+      def apply_filters(url)
+        if @options[:obey_robots_txt]
+          robots_txt_checker = DaimonSkycrawlers::Filter::RobotsTxtChecker.new(base_url: @base_url)
+          unless robots_txt_checker.allowed?(url)
+            skip(url)
+            return
+          end
+        end
+        update_checker = DaimonSkycrawlers::Filter::UpdateChecker.new(storage: storage)
+        unless update_checker.updated?(url.to_s, connection: connection)
+          skip(url)
+          return
+        end
       end
 
-      private
+      def skip(url)
+        log.info("Skip #{url}")
+        @skipped = true
+        schedule_to_process(url.to_s, heartbeat: true)
+      end
 
       def schedule_to_process(url, message = {})
         DaimonSkycrawlers::Processor.enqueue_http_response(url, message)

data/lib/daimon_skycrawlers/crawler/default.rb

@@ -10,43 +10,15 @@ module DaimonSkycrawlers
     # This crawler can GET given URL and store response to storage
     #
     class Default < Base
-      def fetch(path, depth: 3, **kw)
-        @n_processed_urls += 1
-        @skipped = false
-        url = connection.url_prefix + path
-        if @options[:obey_robots_txt]
-          robots_txt_checker = DaimonSkycrawlers::Filter::RobotsTxtChecker.new(base_url: @base_url)
-          unless robots_txt_checker.call(url)
-            skip(url)
-            return
-          end
-        end
-        update_checker = DaimonSkycrawlers::Filter::UpdateChecker.new(storage: storage)
-        unless update_checker.call(url.to_s, connection: connection)
-          skip(url)
-          return
-        end
-        @prepare.call(connection)
-        response = get(path)
+      def fetch(url, message)
+        response = get(url)
         data = [url.to_s, response.headers, response.body]
 
         yield(*data) if block_given?
 
         storage.save(*data)
-        message = {
-          depth: depth
-        }
-        message = message.merge(kw)
         schedule_to_process(url.to_s, message)
       end
-
-      private
-
-      def skip(url)
-        log.info("Skip #{url}")
-        @skipped = true
-        schedule_to_process(url.to_s, heartbeat: true)
-      end
     end
   end
 end

data/lib/daimon_skycrawlers/filter/duplicate_checker.rb

@@ -28,6 +28,15 @@ module DaimonSkycrawlers
         @urls << url
         true
       end
+
+      #
+      # @param [String] url to check duplication. If given URL is
+      #        relative URL, use `@base_url + url` as absolute URL.
+      # @return [true|false] Return true when duplicated, otherwise return false.
+      #
+      def duplicated?(url)
+        !call(url)
+      end
     end
   end
 end

data/lib/daimon_skycrawlers/filter/robots_txt_checker.rb

@@ -24,6 +24,8 @@ module DaimonSkycrawlers
         end
         @webrobots.allowed?(url)
       end
+
+      alias allowed? call
     end
   end
 end

data/lib/daimon_skycrawlers/filter/update_checker.rb

@@ -32,11 +32,17 @@ module DaimonSkycrawlers
         else
           headers = Faraday.head(url)
         end
-        return false if headers["etag"] && page.etag && headers["etag"] == page.etag
-        return false if headers["last-modified"].nil? && page.last_modified_at.nil?
-        return false if headers["last-modified"] <= page.last_modified_at
-        true
+        case
+        when headers.key?("etag") && page.etag
+          headers["etag"] != page.etag
+        when headers.key?("last-modified") && page.last_modified_at
+          headers["last-modified"] > page.last_modified_at
+        else
+          true
+        end
       end
+
+      alias updated? call
     end
   end
 end

data/lib/daimon_skycrawlers/generator/crawler.rb

@@ -0,0 +1,22 @@
+require "thor"
+
+module DaimonSkycrawlers
+  module Generator
+    class Crawler < Thor::Group
+      include Thor::Actions
+
+      argument :name
+
+      def self.source_root
+        File.join(__dir__, "templates")
+      end
+
+      def create_files
+        config = {
+          class_name: name.classify,
+        }
+        template("crawler.rb.erb", "app/crawlers/#{name.underscore}.rb", config)
+      end
+    end
+  end
+end

data/lib/daimon_skycrawlers/generator/generate.rb

@@ -0,0 +1,12 @@
+require "thor"
+require "daimon_skycrawlers/generator/crawler"
+require "daimon_skycrawlers/generator/processor"
+
+module DaimonSkycrawlers
+  module Generator
+    class Generate < Thor
+      register(Crawler, "crawler", "crawler NAME", "Generate new crawler")
+      register(Processor, "processor", "processor NAME", "Generate new processor")
+    end
+  end
+end

data/lib/daimon_skycrawlers/generator/new.rb

@@ -22,16 +22,19 @@ module DaimonSkycrawlers
         ].each do |path|
           template("#{path}.erb", "#{name}/#{path}")
         end
+        migration_options = {
+          destination_root: File.join(destination_root, name),
+          timestamps: true
+        }
         invoke(MigrationGenerator, [
                  "CreatePage",
                  "url:string",
                  "headers:text",
                  "body:binary",
                  "last_modified_at:datetime",
-                 "etag:string",
-                 "timestamps"
+                 "etag:string"
                ],
-          { destination_root: File.join(destination_root, name) })
+               migration_options)
       end
 
       def copy_files
@@ -56,7 +59,7 @@ module DaimonSkycrawlers
         set_local_assigns!
         validate_file_name!
         dest = options[:destination_root]
-        migration_template @migration_template, "#{dest}/db/migrate/#{file_name}.rb"
+        migration_template(@migration_template, "#{dest}/db/migrate/#{file_name}.rb")
       end
     end
   end

data/lib/daimon_skycrawlers/generator/processor.rb

@@ -0,0 +1,22 @@
+require "thor"
+
+module DaimonSkycrawlers
+  module Generator
+    class Processor < Thor::Group
+      include Thor::Actions
+
+      argument :name
+
+      def self.source_root
+        File.join(__dir__, "templates")
+      end
+
+      def create_files
+        config = {
+          class_name: name.classify,
+        }
+        template("processor.rb.erb", "app/processors/#{name.underscore}.rb", config)
+      end
+    end
+  end
+end

data/lib/daimon_skycrawlers/generator/templates/crawler.rb.erb

@@ -0,0 +1,13 @@
+require "daimon_skycrawlers"
+require "daimon_skycrawlers/crawler"
+require "daimon_skycrawlers/crawler/base"
+
+class <%= config[:class_name] %> < DaimonSkycrawlers::Crawler::Base
+  def fetch(path, **kw)
+    # Implement your crawler here
+  end
+end
+
+base_url = ""
+crawler = <%= config[:class_name] %>.new(base_url)
+DaimonSkycrawlers.register_crawler(crawler)

data/lib/daimon_skycrawlers/generator/templates/processor.rb.erb

@@ -0,0 +1,13 @@
+require "daimon_skycrawlers"
+require "daimon_skycrawlers/processor"
+require "daimon_skycrawlers/processor/base"
+
+class <%= config[:class_name] %> < DaimonSkycrawlers::Processor::Base
+  def call(message)
+    # Implement your processor here
+  end
+end
+
+base_url = ""
+processor = <%= config[:class_name] %>.new(base_url)
+DaimonSkycrawlers.register_processor(processor)

data/lib/daimon_skycrawlers/processor/base.rb

@@ -7,6 +7,24 @@ module DaimonSkycrawlers
       include DaimonSkycrawlers::LoggerMixin
       include DaimonSkycrawlers::ConfigMixin
 
+      def initialize
+        super
+        @before_process_filters = []
+      end
+
+      def before_process(filter = nil, &block)
+        if block_given?
+          @before_process_filters << block
+        else
+          @before_process_filters << filter if filter.respond_to?(:call)
+        end
+      end
+
+      def process(message)
+        return unless apply_before_filters(message[:url])
+        call(message)
+      end
+
       def call(message)
         raise "Implement this method in subclass"
       end
@@ -14,6 +32,14 @@ module DaimonSkycrawlers
       def storage
         @storage ||= DaimonSkycrawlers::Storage::RDB.new
       end
+
+      private
+
+      def apply_before_filters(url)
+        @before_process_filters.all? do |filter|
+          filter.call(url)
+        end
+      end
     end
   end
 end

data/lib/daimon_skycrawlers/processor/spider.rb

@@ -8,17 +8,17 @@ module DaimonSkycrawlers
 
       def initialize
         super
-        @filters = []
+        @link_filters = []
         @doc = nil
         @links = nil
         @enqueue = true
       end
 
-      def append_filter(filter = nil, &block)
+      def append_link_filter(filter = nil, &block)
         if block_given?
-          @filters << block
+          @link_filters << block
         else
-          @filters << filter
+          @link_filters << filter if filter.respond_to?(:call)
         end
       end
 
@@ -53,15 +53,15 @@ module DaimonSkycrawlers
           element["href"]
         end
         urls.uniq!
-        apply_filters(urls) || []
+        apply_link_filters(urls) || []
       end
 
-      def apply_filters(urls)
+      def apply_link_filters(urls)
         return if urls.nil?
         return if urls.empty?
         log.debug("Candidate URLs: #{urls.size}")
         urls = urls.select do |url|
-          @filters.inject(true) {|memo, filter| memo & filter.call(url) }
+          @link_filters.all? {|filter| filter.call(url) }
         end
         log.debug("Filtered URLs: #{urls.size}")
         urls

data/lib/daimon_skycrawlers/version.rb

@@ -1,3 +1,3 @@
 module DaimonSkycrawlers
-  VERSION = "0.4.0"
+  VERSION = "0.5.0"
 end

data/sample/spider/app/processors/blog_spider.rb

@@ -6,11 +6,11 @@ require "daimon_skycrawlers/filter/update_checker"
 default_processor = DaimonSkycrawlers::Processor::Default.new
 spider = DaimonSkycrawlers::Processor::Spider.new
 #spider.enqueue = false
-spider.append_filter do |url|
+spider.append_link_filter do |url|
   uri = URI(url)
   uri.host.nil? || uri.host == "www.clear-code.com"
 end
-spider.append_filter do |url|
+spider.append_link_filter do |url|
   case url
   when %r!\A(\.\./|/|#)!
     false
@@ -19,9 +19,9 @@ spider.append_filter do |url|
   end
 end
 duplicate_checker = DaimonSkycrawlers::Filter::DuplicateChecker.new(base_url: "http://www.clear-code.com/blog/")
-spider.append_filter(duplicate_checker)
+spider.append_link_filter(duplicate_checker)
 update_checker = DaimonSkycrawlers::Filter::UpdateChecker.new(base_url: "http://www.clear-code.com/blog/")
-spider.append_filter(update_checker)
+spider.append_link_filter(update_checker)
 
 DaimonSkycrawlers.register_processor(default_processor)
 DaimonSkycrawlers.register_processor(spider)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: daimon_skycrawlers
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Ryunosuke SATO
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-04 00:00:00.000000000 Z
+date: 2016-10-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -324,7 +324,11 @@ files:
 - lib/daimon_skycrawlers/filter/duplicate_checker.rb
 - lib/daimon_skycrawlers/filter/robots_txt_checker.rb
 - lib/daimon_skycrawlers/filter/update_checker.rb
+- lib/daimon_skycrawlers/generator/crawler.rb
+- lib/daimon_skycrawlers/generator/generate.rb
 - lib/daimon_skycrawlers/generator/new.rb
+- lib/daimon_skycrawlers/generator/processor.rb
+- lib/daimon_skycrawlers/generator/templates/crawler.rb.erb
 - lib/daimon_skycrawlers/generator/templates/new/Gemfile
 - lib/daimon_skycrawlers/generator/templates/new/README.md.erb
 - lib/daimon_skycrawlers/generator/templates/new/Rakefile
@@ -332,6 +336,7 @@ files:
 - lib/daimon_skycrawlers/generator/templates/new/app/processors/sample_processor.rb
 - lib/daimon_skycrawlers/generator/templates/new/config/database.yml.erb
 - lib/daimon_skycrawlers/generator/templates/new/config/init.rb
+- lib/daimon_skycrawlers/generator/templates/processor.rb.erb
 - lib/daimon_skycrawlers/logger.rb
 - lib/daimon_skycrawlers/processor.rb
 - lib/daimon_skycrawlers/processor/base.rb