daimon_skycrawlers 0.11.3 → 0.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d26cd9bc437f67de677b2fd929b3df34d5715736
-  data.tar.gz: e2a01ec217a9eac95cd23025286782844d8fc30d
+  metadata.gz: ea06b539d343f1a13e2eb22f3cd2e9662ce3d9da
+  data.tar.gz: 39164028e5902f05a86a4c1e1f1890bbe8065dfa
 SHA512:
-  metadata.gz: d34053642471b25522a768507867d2c21ce4ee5a653c5ab886cdb8c42bfd62558770db354b3c74bc57f14bc9241e3a87488ad65c3634f4dc76705fdf9e1e7c31
-  data.tar.gz: 47937c94c61d1d2a71e8b2c6d5844aaa6dc1f15ac48e450453c1ed54941cd45ff900458de10382d74761ca495be10a7cbe8f1c6dc751b84b8a4925621c7ce965
+  metadata.gz: 22472b47e7a5e87758c2bf8712bd76d227e7c6589afd641060e93e8265fdb628abf8a236dbcf1dab0dc6390f0fd6fb680e96ff4998c4ba479f4a5ff334b15a4f
+  data.tar.gz: 8bdf2df5606221dcb984ab3850b8922f524dfbaf12d80cb38c11f65632327d0567b6c17aaff7a51da47e77e88fa27e360045dbdf38af2f3dc02cec3384f15090

data/README.md

@@ -2,6 +2,9 @@
 
 # DaimonSkycrawlers
 
+[![Gem Version](https://badge.fury.io/rb/daimon_skycrawlers.svg)](https://badge.fury.io/rb/daimon_skycrawlers)
+[![Build Status](https://travis-ci.org/bm-sms/daimon_skycrawlers.svg?branch=master)](https://travis-ci.org/bm-sms/daimon_skycrawlers)
+
 DaimonSkycrawlers is a crawler framework.
 
 ## Requirements
@@ -63,8 +66,8 @@ Or install it yourself as:
     $ daimon_skycrawlers exec processor # on new terminal
     ```
 
-    NOTE: Execute step 5 as soon as possible. Because bin/crawler and
-    bin/processor will stop after 10 seconds by default if their
+    NOTE: Execute step 5 as soon as possible. Because crawler and
+    processor will stop after 10 seconds by default if their
     queues are empty.
 
     NOTE: You can change `shutdown_interval` using following code in config/init.rb:

data/lib/daimon_skycrawlers.rb

@@ -75,6 +75,20 @@ module DaimonSkycrawlers
       exit(false)
     end
 
+    def load_crawlers
+      Dir.glob("app/crawlers/**/*.rb") do |path|
+        require(File.expand_path(path, Dir.pwd)) &&
+          DaimonSkycrawlers.configuration.logger.info("Loaded crawler: #{path}")
+      end
+    end
+
+    def load_processors
+      Dir.glob("app/processors/**/*.rb") do |path|
+        require(File.expand_path(path, Dir.pwd)) &&
+          DaimonSkycrawlers.configuration.logger.info("Loaded processor: #{path}")
+      end
+    end
+
     #
     # Return current environment
     #

data/lib/daimon_skycrawlers/commands/runner.rb

@@ -10,10 +10,8 @@ module DaimonSkycrawlers
       desc "crawler", "Execute crawler"
       def crawler
         load_init
-        Dir.glob("app/crawlers/**/*.rb") do |path|
-          require(File.expand_path(path, Dir.pwd))
-          log.info("Loaded crawler: #{path}")
-        end
+        load_crawlers
+        require(File.expand_path("app/crawler.rb", Dri.pwd))
         DaimonSkycrawlers::Crawler.run
       rescue => ex
         puts ex.message
@@ -23,10 +21,8 @@ module DaimonSkycrawlers
       desc "processor", "Execute processor"
       def processor
         load_init
-        Dir.glob("app/processors/**/*.rb") do |path|
-          require(File.expand_path(path, Dir.pwd))
-          log.info("Loaded processor: #{path}")
-        end
+        load_processors
+        require(File.expand_path("app/processor.rb", Dir.pwd))
         DaimonSkycrawlers::Processor.run
       rescue => ex
         puts ex.message
@@ -39,6 +35,14 @@ module DaimonSkycrawlers
         DaimonSkycrawlers.load_init
       end
 
+      def load_crawlers
+        DaimonSkycrawlers.load_crawlers
+      end
+
+      def load_processors
+        DaimonSkycrawlers.load_processors
+      end
+
       def log
         DaimonSkycrawlers.configuration.logger
       end

data/lib/daimon_skycrawlers/generator/crawler.rb

@@ -17,6 +17,19 @@ module DaimonSkycrawlers
         }
         template("crawler.rb.erb", "app/crawlers/#{name.underscore}.rb", config)
       end
+
+      def display_post_message
+        puts <<MESSAGE
+
+You can register your crawler in `app/crawler.rb` to run your crawler.
+Following code snippet is useful:
+
+    base_url = "https://www.example.com/"
+    crawler = #{name.classify}.new(base_url)
+    DaimonSkycrawlers.register_crawler(crawler)
+
+MESSAGE
+      end
     end
   end
 end

data/lib/daimon_skycrawlers/generator/filter.rb

@@ -0,0 +1,34 @@
+require "thor"
+
+module DaimonSkycrawlers
+  module Generator
+    class Filter < Thor::Group
+      include Thor::Actions
+
+      argument :name
+
+      def self.source_root
+        File.join(__dir__, "templates")
+      end
+
+      def create_files
+        config = {
+          class_name: name.classify,
+        }
+        template("filter.rb.erb", "app/filters/#{name.underscore}.rb", config)
+      end
+
+      def display_post_message
+        puts <<MESSAGE
+
+You can use this filter with both crawlers and processors.
+
+    filter = #{name.classify}.new
+    crawler = DaimonSkycrawlers::Crawler::Default.new
+    crawler.before_process(filter)
+
+MESSAGE
+      end
+    end
+  end
+end

data/lib/daimon_skycrawlers/generator/generate.rb

@@ -1,12 +1,14 @@
 require "thor"
 require "daimon_skycrawlers/generator/crawler"
 require "daimon_skycrawlers/generator/processor"
+require "daimon_skycrawlers/generator/filter"
 
 module DaimonSkycrawlers
   module Generator
     class Generate < Thor
       register(Crawler, "crawler", "crawler NAME", "Generate new crawler")
       register(Processor, "processor", "processor NAME", "Generate new processor")
+      register(Filter, "filter", "filter NAME", "Generate new filter")
     end
   end
 end

data/lib/daimon_skycrawlers/generator/new.rb

@@ -70,9 +70,8 @@ module DaimonSkycrawlers
           "Dockerfile.db",
           "Gemfile",
           "Rakefile",
-          "app/crawlers/sample_crawler.rb",
-          "app/filters/sample_filter.rb",
-          "app/processors/sample_processor.rb",
+          "app/crawler.rb",
+          "app/processor.rb",
           "config/init.rb",
           "services/common/docker-entrypoint.sh",
           "services/db/init-user-db.sh"
@@ -83,6 +82,9 @@ module DaimonSkycrawlers
 
       def create_directories
         [
+          "app/crawlers",
+          "app/filters",
+          "app/processors",
           "vendor/bundle",
           "docker-cache/bundle",
           "docker-cache/.bundle"

data/lib/daimon_skycrawlers/generator/processor.rb

@@ -17,6 +17,18 @@ module DaimonSkycrawlers
         }
         template("processor.rb.erb", "app/processors/#{name.underscore}.rb", config)
       end
+
+      def display_post_message
+        puts <<MESSAGE
+
+You can register your processor in `app/processor.rb` to run your processor.
+Following code snippet is useful:
+
+    processor = #{naem.classify}.new
+    DaimonSkycrawlers.register_processor(processor)
+
+MESSAGE
+      end
     end
   end
 end

data/lib/daimon_skycrawlers/generator/templates/crawler.rb.erb

@@ -7,7 +7,3 @@ class <%= config[:class_name] %> < DaimonSkycrawlers::Crawler::Base
     # Implement your crawler here
   end
 end
-
-base_url = ""
-crawler = <%= config[:class_name] %>.new(base_url)
-DaimonSkycrawlers.register_crawler(crawler)

data/{sample/amazon-ranking/app/filters/sample_filter.rb → lib/daimon_skycrawlers/generator/templates/filter.rb.erb}

@@ -1,6 +1,6 @@
 require "daimon_skycrawlers/filter/base"
 
-class SampleFilter < DaimonSkycrawlers::Filter::Base
+class <%= config[:class_name] %> < DaimonSkycrawlers::Filter::Base
   def call(message)
     # Imprement your filter here.
     # If you want to crawl `url`, return true otherwise false.

data/{sample/amazon-ranking/app/crawlers/sample_crawler.rb → lib/daimon_skycrawlers/generator/templates/new/app/crawler.rb}

@@ -1,6 +1,8 @@
 require "daimon_skycrawlers/crawler"
 require "daimon_skycrawlers/crawler/default"
 
+DaimonSkycrawlers.load_crawlers
+
 base_url = "http://example.com"
 
 crawler = DaimonSkycrawlers::Crawler::Default.new(base_url)

data/lib/daimon_skycrawlers/generator/templates/new/app/{processors/sample_processor.rb → processor.rb}

@@ -1,5 +1,7 @@
 require "daimon_skycrawlers/processor"
 
+DaimonSkycrawlers.load_processors
+
 DaimonSkycrawlers.register_processor do |data|
   p "It works with '#{data[:url]}'"
 end

data/lib/daimon_skycrawlers/generator/templates/processor.rb.erb

@@ -7,6 +7,3 @@ class <%= config[:class_name] %> < DaimonSkycrawlers::Processor::Base
     # Implement your processor here
   end
 end
-
-processor = <%= config[:class_name] %>.new
-DaimonSkycrawlers.register_processor(processor)

data/lib/daimon_skycrawlers/processor/spider.rb

@@ -147,7 +147,7 @@ module DaimonSkycrawlers
         return if urls.empty?
         log.debug("Candidate URLs: #{urls.size}")
         urls = urls.select do |url|
-          @link_filters.all? {|filter| filter.call(url) }
+          @link_filters.all? {|filter| filter.call(url: url) }
         end
         log.debug("Filtered URLs: #{urls.size}")
         urls

data/lib/daimon_skycrawlers/version.rb

@@ -1,3 +1,3 @@
 module DaimonSkycrawlers
-  VERSION = "0.11.3"
+  VERSION = "0.12.0"
 end

data/sample/amazon-ranking/app/processor.rb

@@ -0,0 +1,25 @@
+require "daimon_skycrawlers"
+require "daimon_skycrawlers/processor"
+
+DaimonSkycrawlers.load_processors
+
+DaimonSkycrawlers.register_processor do |data|
+  p "It works with '#{data[:url]}'"
+end
+
+spider = DaimonSkycrawlers::Processor::Spider.new
+spider.configure do |s|
+  s.link_rules = "ul#zg_browseRoot li a"
+  s.link_message = { next_processor: "AmazonRanking" }
+  s.before_process do |message|
+    message[:next_processor] != "AmazonRanking"
+  end
+end
+DaimonSkycrawlers.register_processor(spider)
+
+processor = AmazonRanking.new.configure do |s|
+  s.before_process do |message|
+    message[:next_processor] == "AmazonRanking"
+  end
+end
+DaimonSkycrawlers.register_processor(processor)

data/sample/amazon-ranking/app/processors/amazon_ranking.rb

@@ -19,20 +19,3 @@ class AmazonRanking < DaimonSkycrawlers::Processor::Base
     p ranking
   end
 end
-
-spider = DaimonSkycrawlers::Processor::Spider.new
-spider.configure do |s|
-  s.link_rules = "ul#zg_browseRoot li a"
-  s.link_message = { next_processor: "AmazonRanking" }
-  s.before_process do |message|
-    message[:next_processor] != "AmazonRanking"
-  end
-end
-DaimonSkycrawlers.register_processor(spider)
-
-processor = AmazonRanking.new.configure do |s|
-  s.before_process do |message|
-    message[:next_processor] == "AmazonRanking"
-  end
-end
-DaimonSkycrawlers.register_processor(processor)

data/sample/itp-crawler/app/{crawlers/itp_crawler.rb → crawler.rb}

@@ -2,6 +2,7 @@ require "daimon_skycrawlers"
 require "daimon_skycrawlers/crawler"
 require "daimon_skycrawlers/crawler/default"
 
+DaimonSkycrawlers.load_crawlers
 
 base_url = "http://itp.ne.jp/"
 crawler = DaimonSkycrawlers::Crawler::Default.new(base_url)

data/sample/itp-crawler/app/processor.rb

@@ -0,0 +1,8 @@
+require "daimon_skycrawlers"
+require "daimon_skycrawlers/processor"
+require "daimon_skycrawlers/processor/base"
+
+DaimonSkycrawlers.load_processors
+
+processor = ItpProcessor.new
+DaimonSkycrawlers.register_processor(processor)

data/sample/itp-crawler/app/processors/itp_processor.rb

@@ -90,6 +90,3 @@ class ItpProcessor < DaimonSkycrawlers::Processor::Base
     end
   end
 end
-
-processor = ItpProcessor.new
-DaimonSkycrawlers.register_processor(processor)

data/sample/spider/app/{crawlers/blog_crawler.rb → crawler.rb}

@@ -1,5 +1,7 @@
 require "daimon_skycrawlers/crawler/default"
 
+DaimonSkycrawlers.load_crawlers
+
 base_url = "http://www.clear-code.com/blog/"
 
 crawler = DaimonSkycrawlers::Crawler::Default.new(base_url)

data/sample/spider/app/{processors/blog_spider.rb → processor.rb}

@@ -3,6 +3,8 @@ require "daimon_skycrawlers/filter"
 require "daimon_skycrawlers/filter/duplicate_checker"
 require "daimon_skycrawlers/filter/update_checker"
 
+DaimonSkycrawlers.load_processors
+
 default_processor = DaimonSkycrawlers::Processor::Default.new
 spider = DaimonSkycrawlers::Processor::Spider.new
 #spider.enqueue = false

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: daimon_skycrawlers
 version: !ruby/object:Gem::Version
-  version: 0.11.3
+  version: 0.12.0
 platform: ruby
 authors:
 - daimon developers
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-12-08 00:00:00.000000000 Z
+date: 2016-12-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -340,18 +340,19 @@ files:
 - lib/daimon_skycrawlers/filter/robots_txt_checker.rb
 - lib/daimon_skycrawlers/filter/update_checker.rb
 - lib/daimon_skycrawlers/generator/crawler.rb
+- lib/daimon_skycrawlers/generator/filter.rb
 - lib/daimon_skycrawlers/generator/generate.rb
 - lib/daimon_skycrawlers/generator/new.rb
 - lib/daimon_skycrawlers/generator/processor.rb
 - lib/daimon_skycrawlers/generator/templates/crawler.rb.erb
+- lib/daimon_skycrawlers/generator/templates/filter.rb.erb
 - lib/daimon_skycrawlers/generator/templates/new/Dockerfile
 - lib/daimon_skycrawlers/generator/templates/new/Dockerfile.db
 - lib/daimon_skycrawlers/generator/templates/new/Gemfile
 - lib/daimon_skycrawlers/generator/templates/new/README.md.erb
 - lib/daimon_skycrawlers/generator/templates/new/Rakefile
-- lib/daimon_skycrawlers/generator/templates/new/app/crawlers/sample_crawler.rb
-- lib/daimon_skycrawlers/generator/templates/new/app/filters/sample_filter.rb
-- lib/daimon_skycrawlers/generator/templates/new/app/processors/sample_processor.rb
+- lib/daimon_skycrawlers/generator/templates/new/app/crawler.rb
+- lib/daimon_skycrawlers/generator/templates/new/app/processor.rb
 - lib/daimon_skycrawlers/generator/templates/new/config/database.yml.erb
 - lib/daimon_skycrawlers/generator/templates/new/config/init.rb
 - lib/daimon_skycrawlers/generator/templates/new/docker-compose.yml.erb
@@ -385,10 +386,10 @@ files:
 - sample/amazon-ranking/Gemfile.lock
 - sample/amazon-ranking/README.md
 - sample/amazon-ranking/Rakefile
-- sample/amazon-ranking/app/crawlers/sample_crawler.rb
-- sample/amazon-ranking/app/filters/sample_filter.rb
+- sample/amazon-ranking/app/crawler.rb
+- sample/amazon-ranking/app/crawlers/.gitkeep
+- sample/amazon-ranking/app/processor.rb
 - sample/amazon-ranking/app/processors/amazon_ranking.rb
-- sample/amazon-ranking/app/processors/sample_processor.rb
 - sample/amazon-ranking/config/database.yml
 - sample/amazon-ranking/config/init.rb
 - sample/amazon-ranking/db/migrate/20161206061241_create_pages.rb
@@ -400,9 +401,11 @@ files:
 - sample/itp-crawler/Gemfile.lock
 - sample/itp-crawler/README.md
 - sample/itp-crawler/Rakefile
-- sample/itp-crawler/app/crawlers/itp_crawler.rb
+- sample/itp-crawler/app/crawler.rb
+- sample/itp-crawler/app/crawlers/.gitkeep
 - sample/itp-crawler/app/models/itp_base.rb
 - sample/itp-crawler/app/models/itp_shop.rb
+- sample/itp-crawler/app/processor.rb
 - sample/itp-crawler/app/processors/itp_processor.rb
 - sample/itp-crawler/config/database.yml
 - sample/itp-crawler/config/database_itp.yml
@@ -416,8 +419,10 @@ files:
 - sample/spider/Gemfile
 - sample/spider/README.md
 - sample/spider/Rakefile
-- sample/spider/app/crawlers/blog_crawler.rb
-- sample/spider/app/processors/blog_spider.rb
+- sample/spider/app/crawler.rb
+- sample/spider/app/crawlers/.gitkeep
+- sample/spider/app/processor.rb
+- sample/spider/app/processors/.gitkeep
 - sample/spider/config/database.yml
 - sample/spider/config/init.rb
 - sample/spider/db/migrate/20160830155803_create_pages.rb

data/lib/daimon_skycrawlers/generator/templates/new/app/filters/sample_filter.rb

@@ -1,9 +0,0 @@
-require "daimon_skycrawlers/filter/base"
-
-class SampleFilter < DaimonSkycrawlers::Filter::Base
-  def call(message)
-    # Imprement your filter here.
-    # If you want to crawl `url`, return true otherwise false.
-    true
-  end
-end

data/sample/amazon-ranking/app/processors/sample_processor.rb

@@ -1,5 +0,0 @@
-require "daimon_skycrawlers/processor"
-
-DaimonSkycrawlers.register_processor do |data|
-  p "It works with '#{data[:url]}'"
-end