daimon_skycrawlers 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f03168cc7d465dd69da00eabc00c5bd1d4654455
-  data.tar.gz: e55f902ab4f78340ee5df80499f581cefb2bad96
+  metadata.gz: 411d6b35b2c909712f9f79cd5e45440226ebba28
+  data.tar.gz: afa7bdc3cc98a28742c64ba46d7237fcce981b92
 SHA512:
-  metadata.gz: 0bd1e1e832766d27779e91bba3ae70c70d44863373cd5172d2c67e40ad6ded2dd1329a68f9652aa58fd00bac7e38fd75e3c43092fbeaec4bca42fada779b2ffd
-  data.tar.gz: 99dc774a495bfec4e8b0b0693333340fc57be9321ef9fb5d9b2d64f61856ab6defa10c1e8292dc13d90bcca4a69e80e5f2a4aeec033c7c1a345e64d2c965eaec
+  metadata.gz: f8844e9ebd88e7be7344a607866c15ea74be64a8a405f05f196d54f0125bf66be6a81c4ff27ea3ee4480a82989abbedd79cd934c122c7ac1bee5d83bbfa1b6d3
+  data.tar.gz: 37fe8376a4165aef7f29aa29af51bb637402c48d9d5327c44ea05c37dac2418c39b1f5e55303ec8c66e9cfe8eb3e77f0dc264648caa262abc551c72f06944b55

data/README.md

@@ -33,44 +33,52 @@ Or install it yourself as:
 
 1. Create project
 
-```
-$ bundle exec daimon-skycrawlers new mycrawlers
-$ cd mycrawlers
-```
+     ```
+     $ bundle exec daimon-skycrawlers new mycrawlers
+     $ cd mycrawlers
+     ```
+     or
+     ```
+     $ daimon-skycrawlers new mycrawlers
+     $ cd mycrawlers
+     ```
 
 2. Install dependencies
 
-```
-$ bundle install
-```
+     ```
+     $ bundle install
+     ```
 
 3. Create database
 
-```
-$ bundle exec rake db:create
-$ bundle exec rake db:migrate
-```
+     ```
+     $ bundle exec rake db:create
+     $ bundle exec rake db:migrate
+     ```
 
 4. Open new terminal and run crawler/processor
 
-```
-$ bundle exec ruby crawler.rb # on new terminal
-$ bundle exec ruby processor.rb # on new terminal
-```
+     ```
+     $ bin/crawler   # on new terminal
+     $ bin/processor # on new terminal
+     ```
+
+    NOTE: Execute step 5 as soon as possible. Because bin/crawler and
+    bin/processor will stop after 10 seconds by default if their
+    queues are empty.
 
 5. Enqueue task
 
-```
-$ bundle exec ruby enqueue.rb url http://example.com/
-```
+     ```
+     $ bin/enqueue url http://example.com/
+     ```
 
 6. You'll see `It works with 'http://example.com'` on your terminal which runs your processor!
-
 7. You can re-enqueue task for processor
 
-```
-$ bundle exec ruby enqueue.rb response http://example.com/
-```
+     ```
+     $ bin/enqueue response http://example.com/
+     ```
 
 Display `It works with 'http://example.com'` again on your terminal which runs your processor.
 

data/bin/daimon-skycrawlers

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 
 if File.exist?(File.expand_path("../.git", __dir__))
-  $LOAD_PATH << File.expand_path("../lib", __dir__)
+  $LOAD_PATH.unshift(File.expand_path("../lib", __dir__))
 end
 
 require "daimon_skycrawlers/cli"

data/daimon_skycrawlers.gemspec

@@ -18,16 +18,17 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
+  spec.add_dependency "bundler", "~> 1.11"
   spec.add_dependency "thor"
   spec.add_dependency "songkick_queue"
   spec.add_dependency "faraday"
   spec.add_dependency "faraday_middleware"
   spec.add_dependency "nokogiri"
   spec.add_dependency "activerecord"
+  spec.add_dependency "railties"
   spec.add_dependency "pg"
   spec.add_dependency "timers"
 
-  spec.add_development_dependency "bundler", "~> 1.11"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "test-unit"
   spec.add_development_dependency "test-unit-rr"
@@ -35,4 +36,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "pry"
   spec.add_development_dependency "tapp"
   spec.add_development_dependency "sqlite3"
+  spec.add_development_dependency "yard"
 end

data/lib/daimon_skycrawlers/consumer/http_response.rb

@@ -5,10 +5,26 @@ require "daimon_skycrawlers/processor/default"
 
 module DaimonSkycrawlers
   module Consumer
+    #
+    # HTTP response consumer class
+    #
     class HTTPResponse < Base
       include SongkickQueue::Consumer
 
       class << self
+        #
+        # Register a processor
+        #
+        # @overload register(processor)
+        #   @param [Processor] processor instance which implements `call` method
+        #   @return [void]
+        #
+        # @overload register
+        #   @return [void]
+        #   @yield [message] register given block as a processor
+        #   @yieldparam message [Hash] A message from queue
+        #   @yieldreturn [void]
+        #
         def register(processor = nil, &block)
           if block_given?
             processors << block
@@ -17,14 +33,23 @@ module DaimonSkycrawlers
           end
         end
 
+        #
+        # @private
+        #
         def processors
           @processors ||= []
         end
 
+        #
+        # @private
+        #
         def default_processor
           DaimonSkycrawlers::Processor::Default.new
         end
 
+        #
+        # @private
+        #
         def queue_name
           "#{DaimonSkycrawlers.configuration.queue_name_prefix}.http-response"
         end
@@ -32,6 +57,9 @@ module DaimonSkycrawlers
 
       consume_from_queue queue_name
 
+      #
+      # @private
+      #
       def process(message)
         if self.class.processors.empty?
           processors = [self.class.default_processor]

data/lib/daimon_skycrawlers/consumer/url.rb

@@ -4,18 +4,35 @@ require "daimon_skycrawlers/consumer/base"
 
 module DaimonSkycrawlers
   module Consumer
+    #
+    # URL consumer class
+    #
     class URL < Base
       include SongkickQueue::Consumer
 
       class << self
+        #
+        # Register a given crawler
+        #
+        # @param [Crawler] crawler instance which implements `fetch` method
+        # @return [void]
+        #
         def register(crawler)
           crawlers << crawler
         end
 
+        #
+        # Returns registered crawlers
+        #
+        # @return [Array<Crawler>]
+        #
         def crawlers
           @crawlers ||= []
         end
 
+        #
+        # @private
+        #
         def queue_name
           "#{DaimonSkycrawlers.configuration.queue_name_prefix}.url"
         end
@@ -23,6 +40,9 @@ module DaimonSkycrawlers
 
       consume_from_queue queue_name
 
+      #
+      # @private
+      #
       def process(message)
         url = message[:url]
         depth = Integer(message[:depth] || 0)

data/lib/daimon_skycrawlers/crawler/base.rb

@@ -8,12 +8,22 @@ require "daimon_skycrawlers/processor"
 
 module DaimonSkycrawlers
   module Crawler
+    #
+    # The base class of crawler
+    #
     class Base
       include DaimonSkycrawlers::LoggerMixin
       include DaimonSkycrawlers::ConfigMixin
 
+      # @!attribute [w] storage
+      #   Set storage to crawler instance.
+      #   @return [void]
       attr_writer :storage
 
+      #
+      # @param [String] Base URL for crawler
+      # @param [Hash] options for Faraday
+      #
       def initialize(base_url = nil, options = {})
         super()
         @base_url = base_url
@@ -23,6 +33,13 @@ module DaimonSkycrawlers
         @n_processed_urls = 0
       end
 
+      #
+      # Set up connection
+      #
+      # @param [Hash] options for Faraday
+      # @yield [faraday]
+      # @yieldparam faraday [Faraday]
+      #
       def setup_connection(options = {})
         @connection = Faraday.new(@base_url, @options.merge(options)) do |faraday|
           yield faraday
@@ -37,6 +54,9 @@ module DaimonSkycrawlers
         @prepare = block
       end
 
+      #
+      # Retrieve storage instance
+      #
       def storage
         @storage ||= Storage::RDB.new
       end

data/lib/daimon_skycrawlers/crawler/default.rb

@@ -3,6 +3,11 @@ require "daimon_skycrawlers/filter/update_checker"
 
 module DaimonSkycrawlers
   module Crawler
+    #
+    # The default crawler
+    #
+    # This crawler can GET given URL and store response to storage
+    #
     class Default < Base
       def fetch(path, depth: 3, **kw)
         @n_processed_urls += 1

data/lib/daimon_skycrawlers/crawler.rb

@@ -6,24 +6,50 @@ require "daimon_skycrawlers/consumer/url"
 module DaimonSkycrawlers
   module Crawler
     class << self
+      #
+      # Run registered crawlers
+      #
+      # @param process_name [String] Process name
+      #
       def run(process_name: default_process_name)
         DaimonSkycrawlers::Timer.setup_shutdown_timer(config.queue_name_prefix, interval: config.shutdown_interval)
         SongkickQueue::Worker.new(process_name, [DaimonSkycrawlers::Consumer::URL]).run
       end
 
+      #
+      # Enqueue a URL to crawler queue
+      #
+      # @param [String] Specify absolute URL
+      # @param [Hash] Extra parameters for crawler
+      # @return [void]
       def enqueue_url(url, message = {})
         message[:url] = url
         SongkickQueue.publish(queue_name, message)
       end
 
+      #
+      # Shortcut of DaimonSkycrawlers.configuration
+      #
+      # @return [DaimonSkycrawlers::Configuration]
+      #
       def config
         DaimonSkycrawlers.configuration
       end
 
+      #
+      # Queue name for crawler
+      #
+      # @return [String] Queue name
+      #
       def queue_name
         "#{config.queue_name_prefix}.url"
       end
 
+      #
+      # Default process name
+      #
+      # @return [String] Default process name
+      #
       def default_process_name
         "#{config.queue_name_prefix}:url"
       end

data/lib/daimon_skycrawlers/filter/duplicate_checker.rb

@@ -3,6 +3,9 @@ require "daimon_skycrawlers/filter/base"
 
 module DaimonSkycrawlers
   module Filter
+    #
+    # This filter provides duplication checker for given URL.
+    #
     class DuplicateChecker < Base
       def initialize(base_url: nil)
         @base_url = nil
@@ -10,6 +13,11 @@ module DaimonSkycrawlers
         @urls = Set.new
       end
 
+      #
+      # @param [String] url to check duplication. If given URL is
+      #        relative URL, use `@base_url + url` as absolute URL.
+      # @return [true|false] Return false when duplicated, otherwise return true.
+      #
       def call(url)
         unless URI(url).absolute?
           url = (@base_url + url).to_s

data/lib/daimon_skycrawlers/filter/update_checker.rb

@@ -3,6 +3,9 @@ require "daimon_skycrawlers/filter/base"
 
 module DaimonSkycrawlers
   module Filter
+    #
+    # This filter provides update checker for given URL.
+    #
     class UpdateChecker < Base
       def initialize(storage: nil, base_url: nil)
         super(storage: storage)
@@ -10,6 +13,11 @@ module DaimonSkycrawlers
         @base_url = URI(base_url) if base_url
       end
 
+      #
+      # @param [String] url
+      # @param connection [Faraday]
+      # @return [true|false] Return true when need update, otherwise return false
+      #
       def call(url, connection: nil)
         unless URI(url).absolute?
           url = (@base_url + url).to_s

data/lib/daimon_skycrawlers/generator/new.rb

@@ -1,4 +1,8 @@
 require "thor"
+require "rails/generators"
+require "rails/generators/actions"
+require "rails/generators/active_record"
+require "rails/generators/active_record/migration/migration_generator"
 
 module DaimonSkycrawlers
   module Generator
@@ -18,26 +22,45 @@ module DaimonSkycrawlers
         ].each do |path|
           template("#{path}.erb", "#{name}/#{path}")
         end
+        invoke(MigrationGenerator, [
+                 "CreatePage",
+                 "url:string",
+                 "headers:text",
+                 "body:binary",
+                 "last_modified_at:datetime",
+                 "etag:string",
+                 "timestamps"
+               ],
+          { destination_root: File.join(destination_root, name) })
       end
 
       def copy_files
         [
           "Gemfile",
           "Rakefile",
-          "crawler.rb",
-          "enqueue.rb",
-          "init.rb",
-          "processor.rb",
+          "app/crawlers/sample_crawler.rb",
+          "app/processors/sample_processor.rb",
+          "bin/crawler",
+          "bin/enqueue",
+          "bin/processor",
+          "config/init.rb",
         ].each do |path|
-          copy_file(path, "#{name}/#{path}")
-        end
-        [
-          "db/migrate/create_pages.rb",
-        ].each do |path|
-          migration = "#{Time.now.strftime("%Y%m%d%H%M%S")}_#{File.basename(path)}"
-          copy_file(path, "#{name}/db/migrate/#{migration}")
+          copy_file(path, "#{name}/#{path}", mode: :preserve)
         end
       end
     end
+
+    class MigrationGenerator < ActiveRecord::Generators::MigrationGenerator
+      def self.source_root
+        ActiveRecord::Generators::MigrationGenerator.source_root
+      end
+
+      def create_migration_file
+        set_local_assigns!
+        validate_file_name!
+        dest = options[:destination_root]
+        migration_template @migration_template, "#{dest}/db/migrate/#{file_name}.rb"
+      end
+    end
   end
 end

data/lib/daimon_skycrawlers/generator/templates/new/README.md.erb

@@ -29,14 +29,14 @@ $ bundle exec rake db:migrate
 3. Open new terminal and run crawler/processor
 
 ```
-$ bundle exec ruby crawler.rb # on new terminal
-$ bundle exec ruby processor.rb # on new terminal
+$ bin/crawler   # on new terminal
+$ bin/processor # on new terminal
 ```
 
 4. Enqueue task
 
 ```
-$ bundle exec ruby enqueue.rb http://example.com/
+$ bin/enqueue url http://example.com/
 ```
 
 5. You'll see `It works with 'http://example.com'` on your terminal which runs your processor!
@@ -44,7 +44,7 @@ $ bundle exec ruby enqueue.rb http://example.com/
 6. You can re-enqueue task for processor
 
 ```
-$ bundle exec ruby enqueue.rb response http://example.com/
+$ bin/enqueue response http://example.com/
 ```
 
 Display `It works with 'http://example.com'` again on your terminal which runs your processor.

data/lib/daimon_skycrawlers/generator/templates/new/{crawler.rb → app/crawlers/sample_crawler.rb}

@@ -1,14 +1,8 @@
-#!/usr/bin/env ruby
-
 require "daimon_skycrawlers/crawler"
 require "daimon_skycrawlers/crawler/default"
 
-require_relative "./init"
-
 base_url = "http://example.com"
 
 crawler = DaimonSkycrawlers::Crawler::Default.new(base_url)
 
 DaimonSkycrawlers.register_crawler(crawler)
-
-DaimonSkycrawlers::Crawler.run

data/lib/daimon_skycrawlers/generator/templates/new/{processor.rb → app/processors/sample_processor.rb}

@@ -1,11 +1,5 @@
-#!/usr/bin/env ruby
-
 require "daimon_skycrawlers/processor"
 
-require_relative "./init"
-
 DaimonSkycrawlers.register_processor do |data|
   p "It works with '#{data[:url]}'"
 end
-
-DaimonSkycrawlers::Processor.run

data/lib/daimon_skycrawlers/generator/templates/new/bin/crawler

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+
+require_relative "../config/init"
+require "daimon_skycrawlers/crawler"
+
+Dir.glob("app/crawlers/**/*.rb") do |path|
+  require(File.expand_path(path, File.dirname(__dir__)))
+end
+
+DaimonSkycrawlers::Crawler.run

data/{sample/spider/enqueue.rb → lib/daimon_skycrawlers/generator/templates/new/bin/enqueue}

@@ -2,11 +2,10 @@
 
 require "thor"
 
+require_relative "../config/init"
 require "daimon_skycrawlers/crawler"
 require "daimon_skycrawlers/processor"
 
-require_relative "./init"
-
 class Enqueue < Thor
   desc "url URL [key1:value1 key2:value2...]", "Enqueue URL for URL consumer"
   def url(url, *rest)

data/lib/daimon_skycrawlers/generator/templates/new/bin/processor

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+
+require_relative "../config/init"
+require "daimon_skycrawlers/processor"
+
+Dir.glob("app/processors/**/*.rb") do |path|
+  require(File.expand_path(path, File.dirname(__dir__)))
+end
+
+DaimonSkycrawlers::Processor.run

data/lib/daimon_skycrawlers/generator/templates/new/{init.rb → config/init.rb}

@@ -1,3 +1,4 @@
+require "bundler/setup"
 require "daimon_skycrawlers"
 require "daimon_skycrawlers/logger"
 require "daimon_skycrawlers/queue"

data/lib/daimon_skycrawlers/storage/file.rb

@@ -0,0 +1,46 @@
+require "daimon_skycrawlers/storage/base"
+
+module DaimonSkycrawlers
+  module Storage
+    #
+    # Storage for files
+    #
+    class File < Base
+      def initialize(base_dir)
+        super()
+        @base_dir = Pathname(base_dir)
+      end
+
+      def save(url, headers, body)
+        @base_dir.mkpath
+        body_path(url).dirname.mkpath
+        body_path(url).open("wb+") do |file|
+          file.write(body)
+        end
+        headers_path(url).open("wb+") do |file|
+          file.write(JSON.generate(headers))
+        end
+      end
+
+      def read(url)
+        headers = JSON.parse(headers_path(url).read)
+        body = body_path(url).read
+        Page.new(url, headers, body, headers["last-modified"], headers["etag"])
+      end
+
+      Page = Struct.new(:url, :headers, :body, :last_modified, :etag)
+
+      private
+
+      def body_path(url)
+        url = URI(url)
+        @base_dir + ".#{url.path}"
+      end
+
+      def headers_path(url)
+        url = URI(url)
+        Pathname("#{body_path(url)}-headers.json")
+      end
+    end
+  end
+end

data/lib/daimon_skycrawlers/storage/null.rb

@@ -1,9 +1,21 @@
 module DaimonSkycrawlers
   module Storage
+    #
+    # The null storage.
+    #
+    # This storage is useful for test.
+    #
     class Null < Base
+
+      #
+      # Save nothing
+      #
       def save(url, headers, body)
       end
 
+      #
+      # Find nothing
+      #
       def find(url)
       end
     end

data/lib/daimon_skycrawlers/storage/rdb.rb

@@ -3,13 +3,24 @@ require "active_record"
 
 module DaimonSkycrawlers
   module Storage
+    #
+    # Storage for RDBMS
+    #
     class RDB < Base
       def initialize(config_path = "config/database.yml")
+        super()
         config = YAML.load_file(config_path)
         environment = ENV["SKYCRAWLERS_ENV"] || "development"
         ActiveRecord::Base.establish_connection(config[environment])
       end
 
+      #
+      # Save
+      #
+      # @param [String] url identity of the page
+      # @param [Hash] header of URL
+      # @param [String] body
+      #
       def save(url, headers, body)
         Page.create(url: url,
                     headers: JSON.generate(headers),
@@ -18,6 +29,11 @@ module DaimonSkycrawlers
                     etag: headers["etag"])
       end
 
+      #
+      # Fetch page identified by url
+      #
+      # @param [String] url identity of the page
+      #
       def find(url)
         Page.where(url: url).order(last_modified_at: :desc).limit(1).first
       end

data/lib/daimon_skycrawlers/storage.rb

@@ -6,3 +6,4 @@ end
 require "daimon_skycrawlers/storage/base"
 require "daimon_skycrawlers/storage/rdb"
 require "daimon_skycrawlers/storage/null"
+require "daimon_skycrawlers/storage/file"

data/lib/daimon_skycrawlers/version.rb

@@ -1,3 +1,3 @@
 module DaimonSkycrawlers
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

data/lib/daimon_skycrawlers.rb

@@ -11,14 +11,38 @@ module DaimonSkycrawlers
     :shutdown_interval
   )
   class << self
+    #
+    # Register a processor
+    #
+    # @overload register_processor(processor)
+    #   @param [Processor] processor instance which implements `call` method
+    #   @return [void]
+    #
+    # @overload register_processor
+    #   @return [void]
+    #   @yield [message] Register given block as a processor.
+    #   @yieldparam message [Hash] A message from queue
+    #   @yieldreturn [void]
+    #
     def register_processor(processor = nil, &block)
       DaimonSkycrawlers::Consumer::HTTPResponse.register(processor, &block)
     end
 
+    #
+    # Register a crawler
+    #
+    # @param [Crawler] crawler instance which implements `fetch` method
+    # @return [void]
+    #
     def register_crawler(crawler)
       DaimonSkycrawlers::Consumer::URL.register(crawler)
     end
 
+    #
+    # Retrieve configuration object
+    #
+    # @return [DaimonSkycrawlers::Configuration]
+    #
     def configuration
       @configuration ||= DaimonSkycrawlers::Configuration.new.tap do |config|
         config.logger = DaimonSkycrawlers::Logger.default
@@ -28,6 +52,13 @@ module DaimonSkycrawlers
       end
     end
 
+    #
+    # Configure DaimonSkycrawlers
+    #
+    # @return [void]
+    # @yield [configuration] configure DaimonSkycrawlers
+    # @yieldparam configuration [DaimonSkycrawlers::Configuration] configuration object
+    # @yieldreturn [void]
     def configure
       yield configuration
     end

data/sample/spider/README.md

@@ -29,14 +29,14 @@ $ bundle exec rake db:migrate
 3. Open new terminal and run crawler/processor
 
 ```
-$ bundle exec ruby crawler.rb # on new terminal
-$ bundle exec ruby processor.rb # on new terminal
+$ bin/crawler # on new terminal
+$ bin/processor # on new terminal
 ```
 
 4. Enqueue task
 
 ```
-$ bundle exec ruby enqueue.rb http://example.com/
+$ bin/enqueue url http://example.com/
 ```
 
 5. You'll see `It works with 'http://example.com'` on your terminal which runs your processor!
@@ -44,7 +44,7 @@ $ bundle exec ruby enqueue.rb http://example.com/
 6. You can re-enqueue task for processor
 
 ```
-$ bundle exec ruby enqueue.rb response http://example.com/
+$ bin/enqueue response http://example.com/
 ```
 
 Display `It works with 'http://example.com'` again on your terminal which runs your processor.

data/sample/spider/{crawler.rb → app/crawlers/blog_crawler.rb}

@@ -1,14 +1,7 @@
-#!/usr/bin/env ruby
-
-require "daimon_skycrawlers/crawler"
 require "daimon_skycrawlers/crawler/default"
 
-require_relative "./init"
-
 base_url = "http://www.clear-code.com/blog/"
 
 crawler = DaimonSkycrawlers::Crawler::Default.new(base_url)
 
 DaimonSkycrawlers.register_crawler(crawler)
-
-DaimonSkycrawlers::Crawler.run

data/sample/spider/{processor.rb → app/processors/blog_spider.rb}

@@ -1,13 +1,8 @@
-#!/usr/bin/env ruby
-
-require "daimon_skycrawlers/processor"
 require "daimon_skycrawlers/processor/spider"
 require "daimon_skycrawlers/filter"
 require "daimon_skycrawlers/filter/duplicate_checker"
 require "daimon_skycrawlers/filter/update_checker"
 
-require_relative "./init"
-
 default_processor = DaimonSkycrawlers::Processor::Default.new
 spider = DaimonSkycrawlers::Processor::Spider.new
 #spider.enqueue = false
@@ -30,5 +25,3 @@ spider.append_filter(update_checker)
 
 DaimonSkycrawlers.register_processor(default_processor)
 DaimonSkycrawlers.register_processor(spider)
-
-DaimonSkycrawlers::Processor.run

data/sample/spider/bin/crawler

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+
+require_relative "../config/init"
+require "daimon_skycrawlers/crawler"
+
+Dir.glob("app/crawlers/**/*.rb") do |path|
+  require(File.expand_path(path, File.dirname(__dir__)))
+end
+
+DaimonSkycrawlers::Crawler.run

data/{lib/daimon_skycrawlers/generator/templates/new/enqueue.rb → sample/spider/bin/enqueue}

@@ -2,11 +2,10 @@
 
 require "thor"
 
+require_relative "../config/init"
 require "daimon_skycrawlers/crawler"
 require "daimon_skycrawlers/processor"
 
-require_relative "./init"
-
 class Enqueue < Thor
   desc "url URL [key1:value1 key2:value2...]", "Enqueue URL for URL consumer"
   def url(url, *rest)

data/sample/spider/bin/processor

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+require_relative "../config/init"
+
+require "daimon_skycrawlers/processor"
+
+Dir.glob("app/processors/**/*.rb") do |path|
+  require(File.expand_path(path, File.dirname(__dir__)))
+end
+
+DaimonSkycrawlers::Processor.run

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: daimon_skycrawlers
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Ryunosuke SATO
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-09-15 00:00:00.000000000 Z
+date: 2016-09-29 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -95,7 +109,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: pg
+  name: railties
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -109,7 +123,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: timers
+  name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -123,19 +137,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: timers
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.11'
-  type: :development
+        version: '0'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.11'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -234,6 +248,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: This is a crawler framework.
 email:
 - tricknotes.rs@gmail.com
@@ -269,12 +297,13 @@ files:
 - lib/daimon_skycrawlers/generator/templates/new/Gemfile
 - lib/daimon_skycrawlers/generator/templates/new/README.md.erb
 - lib/daimon_skycrawlers/generator/templates/new/Rakefile
+- lib/daimon_skycrawlers/generator/templates/new/app/crawlers/sample_crawler.rb
+- lib/daimon_skycrawlers/generator/templates/new/app/processors/sample_processor.rb
+- lib/daimon_skycrawlers/generator/templates/new/bin/crawler
+- lib/daimon_skycrawlers/generator/templates/new/bin/enqueue
+- lib/daimon_skycrawlers/generator/templates/new/bin/processor
 - lib/daimon_skycrawlers/generator/templates/new/config/database.yml.erb
-- lib/daimon_skycrawlers/generator/templates/new/crawler.rb
-- lib/daimon_skycrawlers/generator/templates/new/db/migrate/create_pages.rb
-- lib/daimon_skycrawlers/generator/templates/new/enqueue.rb
-- lib/daimon_skycrawlers/generator/templates/new/init.rb
-- lib/daimon_skycrawlers/generator/templates/new/processor.rb
+- lib/daimon_skycrawlers/generator/templates/new/config/init.rb
 - lib/daimon_skycrawlers/logger.rb
 - lib/daimon_skycrawlers/processor.rb
 - lib/daimon_skycrawlers/processor/base.rb
@@ -283,6 +312,7 @@ files:
 - lib/daimon_skycrawlers/queue.rb
 - lib/daimon_skycrawlers/storage.rb
 - lib/daimon_skycrawlers/storage/base.rb
+- lib/daimon_skycrawlers/storage/file.rb
 - lib/daimon_skycrawlers/storage/null.rb
 - lib/daimon_skycrawlers/storage/rdb.rb
 - lib/daimon_skycrawlers/tasks.rb
@@ -292,13 +322,15 @@ files:
 - sample/spider/Gemfile
 - sample/spider/README.md
 - sample/spider/Rakefile
+- sample/spider/app/crawlers/blog_crawler.rb
+- sample/spider/app/processors/blog_spider.rb
+- sample/spider/bin/crawler
+- sample/spider/bin/enqueue
+- sample/spider/bin/processor
 - sample/spider/config/database.yml
-- sample/spider/crawler.rb
+- sample/spider/config/init.rb
 - sample/spider/db/migrate/20160830155803_create_pages.rb
 - sample/spider/db/schema.rb
-- sample/spider/enqueue.rb
-- sample/spider/init.rb
-- sample/spider/processor.rb
 homepage: https://github.com/bm-sms/daimon-skycrawlers
 licenses:
 - MIT

data/lib/daimon_skycrawlers/generator/templates/new/db/migrate/create_pages.rb

@@ -1,13 +0,0 @@
-class CreatePages < ActiveRecord::Migration
-  def change
-    create_table :pages do |t|
-      t.string :url
-      t.text :headers
-      t.binary :body, limit: 10 * 1024 ** 2 # 10MiB
-      t.datetime :last_modified_at
-      t.string :etag
-
-      t.timestamps null: false
-    end
-  end
-end