daimon_skycrawlers 0.1.0 → 0.2.0

data/sample/spider/processor.rb

@@ -0,0 +1,34 @@
+#!/usr/bin/env ruby
+
+require "daimon_skycrawlers/processor"
+require "daimon_skycrawlers/processor/spider"
+require "daimon_skycrawlers/filter"
+require "daimon_skycrawlers/filter/duplicate_checker"
+require "daimon_skycrawlers/filter/update_checker"
+
+require_relative "./init"
+
+default_processor = DaimonSkycrawlers::Processor::Default.new
+spider = DaimonSkycrawlers::Processor::Spider.new
+#spider.enqueue = false
+spider.append_filter do |url|
+  uri = URI(url)
+  uri.host.nil? || uri.host == "www.clear-code.com"
+end
+spider.append_filter do |url|
+  case url
+  when %r!\A(\.\./|/|#)!
+    false
+  else
+    true
+  end
+end
+duplicate_checker = DaimonSkycrawlers::Filter::DuplicateChecker.new(base_url: "http://www.clear-code.com/blog/")
+spider.append_filter(duplicate_checker)
+update_checker = DaimonSkycrawlers::Filter::UpdateChecker.new(base_url: "http://www.clear-code.com/blog/")
+spider.append_filter(update_checker)
+
+DaimonSkycrawlers.register_processor(default_processor)
+DaimonSkycrawlers.register_processor(spider)
+
+DaimonSkycrawlers::Processor.run

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: daimon_skycrawlers
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Ryunosuke SATO
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-27 00:00:00.000000000 Z
+date: 2016-09-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -108,6 +108,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: timers
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -179,7 +193,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: cucumber
+  name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -193,7 +207,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: pry
+  name: tapp
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -207,7 +221,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: tapp
+  name: sqlite3
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -239,8 +253,18 @@ files:
 - db/schema.rb
 - lib/daimon_skycrawlers.rb
 - lib/daimon_skycrawlers/cli.rb
-- lib/daimon_skycrawlers/configure_songkick_queue.rb
+- lib/daimon_skycrawlers/config.rb
+- lib/daimon_skycrawlers/consumer.rb
+- lib/daimon_skycrawlers/consumer/base.rb
+- lib/daimon_skycrawlers/consumer/http_response.rb
+- lib/daimon_skycrawlers/consumer/url.rb
 - lib/daimon_skycrawlers/crawler.rb
+- lib/daimon_skycrawlers/crawler/base.rb
+- lib/daimon_skycrawlers/crawler/default.rb
+- lib/daimon_skycrawlers/filter.rb
+- lib/daimon_skycrawlers/filter/base.rb
+- lib/daimon_skycrawlers/filter/duplicate_checker.rb
+- lib/daimon_skycrawlers/filter/update_checker.rb
 - lib/daimon_skycrawlers/generator/new.rb
 - lib/daimon_skycrawlers/generator/templates/new/Gemfile
 - lib/daimon_skycrawlers/generator/templates/new/README.md.erb
@@ -249,21 +273,32 @@ files:
 - lib/daimon_skycrawlers/generator/templates/new/crawler.rb
 - lib/daimon_skycrawlers/generator/templates/new/db/migrate/create_pages.rb
 - lib/daimon_skycrawlers/generator/templates/new/enqueue.rb
+- lib/daimon_skycrawlers/generator/templates/new/init.rb
 - lib/daimon_skycrawlers/generator/templates/new/processor.rb
-- lib/daimon_skycrawlers/http_response_consumer.rb
-- lib/daimon_skycrawlers/parser.rb
-- lib/daimon_skycrawlers/parser/base.rb
-- lib/daimon_skycrawlers/parser/default.rb
+- lib/daimon_skycrawlers/logger.rb
 - lib/daimon_skycrawlers/processor.rb
+- lib/daimon_skycrawlers/processor/base.rb
 - lib/daimon_skycrawlers/processor/default.rb
+- lib/daimon_skycrawlers/processor/spider.rb
+- lib/daimon_skycrawlers/queue.rb
 - lib/daimon_skycrawlers/storage.rb
 - lib/daimon_skycrawlers/storage/base.rb
 - lib/daimon_skycrawlers/storage/null.rb
 - lib/daimon_skycrawlers/storage/rdb.rb
 - lib/daimon_skycrawlers/tasks.rb
 - lib/daimon_skycrawlers/tasks/database_tasks.rake
-- lib/daimon_skycrawlers/url_consumer.rb
+- lib/daimon_skycrawlers/timer.rb
 - lib/daimon_skycrawlers/version.rb
+- sample/spider/Gemfile
+- sample/spider/README.md
+- sample/spider/Rakefile
+- sample/spider/config/database.yml
+- sample/spider/crawler.rb
+- sample/spider/db/migrate/20160830155803_create_pages.rb
+- sample/spider/db/schema.rb
+- sample/spider/enqueue.rb
+- sample/spider/init.rb
+- sample/spider/processor.rb
 homepage: https://github.com/bm-sms/daimon-skycrawlers
 licenses:
 - MIT
@@ -284,7 +319,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.6.4
 signing_key: 
 specification_version: 4
 summary: This is a crawler framework.

data/lib/daimon_skycrawlers/configure_songkick_queue.rb

@@ -1,12 +0,0 @@
-require 'songkick_queue'
-# TODO Allow to configure from user land
-SongkickQueue.configure do |config|
-  config.logger = Logger.new(STDOUT)
-  config.host = '127.0.0.1'
-  config.port = 5672
-  # config.username = 'guest'
-  # config.password = 'guest'
-  config.vhost = '/'
-  config.max_reconnect_attempts = 10
-  config.network_recovery_interval = 1.0
-end

data/lib/daimon_skycrawlers/http_response_consumer.rb

@@ -1,38 +0,0 @@
-require "daimon_skycrawlers/processor/default"
-
-module DaimonSkycrawlers
-  class HTTPResponseConsumer
-    include SongkickQueue::Consumer
-
-    consume_from_queue 'daimon-skycrawler.http-response'
-
-    class << self
-      def register(processor = nil, &block)
-        if block_given?
-          processors << block
-        else
-          processors << processor
-        end
-      end
-
-      def processors
-        @processors ||= []
-      end
-
-      def default_processor
-        DaimonSkycrawlers::Processor::Default.new
-      end
-    end
-
-    def process(message)
-      if self.class.processors.empty?
-        processors = [self.class.default_processor]
-      else
-        processors = self.class.processors
-      end
-      processors.each do |processor|
-        processor.call(message)
-      end
-    end
-  end
-end

data/lib/daimon_skycrawlers/parser/base.rb

@@ -1,13 +0,0 @@
-module DaimonSkycrawlers
-  module Parser
-    class Base
-      def initialize(html)
-        @html = html
-      end
-
-      def parse
-        raise "Implement this method in subclass"
-      end
-    end
-  end
-end

data/lib/daimon_skycrawlers/parser/default.rb

@@ -1,50 +0,0 @@
-require "nokogiri"
-
-module DaimonSkycrawlers
-  module Parser
-    class Default < Base
-      def initialize
-        @filters = []
-      end
-
-      def append_filter(filter = nil, &block)
-        if block_given?
-          @filters << block
-        else
-          @filters << filter
-        end
-      end
-
-      def parse(html)
-        @html = html
-        @doc = Nokogiri::HTML(html)
-      end
-
-      def links
-        return @links if @links
-        @links = retrieve_links
-        @links
-      end
-
-      private
-
-      def retrieve_links
-        urls = @doc.search("a").map do |element|
-          element["href"]
-        end
-        apply_filters(urls) || []
-      end
-
-      def apply_filters(urls)
-        return if urls.nil?
-        return if urls.empty?
-        @filters.each do |filter|
-          urls = urls.select do |url|
-            filter.call(url)
-          end
-        end
-        urls
-      end
-    end
-  end
-end

data/lib/daimon_skycrawlers/parser.rb

@@ -1,7 +0,0 @@
-module DaimonSkycrawlers
-  module Parser
-  end
-end
-
-require "daimon_skycrawlers/parser/base"
-require "daimon_skycrawlers/parser/default"

data/lib/daimon_skycrawlers/url_consumer.rb

@@ -1,32 +0,0 @@
-require 'daimon_skycrawlers/crawler'
-require 'daimon_skycrawlers/processor'
-
-module DaimonSkycrawlers
-  class URLConsumer
-    include SongkickQueue::Consumer
-
-    consume_from_queue 'daimon-skycrawler.url'
-
-    class << self
-      def register(crawler)
-        crawlers << crawler
-      end
-
-      def crawlers
-        @crawlers ||= []
-      end
-    end
-
-    def process(message)
-      url = message[:url]
-      depth = message[:depth]
-      interval = message[:interval]
-
-      # XXX When several crawlers are registered, how should they behave?
-      self.class.crawlers.each do |crawler|
-        sleep(interval)
-        crawler.fetch(url, depth)
-      end
-    end
-  end
-end