daimon_skycrawlers 1.0.0.pre.rc2 → 1.0.0.pre.rc3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 648bff4fe2019f82836bcf21c75dc6e29d45abc8
-  data.tar.gz: f7691acca87a0686190171806a6a07aaa8f4015f
+  metadata.gz: e324bf361f8fca59312726836b0239decc958752
+  data.tar.gz: c6195b2168b0440a456ed2a3e1b4f18a75e65dca
 SHA512:
-  metadata.gz: 378fe0060bacead511a87f637702cd5d24ce79c5acf3cba0bfac123d9dfc2cfcde41261a813dce69f9e11126d988df609a92df126f82235610db7c5a7332539b
-  data.tar.gz: '0790c05562fc71542a1cf33ca0971bbf356659d20012f7968c6c4ffd9ec9df521f0e6d009aa8256ec05a7d1b3e494baaec187d1faa8bbd34102fb73875edcfd0'
+  metadata.gz: aaff4577b84b6842daf5a308dac8c001618c7fb34f6794bed7434ec253d15dd21b0fd9fa14d2fac2535f33507131cf15f5c950edfca369f02c92f778e79ddd17
+  data.tar.gz: 73c947c3ee5c9e60a0370273ce35128e6cb18e8606a5d8d3efc6bad12bbd3d203aeb5cf594a6eccf0a0231f9e03d1a8c47d35f64e164e0c35720e6a358c715fc

data/lib/daimon_skycrawlers/crawler/base.rb

@@ -111,6 +111,8 @@ module DaimonSkycrawlers
       # 1. Download(fetch) data from given URL
       # 1. Run post processes (store downloaded data to storage)
       #
+      # @param message [Hash] parameters for crawler
+      #
       def process(message, &block)
         @skipped = false
         @n_processed_urls += 1

data/lib/daimon_skycrawlers/filter/update_checker.rb

@@ -23,7 +23,8 @@ module DaimonSkycrawlers
       #
       def call(message, connection: nil)
         url = normalize_url(message[:url])
-        page = storage.read(url, message)
+        message[:url] = url
+        page = storage.read(message)
         return true unless page
         if connection
           response = connection.head(url)

data/lib/daimon_skycrawlers/processor/base.rb

@@ -17,6 +17,11 @@ module DaimonSkycrawlers
       include DaimonSkycrawlers::Callbacks
       include DaimonSkycrawlers::Configurable
 
+      # @!attribute [w] storage
+      #   Set storage to crawler instance.
+      #   @return [void]
+      attr_writer :storage
+
       def initialize
         super
         @skipped = false

data/lib/daimon_skycrawlers/processor/default.rb

@@ -11,8 +11,7 @@ module DaimonSkycrawlers
       # Display page information
       #
       def call(message)
-        url = message[:url]
-        page = storage.read(url, message)
+        page = storage.read(message)
         headers = JSON.parse(page.headers)
         headers_string = headers.map {|key, value| "  #{key}: #{value}" }.join("\n")
         dumped_message = <<LOG

data/lib/daimon_skycrawlers/processor/spider.rb

@@ -99,10 +99,13 @@ module DaimonSkycrawlers
       # @param message [Hash] Must have key :url, :depth
       #
       def call(message)
-        key_url = message[:url]
         depth = Integer(message[:depth] || 2)
         return if depth <= 1
-        page = storage.read(key_url, message)
+        page = storage.read(message)
+        unless page
+          log.warn("Could not read page: url=#{message[:url]}, key=#{message[:key]}")
+          return
+        end
         @doc = Nokogiri::HTML(page.body)
         new_message = {
           depth: depth - 1,
@@ -111,8 +114,8 @@ module DaimonSkycrawlers
         links.each do |url|
           enqueue_url(url, link_message)
         end
-        next_page_url = find_next_page_link
-        if next_page_link
+        next_page_url = next_page_link
+        if next_page_url
           next_page_link_message = new_message.merge(@next_page_link_message)
           enqueue_url(next_page_url, next_page_link_message)
         end
@@ -128,7 +131,7 @@ module DaimonSkycrawlers
 
       def retrieve_links
         urls = @doc.search(*link_rules).map do |element|
-          @extract_next_page_link.call(element)
+          @extract_link.call(element)
         end
         urls.uniq!
         apply_link_filters(urls) || []

data/lib/daimon_skycrawlers/storage/base.rb

@@ -29,9 +29,9 @@ module DaimonSkycrawlers
       #
       # Override this method in subclass
       #
-      # @param url [String] the key to find data in storage
+      # @param message [Hash] this hash can include `:url`, `:key` to find page
       #
-      def read(url, message = {})
+      def read(message = {})
         raise "Implement this in subclass"
       end
     end

data/lib/daimon_skycrawlers/storage/file.rb

@@ -39,9 +39,11 @@ module DaimonSkycrawlers
       #
       # Read data from files under base directory
       #
+      # @param message [Hash] this hash can include `:url`, `:key` to find page
       # @return [DaimonSkycrawlers::Storage::File::Page]
       #
-      def read(url, message)
+      def read(message)
+        url = message[:url]
         key = message[:key]
         headers = JSON.parse(headers_path(url, key).read)
         body = body_path(url, key).read

data/lib/daimon_skycrawlers/storage/null.rb

@@ -16,7 +16,7 @@ module DaimonSkycrawlers
       #
       # Read nothing
       #
-      def read(url, message = {})
+      def read(message = {})
       end
     end
   end

data/lib/daimon_skycrawlers/storage/rdb.rb

@@ -39,10 +39,10 @@ module DaimonSkycrawlers
       #
       # Fetch page identified by url
       #
-      # @param url [String] identity of the page
-      # @param message [Hash] this hash may include `:key` to find page
+      # @param message [Hash] this hash can include `:url`, `:key` to find page
       #
-      def read(url, message = {})
+      def read(message = {})
+        url = message[:url]
         key = message[:key]
         if key
           Page.where(key: key).order(updated_at: :desc).limit(1).first

data/lib/daimon_skycrawlers/version.rb

@@ -2,5 +2,5 @@ module DaimonSkycrawlers
   #
   # Version of this library
   #
-  VERSION = "1.0.0-rc2"
+  VERSION = "1.0.0-rc3"
 end

data/sample/amazon-ranking/app/processors/amazon_ranking.rb

@@ -6,8 +6,7 @@ require "daimon_skycrawlers/processor/spider"
 class AmazonRanking < DaimonSkycrawlers::Processor::Base
   Item = Struct.new(:rank, :name, :url, :star, :review)
   def call(message)
-    url = message[:url]
-    page = storage.read(url)
+    page = storage.read(message)
     doc = Nokogiri::HTML(page.body)
     ranking = []
     doc.search(".zg_itemRow").each do |item|

data/sample/itp-crawler/app/processors/itp_processor.rb

@@ -7,7 +7,7 @@ require_relative "../models/itp_shop"
 class ItpProcessor < DaimonSkycrawlers::Processor::Base
   def call(message)
     key_url = message[:url]
-    page = storage.read(key_url)
+    page = storage.read(message)
     @doc = Nokogiri::HTML(page.body.encode("UTF-8", "CP932"))
     ItpShop.transaction do
       prepare_shops do |shop|

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: daimon_skycrawlers
 version: !ruby/object:Gem::Version
-  version: 1.0.0.pre.rc2
+  version: 1.0.0.pre.rc3
 platform: ruby
 authors:
 - daimon developers
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-07 00:00:00.000000000 Z
+date: 2017-02-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -462,7 +462,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.8
+rubygems_version: 2.6.4
 signing_key: 
 specification_version: 4
 summary: This is a crawler framework.