daimon_skycrawlers 0.9.0 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2fd637d109812fe657a771536f3c3a0041e4868a
-  data.tar.gz: 031bf50b2b72e6320ee748cb9044e177f24f5d2f
+  metadata.gz: 58dd7f91f6e9da8f9388a3364731ab0a543c01cd
+  data.tar.gz: b815a1cdad154eaf1b828568a697887df02bfcb6
 SHA512:
-  metadata.gz: 59f6404ea231ecc337b1658406daa8c1e3b0d557ddc3d46c37efa91b1f7d3e0f8018d524aa3c2cc227fbee57d04ef51d7561ae879fd5d0fc719f4fca1308afb3
-  data.tar.gz: eeb06f74ae722a6cdf18d00f0aeda3e06e06199e02ddd2f0e52fedb3c7a7028c4a99e46a8c19d2a4e6aa0c3979f0602757d5598a5e1de39112457d1b5dce5679
+  metadata.gz: 1ffbdd022a6e6a3a80d292bae8927a716f33f339e19e12f67da536812f9118615be0d8886fc143b529b7854f3b4ef0731962f3102aa6c759790d86dd2df54914
+  data.tar.gz: 1beb4753e8602224c95081651a4da6276659bfe132cdb9dc5fe55e8ae9f2c524681bd9be8137d4e29d52380d0f137ea2630097337bc07aea31de55e3f2785d3c

data/lib/daimon_skycrawlers/processor/spider.rb

@@ -3,8 +3,44 @@ require "daimon_skycrawlers/crawler"
 
 module DaimonSkycrawlers
   module Processor
+    #
+    # Web spider class.
+    # By default extract all links and follow.
+    #
+    # @example Google search result (2016-11-29)
+    #   spider = DaimonSkycrawlers::Processor::Spider.new
+    #   spider.configure do |s|
+    #     s.link_rules = ".g .r a"
+    #     s.extract_link do |element|
+    #       element["data-href"]
+    #     end
+    #     s.link_message = { next: "detail" }
+    #     s.next_page_link_rules = "a#pnnext"
+    #     s.next_page_link_message = { next: "spider" }
+    #   end
+    #
     class Spider < Base
-      attr_accessor :enqueue
+      # @!attribute [rw] enqueue
+      #   If true enqueue found links
+      #
+      # @!attribute [rw] link_rules
+      #   same as Nokogiri::XML::DocumentFragment#search
+      #   In generally, we can set XPath or CSS selector.
+      #
+      # @!attribute [rw] next_page_link_rules
+      #   same as Nokogiri::XML::DocumentFragment#search
+      #   In generally, we can set XPath or CSS selector.
+      #
+      attr_accessor :enqueue, :link_rules, :next_page_link_rules
+
+      # @!attribute [w] link_message
+      #   Specify hash literal to propagate arbitrary data next crawler/processor.
+      #   This is for filtering message before crawler/processor processes the message.
+      #
+      # @!attribute [w] next_page_link_message
+      #   Specify hash literal to propagate arbitrary data next crawler/processor.
+      #   This is for filtering message before crawler/processor processes the message.
+      attr_writer :link_message, :next_page_link_message
 
       def initialize
         super
@@ -12,8 +48,31 @@ module DaimonSkycrawlers
         @doc = nil
         @links = nil
         @enqueue = true
+        @link_rules = ["a"]
+        @extract_link = ->(element) { element["href"] }
+        @link_message = {}
+        @next_page_link_rules = nil
+        @extract_next_page_link = ->(element) { element["href"] }
+        @next_page_link_message = {}
       end
 
+      #
+      # Configure spider instance
+      #
+      # @return [DaimonSkycrawlers::Processor::Spider] self
+      #
+      def configure
+        yield self
+        self
+      end
+
+      #
+      # Append filter to reduce links found by link_rules
+      #
+      # @param filter [Object] Filter object that has call method
+      # @yield [message] Similar to Array#select
+      # @yieldparam message [Hash]
+      #
       def append_link_filter(filter = nil, &block)
         if block_given?
           @link_filters << block
@@ -22,6 +81,30 @@ module DaimonSkycrawlers
         end
       end
 
+      #
+      # Register block to process element found by DaimonSkycrawlers::Processor::Spider#link_rules
+      #
+      # @yield [element]
+      # @yieldparam element [Object]
+      # @example Default
+      #   ->(element) { element["href"] }
+      #
+      def extract_link(&block)
+        @extract_link = block
+      end
+
+      #
+      # Register block to process element found by DaimonSkycrawlers::Processor::Spider#next_page_link_rules
+      #
+      # @yield [element]
+      # @yieldparam element [Object]
+      # @example Default
+      #   ->(element) { element["href"] }
+      #
+      def extract_next_page_link(&block)
+        @extract_next_page_link = block
+      end
+
       #
       # @param [Hash] message Must have key :url, :depth
       #
@@ -35,8 +118,14 @@ module DaimonSkycrawlers
         new_message = {
           depth: depth - 1,
         }
+        link_message = new_message.merge(@link_message)
         links.each do |url|
-          enqueue_url(url, new_message)
+          enqueue_url(url, link_message)
+        end
+        next_page_url = find_next_page_link
+        if next_page_link
+          next_page_link_message = new_message.merge(@next_page_link_message)
+          enqueue_url(next_page_url, next_page_link_message)
         end
       end
 
@@ -49,13 +138,20 @@ module DaimonSkycrawlers
       end
 
       def retrieve_links
-        urls = @doc.search("a").map do |element|
-          element["href"]
+        urls = @doc.search(*link_rules).map do |element|
+          @extract_next_page_link.call(element)
         end
         urls.uniq!
         apply_link_filters(urls) || []
       end
 
+      def next_page_link
+        return unless next_page_link_rules
+        element = @doc.at(*next_page_link_rules)
+        return unless element
+        @extract_next_page_link.call(element)
+      end
+
       def apply_link_filters(urls)
         return if urls.nil?
         return if urls.empty?

data/lib/daimon_skycrawlers/version.rb

@@ -1,3 +1,3 @@
 module DaimonSkycrawlers
-  VERSION = "0.9.0"
+  VERSION = "0.10.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: daimon_skycrawlers
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 0.10.0
 platform: ruby
 authors:
 - daimon developers
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-11-24 00:00:00.000000000 Z
+date: 2016-11-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -407,7 +407,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.4
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: This is a crawler framework.