micro_spider 0.1.19 → 0.1.20

data/lib/micro_spider.rb

@@ -35,13 +35,14 @@ class MicroSpider
   attr_accessor :logger, :actions, :recipe, :skip_set_entrance, :timeout
 
   def initialize(excretion = nil)
-    @paths   = []
-    @actions = []
-    @timeout = 120
-    @excretion = excretion || { status: 'inprogress', results: [] }
-    @logger        = Logger.new(STDOUT)
-    @visited_paths = Set.new
-    @broken_paths  = []
+    @paths            = []
+    @actions          = []
+    @setted_variables = {}
+    @timeout          = 120
+    @excretion        = excretion || { status: 'inprogress', results: [] }
+    @logger           = Logger.new(STDOUT)
+    @visited_paths    = Set.new
+    @broken_paths     = []
   end
 
   # The seconds between each two request.
@@ -70,6 +71,31 @@ class MicroSpider
     logger.info "Current location is #{path}."
   end
 
+  # Set a variable. You can use it later.
+  #
+  # @param name [String]  the variable name
+  # @param value [String] the variable value
+  # @param opts [Hash] the options. can set selector with css or xpath
+  #
+  # @example Set a variable
+  #   spider = MicroSpider.new
+  #   spider.set :id, '645'
+  #   spider.set :table, '.tb a', selector: :css
+  #   spider.set :table, '.tb a', selector: :css do |e|
+  #     e['src']
+  #   end
+  def set(name, value, opts = {}, &block)
+    selector = opts.delete(:selector)
+    if selector.nil?
+      @setted_variables[name.to_s] = value
+    else
+      actions << lambda {
+        elements = scan_all(selector, value, opts)
+        @setted_variables[name.to_s] = block_given? ? yield(elements) : handle_element(elements.first)
+      }
+    end
+  end
+
   # Click the locator. This will trigger visit action and change current location.
   # @params locator [String] the text or id of the link.
   #
@@ -129,7 +155,7 @@ class MicroSpider
     return if @site
     Capybara.app_host = @excretion[:site] = @site = url
   end
-  
+
   # This will be the first path for spider to visit.
   # If more than one entrance, the spider will crawl theme one by one.
   # @param path_or_paths [String] one or more entrances
@@ -248,7 +274,7 @@ class MicroSpider
     spider.instance_variable_set(:@paths, [])
     spider.instance_variable_set(:@actions, [])
     spider.instance_variable_set(:@visited_paths, Set.new)
-    spider.instance_variable_set(:@broken_paths, Set.new)
+    spider.instance_variable_set(:@broken_paths,  Set.new)
     spider.instance_variable_set(:@excretion, { status: 'inprogress', results: [] })
     spider.skip_set_entrance = false
     spider

data/lib/spider_core/behavior.rb

@@ -6,6 +6,7 @@ module SpiderCore
     protected
 
     def scan_all(kind, pattern, opts = {})
+      pattern = handle_pattern(pattern)
       if pattern.is_a?(String)
         elements = all(kind, pattern).lazy
         if opts[:limit] && opts[:limit].to_i > 0
@@ -18,6 +19,7 @@ module SpiderCore
     end
 
     def scan_first(kind, pattern)
+      pattern = handle_pattern(pattern)
       if pattern.is_a?(String)
         first(kind, pattern)
       elsif pattern.is_a?(Regexp)
@@ -25,5 +27,39 @@ module SpiderCore
       end
     end
 
+    def handle_element(element)
+      if element.is_a?(String)
+        element
+      elsif element.tag_name == 'input'
+        element.value
+      else
+        element.text
+      end
+    end
+
+    def handle_elements(elements, &block)
+      if elements.respond_to?(:map) && block_given?
+        elements.map { |element| yield(element) }.force
+      elsif elements.respond_to?(:map)
+        elements.map { |element| handle_element(element) }.force
+      elsif block_given?
+        yield(elements)
+      else
+        handle_element(elements)
+      end
+    end
+
+    # @example Handle pattern
+    #   handle_pattern('.a') # =>'.a'
+    #   set :id, 'a'
+    #   handle_pattern('.%{id}bc') # =>'.abc'
+    def handle_pattern(pattern)
+      scan_results = pattern.scan(/(?<=%{)[^}]*(?=})/)
+      unless scan_results.empty?
+        scan_results.each { |v| pattern = pattern.sub(/%\{#{v}\}/, @setted_variables[v]) }
+      end
+      pattern
+    end
+
   end
 end

data/lib/spider_core/field_dsl.rb

@@ -35,27 +35,6 @@ module SpiderCore
     end
 
     protected
-    def handle_element(element)
-      if element.is_a?(String)
-        element
-      elsif element.tag_name == 'input'
-        element.value
-      else
-        element.text
-      end
-    end
-
-    def handle_elements(elements, &block)
-      if elements.respond_to?(:map) && block_given?
-        elements.map { |element| yield(element) }.force
-      elsif elements.respond_to?(:map)
-        elements.map { |element| handle_element(element) }.force
-      elsif block_given?
-        yield(elements)
-      else
-        handle_element(elements)
-      end
-    end
 
     def action_for(action, action_opts = {}, opts = {}, &block)
       begin

data/lib/spider_core/version.rb

@@ -1,3 +1,3 @@
 module SpiderCore
-  VERSION = "0.1.19"
+  VERSION = "0.1.20"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: micro_spider
 version: !ruby/object:Gem::Version
-  version: 0.1.19
+  version: 0.1.20
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-09-12 00:00:00.000000000 Z
+date: 2013-09-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: capybara