yasuri 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 39e4792feb25676d0f6e6f524d93ece20fb83530
-  data.tar.gz: fefe743c8e4807b69535f998e53d71784685aff6
+  metadata.gz: 61a2aa3974c697ecc14b991521961ec54f3ff5c0
+  data.tar.gz: 03cf5b02e7a646175183725d38d50a9538df7eed
 SHA512:
-  metadata.gz: e532b3ad29681aebaec403bec9c34bcba96f14e6f5b8b6909d89d00938ec1988a4a26396a2fc058abe30a8f04e1c2fdfb8f755daee868c3b9eff6d4daf6c79a2
-  data.tar.gz: 469ac8b9a30715322e2efab60b2665b3da6eab66666bfbfc7af3cfd05d75750aa47e26b8763a373c5691395c93af5a7b35a71704358cf35f96e8662880ada5de
+  metadata.gz: 2108d6b78c8704fa4d99491c1684dfd686c656d7bd039b8c80e52fa2150837958c25f37afabdcf198b10153b2f32216408735182fb0d58b3007f4808ac2226c9
+  data.tar.gz: c956e589ab7676e844110870e18c8f842afb6fd566a5ddd4758ec9e328961ff1b485bac9eebecdac6456f7ed28a9793d9dd4dd9bc2f89dd8f9131003c6db9a5d

data/lib/yasuri/version.rb

@@ -1,3 +1,3 @@
 module Yasuri
-  VERSION = "0.0.5"
+  VERSION = "0.0.6"
 end

data/lib/yasuri/yasuri.rb

@@ -10,7 +10,7 @@ module Yasuri
   module Node
     attr_reader :url, :xpath, :name
 
-    def initialize(xpath, name, children = [])
+    def initialize(xpath, name, children = [], opt: {})
       @xpath, @name, @children = xpath, name, children
     end
 
@@ -21,9 +21,9 @@ module Yasuri
 
   class TextNode
     include Node
-    def initialize(xpath, name, children = [])
+    def initialize(xpath, name, children = [], truncate_regexp: nil, opt: {})
       super(xpath, name, children)
-      @truncate_regexp, dummy = *children
+      @truncate_regexp = truncate_regexp
     end
     def inject(agent, page, retry_count = 5)
       node = page.search(@xpath)
@@ -68,9 +68,9 @@ module Yasuri
   class PaginateNode
     include Node
 
-    def initialize(xpath, name, children = [], limit = Float::INFINITY)
+    def initialize(xpath, name, children = [], limit: nil, opt: {})
       super(xpath, name, children)
-      @limit = limit
+      @limit = limit || opt["limit"] || Float::MAX
     end
 
     def inject(agent, page, retry_count = 5)
@@ -120,8 +120,9 @@ module Yasuri
       when /^links_(.+)$/
         Yasuri::LinksNode.new(xpath, $1, children || [])
       when /^pages_(.+)$/
-        limit = opt || Float::INFINITY
-        Yasuri::PaginateNode.new(xpath, $1, children || [], limit)
+        xpath, limit = *args
+        limit = limit || Float::MAX
+        Yasuri::PaginateNode.new(xpath, $1, children || [], limit: limit)
       else
         nil
       end
@@ -146,16 +147,19 @@ module Yasuri
     "links"  => LinksNode,
     "pages"  => PaginateNode
   }
+  ReservedKeys = %w|node name path children|
   def self.hash2node(node_h)
-    node, name, path, children = %w|node name path children|.map do |key|
+    node, name, path, children = ReservedKeys.map do |key|
       node_h[key]
     end
     children ||= []
 
     childnodes = children.map{|c| Yasuri.hash2node(c) }
+    ReservedKeys.each{|key| node_h.delete(key)}
+    opt = node_h
 
     klass = Text2Node[node]
-    klass ? klass.new(path, name, childnodes) : nil
+    klass ? klass.new(path, name, childnodes, opt: opt) : nil
   end
 
   def self.with_retry(retry_count = 5)

data/spec/yasuri_spec.rb

@@ -54,19 +54,19 @@ describe 'Yasuri' do
     end
 
     it "can be truncated with regexp" do
-      node  = Yasuri.text_title '/html/body/p[1]', /^[^,]+/
+      node  = Yasuri.text_title '/html/body/p[1]', truncate_regexp:/^[^,]+/
       actual = node.inject(@agent, @index_page)
       expect(actual).to eq "Hello"
     end
 
     it "can be truncated with regexp" do
-      node = Yasuri.text_title '/html/body/p[1]', /[^,]+$/
+      node = Yasuri.text_title '/html/body/p[1]', truncate_regexp:/[^,]+$/
       actual = node.inject(@agent, @index_page)
       expect(actual).to eq "Yasuri"
     end
 
     it "return empty string if truncated with no match to regexp" do
-      node = Yasuri.text_title '/html/body/p[1]', /^hoge/
+      node = Yasuri.text_title '/html/body/p[1]', truncate_regexp:/^hoge/
       actual = node.inject(@agent, @index_page)
       expect(actual).to be_empty
     end
@@ -281,7 +281,7 @@ describe 'Yasuri' do
     it "scrape each paginated pages limited" do
       root_node = Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "root", [
         Yasuri::TextNode.new('/html/body/p', "content"),
-      ], 3)
+      ], limit:3)
       actual = root_node.inject(@agent, @page)
       expected = [
         {"content" => "PaginationTest01"},
@@ -327,7 +327,7 @@ describe 'Yasuri' do
       end
       original = Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "root", [
         Yasuri::TextNode.new('/html/body/p', "content"),
-      ], 2)
+      ], limit: 2)
       compare_generated_vs_original(generated, original, @page)
     end
   end
@@ -386,6 +386,26 @@ describe 'Yasuri' do
       compare_generated_vs_original(generated, original, paginate_test_page)
     end
 
+    it "return PaginateNode/TextNode with limit" do
+      src = %q|{ "node"     : "pages",
+                 "name"     : "root",
+                 "path"     : "/html/body/nav/span/a[@class=\'next\']",
+                 "limit"    : 2,
+                 "children" : [ { "node" : "text",
+                                  "name" : "content",
+                                  "path" : "/html/body/p"
+                                } ]
+               }|
+      generated = Yasuri.json2tree(src)
+      original = Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "root", [
+                   Yasuri::TextNode.new('/html/body/p', "content"),
+                 ], limit:2)
+
+      paginate_test_uri  = @uri + "/pagination/page01.html"
+      paginate_test_page = @agent.get(paginate_test_uri)
+      compare_generated_vs_original(generated, original, paginate_test_page)
+    end
+
     it "return StructNode/StructNode/[TextNode,TextNode]" do
      src = %q| { "node"     : "struct",
                  "name"     : "tables",

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: yasuri
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
 platform: ruby
 authors:
 - TAC
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-23 00:00:00.000000000 Z
+date: 2015-02-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler