yasuri 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 765b5141cd81658858ac62ea8306c399f4edda16
-  data.tar.gz: ecde98805bf7d3536e83496e6ac85ec5767f0321
+  metadata.gz: 39e4792feb25676d0f6e6f524d93ece20fb83530
+  data.tar.gz: fefe743c8e4807b69535f998e53d71784685aff6
 SHA512:
-  metadata.gz: 6b0f3466e7017608ec24b75c15ccc8c2f64fff3b173aa2e818f02c1b115c1796d0e8f901879d5dae7c4de312bec3c309a6f530b3f3f7b65e19b5ee58ccf4db4d
-  data.tar.gz: d6551a758c588a3a0c08865551ebef957863643125607cdb1b6f143f56de62691a76444fc3c85dc455793c61903794b61c0eaeb6515dda6536e130760942d2d0
+  metadata.gz: e532b3ad29681aebaec403bec9c34bcba96f14e6f5b8b6909d89d00938ec1988a4a26396a2fc058abe30a8f04e1c2fdfb8f755daee868c3b9eff6d4daf6c79a2
+  data.tar.gz: 469ac8b9a30715322e2efab60b2665b3da6eab66666bfbfc7af3cfd05d75750aa47e26b8763a373c5691395c93af5a7b35a71704358cf35f96e8662880ada5de

data/lib/yasuri/version.rb

@@ -1,3 +1,3 @@
 module Yasuri
-  VERSION = "0.0.4"
+  VERSION = "0.0.5"
 end

data/lib/yasuri/yasuri.rb

@@ -67,6 +67,12 @@ module Yasuri
 
   class PaginateNode
     include Node
+
+    def initialize(xpath, name, children = [], limit = Float::INFINITY)
+      super(xpath, name, children)
+      @limit = limit
+    end
+
     def inject(agent, page, retry_count = 5)
 
       child_results = []
@@ -81,6 +87,7 @@ module Yasuri
 
         link_button = Mechanize::Page::Link.new(link, agent, page)
         page = Yasuri.with_retry(retry_count) { link_button.click }
+        break if (@limit -= 1) <= 0
       end
 
       child_results
@@ -101,19 +108,20 @@ module Yasuri
     end
 
     def self.gen(name, *args, &block)
-      xpath, children = *args
+      xpath, opt = *args
       children = Yasuri::NodeGenerator.new.gen_recursive(&block) if block_given?
 
       case name
       when /^text_(.+)$/
-        truncate_regexp, dummy = children
-        Yasuri::TextNode.new(xpath, $1, children || [])
+        truncate_regexp = opt
+        Yasuri::TextNode.new(xpath, $1, truncate_regexp)
       when /^struct_(.+)$/
         Yasuri::StructNode.new(xpath, $1, children || [])
       when /^links_(.+)$/
         Yasuri::LinksNode.new(xpath, $1, children || [])
       when /^pages_(.+)$/
-        Yasuri::PaginateNode.new(xpath, $1, children || [])
+        limit = opt || Float::INFINITY
+        Yasuri::PaginateNode.new(xpath, $1, children || [], limit)
       else
         nil
       end

data/spec/yasuri_spec.rb

@@ -278,6 +278,20 @@ describe 'Yasuri' do
       expect(actual).to match expected
     end
 
+    it "scrape each paginated pages limited" do
+      root_node = Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "root", [
+        Yasuri::TextNode.new('/html/body/p', "content"),
+      ], 3)
+      actual = root_node.inject(@agent, @page)
+      expected = [
+        {"content" => "PaginationTest01"},
+        {"content" => "PaginationTest02"},
+        {"content" => "PaginationTest03"},
+      ]
+      expect(actual).to match expected
+    end
+
+
     it 'return first content if paginate link node is not found' do
       missing_xpath = "/html/body/nav/span/b[@class='next']"
       root_node = Yasuri::PaginateNode.new(missing_xpath, "root", [
@@ -304,7 +318,17 @@ describe 'Yasuri' do
       original = Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "root", [
         Yasuri::TextNode.new('/html/body/p', "content"),
       ])
-    compare_generated_vs_original(generated, original)
+      compare_generated_vs_original(generated, original, @page)
+    end
+
+    it 'can be defined by DSL, return single PaginateNode content limited' do
+      generated = Yasuri.pages_next "/html/body/nav/span/a[@class='next']", 2 do
+        text_content '/html/body/p'
+      end
+      original = Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "root", [
+        Yasuri::TextNode.new('/html/body/p', "content"),
+      ], 2)
+      compare_generated_vs_original(generated, original, @page)
     end
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: yasuri
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - TAC