yasuri 3.2.0 → 3.3.0

data/spec/yasuri_spec.rb

@@ -8,9 +8,7 @@ describe 'Yasuri' do
   include_context 'httpserver'
 
   before do
-    @agent = Mechanize.new
     @uri = uri
-    @index_page = @agent.get(@uri)
   end
 
 
@@ -29,7 +27,7 @@ describe 'Yasuri' do
       generated = Yasuri.yaml2tree(src)
       original  = Yasuri::TextNode.new('/html/body/p[1]', "content")
 
-      compare_generated_vs_original(generated, original, @index_page)
+      compare_generated_vs_original(generated, original, @uri)
     end
 
     it "return text node as symbol" do
@@ -40,7 +38,7 @@ describe 'Yasuri' do
       generated = Yasuri.yaml2tree(src)
       original  = Yasuri::TextNode.new('/html/body/p[1]', "content")
 
-      compare_generated_vs_original(generated, original, @index_page)
+      compare_generated_vs_original(generated, original, @uri)
     end
 
     it "return LinksNode/TextNode" do
@@ -55,7 +53,7 @@ describe 'Yasuri' do
                     Yasuri::TextNode.new('/html/body/p', "content"),
                   ])
 
-      compare_generated_vs_original(generated, original, @index_page)
+      compare_generated_vs_original(generated, original, @uri)
     end
 
     it "return StructNode/StructNode/[TextNode,TextNode]" do
@@ -75,8 +73,8 @@ describe 'Yasuri' do
           Yasuri::TextNode.new('./td[2]', "pub_date"),
         ])
       ])
-      page = @agent.get(@uri + "/struct/structual_text.html")
-      compare_generated_vs_original(generated, original, page)
+      uri = @uri + "/struct/structual_text.html"
+      compare_generated_vs_original(generated, original, uri)
     end
 
   end # end of describe '.yaml2tree'
@@ -98,7 +96,7 @@ describe 'Yasuri' do
       generated = Yasuri.json2tree(src)
       original  = Yasuri::TextNode.new('/html/body/p[1]', "content")
 
-      compare_generated_vs_original(generated, original, @index_page)
+      compare_generated_vs_original(generated, original, @uri)
     end
 
     it "return TextNode with truncate_regexp" do
@@ -111,7 +109,7 @@ describe 'Yasuri' do
       }|
       generated = Yasuri.json2tree(src)
       original  = Yasuri::TextNode.new('/html/body/p[1]', "content", truncate:/^[^,]+/)
-      compare_generated_vs_original(generated, original, @index_page)
+      compare_generated_vs_original(generated, original, @uri)
     end
 
     it "return MapNode with TextNodes" do
@@ -125,7 +123,7 @@ describe 'Yasuri' do
         Yasuri::TextNode.new('/html/body/p[1]', "content01"),
         Yasuri::TextNode.new('/html/body/p[2]', "content02"),
       ])
-      compare_generated_vs_original(generated, original, @index_page)
+      compare_generated_vs_original(generated, original, @uri)
     end
 
     it "return LinksNode/TextNode" do
@@ -142,7 +140,7 @@ describe 'Yasuri' do
                     Yasuri::TextNode.new('/html/body/p', "content"),
                   ])
 
-      compare_generated_vs_original(generated, original, @index_page)
+      compare_generated_vs_original(generated, original, @uri)
     end
 
     it "return PaginateNode/TextNode" do
@@ -158,9 +156,8 @@ describe 'Yasuri' do
                    Yasuri::TextNode.new('/html/body/p', "content"),
                  ])
 
-      paginate_test_uri  = @uri + "/pagination/page01.html"
-      paginate_test_page = @agent.get(paginate_test_uri)
-      compare_generated_vs_original(generated, original, paginate_test_page)
+      uri  = @uri + "/pagination/page01.html"
+      compare_generated_vs_original(generated, original, uri)
     end
 
     it "return PaginateNode/TextNode with limit" do
@@ -177,9 +174,8 @@ describe 'Yasuri' do
                    Yasuri::TextNode.new('/html/body/p', "content"),
                  ], limit:2)
 
-      paginate_test_uri  = @uri + "/pagination/page01.html"
-      paginate_test_page = @agent.get(paginate_test_uri)
-      compare_generated_vs_original(generated, original, paginate_test_page)
+      uri  = @uri + "/pagination/page01.html"
+      compare_generated_vs_original(generated, original, uri)
     end
 
     it "return StructNode/StructNode/[TextNode,TextNode]" do
@@ -201,8 +197,8 @@ describe 'Yasuri' do
           Yasuri::TextNode.new('./td[2]', "pub_date"),
         ])
       ])
-      page = @agent.get(@uri + "/struct/structual_text.html")
-      compare_generated_vs_original(generated, original, page)
+      uri = @uri + "/struct/structual_text.html"
+      compare_generated_vs_original(generated, original, uri)
     end
   end
 
@@ -299,6 +295,46 @@ describe 'Yasuri' do
     end
   end
 
+  describe '.with_retry' do
+    it "call once if success" do
+      actual = Yasuri.with_retry(0){ 42 }
+      expect(actual).to match 42
+    end
+
+    it "call untile success" do
+      i = [1,1,0,0]
+      actual = Yasuri.with_retry(2){42 / i.pop } # 3 times in max
+      expect(actual).to match 42/1
+    end
+
+    it "raise error when exceed retry count" do
+      i = [1,0,0,0]
+      expect {
+        Yasuri.with_retry(2){42 / i.pop } # do this 3 times
+      }.to raise_error(Exception)
+    end
+
+    it "wait interval before run" do
+      allow(Kernel).to receive(:sleep)
+      Yasuri.with_retry(0){ 42 }
+      expect(Kernel).to have_received(:sleep).once
+    end
+
+    it "wait interval before run" do
+      allow(Kernel).to receive(:sleep)
+      Yasuri.with_retry(0){ 42 }
+      expect(Kernel).to have_received(:sleep).once
+    end
+
+    it "wait interval for each runs" do
+      allow(Kernel).to receive(:sleep)
+
+      i = [1,1,0,0]
+      Yasuri.with_retry(2){42 / i.pop } # 3 times in max
+      expect(Kernel).to have_received(:sleep).exactly(3).times
+    end
+  end
+
   it "return StructNode/StructNode/[TextNode,TextNode]" do
     tree  = Yasuri::StructNode.new('/html/body/table', "tables", [
       Yasuri::StructNode.new('./tr', "table", [

data/spec/yasuri_struct_node_spec.rb

@@ -11,8 +11,7 @@ describe 'Yasuri' do
 
   describe '::StructNode' do
     before do
-      @agent = Mechanize.new
-      @page = @agent.get(uri + "/struct/structual_text.html")
+      @uri = uri + "/struct/structual_text.html"
 
       @table_1996 = [
         { "title"    => "The Perfect Insider",
@@ -53,7 +52,7 @@ describe 'Yasuri' do
         Yasuri::TextNode.new('./td[2]', "pub_date"),
       ])
       expected = @table_1996
-      actual = node.inject(@agent, @page)
+      actual = node.scrape(@uri)
       expect(actual).to match expected
     end
 
@@ -63,7 +62,7 @@ describe 'Yasuri' do
         Yasuri::TextNode.new('./td[2]', "pub_date"),
       ])
       expected = @table_1996.first
-      actual = node.inject(@agent, @page)
+      actual = node.scrape(@uri)
       expect(actual).to match expected
     end
 
@@ -72,7 +71,7 @@ describe 'Yasuri' do
       node = Yasuri::StructNode.new(no_match_xpath, "table", [
         Yasuri::TextNode.new('./td[1]', "title")
       ])
-      actual = node.inject(@agent, @page)
+      actual = node.scrape(@uri)
       expect(actual).to be_empty
     end
 
@@ -81,7 +80,7 @@ describe 'Yasuri' do
       node = Yasuri::StructNode.new(invalid_xpath, "table", [
         Yasuri::TextNode.new('./td[1]', "title")
       ])
-      expect { node.inject(@agent, @page) }.to raise_error(Nokogiri::XML::XPath::SyntaxError)
+      expect { node.scrape(@uri) }.to raise_error(Nokogiri::XML::XPath::SyntaxError)
     end
 
     it 'fail with invalid xpath in children' do
@@ -90,7 +89,7 @@ describe 'Yasuri' do
         Yasuri::TextNode.new(invalid_xpath, "title"),
         Yasuri::TextNode.new('./td[2]', "pub_date"),
       ])
-      expect { node.inject(@agent, @page) }.to raise_error(Nokogiri::XML::XPath::SyntaxError)
+      expect { node.scrape(@uri) }.to raise_error(Nokogiri::XML::XPath::SyntaxError)
     end
 
     it 'scrape all tables' do
@@ -101,7 +100,7 @@ describe 'Yasuri' do
         ])
       ])
       expected = @all_tables
-      actual = node.inject(@agent, @page)
+      actual = node.scrape(@uri)
       expect(actual).to match expected
     end
 
@@ -118,7 +117,7 @@ describe 'Yasuri' do
           Yasuri::TextNode.new('./td[2]', "pub_date"),
         ])
       ])
-      compare_generated_vs_original(generated, original, @page)
+      compare_generated_vs_original(generated, original, @uri)
     end
 
     it 'return child node as symbol' do
@@ -127,7 +126,7 @@ describe 'Yasuri' do
         Yasuri::TextNode.new('./td[2]', "pub_date"),
       ])
       expected = @table_1996.map{|h| h.map{|k,v| [k.to_sym, v] }.to_h }
-      actual = node.inject(@agent, @page, symbolize_names:true)
+      actual = node.scrape(@uri, symbolize_names:true)
       expect(actual).to match expected
     end
 
@@ -135,9 +134,7 @@ describe 'Yasuri' do
 
   describe '::StructNode::Links' do
     before do
-      @agent = Mechanize.new
-      @page = @agent.get(uri + "/struct/structual_links.html")
-
+      @uri = uri + "/struct/structual_links.html"
       @table = [
         { "title" => "Child01,02",
           "child" => [{"p" => "Child 01 page."}, {"p" => "Child 02 page."}] },
@@ -155,22 +152,21 @@ describe 'Yasuri' do
         ])
       ])
       expected = @table
-      actual = node.inject(@agent, @page)
+      actual = node.scrape(@uri)
       expect(actual).to match expected
     end
   end # descrive
 
   describe '::StructNode::Pages' do
     before do
-      @agent = Mechanize.new
-      @page = @agent.get(uri + "/struct/structual_text.html") #dummy
+      @uri = uri + "/struct/structual_text.html"
     end
 
     it 'not supported' do
       node = Yasuri::StructNode.new('/html/body/table[1]/tr', "table", [
         Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "pages", [])
       ])
-      expect{ node.inject(@agent, @page) }.to raise_error(NotImplementedError, "PagenateNode inside StructNode, Not Supported")
+      expect{ node.scrape(@uri) }.to raise_error(NotImplementedError, "PagenateNode inside StructNode, Not Supported")
     end
   end
 end

data/spec/yasuri_text_node_spec.rb

@@ -10,69 +10,68 @@ describe 'Yasuri' do
   include_context 'httpserver'
 
   before do
-    @agent = Mechanize.new
-    @index_page = @agent.get(uri)
+    @uri = uri
   end
 
   describe '::TextNode' do
     before { @node = Yasuri::TextNode.new('/html/body/p[1]', "title") }
 
     it 'scrape text text <p>Hello,Yasuri</p>' do
-      actual = @node.inject(@agent, @index_page)
+      actual = @node.scrape(@uri)
       expect(actual).to eq "Hello,Yasuri"
     end
 
     it 'return empty text if no match node' do
       no_match_node = Yasuri::TextNode.new('/html/body/no_match_node', "title")
-      actual = no_match_node.inject(@agent, @index_page)
+      actual = no_match_node.scrape(@uri)
       expect(actual).to be_empty
     end
 
     it 'fail with invalid xpath' do
       invalid_xpath = '/html/body/no_match_node['
       node = Yasuri::TextNode.new(invalid_xpath, "title")
-      expect { node.inject(@agent, @index_page) }.to raise_error(Nokogiri::XML::XPath::SyntaxError)
+      expect { node.scrape(@uri) }.to raise_error(Nokogiri::XML::XPath::SyntaxError)
     end
 
     it "can be defined by DSL, return single TextNode title" do
       generated = Yasuri.text_title '/html/body/p[1]'
       original  = Yasuri::TextNode.new('/html/body/p[1]', "title")
-      compare_generated_vs_original(generated, original, @index_page)
+      compare_generated_vs_original(generated, original, @uri)
     end
 
     it "can be truncated with regexp" do
       node  = Yasuri.text_title '/html/body/p[1]', truncate:/^[^,]+/
-      actual = node.inject(@agent, @index_page)
+      actual = node.scrape(@uri)
       expect(actual).to eq "Hello"
     end
 
     it "return first captured if matched given capture pattern" do
       node  = Yasuri.text_title '/html/body/p[1]', truncate:/H(.+)i/
-      actual = node.inject(@agent, @index_page)
+      actual = node.scrape(@uri)
       expect(actual).to eq "ello,Yasur"
     end
 
     it "can be truncated with regexp" do
       node = Yasuri.text_title '/html/body/p[1]', truncate:/[^,]+$/
-      actual = node.inject(@agent, @index_page)
+      actual = node.scrape(@uri)
       expect(actual).to eq "Yasuri"
     end
 
     it "return empty string if truncated with no match to regexp" do
       node = Yasuri.text_title '/html/body/p[1]', truncate:/^hoge/
-      actual = node.inject(@agent, @index_page)
+      actual = node.scrape(@uri)
       expect(actual).to be_empty
     end
 
     it "return symbol method applied string" do
       node = Yasuri.text_title '/html/body/p[1]', proc: :upcase
-      actual = node.inject(@agent, @index_page)
+      actual = node.scrape(@uri)
       expect(actual).to eq "HELLO,YASURI"
     end
 
     it "return apply multi arguments" do
       node = Yasuri.text_title '/html/body/p[1]', proc: :upcase, truncate:/H(.+)i/
-      actual = node.inject(@agent, @index_page)
+      actual = node.scrape(@uri)
       expect(actual).to eq "ELLO,YASUR"
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: yasuri
 version: !ruby/object:Gem::Version
-  version: 3.2.0
+  version: 3.3.0
 platform: ruby
 authors:
 - TAC
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-03-25 00:00:00.000000000 Z
+date: 2021-03-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -170,7 +170,10 @@ files:
 - Rakefile
 - USAGE.ja.md
 - USAGE.md
-- app.rb
+- examples/example.rb
+- examples/github.yml
+- examples/sample.json
+- examples/sample.yml
 - exe/yasuri
 - lib/yasuri.rb
 - lib/yasuri/version.rb

data/app.rb

@@ -1,52 +0,0 @@
-#!/usr/bin/env ruby
-# -*- coding: utf-8 -*-
-
-# Author::    TAC (tac@tac42.net)
-
-require 'pp'
-require 'time'
-require 'mechanize'
-
-require_relative 'lib/yasuri/yasuri'
-
-agent = Mechanize.new
-
-uri = "http://www.asahi.com/"
-
-# Node tree constructing by DSL
-root = Yasuri.links_top '//*[@id="MainInner"]/div[1]/ul/li/a' do
-  text_title   '//*[@id="MainInner"]/div[1]/div/h1'
-  text_article '//*[@id="MainInner"]/div/div[@class="ArticleText"]'
-end
-
-# Node tree constructing by JSON
-src = <<-EOJSON
-   { "node"     : "links",
-     "name"     : "root",
-     "path"     : "//*[@id='MainInner']/div[1]/ul/li/a",
-     "children" : [
-                    { "node" : "text",
-                      "name" : "title",
-                      "path" : "//*[@id='MainInner']/div[1]/div/h1"
-                    },
-                    { "node" : "text",
-                      "name" : "article",
-                      "path" : "//*[@id='MainInner']/div/div[@class='ArticleText']"
-                    }
-                  ]
-   }
-EOJSON
-root = Yasuri.json2tree(src)
-
-# Access to parsed resources
-page = agent.get(uri)
-contents = root.inject(agent, page)
-
-contents.each do |h|
-  t = h['title']
-  a = h['article']
-
-  puts t
-  puts a
-  puts "=" * 100
-end