yasuri 2.0.12 → 3.3.0

data/spec/yasuri_map_spec.rb

@@ -0,0 +1,75 @@
+require_relative 'spec_helper'
+
+describe 'Yasuri' do
+  include_context 'httpserver'
+
+  before do
+    @uri = uri
+  end
+
+  describe '::MapNode' do
+    it "multi scrape in singe page" do
+      map = Yasuri.map_sample do
+        text_title  '/html/head/title'
+        text_body_p '/html/body/p[1]'
+      end
+      actual = map.scrape(@uri)
+
+      expected = {
+        "title"  => "Yasuri Test",
+        "body_p" => "Hello,Yasuri"
+      }
+      expect(actual).to include expected
+    end
+
+    it "nested multi scrape in singe page" do
+      map = Yasuri.map_sample do
+        map_group1 { text_child01  '/html/body/a[1]' }
+        map_group2 do
+          text_child01 '/html/body/a[1]'
+          text_child03 '/html/body/a[3]'
+        end
+      end
+      actual = map.scrape(@uri)
+
+      expected = {
+        "group1" => {
+          "child01" => "child01"
+        },
+        "group2" => {
+          "child01" => "child01",
+          "child03" => "child03"
+        }
+      }
+      expect(actual).to include expected
+    end
+
+    it "scrape with links node" do
+      map = Yasuri.map_sample do
+        map_group1 do
+          links_a '/html/body/a' do
+            text_content '/html/body/p'
+          end
+          text_child01  '/html/body/a[1]'
+        end
+        map_group2 do
+          text_child03 '/html/body/a[3]'
+        end
+      end
+      actual = map.scrape(@uri)
+
+      expected = {
+        "group1" => {
+          "a" => [
+            {"content" => "Child 01 page."},
+            {"content" => "Child 02 page."},
+            {"content" => "Child 03 page."},
+          ],
+          "child01" => "child01"
+        },
+        "group2" => { "child03" => "child03" }
+      }
+      expect(actual).to include expected
+    end
+  end
+end

data/spec/yasuri_paginate_node_spec.rb

@@ -11,16 +11,14 @@ describe 'Yasuri' do
 
   describe '::PaginateNode' do
     before do
-      @agent = Mechanize.new
       @uri = uri + "/pagination/page01.html"
-      @page = @agent.get(@uri)
     end
 
     it "scrape each paginated pages" do
       root_node = Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "root", [
         Yasuri::TextNode.new('/html/body/p', "content"),
       ])
-      actual = root_node.inject(@agent, @page)
+      actual = root_node.scrape(@uri)
       expected = [
         {"content" => "PaginationTest01"},
         {"content" => "PaginationTest02"},
@@ -37,7 +35,7 @@ describe 'Yasuri' do
           Yasuri::TextNode.new('./a', "text"),
         ]),
       ], flatten: true)
-      actual = root_node.inject(@agent, @page)
+      actual = root_node.scrape(@uri)
       expected = [
         "PaginationTest01",
         {"text"=>""},
@@ -77,7 +75,7 @@ describe 'Yasuri' do
       root_node = Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "root", [
         Yasuri::TextNode.new('/html/body/p', "content"),
       ], limit:3)
-      actual = root_node.inject(@agent, @page)
+      actual = root_node.scrape(@uri)
       expected = [
         {"content" => "PaginationTest01"},
         {"content" => "PaginationTest02"},
@@ -91,7 +89,7 @@ describe 'Yasuri' do
       root_node = Yasuri::PaginateNode.new(missing_xpath, "root", [
         Yasuri::TextNode.new('/html/body/p', "content"),
       ])
-      actual = root_node.inject(@agent, @page)
+      actual = root_node.scrape(@uri)
       expected = [ {"content" => "PaginationTest01"}, ]
       expect(actual).to match_array expected
     end
@@ -100,7 +98,7 @@ describe 'Yasuri' do
       root_node = Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "root", [
         Yasuri::TextNode.new('/html/body/hoge', "content"),
       ])
-      actual = root_node.inject(@agent, @page)
+      actual = root_node.scrape(@uri)
       expected = [ {"content" => ""}, {"content" => ""}, {"content" => ""}, {"content" => ""},]
       expect(actual).to match_array expected
     end
@@ -112,7 +110,7 @@ describe 'Yasuri' do
       original = Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "root", [
         Yasuri::TextNode.new('/html/body/p', "content"),
       ])
-      compare_generated_vs_original(generated, original, @page)
+      compare_generated_vs_original(generated, original, @uri)
     end
 
     it 'can be defined by DSL, return single PaginateNode content limited' do
@@ -122,14 +120,14 @@ describe 'Yasuri' do
       original = Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "root", [
         Yasuri::TextNode.new('/html/body/p', "content"),
       ], limit: 2)
-      compare_generated_vs_original(generated, original, @page)
+      compare_generated_vs_original(generated, original, @uri)
     end
 
     it "return child node as symbol" do
       root_node = Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "root", [
         Yasuri::TextNode.new('/html/body/p', "content"),
       ])
-      actual = root_node.inject(@agent, @page, symbolize_names:true)
+      actual = root_node.scrape(@uri, symbolize_names:true)
       expected = [
         {:content => "PaginationTest01"},
         {:content => "PaginationTest02"},
@@ -138,5 +136,19 @@ describe 'Yasuri' do
       ]
       expect(actual).to match expected
     end
+
+    it "scrape with interval for each request" do
+      allow(Kernel).to receive(:sleep)
+
+      root_node = Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "root", [
+        Yasuri::TextNode.new('/html/body/p', "content"),
+      ])
+      actual = root_node.scrape(@uri, interval_ms: 1000)
+      expect(actual.size).to match 4
+
+      expect(Kernel).to have_received(:sleep).exactly(4).times do |interval_sec|
+        expect(interval_sec).to match 1.0
+      end
+    end
   end
 end

data/spec/yasuri_spec.rb

@@ -8,11 +8,78 @@ describe 'Yasuri' do
   include_context 'httpserver'
 
   before do
-    @agent = Mechanize.new
     @uri = uri
-    @index_page = @agent.get(@uri)
   end
 
+
+  ############
+  # yam2tree #
+  ############
+  describe '.yaml2tree' do
+    it "fail if empty yaml" do
+      expect { Yasuri.yaml2tree(nil) }.to raise_error(RuntimeError)
+    end
+
+    it "return text node" do
+      src = <<-EOB
+        text_content: "/html/body/p[1]"
+      EOB
+      generated = Yasuri.yaml2tree(src)
+      original  = Yasuri::TextNode.new('/html/body/p[1]', "content")
+
+      compare_generated_vs_original(generated, original, @uri)
+    end
+
+    it "return text node as symbol" do
+      src = <<-EOB
+      :text_content:
+        :path: "/html/body/p[1]"
+      EOB
+      generated = Yasuri.yaml2tree(src)
+      original  = Yasuri::TextNode.new('/html/body/p[1]', "content")
+
+      compare_generated_vs_original(generated, original, @uri)
+    end
+
+    it "return LinksNode/TextNode" do
+
+      src = <<-EOB
+      links_root:
+        path: "/html/body/a"
+        text_content: "/html/body/p"
+      EOB
+      generated = Yasuri.yaml2tree(src)
+      original  = Yasuri::LinksNode.new('/html/body/a', "root", [
+                    Yasuri::TextNode.new('/html/body/p', "content"),
+                  ])
+
+      compare_generated_vs_original(generated, original, @uri)
+    end
+
+    it "return StructNode/StructNode/[TextNode,TextNode]" do
+      src = <<-EOB
+      struct_tables:
+        path: "/html/body/table"
+        struct_table:
+          path: "./tr"
+          text_title: "./td[1]"
+          text_pub_date: "./td[2]"
+      EOB
+
+      generated = Yasuri.yaml2tree(src)
+      original  = Yasuri::StructNode.new('/html/body/table', "tables", [
+        Yasuri::StructNode.new('./tr', "table", [
+          Yasuri::TextNode.new('./td[1]', "title"),
+          Yasuri::TextNode.new('./td[2]', "pub_date"),
+        ])
+      ])
+      uri = @uri + "/struct/structual_text.html"
+      compare_generated_vs_original(generated, original, uri)
+    end
+
+  end # end of describe '.yaml2tree'
+
+
   #############
   # json2tree #
   #############
@@ -22,103 +89,107 @@ describe 'Yasuri' do
     end
 
     it "return TextNode" do
-      src = %q| { "node"  : "text",
-                  "name"  : "content",
-                  "path"  : "/html/body/p[1]"
-                }|
+      src = %q|
+      {
+        "text_content": "/html/body/p[1]"
+      }|
       generated = Yasuri.json2tree(src)
       original  = Yasuri::TextNode.new('/html/body/p[1]', "content")
 
-      compare_generated_vs_original(generated, original, @index_page)
+      compare_generated_vs_original(generated, original, @uri)
     end
 
     it "return TextNode with truncate_regexp" do
-      src = %q| { "node"  : "text",
-                  "name"  : "content",
-                  "path"  : "/html/body/p[1]",
-                  "truncate"  : "^[^,]+"
-                }|
+      src = %q|
+      {
+        "text_content": {
+          "path": "/html/body/p[1]",
+          "truncate"  : "^[^,]+"
+        }
+      }|
       generated = Yasuri.json2tree(src)
       original  = Yasuri::TextNode.new('/html/body/p[1]', "content", truncate:/^[^,]+/)
-      compare_generated_vs_original(generated, original, @index_page)
+      compare_generated_vs_original(generated, original, @uri)
     end
 
+    it "return MapNode with TextNodes" do
+      src = %q|
+      {
+        "text_content01": "/html/body/p[1]",
+        "text_content02": "/html/body/p[2]"
+      }|
+      generated = Yasuri.json2tree(src)
+      original  = Yasuri::MapNode.new('parent', [
+        Yasuri::TextNode.new('/html/body/p[1]', "content01"),
+        Yasuri::TextNode.new('/html/body/p[2]', "content02"),
+      ])
+      compare_generated_vs_original(generated, original, @uri)
+    end
 
     it "return LinksNode/TextNode" do
-      src = %q| { "node"     : "links",
-                  "name"     : "root",
-                  "path"     : "/html/body/a",
-                  "children" : [ { "node" : "text",
-                                   "name" : "content",
-                                   "path" : "/html/body/p"
-                                 } ]
-                }|
+      src = %q|
+      {
+        "links_root": {
+          "path": "/html/body/a",
+          "text_content": "/html/body/p"
+        }
+      }|
+
       generated = Yasuri.json2tree(src)
       original  = Yasuri::LinksNode.new('/html/body/a', "root", [
                     Yasuri::TextNode.new('/html/body/p', "content"),
                   ])
 
-      compare_generated_vs_original(generated, original, @index_page)
+      compare_generated_vs_original(generated, original, @uri)
     end
 
     it "return PaginateNode/TextNode" do
-      src = %q|{ "node"     : "pages",
-                 "name"     : "root",
-                 "path"     : "/html/body/nav/span/a[@class=\'next\']",
-                 "children" : [ { "node" : "text",
-                                  "name" : "content",
-                                  "path" : "/html/body/p"
-                                } ]
-               }|
+      src = %q|
+      {
+        "pages_root": {
+          "path": "/html/body/nav/span/a[@class=\'next\']",
+          "text_content": "/html/body/p"
+        }
+      }|
       generated = Yasuri.json2tree(src)
       original = Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "root", [
                    Yasuri::TextNode.new('/html/body/p', "content"),
                  ])
 
-      paginate_test_uri  = @uri + "/pagination/page01.html"
-      paginate_test_page = @agent.get(paginate_test_uri)
-      compare_generated_vs_original(generated, original, paginate_test_page)
+      uri  = @uri + "/pagination/page01.html"
+      compare_generated_vs_original(generated, original, uri)
     end
 
     it "return PaginateNode/TextNode with limit" do
-      src = %q|{ "node"     : "pages",
-                 "name"     : "root",
-                 "path"     : "/html/body/nav/span/a[@class=\'next\']",
-                 "limit"    : 2,
-                 "children" : [ { "node" : "text",
-                                  "name" : "content",
-                                  "path" : "/html/body/p"
-                                } ]
-               }|
+      src = %q|
+      {
+        "pages_root": {
+          "path": "/html/body/nav/span/a[@class=\'next\']",
+          "limit": 2,
+          "text_content": "/html/body/p"
+        }
+      }|
       generated = Yasuri.json2tree(src)
       original = Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "root", [
                    Yasuri::TextNode.new('/html/body/p', "content"),
                  ], limit:2)
 
-      paginate_test_uri  = @uri + "/pagination/page01.html"
-      paginate_test_page = @agent.get(paginate_test_uri)
-      compare_generated_vs_original(generated, original, paginate_test_page)
+      uri  = @uri + "/pagination/page01.html"
+      compare_generated_vs_original(generated, original, uri)
     end
 
     it "return StructNode/StructNode/[TextNode,TextNode]" do
-     src = %q| { "node"     : "struct",
-                 "name"     : "tables",
-                 "path"     : "/html/body/table",
-                 "children" : [
-                   { "node"       : "struct",
-                     "name"       : "table",
-                     "path"       : "./tr",
-                     "children"   : [
-                       { "node" : "text",
-                         "name" : "title",
-                         "path" : "./td[1]"
-                       },
-                       { "node" : "text",
-                         "name" : "pub_date",
-                         "path" : "./td[2]"
-                       }]
-                   }]
-               }|
+      src = %q|
+      {
+        "struct_tables": {
+          "path": "/html/body/table",
+          "struct_table": {
+            "path": "./tr",
+            "text_title": "./td[1]",
+            "text_pub_date": "./td[2]"
+          }
+        }
+      }|
       generated = Yasuri.json2tree(src)
       original  = Yasuri::StructNode.new('/html/body/table', "tables", [
         Yasuri::StructNode.new('./tr', "table", [
@@ -126,27 +197,27 @@ describe 'Yasuri' do
           Yasuri::TextNode.new('./td[2]', "pub_date"),
         ])
       ])
-      page = @agent.get(@uri + "/struct/structual_text.html")
-      compare_generated_vs_original(generated, original, page)
+      uri = @uri + "/struct/structual_text.html"
+      compare_generated_vs_original(generated, original, uri)
     end
   end
 
+
   #############
   # tree2json #
   #############
   describe '.tree2json' do
     it "return empty json" do
-      json = Yasuri.tree2json(nil)
-      expect(json).to match "{}"
+      expect { Yasuri.tree2json(nil) }.to raise_error(RuntimeError)
     end
 
     it "return text node" do
       node = Yasuri::TextNode.new("/html/head/title", "title")
       json = Yasuri.tree2json(node)
-      expected_str = %q| { "node": "text",
-                           "name": "title",
-                           "path": "/html/head/title"
-                         } |
+      expected_str = %q|
+      {
+        "text_title": "/html/head/title"
+      }|
       expected = JSON.parse(expected_str)
       actual   = JSON.parse(json)
       expect(actual).to match expected
@@ -155,29 +226,49 @@ describe 'Yasuri' do
     it "return text node with truncate_regexp" do
       node = Yasuri::TextNode.new("/html/head/title", "title", truncate:/^[^,]+/)
       json = Yasuri.tree2json(node)
-      expected_str = %q| { "node": "text",
-                           "name": "title",
-                           "path": "/html/head/title",
-                           "truncate": "^[^,]+"
-                         } |
+      expected_str = %q|
+      {
+        "text_title": {
+          "path": "/html/head/title",
+          "truncate": "^[^,]+"
+        }
+      }|
       expected = Yasuri.tree2json(Yasuri.json2tree(expected_str))
       actual   = Yasuri.tree2json(Yasuri.json2tree(json))
       expect(actual).to match expected
     end
 
+    it "return map node with text nodes" do
+      tree = Yasuri::MapNode.new('parent', [
+        Yasuri::TextNode.new('/html/body/p[1]', "content01"),
+        Yasuri::TextNode.new('/html/body/p[2]', "content02"),
+      ])
+      actual_json = Yasuri.tree2json(tree)
+
+      expected_json = %q|
+      {
+        "text_content01": "/html/body/p[1]",
+        "text_content02": "/html/body/p[2]"
+      }|
+
+      expected = Yasuri.tree2json(Yasuri.json2tree(expected_json))
+      actual   = Yasuri.tree2json(Yasuri.json2tree(actual_json))
+      expect(actual).to match expected
+    end
+
     it "return LinksNode/TextNode" do
       tree  = Yasuri::LinksNode.new('/html/body/a', "root", [
                 Yasuri::TextNode.new('/html/body/p', "content"),
               ])
       json   = Yasuri.tree2json(tree)
-      expected_src = %q| { "node"     : "links",
-                           "name"     : "root",
-                           "path"     : "/html/body/a",
-                           "children" : [ { "node" : "text",
-                                            "name" : "content",
-                                            "path" : "/html/body/p"
-                                          } ]
-                         }|
+
+      expected_src = %q|
+      {
+        "links_root": {
+          "path": "/html/body/a",
+          "text_content":"/html/body/p"
+        }
+      }|
       expected  = JSON.parse(expected_src)
       actual    = JSON.parse(json)
       expect(actual).to match expected
@@ -189,25 +280,84 @@ describe 'Yasuri' do
              ], limit:10)
 
       json   = Yasuri.tree2json(tree)
-      expected_src = %q| { "node"     : "pages",
-                           "name"     : "root",
-                           "path"     : "/html/body/nav/span/a[@class='next']",
-                           "limit"    : 10,
-                           "flatten"  : false,
-                           "children" : [ { "node" : "text",
-                                            "name" : "content",
-                                            "path" : "/html/body/p"
-                                          } ]
-                         }|
+      expected_src = %q|
+      {
+        "pages_root": {
+          "path": "/html/body/nav/span/a[@class='next']",
+          "limit": 10,
+          "flatten": false,
+          "text_content": "/html/body/p"
+        }
+      }|
       expected  = JSON.parse(expected_src)
       actual    = JSON.parse(json)
       expect(actual).to match expected
     end
+  end
+
+  describe '.with_retry' do
+    it "call once if success" do
+      actual = Yasuri.with_retry(0){ 42 }
+      expect(actual).to match 42
+    end
+
+    it "call untile success" do
+      i = [1,1,0,0]
+      actual = Yasuri.with_retry(2){42 / i.pop } # 3 times in max
+      expect(actual).to match 42/1
+    end
+
+    it "raise error when exceed retry count" do
+      i = [1,0,0,0]
+      expect {
+        Yasuri.with_retry(2){42 / i.pop } # do this 3 times
+      }.to raise_error(Exception)
+    end
+
+    it "wait interval before run" do
+      allow(Kernel).to receive(:sleep)
+      Yasuri.with_retry(0){ 42 }
+      expect(Kernel).to have_received(:sleep).once
+    end
 
+    it "wait interval before run" do
+      allow(Kernel).to receive(:sleep)
+      Yasuri.with_retry(0){ 42 }
+      expect(Kernel).to have_received(:sleep).once
+    end
 
+    it "wait interval for each runs" do
+      allow(Kernel).to receive(:sleep)
 
+      i = [1,1,0,0]
+      Yasuri.with_retry(2){42 / i.pop } # 3 times in max
+      expect(Kernel).to have_received(:sleep).exactly(3).times
+    end
   end
 
+  it "return StructNode/StructNode/[TextNode,TextNode]" do
+    tree  = Yasuri::StructNode.new('/html/body/table', "tables", [
+      Yasuri::StructNode.new('./tr', "table", [
+        Yasuri::TextNode.new('./td[1]', "title"),
+        Yasuri::TextNode.new('./td[2]', "pub_date"),
+      ])
+    ])
+    json   = Yasuri.tree2json(tree)
+    expected_src = %q|
+    {
+      "struct_tables": {
+        "path": "/html/body/table",
+        "struct_table": {
+          "path": "./tr",
+          "text_title": "./td[1]",
+          "text_pub_date": "./td[2]"
+        }
+      }
+    }|
+    expected  = JSON.parse(expected_src)
+    actual    = JSON.parse(json)
+    expect(actual).to match expected
+  end
 
   it 'has a version number' do
     expect(Yasuri::VERSION).not_to be nil