yasuri 2.0.11 → 2.0.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d8d6bd8c37be444f0c5568bcf20604d7bca5c223
-  data.tar.gz: 8438eee300a7e4f73be7107cbd9417da18f5048d
+  metadata.gz: a4fed4a13bb125758515e3c0ced665b1ca3d20b6
+  data.tar.gz: e9dfb2ed6256a367db2e5b6a78d23fa097c422d7
 SHA512:
-  metadata.gz: 107ddc8cd0310c646841e6fe6a2695313edb9692418a783e133a5d269d4a1ab39385975276ae167ac68863b9760794ebb2738832dccfc4f599686c5a9e50f244
-  data.tar.gz: b6d089de8cd866f137ca58dd779396cd4948e080d3225cc4384f8f9cdb54f5a778cd4be85b89628938ccacbf11dfefe74ea8bd248e835971470e7a64df597411
+  metadata.gz: 8b9d6345f3f49b1f7d9445ce18bca736b8cbeedc69979a45d541b59af4e09092d7c1d12886801a24296e9e3d73f39a7c2d53a7c2de12e1a0ff890623b47cfe84
+  data.tar.gz: 6d755f266062052dd5244599deefefea85f7570c827a898e48eee22c44510dde287b0554ed2cae85e3b94b44fe4eb6f74b512c44047e6cc1bb43fe27a93143b0

data/USAGE.ja.md

@@ -431,3 +431,38 @@ node.inject(agent, page)
 #=> [ {"content" => "Pagination01"}, {"content" => "Pagination02"}]
 ```
 この場合、PaginateNode は最大2つまでのページを開いてパースします．ページネーションは4つのページを持っているようですが、`limit:2`が指定されているため、結果の配列には2つの結果のみが含まれています．
+
+##### `flatten`
+取得した各ページの結果を展開します．
+
+```ruby
+agent = Mechanize.new
+page = agent.get("http://yasuri.example.net/page01.html")
+
+node = Yasuri.pages_root "/html/body/nav/span/a[@class='next']" , flatten:true do
+         text_title   '/html/head/title'
+         text_content '/html/body/p'
+       end
+node.inject(agent, page)
+
+#=> [ {"title" => "Page01",
+       "content" => "Patination01"},
+      {"title"   => "Page01",
+       "content" => "Patination02"},
+      {"title"   => "Page01",
+       "content" => "Patination03"}]
+
+
+node = Yasuri.pages_root "/html/body/nav/span/a[@class='next']" , flatten:true do
+        text_title   '/html/head/title'
+        text_content '/html/body/p'
+      end
+node.inject(agent, page)
+
+#=> [ "Page01",
+      "Patination01",
+      "Page02",
+      "Patination02",
+      "Page03",
+      "Patination03"]
+```

data/USAGE.md

@@ -429,3 +429,38 @@ node.inject(agent, page)
 #=> [ {"content" => "Pagination01"}, {"content" => "Pagination02"}]
 ```
 Paginate Node open upto 2 given by `limit`. In this situation, pagination has 4 pages, but result Array has 2 texts because given `limit:2`.
+
+##### `flatten`
+`flatten` option expands each page results.
+
+```ruby
+agent = Mechanize.new
+page = agent.get("http://yasuri.example.net/page01.html")
+
+node = Yasuri.pages_root "/html/body/nav/span/a[@class='next']" , flatten:true do
+         text_title   '/html/head/title'
+         text_content '/html/body/p'
+       end
+node.inject(agent, page)
+
+#=> [ {"title" => "Page01",
+       "content" => "Patination01"},
+      {"title"   => "Page01",
+       "content" => "Patination02"},
+      {"title"   => "Page01",
+       "content" => "Patination03"}]
+
+
+node = Yasuri.pages_root "/html/body/nav/span/a[@class='next']" , flatten:true do
+        text_title   '/html/head/title'
+        text_content '/html/body/p'
+      end
+node.inject(agent, page)
+
+#=> [ "Page01",
+      "Patination01",
+      "Page02",
+      "Patination02",
+      "Page03",
+      "Patination03"]
+```

data/lib/yasuri/version.rb

@@ -1,3 +1,3 @@
 module Yasuri
-  VERSION = "2.0.11"
+  VERSION = "2.0.12"
 end

data/lib/yasuri/yasuri_links_node.rb

@@ -6,10 +6,10 @@ require_relative 'yasuri_node'
 module Yasuri
   class LinksNode
     include Node
-    def inject(agent, page, opt = {})
+    def inject(agent, page, opt = {}, element = page)
       retry_count = opt[:retry_count] || 5
 
-      links = page.search(@xpath) || [] # links expected
+      links = element.search(@xpath) || [] # links expected
       links.map do |link|
         link_button = Mechanize::Page::Link.new(link, agent, page)
         child_page = Yasuri.with_retry(retry_count) { link_button.click }

data/lib/yasuri/yasuri_node.rb

@@ -11,7 +11,7 @@ module Yasuri
       @xpath, @name, @children = xpath, name, children
     end
 
-    def inject(agent, page, opt = {})
+    def inject(agent, page, opt = {}, element = page)
       fail "#{Kernel.__method__} is not implemented."
     end
     def opts

data/lib/yasuri/yasuri_paginate_node.rb

@@ -7,14 +7,17 @@ module Yasuri
   class PaginateNode
     include Node
 
-    def initialize(xpath, name, children = [], limit: nil)
+    def initialize(xpath, name, children = [], limit: nil, flatten: false)
       super(xpath, name, children)
+      @flatten = flatten
       @limit = limit
     end
 
-    def inject(agent, page, opt = {})
+    def inject(agent, page, opt = {}, element = page)
       retry_count = opt[:retry_count] || 5
 
+      raise NotImplementedError.new("PagenateNode inside StructNode, Not Supported") if page != element
+
       child_results = []
       limit = @limit.nil? ? Float::MAX : @limit
       while page
@@ -32,10 +35,14 @@ module Yasuri
         break if (limit -= 1) <= 0
       end
 
+      if @flatten == true
+        return child_results.map{|h| h.values}.flatten
+      end
+
       child_results
     end
     def opts
-      {limit:@limit}
+      {limit:@limit, flatten:@flatten}
     end
   end
 end

data/lib/yasuri/yasuri_struct_node.rb

@@ -6,12 +6,12 @@ require_relative 'yasuri_node'
 module Yasuri
   class StructNode
     include Node
-    def inject(agent, page, opt = {})
-      sub_tags = page.search(@xpath)
+    def inject(agent, page, opt = {}, element = page)
+      sub_tags = element.search(@xpath)
       tree = sub_tags.map do |sub_tag|
         child_results_kv = @children.map do |child_node|
           child_name = Yasuri.NodeName(child_node.name, opt)
-          [child_name, child_node.inject(agent, sub_tag, opt)]
+          [child_name, child_node.inject(agent, page, opt, sub_tag)]
         end
         Hash[child_results_kv]
       end

data/lib/yasuri/yasuri_text_node.rb

@@ -18,8 +18,8 @@ module Yasuri
 
     end
 
-    def inject(agent, page, opt = {})
-      node = page.search(@xpath)
+    def inject(agent, page, opt = {}, element = page)
+      node = element.search(@xpath)
       text = node.text.to_s
 
       if @truncate

data/spec/htdocs/struct/structual_links.html

@@ -0,0 +1,30 @@
+<html>
+  <head>
+    <title>StructualLinksTest</title>
+  </head>
+  <body>
+
+    <table>
+      <thead>
+        <tr>
+          <th>Title</th>
+          <th>Links</th>
+        </tr>
+      </thead>
+      <tr>
+        <td>Child01,02</td>
+        <td><a href="../child01.html">Child01</a></td>
+        <td><a href="../child02.html">Child02</a></td>
+        <td>../child02.html</td>
+      </tr>
+
+      <tr>
+        <td>Child01,02,03</td>
+        <td><a href="../child01.html">Child01</a></td>
+        <td><a href="../child02.html">Child02</a></td>
+        <td><a href="../child03.html">Child03</a></td>
+      </tr>
+    </table>
+
+  </body>
+</html>

data/spec/yasuri_paginate_node_spec.rb

@@ -30,6 +30,49 @@ describe 'Yasuri' do
       expect(actual).to match expected
     end
 
+    it "scrape each paginated pages with flatten" do
+      root_node = Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "root", [
+        Yasuri::TextNode.new('/html/body/p', "content"),
+        Yasuri::StructNode.new('/html/body/nav/span', "span", [
+          Yasuri::TextNode.new('./a', "text"),
+        ]),
+      ], flatten: true)
+      actual = root_node.inject(@agent, @page)
+      expected = [
+        "PaginationTest01",
+        {"text"=>""},
+        {"text"=>""},
+        {"text" => "2"},
+        {"text" => "3"},
+        {"text" => "4"},
+        {"text"=>"NextPage »"},
+        "PaginationTest02",
+        {"text"=>"« PreviousPage"},
+        {"text" => "1"},
+        {"text"=>""},
+        {"text" => "3"},
+        {"text" => "4"},
+        {"text"=>"NextPage »"},
+        "PaginationTest03",
+        {"text"=>"« PreviousPage"},
+        {"text" => "1"},
+        {"text" => "2"},
+        {"text"=>""},
+        {"text" => "4"},
+        {"text"=>"NextPage »"},
+        "PaginationTest04",
+        {"text"=>"« PreviousPage"},
+        {"text" => "1"},
+        {"text" => "2"},
+        {"text" => "3"},
+        {"text"=>""},
+        {"text"=>""},
+      ]
+
+      expect(actual).to match expected
+    end
+
+
     it "scrape each paginated pages limited" do
       root_node = Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "root", [
         Yasuri::TextNode.new('/html/body/p', "content"),

data/spec/yasuri_spec.rb

@@ -126,7 +126,7 @@ describe 'Yasuri' do
           Yasuri::TextNode.new('./td[2]', "pub_date"),
         ])
       ])
-      page = @agent.get(@uri + "/structual_text.html")
+      page = @agent.get(@uri + "/struct/structual_text.html")
       compare_generated_vs_original(generated, original, page)
     end
   end
@@ -193,6 +193,7 @@ describe 'Yasuri' do
                            "name"     : "root",
                            "path"     : "/html/body/nav/span/a[@class='next']",
                            "limit"    : 10,
+                           "flatten"  : false,
                            "children" : [ { "node" : "text",
                                             "name" : "content",
                                             "path" : "/html/body/p"

data/spec/yasuri_struct_node_spec.rb

@@ -12,7 +12,7 @@ describe 'Yasuri' do
   describe '::StructNode' do
     before do
       @agent = Mechanize.new
-      @page = @agent.get(uri + "/structual_text.html")
+      @page = @agent.get(uri + "/struct/structual_text.html")
 
       @table_1996 = [
         { "title"    => "The Perfect Insider",
@@ -132,4 +132,45 @@ describe 'Yasuri' do
     end
 
   end
+
+  describe '::StructNode::Links' do
+    before do
+      @agent = Mechanize.new
+      @page = @agent.get(uri + "/struct/structual_links.html")
+
+      @table = [
+        { "title" => "Child01,02",
+          "child" => [{"p" => "Child 01 page."}, {"p" => "Child 02 page."}] },
+
+        { "title" => "Child01,02,03",
+          "child" => [{"p" => "Child 01 page."}, {"p" => "Child 02 page."}, {"p" => "Child 03 page."}]}
+      ]
+    end
+
+    it 'return child node in links inside struct' do
+      node = Yasuri::StructNode.new('/html/body/table/tr', "table", [
+        Yasuri::TextNode.new('./td[1]', "title"),
+        Yasuri::LinksNode.new('./td/a', "child", [
+          Yasuri::TextNode.new('/html/body/p', "p"),
+        ])
+      ])
+      expected = @table
+      actual = node.inject(@agent, @page)
+      expect(actual).to match expected
+    end
+  end # descrive
+
+  describe '::StructNode::Pages' do
+    before do
+      @agent = Mechanize.new
+      @page = @agent.get(uri + "/struct/structual_text.html") #dummy
+    end
+
+    it 'not supported' do
+      node = Yasuri::StructNode.new('/html/body/table[1]/tr', "table", [
+        Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "pages", [])
+      ])
+      expect{ node.inject(@agent, @page) }.to raise_error(NotImplementedError, "PagenateNode inside StructNode, Not Supported")
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: yasuri
 version: !ruby/object:Gem::Version
-  version: 2.0.11
+  version: 2.0.12
 platform: ruby
 authors:
 - TAC
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-11-14 00:00:00.000000000 Z
+date: 2016-12-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -174,7 +174,8 @@ files:
 - spec/htdocs/pagination/page02.html
 - spec/htdocs/pagination/page03.html
 - spec/htdocs/pagination/page04.html
-- spec/htdocs/structual_text.html
+- spec/htdocs/struct/structual_links.html
+- spec/htdocs/struct/structual_text.html
 - spec/servers/httpserver.rb
 - spec/spec_helper.rb
 - spec/yasuri_links_node_spec.rb
@@ -204,7 +205,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.5.2
 signing_key: 
 specification_version: 4
 summary: Yasuri is easy scraping library.
@@ -220,7 +221,8 @@ test_files:
 - spec/htdocs/pagination/page02.html
 - spec/htdocs/pagination/page03.html
 - spec/htdocs/pagination/page04.html
-- spec/htdocs/structual_text.html
+- spec/htdocs/struct/structual_links.html
+- spec/htdocs/struct/structual_text.html
 - spec/servers/httpserver.rb
 - spec/spec_helper.rb
 - spec/yasuri_links_node_spec.rb