yasuri 1.9.11 → 1.9.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 28e6a3903cec3d8036b718a7c5de4fb8df8dfbfa
-  data.tar.gz: fbfb4c2b3a042410a7d05b1604e004bdffd82779
+  metadata.gz: 87406480e622911dca3649ba2a8e5b134ccffb36
+  data.tar.gz: 2fde6a481bed02e569a5c08f6df96c09925abee0
 SHA512:
-  metadata.gz: e4312623046ecf451ef261b1d99164b7f79b861ad8d7fe49d7ba8fd2319cd840978ae88235e6fb10b6991cd5131ebec966ec56e3bddece8fbbefd0b53d4a9dfe
-  data.tar.gz: 54639066aa4511309a1f712b8920c81c09a7489e0cab6a91c453d2aeca07d4f1c3bed167ed25746411d5f87fa9cc9fff8c6a32567c4df8d1990f12cb053286e2
+  metadata.gz: 7c274f2316495aea66d737f053119dd71c154f7411b9cd54b102c71ee2e7ac36602dd0e44d78d0d06895ee0e27f934a9ac0e2f45d7e52ce4629f60f9fd905cf3
+  data.tar.gz: a5696ca1fac061c542c7f0586bfecdf623962443e98207f897ff76ff0204cc78e53505b070f18467d2e72c0ddb527383224fb75ff2d05b5ff6e5a5149caaa20a

data/USAGE.ja.md

@@ -431,3 +431,38 @@ node.inject(agent, page)
 #=> [ {"content" => "Pagination01"}, {"content" => "Pagination02"}]
 ```
 この場合、PaginateNode は最大2つまでのページを開いてパースします．ページネーションは4つのページを持っているようですが、`limit:2`が指定されているため、結果の配列には2つの結果のみが含まれています．
+
+##### `flatten`
+取得した各ページの結果を展開します．
+
+```ruby
+agent = Mechanize.new
+page = agent.get("http://yasuri.example.net/page01.html")
+
+node = Yasuri.pages_root "/html/body/nav/span/a[@class='next']" , flatten:true do
+         text_title   '/html/head/title'
+         text_content '/html/body/p'
+       end
+node.inject(agent, page)
+
+#=> [ {"title" => "Page01",
+       "content" => "Patination01"},
+      {"title"   => "Page01",
+       "content" => "Patination02"},
+      {"title"   => "Page01",
+       "content" => "Patination03"}]
+
+
+node = Yasuri.pages_root "/html/body/nav/span/a[@class='next']" , flatten:true do
+        text_title   '/html/head/title'
+        text_content '/html/body/p'
+      end
+node.inject(agent, page)
+
+#=> [ "Page01",
+      "Patination01",
+      "Page02",
+      "Patination02",
+      "Page03",
+      "Patination03"]
+```

data/USAGE.md

@@ -429,3 +429,38 @@ node.inject(agent, page)
 #=> [ {"content" => "Pagination01"}, {"content" => "Pagination02"}]
 ```
 Paginate Node open upto 2 given by `limit`. In this situation, pagination has 4 pages, but result Array has 2 texts because given `limit:2`.
+
+##### `flatten`
+`flatten` option expands each page results.
+
+```ruby
+agent = Mechanize.new
+page = agent.get("http://yasuri.example.net/page01.html")
+
+node = Yasuri.pages_root "/html/body/nav/span/a[@class='next']" , flatten:true do
+         text_title   '/html/head/title'
+         text_content '/html/body/p'
+       end
+node.inject(agent, page)
+
+#=> [ {"title" => "Page01",
+       "content" => "Patination01"},
+      {"title"   => "Page01",
+       "content" => "Patination02"},
+      {"title"   => "Page01",
+       "content" => "Patination03"}]
+
+
+node = Yasuri.pages_root "/html/body/nav/span/a[@class='next']" , flatten:true do
+        text_title   '/html/head/title'
+        text_content '/html/body/p'
+      end
+node.inject(agent, page)
+
+#=> [ "Page01",
+      "Patination01",
+      "Page02",
+      "Patination02",
+      "Page03",
+      "Patination03"]
+```

data/lib/yasuri/version.rb

@@ -1,3 +1,3 @@
 module Yasuri
-  VERSION = "1.9.11"
+  VERSION = "1.9.12"
 end

data/lib/yasuri/yasuri_links_node.rb

@@ -6,10 +6,10 @@ require_relative 'yasuri_node'
 module Yasuri
   class LinksNode
     include Node
-    def inject(agent, page, opt = {})
+    def inject(agent, page, opt = {}, element = page)
       retry_count = opt[:retry_count] || 5
 
-      links = page.search(@xpath) || [] # links expected
+      links = element.search(@xpath) || [] # links expected
       links.map do |link|
         link_button = Mechanize::Page::Link.new(link, agent, page)
         child_page = Yasuri.with_retry(retry_count) { link_button.click }

data/lib/yasuri/yasuri_node.rb

@@ -11,7 +11,7 @@ module Yasuri
       @xpath, @name, @children = xpath, name, children
     end
 
-    def inject(agent, page, opt = {})
+    def inject(agent, page, opt = {}, element = page)
       fail "#{Kernel.__method__} is not implemented."
     end
     def opts

data/lib/yasuri/yasuri_paginate_node.rb

@@ -10,11 +10,14 @@ module Yasuri
     def initialize(xpath, name, children = [], hash = {})
       super(xpath, name, children)
       @limit = hash[:limit]
+      @flatten = hash[:flatten] || false
     end
 
-    def inject(agent, page, opt = {})
+    def inject(agent, page, opt = {}, element = page)
       retry_count = opt[:retry_count] || 5
 
+      raise NotImplementedError.new("PagenateNode inside StructNode, Not Supported") if page != element
+
       child_results = []
       limit = @limit.nil? ? Float::MAX : @limit
       while page
@@ -32,10 +35,14 @@ module Yasuri
         break if (limit -= 1) <= 0
       end
 
+      if @flatten == true
+        return child_results.map{|h| h.values}.flatten
+      end
+
       child_results
     end
     def opts
-      {limit:@limit}
+      {limit:@limit, flatten:@flatten}
     end
   end
 end

data/lib/yasuri/yasuri_struct_node.rb

@@ -6,12 +6,12 @@ require_relative 'yasuri_node'
 module Yasuri
   class StructNode
     include Node
-    def inject(agent, page, opt = {})
-      sub_tags = page.search(@xpath)
+    def inject(agent, page, opt = {}, element = page)
+      sub_tags = element.search(@xpath)
       tree = sub_tags.map do |sub_tag|
         child_results_kv = @children.map do |child_node|
           child_name = Yasuri.NodeName(child_node.name, opt)
-          [child_name, child_node.inject(agent, sub_tag, opt)]
+          [child_name, child_node.inject(agent, page, opt, sub_tag)]
         end
         Hash[child_results_kv]
       end

data/lib/yasuri/yasuri_text_node.rb

@@ -21,8 +21,8 @@ module Yasuri
 
     end
 
-    def inject(agent, page, opt = {})
-      node = page.search(@xpath)
+    def inject(agent, page, opt = {}, element = page)
+      node = element.search(@xpath)
       text = node.text.to_s
 
       if @truncate

data/spec/htdocs/struct/structual_links.html

@@ -0,0 +1,30 @@
+<html>
+  <head>
+    <title>StructualLinksTest</title>
+  </head>
+  <body>
+
+    <table>
+      <thead>
+        <tr>
+          <th>Title</th>
+          <th>Links</th>
+        </tr>
+      </thead>
+      <tr>
+        <td>Child01,02</td>
+        <td><a href="../child01.html">Child01</a></td>
+        <td><a href="../child02.html">Child02</a></td>
+        <td>../child02.html</td>
+      </tr>
+
+      <tr>
+        <td>Child01,02,03</td>
+        <td><a href="../child01.html">Child01</a></td>
+        <td><a href="../child02.html">Child02</a></td>
+        <td><a href="../child03.html">Child03</a></td>
+      </tr>
+    </table>
+
+  </body>
+</html>

data/spec/yasuri_paginate_node_spec.rb

@@ -30,6 +30,49 @@ describe 'Yasuri' do
       expect(actual).to match expected
     end
 
+    it "scrape each paginated pages with flatten" do
+      root_node = Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "root", [
+        Yasuri::TextNode.new('/html/body/p', "content"),
+        Yasuri::StructNode.new('/html/body/nav/span', "span", [
+          Yasuri::TextNode.new('./a', "text"),
+        ]),
+      ], flatten: true)
+      actual = root_node.inject(@agent, @page)
+      expected = [
+        "PaginationTest01",
+        {"text"=>""},
+        {"text"=>""},
+        {"text" => "2"},
+        {"text" => "3"},
+        {"text" => "4"},
+        {"text"=>"NextPage »"},
+        "PaginationTest02",
+        {"text"=>"« PreviousPage"},
+        {"text" => "1"},
+        {"text"=>""},
+        {"text" => "3"},
+        {"text" => "4"},
+        {"text"=>"NextPage »"},
+        "PaginationTest03",
+        {"text"=>"« PreviousPage"},
+        {"text" => "1"},
+        {"text" => "2"},
+        {"text"=>""},
+        {"text" => "4"},
+        {"text"=>"NextPage »"},
+        "PaginationTest04",
+        {"text"=>"« PreviousPage"},
+        {"text" => "1"},
+        {"text" => "2"},
+        {"text" => "3"},
+        {"text"=>""},
+        {"text"=>""},
+      ]
+
+      expect(actual).to match expected
+    end
+
+
     it "scrape each paginated pages limited" do
       root_node = Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "root", [
         Yasuri::TextNode.new('/html/body/p', "content"),

data/spec/yasuri_spec.rb

@@ -126,7 +126,7 @@ describe 'Yasuri' do
           Yasuri::TextNode.new('./td[2]', "pub_date"),
         ])
       ])
-      page = @agent.get(@uri + "/structual_text.html")
+      page = @agent.get(@uri + "/struct/structual_text.html")
       compare_generated_vs_original(generated, original, page)
     end
   end
@@ -193,6 +193,7 @@ describe 'Yasuri' do
                            "name"     : "root",
                            "path"     : "/html/body/nav/span/a[@class='next']",
                            "limit"    : 10,
+                           "flatten"  : false,
                            "children" : [ { "node" : "text",
                                             "name" : "content",
                                             "path" : "/html/body/p"

data/spec/yasuri_struct_node_spec.rb

@@ -12,7 +12,7 @@ describe 'Yasuri' do
   describe '::StructNode' do
     before do
       @agent = Mechanize.new
-      @page = @agent.get(uri + "/structual_text.html")
+      @page = @agent.get(uri + "/struct/structual_text.html")
 
       @table_1996 = [
         { "title"    => "The Perfect Insider",
@@ -132,4 +132,45 @@ describe 'Yasuri' do
     end
 
   end
+
+  describe '::StructNode::Links' do
+    before do
+      @agent = Mechanize.new
+      @page = @agent.get(uri + "/struct/structual_links.html")
+
+      @table = [
+        { "title" => "Child01,02",
+          "child" => [{"p" => "Child 01 page."}, {"p" => "Child 02 page."}] },
+
+        { "title" => "Child01,02,03",
+          "child" => [{"p" => "Child 01 page."}, {"p" => "Child 02 page."}, {"p" => "Child 03 page."}]}
+      ]
+    end
+
+    it 'return child node in links inside struct' do
+      node = Yasuri::StructNode.new('/html/body/table/tr', "table", [
+        Yasuri::TextNode.new('./td[1]', "title"),
+        Yasuri::LinksNode.new('./td/a', "child", [
+          Yasuri::TextNode.new('/html/body/p', "p"),
+        ])
+      ])
+      expected = @table
+      actual = node.inject(@agent, @page)
+      expect(actual).to match expected
+    end
+  end # descrive
+
+  describe '::StructNode::Pages' do
+    before do
+      @agent = Mechanize.new
+      @page = @agent.get(uri + "/struct/structual_text.html") #dummy
+    end
+
+    it 'not supported' do
+      node = Yasuri::StructNode.new('/html/body/table[1]/tr', "table", [
+        Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "pages", [])
+      ])
+      expect{ node.inject(@agent, @page) }.to raise_error(NotImplementedError, "PagenateNode inside StructNode, Not Supported")
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: yasuri
 version: !ruby/object:Gem::Version
-  version: 1.9.11
+  version: 1.9.12
 platform: ruby
 authors:
 - TAC
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-11-14 00:00:00.000000000 Z
+date: 2016-12-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -174,7 +174,8 @@ files:
 - spec/htdocs/pagination/page02.html
 - spec/htdocs/pagination/page03.html
 - spec/htdocs/pagination/page04.html
-- spec/htdocs/structual_text.html
+- spec/htdocs/struct/structual_links.html
+- spec/htdocs/struct/structual_text.html
 - spec/servers/httpserver.rb
 - spec/spec_helper.rb
 - spec/yasuri_links_node_spec.rb
@@ -204,7 +205,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.5.2
 signing_key: 
 specification_version: 4
 summary: Yasuri is easy scraping library.
@@ -220,7 +221,8 @@ test_files:
 - spec/htdocs/pagination/page02.html
 - spec/htdocs/pagination/page03.html
 - spec/htdocs/pagination/page04.html
-- spec/htdocs/structual_text.html
+- spec/htdocs/struct/structual_links.html
+- spec/htdocs/struct/structual_text.html
 - spec/servers/httpserver.rb
 - spec/spec_helper.rb
 - spec/yasuri_links_node_spec.rb