yasuri 1.9.12 → 2.0.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 87406480e622911dca3649ba2a8e5b134ccffb36
-  data.tar.gz: 2fde6a481bed02e569a5c08f6df96c09925abee0
+  metadata.gz: d8d6bd8c37be444f0c5568bcf20604d7bca5c223
+  data.tar.gz: 8438eee300a7e4f73be7107cbd9417da18f5048d
 SHA512:
-  metadata.gz: 7c274f2316495aea66d737f053119dd71c154f7411b9cd54b102c71ee2e7ac36602dd0e44d78d0d06895ee0e27f934a9ac0e2f45d7e52ce4629f60f9fd905cf3
-  data.tar.gz: a5696ca1fac061c542c7f0586bfecdf623962443e98207f897ff76ff0204cc78e53505b070f18467d2e72c0ddb527383224fb75ff2d05b5ff6e5a5149caaa20a
+  metadata.gz: 107ddc8cd0310c646841e6fe6a2695313edb9692418a783e133a5d269d4a1ab39385975276ae167ac68863b9760794ebb2738832dccfc4f599686c5a9e50f244
+  data.tar.gz: b6d089de8cd866f137ca58dd779396cd4948e080d3225cc4384f8f9cdb54f5a778cd4be85b89628938ccacbf11dfefe74ea8bd248e835971470e7a64df597411

data/USAGE.ja.md

@@ -431,38 +431,3 @@ node.inject(agent, page)
 #=> [ {"content" => "Pagination01"}, {"content" => "Pagination02"}]
 ```
 この場合、PaginateNode は最大2つまでのページを開いてパースします．ページネーションは4つのページを持っているようですが、`limit:2`が指定されているため、結果の配列には2つの結果のみが含まれています．
-
-##### `flatten`
-取得した各ページの結果を展開します．
-
-```ruby
-agent = Mechanize.new
-page = agent.get("http://yasuri.example.net/page01.html")
-
-node = Yasuri.pages_root "/html/body/nav/span/a[@class='next']" , flatten:true do
-         text_title   '/html/head/title'
-         text_content '/html/body/p'
-       end
-node.inject(agent, page)
-
-#=> [ {"title" => "Page01",
-       "content" => "Patination01"},
-      {"title"   => "Page01",
-       "content" => "Patination02"},
-      {"title"   => "Page01",
-       "content" => "Patination03"}]
-
-
-node = Yasuri.pages_root "/html/body/nav/span/a[@class='next']" , flatten:true do
-        text_title   '/html/head/title'
-        text_content '/html/body/p'
-      end
-node.inject(agent, page)
-
-#=> [ "Page01",
-      "Patination01",
-      "Page02",
-      "Patination02",
-      "Page03",
-      "Patination03"]
-```

data/USAGE.md

@@ -429,38 +429,3 @@ node.inject(agent, page)
 #=> [ {"content" => "Pagination01"}, {"content" => "Pagination02"}]
 ```
 Paginate Node open upto 2 given by `limit`. In this situation, pagination has 4 pages, but result Array has 2 texts because given `limit:2`.
-
-##### `flatten`
-`flatten` option expands each page results.
-
-```ruby
-agent = Mechanize.new
-page = agent.get("http://yasuri.example.net/page01.html")
-
-node = Yasuri.pages_root "/html/body/nav/span/a[@class='next']" , flatten:true do
-         text_title   '/html/head/title'
-         text_content '/html/body/p'
-       end
-node.inject(agent, page)
-
-#=> [ {"title" => "Page01",
-       "content" => "Patination01"},
-      {"title"   => "Page01",
-       "content" => "Patination02"},
-      {"title"   => "Page01",
-       "content" => "Patination03"}]
-
-
-node = Yasuri.pages_root "/html/body/nav/span/a[@class='next']" , flatten:true do
-        text_title   '/html/head/title'
-        text_content '/html/body/p'
-      end
-node.inject(agent, page)
-
-#=> [ "Page01",
-      "Patination01",
-      "Page02",
-      "Patination02",
-      "Page03",
-      "Patination03"]
-```

data/lib/yasuri/version.rb

@@ -1,3 +1,3 @@
 module Yasuri
-  VERSION = "1.9.12"
+  VERSION = "2.0.11"
 end

data/lib/yasuri/yasuri.rb

@@ -37,7 +37,7 @@ module Yasuri
   }
   Node2Text = Text2Node.invert
 
-  ReservedKeys = [:node, :name, :path, :children]
+  ReservedKeys = %i|node name path children|
   def self.hash2node(node_h)
     node, name, path, children = ReservedKeys.map do |key|
       node_h[key]
@@ -78,8 +78,7 @@ module Yasuri
     json
   end
 
-  def self.NodeName(name, hash = {})
-    symbolize_names = hash[:symbolize_names] || false
+  def self.NodeName(name, symbolize_names:false)
     symbolize_names ? name.to_sym : name
   end
 

data/lib/yasuri/yasuri_links_node.rb

@@ -6,10 +6,10 @@ require_relative 'yasuri_node'
 module Yasuri
   class LinksNode
     include Node
-    def inject(agent, page, opt = {}, element = page)
+    def inject(agent, page, opt = {})
       retry_count = opt[:retry_count] || 5
 
-      links = element.search(@xpath) || [] # links expected
+      links = page.search(@xpath) || [] # links expected
       links.map do |link|
         link_button = Mechanize::Page::Link.new(link, agent, page)
         child_page = Yasuri.with_retry(retry_count) { link_button.click }

data/lib/yasuri/yasuri_node.rb

@@ -7,11 +7,11 @@ module Yasuri
   module Node
     attr_reader :url, :xpath, :name, :children
 
-    def initialize(xpath, name, children = [], opt = {})
+    def initialize(xpath, name, children = [], opt: {})
       @xpath, @name, @children = xpath, name, children
     end
 
-    def inject(agent, page, opt = {}, element = page)
+    def inject(agent, page, opt = {})
       fail "#{Kernel.__method__} is not implemented."
     end
     def opts

data/lib/yasuri/yasuri_paginate_node.rb

@@ -7,17 +7,14 @@ module Yasuri
   class PaginateNode
     include Node
 
-    def initialize(xpath, name, children = [], hash = {})
+    def initialize(xpath, name, children = [], limit: nil)
       super(xpath, name, children)
-      @limit = hash[:limit]
-      @flatten = hash[:flatten] || false
+      @limit = limit
     end
 
-    def inject(agent, page, opt = {}, element = page)
+    def inject(agent, page, opt = {})
       retry_count = opt[:retry_count] || 5
 
-      raise NotImplementedError.new("PagenateNode inside StructNode, Not Supported") if page != element
-
       child_results = []
       limit = @limit.nil? ? Float::MAX : @limit
       while page
@@ -35,14 +32,10 @@ module Yasuri
         break if (limit -= 1) <= 0
       end
 
-      if @flatten == true
-        return child_results.map{|h| h.values}.flatten
-      end
-
       child_results
     end
     def opts
-      {limit:@limit, flatten:@flatten}
+      {limit:@limit}
     end
   end
 end

data/lib/yasuri/yasuri_struct_node.rb

@@ -6,12 +6,12 @@ require_relative 'yasuri_node'
 module Yasuri
   class StructNode
     include Node
-    def inject(agent, page, opt = {}, element = page)
-      sub_tags = element.search(@xpath)
+    def inject(agent, page, opt = {})
+      sub_tags = page.search(@xpath)
       tree = sub_tags.map do |sub_tag|
         child_results_kv = @children.map do |child_node|
           child_name = Yasuri.NodeName(child_node.name, opt)
-          [child_name, child_node.inject(agent, page, opt, sub_tag)]
+          [child_name, child_node.inject(agent, sub_tag, opt)]
         end
         Hash[child_results_kv]
       end

data/lib/yasuri/yasuri_text_node.rb

@@ -7,12 +7,9 @@ module Yasuri
   class TextNode
     include Node
 
-    def initialize(xpath, name, children = [], hash = {})
+    def initialize(xpath, name, children = [], truncate: nil, proc:nil)
       super(xpath, name, children)
 
-      truncate = hash[:truncate]
-      proc     = hash[:proc]
-
       truncate = Regexp.new(truncate) if not truncate.nil? # regexp or nil
       @truncate = truncate
       @truncate = Regexp.new(@truncate.to_s) if not @truncate.nil?
@@ -21,8 +18,8 @@ module Yasuri
 
     end
 
-    def inject(agent, page, opt = {}, element = page)
-      node = element.search(@xpath)
+    def inject(agent, page, opt = {})
+      node = page.search(@xpath)
       text = node.text.to_s
 
       if @truncate

data/spec/yasuri_paginate_node_spec.rb

@@ -30,49 +30,6 @@ describe 'Yasuri' do
       expect(actual).to match expected
     end
 
-    it "scrape each paginated pages with flatten" do
-      root_node = Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "root", [
-        Yasuri::TextNode.new('/html/body/p', "content"),
-        Yasuri::StructNode.new('/html/body/nav/span', "span", [
-          Yasuri::TextNode.new('./a', "text"),
-        ]),
-      ], flatten: true)
-      actual = root_node.inject(@agent, @page)
-      expected = [
-        "PaginationTest01",
-        {"text"=>""},
-        {"text"=>""},
-        {"text" => "2"},
-        {"text" => "3"},
-        {"text" => "4"},
-        {"text"=>"NextPage »"},
-        "PaginationTest02",
-        {"text"=>"« PreviousPage"},
-        {"text" => "1"},
-        {"text"=>""},
-        {"text" => "3"},
-        {"text" => "4"},
-        {"text"=>"NextPage »"},
-        "PaginationTest03",
-        {"text"=>"« PreviousPage"},
-        {"text" => "1"},
-        {"text" => "2"},
-        {"text"=>""},
-        {"text" => "4"},
-        {"text"=>"NextPage »"},
-        "PaginationTest04",
-        {"text"=>"« PreviousPage"},
-        {"text" => "1"},
-        {"text" => "2"},
-        {"text" => "3"},
-        {"text"=>""},
-        {"text"=>""},
-      ]
-
-      expect(actual).to match expected
-    end
-
-
     it "scrape each paginated pages limited" do
       root_node = Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "root", [
         Yasuri::TextNode.new('/html/body/p', "content"),

data/spec/yasuri_spec.rb

@@ -39,7 +39,7 @@ describe 'Yasuri' do
                   "truncate"  : "^[^,]+"
                 }|
       generated = Yasuri.json2tree(src)
-      original  = Yasuri::TextNode.new('/html/body/p[1]', "content", {}, truncate:/^[^,]+/)
+      original  = Yasuri::TextNode.new('/html/body/p[1]', "content", truncate:/^[^,]+/)
       compare_generated_vs_original(generated, original, @index_page)
     end
 
@@ -126,7 +126,7 @@ describe 'Yasuri' do
           Yasuri::TextNode.new('./td[2]', "pub_date"),
         ])
       ])
-      page = @agent.get(@uri + "/struct/structual_text.html")
+      page = @agent.get(@uri + "/structual_text.html")
       compare_generated_vs_original(generated, original, page)
     end
   end
@@ -153,7 +153,7 @@ describe 'Yasuri' do
     end
 
     it "return text node with truncate_regexp" do
-      node = Yasuri::TextNode.new("/html/head/title", "title", {}, truncate:/^[^,]+/)
+      node = Yasuri::TextNode.new("/html/head/title", "title", truncate:/^[^,]+/)
       json = Yasuri.tree2json(node)
       expected_str = %q| { "node": "text",
                            "name": "title",
@@ -193,7 +193,6 @@ describe 'Yasuri' do
                            "name"     : "root",
                            "path"     : "/html/body/nav/span/a[@class='next']",
                            "limit"    : 10,
-                           "flatten"  : false,
                            "children" : [ { "node" : "text",
                                             "name" : "content",
                                             "path" : "/html/body/p"

data/spec/yasuri_struct_node_spec.rb

@@ -12,7 +12,7 @@ describe 'Yasuri' do
   describe '::StructNode' do
     before do
       @agent = Mechanize.new
-      @page = @agent.get(uri + "/struct/structual_text.html")
+      @page = @agent.get(uri + "/structual_text.html")
 
       @table_1996 = [
         { "title"    => "The Perfect Insider",
@@ -126,51 +126,10 @@ describe 'Yasuri' do
         Yasuri::TextNode.new('./td[1]', "title"),
         Yasuri::TextNode.new('./td[2]', "pub_date"),
       ])
-      expected = @table_1996.map{|h| Hash[h.map{|k,v| [k.to_sym, v] }] }
+      expected = @table_1996.map{|h| h.map{|k,v| [k.to_sym, v] }.to_h }
       actual = node.inject(@agent, @page, symbolize_names:true)
       expect(actual).to match expected
     end
 
   end
-
-  describe '::StructNode::Links' do
-    before do
-      @agent = Mechanize.new
-      @page = @agent.get(uri + "/struct/structual_links.html")
-
-      @table = [
-        { "title" => "Child01,02",
-          "child" => [{"p" => "Child 01 page."}, {"p" => "Child 02 page."}] },
-
-        { "title" => "Child01,02,03",
-          "child" => [{"p" => "Child 01 page."}, {"p" => "Child 02 page."}, {"p" => "Child 03 page."}]}
-      ]
-    end
-
-    it 'return child node in links inside struct' do
-      node = Yasuri::StructNode.new('/html/body/table/tr', "table", [
-        Yasuri::TextNode.new('./td[1]', "title"),
-        Yasuri::LinksNode.new('./td/a', "child", [
-          Yasuri::TextNode.new('/html/body/p', "p"),
-        ])
-      ])
-      expected = @table
-      actual = node.inject(@agent, @page)
-      expect(actual).to match expected
-    end
-  end # descrive
-
-  describe '::StructNode::Pages' do
-    before do
-      @agent = Mechanize.new
-      @page = @agent.get(uri + "/struct/structual_text.html") #dummy
-    end
-
-    it 'not supported' do
-      node = Yasuri::StructNode.new('/html/body/table[1]/tr', "table", [
-        Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "pages", [])
-      ])
-      expect{ node.inject(@agent, @page) }.to raise_error(NotImplementedError, "PagenateNode inside StructNode, Not Supported")
-    end
-  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: yasuri
 version: !ruby/object:Gem::Version
-  version: 1.9.12
+  version: 2.0.11
 platform: ruby
 authors:
 - TAC
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-12-15 00:00:00.000000000 Z
+date: 2016-11-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -174,8 +174,7 @@ files:
 - spec/htdocs/pagination/page02.html
 - spec/htdocs/pagination/page03.html
 - spec/htdocs/pagination/page04.html
-- spec/htdocs/struct/structual_links.html
-- spec/htdocs/struct/structual_text.html
+- spec/htdocs/structual_text.html
 - spec/servers/httpserver.rb
 - spec/spec_helper.rb
 - spec/yasuri_links_node_spec.rb
@@ -205,7 +204,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.2
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: Yasuri is easy scraping library.
@@ -221,8 +220,7 @@ test_files:
 - spec/htdocs/pagination/page02.html
 - spec/htdocs/pagination/page03.html
 - spec/htdocs/pagination/page04.html
-- spec/htdocs/struct/structual_links.html
-- spec/htdocs/struct/structual_text.html
+- spec/htdocs/structual_text.html
 - spec/servers/httpserver.rb
 - spec/spec_helper.rb
 - spec/yasuri_links_node_spec.rb

data/spec/htdocs/struct/structual_links.html

@@ -1,30 +0,0 @@
-<html>
-  <head>
-    <title>StructualLinksTest</title>
-  </head>
-  <body>
-
-    <table>
-      <thead>
-        <tr>
-          <th>Title</th>
-          <th>Links</th>
-        </tr>
-      </thead>
-      <tr>
-        <td>Child01,02</td>
-        <td><a href="../child01.html">Child01</a></td>
-        <td><a href="../child02.html">Child02</a></td>
-        <td>../child02.html</td>
-      </tr>
-
-      <tr>
-        <td>Child01,02,03</td>
-        <td><a href="../child01.html">Child01</a></td>
-        <td><a href="../child02.html">Child02</a></td>
-        <td><a href="../child03.html">Child03</a></td>
-      </tr>
-    </table>
-
-  </body>
-</html>