yasuri 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1573790f94cdd8bf8621b1178f7c5ef5e00eee15
-  data.tar.gz: b37d3a38cf679b38a1d5d9dba2bc785e54d6d2b3
+  metadata.gz: 66449daf368bdf42e7406cdf1b7db1eedc9625c9
+  data.tar.gz: d0bbef804af0da5228594df407f5e47edf5cf14e
 SHA512:
-  metadata.gz: 8c8da39e14541e21ed520c051a4e425d6a5aed2f530b7b50e1652894ead3678d4717db7aa686a64f6c6f905d3df1e1bbfe71d1226381b0f08778a043fafaa2ba
-  data.tar.gz: 1ee0447e8757ea26ff7b4dc36cc089dda9003be8de746259e8f430d27c608e0519e2558daf37b145f009e69b909c6647990f1e46876c47d47df7ba24b75f3862
+  metadata.gz: f2fce89a90d0d878d12d48b1e803d58581e1b3442efc76cfe2e3b938c048ada3fd558099634a3d25a2f48851771ee1b047e3c8303eed51ebf6dd9e5bf1eae122
+  data.tar.gz: 58f748c39156abade472ee3c8c0b9332c5321fc6562dad5a35cb3a01e80c9e7cafb9abb327ae5626b50aaf4afd4d4f2e67e2da5b341ca575bf60514b924ad19e

data/lib/yasuri/version.rb

@@ -1,3 +1,3 @@
 module Yasuri
-  VERSION = "0.0.2"
+  VERSION = "0.0.3"
 end

data/lib/yasuri/yasuri.rb

@@ -21,19 +21,27 @@ module Yasuri
 
   class TextNode
     include Node
-    def inject(agent, page)
+    def initialize(xpath, name, children = [])
+      super(xpath, name, children)
+      @truncate_regexp, dummy = *children
+    end
+    def inject(agent, page, retry_count = 5)
       node = page.search(@xpath)
-      node.text.to_s
+      text = node.text.to_s
+
+      text = text[@truncate_regexp, 0] if @truncate_regexp
+
+      text.to_s
     end
   end
 
   class StructNode
     include Node
-    def inject(agent, page)
+    def inject(agent, page, retry_count = 5)
       sub_tags = page.search(@xpath)
       sub_tags.map do |sub_tag|
         child_results_kv = @children.map do |child_node|
-          [child_node.name, child_node.inject(agent, sub_tag)]
+          [child_node.name, child_node.inject(agent, sub_tag, retry_count)]
         end
         Hash[child_results_kv]
       end
@@ -42,14 +50,14 @@ module Yasuri
 
   class LinksNode
     include Node
-    def inject(agent, page)
+    def inject(agent, page, retry_count = 5)
       links = page.search(@xpath) || [] # links expected
       links.map do |link|
         link_button = Mechanize::Page::Link.new(link, agent, page)
-        child_page = link_button.click
+        child_page = Yasuri.with_retry(retry_count) { link_button.click }
 
         child_results_kv = @children.map do |child_node|
-          [child_node.name, child_node.inject(agent, child_page)]
+          [child_node.name, child_node.inject(agent, child_page, retry_count)]
         end
 
         Hash[child_results_kv]
@@ -59,12 +67,12 @@ module Yasuri
 
   class PaginateNode
     include Node
-    def inject(agent, page)
+    def inject(agent, page, retry_count = 5)
 
       child_results = []
       while page
         child_results_kv = @children.map do |child_node|
-          [child_node.name, child_node.inject(agent, page)]
+          [child_node.name, child_node.inject(agent, page, retry_count)]
         end
         child_results << Hash[child_results_kv]
 
@@ -72,7 +80,7 @@ module Yasuri
         break if link == nil
 
         link_button = Mechanize::Page::Link.new(link, agent, page)
-        page = link_button.click
+        page = Yasuri.with_retry(retry_count) { link_button.click }
       end
 
       child_results
@@ -98,6 +106,7 @@ module Yasuri
 
       case name
       when /^text_(.+)$/
+        truncate_regexp, dummy = children
         Yasuri::TextNode.new(xpath, $1, children || [])
       when /^struct_(.+)$/
         Yasuri::StructNode.new(xpath, $1, children || [])
@@ -134,6 +143,19 @@ module Yasuri
     klass = Text2Node[node]
     klass ? klass.new(path, name, childnodes) : nil
   end
+
+  def self.with_retry(retry_count = 5)
+    begin
+      return yield() if block_given?
+    rescue => e
+      if retry_count > 0
+        pp "retry #{retry_count}"
+        retry_count -= 1
+        retry
+      end
+      fail e
+    end
+  end
 end
 
 # alias for DSL

data/spec/yasuri_spec.rb

@@ -24,6 +24,9 @@ describe 'Yasuri' do
     expect(actual).to match expected
   end
 
+  ########
+  # Text #
+  ########
   describe '::TextNode' do
     before { @node = Yasuri::TextNode.new('/html/body/p[1]', "title") }
 
@@ -32,13 +35,46 @@ describe 'Yasuri' do
       expect(actual).to eq "Hello,Yasuri"
     end
 
+    it 'return empty text if no match node' do
+      no_match_node = Yasuri::TextNode.new('/html/body/no_match_node', "title")
+      actual = no_match_node.inject(@agent, @index_page)
+      expect(actual).to be_empty
+    end
+
+    it 'fail with invalid xpath' do
+      invalid_xpath = '/html/body/no_match_node['
+      node = Yasuri::TextNode.new(invalid_xpath, "title")
+      expect { node.inject(@agent, @index_page) }.to raise_error
+    end
+
     it "can be defined by DSL, return single TextNode title" do
       generated = text_title '/html/body/p[1]'
       original  = Yasuri::TextNode.new('/html/body/p[1]', "title")
       compare_generated_vs_original(generated, original)
     end
+
+    it "can be truncated with regexp" do
+      node  = text_title '/html/body/p[1]', /^[^,]+/
+      actual = node.inject(@agent, @index_page)
+      expect(actual).to eq "Hello"
+    end
+
+    it "can be truncated with regexp" do
+      node = text_title '/html/body/p[1]', /[^,]+$/
+      actual = node.inject(@agent, @index_page)
+      expect(actual).to eq "Yasuri"
+    end
+
+    it "return empty string if truncated with no match to regexp" do
+      node = text_title '/html/body/p[1]', /^hoge/
+      actual = node.inject(@agent, @index_page)
+      expect(actual).to be_empty
+    end
   end
 
+  ##########
+  # Struct #
+  ##########
   describe '::StructNode' do
     before do
       @page = @agent.get(@uri + "/structual_text.html")
@@ -84,6 +120,32 @@ describe 'Yasuri' do
       expect(actual).to match expected
     end
 
+    it 'return empty text if no match node' do
+      no_match_xpath = '/html/body/table[1]/t'
+      node = Yasuri::StructNode.new(no_match_xpath, "table", [
+        Yasuri::TextNode.new('./td[1]', "title")
+      ])
+      actual = node.inject(@agent, @page)
+      expect(actual).to be_empty
+    end
+
+    it 'fail with invalid xpath' do
+      invalid_xpath = '/html/body/table[1]/table[1]/tr['
+      node = Yasuri::StructNode.new(invalid_xpath, "table", [
+        Yasuri::TextNode.new('./td[1]', "title")
+      ])
+      expect { node.inject(@agent, @page) }.to raise_error
+    end
+
+    it 'fail with invalid xpath in children' do
+      invalid_xpath = './td[1]['
+      node = Yasuri::StructNode.new('/html/body/table[1]/tr', "table", [
+        Yasuri::TextNode.new(invalid_xpath, "title"),
+        Yasuri::TextNode.new('./td[2]', "pub_date"),
+      ])
+      expect { node.inject(@agent, @page) }.to raise_error
+    end
+
     it 'scrape all tables' do
       node = Yasuri::StructNode.new('/html/body/table', "tables", [
         Yasuri::StructNode.new('./tr', "table", [
@@ -113,6 +175,9 @@ describe 'Yasuri' do
     end
   end
 
+  #########
+  # Links #
+  #########
   describe '::LinksNode' do
     it 'scrape links' do
       root_node = Yasuri::LinksNode.new('/html/body/a', "root", [
@@ -128,6 +193,16 @@ describe 'Yasuri' do
       expect(actual).to match expected
     end
 
+    it 'return empty set if no match node' do
+      missing_xpath = '/html/body/b'
+      root_node = Yasuri::LinksNode.new(missing_xpath, "root", [
+        Yasuri::TextNode.new('/html/body/p', "content"),
+      ])
+
+      actual = root_node.inject(@agent, @index_page)
+      expect(actual).to be_empty
+    end
+
     it 'scrape links, recursive' do
       root_node = Yasuri::LinksNode.new('/html/body/a', "root", [
         Yasuri::TextNode.new('/html/body/p', "content"),
@@ -180,6 +255,9 @@ describe 'Yasuri' do
     end
   end
 
+  ############
+  # Paginate #
+  ############
   describe '::PaginateNode' do
     before do
       @uri += "/pagination/page01.html"
@@ -200,6 +278,25 @@ describe 'Yasuri' do
       expect(actual).to match expected
     end
 
+    it 'return first content if paginate link node is not found' do
+      missing_xpath = "/html/body/nav/span/b[@class='next']"
+      root_node = Yasuri::PaginateNode.new(missing_xpath, "root", [
+        Yasuri::TextNode.new('/html/body/p', "content"),
+      ])
+      actual = root_node.inject(@agent, @page)
+      expected = [ {"content" => "PaginationTest01"}, ]
+      expect(actual).to match_array expected
+    end
+
+    it 'return empty hashes if content node is not found' do
+      root_node = Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "root", [
+        Yasuri::TextNode.new('/html/body/hoge', "content"),
+      ])
+      actual = root_node.inject(@agent, @page)
+      expected = [ {"content" => ""}, {"content" => ""}, {"content" => ""}, {"content" => ""},]
+      expect(actual).to match_array expected
+    end
+
     it 'can be defined by DSL, return single PaginateNode content' do
       generated = pages_next "/html/body/nav/span/a[@class='next']" do
         text_content '/html/body/p'
@@ -211,6 +308,9 @@ describe 'Yasuri' do
     end
   end
 
+  #############
+  # json2tree #
+  #############
   describe '.json2tree' do
     it "return empty tree" do
       tree = Yasuri.json2tree("{}")

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: yasuri
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - TAC
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-21 00:00:00.000000000 Z
+date: 2015-02-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler