yasuri 2.0.13 → 3.3.1

data/spec/yasuri_struct_node_spec.rb

@@ -1,107 +1,121 @@
 
-# Author::    TAC (tac@tac42.net)
-
 require_relative 'spec_helper'
 
-##########
-# Struct #
-##########
 describe 'Yasuri' do
   include_context 'httpserver'
 
   describe '::StructNode' do
-    before do
-      @agent = Mechanize.new
-      @page = @agent.get(uri + "/struct/structual_text.html")
-
-      @table_1996 = [
-        { "title"    => "The Perfect Insider",
+    let(:uri_struct) { "#{uri}/struct/structual_text.html" }
+    let(:table1996) do
+      [
+        { "title" => "The Perfect Insider",
           "pub_date" => "1996/4/5" },
-        { "title"    => "Doctors in Isolated Room",
+        { "title" => "Doctors in Isolated Room",
           "pub_date" => "1996/7/5" },
-        { "title"    => "Mathematical Goodbye",
-          "pub_date" => "1996/9/5" },
+        { "title" => "Mathematical Goodbye",
+          "pub_date" => "1996/9/5" }
       ]
-      @table_1997 = [
-        { "title"    => "Jack the Poetical Private",
+    end
+    let(:table1997) do
+      [
+        { "title" => "Jack the Poetical Private",
           "pub_date" => "1997/1/5" },
-        { "title"    => "Who Inside",
+        { "title" => "Who Inside",
           "pub_date" => "1997/4/5" },
-        { "title"    => "Illusion Acts Like Magic",
-          "pub_date" => "1997/10/5" },
+        { "title" => "Illusion Acts Like Magic",
+          "pub_date" => "1997/10/5" }
       ]
-      @table_1998 = [
-        { "title"    => "Replaceable Summer",
+    end
+    let(:table1998) do
+      [
+        { "title" => "Replaceable Summer",
           "pub_date" => "1998/1/7" },
-        { "title"    => "Switch Back",
+        { "title" => "Switch Back",
           "pub_date" => "1998/4/5" },
-        { "title"    => "Numerical Models",
+        { "title" => "Numerical Models",
           "pub_date" => "1998/7/5" },
-        { "title"    => "The Perfect Outsider",
-          "pub_date" => "1998/10/5" },
+        { "title" => "The Perfect Outsider",
+          "pub_date" => "1998/10/5" }
       ]
-      @all_tables = [
-        {"table" => @table_1996},
-        {"table" => @table_1997},
-        {"table" => @table_1998},
+    end
+
+    let(:all_tables) do
+      [
+        { "table" => table1996 },
+        { "table" => table1997 },
+        { "table" => table1998 }
       ]
     end
 
     it 'scrape single table contents' do
-      node = Yasuri::StructNode.new('/html/body/table[1]/tr', "table", [
-        Yasuri::TextNode.new('./td[1]', "title"),
-        Yasuri::TextNode.new('./td[2]', "pub_date"),
-      ])
-      expected = @table_1996
-      actual = node.inject(@agent, @page)
+      node = Yasuri::StructNode.new(
+        '/html/body/table[1]/tr', "table", [
+          Yasuri::TextNode.new('./td[1]', "title"),
+          Yasuri::TextNode.new('./td[2]', "pub_date")
+        ]
+      )
+      expected = table1996
+      actual = node.scrape(uri_struct)
       expect(actual).to match expected
     end
 
     it 'return single result without array' do
-      node = Yasuri::StructNode.new('/html/body/table[1]/tr[1]', "table_first_tr", [
-        Yasuri::TextNode.new('./td[1]', "title"),
-        Yasuri::TextNode.new('./td[2]', "pub_date"),
-      ])
-      expected = @table_1996.first
-      actual = node.inject(@agent, @page)
+      node = Yasuri::StructNode.new(
+        '/html/body/table[1]/tr[1]', "table_first_tr", [
+          Yasuri::TextNode.new('./td[1]', "title"),
+          Yasuri::TextNode.new('./td[2]', "pub_date")
+        ]
+      )
+      expected = table1996.first
+      actual = node.scrape(uri_struct)
       expect(actual).to match expected
     end
 
     it 'return empty text if no match node' do
       no_match_xpath = '/html/body/table[1]/t'
-      node = Yasuri::StructNode.new(no_match_xpath, "table", [
-        Yasuri::TextNode.new('./td[1]', "title")
-      ])
-      actual = node.inject(@agent, @page)
+      node = Yasuri::StructNode.new(
+        no_match_xpath, "table", [
+          Yasuri::TextNode.new('./td[1]', "title")
+        ]
+      )
+      actual = node.scrape(uri_struct)
       expect(actual).to be_empty
     end
 
     it 'fail with invalid xpath' do
       invalid_xpath = '/html/body/table[1]/table[1]/tr['
-      node = Yasuri::StructNode.new(invalid_xpath, "table", [
-        Yasuri::TextNode.new('./td[1]', "title")
-      ])
-      expect { node.inject(@agent, @page) }.to raise_error(Nokogiri::XML::XPath::SyntaxError)
+      node = Yasuri::StructNode.new(
+        invalid_xpath, "table", [
+          Yasuri::TextNode.new('./td[1]', "title")
+        ]
+      )
+      expect { node.scrape(uri_struct) }.to raise_error(Nokogiri::XML::XPath::SyntaxError)
     end
 
     it 'fail with invalid xpath in children' do
       invalid_xpath = './td[1]['
-      node = Yasuri::StructNode.new('/html/body/table[1]/tr', "table", [
-        Yasuri::TextNode.new(invalid_xpath, "title"),
-        Yasuri::TextNode.new('./td[2]', "pub_date"),
-      ])
-      expect { node.inject(@agent, @page) }.to raise_error(Nokogiri::XML::XPath::SyntaxError)
+      node = Yasuri::StructNode.new(
+        '/html/body/table[1]/tr', "table", [
+          Yasuri::TextNode.new(invalid_xpath, "title"),
+          Yasuri::TextNode.new('./td[2]', "pub_date")
+        ]
+      )
+      expect { node.scrape(uri_struct) }.to raise_error(Nokogiri::XML::XPath::SyntaxError)
     end
 
     it 'scrape all tables' do
-      node = Yasuri::StructNode.new('/html/body/table', "tables", [
-        Yasuri::StructNode.new('./tr', "table", [
-          Yasuri::TextNode.new('./td[1]', "title"),
-          Yasuri::TextNode.new('./td[2]', "pub_date"),
-        ])
-      ])
-      expected = @all_tables
-      actual = node.inject(@agent, @page)
+      node = Yasuri::StructNode.new(
+        '/html/body/table', "tables", [
+          Yasuri::StructNode.new(
+            './tr', "table", [
+              Yasuri::TextNode.new('./td[1]', "title"),
+              Yasuri::TextNode.new('./td[2]', "pub_date")
+            ]
+          )
+        ]
+      )
+      expected = all_tables
+      actual = node.scrape(uri_struct)
       expect(actual).to match expected
     end
 
@@ -112,65 +126,71 @@ describe 'Yasuri' do
           text_pub_date './td[2]'
         end
       end
-      original = Yasuri::StructNode.new('/html/body/table', "tables", [
-        Yasuri::StructNode.new('./tr', "table", [
-          Yasuri::TextNode.new('./td[1]', "title"),
-          Yasuri::TextNode.new('./td[2]', "pub_date"),
-        ])
-      ])
-      compare_generated_vs_original(generated, original, @page)
+      original = Yasuri::StructNode.new(
+        '/html/body/table', "tables", [
+          Yasuri::StructNode.new(
+            './tr', "table", [
+              Yasuri::TextNode.new('./td[1]', "title"),
+              Yasuri::TextNode.new('./td[2]', "pub_date")
+            ]
+          )
+        ]
+      )
+      compare_generated_vs_original(generated, original, uri_struct)
     end
 
     it 'return child node as symbol' do
-      node = Yasuri::StructNode.new('/html/body/table[1]/tr', "table", [
-        Yasuri::TextNode.new('./td[1]', "title"),
-        Yasuri::TextNode.new('./td[2]', "pub_date"),
-      ])
-      expected = @table_1996.map{|h| h.map{|k,v| [k.to_sym, v] }.to_h }
-      actual = node.inject(@agent, @page, symbolize_names:true)
+      node = Yasuri::StructNode.new(
+        '/html/body/table[1]/tr', "table", [
+          Yasuri::TextNode.new('./td[1]', "title"),
+          Yasuri::TextNode.new('./td[2]', "pub_date")
+        ]
+      )
+      expected = table1996.map { |h| h.transform_keys(&:to_sym) }
+      actual = node.scrape(uri_struct, symbolize_names: true)
       expect(actual).to match expected
     end
-
   end
 
   describe '::StructNode::Links' do
-    before do
-      @agent = Mechanize.new
-      @page = @agent.get(uri + "/struct/structual_links.html")
-
-      @table = [
+    let(:uri_struct) { "#{uri}/struct/structual_links.html" }
+    let(:table) do
+      [
         { "title" => "Child01,02",
-          "child" => [{"p" => "Child 01 page."}, {"p" => "Child 02 page."}] },
+          "child" => [{ "p" => "Child 01 page." }, { "p" => "Child 02 page." }] },
 
         { "title" => "Child01,02,03",
-          "child" => [{"p" => "Child 01 page."}, {"p" => "Child 02 page."}, {"p" => "Child 03 page."}]}
+          "child" => [{ "p" => "Child 01 page." }, { "p" => "Child 02 page." }, { "p" => "Child 03 page." }] }
       ]
     end
 
     it 'return child node in links inside struct' do
-      node = Yasuri::StructNode.new('/html/body/table/tr', "table", [
-        Yasuri::TextNode.new('./td[1]', "title"),
-        Yasuri::LinksNode.new('./td/a', "child", [
-          Yasuri::TextNode.new('/html/body/p', "p"),
-        ])
-      ])
-      expected = @table
-      actual = node.inject(@agent, @page)
+      node = Yasuri::StructNode.new(
+        '/html/body/table/tr', "table", [
+          Yasuri::TextNode.new('./td[1]', "title"),
+          Yasuri::LinksNode.new(
+            './td/a', "child", [
+              Yasuri::TextNode.new('/html/body/p', "p")
+            ]
+          )
+        ]
+      )
+      expected = table
+      actual = node.scrape(uri_struct)
       expect(actual).to match expected
     end
-  end # descrive
+  end
 
   describe '::StructNode::Pages' do
-    before do
-      @agent = Mechanize.new
-      @page = @agent.get(uri + "/struct/structual_text.html") #dummy
-    end
+    let(:uri_struct) { "#{uri}/struct/structual_text.html" }
 
     it 'not supported' do
-      node = Yasuri::StructNode.new('/html/body/table[1]/tr', "table", [
-        Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "pages", [])
-      ])
-      expect{ node.inject(@agent, @page) }.to raise_error(NotImplementedError, "PagenateNode inside StructNode, Not Supported")
+      node = Yasuri::StructNode.new(
+        '/html/body/table[1]/tr', "table", [
+          Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "pages", [])
+        ]
+      )
+      expect { node.scrape(uri_struct) }.to raise_error(NotImplementedError, "PagenateNode inside StructNode, Not Supported")
     end
   end
 end

data/spec/yasuri_text_node_spec.rb

@@ -1,78 +1,68 @@
 
-# Author::    TAC (tac@tac42.net)
 
 require_relative 'spec_helper'
 
-########
-# Text #
-########
 describe 'Yasuri' do
   include_context 'httpserver'
 
-  before do
-    @agent = Mechanize.new
-    @index_page = @agent.get(uri)
-  end
-
   describe '::TextNode' do
-    before { @node = Yasuri::TextNode.new('/html/body/p[1]', "title") }
-
     it 'scrape text text <p>Hello,Yasuri</p>' do
-      actual = @node.inject(@agent, @index_page)
+      node = Yasuri::TextNode.new('/html/body/p[1]', "title")
+      actual = node.scrape(uri)
       expect(actual).to eq "Hello,Yasuri"
     end
 
     it 'return empty text if no match node' do
       no_match_node = Yasuri::TextNode.new('/html/body/no_match_node', "title")
-      actual = no_match_node.inject(@agent, @index_page)
+      actual = no_match_node.scrape(uri)
       expect(actual).to be_empty
     end
 
     it 'fail with invalid xpath' do
       invalid_xpath = '/html/body/no_match_node['
       node = Yasuri::TextNode.new(invalid_xpath, "title")
-      expect { node.inject(@agent, @index_page) }.to raise_error(Nokogiri::XML::XPath::SyntaxError)
+      expect { node.scrape(uri) }.to raise_error(Nokogiri::XML::XPath::SyntaxError)
     end
 
     it "can be defined by DSL, return single TextNode title" do
       generated = Yasuri.text_title '/html/body/p[1]'
-      original  = Yasuri::TextNode.new('/html/body/p[1]', "title")
-      compare_generated_vs_original(generated, original, @index_page)
+      original = Yasuri::TextNode.new('/html/body/p[1]', "title")
+      compare_generated_vs_original(generated, original, uri)
     end
 
-    it "can be truncated with regexp" do
-      node  = Yasuri.text_title '/html/body/p[1]', truncate:/^[^,]+/
-      actual = node.inject(@agent, @index_page)
+    it "can truncate head by regexp" do
+      node = Yasuri.text_title '/html/body/p[1]', truncate: /^[^,]+/
+      actual = node.scrape(uri)
       expect(actual).to eq "Hello"
     end
 
-    it "return first captured if matched given capture pattern" do
-      node  = Yasuri.text_title '/html/body/p[1]', truncate:/H(.+)i/
-      actual = node.inject(@agent, @index_page)
-      expect(actual).to eq "ello,Yasur"
+    it "can truncate tail by regexp" do
+      node = Yasuri.text_title '/html/body/p[1]', truncate: /[^,]+$/
+      actual = node.scrape(uri)
+      expect(actual).to eq "Yasuri"
     end
 
-    it "can be truncated with regexp" do
-      node = Yasuri.text_title '/html/body/p[1]', truncate:/[^,]+$/
-      actual = node.inject(@agent, @index_page)
-      expect(actual).to eq "Yasuri"
+    it "return first captured if matched given capture pattern" do
+      node = Yasuri.text_title '/html/body/p[1]', truncate: /H(.+)i/
+      actual = node.scrape(uri)
+      expect(actual).to eq "ello,Yasur"
     end
 
     it "return empty string if truncated with no match to regexp" do
-      node = Yasuri.text_title '/html/body/p[1]', truncate:/^hoge/
-      actual = node.inject(@agent, @index_page)
+      node = Yasuri.text_title '/html/body/p[1]', truncate: /^hoge/
+      actual = node.scrape(uri)
       expect(actual).to be_empty
     end
 
     it "return symbol method applied string" do
       node = Yasuri.text_title '/html/body/p[1]', proc: :upcase
-      actual = node.inject(@agent, @index_page)
+      actual = node.scrape(uri)
       expect(actual).to eq "HELLO,YASURI"
     end
 
     it "return apply multi arguments" do
-      node = Yasuri.text_title '/html/body/p[1]', proc: :upcase, truncate:/H(.+)i/
-      actual = node.inject(@agent, @index_page)
+      node = Yasuri.text_title '/html/body/p[1]', proc: :upcase, truncate: /H(.+)i/
+      actual = node.scrape(uri)
       expect(actual).to eq "ELLO,YASUR"
     end
   end

data/yasuri.gemspec

@@ -1,31 +1,38 @@
-# coding: utf-8
+
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'yasuri/version'
 
 Gem::Specification.new do |spec|
-  spec.name          = "yasuri"
-  spec.version       = Yasuri::VERSION
-  spec.authors       = ["TAC"]
-  spec.email         = ["tac@tac42.net"]
-  spec.summary       = %q{Yasuri is easy scraping library.}
-  spec.description   = %q{Yasuri is an easy web-scraping library for supporting "Mechanize".}
-  spec.homepage      = "https://github.com/tac0x2a/yasuri"
-  spec.license       = "MIT"
+  spec.name                  = 'yasuri'
+  spec.version               = Yasuri::VERSION
+  spec.authors               = ['TAC']
+  spec.email                 = ['tac@tac42.net']
+  spec.summary               = %q{Yasuri is easy scraping library.}
+  spec.description           = %q{Yasuri is an easy web-scraping library for supporting 'Mechanize'.}
+  spec.homepage              = 'https://github.com/tac0x2a/yasuri'
+  spec.license               = 'MIT'
 
-  spec.files         = `git ls-files -z`.split("\x0")
-  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
-  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
-  spec.require_paths = ["lib"]
+  spec.files                 = `git ls-files -z`.split("\x0")
+  spec.executables           = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.bindir                = 'exe'
+  spec.test_files            = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths         = ['lib']
+  spec.required_ruby_version = '>= 2.7.0'
 
-  spec.add_development_dependency "bundler"
-  spec.add_development_dependency "rake"
-  spec.add_development_dependency "rspec"
-  spec.add_development_dependency "fuubar"
-  spec.add_development_dependency "glint"
-  spec.add_development_dependency "coveralls"
-  spec.add_development_dependency "simplecov"
-  spec.add_development_dependency "codeclimate-test-reporter"
+  spec.add_development_dependency 'bundler'
+  spec.add_development_dependency 'codeclimate-test-reporter'
+  spec.add_development_dependency 'coveralls'
+  spec.add_development_dependency 'fuubar'
+  spec.add_development_dependency 'glint'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'rubocop'
+  spec.add_development_dependency 'rubocop-performance'
+  spec.add_development_dependency 'rubocop-rspec'
+  spec.add_development_dependency 'rubocop-rubycw'
+  spec.add_development_dependency 'simplecov'
 
-  spec.add_dependency "mechanize"
+  spec.add_dependency 'mechanize'
+  spec.add_dependency 'thor'
 end