yasuri 2.0.12 → 3.3.0

data/spec/yasuri_struct_node_spec.rb

@@ -11,8 +11,7 @@ describe 'Yasuri' do
 
   describe '::StructNode' do
     before do
-      @agent = Mechanize.new
-      @page = @agent.get(uri + "/struct/structual_text.html")
+      @uri = uri + "/struct/structual_text.html"
 
       @table_1996 = [
         { "title"    => "The Perfect Insider",
@@ -53,7 +52,7 @@ describe 'Yasuri' do
         Yasuri::TextNode.new('./td[2]', "pub_date"),
       ])
       expected = @table_1996
-      actual = node.inject(@agent, @page)
+      actual = node.scrape(@uri)
       expect(actual).to match expected
     end
 
@@ -63,7 +62,7 @@ describe 'Yasuri' do
         Yasuri::TextNode.new('./td[2]', "pub_date"),
       ])
       expected = @table_1996.first
-      actual = node.inject(@agent, @page)
+      actual = node.scrape(@uri)
       expect(actual).to match expected
     end
 
@@ -72,7 +71,7 @@ describe 'Yasuri' do
       node = Yasuri::StructNode.new(no_match_xpath, "table", [
         Yasuri::TextNode.new('./td[1]', "title")
       ])
-      actual = node.inject(@agent, @page)
+      actual = node.scrape(@uri)
       expect(actual).to be_empty
     end
 
@@ -81,7 +80,7 @@ describe 'Yasuri' do
       node = Yasuri::StructNode.new(invalid_xpath, "table", [
         Yasuri::TextNode.new('./td[1]', "title")
       ])
-      expect { node.inject(@agent, @page) }.to raise_error(Nokogiri::XML::XPath::SyntaxError)
+      expect { node.scrape(@uri) }.to raise_error(Nokogiri::XML::XPath::SyntaxError)
     end
 
     it 'fail with invalid xpath in children' do
@@ -90,7 +89,7 @@ describe 'Yasuri' do
         Yasuri::TextNode.new(invalid_xpath, "title"),
         Yasuri::TextNode.new('./td[2]', "pub_date"),
       ])
-      expect { node.inject(@agent, @page) }.to raise_error(Nokogiri::XML::XPath::SyntaxError)
+      expect { node.scrape(@uri) }.to raise_error(Nokogiri::XML::XPath::SyntaxError)
     end
 
     it 'scrape all tables' do
@@ -101,7 +100,7 @@ describe 'Yasuri' do
         ])
       ])
       expected = @all_tables
-      actual = node.inject(@agent, @page)
+      actual = node.scrape(@uri)
       expect(actual).to match expected
     end
 
@@ -118,7 +117,7 @@ describe 'Yasuri' do
           Yasuri::TextNode.new('./td[2]', "pub_date"),
         ])
       ])
-      compare_generated_vs_original(generated, original, @page)
+      compare_generated_vs_original(generated, original, @uri)
     end
 
     it 'return child node as symbol' do
@@ -127,7 +126,7 @@ describe 'Yasuri' do
         Yasuri::TextNode.new('./td[2]', "pub_date"),
       ])
       expected = @table_1996.map{|h| h.map{|k,v| [k.to_sym, v] }.to_h }
-      actual = node.inject(@agent, @page, symbolize_names:true)
+      actual = node.scrape(@uri, symbolize_names:true)
       expect(actual).to match expected
     end
 
@@ -135,9 +134,7 @@ describe 'Yasuri' do
 
   describe '::StructNode::Links' do
     before do
-      @agent = Mechanize.new
-      @page = @agent.get(uri + "/struct/structual_links.html")
-
+      @uri = uri + "/struct/structual_links.html"
       @table = [
         { "title" => "Child01,02",
           "child" => [{"p" => "Child 01 page."}, {"p" => "Child 02 page."}] },
@@ -155,22 +152,21 @@ describe 'Yasuri' do
         ])
       ])
       expected = @table
-      actual = node.inject(@agent, @page)
+      actual = node.scrape(@uri)
       expect(actual).to match expected
     end
   end # descrive
 
   describe '::StructNode::Pages' do
     before do
-      @agent = Mechanize.new
-      @page = @agent.get(uri + "/struct/structual_text.html") #dummy
+      @uri = uri + "/struct/structual_text.html"
     end
 
     it 'not supported' do
       node = Yasuri::StructNode.new('/html/body/table[1]/tr', "table", [
         Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "pages", [])
       ])
-      expect{ node.inject(@agent, @page) }.to raise_error(NotImplementedError, "PagenateNode inside StructNode, Not Supported")
+      expect{ node.scrape(@uri) }.to raise_error(NotImplementedError, "PagenateNode inside StructNode, Not Supported")
     end
   end
 end

data/spec/yasuri_text_node_spec.rb

@@ -10,69 +10,68 @@ describe 'Yasuri' do
   include_context 'httpserver'
 
   before do
-    @agent = Mechanize.new
-    @index_page = @agent.get(uri)
+    @uri = uri
   end
 
   describe '::TextNode' do
     before { @node = Yasuri::TextNode.new('/html/body/p[1]', "title") }
 
     it 'scrape text text <p>Hello,Yasuri</p>' do
-      actual = @node.inject(@agent, @index_page)
+      actual = @node.scrape(@uri)
       expect(actual).to eq "Hello,Yasuri"
     end
 
     it 'return empty text if no match node' do
       no_match_node = Yasuri::TextNode.new('/html/body/no_match_node', "title")
-      actual = no_match_node.inject(@agent, @index_page)
+      actual = no_match_node.scrape(@uri)
       expect(actual).to be_empty
     end
 
     it 'fail with invalid xpath' do
       invalid_xpath = '/html/body/no_match_node['
       node = Yasuri::TextNode.new(invalid_xpath, "title")
-      expect { node.inject(@agent, @index_page) }.to raise_error(Nokogiri::XML::XPath::SyntaxError)
+      expect { node.scrape(@uri) }.to raise_error(Nokogiri::XML::XPath::SyntaxError)
     end
 
     it "can be defined by DSL, return single TextNode title" do
       generated = Yasuri.text_title '/html/body/p[1]'
       original  = Yasuri::TextNode.new('/html/body/p[1]', "title")
-      compare_generated_vs_original(generated, original, @index_page)
+      compare_generated_vs_original(generated, original, @uri)
     end
 
     it "can be truncated with regexp" do
       node  = Yasuri.text_title '/html/body/p[1]', truncate:/^[^,]+/
-      actual = node.inject(@agent, @index_page)
+      actual = node.scrape(@uri)
       expect(actual).to eq "Hello"
     end
 
     it "return first captured if matched given capture pattern" do
       node  = Yasuri.text_title '/html/body/p[1]', truncate:/H(.+)i/
-      actual = node.inject(@agent, @index_page)
+      actual = node.scrape(@uri)
       expect(actual).to eq "ello,Yasur"
     end
 
     it "can be truncated with regexp" do
       node = Yasuri.text_title '/html/body/p[1]', truncate:/[^,]+$/
-      actual = node.inject(@agent, @index_page)
+      actual = node.scrape(@uri)
       expect(actual).to eq "Yasuri"
     end
 
     it "return empty string if truncated with no match to regexp" do
       node = Yasuri.text_title '/html/body/p[1]', truncate:/^hoge/
-      actual = node.inject(@agent, @index_page)
+      actual = node.scrape(@uri)
       expect(actual).to be_empty
     end
 
     it "return symbol method applied string" do
       node = Yasuri.text_title '/html/body/p[1]', proc: :upcase
-      actual = node.inject(@agent, @index_page)
+      actual = node.scrape(@uri)
       expect(actual).to eq "HELLO,YASURI"
     end
 
     it "return apply multi arguments" do
       node = Yasuri.text_title '/html/body/p[1]', proc: :upcase, truncate:/H(.+)i/
-      actual = node.inject(@agent, @index_page)
+      actual = node.scrape(@uri)
       expect(actual).to eq "ELLO,YASUR"
     end
   end

data/yasuri.gemspec

@@ -14,12 +14,13 @@ Gem::Specification.new do |spec|
   spec.license       = "MIT"
 
   spec.files         = `git ls-files -z`.split("\x0")
-  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.bindir        = "exe"
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.7"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "bundler"
+  spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec"
   spec.add_development_dependency "fuubar"
   spec.add_development_dependency "glint"
@@ -28,4 +29,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "codeclimate-test-reporter"
 
   spec.add_dependency "mechanize"
+  spec.add_dependency "thor"
 end

metadata

@@ -1,43 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: yasuri
 version: !ruby/object:Gem::Version
-  version: 2.0.12
+  version: 3.3.0
 platform: ruby
 authors:
 - TAC
-autorequire: 
-bindir: bin
+autorequire:
+bindir: exe
 cert_chain: []
-date: 2016-12-15 00:00:00.000000000 Z
+date: 2021-03-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -136,16 +136,33 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Yasuri is an easy web-scraping library for supporting "Mechanize".
 email:
 - tac@tac42.net
-executables: []
+executables:
+- yasuri
 extensions: []
 extra_rdoc_files: []
 files:
 - ".coveralls.yml"
+- ".github/workflows/ruby.yml"
 - ".gitignore"
 - ".rspec"
+- ".ruby-version"
 - ".travis.yml"
 - Gemfile
 - LICENSE
@@ -153,16 +170,26 @@ files:
 - Rakefile
 - USAGE.ja.md
 - USAGE.md
-- app.rb
+- examples/example.rb
+- examples/github.yml
+- examples/sample.json
+- examples/sample.yml
+- exe/yasuri
 - lib/yasuri.rb
 - lib/yasuri/version.rb
 - lib/yasuri/yasuri.rb
+- lib/yasuri/yasuri_cli.rb
 - lib/yasuri/yasuri_links_node.rb
+- lib/yasuri/yasuri_map_node.rb
 - lib/yasuri/yasuri_node.rb
 - lib/yasuri/yasuri_node_generator.rb
 - lib/yasuri/yasuri_paginate_node.rb
 - lib/yasuri/yasuri_struct_node.rb
 - lib/yasuri/yasuri_text_node.rb
+- spec/cli_resources/tree.json
+- spec/cli_resources/tree.yml
+- spec/cli_resources/tree_wrong.json
+- spec/cli_resources/tree_wrong.yml
 - spec/htdocs/child01.html
 - spec/htdocs/child01_sub.html
 - spec/htdocs/child02.html
@@ -178,7 +205,9 @@ files:
 - spec/htdocs/struct/structual_text.html
 - spec/servers/httpserver.rb
 - spec/spec_helper.rb
+- spec/yasuri_cli_spec.rb
 - spec/yasuri_links_node_spec.rb
+- spec/yasuri_map_spec.rb
 - spec/yasuri_node_spec.rb
 - spec/yasuri_paginate_node_spec.rb
 - spec/yasuri_spec.rb
@@ -189,7 +218,7 @@ homepage: https://github.com/tac0x2a/yasuri
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -204,12 +233,15 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.2
-signing_key: 
+rubygems_version: 3.2.3
+signing_key:
 specification_version: 4
 summary: Yasuri is easy scraping library.
 test_files:
+- spec/cli_resources/tree.json
+- spec/cli_resources/tree.yml
+- spec/cli_resources/tree_wrong.json
+- spec/cli_resources/tree_wrong.yml
 - spec/htdocs/child01.html
 - spec/htdocs/child01_sub.html
 - spec/htdocs/child02.html
@@ -225,7 +257,9 @@ test_files:
 - spec/htdocs/struct/structual_text.html
 - spec/servers/httpserver.rb
 - spec/spec_helper.rb
+- spec/yasuri_cli_spec.rb
 - spec/yasuri_links_node_spec.rb
+- spec/yasuri_map_spec.rb
 - spec/yasuri_node_spec.rb
 - spec/yasuri_paginate_node_spec.rb
 - spec/yasuri_spec.rb

data/app.rb

@@ -1,52 +0,0 @@
-#!/usr/bin/env ruby
-# -*- coding: utf-8 -*-
-
-# Author::    TAC (tac@tac42.net)
-
-require 'pp'
-require 'time'
-require 'mechanize'
-
-require_relative 'lib/yasuri/yasuri'
-
-agent = Mechanize.new
-
-uri = "http://www.asahi.com/"
-
-# Node tree constructing by DSL
-root = Yasuri.links_top '//*[@id="MainInner"]/div[1]/ul/li/a' do
-  text_title   '//*[@id="MainInner"]/div[1]/div/h1'
-  text_article '//*[@id="MainInner"]/div/div[@class="ArticleText"]'
-end
-
-# Node tree constructing by JSON
-src = <<-EOJSON
-   { "node"     : "links",
-     "name"     : "root",
-     "path"     : "//*[@id='MainInner']/div[1]/ul/li/a",
-     "children" : [
-                    { "node" : "text",
-                      "name" : "title",
-                      "path" : "//*[@id='MainInner']/div[1]/div/h1"
-                    },
-                    { "node" : "text",
-                      "name" : "article",
-                      "path" : "//*[@id='MainInner']/div/div[@class='ArticleText']"
-                    }
-                  ]
-   }
-EOJSON
-root = Yasuri.json2tree(src)
-
-# Access to parsed resources
-page = agent.get(uri)
-contents = root.inject(agent, page)
-
-contents.each do |h|
-  t = h['title']
-  a = h['article']
-
-  puts t
-  puts a
-  puts "=" * 100
-end