yasuri 3.3.0 → 3.3.1

data/lib/yasuri/yasuri_node_generator.rb

@@ -1,6 +1,4 @@
 
-# Author::    TAC (tac@tac42.net)
-
 require_relative 'yasuri_node'
 require_relative 'yasuri_text_node'
 require_relative 'yasuri_struct_node'
@@ -23,7 +21,7 @@ module Yasuri
     end
 
     def self.gen(method_name, xpath, **opt, &block)
-      children = Yasuri::NodeGenerator.new.gen_recursive(&block) if block_given?
+      children = Yasuri::NodeGenerator.new.gen_recursive(&block) if block
 
       case method_name
       when /^text_(.+)$/

data/lib/yasuri/yasuri_paginate_node.rb

@@ -1,6 +1,4 @@
 
-# Author::    TAC (tac@tac42.net)
-
 require_relative 'yasuri_node'
 
 module Yasuri
@@ -14,13 +12,31 @@ module Yasuri
     end
 
     def inject(agent, page, opt = {}, element = page)
+      raise NotImplementedError.new("PagenateNode inside StructNode, Not Supported") if page != element
+
+      limit = @limit.nil? ? Float::MAX : @limit
+      child_results = inject_child(agent, page, limit, opt)
+
+      return child_results.map(&:values).flatten if @flatten == true
+
+      child_results
+    end
+
+    def opts
+      { limit: @limit, flatten: @flatten }
+    end
+
+    def node_type_str
+      "pages".freeze
+    end
+
+    private
+
+    def inject_child(agent, page, limit, opt)
       retry_count = opt[:retry_count] || Yasuri::DefaultRetryCount
       interval_ms = opt[:interval_ms] || Yasuri::DefaultInterval_ms
 
-      raise NotImplementedError.new("PagenateNode inside StructNode, Not Supported") if page != element
-
       child_results = []
-      limit = @limit.nil? ? Float::MAX : @limit
       while page
         child_results_kv = @children.map do |child_node|
           child_name = Yasuri.node_name(child_node.name, opt)
@@ -29,26 +45,14 @@ module Yasuri
         child_results << Hash[child_results_kv]
 
         link = page.search(@xpath).first # Todo raise:  link is not found
-        break if link == nil
+        break if link.nil?
 
         link_button = Mechanize::Page::Link.new(link, agent, page)
         page = Yasuri.with_retry(retry_count, interval_ms) { link_button.click }
         break if (limit -= 1) <= 0
       end
 
-      if @flatten == true
-        return child_results.map{|h| h.values}.flatten
-      end
-
       child_results
     end
-
-    def opts
-      {limit:@limit, flatten:@flatten}
-    end
-
-    def node_type_str
-      "pages".freeze
-    end
   end
 end

data/lib/yasuri/yasuri_struct_node.rb

@@ -1,6 +1,4 @@
 
-# Author::    TAC (tac@tac42.net)
-
 require_relative 'yasuri_node'
 
 module Yasuri
@@ -16,7 +14,7 @@ module Yasuri
         Hash[child_results_kv]
       end
       tree.size == 1 ? tree.first : tree
-    end # inject
+    end
 
     def node_type_str
       "struct".freeze

data/lib/yasuri/yasuri_text_node.rb

@@ -1,6 +1,4 @@
 
-# Author::    TAC (tac@tac42.net)
-
 require_relative 'yasuri_node'
 
 module Yasuri
@@ -13,14 +11,14 @@ module Yasuri
       truncate = opt[:truncate]
       proc = opt[:proc]
 
-      truncate = Regexp.new(truncate) if not truncate.nil? # regexp or nil
+      truncate = Regexp.new(truncate) unless truncate.nil? # regexp or nil
       @truncate = truncate
-      @truncate = Regexp.new(@truncate.to_s) if not @truncate.nil?
+      @truncate = Regexp.new(@truncate.to_s) unless @truncate.nil?
 
       @proc = proc.nil? ? nil : proc.to_sym
     end
 
-    def inject(agent, page, opt = {}, element = page)
+    def inject(_agent, page, _opt = {}, element = page)
       node = element.search(@xpath)
       text = node.text.to_s
 
@@ -35,7 +33,7 @@ module Yasuri
     end
 
     def opts
-      {truncate:@truncate, proc:@proc}
+      { truncate: @truncate, proc: @proc }
     end
 
     def node_type_str

data/spec/servers/httpserver.rb

@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-# Author::    TAC (tac@tac42.net)
 
 require 'glint'
 

data/spec/spec_helper.rb

@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-# Author::    TAC (tac@tac42.net)
 
 require 'glint'
 Dir[File.expand_path("../servers/*.rb", __FILE__)].each {|f| require f}

data/spec/yasuri_cli_spec.rb

@@ -2,95 +2,113 @@ require_relative 'spec_helper'
 
 describe 'Yasuri' do
   include_context 'httpserver'
-
-  before do
-    @agent = Mechanize.new
-    @index_page = @agent.get(uri)
-
-    @res_dir = File.expand_path('../cli_resources', __FILE__)
-  end
+  let(:res_dir) { File.expand_path('cli_resources', __dir__) }
 
   describe 'cli scrape' do
-    it "require --file or --json option" do
-      expect {
+    it 'require --file or --json option' do
+      expect do
         Yasuri::CLI.new.invoke(:scrape, [uri], {})
-      }.to output("ERROR: Only one of `--file` or `--json` option should be specified.\n").to_stderr
+      end.to output("ERROR: Only one of `--file` or `--json` option should be specified.\n").to_stderr
     end
 
-    it "only one of --file or --json option" do
-      expect {
-        Yasuri::CLI.new.invoke(:scrape, [uri], {file: "path.json", json: '{"text_title": "/html/head/title"}'})
-      }.to output("ERROR: Only one of `--file` or `--json` option should be specified.\n").to_stderr
+    it 'only one of --file or --json option' do
+      expect do
+        Yasuri::CLI.new.invoke(:scrape, [uri], { file: 'path.json', json: '{"text_title": "/html/head/title"}' })
+      end.to output("ERROR: Only one of `--file` or `--json` option should be specified.\n").to_stderr
     end
 
-    it "require --file option is not empty string" do
-      expect {
-        Yasuri::CLI.new.invoke(:scrape, [uri], {file: "file"})
-      }.to output("ERROR: --file option require not empty argument.\n").to_stderr
+    it 'require --file option is not empty string' do
+      expect do
+        Yasuri::CLI.new.invoke(:scrape, [uri], { file: 'file' })
+      end.to output("ERROR: --file option require not empty argument.\n").to_stderr
     end
 
-    it "require --json option is not empty string" do
-      expect {
-        Yasuri::CLI.new.invoke(:scrape, [uri], {json: "json"})
-      }.to output("ERROR: --json option require not empty argument.\n").to_stderr
+    it 'require --json option is not empty string' do
+      expect do
+        Yasuri::CLI.new.invoke(:scrape, [uri], { json: 'json' })
+      end.to output("ERROR: --json option require not empty argument.\n").to_stderr
     end
 
-
-    it "display text node as simple string" do
-      expect {
-        Yasuri::CLI.new.invoke(:scrape, [uri], {json: '{"text_title": "/html/head/title"}'})
-      }.to output("Yasuri Test\n").to_stdout
+    it 'display text node as simple string' do
+      expect do
+        Yasuri::CLI.new.invoke(:scrape, [uri], { json: '{"text_title": "/html/head/title"}' })
+      end.to output("Yasuri Test\n").to_stdout
     end
 
-    it "display texts in single json" do
-      expect {
-        Yasuri::CLI.new.invoke(:scrape, [uri], {json: '{"text_c1":"/html/body/p[1]", "text_c2":"/html/body/p[2]"}'})
-      }.to output('{"c1":"Hello,Yasuri","c2":"Last Modify - 2015/02/14"}'+"\n").to_stdout
+    it 'display texts in single json' do
+      expect do
+        Yasuri::CLI.new.invoke(:scrape, [uri], { json: '{"text_c1":"/html/body/p[1]", "text_c2":"/html/body/p[2]"}' })
+      end.to output('{"c1":"Hello,Yasuri","c2":"Last Modify - 2015/02/14"}' << "\n").to_stdout
     end
 
-
-    it "display text node as simple string via json file" do
-      expect {
-        Yasuri::CLI.new.invoke(:scrape, [uri+"/pagination/page01.html"], {file: "#{@res_dir}/tree.json"})
-      }.to output('[{"content":"PaginationTest01"},{"content":"PaginationTest02"},{"content":"PaginationTest03"},{"content":"PaginationTest04"}]' + "\n").to_stdout
+    it 'display text node as simple string via json file' do
+      expect do
+        Yasuri::CLI.new.invoke(:scrape, ["#{uri}/pagination/page01.html"], { file: "#{res_dir}/tree.json" })
+      end.to output(
+        '[{"content":"PaginationTest01"},{"content":"PaginationTest02"},' \
+        '{"content":"PaginationTest03"},{"content":"PaginationTest04"}]' \
+        "\n"
+      ).to_stdout
     end
-    it "display text node as simple string via yaml file" do
-      expect {
-        Yasuri::CLI.new.invoke(:scrape, [uri+"/pagination/page01.html"], {file: "#{@res_dir}/tree.yml"})
-      }.to output('[{"content":"PaginationTest01"},{"content":"PaginationTest02"},{"content":"PaginationTest03"},{"content":"PaginationTest04"}]' + "\n").to_stdout
+
+    it 'display text node as simple string via yaml file' do
+      expect do
+        Yasuri::CLI.new.invoke(:scrape, ["#{uri}/pagination/page01.html"], { file: "#{res_dir}/tree.yml" })
+      end.to output(
+        '[{"content":"PaginationTest01"},{"content":"PaginationTest02"},' \
+        '{"content":"PaginationTest03"},{"content":"PaginationTest04"}]' \
+        "\n"
+      ).to_stdout
     end
 
-    it "interval option is effect for each request" do
+    it 'interval option is effect for each request' do
       allow(Kernel).to receive(:sleep)
 
-      Yasuri::CLI.new.invoke(
-        :scrape,
-        [uri+"/pagination/page01.html"],
-        {file: "#{@res_dir}/tree.yml", interval: 500}
-      )
+      expect do
+        Yasuri::CLI.new.invoke(
+          :scrape,
+          ["#{uri}/pagination/page01.html"],
+          { file: "#{res_dir}/tree.yml", interval: 500 }
+        )
+      end.to output(
+        '[{"content":"PaginationTest01"},{"content":"PaginationTest02"},' \
+        '{"content":"PaginationTest03"},{"content":"PaginationTest04"}]' \
+        "\n"
+      ).to_stdout
 
       expect(Kernel).to have_received(:sleep).exactly(4).times do |interval_sec|
         expect(interval_sec).to match 0.5
       end
     end
 
-    it "display ERROR when json string is wrong" do
+    it 'display ERROR when json string is wrong' do
       wrong_json = '{,,}'
-      expect {
-        Yasuri::CLI.new.invoke(:scrape, [uri], {json: wrong_json})
-      }.to output("ERROR: Failed to convert json to yasuri tree. 809: unexpected token at '#{wrong_json}'\n").to_stderr
+      expect do
+        Yasuri::CLI.new.invoke(:scrape, [uri], { json: wrong_json })
+      end.to output(
+        'ERROR: Failed to convert json to yasuri tree. ' \
+        "809: unexpected token at '#{wrong_json}'\n"
+      ).to_stderr
     end
-    it "display ERROR when json file contains is wrong" do
-      file_path = "#{@res_dir}/tree_wrong.json"
-      expect {
-        Yasuri::CLI.new.invoke(:scrape, [uri], {file: file_path})
-      }.to output("ERROR: Failed to convert to yasuri tree `#{file_path}`. (<unknown>): did not find expected node content while parsing a flow node at line 2 column 3\n").to_stderr
+
+    it 'display ERROR when json file contains is wrong' do
+      file_path = "#{res_dir}/tree_wrong.json"
+      expect do
+        Yasuri::CLI.new.invoke(:scrape, [uri], { file: file_path })
+      end.to output(
+        "ERROR: Failed to convert to yasuri tree `#{file_path}`. " \
+        "(<unknown>): did not find expected node content while parsing a flow node at line 2 column 3\n"
+      ).to_stderr
     end
-    it "display ERROR when yaml file contains is wrong" do
-      file_path = "#{@res_dir}/tree_wrong.yml"
-      expect {
-        Yasuri::CLI.new.invoke(:scrape, [uri], {file: file_path})
-      }.to output("ERROR: Failed to convert to yasuri tree `#{file_path}`. (<unknown>): did not find expected node content while parsing a block node at line 1 column 1\n").to_stderr
+
+    it 'display ERROR when yaml file contains is wrong' do
+      file_path = "#{res_dir}/tree_wrong.yml"
+      expect do
+        Yasuri::CLI.new.invoke(:scrape, [uri], { file: file_path })
+      end.to output(
+        "ERROR: Failed to convert to yasuri tree `#{file_path}`. " \
+        "(<unknown>): did not find expected node content while parsing a block node at line 1 column 1\n"
+      ).to_stderr
     end
   end
-end
+end

data/spec/yasuri_links_node_spec.rb

@@ -1,83 +1,86 @@
 
-# Author::    TAC (tac@tac42.net)
-
 require_relative 'spec_helper'
 
-#########
-# Links #
-#########
 describe 'Yasuri' do
   include_context 'httpserver'
 
   describe '::LinksNode' do
-    before do
-      @uri = uri
-    end
-
     it 'scrape links' do
-      root_node = Yasuri::LinksNode.new('/html/body/a', "root", [
-        Yasuri::TextNode.new('/html/body/p', "content"),
-      ])
+      root_node = Yasuri::LinksNode.new(
+        '/html/body/a', "root", [
+          Yasuri::TextNode.new('/html/body/p', "content")
+        ]
+      )
 
-      actual = root_node.scrape(@uri)
+      actual = root_node.scrape(uri)
       expected = [
-        {"content" => "Child 01 page."},
-        {"content" => "Child 02 page."},
-        {"content" => "Child 03 page."},
+        { "content" => "Child 01 page." },
+        { "content" => "Child 02 page." },
+        { "content" => "Child 03 page." }
       ]
       expect(actual).to match expected
     end
 
     it 'return empty set if no match node' do
       missing_xpath = '/html/body/b'
-      root_node = Yasuri::LinksNode.new(missing_xpath, "root", [
-        Yasuri::TextNode.new('/html/body/p', "content"),
-      ])
+      root_node = Yasuri::LinksNode.new(
+        missing_xpath, "root", [
+          Yasuri::TextNode.new('/html/body/p', "content")
+        ]
+      )
 
-      actual = root_node.scrape(@uri)
+      actual = root_node.scrape(uri)
       expect(actual).to be_empty
     end
 
     it 'scrape links, recursive' do
-      root_node = Yasuri::LinksNode.new('/html/body/a', "root", [
-        Yasuri::TextNode.new('/html/body/p', "content"),
-        Yasuri::LinksNode.new('/html/body/ul/li/a', "sub_link", [
-          Yasuri::TextNode.new('/html/head/title', "sub_page_title"),
-        ]),
-      ])
-      actual = root_node.scrape(@uri)
+      root_node = Yasuri::LinksNode.new(
+        '/html/body/a', "root", [
+          Yasuri::TextNode.new('/html/body/p', "content"),
+          Yasuri::LinksNode.new(
+            '/html/body/ul/li/a', "sub_link", [
+              Yasuri::TextNode.new('/html/head/title', "sub_page_title")
+            ]
+          )
+        ]
+      )
+      actual = root_node.scrape(uri)
       expected = [
-        {"content"  => "Child 01 page.",
-         "sub_link" => [{"sub_page_title" => "Child 01 SubPage Test"},
-                        {"sub_page_title" => "Child 02 SubPage Test"}],},
-        {"content" => "Child 02 page.",
-         "sub_link" => [],},
-        {"content" => "Child 03 page.",
-         "sub_link" => [{"sub_page_title" => "Child 03 SubPage Test"}],},
+        { "content" => "Child 01 page.",
+          "sub_link" => [{ "sub_page_title" => "Child 01 SubPage Test" },
+                         { "sub_page_title" => "Child 02 SubPage Test" }] },
+        { "content" => "Child 02 page.",
+          "sub_link" => [] },
+        { "content" => "Child 03 page.",
+          "sub_link" => [{ "sub_page_title" => "Child 03 SubPage Test" }] }
       ]
       expect(actual).to match expected
     end
+
     it 'can be defined by DSL, return no contains if no child node' do
       root_node = Yasuri.links_title '/html/body/a'
-      actual = root_node.scrape(@uri)
+      actual = root_node.scrape(uri)
       expected = [{}, {}, {}] # Empty if no child node under links node.
       expect(actual).to match expected
     end
 
     it 'can be defined return no contains if no child node' do
       root_node = Yasuri::LinksNode.new('/html/body/a', "title")
-      actual = root_node.scrape(@uri)
+      actual = root_node.scrape(uri)
       expected = [{}, {}, {}] # Empty if no child node under links node.
       expect(actual).to match expected
     end
+
     it 'can be defined by DSL, return nested contents under link' do
       generated = Yasuri.links_title '/html/body/a' do
-                     text_name '/html/body/p'
-                  end
-      original = Yasuri::LinksNode.new('/html/body/a', "root", [
-        Yasuri::TextNode.new('/html/body/p', "name"),
-      ])
-      compare_generated_vs_original(generated, original, @uri)
+        text_name '/html/body/p'
+      end
+      original = Yasuri::LinksNode.new(
+        '/html/body/a', "root", [
+          Yasuri::TextNode.new('/html/body/p', "name")
+        ]
+      )
+      compare_generated_vs_original(generated, original, uri)
     end
 
     it 'can be defined by DSL, return recursive links node' do
@@ -88,25 +91,31 @@ describe 'Yasuri' do
         end
       end
 
-      original = Yasuri::LinksNode.new('/html/body/a', "root", [
-        Yasuri::TextNode.new('/html/body/p', "content"),
-        Yasuri::LinksNode.new('/html/body/ul/li/a', "sub_link", [
-          Yasuri::TextNode.new('/html/head/title', "sub_page_title"),
-        ]),
-      ])
-      compare_generated_vs_original(generated, original, @uri)
+      original = Yasuri::LinksNode.new(
+        '/html/body/a', "root", [
+          Yasuri::TextNode.new('/html/body/p', "content"),
+          Yasuri::LinksNode.new(
+            '/html/body/ul/li/a', "sub_link", [
+              Yasuri::TextNode.new('/html/head/title', "sub_page_title")
+            ]
+          )
+        ]
+      )
+      compare_generated_vs_original(generated, original, uri)
     end
 
     it 'return child node as symbol' do
-      root_node = Yasuri::LinksNode.new('/html/body/a', "root", [
-        Yasuri::TextNode.new('/html/body/p', "content"),
-      ])
+      root_node = Yasuri::LinksNode.new(
+        '/html/body/a', "root", [
+          Yasuri::TextNode.new('/html/body/p', "content")
+        ]
+      )
 
-      actual = root_node.scrape(@uri, symbolize_names: true )
+      actual = root_node.scrape(uri, symbolize_names: true)
       expected = [
-        {:content => "Child 01 page."},
-        {:content => "Child 02 page."},
-        {:content => "Child 03 page."},
+        { content: "Child 01 page." },
+        { content: "Child 02 page." },
+        { content: "Child 03 page." }
       ]
       expect(actual).to match expected
     end
@@ -114,15 +123,16 @@ describe 'Yasuri' do
     it 'scrape with interval for each request' do
       allow(Kernel).to receive(:sleep)
 
-      root_node = Yasuri::LinksNode.new('/html/body/a', "root", [
-        Yasuri::TextNode.new('/html/body/p', "content"),
-      ])
-      actual = root_node.scrape(@uri, interval_ms: 100)
-
+      root_node = Yasuri::LinksNode.new(
+        '/html/body/a', "root", [
+          Yasuri::TextNode.new('/html/body/p', "content")
+        ]
+      )
+      actual = root_node.scrape(uri, interval_ms: 100)
       expect(actual.size).to match 3
 
       # request will be run 4(1+3) times because root page will be requested
-      expect(Kernel).to have_received(:sleep).exactly(1+3).times do |interval_sec|
+      expect(Kernel).to have_received(:sleep).exactly(1 + 3).times do |interval_sec|
         expect(interval_sec).to match 0.1
       end
     end