yasuri 2.0.12 → 3.3.0

data/lib/yasuri/yasuri_node.rb

@@ -7,15 +7,50 @@ module Yasuri
   module Node
     attr_reader :url, :xpath, :name, :children
 
-    def initialize(xpath, name, children = [], opt: {})
+    def initialize(xpath, name, children = [], **opt)
       @xpath, @name, @children = xpath, name, children
     end
 
+    def scrape(uri, opt = {})
+      retry_count = opt[:retry_count] || Yasuri::DefaultRetryCount
+      interval_ms = opt[:interval_ms] || Yasuri::DefaultInterval_ms
+
+      agent = Mechanize.new
+      page = Yasuri.with_retry(retry_count, interval_ms) { agent.get(uri) }
+      scrape_with_agent(uri, agent, opt)
+    end
+
+    def scrape_with_agent(uri, agent, opt = {})
+      page = agent.get(uri)
+      inject(agent, page, opt)
+    end
+
     def inject(agent, page, opt = {}, element = page)
-      fail "#{Kernel.__method__} is not implemented."
+      fail "#{Kernel.__method__} is not implemented in included class."
+    end
+
+    def to_h
+      return @xpath if @xpath and @children.empty? and self.opts.values.compact.empty?
+
+      node_hash = {}
+      self.opts.each{|k, v| node_hash[k] = v if not v.nil?}
+
+      node_hash[:path] = @xpath if @xpath
+
+      children.each do |child|
+        child_node_name = "#{child.node_type_str}_#{child.name}"
+        node_hash[child_node_name] = child.to_h
+      end
+
+      node_hash
     end
+
     def opts
       {}
     end
+
+    def node_type_str
+      fail "#{Kernel.__method__} is not implemented in included class."
+    end
   end
 end

data/lib/yasuri/yasuri_node_generator.rb

@@ -6,6 +6,7 @@ require_relative 'yasuri_text_node'
 require_relative 'yasuri_struct_node'
 require_relative 'yasuri_links_node'
 require_relative 'yasuri_paginate_node'
+require_relative 'yasuri_map_node'
 
 module Yasuri
   class NodeGenerator
@@ -15,29 +16,33 @@ module Yasuri
       @nodes
     end
 
-    def method_missing(name, *args, &block)
-      node = NodeGenerator.gen(name, *args, &block)
+    def method_missing(name, pattern=nil, **args, &block)
+      node = NodeGenerator.gen(name, pattern, **args, &block)
       raise "Undefined Node Name '#{name}'" if node == nil
       @nodes << node
     end
 
-    def self.gen(name, *args, &block)
-      xpath, opt = *args
-      opt = [opt].flatten.compact
+    def self.gen(method_name, xpath, **opt, &block)
       children = Yasuri::NodeGenerator.new.gen_recursive(&block) if block_given?
 
-      case name
+      case method_name
       when /^text_(.+)$/
-        Yasuri::TextNode.new(xpath,   $1, children || [], *opt)
+        # Todo raise error xpath is not valid
+        Yasuri::TextNode.new(xpath, $1, children || [], **opt)
       when /^struct_(.+)$/
-        Yasuri::StructNode.new(xpath, $1, children || [], *opt)
+        # Todo raise error xpath is not valid
+        Yasuri::StructNode.new(xpath, $1, children || [], **opt)
       when /^links_(.+)$/
-        Yasuri::LinksNode.new(xpath,  $1, children || [], *opt)
+        # Todo raise error xpath is not valid
+        Yasuri::LinksNode.new(xpath, $1, children || [], **opt)
       when /^pages_(.+)$/
-        Yasuri::PaginateNode.new(xpath, $1, children || [], *opt)
+        # Todo raise error xpath is not valid
+        Yasuri::PaginateNode.new(xpath, $1, children || [], **opt)
+      when /^map_(.+)$/
+        Yasuri::MapNode.new($1, children, **opt)
       else
         nil
       end
-    end # of self.gen(name, *args, &block)
+    end # of self.gen(method_name, xpath, **opt, &block)
   end # of class NodeGenerator
 end

data/lib/yasuri/yasuri_paginate_node.rb

@@ -14,7 +14,8 @@ module Yasuri
     end
 
     def inject(agent, page, opt = {}, element = page)
-      retry_count = opt[:retry_count] || 5
+      retry_count = opt[:retry_count] || Yasuri::DefaultRetryCount
+      interval_ms = opt[:interval_ms] || Yasuri::DefaultInterval_ms
 
       raise NotImplementedError.new("PagenateNode inside StructNode, Not Supported") if page != element
 
@@ -22,16 +23,16 @@ module Yasuri
       limit = @limit.nil? ? Float::MAX : @limit
       while page
         child_results_kv = @children.map do |child_node|
-          child_name = Yasuri.NodeName(child_node.name, opt)
+          child_name = Yasuri.node_name(child_node.name, opt)
           [child_name, child_node.inject(agent, page, opt)]
         end
         child_results << Hash[child_results_kv]
 
-        link = page.search(@xpath).first
+        link = page.search(@xpath).first # Todo raise:  link is not found
         break if link == nil
 
         link_button = Mechanize::Page::Link.new(link, agent, page)
-        page = Yasuri.with_retry(retry_count) { link_button.click }
+        page = Yasuri.with_retry(retry_count, interval_ms) { link_button.click }
         break if (limit -= 1) <= 0
       end
 
@@ -41,8 +42,13 @@ module Yasuri
 
       child_results
     end
+
     def opts
       {limit:@limit, flatten:@flatten}
     end
+
+    def node_type_str
+      "pages".freeze
+    end
   end
 end

data/lib/yasuri/yasuri_struct_node.rb

@@ -10,12 +10,16 @@ module Yasuri
       sub_tags = element.search(@xpath)
       tree = sub_tags.map do |sub_tag|
         child_results_kv = @children.map do |child_node|
-          child_name = Yasuri.NodeName(child_node.name, opt)
+          child_name = Yasuri.node_name(child_node.name, opt)
           [child_name, child_node.inject(agent, page, opt, sub_tag)]
         end
         Hash[child_results_kv]
       end
       tree.size == 1 ? tree.first : tree
     end # inject
+
+    def node_type_str
+      "struct".freeze
+    end
   end
 end

data/lib/yasuri/yasuri_text_node.rb

@@ -7,15 +7,17 @@ module Yasuri
   class TextNode
     include Node
 
-    def initialize(xpath, name, children = [], truncate: nil, proc:nil)
+    def initialize(xpath, name, children = [], **opt)
       super(xpath, name, children)
 
+      truncate = opt[:truncate]
+      proc = opt[:proc]
+
       truncate = Regexp.new(truncate) if not truncate.nil? # regexp or nil
       @truncate = truncate
       @truncate = Regexp.new(@truncate.to_s) if not @truncate.nil?
 
       @proc = proc.nil? ? nil : proc.to_sym
-
     end
 
     def inject(agent, page, opt = {}, element = page)
@@ -28,11 +30,16 @@ module Yasuri
       end
 
       text = text.__send__(@proc) if @proc && text.respond_to?(@proc)
+
       text
     end
 
     def opts
       {truncate:@truncate, proc:@proc}
     end
+
+    def node_type_str
+      "text".freeze
+    end
   end
 end

data/spec/cli_resources/tree.json

@@ -0,0 +1,8 @@
+{
+  "pages_root": {
+    "path": "/html/body/nav/span/a[@class='next']",
+    "limit": 10,
+    "flatten": false,
+    "text_content": "/html/body/p"
+  }
+}

data/spec/cli_resources/tree.yml

@@ -0,0 +1,5 @@
+pages_root:
+  path: "/html/body/nav/span/a[@class='next']"
+  limit: 10
+  flatten: false
+  text_content: "/html/body/p"

data/spec/cli_resources/tree_wrong.json

@@ -0,0 +1,9 @@
+{
+  ,,,
+  "pages_root": {
+    "path": "/html/body/nav/span/a[@class='next']",
+    "limit": 10,
+    "flatten": false,
+    "text_content": "/html/body/p"
+  }
+}

data/spec/cli_resources/tree_wrong.yml

@@ -0,0 +1,6 @@
+,,,
+pages_root:
+  path: "/html/body/nav/span/a[@class='next']"
+  limit: 10
+  flatten: false
+  text_content: "/html/body/p"

data/spec/spec_helper.rb

@@ -12,16 +12,11 @@ shared_context 'httpserver' do
   }
 end
 
-
-# ENV['CODECLIMATE_REPO_TOKEN'] = "0dc78d33107a7f11f257c0218ac1a37e0073005bb9734f2fd61d0f7e803fc151"
-# require "codeclimate-test-reporter"
-# CodeClimate::TestReporter.start
-
 require 'simplecov'
 require 'coveralls'
 Coveralls.wear!
 
-SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter[
+SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter.new [
   SimpleCov::Formatter::HTMLFormatter,
   Coveralls::SimpleCov::Formatter
 ]
@@ -31,8 +26,8 @@ SimpleCov.start
 $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
 require 'yasuri'
 
-def compare_generated_vs_original(generated, original, page)
-  expected = original.inject(@agent, page)
-  actual   = generated.inject(@agent, page)
+def compare_generated_vs_original(generated, original, uri)
+  expected = original.scrape(uri)
+  actual   = generated.scrape(uri)
   expect(actual).to match expected
 end

data/spec/yasuri_cli_spec.rb

@@ -0,0 +1,96 @@
+require_relative 'spec_helper'
+
+describe 'Yasuri' do
+  include_context 'httpserver'
+
+  before do
+    @agent = Mechanize.new
+    @index_page = @agent.get(uri)
+
+    @res_dir = File.expand_path('../cli_resources', __FILE__)
+  end
+
+  describe 'cli scrape' do
+    it "require --file or --json option" do
+      expect {
+        Yasuri::CLI.new.invoke(:scrape, [uri], {})
+      }.to output("ERROR: Only one of `--file` or `--json` option should be specified.\n").to_stderr
+    end
+
+    it "only one of --file or --json option" do
+      expect {
+        Yasuri::CLI.new.invoke(:scrape, [uri], {file: "path.json", json: '{"text_title": "/html/head/title"}'})
+      }.to output("ERROR: Only one of `--file` or `--json` option should be specified.\n").to_stderr
+    end
+
+    it "require --file option is not empty string" do
+      expect {
+        Yasuri::CLI.new.invoke(:scrape, [uri], {file: "file"})
+      }.to output("ERROR: --file option require not empty argument.\n").to_stderr
+    end
+
+    it "require --json option is not empty string" do
+      expect {
+        Yasuri::CLI.new.invoke(:scrape, [uri], {json: "json"})
+      }.to output("ERROR: --json option require not empty argument.\n").to_stderr
+    end
+
+
+    it "display text node as simple string" do
+      expect {
+        Yasuri::CLI.new.invoke(:scrape, [uri], {json: '{"text_title": "/html/head/title"}'})
+      }.to output("Yasuri Test\n").to_stdout
+    end
+
+    it "display texts in single json" do
+      expect {
+        Yasuri::CLI.new.invoke(:scrape, [uri], {json: '{"text_c1":"/html/body/p[1]", "text_c2":"/html/body/p[2]"}'})
+      }.to output('{"c1":"Hello,Yasuri","c2":"Last Modify - 2015/02/14"}'+"\n").to_stdout
+    end
+
+
+    it "display text node as simple string via json file" do
+      expect {
+        Yasuri::CLI.new.invoke(:scrape, [uri+"/pagination/page01.html"], {file: "#{@res_dir}/tree.json"})
+      }.to output('[{"content":"PaginationTest01"},{"content":"PaginationTest02"},{"content":"PaginationTest03"},{"content":"PaginationTest04"}]' + "\n").to_stdout
+    end
+    it "display text node as simple string via yaml file" do
+      expect {
+        Yasuri::CLI.new.invoke(:scrape, [uri+"/pagination/page01.html"], {file: "#{@res_dir}/tree.yml"})
+      }.to output('[{"content":"PaginationTest01"},{"content":"PaginationTest02"},{"content":"PaginationTest03"},{"content":"PaginationTest04"}]' + "\n").to_stdout
+    end
+
+    it "interval option is effect for each request" do
+      allow(Kernel).to receive(:sleep)
+
+      Yasuri::CLI.new.invoke(
+        :scrape,
+        [uri+"/pagination/page01.html"],
+        {file: "#{@res_dir}/tree.yml", interval: 500}
+      )
+
+      expect(Kernel).to have_received(:sleep).exactly(4).times do |interval_sec|
+        expect(interval_sec).to match 0.5
+      end
+    end
+
+    it "display ERROR when json string is wrong" do
+      wrong_json = '{,,}'
+      expect {
+        Yasuri::CLI.new.invoke(:scrape, [uri], {json: wrong_json})
+      }.to output("ERROR: Failed to convert json to yasuri tree. 809: unexpected token at '#{wrong_json}'\n").to_stderr
+    end
+    it "display ERROR when json file contains is wrong" do
+      file_path = "#{@res_dir}/tree_wrong.json"
+      expect {
+        Yasuri::CLI.new.invoke(:scrape, [uri], {file: file_path})
+      }.to output("ERROR: Failed to convert to yasuri tree `#{file_path}`. (<unknown>): did not find expected node content while parsing a flow node at line 2 column 3\n").to_stderr
+    end
+    it "display ERROR when yaml file contains is wrong" do
+      file_path = "#{@res_dir}/tree_wrong.yml"
+      expect {
+        Yasuri::CLI.new.invoke(:scrape, [uri], {file: file_path})
+      }.to output("ERROR: Failed to convert to yasuri tree `#{file_path}`. (<unknown>): did not find expected node content while parsing a block node at line 1 column 1\n").to_stderr
+    end
+  end
+end

data/spec/yasuri_links_node_spec.rb

@@ -11,9 +11,7 @@ describe 'Yasuri' do
 
   describe '::LinksNode' do
     before do
-      @agent = Mechanize.new
       @uri = uri
-      @index_page = @agent.get(@uri)
     end
 
     it 'scrape links' do
@@ -21,7 +19,7 @@ describe 'Yasuri' do
         Yasuri::TextNode.new('/html/body/p', "content"),
       ])
 
-      actual = root_node.inject(@agent, @index_page)
+      actual = root_node.scrape(@uri)
       expected = [
         {"content" => "Child 01 page."},
         {"content" => "Child 02 page."},
@@ -36,7 +34,7 @@ describe 'Yasuri' do
         Yasuri::TextNode.new('/html/body/p', "content"),
       ])
 
-      actual = root_node.inject(@agent, @index_page)
+      actual = root_node.scrape(@uri)
       expect(actual).to be_empty
     end
 
@@ -47,7 +45,7 @@ describe 'Yasuri' do
           Yasuri::TextNode.new('/html/head/title', "sub_page_title"),
         ]),
       ])
-      actual = root_node.inject(@agent, @index_page)
+      actual = root_node.scrape(@uri)
       expected = [
         {"content"  => "Child 01 page.",
          "sub_link" => [{"sub_page_title" => "Child 01 SubPage Test"},
@@ -59,10 +57,18 @@ describe 'Yasuri' do
       ]
       expect(actual).to match expected
     end
-    it 'can be defined by DSL, return single LinkNode title' do
-      generated = Yasuri.links_title '/html/body/a'
-      original  = Yasuri::LinksNode.new('/html/body/a', "title")
-      compare_generated_vs_original(generated, original, @index_page)
+    it 'can be defined by DSL, return no contains if no child node' do
+      root_node = Yasuri.links_title '/html/body/a'
+      actual = root_node.scrape(@uri)
+      expected = [{}, {}, {}] # Empty if no child node under links node.
+      expect(actual).to match expected
+    end
+
+    it 'can be defined return no contains if no child node' do
+      root_node = Yasuri::LinksNode.new('/html/body/a', "title")
+      actual = root_node.scrape(@uri)
+      expected = [{}, {}, {}] # Empty if no child node under links node.
+      expect(actual).to match expected
     end
     it 'can be defined by DSL, return nested contents under link' do
       generated = Yasuri.links_title '/html/body/a' do
@@ -71,7 +77,7 @@ describe 'Yasuri' do
       original = Yasuri::LinksNode.new('/html/body/a', "root", [
         Yasuri::TextNode.new('/html/body/p', "name"),
       ])
-      compare_generated_vs_original(generated, original, @index_page)
+      compare_generated_vs_original(generated, original, @uri)
     end
 
     it 'can be defined by DSL, return recursive links node' do
@@ -88,7 +94,7 @@ describe 'Yasuri' do
           Yasuri::TextNode.new('/html/head/title', "sub_page_title"),
         ]),
       ])
-      compare_generated_vs_original(generated, original, @index_page)
+      compare_generated_vs_original(generated, original, @uri)
     end
 
     it 'return child node as symbol' do
@@ -96,7 +102,7 @@ describe 'Yasuri' do
         Yasuri::TextNode.new('/html/body/p', "content"),
       ])
 
-      actual = root_node.inject(@agent, @index_page, symbolize_names: true )
+      actual = root_node.scrape(@uri, symbolize_names: true )
       expected = [
         {:content => "Child 01 page."},
         {:content => "Child 02 page."},
@@ -104,5 +110,21 @@ describe 'Yasuri' do
       ]
       expect(actual).to match expected
     end
+
+    it 'scrape with interval for each request' do
+      allow(Kernel).to receive(:sleep)
+
+      root_node = Yasuri::LinksNode.new('/html/body/a', "root", [
+        Yasuri::TextNode.new('/html/body/p', "content"),
+      ])
+      actual = root_node.scrape(@uri, interval_ms: 100)
+
+      expect(actual.size).to match 3
+
+      # request will be run 4(1+3) times because root page will be requested
+      expect(Kernel).to have_received(:sleep).exactly(1+3).times do |interval_sec|
+        expect(interval_sec).to match 0.1
+      end
+    end
   end
 end