yasuri 3.0.0 → 3.3.2

data/lib/yasuri/yasuri_map_node.rb

@@ -0,0 +1,40 @@
+
+module Yasuri
+  class MapNode
+    include Node
+    attr_reader :name, :children
+
+    def initialize(name, children, **opt)
+      @name = name
+      @children = children
+      @opt = opt
+    end
+
+    def inject(agent, page, opt = {}, _element = page)
+      child_results_kv = @children.map do |node|
+        [node.name, node.inject(agent, page, opt)]
+      end
+      Hash[child_results_kv]
+    end
+
+    def to_h
+      node_hash = {}
+      self.opts.each { |k, v| node_hash[k] = v unless v.nil? }
+
+      children.each do |child|
+        child_node_name = "#{child.node_type_str}_#{child.name}"
+        node_hash[child_node_name] = child.to_h
+      end
+
+      node_hash
+    end
+
+    def opts
+      {}
+    end
+
+    def node_type_str
+      "map".freeze
+    end
+  end
+end

data/lib/yasuri/yasuri_node.rb

@@ -1,21 +1,53 @@
 
-# Author::    TAC (tac@tac42.net)
-
 require_relative 'yasuri_node'
 
 module Yasuri
   module Node
     attr_reader :url, :xpath, :name, :children
 
-    def initialize(xpath, name, children = [], opt: {})
+    def initialize(xpath, name, children = [], **_opt)
       @xpath, @name, @children = xpath, name, children
     end
 
+    def scrape(uri, opt = {})
+      agent = Mechanize.new
+      scrape_with_agent(uri, agent, opt)
+    end
+
+    def scrape_with_agent(uri, agent, opt = {})
+      retry_count = opt[:retry_count] || Yasuri::DefaultRetryCount
+      interval_ms = opt[:interval_ms] || Yasuri::DefaultInterval_ms
+
+      page = Yasuri.with_retry(retry_count, interval_ms) { agent.get(uri) }
+      inject(agent, page, opt)
+    end
+
     def inject(agent, page, opt = {}, element = page)
-      fail "#{Kernel.__method__} is not implemented."
+      fail "#{Kernel.__method__} is not implemented in included class."
     end
+
+    def to_h
+      return @xpath if @xpath and @children.empty? and self.opts.values.compact.empty?
+
+      node_hash = {}
+      self.opts.each{|k, v| node_hash[k] = v if not v.nil?}
+
+      node_hash[:path] = @xpath if @xpath
+
+      children.each do |child|
+        child_node_name = "#{child.node_type_str}_#{child.name}"
+        node_hash[child_node_name] = child.to_h
+      end
+
+      node_hash
+    end
+
     def opts
       {}
     end
+
+    def node_type_str
+      fail "#{Kernel.__method__} is not implemented in included class."
+    end
   end
 end

data/lib/yasuri/yasuri_node_generator.rb

@@ -1,11 +1,10 @@
 
-# Author::    TAC (tac@tac42.net)
-
 require_relative 'yasuri_node'
 require_relative 'yasuri_text_node'
 require_relative 'yasuri_struct_node'
 require_relative 'yasuri_links_node'
 require_relative 'yasuri_paginate_node'
+require_relative 'yasuri_map_node'
 
 module Yasuri
   class NodeGenerator
@@ -15,27 +14,33 @@ module Yasuri
       @nodes
     end
 
-    def method_missing(name, pattern, **args, &block)
+    def method_missing(name, pattern=nil, **args, &block)
       node = NodeGenerator.gen(name, pattern, **args, &block)
       raise "Undefined Node Name '#{name}'" if node == nil
       @nodes << node
     end
 
-    def self.gen(name, xpath, **opt, &block)
-      children = Yasuri::NodeGenerator.new.gen_recursive(&block) if block_given?
+    def self.gen(method_name, xpath, **opt, &block)
+      children = Yasuri::NodeGenerator.new.gen_recursive(&block) if block
 
-      case name
+      case method_name
       when /^text_(.+)$/
-        Yasuri::TextNode.new(xpath,   $1, children || [], **opt)
+        # Todo raise error xpath is not valid
+        Yasuri::TextNode.new(xpath, $1, children || [], **opt)
       when /^struct_(.+)$/
+        # Todo raise error xpath is not valid
         Yasuri::StructNode.new(xpath, $1, children || [], **opt)
       when /^links_(.+)$/
-        Yasuri::LinksNode.new(xpath,  $1, children || [], **opt)
+        # Todo raise error xpath is not valid
+        Yasuri::LinksNode.new(xpath, $1, children || [], **opt)
       when /^pages_(.+)$/
+        # Todo raise error xpath is not valid
         Yasuri::PaginateNode.new(xpath, $1, children || [], **opt)
+      when /^map_(.+)$/
+        Yasuri::MapNode.new($1, children, **opt)
       else
         nil
       end
-    end # of self.gen(name, *args, &block)
+    end # of self.gen(method_name, xpath, **opt, &block)
   end # of class NodeGenerator
 end

data/lib/yasuri/yasuri_paginate_node.rb

@@ -1,6 +1,4 @@
 
-# Author::    TAC (tac@tac42.net)
-
 require_relative 'yasuri_node'
 
 module Yasuri
@@ -14,35 +12,47 @@ module Yasuri
     end
 
     def inject(agent, page, opt = {}, element = page)
-      retry_count = opt[:retry_count] || 5
-
       raise NotImplementedError.new("PagenateNode inside StructNode, Not Supported") if page != element
 
-      child_results = []
       limit = @limit.nil? ? Float::MAX : @limit
+      child_results = inject_child(agent, page, limit, opt)
+
+      return child_results.map(&:values).flatten if @flatten == true
+
+      child_results
+    end
+
+    def opts
+      { limit: @limit, flatten: @flatten }
+    end
+
+    def node_type_str
+      "pages".freeze
+    end
+
+    private
+
+    def inject_child(agent, page, limit, opt)
+      retry_count = opt[:retry_count] || Yasuri::DefaultRetryCount
+      interval_ms = opt[:interval_ms] || Yasuri::DefaultInterval_ms
+
+      child_results = []
       while page
         child_results_kv = @children.map do |child_node|
-          child_name = Yasuri.NodeName(child_node.name, opt)
+          child_name = Yasuri.node_name(child_node.name, opt)
           [child_name, child_node.inject(agent, page, opt)]
         end
         child_results << Hash[child_results_kv]
 
-        link = page.search(@xpath).first
-        break if link == nil
+        link = page.search(@xpath).first # Todo raise:  link is not found
+        break if link.nil?
 
         link_button = Mechanize::Page::Link.new(link, agent, page)
-        page = Yasuri.with_retry(retry_count) { link_button.click }
+        page = Yasuri.with_retry(retry_count, interval_ms) { link_button.click }
         break if (limit -= 1) <= 0
       end
 
-      if @flatten == true
-        return child_results.map{|h| h.values}.flatten
-      end
-
       child_results
     end
-    def opts
-      {limit:@limit, flatten:@flatten}
-    end
   end
 end

data/lib/yasuri/yasuri_struct_node.rb

@@ -1,6 +1,4 @@
 
-# Author::    TAC (tac@tac42.net)
-
 require_relative 'yasuri_node'
 
 module Yasuri
@@ -10,12 +8,16 @@ module Yasuri
       sub_tags = element.search(@xpath)
       tree = sub_tags.map do |sub_tag|
         child_results_kv = @children.map do |child_node|
-          child_name = Yasuri.NodeName(child_node.name, opt)
+          child_name = Yasuri.node_name(child_node.name, opt)
           [child_name, child_node.inject(agent, page, opt, sub_tag)]
         end
         Hash[child_results_kv]
       end
       tree.size == 1 ? tree.first : tree
-    end # inject
+    end
+
+    def node_type_str
+      "struct".freeze
+    end
   end
 end

data/lib/yasuri/yasuri_text_node.rb

@@ -1,6 +1,4 @@
 
-# Author::    TAC (tac@tac42.net)
-
 require_relative 'yasuri_node'
 
 module Yasuri
@@ -13,15 +11,14 @@ module Yasuri
       truncate = opt[:truncate]
       proc = opt[:proc]
 
-      truncate = Regexp.new(truncate) if not truncate.nil? # regexp or nil
+      truncate = Regexp.new(truncate) unless truncate.nil? # regexp or nil
       @truncate = truncate
-      @truncate = Regexp.new(@truncate.to_s) if not @truncate.nil?
+      @truncate = Regexp.new(@truncate.to_s) unless @truncate.nil?
 
       @proc = proc.nil? ? nil : proc.to_sym
-
     end
 
-    def inject(agent, page, opt = {}, element = page)
+    def inject(_agent, page, _opt = {}, element = page)
       node = element.search(@xpath)
       text = node.text.to_s
 
@@ -31,11 +28,16 @@ module Yasuri
       end
 
       text = text.__send__(@proc) if @proc && text.respond_to?(@proc)
+
       text
     end
 
     def opts
-      {truncate:@truncate, proc:@proc}
+      { truncate: @truncate, proc: @proc }
+    end
+
+    def node_type_str
+      "text".freeze
     end
   end
 end

data/spec/cli_resources/tree.json

@@ -0,0 +1,8 @@
+{
+  "pages_root": {
+    "path": "/html/body/nav/span/a[@class='next']",
+    "limit": 10,
+    "flatten": false,
+    "text_content": "/html/body/p"
+  }
+}

data/spec/cli_resources/tree.yml

@@ -0,0 +1,5 @@
+pages_root:
+  path: "/html/body/nav/span/a[@class='next']"
+  limit: 10
+  flatten: false
+  text_content: "/html/body/p"

data/spec/cli_resources/tree_wrong.json

@@ -0,0 +1,9 @@
+{
+  ,,,
+  "pages_root": {
+    "path": "/html/body/nav/span/a[@class='next']",
+    "limit": 10,
+    "flatten": false,
+    "text_content": "/html/body/p"
+  }
+}

data/spec/cli_resources/tree_wrong.yml

@@ -0,0 +1,6 @@
+,,,
+pages_root:
+  path: "/html/body/nav/span/a[@class='next']"
+  limit: 10
+  flatten: false
+  text_content: "/html/body/p"

data/spec/servers/httpserver.rb

@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-# Author::    TAC (tac@tac42.net)
 
 require 'glint'
 

data/spec/spec_helper.rb

@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-# Author::    TAC (tac@tac42.net)
 
 require 'glint'
 Dir[File.expand_path("../servers/*.rb", __FILE__)].each {|f| require f}
@@ -16,7 +14,7 @@ require 'simplecov'
 require 'coveralls'
 Coveralls.wear!
 
-SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter[
+SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter.new [
   SimpleCov::Formatter::HTMLFormatter,
   Coveralls::SimpleCov::Formatter
 ]
@@ -26,8 +24,8 @@ SimpleCov.start
 $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
 require 'yasuri'
 
-def compare_generated_vs_original(generated, original, page)
-  expected = original.inject(@agent, page)
-  actual   = generated.inject(@agent, page)
+def compare_generated_vs_original(generated, original, uri)
+  expected = original.scrape(uri)
+  actual   = generated.scrape(uri)
   expect(actual).to match expected
 end

data/spec/yasuri_cli_spec.rb

@@ -0,0 +1,114 @@
+require_relative 'spec_helper'
+
+describe 'Yasuri' do
+  include_context 'httpserver'
+  let(:res_dir) { File.expand_path('cli_resources', __dir__) }
+
+  describe 'cli scrape' do
+    it 'require --file or --json option' do
+      expect do
+        Yasuri::CLI.new.invoke(:scrape, [uri], {})
+      end.to output("ERROR: Only one of `--file` or `--json` option should be specified.\n").to_stderr
+    end
+
+    it 'only one of --file or --json option' do
+      expect do
+        Yasuri::CLI.new.invoke(:scrape, [uri], { file: 'path.json', json: '{"text_title": "/html/head/title"}' })
+      end.to output("ERROR: Only one of `--file` or `--json` option should be specified.\n").to_stderr
+    end
+
+    it 'require --file option is not empty string' do
+      expect do
+        Yasuri::CLI.new.invoke(:scrape, [uri], { file: 'file' })
+      end.to output("ERROR: --file option require not empty argument.\n").to_stderr
+    end
+
+    it 'require --json option is not empty string' do
+      expect do
+        Yasuri::CLI.new.invoke(:scrape, [uri], { json: 'json' })
+      end.to output("ERROR: --json option require not empty argument.\n").to_stderr
+    end
+
+    it 'display text node as simple string' do
+      expect do
+        Yasuri::CLI.new.invoke(:scrape, [uri], { json: '{"text_title": "/html/head/title"}' })
+      end.to output("Yasuri Test\n").to_stdout
+    end
+
+    it 'display texts in single json' do
+      expect do
+        Yasuri::CLI.new.invoke(:scrape, [uri], { json: '{"text_c1":"/html/body/p[1]", "text_c2":"/html/body/p[2]"}' })
+      end.to output('{"c1":"Hello,Yasuri","c2":"Last Modify - 2015/02/14"}' << "\n").to_stdout
+    end
+
+    it 'display text node as simple string via json file' do
+      expect do
+        Yasuri::CLI.new.invoke(:scrape, ["#{uri}/pagination/page01.html"], { file: "#{res_dir}/tree.json" })
+      end.to output(
+        '[{"content":"PaginationTest01"},{"content":"PaginationTest02"},' \
+        '{"content":"PaginationTest03"},{"content":"PaginationTest04"}]' \
+        "\n"
+      ).to_stdout
+    end
+
+    it 'display text node as simple string via yaml file' do
+      expect do
+        Yasuri::CLI.new.invoke(:scrape, ["#{uri}/pagination/page01.html"], { file: "#{res_dir}/tree.yml" })
+      end.to output(
+        '[{"content":"PaginationTest01"},{"content":"PaginationTest02"},' \
+        '{"content":"PaginationTest03"},{"content":"PaginationTest04"}]' \
+        "\n"
+      ).to_stdout
+    end
+
+    it 'interval option is effect for each request' do
+      allow(Kernel).to receive(:sleep)
+
+      expect do
+        Yasuri::CLI.new.invoke(
+          :scrape,
+          ["#{uri}/pagination/page01.html"],
+          { file: "#{res_dir}/tree.yml", interval: 500 }
+        )
+      end.to output(
+        '[{"content":"PaginationTest01"},{"content":"PaginationTest02"},' \
+        '{"content":"PaginationTest03"},{"content":"PaginationTest04"}]' \
+        "\n"
+      ).to_stdout
+
+      expect(Kernel).to have_received(:sleep).exactly(4).times do |interval_sec|
+        expect(interval_sec).to match 0.5
+      end
+    end
+
+    it 'display ERROR when json string is wrong' do
+      wrong_json = '{,,}'
+      expect do
+        Yasuri::CLI.new.invoke(:scrape, [uri], { json: wrong_json })
+      end.to output(
+        'ERROR: Failed to convert json to yasuri tree. ' \
+        "809: unexpected token at '#{wrong_json}'\n"
+      ).to_stderr
+    end
+
+    it 'display ERROR when json file contains is wrong' do
+      file_path = "#{res_dir}/tree_wrong.json"
+      expect do
+        Yasuri::CLI.new.invoke(:scrape, [uri], { file: file_path })
+      end.to output(
+        "ERROR: Failed to convert to yasuri tree `#{file_path}`. " \
+        "(<unknown>): did not find expected node content while parsing a flow node at line 2 column 3\n"
+      ).to_stderr
+    end
+
+    it 'display ERROR when yaml file contains is wrong' do
+      file_path = "#{res_dir}/tree_wrong.yml"
+      expect do
+        Yasuri::CLI.new.invoke(:scrape, [uri], { file: file_path })
+      end.to output(
+        "ERROR: Failed to convert to yasuri tree `#{file_path}`. " \
+        "(<unknown>): did not find expected node content while parsing a block node at line 1 column 1\n"
+      ).to_stderr
+    end
+  end
+end