yasuri 3.2.0 → 3.3.0

data/examples/github.yml

@@ -0,0 +1,15 @@
+
+# yasuri scrape "https://github.com/tac0x2a?tab=repositories" -f github.yml
+text_title: /html/head/title
+links_repo:
+  path: //*[@id="user-repositories-list"]/ul/li/div[1]/div[1]/h3/a
+  text_name: //*[@id="js-repo-pjax-container"]/div[1]/div[1]/div/h1/strong/a
+  text_desc:
+    path: //*[@id="repo-content-pjax-container"]/div/div[2]/div[2]/div/div[1]/div/p
+    proc: :strip
+  text_stars:
+    path: //*[@id="js-repo-pjax-container"]/div[1]/div[2]/div[2]/a[1]
+    proc: :to_i
+  text_forks:
+    path: //*[@id="js-repo-pjax-container"]/div[1]/div[2]/div[2]/a[2]/span
+    proc: :to_i

data/examples/sample.json

@@ -0,0 +1,4 @@
+{
+  "text_title": "/html/head/title",
+  "text_desc": "//*[@id=\"intro\"]/p"
+}

data/examples/sample.yml

@@ -0,0 +1,11 @@
+# yasuri scrape "https://www.tac42.net/" -f sample.yml
+links_each:
+  path: //*[@id="posts"]/article/header/h1/a
+  text_title: //*[@id="content"]/article/header/h1
+  text_description: /html/head/meta[12]/@content
+  text_date:
+    path: //*[@id="content"]/article/header/div/span
+    proc: :strip
+  text_length:
+    path: //*[@id="content"]
+    proc: :size

data/lib/yasuri/version.rb

@@ -1,3 +1,3 @@
 module Yasuri
-  VERSION = "3.2.0"
+  VERSION = "3.3.0"
 end

data/lib/yasuri/yasuri.rb

@@ -17,6 +17,7 @@ require_relative 'yasuri_node_generator'
 module Yasuri
 
   DefaultRetryCount = 5
+  DefaultInterval_ms = 0
 
   def self.json2tree(json_string)
     raise RuntimeError if json_string.nil? or json_string.empty?
@@ -112,12 +113,15 @@ module Yasuri
     symbolize_names ? name.to_sym : name
   end
 
-  def self.with_retry(retry_count = 5)
+  def self.with_retry(
+    retry_count = DefaultRetryCount,
+    interval_ms = DefaultInterval_ms)
+
     begin
+      Kernel.sleep(interval_ms * 0.001)
       return yield() if block_given?
     rescue => e
       if retry_count > 0
-        pp "retry #{retry_count}"
         retry_count -= 1
         retry
       end

data/lib/yasuri/yasuri_cli.rb

@@ -1,7 +1,6 @@
 require 'thor'
 require 'json'
 require 'yasuri'
-require 'mechanize'
 
 module Yasuri
   class CLI < Thor
@@ -9,8 +8,9 @@ module Yasuri
 
     default_command :scrape
     desc "scrape <URI> [[--file <TREE_FILE>] or [--json <JSON>]]", "Getting from <URI> and scrape it. with <JSON> or json/yml from <TREE_FILE>. They should be Yasuri's format json or yaml string."
-    option :file, {aliases: 'f', desc: "path to file that written yasuri tree as json or yaml", type: :string}
-    option :json, {aliases: 'j', desc: "yasuri tree format json string", type: :string}
+    option :file,     {aliases: 'f', desc: "path to file that written yasuri tree as json or yaml", type: :string}
+    option :json,     {aliases: 'j', desc: "yasuri tree format json string", type: :string}
+    option :interval, {aliases: 'i', desc: "interval each request [ms]", type: :numeric}
     def scrape(uri)
       # argument validations
       if [options[:file], options[:json]].compact.count != 1
@@ -26,6 +26,8 @@ module Yasuri
         return -1
       end
 
+      interval_ms = options[:interval] || Yasuri::DefaultInterval_ms
+
       tree = if options[:file]
               src = File.read(options[:file])
 
@@ -48,9 +50,7 @@ module Yasuri
               end
             end
 
-      agent = Mechanize.new
-      root_page = agent.get(uri)
-      result = tree.inject(agent, root_page)
+      result = tree.scrape(uri, interval_ms: interval_ms)
 
       if result.instance_of?(String)
         puts result

data/lib/yasuri/yasuri_links_node.rb

@@ -6,13 +6,15 @@ require_relative 'yasuri_node'
 module Yasuri
   class LinksNode
     include Node
+
     def inject(agent, page, opt = {}, element = page)
       retry_count = opt[:retry_count] || Yasuri::DefaultRetryCount
+      interval_ms = opt[:interval_ms] || Yasuri::DefaultInterval_ms
 
       links = element.search(@xpath) || [] # links expected
       links.map do |link|
         link_button = Mechanize::Page::Link.new(link, agent, page)
-        child_page = Yasuri.with_retry(retry_count) { link_button.click }
+        child_page = Yasuri.with_retry(retry_count, interval_ms) { link_button.click }
 
         child_results_kv = @children.map do |child_node|
           child_name = Yasuri.node_name(child_node.name, opt)

data/lib/yasuri/yasuri_map_node.rb

@@ -1,6 +1,7 @@
 
 module Yasuri
   class MapNode
+    include Node
     attr_reader :name, :children
 
     def initialize(name, children, **opt)

data/lib/yasuri/yasuri_node.rb

@@ -11,6 +11,20 @@ module Yasuri
       @xpath, @name, @children = xpath, name, children
     end
 
+    def scrape(uri, opt = {})
+      retry_count = opt[:retry_count] || Yasuri::DefaultRetryCount
+      interval_ms = opt[:interval_ms] || Yasuri::DefaultInterval_ms
+
+      agent = Mechanize.new
+      page = Yasuri.with_retry(retry_count, interval_ms) { agent.get(uri) }
+      scrape_with_agent(uri, agent, opt)
+    end
+
+    def scrape_with_agent(uri, agent, opt = {})
+      page = agent.get(uri)
+      inject(agent, page, opt)
+    end
+
     def inject(agent, page, opt = {}, element = page)
       fail "#{Kernel.__method__} is not implemented in included class."
     end

data/lib/yasuri/yasuri_paginate_node.rb

@@ -15,6 +15,7 @@ module Yasuri
 
     def inject(agent, page, opt = {}, element = page)
       retry_count = opt[:retry_count] || Yasuri::DefaultRetryCount
+      interval_ms = opt[:interval_ms] || Yasuri::DefaultInterval_ms
 
       raise NotImplementedError.new("PagenateNode inside StructNode, Not Supported") if page != element
 
@@ -31,7 +32,7 @@ module Yasuri
         break if link == nil
 
         link_button = Mechanize::Page::Link.new(link, agent, page)
-        page = Yasuri.with_retry(retry_count) { link_button.click }
+        page = Yasuri.with_retry(retry_count, interval_ms) { link_button.click }
         break if (limit -= 1) <= 0
       end
 

data/spec/spec_helper.rb

@@ -26,8 +26,8 @@ SimpleCov.start
 $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
 require 'yasuri'
 
-def compare_generated_vs_original(generated, original, page)
-  expected = original.inject(@agent, page)
-  actual   = generated.inject(@agent, page)
+def compare_generated_vs_original(generated, original, uri)
+  expected = original.scrape(uri)
+  actual   = generated.scrape(uri)
   expect(actual).to match expected
 end

data/spec/yasuri_cli_spec.rb

@@ -51,15 +51,28 @@ describe 'Yasuri' do
 
     it "display text node as simple string via json file" do
       expect {
-        Yasuri::CLI.new.invoke(:scrape, [uri], {file: "#{@res_dir}/tree.json"})
-      }.to output('[{"content":"Hello,YasuriLast Modify - 2015/02/14"}]' + "\n").to_stdout
+        Yasuri::CLI.new.invoke(:scrape, [uri+"/pagination/page01.html"], {file: "#{@res_dir}/tree.json"})
+      }.to output('[{"content":"PaginationTest01"},{"content":"PaginationTest02"},{"content":"PaginationTest03"},{"content":"PaginationTest04"}]' + "\n").to_stdout
     end
     it "display text node as simple string via yaml file" do
       expect {
-        Yasuri::CLI.new.invoke(:scrape, [uri], {file: "#{@res_dir}/tree.yml"})
-      }.to output('[{"content":"Hello,YasuriLast Modify - 2015/02/14"}]' + "\n").to_stdout
+        Yasuri::CLI.new.invoke(:scrape, [uri+"/pagination/page01.html"], {file: "#{@res_dir}/tree.yml"})
+      }.to output('[{"content":"PaginationTest01"},{"content":"PaginationTest02"},{"content":"PaginationTest03"},{"content":"PaginationTest04"}]' + "\n").to_stdout
     end
 
+    it "interval option is effect for each request" do
+      allow(Kernel).to receive(:sleep)
+
+      Yasuri::CLI.new.invoke(
+        :scrape,
+        [uri+"/pagination/page01.html"],
+        {file: "#{@res_dir}/tree.yml", interval: 500}
+      )
+
+      expect(Kernel).to have_received(:sleep).exactly(4).times do |interval_sec|
+        expect(interval_sec).to match 0.5
+      end
+    end
 
     it "display ERROR when json string is wrong" do
       wrong_json = '{,,}'

data/spec/yasuri_links_node_spec.rb

@@ -11,9 +11,7 @@ describe 'Yasuri' do
 
   describe '::LinksNode' do
     before do
-      @agent = Mechanize.new
       @uri = uri
-      @index_page = @agent.get(@uri)
     end
 
     it 'scrape links' do
@@ -21,7 +19,7 @@ describe 'Yasuri' do
         Yasuri::TextNode.new('/html/body/p', "content"),
       ])
 
-      actual = root_node.inject(@agent, @index_page)
+      actual = root_node.scrape(@uri)
       expected = [
         {"content" => "Child 01 page."},
         {"content" => "Child 02 page."},
@@ -36,7 +34,7 @@ describe 'Yasuri' do
         Yasuri::TextNode.new('/html/body/p', "content"),
       ])
 
-      actual = root_node.inject(@agent, @index_page)
+      actual = root_node.scrape(@uri)
       expect(actual).to be_empty
     end
 
@@ -47,7 +45,7 @@ describe 'Yasuri' do
           Yasuri::TextNode.new('/html/head/title', "sub_page_title"),
         ]),
       ])
-      actual = root_node.inject(@agent, @index_page)
+      actual = root_node.scrape(@uri)
       expected = [
         {"content"  => "Child 01 page.",
          "sub_link" => [{"sub_page_title" => "Child 01 SubPage Test"},
@@ -61,14 +59,14 @@ describe 'Yasuri' do
     end
     it 'can be defined by DSL, return no contains if no child node' do
       root_node = Yasuri.links_title '/html/body/a'
-      actual = root_node.inject(@agent, @index_page)
+      actual = root_node.scrape(@uri)
       expected = [{}, {}, {}] # Empty if no child node under links node.
       expect(actual).to match expected
     end
 
     it 'can be defined return no contains if no child node' do
       root_node = Yasuri::LinksNode.new('/html/body/a', "title")
-      actual = root_node.inject(@agent, @index_page)
+      actual = root_node.scrape(@uri)
       expected = [{}, {}, {}] # Empty if no child node under links node.
       expect(actual).to match expected
     end
@@ -79,7 +77,7 @@ describe 'Yasuri' do
       original = Yasuri::LinksNode.new('/html/body/a', "root", [
         Yasuri::TextNode.new('/html/body/p', "name"),
       ])
-      compare_generated_vs_original(generated, original, @index_page)
+      compare_generated_vs_original(generated, original, @uri)
     end
 
     it 'can be defined by DSL, return recursive links node' do
@@ -96,7 +94,7 @@ describe 'Yasuri' do
           Yasuri::TextNode.new('/html/head/title', "sub_page_title"),
         ]),
       ])
-      compare_generated_vs_original(generated, original, @index_page)
+      compare_generated_vs_original(generated, original, @uri)
     end
 
     it 'return child node as symbol' do
@@ -104,7 +102,7 @@ describe 'Yasuri' do
         Yasuri::TextNode.new('/html/body/p', "content"),
       ])
 
-      actual = root_node.inject(@agent, @index_page, symbolize_names: true )
+      actual = root_node.scrape(@uri, symbolize_names: true )
       expected = [
         {:content => "Child 01 page."},
         {:content => "Child 02 page."},
@@ -112,5 +110,21 @@ describe 'Yasuri' do
       ]
       expect(actual).to match expected
     end
+
+    it 'scrape with interval for each request' do
+      allow(Kernel).to receive(:sleep)
+
+      root_node = Yasuri::LinksNode.new('/html/body/a', "root", [
+        Yasuri::TextNode.new('/html/body/p', "content"),
+      ])
+      actual = root_node.scrape(@uri, interval_ms: 100)
+
+      expect(actual.size).to match 3
+
+      # request will be run 4(1+3) times because root page will be requested
+      expect(Kernel).to have_received(:sleep).exactly(1+3).times do |interval_sec|
+        expect(interval_sec).to match 0.1
+      end
+    end
   end
 end

data/spec/yasuri_map_spec.rb

@@ -4,8 +4,7 @@ describe 'Yasuri' do
   include_context 'httpserver'
 
   before do
-    @agent = Mechanize.new
-    @index_page = @agent.get(uri)
+    @uri = uri
   end
 
   describe '::MapNode' do
@@ -14,7 +13,7 @@ describe 'Yasuri' do
         text_title  '/html/head/title'
         text_body_p '/html/body/p[1]'
       end
-      actual = map.inject(@agent, @index_page)
+      actual = map.scrape(@uri)
 
       expected = {
         "title"  => "Yasuri Test",
@@ -31,7 +30,7 @@ describe 'Yasuri' do
           text_child03 '/html/body/a[3]'
         end
       end
-      actual = map.inject(@agent, @index_page)
+      actual = map.scrape(@uri)
 
       expected = {
         "group1" => {
@@ -57,7 +56,7 @@ describe 'Yasuri' do
           text_child03 '/html/body/a[3]'
         end
       end
-      actual = map.inject(@agent, @index_page)
+      actual = map.scrape(@uri)
 
       expected = {
         "group1" => {

data/spec/yasuri_paginate_node_spec.rb

@@ -11,16 +11,14 @@ describe 'Yasuri' do
 
   describe '::PaginateNode' do
     before do
-      @agent = Mechanize.new
       @uri = uri + "/pagination/page01.html"
-      @page = @agent.get(@uri)
     end
 
     it "scrape each paginated pages" do
       root_node = Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "root", [
         Yasuri::TextNode.new('/html/body/p', "content"),
       ])
-      actual = root_node.inject(@agent, @page)
+      actual = root_node.scrape(@uri)
       expected = [
         {"content" => "PaginationTest01"},
         {"content" => "PaginationTest02"},
@@ -37,7 +35,7 @@ describe 'Yasuri' do
           Yasuri::TextNode.new('./a', "text"),
         ]),
       ], flatten: true)
-      actual = root_node.inject(@agent, @page)
+      actual = root_node.scrape(@uri)
       expected = [
         "PaginationTest01",
         {"text"=>""},
@@ -77,7 +75,7 @@ describe 'Yasuri' do
       root_node = Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "root", [
         Yasuri::TextNode.new('/html/body/p', "content"),
       ], limit:3)
-      actual = root_node.inject(@agent, @page)
+      actual = root_node.scrape(@uri)
       expected = [
         {"content" => "PaginationTest01"},
         {"content" => "PaginationTest02"},
@@ -91,7 +89,7 @@ describe 'Yasuri' do
       root_node = Yasuri::PaginateNode.new(missing_xpath, "root", [
         Yasuri::TextNode.new('/html/body/p', "content"),
       ])
-      actual = root_node.inject(@agent, @page)
+      actual = root_node.scrape(@uri)
       expected = [ {"content" => "PaginationTest01"}, ]
       expect(actual).to match_array expected
     end
@@ -100,7 +98,7 @@ describe 'Yasuri' do
       root_node = Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "root", [
         Yasuri::TextNode.new('/html/body/hoge', "content"),
       ])
-      actual = root_node.inject(@agent, @page)
+      actual = root_node.scrape(@uri)
       expected = [ {"content" => ""}, {"content" => ""}, {"content" => ""}, {"content" => ""},]
       expect(actual).to match_array expected
     end
@@ -112,7 +110,7 @@ describe 'Yasuri' do
       original = Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "root", [
         Yasuri::TextNode.new('/html/body/p', "content"),
       ])
-      compare_generated_vs_original(generated, original, @page)
+      compare_generated_vs_original(generated, original, @uri)
     end
 
     it 'can be defined by DSL, return single PaginateNode content limited' do
@@ -122,14 +120,14 @@ describe 'Yasuri' do
       original = Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "root", [
         Yasuri::TextNode.new('/html/body/p', "content"),
       ], limit: 2)
-      compare_generated_vs_original(generated, original, @page)
+      compare_generated_vs_original(generated, original, @uri)
     end
 
     it "return child node as symbol" do
       root_node = Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "root", [
         Yasuri::TextNode.new('/html/body/p', "content"),
       ])
-      actual = root_node.inject(@agent, @page, symbolize_names:true)
+      actual = root_node.scrape(@uri, symbolize_names:true)
       expected = [
         {:content => "PaginationTest01"},
         {:content => "PaginationTest02"},
@@ -138,5 +136,19 @@ describe 'Yasuri' do
       ]
       expect(actual).to match expected
     end
+
+    it "scrape with interval for each request" do
+      allow(Kernel).to receive(:sleep)
+
+      root_node = Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "root", [
+        Yasuri::TextNode.new('/html/body/p', "content"),
+      ])
+      actual = root_node.scrape(@uri, interval_ms: 1000)
+      expect(actual.size).to match 4
+
+      expect(Kernel).to have_received(:sleep).exactly(4).times do |interval_sec|
+        expect(interval_sec).to match 1.0
+      end
+    end
   end
 end