yasuri 2.0.13 → 3.3.1

data/examples/example.rb

@@ -0,0 +1,78 @@
+#!/usr/bin/env ruby
+
+require 'yasuri'
+uri = "https://github.com/tac0x2a?tab=repositories"
+
+# Node tree constructing by DSL
+root = Yasuri.map_root do
+  text_title '/html/head/title'
+  links_repo '//*[@id="user-repositories-list"]/ul/li/div[1]/div[1]/h3/a' do
+    text_name '//*[@id="js-repo-pjax-container"]/div[1]/div[1]/div/h1/strong/a'
+    text_desc '//*[@id="repo-content-pjax-container"]/div/div[2]/div[2]/div/div[1]/div/p', proc: :strip
+    text_stars '//*[@id="js-repo-pjax-container"]/div[1]/div[2]/div[2]/a[1]', proc: :to_i
+    text_forks '//*[@id="js-repo-pjax-container"]/div[1]/div[2]/div[2]/a[2]/span', proc: :to_i
+  end
+end
+
+# Node tree constructing by YAML
+# src = <<-EOYML
+# text_title: /html/head/title
+# links_repo:
+#   path: //*[@id="user-repositories-list"]/ul/li/div[1]/div[1]/h3/a
+#   text_name: //*[@id="js-repo-pjax-container"]/div[1]/div[1]/div/h1/strong/a
+#   text_desc:
+#     path: //*[@id="repo-content-pjax-container"]/div/div[2]/div[2]/div/div[1]/div/p
+#     proc: :strip
+#   text_stars:
+#     path: //*[@id="js-repo-pjax-container"]/div[1]/div[2]/div[2]/a[1]
+#     proc: :to_i
+#   text_forks:
+#     path: //*[@id="js-repo-pjax-container"]/div[1]/div[2]/div[2]/a[2]/span
+#     proc: :to_i
+# EOYML
+# root = Yasuri.yaml2tree(src)
+
+contents = root.scrape(uri, interval_ms: 100)
+# jj contents
+# {
+#   "title": "tac0x2a (TAC) / Repositories · GitHub",
+#   "repo": [
+#     {
+#       "name": "o-namazu",
+#       "desc": "Oh Namazu (Catfish) in datalake",
+#       "stars": 1,
+#       "forks": 0
+#     },
+#     {
+#       "name": "grebe",
+#       "desc": "grebe in datalake",
+#       "stars": 2,
+#       "forks": 0
+#     },
+#     {
+#       "name": "yasuri",
+#       "desc": "Yasuri (鑢) is easy web scraping library.",
+#       "stars": 43,
+#       "forks": 1
+#     },
+#     {
+#       "name": "dotfiles",
+#       "desc": "dotfiles",
+#       "stars": 0,
+#       "forks": 0
+#     }
+#     ...
+#   ]
+# }
+
+# Output as markdown
+puts "# #{contents['title']}"
+contents['repo'].each do |h|
+  puts "-----"
+  puts "## #{h['name']}"
+  puts h['desc']
+  puts ""
+  puts "* Stars: #{h['stars']}"
+  puts "* Forks: #{h['forks']}"
+  puts ""
+end

data/examples/github.yml

@@ -0,0 +1,15 @@
+
+# yasuri scrape "https://github.com/tac0x2a?tab=repositories" -f github.yml
+text_title: /html/head/title
+links_repo:
+  path: //*[@id="user-repositories-list"]/ul/li/div[1]/div[1]/h3/a
+  text_name: //*[@id="js-repo-pjax-container"]/div[1]/div[1]/div/h1/strong/a
+  text_desc:
+    path: //*[@id="repo-content-pjax-container"]/div/div[2]/div[2]/div/div[1]/div/p
+    proc: :strip
+  text_stars:
+    path: //*[@id="js-repo-pjax-container"]/div[1]/div[2]/div[2]/a[1]
+    proc: :to_i
+  text_forks:
+    path: //*[@id="js-repo-pjax-container"]/div[1]/div[2]/div[2]/a[2]/span
+    proc: :to_i

data/examples/sample.json

@@ -0,0 +1,4 @@
+{
+  "text_title": "/html/head/title",
+  "text_desc": "//*[@id=\"intro\"]/p"
+}

data/examples/sample.yml

@@ -0,0 +1,11 @@
+# yasuri scrape "https://www.tac42.net/" -f sample.yml
+links_each:
+  path: //*[@id="posts"]/article/header/h1/a
+  text_title: //*[@id="content"]/article/header/h1
+  text_description: /html/head/meta[12]/@content
+  text_date:
+    path: //*[@id="content"]/article/header/div/span
+    proc: :strip
+  text_length:
+    path: //*[@id="content"]
+    proc: :size

data/exe/yasuri

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require "yasuri"
+
+Yasuri::CLI.start

data/lib/yasuri.rb

@@ -1,5 +1,6 @@
 require "yasuri/version"
 require "yasuri/yasuri"
+require "yasuri/yasuri_cli"
 
 module Yasuri
   # Your code goes here...

data/lib/yasuri/version.rb

@@ -1,3 +1,3 @@
 module Yasuri
-  VERSION = "2.0.13"
+  VERSION = "3.3.1"
 end

data/lib/yasuri/yasuri.rb

@@ -1,6 +1,3 @@
-# -*- coding: utf-8 -*-
-
-# Author::    TAC (tac@tac42.net)
 
 require 'mechanize'
 require 'json'
@@ -11,118 +8,142 @@ require_relative 'yasuri_text_node'
 require_relative 'yasuri_struct_node'
 require_relative 'yasuri_paginate_node'
 require_relative 'yasuri_links_node'
+require_relative 'yasuri_map_node'
 require_relative 'yasuri_node_generator'
 
 module Yasuri
 
+  DefaultRetryCount = 5
+  DefaultInterval_ms = 0
+
   def self.json2tree(json_string)
-    json = JSON.parse(json_string, {symbolize_names: true})
-    Yasuri.hash2node(json)
-  end
+    raise RuntimeError if json_string.nil? or json_string.empty?
 
-  def self.tree2json(node)
-    Yasuri.node2hash(node).to_json
+    node_hash = JSON.parse(json_string, {symbolize_names: true})
+    self.hash2node(node_hash)
   end
 
   def self.yaml2tree(yaml_string)
     raise RuntimeError if yaml_string.nil? or yaml_string.empty?
 
-    yaml = YAML.load(yaml_string)
-    raise RuntimeError if yaml.keys.size < 1
+    node_hash = YAML.safe_load(yaml_string, [Symbol], symbolize_names: true)
+    self.hash2node(node_hash.deep_symbolize_keys)
+  end
 
-    root_key, root = yaml.keys.first, yaml.values.first
-    hash = Yasuri.yaml2tree_sub(root_key, root)
+  def self.tree2json(node)
+    raise RuntimeError if node.nil?
 
-    Yasuri.hash2node(hash)
+    self.node2hash(node).to_json
   end
 
-  private
-  def self.yaml2tree_sub(name, body)
-    return nil if name.nil? or body.nil?
+  def self.with_retry(
+    retry_count = DefaultRetryCount,
+    interval_ms = DefaultInterval_ms)
 
-    new_body = Hash[:name, name]
-    body.each{|k,v| new_body[k.to_sym] = v}
-    body = new_body
+    begin
+      Kernel.sleep(interval_ms * 0.001)
+      return yield() if block_given?
+    rescue => e
+      if retry_count > 0
+        retry_count -= 1
+        retry
+      end
+      fail e
+    end
+  end
 
-    return body if body[:children].nil?
+  def self.node_name(name, opt)
+    symbolize_names = opt[:symbolize_names]
+    symbolize_names ? name.to_sym : name
+  end
 
-    body[:children] = body[:children].map do |c|
-      k, b = c.keys.first, c.values.first
-      Yasuri.yaml2tree_sub(k, b)
+  # private
+
+  def self.hash2node(node_hash, node_name = nil, node_type_class = nil)
+    raise RuntimeError.new("") if node_name.nil? and node_hash.empty?
+
+    child_nodes = []
+    opt = {}
+    path = nil
+
+    if node_hash.is_a?(String)
+      path = node_hash
+    else
+      child_nodes, opt, path = self.hash2child_node(node_hash)
     end
 
-    body
-  end
+    # If only single node under root, return only the node.
+    return child_nodes.first if node_name.nil? and child_nodes.size == 1
 
-  def self.method_missing(name, *args, &block)
-    generated = Yasuri::NodeGenerator.gen(name, *args, &block)
-    generated || super(name, args)
+    node = if node_type_class.nil?
+      Yasuri::MapNode.new(node_name, child_nodes, **opt)
+    else
+      node_type_class::new(path, node_name, child_nodes, **opt)
+    end
+
+    node
   end
 
-  private
   Text2Node = {
     text:   Yasuri::TextNode,
     struct: Yasuri::StructNode,
     links:  Yasuri::LinksNode,
-    pages:  Yasuri::PaginateNode
+    pages:  Yasuri::PaginateNode,
+    map:    Yasuri::MapNode
   }
-  Node2Text = Text2Node.invert
-
-  ReservedKeys = %i|node name path children|
-  def self.hash2node(node_h)
-    node, name, path, children = ReservedKeys.map do |key|
-      node_h[key]
-    end
-    children ||= []
 
-    fail "Not found 'node' value in map" if node.nil?
-    fail "Not found 'name' value in map" if name.nil?
-    fail "Not found 'path' value in map" if path.nil?
+  NodeRegexps = Text2Node.keys.map { |node_type_sym| /^(#{node_type_sym})_(.+)$/ }
 
-    childnodes = children.map{|c| Yasuri.hash2node(c) }
-    ReservedKeys.each{|key| node_h.delete(key)}
-    opt = node_h
+  def self.hash2child_node(node_hash)
+    child_nodes = []
+    opt = {}
+    path = nil
 
-    klass = Text2Node[node.to_sym]
-    fail "Undefined node type #{node}" if klass.nil?
-    klass.new(path, name, childnodes, opt)
-  end
-
-  def self.node2hash(node)
-    json = JSON.parse("{}")
-    return json if node.nil?
+    node_hash.each do |key, value|
+      # is node?
 
-    klass = node.class
-    klass_str = Node2Text[klass]
+      node_regexp = NodeRegexps.find { |r| key =~ r }
 
-    json["node"] = klass_str
-    json["name"] = node.name
-    json["path"] = node.xpath
+      case key
+      when node_regexp
+        node_type_sym = $1.to_sym
+        child_node_name = $2
+        child_node_type = Text2Node[node_type_sym]
+        child_nodes << self.hash2node(value, child_node_name, child_node_type)
+      when :path
+        path = value
+      else
+        opt[key] = value
+      end
+    end
 
-    children = node.children.map{|c| Yasuri.node2hash(c)}
-    json["children"] = children if not children.empty?
+    [child_nodes, opt, path]
+  end
 
-    node.opts.each do |key,value|
-      json[key] = value if not value.nil?
-    end
+  def self.node2hash(node)
+    return node.to_h if node.instance_of?(Yasuri::MapNode)
 
-    json
+    {
+      "#{node.node_type_str}_#{node.name}" => node.to_h
+    }
   end
 
-  def self.NodeName(name, symbolize_names:false)
-    symbolize_names ? name.to_sym : name
+  def self.method_missing(method_name, pattern=nil, **opt, &block)
+    generated = Yasuri::NodeGenerator.gen(method_name, pattern, **opt, &block)
+    generated || super(method_name, **opt)
   end
 
-  def self.with_retry(retry_count = 5)
-    begin
-      return yield() if block_given?
-    rescue => e
-      if retry_count > 0
-        pp "retry #{retry_count}"
-        retry_count -= 1
-        retry
+  private_constant :Text2Node, :NodeRegexps
+  private_class_method :method_missing, :hash2child_node, :hash2node, :node2hash
+end
+
+class Hash
+  def deep_symbolize_keys
+    Hash[
+      self.map do |k, v|
+        v = v.deep_symbolize_keys if v.kind_of?(Hash)
+        [k.to_sym, v]
       end
-      fail e
-    end
+    ]
   end
 end

data/lib/yasuri/yasuri_cli.rb

@@ -0,0 +1,78 @@
+require 'thor'
+require 'json'
+require 'yasuri'
+
+module Yasuri
+  class CLI < Thor
+    package_name "yasuri"
+
+    default_command :scrape
+    desc "scrape <URI> [[--file <TREE_FILE>] or [--json <JSON>]]",
+         "Getting from <URI> and scrape it. with <JSON> or json/yml from <TREE_FILE>. They should be Yasuri's format json or yaml string."
+    option :file,     {aliases: 'f', desc: "path to file that written yasuri tree as json or yaml", type: :string}
+    option :json,     {aliases: 'j', desc: "yasuri tree format json string", type: :string}
+    option :interval, {aliases: 'i', desc: "interval each request [ms]", type: :numeric}
+    def scrape(uri)
+      begin
+        test_arguments(options)
+      rescue => e
+        $stderr.puts e.message
+        return -1
+      end
+
+      interval_ms = options[:interval] || Yasuri::DefaultInterval_ms
+      file_path = options[:file]
+      json_string = options[:json]
+
+      begin
+        tree = make_tree(file_path, json_string)
+        result = tree.scrape(uri, interval_ms: interval_ms)
+      rescue => e
+        $stderr.puts e.message
+        return -1
+      end
+
+      if result.instance_of?(String)
+        puts result
+      else
+        j result
+      end
+
+      return 0
+    end
+
+    private
+
+    def test_arguments(options)
+      too_many_options = [options[:file], options[:json]].compact.count != 1
+      raise "ERROR: Only one of `--file` or `--json` option should be specified." if too_many_options
+
+      empty_file_argument = options[:file]&.empty? || options[:file] == "file" || options[:json]&.empty?
+      raise "ERROR: --file option require not empty argument." if empty_file_argument
+
+      empty_json_string_argument = options[:json]&.empty? || options[:json] == "json"
+      raise "ERROR: --json option require not empty argument." if empty_json_string_argument
+    end
+
+    def make_tree(file_path, json_string)
+      if file_path
+        begin
+          src = File.read(file_path)
+          make_tree_from_file(src)
+        rescue => e
+          raise "ERROR: Failed to convert to yasuri tree `#{file_path}`. #{e.message}"
+        end
+      else
+        begin
+          Yasuri.json2tree(json_string)
+        rescue => e
+          raise "ERROR: Failed to convert json to yasuri tree. #{e.message}"
+        end
+      end
+    end
+
+    def make_tree_from_file(src)
+        Yasuri.json2tree(src) rescue Yasuri.yaml2tree(src)
+    end
+  end
+end

data/lib/yasuri/yasuri_links_node.rb

@@ -1,26 +1,30 @@
 
-# Author::    TAC (tac@tac42.net)
-
 require_relative 'yasuri_node'
 
 module Yasuri
   class LinksNode
     include Node
+
     def inject(agent, page, opt = {}, element = page)
-      retry_count = opt[:retry_count] || 5
+      retry_count = opt[:retry_count] || Yasuri::DefaultRetryCount
+      interval_ms = opt[:interval_ms] || Yasuri::DefaultInterval_ms
 
       links = element.search(@xpath) || [] # links expected
       links.map do |link|
         link_button = Mechanize::Page::Link.new(link, agent, page)
-        child_page = Yasuri.with_retry(retry_count) { link_button.click }
+        child_page = Yasuri.with_retry(retry_count, interval_ms) { link_button.click }
 
         child_results_kv = @children.map do |child_node|
-          child_name = Yasuri.NodeName(child_node.name, opt)
+          child_name = Yasuri.node_name(child_node.name, opt)
           [child_name, child_node.inject(agent, child_page, opt)]
         end
 
         Hash[child_results_kv]
-      end # each named child node
+      end
+    end
+
+    def node_type_str
+      "links".freeze
     end
   end
 end