yasuri 2.0.12 → 3.3.0

data/examples/example.rb

@@ -0,0 +1,79 @@
+#!/usr/bin/env ruby
+# Author::    TAC (tac@tac42.net)
+
+require 'yasuri'
+uri = "https://github.com/tac0x2a?tab=repositories"
+
+# Node tree constructing by DSL
+root = Yasuri.map_root do
+  text_title '/html/head/title'
+  links_repo '//*[@id="user-repositories-list"]/ul/li/div[1]/div[1]/h3/a' do
+    text_name '//*[@id="js-repo-pjax-container"]/div[1]/div[1]/div/h1/strong/a'
+    text_desc '//*[@id="repo-content-pjax-container"]/div/div[2]/div[2]/div/div[1]/div/p', proc: :strip
+    text_stars '//*[@id="js-repo-pjax-container"]/div[1]/div[2]/div[2]/a[1]', proc: :to_i
+    text_forks '//*[@id="js-repo-pjax-container"]/div[1]/div[2]/div[2]/a[2]/span', proc: :to_i
+  end
+end
+
+# Node tree constructing by YAML
+# src = <<-EOYML
+# text_title: /html/head/title
+# links_repo:
+#   path: //*[@id="user-repositories-list"]/ul/li/div[1]/div[1]/h3/a
+#   text_name: //*[@id="js-repo-pjax-container"]/div[1]/div[1]/div/h1/strong/a
+#   text_desc:
+#     path: //*[@id="repo-content-pjax-container"]/div/div[2]/div[2]/div/div[1]/div/p
+#     proc: :strip
+#   text_stars:
+#     path: //*[@id="js-repo-pjax-container"]/div[1]/div[2]/div[2]/a[1]
+#     proc: :to_i
+#   text_forks:
+#     path: //*[@id="js-repo-pjax-container"]/div[1]/div[2]/div[2]/a[2]/span
+#     proc: :to_i
+# EOYML
+# root = Yasuri.yaml2tree(src)
+
+contents = root.scrape(uri, interval_ms: 100)
+# jj contents
+# {
+#   "title": "tac0x2a (TAC) / Repositories · GitHub",
+#   "repo": [
+#     {
+#       "name": "o-namazu",
+#       "desc": "Oh Namazu (Catfish) in datalake",
+#       "stars": 1,
+#       "forks": 0
+#     },
+#     {
+#       "name": "grebe",
+#       "desc": "grebe in datalake",
+#       "stars": 2,
+#       "forks": 0
+#     },
+#     {
+#       "name": "yasuri",
+#       "desc": "Yasuri (鑢) is easy web scraping library.",
+#       "stars": 43,
+#       "forks": 1
+#     },
+#     {
+#       "name": "dotfiles",
+#       "desc": "dotfiles",
+#       "stars": 0,
+#       "forks": 0
+#     }
+#     ...
+#   ]
+# }
+
+# Output as markdown
+puts "# #{contents['title']}"
+contents['repo'].each do |h|
+  puts "-----"
+  puts "## #{h['name']}"
+  puts h['desc']
+  puts ""
+  puts "* Stars: #{h['stars']}"
+  puts "* Forks: #{h['forks']}"
+  puts ""
+end

data/examples/github.yml

@@ -0,0 +1,15 @@
+
+# yasuri scrape "https://github.com/tac0x2a?tab=repositories" -f github.yml
+text_title: /html/head/title
+links_repo:
+  path: //*[@id="user-repositories-list"]/ul/li/div[1]/div[1]/h3/a
+  text_name: //*[@id="js-repo-pjax-container"]/div[1]/div[1]/div/h1/strong/a
+  text_desc:
+    path: //*[@id="repo-content-pjax-container"]/div/div[2]/div[2]/div/div[1]/div/p
+    proc: :strip
+  text_stars:
+    path: //*[@id="js-repo-pjax-container"]/div[1]/div[2]/div[2]/a[1]
+    proc: :to_i
+  text_forks:
+    path: //*[@id="js-repo-pjax-container"]/div[1]/div[2]/div[2]/a[2]/span
+    proc: :to_i

data/examples/sample.json

@@ -0,0 +1,4 @@
+{
+  "text_title": "/html/head/title",
+  "text_desc": "//*[@id=\"intro\"]/p"
+}

data/examples/sample.yml

@@ -0,0 +1,11 @@
+# yasuri scrape "https://www.tac42.net/" -f sample.yml
+links_each:
+  path: //*[@id="posts"]/article/header/h1/a
+  text_title: //*[@id="content"]/article/header/h1
+  text_description: /html/head/meta[12]/@content
+  text_date:
+    path: //*[@id="content"]/article/header/div/span
+    proc: :strip
+  text_length:
+    path: //*[@id="content"]
+    proc: :size

data/exe/yasuri

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require "yasuri"
+
+Yasuri::CLI.start

data/lib/yasuri.rb

@@ -1,5 +1,6 @@
 require "yasuri/version"
 require "yasuri/yasuri"
+require "yasuri/yasuri_cli"
 
 module Yasuri
   # Your code goes here...

data/lib/yasuri/version.rb

@@ -1,3 +1,3 @@
 module Yasuri
-  VERSION = "2.0.12"
+  VERSION = "3.3.0"
 end

data/lib/yasuri/yasuri.rb

@@ -4,28 +4,45 @@
 
 require 'mechanize'
 require 'json'
+require 'yaml'
 
 require_relative 'yasuri_node'
 require_relative 'yasuri_text_node'
 require_relative 'yasuri_struct_node'
 require_relative 'yasuri_paginate_node'
 require_relative 'yasuri_links_node'
+require_relative 'yasuri_map_node'
 require_relative 'yasuri_node_generator'
 
 module Yasuri
 
+  DefaultRetryCount = 5
+  DefaultInterval_ms = 0
+
   def self.json2tree(json_string)
-    json = JSON.parse(json_string, {symbolize_names: true})
-    Yasuri.hash2node(json)
+    raise RuntimeError if json_string.nil? or json_string.empty?
+
+    node_hash = JSON.parse(json_string, {symbolize_names: true})
+    Yasuri.hash2node(node_hash)
   end
 
   def self.tree2json(node)
+    raise RuntimeError if node.nil?
+
     Yasuri.node2hash(node).to_json
   end
 
-  def self.method_missing(name, *args, &block)
-    generated = Yasuri::NodeGenerator.gen(name, *args, &block)
-    generated || super(name, args)
+  def self.yaml2tree(yaml_string)
+    raise RuntimeError if yaml_string.nil? or yaml_string.empty?
+
+    node_hash = YAML.load(yaml_string)
+    Yasuri.hash2node(node_hash.deep_symbolize_keys)
+  end
+
+  private
+  def self.method_missing(method_name, pattern=nil, **opt, &block)
+    generated = Yasuri::NodeGenerator.gen(method_name, pattern, **opt, &block)
+    generated || super(method_name, **opt)
   end
 
   private
@@ -33,61 +50,78 @@ module Yasuri
     text:   Yasuri::TextNode,
     struct: Yasuri::StructNode,
     links:  Yasuri::LinksNode,
-    pages:  Yasuri::PaginateNode
+    pages:  Yasuri::PaginateNode,
+    map:    Yasuri::MapNode
   }
-  Node2Text = Text2Node.invert
 
-  ReservedKeys = %i|node name path children|
-  def self.hash2node(node_h)
-    node, name, path, children = ReservedKeys.map do |key|
-      node_h[key]
+  def self.hash2node(node_hash, node_name = nil, node_type_class = nil)
+    raise RuntimeError.new("") if node_name.nil? and node_hash.empty?
+
+    node_prefixes = Text2Node.keys.freeze
+    child_nodes = []
+    opt = {}
+    path = nil
+
+    if node_hash.is_a?(String)
+      path = node_hash
+    else
+      node_hash.each do |key, value|
+        # is node?
+        node_regexps = Text2Node.keys.map do |node_type_sym|
+          /^(#{node_type_sym.to_s})_(.+)$/
+        end
+        node_regexp = node_regexps.find do |node_regexp|
+          key =~ node_regexp
+        end
+
+        case key
+        when node_regexp
+          node_type_sym = $1.to_sym
+          child_node_name = $2
+          child_node_type = Text2Node[node_type_sym]
+          child_nodes << self.hash2node(value, child_node_name, child_node_type)
+        when :path
+          path = value
+        else
+          opt[key] = value
+        end
+      end
     end
-    children ||= []
 
-    fail "Not found 'node' value in json" if node.nil?
-    fail "Not found 'name' value in json" if name.nil?
-    fail "Not found 'path' value in json" if path.nil?
+    # If only single node under root, return only the node.
+    return child_nodes.first if node_name.nil? and child_nodes.size == 1
 
-    childnodes = children.map{|c| Yasuri.hash2node(c) }
-    ReservedKeys.each{|key| node_h.delete(key)}
-    opt = node_h
+    node = if node_type_class.nil?
+      Yasuri::MapNode.new(node_name, child_nodes, **opt)
+    else
+      node_type_class::new(path, node_name, child_nodes, **opt)
+    end
 
-    klass = Text2Node[node.to_sym]
-    fail "Undefined node type #{node}" if klass.nil?
-    klass.new(path, name, childnodes, opt)
+    node
   end
 
   def self.node2hash(node)
-    json = JSON.parse("{}")
-    return json if node.nil?
-
-    klass = node.class
-    klass_str = Node2Text[klass]
+    return node.to_h if node.instance_of?(Yasuri::MapNode)
 
-    json["node"] = klass_str
-    json["name"] = node.name
-    json["path"] = node.xpath
-
-    children = node.children.map{|c| Yasuri.node2hash(c)}
-    json["children"] = children if not children.empty?
-
-    node.opts.each do |key,value|
-      json[key] = value if not value.nil?
-    end
-
-    json
+    {
+      "#{node.node_type_str}_#{node.name}" => node.to_h
+    }
   end
 
-  def self.NodeName(name, symbolize_names:false)
+  def self.node_name(name, opt)
+    symbolize_names = opt[:symbolize_names]
     symbolize_names ? name.to_sym : name
   end
 
-  def self.with_retry(retry_count = 5)
+  def self.with_retry(
+    retry_count = DefaultRetryCount,
+    interval_ms = DefaultInterval_ms)
+
     begin
+      Kernel.sleep(interval_ms * 0.001)
       return yield() if block_given?
     rescue => e
       if retry_count > 0
-        pp "retry #{retry_count}"
         retry_count -= 1
         retry
       end
@@ -95,3 +129,14 @@ module Yasuri
     end
   end
 end
+
+class Hash
+  def deep_symbolize_keys
+    Hash[
+      self.map do |k, v|
+        v = v.deep_symbolize_keys if v.kind_of?(Hash)
+        [k.to_sym, v]
+      end
+    ]
+  end
+end

data/lib/yasuri/yasuri_cli.rb

@@ -0,0 +1,64 @@
+require 'thor'
+require 'json'
+require 'yasuri'
+
+module Yasuri
+  class CLI < Thor
+    package_name "yasuri"
+
+    default_command :scrape
+    desc "scrape <URI> [[--file <TREE_FILE>] or [--json <JSON>]]", "Getting from <URI> and scrape it. with <JSON> or json/yml from <TREE_FILE>. They should be Yasuri's format json or yaml string."
+    option :file,     {aliases: 'f', desc: "path to file that written yasuri tree as json or yaml", type: :string}
+    option :json,     {aliases: 'j', desc: "yasuri tree format json string", type: :string}
+    option :interval, {aliases: 'i', desc: "interval each request [ms]", type: :numeric}
+    def scrape(uri)
+      # argument validations
+      if [options[:file], options[:json]].compact.count != 1
+        $stderr.puts "ERROR: Only one of `--file` or `--json` option should be specified."
+        return -1
+      end
+      if options[:file]&.empty? or options[:file] == "file" or options[:json]&.empty?
+        $stderr.puts "ERROR: --file option require not empty argument."
+        return -1
+      end
+      if options[:json]&.empty? or options[:json] == "json"
+        $stderr.puts "ERROR: --json option require not empty argument."
+        return -1
+      end
+
+      interval_ms = options[:interval] || Yasuri::DefaultInterval_ms
+
+      tree = if options[:file]
+              src = File.read(options[:file])
+
+              begin
+                Yasuri.json2tree(src)
+              rescue
+                begin
+                  Yasuri.yaml2tree(src)
+                rescue => e
+                  $stderr.puts "ERROR: Failed to convert to yasuri tree `#{options[:file]}`. #{e.message}"
+                  return -1
+                end
+              end
+            else
+              begin
+                Yasuri.json2tree(options[:json])
+              rescue => e
+                $stderr.puts "ERROR: Failed to convert json to yasuri tree. #{e.message}"
+                return -1
+              end
+            end
+
+      result = tree.scrape(uri, interval_ms: interval_ms)
+
+      if result.instance_of?(String)
+        puts result
+      else
+        j result
+      end
+
+      return 0
+    end
+  end
+end

data/lib/yasuri/yasuri_links_node.rb

@@ -6,21 +6,27 @@ require_relative 'yasuri_node'
 module Yasuri
   class LinksNode
     include Node
+
     def inject(agent, page, opt = {}, element = page)
-      retry_count = opt[:retry_count] || 5
+      retry_count = opt[:retry_count] || Yasuri::DefaultRetryCount
+      interval_ms = opt[:interval_ms] || Yasuri::DefaultInterval_ms
 
       links = element.search(@xpath) || [] # links expected
       links.map do |link|
         link_button = Mechanize::Page::Link.new(link, agent, page)
-        child_page = Yasuri.with_retry(retry_count) { link_button.click }
+        child_page = Yasuri.with_retry(retry_count, interval_ms) { link_button.click }
 
         child_results_kv = @children.map do |child_node|
-          child_name = Yasuri.NodeName(child_node.name, opt)
+          child_name = Yasuri.node_name(child_node.name, opt)
           [child_name, child_node.inject(agent, child_page, opt)]
         end
 
         Hash[child_results_kv]
       end # each named child node
     end
-  end
-end
+
+    def node_type_str
+      "links".freeze
+    end
+  end # class
+end # module

data/lib/yasuri/yasuri_map_node.rb

@@ -0,0 +1,40 @@
+
+module Yasuri
+  class MapNode
+    include Node
+    attr_reader :name, :children
+
+    def initialize(name, children, **opt)
+      @name = name
+      @children = children
+      @opt = opt
+    end
+
+    def inject(agent, page, opt = {}, element = page)
+      child_results_kv = @children.map do |node|
+        [node.name, node.inject(agent, page, opt)]
+      end
+      Hash[child_results_kv]
+    end
+
+    def to_h
+      node_hash = {}
+      self.opts.each{|k, v| node_hash[k] = v if not v.nil?}
+
+      children.each do |child|
+        child_node_name = "#{child.node_type_str}_#{child.name}"
+        node_hash[child_node_name] = child.to_h
+      end
+
+      node_hash
+    end
+
+    def opts
+      {}
+    end
+
+    def node_type_str
+      "map".freeze
+    end
+  end
+end