yasuri 2.0.11 → 3.2.0

data/exe/yasuri

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require "yasuri"
+
+Yasuri::CLI.start

data/lib/yasuri.rb

@@ -1,5 +1,6 @@
 require "yasuri/version"
 require "yasuri/yasuri"
+require "yasuri/yasuri_cli"
 
 module Yasuri
   # Your code goes here...

data/lib/yasuri/version.rb

@@ -1,3 +1,3 @@
 module Yasuri
-  VERSION = "2.0.11"
+  VERSION = "3.2.0"
 end

data/lib/yasuri/yasuri.rb

@@ -4,28 +4,44 @@
 
 require 'mechanize'
 require 'json'
+require 'yaml'
 
 require_relative 'yasuri_node'
 require_relative 'yasuri_text_node'
 require_relative 'yasuri_struct_node'
 require_relative 'yasuri_paginate_node'
 require_relative 'yasuri_links_node'
+require_relative 'yasuri_map_node'
 require_relative 'yasuri_node_generator'
 
 module Yasuri
 
+  DefaultRetryCount = 5
+
   def self.json2tree(json_string)
-    json = JSON.parse(json_string, {symbolize_names: true})
-    Yasuri.hash2node(json)
+    raise RuntimeError if json_string.nil? or json_string.empty?
+
+    node_hash = JSON.parse(json_string, {symbolize_names: true})
+    Yasuri.hash2node(node_hash)
   end
 
   def self.tree2json(node)
+    raise RuntimeError if node.nil?
+
     Yasuri.node2hash(node).to_json
   end
 
-  def self.method_missing(name, *args, &block)
-    generated = Yasuri::NodeGenerator.gen(name, *args, &block)
-    generated || super(name, args)
+  def self.yaml2tree(yaml_string)
+    raise RuntimeError if yaml_string.nil? or yaml_string.empty?
+
+    node_hash = YAML.load(yaml_string)
+    Yasuri.hash2node(node_hash.deep_symbolize_keys)
+  end
+
+  private
+  def self.method_missing(method_name, pattern=nil, **opt, &block)
+    generated = Yasuri::NodeGenerator.gen(method_name, pattern, **opt, &block)
+    generated || super(method_name, **opt)
   end
 
   private
@@ -33,52 +49,66 @@ module Yasuri
     text:   Yasuri::TextNode,
     struct: Yasuri::StructNode,
     links:  Yasuri::LinksNode,
-    pages:  Yasuri::PaginateNode
+    pages:  Yasuri::PaginateNode,
+    map:    Yasuri::MapNode
   }
-  Node2Text = Text2Node.invert
 
-  ReservedKeys = %i|node name path children|
-  def self.hash2node(node_h)
-    node, name, path, children = ReservedKeys.map do |key|
-      node_h[key]
+  def self.hash2node(node_hash, node_name = nil, node_type_class = nil)
+    raise RuntimeError.new("") if node_name.nil? and node_hash.empty?
+
+    node_prefixes = Text2Node.keys.freeze
+    child_nodes = []
+    opt = {}
+    path = nil
+
+    if node_hash.is_a?(String)
+      path = node_hash
+    else
+      node_hash.each do |key, value|
+        # is node?
+        node_regexps = Text2Node.keys.map do |node_type_sym|
+          /^(#{node_type_sym.to_s})_(.+)$/
+        end
+        node_regexp = node_regexps.find do |node_regexp|
+          key =~ node_regexp
+        end
+
+        case key
+        when node_regexp
+          node_type_sym = $1.to_sym
+          child_node_name = $2
+          child_node_type = Text2Node[node_type_sym]
+          child_nodes << self.hash2node(value, child_node_name, child_node_type)
+        when :path
+          path = value
+        else
+          opt[key] = value
+        end
+      end
     end
-    children ||= []
 
-    fail "Not found 'node' value in json" if node.nil?
-    fail "Not found 'name' value in json" if name.nil?
-    fail "Not found 'path' value in json" if path.nil?
+    # If only single node under root, return only the node.
+    return child_nodes.first if node_name.nil? and child_nodes.size == 1
 
-    childnodes = children.map{|c| Yasuri.hash2node(c) }
-    ReservedKeys.each{|key| node_h.delete(key)}
-    opt = node_h
+    node = if node_type_class.nil?
+      Yasuri::MapNode.new(node_name, child_nodes, **opt)
+    else
+      node_type_class::new(path, node_name, child_nodes, **opt)
+    end
 
-    klass = Text2Node[node.to_sym]
-    fail "Undefined node type #{node}" if klass.nil?
-    klass.new(path, name, childnodes, opt)
+    node
   end
 
   def self.node2hash(node)
-    json = JSON.parse("{}")
-    return json if node.nil?
+    return node.to_h if node.instance_of?(Yasuri::MapNode)
 
-    klass = node.class
-    klass_str = Node2Text[klass]
-
-    json["node"] = klass_str
-    json["name"] = node.name
-    json["path"] = node.xpath
-
-    children = node.children.map{|c| Yasuri.node2hash(c)}
-    json["children"] = children if not children.empty?
-
-    node.opts.each do |key,value|
-      json[key] = value if not value.nil?
-    end
-
-    json
+    {
+      "#{node.node_type_str}_#{node.name}" => node.to_h
+    }
   end
 
-  def self.NodeName(name, symbolize_names:false)
+  def self.node_name(name, opt)
+    symbolize_names = opt[:symbolize_names]
     symbolize_names ? name.to_sym : name
   end
 
@@ -95,3 +125,14 @@ module Yasuri
     end
   end
 end
+
+class Hash
+  def deep_symbolize_keys
+    Hash[
+      self.map do |k, v|
+        v = v.deep_symbolize_keys if v.kind_of?(Hash)
+        [k.to_sym, v]
+      end
+    ]
+  end
+end

data/lib/yasuri/yasuri_cli.rb

@@ -0,0 +1,64 @@
+require 'thor'
+require 'json'
+require 'yasuri'
+require 'mechanize'
+
+module Yasuri
+  class CLI < Thor
+    package_name "yasuri"
+
+    default_command :scrape
+    desc "scrape <URI> [[--file <TREE_FILE>] or [--json <JSON>]]", "Getting from <URI> and scrape it. with <JSON> or json/yml from <TREE_FILE>. They should be Yasuri's format json or yaml string."
+    option :file, {aliases: 'f', desc: "path to file that written yasuri tree as json or yaml", type: :string}
+    option :json, {aliases: 'j', desc: "yasuri tree format json string", type: :string}
+    def scrape(uri)
+      # argument validations
+      if [options[:file], options[:json]].compact.count != 1
+        $stderr.puts "ERROR: Only one of `--file` or `--json` option should be specified."
+        return -1
+      end
+      if options[:file]&.empty? or options[:file] == "file" or options[:json]&.empty?
+        $stderr.puts "ERROR: --file option require not empty argument."
+        return -1
+      end
+      if options[:json]&.empty? or options[:json] == "json"
+        $stderr.puts "ERROR: --json option require not empty argument."
+        return -1
+      end
+
+      tree = if options[:file]
+              src = File.read(options[:file])
+
+              begin
+                Yasuri.json2tree(src)
+              rescue
+                begin
+                  Yasuri.yaml2tree(src)
+                rescue => e
+                  $stderr.puts "ERROR: Failed to convert to yasuri tree `#{options[:file]}`. #{e.message}"
+                  return -1
+                end
+              end
+            else
+              begin
+                Yasuri.json2tree(options[:json])
+              rescue => e
+                $stderr.puts "ERROR: Failed to convert json to yasuri tree. #{e.message}"
+                return -1
+              end
+            end
+
+      agent = Mechanize.new
+      root_page = agent.get(uri)
+      result = tree.inject(agent, root_page)
+
+      if result.instance_of?(String)
+        puts result
+      else
+        j result
+      end
+
+      return 0
+    end
+  end
+end

data/lib/yasuri/yasuri_links_node.rb

@@ -6,21 +6,25 @@ require_relative 'yasuri_node'
 module Yasuri
   class LinksNode
     include Node
-    def inject(agent, page, opt = {})
-      retry_count = opt[:retry_count] || 5
+    def inject(agent, page, opt = {}, element = page)
+      retry_count = opt[:retry_count] || Yasuri::DefaultRetryCount
 
-      links = page.search(@xpath) || [] # links expected
+      links = element.search(@xpath) || [] # links expected
       links.map do |link|
         link_button = Mechanize::Page::Link.new(link, agent, page)
         child_page = Yasuri.with_retry(retry_count) { link_button.click }
 
         child_results_kv = @children.map do |child_node|
-          child_name = Yasuri.NodeName(child_node.name, opt)
+          child_name = Yasuri.node_name(child_node.name, opt)
           [child_name, child_node.inject(agent, child_page, opt)]
         end
 
         Hash[child_results_kv]
       end # each named child node
     end
-  end
-end
+
+    def node_type_str
+      "links".freeze
+    end
+  end # class
+end # module

data/lib/yasuri/yasuri_map_node.rb

@@ -0,0 +1,39 @@
+
+module Yasuri
+  class MapNode
+    attr_reader :name, :children
+
+    def initialize(name, children, **opt)
+      @name = name
+      @children = children
+      @opt = opt
+    end
+
+    def inject(agent, page, opt = {}, element = page)
+      child_results_kv = @children.map do |node|
+        [node.name, node.inject(agent, page, opt)]
+      end
+      Hash[child_results_kv]
+    end
+
+    def to_h
+      node_hash = {}
+      self.opts.each{|k, v| node_hash[k] = v if not v.nil?}
+
+      children.each do |child|
+        child_node_name = "#{child.node_type_str}_#{child.name}"
+        node_hash[child_node_name] = child.to_h
+      end
+
+      node_hash
+    end
+
+    def opts
+      {}
+    end
+
+    def node_type_str
+      "map".freeze
+    end
+  end
+end

data/lib/yasuri/yasuri_node.rb

@@ -7,15 +7,36 @@ module Yasuri
   module Node
     attr_reader :url, :xpath, :name, :children
 
-    def initialize(xpath, name, children = [], opt: {})
+    def initialize(xpath, name, children = [], **opt)
       @xpath, @name, @children = xpath, name, children
     end
 
-    def inject(agent, page, opt = {})
-      fail "#{Kernel.__method__} is not implemented."
+    def inject(agent, page, opt = {}, element = page)
+      fail "#{Kernel.__method__} is not implemented in included class."
     end
+
+    def to_h
+      return @xpath if @xpath and @children.empty? and self.opts.values.compact.empty?
+
+      node_hash = {}
+      self.opts.each{|k, v| node_hash[k] = v if not v.nil?}
+
+      node_hash[:path] = @xpath if @xpath
+
+      children.each do |child|
+        child_node_name = "#{child.node_type_str}_#{child.name}"
+        node_hash[child_node_name] = child.to_h
+      end
+
+      node_hash
+    end
+
     def opts
       {}
     end
+
+    def node_type_str
+      fail "#{Kernel.__method__} is not implemented in included class."
+    end
   end
 end

data/lib/yasuri/yasuri_node_generator.rb

@@ -6,6 +6,7 @@ require_relative 'yasuri_text_node'
 require_relative 'yasuri_struct_node'
 require_relative 'yasuri_links_node'
 require_relative 'yasuri_paginate_node'
+require_relative 'yasuri_map_node'
 
 module Yasuri
   class NodeGenerator
@@ -15,29 +16,33 @@ module Yasuri
       @nodes
     end
 
-    def method_missing(name, *args, &block)
-      node = NodeGenerator.gen(name, *args, &block)
+    def method_missing(name, pattern=nil, **args, &block)
+      node = NodeGenerator.gen(name, pattern, **args, &block)
       raise "Undefined Node Name '#{name}'" if node == nil
       @nodes << node
     end
 
-    def self.gen(name, *args, &block)
-      xpath, opt = *args
-      opt = [opt].flatten.compact
+    def self.gen(method_name, xpath, **opt, &block)
       children = Yasuri::NodeGenerator.new.gen_recursive(&block) if block_given?
 
-      case name
+      case method_name
       when /^text_(.+)$/
-        Yasuri::TextNode.new(xpath,   $1, children || [], *opt)
+        # Todo raise error xpath is not valid
+        Yasuri::TextNode.new(xpath, $1, children || [], **opt)
       when /^struct_(.+)$/
-        Yasuri::StructNode.new(xpath, $1, children || [], *opt)
+        # Todo raise error xpath is not valid
+        Yasuri::StructNode.new(xpath, $1, children || [], **opt)
       when /^links_(.+)$/
-        Yasuri::LinksNode.new(xpath,  $1, children || [], *opt)
+        # Todo raise error xpath is not valid
+        Yasuri::LinksNode.new(xpath, $1, children || [], **opt)
       when /^pages_(.+)$/
-        Yasuri::PaginateNode.new(xpath, $1, children || [], *opt)
+        # Todo raise error xpath is not valid
+        Yasuri::PaginateNode.new(xpath, $1, children || [], **opt)
+      when /^map_(.+)$/
+        Yasuri::MapNode.new($1, children, **opt)
       else
         nil
       end
-    end # of self.gen(name, *args, &block)
+    end # of self.gen(method_name, xpath, **opt, &block)
   end # of class NodeGenerator
 end

data/lib/yasuri/yasuri_paginate_node.rb

@@ -7,24 +7,27 @@ module Yasuri
   class PaginateNode
     include Node
 
-    def initialize(xpath, name, children = [], limit: nil)
+    def initialize(xpath, name, children = [], limit: nil, flatten: false)
       super(xpath, name, children)
+      @flatten = flatten
       @limit = limit
     end
 
-    def inject(agent, page, opt = {})
-      retry_count = opt[:retry_count] || 5
+    def inject(agent, page, opt = {}, element = page)
+      retry_count = opt[:retry_count] || Yasuri::DefaultRetryCount
+
+      raise NotImplementedError.new("PagenateNode inside StructNode, Not Supported") if page != element
 
       child_results = []
       limit = @limit.nil? ? Float::MAX : @limit
       while page
         child_results_kv = @children.map do |child_node|
-          child_name = Yasuri.NodeName(child_node.name, opt)
+          child_name = Yasuri.node_name(child_node.name, opt)
           [child_name, child_node.inject(agent, page, opt)]
         end
         child_results << Hash[child_results_kv]
 
-        link = page.search(@xpath).first
+        link = page.search(@xpath).first # Todo raise:  link is not found
         break if link == nil
 
         link_button = Mechanize::Page::Link.new(link, agent, page)
@@ -32,10 +35,19 @@ module Yasuri
         break if (limit -= 1) <= 0
       end
 
+      if @flatten == true
+        return child_results.map{|h| h.values}.flatten
+      end
+
       child_results
     end
+
     def opts
-      {limit:@limit}
+      {limit:@limit, flatten:@flatten}
+    end
+
+    def node_type_str
+      "pages".freeze
     end
   end
 end