yasuri 3.3.0 → 3.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a7bf438a08fc83fec7e78cb5543577c98f6cc98b4f5fae7b0dd969f2049c0531
-  data.tar.gz: e399c6b57589b7d8ba2e8eff7a1d204fa7f8e676f82f631057e19a9377333060
+  metadata.gz: 0ec1d3a8cc766976c1d1329448a52df1696ccb50c41bf7c714b28cd265470d54
+  data.tar.gz: a87a1cd109c0e3dd8d820d3f35e34ef3096ef493d53d9c6bab0a79f1ccd7df9a
 SHA512:
-  metadata.gz: 56f39994972657712cb7d95e5ceaadefca8de41e06c2cd4759363b496d7c8531fad7517f9df99bf2446c144f01c5cd82cbc94146c432d6b5b552f092b975ecd7
-  data.tar.gz: cf74a25615187ecbe5f8ca5f2072679fa9cc1902dfa3bf2190b87e11104f332688cdaee16f4be6cb00f9ed63fa18f2ec8f27cf32b0b27389f81d98229fa212e6
+  metadata.gz: 5f7787bfdc549e70b9e5a5ae45f84f3a0316142deea3c75f17d9781c8140e597d41ccbfd1679e8186799f76e771a24a1d6129402c818f5d6e3ea0b94128a2185
+  data.tar.gz: 61e430d7dae6fda7c28a9456f1751206cfcd01321ff71feeeba7447606a1eb10d7270dda5c8947a4cc8edb3dae60d6d57fc79f127e61920249fcab0ca47f4e37

data/.github/workflows/ruby.yml

@@ -19,7 +19,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        ruby-version: ['2.6', '2.7', '3.0']
+        ruby-version: ['2.7', '3.0']
 
     steps:
     - uses: actions/checkout@v2

data/.rubocop.yml

@@ -0,0 +1,49 @@
+
+# inherit_from: .rubocop_todo.yml
+
+inherit_mode:
+  merge:
+    - Exclude
+
+require:
+  - rubocop-performance
+  - rubocop-rspec
+  - rubocop-rubycw
+
+AllCops:
+  DisabledByDefault: true
+  DisplayCopNames: true
+  Exclude:
+    - 'gems/**/*'
+    - 'pkg/**/*'
+    - 'coverage/**/*'
+    - 'exe/**/*'
+
+  NewCops: enable
+
+Bundler:
+  Enabled: true
+
+Gemspec:
+  Enabled: true
+
+Lint:
+  Enabled: true
+
+Performance:
+  Enabled: true
+
+Rubycw:
+  Enabled: true
+
+Security:
+  Enabled: true
+
+Style/HashSyntax:
+  EnforcedStyle: ruby19
+Style/HashEachMethods:
+  Enabled: true
+Style/HashTransformKeys:
+  Enabled: true
+Style/HashTransformValues:
+  Enabled: true

data/README.md

@@ -2,9 +2,11 @@
 [![Build Status](https://github.com/tac0x2a/yasuri/actions/workflows/ruby.yml/badge.svg)](https://github.com/tac0x2a/yasuri/actions/workflows/ruby.yml)
 [![Coverage Status](https://coveralls.io/repos/tac0x2a/yasuri/badge.svg?branch=master)](https://coveralls.io/r/tac0x2a/yasuri?branch=master) [![Maintainability](https://api.codeclimate.com/v1/badges/c29480fea1305afe999f/maintainability)](https://codeclimate.com/github/tac0x2a/yasuri/maintainability)
 
-Yasuri (鑢) is an easy web-scraping library for supporting "[Mechanize](https://github.com/sparklemotion/mechanize)", and CLI tool using it.
+Yasuri (鑢) is a library for declarative web scraping and a command line tool for scraping with it.
+It performs scraping by simply describing the expected result in a simple declarative notation.
 
-Yasuri can reduce frequently processes in Scraping.
+Yasuri makes it easy to write common scraping operations.
+For example, the following processes can be easily implemented.
 
 For example,
 

data/Rakefile

@@ -3,5 +3,5 @@ require "rspec/core/rake_task"
 
 RSpec::Core::RakeTask.new(:spec)
 
-task :default => :spec
+task default: :spec
 

data/examples/example.rb

@@ -1,5 +1,4 @@
 #!/usr/bin/env ruby
-# Author::    TAC (tac@tac42.net)
 
 require 'yasuri'
 uri = "https://github.com/tac0x2a?tab=repositories"

data/lib/yasuri/version.rb

@@ -1,3 +1,3 @@
 module Yasuri
-  VERSION = "3.3.0"
+  VERSION = "3.3.1"
 end

data/lib/yasuri/yasuri.rb

@@ -1,6 +1,3 @@
-# -*- coding: utf-8 -*-
-
-# Author::    TAC (tac@tac42.net)
 
 require 'mechanize'
 require 'json'
@@ -23,41 +20,48 @@ module Yasuri
     raise RuntimeError if json_string.nil? or json_string.empty?
 
     node_hash = JSON.parse(json_string, {symbolize_names: true})
-    Yasuri.hash2node(node_hash)
+    self.hash2node(node_hash)
+  end
+
+  def self.yaml2tree(yaml_string)
+    raise RuntimeError if yaml_string.nil? or yaml_string.empty?
+
+    node_hash = YAML.safe_load(yaml_string, [Symbol], symbolize_names: true)
+    self.hash2node(node_hash.deep_symbolize_keys)
   end
 
   def self.tree2json(node)
     raise RuntimeError if node.nil?
 
-    Yasuri.node2hash(node).to_json
+    self.node2hash(node).to_json
   end
 
-  def self.yaml2tree(yaml_string)
-    raise RuntimeError if yaml_string.nil? or yaml_string.empty?
+  def self.with_retry(
+    retry_count = DefaultRetryCount,
+    interval_ms = DefaultInterval_ms)
 
-    node_hash = YAML.load(yaml_string)
-    Yasuri.hash2node(node_hash.deep_symbolize_keys)
+    begin
+      Kernel.sleep(interval_ms * 0.001)
+      return yield() if block_given?
+    rescue => e
+      if retry_count > 0
+        retry_count -= 1
+        retry
+      end
+      fail e
+    end
   end
 
-  private
-  def self.method_missing(method_name, pattern=nil, **opt, &block)
-    generated = Yasuri::NodeGenerator.gen(method_name, pattern, **opt, &block)
-    generated || super(method_name, **opt)
+  def self.node_name(name, opt)
+    symbolize_names = opt[:symbolize_names]
+    symbolize_names ? name.to_sym : name
   end
 
-  private
-  Text2Node = {
-    text:   Yasuri::TextNode,
-    struct: Yasuri::StructNode,
-    links:  Yasuri::LinksNode,
-    pages:  Yasuri::PaginateNode,
-    map:    Yasuri::MapNode
-  }
+  # private
 
   def self.hash2node(node_hash, node_name = nil, node_type_class = nil)
     raise RuntimeError.new("") if node_name.nil? and node_hash.empty?
 
-    node_prefixes = Text2Node.keys.freeze
     child_nodes = []
     opt = {}
     path = nil
@@ -65,27 +69,7 @@ module Yasuri
     if node_hash.is_a?(String)
       path = node_hash
     else
-      node_hash.each do |key, value|
-        # is node?
-        node_regexps = Text2Node.keys.map do |node_type_sym|
-          /^(#{node_type_sym.to_s})_(.+)$/
-        end
-        node_regexp = node_regexps.find do |node_regexp|
-          key =~ node_regexp
-        end
-
-        case key
-        when node_regexp
-          node_type_sym = $1.to_sym
-          child_node_name = $2
-          child_node_type = Text2Node[node_type_sym]
-          child_nodes << self.hash2node(value, child_node_name, child_node_type)
-        when :path
-          path = value
-        else
-          opt[key] = value
-        end
-      end
+      child_nodes, opt, path = self.hash2child_node(node_hash)
     end
 
     # If only single node under root, return only the node.
@@ -100,6 +84,42 @@ module Yasuri
     node
   end
 
+  Text2Node = {
+    text:   Yasuri::TextNode,
+    struct: Yasuri::StructNode,
+    links:  Yasuri::LinksNode,
+    pages:  Yasuri::PaginateNode,
+    map:    Yasuri::MapNode
+  }
+
+  NodeRegexps = Text2Node.keys.map { |node_type_sym| /^(#{node_type_sym})_(.+)$/ }
+
+  def self.hash2child_node(node_hash)
+    child_nodes = []
+    opt = {}
+    path = nil
+
+    node_hash.each do |key, value|
+      # is node?
+
+      node_regexp = NodeRegexps.find { |r| key =~ r }
+
+      case key
+      when node_regexp
+        node_type_sym = $1.to_sym
+        child_node_name = $2
+        child_node_type = Text2Node[node_type_sym]
+        child_nodes << self.hash2node(value, child_node_name, child_node_type)
+      when :path
+        path = value
+      else
+        opt[key] = value
+      end
+    end
+
+    [child_nodes, opt, path]
+  end
+
   def self.node2hash(node)
     return node.to_h if node.instance_of?(Yasuri::MapNode)
 
@@ -108,26 +128,13 @@ module Yasuri
     }
   end
 
-  def self.node_name(name, opt)
-    symbolize_names = opt[:symbolize_names]
-    symbolize_names ? name.to_sym : name
+  def self.method_missing(method_name, pattern=nil, **opt, &block)
+    generated = Yasuri::NodeGenerator.gen(method_name, pattern, **opt, &block)
+    generated || super(method_name, **opt)
   end
 
-  def self.with_retry(
-    retry_count = DefaultRetryCount,
-    interval_ms = DefaultInterval_ms)
-
-    begin
-      Kernel.sleep(interval_ms * 0.001)
-      return yield() if block_given?
-    rescue => e
-      if retry_count > 0
-        retry_count -= 1
-        retry
-      end
-      fail e
-    end
-  end
+  private_constant :Text2Node, :NodeRegexps
+  private_class_method :method_missing, :hash2child_node, :hash2node, :node2hash
 end
 
 class Hash

data/lib/yasuri/yasuri_cli.rb

@@ -7,50 +7,30 @@ module Yasuri
     package_name "yasuri"
 
     default_command :scrape
-    desc "scrape <URI> [[--file <TREE_FILE>] or [--json <JSON>]]", "Getting from <URI> and scrape it. with <JSON> or json/yml from <TREE_FILE>. They should be Yasuri's format json or yaml string."
+    desc "scrape <URI> [[--file <TREE_FILE>] or [--json <JSON>]]",
+         "Getting from <URI> and scrape it. with <JSON> or json/yml from <TREE_FILE>. They should be Yasuri's format json or yaml string."
     option :file,     {aliases: 'f', desc: "path to file that written yasuri tree as json or yaml", type: :string}
     option :json,     {aliases: 'j', desc: "yasuri tree format json string", type: :string}
     option :interval, {aliases: 'i', desc: "interval each request [ms]", type: :numeric}
     def scrape(uri)
-      # argument validations
-      if [options[:file], options[:json]].compact.count != 1
-        $stderr.puts "ERROR: Only one of `--file` or `--json` option should be specified."
-        return -1
-      end
-      if options[:file]&.empty? or options[:file] == "file" or options[:json]&.empty?
-        $stderr.puts "ERROR: --file option require not empty argument."
-        return -1
-      end
-      if options[:json]&.empty? or options[:json] == "json"
-        $stderr.puts "ERROR: --json option require not empty argument."
+      begin
+        test_arguments(options)
+      rescue => e
+        $stderr.puts e.message
         return -1
       end
 
       interval_ms = options[:interval] || Yasuri::DefaultInterval_ms
+      file_path = options[:file]
+      json_string = options[:json]
 
-      tree = if options[:file]
-              src = File.read(options[:file])
-
-              begin
-                Yasuri.json2tree(src)
-              rescue
-                begin
-                  Yasuri.yaml2tree(src)
-                rescue => e
-                  $stderr.puts "ERROR: Failed to convert to yasuri tree `#{options[:file]}`. #{e.message}"
-                  return -1
-                end
-              end
-            else
-              begin
-                Yasuri.json2tree(options[:json])
-              rescue => e
-                $stderr.puts "ERROR: Failed to convert json to yasuri tree. #{e.message}"
-                return -1
-              end
-            end
-
-      result = tree.scrape(uri, interval_ms: interval_ms)
+      begin
+        tree = make_tree(file_path, json_string)
+        result = tree.scrape(uri, interval_ms: interval_ms)
+      rescue => e
+        $stderr.puts e.message
+        return -1
+      end
 
       if result.instance_of?(String)
         puts result
@@ -60,5 +40,39 @@ module Yasuri
 
       return 0
     end
+
+    private
+
+    def test_arguments(options)
+      too_many_options = [options[:file], options[:json]].compact.count != 1
+      raise "ERROR: Only one of `--file` or `--json` option should be specified." if too_many_options
+
+      empty_file_argument = options[:file]&.empty? || options[:file] == "file" || options[:json]&.empty?
+      raise "ERROR: --file option require not empty argument." if empty_file_argument
+
+      empty_json_string_argument = options[:json]&.empty? || options[:json] == "json"
+      raise "ERROR: --json option require not empty argument." if empty_json_string_argument
+    end
+
+    def make_tree(file_path, json_string)
+      if file_path
+        begin
+          src = File.read(file_path)
+          make_tree_from_file(src)
+        rescue => e
+          raise "ERROR: Failed to convert to yasuri tree `#{file_path}`. #{e.message}"
+        end
+      else
+        begin
+          Yasuri.json2tree(json_string)
+        rescue => e
+          raise "ERROR: Failed to convert json to yasuri tree. #{e.message}"
+        end
+      end
+    end
+
+    def make_tree_from_file(src)
+        Yasuri.json2tree(src) rescue Yasuri.yaml2tree(src)
+    end
   end
 end

data/lib/yasuri/yasuri_links_node.rb

@@ -1,6 +1,4 @@
 
-# Author::    TAC (tac@tac42.net)
-
 require_relative 'yasuri_node'
 
 module Yasuri
@@ -22,11 +20,11 @@ module Yasuri
         end
 
         Hash[child_results_kv]
-      end # each named child node
+      end
     end
 
     def node_type_str
       "links".freeze
     end
-  end # class
-end # module
+  end
+end

data/lib/yasuri/yasuri_map_node.rb

@@ -10,7 +10,7 @@ module Yasuri
       @opt = opt
     end
 
-    def inject(agent, page, opt = {}, element = page)
+    def inject(agent, page, opt = {}, _element = page)
       child_results_kv = @children.map do |node|
         [node.name, node.inject(agent, page, opt)]
       end
@@ -19,7 +19,7 @@ module Yasuri
 
     def to_h
       node_hash = {}
-      self.opts.each{|k, v| node_hash[k] = v if not v.nil?}
+      self.opts.each { |k, v| node_hash[k] = v unless v.nil? }
 
       children.each do |child|
         child_node_name = "#{child.node_type_str}_#{child.name}"

data/lib/yasuri/yasuri_node.rb

@@ -1,27 +1,24 @@
 
-# Author::    TAC (tac@tac42.net)
-
 require_relative 'yasuri_node'
 
 module Yasuri
   module Node
     attr_reader :url, :xpath, :name, :children
 
-    def initialize(xpath, name, children = [], **opt)
+    def initialize(xpath, name, children = [], **_opt)
       @xpath, @name, @children = xpath, name, children
     end
 
     def scrape(uri, opt = {})
-      retry_count = opt[:retry_count] || Yasuri::DefaultRetryCount
-      interval_ms = opt[:interval_ms] || Yasuri::DefaultInterval_ms
-
       agent = Mechanize.new
-      page = Yasuri.with_retry(retry_count, interval_ms) { agent.get(uri) }
       scrape_with_agent(uri, agent, opt)
     end
 
     def scrape_with_agent(uri, agent, opt = {})
-      page = agent.get(uri)
+      retry_count = opt[:retry_count] || Yasuri::DefaultRetryCount
+      interval_ms = opt[:interval_ms] || Yasuri::DefaultInterval_ms
+
+      page = Yasuri.with_retry(retry_count, interval_ms) { agent.get(uri) }
       inject(agent, page, opt)
     end