yasuri 1.9.12 → 3.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 87406480e622911dca3649ba2a8e5b134ccffb36
-  data.tar.gz: 2fde6a481bed02e569a5c08f6df96c09925abee0
+SHA256:
+  metadata.gz: f3542a2cc0959a4534520f6104fc2922bdf0dbd368fcd4c149c3d251c2fc2198
+  data.tar.gz: 6fdb960db697e9a4ec1d87f2b83bf0e9914e3c9efe90764536bbee6d68774353
 SHA512:
-  metadata.gz: 7c274f2316495aea66d737f053119dd71c154f7411b9cd54b102c71ee2e7ac36602dd0e44d78d0d06895ee0e27f934a9ac0e2f45d7e52ce4629f60f9fd905cf3
-  data.tar.gz: a5696ca1fac061c542c7f0586bfecdf623962443e98207f897ff76ff0204cc78e53505b070f18467d2e72c0ddb527383224fb75ff2d05b5ff6e5a5149caaa20a
+  metadata.gz: 9df576243bea289f4c285c46f1bd2137b7b69b79b24e0c657e4ac952114dd7bcf82a5f95cd2dae88c6eac4e3e468273b7dbd6ead9d05ffdc8d25861921702333
+  data.tar.gz: 13f2ae72b3e8fa6d3ef58932daa2acad49f5d4f57c80f34e5215394940fc2305bc016d949760efe9f43ae2b8c3796064a1b0bd9bccf236cfe3789c2c291dfd8b

data/.github/workflows/ruby.yml

@@ -0,0 +1,35 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+# This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake
+# For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby
+
+name: Ruby
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  test:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby-version: ['2.6', '2.7', '3.0']
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+    # To automatically get bug fixes and new Ruby versions for ruby/setup-ruby,
+    # change this to (see https://github.com/ruby/setup-ruby#versioning):
+    # uses: ruby/setup-ruby@v1
+      uses: ruby/setup-ruby@473e4d8fe5dd94ee328fdfca9f8c9c7afc9dae5e
+      with:
+        ruby-version: ${{ matrix.ruby-version }}
+        bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+    - name: Run tests
+      run: bundle exec rake

data/.gitignore

@@ -66,5 +66,4 @@ tramp
 # cask packages
 .cask/
 
-.ruby-version
-Gemfile.lock
+Gemfile.lock

data/.ruby-version

@@ -0,0 +1 @@
+3.0.0

data/.travis.yml

@@ -1,9 +1,7 @@
 language: ruby
-rvm:
-  - 2.2.0
 script:
   - ruby --version
   - rspec spec
 addons:
   code_climate:
-    repo_token: 0dc78d33107a7f11f257c0218ac1a37e0073005bb9734f2fd61d0f7e803fc151
+    repo_token: 0dc78d33107a7f11f257c0218ac1a37e0073005bb9734f2fd61d0f7e803fc151

data/README.md

@@ -1,4 +1,6 @@
-# Yasuri [![Build Status](https://travis-ci.org/tac0x2a/yasuri.svg?branch=master)](https://travis-ci.org/tac0x2a/yasuri) [![Coverage Status](https://coveralls.io/repos/tac0x2a/yasuri/badge.svg?branch=master)](https://coveralls.io/r/tac0x2a/yasuri?branch=master) [![Code Climate](https://codeclimate.com/github/tac0x2a/yasuri/badges/gpa.svg)](https://codeclimate.com/github/tac0x2a/yasuri)
+# Yasuri
+[![Build Status](https://github.com/tac0x2a/yasuri/actions/workflows/ruby.yml/badge.svg)](https://github.com/tac0x2a/yasuri/actions/workflows/ruby.yml)
+[![Coverage Status](https://coveralls.io/repos/tac0x2a/yasuri/badge.svg?branch=master)](https://coveralls.io/r/tac0x2a/yasuri?branch=master) [![Maintainability](https://api.codeclimate.com/v1/badges/c29480fea1305afe999f/maintainability)](https://codeclimate.com/github/tac0x2a/yasuri/maintainability)
 
 Yasuri (鑢) is an easy web-scraping library for supporting "[Mechanize](https://github.com/sparklemotion/mechanize)".
 
@@ -32,6 +34,9 @@ or
 ```ruby
 # for Ruby 1.9.3 or lower
 gem 'yasuri', '~> 1.9'
+
+# for Ruby 3.0.0 or lower
+gem 'yasuri', '~> 3.0.1'
 ```
 
 
@@ -52,6 +57,19 @@ root = Yasuri.links_root '//*[@id="menu"]/ul/li/a' do
          text_content '//*[@id="contents"]/p[1]'
        end
 
+
+# Node tree constructing by YAML
+src = <<-EOYAML
+root:
+  node: links
+  path: "//*[@id='menu']/ul/li/a"
+  children:
+    - title:   { node: text, path: "//*[@id='contents']/h2" }
+    - content: { node: text, path: "//*[@id='contents']/p[1]" }
+EOYAML
+root = Yasuri.yaml2tree(src)
+
+
 # Node tree constructing by JSON
 src = <<-EOJSON
    { "node"     : "links",
@@ -78,6 +96,27 @@ result = root.inject(agent, root_page)
 # => [ {"title" => "PageTitle", "content" => "Page Contents" }, ...  ]
 ```
 
+## Dev
+```sh
+$ gem install bundler
+$ bundle install
+```
+### Test
+```sh
+$ rake
+# or
+$ rspec spec/*spec.rb
+```
+
+### Release RubyGems
+```sh
+# Only first time
+$ curl -u <user_name> https://rubygems.org/api/v1/api_key.yaml > ~/.gem/credentials
+$ chmod 0600 ~/.gem/credentials
+
+$ nano lib/yasuri/version.rb # edit gem version
+$ rake release
+```
 
 ## Contributing
 

data/USAGE.ja.md

@@ -67,7 +67,7 @@ page = agent.get(uri)
 tree.inject(agent, page)
 ```
 
-ツリーは、DSLまたはjsonで定義することができます．上の例ではDSLで定義しています．
+ツリーは、json，yaml，またはDSLで定義することができます．上の例ではDSLで定義しています．
 以下は、jsonで上記と等価な解析ツリーを定義した例です．
 
 ```ruby
@@ -87,25 +87,54 @@ EOJSON
 tree = Yasuri.json2tree(src)
 ```
 
+```ruby
+# yaml で構成する場合
+src = <<-EOYAML
+title:
+  node: links
+  path: "/html/body/a"
+  children:
+    - name:
+        node: text
+        path: "/html/body/p"
+EOYAML
+tree = Yasuri.yaml2tree(src)
+```
 
 ### Node
 ツリーは入れ子になった *Node* で構成されます．
 Node は `Type`, `Name`, `Path`, `Childlen`, `Options` を持っています．
+(ただし、`MapNode` のみ `Path` を持ちません)
 
 Nodeは以下のフォーマットで定義されます．
 
 ```ruby
-# トップレベル
 Yasuri.<Type>_<Name> <Path> [,<Options>]
 
 # 入れ子になっている場合
 Yasuri.<Type>_<Name> <Path> [,<Options>] do
   <Type>_<Name> <Path> [,<Options>] do
-    <Children>
+    <Type>_<Name> <Path> [,<Options>]
+    ...
   end
 end
 ```
 
+例
+
+```ruby
+Yasuri.text_title '/html/head/title', truncate:/^[^,]+/
+
+# 入れ子になっている場合
+Yasuri.links_root '//*[@id="menu"]/ul/li/a' do
+  struct_table './tr' do
+    text_title    './td[1]'
+    text_pub_date './td[2]'
+  end
+end
+```
+
+
 #### Type
 *Type* は Nodeの振る舞いを示します．Typeには以下のものがあります．
 
@@ -113,18 +142,19 @@ end
 - *Struct*
 - *Links*
 - *Paginate*
+- *Map*
 
-### Name
+#### Name
 *Name* は 解析結果のHashにおけるキーになります．
 
-### Path
+#### Path
 *Path* は xpath あるいは css セレクタによって、HTML上の特定のノードを指定します．
 これは Machinize の `search` で使用されます．
 
-### Childlen
+#### Childlen
 入れ子になっているノードの子ノードです．TextNodeはツリーの葉に当たるため、子ノードを持ちません．
 
-### Options
+#### Options
 パースのオプションです．オプションはTypeごとに異なります．
 各ノードに対して、`opt`メソッドをコールすることで、利用可能なオプションを取得できます．
 
@@ -156,13 +186,15 @@ page = agent.get("http://yasuri.example.net")
 
 p1  = Yasuri.text_title '/html/body/p[1]'
 p1t = Yasuri.text_title '/html/body/p[1]', truncate:/^[^,]+/
-p2u = Yasuri.text_title '/html/body/p[2]', proc: :upcase
+p2u = Yasuri.text_title '/html/body/p[1]', proc: :upcase
 
-p1.inject(agent, page)   #=> { "title" => "Hello,World" }
-p1t.inject(agent, page)  #=> { "title" => "Hello" }
-node.inject(agent, page) #=> { "title" => "HELLO,YASURI" }
+p1.inject(agent, page)   #=> "Hello,World"
+p1t.inject(agent, page)  #=> "Hello"
+p2u.inject(agent, page)  #=> "HELLO,WORLD"
 ```
 
+なお、同じページ内の複数の要素を一度にスクレイピングする場合は、`MapNode`を使用します。
+
 ### オプション
 ##### `truncate`
 正規表現にマッチした文字列を取り出します．グループを指定した場合、最初にマッチしたグループだけを返します．
@@ -466,3 +498,54 @@ node.inject(agent, page)
       "Page03",
       "Patination03"]
 ```
+
+## Map Node
+*MapNode* はスクレイピングした結果をまとめるノードです．このノードはパースツリーにおいて常に節です．
+
+### 例
+
+```html
+<!-- http://yasuri.example.net -->
+<html>
+  <head><title>Yasuri Example</title></head>
+  <body>
+    <p>Hello,World</p>
+    <p>Hello,Yasuri</p>
+  </body>
+</html>
+```
+
+```ruby
+agent = Mechanize.new
+page = agent.get("http://yasuri.example.net")
+
+
+tree = Yasuri.map_root do
+  text_title  '/html/head/title'
+  text_body_p '/html/body/p[1]'
+end
+
+tree.inject(agent, page) #=> { "title" => "Yasuri Example", "body_p" => "Hello,World" }
+
+
+tree = Yasuri.map_root do
+  map_group1 { text_child01  '/html/body/a[1]' }
+  map_group2 do
+    text_child01 '/html/body/a[1]'
+    text_child03 '/html/body/a[3]'
+  end
+end
+
+tree.inject(agent, page) #=> {
+#   "group1" => {
+#           "child01" => "child01"
+#         },
+#         "group2" => {
+#           "child01" => "child01",
+#           "child03" => "child03"
+#         }
+# }
+```
+
+### オプション
+なし

data/USAGE.md

@@ -69,7 +69,7 @@ page = agent.get(uri)
 tree.inject(agent, page)
 ```
 
-Tree is definable by 2(+1) ways, DSL and json (and basic ruby code). In above example, DSL.
+Tree is definable by 3(+1) ways, json, yaml, and DSL (or basic ruby code). In above example, DSL.
 
 ```ruby
 # Construct by json.
@@ -88,21 +88,51 @@ EOJSON
 tree = Yasuri.json2tree(src)
 ```
 
+```ruby
+# Construct by yaml.
+src = <<-EOYAML
+title:
+  node: links
+  path: "/html/body/a"
+  children:
+    - name:
+        node: text
+        path: "/html/body/p"
+EOYAML
+tree = Yasuri.yaml2tree(src)
+```
+
+
 ### Node
 Tree is constructed by nested Nodes.
 Node has `Type`, `Name`, `Path`, `Childlen`, and `Options`.
+(But only `MapNode` does not have `Path`.)
 
 Node is defined by this format.
 
 
 ```ruby
-# Top Level
 Yasuri.<Type>_<Name> <Path> [,<Options>]
 
-# Nested
+# Nested case
 Yasuri.<Type>_<Name> <Path> [,<Options>] do
   <Type>_<Name> <Path> [,<Options>] do
-    <Children>
+    <Type>_<Name> <Path> [,<Options>]
+    ...
+  end
+end
+```
+
+Example
+
+```ruby
+Yasuri.text_title '/html/head/title', truncate:/^[^,]+/
+
+# Nested case
+Yasuri.links_root '//*[@id="menu"]/ul/li/a' do
+  struct_table './tr' do
+    text_title    './td[1]'
+    text_pub_date './td[2]'
   end
 end
 ```
@@ -114,17 +144,18 @@ Type meen behavior of Node.
 - *Struct*
 - *Links*
 - *Paginate*
+- *Map*
 
-### Name
+#### Name
 Name is used keys in returned hash.
 
-### Path
+#### Path
 Path determine target node by xpath or css selector. It given by Machinize `search`.
 
-### Childlen
+#### Childlen
 Child nodes. TextNode has always empty set, because TextNode is leaf.
 
-### Options
+#### Options
 Parse options. It different in each types. You can get options and values by `opt` method.
 
 ```ruby
@@ -155,13 +186,15 @@ page = agent.get("http://yasuri.example.net")
 
 p1  = Yasuri.text_title '/html/body/p[1]'
 p1t = Yasuri.text_title '/html/body/p[1]', truncate:/^[^,]+/
-p2u = Yasuri.text_title '/html/body/p[2]', proc: :upcase
+p2u = Yasuri.text_title '/html/body/p[1]', proc: :upcase
 
-p1.inject(agent, page)   #=> { "title" => "Hello,World" }
-p1t.inject(agent, page)  #=> { "title" => "Hello" }
-node.inject(agent, page) #=> { "title" => "HELLO,YASURI" }
+p1.inject(agent, page)   #=> "Hello,World"
+p1t.inject(agent, page)  #=> "Hello"
+p2u.inject(agent, page)  #=> "HELLO,WORLD"
 ```
 
+Note that if you want to scrape multiple elements in the same page at once, use `MapNode`. See the `MapNode` example for details.
+
 ### Options
 ##### `truncate`
 Match to regexp, and truncate text. When you use group, it will return first matched group only.
@@ -464,3 +497,54 @@ node.inject(agent, page)
       "Page03",
       "Patination03"]
 ```
+
+## Map Node
+*MapNode* is a node that summarizes the results of scraping. This node is always a branch node in the parse tree.
+
+### Example
+
+```html
+<!-- http://yasuri.example.net -->
+<html>
+  <head><title>Yasuri Example</title></head>
+  <body>
+    <p>Hello,World</p>
+    <p>Hello,Yasuri</p>
+  </body>
+</html>
+```
+
+```ruby
+agent = Mechanize.new
+page = agent.get("http://yasuri.example.net")
+
+
+tree = Yasuri.map_root do
+  text_title  '/html/head/title'
+  text_body_p '/html/body/p[1]'
+end
+
+tree.inject(agent, page) #=> { "title" => "Yasuri Example", "body_p" => "Hello,World" }
+
+
+tree = Yasuri.map_root do
+  map_group1 { text_child01  '/html/body/a[1]' }
+  map_group2 do
+    text_child01 '/html/body/a[1]'
+    text_child03 '/html/body/a[3]'
+  end
+end
+
+tree.inject(agent, page) #=> {
+#   "group1" => {
+#           "child01" => "child01"
+#         },
+#         "group2" => {
+#           "child01" => "child01",
+#           "child03" => "child03"
+#         }
+# }
+```
+
+### Options
+None.

data/lib/yasuri/version.rb

@@ -1,3 +1,3 @@
 module Yasuri
-  VERSION = "1.9.12"
+  VERSION = "3.1.0"
 end

data/lib/yasuri/yasuri.rb

@@ -4,12 +4,14 @@
 
 require 'mechanize'
 require 'json'
+require 'yaml'
 
 require_relative 'yasuri_node'
 require_relative 'yasuri_text_node'
 require_relative 'yasuri_struct_node'
 require_relative 'yasuri_paginate_node'
 require_relative 'yasuri_links_node'
+require_relative 'yasuri_map_node'
 require_relative 'yasuri_node_generator'
 
 module Yasuri
@@ -23,9 +25,39 @@ module Yasuri
     Yasuri.node2hash(node).to_json
   end
 
-  def self.method_missing(name, *args, &block)
-    generated = Yasuri::NodeGenerator.gen(name, *args, &block)
-    generated || super(name, args)
+  def self.yaml2tree(yaml_string)
+    raise RuntimeError if yaml_string.nil? or yaml_string.empty?
+
+    yaml = YAML.load(yaml_string)
+    raise RuntimeError if yaml.keys.size < 1
+
+    root_key, root = yaml.keys.first, yaml.values.first
+    hash = Yasuri.yaml2tree_sub(root_key, root)
+
+    Yasuri.hash2node(hash)
+  end
+
+  private
+  def self.yaml2tree_sub(name, body)
+    return nil if name.nil? or body.nil?
+
+    new_body = Hash[:name, name]
+    body.each{|k,v| new_body[k.to_sym] = v}
+    body = new_body
+
+    return body if body[:children].nil?
+
+    body[:children] = body[:children].map do |c|
+      k, b = c.keys.first, c.values.first
+      Yasuri.yaml2tree_sub(k, b)
+    end
+
+    body
+  end
+
+  def self.method_missing(method_name, pattern=nil, **opt, &block)
+    generated = Yasuri::NodeGenerator.gen(method_name, pattern, **opt, &block)
+    generated || super(method_name, **opt)
   end
 
   private
@@ -33,53 +65,26 @@ module Yasuri
     text:   Yasuri::TextNode,
     struct: Yasuri::StructNode,
     links:  Yasuri::LinksNode,
-    pages:  Yasuri::PaginateNode
+    pages:  Yasuri::PaginateNode,
+    map:   Yasuri::MapNode
   }
   Node2Text = Text2Node.invert
 
-  ReservedKeys = [:node, :name, :path, :children]
+  ReservedKeys = %i|node name path children|
   def self.hash2node(node_h)
-    node, name, path, children = ReservedKeys.map do |key|
-      node_h[key]
-    end
-    children ||= []
-
-    fail "Not found 'node' value in json" if node.nil?
-    fail "Not found 'name' value in json" if name.nil?
-    fail "Not found 'path' value in json" if path.nil?
-
-    childnodes = children.map{|c| Yasuri.hash2node(c) }
-    ReservedKeys.each{|key| node_h.delete(key)}
-    opt = node_h
+    node = node_h[:node]
 
+    fail "Not found 'node' value in map" if node.nil?
     klass = Text2Node[node.to_sym]
-    fail "Undefined node type #{node}" if klass.nil?
-    klass.new(path, name, childnodes, opt)
+    klass::hash2node(node_h)
   end
 
   def self.node2hash(node)
-    json = JSON.parse("{}")
-    return json if node.nil?
-
-    klass = node.class
-    klass_str = Node2Text[klass]
-
-    json["node"] = klass_str
-    json["name"] = node.name
-    json["path"] = node.xpath
-
-    children = node.children.map{|c| Yasuri.node2hash(c)}
-    json["children"] = children if not children.empty?
-
-    node.opts.each do |key,value|
-      json[key] = value if not value.nil?
-    end
-
-    json
+    node.to_h
   end
 
-  def self.NodeName(name, hash = {})
-    symbolize_names = hash[:symbolize_names] || false
+  def self.NodeName(name, opt)
+    symbolize_names = opt[:symbolize_names]
     symbolize_names ? name.to_sym : name
   end
 

data/lib/yasuri/yasuri_links_node.rb

@@ -22,5 +22,9 @@ module Yasuri
         Hash[child_results_kv]
       end # each named child node
     end
-  end
-end
+
+    def node_type_str
+      "links"
+    end
+  end # class
+end # module

data/lib/yasuri/yasuri_map_node.rb

@@ -0,0 +1,54 @@
+
+module Yasuri
+  class MapNode
+    attr_reader :name, :children
+
+    def initialize(name, children, opt: {})
+      @name = name
+      @children = children
+      @opt = opt
+    end
+
+    def inject(agent, page, opt = {}, element = page)
+      child_results_kv = @children.map do |node|
+        [node.name, node.inject(agent, page, opt)]
+      end
+      Hash[child_results_kv]
+    end
+
+    def opts
+      {}
+    end
+
+    def to_h
+      h = {}
+      h["node"] = "map"
+      h["name"] = self.name
+      h["children"] = self.children.map{|c| c.to_h} if not children.empty?
+
+      self.opts.each do |key,value|
+        h[key] = value if not value.nil?
+      end
+
+      h
+    end
+
+    def self.hash2node(node_h)
+      reservedKeys = %i|node name children|
+
+      node, name, children = reservedKeys.map do |key|
+        node_h[key]
+      end
+
+      fail "Not found 'name' value in map" if name.nil?
+      fail "Not found 'children' value in map" if children.nil?
+      children ||= []
+
+      childnodes = children.map{|c| Yasuri.hash2node(c) }
+      reservedKeys.each{|key| node_h.delete(key)}
+      opt = node_h
+
+      self.new(name, childnodes, **opt)
+    end
+  end
+end

data/lib/yasuri/yasuri_node.rb

@@ -7,15 +7,58 @@ module Yasuri
   module Node
     attr_reader :url, :xpath, :name, :children
 
-    def initialize(xpath, name, children = [], opt = {})
+    def initialize(xpath, name, children = [], opt: {})
       @xpath, @name, @children = xpath, name, children
     end
 
     def inject(agent, page, opt = {}, element = page)
-      fail "#{Kernel.__method__} is not implemented."
+      fail "#{Kernel.__method__} is not implemented in included class."
     end
+
     def opts
       {}
     end
+
+    def to_h
+      h = {}
+      h["node"] = self.node_type_str
+      h["name"] = self.name
+      h["path"] = self.xpath
+      h["children"] = self.children.map{|c| c.to_h} if not children.empty?
+
+      self.opts.each do |key,value|
+        h[key] = value if not value.nil?
+      end
+
+      h
+    end
+
+    module ClassMethods
+      def hash2node(node_h)
+        reservedKeys = %i|node name path children|
+
+        node, name, path, children = ReservedKeys.map do |key|
+          node_h[key]
+        end
+
+        fail "Not found 'name' value in map" if name.nil?
+        fail "Not found 'path' value in map" if path.nil?
+        children ||= []
+
+        childnodes = children.map{|c| Yasuri.hash2node(c) }
+        reservedKeys.each{|key| node_h.delete(key)}
+        opt = node_h
+
+        self.new(path, name, childnodes, **opt)
+      end
+
+      def node_type_str
+        fail "#{Kernel.__method__} is not implemented in included class."
+      end
+    end
+
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
   end
 end

data/lib/yasuri/yasuri_node_generator.rb

@@ -6,6 +6,7 @@ require_relative 'yasuri_text_node'
 require_relative 'yasuri_struct_node'
 require_relative 'yasuri_links_node'
 require_relative 'yasuri_paginate_node'
+require_relative 'yasuri_map_node'
 
 module Yasuri
   class NodeGenerator
@@ -15,29 +16,33 @@ module Yasuri
       @nodes
     end
 
-    def method_missing(name, *args, &block)
-      node = NodeGenerator.gen(name, *args, &block)
+    def method_missing(name, pattern=nil, **args, &block)
+      node = NodeGenerator.gen(name, pattern, **args, &block)
       raise "Undefined Node Name '#{name}'" if node == nil
       @nodes << node
     end
 
-    def self.gen(name, *args, &block)
-      xpath, opt = *args
-      opt = [opt].flatten.compact
+    def self.gen(method_name, xpath, **opt, &block)
       children = Yasuri::NodeGenerator.new.gen_recursive(&block) if block_given?
 
-      case name
+      case method_name
       when /^text_(.+)$/
-        Yasuri::TextNode.new(xpath,   $1, children || [], *opt)
+        # Todo raise error xpath is not valid
+        Yasuri::TextNode.new(xpath, $1, children || [], **opt)
       when /^struct_(.+)$/
-        Yasuri::StructNode.new(xpath, $1, children || [], *opt)
+        # Todo raise error xpath is not valid
+        Yasuri::StructNode.new(xpath, $1, children || [], **opt)
       when /^links_(.+)$/
-        Yasuri::LinksNode.new(xpath,  $1, children || [], *opt)
+        # Todo raise error xpath is not valid
+        Yasuri::LinksNode.new(xpath, $1, children || [], **opt)
       when /^pages_(.+)$/
-        Yasuri::PaginateNode.new(xpath, $1, children || [], *opt)
+        # Todo raise error xpath is not valid
+        Yasuri::PaginateNode.new(xpath, $1, children || [], **opt)
+      when /^map_(.+)$/
+        Yasuri::MapNode.new($1, children, **opt)
       else
         nil
       end
-    end # of self.gen(name, *args, &block)
+    end # of self.gen(method_name, xpath, **opt, &block)
   end # of class NodeGenerator
 end

data/lib/yasuri/yasuri_paginate_node.rb

@@ -7,10 +7,10 @@ module Yasuri
   class PaginateNode
     include Node
 
-    def initialize(xpath, name, children = [], hash = {})
+    def initialize(xpath, name, children = [], limit: nil, flatten: false)
       super(xpath, name, children)
-      @limit = hash[:limit]
-      @flatten = hash[:flatten] || false
+      @flatten = flatten
+      @limit = limit
     end
 
     def inject(agent, page, opt = {}, element = page)
@@ -44,5 +44,9 @@ module Yasuri
     def opts
       {limit:@limit, flatten:@flatten}
     end
+
+    def node_type_str
+      "pages"
+    end
   end
 end

data/lib/yasuri/yasuri_text_node.rb

@@ -7,11 +7,11 @@ module Yasuri
   class TextNode
     include Node
 
-    def initialize(xpath, name, children = [], hash = {})
+    def initialize(xpath, name, children = [], **opt)
       super(xpath, name, children)
 
-      truncate = hash[:truncate]
-      proc     = hash[:proc]
+      truncate = opt[:truncate]
+      proc = opt[:proc]
 
       truncate = Regexp.new(truncate) if not truncate.nil? # regexp or nil
       @truncate = truncate
@@ -34,6 +34,10 @@ module Yasuri
       text
     end
 
+    def node_type_str
+      "text"
+    end
+
     def opts
       {truncate:@truncate, proc:@proc}
     end

data/spec/spec_helper.rb

@@ -12,11 +12,6 @@ shared_context 'httpserver' do
   }
 end
 
-
-# ENV['CODECLIMATE_REPO_TOKEN'] = "0dc78d33107a7f11f257c0218ac1a37e0073005bb9734f2fd61d0f7e803fc151"
-# require "codeclimate-test-reporter"
-# CodeClimate::TestReporter.start
-
 require 'simplecov'
 require 'coveralls'
 Coveralls.wear!

data/spec/yasuri_links_node_spec.rb

@@ -59,10 +59,18 @@ describe 'Yasuri' do
       ]
       expect(actual).to match expected
     end
-    it 'can be defined by DSL, return single LinkNode title' do
-      generated = Yasuri.links_title '/html/body/a'
-      original  = Yasuri::LinksNode.new('/html/body/a', "title")
-      compare_generated_vs_original(generated, original, @index_page)
+    it 'can be defined by DSL, return no contains if no child node' do
+      root_node = Yasuri.links_title '/html/body/a'
+      actual = root_node.inject(@agent, @index_page)
+      expected = [{}, {}, {}] # Empty if no child node under links node.
+      expect(actual).to match expected
+    end
+
+    it 'can be defined return no contains if no child node' do
+      root_node = Yasuri::LinksNode.new('/html/body/a', "title")
+      actual = root_node.inject(@agent, @index_page)
+      expected = [{}, {}, {}] # Empty if no child node under links node.
+      expect(actual).to match expected
     end
     it 'can be defined by DSL, return nested contents under link' do
       generated = Yasuri.links_title '/html/body/a' do

data/spec/yasuri_map_spec.rb

@@ -0,0 +1,76 @@
+require_relative 'spec_helper'
+
+describe 'Yasuri' do
+  include_context 'httpserver'
+
+  before do
+    @agent = Mechanize.new
+    @index_page = @agent.get(uri)
+  end
+
+  describe '::MapNode' do
+    it "multi scrape in singe page" do
+      map = Yasuri.map_sample do
+        text_title  '/html/head/title'
+        text_body_p '/html/body/p[1]'
+      end
+      actual = map.inject(@agent, @index_page)
+
+      expected = {
+        "title"  => "Yasuri Test",
+        "body_p" => "Hello,Yasuri"
+      }
+      expect(actual).to include expected
+    end
+
+    it "nested multi scrape in singe page" do
+      map = Yasuri.map_sample do
+        map_group1 { text_child01  '/html/body/a[1]' }
+        map_group2 do
+          text_child01 '/html/body/a[1]'
+          text_child03 '/html/body/a[3]'
+        end
+      end
+      actual = map.inject(@agent, @index_page)
+
+      expected = {
+        "group1" => {
+          "child01" => "child01"
+        },
+        "group2" => {
+          "child01" => "child01",
+          "child03" => "child03"
+        }
+      }
+      expect(actual).to include expected
+    end
+
+    it "scrape with links node" do
+      map = Yasuri.map_sample do
+        map_group1 do
+          links_a '/html/body/a' do
+            text_content '/html/body/p'
+          end
+          text_child01  '/html/body/a[1]'
+        end
+        map_group2 do
+          text_child03 '/html/body/a[3]'
+        end
+      end
+      actual = map.inject(@agent, @index_page)
+
+      expected = {
+        "group1" => {
+          "a" => [
+            {"content" => "Child 01 page."},
+            {"content" => "Child 02 page."},
+            {"content" => "Child 03 page."},
+          ],
+          "child01" => "child01"
+        },
+        "group2" => { "child03" => "child03" }
+      }
+      expect(actual).to include expected
+    end
+  end
+end

data/spec/yasuri_spec.rb

@@ -13,6 +13,89 @@ describe 'Yasuri' do
     @index_page = @agent.get(@uri)
   end
 
+  ############
+  # yam2tree #
+  ############
+  describe '.yaml2tree' do
+    it "fail if empty yaml" do
+      expect { Yasuri.yaml2tree(nil) }.to raise_error(RuntimeError)
+    end
+
+    it "return text node" do
+      src = <<-EOB
+content:
+  node: text
+  path: "/html/body/p[1]"
+EOB
+      generated = Yasuri.yaml2tree(src)
+      original  = Yasuri::TextNode.new('/html/body/p[1]', "content")
+
+      compare_generated_vs_original(generated, original, @index_page)
+    end
+
+    it "return text node as symbol" do
+      src = <<-EOB
+:content:
+  :node: text
+  :path: "/html/body/p[1]"
+EOB
+      generated = Yasuri.yaml2tree(src)
+      original  = Yasuri::TextNode.new('/html/body/p[1]', "content")
+
+      compare_generated_vs_original(generated, original, @index_page)
+    end
+
+    it "return LinksNode/TextNode" do
+
+      src = <<-EOB
+root:
+  node: links
+  path: "/html/body/a"
+  children:
+    - content:
+        node: text
+        path: "/html/body/p"
+EOB
+      generated = Yasuri.yaml2tree(src)
+      original  = Yasuri::LinksNode.new('/html/body/a', "root", [
+                    Yasuri::TextNode.new('/html/body/p', "content"),
+                  ])
+
+      compare_generated_vs_original(generated, original, @index_page)
+    end
+
+    it "return StructNode/StructNode/[TextNode,TextNode]" do
+      src = <<-EOB
+tables:
+  node: struct
+  path: "/html/body/table"
+  children:
+    - table:
+        node: struct
+        path: "./tr"
+        children:
+          - title:
+              node: text
+              path: "./td[1]"
+          - pub_date:
+              node: text
+              path: "./td[2]"
+EOB
+
+      generated = Yasuri.yaml2tree(src)
+      original  = Yasuri::StructNode.new('/html/body/table', "tables", [
+        Yasuri::StructNode.new('./tr', "table", [
+          Yasuri::TextNode.new('./td[1]', "title"),
+          Yasuri::TextNode.new('./td[2]', "pub_date"),
+        ])
+      ])
+      page = @agent.get(@uri + "/struct/structual_text.html")
+      compare_generated_vs_original(generated, original, page)
+    end
+
+  end # end of describe '.yaml2tree'
+
+
   #############
   # json2tree #
   #############
@@ -39,10 +122,31 @@ describe 'Yasuri' do
                   "truncate"  : "^[^,]+"
                 }|
       generated = Yasuri.json2tree(src)
-      original  = Yasuri::TextNode.new('/html/body/p[1]', "content", {}, truncate:/^[^,]+/)
+      original  = Yasuri::TextNode.new('/html/body/p[1]', "content", truncate:/^[^,]+/)
       compare_generated_vs_original(generated, original, @index_page)
     end
 
+    it "return MapNode with TextNodes" do
+      src = %q| { "node" : "map",
+                  "name"  : "parent",
+                  "children" : [
+                    { "node"  : "text",
+                      "name"  : "content01",
+                      "path"  : "/html/body/p[1]"
+                    },
+                    { "node"  : "text",
+                      "name"  : "content02",
+                      "path"  : "/html/body/p[2]"
+                    }
+                  ]
+                }|
+      generated = Yasuri.json2tree(src)
+      original  = Yasuri::MapNode.new('parent', [
+        Yasuri::TextNode.new('/html/body/p[1]', "content01"),
+        Yasuri::TextNode.new('/html/body/p[2]', "content02"),
+      ])
+      compare_generated_vs_original(generated, original, @index_page)
+    end
 
     it "return LinksNode/TextNode" do
       src = %q| { "node"     : "links",
@@ -153,7 +257,7 @@ describe 'Yasuri' do
     end
 
     it "return text node with truncate_regexp" do
-      node = Yasuri::TextNode.new("/html/head/title", "title", {}, truncate:/^[^,]+/)
+      node = Yasuri::TextNode.new("/html/head/title", "title", truncate:/^[^,]+/)
       json = Yasuri.tree2json(node)
       expected_str = %q| { "node": "text",
                            "name": "title",
@@ -165,6 +269,31 @@ describe 'Yasuri' do
       expect(actual).to match expected
     end
 
+    it "return map node with text nodes" do
+      tree = Yasuri::MapNode.new('parent', [
+        Yasuri::TextNode.new('/html/body/p[1]', "content01"),
+        Yasuri::TextNode.new('/html/body/p[2]', "content02"),
+      ])
+      actual_json = Yasuri.tree2json(tree)
+
+      expected_json = %q| { "node" : "map",
+        "name"  : "parent",
+        "children" : [
+          { "node"  : "text",
+            "name"  : "content01",
+            "path"  : "/html/body/p[1]"
+          },
+          { "node"  : "text",
+            "name"  : "content02",
+            "path"  : "/html/body/p[2]"
+          }
+        ]
+      }|
+      expected = Yasuri.tree2json(Yasuri.json2tree(expected_json))
+      actual   = Yasuri.tree2json(Yasuri.json2tree(actual_json))
+      expect(actual).to match expected
+    end
+
     it "return LinksNode/TextNode" do
       tree  = Yasuri::LinksNode.new('/html/body/a', "root", [
                 Yasuri::TextNode.new('/html/body/p', "content"),

data/spec/yasuri_struct_node_spec.rb

@@ -126,7 +126,7 @@ describe 'Yasuri' do
         Yasuri::TextNode.new('./td[1]', "title"),
         Yasuri::TextNode.new('./td[2]', "pub_date"),
       ])
-      expected = @table_1996.map{|h| Hash[h.map{|k,v| [k.to_sym, v] }] }
+      expected = @table_1996.map{|h| h.map{|k,v| [k.to_sym, v] }.to_h }
       actual = node.inject(@agent, @page, symbolize_names:true)
       expect(actual).to match expected
     end

data/yasuri.gemspec

@@ -18,8 +18,8 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.7"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "bundler"
+  spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec"
   spec.add_development_dependency "fuubar"
   spec.add_development_dependency "glint"

metadata

@@ -1,43 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: yasuri
 version: !ruby/object:Gem::Version
-  version: 1.9.12
+  version: 3.1.0
 platform: ruby
 authors:
 - TAC
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-12-15 00:00:00.000000000 Z
+date: 2021-03-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -144,8 +144,10 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".coveralls.yml"
+- ".github/workflows/ruby.yml"
 - ".gitignore"
 - ".rspec"
+- ".ruby-version"
 - ".travis.yml"
 - Gemfile
 - LICENSE
@@ -158,6 +160,7 @@ files:
 - lib/yasuri/version.rb
 - lib/yasuri/yasuri.rb
 - lib/yasuri/yasuri_links_node.rb
+- lib/yasuri/yasuri_map_node.rb
 - lib/yasuri/yasuri_node.rb
 - lib/yasuri/yasuri_node_generator.rb
 - lib/yasuri/yasuri_paginate_node.rb
@@ -179,6 +182,7 @@ files:
 - spec/servers/httpserver.rb
 - spec/spec_helper.rb
 - spec/yasuri_links_node_spec.rb
+- spec/yasuri_map_spec.rb
 - spec/yasuri_node_spec.rb
 - spec/yasuri_paginate_node_spec.rb
 - spec/yasuri_spec.rb
@@ -189,7 +193,7 @@ homepage: https://github.com/tac0x2a/yasuri
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -204,9 +208,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.2
-signing_key: 
+rubygems_version: 3.2.3
+signing_key:
 specification_version: 4
 summary: Yasuri is easy scraping library.
 test_files:
@@ -226,6 +229,7 @@ test_files:
 - spec/servers/httpserver.rb
 - spec/spec_helper.rb
 - spec/yasuri_links_node_spec.rb
+- spec/yasuri_map_spec.rb
 - spec/yasuri_node_spec.rb
 - spec/yasuri_paginate_node_spec.rb
 - spec/yasuri_spec.rb