yasuri 1.9.11 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 28e6a3903cec3d8036b718a7c5de4fb8df8dfbfa
-  data.tar.gz: fbfb4c2b3a042410a7d05b1604e004bdffd82779
+SHA256:
+  metadata.gz: 7f360d6efb02954a5a54e2fc308d0cd0c2e5c129c52eba727fb0dfe4a40ce502
+  data.tar.gz: 8d8805a55c7ce16c76eb50945b954ad19327a3a63183eca098dac6ac93d2203b
 SHA512:
-  metadata.gz: e4312623046ecf451ef261b1d99164b7f79b861ad8d7fe49d7ba8fd2319cd840978ae88235e6fb10b6991cd5131ebec966ec56e3bddece8fbbefd0b53d4a9dfe
-  data.tar.gz: 54639066aa4511309a1f712b8920c81c09a7489e0cab6a91c453d2aeca07d4f1c3bed167ed25746411d5f87fa9cc9fff8c6a32567c4df8d1990f12cb053286e2
+  metadata.gz: ffe02aee78de5f30f1e583b2aca8c0617324bdbf62d7c64e371e90d139bac8b1d26df23e9725df0b81b946c6a465283f88a7d51945872c56e7be892eac1b5e4e
+  data.tar.gz: c8983dc2cd283c7de0d97357d2a8164426ee3e1017e73c498c0676716a1c9ab4c42cc02a836bf7e559877d50ca23df6fa656c0197b5018a4881997e2fb4c57d0

data/.github/workflows/ruby.yml

@@ -0,0 +1,35 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+# This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake
+# For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby
+
+name: Ruby
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  test:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby-version: ['2.6', '2.7', '3.0']
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+    # To automatically get bug fixes and new Ruby versions for ruby/setup-ruby,
+    # change this to (see https://github.com/ruby/setup-ruby#versioning):
+    # uses: ruby/setup-ruby@v1
+      uses: ruby/setup-ruby@473e4d8fe5dd94ee328fdfca9f8c9c7afc9dae5e
+      with:
+        ruby-version: ${{ matrix.ruby-version }}
+        bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+    - name: Run tests
+      run: bundle exec rake

data/.gitignore

@@ -66,5 +66,4 @@ tramp
 # cask packages
 .cask/
 
-.ruby-version
-Gemfile.lock
+Gemfile.lock

data/.ruby-version

@@ -0,0 +1 @@
+3.0.0

data/.travis.yml

@@ -1,9 +1,7 @@
 language: ruby
-rvm:
-  - 2.2.0
 script:
   - ruby --version
   - rspec spec
 addons:
   code_climate:
-    repo_token: 0dc78d33107a7f11f257c0218ac1a37e0073005bb9734f2fd61d0f7e803fc151
+    repo_token: 0dc78d33107a7f11f257c0218ac1a37e0073005bb9734f2fd61d0f7e803fc151

data/README.md

@@ -1,4 +1,5 @@
-# Yasuri [![Build Status](https://travis-ci.org/tac0x2a/yasuri.svg?branch=master)](https://travis-ci.org/tac0x2a/yasuri) [![Coverage Status](https://coveralls.io/repos/tac0x2a/yasuri/badge.svg?branch=master)](https://coveralls.io/r/tac0x2a/yasuri?branch=master) [![Code Climate](https://codeclimate.com/github/tac0x2a/yasuri/badges/gpa.svg)](https://codeclimate.com/github/tac0x2a/yasuri)
+# Yasuri
+[![Build Status](https://travis-ci.org/tac0x2a/yasuri.svg?branch=master)](https://travis-ci.org/tac0x2a/yasuri) [![Coverage Status](https://coveralls.io/repos/tac0x2a/yasuri/badge.svg?branch=master)](https://coveralls.io/r/tac0x2a/yasuri?branch=master) [![Maintainability](https://api.codeclimate.com/v1/badges/c29480fea1305afe999f/maintainability)](https://codeclimate.com/github/tac0x2a/yasuri/maintainability)
 
 Yasuri (鑢) is an easy web-scraping library for supporting "[Mechanize](https://github.com/sparklemotion/mechanize)".
 
@@ -52,6 +53,19 @@ root = Yasuri.links_root '//*[@id="menu"]/ul/li/a' do
          text_content '//*[@id="contents"]/p[1]'
        end
 
+
+# Node tree constructing by YAML
+src = <<-EOYAML
+root:
+  node: links
+  path: "//*[@id='menu']/ul/li/a"
+  children:
+    - title:   { node: text, path: "//*[@id='contents']/h2" }
+    - content: { node: text, path: "//*[@id='contents']/p[1]" }
+EOYAML
+root = Yasuri.yaml2tree(src)
+
+
 # Node tree constructing by JSON
 src = <<-EOJSON
    { "node"     : "links",
@@ -78,6 +92,17 @@ result = root.inject(agent, root_page)
 # => [ {"title" => "PageTitle", "content" => "Page Contents" }, ...  ]
 ```
 
+## Dev
+```sh
+$ gem install bundler
+$ bundle install
+```
+### Test
+```sh
+$ rake
+# or
+$ rspec spec/*spec.rb
+```
 
 ## Contributing
 

data/USAGE.ja.md

@@ -67,7 +67,7 @@ page = agent.get(uri)
 tree.inject(agent, page)
 ```
 
-ツリーは、DSLまたはjsonで定義することができます．上の例ではDSLで定義しています．
+ツリーは、json，yaml，またはDSLで定義することができます．上の例ではDSLで定義しています．
 以下は、jsonで上記と等価な解析ツリーを定義した例です．
 
 ```ruby
@@ -87,6 +87,19 @@ EOJSON
 tree = Yasuri.json2tree(src)
 ```
 
+```ruby
+# yaml で構成する場合
+src = <<-EOYAML
+title:
+  node: links
+  path: "/html/body/a"
+  children:
+    - name:
+        node: text
+        path: "/html/body/p"
+EOYAML
+tree = Yasuri.yaml2tree(src)
+```
 
 ### Node
 ツリーは入れ子になった *Node* で構成されます．
@@ -431,3 +444,38 @@ node.inject(agent, page)
 #=> [ {"content" => "Pagination01"}, {"content" => "Pagination02"}]
 ```
 この場合、PaginateNode は最大2つまでのページを開いてパースします．ページネーションは4つのページを持っているようですが、`limit:2`が指定されているため、結果の配列には2つの結果のみが含まれています．
+
+##### `flatten`
+取得した各ページの結果を展開します．
+
+```ruby
+agent = Mechanize.new
+page = agent.get("http://yasuri.example.net/page01.html")
+
+node = Yasuri.pages_root "/html/body/nav/span/a[@class='next']" , flatten:true do
+         text_title   '/html/head/title'
+         text_content '/html/body/p'
+       end
+node.inject(agent, page)
+
+#=> [ {"title" => "Page01",
+       "content" => "Patination01"},
+      {"title"   => "Page01",
+       "content" => "Patination02"},
+      {"title"   => "Page01",
+       "content" => "Patination03"}]
+
+
+node = Yasuri.pages_root "/html/body/nav/span/a[@class='next']" , flatten:true do
+        text_title   '/html/head/title'
+        text_content '/html/body/p'
+      end
+node.inject(agent, page)
+
+#=> [ "Page01",
+      "Patination01",
+      "Page02",
+      "Patination02",
+      "Page03",
+      "Patination03"]
+```

data/USAGE.md

@@ -69,7 +69,7 @@ page = agent.get(uri)
 tree.inject(agent, page)
 ```
 
-Tree is definable by 2(+1) ways, DSL and json (and basic ruby code). In above example, DSL.
+Tree is definable by 3(+1) ways, json, yaml, and DSL (or basic ruby code). In above example, DSL.
 
 ```ruby
 # Construct by json.
@@ -88,6 +88,21 @@ EOJSON
 tree = Yasuri.json2tree(src)
 ```
 
+```ruby
+# Construct by yaml.
+src = <<-EOYAML
+title:
+  node: links
+  path: "/html/body/a"
+  children:
+    - name:
+        node: text
+        path: "/html/body/p"
+EOYAML
+tree = Yasuri.yaml2tree(src)
+```
+
+
 ### Node
 Tree is constructed by nested Nodes.
 Node has `Type`, `Name`, `Path`, `Childlen`, and `Options`.
@@ -429,3 +444,38 @@ node.inject(agent, page)
 #=> [ {"content" => "Pagination01"}, {"content" => "Pagination02"}]
 ```
 Paginate Node open upto 2 given by `limit`. In this situation, pagination has 4 pages, but result Array has 2 texts because given `limit:2`.
+
+##### `flatten`
+`flatten` option expands each page results.
+
+```ruby
+agent = Mechanize.new
+page = agent.get("http://yasuri.example.net/page01.html")
+
+node = Yasuri.pages_root "/html/body/nav/span/a[@class='next']" , flatten:true do
+         text_title   '/html/head/title'
+         text_content '/html/body/p'
+       end
+node.inject(agent, page)
+
+#=> [ {"title" => "Page01",
+       "content" => "Patination01"},
+      {"title"   => "Page01",
+       "content" => "Patination02"},
+      {"title"   => "Page01",
+       "content" => "Patination03"}]
+
+
+node = Yasuri.pages_root "/html/body/nav/span/a[@class='next']" , flatten:true do
+        text_title   '/html/head/title'
+        text_content '/html/body/p'
+      end
+node.inject(agent, page)
+
+#=> [ "Page01",
+      "Patination01",
+      "Page02",
+      "Patination02",
+      "Page03",
+      "Patination03"]
+```

data/lib/yasuri/version.rb

@@ -1,3 +1,3 @@
 module Yasuri
-  VERSION = "1.9.11"
+  VERSION = "3.0.0"
 end

data/lib/yasuri/yasuri.rb

@@ -4,6 +4,7 @@
 
 require 'mechanize'
 require 'json'
+require 'yaml'
 
 require_relative 'yasuri_node'
 require_relative 'yasuri_text_node'
@@ -23,9 +24,39 @@ module Yasuri
     Yasuri.node2hash(node).to_json
   end
 
-  def self.method_missing(name, *args, &block)
-    generated = Yasuri::NodeGenerator.gen(name, *args, &block)
-    generated || super(name, args)
+  def self.yaml2tree(yaml_string)
+    raise RuntimeError if yaml_string.nil? or yaml_string.empty?
+
+    yaml = YAML.load(yaml_string)
+    raise RuntimeError if yaml.keys.size < 1
+
+    root_key, root = yaml.keys.first, yaml.values.first
+    hash = Yasuri.yaml2tree_sub(root_key, root)
+
+    Yasuri.hash2node(hash)
+  end
+
+  private
+  def self.yaml2tree_sub(name, body)
+    return nil if name.nil? or body.nil?
+
+    new_body = Hash[:name, name]
+    body.each{|k,v| new_body[k.to_sym] = v}
+    body = new_body
+
+    return body if body[:children].nil?
+
+    body[:children] = body[:children].map do |c|
+      k, b = c.keys.first, c.values.first
+      Yasuri.yaml2tree_sub(k, b)
+    end
+
+    body
+  end
+
+  def self.method_missing(node_name, pattern, **opt, &block)
+    generated = Yasuri::NodeGenerator.gen(node_name, pattern, **opt, &block)
+    generated || super(node_name, **opt)
   end
 
   private
@@ -37,16 +68,16 @@ module Yasuri
   }
   Node2Text = Text2Node.invert
 
-  ReservedKeys = [:node, :name, :path, :children]
+  ReservedKeys = %i|node name path children|
   def self.hash2node(node_h)
     node, name, path, children = ReservedKeys.map do |key|
       node_h[key]
     end
     children ||= []
 
-    fail "Not found 'node' value in json" if node.nil?
-    fail "Not found 'name' value in json" if name.nil?
-    fail "Not found 'path' value in json" if path.nil?
+    fail "Not found 'node' value in map" if node.nil?
+    fail "Not found 'name' value in map" if name.nil?
+    fail "Not found 'path' value in map" if path.nil?
 
     childnodes = children.map{|c| Yasuri.hash2node(c) }
     ReservedKeys.each{|key| node_h.delete(key)}
@@ -54,7 +85,7 @@ module Yasuri
 
     klass = Text2Node[node.to_sym]
     fail "Undefined node type #{node}" if klass.nil?
-    klass.new(path, name, childnodes, opt)
+    klass.new(path, name, childnodes, **opt)
   end
 
   def self.node2hash(node)
@@ -78,8 +109,8 @@ module Yasuri
     json
   end
 
-  def self.NodeName(name, hash = {})
-    symbolize_names = hash[:symbolize_names] || false
+  def self.NodeName(name, opt)
+    symbolize_names = opt[:symbolize_names]
     symbolize_names ? name.to_sym : name
   end
 

data/lib/yasuri/yasuri_links_node.rb

@@ -6,10 +6,10 @@ require_relative 'yasuri_node'
 module Yasuri
   class LinksNode
     include Node
-    def inject(agent, page, opt = {})
+    def inject(agent, page, opt = {}, element = page)
       retry_count = opt[:retry_count] || 5
 
-      links = page.search(@xpath) || [] # links expected
+      links = element.search(@xpath) || [] # links expected
       links.map do |link|
         link_button = Mechanize::Page::Link.new(link, agent, page)
         child_page = Yasuri.with_retry(retry_count) { link_button.click }

data/lib/yasuri/yasuri_node.rb

@@ -7,11 +7,11 @@ module Yasuri
   module Node
     attr_reader :url, :xpath, :name, :children
 
-    def initialize(xpath, name, children = [], opt = {})
+    def initialize(xpath, name, children = [], opt: {})
       @xpath, @name, @children = xpath, name, children
     end
 
-    def inject(agent, page, opt = {})
+    def inject(agent, page, opt = {}, element = page)
       fail "#{Kernel.__method__} is not implemented."
     end
     def opts

data/lib/yasuri/yasuri_node_generator.rb

@@ -15,26 +15,24 @@ module Yasuri
       @nodes
     end
 
-    def method_missing(name, *args, &block)
-      node = NodeGenerator.gen(name, *args, &block)
+    def method_missing(name, pattern, **args, &block)
+      node = NodeGenerator.gen(name, pattern, **args, &block)
       raise "Undefined Node Name '#{name}'" if node == nil
       @nodes << node
     end
 
-    def self.gen(name, *args, &block)
-      xpath, opt = *args
-      opt = [opt].flatten.compact
+    def self.gen(name, xpath, **opt, &block)
       children = Yasuri::NodeGenerator.new.gen_recursive(&block) if block_given?
 
       case name
       when /^text_(.+)$/
-        Yasuri::TextNode.new(xpath,   $1, children || [], *opt)
+        Yasuri::TextNode.new(xpath,   $1, children || [], **opt)
       when /^struct_(.+)$/
-        Yasuri::StructNode.new(xpath, $1, children || [], *opt)
+        Yasuri::StructNode.new(xpath, $1, children || [], **opt)
       when /^links_(.+)$/
-        Yasuri::LinksNode.new(xpath,  $1, children || [], *opt)
+        Yasuri::LinksNode.new(xpath,  $1, children || [], **opt)
       when /^pages_(.+)$/
-        Yasuri::PaginateNode.new(xpath, $1, children || [], *opt)
+        Yasuri::PaginateNode.new(xpath, $1, children || [], **opt)
       else
         nil
       end

data/lib/yasuri/yasuri_paginate_node.rb

@@ -7,14 +7,17 @@ module Yasuri
   class PaginateNode
     include Node
 
-    def initialize(xpath, name, children = [], hash = {})
+    def initialize(xpath, name, children = [], limit: nil, flatten: false)
       super(xpath, name, children)
-      @limit = hash[:limit]
+      @flatten = flatten
+      @limit = limit
     end
 
-    def inject(agent, page, opt = {})
+    def inject(agent, page, opt = {}, element = page)
       retry_count = opt[:retry_count] || 5
 
+      raise NotImplementedError.new("PagenateNode inside StructNode, Not Supported") if page != element
+
       child_results = []
       limit = @limit.nil? ? Float::MAX : @limit
       while page
@@ -32,10 +35,14 @@ module Yasuri
         break if (limit -= 1) <= 0
       end
 
+      if @flatten == true
+        return child_results.map{|h| h.values}.flatten
+      end
+
       child_results
     end
     def opts
-      {limit:@limit}
+      {limit:@limit, flatten:@flatten}
     end
   end
 end

data/lib/yasuri/yasuri_struct_node.rb

@@ -6,12 +6,12 @@ require_relative 'yasuri_node'
 module Yasuri
   class StructNode
     include Node
-    def inject(agent, page, opt = {})
-      sub_tags = page.search(@xpath)
+    def inject(agent, page, opt = {}, element = page)
+      sub_tags = element.search(@xpath)
       tree = sub_tags.map do |sub_tag|
         child_results_kv = @children.map do |child_node|
           child_name = Yasuri.NodeName(child_node.name, opt)
-          [child_name, child_node.inject(agent, sub_tag, opt)]
+          [child_name, child_node.inject(agent, page, opt, sub_tag)]
         end
         Hash[child_results_kv]
       end

data/lib/yasuri/yasuri_text_node.rb

@@ -7,11 +7,11 @@ module Yasuri
   class TextNode
     include Node
 
-    def initialize(xpath, name, children = [], hash = {})
+    def initialize(xpath, name, children = [], **opt)
       super(xpath, name, children)
 
-      truncate = hash[:truncate]
-      proc     = hash[:proc]
+      truncate = opt[:truncate]
+      proc = opt[:proc]
 
       truncate = Regexp.new(truncate) if not truncate.nil? # regexp or nil
       @truncate = truncate
@@ -21,8 +21,8 @@ module Yasuri
 
     end
 
-    def inject(agent, page, opt = {})
-      node = page.search(@xpath)
+    def inject(agent, page, opt = {}, element = page)
+      node = element.search(@xpath)
       text = node.text.to_s
 
       if @truncate

data/spec/htdocs/struct/structual_links.html

@@ -0,0 +1,30 @@
+<html>
+  <head>
+    <title>StructualLinksTest</title>
+  </head>
+  <body>
+
+    <table>
+      <thead>
+        <tr>
+          <th>Title</th>
+          <th>Links</th>
+        </tr>
+      </thead>
+      <tr>
+        <td>Child01,02</td>
+        <td><a href="../child01.html">Child01</a></td>
+        <td><a href="../child02.html">Child02</a></td>
+        <td>../child02.html</td>
+      </tr>
+
+      <tr>
+        <td>Child01,02,03</td>
+        <td><a href="../child01.html">Child01</a></td>
+        <td><a href="../child02.html">Child02</a></td>
+        <td><a href="../child03.html">Child03</a></td>
+      </tr>
+    </table>
+
+  </body>
+</html>

data/spec/spec_helper.rb

@@ -12,11 +12,6 @@ shared_context 'httpserver' do
   }
 end
 
-
-# ENV['CODECLIMATE_REPO_TOKEN'] = "0dc78d33107a7f11f257c0218ac1a37e0073005bb9734f2fd61d0f7e803fc151"
-# require "codeclimate-test-reporter"
-# CodeClimate::TestReporter.start
-
 require 'simplecov'
 require 'coveralls'
 Coveralls.wear!

data/spec/yasuri_links_node_spec.rb

@@ -59,10 +59,18 @@ describe 'Yasuri' do
       ]
       expect(actual).to match expected
     end
-    it 'can be defined by DSL, return single LinkNode title' do
-      generated = Yasuri.links_title '/html/body/a'
-      original  = Yasuri::LinksNode.new('/html/body/a', "title")
-      compare_generated_vs_original(generated, original, @index_page)
+    it 'can be defined by DSL, return no contains if no child node' do
+      root_node = Yasuri.links_title '/html/body/a'
+      actual = root_node.inject(@agent, @index_page)
+      expected = [{}, {}, {}] # Empty if no child node under links node.
+      expect(actual).to match expected
+    end
+
+    it 'can be defined return no contains if no child node' do
+      root_node = Yasuri::LinksNode.new('/html/body/a', "title")
+      actual = root_node.inject(@agent, @index_page)
+      expected = [{}, {}, {}] # Empty if no child node under links node.
+      expect(actual).to match expected
     end
     it 'can be defined by DSL, return nested contents under link' do
       generated = Yasuri.links_title '/html/body/a' do

data/spec/yasuri_paginate_node_spec.rb

@@ -30,6 +30,49 @@ describe 'Yasuri' do
       expect(actual).to match expected
     end
 
+    it "scrape each paginated pages with flatten" do
+      root_node = Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "root", [
+        Yasuri::TextNode.new('/html/body/p', "content"),
+        Yasuri::StructNode.new('/html/body/nav/span', "span", [
+          Yasuri::TextNode.new('./a', "text"),
+        ]),
+      ], flatten: true)
+      actual = root_node.inject(@agent, @page)
+      expected = [
+        "PaginationTest01",
+        {"text"=>""},
+        {"text"=>""},
+        {"text" => "2"},
+        {"text" => "3"},
+        {"text" => "4"},
+        {"text"=>"NextPage »"},
+        "PaginationTest02",
+        {"text"=>"« PreviousPage"},
+        {"text" => "1"},
+        {"text"=>""},
+        {"text" => "3"},
+        {"text" => "4"},
+        {"text"=>"NextPage »"},
+        "PaginationTest03",
+        {"text"=>"« PreviousPage"},
+        {"text" => "1"},
+        {"text" => "2"},
+        {"text"=>""},
+        {"text" => "4"},
+        {"text"=>"NextPage »"},
+        "PaginationTest04",
+        {"text"=>"« PreviousPage"},
+        {"text" => "1"},
+        {"text" => "2"},
+        {"text" => "3"},
+        {"text"=>""},
+        {"text"=>""},
+      ]
+
+      expect(actual).to match expected
+    end
+
+
     it "scrape each paginated pages limited" do
       root_node = Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "root", [
         Yasuri::TextNode.new('/html/body/p', "content"),

data/spec/yasuri_spec.rb

@@ -13,6 +13,89 @@ describe 'Yasuri' do
     @index_page = @agent.get(@uri)
   end
 
+  ############
+  # yam2tree #
+  ############
+  describe '.yaml2tree' do
+    it "fail if empty yaml" do
+      expect { Yasuri.yaml2tree(nil) }.to raise_error(RuntimeError)
+    end
+
+    it "return text node" do
+      src = <<-EOB
+content:
+  node: text
+  path: "/html/body/p[1]"
+EOB
+      generated = Yasuri.yaml2tree(src)
+      original  = Yasuri::TextNode.new('/html/body/p[1]', "content")
+
+      compare_generated_vs_original(generated, original, @index_page)
+    end
+
+    it "return text node as symbol" do
+      src = <<-EOB
+:content:
+  :node: text
+  :path: "/html/body/p[1]"
+EOB
+      generated = Yasuri.yaml2tree(src)
+      original  = Yasuri::TextNode.new('/html/body/p[1]', "content")
+
+      compare_generated_vs_original(generated, original, @index_page)
+    end
+
+    it "return LinksNode/TextNode" do
+
+      src = <<-EOB
+root:
+  node: links
+  path: "/html/body/a"
+  children:
+    - content:
+        node: text
+        path: "/html/body/p"
+EOB
+      generated = Yasuri.yaml2tree(src)
+      original  = Yasuri::LinksNode.new('/html/body/a', "root", [
+                    Yasuri::TextNode.new('/html/body/p', "content"),
+                  ])
+
+      compare_generated_vs_original(generated, original, @index_page)
+    end
+
+    it "return StructNode/StructNode/[TextNode,TextNode]" do
+      src = <<-EOB
+tables:
+  node: struct
+  path: "/html/body/table"
+  children:
+    - table:
+        node: struct
+        path: "./tr"
+        children:
+          - title:
+              node: text
+              path: "./td[1]"
+          - pub_date:
+              node: text
+              path: "./td[2]"
+EOB
+
+      generated = Yasuri.yaml2tree(src)
+      original  = Yasuri::StructNode.new('/html/body/table', "tables", [
+        Yasuri::StructNode.new('./tr', "table", [
+          Yasuri::TextNode.new('./td[1]', "title"),
+          Yasuri::TextNode.new('./td[2]', "pub_date"),
+        ])
+      ])
+      page = @agent.get(@uri + "/struct/structual_text.html")
+      compare_generated_vs_original(generated, original, page)
+    end
+
+  end # end of describe '.yaml2tree'
+
+
   #############
   # json2tree #
   #############
@@ -39,7 +122,7 @@ describe 'Yasuri' do
                   "truncate"  : "^[^,]+"
                 }|
       generated = Yasuri.json2tree(src)
-      original  = Yasuri::TextNode.new('/html/body/p[1]', "content", {}, truncate:/^[^,]+/)
+      original  = Yasuri::TextNode.new('/html/body/p[1]', "content", truncate:/^[^,]+/)
       compare_generated_vs_original(generated, original, @index_page)
     end
 
@@ -126,7 +209,7 @@ describe 'Yasuri' do
           Yasuri::TextNode.new('./td[2]', "pub_date"),
         ])
       ])
-      page = @agent.get(@uri + "/structual_text.html")
+      page = @agent.get(@uri + "/struct/structual_text.html")
       compare_generated_vs_original(generated, original, page)
     end
   end
@@ -153,7 +236,7 @@ describe 'Yasuri' do
     end
 
     it "return text node with truncate_regexp" do
-      node = Yasuri::TextNode.new("/html/head/title", "title", {}, truncate:/^[^,]+/)
+      node = Yasuri::TextNode.new("/html/head/title", "title", truncate:/^[^,]+/)
       json = Yasuri.tree2json(node)
       expected_str = %q| { "node": "text",
                            "name": "title",
@@ -193,6 +276,7 @@ describe 'Yasuri' do
                            "name"     : "root",
                            "path"     : "/html/body/nav/span/a[@class='next']",
                            "limit"    : 10,
+                           "flatten"  : false,
                            "children" : [ { "node" : "text",
                                             "name" : "content",
                                             "path" : "/html/body/p"

data/spec/yasuri_struct_node_spec.rb

@@ -12,7 +12,7 @@ describe 'Yasuri' do
   describe '::StructNode' do
     before do
       @agent = Mechanize.new
-      @page = @agent.get(uri + "/structual_text.html")
+      @page = @agent.get(uri + "/struct/structual_text.html")
 
       @table_1996 = [
         { "title"    => "The Perfect Insider",
@@ -126,10 +126,51 @@ describe 'Yasuri' do
         Yasuri::TextNode.new('./td[1]', "title"),
         Yasuri::TextNode.new('./td[2]', "pub_date"),
       ])
-      expected = @table_1996.map{|h| Hash[h.map{|k,v| [k.to_sym, v] }] }
+      expected = @table_1996.map{|h| h.map{|k,v| [k.to_sym, v] }.to_h }
       actual = node.inject(@agent, @page, symbolize_names:true)
       expect(actual).to match expected
     end
 
   end
+
+  describe '::StructNode::Links' do
+    before do
+      @agent = Mechanize.new
+      @page = @agent.get(uri + "/struct/structual_links.html")
+
+      @table = [
+        { "title" => "Child01,02",
+          "child" => [{"p" => "Child 01 page."}, {"p" => "Child 02 page."}] },
+
+        { "title" => "Child01,02,03",
+          "child" => [{"p" => "Child 01 page."}, {"p" => "Child 02 page."}, {"p" => "Child 03 page."}]}
+      ]
+    end
+
+    it 'return child node in links inside struct' do
+      node = Yasuri::StructNode.new('/html/body/table/tr', "table", [
+        Yasuri::TextNode.new('./td[1]', "title"),
+        Yasuri::LinksNode.new('./td/a', "child", [
+          Yasuri::TextNode.new('/html/body/p', "p"),
+        ])
+      ])
+      expected = @table
+      actual = node.inject(@agent, @page)
+      expect(actual).to match expected
+    end
+  end # descrive
+
+  describe '::StructNode::Pages' do
+    before do
+      @agent = Mechanize.new
+      @page = @agent.get(uri + "/struct/structual_text.html") #dummy
+    end
+
+    it 'not supported' do
+      node = Yasuri::StructNode.new('/html/body/table[1]/tr', "table", [
+        Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "pages", [])
+      ])
+      expect{ node.inject(@agent, @page) }.to raise_error(NotImplementedError, "PagenateNode inside StructNode, Not Supported")
+    end
+  end
 end

data/yasuri.gemspec

@@ -18,8 +18,8 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.7"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "bundler"
+  spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec"
   spec.add_development_dependency "fuubar"
   spec.add_development_dependency "glint"

metadata

@@ -1,43 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: yasuri
 version: !ruby/object:Gem::Version
-  version: 1.9.11
+  version: 3.0.0
 platform: ruby
 authors:
 - TAC
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-11-14 00:00:00.000000000 Z
+date: 2021-03-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -144,8 +144,10 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".coveralls.yml"
+- ".github/workflows/ruby.yml"
 - ".gitignore"
 - ".rspec"
+- ".ruby-version"
 - ".travis.yml"
 - Gemfile
 - LICENSE
@@ -174,7 +176,8 @@ files:
 - spec/htdocs/pagination/page02.html
 - spec/htdocs/pagination/page03.html
 - spec/htdocs/pagination/page04.html
-- spec/htdocs/structual_text.html
+- spec/htdocs/struct/structual_links.html
+- spec/htdocs/struct/structual_text.html
 - spec/servers/httpserver.rb
 - spec/spec_helper.rb
 - spec/yasuri_links_node_spec.rb
@@ -188,7 +191,7 @@ homepage: https://github.com/tac0x2a/yasuri
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -203,9 +206,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.4.5
-signing_key: 
+rubygems_version: 3.2.3
+signing_key:
 specification_version: 4
 summary: Yasuri is easy scraping library.
 test_files:
@@ -220,7 +222,8 @@ test_files:
 - spec/htdocs/pagination/page02.html
 - spec/htdocs/pagination/page03.html
 - spec/htdocs/pagination/page04.html
-- spec/htdocs/structual_text.html
+- spec/htdocs/struct/structual_links.html
+- spec/htdocs/struct/structual_text.html
 - spec/servers/httpserver.rb
 - spec/spec_helper.rb
 - spec/yasuri_links_node_spec.rb