yasuri 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 1573790f94cdd8bf8621b1178f7c5ef5e00eee15
+  data.tar.gz: b37d3a38cf679b38a1d5d9dba2bc785e54d6d2b3
+SHA512:
+  metadata.gz: 8c8da39e14541e21ed520c051a4e425d6a5aed2f530b7b50e1652894ead3678d4717db7aa686a64f6c6f905d3df1e1bbfe71d1226381b0f08778a043fafaa2ba
+  data.tar.gz: 1ee0447e8757ea26ff7b4dc36cc089dda9003be8de746259e8f430d27c608e0519e2558daf37b145f009e69b909c6647990f1e46876c47d47df7ba24b75f3862

data/.coveralls.yml

@@ -0,0 +1 @@
+service_name: travis-ci

data/.gitignore

@@ -0,0 +1,70 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalisation:
+/.bundle/
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc
+
+# -*- mode: gitignore; -*-
+*~
+\#*\#
+/.emacs.desktop
+/.emacs.desktop.lock
+*.elc
+auto-save-list
+tramp
+.\#*
+
+# Org-mode
+.org-id-locations
+*_archive
+
+# flymake-mode
+*_flymake.*
+
+# eshell files
+/eshell/history
+/eshell/lastdir
+
+# elpa packages
+/elpa/
+
+# reftex files
+*.rel
+
+# AUCTeX auto folder
+/auto/
+
+# cask packages
+.cask/
+
+.ruby-version
+Gemfile.lock

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,9 @@
+language: ruby
+rvm:
+  - 2.2.0
+script:
+  - ruby --version
+  - rspec spec
+addons:
+  code_climate:
+    repo_token: 0dc78d33107a7f11f257c0218ac1a37e0073005bb9734f2fd61d0f7e803fc151

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in yasuri.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 TAC
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

data/README.md

@@ -0,0 +1,63 @@
+# Yasuri [![Build Status](https://travis-ci.org/tac0x2a/yasuri.svg?branch=master)](https://travis-ci.org/tac0x2a/yasuri) [![Coverage Status](https://coveralls.io/repos/tac0x2a/yasuri/badge.svg?branch=master)](https://coveralls.io/r/tac0x2a/yasuri?branch=master) [![Code Climate](https://codeclimate.com/github/tac0x2a/yasuri/badges/gpa.svg)](https://codeclimate.com/github/tac0x2a/yasuri)
+
+Yasuri (鑢) is an easy web-scraping library for supporting "[Mechanize](https://github.com/sparklemotion/mechanize)".
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'yasuri'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install yasuri
+
+## Usage
+
+```ruby
+# Node tree constructing by DSL
+root = links_root '//*[@id="menu"]/ul/li/a' do
+         text_title '//*[@id="contents"]/h2'
+         text_content '//*[@id="contents"]/p[1]'
+       end
+
+# Node tree constructing by JSON
+src = <<-EOJSON
+   { "node"     : "links",
+     "name"     : "root",
+     "path"     : "//*[@id='menu']/ul/li/a",
+     "children" : [
+                    { "node" : "text",
+                      "name" : "title",
+                      "path" : "//*[@id='contents']/h2"
+                    },
+                    { "node" : "text",
+                      "name" : "content",
+                      "path" : "//*[@id='contents']/p[1]"
+                    }
+                  ]
+   }
+EOJSON
+root = Yasuri.json2tree(src)
+
+agent = Mechanize.new
+root_page = agent.get("http://some.scraping.page.net/")
+
+result = root.inject(agent, root_page)
+# => [ {"title" => "PageTitle", "content" => "Page Contents" }, ...  ]
+```
+
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/yasuri/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,7 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec
+

data/app.rb

@@ -0,0 +1,52 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+
+# Author::    TAC (tac@tac42.net)
+
+require 'pp'
+require 'time'
+require 'mechanize'
+
+require_relative 'lib/yasuri/yasuri'
+
+agent = Mechanize.new
+
+uri = "http://www.asahi.com/"
+
+# Node tree constructing by DSL
+root = links_top '//*[@id="MainInner"]/div[1]/ul/li/a' do
+  text_title   '//*[@id="MainInner"]/div[1]/div/h1'
+  text_article '//*[@id="MainInner"]/div/div[@class="ArticleText"]'
+end
+
+# Node tree constructing by JSON
+src = <<-EOJSON
+   { "node"     : "links",
+     "name"     : "root",
+     "path"     : "//*[@id='MainInner']/div[1]/ul/li/a",
+     "children" : [
+                    { "node" : "text",
+                      "name" : "title",
+                      "path" : "//*[@id='MainInner']/div[1]/div/h1"
+                    },
+                    { "node" : "text",
+                      "name" : "article",
+                      "path" : "//*[@id='MainInner']/div/div[@class='ArticleText']"
+                    }
+                  ]
+   }
+EOJSON
+root = Yasuri.json2tree(src)
+
+# Access to parsed resources
+page = agent.get(uri)
+contents = root.inject(agent, page)
+
+contents.each do |h|
+  t = h['title']
+  a = h['article']
+
+  puts t
+  puts a
+  puts "=" * 100
+end

data/lib/yasuri/version.rb

@@ -0,0 +1,3 @@
+module Yasuri
+  VERSION = "0.0.2"
+end

data/lib/yasuri/yasuri.rb

@@ -0,0 +1,143 @@
+# -*- coding: utf-8 -*-
+
+# Author::    TAC (tac@tac42.net)
+
+require 'mechanize'
+require 'json'
+
+module Yasuri
+
+  module Node
+    attr_reader :url, :xpath, :name
+
+    def initialize(xpath, name, children = [])
+      @xpath, @name, @children = xpath, name, children
+    end
+
+    def inject(agent, page)
+      fail "#{Kernel.__method__} is not implemented."
+    end
+  end
+
+  class TextNode
+    include Node
+    def inject(agent, page)
+      node = page.search(@xpath)
+      node.text.to_s
+    end
+  end
+
+  class StructNode
+    include Node
+    def inject(agent, page)
+      sub_tags = page.search(@xpath)
+      sub_tags.map do |sub_tag|
+        child_results_kv = @children.map do |child_node|
+          [child_node.name, child_node.inject(agent, sub_tag)]
+        end
+        Hash[child_results_kv]
+      end
+    end
+  end
+
+  class LinksNode
+    include Node
+    def inject(agent, page)
+      links = page.search(@xpath) || [] # links expected
+      links.map do |link|
+        link_button = Mechanize::Page::Link.new(link, agent, page)
+        child_page = link_button.click
+
+        child_results_kv = @children.map do |child_node|
+          [child_node.name, child_node.inject(agent, child_page)]
+        end
+
+        Hash[child_results_kv]
+      end # each named child node
+    end
+  end
+
+  class PaginateNode
+    include Node
+    def inject(agent, page)
+
+      child_results = []
+      while page
+        child_results_kv = @children.map do |child_node|
+          [child_node.name, child_node.inject(agent, page)]
+        end
+        child_results << Hash[child_results_kv]
+
+        link = page.search(@xpath).first
+        break if link == nil
+
+        link_button = Mechanize::Page::Link.new(link, agent, page)
+        page = link_button.click
+      end
+
+      child_results
+    end
+  end
+
+  class NodeGenerator
+    def gen_recursive(&block)
+      @nodes = []
+      instance_eval(&block)
+      @nodes
+    end
+
+    def method_missing(name, *args, &block)
+      node = NodeGenerator.gen(name, *args, &block)
+      raise "Undefined Node Name '#{name}'" if node == nil
+      @nodes << node
+    end
+
+    def self.gen(name, *args, &block)
+      xpath, children = *args
+      children = Yasuri::NodeGenerator.new.gen_recursive(&block) if block_given?
+
+      case name
+      when /^text_(.+)$/
+        Yasuri::TextNode.new(xpath, $1, children || [])
+      when /^struct_(.+)$/
+        Yasuri::StructNode.new(xpath, $1, children || [])
+      when /^links_(.+)$/
+        Yasuri::LinksNode.new(xpath, $1, children || [])
+      when /^pages_(.+)$/
+        Yasuri::PaginateNode.new(xpath, $1, children || [])
+      else
+        nil
+      end
+    end # of self.gen(name, *args, &block)
+  end # of class NodeGenerator
+
+  def self.json2tree(json_string)
+    json = JSON.parse(json_string)
+    Yasuri.hash2node(json)
+  end
+
+  private
+  Text2Node = {
+    "text"   => TextNode,
+    "struct" => StructNode,
+    "links"  => LinksNode,
+    "pages"  => PaginateNode
+  }
+  def self.hash2node(node_h)
+    node, name, path, children = %w|node name path children|.map do |key|
+      node_h[key]
+    end
+    children ||= []
+
+    childnodes = children.map{|c| Yasuri.hash2node(c) }
+
+    klass = Text2Node[node]
+    klass ? klass.new(path, name, childnodes) : nil
+  end
+end
+
+# alias for DSL
+def method_missing(name, *args, &block)
+  generated = Yasuri::NodeGenerator.gen(name, *args, &block)
+  generated || super(name, args)
+end

data/lib/yasuri.rb

@@ -0,0 +1,6 @@
+require "yasuri/version"
+require "yasuri/yasuri"
+
+module Yasuri
+  # Your code goes here...
+end

data/spec/htdocs/child01.html

@@ -0,0 +1,10 @@
+<html>
+  <head><title>Child 01 Test</title></head>
+  <body>
+    <p>Child 01 page.</p>
+    <ul>
+      <li><a href="./child01_sub.html">Child01_Sub</a></li>
+      <li><a href="./child02_sub.html">Child02_Sub</a></li>
+    </ul>
+  </body>
+<title>

data/spec/htdocs/child01_sub.html

@@ -0,0 +1,6 @@
+<html>
+  <head><title>Child 01 SubPage Test</title></head>
+  <body>
+    <p>Child 01 sub page.</p>
+  </body>
+<title>

data/spec/htdocs/child02.html

@@ -0,0 +1,6 @@
+<html>
+  <head><title>Child 02 Test</title></head>
+  <body>
+    <p>Child 02 page.</p>
+  </body>
+<title>

data/spec/htdocs/child02_sub.html

@@ -0,0 +1,6 @@
+<html>
+  <head><title>Child 02 SubPage Test</title></head>
+  <body>
+    <p>Child 02 sub page.</p>
+  </body>
+<title>

data/spec/htdocs/child03.html

@@ -0,0 +1,9 @@
+<html>
+  <head><title>Child 03 Test</title></head>
+  <body>
+    <p>Child 03 page.</p>
+    <ul>
+      <li><a href="./child03_sub.html">Child03_Sub</a></li>
+    </ul>
+  </body>
+<title>

data/spec/htdocs/child03_sub.html

@@ -0,0 +1,6 @@
+<html>
+  <head><title>Child 03 SubPage Test</title></head>
+  <body>
+    <p>Child 03 sub page.</p>
+  </body>
+<title>

data/spec/htdocs/index.html

@@ -0,0 +1,10 @@
+<html>
+  <head><title>Yasuri Test</title></head>
+  <body>
+    <p>Hello,Yasuri</p>
+    <a href="./child01.html">child01</a>
+    <a href="./child02.html">child02</a>
+    <a href="./child03.html">child03</a>
+    <p>Last Modify - 2015/02/14</p>
+  </body>
+<title>

data/spec/htdocs/pagination/page01.html

@@ -0,0 +1,26 @@
+<html>
+  <head><title>Page01Test</title></head>
+  <body>
+    <p>PaginationTest01</p>
+    <nav class='pagination'>
+      <span class='prev'>
+        &laquo; PreviousPage
+      </span>
+      <span class='page'>
+        1
+      </span>
+      <span class='page'>
+        <a href="./page02.html">2</a>
+      </span>
+      <span class='page'>
+        <a href="./page03.html">3</a>
+      </span>
+      <span class='page'>
+        <a href="./page04.html">4</a>
+      </span>
+      <span class='next'>
+        <a href="./page02.html" class="next" rel="next">NextPage &raquo;</a>
+      </span>
+    </nav>
+  </body>
+<title>

data/spec/htdocs/pagination/page02.html

@@ -0,0 +1,26 @@
+<html>
+  <head><title>Page02Test</title></head>
+  <body>
+    <p>PaginationTest02</p>
+    <nav class='pagination'>
+      <span class='prev'>
+        <a href="./page01.html" class="prev" rel="prev">&laquo; PreviousPage</a>
+      </span>
+      <span class='page'>
+        <a href="./page01.html">1</a>
+      </span>
+      <span class='page'>
+        2
+      </span>
+      <span class='page'>
+        <a href="./page03.html">3</a>
+      </span>
+      <span class='page'>
+        <a href="./page04.html">4</a>
+      </span>
+      <span class='next'>
+        <a href="./page03.html" class="next" rel="next">NextPage &raquo;</a>
+      </span>
+    </nav>
+  </body>
+<title>

data/spec/htdocs/pagination/page03.html

@@ -0,0 +1,26 @@
+<html>
+  <head><title>Page03Test</title></head>
+  <body>
+    <p>PaginationTest03</p>
+    <nav class='pagination'>
+      <span class='prev'>
+        <a href="./page02.html" class="prev" rel="prev">&laquo; PreviousPage</a>
+      </span>
+      <span class='page'>
+        <a href="./page01.html">1</a>
+      </span>
+      <span class='page'>
+        <a href="./page02.html">2</a>
+      </span>
+      <span class='page'>
+        3
+      </span>
+      <span class='page'>
+        <a href="./page04.html">4</a>
+      </span>
+      <span class='next'>
+        <a href="./page04.html" class="next" rel="next">NextPage &raquo;</a>
+      </span>
+    </nav>
+  </body>
+<title>

data/spec/htdocs/pagination/page04.html

@@ -0,0 +1,26 @@
+<html>
+  <head><title>Page04Test</title></head>
+  <body>
+    <p>PaginationTest04</p>
+    <nav class='pagination'>
+      <span class='prev'>
+        <a href="./page03.html" class="prev" rel="prev">&laquo; PreviousPage</a>
+      </span>
+      <span class='page'>
+        <a href="./page01.html">1</a>
+      </span>
+      <span class='page'>
+        <a href="./page02.html">2</a>
+      </span>
+      <span class='page'>
+        <a href="./page03.html">3</a>
+      </span>
+      <span class='page'>
+        4
+      </span>
+      <span class='next'>
+        NextPage &raquo;
+      </span>
+    </nav>
+  </body>
+<title>

data/spec/htdocs/structual_text.html

@@ -0,0 +1,77 @@
+<html>
+  <head>
+    <title>StructualTextTest</title>
+  </head>
+  <body>
+
+    <h1>1996</h1>
+    <table>
+      <thead>
+        <tr>
+          <th>Title</th>
+          <th>Publication Date</th>
+        </tr>
+      </thead>
+      <tr>
+        <td>The Perfect Insider</td>
+        <td>1996/4/5</td>
+      </tr>
+      <tr>
+        <td>Doctors in Isolated Room</td>
+        <td>1996/7/5</td>
+      </tr>
+      <tr>
+        <td>Mathematical Goodbye</td>
+        <td>1996/9/5</td>
+      </tr>
+    </table>
+
+    <h1>1997</h1>
+    <table>
+      <thead>
+        <tr>
+          <th>Title</th>
+          <th>Publication Date</th>
+        </tr>
+      </thead>
+      <tr>
+        <td>Jack the Poetical Private</td>
+        <td>1997/1/5</td>
+      </tr>
+      <tr>
+        <td>Who Inside</td>
+        <td>1997/4/5</td>
+      </tr>
+      <tr>
+        <td>Illusion Acts Like Magic</td>
+        <td>1997/10/5</td>
+      </tr>
+    </table>
+
+    <h1>1998</h1>
+    <table>
+      <thead>
+        <tr>
+          <th>Title</th>
+          <th>Publication Date</th>
+        </tr>
+      </thead>
+      <tr>
+        <td>Replaceable Summer</td>
+        <td>1998/1/7</td>
+      </tr>
+      <tr>
+        <td>Switch Back</td>
+        <td>1998/4/5</td>
+      </tr>
+      <tr>
+        <td>Numerical Models</td>
+        <td>1998/7/5</td>
+      </tr>
+      <tr>
+        <td>The Perfect Outsider</td>
+        <td>1998/10/5</td>
+      </tr>
+    </table>
+  </body>
+</html>

data/spec/servers/httpserver.rb

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+# Author::    TAC (tac@tac42.net)
+
+require 'glint'
+
+server = Glint::Server.new do |port|
+
+  require 'webrick'
+  http = WEBrick::HTTPServer.new({
+    DocumentRoot: 'spec/htdocs',
+    BindAddress:  '127.0.0.1',
+    Port:         port,
+    AccessLog: []
+  })
+
+  trap(:INT)  { http.shutdown }
+  trap(:TERM) { http.shutdown }
+  http.start
+end
+
+server.start
+
+Glint::Server.info[:httpserver] = {
+  host: "127.0.0.1",
+  port: server.port
+}

data/spec/spec_helper.rb

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+# Author::    TAC (tac@tac42.net)
+
+require 'glint'
+Dir[File.expand_path("../servers/*.rb", __FILE__)].each {|f| require f}
+
+require 'rspec'
+shared_context 'httpserver' do
+  require 'net/http'
+  let(:uri) {
+    "http://#{Glint::Server.info[:httpserver][:host]}:#{Glint::Server.info[:httpserver][:port]}"
+  }
+end
+
+
+# ENV['CODECLIMATE_REPO_TOKEN'] = "0dc78d33107a7f11f257c0218ac1a37e0073005bb9734f2fd61d0f7e803fc151"
+require "codeclimate-test-reporter"
+CodeClimate::TestReporter.start
+
+require 'simplecov'
+require 'coveralls'
+Coveralls.wear!
+
+SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter[
+  SimpleCov::Formatter::HTMLFormatter,
+  Coveralls::SimpleCov::Formatter
+]
+SimpleCov.start
+
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+require 'yasuri'

data/spec/yasuri_spec.rb

@@ -0,0 +1,299 @@
+# -*- coding: utf-8 -*-
+
+# Author::    TAC (tac@tac42.net)
+
+require_relative 'spec_helper'
+
+#require_relative '../lib/yasuri/yasuri'
+
+describe 'Yasuri' do
+  include_context 'httpserver'
+
+  before do
+    @agent = Mechanize.new
+    @uri = uri
+    @index_page = @agent.get(@uri)
+  end
+
+  ########
+  # Node #
+  ########
+  def compare_generated_vs_original(generated, original, page = @index_page)
+    expected = original.inject(@agent, page)
+    actual   = generated.inject(@agent, page)
+    expect(actual).to match expected
+  end
+
+  describe '::TextNode' do
+    before { @node = Yasuri::TextNode.new('/html/body/p[1]', "title") }
+
+    it 'scrape text text <p>Hello,Yasuri</p>' do
+      actual = @node.inject(@agent, @index_page)
+      expect(actual).to eq "Hello,Yasuri"
+    end
+
+    it "can be defined by DSL, return single TextNode title" do
+      generated = text_title '/html/body/p[1]'
+      original  = Yasuri::TextNode.new('/html/body/p[1]', "title")
+      compare_generated_vs_original(generated, original)
+    end
+  end
+
+  describe '::StructNode' do
+    before do
+      @page = @agent.get(@uri + "/structual_text.html")
+      @table_1996 = [
+        { "title"    => "The Perfect Insider",
+          "pub_date" => "1996/4/5" },
+        { "title"    => "Doctors in Isolated Room",
+          "pub_date" => "1996/7/5" },
+        { "title"    => "Mathematical Goodbye",
+          "pub_date" => "1996/9/5" },
+      ]
+      @table_1997 = [
+        { "title"    => "Jack the Poetical Private",
+          "pub_date" => "1997/1/5" },
+        { "title"    => "Who Inside",
+          "pub_date" => "1997/4/5" },
+        { "title"    => "Illusion Acts Like Magic",
+          "pub_date" => "1997/10/5" },
+      ]
+      @table_1998 = [
+        { "title"    => "Replaceable Summer",
+          "pub_date" => "1998/1/7" },
+        { "title"    => "Switch Back",
+          "pub_date" => "1998/4/5" },
+        { "title"    => "Numerical Models",
+          "pub_date" => "1998/7/5" },
+        { "title"    => "The Perfect Outsider",
+          "pub_date" => "1998/10/5" },
+      ]
+      @all_tables = [
+        {"table" => @table_1996},
+        {"table" => @table_1997},
+        {"table" => @table_1998},
+      ]
+    end
+    it 'scrape single table contents' do
+      node = Yasuri::StructNode.new('/html/body/table[1]/tr', "table", [
+        Yasuri::TextNode.new('./td[1]', "title"),
+        Yasuri::TextNode.new('./td[2]', "pub_date"),
+      ])
+      expected = @table_1996
+      actual = node.inject(@agent, @page)
+      expect(actual).to match expected
+    end
+
+    it 'scrape all tables' do
+      node = Yasuri::StructNode.new('/html/body/table', "tables", [
+        Yasuri::StructNode.new('./tr', "table", [
+          Yasuri::TextNode.new('./td[1]', "title"),
+          Yasuri::TextNode.new('./td[2]', "pub_date"),
+        ])
+      ])
+      expected = @all_tables
+      actual = node.inject(@agent, @page)
+      expect(actual).to match expected
+    end
+
+    it 'can be defined by DSL, scrape all tables' do
+      generated = struct_tables '/html/body/table' do
+        struct_table './tr' do
+          text_title    './td[1]'
+          text_pub_date './td[2]'
+        end
+      end
+      original = Yasuri::StructNode.new('/html/body/table', "tables", [
+        Yasuri::StructNode.new('./tr', "table", [
+          Yasuri::TextNode.new('./td[1]', "title"),
+          Yasuri::TextNode.new('./td[2]', "pub_date"),
+        ])
+      ])
+      compare_generated_vs_original(generated, original)
+    end
+  end
+
+  describe '::LinksNode' do
+    it 'scrape links' do
+      root_node = Yasuri::LinksNode.new('/html/body/a', "root", [
+        Yasuri::TextNode.new('/html/body/p', "content"),
+      ])
+
+      actual = root_node.inject(@agent, @index_page)
+      expected = [
+        {"content" => "Child 01 page."},
+        {"content" => "Child 02 page."},
+        {"content" => "Child 03 page."},
+      ]
+      expect(actual).to match expected
+    end
+
+    it 'scrape links, recursive' do
+      root_node = Yasuri::LinksNode.new('/html/body/a', "root", [
+        Yasuri::TextNode.new('/html/body/p', "content"),
+        Yasuri::LinksNode.new('/html/body/ul/li/a', "sub_link", [
+          Yasuri::TextNode.new('/html/head/title', "sub_page_title"),
+        ]),
+      ])
+      actual = root_node.inject(@agent, @index_page)
+      expected = [
+        {"content"  => "Child 01 page.",
+         "sub_link" => [{"sub_page_title" => "Child 01 SubPage Test"},
+                        {"sub_page_title" => "Child 02 SubPage Test"}],},
+        {"content" => "Child 02 page.",
+         "sub_link" => [],},
+        {"content" => "Child 03 page.",
+         "sub_link" => [{"sub_page_title" => "Child 03 SubPage Test"}],},
+      ]
+      expect(actual).to match expected
+    end
+    it 'can be defined by DSL, return single LinkNode title' do
+      generated = links_title     '/html/body/a'
+      original  = Yasuri::LinksNode.new('/html/body/a', "title")
+      compare_generated_vs_original(generated, original)
+    end
+    it 'can be defined by DSL, return nested contents under link' do
+      generated = links_title '/html/body/a' do
+                     text_name '/html/body/p'
+                  end
+      original = Yasuri::LinksNode.new('/html/body/a', "root", [
+        Yasuri::TextNode.new('/html/body/p', "name"),
+      ])
+      compare_generated_vs_original(generated, original)
+    end
+
+    it 'can be defined by DSL, return recursive links node' do
+      generated = links_root '/html/body/a' do
+        text_content '/html/body/p'
+        links_sub_link '/html/body/ul/li/a' do
+          text_sub_page_title '/html/head/title'
+        end
+      end
+
+      original = Yasuri::LinksNode.new('/html/body/a', "root", [
+        Yasuri::TextNode.new('/html/body/p', "content"),
+        Yasuri::LinksNode.new('/html/body/ul/li/a', "sub_link", [
+          Yasuri::TextNode.new('/html/head/title', "sub_page_title"),
+        ]),
+      ])
+      compare_generated_vs_original(generated, original)
+    end
+  end
+
+  describe '::PaginateNode' do
+    before do
+      @uri += "/pagination/page01.html"
+      @page = @agent.get(@uri)
+    end
+
+    it "scrape each paginated pages" do
+      root_node = Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "root", [
+        Yasuri::TextNode.new('/html/body/p', "content"),
+      ])
+      actual = root_node.inject(@agent, @page)
+      expected = [
+        {"content" => "PaginationTest01"},
+        {"content" => "PaginationTest02"},
+        {"content" => "PaginationTest03"},
+        {"content" => "PaginationTest04"},
+      ]
+      expect(actual).to match expected
+    end
+
+    it 'can be defined by DSL, return single PaginateNode content' do
+      generated = pages_next "/html/body/nav/span/a[@class='next']" do
+        text_content '/html/body/p'
+      end
+      original = Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "root", [
+        Yasuri::TextNode.new('/html/body/p', "content"),
+      ])
+    compare_generated_vs_original(generated, original)
+    end
+  end
+
+  describe '.json2tree' do
+    it "return empty tree" do
+      tree = Yasuri.json2tree("{}")
+      expect(tree).to be_nil
+    end
+
+    it "return TextNode" do
+      src = %q| { "node"  : "text",
+                  "name"  : "content",
+                  "path"  : "/html/body/p[1]"
+                }|
+      generated = Yasuri.json2tree(src)
+      original  = Yasuri::TextNode.new('/html/body/p[1]', "content")
+      compare_generated_vs_original(generated, original)
+    end
+
+    it "return LinksNode/TextNode" do
+      src = %q| { "node"     : "links",
+                  "name"     : "root",
+                  "path"     : "/html/body/a",
+                  "children" : [ { "node" : "text",
+                                   "name" : "content",
+                                   "path" : "/html/body/p"
+                                 } ]
+                }|
+      generated = Yasuri.json2tree(src)
+      original  = Yasuri::LinksNode.new('/html/body/a', "root", [
+                    Yasuri::TextNode.new('/html/body/p', "content"),
+                  ])
+      compare_generated_vs_original(generated, original)
+    end
+
+    it "return PaginateNode/TextNode" do
+      src = %q|{ "node"     : "pages",
+                 "name"     : "root",
+                 "path"     : "/html/body/nav/span/a[@class=\'next\']",
+                 "children" : [ { "node" : "text",
+                                  "name" : "content",
+                                  "path" : "/html/body/p"
+                                } ]
+               }|
+      generated = Yasuri.json2tree(src)
+      original = Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "root", [
+                   Yasuri::TextNode.new('/html/body/p', "content"),
+                 ])
+
+      paginate_test_uri  = @uri + "/pagination/page01.html"
+      paginate_test_page = @agent.get(paginate_test_uri)
+      compare_generated_vs_original(generated, original, paginate_test_page)
+    end
+
+    it "return StructNode/StructNode/[TextNode,TextNode]" do
+     src = %q| { "node"     : "struct",
+                 "name"     : "tables",
+                 "path"     : "/html/body/table",
+                 "children" : [
+                   { "node"       : "struct",
+                     "name"       : "table",
+                     "path"       : "./tr",
+                     "children"   : [
+                       { "node" : "text",
+                         "name" : "title",
+                         "path" : "./td[1]"
+                       },
+                       { "node" : "text",
+                         "name" : "pub_date",
+                         "path" : "./td[2]"
+                       }]
+                   }]
+               }|
+      generated = Yasuri.json2tree(src)
+      original  = Yasuri::StructNode.new('/html/body/table', "tables", [
+        Yasuri::StructNode.new('./tr', "table", [
+          Yasuri::TextNode.new('./td[1]', "title"),
+          Yasuri::TextNode.new('./td[2]', "pub_date"),
+        ])
+      ])
+      page = @agent.get(@uri + "/structual_text.html")
+      compare_generated_vs_original(generated, original, page)
+    end
+  end
+
+  it 'has a version number' do
+    expect(Yasuri::VERSION).not_to be nil
+  end
+end

data/yasuri.gemspec

@@ -0,0 +1,31 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'yasuri/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "yasuri"
+  spec.version       = Yasuri::VERSION
+  spec.authors       = ["TAC"]
+  spec.email         = ["tac@tac42.net"]
+  spec.summary       = %q{Yasuri is easy scraping library.}
+  spec.description   = %q{Yasuri is an easy web-scraping library for supporting "Mechanize".}
+  spec.homepage      = "https://github.com/tac0x2a/yasuri"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "fuubar"
+  spec.add_development_dependency "glint"
+  spec.add_development_dependency "coveralls"
+  spec.add_development_dependency "simplecov"
+  spec.add_development_dependency "codeclimate-test-reporter"
+
+  spec.add_dependency "mechanize"
+end

metadata

@@ -0,0 +1,213 @@
+--- !ruby/object:Gem::Specification
+name: yasuri
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: ruby
+authors:
+- TAC
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-02-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fuubar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: glint
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: coveralls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: codeclimate-test-reporter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mechanize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Yasuri is an easy web-scraping library for supporting "Mechanize".
+email:
+- tac@tac42.net
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".coveralls.yml"
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- app.rb
+- lib/yasuri.rb
+- lib/yasuri/version.rb
+- lib/yasuri/yasuri.rb
+- spec/htdocs/child01.html
+- spec/htdocs/child01_sub.html
+- spec/htdocs/child02.html
+- spec/htdocs/child02_sub.html
+- spec/htdocs/child03.html
+- spec/htdocs/child03_sub.html
+- spec/htdocs/index.html
+- spec/htdocs/pagination/page01.html
+- spec/htdocs/pagination/page02.html
+- spec/htdocs/pagination/page03.html
+- spec/htdocs/pagination/page04.html
+- spec/htdocs/structual_text.html
+- spec/servers/httpserver.rb
+- spec/spec_helper.rb
+- spec/yasuri_spec.rb
+- yasuri.gemspec
+homepage: https://github.com/tac0x2a/yasuri
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Yasuri is easy scraping library.
+test_files:
+- spec/htdocs/child01.html
+- spec/htdocs/child01_sub.html
+- spec/htdocs/child02.html
+- spec/htdocs/child02_sub.html
+- spec/htdocs/child03.html
+- spec/htdocs/child03_sub.html
+- spec/htdocs/index.html
+- spec/htdocs/pagination/page01.html
+- spec/htdocs/pagination/page02.html
+- spec/htdocs/pagination/page03.html
+- spec/htdocs/pagination/page04.html
+- spec/htdocs/structual_text.html
+- spec/servers/httpserver.rb
+- spec/spec_helper.rb
+- spec/yasuri_spec.rb