yasuri 0.0.6 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 61a2aa3974c697ecc14b991521961ec54f3ff5c0
-  data.tar.gz: 03cf5b02e7a646175183725d38d50a9538df7eed
+  metadata.gz: 18c8b6da6ca1f9d5433adc128b83ed5d5a8e353e
+  data.tar.gz: 07ba467f8d62982e4a8e969da42b839d8ee07664
 SHA512:
-  metadata.gz: 2108d6b78c8704fa4d99491c1684dfd686c656d7bd039b8c80e52fa2150837958c25f37afabdcf198b10153b2f32216408735182fb0d58b3007f4808ac2226c9
-  data.tar.gz: c956e589ab7676e844110870e18c8f842afb6fd566a5ddd4758ec9e328961ff1b485bac9eebecdac6456f7ed28a9793d9dd4dd9bc2f89dd8f9131003c6db9a5d
+  metadata.gz: 232a5893c4511b0ef80b34a95d58af4d1bb8683512a1cb41b7f9a6d19def75d3c825b12c383bf51bab09e12fe8bd54cd91b79b32640319b60d6185a46ed7f086
+  data.tar.gz: 4cb31e01b60861d13770b8b9d033cef35fe6bbdd226d11703826040840ac979fcfd98c162b18322baa8f9102fd2ed059ac0c15c277c49217feaf39173800dea6

data/README.md

@@ -2,6 +2,13 @@
 
 Yasuri (鑢) is an easy web-scraping library for supporting "[Mechanize](https://github.com/sparklemotion/mechanize)".
 
+
+## Sample
+
+https://yasuri-sample.herokuapp.com/
+
+(source code: https://github.com/tac0x2a/yasuri-sample)
+
 ## Installation
 
 Add this line to your application's Gemfile:

data/lib/yasuri/version.rb

@@ -1,3 +1,3 @@
 module Yasuri
-  VERSION = "0.0.6"
+  VERSION = "0.0.7"
 end

data/lib/yasuri/yasuri.rb

@@ -5,148 +5,38 @@
 require 'mechanize'
 require 'json'
 
-module Yasuri
-
-  module Node
-    attr_reader :url, :xpath, :name
-
-    def initialize(xpath, name, children = [], opt: {})
-      @xpath, @name, @children = xpath, name, children
-    end
-
-    def inject(agent, page)
-      fail "#{Kernel.__method__} is not implemented."
-    end
-  end
-
-  class TextNode
-    include Node
-    def initialize(xpath, name, children = [], truncate_regexp: nil, opt: {})
-      super(xpath, name, children)
-      @truncate_regexp = truncate_regexp
-    end
-    def inject(agent, page, retry_count = 5)
-      node = page.search(@xpath)
-      text = node.text.to_s
-
-      text = text[@truncate_regexp, 0] if @truncate_regexp
-
-      text.to_s
-    end
-  end
-
-  class StructNode
-    include Node
-    def inject(agent, page, retry_count = 5)
-      sub_tags = page.search(@xpath)
-      sub_tags.map do |sub_tag|
-        child_results_kv = @children.map do |child_node|
-          [child_node.name, child_node.inject(agent, sub_tag, retry_count)]
-        end
-        Hash[child_results_kv]
-      end
-    end
-  end
-
-  class LinksNode
-    include Node
-    def inject(agent, page, retry_count = 5)
-      links = page.search(@xpath) || [] # links expected
-      links.map do |link|
-        link_button = Mechanize::Page::Link.new(link, agent, page)
-        child_page = Yasuri.with_retry(retry_count) { link_button.click }
-
-        child_results_kv = @children.map do |child_node|
-          [child_node.name, child_node.inject(agent, child_page, retry_count)]
-        end
-
-        Hash[child_results_kv]
-      end # each named child node
-    end
-  end
-
-  class PaginateNode
-    include Node
-
-    def initialize(xpath, name, children = [], limit: nil, opt: {})
-      super(xpath, name, children)
-      @limit = limit || opt["limit"] || Float::MAX
-    end
-
-    def inject(agent, page, retry_count = 5)
+require_relative 'yasuri_node'
+require_relative 'yasuri_text_node'
+require_relative 'yasuri_struct_node'
+require_relative 'yasuri_paginate_node'
+require_relative 'yasuri_links_node'
+require_relative 'yasuri_node_generator'
 
-      child_results = []
-      while page
-        child_results_kv = @children.map do |child_node|
-          [child_node.name, child_node.inject(agent, page, retry_count)]
-        end
-        child_results << Hash[child_results_kv]
-
-        link = page.search(@xpath).first
-        break if link == nil
-
-        link_button = Mechanize::Page::Link.new(link, agent, page)
-        page = Yasuri.with_retry(retry_count) { link_button.click }
-        break if (@limit -= 1) <= 0
-      end
-
-      child_results
-    end
-  end
-
-  class NodeGenerator
-    def gen_recursive(&block)
-      @nodes = []
-      instance_eval(&block)
-      @nodes
-    end
-
-    def method_missing(name, *args, &block)
-      node = NodeGenerator.gen(name, *args, &block)
-      raise "Undefined Node Name '#{name}'" if node == nil
-      @nodes << node
-    end
-
-    def self.gen(name, *args, &block)
-      xpath, opt = *args
-      children = Yasuri::NodeGenerator.new.gen_recursive(&block) if block_given?
-
-      case name
-      when /^text_(.+)$/
-        truncate_regexp = opt
-        Yasuri::TextNode.new(xpath, $1, truncate_regexp)
-      when /^struct_(.+)$/
-        Yasuri::StructNode.new(xpath, $1, children || [])
-      when /^links_(.+)$/
-        Yasuri::LinksNode.new(xpath, $1, children || [])
-      when /^pages_(.+)$/
-        xpath, limit = *args
-        limit = limit || Float::MAX
-        Yasuri::PaginateNode.new(xpath, $1, children || [], limit: limit)
-      else
-        nil
-      end
-    end # of self.gen(name, *args, &block)
-  end # of class NodeGenerator
+module Yasuri
 
   def self.json2tree(json_string)
     json = JSON.parse(json_string)
     Yasuri.hash2node(json)
   end
 
+  def self.tree2json(node)
+    Yasuri.node2hash(node).to_json
+  end
+
   def self.method_missing(name, *args, &block)
     generated = Yasuri::NodeGenerator.gen(name, *args, &block)
     generated || super(name, args)
   end
 
-
   private
   Text2Node = {
-    "text"   => TextNode,
-    "struct" => StructNode,
-    "links"  => LinksNode,
-    "pages"  => PaginateNode
+    "text"   => Yasuri::TextNode,
+    "struct" => Yasuri::StructNode,
+    "links"  => Yasuri::LinksNode,
+    "pages"  => Yasuri::PaginateNode
   }
+  Node2Text = Text2Node.invert
+
   ReservedKeys = %w|node name path children|
   def self.hash2node(node_h)
     node, name, path, children = ReservedKeys.map do |key|
@@ -162,6 +52,27 @@ module Yasuri
     klass ? klass.new(path, name, childnodes, opt: opt) : nil
   end
 
+  def self.node2hash(node)
+    json = JSON.parse("{}")
+    return json if node.nil?
+
+    klass = node.class
+    klass_str = Node2Text[klass]
+
+    json["node"] = klass_str
+    json["name"] = node.name
+    json["path"] = node.xpath
+
+    children = node.children.map{|c| Yasuri.node2hash(c)}
+    json["children"] = children if not children.empty?
+
+    node.opts.each do |key,value|
+      json[key] = value if not value.nil?
+    end
+
+    json
+  end
+
   def self.with_retry(retry_count = 5)
     begin
       return yield() if block_given?

data/lib/yasuri/yasuri_links_node.rb

@@ -0,0 +1,23 @@
+
+# Author::    TAC (tac@tac42.net)
+
+require_relative 'yasuri_node'
+
+module Yasuri
+  class LinksNode
+    include Node
+    def inject(agent, page, retry_count = 5)
+      links = page.search(@xpath) || [] # links expected
+      links.map do |link|
+        link_button = Mechanize::Page::Link.new(link, agent, page)
+        child_page = Yasuri.with_retry(retry_count) { link_button.click }
+
+        child_results_kv = @children.map do |child_node|
+          [child_node.name, child_node.inject(agent, child_page, retry_count)]
+        end
+
+        Hash[child_results_kv]
+      end # each named child node
+    end
+  end
+end

data/lib/yasuri/yasuri_node.rb

@@ -0,0 +1,21 @@
+
+# Author::    TAC (tac@tac42.net)
+
+require_relative 'yasuri_node'
+
+module Yasuri
+  module Node
+    attr_reader :url, :xpath, :name, :children
+
+    def initialize(xpath, name, children = [], opt: {})
+      @xpath, @name, @children = xpath, name, children
+    end
+
+    def inject(agent, page)
+      fail "#{Kernel.__method__} is not implemented."
+    end
+    def opts
+      {}
+    end
+  end
+end

data/lib/yasuri/yasuri_node_generator.rb

@@ -0,0 +1,46 @@
+
+# Author::    TAC (tac@tac42.net)
+
+require_relative 'yasuri_node'
+require_relative 'yasuri_text_node'
+require_relative 'yasuri_struct_node'
+require_relative 'yasuri_links_node'
+require_relative 'yasuri_paginate_node'
+
+module Yasuri
+  class NodeGenerator
+    def gen_recursive(&block)
+      @nodes = []
+      instance_eval(&block)
+      @nodes
+    end
+
+    def method_missing(name, *args, &block)
+      node = NodeGenerator.gen(name, *args, &block)
+      raise "Undefined Node Name '#{name}'" if node == nil
+      @nodes << node
+    end
+
+    def self.gen(name, *args, &block)
+      xpath, opt = *args
+      opt = [opt].flatten.compact
+      children = Yasuri::NodeGenerator.new.gen_recursive(&block) if block_given?
+
+      case name
+      when /^text_(.+)$/
+        truncate, dummy = *opt
+        Yasuri::TextNode.new(xpath,   $1, children || [], truncate: truncate)
+      when /^struct_(.+)$/
+        Yasuri::StructNode.new(xpath, $1, children || [])
+      when /^links_(.+)$/
+        Yasuri::LinksNode.new(xpath,  $1, children || [])
+      when /^pages_(.+)$/
+        limit, dummy = *opt
+        limit = limit || Float::MAX
+        Yasuri::PaginateNode.new(xpath, $1, children || [], limit: limit)
+      else
+        nil
+      end
+    end # of self.gen(name, *args, &block)
+  end # of class NodeGenerator
+end

data/lib/yasuri/yasuri_paginate_node.rb

@@ -0,0 +1,39 @@
+
+# Author::    TAC (tac@tac42.net)
+
+require_relative 'yasuri_node'
+
+module Yasuri
+  class PaginateNode
+    include Node
+
+    def initialize(xpath, name, children = [], limit: nil, opt: {})
+      super(xpath, name, children)
+      @limit = limit || opt["limit"]
+    end
+
+    def inject(agent, page, retry_count = 5)
+
+      child_results = []
+      limit = @limit.nil? ? Float::MAX : @limit
+      while page
+        child_results_kv = @children.map do |child_node|
+          [child_node.name, child_node.inject(agent, page, retry_count)]
+        end
+        child_results << Hash[child_results_kv]
+
+        link = page.search(@xpath).first
+        break if link == nil
+
+        link_button = Mechanize::Page::Link.new(link, agent, page)
+        page = Yasuri.with_retry(retry_count) { link_button.click }
+        break if (limit -= 1) <= 0
+      end
+
+      child_results
+    end
+    def opts
+      {limit:@limit}
+    end
+  end
+end

data/lib/yasuri/yasuri_struct_node.rb

@@ -0,0 +1,19 @@
+
+# Author::    TAC (tac@tac42.net)
+
+require_relative 'yasuri_node'
+
+module Yasuri
+  class StructNode
+    include Node
+    def inject(agent, page, retry_count = 5)
+      sub_tags = page.search(@xpath)
+      sub_tags.map do |sub_tag|
+        child_results_kv = @children.map do |child_node|
+          [child_node.name, child_node.inject(agent, sub_tag, retry_count)]
+        end
+        Hash[child_results_kv]
+      end
+    end
+  end
+end

data/lib/yasuri/yasuri_text_node.rb

@@ -0,0 +1,32 @@
+
+# Author::    TAC (tac@tac42.net)
+
+require_relative 'yasuri_node'
+
+module Yasuri
+  class TextNode
+    include Node
+    def initialize(xpath, name, children = [], truncate: nil, opt: {})
+      super(xpath, name, children)
+
+      truncate_opt = opt["truncate"] #str
+      truncate_opt = Regexp.new(truncate_opt) if not truncate_opt.nil? # regexp or nil
+
+      @truncate = truncate || truncate_opt || nil # regexp or nil
+
+      @truncate = Regexp.new(@truncate.to_s) if not @truncate.nil?
+
+    end
+    def inject(agent, page, retry_count = 5)
+      node = page.search(@xpath)
+      text = node.text.to_s
+
+      text = text[@truncate, 0] if @truncate
+
+      text.to_s
+    end
+    def opts
+      {truncate:@truncate}
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -29,3 +29,9 @@ SimpleCov.start
 
 $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
 require 'yasuri'
+
+def compare_generated_vs_original(generated, original, page)
+  expected = original.inject(@agent, page)
+  actual   = generated.inject(@agent, page)
+  expect(actual).to match expected
+end

data/spec/yasuri_links_node_spec.rb

@@ -0,0 +1,94 @@
+
+# Author::    TAC (tac@tac42.net)
+
+require_relative 'spec_helper'
+
+#########
+# Links #
+#########
+describe 'Yasuri' do
+  include_context 'httpserver'
+
+  describe '::LinksNode' do
+    before do
+      @agent = Mechanize.new
+      @uri = uri
+      @index_page = @agent.get(@uri)
+    end
+
+    it 'scrape links' do
+      root_node = Yasuri::LinksNode.new('/html/body/a', "root", [
+        Yasuri::TextNode.new('/html/body/p', "content"),
+      ])
+
+      actual = root_node.inject(@agent, @index_page)
+      expected = [
+        {"content" => "Child 01 page."},
+        {"content" => "Child 02 page."},
+        {"content" => "Child 03 page."},
+      ]
+      expect(actual).to match expected
+    end
+
+    it 'return empty set if no match node' do
+      missing_xpath = '/html/body/b'
+      root_node = Yasuri::LinksNode.new(missing_xpath, "root", [
+        Yasuri::TextNode.new('/html/body/p', "content"),
+      ])
+
+      actual = root_node.inject(@agent, @index_page)
+      expect(actual).to be_empty
+    end
+
+    it 'scrape links, recursive' do
+      root_node = Yasuri::LinksNode.new('/html/body/a', "root", [
+        Yasuri::TextNode.new('/html/body/p', "content"),
+        Yasuri::LinksNode.new('/html/body/ul/li/a', "sub_link", [
+          Yasuri::TextNode.new('/html/head/title', "sub_page_title"),
+        ]),
+      ])
+      actual = root_node.inject(@agent, @index_page)
+      expected = [
+        {"content"  => "Child 01 page.",
+         "sub_link" => [{"sub_page_title" => "Child 01 SubPage Test"},
+                        {"sub_page_title" => "Child 02 SubPage Test"}],},
+        {"content" => "Child 02 page.",
+         "sub_link" => [],},
+        {"content" => "Child 03 page.",
+         "sub_link" => [{"sub_page_title" => "Child 03 SubPage Test"}],},
+      ]
+      expect(actual).to match expected
+    end
+    it 'can be defined by DSL, return single LinkNode title' do
+      generated = Yasuri.links_title '/html/body/a'
+      original  = Yasuri::LinksNode.new('/html/body/a', "title")
+      compare_generated_vs_original(generated, original, @index_page)
+    end
+    it 'can be defined by DSL, return nested contents under link' do
+      generated = Yasuri.links_title '/html/body/a' do
+                     text_name '/html/body/p'
+                  end
+      original = Yasuri::LinksNode.new('/html/body/a', "root", [
+        Yasuri::TextNode.new('/html/body/p', "name"),
+      ])
+      compare_generated_vs_original(generated, original, @index_page)
+    end
+
+    it 'can be defined by DSL, return recursive links node' do
+      generated = Yasuri.links_root '/html/body/a' do
+        text_content '/html/body/p'
+        links_sub_link '/html/body/ul/li/a' do
+          text_sub_page_title '/html/head/title'
+        end
+      end
+
+      original = Yasuri::LinksNode.new('/html/body/a', "root", [
+        Yasuri::TextNode.new('/html/body/p', "content"),
+        Yasuri::LinksNode.new('/html/body/ul/li/a', "sub_link", [
+          Yasuri::TextNode.new('/html/head/title', "sub_page_title"),
+        ]),
+      ])
+      compare_generated_vs_original(generated, original, @index_page)
+    end
+  end
+end

data/spec/yasuri_node_spec.rb

@@ -0,0 +1,11 @@
+
+# Author::    TAC (tac@tac42.net)
+
+require_relative 'spec_helper'
+
+########
+# Node #
+########
+describe 'Yasuri' do
+
+end

data/spec/yasuri_paginate_node_spec.rb

@@ -0,0 +1,85 @@
+
+# Author::    TAC (tac@tac42.net)
+
+require_relative 'spec_helper'
+
+############
+# Paginate #
+############
+describe 'Yasuri' do
+  include_context 'httpserver'
+
+  describe '::PaginateNode' do
+    before do
+      @agent = Mechanize.new
+      @uri = uri + "/pagination/page01.html"
+      @page = @agent.get(@uri)
+    end
+
+    it "scrape each paginated pages" do
+      root_node = Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "root", [
+        Yasuri::TextNode.new('/html/body/p', "content"),
+      ])
+      actual = root_node.inject(@agent, @page)
+      expected = [
+        {"content" => "PaginationTest01"},
+        {"content" => "PaginationTest02"},
+        {"content" => "PaginationTest03"},
+        {"content" => "PaginationTest04"},
+      ]
+      expect(actual).to match expected
+    end
+
+    it "scrape each paginated pages limited" do
+      root_node = Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "root", [
+        Yasuri::TextNode.new('/html/body/p', "content"),
+      ], limit:3)
+      actual = root_node.inject(@agent, @page)
+      expected = [
+        {"content" => "PaginationTest01"},
+        {"content" => "PaginationTest02"},
+        {"content" => "PaginationTest03"},
+      ]
+      expect(actual).to match expected
+    end
+
+    it 'return first content if paginate link node is not found' do
+      missing_xpath = "/html/body/nav/span/b[@class='next']"
+      root_node = Yasuri::PaginateNode.new(missing_xpath, "root", [
+        Yasuri::TextNode.new('/html/body/p', "content"),
+      ])
+      actual = root_node.inject(@agent, @page)
+      expected = [ {"content" => "PaginationTest01"}, ]
+      expect(actual).to match_array expected
+    end
+
+    it 'return empty hashes if content node is not found' do
+      root_node = Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "root", [
+        Yasuri::TextNode.new('/html/body/hoge', "content"),
+      ])
+      actual = root_node.inject(@agent, @page)
+      expected = [ {"content" => ""}, {"content" => ""}, {"content" => ""}, {"content" => ""},]
+      expect(actual).to match_array expected
+    end
+
+    it 'can be defined by DSL, return single PaginateNode content' do
+      generated = Yasuri.pages_next "/html/body/nav/span/a[@class='next']" do
+        text_content '/html/body/p'
+      end
+      original = Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "root", [
+        Yasuri::TextNode.new('/html/body/p', "content"),
+      ])
+      compare_generated_vs_original(generated, original, @page)
+    end
+
+    it 'can be defined by DSL, return single PaginateNode content limited' do
+      generated = Yasuri.pages_next "/html/body/nav/span/a[@class='next']", 2 do
+        text_content '/html/body/p'
+      end
+      original = Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "root", [
+        Yasuri::TextNode.new('/html/body/p', "content"),
+      ], limit: 2)
+      compare_generated_vs_original(generated, original, @page)
+    end
+  end
+end

data/spec/yasuri_spec.rb

@@ -3,6 +3,7 @@
 # Author::    TAC (tac@tac42.net)
 
 require_relative 'spec_helper'
+require_relative 'yasuri_text_node_spec'
 
 #require_relative '../lib/yasuri/yasuri'
 
@@ -15,323 +16,6 @@ describe 'Yasuri' do
     @index_page = @agent.get(@uri)
   end
 
-  ########
-  # Node #
-  ########
-  def compare_generated_vs_original(generated, original, page = @index_page)
-    expected = original.inject(@agent, page)
-    actual   = generated.inject(@agent, page)
-    expect(actual).to match expected
-  end
-
-  ########
-  # Text #
-  ########
-  describe '::TextNode' do
-    before { @node = Yasuri::TextNode.new('/html/body/p[1]', "title") }
-
-    it 'scrape text text <p>Hello,Yasuri</p>' do
-      actual = @node.inject(@agent, @index_page)
-      expect(actual).to eq "Hello,Yasuri"
-    end
-
-    it 'return empty text if no match node' do
-      no_match_node = Yasuri::TextNode.new('/html/body/no_match_node', "title")
-      actual = no_match_node.inject(@agent, @index_page)
-      expect(actual).to be_empty
-    end
-
-    it 'fail with invalid xpath' do
-      invalid_xpath = '/html/body/no_match_node['
-      node = Yasuri::TextNode.new(invalid_xpath, "title")
-      expect { node.inject(@agent, @index_page) }.to raise_error
-    end
-
-    it "can be defined by DSL, return single TextNode title" do
-      generated = Yasuri.text_title '/html/body/p[1]'
-      original  = Yasuri::TextNode.new('/html/body/p[1]', "title")
-      compare_generated_vs_original(generated, original)
-    end
-
-    it "can be truncated with regexp" do
-      node  = Yasuri.text_title '/html/body/p[1]', truncate_regexp:/^[^,]+/
-      actual = node.inject(@agent, @index_page)
-      expect(actual).to eq "Hello"
-    end
-
-    it "can be truncated with regexp" do
-      node = Yasuri.text_title '/html/body/p[1]', truncate_regexp:/[^,]+$/
-      actual = node.inject(@agent, @index_page)
-      expect(actual).to eq "Yasuri"
-    end
-
-    it "return empty string if truncated with no match to regexp" do
-      node = Yasuri.text_title '/html/body/p[1]', truncate_regexp:/^hoge/
-      actual = node.inject(@agent, @index_page)
-      expect(actual).to be_empty
-    end
-  end
-
-  ##########
-  # Struct #
-  ##########
-  describe '::StructNode' do
-    before do
-      @page = @agent.get(@uri + "/structual_text.html")
-      @table_1996 = [
-        { "title"    => "The Perfect Insider",
-          "pub_date" => "1996/4/5" },
-        { "title"    => "Doctors in Isolated Room",
-          "pub_date" => "1996/7/5" },
-        { "title"    => "Mathematical Goodbye",
-          "pub_date" => "1996/9/5" },
-      ]
-      @table_1997 = [
-        { "title"    => "Jack the Poetical Private",
-          "pub_date" => "1997/1/5" },
-        { "title"    => "Who Inside",
-          "pub_date" => "1997/4/5" },
-        { "title"    => "Illusion Acts Like Magic",
-          "pub_date" => "1997/10/5" },
-      ]
-      @table_1998 = [
-        { "title"    => "Replaceable Summer",
-          "pub_date" => "1998/1/7" },
-        { "title"    => "Switch Back",
-          "pub_date" => "1998/4/5" },
-        { "title"    => "Numerical Models",
-          "pub_date" => "1998/7/5" },
-        { "title"    => "The Perfect Outsider",
-          "pub_date" => "1998/10/5" },
-      ]
-      @all_tables = [
-        {"table" => @table_1996},
-        {"table" => @table_1997},
-        {"table" => @table_1998},
-      ]
-    end
-    it 'scrape single table contents' do
-      node = Yasuri::StructNode.new('/html/body/table[1]/tr', "table", [
-        Yasuri::TextNode.new('./td[1]', "title"),
-        Yasuri::TextNode.new('./td[2]', "pub_date"),
-      ])
-      expected = @table_1996
-      actual = node.inject(@agent, @page)
-      expect(actual).to match expected
-    end
-
-    it 'return empty text if no match node' do
-      no_match_xpath = '/html/body/table[1]/t'
-      node = Yasuri::StructNode.new(no_match_xpath, "table", [
-        Yasuri::TextNode.new('./td[1]', "title")
-      ])
-      actual = node.inject(@agent, @page)
-      expect(actual).to be_empty
-    end
-
-    it 'fail with invalid xpath' do
-      invalid_xpath = '/html/body/table[1]/table[1]/tr['
-      node = Yasuri::StructNode.new(invalid_xpath, "table", [
-        Yasuri::TextNode.new('./td[1]', "title")
-      ])
-      expect { node.inject(@agent, @page) }.to raise_error
-    end
-
-    it 'fail with invalid xpath in children' do
-      invalid_xpath = './td[1]['
-      node = Yasuri::StructNode.new('/html/body/table[1]/tr', "table", [
-        Yasuri::TextNode.new(invalid_xpath, "title"),
-        Yasuri::TextNode.new('./td[2]', "pub_date"),
-      ])
-      expect { node.inject(@agent, @page) }.to raise_error
-    end
-
-    it 'scrape all tables' do
-      node = Yasuri::StructNode.new('/html/body/table', "tables", [
-        Yasuri::StructNode.new('./tr', "table", [
-          Yasuri::TextNode.new('./td[1]', "title"),
-          Yasuri::TextNode.new('./td[2]', "pub_date"),
-        ])
-      ])
-      expected = @all_tables
-      actual = node.inject(@agent, @page)
-      expect(actual).to match expected
-    end
-
-    it 'can be defined by DSL, scrape all tables' do
-      generated = Yasuri.struct_tables '/html/body/table' do
-        struct_table './tr' do
-          text_title    './td[1]'
-          text_pub_date './td[2]'
-        end
-      end
-      original = Yasuri::StructNode.new('/html/body/table', "tables", [
-        Yasuri::StructNode.new('./tr', "table", [
-          Yasuri::TextNode.new('./td[1]', "title"),
-          Yasuri::TextNode.new('./td[2]', "pub_date"),
-        ])
-      ])
-      compare_generated_vs_original(generated, original)
-    end
-  end
-
-  #########
-  # Links #
-  #########
-  describe '::LinksNode' do
-    it 'scrape links' do
-      root_node = Yasuri::LinksNode.new('/html/body/a', "root", [
-        Yasuri::TextNode.new('/html/body/p', "content"),
-      ])
-
-      actual = root_node.inject(@agent, @index_page)
-      expected = [
-        {"content" => "Child 01 page."},
-        {"content" => "Child 02 page."},
-        {"content" => "Child 03 page."},
-      ]
-      expect(actual).to match expected
-    end
-
-    it 'return empty set if no match node' do
-      missing_xpath = '/html/body/b'
-      root_node = Yasuri::LinksNode.new(missing_xpath, "root", [
-        Yasuri::TextNode.new('/html/body/p', "content"),
-      ])
-
-      actual = root_node.inject(@agent, @index_page)
-      expect(actual).to be_empty
-    end
-
-    it 'scrape links, recursive' do
-      root_node = Yasuri::LinksNode.new('/html/body/a', "root", [
-        Yasuri::TextNode.new('/html/body/p', "content"),
-        Yasuri::LinksNode.new('/html/body/ul/li/a', "sub_link", [
-          Yasuri::TextNode.new('/html/head/title', "sub_page_title"),
-        ]),
-      ])
-      actual = root_node.inject(@agent, @index_page)
-      expected = [
-        {"content"  => "Child 01 page.",
-         "sub_link" => [{"sub_page_title" => "Child 01 SubPage Test"},
-                        {"sub_page_title" => "Child 02 SubPage Test"}],},
-        {"content" => "Child 02 page.",
-         "sub_link" => [],},
-        {"content" => "Child 03 page.",
-         "sub_link" => [{"sub_page_title" => "Child 03 SubPage Test"}],},
-      ]
-      expect(actual).to match expected
-    end
-    it 'can be defined by DSL, return single LinkNode title' do
-      generated = Yasuri.links_title '/html/body/a'
-      original  = Yasuri::LinksNode.new('/html/body/a', "title")
-      compare_generated_vs_original(generated, original)
-    end
-    it 'can be defined by DSL, return nested contents under link' do
-      generated = Yasuri.links_title '/html/body/a' do
-                     text_name '/html/body/p'
-                  end
-      original = Yasuri::LinksNode.new('/html/body/a', "root", [
-        Yasuri::TextNode.new('/html/body/p', "name"),
-      ])
-      compare_generated_vs_original(generated, original)
-    end
-
-    it 'can be defined by DSL, return recursive links node' do
-      generated = Yasuri.links_root '/html/body/a' do
-        text_content '/html/body/p'
-        links_sub_link '/html/body/ul/li/a' do
-          text_sub_page_title '/html/head/title'
-        end
-      end
-
-      original = Yasuri::LinksNode.new('/html/body/a', "root", [
-        Yasuri::TextNode.new('/html/body/p', "content"),
-        Yasuri::LinksNode.new('/html/body/ul/li/a', "sub_link", [
-          Yasuri::TextNode.new('/html/head/title', "sub_page_title"),
-        ]),
-      ])
-      compare_generated_vs_original(generated, original)
-    end
-  end
-
-  ############
-  # Paginate #
-  ############
-  describe '::PaginateNode' do
-    before do
-      @uri += "/pagination/page01.html"
-      @page = @agent.get(@uri)
-    end
-
-    it "scrape each paginated pages" do
-      root_node = Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "root", [
-        Yasuri::TextNode.new('/html/body/p', "content"),
-      ])
-      actual = root_node.inject(@agent, @page)
-      expected = [
-        {"content" => "PaginationTest01"},
-        {"content" => "PaginationTest02"},
-        {"content" => "PaginationTest03"},
-        {"content" => "PaginationTest04"},
-      ]
-      expect(actual).to match expected
-    end
-
-    it "scrape each paginated pages limited" do
-      root_node = Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "root", [
-        Yasuri::TextNode.new('/html/body/p', "content"),
-      ], limit:3)
-      actual = root_node.inject(@agent, @page)
-      expected = [
-        {"content" => "PaginationTest01"},
-        {"content" => "PaginationTest02"},
-        {"content" => "PaginationTest03"},
-      ]
-      expect(actual).to match expected
-    end
-
-
-    it 'return first content if paginate link node is not found' do
-      missing_xpath = "/html/body/nav/span/b[@class='next']"
-      root_node = Yasuri::PaginateNode.new(missing_xpath, "root", [
-        Yasuri::TextNode.new('/html/body/p', "content"),
-      ])
-      actual = root_node.inject(@agent, @page)
-      expected = [ {"content" => "PaginationTest01"}, ]
-      expect(actual).to match_array expected
-    end
-
-    it 'return empty hashes if content node is not found' do
-      root_node = Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "root", [
-        Yasuri::TextNode.new('/html/body/hoge', "content"),
-      ])
-      actual = root_node.inject(@agent, @page)
-      expected = [ {"content" => ""}, {"content" => ""}, {"content" => ""}, {"content" => ""},]
-      expect(actual).to match_array expected
-    end
-
-    it 'can be defined by DSL, return single PaginateNode content' do
-      generated = Yasuri.pages_next "/html/body/nav/span/a[@class='next']" do
-        text_content '/html/body/p'
-      end
-      original = Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "root", [
-        Yasuri::TextNode.new('/html/body/p', "content"),
-      ])
-      compare_generated_vs_original(generated, original, @page)
-    end
-
-    it 'can be defined by DSL, return single PaginateNode content limited' do
-      generated = Yasuri.pages_next "/html/body/nav/span/a[@class='next']", 2 do
-        text_content '/html/body/p'
-      end
-      original = Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "root", [
-        Yasuri::TextNode.new('/html/body/p', "content"),
-      ], limit: 2)
-      compare_generated_vs_original(generated, original, @page)
-    end
-  end
-
   #############
   # json2tree #
   #############
@@ -348,9 +32,21 @@ describe 'Yasuri' do
                 }|
       generated = Yasuri.json2tree(src)
       original  = Yasuri::TextNode.new('/html/body/p[1]', "content")
-      compare_generated_vs_original(generated, original)
+      compare_generated_vs_original(generated, original, @index_page)
+    end
+
+    it "return TextNode with truncate_regexp" do
+      src = %q| { "node"  : "text",
+                  "name"  : "content",
+                  "path"  : "/html/body/p[1]",
+                  "truncate"  : "^[^,]+"
+                }|
+      generated = Yasuri.json2tree(src)
+      original  = Yasuri::TextNode.new('/html/body/p[1]', "content", truncate:/^[^,]+/)
+      compare_generated_vs_original(generated, original, @index_page)
     end
 
+
     it "return LinksNode/TextNode" do
       src = %q| { "node"     : "links",
                   "name"     : "root",
@@ -364,7 +60,8 @@ describe 'Yasuri' do
       original  = Yasuri::LinksNode.new('/html/body/a', "root", [
                     Yasuri::TextNode.new('/html/body/p', "content"),
                   ])
-      compare_generated_vs_original(generated, original)
+
+      compare_generated_vs_original(generated, original, @index_page)
     end
 
     it "return PaginateNode/TextNode" do
@@ -437,6 +134,83 @@ describe 'Yasuri' do
     end
   end
 
+  #############
+  # tree2json #
+  #############
+  describe '.tree2json' do
+    it "return empty json" do
+      json = Yasuri.tree2json(nil)
+      expect(json).to match "{}"
+    end
+
+    it "return text node" do
+      node = Yasuri::TextNode.new("/html/head/title", "title")
+      json = Yasuri.tree2json(node)
+      expected_str = %q| { "node": "text",
+                           "name": "title",
+                           "path": "/html/head/title"
+                         } |
+      expected = JSON.parse(expected_str)
+      actual   = JSON.parse(json)
+      expect(actual).to match expected
+    end
+
+    it "return text node with truncate_regexp" do
+      node = Yasuri::TextNode.new("/html/head/title", "title", truncate:/^[^,]+/)
+      json = Yasuri.tree2json(node)
+      expected_str = %q| { "node": "text",
+                           "name": "title",
+                           "path": "/html/head/title",
+                           "truncate": "^[^,]+"
+                         } |
+      expected = Yasuri.tree2json(Yasuri.json2tree(expected_str))
+      actual   = Yasuri.tree2json(Yasuri.json2tree(json))
+      expect(actual).to match expected
+    end
+
+    it "return LinksNode/TextNode" do
+      tree  = Yasuri::LinksNode.new('/html/body/a', "root", [
+                Yasuri::TextNode.new('/html/body/p', "content"),
+              ])
+      json   = Yasuri.tree2json(tree)
+      expected_src = %q| { "node"     : "links",
+                           "name"     : "root",
+                           "path"     : "/html/body/a",
+                           "children" : [ { "node" : "text",
+                                            "name" : "content",
+                                            "path" : "/html/body/p"
+                                          } ]
+                         }|
+      expected  = JSON.parse(expected_src)
+      actual    = JSON.parse(json)
+      expect(actual).to match expected
+    end
+
+    it "return PaginateNode/TextNode with limit" do
+      tree = Yasuri::PaginateNode.new("/html/body/nav/span/a[@class='next']", "root", [
+                Yasuri::TextNode.new('/html/body/p', "content"),
+             ], limit:10)
+
+      json   = Yasuri.tree2json(tree)
+      expected_src = %q| { "node"     : "pages",
+                           "name"     : "root",
+                           "path"     : "/html/body/nav/span/a[@class='next']",
+                           "limit"    : 10,
+                           "children" : [ { "node" : "text",
+                                            "name" : "content",
+                                            "path" : "/html/body/p"
+                                          } ]
+                         }|
+      expected  = JSON.parse(expected_src)
+      actual    = JSON.parse(json)
+      expect(actual).to match expected
+    end
+
+
+
+  end
+
+
   it 'has a version number' do
     expect(Yasuri::VERSION).not_to be nil
   end

data/spec/yasuri_struct_node_spec.rb

@@ -0,0 +1,114 @@
+
+# Author::    TAC (tac@tac42.net)
+
+require_relative 'spec_helper'
+
+##########
+# Struct #
+##########
+describe 'Yasuri' do
+  include_context 'httpserver'
+
+  describe '::StructNode' do
+    before do
+      @agent = Mechanize.new
+      @page = @agent.get(uri + "/structual_text.html")
+
+      @table_1996 = [
+        { "title"    => "The Perfect Insider",
+          "pub_date" => "1996/4/5" },
+        { "title"    => "Doctors in Isolated Room",
+          "pub_date" => "1996/7/5" },
+        { "title"    => "Mathematical Goodbye",
+          "pub_date" => "1996/9/5" },
+      ]
+      @table_1997 = [
+        { "title"    => "Jack the Poetical Private",
+          "pub_date" => "1997/1/5" },
+        { "title"    => "Who Inside",
+          "pub_date" => "1997/4/5" },
+        { "title"    => "Illusion Acts Like Magic",
+          "pub_date" => "1997/10/5" },
+      ]
+      @table_1998 = [
+        { "title"    => "Replaceable Summer",
+          "pub_date" => "1998/1/7" },
+        { "title"    => "Switch Back",
+          "pub_date" => "1998/4/5" },
+        { "title"    => "Numerical Models",
+          "pub_date" => "1998/7/5" },
+        { "title"    => "The Perfect Outsider",
+          "pub_date" => "1998/10/5" },
+      ]
+      @all_tables = [
+        {"table" => @table_1996},
+        {"table" => @table_1997},
+        {"table" => @table_1998},
+      ]
+    end
+
+    it 'scrape single table contents' do
+      node = Yasuri::StructNode.new('/html/body/table[1]/tr', "table", [
+        Yasuri::TextNode.new('./td[1]', "title"),
+        Yasuri::TextNode.new('./td[2]', "pub_date"),
+      ])
+      expected = @table_1996
+      actual = node.inject(@agent, @page)
+      expect(actual).to match expected
+    end
+
+    it 'return empty text if no match node' do
+      no_match_xpath = '/html/body/table[1]/t'
+      node = Yasuri::StructNode.new(no_match_xpath, "table", [
+        Yasuri::TextNode.new('./td[1]', "title")
+      ])
+      actual = node.inject(@agent, @page)
+      expect(actual).to be_empty
+    end
+
+    it 'fail with invalid xpath' do
+      invalid_xpath = '/html/body/table[1]/table[1]/tr['
+      node = Yasuri::StructNode.new(invalid_xpath, "table", [
+        Yasuri::TextNode.new('./td[1]', "title")
+      ])
+      expect { node.inject(@agent, @page) }.to raise_error
+    end
+
+    it 'fail with invalid xpath in children' do
+      invalid_xpath = './td[1]['
+      node = Yasuri::StructNode.new('/html/body/table[1]/tr', "table", [
+        Yasuri::TextNode.new(invalid_xpath, "title"),
+        Yasuri::TextNode.new('./td[2]', "pub_date"),
+      ])
+      expect { node.inject(@agent, @page) }.to raise_error
+    end
+
+    it 'scrape all tables' do
+      node = Yasuri::StructNode.new('/html/body/table', "tables", [
+        Yasuri::StructNode.new('./tr', "table", [
+          Yasuri::TextNode.new('./td[1]', "title"),
+          Yasuri::TextNode.new('./td[2]', "pub_date"),
+        ])
+      ])
+      expected = @all_tables
+      actual = node.inject(@agent, @page)
+      expect(actual).to match expected
+    end
+
+    it 'can be defined by DSL, scrape all tables' do
+      generated = Yasuri.struct_tables '/html/body/table' do
+        struct_table './tr' do
+          text_title    './td[1]'
+          text_pub_date './td[2]'
+        end
+      end
+      original = Yasuri::StructNode.new('/html/body/table', "tables", [
+        Yasuri::StructNode.new('./tr', "table", [
+          Yasuri::TextNode.new('./td[1]', "title"),
+          Yasuri::TextNode.new('./td[2]', "pub_date"),
+        ])
+      ])
+      compare_generated_vs_original(generated, original, @page)
+    end
+  end
+end

data/spec/yasuri_text_node_spec.rb

@@ -0,0 +1,61 @@
+
+# Author::    TAC (tac@tac42.net)
+
+require_relative 'spec_helper'
+
+########
+# Text #
+########
+describe 'Yasuri' do
+  include_context 'httpserver'
+
+  before do
+    @agent = Mechanize.new
+    @index_page = @agent.get(uri)
+  end
+
+  describe '::TextNode' do
+    before { @node = Yasuri::TextNode.new('/html/body/p[1]', "title") }
+
+    it 'scrape text text <p>Hello,Yasuri</p>' do
+      actual = @node.inject(@agent, @index_page)
+      expect(actual).to eq "Hello,Yasuri"
+    end
+
+    it 'return empty text if no match node' do
+      no_match_node = Yasuri::TextNode.new('/html/body/no_match_node', "title")
+      actual = no_match_node.inject(@agent, @index_page)
+      expect(actual).to be_empty
+    end
+
+    it 'fail with invalid xpath' do
+      invalid_xpath = '/html/body/no_match_node['
+      node = Yasuri::TextNode.new(invalid_xpath, "title")
+      expect { node.inject(@agent, @index_page) }.to raise_error
+    end
+
+    it "can be defined by DSL, return single TextNode title" do
+      generated = Yasuri.text_title '/html/body/p[1]'
+      original  = Yasuri::TextNode.new('/html/body/p[1]', "title")
+      compare_generated_vs_original(generated, original, @index_page)
+    end
+
+    it "can be truncated with regexp" do
+      node  = Yasuri.text_title '/html/body/p[1]', /^[^,]+/
+      actual = node.inject(@agent, @index_page)
+      expect(actual).to eq "Hello"
+    end
+
+    it "can be truncated with regexp" do
+      node = Yasuri.text_title '/html/body/p[1]', /[^,]+$/
+      actual = node.inject(@agent, @index_page)
+      expect(actual).to eq "Yasuri"
+    end
+
+    it "return empty string if truncated with no match to regexp" do
+      node = Yasuri.text_title '/html/body/p[1]', /^hoge/
+      actual = node.inject(@agent, @index_page)
+      expect(actual).to be_empty
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: yasuri
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.7
 platform: ruby
 authors:
 - TAC
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-24 00:00:00.000000000 Z
+date: 2015-02-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -155,6 +155,12 @@ files:
 - lib/yasuri.rb
 - lib/yasuri/version.rb
 - lib/yasuri/yasuri.rb
+- lib/yasuri/yasuri_links_node.rb
+- lib/yasuri/yasuri_node.rb
+- lib/yasuri/yasuri_node_generator.rb
+- lib/yasuri/yasuri_paginate_node.rb
+- lib/yasuri/yasuri_struct_node.rb
+- lib/yasuri/yasuri_text_node.rb
 - spec/htdocs/child01.html
 - spec/htdocs/child01_sub.html
 - spec/htdocs/child02.html
@@ -169,7 +175,12 @@ files:
 - spec/htdocs/structual_text.html
 - spec/servers/httpserver.rb
 - spec/spec_helper.rb
+- spec/yasuri_links_node_spec.rb
+- spec/yasuri_node_spec.rb
+- spec/yasuri_paginate_node_spec.rb
 - spec/yasuri_spec.rb
+- spec/yasuri_struct_node_spec.rb
+- spec/yasuri_text_node_spec.rb
 - yasuri.gemspec
 homepage: https://github.com/tac0x2a/yasuri
 licenses:
@@ -210,4 +221,9 @@ test_files:
 - spec/htdocs/structual_text.html
 - spec/servers/httpserver.rb
 - spec/spec_helper.rb
+- spec/yasuri_links_node_spec.rb
+- spec/yasuri_node_spec.rb
+- spec/yasuri_paginate_node_spec.rb
 - spec/yasuri_spec.rb
+- spec/yasuri_struct_node_spec.rb
+- spec/yasuri_text_node_spec.rb