skyscraper 0.0.4 → 0.0.5

data/lib/skyscraper.rb

@@ -16,6 +16,7 @@ module Skyscraper
   autoload :Node
   autoload :Pages
   autoload :Path
+  autoload :Resource
   autoload :Results
   
   mattr_accessor :defaults
@@ -24,7 +25,6 @@ module Skyscraper
     limit: nil,
     encoding: "utf-8",
     download_path: "/tmp/skyscraper/:sequence/:file_name", 
-#    reattempt_times: 1,
     noise_errors: true,
     skip_on_error: true
   }
@@ -35,7 +35,7 @@ module Skyscraper
 
   def self.fetch path, encoding = Skyscraper.config.encoding
     document = Skyscraper::Document.load path, encoding
-    Node::Base.new document.css("html")
+    Node.new document.css("html")
   end
 
   def fetch

data/lib/skyscraper/node.rb

@@ -1,8 +1,101 @@
 module Skyscraper
-  module Node
-    extend ActiveSupport::Autoload
+  class Node
+    alias :original_class :class
 
-    autoload :Base
-    autoload :Resource
+    attr_accessor :element
+
+    def initialize element
+      @element = element
+    end
+
+    def first selector
+      self.find(selector).first
+    end
+
+    def find selector
+      @element.css(selector).map do |element|
+        Node.new(element)
+      end
+    end
+
+    def children selector = nil
+      if selector
+        children = @element.css(selector)
+      else
+        children = @element.children
+      end
+
+      children.select do |element|
+        element.parent == @element and element.is_a?(Nokogiri::XML::Element)
+      end.map do |child|
+        Node.new(child)
+      end 
+    end
+
+    def parent
+      if @element.parent.is_a? Nokogiri::XML::Element
+        Node.new @element.parent
+      end
+    end
+
+    def have_parent?
+      self.parent.present? 
+    end
+
+    def parents selector = nil
+      node = self
+      parents = []
+
+      while node.have_parent?
+        node = node.parent
+        parents << node
+      end
+
+      parents.select! do |item|
+        item.element.matches? selector
+      end if selector
+
+      parents
+    end
+
+    def siblings
+      self.parent.children.select do |node|
+        node.element != self.element
+      end
+    end
+
+    def follow 
+      if self.href
+        Skyscraper::fetch(self.uri)
+      end
+    end
+
+    def html
+      @element.children.to_html
+    end
+
+    def class
+      @element.attribute("class").to_s
+    end
+
+    def download options = {}
+      Resource.new(self).download(options)
+    end
+
+    def uri
+      @element.document.path.full_path_for(self.href)
+    end
+
+    def method_missing name
+      @element.attribute(name.to_s).to_s
+    end
+
+    def text
+      @element.content.to_s.strip
+    end
+
+    def tag
+      @element.name
+    end
   end
 end

data/lib/skyscraper/resource.rb

@@ -0,0 +1,55 @@
+module Skyscraper
+  class Resource
+    def initialize node
+      @node           = node
+      @path           = extract_path_from_node(@node)
+    end
+
+    def download options = {}
+      name          = options[:file_name] || @path.file_name
+      new_file_path = replace_path_variables(options[:path] || Skyscraper.config.download_path, name)
+      temp_file     = open(@path.full_path)
+
+      copy temp_file.path, new_file_path
+      new_file_path
+    end
+
+    private 
+
+    def copy from, to 
+      create_path_if_not_exists to
+      `cp #{from} #{to}`
+    end
+
+    def create_path_if_not_exists path
+      `mkdir -p #{path}` unless File.directory?(path)
+    end
+
+    def replace_path_variables path, name
+      new_path = path.dup
+      new_path.gsub! /:file_name/, name
+      new_path.gsub! /:sequence/, get_sequence_number_for(new_path)
+      new_path
+    end
+
+    def get_sequence_number_for path
+      new_path = path.split(":sequence")[0]
+      if File.directory?(new_path)
+        entries = Dir.entries(new_path).select { |i| i =~ /^\d+$/ } || []
+        last = entries.sort.last.to_i
+        last += 1
+        last.to_s
+      else
+        "1"
+      end
+    end
+
+    def extract_path_from_node node
+      if href_or_src = node.href.present? ? node.href : node.src
+        node.element.document.path.path_for(href_or_src)
+      else
+        throw Exception.new("no href no src")
+      end
+    end
+  end
+end

data/lib/skyscraper/version.rb

@@ -1,3 +1,3 @@
 module Skyscraper
-  VERSION = "0.0.4"
+  VERSION = "0.0.5"
 end

data/skyscraper.gemspec

@@ -4,8 +4,8 @@ require File.expand_path('../lib/skyscraper/version', __FILE__)
 Gem::Specification.new do |gem|
   gem.authors       = ["Adam Dratwinski"]
   gem.email         = ["arboooz@gmail.com"]
-  gem.summary       = %q{Library that helps scraping data from websites in easy way}
-  gem.description   = %q{Library that helps scraping data from websites in easy way. Skyscraper allows you to traversing through html nodes, similary to jquery, it provides methods like parent, children, first, find, siblings etc. Thanks to Skyscraper you can fetch all HTML attributes on any node. Furthermore it's allow to download images, webpages, and store content in the database. Please visit Github account for more details.}
+  gem.summary       = %q{Easy to use DSL that helps scraping data from websites}
+  gem.description   = %q{Easy to use DSL that helps scraping data from websites. Thanks to it, writing web crawlers would be very fast and intuitive. Traversing through html nodes and fetching all of the HTML attributes, would be possible. Just like in jQuery - you will find methods like parent, children, first, find, siblings etc. Furthermore, you are able to download images, web pages, and store all content in the database. Please visit my Github account for more details.}
   gem.homepage      = "https://github.com/boooz/skyscraper"
 
   gem.files         = `git ls-files`.split($\)

data/spec/skyscraper/skyscraper/node_spec.rb

@@ -1,2 +1,87 @@
 describe Skyscraper::Node do
+  describe "when is initialized" do
+    before(:each) do
+      @node = Skyscraper::fetch(path_to("skyscraper-node.html")).first("div.item")
+    end
+    
+    it "should returns html code" do
+      @node.html.should include "<strong class=\"name\">Name value</strong>"
+    end
+
+    it "should returns class name" do
+      @node.class.should == "item"
+    end
+
+    it "should be auto converted to string with stripped tags" do
+      @node.text.should == "Name value"
+    end
+  end
+
+  it "should follow links" do
+    Skyscraper::fetch(path_to("skyscraper-node.html")).first("li a").follow.first("h1").text.should == "Hello from A"
+  end
+
+  it "should deep follow links" do
+    Skyscraper::fetch(path_to("skyscraper-node.html")).first("li a").follow.first("a").follow.first("h1").text.should == "Hello from B"
+  end
+
+  it "should download page" do
+    remove_test_directory
+    Skyscraper.config.download_path = "/tmp/skyscraper_test/nodes/:file_name"
+    file = Skyscraper::fetch(path_to("skyscraper-node.html")).first("li a").follow.first("a").download
+    File.exists?(file).should == true
+  end
+  describe "traversing" do
+    before(:each) do
+      @node = Skyscraper::fetch(path_to("skyscraper-node-traversing.html")).first(".menu")
+    end
+
+    it "should find descendands items" do
+      result = @node.find("li")
+      result.length.should == 5
+      result.map(&:text).should include "Item 4 1"
+    end
+
+    it "should returns children of element with selector" do
+      node = Skyscraper::fetch(path_to("skyscraper-node-traversing.html")).first("#parent-3")
+      node.children(".a").length.should == 4
+      node.children(".b").length.should == 2
+    end
+
+    it "should returns children of element without selector" do
+      result = @node.children
+      result.length.should == 4
+      result.map(&:to_s).should_not include "Item 4 1"
+    end
+
+    it "should returns first element" do
+      @node.first("li").class.should == "item-1"
+    end
+
+    it "should returns parent of item" do
+      @node.parent.class.should == "parent-2"
+    end
+
+    it "should tells if element have parent" do
+      @node.have_parent?.should == true
+      @node.parents("html").first.have_parent?.should == false
+    end
+
+    it "should returns parents of item" do
+      @node.parents.length.should == 4
+    end
+    
+    it "should returns parents of item matched by selector" do
+      @node.parents("div").length.should == 2
+    end
+
+    it "should returns siblings of item" do
+      @node.first(".item-3").siblings.length.should == 3
+    end
+
+    it "should returns node tag" do
+      @node.tag.should == "ul"
+    end
+  end
 end
+

data/spec/skyscraper/skyscraper/{node/resource_spec.rb → resource_spec.rb}

@@ -1,12 +1,12 @@
-describe Skyscraper::Node::Resource do
+describe Skyscraper::Resource do
   def should_download_resource_to node, path, options = {}
-    resource = Skyscraper::Node::Resource.new(node)
+    resource = Skyscraper::Resource.new(node)
     resource.download(options).should == path
   end
 
   before(:all) do
     Skyscraper.config.download_path = "/tmp/skyscraper_test/:sequence/:file_name"
-    @node = Skyscraper::fetch(path_to("skyscraper-node-resource.html")).first("a")
+    @node = Skyscraper::fetch(path_to("skyscraper-resource.html")).first("a")
   end
 
   before(:each) do
@@ -15,29 +15,29 @@ describe Skyscraper::Node::Resource do
 
   it "should create path if not exists when downloaded" do
     File.directory?("/tmp/skyscraper_test/1").should == false
-    Skyscraper::Node::Resource.new(@node)
+    Skyscraper::Resource.new(@node)
     File.directory?("/tmp/skyscraper_test/1").should == false
-    Skyscraper::Node::Resource.new(@node).download
+    Skyscraper::Resource.new(@node).download
     File.directory?("/tmp/skyscraper_test/1").should == true
   end
 
   it "should not fail if path already exists" do
-    Skyscraper::Node::Resource.new(@node).download path: "/tmp/skyscraper_test/some_directory/:file_name"
+    Skyscraper::Resource.new(@node).download path: "/tmp/skyscraper_test/some_directory/:file_name"
     File.directory?("/tmp/skyscraper_test/some_directory").should == true
-    Skyscraper::Node::Resource.new(@node).download
+    Skyscraper::Resource.new(@node).download
     File.directory?("/tmp/skyscraper_test/some_directory").should == true
   end
 
   it "should have file name" do
-    resource = Skyscraper::Node::Resource.new(@node)
-    resource.download.should == "/tmp/skyscraper_test/1/skyscraper-node-resource-b.html"
+    resource = Skyscraper::Resource.new(@node)
+    resource.download.should == "/tmp/skyscraper_test/1/skyscraper-resource-b.html"
   end
 
   it "should create path with :sequence variable" do
     download_to = "/tmp/skyscraper_test/sequences/:sequence/:file_name"
-    should_download_resource_to @node, "/tmp/skyscraper_test/sequences/1/skyscraper-node-resource-b.html", path: download_to
-    should_download_resource_to @node, "/tmp/skyscraper_test/sequences/2/skyscraper-node-resource-b.html", path: download_to
-    should_download_resource_to @node, "/tmp/skyscraper_test/sequences/3/skyscraper-node-resource-b.html", path: download_to
+    should_download_resource_to @node, "/tmp/skyscraper_test/sequences/1/skyscraper-resource-b.html", path: download_to
+    should_download_resource_to @node, "/tmp/skyscraper_test/sequences/2/skyscraper-resource-b.html", path: download_to
+    should_download_resource_to @node, "/tmp/skyscraper_test/sequences/3/skyscraper-resource-b.html", path: download_to
   end
 
   it "should create custom file name if provided" do
@@ -46,13 +46,13 @@ describe Skyscraper::Node::Resource do
   end
 
   it "should download resource" do
-    Skyscraper::Node::Resource.new(@node).download
-    File.exists?("/tmp/skyscraper_test/1/skyscraper-node-resource-b.html").should == true
+    Skyscraper::Resource.new(@node).download
+    File.exists?("/tmp/skyscraper_test/1/skyscraper-resource-b.html").should == true
   end
 
   it "should download image" do
-    image_node = Skyscraper::fetch(path_to("skyscraper-node-resource.html")).first("img")
-    Skyscraper::Node::Resource.new(image_node).download
-    File.exists?("/tmp/skyscraper_test/1/skyscraper-node-resource-image.png").should == true
+    image_node = Skyscraper::fetch(path_to("skyscraper-resource.html")).first("img")
+    Skyscraper::Resource.new(image_node).download
+    File.exists?("/tmp/skyscraper_test/1/skyscraper-resource-image.png").should == true
   end
 end

data/spec/skyscraper/skyscraper/results_spec.rb

@@ -69,7 +69,7 @@ describe Skyscraper::Results do
       @call_count = 0
       callback = proc do |result, page|  
         result.should be_an_instance_of(Hash)
-        page.should be_an_instance_of(Skyscraper::Node::Base)
+        page.should be_an_instance_of(Skyscraper::Node)
         @call_count += 1
       end
 

data/spec/skyscraper/skyscraper_spec.rb

@@ -16,7 +16,7 @@ describe Skyscraper do
   end
 
   it "should fetch remote page" do
-    Skyscraper::fetch("http://google.com").should be_an Skyscraper::Node::Base
+    Skyscraper::fetch("http://google.com").should be_an Skyscraper::Node
   end
 
   it "static method fetch should works" do

data/spec/test_files/{skyscraper-node-base-a.html → skyscraper-node-a.html}

@@ -6,6 +6,6 @@
 
     <body>
 	<h1>Hello from A</h1>
-	<a href="skyscraper-node-base-b.html">b</a>
+	<a href="skyscraper-node-b.html">b</a>
     </body>
 </html>

data/spec/test_files/{skyscraper-node-base.html → skyscraper-node.html}

@@ -8,7 +8,7 @@
         <h1>Hello world</h1>
         <ul class="menu">
             <li>
-                <a href="skyscraper-node-base-a.html">A</a>
+                <a href="skyscraper-node-a.html">A</a>
                 <a href="b.html">A</a>
                 <a href="c.html">A</a>
                 <a href="d.html">A</a>

data/spec/test_files/{skyscraper-node-resource.html → skyscraper-resource.html}

@@ -6,7 +6,7 @@
 
     <body>
 	<h1>Hello from A</h1>
-	<a href="skyscraper-node-resource-b.html">B</a>
-	<img src="skyscraper-node-resource-image.png">
+	<a href="skyscraper-resource-b.html">B</a>
+	<img src="skyscraper-resource-image.png">
     </body>
 </html>

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: skyscraper
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-05-17 00:00:00.000000000 Z
+date: 2012-05-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &71541210 !ruby/object:Gem::Requirement
+  requirement: &77062930 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *71541210
+  version_requirements: *77062930
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &71603060 !ruby/object:Gem::Requirement
+  requirement: &77124830 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *71603060
+  version_requirements: *77124830
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &71600890 !ruby/object:Gem::Requirement
+  requirement: &77122840 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *71600890
+  version_requirements: *77122840
 - !ruby/object:Gem::Dependency
   name: actionpack
-  requirement: &71599790 !ruby/object:Gem::Requirement
+  requirement: &77119590 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,12 +54,13 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *71599790
-description: Library that helps scraping data from websites in easy way. Skyscraper
-  allows you to traversing through html nodes, similary to jquery, it provides methods
-  like parent, children, first, find, siblings etc. Thanks to Skyscraper you can fetch
-  all HTML attributes on any node. Furthermore it's allow to download images, webpages,
-  and store content in the database. Please visit Github account for more details.
+  version_requirements: *77119590
+description: Easy to use DSL that helps scraping data from websites. Thanks to it,
+  writing web crawlers would be very fast and intuitive. Traversing through html nodes
+  and fetching all of the HTML attributes, would be possible. Just like in jQuery
+  - you will find methods like parent, children, first, find, siblings etc. Furthermore,
+  you are able to download images, web pages, and store all content in the database.
+  Please visit my Github account for more details.
 email:
 - arboooz@gmail.com
 executables: []
@@ -78,13 +79,12 @@ files:
 - lib/skyscraper/document.rb
 - lib/skyscraper/field.rb
 - lib/skyscraper/node.rb
-- lib/skyscraper/node/base.rb
-- lib/skyscraper/node/resource.rb
 - lib/skyscraper/pages.rb
 - lib/skyscraper/path.rb
 - lib/skyscraper/path/base.rb
 - lib/skyscraper/path/local.rb
 - lib/skyscraper/path/remote.rb
+- lib/skyscraper/resource.rb
 - lib/skyscraper/results.rb
 - lib/skyscraper/version.rb
 - skyscraper.gemspec
@@ -92,11 +92,10 @@ files:
 - spec/skyscraper/skyscraper/config_spec.rb
 - spec/skyscraper/skyscraper/document_spec.rb
 - spec/skyscraper/skyscraper/field_spec.rb
-- spec/skyscraper/skyscraper/node/base_spec.rb
-- spec/skyscraper/skyscraper/node/resource_spec.rb
 - spec/skyscraper/skyscraper/node_spec.rb
 - spec/skyscraper/skyscraper/pages_spec.rb
 - spec/skyscraper/skyscraper/path_spec.rb
+- spec/skyscraper/skyscraper/resource_spec.rb
 - spec/skyscraper/skyscraper/results_spec.rb
 - spec/skyscraper/skyscraper_spec.rb
 - spec/spec_helper.rb
@@ -108,14 +107,14 @@ files:
 - spec/test_files/skyscraper-fetch-2.html
 - spec/test_files/skyscraper-fetch.html
 - spec/test_files/skyscraper-field.html
-- spec/test_files/skyscraper-node-base-a.html
-- spec/test_files/skyscraper-node-base-b.html
-- spec/test_files/skyscraper-node-base-traversing.html
-- spec/test_files/skyscraper-node-base.html
-- spec/test_files/skyscraper-node-resource-b.html
-- spec/test_files/skyscraper-node-resource-image.png
-- spec/test_files/skyscraper-node-resource.html
+- spec/test_files/skyscraper-node-a.html
+- spec/test_files/skyscraper-node-b.html
+- spec/test_files/skyscraper-node-traversing.html
+- spec/test_files/skyscraper-node.html
 - spec/test_files/skyscraper-pages.html
+- spec/test_files/skyscraper-resource-b.html
+- spec/test_files/skyscraper-resource-image.png
+- spec/test_files/skyscraper-resource.html
 - spec/test_files/skyscraper.html
 homepage: https://github.com/boooz/skyscraper
 licenses: []
@@ -140,17 +139,16 @@ rubyforge_project:
 rubygems_version: 1.8.15
 signing_key: 
 specification_version: 3
-summary: Library that helps scraping data from websites in easy way
+summary: Easy to use DSL that helps scraping data from websites
 test_files:
 - spec/skyscraper/skyscraper/base_spec.rb
 - spec/skyscraper/skyscraper/config_spec.rb
 - spec/skyscraper/skyscraper/document_spec.rb
 - spec/skyscraper/skyscraper/field_spec.rb
-- spec/skyscraper/skyscraper/node/base_spec.rb
-- spec/skyscraper/skyscraper/node/resource_spec.rb
 - spec/skyscraper/skyscraper/node_spec.rb
 - spec/skyscraper/skyscraper/pages_spec.rb
 - spec/skyscraper/skyscraper/path_spec.rb
+- spec/skyscraper/skyscraper/resource_spec.rb
 - spec/skyscraper/skyscraper/results_spec.rb
 - spec/skyscraper/skyscraper_spec.rb
 - spec/spec_helper.rb
@@ -162,12 +160,12 @@ test_files:
 - spec/test_files/skyscraper-fetch-2.html
 - spec/test_files/skyscraper-fetch.html
 - spec/test_files/skyscraper-field.html
-- spec/test_files/skyscraper-node-base-a.html
-- spec/test_files/skyscraper-node-base-b.html
-- spec/test_files/skyscraper-node-base-traversing.html
-- spec/test_files/skyscraper-node-base.html
-- spec/test_files/skyscraper-node-resource-b.html
-- spec/test_files/skyscraper-node-resource-image.png
-- spec/test_files/skyscraper-node-resource.html
+- spec/test_files/skyscraper-node-a.html
+- spec/test_files/skyscraper-node-b.html
+- spec/test_files/skyscraper-node-traversing.html
+- spec/test_files/skyscraper-node.html
 - spec/test_files/skyscraper-pages.html
+- spec/test_files/skyscraper-resource-b.html
+- spec/test_files/skyscraper-resource-image.png
+- spec/test_files/skyscraper-resource.html
 - spec/test_files/skyscraper.html

data/lib/skyscraper/node/base.rb

@@ -1,103 +0,0 @@
-module Skyscraper
-  module Node
-    class Base
-      alias :original_class :class
-
-      attr_accessor :element
-
-      def initialize element
-        @element = element
-      end
-
-      def first selector
-        self.find(selector).first
-      end
-
-      def find selector
-        @element.css(selector).map do |element|
-          Base.new(element)
-        end
-      end
-
-      def children selector = nil
-        if selector
-          children = @element.css(selector)
-        else
-          children = @element.children
-        end
-
-        children.select do |element|
-          element.parent == @element and element.is_a?(Nokogiri::XML::Element)
-        end.map do |child|
-          Base.new(child)
-        end 
-      end
-
-      def parent
-        if @element.parent.is_a? Nokogiri::XML::Element
-          Base.new @element.parent
-        end
-      end
-
-      def have_parent?
-        self.parent.present? 
-      end
-
-      def parents selector = nil
-        node = self
-        parents = []
-
-        while node.have_parent?
-          node = node.parent
-          parents << node
-        end
-
-        parents.select! do |item|
-          item.element.matches? selector
-        end if selector
-
-        parents
-      end
-
-      def siblings
-        self.parent.children.select do |node|
-          node.element != self.element
-        end
-      end
-
-      def follow 
-        if self.href
-          Skyscraper::fetch(self.uri)
-        end
-      end
-
-      def html
-        @element.children.to_html
-      end
-
-      def class
-        @element.attribute("class").to_s
-      end
-
-      def download options = {}
-        Resource.new(self).download(options)
-      end
-
-      def uri
-        @element.document.path.full_path_for(self.href)
-      end
-
-      def method_missing name
-        @element.attribute(name.to_s).to_s
-      end
-
-      def text
-        @element.content.to_s.strip
-      end
-
-      def tag
-	@element.name
-      end
-    end
-  end
-end

data/lib/skyscraper/node/resource.rb

@@ -1,57 +0,0 @@
-module Skyscraper
-  module Node
-    class Resource
-      def initialize node
-        @node           = node
-        @path           = extract_path_from_node(@node)
-      end
-
-      def download options = {}
-        @name          = options[:file_name] || @path.file_name
-        @new_file_path = replace_path_variables(options[:path] || Skyscraper.config.download_path)
-        @temp_file     = open(@path.full_path)
-
-        copy @temp_file.path, @new_file_path
-        @new_file_path
-      end
-
-      private 
-
-      def copy from, to 
-        create_path_if_not_exists to
-        `cp #{from} #{to}`
-      end
-
-      def create_path_if_not_exists path
-        `mkdir -p #{path}` unless File.directory?(path)
-      end
-
-      def replace_path_variables path
-        new_path = path.dup
-        new_path.gsub! /:file_name/, @name
-        new_path.gsub! /:sequence/, get_sequence_number_for(new_path)
-        new_path
-      end
-
-      def get_sequence_number_for path
-        new_path = path.split(":sequence")[0]
-        if File.directory?(new_path)
-          entries = Dir.entries(new_path).select { |i| i =~ /^\d+$/ } || []
-          last = entries.sort.last.to_i
-          last += 1
-          last.to_s
-        else
-          "1"
-        end
-      end
-
-      def extract_path_from_node node
-        if href_or_src = node.href.present? ? node.href : node.src
-          node.element.document.path.path_for(href_or_src)
-        else
-          throw Exception.new("no href no src")
-        end
-      end
-    end
-  end
-end

data/spec/skyscraper/skyscraper/node/base_spec.rb

@@ -1,87 +0,0 @@
-describe Skyscraper::Node::Base do
-  describe "when is initialized" do
-    before(:each) do
-      @node = Skyscraper::fetch(path_to("skyscraper-node-base.html")).first("div.item")
-    end
-    
-    it "should returns html code" do
-      @node.html.should include "<strong class=\"name\">Name value</strong>"
-    end
-
-    it "should returns class name" do
-      @node.class.should == "item"
-    end
-
-    it "should be auto converted to string with stripped tags" do
-      @node.text.should == "Name value"
-    end
-  end
-
-  it "should follow links" do
-    Skyscraper::fetch(path_to("skyscraper-node-base.html")).first("li a").follow.first("h1").text.should == "Hello from A"
-  end
-
-  it "should deep follow links" do
-    Skyscraper::fetch(path_to("skyscraper-node-base.html")).first("li a").follow.first("a").follow.first("h1").text.should == "Hello from B"
-  end
-
-  it "should download page" do
-    remove_test_directory
-    Skyscraper.config.download_path = "/tmp/skyscraper_test/nodes/:file_name"
-    file = Skyscraper::fetch(path_to("skyscraper-node-base.html")).first("li a").follow.first("a").download
-    File.exists?(file).should == true
-  end
-  describe "traversing" do
-    before(:each) do
-      @node = Skyscraper::fetch(path_to("skyscraper-node-base-traversing.html")).first(".menu")
-    end
-
-    it "should find descendands items" do
-      result = @node.find("li")
-      result.length.should == 5
-      result.map(&:text).should include "Item 4 1"
-    end
-
-    it "should returns children of element with selector" do
-      node = Skyscraper::fetch(path_to("skyscraper-node-base-traversing.html")).first("#parent-3")
-      node.children(".a").length.should == 4
-      node.children(".b").length.should == 2
-    end
-
-    it "should returns children of element without selector" do
-      result = @node.children
-      result.length.should == 4
-      result.map(&:to_s).should_not include "Item 4 1"
-    end
-
-    it "should returns first element" do
-      @node.first("li").class.should == "item-1"
-    end
-
-    it "should returns parent of item" do
-      @node.parent.class.should == "parent-2"
-    end
-
-    it "should tells if element have parent" do
-      @node.have_parent?.should == true
-      @node.parents("html").first.have_parent?.should == false
-    end
-
-    it "should returns parents of item" do
-      @node.parents.length.should == 4
-    end
-    
-    it "should returns parents of item matched by selector" do
-      @node.parents("div").length.should == 2
-    end
-
-    it "should returns siblings of item" do
-      @node.first(".item-3").siblings.length.should == 3
-    end
-
-    it "should returns node tag" do
-      @node.tag.should == "ul"
-    end
-  end
-end
-