skyscraper 0.0.1

data/spec/skyscraper/skyscraper/field_spec.rb

@@ -0,0 +1,36 @@
+describe Skyscraper::Field do
+  before(:all) do
+    @page = Skyscraper::fetch(path_to("skyscraper-field.html"))
+  end
+
+  it "should find field value using css selector" do
+    field = Skyscraper::Field.new name: :name, selector: ".item strong.name"
+    field.find_in_document @page
+    field.value.should == "Name value"
+  end
+
+  it "should apply callback" do
+    callback = proc { |item| item.href * 2 } 
+    field = Skyscraper::Field.new name: :name, selector: "a", callback: callback
+    field.find_in_document @page
+    field.value.should == "a.htmla.html"
+  end
+
+  it "should read attributes from elements" do
+    field = Skyscraper::Field.new name: :name, selector: "a", attribute: :href
+    field.find_in_document @page
+    field.value.should == "a.html"
+  end
+
+  it "should returns text code of inner element by default" do
+    field = Skyscraper::Field.new name: :name, selector: ".item"
+    field.find_in_document @page
+    field.value.should include "Name value"
+  end
+
+  it "should returns html code of inner element" do
+    field = Skyscraper::Field.new name: :name, selector: ".item", attribute: "html"
+    field.find_in_document @page
+    field.value.should include "<strong class=\"name\">Name value</strong>"
+  end
+end

data/spec/skyscraper/skyscraper/node/base_spec.rb

@@ -0,0 +1,87 @@
+describe Skyscraper::Node::Base do
+  describe "when is initialized" do
+    before(:each) do
+      @node = Skyscraper::fetch(path_to("skyscraper-node-base.html")).first("div.item")
+    end
+    
+    it "should returns html code" do
+      @node.html.should include "<strong class=\"name\">Name value</strong>"
+    end
+
+    it "should returns class name" do
+      @node.class.should == "item"
+    end
+
+    it "should be auto converted to string with stripped tags" do
+      @node.text.should == "Name value"
+    end
+  end
+
+  it "should follow links" do
+    Skyscraper::fetch(path_to("skyscraper-node-base.html")).first("li a").follow.first("h1").text.should == "Hello from A"
+  end
+
+  it "should deep follow links" do
+    Skyscraper::fetch(path_to("skyscraper-node-base.html")).first("li a").follow.first("a").follow.first("h1").text.should == "Hello from B"
+  end
+
+  it "should download page" do
+    remove_test_directory
+    Skyscraper.config.download_path = "/tmp/skyscraper_test/nodes/:file_name"
+    file = Skyscraper::fetch(path_to("skyscraper-node-base.html")).first("li a").follow.first("a").download
+    File.exists?(file).should == true
+  end
+  describe "traversing" do
+    before(:each) do
+      @node = Skyscraper::fetch(path_to("skyscraper-node-base-traversing.html")).first(".menu")
+    end
+
+    it "should find descendands items" do
+      result = @node.find("li")
+      result.length.should == 5
+      result.map(&:text).should include "Item 4 1"
+    end
+
+    it "should returns children of element with selector" do
+      node = Skyscraper::fetch(path_to("skyscraper-node-base-traversing.html")).first("#parent-3")
+      node.children(".a").length.should == 4
+      node.children(".b").length.should == 2
+    end
+
+    it "should returns children of element without selector" do
+      result = @node.children
+      result.length.should == 4
+      result.map(&:to_s).should_not include "Item 4 1"
+    end
+
+    it "should returns first element" do
+      @node.first("li").class.should == "item-1"
+    end
+
+    it "should returns parent of item" do
+      @node.parent.class.should == "parent-2"
+    end
+
+    it "should tells if element have parent" do
+      @node.have_parent?.should == true
+      @node.parents("html").first.have_parent?.should == false
+    end
+
+    it "should returns parents of item" do
+      @node.parents.length.should == 4
+    end
+    
+    it "should returns parents of item matched by selector" do
+      @node.parents("div").length.should == 2
+    end
+
+    it "should returns siblings of item" do
+      @node.first(".item-3").siblings.length.should == 3
+    end
+
+    it "should returns node tag" do
+      @node.tag.should == "ul"
+    end
+  end
+end
+

data/spec/skyscraper/skyscraper/node/resource_spec.rb

@@ -0,0 +1,58 @@
+describe Skyscraper::Node::Resource do
+  def should_download_resource_to node, path, options = {}
+    resource = Skyscraper::Node::Resource.new(node)
+    resource.download(options).should == path
+  end
+
+  before(:all) do
+    Skyscraper.config.download_path = "/tmp/skyscraper_test/:sequence/:file_name"
+    @node = Skyscraper::fetch(path_to("skyscraper-node-resource.html")).first("a")
+  end
+
+  before(:each) do
+    remove_test_directory
+  end
+
+  it "should create path if not exists when downloaded" do
+    File.directory?("/tmp/skyscraper_test/1").should == false
+    Skyscraper::Node::Resource.new(@node)
+    File.directory?("/tmp/skyscraper_test/1").should == false
+    Skyscraper::Node::Resource.new(@node).download
+    File.directory?("/tmp/skyscraper_test/1").should == true
+  end
+
+  it "should not fail if path already exists" do
+    Skyscraper::Node::Resource.new(@node).download path: "/tmp/skyscraper_test/some_directory/:file_name"
+    File.directory?("/tmp/skyscraper_test/some_directory").should == true
+    Skyscraper::Node::Resource.new(@node).download
+    File.directory?("/tmp/skyscraper_test/some_directory").should == true
+  end
+
+  it "should have file name" do
+    resource = Skyscraper::Node::Resource.new(@node)
+    resource.download.should == "/tmp/skyscraper_test/1/skyscraper-node-resource-b.html"
+  end
+
+  it "should create path with :sequence variable" do
+    download_to = "/tmp/skyscraper_test/sequences/:sequence/:file_name"
+    should_download_resource_to @node, "/tmp/skyscraper_test/sequences/1/skyscraper-node-resource-b.html", path: download_to
+    should_download_resource_to @node, "/tmp/skyscraper_test/sequences/2/skyscraper-node-resource-b.html", path: download_to
+    should_download_resource_to @node, "/tmp/skyscraper_test/sequences/3/skyscraper-node-resource-b.html", path: download_to
+  end
+
+  it "should create custom file name if provided" do
+    download_to = "/tmp/skyscraper_test/custom_name/:file_name"
+    should_download_resource_to @node, "/tmp/skyscraper_test/custom_name/test.html", path: download_to, file_name: "test.html"
+  end
+
+  it "should download resource" do
+    Skyscraper::Node::Resource.new(@node).download
+    File.exists?("/tmp/skyscraper_test/1/skyscraper-node-resource-b.html").should == true
+  end
+
+  it "should download image" do
+    image_node = Skyscraper::fetch(path_to("skyscraper-node-resource.html")).first("img")
+    Skyscraper::Node::Resource.new(image_node).download
+    File.exists?("/tmp/skyscraper_test/1/skyscraper-node-resource-image.png").should == true
+  end
+end

data/spec/skyscraper/skyscraper/node_spec.rb

@@ -0,0 +1,2 @@
+describe Skyscraper::Node do
+end

data/spec/skyscraper/skyscraper/pages_spec.rb

@@ -0,0 +1,46 @@
+describe Skyscraper::Pages do
+  it "should set convert string to items array" do
+    Skyscraper::Pages.new("http://google.com").items.should == ["http://google.com"]
+  end
+
+  it "should set items array from array" do
+    Skyscraper::Pages.new(["http://google.com"]).items.should == ["http://google.com"]
+  end
+
+  it "should flat pages from nested arrays" do
+    Skyscraper::Pages.new(["http://google.com", ["http://yahoo.com"]]).items.should == ["http://google.com", "http://yahoo.com"]
+  end
+
+  it "should set array from block" do
+    Skyscraper::Pages.new do
+      2.times.map { |i| "http://google.com/#{i}.html"}
+    end.items.should == ["http://google.com/0.html", "http://google.com/1.html"]
+  end
+
+  it "should pass scraper instance to block" do
+    Skyscraper::Pages.new do |scraper|
+      scraper.fetch(path_to("skyscraper-pages.html")).first("a").href
+    end.items.should == ["a.html"]
+  end
+
+  it "should works when block is passed without arguments" do
+    Skyscraper::Pages.new do 
+      "a.html"
+    end.items.should == ["a.html"]
+  end
+
+  it "should return next item" do
+    pages = Skyscraper::Pages.new(["a", "b", "c"])
+    pages.next.should == "a"
+    pages.next.should == "b"
+    pages.next.should == "c"
+  end
+
+  it "should reset pages" do
+    pages = Skyscraper::Pages.new(["a", "b", "c"])
+    pages.next.should == "a"
+    pages.next.should == "b"
+    pages.reset
+    pages.next.should == "a"
+  end
+end

data/spec/skyscraper/skyscraper/path_spec.rb

@@ -0,0 +1,110 @@
+require "spec_helper"
+
+describe Skyscraper::Path do
+  describe "when path is REMOTE" do
+    before(:each) do
+      @path = Skyscraper::Path.factory("http://google.com/index.php?q=e")
+    end
+
+    it "should returns domain" do
+      @path.domain.should == "http://google.com"
+    end
+
+    it "should returns domain with no scheme" do
+      @path = Skyscraper::Path.factory("google.com/index.php?q=e")
+      @path.full_path.should == "google.com/index.php?q=e"
+    end
+
+    it "should returns path" do
+      @path.path.should == "/index.php"
+    end
+
+    it "should returns query" do
+      @path.query.should == "?q=e"
+    end
+
+    it "should returns base" do
+      @path.base.should == "http://google.com/"
+    end
+
+    it "should returns full path" do
+      @path.full_path.should == "http://google.com/index.php?q=e"
+    end
+
+    it "should be converted to string" do
+      @path.to_s.should == @path.full_path
+    end
+
+    it "should returns full path for full different" do
+      path = Skyscraper::Path.factory("http://google.com/a/index.php")
+      path.full_path_for("http://yahoo.com/b.html").should == "http://yahoo.com/b.html"
+    end
+
+    it "should returns full path for relative" do
+      path = Skyscraper::Path.factory("http://google.com/a/index.php")
+      path.full_path_for("b.html").should == "http://google.com/a/b.html"
+    end
+   
+    it "should returns full path for absolute" do
+      path = Skyscraper::Path.factory("http://google.com/a/index.php")
+      path.full_path_for("/b.html").should == "http://google.com/b.html"
+    end
+
+    it "should returns file name" do
+      path = Skyscraper::Path.factory("http://google.com/a/index.php")
+      path.file_name.should == "index.php"
+    end
+  end
+
+  describe "when path is LOCAL" do
+    before(:each) do
+      @path = Skyscraper::Path.factory("/var/www/files/file.ext")
+    end
+
+    it "should returns folder" do
+      @path.folder.should == "/var/www/files/"
+    end
+
+    it "should returns file name" do
+      @path.file_name.should == "file.ext"
+    end
+
+    it "should returns full path" do
+      @path.full_path.should == "/var/www/files/file.ext"
+    end
+
+    it "should returns base" do
+      @path.base.should == "/var/www/files/"
+    end
+
+    it "should returns full path for relative" do
+      path = Skyscraper::Path.factory("/var/www/public/index.html")
+      path.full_path_for("../b.html").should == "/var/www/public/../b.html"
+      path.full_path_for("b.html").should == "/var/www/public/b.html"
+    end
+   
+    it "should returns full path for absolute full" do
+      path = Skyscraper::Path.factory("/var/www/public/index.html")
+      path.full_path_for("/var/www/test.html").should == "/var/www/test.html"
+    end
+  end
+
+  it "should detect if string is remote " do
+    Skyscraper::Path.remote?("http://google.com").should == true
+    Skyscraper::Path.remote?("google.com").should == true
+  end
+
+  it "should detect if string is not remote " do
+    Skyscraper::Path.remote?("/var/www/projects").should == false
+    Skyscraper::Path.remote?("/var/www/projects/file.ext").should == false
+  end
+
+  it "should check if is absolute address" do
+    Skyscraper::Path.absolute?("/some/relative/path").should == true
+  end
+
+  it "should returns nil for wrong path" do
+    path = Skyscraper::Path.factory("/var/www/files/")
+    path.file_name.should == nil
+  end
+end

data/spec/skyscraper/skyscraper/results_spec.rb

@@ -0,0 +1,151 @@
+describe Skyscraper::Results do
+  def fetch options = {}
+    options.reverse_merge! fields: {}, options: {} 
+    base = Skyscraper::Base.new
+    base.pages options[:path]
+
+    options[:fields].each_pair do |key, value|
+      base.field key, value
+    end
+
+    Skyscraper::Results.new(base, options[:options]).fetch
+  end
+
+  it "should fetch file content" do
+    results = fetch path: path_to("skyscraper-fetch.html"), fields: { h1: "h1" }
+    results[0][:h1].should == "Hello world"
+  end
+
+  it "should fetch with delay" do
+    time = Time.now
+    results = fetch path: [path_to("skyscraper-fetch.html")] * 2, fields: { title: "title" }, options: { delay: 1 }
+    time_diff = Time.now - time
+    time_diff.should > 1
+  end
+
+  it "should fetch with delay after" do
+    time = Time.now
+    results = fetch path: [path_to("skyscraper-fetch.html")] * 10, options: { delay: { sleep: 1, after: 7 } }
+    time_diff = Time.now - time
+    time_diff.should > 1
+    time_diff.should < 3
+
+    time = Time.now
+    results = fetch path: [path_to("skyscraper-fetch.html")] * 10, options: { delay: { sleep: 1, after: 11 } }
+    time_diff = Time.now - time
+    time_diff.should < 1
+  end
+
+  it "should fetch with results limit" do
+    results = fetch path: [path_to("skyscraper-fetch.html")] * 11, options: { limit: 10 }
+    results.length.should == 10
+  end
+
+  it "should apply config defaults" do
+    base = Skyscraper::Base.new
+    base.config.limit = 2
+    base.pages [path_to("skyscraper-fetch.html")] * 10
+
+    results = Skyscraper::Results.new(base).fetch
+    results.length.should == 2
+  end
+
+  it "should continue after limit reached" do
+    base = Skyscraper::Base.new
+    base.config.limit = 1
+    base.pages [path_to("skyscraper-fetch.html"), path_to("skyscraper-fetch-2.html")]
+    base.field :h1, "h1"
+
+    results = Skyscraper::Results.new(base)
+    results.fetch.length.should == 1
+    results.records.length.should == 1
+    results.continue.length.should == 1
+    results.records.length.should == 2
+    results.records[1][:h1].should == "Hello from A"
+  end
+
+  describe "callbacks" do
+    it "should calls after each page callback" do
+      @call_count = 0
+      callback = proc do |result, page|  
+        result.should be_an_instance_of(Hash)
+        page.should be_an_instance_of(Skyscraper::Node::Base)
+        @call_count += 1
+      end
+
+      results = fetch path: [path_to("skyscraper-fetch.html")] * 10, options: { after_each: [callback] }
+
+      @call_count.should == 10
+    end
+
+    it "should calls after all callback" do
+      @call_count = 0
+      callback = proc do |results|  
+        results.should be_an_instance_of(Array)
+        @call_count += 1
+      end
+
+      results = fetch path: [path_to("skyscraper-fetch.html")] * 10, options: { after_all: [callback] }
+
+      @call_count.should == 1
+    end
+
+    it "should change result value for each" do
+      callback = proc do |result, page|  
+        result[:h1] += " with callback"
+      end
+
+      results = fetch path: [path_to("skyscraper-fetch.html")] * 10, fields: { h1: "h1" }, options: { after_each: [callback] }
+      results[0][:h1].should == "Hello world with callback"
+    end
+
+    it "should change results values for all" do
+      callback = proc do |results|  
+        results << "test"
+      end
+
+      results = fetch path: [path_to("skyscraper-fetch.html")] * 10, fields: { h1: "h1" }, options: { after_all: [callback] }
+      results.last.should == "test"
+    end
+
+    it "should doesn't requires callback arguments" do
+      callback = proc do 
+        "with callback"
+      end
+
+      results = fetch path: [path_to("skyscraper-fetch.html")] * 10, fields: { h1: "h1" }, options: { after_each: [callback] }
+      results[0][:h1].should == "Hello world"
+    end
+
+  end
+
+  describe "errors" do
+    before(:all) do
+      Skyscraper.config.noise_errors = false
+    end
+
+    it "should catch invalid url exception" do
+      Skyscraper.config.skip_on_error = false
+      lambda do
+        fetch path: "http://google.wrong" 
+      end.should raise_error Skyscraper::NoResourceException
+    end
+
+    it "should catch file not exists exception" do
+      Skyscraper.config.skip_on_error = false
+      lambda do
+        fetch path: "/tmp/skyscraper/unknow_file" 
+      end.should raise_error Skyscraper::NoResourceException
+    end
+
+    it "should skip on error" do
+      Skyscraper.config.skip_on_error = true
+      begin 
+        fetch path: "http://google.wrong"
+      rescue Skyscraper::NoResourceException
+        catched = true
+      end
+      catched.should == nil
+    end
+  end
+end