skyscraper 0.0.5 → 0.1.0

data/.rspec

@@ -1 +1 @@
---color
+--color --fail-fast --format nested 

data/lib/skyscraper.rb

@@ -2,6 +2,7 @@ require "open-uri"
 require "uri"
 require "nokogiri"
 require "active_support/core_ext"
+require "net/http"
 
 include ActiveSupport
 
@@ -38,6 +39,7 @@ module Skyscraper
     Node.new document.css("html")
   end
 
+
   def fetch
     self.class.send(:base).fetch 
   end

data/lib/skyscraper/document.rb

@@ -3,9 +3,29 @@ module Skyscraper
     attr_accessor :path
 
     def self.load path, encoding = 'utf-8'
-      document = Skyscraper::Document.parse open(path), nil, encoding
+      document = Skyscraper::Document.parse open_from_path(path), nil, encoding
       document.path = Skyscraper::Path.factory(path)
       document
     end
+
+    def self.load_post path, params = {}, encoding = 'utf-8'
+      file = Net::HTTP.post_form(URI.parse(path), params).body
+
+      document = Skyscraper::Document.parse file
+      document.path = path
+      document
+    end
+
+    def self.open_from_path path
+      begin 
+        file = open(path)
+      rescue RuntimeError
+        https_path = path.gsub /http:/, "https:"
+        file = open(https_path) unless https_path == path
+      end
+
+      file
+    end
+
   end
 end

data/lib/skyscraper/node.rb

@@ -97,5 +97,20 @@ module Skyscraper
     def tag
       @element.name
     end
+
+    def submit params = {}
+      raise Skyscraper::LocalFormException if @element.document.path.local?
+      raise Skyscraper::NotActionException if self.action.blank?
+
+      path = @element.document.path.full_path_for(self.action)
+      document  = Skyscraper::Document.load_post path, params
+
+      Node.new(document.css("html"))
+    end
+  end
+
+  class LocalFormException < Exception
+  end
+  class NotActionException < Exception
   end
 end

data/lib/skyscraper/path/base.rb

@@ -5,6 +5,14 @@ module Skyscraper
         Path::factory(self.full_path_for(path))
       end
 
+      def local?
+        self.is_a? Skyscraper::Path::Local
+      end
+
+      def remote?
+        self.is_a? Skyscraper::Path::Remote
+      end
+
       private
 
       def get_file_name path

data/lib/skyscraper/results.rb

@@ -40,8 +40,9 @@ module Skyscraper
             result[field.name] = field.find_in_document document 
           end
 
-          call_callbacks @after_each, result, document
+          callback_result = call_callbacks @after_each, result, document
           results << result
+          break if callback_result === false
           sleep @delay[:sleep] if (i+1) % @delay[:after] == 0
 
         rescue SocketError, Errno::ENOENT
@@ -82,9 +83,11 @@ module Skyscraper
     end
 
     def call_callbacks callbacks, *args
+      result = true
       callbacks.each do |callback|
-        callback.call(*args)
+        result = callback.call(*args)
       end
+      result
     end
   end
 

data/lib/skyscraper/version.rb

@@ -1,3 +1,3 @@
 module Skyscraper
-  VERSION = "0.0.5"
+  VERSION = "0.1.0"
 end

data/spec/skyscraper/skyscraper/base_spec.rb

@@ -1,3 +1,5 @@
+require "spec_helper"
+
 describe Skyscraper::Base do
 
   it "should set pages" do

data/spec/skyscraper/skyscraper/config_spec.rb

@@ -1,3 +1,5 @@
+require "spec_helper"
+
 describe Skyscraper::Config do
   it "should set variable on initialize" do
     config = Skyscraper::Config.new foo: "bar"

data/spec/skyscraper/skyscraper/document_spec.rb

@@ -1,4 +1,5 @@
 #encoding: utf-8
+require "spec_helper"
 
 describe Skyscraper::Document do
   it "should support utf-8 encoding by default in remote pages" do
@@ -11,4 +12,11 @@ describe Skyscraper::Document do
     document = Skyscraper::Document::load(path_to("skyscraper-document.html"))
     document.path.should be_an Skyscraper::Path::Base
   end
+
+  describe "when is opening" do
+    it "should handle https redirects" do
+      Skyscraper::Document.open_from_path("http://github.com").should be_an_instance_of Tempfile
+    end
+  end
+
 end

data/spec/skyscraper/skyscraper/field_spec.rb

@@ -1,3 +1,5 @@
+require "spec_helper"
+
 describe Skyscraper::Field do
   before(:all) do
     @page = Skyscraper::fetch(path_to("skyscraper-field.html"))

data/spec/skyscraper/skyscraper/node_spec.rb

@@ -1,3 +1,5 @@
+require "spec_helper"
+
 describe Skyscraper::Node do
   describe "when is initialized" do
     before(:each) do
@@ -83,5 +85,32 @@ describe Skyscraper::Node do
       @node.tag.should == "ul"
     end
   end
+
+  describe "Submit post data" do
+    it "should submit form post data" do
+      node = Skyscraper::fetch("http://www.balticplaza.eu/kontakt").first("#new_inquiry") 
+      submited_page = node.submit(:"inquiry[name]" => "Example name")
+      submited_page.first("#inquiry_name").value.should == "Example name"
+    end
+
+    it "should throws an LocalFormException" do
+      lambda do
+        node = Skyscraper::fetch(path_to("skyscraper-node-form.html"))
+        node.first("form").submit
+      end.should raise_error Skyscraper::LocalFormException
+    end
+
+    it "should throws NotActionException" do
+      Skyscraper::Path::Local.any_instance.stub(:local? => false)
+
+      lambda do
+        Skyscraper::fetch(path_to("skyscraper-node-traversing.html")).first(".menu").submit
+      end.should raise_error Skyscraper::NotActionException
+    end
+
+    it "should handle GET form method" do
+      pending "todo"
+    end
+  end
 end
 

data/spec/skyscraper/skyscraper/pages_spec.rb

@@ -1,3 +1,5 @@
+require "spec_helper"
+
 describe Skyscraper::Pages do
   it "should set convert string to items array" do
     Skyscraper::Pages.new("http://google.com").items.should == ["http://google.com"]

data/spec/skyscraper/skyscraper/path_spec.rb

@@ -6,6 +6,14 @@ describe Skyscraper::Path do
       @path = Skyscraper::Path.factory("http://google.com/index.php?q=e")
     end
 
+    it "local? method should returns false" do
+      @path.local?.should == false
+    end
+
+    it "remote? method should returns true" do
+      @path.remote?.should == true
+    end
+
     it "should returns domain" do
       @path.domain.should == "http://google.com"
     end
@@ -65,6 +73,14 @@ describe Skyscraper::Path do
       @path.folder.should == "/var/www/files/"
     end
 
+    it "local? method should returns true" do
+      @path.local?.should == true
+    end
+
+    it "remote? method should returns false" do
+      @path.remote?.should == false
+    end
+
     it "should returns file name" do
       @path.file_name.should == "file.ext"
     end

data/spec/skyscraper/skyscraper/resource_spec.rb

@@ -1,3 +1,5 @@
+require "spec_helper"
+
 describe Skyscraper::Resource do
   def should_download_resource_to node, path, options = {}
     resource = Skyscraper::Resource.new(node)

data/spec/skyscraper/skyscraper/results_spec.rb

@@ -1,3 +1,5 @@
+require "spec_helper"
+
 describe Skyscraper::Results do
   def fetch options = {}
     options.reverse_merge! fields: {}, options: {} 
@@ -72,7 +74,7 @@ describe Skyscraper::Results do
         page.should be_an_instance_of(Skyscraper::Node)
         @call_count += 1
       end
-
+      
       results = fetch path: [path_to("skyscraper-fetch.html")] * 10, options: { after_each: [callback] }
 
       @call_count.should == 10
@@ -117,6 +119,21 @@ describe Skyscraper::Results do
       results[0][:h1].should == "Hello world"
     end
 
+    it "should stops when after each callback returns false" do
+      counter = 0
+
+      callback = proc do 
+        if counter == 1
+          counter = 0
+          false
+        else
+          counter += 1
+        end
+      end
+
+      results = fetch path: [path_to("skyscraper-fetch.html")] * 10, fields: { h1: "h1" }, options: { after_each: [callback] }
+      results.length.should == 2
+    end
   end
 
   describe "errors" do

data/spec/skyscraper/skyscraper_spec.rb

@@ -1,5 +1,7 @@
 #encoding: utf-8
 
+require "spec_helper"
+
 class TestScraper
   include Skyscraper
 
@@ -27,6 +29,10 @@ describe Skyscraper do
     Skyscraper::fetch(path_to("skyscraper-encoding.html")).first(".utf-8").text.should == "ąśćżół"
   end
 
+  it "should handle http -> https redirects" do
+    Skyscraper::fetch("http://github.com").first("title").text.should =~ /GitHub/
+  end
+
   it "should works when included" do
     TestScraper.new.fetch[0][:h1].should == "Hello world"
     TestScraper.new.fetch[1][:h1].should == "Hello world"

data/spec/test_files/skyscraper-node-form.html

@@ -0,0 +1,13 @@
+<!DOCTYPE HTML>
+<html>
+    <head>
+        <title>Title</title>
+    </head>
+
+    <body>
+        <form method="post">
+
+        </form>
+    </body>
+</html>
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: skyscraper
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.1.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-05-21 00:00:00.000000000 Z
+date: 2012-05-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &77062930 !ruby/object:Gem::Requirement
+  requirement: &78066490 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *77062930
+  version_requirements: *78066490
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &77124830 !ruby/object:Gem::Requirement
+  requirement: &78128330 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *77124830
+  version_requirements: *78128330
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &77122840 !ruby/object:Gem::Requirement
+  requirement: &78126140 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *77122840
+  version_requirements: *78126140
 - !ruby/object:Gem::Dependency
   name: actionpack
-  requirement: &77119590 !ruby/object:Gem::Requirement
+  requirement: &78123100 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,7 +54,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *77119590
+  version_requirements: *78123100
 description: Easy to use DSL that helps scraping data from websites. Thanks to it,
   writing web crawlers would be very fast and intuitive. Traversing through html nodes
   and fetching all of the HTML attributes, would be possible. Just like in jQuery
@@ -109,6 +109,7 @@ files:
 - spec/test_files/skyscraper-field.html
 - spec/test_files/skyscraper-node-a.html
 - spec/test_files/skyscraper-node-b.html
+- spec/test_files/skyscraper-node-form.html
 - spec/test_files/skyscraper-node-traversing.html
 - spec/test_files/skyscraper-node.html
 - spec/test_files/skyscraper-pages.html
@@ -162,6 +163,7 @@ test_files:
 - spec/test_files/skyscraper-field.html
 - spec/test_files/skyscraper-node-a.html
 - spec/test_files/skyscraper-node-b.html
+- spec/test_files/skyscraper-node-form.html
 - spec/test_files/skyscraper-node-traversing.html
 - spec/test_files/skyscraper-node.html
 - spec/test_files/skyscraper-pages.html