klepto 0.2.3 → 0.2.5

data/README.md

@@ -11,6 +11,47 @@ A mean little DSL'd capybara (poltergeist) based web scraper that structures dat
 * Pretty nifty DSL
 * Test coverage!
 
+## Installing
+You need at least PhantomJS 1.8.1.  There are *no other external
+dependencies* (you don't need Qt, or a running X server, etc.)
+
+### Mac ###
+
+* *Homebrew*: `brew install phantomjs`
+* *MacPorts*: `sudo port install phantomjs`
+* *Manual install*: [Download this](http://code.google.com/p/phantomjs/downloads/detail?name=phantomjs-1.8.1-macosx.zip&can=2&q=)
+
+### Linux ###
+
+* Download the [32
+bit](http://code.google.com/p/phantomjs/downloads/detail?name=phantomjs-1.8.1-linux-i686.tar.bz2&can=2&q=)
+or [64
+bit](http://code.google.com/p/phantomjs/downloads/detail?name=phantomjs-1.8.1-linux-x86_64.tar.bz2&can=2&q=)
+binary.
+* Extract the tarball and copy `bin/phantomjs` into your `PATH`
+
+### Windows ###
+* Download the [precompiled binary](http://phantomjs.org/download.html) for Windows
+
+### Manual compilation ###
+
+Do this as a last resort if the binaries don't work for you. It will
+take quite a long time as it has to build WebKit.
+
+* Download [the source tarball](http://code.google.com/p/phantomjs/downloads/detail?name=phantomjs-1.8.1-source.zip&can=2&q=)
+* Extract and cd in
+* `./build.sh`
+
+(See also the [PhantomJS building guide](http://phantomjs.org/build.html).)
+
+Then put klepto in your gemfile.
+
+```ruby
+gem 'klepto', '>= 0.2.5'
+```
+
+
+
 ## Usage (All your content are belong to us)
 Say you want a bunch of Bieb tweets! How is there not profit in that?
 
@@ -121,6 +162,11 @@ end
 }
 ```
 
+## Configuration Options
+* config.headers - Hash; Sets request headers
+* config.urls    - Array(String); Sets URLs to structure
+* config.abort_on_failure - Boolean(Default: true); Should structuring be aborted on 4xx or 5xx
+
 ## Callbacks & Processing
 
 * before
@@ -128,6 +174,8 @@ end
 * after
   * :each (resource, Hash) - called for each resource parsed
   * :get (page, Capybara::Node) - called after each HTTP GET
+  * :abort (page, Capybara::Node) - called after a 4xx or 5xx if config.abort_on_failure is true (default)
+
 
 ## Stuff I'm going to add.
 Async 

data/lib/klepto/bot.rb

@@ -12,6 +12,9 @@ module Klepto
       instance_eval <<-EOS
 def queue; @queue; end;
 def resources; @resources; end;
+def method_missing(meth, *args, &block)
+  raise NoMethodError.new("undefined method: Klepto::Bot#" + meth.to_s)
+end
 EOS
 
       __process!
@@ -36,24 +39,34 @@ EOS
           config.dispatch_status_handlers(status, browser.page)
         end
 
-        structure = Structure.new(browser.page)
-
-        queue.each do |instruction|
-          if instruction[2]
-            structure.send instruction[0], *instruction[1], &instruction[2]
-          else
-            structure.send instruction[0], *instruction[1]
+        if !browser.failure? || (browser.failure? && !config.abort_on_failure?)
+          resources << __structure(browser.page)
+        else
+          config.after_handlers[:abort].each do |ah|
+            ah.call(browser.page)
           end
         end
+      end
+
+      @resources
+    end
+
+    def __structure(context)
+      structure = Structure.new(context)
 
-        config.after_handlers[:each].each do |ah|
-          ah.call(structure._hash)
+      queue.each do |instruction|
+        if instruction[2]
+          structure.send instruction[0], *instruction[1], &instruction[2]
+        else
+          structure.send instruction[0], *instruction[1]
         end
+      end
 
-        resources << structure._hash
+      config.after_handlers[:each].each do |ah|
+        ah.call(structure._hash)
       end
 
-      @resources
+      structure._hash
     end
 
     def method_missing(meth, *args, &block)

data/lib/klepto/browser.rb

@@ -14,6 +14,11 @@ module Klepto
       page.status_code
     end
 
+    # 4xx or 5xx
+    def failure?
+      ['4xx', '5xx'].include? statusx
+    end
+
     def statusx
       page.status_code.to_s[0..-3] + "xx"
     end

data/lib/klepto/config.rb

@@ -6,7 +6,11 @@ module Klepto
       @headers = {}
       @abort_on_failure = true
       @urls    = []
-      @after_handlers   = {:each => [], :get => []}
+      @after_handlers   = {
+        :each => [], 
+        :get  => [],
+        :abort=> []
+      }
       @before_handlers  = {:each => []}
       @status_handlers  = {}
     end
@@ -16,6 +20,10 @@ module Klepto
       @headers
     end
 
+    def abort_on_failure?
+      !!@abort_on_failure
+    end
+
     # 4xx, 5xx
     def abort_on_failure(aof)
       @abort_on_failure = aof

data/lib/klepto/version.rb

@@ -1,3 +1,3 @@
 module Klepto
-  VERSION = "0.2.3"
+  VERSION = "0.2.5"
 end

data/spec/lib/klepto/bot_spec.rb

@@ -27,6 +27,44 @@ describe Klepto::Bot do
       end
     end
 
+    describe 'aborting after a failure' do
+      before(:each) do
+        @bot = Klepto::Bot.new("https://github.com/coryodaniel/nonexistentproject"){
+          name 'h1.fullname'
+          config.abort_on_failure true
+          config.after(:abort) do |page|
+            StatusLog.create message: 'Aborted.'
+          end
+        }
+        @structure = @bot.resources
+      end      
+
+      it 'should abort after a 4xx or 5xx' do
+        StatusLog.first.message.should eq("Aborted.")
+      end
+    end
+
+    describe 'structuring a 4xx or 5xx response' do
+      before(:each) do
+        @bot = Klepto::Bot.new("https://github.com/coryodaniel/nonexistentproject"){
+          cat 'img#parallax_octocat', :attr => :alt
+          config.abort_on_failure false
+          config.after(:abort) do |page|
+            StatusLog.create message: 'Aborted.'
+          end
+        }
+        @structure = @bot.resources
+      end      
+
+      it 'should perform structuring' do
+        @structure.first[:cat].should eq('Octobi Wan Catnobi')
+      end
+
+      it 'should not abort after a 4xx or 5xx' do
+        StatusLog.first.should be(nil)
+      end
+    end
+
     describe 'structuring multiple pages' do
       before(:each) do
         @bot = Klepto::Bot.new("https://twitter.com/justinbieber"){
@@ -118,6 +156,13 @@ describe Klepto::Bot do
         Tweet.count.should_not be(0)
       end
 
+      it 'should not have the DSL once its been processed' do
+        lambda{
+          @bot.i_dont_exist
+        }.should raise_error(NoMethodError)
+        
+      end
+
       it 'should have dispatched status handlers' do
         statuses = StatusLog.all.map(&:message)
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: klepto
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.2.5
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2013-04-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: poltergeist
-  requirement: &70237982927260 !ruby/object:Gem::Requirement
+  requirement: &70341397514440 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - =
@@ -21,10 +21,10 @@ dependencies:
         version: 1.1.0
   type: :runtime
   prerelease: false
-  version_requirements: *70237982927260
+  version_requirements: *70341397514440
 - !ruby/object:Gem::Dependency
   name: capybara
-  requirement: &70237982925860 !ruby/object:Gem::Requirement
+  requirement: &70341397512360 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - =
@@ -32,10 +32,10 @@ dependencies:
         version: 2.0.2
   type: :runtime
   prerelease: false
-  version_requirements: *70237982925860
+  version_requirements: *70341397512360
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70237982924820 !ruby/object:Gem::Requirement
+  requirement: &70341397509940 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: 1.5.6
   type: :runtime
   prerelease: false
-  version_requirements: *70237982924820
+  version_requirements: *70341397509940
 - !ruby/object:Gem::Dependency
   name: activesupport
-  requirement: &70237982923900 !ruby/object:Gem::Requirement
+  requirement: &70341397506060 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70237982923900
+  version_requirements: *70341397506060
 - !ruby/object:Gem::Dependency
   name: multi_json
-  requirement: &70237982922600 !ruby/object:Gem::Requirement
+  requirement: &70341397504900 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,7 +65,7 @@ dependencies:
         version: '1.0'
   type: :runtime
   prerelease: false
-  version_requirements: *70237982922600
+  version_requirements: *70341397504900
 description: Tearing up web pages into ActiveRecord resources
 email:
 - github@coryodaniel.com