klepto 0.2.2 → 0.2.3

data/README.md

@@ -1,6 +1,6 @@
 # Klepto
 
-A mean little DSL'd capybara (poltergeist) based web crawler that stuffs data into ActiveRecord or wherever(TM).
+A mean little DSL'd capybara (poltergeist) based web scraper that structures data into ActiveRecord or wherever(TM).
 
 ## Features 
 
@@ -8,7 +8,6 @@ A mean little DSL'd capybara (poltergeist) based web crawler that stuffs data in
 * Full javascript processing via phantomjs / poltergeist
 * All the fun of capybara
 * Scrape multiple pages with a single bot
-* Scrape individuals pages with multiple 'crawlers', see Bieber example.
 * Pretty nifty DSL
 * Test coverage!
 
@@ -16,8 +15,8 @@ A mean little DSL'd capybara (poltergeist) based web crawler that stuffs data in
 Say you want a bunch of Bieb tweets! How is there not profit in that?
 
 ```ruby
-# Crawl a web site or multiple. Structure#crawl takes a *splat!
-@structures = Klepto::Bot.new("https://twitter.com/justinbieber"){
+# Fetch a web site or multiple. Bot#new takes a *splat!
+@bot = Klepto::Bot.new("https://twitter.com/justinbieber"){
   # By default, it uses CSS selectors
   name      'h1.fullname'
 
@@ -73,6 +72,10 @@ Say you want a bunch of Bieb tweets! How is there not profit in that?
     puts "HOLY CRAP!"
   }
 
+  config.after(:get) do |page|
+    # This is fired after each HTTP GET. It receives a Capybara::Node
+  end  
+
   # If you want to do something with each resource, like stick it in AR
   #   go for it here...
   config.after do |resource|
@@ -87,10 +90,10 @@ Say you want a bunch of Bieb tweets! How is there not profit in that?
   end #=> Profit!
 }
 
-# An array of hashes is returned, so if you wanted to do something else 
+# You can get an array of hashes(resources), so if you wanted to do something else 
 # you could do it here...
-@structures.each do |structure|
-  pp structure
+@bot.resources.each do |resource|
+  pp resource
 end
 ```
 
@@ -118,8 +121,19 @@ end
 }
 ```
 
+## Callbacks & Processing
+
+* before
+  * n/a
+* after
+  * :each (resource, Hash) - called for each resource parsed
+  * :get (page, Capybara::Node) - called after each HTTP GET
 
 ## Stuff I'm going to add.
+Async 
+--------
+-> https://github.com/igrigorik/em-synchrony
+
 Cookie Stuffing
 -------------------
 ```ruby

data/lib/klepto/bot.rb

@@ -17,13 +17,6 @@ EOS
       __process!
     end
 
-    def __dispatch_handlers_for(status_code)
-      if status_code.is_a?(Fixnum)
-      elsif status_code.is_a?(Symbol)
-      elsif status_code.is_a?(String)
-      end
-    end
-
     def __process!
       @resources = []
 
@@ -32,6 +25,10 @@ EOS
 
         browser.set_headers config.headers
         browser.fetch! url
+
+        config.after_handlers[:get].each do |ah|
+          ah.call(browser.page)
+        end
         
         statuses = [browser.status, browser.statusx]
         statuses.push :redirect if url != browser.page.current_url

data/lib/klepto/config.rb

@@ -4,8 +4,9 @@ module Klepto
 
     def initialize
       @headers = {}
+      @abort_on_failure = true
       @urls    = []
-      @after_handlers   = {:each => []}
+      @after_handlers   = {:each => [], :get => []}
       @before_handlers  = {:each => []}
       @status_handlers  = {}
     end
@@ -15,6 +16,11 @@ module Klepto
       @headers
     end
 
+    # 4xx, 5xx
+    def abort_on_failure(aof)
+      @abort_on_failure = aof
+    end
+
     def on_http_status(*statuses,&block)
       statuses.each do |status|
         @status_handlers[status] ||= []
@@ -36,6 +42,11 @@ module Klepto
       @after_handlers[which].push block
     end
 
+    def before(which, &block)
+      @before_handlers[which] ||= []
+      @before_handlers[which].push block
+    end    
+
     def url(*args)
       @urls += args
       @urls.flatten!

data/lib/klepto/version.rb

@@ -1,3 +1,3 @@
 module Klepto
-  VERSION = "0.2.2"
+  VERSION = "0.2.3"
 end

data/spec/lib/klepto/bot_spec.rb

@@ -27,7 +27,7 @@ describe Klepto::Bot do
       end
     end
 
-    describe 'crawling multiple pages' do
+    describe 'structuring multiple pages' do
       before(:each) do
         @bot = Klepto::Bot.new("https://twitter.com/justinbieber"){
           config.urls "https://twitter.com/ladygaga"
@@ -76,6 +76,10 @@ describe Klepto::Bot do
             permalink '.time a', :css, :attr => :href
           end
 
+          config.after(:get) do |page|
+            StatusLog.create message: 'got a page'
+          end
+
           config.on_http_status('2xx'){
             StatusLog.create message: '2xx'
           }
@@ -120,6 +124,7 @@ describe Klepto::Bot do
         statuses.should_not include 'redirect'
         statuses.should include '200'
         statuses.should include '2xx'
+        statuses.should include 'got a page'
       end
     end
   end

data/spec/lib/klepto/config_spec.rb

@@ -14,6 +14,7 @@ describe Klepto::Config do
     @config.on_http_status('5xx','4xx'){
       "Its crazy."
     }
+    @config.abort_on_failure(false)
   end
 
   it 'should be able to set headers' do
@@ -37,7 +38,10 @@ describe Klepto::Config do
     @config.urls.should == ['http://example.com', 'http://www.iana.org']
   end
 
-  pending 'should be able to set before handlers'  
+  it 'should have an abort on 4xx/5xx option' do
+    @config.instance_variable_get("@abort_on_failure").should be false
+  end
+
   pending 'should be able to set cookies'
   pending 'should be able to set steps'
   pending 'should be able to set assertions'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: klepto
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.3
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2013-04-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: poltergeist
-  requirement: &70235243869100 !ruby/object:Gem::Requirement
+  requirement: &70237982927260 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - =
@@ -21,10 +21,10 @@ dependencies:
         version: 1.1.0
   type: :runtime
   prerelease: false
-  version_requirements: *70235243869100
+  version_requirements: *70237982927260
 - !ruby/object:Gem::Dependency
   name: capybara
-  requirement: &70235243868100 !ruby/object:Gem::Requirement
+  requirement: &70237982925860 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - =
@@ -32,10 +32,10 @@ dependencies:
         version: 2.0.2
   type: :runtime
   prerelease: false
-  version_requirements: *70235243868100
+  version_requirements: *70237982925860
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70235243867320 !ruby/object:Gem::Requirement
+  requirement: &70237982924820 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: 1.5.6
   type: :runtime
   prerelease: false
-  version_requirements: *70235243867320
+  version_requirements: *70237982924820
 - !ruby/object:Gem::Dependency
   name: activesupport
-  requirement: &70235243866680 !ruby/object:Gem::Requirement
+  requirement: &70237982923900 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70235243866680
+  version_requirements: *70237982923900
 - !ruby/object:Gem::Dependency
   name: multi_json
-  requirement: &70235243865240 !ruby/object:Gem::Requirement
+  requirement: &70237982922600 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,7 +65,7 @@ dependencies:
         version: '1.0'
   type: :runtime
   prerelease: false
-  version_requirements: *70235243865240
+  version_requirements: *70237982922600
 description: Tearing up web pages into ActiveRecord resources
 email:
 - github@coryodaniel.com