klepto 0.2.5 → 0.2.6

data/README.md

@@ -84,7 +84,8 @@ Say you want a bunch of Bieb tweets! How is there not profit in that?
   end      
 
   # Multiple Nested structures? Let klepto know this is a collection of resources
-  tweets    'li.stream-item', :as => :collection do
+  # Does bieber, tweet to much? Maybe. Lets only get the new stuff kids crave.
+  tweets    'li.stream-item', :as => :collection, :limit => 10 do
     twitter_id do |node|
       node['data-item-id']
     end
@@ -190,16 +191,6 @@ cookies({
 })  
 ```
 
-event handlers...
---------------------
-```ruby
-on_http_status(500,404) do |response, bot|
-  email('admin@example.com', bot.status, bot.summary)
-end
-on_http_status('3xx') do |response, bot|
-end
-```
-
 Pre-req Steps
 --------------------  
 ```ruby

data/lib/klepto/bot.rb

@@ -7,8 +7,12 @@ module Klepto
       @config.urls urls
       @queue  = []
 
+      # Evaluate the block as DSL, proxy off anything that isn't on #config
+      #   to a queue, then apply that queue to the top-level Klepto::Structure
       instance_eval &block
 
+      # After DSL evaluation is queued up, put some methods onto this instance
+      # and restore method_missing (for sanity sake)
       instance_eval <<-EOS
 def queue; @queue; end;
 def resources; @resources; end;
@@ -20,6 +24,7 @@ EOS
       __process!
     end
 
+    # Structure all the pages
     def __process!
       @resources = []
 
@@ -29,16 +34,24 @@ EOS
         browser.set_headers config.headers
         browser.fetch! url
 
+        # Fire callbacks on GET
         config.after_handlers[:get].each do |ah|
           ah.call(browser.page)
         end
         
+        # Capybara automatically follows redirects... Checking the page here
+        # to see if it has changed, and if so add it on to the stack of statuses.
+        # statuses is an array because it holds the actually HTTP response code and an
+        # approximate code (2xx for example). :redirect will be pushed onto the stack if a
+        # redirect happened.
         statuses = [browser.status, browser.statusx]
         statuses.push :redirect if url != browser.page.current_url
+        # Dispatch all the handlers for HTTP Status Codes.
         statuses.each do |status|
           config.dispatch_status_handlers(status, browser.page)
         end
 
+        # If the page was not a failure or if not aborting, structure that bad boy.
         if !browser.failure? || (browser.failure? && !config.abort_on_failure?)
           resources << __structure(browser.page)
         else
@@ -54,6 +67,7 @@ EOS
     def __structure(context)
       structure = Structure.new(context)
 
+      # A queue of DSL instructions
       queue.each do |instruction|
         if instruction[2]
           structure.send instruction[0], *instruction[1], &instruction[2]
@@ -62,10 +76,9 @@ EOS
         end
       end
 
-      config.after_handlers[:each].each do |ah|
-        ah.call(structure._hash)
-      end
-
+      # Call after(:each) handlers...
+      config.after_handlers[:each].each { |ah| ah.call(structure._hash) }
+    
       structure._hash
     end
 

data/lib/klepto/config.rb

@@ -13,8 +13,16 @@ module Klepto
       }
       @before_handlers  = {:each => []}
       @status_handlers  = {}
+      @handlers = {}
     end
 
+    # TODO: DRY up handlers...
+    # def dispatch(group, handler, *args)
+    #   @handlers[group] ||= {}
+    #   @handlers[group][handler] ||= []
+    #   @handlers[group][handler].each{|handler| handler.call(*args)}
+    # end
+
     def headers(_headers=nil)
       @headers = _headers if _headers
       @headers

data/lib/klepto/structure.rb

@@ -20,17 +20,20 @@ module Klepto
     #options[:as]     :collection, :resource
     #options[:match]  :first, :all
     #options[:syntax] :xpath, :css
+    #options[:limit]  Integer elements to structure when :match => :all or :as => :collection
     def method_missing(meth, *args, &block)
       options = args.last.is_a?(Hash) ? args.pop : {}
       options[:syntax]  ||= :css
       options[:match]   ||= :first
       options[:attr]    ||= nil
+      options[:limit]   ||= nil
       selector          = args.shift
 
       if options[:as] == :collection
         @_hash[meth] = []
         result = _context.all( options[:syntax], selector )      
-        result.each do |ele|
+        options[:limit] ||= result.length
+        result[0, options[:limit]].each do |ele|
           @_hash[meth].push Structure.build(ele, self, &block)
         end 
       elsif options[:as] == :resource
@@ -45,7 +48,8 @@ module Klepto
 
         if options[:match] == :all
           @_hash[meth] = []
-          result.each do |node|
+          options[:limit] ||= result.length
+          result[0, options[:limit]].each do |node|
             @_hash[meth] << block.call( node )
           end
         else
@@ -55,7 +59,8 @@ module Klepto
         result = _context.send( options[:match], options[:syntax], selector )
         if options[:match] == :all
           @_hash[meth] = []
-          result.each do |node|
+          options[:limit] ||= result.length
+          result[0, options[:limit]].each do |node|
             @_hash[meth] << (node[options[:attr]] || node.try(:text))
           end        
         else

data/lib/klepto/version.rb

@@ -1,3 +1,3 @@
 module Klepto
-  VERSION = "0.2.5"
+  VERSION = "0.2.6"
 end

data/spec/lib/klepto/bot_spec.rb

@@ -172,5 +172,49 @@ describe Klepto::Bot do
         statuses.should include 'got a page'
       end
     end
+
+
+    describe 'creating a bot with a node limit' do
+      before(:each) do
+        @bot = Klepto::Bot.new("https://twitter.com/justinbieber"){
+          config.headers({
+            'Referer'     => 'http://www.twitter.com',
+            'X-Sup-Dawg'  => "Yo, What's up?"
+          })
+
+          # Structure that stuff
+          name      'h1.fullname'
+          username  "span.screen-name"
+          
+          tweets    'li.stream-item', :as => :collection, :limit => 5 do
+            twitter_id do |node|
+              node['data-item-id']
+            end
+            tweet '.content p', :css
+            timestamp '._timestamp', :attr => 'data-time'
+            permalink '.time a', :css, :attr => :href
+          end
+
+          config.after(:each) do |resource|
+            @user = User.new
+            @user.name = resource[:name]
+            @user.username = resource[:username]
+            @user.save
+
+            resource[:tweets].each do |tweet|
+              Tweet.create(tweet)
+            end
+          end 
+        }
+        @structure = @bot.resources
+      end
+
+      it 'should limit the nodes structured' do
+        User.count.should be(1)
+        Tweet.count.should be(5)
+      end
+    end
+
+
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: klepto
 version: !ruby/object:Gem::Version
-  version: 0.2.5
+  version: 0.2.6
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2013-04-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: poltergeist
-  requirement: &70341397514440 !ruby/object:Gem::Requirement
+  requirement: &70115769000140 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - =
@@ -21,10 +21,10 @@ dependencies:
         version: 1.1.0
   type: :runtime
   prerelease: false
-  version_requirements: *70341397514440
+  version_requirements: *70115769000140
 - !ruby/object:Gem::Dependency
   name: capybara
-  requirement: &70341397512360 !ruby/object:Gem::Requirement
+  requirement: &70115768999120 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - =
@@ -32,10 +32,10 @@ dependencies:
         version: 2.0.2
   type: :runtime
   prerelease: false
-  version_requirements: *70341397512360
+  version_requirements: *70115768999120
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70341397509940 !ruby/object:Gem::Requirement
+  requirement: &70115768997540 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: 1.5.6
   type: :runtime
   prerelease: false
-  version_requirements: *70341397509940
+  version_requirements: *70115768997540
 - !ruby/object:Gem::Dependency
   name: activesupport
-  requirement: &70341397506060 !ruby/object:Gem::Requirement
+  requirement: &70115768995980 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70341397506060
+  version_requirements: *70115768995980
 - !ruby/object:Gem::Dependency
   name: multi_json
-  requirement: &70341397504900 !ruby/object:Gem::Requirement
+  requirement: &70115768994800 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,7 +65,7 @@ dependencies:
         version: '1.0'
   type: :runtime
   prerelease: false
-  version_requirements: *70341397504900
+  version_requirements: *70115768994800
 description: Tearing up web pages into ActiveRecord resources
 email:
 - github@coryodaniel.com