klepto 0.5.3 → 0.5.5

data/README.md

@@ -165,7 +165,7 @@ end
 
 ## Configuration Options
 * config.headers - Hash; Sets request headers
-* config.urls    - Array(String); Sets URLs to structure
+* config.url    - String; Set URL to structure
 * config.abort_on_failure - Boolean(Default: true); Should structuring be aborted on 4xx or 5xx
 
 ## Callbacks & Processing

data/lib/klepto/bot.rb

@@ -1,24 +1,13 @@
 module Klepto
   class Bot
     attr_reader :config
-    @@_bots = {}
-    class << self
-      def run(name,*urls)
-        urls.each do |url|
-          @@_bots[name].parse! url
-        end
-      end
-      def make(name, &block)
-        @@_bots[name] = Klepto::Bot.new(&block)
-      end
-    end
 
-    def initialize(*urls, &block)
+    def initialize(url=nil, &block)
       @config = Klepto::Config.new
-      @config.urls urls
+      @config.url url
       @queue  = []
-      @pages  = {}
-
+      @browser = Klepto::Browser.new
+      
       # Evaluate the block as DSL, proxy off anything that isn't on #config
       #   to a queue, then apply that queue to the top-level Klepto::Structure
       instance_eval &block
@@ -27,9 +16,9 @@ module Klepto
       # and restore method_missing (for sanity sake)
       instance_eval <<-EOS
 def queue; @queue; end;
-def pages; @pages; end;
-def parse!(*_urls); __process!(*_urls); end;
-def resources; @resources; end;
+def browser; @browser; end;
+def url=(_url); @config.url(_url); end;
+def structure; @structure; end;
 def method_missing(meth, *args, &block)
   raise NoMethodError.new("undefined method: Klepto::Bot#" + meth.to_s)
 end
@@ -39,54 +28,47 @@ EOS
     end
 
     # Structure all the pages
-    def __process!(*_urls)
-      @resources = []
-
-      (_urls + config.urls).each do |url|
-        browser   = Klepto::Browser.new
+    def __process!
+      @structure = nil
+      @browser.set_headers @config.headers
+      #browser.set_driver  config.driver
 
-        browser.set_headers config.headers
-        #browser.set_driver  config.driver
+      # Call before(:each) handlers...
+      @config.before_handlers[:each].each { |bh| 
+        bh.call(url, browser) 
+      }
+      
+      begin
+        @browser.fetch! @config.url
 
-        # Call before(:each) handlers...
-        config.before_handlers[:each].each { |bh| 
-          bh.call(url, browser) 
-        }
+        # Fire callbacks on GET
+        @config.after_handlers[:get].each do |ah|
+          ah.call(@browser.page, @browser, @config.url)
+        end
+                
+        # Dispatch all the handlers for HTTP Status Codes.
+        @browser.statuses.each do |status|
+          @config.dispatch_status_handlers(status, @browser.page)
+        end
         
-        begin
-          browser.fetch! url
-
-          @pages[url] = browser.page if config.keep_pages
-
-          # Fire callbacks on GET
-          config.after_handlers[:get].each do |ah|
-            ah.call(browser.page, browser, url)
-          end
-                  
-          # Dispatch all the handlers for HTTP Status Codes.
-          browser.statuses.each do |status|
-            config.dispatch_status_handlers(status, browser.page)
-          end
-          
-          # If the page was not a failure or if not aborting, structure that bad boy.
-          if (browser.failure? && config.abort_on_failure?) || (config.abort_on_redirect? && browser.was_redirected?)
-            config.after_handlers[:abort].each do |ah|
-              ah.call(browser.page,{
-                browser_failure:     browser.failure?,
-                abort_on_failure:   config.abort_on_failure?,
-                abort_on_redirect:  config.abort_on_redirect?,
-                redirect:           browser.was_redirected?
-              })
-            end          
-          else
-            @resources << __structure(browser.page)
+        # If the page was not a failure or if not aborting, structure that bad boy.
+        if (@browser.failure? && @config.abort_on_failure?) || (@config.abort_on_redirect? && @browser.was_redirected?)
+          @config.after_handlers[:abort].each do |ah|
+            ah.call(browser.page,{
+              browser_failure:    @browser.failure?,
+              abort_on_failure:   @config.abort_on_failure?,
+              abort_on_redirect:  @config.abort_on_redirect?,
+              redirect:           @browser.was_redirected?
+            })
           end          
-        rescue Capybara::Poltergeist::TimeoutError => ex
-          config.dispatch_timeout_handler(ex, url)
-        end
+        else
+          @structure = __structure(@browser.page)
+        end          
+      rescue Capybara::Poltergeist::TimeoutError => ex
+        config.dispatch_timeout_handler(ex, url)
       end
 
-      @resources
+      @structure
     end
 
     def __structure(context)

data/lib/klepto/config.rb

@@ -2,14 +2,11 @@ module Klepto
   class Config
     attr_reader :after_handlers
     attr_reader :before_handlers
-    attr_reader :keep_pages
 
     def initialize
       @headers = {}
-      @keep_pages = false
       @abort_on_failure = true
       @abort_on_redirect = false
-      @urls    = []
       @after_handlers   = {
         :each => [], #after each call to
         :get  => [], #after GET, before structure
@@ -32,11 +29,6 @@ module Klepto
     #   @default_driver
     # end
 
-    def keep_pages(_keep = nil)
-      @keep_pages = _keep if _keep != nil
-      @keep_pages
-    end
-
     def headers(_headers=nil)
       @headers = _headers if _headers
       @headers
@@ -103,12 +95,9 @@ module Klepto
       @before_handlers[which].push block
     end    
 
-    def url(*args)
-      @urls += args
-      @urls.flatten!
-      @urls.uniq!
-      @urls
+    def url(url=nil)
+      @url = url if url
+      @url
     end
-    alias :urls :url    
   end
 end

data/lib/klepto/structure.rb

@@ -61,8 +61,8 @@ module Klepto
           Klepto.logger.debug("\t\t\tAs: block (match all), Result? #{!result.nil?}")
           @_hash[meth] = []
           options[:limit] ||= result.length
-          result[0, options[:limit]].each do |node|
-            @_hash[meth] << block.call( node )
+          result[0, options[:limit]].each do |_node|
+            @_hash[meth] << block.call( _node )
           end
         else
           if result
@@ -81,8 +81,8 @@ module Klepto
           Klepto.logger.debug("\t\t\tAs: simple (match all), Result? #{!result.nil?}")
           @_hash[meth] = []
           options[:limit] ||= result.length
-          result[0, options[:limit]].each do |node|
-            @_hash[meth] << (node[options[:attr]] || node.try(:text))
+          result[0, options[:limit]].each do |_node|
+            @_hash[meth] << (_node[options[:attr]] || _node.try(:text))
           end        
         elsif result
           Klepto.logger.debug("\t\t\tAs: block (match one)")

data/lib/klepto/version.rb

@@ -1,3 +1,3 @@
 module Klepto
-  VERSION = "0.5.3"
+  VERSION = "0.5.5"
 end

data/spec/lib/klepto/bot_spec.rb

@@ -13,11 +13,10 @@ describe Klepto::Bot do
               StatusLog.create message: 'Abort!'
             }            
           }      
-          @structure = @bot.resources
         end
 
         it 'should structure not have structured the data' do
-          @structure.should be_empty
+          @bot.structure.should be_nil
         end      
 
         it 'should have dispatched abort handlers' do
@@ -37,11 +36,10 @@ describe Klepto::Bot do
               StatusLog.create message: '200'
             }            
           }      
-          @structure = @bot.resources
         end
 
         it 'should structure the data' do
-          @structure.first[:name].should match(/Justin/i)
+          @bot.structure[:name].should match(/Justin/i)
         end      
 
         it 'should have dispatched status handlers' do
@@ -62,7 +60,6 @@ describe Klepto::Bot do
             StatusLog.create message: 'Aborted.'
           end
         }
-        @structure = @bot.resources
       end      
 
       it 'should abort after a 4xx or 5xx' do
@@ -79,11 +76,10 @@ describe Klepto::Bot do
             StatusLog.create message: 'Aborted.'
           end
         }
-        @structure = @bot.resources
       end      
 
       it 'should perform structuring' do
-        @structure.first[:title].should == 'Not Found'
+        @bot.structure[:title].should == 'Not Found'
       end
 
       it 'should not abort after a 4xx or 5xx' do
@@ -91,21 +87,6 @@ describe Klepto::Bot do
       end
     end
 
-    describe 'structuring multiple pages' do
-      before(:each) do
-        @bot = Klepto::Bot.new("https://twitter.com/justinbieber"){
-          config.urls "https://twitter.com/ladygaga"
-          name 'h1.fullname'
-        }
-        @structure = @bot.resources
-      end
-
-      it 'should have both pages data' do
-        @structure.first[:name].should match(/Justin/i)
-        @structure.last[:name].should match(/Lady/i)
-      end
-    end
-
     describe 'creating a bot' do
       before(:each) do
         @bot = Klepto::Bot.new("https://twitter.com/justinbieber"){
@@ -114,8 +95,6 @@ describe Klepto::Bot do
             'X-Sup-Dawg'  => "Yo, What's up?"
           })
 
-          config.keep_pages true
-
           # Structure that stuff
           name      'h1.fullname'
           username "//span[contains(concat(' ',normalize-space(@class),' '),' screen-name ')]", :syntax => :xpath
@@ -169,28 +148,18 @@ describe Klepto::Bot do
             end
           end 
         }
-        @structure = @bot.resources
       end
 
       it 'should structure the data' do
-        @structure.first[:name].should match(/Justin/i)
-        @structure.first[:links].first.should match(/^http:/i)
-        #@structure.first[:links].should == ["http://t.co/2oSNE36kNM"]
-        @structure.first[:username].should eq '@justinbieber'
-        @structure.first[:last_tweet][:twitter_id].should == @structure.first[:tweets].first[:twitter_id]
-      end
-
-      it 'should have the pages stored' do
-        @bot.pages["https://twitter.com/justinbieber"].should_not be_nil
-      end
-
-      it 'should be able to #parse! a url' do
-        @new_structure = @bot.parse!("https://twitter.com/justinbieber")
-        @new_structure.first[:name].should match(/Justin/i)
+        @bot.structure[:name].should match(/Justin/i)
+        @bot.structure[:links].first.should match(/^http:/i)
+        #@bot.structure[:links].should == ["http://t.co/2oSNE36kNM"]
+        @bot.structure[:username].should eq '@justinbieber'
+        @bot.structure[:last_tweet][:twitter_id].should == @bot.structure[:tweets].first[:twitter_id]
       end
 
       it 'should store the data' do
-        User.first.name.should eq( @structure.first[:name] )
+        User.first.name.should eq( @bot.structure[:name] )
         User.count.should be(1)
         Tweet.count.should_not be(0)
       end
@@ -229,13 +198,12 @@ describe Klepto::Bot do
     #         end
     #       end          
     #     }
-    #     @structure = @bot.resources
     #   end
 
     #   it 'should set the value to nil when an exception is raised' do
-    #     @structure.first[:name].should match(/Justin/i)
-    #     @structure.first[:tweets].first.keys.should include(:timestamp)
-    #     @structure.first[:tweets].first[:timestamp].should be(nil)
+    #     @bot.structure[:name].should match(/Justin/i)
+    #     @bot.structure[:tweets].first.keys.should include(:timestamp)
+    #     @bot.structure[:tweets].first[:timestamp].should be(nil)
     #   end
     # end
 
@@ -245,11 +213,10 @@ describe Klepto::Bot do
           name      'h1.fullname'
           username  "span.screen-NOPE", default: "CHICKENS"
         }
-        @structure = @bot.resources
       end
 
       it 'should have a sensible default for the structure' do
-        @structure.first[:username].should eq('CHICKENS')
+        @bot.structure[:username].should eq('CHICKENS')
       end
     end
 
@@ -259,12 +226,11 @@ describe Klepto::Bot do
           name      'h1.fullname', parser: TextParser
           links 'span.url a', :match => :all, :parser => HrefParser
         }
-        @structure = @bot.resources
       end
 
       it 'should structure the data' do
-        @structure.first[:name].should match(/Justin/i)
-        @structure.first[:links].first.should match(/^http:/i)
+        @bot.structure[:name].should match(/Justin/i)
+        @bot.structure[:links].first.should match(/^http:/i)
       end     
     end
 
@@ -300,7 +266,6 @@ describe Klepto::Bot do
             end
           end 
         }
-        @structure = @bot.resources
       end
 
       it 'should limit the nodes structured' do
@@ -308,7 +273,5 @@ describe Klepto::Bot do
         Tweet.count.should be(5)
       end
     end
-
-
   end
 end

data/spec/lib/klepto/config_spec.rb

@@ -4,7 +4,7 @@ describe Klepto::Config do
   before(:each) do
     @config = Klepto::Config.new
     @config.headers({'Referer' => 'http://example.com'})
-    @config.urls 'http://example.com', 'http://www.iana.org'
+    @config.url 'http://example.com'
     @config.on_http_status(200){
       "Its 200"
     }
@@ -39,8 +39,8 @@ describe Klepto::Config do
     @config.instance_variable_get("@status_handlers")['4xx'].first.call.should eq ('Its crazy.')
   end
 
-  it 'should be able to set URLs' do
-    @config.urls.should == ['http://example.com', 'http://www.iana.org']
+  it 'should be able to set a URL' do
+    @config.url.should == 'http://example.com'
   end
 
   it 'should have an abort on 4xx/5xx option' do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: klepto
 version: !ruby/object:Gem::Version
-  version: 0.5.3
+  version: 0.5.5
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2013-05-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: poltergeist
-  requirement: &70314273924640 !ruby/object:Gem::Requirement
+  requirement: &70303258668580 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - =
@@ -21,10 +21,10 @@ dependencies:
         version: 1.1.0
   type: :runtime
   prerelease: false
-  version_requirements: *70314273924640
+  version_requirements: *70303258668580
 - !ruby/object:Gem::Dependency
   name: capybara
-  requirement: &70314260798000 !ruby/object:Gem::Requirement
+  requirement: &70303258666700 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - =
@@ -32,10 +32,10 @@ dependencies:
         version: 2.0.2
   type: :runtime
   prerelease: false
-  version_requirements: *70314260798000
+  version_requirements: *70303258666700
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70314260796320 !ruby/object:Gem::Requirement
+  requirement: &70303259419200 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: 1.5.6
   type: :runtime
   prerelease: false
-  version_requirements: *70314260796320
+  version_requirements: *70303259419200
 - !ruby/object:Gem::Dependency
   name: activesupport
-  requirement: &70314260795500 !ruby/object:Gem::Requirement
+  requirement: &70303259418820 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70314260795500
+  version_requirements: *70303259418820
 - !ruby/object:Gem::Dependency
   name: multi_json
-  requirement: &70314260792000 !ruby/object:Gem::Requirement
+  requirement: &70303259418280 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,7 +65,7 @@ dependencies:
         version: '1.0'
   type: :runtime
   prerelease: false
-  version_requirements: *70314260792000
+  version_requirements: *70303259418280
 description: Tearing up web pages into ActiveRecord resources
 email:
 - github@coryodaniel.com