klepto 0.5.1 → 0.5.3

data/lib/klepto/bot.rb

@@ -17,6 +17,7 @@ module Klepto
       @config = Klepto::Config.new
       @config.urls urls
       @queue  = []
+      @pages  = {}
 
       # Evaluate the block as DSL, proxy off anything that isn't on #config
       #   to a queue, then apply that queue to the top-level Klepto::Structure
@@ -26,6 +27,7 @@ module Klepto
       # and restore method_missing (for sanity sake)
       instance_eval <<-EOS
 def queue; @queue; end;
+def pages; @pages; end;
 def parse!(*_urls); __process!(*_urls); end;
 def resources; @resources; end;
 def method_missing(meth, *args, &block)
@@ -53,7 +55,9 @@ EOS
         
         begin
           browser.fetch! url
-          
+
+          @pages[url] = browser.page if config.keep_pages
+
           # Fire callbacks on GET
           config.after_handlers[:get].each do |ah|
             ah.call(browser.page, browser, url)
@@ -75,7 +79,7 @@ EOS
               })
             end          
           else
-            resources << __structure(browser.page)
+            @resources << __structure(browser.page)
           end          
         rescue Capybara::Poltergeist::TimeoutError => ex
           config.dispatch_timeout_handler(ex, url)

data/lib/klepto/config.rb

@@ -2,9 +2,11 @@ module Klepto
   class Config
     attr_reader :after_handlers
     attr_reader :before_handlers
+    attr_reader :keep_pages
 
     def initialize
       @headers = {}
+      @keep_pages = false
       @abort_on_failure = true
       @abort_on_redirect = false
       @urls    = []
@@ -30,6 +32,11 @@ module Klepto
     #   @default_driver
     # end
 
+    def keep_pages(_keep = nil)
+      @keep_pages = _keep if _keep != nil
+      @keep_pages
+    end
+
     def headers(_headers=nil)
       @headers = _headers if _headers
       @headers

data/lib/klepto/version.rb

@@ -1,3 +1,3 @@
 module Klepto
-  VERSION = "0.5.1"
-end
+  VERSION = "0.5.3"
+end

data/spec/lib/klepto/bot_spec.rb

@@ -114,6 +114,8 @@ describe Klepto::Bot do
             'X-Sup-Dawg'  => "Yo, What's up?"
           })
 
+          config.keep_pages true
+
           # Structure that stuff
           name      'h1.fullname'
           username "//span[contains(concat(' ',normalize-space(@class),' '),' screen-name ')]", :syntax => :xpath
@@ -178,6 +180,10 @@ describe Klepto::Bot do
         @structure.first[:last_tweet][:twitter_id].should == @structure.first[:tweets].first[:twitter_id]
       end
 
+      it 'should have the pages stored' do
+        @bot.pages["https://twitter.com/justinbieber"].should_not be_nil
+      end
+
       it 'should be able to #parse! a url' do
         @new_structure = @bot.parse!("https://twitter.com/justinbieber")
         @new_structure.first[:name].should match(/Justin/i)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: klepto
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.5.3
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2013-05-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: poltergeist
-  requirement: &70221809709220 !ruby/object:Gem::Requirement
+  requirement: &70314273924640 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - =
@@ -21,10 +21,10 @@ dependencies:
         version: 1.1.0
   type: :runtime
   prerelease: false
-  version_requirements: *70221809709220
+  version_requirements: *70314273924640
 - !ruby/object:Gem::Dependency
   name: capybara
-  requirement: &70221809708720 !ruby/object:Gem::Requirement
+  requirement: &70314260798000 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - =
@@ -32,10 +32,10 @@ dependencies:
         version: 2.0.2
   type: :runtime
   prerelease: false
-  version_requirements: *70221809708720
+  version_requirements: *70314260798000
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70221809708260 !ruby/object:Gem::Requirement
+  requirement: &70314260796320 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: 1.5.6
   type: :runtime
   prerelease: false
-  version_requirements: *70221809708260
+  version_requirements: *70314260796320
 - !ruby/object:Gem::Dependency
   name: activesupport
-  requirement: &70221809707880 !ruby/object:Gem::Requirement
+  requirement: &70314260795500 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70221809707880
+  version_requirements: *70314260795500
 - !ruby/object:Gem::Dependency
   name: multi_json
-  requirement: &70221797327360 !ruby/object:Gem::Requirement
+  requirement: &70314260792000 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,7 +65,7 @@ dependencies:
         version: '1.0'
   type: :runtime
   prerelease: false
-  version_requirements: *70221797327360
+  version_requirements: *70314260792000
 description: Tearing up web pages into ActiveRecord resources
 email:
 - github@coryodaniel.com