epitools 0.4.32 → 0.4.33

data/VERSION

@@ -1 +1 @@
-0.4.32
+0.4.33

data/epitools.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{epitools}
-  s.version = "0.4.32"
+  s.version = "0.4.33"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["epitron"]
-  s.date = %q{2011-05-15}
+  s.date = %q{2011-05-16}
   s.description = %q{Miscellaneous utility libraries to make my life easier.}
   s.email = %q{chris@ill-logic.com}
   s.extra_rdoc_files = [

data/lib/epitools/basetypes.rb

@@ -194,6 +194,14 @@ class String
   def endswith(substring)
     self[-substring.size..-1] == substring
   end
+
+  #
+  # Parse object as JSON
+  #
+  def from_json
+    require 'json' unless defined? JSON
+    JSON.parse self
+  end
   
 end
 

data/lib/epitools/browser.rb

@@ -1,7 +1,7 @@
-
 require 'mechanize'
 require 'uri'
 require 'fileutils'
+require 'json'
 
 require 'epitools'
 require 'epitools/browser/cache'
@@ -39,6 +39,14 @@ class BrowserOptions < OpenStruct
 end
 =end
 
+# Monkeypatches!
+class Mechanize::File
+  def content_type
+    response['content-type']
+  end
+end
+  
+
 #
 # A mechanize class that emulates a web-browser, with cache and everything.
 # Progress bars are enabled by default.
@@ -74,7 +82,7 @@ class Browser
     init_agent!
     init_cache!
   end
-
+                                        
 
   def init_agent!
     @agent = Mechanize.new do |a|
@@ -106,18 +114,46 @@ class Browser
   end
 
 
+  def load_cookies!
+    if File.exists? @cookie_file
+      agent.cookie_jar.load @cookie_file
+      true
+    else
+      false
+    end
+  end
+
+  
+  def save_cookies!
+    agent.cookie_jar.save_as @cookie_file
+    true
+  end
+
+  
+  
   def relative?(url)
     not url[ %r{^https?://} ]
   end
 
+  
+  def cacheable?(page)
+    case page.content_type
+    when %r{^(text|application)}
+      true
+    end
+  end    
 
+  
   def cache_put(page, url)
-    if page.is_a? Mechanize::Page and page.content_type =~ %r{^text/}
-      puts "  |_ writing to cache"
-      cache.put(page, url, :overwrite=>true)
+    if cache.valid_page?(page)
+      if page.content_type =~ %r{(^text/|^application/javascript|javascript)}
+        puts "  |_ writing to cache"
+        cache.put(page, url, :overwrite=>true)
+      end
     end
   end
 
+  
   #
   # Retrieve an URL, and return a Mechanize::Page instance (which acts a 
   # bit like a Nokogiri::HTML::Document instance.)
@@ -136,9 +172,7 @@ class Browser
     #end
 
     # Determine the cache setting
-
-
-    use_cache = options[:use_cache] || @use_cache
+    use_cache = options[:use_cache] || options[:cache] || options[:cached] || @use_cache
 
     cached_already = cache.include?(url)
 
@@ -149,20 +183,14 @@ class Browser
 
     begin
       
-      if cached_already
-        page = cache.get(url)
-        if page.nil?
-          puts "  |_ CACHE FAIL! Re-getting page."
-          page = get(url, false)
-        end
+      if page = cache.get(url)
         puts "  |_ cached (#{page.content_type})"
       else
-        page = agent.get url
+        page = agent.get(url)
         @last_get = Time.now
+        cache_put(page, url)
       end
 
-      cache_put(page, url) unless cached_already
-
       puts
 
     rescue Net::HTTPBadResponse, Errno::ECONNRESET, SocketError, Timeout::Error, SOCKSError => e
@@ -196,22 +224,14 @@ class Browser
   end
 
   
-  # Delegation
-  [:head, :post, :put].each do |meth|
+  #
+  # Delegate certain methods to @agent
+  #
+  [:head, :post, :put, :submit].each do |meth|
     define_method meth do |*args|
       agent.send(meth, *args)
     end
   end
   
-private
-
-  def load_cookies!
-    agent.cookie_jar.load @cookie_file if File.exists? @cookie_file
-  end
-
-  def save_cookies!
-    agent.cookie_jar.save_as @cookie_file
-  end
-
 end
 

data/lib/epitools/browser/cache.rb

@@ -48,10 +48,15 @@ class Browser
     
     alias_method :size, :count
   
+    def valid_page?(page)
+      [:body, :content_type, :uri].all?{|m| page.respond_to? m }
+    end
+       
+    
     def put(page, original_url=nil, options={})
       dmsg [:put, original_url]
   
-      raise "Invalid page" unless [:body, :content_type, :uri].all?{|m| page.respond_to? m }
+      raise "Invalid page" unless valid_page?(page) 
   
       url = page.uri.to_s
   
@@ -98,13 +103,25 @@ class Browser
         #body = compressed_body
         body = Zlib::Inflate.inflate(compressed_body)
   
-        Mechanize::Page.new(
-          URI.parse(url),
-          {'content-type'=>content_type},
-          body,
-          nil,
-          agent
-        )
+        if content_type =~ /^(text\/html)|(application\/xhtml\+xml)/i
+          Mechanize::Page.new(
+            #initialize(uri=nil, response=nil, body=nil, code=nil, mech=nil)
+            URI.parse(url),
+            {'content-type'=>content_type},
+            body,
+            nil,
+            agent
+          )
+        else
+          Mechanize::File.new(
+            #initialize(uri=nil, response=nil, body=nil, code=nil
+            URI.parse(url),
+            {'content-type'=>content_type},
+            body,
+            nil
+          )
+        end
+            
       end
     end
   

data/spec/browser_spec.rb

@@ -1,3 +1,4 @@
+require 'pp'
 require 'epitools/browser'
 
 class Mechanize::Page
@@ -7,7 +8,6 @@ class Mechanize::Page
 end
 
 
-
 describe Browser do
 
   before :all do
@@ -18,6 +18,12 @@ describe Browser do
     @browser.cache.delete!
   end
 
+  it "caches javascript" do
+    url = "http://code.jquery.com/jquery-1.0.pack.js"
+    page = @browser.get(url)
+    @browser.cache.get(url).should_not == nil
+  end
+  
   it "googles" do
     page = @browser.get("http://google.com")
     page.body["Feeling Lucky"].should_not be_empty

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: epitools
 version: !ruby/object:Gem::Version 
-  hash: 79
+  hash: 77
   prerelease: 
   segments: 
   - 0
   - 4
-  - 32
-  version: 0.4.32
+  - 33
+  version: 0.4.33
 platform: ruby
 authors: 
 - epitron
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-05-15 00:00:00 Z
+date: 2011-05-16 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: rspec