epitools 0.4.10 → 0.4.13

data/README.rdoc

@@ -1,7 +1,17 @@
 = epitools
 
-Awesome collection of Ruby tools!
-
+Useful miscellaneous improvements for base Ruby objects, plus some extra
+data structures and handy wrappers.
+
+Base classess: Object, Enumerable, Hash, String, Array, Integer, etc.
+
+Extras:
+  * Path (a better Pathname)
+  * Rash (a hash which can have Regexps as keys, allowing a single (key,value) pair to match many keys.)
+  * Progressbar (better than the progressbar gem)
+  * Colored (enhanced version of defunkt's colored -- adds ANSI colouring methods to String, eg: #red, #green, #light_blue, etc.)
+  * Browser (a fake browser, using mechanize, Progressbar, and CacheDB)
+  
 == Installing
 
   gem install epitools
@@ -10,7 +20,7 @@ Awesome collection of Ruby tools!
 
 This is basically a collection of enhancements to the ruby base types (Hash, Array,
 Enumerable, etc.). To learn how it works, read the specs in spec/*.rb,
-or check out the rdoc: http://rdoc.info/github/epitron/epitools
+or check out the rdoc: http://rdoc.info/gems/epitools/frames
 
 == Copyright
 

data/Rakefile

@@ -1,17 +1,22 @@
-require 'rubygems'
-
-#require 'bundler'
-#begin
-#  Bundler.setup(:default, :development)
-#rescue Bundler::BundlerError => e
-#  $stderr.puts e.message
-#  $stderr.puts "Run `bundle install` to install missing gems"
-#  exit e.status_code
-#end
-
-require 'rake'
+### Require all gems...
+%w[
+
+  rubygems
+  
+  rake
+  rake/rdoctask
+  rspec/core
+  rspec/core/rake_task
+  jeweler
+  
+].each { |mod| require mod }
+
+desc 'Default: specs.'
+task :default => :spec
 
-require 'jeweler'
+#
+# Jewelerrrr
+#
 Jeweler::Tasks.new do |gem|
   # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
   gem.name = "epitools"
@@ -32,20 +37,19 @@ Jeweler::Tasks.new do |gem|
 end
 Jeweler::RubygemsDotOrgTasks.new
 
-require 'rspec/core'
-require 'rspec/core/rake_task'
+desc 'Run all the specs.'
 RSpec::Core::RakeTask.new(:spec) do |spec|
   spec.pattern = FileList['spec/**/*_spec.rb']
 end
 
+desc 'Run rcov code coverage'
 RSpec::Core::RakeTask.new(:rcov) do |spec|
   spec.pattern = 'spec/**/*_spec.rb'
   spec.rcov = true
 end
 
-task :default => :spec
 
-require 'rake/rdoctask'
+desc 'Generate documentation for rdoc.'
 Rake::RDocTask.new do |rdoc|
   version = File.exist?('VERSION') ? File.read('VERSION') : ""
 

data/VERSION

@@ -1 +1 @@
-0.4.10
+0.4.13

data/epitools.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{epitools}
-  s.version = "0.4.10"
+  s.version = "0.4.13"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["epitron"]
-  s.date = %q{2011-02-02}
+  s.date = %q{2011-02-14}
   s.description = %q{Miscellaneous utility libraries to make my life easier.}
   s.email = %q{chris@ill-logic.com}
   s.extra_rdoc_files = [
@@ -26,7 +26,7 @@ Gem::Specification.new do |s|
     "lib/epitools.rb",
     "lib/epitools/basetypes.rb",
     "lib/epitools/browser.rb",
-    "lib/epitools/browser/browser_cache.rb",
+    "lib/epitools/browser/cache.rb",
     "lib/epitools/browser/mechanize_progressbar.rb",
     "lib/epitools/clitools.rb",
     "lib/epitools/colored.rb",
@@ -62,7 +62,7 @@ Gem::Specification.new do |s|
   s.homepage = %q{http://github.com/epitron/epitools}
   s.licenses = ["WTFPL"]
   s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.3.7}
+  s.rubygems_version = %q{1.5.2}
   s.summary = %q{NOT UTILS... METILS!}
   s.test_files = [
     "spec/basetypes_spec.rb",
@@ -80,7 +80,6 @@ Gem::Specification.new do |s|
   ]
 
   if s.respond_to? :specification_version then
-    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
     s.specification_version = 3
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then

data/lib/epitools/browser.rb

@@ -1,8 +1,9 @@
+
 require 'mechanize'
 require 'uri'
 require 'fileutils'
 
-require 'epitools/browser/browser_cache'
+require 'epitools/browser/cache'
 require 'epitools/browser/mechanize_progressbar'
 
 # TODO: Make socksify optional (eg: if proxy is specified)
@@ -100,12 +101,12 @@ class Browser
 
   def init_cache!
     # TODO: Rescue "couldn't load" exception and disable caching
-    @cache = CacheDB.new(agent) if @use_cache
+    @cache = Cache.new(agent) if @use_cache
   end
 
 
   def relative?(url)
-    not url =~ %r{^https?://}
+    not url[ %r{^https?://} ]
   end
 
 
@@ -141,7 +142,7 @@ class Browser
     cached_already = cache.include?(url)
 
     puts
-    puts "[ #{url.inspect} (use_cache=#{use_cache}) ]"
+    puts "[ GET #{url} (using cache: #{use_cache}) ]"
     
     delay unless cached_already
 

data/lib/epitools/browser/cache.rb

@@ -0,0 +1,196 @@
+require 'mechanize'
+require 'sqlite3'
+
+def dmsg(msg)
+  
+  if $DEBUG
+    case msg
+      when String
+        puts msg
+      else
+        puts msg.inspect
+    end
+  end
+  
+end
+
+class Browser
+  
+  #
+  # An SQLite3-backed browser cache (with gzip compressed pages)
+  #
+  class Cache
+  
+    include Enumerable
+  
+    attr_reader :db, :agent
+  
+    def initialize(agent, filename="browsercache.db")
+      @agent    = agent
+      @filename = filename
+  
+      @db = SQLite3::Database.new(filename)
+      @db.busy_timeout(50)
+  
+      create_tables
+    end
+  
+    def inspect
+      "#<Browser::Cache filename=#{@filename.inspect}, count=#{count}, size=#{File.size @filename} bytes>"
+    end
+  
+    def count
+      db.execute("SELECT COUNT(1) FROM cache").first.first.to_i
+    end
+    
+    alias_method :size, :count
+  
+    def put(page, original_url=nil, options={})
+      dmsg [:put, original_url]
+  
+      raise "Invalid page" unless [:body, :content_type, :uri].all?{|m| page.respond_to? m }
+  
+      url = page.uri.to_s
+  
+      dmsg [:page_uri, url]
+      dmsg [:original_url, url]
+  
+      if url != original_url
+        # redirect original_url to url
+        expire(original_url) if options[:overwrite]
+        db.execute(
+          "INSERT INTO cache VALUES ( ?, ?, ?, ? )",
+          original_url,
+          page.content_type,
+          nil,
+          url
+        )
+      end
+  
+      #compressed_body = page.body
+      compressed_body = Zlib::Deflate.deflate(page.body)
+
+      expire(url) if options[:overwrite]
+      db.execute(
+        "INSERT INTO cache VALUES ( ?, ?, ?, ? )",
+        url,
+        page.content_type,
+        SQLite3::Blob.new( compressed_body  ),
+        nil
+      )
+  
+      true
+      
+    rescue SQLite3::SQLException => e
+      p [:exception, e]
+      false
+    end
+  
+    def row_to_page(row)
+      url, content_type, compressed_body, redirect = row
+  
+      if redirect
+        get(redirect)
+      else
+        #body = compressed_body
+        body = Zlib::Inflate.inflate(compressed_body)
+  
+        Mechanize::Page.new(
+          URI.parse(url),
+          {'content-type'=>content_type},
+          body,
+          nil,
+          agent
+        )
+      end
+    end
+  
+    def pages_via_sql(*args, &block)
+      dmsg [:pages_via_sql, args]
+      if block_given?
+        db.execute(*args) do |row|
+          yield row_to_page(row)
+        end
+      else
+        db.execute(*args).map{|row| row_to_page(row) }
+      end
+    end
+  
+    def grep(pattern, &block)
+      pages_via_sql("SELECT * FROM cache WHERE url like '%#{pattern}%'", &block)
+    end
+  
+    def get(url)
+      pages = pages_via_sql("SELECT * FROM cache WHERE url = ?", url.to_s)
+      
+      if pages.any?
+        pages.first
+      else
+        nil
+      end
+    end
+  
+    def includes?(url)
+      db.execute("SELECT url FROM cache WHERE url = ?", url.to_s).any?
+    end
+  
+    alias_method :include?, :includes?
+  
+    def urls(pattern=nil)
+      if pattern
+        rows = db.execute("SELECT url FROM cache WHERE url LIKE '%#{pattern}%'")
+      else
+        rows = db.execute('SELECT url FROM cache')
+      end
+      rows.map{|row| row.first}
+    end
+  
+    def clear(pattern=nil)
+      if pattern
+        db.execute("DELETE FROM cache WHERE url LIKE '%#{pattern}%'")
+      else
+        db.execute("DELETE FROM cache") 
+      end
+    end
+  
+    def each(&block)
+      pages_via_sql("SELECT * FROM cache", &block)
+    end
+  
+    def each_url
+      db.execute("SELECT url FROM cache") do |row|
+        yield row.first
+      end
+    end
+  
+    def expire(url)
+      db.execute("DELETE FROM cache WHERE url = ?", url)
+    end
+  
+    def recreate_tables
+      drop_tables rescue nil
+      create_tables
+    end
+  
+    def delete!
+      File.unlink @filename  
+    end
+  
+  private
+  
+    def list_tables
+      db.execute("SELECT name FROM SQLITE_MASTER WHERE type='table'")
+    end
+  
+    def create_tables
+      db.execute("CREATE TABLE IF NOT EXISTS cache ( url varchar(2048), content_type varchar(255), body blob, redirect varchar(2048) )")
+      db.execute("CREATE UNIQUE INDEX IF NOT EXISTS url_index ON cache ( url )")
+    end
+  
+    def drop_tables
+      db.execute("DROP TABLE cache")
+    end
+  
+  end
+  
+end

data/lib/epitools/browser/mechanize_progressbar.rb

@@ -1,5 +1,10 @@
 require 'epitools/progressbar'
 
+#
+# Mechanize Progress Bar extension
+#
+# (Displays a progress bar whenever an url is retrieved.)
+#
 class Mechanize
   class Chain
     class ResponseReader

data/spec/browser_spec.rb

@@ -1,14 +1,77 @@
 require 'epitools/browser'
 
+class Mechanize::Page
+  def url
+    uri.to_s
+  end
+end
+
+
+
 describe Browser do
 
   before :all do
     @browser = Browser.new
   end
-  
+
+  after :all do
+    @browser.cache.delete!
+  end
+
   it "googles" do
     page = @browser.get("http://google.com")
     page.body["Feeling Lucky"].should_not be_empty
   end
+  
+  it "googles (cached)" do
+    @browser.get("http://google.com").body
+  end
+
+end
+
+
+
+describe Browser::Cache do
+
+  before :all do
+    @agent = Mechanize.new
+    Browser::Cache.new(@agent).delete!
+    @cache = Browser::Cache.new(@agent)
+  end
+
+  def new_page(body, url)
+    Mechanize::Page.new(
+      URI.parse(url),
+      {'content-type'=>'text/html'},
+      body,
+      nil,
+      @agent
+    )
+  end
+
+  after :all do
+    @cache.delete!
+  end
+
+  it "writes and reads" do
+    body = "Blah blah blah."
+    url  = "http://example.com/url.html"
+
+    page = new_page(body, url)
+
+    page.body.should == body
+    page.url.should == url
+
+    @cache.put page, url
+    @cache.urls.size.should == 1
+    @cache.includes?(url).should == true
+
+    result = @cache.get url
+
+    body.should == page.body
+    body.should == result.body
+    url.should == page.url
+    url.should == result.url
+  end
 
 end

metadata

@@ -1,13 +1,8 @@
 --- !ruby/object:Gem::Specification 
 name: epitools
 version: !ruby/object:Gem::Version 
-  hash: 27
-  prerelease: false
-  segments: 
-  - 0
-  - 4
-  - 10
-  version: 0.4.10
+  prerelease: 
+  version: 0.4.13
 platform: ruby
 authors: 
 - epitron
@@ -15,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-02-02 00:00:00 -05:00
+date: 2011-02-14 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -26,11 +21,6 @@ dependencies:
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        hash: 7
-        segments: 
-        - 2
-        - 2
-        - 0
         version: 2.2.0
   type: :development
   version_requirements: *id001
@@ -42,11 +32,6 @@ dependencies:
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        hash: 23
-        segments: 
-        - 1
-        - 0
-        - 0
         version: 1.0.0
   type: :development
   version_requirements: *id002
@@ -58,9 +43,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
         version: "0"
   type: :development
   version_requirements: *id003
@@ -83,7 +65,7 @@ files:
 - lib/epitools.rb
 - lib/epitools/basetypes.rb
 - lib/epitools/browser.rb
-- lib/epitools/browser/browser_cache.rb
+- lib/epitools/browser/cache.rb
 - lib/epitools/browser/mechanize_progressbar.rb
 - lib/epitools/clitools.rb
 - lib/epitools/colored.rb
@@ -129,23 +111,17 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
       version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
       version: "0"
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.7
+rubygems_version: 1.5.2
 signing_key: 
 specification_version: 3
 summary: NOT UTILS... METILS!

data/lib/epitools/browser/browser_cache.rb

@@ -1,169 +0,0 @@
-require 'mechanize'
-require 'sqlite3'
-
-class CacheDB
-
-  include Enumerable
-
-  attr_reader :db, :agent
-
-  def initialize(agent, filename="browsercache.db")
-    @agent    = agent
-    @filename = filename
-
-    @db = SQLite3::Database.new(filename)
-    @db.busy_timeout(50)
-
-    create_tables
-  end
-
-  def inspect
-    "#<CacheDB filename=#{@filename.inspect}, count=#{count}, size=#{File.size @filename} bytes>"
-  end
-
-  def count
-    db.execute("SELECT COUNT(1) FROM cache").first.first.to_i
-  end
-  
-  alias_method :size, :count
-
-  def put(page, original_url=nil, options={})
-    p [:put, original_url]
-
-    raise "Invalid page" unless [:body, :content_type, :uri].all?{|m| page.respond_to? m }
-
-    url = page.uri.to_s
-
-    p [:page_uri, url]
-
-    if url != original_url
-      # redirect original_url to url
-      expire(original_url) if options[:overwrite]
-      db.execute(
-        "INSERT INTO cache VALUES ( ?, ?, ?, ? )",
-        original_url,
-        page.content_type,
-        nil,
-        url
-      )
-    end
-
-    compressed_body = Zlib::Deflate.deflate(page.body)
-    expire(url) if options[:overwrite]
-    db.execute(
-      "INSERT INTO cache VALUES ( ?, ?, ?, ? )",
-      url,
-      page.content_type,
-      SQLite3::Blob.new( compressed_body  ),
-      nil
-    )
-
-    true
-    
-  rescue SQLite3::SQLException => e
-    p [:exception, e]
-    false
-  end
-
-  def row_to_page(row)
-    url, content_type, compressed_body, redirect = row
-
-    if redirect
-      get(redirect)
-    else
-      body = Zlib::Inflate.inflate(compressed_body)
-
-      Mechanize::Page.new(
-        URI.parse(url),
-        {'content-type'=>content_type},
-        body,
-        nil,
-        agent
-      )
-    end
-  end
-
-  def pages_via_sql(*args, &block)
-    p [:pages_via_sql, args]
-    if block_given?
-      db.execute(*args) do |row|
-        yield row_to_page(row)
-      end
-    else
-      db.execute(*args).map{|row| row_to_page(row) }
-    end
-  end
-
-  def grep(pattern, &block)
-    pages_via_sql("SELECT * FROM cache WHERE url like '%#{pattern}%'", &block)
-  end
-
-  def get(url)
-    pages = pages_via_sql("SELECT * FROM cache WHERE url = ?", url.to_s)
-    
-    if pages.any?
-      pages.first
-    else
-      nil
-    end
-  end
-
-  def includes?(url)
-    db.execute("SELECT url FROM cache WHERE url = ?", url.to_s).any?
-  end
-
-  alias_method :include?, :includes?
-
-  def urls(pattern=nil)
-    if pattern
-      rows = db.execute("SELECT url FROM cache WHERE url LIKE '%#{pattern}%'")
-    else
-      rows = db.execute('SELECT url FROM cache')
-    end
-    rows.map{|row| row.first}
-  end
-
-  def clear(pattern=nil)
-    if pattern
-      db.execute("DELETE FROM cache WHERE url LIKE '%#{pattern}%'")
-    else
-      db.execute("DELETE FROM cache") 
-    end
-  end
-
-  def each(&block)
-    pages_via_sql("SELECT * FROM cache", &block)
-  end
-
-  def each_url
-    db.execute("SELECT url FROM cache") do |row|
-      yield row.first
-    end
-  end
-
-  def expire(url)
-    db.execute("DELETE FROM cache WHERE url = ?", url)
-  end
-
-  def recreate_tables
-    drop_tables rescue nil
-    create_tables
-  end
-
-private
-
-  def list_tables
-    db.execute("SELECT name FROM SQLITE_MASTER WHERE type='table'")
-  end
-
-  def create_tables
-    db.execute("CREATE TABLE IF NOT EXISTS cache ( url varchar(2048), content_type varchar(255), body blob, redirect varchar(2048) )")
-    db.execute("CREATE UNIQUE INDEX IF NOT EXISTS url_index ON cache ( url )")
-  end
-
-  def drop_tables
-    db.execute("DROP TABLE cache")
-  end
-
-
-end