faster_require 0.5.0

data/README

@@ -0,0 +1,51 @@
+A little utility to make
+
+require 'xxx'
+
+take much less time.
+
+Well, mostly on windows--on linux it's a speedup of only 0.41 to 0.45s, or so. [1]
+
+If you've ever wondered why ruby feels slow on doze...sometimes it's just the startup time.  This helps.
+
+Benchmarks:
+
+loading a spec file:
+
+  1.9.1   
+    without 3.20s
+    with 0.34s (10x improvement)
+
+  1.8.6
+    without 3.6s
+    with 1.25s
+    
+
+rails app, running script/console "puts 333"
+
+ 1.9.1
+   without:
+      20s
+    with:
+      10s
+ 
+ 1.8.6 
+   without:
+       9s
+   with:
+       6s
+       
+rake -T
+
+ 1.9.1
+   without: 3.75s
+   with: 1.5s       
+
+ 1.8.6
+   without: 1.37s
+   with:    1.25s
+   
+Note: in reality what we should do is fix core so that it doesn't have such awful I/O time in windows.  There may be some gross inefficiency in there.  For now, this is a work-around.
+   
+[1] A sister project to this one, faster_gem_script, can make ruby scripts in linux run faster by 0.1s :) http://github.com/rdp/faster_gem_script
+(in windows it's a much higher gain).  Eventually they'll be combined into one "gem optimizer" gem.

data/Rakefile

@@ -0,0 +1,13 @@
+  require 'faster_rubygems' if RUBY_VERSION < "1.9'"
+  require 'jeweler'
+  Jeweler::Tasks.new do |s|
+    s.name = "faster_require"
+    s.summary = "A tool designed to speedup library loading in Ruby by caching library locations"
+    s.email = "rogerdpack@gmail.com"
+    s.homepage = "http://github.com/rdp/faster_require"
+    s.authors = ["Roger Pack"]
+    s.add_development_dependency 'rspec'
+    s.add_development_dependency 'sane'
+    s.add_development_dependency 'ruby-prof'
+    # s.add_dependency
+  end

data/VERSION

@@ -0,0 +1 @@
+0.5.0

data/bin/fast_require

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require 'faster_require'
+FastRequire.clear_all!
+puts 'successfully cleared faster_require cache: (' + FastRequire.dir + ')'

data/lib/faster_require.rb

@@ -0,0 +1,213 @@
+module FastRequire
+  $FAST_REQUIRE_DEBUG ||= false
+
+  def self.setup
+    @@dir = File.expand_path('~/.ruby_faster_require_cache')
+
+    Dir.mkdir @@dir unless File.directory?(@@dir)
+    @@loc = @@dir + '/' + RUBY_VERSION + '-' + RUBY_PLATFORM + '-' +
+    sanitize(File.expand_path($0).gsub(/[\/:]/, '_')) +
+    sanitize(Dir.pwd)
+  end
+
+  def self.sanitize filename
+    filename.gsub(/[\/:]/, '_')
+  end
+
+  FastRequire.setup
+
+  def self.load filename
+    @@require_locs = Marshal.restore( File.open(filename, 'rb') {|f| f.read})
+  end
+
+  if File.exist?(@@loc)
+    FastRequire.load @@loc
+  else
+    @@require_locs = {}
+  end
+
+  @@already_loaded = {}
+
+  # try to see where this file was loaded from, from $:
+  # partial_name might be abc.rb, or might be abc
+  # partial_name might be a full path, too
+  def self.guess_discover partial_name, add_dot_rb = false
+
+    # test for full path first
+    # unfortunately it has to be a full separate test
+    # for windoze sake, as drive letter could be different than slapping a '/' on the dir to test list...
+    tests = [partial_name]
+
+    if add_dot_rb
+      tests << partial_name + '.rb'
+      tests << partial_name + '.so'
+    end
+
+    tests.each{|b|
+      # assume that .rb.rb is...valid...
+      if File.file?(b) && ((b[-3..-1] == '.rb') || (b[-3..-1] == '.so'))
+        return File.expand_path(b)
+      end
+    }
+
+    for dir in $:
+      if File.file?(b = (dir + '/' + partial_name))
+        # make sure we require a file that has the right suffix...
+        if (b[-3..-1] == '.rb')  || (b[-3..-1] == '.so')
+          return File.expand_path(b)
+        end
+
+      end
+    end
+
+    if add_dot_rb && (partial_name[-3..-1] != '.rb') && (partial_name[-3..-1] != '.so')
+      guess_discover(partial_name + '.rb') || guess_discover(partial_name + '.so')
+    else
+      nil
+    end
+  end
+
+  $LOADED_FEATURES.each{|already_loaded|
+    # in 1.8 they might be partial paths
+    # in 1.9, they might be non collapsed paths
+    # so we have to sanitize them here...
+    # XXXX File.exist? is a bit too loose, here...
+    if File.exist?(already_loaded)
+      key = File.expand_path(already_loaded)
+    else
+      key = FastRequire.guess_discover(already_loaded) || already_loaded
+    end
+    @@already_loaded[key] = true
+  }
+
+  @@already_loaded[File.expand_path(__FILE__)] = true # this file itself isn't in loaded features, yet, but very soon will be..
+  # special case--I hope...
+
+  # disallow re-requiring $0
+  @@require_locs[$0] = File.expand_path($0) # so when we run into it on a require, we will skip it...
+  @@already_loaded[File.expand_path($0)] = true
+
+
+  # XXXX within a very long depth to require fast_require,
+  # require 'a' => 'b' => 'c' => 'd' & fast_require
+  #             then
+  #             => 'b.rb'
+  # it works always
+
+  def self.already_loaded
+    @@already_loaded
+  end
+
+  def self.require_locs
+    @@require_locs
+  end
+
+  def self.dir
+    @@dir
+  end
+
+  at_exit {
+    FastRequire.default_save
+  }
+
+  def self.default_save
+    self.save @@loc
+  end
+
+  def self.save to_file
+    File.open(to_file, 'wb'){|f| f.write Marshal.dump(@@require_locs)}
+  end
+
+  def self.clear_all!
+    require 'fileutils'
+    FileUtils.rm_rf @@dir if File.exist? @@dir
+    @@require_locs.clear
+    setup
+  end
+
+  def require_cached lib
+    lib = lib.to_s unless lib.is_a?(String) # might not be zactly 1.9 compat...
+    if known_loc = @@require_locs[lib]
+      return false if @@already_loaded[known_loc]
+      @@already_loaded[known_loc] = true
+      if known_loc =~ /.so$/
+        puts 'doing original_non_cached_require on .so full path ' + known_loc if $FAST_REQUIRE_DEBUG
+        original_non_cached_require known_loc # not much we can do there...too bad...
+      else
+        puts 'doing eval on ' + lib + '=>' + known_loc if $FAST_REQUIRE_DEBUG
+        $LOADED_FEATURES << known_loc # *must*
+        return eval(File.open(known_loc, 'rb') {|f| f.read}, TOPLEVEL_BINDING, known_loc) || true # note the b here--this means it's reading .rb files as binary, which *typically* works--if it breaks re-save the offending file in binary mode, or file an issue on the tracker...
+      end
+    else
+      # we don't know the location--let Ruby's original require do the heavy lifting for us here
+      old = $LOADED_FEATURES.dup
+      if(original_non_cached_require lib)
+        # debugger might land here the first time you run a script and it doesn't have a require
+        # cached yet...
+        new = $LOADED_FEATURES - old
+        found = new.last
+
+        # incredibly, in 1.8.6, this doesn't always get set to a full path
+        if RUBY_VERSION < '1.9'
+          if !File.file?(found)
+            # discover the full path.
+            dir = $:.find{|path| File.file?(path + '/' + found)}
+            found = dir + '/' + found
+          end
+          found = File.expand_path(found);
+        end
+        puts 'found new loc:' + lib + '=>' + found if $FAST_REQUIRE_DEBUG
+        @@require_locs[lib] = found
+        @@already_loaded[found] = true
+        return true
+      else
+        puts 'already loaded, apparently' + lib if $FAST_REQUIRE_DEBUG
+        # this probably was something like
+        # the first pass was require 'regdeferred'
+        # now it's a different require 'regdeferred.rb'
+        # which fails (or vice versa)
+        # so figure out why
+        # calc location, expand, map back
+        where_found = FastRequire.guess_discover(lib, true)
+        if where_found
+          puts 'inferred ghost loc:' + lib + '=>' + where_found if $FAST_REQUIRE_DEBUG
+          @@require_locs[lib] = where_found
+          # unfortunately if it's our first pass
+          # and we are in the middle of a "real" require
+          # that is circular
+          # then $LOADED_FEATURES or (AFAIK) nothing will have been set
+          # for us to be able to assert that
+          # so...I think we'll end up
+          # just fudging for a bit
+          #	raise 'not found' unless @@already_loaded[where_found] # should have already been set...I think...
+        else
+          if $FAST_REQUIRE_DEBUG
+            # happens for enumerator XXXX
+            puts 'unable to infer' + lib + ' in ' if $FAST_REQUIRE_DEBUG
+            @@already_loaded[found] = true # hacky
+          end
+        end
+        return false # XXXX test all these return values
+      end
+    end
+  end
+
+  def self.resetup!
+    eval "module ::Kernel; alias :require :require_cached; end"
+  end
+end
+
+module Kernel
+
+  if(defined?(@already_using_faster_require))
+    raise 'twice not allowed...'
+    # *shouldn't* get here...unless I'm wrong...
+  else
+    @already_using_faster_require = true
+    include FastRequire
+    # overwrite old require...
+    alias :original_non_cached_require :require
+    FastRequire.resetup!
+  end
+
+end

data/lib/rubygems_plugin.rb

@@ -0,0 +1,5 @@
+Gem.post_install { |gem_installer_instance|
+  require 'faster_require'
+  FastRequire.clear_all!
+  puts 'cleared faster_require caches due to new gem install...'
+}

data/spec/files/a_requires_b.rb

@@ -0,0 +1 @@
+require 'b'

data/spec/files/attempt_double_load.rb

@@ -0,0 +1,4 @@
+$FAST_REQUIRE_DEBUG = 1
+a = File.dirname(File.expand_path(__FILE__)) + '/../../lib/fast_require.rb'
+load a
+load a

data/spec/files/b.rb

@@ -0,0 +1,4 @@
+if defined?($b) && $b > 0   
+  raise 'cannot require b twice'
+end
+$b = 1

data/spec/files/c.rb

@@ -0,0 +1,5 @@
+if defined?($c)
+  raise 'cannot require twice'
+else
+  $c = true
+end

data/spec/files/d.rb

@@ -0,0 +1 @@
+require 'faster_require'

data/spec/files/e.rb

@@ -0,0 +1 @@
+require 'faster_require'

data/spec/files/fast.rb

@@ -0,0 +1,4 @@
+require 'rubygems'
+require File.dirname(__FILE__) + "/../../lib/faster_require.rb"
+require 'spec/autorun'
+a = Spec

data/spec/files/gem_after.rb

@@ -0,0 +1,8 @@
+require '../lib/faster_require.rb'
+require 'rubygems'
+Gem::Specification
+
+raise if FastRequire.already_loaded.to_a.flatten.grep(/files\/b.rb/).length > 0
+require 'files/b.rb'
+raise if(require 'files/b.rb')
+raise unless  FastRequire.already_loaded.to_a.flatten.grep(/files\/b.rb/).length > 0

data/spec/files/gem_before.rb

@@ -0,0 +1,9 @@
+require 'rubygems'
+Gem::Specification
+
+require '../lib/faster_require.rb'
+
+raise if FastRequire.already_loaded.to_a.flatten.grep(/files\/b.rb/).length > 0
+require 'files/b.rb'
+raise if(require 'files/b.rb')
+raise unless  FastRequire.already_loaded.to_a.flatten.grep(/files\/b.rb/).length > 0

data/spec/files/large.rb

@@ -0,0 +1,5 @@
+require 'rubygems'
+require 'ruby-debug'
+require File.dirname(File.expand_path(__FILE__)) + '/../../lib/faster_require.rb'
+require 'stringio'
+require 'redparse'

data/spec/files/require_full_path.rb

@@ -0,0 +1,6 @@
+require File.dirname(File.expand_path(__FILE__)) + '/../../lib/faster_require.rb'
+raise if (require File.dirname(File.expand_path(__FILE__)) + '/../../lib/faster_require.rb') # another test, why not? :)
+paths = [File.expand_path('b.rb'), File.expand_path('b')]
+paths.each{|p| require p}
+paths.each{|p| raise unless FastRequire.require_locs.keys.include?(p)}
+

data/spec/files/require_non_dot_rb_fails.rb

@@ -0,0 +1,6 @@
+$:.unshift 'bin'
+require File.dirname(File.expand_path(__FILE__)) + '/../../lib/faster_require.rb'
+require 'non_dot_rb.rb' # succeed
+require 'non_dot_rb' # fails (well, returns false), we think it succeeds, but it in the wrong place, so when we run the second time, it loads the wrong file
+
+# unless my gem is working right

data/spec/files/require_twice_in_dir_pwd.rb

@@ -0,0 +1,6 @@
+require File.dirname(File.expand_path(__FILE__)) + '/../../lib/faster_require.rb'
+raise if (require File.dirname(File.expand_path(__FILE__)) + '/../../lib/faster_require.rb') # another test, why not? :)
+require 'b.rb'
+require 'b'
+raise unless FastRequire.require_locs.keys.include?('b.rb') && FastRequire.require_locs.keys.include?('b')
+

data/spec/files/requires_itself.rb

@@ -0,0 +1,4 @@
+require '../lib/faster_require.rb'
+raise if $already_here
+$already_here = true
+require $0

data/spec/files/slow.rb

@@ -0,0 +1,3 @@
+require 'rubygems'
+require 'spec/autorun'
+a = Spec

data/spec/spec.fast_require.rb

@@ -0,0 +1,153 @@
+# $FAST_REQUIRE_DEBUG
+if RUBY_VERSION < '1.9'
+  require 'faster_rubygems'
+end
+require 'sane'
+
+cached = '.cached_spec_locs' + RUBY_VERSION
+require_relative '../lib/faster_require'
+# use it for our own local test specs
+FastRequire.load cached if File.exist? cached
+require 'spec/autorun'
+require 'benchmark'
+FastRequire.save cached
+
+describe "faster requires" do
+
+  before do
+    FastRequire.clear_all!
+    @old_length = $LOADED_FEATURES.length
+    $b = 0
+    @ruby = OS.ruby_bin + " "
+  end
+
+  def with_file(filename = 'test')
+    FileUtils.touch filename + '.rb'
+    yield
+    FileUtils.rm filename + '.rb'
+  end  
+  
+  it "should be able to do a single require" do
+    Dir.chdir('files') do
+      old = $LOADED_FEATURES.dup
+      assert require('c')
+      assert !(require 'c')
+      new = $LOADED_FEATURES - old
+      raise new.inspect if new.length != 1    
+    end
+  end
+
+  it "should be able to go two sub-requires deep appropriately" do
+    Dir.chdir('files') do
+      assert(require('a_requires_b'))
+      assert !(require 'a_requires_b')
+      assert !(require 'a_requires_b')
+      $b.should == 1
+    end
+  end
+  
+  it "should be faster" do
+    Dir.chdir('files') do
+      slow = Benchmark.realtime { assert system("#{OS.ruby_bin} slow.rb")}
+      Benchmark.realtime { assert system("#{OS.ruby_bin} fast.rb")} # warmup
+      fast = Benchmark.realtime { assert system("#{OS.ruby_bin} fast.rb")}
+      pps 'fast', fast, 'slow', slow
+      assert fast < slow
+    end
+  end
+  
+  it "should work with large complex gem" do
+  	Dir.chdir('files') do
+  		assert(system("#{OS.ruby_bin} large.rb"))
+  		assert(system("#{OS.ruby_bin} large.rb"))
+  		assert(system("#{OS.ruby_bin} large.rb"))
+  	end
+  end
+
+  it "should not re-save the cache file if it hasn't changed [?]"
+  
+  it "should require .so files still, and only load them once" do
+    # ruby-prof gem
+    2.times { require 'ruby_prof'; RubyProf } # .so
+    assert $LOADED_FEATURES.length == (@old_length + 1)
+  end
+
+  it "should add requires to $LOADED_FEATURES" do
+  	with_file('file2') {require 'file2'}
+    assert ($LOADED_FEATURES.grep(/file2.rb/)).length > 0
+    assert $LOADED_FEATURES.length == (@old_length + 1)
+  end
+
+  it "should save a file as a cache in a dir" do    
+    assert Dir[FastRequire.dir + '/*'].length == 0 # all clear
+    FastRequire.default_save
+    assert Dir[FastRequire.dir + '/*'].length > 0
+  end
+  
+  it "should have different caches based on the file being run, and Dir.pwd" do
+   # that wouldn't help much at all for ruby-prof runs, but...we do what we can 
+   assert Dir[FastRequire.dir + '/*'].length == 0 # all clear
+   Dir.chdir('files') do
+   	  assert system("ruby -I../../lib d.rb")
+   	  assert system("ruby -I../../lib e.rb")
+   	  assert system("ruby -C.. -I../lib files/e.rb")
+   end
+   assert Dir[FastRequire.dir + '/*'].length == 3    
+   assert Dir[FastRequire.dir + '/*spec_files_d*'].length == 1 # use full path
+   assert Dir[FastRequire.dir + '/*spec_files_e*'].length == 2 # different dirs
+  end
+    
+  context "should work with ascii files well" do # most are binary, so...low prio
+    it "could cache the converted file, if that speeds things up"
+  end
+  
+  private
+  
+  def ruby filename
+    3.times { assert system(@ruby  + " " + filename) }    
+  end
+  
+  it "should override rubygems' require if rubygems is loaded after the fact...maybe by hooking to Gem::const_defined or something" do
+    ruby "files/gem_after.rb"
+  end
+  
+  it "should override rubygems' require if rubygems is loaded before the fact" do
+    ruby "files/gem_before.rb"    
+  end  
+  
+  it "should not double load gems" do
+    a = `#{@ruby} files/gem_after.rb 2>&1`
+    a.should_not match('already initialized')
+ end
+  
+  it "should throw if you require it twice" do
+    Dir.chdir('files') do
+      assert !system(@ruby + 'attempt_double_load.rb')
+    end
+  end
+  
+  it "should force require 'abc' to not load file called exactly abc" do
+    Dir.chdir('files') do
+      ruby 'require_non_dot_rb_fails.rb'
+    end
+  end
+  
+  it "should handle full path requires" do
+    Dir.chdir('files') do
+     ruby 'require_full_path.rb' 
+    end
+  end
+  
+  it "should handle Pathname requires, too" do
+    require 'pathname'
+    require Pathname.new('pathname')
+  end  
+  
+  it "should work well with rubygems for gem libs (installed), themselves"
+  
+  it "should disallow a file requiring itself" do
+    ruby 'files/requires_itself.rb'
+    ruby 'files/requires_itself.rb'    
+  end
+  
+end

metadata

@@ -0,0 +1,122 @@
+--- !ruby/object:Gem::Specification 
+name: faster_require
+version: !ruby/object:Gem::Version 
+  version: 0.5.0
+platform: ruby
+authors: 
+- Roger Pack
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-01-29 00:00:00 -07:00
+default_executable: fast_require
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: sane
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: ruby-prof
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+description: 
+email: rogerdpack@gmail.com
+executables: 
+- fast_require
+extensions: []
+
+extra_rdoc_files: 
+- README
+files: 
+- README
+- Rakefile
+- VERSION
+- bin/fast_require
+- lib/faster_require.rb
+- lib/rubygems_plugin.rb
+- spec/files/a_requires_b.rb
+- spec/files/attempt_double_load.rb
+- spec/files/b.rb
+- spec/files/c.rb
+- spec/files/d.rb
+- spec/files/e.rb
+- spec/files/fast.rb
+- spec/files/gem_after.rb
+- spec/files/gem_before.rb
+- spec/files/large.rb
+- spec/files/non_dot_rb.rb
+- spec/files/require_full_path.rb
+- spec/files/require_non_dot_rb_fails.rb
+- spec/files/require_twice_in_dir_pwd.rb
+- spec/files/requires_itself.rb
+- spec/files/slow.rb
+- spec/spec.fast_require.rb
+has_rdoc: true
+homepage: http://github.com/rdp/faster_require
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: A tool designed to speedup library loading in Ruby by caching library locations
+test_files: 
+- spec/files/attempt_double_load.rb
+- spec/files/a_requires_b.rb
+- spec/files/b.rb
+- spec/files/c.rb
+- spec/files/d.rb
+- spec/files/e.rb
+- spec/files/fast.rb
+- spec/files/gem_after.rb
+- spec/files/gem_before.rb
+- spec/files/large.rb
+- spec/files/non_dot_rb.rb
+- spec/files/requires_itself.rb
+- spec/files/require_full_path.rb
+- spec/files/require_non_dot_rb_fails.rb
+- spec/files/require_twice_in_dir_pwd.rb
+- spec/files/slow.rb
+- spec/spec.fast_require.rb