gauntlet 1.1.0 → 2.0.0

data/History.txt

@@ -1,3 +1,24 @@
+=== 2.0.0 / 2011-02-18
+
+* 1 major enhancement:
+
+  * Dropped dependencies_of, dependent_upon, gems_by_name, find_stupid_gems.
+
+* 4 minor enhancements:
+
+  * Added support for new rubygem server changes
+  * Separate downloads from conversion to .tar.gz
+  * Stripped down gauntlet_grep to pass all args through to grep
+  * Switched to net-http-persistent to clean up and speed up downloads.
+
+* 5 bug fixes:
+
+  * 1.9 fix for shadowed variables
+  * Add TERM and KILL handlers that shutdown nicely
+  * Fixes for stupidly named gems (--1)
+  * Handle SocketError and 404s more elegantly
+  * Switch to SpecFetcher for latest RubyGems
+
 === 1.1.0 / 2009-06-23
 
 * 6 minor enhancements:

data/Rakefile

@@ -4,11 +4,19 @@ require 'rubygems'
 require 'hoe'
 
 Hoe.plugin :seattlerb
+Hoe.plugin :isolate
 
 Hoe.spec 'gauntlet' do
   developer 'Ryan Davis', 'ryand-ruby@zenspider.com'
 
   self.rubyforge_name = 'seattlerb'
+
+  extra_deps << ["net-http-persistent", "~> 1.4.1"]
+end
+
+desc "update your gauntlet gems"
+task :update do
+  sh "ruby -I lib ./bin/gauntlet update"
 end
 
 # vim: syntax=ruby

data/bin/gauntlet

@@ -18,6 +18,7 @@ when "help" then
   puts "  cmds:"
   puts "    help          - show help"
   puts "    list          - list known gauntlet data for reports"
+  puts "    update        - update the gems, but don't run anything"
   puts "    report <name> - show a report on <name>, requires specific data format"
   puts "    <name>        - run the gauntlet for named plugin"
   Gem.find_files('gauntlet_*.rb').each do |path|
@@ -51,7 +52,7 @@ when "report" then
       good += 1
       next
     end
-    files.each do |path, result|
+    files.each do |dir, result|
       case result
       when true then
         good += 1
@@ -62,12 +63,12 @@ when "report" then
       when String then
         bad += 1
 
-        names[File.basename(path)] += 1
+        names[File.basename(dir)] += 1
 
         loop do
-          path = File.dirname path
-          break if path.empty? or path == "."
-          paths[path] += 1
+          dir = File.dirname dir
+          break if dir.empty? or dir == "."
+          dirs[dir] += 1
         end
       end
     end

data/lib/gauntlet.rb

@@ -1,6 +1,7 @@
-require 'rubygems/remote_fetcher'
+require 'rubygems'
 require 'thread'
 require 'yaml'
+require 'net/http/persistent'
 
 $u ||= false
 $f ||= false
@@ -9,36 +10,18 @@ $F ||= false
 Thread.abort_on_exception = true
 
 class Gauntlet
-  VERSION = '1.1.0'
-
-  GEMURL = URI.parse 'http://gems.rubyforge.org'
-  GEMDIR = File.expand_path "~/.gauntlet"
+  VERSION = '2.0.0'
+  GEMURL  = URI.parse 'http://gems.rubyforge.org'
+  GEMDIR  = File.expand_path "~/.gauntlet"
   DATADIR = File.expand_path "~/.gauntlet/data"
 
-  # stupid dups usually because of "dash" renames
-  STUPID_GEMS = %w(ajax-scaffold-generator
-                   extract_curves
-                   flickr-fu
-                   hpricot-scrub
-                   html_me
-                   merb-builder
-                   merb-jquery
-                   merb-parts
-                   merb_exceptions
-                   merb_helpers
-                   merb_param_protection
-                   model_graph
-                   not_naughty
-                   rfeedparser-ictv
-                   spec-converter
-                   spec_unit)
-
   attr_accessor :dirty, :data_file, :data
 
   def initialize
     name = self.class.name.downcase.sub(/gauntlet/, '')
     self.data_file = "#{DATADIR}/#{name}-data.yml"
     self.dirty = false
+    @cache = nil
   end
 
   def initialize_dir
@@ -69,48 +52,19 @@ class Gauntlet
   end
 
   def latest_gems
-    @cache ||= source_index.latest_specs
-  end
-
-  def gems_by_name
-    @by_name ||= Hash[*latest_gems.map { |gem|
-                        [gem.name, gem, gem.full_name, gem]
-                      }.flatten]
-  end
-
-  def dependencies_of name
-    index = source_index
-    gems_by_name[name].dependencies.map { |dep| index.search(dep).last }
-  end
-
-  def dependent_upon name
-    latest_gems.find_all { |gem|
-      gem.dependencies.any? { |dep| dep.name == name }
-    }
-  end
-
-  def find_stupid_gems
-    gems   = Hash.new { |h,k| h[k] = [] }
-    stupid = []
-    latest = {}
-
-    latest_gems.each do |spec|
-      name = spec.name.gsub(/-/, '_')
-      next unless name =~ /_/
-      gems[name] << spec
-    end
-
-    gems.reject! { |k,v| v.size == 1 || v.map { |s| s.name }.uniq.size == 1 }
-
-    gems.each do |k,v|
-      sorted = v.sort_by { |spec| spec.version }
-      latest[sorted.last.name] = true
-      sorted.each do |spec|
-        stupid << spec.name unless latest[spec.name]
-      end
-    end
-
-    stupid.uniq
+    Gem::SpecFetcher.new.list.map { |source_uri, gems|
+      base_url = source_uri.to_s
+      gems.map { |(name, version, platform)|
+        gem_name = case platform
+                   when Gem::Platform::RUBY then
+                     [name, version].join '-'
+                   else
+                     [name, version, platform].join '-'
+                   end
+
+        [gem_name, File.join(base_url, "/gems/#{gem_name}.gem")]
+      }
+    }.flatten(1)
   end
 
   def update_gem_tarballs
@@ -124,7 +78,7 @@ class Gauntlet
       gems = Dir["*.gem"]
       tgzs = Dir["*.tgz"]
 
-      old = tgzs - latest.map { |spec| "#{spec.full_name}.tgz" }
+      old = tgzs - latest.map { |(full_name, url)| "#{full_name}.tgz" }
       unless old.empty? then
         warn "deleting #{old.size} tgzs"
         old.each do |tgz|
@@ -132,54 +86,106 @@ class Gauntlet
         end
       end
 
-      tasks = Queue.new
-      latest.sort!
-      latest.reject! { |spec| tgzs.include? "#{spec.full_name}.tgz" }
-      tasks.push(latest.shift) until latest.empty? # LAME
-
-      warn "fetching #{tasks.size} gems"
-
-      threads = []
-      10.times do
-        threads << Thread.new do
-          fetcher = Gem::RemoteFetcher.new nil # fuck proxies
-          until tasks.empty? do
-            spec      = tasks.shift
-            full_name = spec.full_name
-            tgz_name  = "#{full_name}.tgz"
-            gem_name  = "#{full_name}.gem"
-
-            unless gems.include? gem_name then
-              begin
-                warn "downloading  #{full_name}"
-                fetcher.download(spec, GEMURL, Dir.pwd)
-              rescue Gem::RemoteFetcher::FetchError => e
-                warn "  failed #{full_name}: #{e.message}"
-                next
-              end
-            end
+      conversions = Queue.new
+      gem_names = gems.map { |gem| File.basename gem, '.gem' }
+      tgz_names = tgzs.map { |tgz| File.basename tgz, '.tgz' }
+      to_convert = gem_names - tgz_names
 
-            warn "  converting #{gem_name} to tarball"
+      seen_tgzs = Hash[*tgzs.map { |name| [name, true] }.flatten]
 
-            unless File.directory? full_name then
-              system "gem unpack cache/#{gem_name} &> /dev/null"
-              system "gem spec -l cache/#{gem_name} > #{full_name}/gemspec"
-            end
+      warn "adding #{to_convert.size} unconverted gems" unless to_convert.empty?
+
+      conversions.push to_convert.shift until to_convert.empty? # LAME
+
+      downloads = Queue.new
+      latest = latest.sort_by { |(full_name, url)| full_name.downcase }
+      latest.reject! { |(full_name, url)| seen_tgzs["#{full_name}.tgz"] }
+
+      downloads.push(latest.shift) until latest.empty? # LAME
+
+      converter = Thread.start do
+        while payload = conversions.shift do
+          full_name, _ = payload
+          tgz_name  = "#{full_name}.tgz"
+          gem_name  = "#{full_name}.gem"
+
+          warn " converting #{gem_name} to tarball"
 
-            system "chmod -R u+rwX #{full_name} && tar zmcf #{tgz_name} #{full_name} && rm -rf   #{full_name} #{gem_name}"
+          unless File.directory? full_name then
+            system "gem unpack cache/#{gem_name} &> /dev/null"
+            system "gem spec -l cache/#{gem_name} > #{full_name}/gemspec"
           end
+
+          system ["chmod -R u+rwX #{full_name}",
+                  "tar zmcf #{tgz_name} -- #{full_name}",
+                  "rm -rf -- #{full_name} #{gem_name}"].join(" && ")
         end
       end
 
-      threads.each do |thread|
-        thread.join
+      warn "fetching #{downloads.size} gems"
+
+      http = Net::HTTP::Persistent.new
+
+      workers downloads do |full_name, url|
+        gem_name  = "#{full_name}.gem"
+
+        unless gems.include? gem_name then
+          limit = 3
+          begin
+            warn "downloading #{full_name}"
+            while limit > 0 do
+              http.request URI.parse(url) do |response|
+                case response
+                when Net::HTTPSuccess
+                  File.open gem_name, "wb" do |f|
+                    response.read_body do |chunk|
+                      f.write chunk
+                    end
+                  end
+                  limit = 0 # kinda lame.
+                when Net::HTTPRedirection
+                  url = response['location']
+                  limit -= 1
+                else
+                  warn "  #{full_name} got #{response.code}. skipping."
+                  limit = 0
+                end
+              end
+            end
+          rescue SocketError, Net::HTTP::Persistent::Error => e
+            warn "  #{full_name} raised #{e.message}. skipping."
+          end
+        end
+
+        conversions.push full_name
       end
+
+      conversions.push nil
+
+      converter.join
     end
+
   rescue Interrupt
     warn "user cancelled... quitting"
     exit 1
   end
 
+  def workers tasks, count = 10
+    threads = []
+    count.times do
+      threads << Thread.new do
+        until tasks.empty? do
+          task = tasks.shift
+          yield task
+        end
+      end
+    end
+
+    threads.each do |thread|
+      thread.join
+    end
+  end
+
   def each_gem filter = nil
     filter ||= /^[\w-]+-\d+(\.\d+)*\.tgz$/
     in_gem_dir do
@@ -203,7 +209,7 @@ class Gauntlet
           end
         end
       ensure
-        system "rm -rf #{process_dir}"
+        system "rm -rf -- #{process_dir}"
       end
     end
   end
@@ -262,6 +268,13 @@ class Gauntlet
       self.data.delete outdated
     end
 
+    %w[TERM KILL].each do |signal|
+      trap signal do
+        shutdown
+        exit
+      end
+    end
+
     each_gem filter do |name|
       next if should_skip? name
       with_gem name do
@@ -275,6 +288,10 @@ class Gauntlet
   rescue Interrupt
     warn "user cancelled. quitting"
   ensure
+    shutdown
+  end
+
+  def shutdown
     save_yaml data_file, data if dirty
   end
 end
@@ -282,68 +299,15 @@ end
 ############################################################
 # Extensions and Overrides
 
-# bug in RemoteFetcher#download prevents multithreading. Remove after 1.3.2
-class Gem::RemoteFetcher
-  alias :old_download :download
-  def download(spec, source_uri, install_dir = Gem.dir)
-    if File.writable?(install_dir)
-      cache_dir = File.join install_dir, 'cache'
-    else
-      cache_dir = File.join(Gem.user_dir, 'cache')
-    end
-
-    gem_file_name = "#{spec.full_name}.gem"
-    local_gem_path = File.join cache_dir, gem_file_name
-
-    FileUtils.mkdir_p cache_dir rescue nil unless File.exist? cache_dir
+class Gem::SpecFetcher
+  attr_writer :fetcher
 
-    source_uri = URI.parse source_uri unless URI::Generic === source_uri
-    scheme = source_uri.scheme
+  alias :old_initialize :initialize
 
-    # URI.parse gets confused by MS Windows paths with forward slashes.
-    scheme = nil if scheme =~ /^[a-z]$/i
+  def initialize fetcher = Gem::RemoteFetcher.fetcher
+    old_initialize
 
-    case scheme
-    when 'http', 'https' then
-      unless File.exist? local_gem_path then
-        begin
-          say "Downloading gem #{gem_file_name}" if
-            Gem.configuration.really_verbose
-
-          remote_gem_path = source_uri + "gems/#{gem_file_name}"
-
-          gem = self.fetch_path remote_gem_path
-        rescue Gem::RemoteFetcher::FetchError
-          raise if spec.original_platform == spec.platform
-
-          alternate_name = "#{spec.original_name}.gem"
-
-          say "Failed, downloading gem #{alternate_name}" if
-            Gem.configuration.really_verbose
-
-          remote_gem_path = source_uri + "gems/#{alternate_name}"
-
-          gem = self.fetch_path remote_gem_path
-        end
-
-        File.open local_gem_path, 'wb' do |fp|
-          fp.write gem
-        end
-      end
-    when nil, 'file' then # TODO test for local overriding cache
-      begin
-        FileUtils.cp source_uri.to_s, local_gem_path
-      rescue Errno::EACCES
-        local_gem_path = source_uri.to_s
-      end
-
-      say "Using local gem #{local_gem_path}" if
-        Gem.configuration.really_verbose
-    else
-      raise Gem::InstallError, "unsupported URI scheme #{source_uri.scheme}"
-    end
-
-    local_gem_path
+    self.fetcher = fetcher
   end
 end
 
@@ -369,3 +333,4 @@ class Array
     return Math.sqrt(self.sample_variance)
   end
 end
+

data/lib/gauntlet_grep.rb

@@ -6,18 +6,17 @@ require 'rubygems'
 require 'gauntlet'
 
 class GrepGauntlet < Gauntlet
-  attr_accessor :pattern
+  attr_accessor :args
 
-  def initialize pattern
-    self.pattern = pattern
+  def initialize args
+    self.args = args
   end
 
   def run name
-    system "find . -type f -print0 | xargs -0 grep #{pattern}"
+    system "grep", *args
   end
 end
 
-pattern = ARGV.shift
-flogger = GrepGauntlet.new pattern
+flogger = GrepGauntlet.new ARGV
 flogger.run_the_gauntlet
 

metadata

@@ -1,7 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: gauntlet
 version: !ruby/object:Gem::Version 
-  version: 1.1.0
+  hash: 15
+  prerelease: 
+  segments: 
+  - 2
+  - 0
+  - 0
+  version: 2.0.0
 platform: ruby
 authors: 
 - Ryan Davis
@@ -30,19 +36,57 @@ cert_chain:
   FBHgymkyj/AOSqKRIpXPhjC6
   -----END CERTIFICATE-----
 
-date: 2009-06-23 00:00:00 -07:00
+date: 2011-02-18 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
-  name: hoe
+  name: net-http-persistent
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 5
+        segments: 
+        - 1
+        - 4
+        - 1
+        version: 1.4.1
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: minitest
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 11
+        segments: 
+        - 2
+        - 0
+        - 2
+        version: 2.0.2
   type: :development
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 2.3.0
-    version: 
+        hash: 41
+        segments: 
+        - 2
+        - 9
+        - 1
+        version: 2.9.1
+  type: :development
+  version_requirements: *id003
 description: |-
   Gauntlet is a pluggable means of running code against all the latest
   gems and storing off the data.
@@ -65,6 +109,7 @@ files:
 - lib/gauntlet.rb
 - lib/gauntlet_grep.rb
 - test/test_gauntlet.rb
+- .gemtest
 has_rdoc: true
 homepage: http://rubyforge.org/projects/seattlerb
 licenses: []
@@ -76,21 +121,27 @@ rdoc_options:
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
       version: "0"
-  version: 
 required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
       version: "0"
-  version: 
 requirements: []
 
 rubyforge_project: seattlerb
-rubygems_version: 1.3.4
+rubygems_version: 1.4.2
 signing_key: 
 specification_version: 3
 summary: Gauntlet is a pluggable means of running code against all the latest gems and storing off the data.