rubycut-babushka 0.10.8 → 0.15.6.1

data/lib/babushka/resource.rb

@@ -1,18 +1,15 @@
 module Babushka
-  class ResourceError < StandardError
-  end
   class Resource
-    include LogHelpers
     extend LogHelpers
-    include ShellHelpers
     extend ShellHelpers
-    include PathHelpers
     extend PathHelpers
 
     def self.get url, &block
       filename = URI.unescape(url.to_s).p.basename
       if filename.to_s.blank?
         log_error "Not a valid URL to download: #{url}"
+      elsif url.to_s[%r{^git://}]
+        GitHelpers.git(url, &block)
       else
         download_path = in_download_dir {|path|
           downloaded_file = download(url, filename)
@@ -25,7 +22,7 @@ module Babushka
     def self.extract url, &block
       get url do |download_path|
         in_build_dir {
-          Resource.for(download_path).extract(&block)
+          Asset.for(download_path).extract(&block)
         }
       end
     end
@@ -45,171 +42,13 @@ module Babushka
           download URI.escape(location), location.p.basename
         else
           success = log_block "Downloading #{url}" do
-            shell %Q{curl -# -o "#{filename}.tmp" "#{url}" && mv -f "#{filename}.tmp" "#{filename}"}, :progress => /[\d\.]+%/
+            shell('curl', '-#', '-o', "#{filename}.tmp", url.to_s, :progress => /[\d\.]+%/) &&
+            shell('mv', '-f', "#{filename}.tmp", filename)
           end
           filename if success
         end
       end
     end
 
-    def self.detect_type_by_extension path
-      TYPES.keys.detect {|key|
-        TYPES[key][:exts].any? {|extension|
-          path.has_extension? extension
-        }
-      }
-    end
-    
-    def self.detect_type_by_contents path
-      TYPES.keys.detect {|key|
-        shell("file '#{path}'")[TYPES[key][:file_match]]
-      }
-    end
-
-    def self.type path
-      detect_type_by_extension(path) || detect_type_by_contents(path)
-    end
-
-    TYPES = {
-      :deb => {:file_match => 'Debian binary package', :exts => %w[deb]},
-      :pkg => {:file_match => 'xar archive', :exts => %w[pkg]},
-      :tar => {:file_match => 'tar archive', :exts => %w[tar]},
-      :gzip => {:file_match => 'gzip compressed data', :exts => %w[tgz tar.gz]},
-      :bzip2 => {:file_match => 'bzip2 compressed data', :exts => %w[tbz2 tar.bz2]},
-      :zip => {:file_match => 'Zip archive data', :exts => %w[zip]},
-      :dmg => {:file_match => 'VAX COFF executable not stripped', :exts => %w[dmg]}
-    }
-
-    attr_reader :path, :name
-
-    def initialize path, opts = {}
-      @path = path.p
-      @name = TYPES[type][:exts].inject(filename) {|fn,t| fn.gsub(/\.#{t}$/, '') }
-    end
-
-    def filename
-      path.basename.to_s
-    end
-
-    def type
-      self.class.type path
-    end
-
-    def supported?
-      !type.nil?
-    end
-
-    def extract &block
-      cd(archive_prefix, :create => true) { process_extract(&block) }
-    end
-
-    def process_extract &block
-      shell("mkdir -p '#{name}'") and
-      cd(name) {
-        unless log_shell("Extracting #{filename}", extract_command)
-          log_error "Couldn't extract #{path} - probably a bad download."
-        else
-          cd(content_subdir) {
-            block.nil? or block.call(self)
-          }
-        end
-      }
-    end
-
-    def content_subdir
-      identity_dirs.reject {|dir|
-        %w[app pkg bundle tmbundle prefPane].map {|i|
-          /\.#{i}$/
-        }.any? {|dont_descend|
-          dir[dont_descend]
-        }
-      }.first
-    end
-
-    def identity_dirs
-      everything = Dir.glob('*')
-      if everything.length == 1 && File.directory?(everything.first)
-        everything
-      else
-        Dir.glob('*/').map {|dir| dir.chomp('/') }.select {|dir|
-          dir.downcase.gsub(/[ \-_\.]/, '') == name.downcase.gsub(/[ \-_\.]/, '')
-        }
-      end
-    end
-
-    def archive_prefix
-      BuildPrefix
-    end
-  end
-
-  class FileResource < Resource
-    def extract &block
-      in_download_dir {
-        block.call(self)
-      }
-    end
-  end
-
-  class TarResource < Resource
-    def extract_command
-      "tar -#{extract_option(type)}xf '#{path}'"
-    end
-    def extract_option type
-      {
-        :tar => '',
-        :gzip => 'z',
-        :bzip2 => 'j'
-      }[type]
-    end
-  end
-
-  class ZipResource < Resource
-    def extract_command
-      "unzip -o '#{path}'"
-    end
-  end
-
-  class DmgResource < Resource
-    def extract &block
-      in_download_dir {
-        output = log_shell "Attaching #{filename}", "hdiutil attach '#{filename.p.basename}'"
-        if output.nil?
-          log_error "Couldn't mount #{filename.p}."
-        elsif (path = mountpoint_for(output)).nil?
-          raise "Couldn't find where `hdiutil` mounted #{filename.p}."
-        else
-          cd(path) {
-            block.call(self)
-          }.tap {
-            log_shell "Detaching #{filename}", "hdiutil detach '#{path}'"
-          }
-        end
-      }
-    end
-
-    def mountpoint_for output
-      output.scan(/\s+(\/Volumes\/[^\n]+)/).flatten.first
-    end
-  end
-  
-  class Resource
-    CLASSES = {
-      :deb => FileResource,
-      :pkg => FileResource,
-      :tar => TarResource,
-      :gzip => TarResource,
-      :bzip2 => TarResource,
-      :zip => ZipResource,
-      :dmg => DmgResource
-    }
-
-    def self.for path, opts = {}
-      path = path.p
-      filename = path.basename.to_s
-      raise ResourceError, "The archive #{filename} does not exist." unless path.exists?
-      klass = CLASSES[type(path)]
-      raise ResourceError, "Don't know how to extract #{filename}." if klass.nil?
-      klass.new(path, opts)
-    end
   end
 end

data/lib/babushka/run_reporter.rb

@@ -24,7 +24,16 @@ module Babushka
 
     def post_report report
       submit_report_to_webservice(report.p.read).tap {|result|
-        report.p.rm if result
+        if result.is_a?(Net::HTTPSuccess) || result.is_a?(Net::HTTPNotAcceptable)
+          # Remove the run on success, and on validation error: retrying
+          # won't help that anyway.
+          report.p.rm
+        else
+          # Wait for a moment before trying again, so persistent problems don't
+          # slam babushka.me (if it's rejecting the data) or peg our CPU (if
+          # the network is down).
+          sleep 1
+        end
       }
     end
 
@@ -33,8 +42,8 @@ module Babushka
       Net::HTTP.start('babushka.me') {|http|
         http.open_timeout = http.read_timeout = 5
         http.post '/runs.json', data
-      }.is_a?(Net::HTTPSuccess)
-    rescue Errno::ECONNREFUSED, SocketError
+      }
+    rescue Errno::EADDRNOTAVAIL, Errno::ECONNREFUSED, SocketError
       log_error "Couldn't connect to the babushka webservice." unless Base.task.running?
     rescue Timeout::Error, Errno::ETIMEDOUT
       debug "Timeout while submitting run report."

data/lib/babushka/shell.rb

@@ -2,6 +2,8 @@ module Babushka
   class Shell
     include LogHelpers
 
+    BUF_SIZE = 1024 * 16
+
     class ShellCommandFailed < StandardError
       attr_reader :cmd, :stdout, :stderr
       def initialize cmd, stdout, stderr
@@ -15,17 +17,28 @@ module Babushka
       end
     end
 
-    attr_reader :cmd, :opts, :env, :result, :stdout, :stderr
+    attr_reader :cmd, :opts, :env, :input, :result, :stdout, :stderr
 
     def initialize *cmd
       @opts = cmd.extract_options!
-      raise "You can't use :spinner and :progress together in Babushka::Shell." if opts[:spinner] && opts[:progress]
+      raise ArgumentError, "You can't use :spinner and :progress together in Babushka::Shell." if opts[:spinner] && opts[:progress]
+      raise ArgumentError, "wrong number of arguments (0 for 1+)" if cmd.empty?
       @env = cmd.first.is_a?(Hash) ? cmd.shift : {}
       @cmd = cmd
+      @input = if opts[:input].respond_to?(:read)
+        opts[:input]
+      elsif !opts[:input].nil?
+        StringIO.new(opts[:input])
+      end
+
       @progress = nil
+      @spinner_offset = -1
+      @should_spin = opts[:spinner] && !Base.task.opt(:debug)
     end
 
-    def ok?; result == 0 end
+    def ok?
+      result == 0
+    end
 
     def run &block
       @stdout, @stderr = '', ''
@@ -45,63 +58,60 @@ module Babushka
 
     def invoke
       debug "$ #{@cmd.join(' ')}".colorize('grey')
-      Babushka::Open3.popen3 @cmd, popen_opts do |stdin,stdout,stderr,thread|
-        unless @opts[:input].nil?
-          stdin << @opts[:input]
-          stdin.close
+      Babushka::Open3.popen3 @cmd, popen_opts do |pipe_in, pipe_out, pipe_err, thread|
+        read_fds = [pipe_err, pipe_out].reject {|p| p.closed? }
+        write_fds = if input.nil?
+          pipe_in.close
+          []
+        else
+          [pipe_in]
         end
 
-        spinner_offset = -1
-        should_spin = @opts[:spinner] && !Base.task.opt(:debug)
+        until read_fds.empty?
+          to_read, to_write, _ = IO.select(read_fds, write_fds, [])
 
-        # For very short-running commands, check for output in a tight loop.
-        # The sleep below would at least halve the speed of quick #shell calls.
-        # This means really quick calls (e.g. `whoami`, `pwd`, etc) aren't
-        # delayed, but the CPU is only pegged for a fraction of a second on
-        # slower calls (e.g. `gem env`, `make`, etc).
-        1_000.times { break if stdout.ready_for_read? || stderr.ready_for_read? }
+          read_from(pipe_out, stdout) if to_read.include?(pipe_out)
+          read_from(pipe_err, stderr, :stderr) if to_read.include?(pipe_err)
 
-        loop {
-          read_from stdout, @stdout do
-            print " #{%w[| / - \\][spinner_offset = ((spinner_offset + 1) % 4)]}\b\b" if should_spin
-          end
-          read_from stderr, @stderr, :stderr
+          read_fds.reject! {|p| p.closed? }
 
-          if stdout.closed? && stderr.closed?
-            break
+          if input.nil? || to_write.empty?
+            # Nothing to write.
+          elsif (buf = input.read(BUF_SIZE)).nil?
+            pipe_in.close
+            write_fds.delete(pipe_in)
           else
-            # We sleep here because otherwise babushka itself would peg the CPU
-            # while waiting for output from long-running shell commands.
-            sleep 0.05
+            pipe_in.write(buf)
           end
-        }
+        end
       end
     end
 
     def read_from io, buf, log_as = nil
-      while !io.closed? && io.ready_for_read?
-        output = nil
-        # Only try reading up to a backspace if we're looking for progress output.
-        output = io.gets("\r") if @opts[:progress]
-        output = io.gets if output.nil?
-
-        if output.nil?
-          io.close
-        else
-          debug output.chomp, :log => @opts[:log], :as => log_as
-          buf << output
-          if @opts[:progress] && (@progress = output[@opts[:progress]])
-            print " #{@progress}#{"\b" * (@progress.length + 1)}"
-          end
-          yield if block_given?
+      output = nil
+      # Try reading less than a full line (up to just a backspace) if we're
+      # looking for progress output.
+      output = io.gets("\r") if opts[:progress]
+      output = io.gets if output.nil?
+
+      if output.nil?
+        io.close
+      else
+        debug output.chomp, :log => opts[:log], :as => log_as
+        buf << output
+
+        if @should_spin
+          print " #{%w[| / - \\][@spinner_offset = ((@spinner_offset + 1) % 4)]}\b\b"
+        elsif opts[:progress] && (@progress = output[opts[:progress]])
+          print " #{@progress}#{"\b" * (@progress.length + 1)}"
         end
       end
     end
 
     def popen_opts
-      {}.tap {|opts|
-        opts[:chdir] = @opts[:cd].p.to_s if @opts[:cd]
-        opts[:env] = @env if @env
+      {}.tap {|opts_to_pass|
+        opts_to_pass[:chdir] = opts[:cd].p.to_s if opts[:cd]
+        opts_to_pass[:env] = env if env
       }
     end
   end

data/lib/babushka/source.rb

@@ -1,8 +1,9 @@
 module Babushka
   class SourceError < StandardError
   end
+  class SourceLoadError < LoadError
+  end
   class Source
-    include GitHelpers
     include LogHelpers
     extend LogHelpers
     extend ShellHelpers
@@ -22,11 +23,14 @@ module Babushka
     end
 
     def self.for_path path
-      remote = cd(path) { shell "git config remote.origin.url" }
-      if remote.nil?
-        Source.new path # local source
-      else
-        Source.new remote, :name => path.basename # remote source with custom path
+      @sources ||= {}
+      @sources[default_name_for_uri(path)] ||= begin
+        remote = shell "git config remote.origin.url", :cd => path
+        if remote.nil?
+          Source.new path # local source
+        else
+          Source.new remote, :name => default_name_for_uri(path) # remote source with custom path
+        end
       end
     end
 
@@ -44,8 +48,10 @@ module Babushka
         [nil, :implicit]
       elsif path.to_s.sub(/^\w+:\/\//, '')[/^[^\/]+[@:]/]
         [path.to_s, :private]
-      elsif path.to_s[/^(git|https?|file):\/\//]
+      elsif path.to_s[/^git:\/\//]
         [path.to_s, :public]
+      elsif path.to_s[/^\w+:\/\//]
+        [path.to_s, :private]
       else
         [path.p, :local]
       end
@@ -85,8 +91,9 @@ module Babushka
     def prefix
       self.class.source_prefix
     end
+
     def path
-      if implicit? || local?
+      @path ||= if implicit? || local?
         @uri
       else
         prefix / name
@@ -100,6 +107,7 @@ module Babushka
     def updated_at
       Time.now - File.mtime(path)
     end
+
     def description_pieces
       [
         name,
@@ -108,21 +116,27 @@ module Babushka
         ("#{updated_at.round.xsecs} ago" if cloneable?)
       ]
     end
+
     def type
       @type
     end
+
     def cloneable?
       [:public, :private].include? type
     end
+
     def cloned?
-      File.directory? path / '.git'
+      cloneable? && File.directory?(path / '.git')
     end
+
     def present?
       cloneable? ? cloned? : path.exists?
     end
+
     def local?
       type == :local
     end
+
     def implicit?
       type == :implicit
     end
@@ -145,10 +159,15 @@ module Babushka
       end
     end
 
-    def load!
+    def clear!
+      deps.clear!
+      templates.clear!
+    end
+
+    def load! should_update = false
       unless @currently_loading
         @currently_loading = true
-        update! if cloneable?
+        update! if cloneable? && (!cloned? || should_update)
         load_deps! unless implicit? # implicit sources can't be loaded.
         @currently_loading = false
       end
@@ -158,18 +177,15 @@ module Babushka
       unless @loaded
         path.p.glob('**/*.rb').each {|f|
           Base.sources.load_context :source => self, :path => f do
-            begin
-              load f
-            rescue Exception => e
-              log_error "#{e.backtrace.first}: #{e.message}"
-              log "Check #{(e.backtrace.detect {|l| l[f] } || f).sub(/\:in [^:]+$/, '')}."
-              debug e.backtrace * "\n"
-            end
+            load f
           end
         }
-        debug "Loaded #{deps.count} deps from #{path}." unless deps.count.zero?
+        debug "Loaded #{deps.count} deps from #{path}."
         @loaded = true
       end
+    rescue StandardError, SyntaxError => e
+      clear!
+      raise SourceLoadError.new(e.message).tap {|raised| raised.set_backtrace(e.backtrace) }
     end
 
     def update!
@@ -184,15 +200,20 @@ module Babushka
       elsif repo.exists? && repo.dirty?
         log "Not updating #{name} (#{path}) because there are local changes."
       elsif repo.exists? && repo.ahead?
+        @updated = false # So the ahead? check doesn't run again, for when there's no network.
         log "Not updating #{name} (#{path}) because it's ahead of origin."
       else
-        git(uri, :to => path, :log => true).tap {|result|
+        GitHelpers.git(uri, :to => path, :log => true).tap {|result|
           log "Marking #{uri} as offline for this run." unless result
           @updated = result || false
         }
       end
     end
 
+    def remove!
+      !cloneable? || !path.exists? || path.rm
+    end
+
     private
 
     def raise_unless_addable!