right_scraper 3.0.4 → 3.2.3

data/Gemfile

@@ -1,15 +1,14 @@
-source :rubygems 
+source 'https://rubygems.org'
 
-gem "json", "~> 1.4.5"
-gem "blackwinter-git", "~> 1.2.7"
-gem "libarchive", "~> 0.1.2"
-gem "right_aws", "~> 2.0"
-gem "process_watcher", "~> 0.3"
+gemspec
+
+gem 'right_popen', :git => 'git@github.com:rightscale/right_popen.git',
+                   :branch => "master"
+
+gem 'right_support', '2.7' 
 
 group :development do
-  gem "rspec", "~> 2.3"
   gem "rake", "0.8.7"
-  gem "flexmock"
   gem "ruby-debug", :platform=>"ruby_18"
   gem "ruby-debug19", :platform=>"ruby_19"
   gem "rdoc", "~> 2.4"

data/README.rdoc

@@ -10,6 +10,8 @@ may specify only the functionality (and required libraries and gems) you require
 This gem depends on all available RightScraper modules, enabling full support at the
 cost of requiring some systems administration work external to Ruby.
 
+Maintained by the RightScale Teal Team
+
 == USAGE
 
 === Simple Example
@@ -17,7 +19,7 @@ cost of requiring some systems administration work external to Ruby.
  require 'rubygems'
  require 'right_scraper'
 
- scraper = RightScale::Scraper.new(:basedir => '/tmp', :kind => :cookbook)
+ scraper = RightScraper::Scraper.new(:basedir => '/tmp', :kind => :cookbook)
  scraper.scrape(:type => :git, :url => 'git://github.com/rightscale/right_scraper.git')
 
 == INSTALLATION

data/lib/right_scraper/logger.rb

@@ -53,8 +53,8 @@ module RightScraper
         result = yield
         note_phase(:commit, type, explanation)
         result
-      rescue
-        note_phase(:abort, type, explanation, $!)
+      rescue Exception => e
+        note_phase(:abort, type, explanation, e)
         raise
       end
     end

data/lib/right_scraper/processes/ssh.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright: Copyright (c) 2010 RightScale, Inc.
+# Copyright: Copyright (c) 2010-2013 RightScale, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -20,14 +20,19 @@
 # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #++
+
 require 'tempfile'
-require 'process_watcher'
 require 'tmpdir'
+require 'right_popen'
+require 'right_popen/safe_output_buffer'
 
 module RightScraper
   module Processes
     # Manage a dedicated SSH agent.
     class SSHAgent
+
+      class SSHAgentError < Exception; end
+
       def initialize
         @display = ENV['DISPLAY']
         @askpass = ENV['SSH_ASKPASS']
@@ -46,26 +51,31 @@ module RightScraper
         ENV['HOME'] = "/dev/null"
         @dir = Dir.mktmpdir
         @socketfile = File.join(@dir, "agent")
-        @monitor = ProcessWatcher::ProcessMonitor.new
-        @pid = @monitor.spawn('ssh-agent', '-a', @socketfile, '-d') {}
-        timeout = 0
-        until File.exists?(@socketfile)
-          timeout += 1
-          sleep 0.1
-          if timeout > 100
-            raise "Couldn't find SSH agent control socket in time.  Timing out"
-          end
+        @process = nil
+        @interupted_to_close = false
+        @ssh_agent_output = ::RightScale::RightPopen::SafeOutputBuffer.new
+        cmd = ['ssh-agent', '-a', @socketfile, '-d']
+        ::RightScale::RightPopen.popen3_sync(
+          cmd,
+          :target          => self,
+          :inherit_io      => true,  # avoid killing any rails connection
+          :watch_handler   => :watch_ssh_agent,
+          :stderr_handler  => :output_ssh_agent,
+          :stdout_handler  => :output_ssh_agent,
+          :timeout_handler => :timeout_ssh_agent,
+          :exit_handler    => :exit_ssh_agent,
+          :timeout_seconds => 10)
+        if @process
+          ENV['SSH_AGENT_PID'] = @process.pid.to_s
+          ENV['SSH_AUTH_SOCK'] = @socketfile
         end
-        ENV['SSH_AGENT_PID'] = @pid.to_s
-        ENV['SSH_AUTH_SOCK'] = @socketfile
       end
 
       # Close the connection to the SSH agent, and restore +ENV+.
       def close
         begin
           FileUtils.remove_entry_secure @dir
-          lay_to_rest(@pid) if @pid
-          @monitor.cleanup if @monitor
+          lay_to_rest
         ensure
           setvar 'SSH_AGENT_PID', @agentpid
           setvar 'DISPLAY', @display
@@ -75,71 +85,35 @@ module RightScraper
         end
       end
 
-      # Kill +pid+.  Initially use SIGTERM to be kind and a good
-      # citizen.  If it doesn't die after +timeout+ seconds, use
-      # SIGKILL instead.  In any case, the process will die.  The
-      # status information is accessible in $?.
-      #
-      # === Parameters
-      # pid(Fixnum):: pid of process to kill
-      # timeout(Fixnum):: time in seconds to wait before forcing
-      #                   process to die.  Defaults to 10 seconds.
-      def lay_to_rest(pid, timeout=10)
-        #refuse to kill ourselves, or to pass a bad arg to Process.kill
-        return 0 unless pid.is_a?(Integer) && pid > 0
-
-        Process.kill('TERM', pid)
-        time_waited = 0
-        loop do
-          if time_waited >= timeout
-            Process.kill('KILL', pid)
-            # can't waitpid here, because the ssh-agent isn't our
-            # child.  Still, after SIGKILL it will die and init will
-            # reap it, so continue
-            return
-          end
-          # still can't waitpid here, so we see if it's still alive
-          return unless still_alive?(pid)
-          sleep 1
-          time_waited += 1
-        end
+      def output_ssh_agent(data)
+        @ssh_agent_output.safe_buffer_data(data)
       end
 
-      # Check to see if the process +pid+ is still alive, by sending
-      # the 0 signal and checking for an exception.
+      # abandons watch when socket file appears
       #
-      # === Parameters
-      # pid(Fixnum):: pid of process to check on
-      #
-      # === Return
-      # Boolean:: true if process is still alive
-      def still_alive?(pid)
-        begin
-          Process.kill(0, pid)
+      # @return [TrueClass|FalseClass] true to continue watch, false to abandon
+      def watch_ssh_agent(process)
+        if @interupted_to_close
           true
-        rescue Errno::ESRCH
-          false
+        else
+          @process = process
+          !::File.exists?(@socketfile)
         end
       end
 
-      # Set an environment variable to a value.  If +value+ is nil,
-      # delete the variable instead.
-      #
-      # === Parameters
-      # key(String):: environment variable name
-      # value(String or nil):: proposed new value
-      #
-      # === Return
-      # true
-      def setvar(key, value)
-        if value.nil?
-          ENV.delete(key)
-        else
-          ENV[key] = value
+      def timeout_ssh_agent
+        unless @interupted_to_close
+          raise SSHAgentError, 'Timed out waiting for ssh-agent control socket to appear'
+        end
+      end
+
+      def exit_ssh_agent(status)
+        unless @interupted_to_close || status.success?
+          @ssh_agent_output.safe_buffer_data("Exit code = #{status.exitstatus}")
+          raise SSHAgentError, "ssh-agent failed: #{@ssh_agent_output.display_text}"
         end
         true
       end
-      private :setvar
 
       # Add the given key data to the ssh agent.
       #
@@ -162,7 +136,33 @@ module RightScraper
       # === Parameters
       # file(String):: file containing key data
       def add_keyfile(file)
-        ProcessWatcher.watch("ssh-add", [file], nil, -1, 10)
+        @ssh_add_output = ::RightScale::RightPopen::SafeOutputBuffer.new
+        cmd = ['ssh-add', file]
+        ::RightScale::RightPopen.popen3_sync(
+          cmd,
+          :target          => self,
+          :inherit_io      => true,  # avoid killing any rails connection
+          :stderr_handler  => :output_ssh_add,
+          :stdout_handler  => :output_ssh_add,
+          :timeout_handler => :timeout_ssh_add,
+          :exit_handler    => :exit_ssh_add,
+          :timeout_seconds => 10)
+      end
+
+      def output_ssh_add(data)
+        @ssh_add_output.safe_buffer_data(data)
+      end
+
+      def timeout_ssh_add
+        raise SSHAgentError, 'ssh-add timed out'
+      end
+
+      def exit_ssh_add(status)
+        unless status.success?
+          @ssh_add_output.safe_buffer_data("Exit code = #{status.exitstatus}")
+          raise SSHAgentError, "ssh-add failed: #{@ssh_add_output.display_text}"
+        end
+        true
       end
 
       # Execute the block in a new ssh agent.
@@ -183,6 +183,38 @@ module RightScraper
           agent.close
         end
       end
+
+      private
+
+      def lay_to_rest
+        if @process
+          if @process.interrupt
+            @interupted_to_close = true
+            @process.sync_exit_with_target
+          else
+            @process.safe_close_io
+          end
+        end
+      end
+
+      # Set an environment variable to a value.  If +value+ is nil,
+      # delete the variable instead.
+      #
+      # === Parameters
+      # key(String):: environment variable name
+      # value(String or nil):: proposed new value
+      #
+      # === Return
+      # true
+      def setvar(key, value)
+        if value.nil?
+          ENV.delete(key)
+        else
+          ENV[key] = value
+        end
+        true
+      end
+
     end
   end
 end

data/lib/right_scraper/resources/cookbook.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
+# Copyright: Copyright (c) 2010-2012 RightScale, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -21,12 +21,45 @@
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #++
 
+require 'digest/md5'
+require 'json'
+require 'right_support'
+
 module RightScraper
 
   module Resources
 
     class Cookbook < Base
-    end
 
+      EMPTY_MANIFEST_JSON = ::JSON.dump(:manifest => {}).freeze
+
+      def manifest=(value)
+        @manifest_json = nil
+        @resource_hash = nil
+        @manifest      = value
+      end
+
+      def manifest_json
+        unless @manifest_json
+          if manifest && !manifest.empty?
+            # note that we are preserving the :manifest key at the root only to
+            # avoid having to change how the manifest is interpreted by Repose.
+            manifest_hash = { :manifest => manifest }
+            @manifest_json = ::RightSupport::Data::HashTools.deep_sorted_json(manifest_hash, pretty=true).freeze
+          else
+            @manifest_json = EMPTY_MANIFEST_JSON
+          end
+        end
+        @manifest_json
+      end
+
+      def resource_hash
+        unless @resource_hash
+          @resource_hash = ::Digest::MD5.hexdigest(manifest_json).freeze
+        end
+        @resource_hash
+      end
+
+    end
   end
 end

data/lib/right_scraper/retrievers/checkout.rb

@@ -40,9 +40,9 @@ module RightScraper
             @logger.operation(:updating) do
               do_update
             end
-          rescue
-            @logger.note_error($!, :updating, "switching to using checkout")
-            FileUtils.remove_entry_secure @repo_dir
+          rescue Exception => e
+            @logger.note_error(e, :updating, "switching to using checkout")
+            FileUtils.remove_entry_secure basedir
             @logger.operation(:checkout) do
               do_checkout
             end

data/lib/right_scraper/retrievers/download.rb

@@ -20,9 +20,11 @@
 # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #++
-require 'process_watcher'
+
 require 'tempfile'
 require 'digest/sha1'
+require 'right_popen'
+require 'right_popen/safe_output_buffer'
 
 module RightScraper
   module Retrievers
@@ -30,6 +32,8 @@ module RightScraper
     # somewhere.  Uses command line curl and command line tar.
     class Download < Base
 
+      class DownloadError < Exception; end
+
       @@available = false
 
       # Determines if downloader is available.
@@ -54,32 +58,48 @@ module RightScraper
 
       # Directory used to download tarballs
       def workdir
-        File.join(@basedir, @repository.repository_hash)
-      end
-
-      # Path to directory where files are retrieved
-      def repo_dir
-        File.join(workdir, "archive")
+        @workdir ||= ::File.join(::File.dirname(@repo_dir), 'download')
       end
 
       # Download tarball and unpack it
       def retrieve
         raise RetrieverError.new("download retriever is unavailable") unless available?
+        FileUtils.remove_entry_secure @repo_dir if File.exists?(@repo_dir)
         FileUtils.remove_entry_secure workdir if File.exists?(workdir)
-        FileUtils.mkdir_p repo_dir
+        FileUtils.mkdir_p @repo_dir
+        FileUtils.mkdir_p workdir
         file = File.join(workdir, "package")
 
         @logger.operation(:downloading) do
           credential_command = if @repository.first_credential && @repository.second_credential
-            ["-u", "#{@repository.first_credential}:#{@repository.second_credential}"]
+            ['-u', "#{@repository.first_credential}:#{@repository.second_credential}"]
           else
             []
           end
-          ProcessWatcher.watch("curl", ["--silent", "--show-error", "--location", "--fail",
-                                        "--location-trusted", "-o", file,
-                                        credential_command, @repository.url].flatten,
-                               workdir, @max_bytes || -1, @max_seconds || -1) do |phase, command, exception|
-            @logger.note_phase(phase, :running_command, command, exception)
+          @output = ::RightScale::RightPopen::SafeOutputBuffer.new
+          @cmd = [
+            'curl',
+            '--silent', '--show-error', '--location', '--fail',
+            '--location-trusted', '-o', file, credential_command,
+            @repository.url
+          ].flatten
+          begin
+            ::RightScale::RightPopen.popen3_sync(
+              @cmd,
+              :target             => self,
+              :pid_handler        => :pid_download,
+              :timeout_handler    => :timeout_download,
+              :size_limit_handler => :size_limit_download,
+              :exit_handler       => :exit_download,
+              :stderr_handler     => :output_download,
+              :stdout_handler     => :output_download,
+              :inherit_io         => true,  # avoid killing any rails connection
+              :watch_directory    => workdir,
+              :size_limit_bytes   => @max_bytes,
+              :timeout_seconds    => @max_seconds)
+          rescue Exception => e
+            @logger.note_phase(:abort, :running_command, 'curl', e)
+            raise
           end
         end
 
@@ -94,15 +114,58 @@ module RightScraper
           else
             extraction = "xf"
           end
-          Dir.chdir(repo_dir) do
-            ProcessWatcher.watch("tar", [extraction, file], repo_dir,
-                                 @max_bytes || -1, @max_seconds || -1) do |phase, command, exception|
-              @logger.note_phase(phase, :running_command, command, exception)
+          Dir.chdir(@repo_dir) do
+            @output = ::RightScale::RightPopen::SafeOutputBuffer.new
+            @cmd = ['tar', extraction, file]
+            begin
+              ::RightScale::RightPopen.popen3_sync(
+                @cmd,
+                :target             => self,
+                :pid_handler        => :pid_download,
+                :timeout_handler    => :timeout_download,
+                :size_limit_handler => :size_limit_download,
+                :exit_handler       => :exit_download,
+                :stderr_handler     => :output_download,
+                :stdout_handler     => :output_download,
+                :inherit_io         => true,  # avoid killing any rails connection
+                :watch_directory    => @repo_dir,
+                :size_limit_bytes   => @max_bytes,
+                :timeout_seconds    => @max_seconds)
+            rescue Exception => e
+              @logger.note_phase(:abort, :running_command, @cmd.first, e)
+              raise
             end
           end
         end
       end
 
+      def pid_download(pid)
+        @logger.note_phase(:begin, :running_command, @cmd.first)
+        true
+      end
+
+      def output_download(data)
+        @output.safe_buffer_data(data)
+      end
+
+      def timeout_download
+        raise DownloadError, "Downloader timed out"
+      end
+
+      def size_limit_download
+        raise DownloadError, "Downloader exceeded size limit"
+      end
+
+      def exit_download(status)
+        unless status.success?
+          @output.safe_buffer_data("Exit code = #{status.exitstatus}")
+          raise DownloadError, "Downloader failed: #{@output.display_text}"
+        end
+        @logger.note_phase(:commit, :running_command, @cmd.first)
+        true
+      end
+
+
       # Amend @repository with the tag information from the downloaded
       # file.
       #

data/lib/right_scraper/retrievers/git.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
+# Copyright: Copyright (c) 2010-2013 RightScale, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -23,6 +23,35 @@
 
 require 'tmpdir'
 
+# TEAL FIX: figure out a way to do this monkey-patch without always rquiring the
+# blackwinter gem and/or create a rightscale-git fork with this fix.
+#
+# ADDENDUM: we can't unconditionally require the git gem because git is not
+# always available.
+begin
+  require 'git'
+  require 'git/lib'
+
+  module Git
+    class Lib
+      # Monkey patch to blackwinter-git that strips ANSI escape sequences
+      # from command output to avoid confusing the parser.
+      def run_command_with_color_stripping(git_cmd, &block)
+        out = run_command_without_color_stripping(git_cmd, &block)
+        out.gsub!(/\e\[[^m]*m/, '')
+        out
+      end
+
+      unless self.methods.include?('run_command_without_color_stripping')
+        alias :run_command_without_color_stripping :run_command
+        alias :run_command :run_command_with_color_stripping
+      end
+    end
+  end
+rescue ::Git::GitExecuteError
+  # silently ignore git gem's failed attempt to execute git on load.
+end
+
 module RightScraper
   module Retrievers
     # Retriever for resources stored in a git repository.
@@ -33,7 +62,6 @@ module RightScraper
       def available?
         unless @@available
           begin
-            require 'git'
             # note that require 'git' does the same version check on load but
             # we don't want to assume any particular implementation.
             #
@@ -95,7 +123,13 @@ module RightScraper
       def do_update
         git = ::Git.open(@repo_dir)
         do_fetch(git)
-        git.reset_hard
+        @logger.operation(:cleanup, "ensure no untracked files in #{@repo_dir}") do
+          git.reset_hard
+          Dir.chdir(@repo_dir) do
+            # ignore outcome; there is no way to record 'warnings'
+            system("git clean -f")
+          end
+        end
         do_checkout_revision(git)
         do_update_tag(git)
       end

data/lib/right_scraper/retrievers/svn.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
+# Copyright: Copyright (c) 2010-2013 RightScale, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -22,8 +22,6 @@
 #++
 require File.join(File.dirname(__FILE__), '..', 'svn_client')
 
-require 'process_watcher'
-
 module RightScraper
   module Retrievers
     # Retriever for svn repositories
@@ -37,17 +35,10 @@ module RightScraper
       def available?
         unless @@available
           begin
-            # FIX: we might want to parse the result and require a minimum svn
-            # client version.
-            cmd = "svn --version"
-            `#{cmd}`
-            if $?.success?
-              @@available = true
-            else
-              raise RetrieverError, "\"#{cmd}\" exited with #{$?.exitstatus}"
-            end
-          rescue
-            @logger.note_error($!, :available, "svn retriever is unavailable")
+            calculate_version
+            @@available = true
+          rescue SvnClientError => e
+            @logger.note_error(e, :available, "svn retriever is unavailable")
           end
         end
         @@available
@@ -81,8 +72,8 @@ module RightScraper
       # pick out the first tag.
       def do_update_tag
         @repository = @repository.clone
-        log = run_svn("log", "-r", 'HEAD')
-        log.split(/\n/).each do |line|
+        lines = run_svn_with_buffered_output("log", "-r", 'HEAD')
+        lines.each do |line|
           if line =~ /^r(\d+)/
             @repository.tag = $1
             break

data/lib/right_scraper/scanners/cookbook_manifest.rb

@@ -22,7 +22,7 @@
 #++
 
 require File.expand_path(File.join(File.dirname(__FILE__), 'base'))
-require 'digest/sha1'
+require 'digest/md5'
 
 module RightScraper
   module Scanners
@@ -52,7 +52,7 @@ module RightScraper
       # === Parameters ===
       # relative_position(String):: relative pathname for file from root of resource
       def notice(relative_position)
-        @manifest[relative_position] = Digest::SHA1.hexdigest(yield)
+        @manifest[relative_position] = Digest::MD5.hexdigest(yield)
       end
     end
   end

data/lib/right_scraper/scanners/cookbook_s3_upload.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
+# Copyright: Copyright (c) 2010-2012 RightScale, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -22,6 +22,7 @@
 #++
 require 'right_aws'
 require 'json'
+require 'digest/md5'
 
 module RightScraper
   module Scanners
@@ -56,11 +57,11 @@ module RightScraper
       # === Parameters
       # cookbook(RightScraper::Cookbook):: cookbook to scan
       def end(cookbook)
-        @bucket.put(File.join('Cooks', cookbook.resource_hash),
-                    {
-                      :metadata => cookbook.metadata,
-                      :manifest => cookbook.manifest
-                    }.to_json)
+        path = File.join('Cooks', cookbook.resource_hash)
+        unless @bucket.key(path).exists?
+          contents = cookbook.manifest_json
+          @bucket.put(path, contents)
+        end
       end
 
       # Upload a file during scanning.
@@ -73,7 +74,7 @@ module RightScraper
       # relative_position(String):: relative pathname for file from root of cookbook
       def notice(relative_position)
         contents = yield
-        name = Digest::SHA1.hexdigest(contents)
+        name = Digest::MD5.hexdigest(contents)
         path = File.join('Files', name)
         unless @bucket.key(path).exists?
           @bucket.put(path, contents)