right_scraper 1.0.23 → 1.0.24

data/lib/right_scraper/linux/process_monitor.rb

@@ -25,11 +25,16 @@ module RightScale
 
   # *nix specific watcher implementation
   class ProcessMonitor
-
-    # Spawn given process and callback given block with output and exit code
+    # Spawn given process and callback given block with output and exit code. This method
+    # accepts a variable number of parameters; the first param is always the command to
+    # run; successive parameters are command-line arguments for the process.
     #
     # === Parameters
-    # cmd(String):: Process command line (including arguments)
+    # cmd(String):: Name of the command to run
+    # arg1(String):: Optional, first command-line argumument
+    # arg2(String):: Optional, first command-line argumument
+    # ...
+    # argN(String):: Optional, Nth command-line argumument
     #
     # === Block
     # Given block should take one argument which is a hash which may contain
@@ -39,19 +44,31 @@ module RightScale
     #
     # === Return
     # pid(Integer):: Spawned process pid
-    def spawn(cmd)
-      # Run external process and monitor it in a new thread
-      @io = IO.popen(cmd)
+    def spawn(cmd, *args)
+      args = args.map { |a| a.to_s } #exec only likes string arguments
+      
+      #Run subprocess; capture its output using a pipe
+      pr, pw = IO::pipe
+      @pid = fork do
+        pr.close
+        STDIN.reopen(File.open('/dev/null', 'r'))
+        STDOUT.reopen(pw)
+        STDERR.reopen(pw)
+        exec(cmd, *args)
+      end
+
+      #Monitor subprocess output and status in a dedicated thread
+      pw.close
+      @io = pr
       @reader = Thread.new do
-        o = @io.read
-        until o == ''
-          yield(:output => o)
-          o = @io.read
+        until @io.eof?
+          yield(:output => @io.read)
         end
-        Process.wait(@io.pid)
+        Process.wait(@pid)
         yield(:exit_code => $?.exitstatus)
       end
-      @io.pid
+
+      return @pid
     end
 
     # Close io and join reader thread

data/lib/right_scraper/scraper_base.rb

@@ -140,5 +140,36 @@ module RightScale
       true
     end
 
+    # Spawn given process, wait for it to complete, and return its output The exit status
+    # of the process is available in the $? global. Functions similarly to the backtick
+    # operator, only it avoids invoking the command interpreter under operating systems
+    # that support fork-and-exec.
+    #
+    # This method accepts a variable number of parameters; the first param is always the
+    # command to run; successive parameters are command-line arguments for the process.
+    #
+    # === Parameters
+    # cmd(String):: Name of the command to run
+    # arg1(String):: Optional, first command-line argumument
+    # arg2(String):: Optional, first command-line argumument
+    # ...
+    # argN(String):: Optional, Nth command-line argumument
+    #
+    # === Return
+    # output(String):: The process' output
+    def run(cmd, *args)
+      pm = ProcessMonitor.new
+      output = StringIO.new
+
+      pm.spawn(cmd, *args) do |options|
+        output << options[:output] if options[:output]
+      end
+
+      pm.cleanup
+      output.close
+      output = output.string
+      return output
+    end
+
   end
 end

data/lib/right_scraper/scrapers/download_scraper.rb

@@ -33,10 +33,19 @@ module RightScale
       msg = "Downloading repository '#{@repo.display_name}'"
       @callback.call(msg, is_step=true) if @callback
       filename = @repo.url.split('/').last
-      user_opt = @repo.first_credential && @repo.second_credential ? "--user #{@repo.first_credential}:#{@repo.second_credential}" : ''
-      cmd = "curl --fail --silent --show-error --insecure --location #{user_opt} --output \"#{@current_repo_dir}/#{filename}\" '#{@repo.url}' 2>&1"
+
+      if @repo.first_credential && @repo.second_credential
+        user_opt = ['--user', "#{@repo.first_credential}:#{@repo.second_credential}"]
+      else
+        user_opt = []
+      end
+
+      args = ['--fail', '--silent', '--show-error', '--insecure', '--location']
+      args += user_opt
+      args += ['--output', "#{@current_repo_dir}/#{filename}", @repo.url]
+
       FileUtils.mkdir_p(@current_repo_dir)
-      res = @watcher.launch_and_watch(cmd, @current_repo_dir)
+      res = @watcher.launch_and_watch('curl', args, @current_repo_dir)
       handle_watcher_result(res, 'Download')
       if succeeded?
         unzip_opt = case @repo.url[/\.(.*)$/]
@@ -45,9 +54,8 @@ module RightScale
           else ''
         end
         Dir.chdir(@current_repo_dir) do
-          cmd = "tar x#{unzip_opt}f #{filename} 2>&1"
-          res = `#{cmd}`
-          @errors << res if $? != 0
+          res = run('tar', "x#{unzip_opt}f", filename)
+          @errors << res unless $?.success?
           File.delete(filename)
         end
       end

data/lib/right_scraper/scrapers/git_scraper.rb

@@ -37,7 +37,7 @@ module RightScale
       # msysgit or else a native Windows implementation such as Git#
       return false if (is_windows? || !File.directory?(@current_repo_dir))
       Dir.chdir(@current_repo_dir) do
-        remote_url = `git config --get remote.origin.url`.chomp
+        remote_url = run('git', 'config', '--get', 'remote.origin.url').chomp
         $?.success? && remote_url == @repo.url
       end
     end
@@ -67,7 +67,7 @@ module RightScale
               if is_tag && is_branch
                 @errors << 'Repository tag ambiguous: could be git tag or git branch'
               elsif !is_tag && !is_branch
-                current_sha = `git rev-parse HEAD`.chomp
+                current_sha = run('git', 'rev-parse', 'HEAD').chomp
                 if current_sha == @repo.tag
                   @callback.call("Nothing to update: already using #{@repo.tag}", is_step=false) if @callback
                   return true
@@ -93,8 +93,10 @@ module RightScale
       end
 
       if !@incremental && succeeded?
-        git_cmd = "#{@ssh_cmd} git clone --quiet --depth 1 \"#{@repo.url}\" \"#{@current_repo_dir}\" 2>&1"
-        res = @watcher.launch_and_watch(git_cmd, @current_repo_dir)
+        args = ['clone', '--quiet', '--depth', '1', @repo.url, @current_repo_dir] 
+        ENV['GIT_SSH'] = @ssh_cmd
+        res = @watcher.launch_and_watch('git', args, @current_repo_dir)
+        ENV['GIT_SSH'] = nil
         handle_watcher_result(res, 'git clone')
         if has_tag && succeeded?
           Dir.chdir(@current_repo_dir) do
@@ -109,8 +111,8 @@ module RightScale
                 @errors << 'Repository tag ambiguous: could be git tag or git branch'
               elsif is_branch 
                 if !on_branch
-                  output = `git branch #{@repo.tag} origin/#{@repo.tag} 2>&1`
-                  @errors << output if $? != 0
+                  output = run('git', 'branch', @repo.tag, "origin/#{repo.tag}")
+                  @errors << output unless $?.success?
                 end
               elsif !is_tag # Not a branch nor a tag, SHA ref? fetch everything so we have all SHAs
                 git_fetch(:depth => 2**31 -1)
@@ -167,7 +169,8 @@ module RightScale
       ssh = File.join(ssh_dir, 'ssh')
       File.open(ssh, 'w') { |f| f.puts("ssh -F #{ssh_config} $*") }
       File.chmod(0755, ssh)
-      "GIT_SSH=#{ssh}"
+
+      return ssh
     end
 
     # Prepare SSH for git on Windows
@@ -200,7 +203,7 @@ module RightScale
         # "-o StrictHostKeyChecking=no" in the GIT_SSH executable, but it is
         # still a mystery why this doesn't work properly in windows.
         # so make a ssh call which creates the proper "known_hosts" file.
-        out = `ssh -o StrictHostKeyChecking=no #{repo.url.split(':').first} exit 2>&1`
+        run('ssh', '-o', 'StrictHostKeyChecking=no', repo.url.split(':').first)
       end
       return ''
     end
@@ -222,8 +225,10 @@ module RightScale
       remote  = opts[:remote_tag] 
       remote  = 'master' if remote.nil? || remote.rstrip.empty?
       action  = (opts[:merge] ? 'pull' : 'fetch')
-      git_cmd = "#{@ssh_cmd} git #{action} --tags --depth #{depth} origin #{remote} 2>&1"
-      res = @watcher.launch_and_watch(git_cmd, @current_repo_dir)
+      args = [action, '--tags', '--depth', depth, 'origin', remote]
+      ENV['GIT_SSH'] = @ssh_cmd
+      res = @watcher.launch_and_watch('git', args, @current_repo_dir)
+      ENV['GIT_SSH'] = nil
       handle_watcher_result(res, "git #{action}")
     end
 
@@ -237,8 +242,8 @@ module RightScale
     # === Return
     # output(String):: Output of git command
     def git_checkout(tag)
-      output = `git checkout #{tag} 2>&1`
-      @errors << output if $? != 0
+      output = run('git', 'checkout', tag)
+      @errors << output unless $?.success?
       output
     end
 
@@ -255,9 +260,9 @@ module RightScale
     def analyze_repo_tag
       is_tag = is_branch = on_branch = nil
       begin
-        is_tag = `git tag`.split("\n").include?(@repo.tag)
-        is_branch = `git branch -r`.split("\n").map { |t| t.strip }.include?("origin/#{@repo.tag}")
-        on_branch = is_branch && !!`git branch`.split("\n").include?("* #{@repo.tag}")
+        is_tag = run('git', 'tag').split("\n").include?(@repo.tag)
+        is_branch = run('git', 'branch', '-r').split("\n").map { |t| t.strip }.include?("origin/#{@repo.tag}")
+        on_branch = is_branch && !!run('git', 'branch').split("\n").include?("* #{@repo.tag}")
       rescue Exception => e
         @errors << "Analysis of repository tag failed with: #{e.message}"
       end

data/lib/right_scraper/scrapers/svn_scraper.rb

@@ -38,13 +38,13 @@ module RightScale
       cookbooks_path = [ cookbooks_path ] unless cookbooks_path.is_a?(Array)
       if cookbooks_path.empty?
         Dir.chdir(@current_repo_dir) do
-          info = `svn info`
+          info = run('svn', 'info')
           inc = $?.success? && info =~ (/^URL: (.*)$/) && $1 == @repo.url
         end
       else
         cookbooks_path.each do |path|
           Dir.chdir(File.join(@current_repo_dir, path)) do
-            info = `svn info`
+            info = run('svn', 'info')
             inc = $?.success? && info =~ (/^URL: (.*)$/) && $1 == File.join(@repo.url, path)
             break unless inc
           end
@@ -64,14 +64,14 @@ module RightScale
       cookbooks_path = repo.cookbooks_path || []
       cookbooks_path = [ cookbooks_path ] unless cookbooks_path.is_a?(Array)
       if @incremental
-        svn_cmd = "svn update --no-auth-cache --non-interactive --quiet" +
-        (!@repo.tag.nil? && !@repo.tag.empty? ? " --revision #{@repo.tag}" : '') +
-        (@repo.first_credential ? " --username #{@repo.first_credential}" : '') +
-        (@repo.second_credential ? " --password #{@repo.second_credential}" : '') +
-        ' 2>&1'
+        args = ['update', '--no-auth-cache', '--non-interactive', '--quiet']        
+        args += ['--revision', @repo.tag] if (!@repo.tag.nil? && !@repo.tag.empty?)
+        args += ['--username', @repo.first_credential] if @repo.first_credential
+        args += ['--password', @repo.second_credential] if @repo.second_credential
+        
         if cookbooks_path.empty?
           Dir.chdir(@current_repo_dir) do
-            res = @watcher.launch_and_watch(svn_cmd, @current_repo_dir)
+            res = @watcher.launch_and_watch('svn', args, @current_repo_dir)
             handle_watcher_result(res, 'SVN update')
           end
         else
@@ -79,7 +79,7 @@ module RightScale
             break unless succeeded?
             full_path = File.join(@current_repo_dir, path)
             Dir.chdir(full_path) do
-              res = @watcher.launch_and_watch(svn_cmd, @current_repo_dir)
+              res = @watcher.launch_and_watch('svn', args, @current_repo_dir)
               handle_watcher_result(res, 'SVN update')
             end
           end
@@ -87,12 +87,12 @@ module RightScale
       end
       if !@incremental && succeeded?
         if cookbooks_path.empty?
-          res = @watcher.launch_and_watch(svn_checkout_cmd, @current_repo_dir)
+          res = @watcher.launch_and_watch('svn', svn_checkout_args, @current_repo_dir)
           handle_watcher_result(res, 'SVN checkout')
         else
           cookbooks_path.each do |path|
             break unless succeeded?
-            res = @watcher.launch_and_watch(svn_checkout_cmd(path), @current_repo_dir)
+            res = @watcher.launch_and_watch('svn', svn_checkout_args(path), @current_repo_dir)
             handle_watcher_result(res, 'SVN checkout')
           end
         end
@@ -107,12 +107,13 @@ module RightScale
     #
     # === Return
     # svn_cmd(String):: Corresponding SVN command line
-    def svn_checkout_cmd(path='')
-      svn_cmd = "svn checkout \"#{File.join(@repo.url, path)}\" \"#{File.join(@current_repo_dir, path)}\" --no-auth-cache --non-interactive --quiet" +
-      (!@repo.tag.nil? && !@repo.tag.empty? ? " --revision #{@repo.tag}" : '') +
-      (@repo.first_credential ? " --username #{@repo.first_credential}" : '') +
-      (@repo.second_credential ? " --password #{@repo.second_credential}" : '') +
-      ' 2>&1'
+    def svn_checkout_args(path='')
+      args = ['checkout', File.join(@repo.url, path), File.join(@current_repo_dir, path),
+              '--no-auth-cache', '--non-interactive', '--quiet']
+      args += ['--revision', @repo.tag] if !@repo.tag.nil? && !@repo.tag.empty?
+      args += ['--username', @repo.first_credential] if @repo.first_credential
+      args += ['--password', @repo.second_credential] if @repo.second_credential
+      return args
     end
   end
 end

data/lib/right_scraper/watcher.rb

@@ -75,13 +75,13 @@ module RightScale
     #
     # === Return
     # res(RightScale::WatchStatus):: Outcome of watch, see RightScale::WatchStatus
-    def launch_and_watch(cmd, dest_dir)
+    def launch_and_watch(cmd, args, dest_dir)
       exit_code = nil
       output    = ''
       monitor   = ProcessMonitor.new
 
       # Run external process and monitor it in a new thread, platform specific
-      pid = monitor.spawn(cmd) do |data|
+      pid = monitor.spawn(cmd, *args) do |data|
         output << data[:output] if data[:output]
         exit_code = data[:exit_code] if data.include?(:exit_code)
       end

data/lib/right_scraper/win32/process_monitor.rb

@@ -35,7 +35,11 @@ module RightScale
     # Spawn given process and callback given block with output and exit code
     #
     # === Parameters
-    # cmd(String):: Process command line (including arguments)
+    # cmd(String):: Name of the command to run
+    # arg1(String):: Optional, first command-line argumument
+    # arg2(String):: Optional, first command-line argumument
+    # ...
+    # argN(String):: Optional, Nth command-line argumument
     #
     # === Block
     # Given block should take one argument which is a hash which may contain
@@ -45,7 +49,10 @@ module RightScale
     #
     # === Return
     # pid(Integer):: Spawned process pid
-    def spawn(cmd)
+    def spawn(cmd, *args)
+      args = args.map { |a| a.to_s }
+      cmd = ([cmd] + args).join(' ')
+
       # Run external process and monitor it in a new thread
       @io = IO.popen(cmd)
       @handle = OpenProcess(PROCESS_ALL_ACCESS, 0, @io.pid)

data/right_scraper.gemspec

@@ -23,7 +23,7 @@ require 'rubygems'
 
 spec = Gem::Specification.new do |spec|
   spec.name      = 'right_scraper'
-  spec.version   = '1.0.23'
+  spec.version   = '1.0.24'
   spec.authors   = ['Raphael Simon']
   spec.email     = 'raphael@rightscale.com'
   spec.homepage  = 'https://github.com/rightscale/right_scraper'

data/spec/watcher_spec.rb

@@ -37,7 +37,8 @@ describe RightScale::Watcher do
 
   it 'should launch and watch well-behaved processes' do
     watcher = RightScale::Watcher.new(max_bytes=1, max_seconds=5)
-    status = watcher.launch_and_watch('ruby -e "puts 42; exit 42"', @dest_dir)
+    ruby = "trap('INT', 'IGNORE'); puts 42; exit 42"
+    status = watcher.launch_and_watch('ruby', ['-e', ruby], @dest_dir)
     status.status.should == :success
     status.exit_code.should == 42
     status.output.should == "42\n"
@@ -45,7 +46,8 @@ describe RightScale::Watcher do
 
   it 'should report timeouts' do
     watcher = RightScale::Watcher.new(max_bytes=1, max_seconds=2)
-    status = watcher.launch_and_watch('ruby -e "STDOUT.sync = true; puts 42; sleep 5" 2>&1', @dest_dir)
+    ruby = "trap('INT', 'IGNORE'); STDOUT.sync = true; puts 42; sleep 5"
+    status = watcher.launch_and_watch('ruby',  ['-e', ruby], @dest_dir)
     status.status.should == :timeout
     status.exit_code.should == -1
     status.output.should == "42\n"
@@ -53,7 +55,8 @@ describe RightScale::Watcher do
 
   it 'should report size exceeded' do
     watcher = RightScale::Watcher.new(max_bytes=1, max_seconds=5)
-    status = watcher.launch_and_watch("ruby -e 'STDOUT.sync = true; puts 42; File.open(File.join(\"#{@dest_dir}\", \"test\"), \"w\") { |f| f.puts \"MORE THAN 2 CHARS\" }'; sleep 5", @dest_dir)
+    ruby = "trap('INT', 'IGNORE'); STDOUT.sync = true; puts 42; File.open(File.join('#{@dest_dir}', 'test'), 'w') { |f| f.puts 'MORE THAN 2 CHARS' }; sleep 5 rescue nil"
+    status = watcher.launch_and_watch('ruby', ['-e', ruby], @dest_dir)
     status.status.should == :size_exceeded
     status.exit_code.should == -1
     status.output.should == "42\n"
@@ -61,7 +64,8 @@ describe RightScale::Watcher do
 
   it 'should allow infinite size and timeout' do
     watcher = RightScale::Watcher.new(max_bytes=-1, max_seconds=-1)
-    status = watcher.launch_and_watch("ruby -e 'STDOUT.sync = true; puts 42; File.open(File.join(\"#{@dest_dir}\", \"test\"), \"w\") { |f| f.puts \"MORE THAN 2 CHARS\" };sleep 2'", @dest_dir)
+    ruby = "trap('INT', 'IGNORE'); STDOUT.sync = true; puts 42; File.open(File.join('#{@dest_dir}', 'test'), 'w') { |f| f.puts 'MORE THAN 2 CHARS' }; sleep 2 rescue nil"
+    status = watcher.launch_and_watch('ruby', ['-e', ruby], @dest_dir)
     status.status.should == :success
     status.exit_code.should == 0
     status.output.should == "42\n"

metadata

@@ -1,7 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: right_scraper
 version: !ruby/object:Gem::Version 
-  version: 1.0.23
+  hash: 39
+  prerelease: false
+  segments: 
+  - 1
+  - 0
+  - 24
+  version: 1.0.24
 platform: ruby
 authors: 
 - Raphael Simon
@@ -9,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-08-02 00:00:00 -07:00
+date: 2010-08-31 00:00:00 -07:00
 default_executable: 
 dependencies: []
 
@@ -62,21 +68,29 @@ rdoc_options:
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 59
+      segments: 
+      - 1
+      - 8
+      - 6
       version: 1.8.6
-  version: 
 required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
       version: "0"
-  version: 
 requirements: []
 
 rubyforge_project: right_scraper
-rubygems_version: 1.3.5
+rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
 summary: Download and update remote repositories