right_scraper 1.0.9 → 1.0.10

data/lib/right_scraper/linux/process_monitor.rb

@@ -0,0 +1,67 @@
+#--
+# Copyright: Copyright (c) 2010 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+module RightScale
+
+  # *nix specific watcher implementation
+  class ProcessMonitor
+
+    # Spawn given process and callback given block with output and exit code
+    #
+    # === Parameters
+    # cmd(String):: Process command line (including arguments)
+    #
+    # === Block
+    # Given block should take one argument which is a hash which may contain
+    # the keys :output and :exit_code. The value associated with :output is a chunk
+    # of output while the value associated with :exit_code is the process exit code
+    # This block won't be called anymore once the :exit_code key has associated value
+    #
+    # === Return
+    # pid(Integer):: Spawned process pid
+    def spawn(cmd)
+      # Run external process and monitor it in a new thread
+      @io = IO.popen(cmd)
+      @reader = Thread.new do
+        o = @io.read
+        until o == ''
+          yield(:output => o)
+          o = @io.read
+        end
+        Process.wait(@io.pid)
+        yield(:exit_code => $?.exitstatus)
+      end
+      @io.pid
+    end
+
+    # Close io and join reader thread
+    #
+    # === Return
+    # true:: Always return true
+    def cleanup
+      @reader.join
+      @io.close
+    end
+
+  end
+end

data/lib/right_scraper/scraper_base.rb

@@ -56,7 +56,7 @@ module RightScale
     # Each scraper implementation should override scrape_imp which is called
     # after this method initializes all the scraper attributes properly.
     # See RightScale::Scraper#scrape
-    def scrape(repo, incremental, &callback)
+    def scrape(repo, incremental=true, &callback)
       @repo             = repo
       @callback         = callback
       @current_repo_dir = ScraperBase.repo_dir(root_dir, repo)

data/lib/right_scraper/scrapers/git_scraper.rb

@@ -182,7 +182,7 @@ module RightScale
 		# "-o StrictHostKeyChecking=no" in the GIT_SSH executable, but it is
 		# still a mystery why this doesn't work properly in windows.
 		# so make a ssh call which creates the proper "known_hosts" file.
-		system("ssh -o StrictHostKeyChecking=no #{repo.url.split(':').first} exit")
+		system("ssh -o StrictHostKeyChecking=no #{repo.url.split(':').first} exit 2>&1")
 	  end
       return ''
     end

data/lib/right_scraper/watcher.rb

@@ -22,6 +22,11 @@
 #++
 
 require 'find'
+if RUBY_PLATFORM =~ /mswin/
+  require File.join(File.dirname(__FILE__), 'win32', 'process_monitor')
+else
+  require File.join(File.dirname(__FILE__), 'linux', 'process_monitor')
+end
 
 module RightScale
 
@@ -72,18 +77,13 @@ module RightScale
     # res(RightScale::WatchStatus):: Outcome of watch, see RightScale::WatchStatus
     def launch_and_watch(cmd, dest_dir)
       exit_code = nil
-      output = ''
-
-      # Run external process and monitor it in a new thread
-      io = IO.popen(cmd)
-      reader = Thread.new do
-        o = io.read
-        until o == ''
-          output += o
-          o = io.read
-        end
-        Process.wait(io.pid)
-        exit_code = $?.exitstatus
+      output    = ''
+      monitor   = ProcessMonitor.new
+
+      # Run external process and monitor it in a new thread, platform specific
+      pid = monitor.spawn(cmd) do |data|
+        output << data[:output] if data.include?(:output)
+        exit_code = data[:exit_code] if data.include?(:exit_code)
       end
 
       # Loop until process is done or times out or takes too much space
@@ -106,10 +106,12 @@ module RightScale
       else
         exit_status = -1
         outcome = (timed_out ? :timeout : :size_exceeded)
-        Process.kill('INT', io.pid)
+        Process.kill('INT', pid)
       end
-      reader.join
-      io.close
+
+      # Cleanup any open handle etc., platform specific
+      monitor.cleanup
+
       res = WatchStatus.new(outcome, exit_status, output)
     end
 

data/lib/right_scraper/win32/process_monitor.rb

@@ -0,0 +1,91 @@
+#--
+# Copyright: Copyright (c) 2010 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+require 'win32/process'
+
+module RightScale
+
+  # Windows specific watcher implementation
+  class ProcessMonitor
+
+    include Windows::Process
+    include Windows::Synchronize
+    include Windows::Handle
+
+    # Spawn given process and callback given block with output and exit code
+    #
+    # === Parameters
+    # cmd(String):: Process command line (including arguments)
+    #
+    # === Block
+    # Given block should take one argument which is a hash which may contain
+    # the keys :output and :exit_code. The value associated with :output is a chunk
+    # of output while the value associated with :exit_code is the process exit code
+    # This block won't be called anymore once the :exit_code key has associated value
+    #
+    # === Return
+    # pid(Integer):: Spawned process pid
+    def spawn(cmd)
+      # Run external process and monitor it in a new thread
+      @io = IO.popen(cmd)
+      @handle = OpenProcess(PROCESS_ALL_ACCESS, 0, @io.pid)
+      case @handle
+      when INVALID_HANDLE_VALUE
+        # Something bad happened
+        yield(:exit_code => 1)
+      when 0
+        # Process already finished
+        yield(:exit_code => 0)
+      else
+        # Start output read
+        @reader = Thread.new do
+          o = @io.read
+          until o == ''
+            yield(:output => o)
+            o = @io.read
+          end
+          status = WaitForSingleObject(@handle, INFINITE)
+          exit_code = [0].pack('L')
+          if GetExitCodeProcess(@handle, exit_code)
+            exit_code = exit_code.unpack('L').first
+          else
+            exit_code = 1
+          end
+          yield(:exit_code => exit_code)
+        end
+      end
+      @io.pid
+    end
+
+    # Cleanup underlying handle
+    #
+    # === Return
+    # true:: Always return true
+    def cleanup
+      @reader.join
+      CloseHandle(@handle) if @handle > 0
+      @io.close
+    end
+
+  end
+end

data/right_scraper.gemspec

@@ -23,7 +23,7 @@ require 'rubygems'
 
 spec = Gem::Specification.new do |spec|
   spec.name      = 'right_scraper'
-  spec.version   = '1.0.9'
+  spec.version   = '1.0.10'
   spec.authors   = ['Raphael Simon']
   spec.email     = 'raphael@rightscale.com'
   spec.homepage  = 'https://github.com/rightscale/right_scraper'

data/spec/download_scraper_spec.rb

@@ -27,6 +27,7 @@ unless RUBY_PLATFORM=~/mswin/
 require File.join(File.dirname(__FILE__), 'spec_helper')
 require 'scraper_base'
 require 'repository'
+require 'watcher'
 require File.join('scrapers', 'download_scraper')
 
 describe RightScale::DownloadScraper do

data/spec/scraper_spec.rb

@@ -22,10 +22,12 @@
 #++
 
 require File.join(File.dirname(__FILE__), 'spec_helper')
-require File.join('scrapers', 'git_scraper')
-require File.join('scrapers', 'svn_scraper')
-require File.join('scrapers', 'download_scraper')
-require 'scraper'
+require File.join(File.dirname(__FILE__), '..', 'lib', 'right_scraper', 'scraper_base')
+require File.join(File.dirname(__FILE__), '..', 'lib', 'right_scraper', 'scrapers', 'git_scraper')
+require File.join(File.dirname(__FILE__), '..', 'lib', 'right_scraper', 'scrapers', 'svn_scraper')
+require File.join(File.dirname(__FILE__), '..', 'lib', 'right_scraper', 'scrapers', 'download_scraper')
+require File.join(File.dirname(__FILE__), '..', 'lib', 'right_scraper', 'repository')
+require File.join(File.dirname(__FILE__), '..', 'lib', 'right_scraper', 'scraper')
 
 describe RightScale::Scraper do
 
@@ -33,17 +35,17 @@ describe RightScale::Scraper do
     @scraper = RightScale::Scraper.new('/tmp')
     @mock_scraper = flexmock('MockScraper')
     mock_scraper_klass = flexmock('MockScraperClass', :new => @mock_scraper)
-    RightScale::SCRAPERS.merge!(:mock => mock_scraper_klass)
+    RightScale::SCRAPERS.merge!('mock' => mock_scraper_klass)
   end
   
   after(:all) do
-    RightScale::SCRAPERS.delete(:mock)
+    RightScale::SCRAPERS.delete('mock')
   end
   
   it 'should scrape' do
     repo = RightScale::Repository.new
     repo.repo_type = :mock
-    @mock_scraper.should_receive(:scrape).with(repo, Proc).and_return(true)
+    @mock_scraper.should_receive(:scrape).with(repo, true, Proc).and_return(true)
     @mock_scraper.should_receive(:succeeded?).and_return(true)
     @mock_scraper.should_receive(:current_repo_dir).and_return('42')
     @scraper.scrape(repo) { }.should be_true
@@ -51,14 +53,14 @@ describe RightScale::Scraper do
   end
   
   it 'should scrape from a hash' do
-    @mock_scraper.should_receive(:scrape).with(RightScale::Repository, Proc).and_return(true)
+    @mock_scraper.should_receive(:scrape).with(RightScale::Repository, true, Proc).and_return(true)
     @mock_scraper.should_receive(:succeeded?).and_return(true)
     @mock_scraper.should_receive(:current_repo_dir).and_return('42')
     @scraper.scrape({:repo_type => :mock}) { }.should be_true
   end
   
   it 'should report failures' do
-    @mock_scraper.should_receive(:scrape).with(RightScale::Repository, Proc).and_return(true)
+    @mock_scraper.should_receive(:scrape).with(RightScale::Repository, true, Proc).and_return(true)
     @mock_scraper.should_receive(:succeeded?).and_return(false)
     @mock_scraper.should_receive(:current_repo_dir).and_return('42')
     @scraper.scrape({:repo_type => :mock}) { }.should be_false

data/spec/svn_scraper_spec.rb

@@ -24,6 +24,7 @@
 require File.join(File.dirname(__FILE__), 'spec_helper')
 require 'scraper_base'
 require 'repository'
+require 'watcher'
 require File.join('scrapers', 'svn_scraper')
 
 describe RightScale::SvnScraper do

data/spec/watcher_spec.rb

@@ -45,7 +45,7 @@ describe RightScale::Watcher do
 
   it 'should report timeouts' do
     watcher = RightScale::Watcher.new(max_bytes=1, max_seconds=2)
-    status = watcher.launch_and_watch('ruby -e "STDOUT.sync = true; puts 42; sleep 5"', @dest_dir)
+    status = watcher.launch_and_watch('ruby -e "STDOUT.sync = true; puts 42; sleep 5" 2>&1', @dest_dir)
     status.status.should == :timeout
     status.exit_code.should == -1
     status.output.should == "42\n"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: right_scraper
 version: !ruby/object:Gem::Version 
-  version: 1.0.9
+  version: 1.0.10
 platform: ruby
 authors: 
 - Raphael Simon
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-02-12 00:00:00 -08:00
+date: 2010-02-15 00:00:00 -08:00
 default_executable: 
 dependencies: []
 
@@ -26,6 +26,7 @@ files:
 - README.rdoc
 - Rakefile
 - lib/right_scraper.rb
+- lib/right_scraper/linux/process_monitor.rb
 - lib/right_scraper/repository.rb
 - lib/right_scraper/scraper.rb
 - lib/right_scraper/scraper_base.rb
@@ -33,6 +34,7 @@ files:
 - lib/right_scraper/scrapers/git_scraper.rb
 - lib/right_scraper/scrapers/svn_scraper.rb
 - lib/right_scraper/watcher.rb
+- lib/right_scraper/win32/process_monitor.rb
 - right_scraper.gemspec
 - spec/download_scraper_spec.rb
 - spec/git_scraper_spec.rb