right_scraper 1.0.4 → 1.0.5

data/lib/right_scraper/scraper.rb

@@ -40,8 +40,12 @@ module RightScale
     #
     # === Parameters
     # scrape_dir(String):: Scrape destination directory
-    def initialize(scrape_dir)
+    # max_bytes(Integer):: Maximum size allowed for repos, -1 for no limit (default)
+    # max_seconds(Integer):: Maximum number of seconds a single scrape operation should take, -1 for no limit (default)
+    def initialize(scrape_dir, max_bytes = -1, max_seconds = -1)
       @scrape_dir = scrape_dir
+      @max_bytes = max_bytes
+      @max_seconds = max_seconds
       @scrapers = {}
     end
 
@@ -70,7 +74,7 @@ module RightScale
     def scrape(repo, &callback)
       repo = RightScale::Repository.from_hash(repo) if repo.is_a?(Hash)
       raise "Invalid repository type" unless SCRAPERS.include?(repo.repo_type)
-      @scraper = @scrapers[repo.repo_type] ||= SCRAPERS[repo.repo_type].new(@scrape_dir)
+      @scraper = @scrapers[repo.repo_type] ||= SCRAPERS[repo.repo_type].new(@scrape_dir, @max_bytes, @max_seconds)
       @scraper.scrape(repo, &callback)
       @last_repo_dir = @scraper.current_repo_dir
       @scraper.succeeded?

data/lib/right_scraper/scraper_base.rb

@@ -41,12 +41,15 @@ module RightScale
     # (String) Path to local directory where repository was downloaded
     attr_reader :current_repo_dir
 
-    # Set path to directory containing all scraped repos
+    # Set path to directory containing all scraped repos as well as space and time upperbounds
     #
     # === Parameters
     # root_dir(String):: Path to scraped repos parent directory
-    def initialize(root_dir)
+    # max_bytes(Integer):: Maximum size allowed for repos, -1 for no limit (default)
+    # max_seconds(Integer):: Maximum number of seconds a single scrape operation should take, -1 for no limit (default)
+    def initialize(root_dir, max_bytes, max_seconds)
       @root_dir = root_dir
+      @watcher  = Watcher.new(max_bytes, max_seconds)
     end
 
     # Common implementation of scrape method for all repository types.
@@ -108,6 +111,35 @@ module RightScale
     def scrape_imp
       raise "Method not implemented"
     end
-    
+
+    # Update state of scraper according to status returned by watcher
+    #
+    # === Parameters
+    # res(RightScale::WatchResult):: Watcher status to be analyzed
+    # msg_title(String):: Error message title in case of failure
+    # update(TrueClass|FalseClass):: Whether the process was launch to incrementally update the repo
+    # ok_codes:: Successful process return codes, only 0 by default
+    #
+    # === Return
+    # true:: Always return true
+    def handle_watcher_result(res, msg_title, update, ok_codes=[0])
+      if res.status == :timeout
+        @errors << "#{msg_title} is taking more time than #{@watcher.max_seconds / 60} minutes, aborting..."
+        FileUtils.rm_rf(@current_repo_dir)
+      elsif res.status == :size_exceeded
+        @errors << "#{msg_title} is taking more space than #{@watcher.max_bytes / 1048576} MB, aborting..."
+        FileUtils.rm_rf(@current_repo_dir)
+      elsif !ok_codes.include?(res.exit_code)
+        if update
+          @callback.call("#{msg_title} failed: #{res.output}, reverting to non incremental update", is_step=false) if @callback
+          FileUtils.rm_rf(@current_repo_dir)
+          @incremental = false
+        else
+          @errors << "#{msg_title} failed: #{res.output}"
+        end
+      end
+      true
+    end
+
   end
 end

data/lib/right_scraper/scrapers/download_scraper.rb

@@ -36,8 +36,8 @@ module RightScale
       user_opt = @repo.first_credential && @repo.second_credential ? "--user #{@repo.first_credential}:#{@repo.second_credential}" : ''
       cmd = "curl --fail --silent --show-error --insecure --location #{user_opt} --output \"#{@current_repo_dir}/#{filename}\" '#{@repo.url}' 2>&1"
       FileUtils.mkdir_p(@current_repo_dir)
-      res = `#{cmd}`
-      @errors << res if $? != 0
+      res = @watcher.launch_and_watch(cmd, @current_repo_dir)
+      handle_watcher_result(res, 'Download', update=false)
       if succeeded?
         unzip_opt = case @repo.url[/\.(.*)$/]
           when 'bzip', 'bzip2', 'bz2' then 'j'

data/lib/right_scraper/scrapers/git_scraper.rb

@@ -48,19 +48,17 @@ module RightScale
       msg += "git repository '#{@repo.display_name}'"
       @callback.call(msg, is_step=true) if @callback
       ssh_cmd = ssh_command
-      res = ''
       is_tag = nil
       is_branch = nil
-      update_failed = false
 
       if @incremental
         Dir.chdir(@current_repo_dir) do
-          analysis = git_fetch_and_analyze(ssh_cmd)
-          if analysis[:success]
+          analysis = git_fetch_and_analyze(ssh_cmd, update=true)
+          if succeeded? && @incremental
             is_tag = analysis[:tag]
             is_branch = analysis[:branch]
             if !is_tag && !is_branch
-              @callback.call('Nothing to update: repo tag refers to neither a branch nor a tag', is_step=false)
+              @callback.call('Nothing to update: repo tag refers to neither a branch nor a tag', is_step=false) if @callback
               return true
             end
             if is_tag && is_branch
@@ -70,35 +68,25 @@ module RightScale
               res = `git checkout #{tag} 2>&1`
               if $? != 0
                 @callback.call("Failed to checkout #{tag}: #{res}, falling back to cloning", is_step=false) if @callback
-                update_failed = true
+                FileUtils.rm_rf(@current_repo_dir)
+                @incremental = false
               end
             end
-          else
-            @callback.call("Failed to update repo: #{analysis[:output]}, falling back to cloning", is_step=false) if @callback
-            update_failed = true
           end
         end
       end
-      if update_failed
-        FileUtils.rm_rf(@current_repo_dir)
-        @incremental = false
-      end
       if !@incremental && succeeded?
-        res += `#{ssh_cmd} git clone --quiet --depth 1 "#{@repo.url}" "#{@current_repo_dir}" 2>&1`
-        @errors << "Failed to clone repo: #{res}" if $? != 0
+        git_cmd = "#{ssh_cmd} git clone --quiet --depth 1 \"#{@repo.url}\" \"#{@current_repo_dir}\" 2>&1"
+        res = @watcher.launch_and_watch(git_cmd, @current_repo_dir)
+        handle_watcher_result(res, 'git clone', update=false)
         if !@repo.tag.nil? && !@repo.tag.empty? && @repo.tag != 'master' && succeeded?
           Dir.chdir(@current_repo_dir) do
             if is_tag.nil?
-              analysis = git_fetch_and_analyze(ssh_cmd)
-              if analysis[:success]
-                is_tag = analysis[:tag]
-                is_branch = analysis[:branch]
-                res += analysis[:output]
-              else
-                @errors << "Failed to analyze repo: #{res}"
-              end
+              analysis = git_fetch_and_analyze(ssh_cmd, update=false)
+              is_tag = analysis[:tag]
+              is_branch = analysis[:branch]
             end
-            if succeded?
+            if succeeded?
               if is_tag && is_branch
                 @errors << 'Repository tag ambiguous: could be git tag or git branch'
               elsif is_branch
@@ -202,26 +190,27 @@ module RightScale
     # Shallow fetch
     # Resolves whehter repository tag is a git tag or a git branch
     # Return output of run commands too
+    # Update errors collection upon failure (check for succeeded? after call)
     # Note: Assume that current working directory is a git directory
     #
     # === Parameters
     # ssh_cmd(String):: SSH command to be used with git if any
+    # update(FalseClass|TrueClass):: Whether analysis is done as part of an update
     #
     # === Return
     # res(Hash)::
-    #   - resp[:success]:: true if fetch was successful, false otherwise
-    #   - res[:output] contains the git output
     #   - res[:tag]:: is true if git repo has a tag with a name corresponding to the repository tag
     #   - res[:branch] is true if git repo has a branch with a name corresponding to the repository tag
-    def git_fetch_and_analyze(ssh_cmd)
-      output = `#{ssh_cmd} git fetch --tags --depth 1 2>&1`
-      success = [0, 1].include? $?.exitstatus # git fetch returns 1 when there is nothing to fetch
+    def git_fetch_and_analyze(ssh_cmd, update)
+      git_cmd = "#{ssh_cmd} git fetch --tags --depth 1 2>&1"
+      res = @watcher.launch_and_watch(git_cmd, @current_repo_dir)
+      handle_watcher_result(res, 'git fetch', update, ok_codes=[0, 1]) # git fetch returns 1 when there is nothing to fetch
       is_tag = is_branch = false
-      if success
+      if succeeded? && (!update || @incremental)
         is_tag = `git tag`.split("\n").include?(@repo.tag)
         is_branch = `git branch -r`.split("\n").map { |t| t.strip }.include?("origin/#{@repo.tag}")
       end
-      { :success => success, :output => output, :tag => is_tag, :branch => is_branch }
+      { :tag => is_tag, :branch => is_branch }
     end
 
   end

data/lib/right_scraper/scrapers/svn_scraper.rb

@@ -53,22 +53,18 @@ module RightScale
         (@repo.second_credential ? " --password #{@repo.second_credential}" : '') +
         ' 2>&1'
         Dir.chdir(@current_repo_dir) do
-          res = `#{svn_cmd}`
-          if $? != 0
-            @callback.call("Failed to update repo: #{res}, falling back to checkout", is_step=false) if @callback
-            FileUtils.rm_rf(@current_repo_dir)
-            @incremental = false
-          end
+          res = @watcher.launch_and_watch(svn_cmd, @current_repo_dir)
+          handle_watcher_result(res, 'SVN update', update=true)
         end
       end
-      if !@incremental
+      if !@incremental && succeeded?
         svn_cmd = "svn checkout \"#{@repo.url}\" \"#{@current_repo_dir}\" --no-auth-cache --non-interactive --quiet" +
         (!@repo.tag.nil? && !@repo.tag.empty? ? " --revision #{@repo.tag}" : '') +
         (@repo.first_credential ? " --username #{@repo.first_credential}" : '') +
         (@repo.second_credential ? " --password #{@repo.second_credential}" : '') +
         ' 2>&1'
-        res = `#{svn_cmd}`
-        @errors << res if $? != 0
+        res = @watcher.launch_and_watch(svn_cmd, @current_repo_dir)
+        handle_watcher_result(res, 'SVN checkout', update=false)
       end
       true
     end

data/lib/right_scraper/watcher.rb

@@ -0,0 +1,141 @@
+#--
+# Copyright: Copyright (c) 2010 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+require 'find'
+
+module RightScale
+
+  # Encapsulate information returned by watcher
+  class WatchStatus
+
+    # Potential outcome of watcher
+    VALID_STATUSES = [ :success, :timeout, :size_exceeded ]
+
+    attr_reader :status    # One of VALID_STATUSES
+    attr_reader :exit_code # Watched process exit code or -1 if process was killed
+    attr_reader :output    # Watched process combined output
+
+    # Initialize attibutes
+    def initialize(status, exit_code, output)
+      @status    = status
+      @exit_code = exit_code
+      @output    = output
+    end
+
+  end
+
+  class Watcher
+
+    attr_reader :max_bytes   # Maximum size in bytes of watched directory before process is killed
+    attr_reader :max_seconds # Maximum number of elapased seconds before external process is killed
+
+    # Initialize attributes
+    #
+    # max_bytes(Integer):: Maximum size in bytes of watched directory before process is killed
+    # max_seconds(Integer):: Maximum number of elapased seconds before external process is killed
+    def initialize(max_bytes, max_seconds)
+      @max_bytes   = max_bytes
+      @max_seconds = max_seconds
+    end
+
+    # Launch given command as external process and watch given directory
+    # so it doesn't exceed given size. Also watch time elapsed and kill
+    # external process if either the size of the watched directory exceed
+    # @max_bytes or the time elapsed exceeds @max_seconds.
+    # Note: This method is not thread-safe, instantiate one watcher per thread
+    #
+    # === Parameters
+    # cmd(String):: Command line to be launched
+    # dest_dir(String):: Watched directory
+    #
+    # === Return
+    # res(RightScale::WatchStatus):: Outcome of watch, see RightScale::WatchStatus
+    def launch_and_watch(cmd, dest_dir)
+      status = nil
+      output = ''
+
+      # Run external process and monitor it in a new thread
+      r = IO.popen(cmd)
+      Thread.new do
+        Process.wait(r.pid)
+        status = $?
+      end
+
+      # Loop until process is done or times out or takes too much space
+      timed_out = repeat(1, @max_seconds) do
+        output += r.readlines.join
+        size = 0
+        Find.find(dest_dir) { |f| size += File.stat(f).size unless File.directory?(f) } if File.directory?(dest_dir)
+        size > @max_bytes || status
+      end
+
+      # Cleanup and report status
+      output += r.readlines.join
+      Process.kill('TERM', r.pid) unless status
+      r.close
+      s = status ? :success : (timed_out ? :timeout : :size_exceeded)
+      exit_code = status && status.exitstatus || -1
+      res = WatchStatus.new(s, exit_code, output)
+    end
+
+    protected
+
+    # Run given block in thread and time execution
+    #
+    # === Block
+    # Block whose execution is timed
+    #
+    # === Return
+    # elapsed(Integer):: Number of seconds elapsed while running given block
+    def timed
+      start_at = Time.now
+      yield
+      elapsed = Time.now - start_at
+    end
+
+    # Repeat given block at regular intervals
+    #
+    # === Parameters
+    # seconds(Integer):: Number of seconds between executions
+    # timeout(Integer):: Timeout after which execution stops and method returns
+    #
+    # === Block
+    # Given block gets executed every period seconds until timeout is reached
+    # *or* block returns true
+    #
+    # === Return
+    # res(TrueClass|FalseClass):: true if timeout is reached, false otherwise.
+    def repeat(period, timeout)
+      end_at = Time.now + timeout
+      while res = (Time.now < end_at)
+        exit = false
+        elapsed = timed { exit = yield }
+        break if exit
+        sleep(period - elapsed) if elapsed < period
+      end
+      !res
+    end
+
+  end
+
+end

data/right_scraper.gemspec

@@ -23,7 +23,7 @@ require 'rubygems'
 
 spec = Gem::Specification.new do |spec|
   spec.name      = 'right_scraper'
-  spec.version   = '1.0.4'
+  spec.version   = '1.0.5'
   spec.authors   = ['Raphael Simon']
   spec.email     = 'raphael@rightscale.com'
   spec.homepage  = 'https://github.com/rightscale/right_scraper'

data/spec/__origin/config

@@ -0,0 +1,5 @@
+[core]
+	repositoryformatversion = 0
+	filemode = true
+	bare = true
+	ignorecase = true

data/spec/download_scraper_spec.rb

@@ -65,7 +65,7 @@ describe RightScale::DownloadScraper do
     end
 
     before(:each) do
-      @scraper = RightScale::DownloadScraper.new(@repo_path)
+      @scraper = RightScale::DownloadScraper.new(@repo_path, max_bytes=1024**2, max_seconds=20)
       @repo = RightScale::Repository.from_hash(:display_name => 'test repo',
                                                :repo_type    => :download,
                                                :url          => "file:///#{@download_file}")

data/spec/git_scraper_spec.rb

@@ -24,6 +24,7 @@
 require File.join(File.dirname(__FILE__), 'spec_helper')
 require 'scraper_base'
 require 'repository'
+require 'watcher'
 require File.join('scrapers', 'git_scraper')
 
 describe RightScale::GitScraper do
@@ -70,7 +71,7 @@ describe RightScale::GitScraper do
     end
 
     before(:each) do
-      @scraper = RightScale::GitScraper.new(@repo_path)      
+      @scraper = RightScale::GitScraper.new(@repo_path, max_bytes=1024**2, max_seconds=20)      
       @repo = RightScale::Repository.from_hash(:display_name => 'test repo',
                                                :repo_type    => :git,
                                                :url          => @origin_path)

data/spec/scraper_base_spec.rb

@@ -27,7 +27,7 @@ require 'scraper_base'
 describe RightScale::ScraperBase do
 
   before(:each) do
-    @base = RightScale::ScraperBase.new('/tmp')
+    @base = RightScale::ScraperBase.new('/tmp', max_bytes=1024**2, max_seconds=20)
   end
   
   it 'should initialize the scrape directory' do

data/spec/svn_scraper_spec.rb

@@ -66,7 +66,7 @@ describe RightScale::SvnScraper do
     end
 
     before(:each) do
-      @scraper = RightScale::SvnScraper.new(@repo_path)      
+      @scraper = RightScale::SvnScraper.new(@repo_path, max_bytes=1024**2, max_seconds=20)      
       @repo = RightScale::Repository.from_hash(:display_name => 'test repo',
                                                :repo_type    => :svn,
                                                :url          => "file://#{@svn_repo_path}")

data/spec/watcher_spec.rb

@@ -0,0 +1,62 @@
+#--
+# Copyright: Copyright (c) 2010 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+require File.join(File.dirname(__FILE__), 'spec_helper')
+require 'watcher'
+
+describe RightScale::Watcher do
+
+  before(:each) do
+    @dest_dir = File.join(File.dirname(__FILE__), '__destdir')
+    FileUtils.mkdir_p(@dest_dir)
+  end
+
+  after(:each) do
+    FileUtils.rm_rf(@dest_dir)
+  end
+
+  it 'should launch and watch well-behaved processes' do
+    watcher = RightScale::Watcher.new(max_bytes=1, max_seconds=5)
+    status = watcher.launch_and_watch('ruby -e "puts 42; exit 42"', @dest_dir)
+    status.status.should == :success
+    status.exit_code.should == 42
+    status.output.should == "42\n"
+  end
+
+  it 'should report timeouts' do
+    watcher = RightScale::Watcher.new(max_bytes=1, max_seconds=2)
+    status = watcher.launch_and_watch('ruby -e "puts 42; sleep 3"', @dest_dir)
+    status.status.should == :timeout
+    status.exit_code.should == -1
+    status.output.should == "42\n"
+  end
+
+  it 'should report size exceeded' do
+    watcher = RightScale::Watcher.new(max_bytes=1, max_seconds=5)
+    status = watcher.launch_and_watch("ruby -e 'puts 42; File.open(File.join(\"#{@dest_dir}\", \"test\"), \"w\") { |f| f.puts \"MORE THAN 2 CHARS\" }'", @dest_dir)
+    status.status.should == :size_exceeded
+    status.exit_code.should == -1
+    status.output.should == "42\n"
+  end
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: right_scraper
 version: !ruby/object:Gem::Version 
-  version: 1.0.4
+  version: 1.0.5
 platform: ruby
 authors: 
 - Raphael Simon
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-02-05 00:00:00 -08:00
+date: 2010-02-08 00:00:00 -08:00
 default_executable: 
 dependencies: []
 
@@ -32,7 +32,9 @@ files:
 - lib/right_scraper/scrapers/download_scraper.rb
 - lib/right_scraper/scrapers/git_scraper.rb
 - lib/right_scraper/scrapers/svn_scraper.rb
+- lib/right_scraper/watcher.rb
 - right_scraper.gemspec
+- spec/__origin/config
 - spec/download_scraper_spec.rb
 - spec/git_scraper_spec.rb
 - spec/rcov.opts
@@ -42,6 +44,7 @@ files:
 - spec/spec.opts
 - spec/spec_helper.rb
 - spec/svn_scraper_spec.rb
+- spec/watcher_spec.rb
 has_rdoc: true
 homepage: https://github.com/rightscale/right_scraper
 licenses: []