right_scraper 3.2.3 → 3.2.4

data/lib/right_scraper/logger.rb

@@ -90,6 +90,10 @@ module RightScraper
       error("Saw #{exception} during #{type}#{maybe_explain(explanation)}")
     end
 
+    def note_warning(message)
+      warn(message)
+    end
+
     protected
     def maybe_explain(explanation)
       if explanation.nil? || explanation.empty?

data/lib/right_scraper/retrievers/git.rb

@@ -34,6 +34,17 @@ begin
 
   module Git
     class Lib
+      # Monkey patch to prevent screw up any subsequent shell out to git
+      def command_with_preserved_env(cmd, opts = [], chdir = true, redirect = '', &block)
+        variables_to_preserve = ['GIT_DIR', 'GIT_INDEX_FILE', 'GIT_WORK_TREE']
+        preserved_env = Hash[variables_to_preserve.map { |var| [var, ENV[var]] }]
+        begin
+          command_without_preserved_env(cmd, opts, chdir, redirect, &block)
+        ensure
+          preserved_env.each { |var, value| ENV[var] = value }
+        end
+      end
+
       # Monkey patch to blackwinter-git that strips ANSI escape sequences
       # from command output to avoid confusing the parser.
       def run_command_with_color_stripping(git_cmd, &block)
@@ -46,6 +57,11 @@ begin
         alias :run_command_without_color_stripping :run_command
         alias :run_command :run_command_with_color_stripping
       end
+
+      unless self.methods.include?('command_without_preserved_env')
+        alias :command_without_preserved_env :command
+        alias :command :command_with_preserved_env
+      end
     end
   end
 rescue ::Git::GitExecuteError
@@ -86,12 +102,61 @@ module RightScraper
       # fresh SSHAgent and add the credential to it.
       def retrieve
         raise RetrieverError.new("git retriever is unavailable") unless available?
+
+        start_time = nil
+        end_time = nil
         RightScraper::Processes::SSHAgent.with do |agent|
           unless @repository.first_credential.nil? || @repository.first_credential.empty?
             agent.add_key(@repository.first_credential)
           end
+          start_time = ::Time.now
           super
+          end_time = ::Time.now
+        end
+
+        # TEAL FIX: the use of blackwinter-git has defeated the logic that
+        # ensured the max bytes was not exceeded during checkout. we will need
+        # to replace blackwinter-git in future but in the interim our only
+        # solution is to warn the user after the checkout has completed that we
+        # are going to restrict their repo size/time in an upcoming release.
+        if size_limit_exceeded?
+          message =
+            "The size of the downloaded repository exceeded a soft limit of" +
+            " #{@max_bytes / (1024 * 1024)} MB. This will become a hard limit" +
+            " in an upcoming release. You may avoid retrieval failure by" +
+            " moving some of your files to seperate repositories."
+          @logger.note_warning(message)
+        end
+        if @max_seconds && (end_time >= start_time + @max_seconds)
+          message =
+            "The time to download the repository exceeded a soft limit of" +
+            " #{@max_seconds} seconds. This will become a hard limit" +
+            " in an upcoming release. You may avoid retrieval failure by" +
+            " moving some of your files to seperate repositories."
+          @logger.note_warning(message)
+        end
+        true
+      end
+
+      # Determines if total size of files created by child process has exceeded
+      # the limit specified, if any.
+      #
+      # === Return
+      # @return [TrueClass|FalseClass] true if size limit exceeded
+      def size_limit_exceeded?
+        exceeded = false
+        if @max_bytes
+          globbie = ::File.join(@repo_dir, '**/*')
+          size = 0
+          ::Dir.glob(globbie) do |f|
+            size += ::File.stat(f).size rescue 0 if ::File.file?(f)
+            if size > @max_bytes
+              exceeded = true
+              break
+            end
+          end
         end
+        exceeded
       end
 
       # Return true if a checkout exists.  Currently tests for .git in

data/lib/right_scraper/scraper.rb

@@ -122,6 +122,11 @@ module RightScraper
       @logger.errors
     end
 
+    # (Array):: Warnings or empty
+    def warnings
+      @logger.warnings
+    end
+
    # Was scraping successful?
     # Call errors to get error messages if false
     #

data/lib/right_scraper/scraper_logger.rb

@@ -24,8 +24,7 @@
 module RightScraper
 
   class ScraperLogger < Logger
-    attr_accessor :errors
-    attr_accessor :callback
+    attr_accessor :callback, :errors, :warnings
 
     def add(severity, message=nil, progname=nil)
       if severity >= (self.level || Logger::WARN)
@@ -46,6 +45,7 @@ module RightScraper
 
     def initialize
       @errors = []
+      @warnings = []
     end
 
     def note_phase(phase, type, explanation, exception=nil)
@@ -56,6 +56,11 @@ module RightScraper
     def note_error(exception, type, explanation="")
       @errors << [exception, type, explanation]
     end
+
+    def note_warning(message)
+      @warnings << message
+    end
+
   end
 
 end

data/right_scraper.gemspec

@@ -24,7 +24,7 @@ require 'rubygems'
 
 Gem::Specification.new do |spec|
   spec.name      = 'right_scraper'
-  spec.version   = '3.2.3'
+  spec.version   = '3.2.4'
   spec.authors   = ['Graham Hughes', 'Raphael Simon', 'Scott Messier']
   spec.email     = 'raphael@rightscale.com'
   spec.homepage  = 'https://github.com/rightscale/right_scraper'

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: right_scraper
 version: !ruby/object:Gem::Version 
-  hash: 9
+  hash: 7
   prerelease: false
   segments: 
   - 3
   - 2
-  - 3
-  version: 3.2.3
+  - 4
+  version: 3.2.4
 platform: ruby
 authors: 
 - Graham Hughes
@@ -17,7 +17,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2013-07-22 00:00:00 -07:00
+date: 2013-08-13 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency