right_scraper 3.2.6 → 5.0.1

data/lib/right_scraper/retrievers/checkout_base.rb

@@ -0,0 +1,178 @@
+#--
+# Copyright: Copyright (c) 2010-2013 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+# ancestor
+require 'right_scraper/retrievers'
+
+require 'fileutils'
+
+module RightScraper::Retrievers
+
+  # Base class for retrievers that want to do version control operations
+  # (CVS, SVN, etc.). Subclasses can get away with implementing only
+  # Retrievers::Base#available? and #do_checkout but to support incremental
+  # operation need to implement #exists? and #do_update, in addition to
+  # Retrievers::Base#ignorable_paths.
+  class CheckoutBase < ::RightScraper::Retrievers::Base
+
+    # Attempts to update and then resorts to clean checkout for repository.
+    def retrieve
+      raise RetrieverError.new("retriever is unavailable") unless available?
+      updated = false
+      explanation = ''
+      if exists?
+        @logger.operation(:updating) do
+          # a retriever may be able to determine that the repo directory is
+          # already pointing to the same commit as the revision. in that case
+          # we can return quickly.
+          if remote_differs?
+            # there is no point in updating and failing the size check when the
+            # directory on disk already exceeds size limit; fall back to a clean
+            # checkout in hopes that the latest revision corrects the issue.
+            if size_limit_exceeded?
+              explanation = 'switching to checkout due to existing directory exceeding size limimt'
+            else
+              # attempt update.
+              begin
+                do_update
+                updated = true
+              rescue ::RightScraper::Processes::Shell::LimitError
+                # update exceeded a limitation; requires user intervention
+                raise
+              rescue Exception => e
+                # retry with clean checkout after discarding repo dir.
+                explanation = 'switching to checkout after unsuccessful update'
+              end
+            end
+          else
+            # no retrieval needed but warn exactly why we didn't do full
+            # checkout to avoid being challenged about it.
+            repo_ref = @repository.tag
+            do_update_tag
+            full_head_ref = @repository.tag
+            abbreviated_head_ref = full_head_ref[0..6]
+            if repo_ref == full_head_ref || repo_ref == abbreviated_head_ref
+              detail = abbreviated_head_ref
+            else
+              detail = "#{repo_ref} = #{abbreviated_head_ref}"
+            end
+            message =
+              "Skipped updating local directory due to the HEAD commit SHA " +
+              "on local matching the remote repository reference (#{detail})."
+            @logger.note_warning(message)
+            return false
+          end
+        end
+      end
+
+      # Clean checkout only if not updated.
+      unless updated
+        @logger.operation(:checkout, explanation) do
+          # remove any full or partial directory before attempting a clean
+          # checkout in case repo_dir is in a bad state.
+          if exists?
+            ::FileUtils.remove_entry_secure(@repo_dir)
+          end
+          ::FileUtils.mkdir_p(@repo_dir)
+          begin
+            do_checkout
+          rescue Exception
+            # clean checkout failed; repo directory is in an undetermined
+            # state and must be deleted to prevent a future update attempt.
+            if exists?
+              ::FileUtils.remove_entry_secure(@repo_dir) rescue nil
+            end
+            raise
+          end
+        end
+      end
+      true
+    end
+
+    # Return true if a checkout exists.
+    #
+    # === Returns
+    # Boolean:: true if the checkout already exists (and thus
+    #           incremental updating can occur).
+    def exists?
+      false
+    end
+
+    # Determines if the remote SHA/tag/branch referenced by the repostory
+    # differs from what appears on disk, if possible. Not all retrievers will
+    # have this capability. If not, the retriever should default to returning
+    # true to indicate that the remote is changed.
+    #
+    # @return [TrueClass|FalseClass] true if changed
+    def remote_differs?
+      true
+    end
+
+    # Determines if total size of files in repo_dir has exceeded size limit.
+    #
+    # === Return
+    # @return [TrueClass|FalseClass] true if size limit exceeded
+    def size_limit_exceeded?
+      if @max_bytes
+        # note that Dir.glob ignores hidden directories (e.g. ".git") so the
+        # size total correctly excludes those hidden contents that are not to
+        # be uploaded after scrape. this may cause the on-disk directory size
+        # to far exceed the upload size.
+        globbie = ::File.join(@repo_dir, '**/*')
+        size = 0
+        ::Dir.glob(globbie) do |f|
+          size += ::File.stat(f).size rescue 0 if ::File.file?(f)
+          break if size > @max_bytes
+        end
+        size > @max_bytes
+      else
+        false
+      end
+    end
+
+    # Perform a de novo full checkout of the repository.  Subclasses
+    # must override this to do anything useful.
+    #
+    # @return [TrueClass] always true
+    def do_checkout
+      raise NotImplementedError
+    end
+
+    # Perform an incremental update of the checkout.  Subclasses that
+    # want to handle incremental updating need to override this.
+    #
+    # @return [TrueClass] always true
+    def do_update
+      raise NotImplementedError
+    end
+
+    # Updates the tag of the repository associated with this retriever to refer
+    # to the HEAD commit (SHA) on disk after retrieval.
+    #
+    # @return [TrueClass] always true
+    def do_update_tag
+      raise NotImplementedError
+    end
+
+  end
+end

data/lib/right_scraper/retrievers/download.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
+# Copyright: Copyright (c) 2010-2013 RightScale, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -21,68 +21,110 @@
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #++
 
+# ancestor
+require 'right_scraper/retrievers'
+
+require 'fileutils'
 require 'tempfile'
 require 'digest/sha1'
 require 'right_popen'
 require 'right_popen/safe_output_buffer'
 
-module RightScraper
-  module Retrievers
-    # A retriever for resources stored in archives on a web server
-    # somewhere.  Uses command line curl and command line tar.
-    class Download < Base
+module RightScraper::Retrievers
 
-      class DownloadError < Exception; end
+  # A retriever for resources stored in archives on a web server
+  # somewhere.  Uses command line curl and command line tar.
+  class Download < ::RightScraper::Retrievers::Base
 
-      @@available = false
+    class DownloadError < Exception; end
 
-      # Determines if downloader is available.
-      def available?
-        unless @@available
-          begin
-            # FIX: we might want to parse the result and require a minimum curl
-            # version.
-            cmd = "curl --version"
-            `#{cmd}`
-            if $?.success?
-              @@available = true
-            else
-              raise RetrieverError, "\"#{cmd}\" exited with #{$?.exitstatus}"
-            end
-          rescue
-            @logger.note_error($!, :available, "download retriever is unavailable")
+    @@available = false
+
+    # Determines if downloader is available.
+    def available?
+      unless @@available
+        begin
+          # FIX: we might want to parse the result and require a minimum curl
+          # version.
+          cmd = "curl --version"
+          `#{cmd}`
+          if $?.success?
+            @@available = true
+          else
+            raise RetrieverError, "\"#{cmd}\" exited with #{$?.exitstatus}"
           end
+        rescue
+          @logger.note_error($!, :available, "download retriever is unavailable")
         end
-        @@available
       end
+      @@available
+    end
+
+    # Directory used to download tarballs
+    def workdir
+      @workdir ||= ::File.join(::File.dirname(@repo_dir), 'download')
+    end
 
-      # Directory used to download tarballs
-      def workdir
-        @workdir ||= ::File.join(::File.dirname(@repo_dir), 'download')
+    # Download tarball and unpack it
+    def retrieve
+      raise RetrieverError.new("download retriever is unavailable") unless available?
+      ::FileUtils.remove_entry_secure @repo_dir if File.exists?(@repo_dir)
+      ::FileUtils.remove_entry_secure workdir if File.exists?(workdir)
+      ::FileUtils.mkdir_p @repo_dir
+      ::FileUtils.mkdir_p workdir
+      file = ::File.join(workdir, "package")
+
+      # TEAL FIX: we have to always-download the tarball before we can
+      # determine if contents have changed, but afterward we can compare the
+      # previous download against the latest downloaded and short-circuit the
+      # remaining flow for the no-difference case.
+      @logger.operation(:downloading) do
+        credential_command = if @repository.first_credential && @repository.second_credential
+          ['-u', "#{@repository.first_credential}:#{@repository.second_credential}"]
+        else
+          []
+        end
+        @output = ::RightScale::RightPopen::SafeOutputBuffer.new
+        @cmd = [
+          'curl',
+          '--silent', '--show-error', '--location', '--fail',
+          '--location-trusted', '-o', file, credential_command,
+          @repository.url
+        ].flatten
+        begin
+          ::RightScale::RightPopen.popen3_sync(
+            @cmd,
+            :target             => self,
+            :pid_handler        => :pid_download,
+            :timeout_handler    => :timeout_download,
+            :size_limit_handler => :size_limit_download,
+            :exit_handler       => :exit_download,
+            :stderr_handler     => :output_download,
+            :stdout_handler     => :output_download,
+            :inherit_io         => true,  # avoid killing any rails connection
+            :watch_directory    => workdir,
+            :size_limit_bytes   => @max_bytes,
+            :timeout_seconds    => @max_seconds)
+        rescue Exception => e
+          @logger.note_phase(:abort, :running_command, 'curl', e)
+          raise
+        end
       end
 
-      # Download tarball and unpack it
-      def retrieve
-        raise RetrieverError.new("download retriever is unavailable") unless available?
-        FileUtils.remove_entry_secure @repo_dir if File.exists?(@repo_dir)
-        FileUtils.remove_entry_secure workdir if File.exists?(workdir)
-        FileUtils.mkdir_p @repo_dir
-        FileUtils.mkdir_p workdir
-        file = File.join(workdir, "package")
-
-        @logger.operation(:downloading) do
-          credential_command = if @repository.first_credential && @repository.second_credential
-            ['-u', "#{@repository.first_credential}:#{@repository.second_credential}"]
-          else
-            []
-          end
+      note_tag(file)
+
+      @logger.operation(:unpacking) do
+        path = @repository.to_url.path
+        if path =~ /\.gz$/
+          extraction = "xzf"
+        elsif path =~ /\.bz2$/
+          extraction = "xjf"
+        else
+          extraction = "xf"
+        end
+        Dir.chdir(@repo_dir) do
           @output = ::RightScale::RightPopen::SafeOutputBuffer.new
-          @cmd = [
-            'curl',
-            '--silent', '--show-error', '--location', '--fail',
-            '--location-trusted', '-o', file, credential_command,
-            @repository.url
-          ].flatten
+          @cmd = ['tar', extraction, file]
           begin
             ::RightScale::RightPopen.popen3_sync(
               @cmd,
@@ -94,90 +136,56 @@ module RightScraper
               :stderr_handler     => :output_download,
               :stdout_handler     => :output_download,
               :inherit_io         => true,  # avoid killing any rails connection
-              :watch_directory    => workdir,
+              :watch_directory    => @repo_dir,
               :size_limit_bytes   => @max_bytes,
               :timeout_seconds    => @max_seconds)
           rescue Exception => e
-            @logger.note_phase(:abort, :running_command, 'curl', e)
+            @logger.note_phase(:abort, :running_command, @cmd.first, e)
             raise
           end
         end
-
-        note_tag(file)
-
-        @logger.operation(:unpacking) do
-          path = @repository.to_url.path
-          if path =~ /\.gz$/
-            extraction = "xzf"
-          elsif path =~ /\.bz2$/
-            extraction = "xjf"
-          else
-            extraction = "xf"
-          end
-          Dir.chdir(@repo_dir) do
-            @output = ::RightScale::RightPopen::SafeOutputBuffer.new
-            @cmd = ['tar', extraction, file]
-            begin
-              ::RightScale::RightPopen.popen3_sync(
-                @cmd,
-                :target             => self,
-                :pid_handler        => :pid_download,
-                :timeout_handler    => :timeout_download,
-                :size_limit_handler => :size_limit_download,
-                :exit_handler       => :exit_download,
-                :stderr_handler     => :output_download,
-                :stdout_handler     => :output_download,
-                :inherit_io         => true,  # avoid killing any rails connection
-                :watch_directory    => @repo_dir,
-                :size_limit_bytes   => @max_bytes,
-                :timeout_seconds    => @max_seconds)
-            rescue Exception => e
-              @logger.note_phase(:abort, :running_command, @cmd.first, e)
-              raise
-            end
-          end
-        end
       end
+      true
+    end
 
-      def pid_download(pid)
-        @logger.note_phase(:begin, :running_command, @cmd.first)
-        true
-      end
+    def pid_download(pid)
+      @logger.note_phase(:begin, :running_command, @cmd.first)
+      true
+    end
 
-      def output_download(data)
-        @output.safe_buffer_data(data)
-      end
+    def output_download(data)
+      @output.safe_buffer_data(data)
+    end
 
-      def timeout_download
-        raise DownloadError, "Downloader timed out"
-      end
+    def timeout_download
+      raise DownloadError, "Downloader timed out"
+    end
 
-      def size_limit_download
-        raise DownloadError, "Downloader exceeded size limit"
-      end
+    def size_limit_download
+      raise DownloadError, "Downloader exceeded size limit"
+    end
 
-      def exit_download(status)
-        unless status.success?
-          @output.safe_buffer_data("Exit code = #{status.exitstatus}")
-          raise DownloadError, "Downloader failed: #{@output.display_text}"
-        end
-        @logger.note_phase(:commit, :running_command, @cmd.first)
-        true
+    def exit_download(status)
+      unless status.success?
+        @output.safe_buffer_data("Exit code = #{status.exitstatus}")
+        raise DownloadError, "Downloader failed: #{@output.display_text}"
       end
+      @logger.note_phase(:commit, :running_command, @cmd.first)
+      true
+    end
 
 
-      # Amend @repository with the tag information from the downloaded
-      # file.
-      #
-      # === Parameters
-      # file(String):: file that was downloaded
-      def note_tag(file)
-        digest = Digest::SHA1.new
-        File.open(file) {|f| digest << f.read(4096) }
-        repo = @repository.clone
-        repo.tag = digest.hexdigest
-        @repository = repo
-      end
+    # Amend @repository with the tag information from the downloaded
+    # file.
+    #
+    # === Parameters
+    # file(String):: file that was downloaded
+    def note_tag(file)
+      digest = Digest::SHA1.new
+      File.open(file) {|f| digest << f.read(4096) }
+      repo = @repository.clone
+      repo.tag = digest.hexdigest
+      @repository = repo
     end
   end
 end