autobuild 1.8.3 → 1.9.0.b1

data/lib/autobuild/import/hg.rb

@@ -49,16 +49,14 @@ module Autobuild
             end
         end
 
-        def update(package,only_local=false)
-            if only_local
-                Autobuild.warn "The importer #{self.class} does not support local updates, skipping #{self}"
+        def update(package, options = Hash.new)
+            if options[:only_local]
+                package.warn "%s: the Mercurial importer does not support local updates, skipping"
                 return
             end
             validate_importdir(package)
-            Dir.chdir(package.importdir) do
-                Subprocess.run(package, :import, Autobuild.tool('hg'), 'pull', repository)
-                Subprocess.run(package, :import, Autobuild.tool('hg'), 'update', branch)
-            end
+            package.run(:import, Autobuild.tool('hg'), 'pull', repository, retry: true, working_directory: package.importdir)
+            package.run(:import, Autobuild.tool('hg'), 'update', branch, working_directory: package.importdir)
         end
 
         def checkout(package)
@@ -67,8 +65,7 @@ module Autobuild
                 FileUtils.mkdir_p base_dir
             end
 
-            Subprocess.run(package, :import,
-                Autobuild.tool('hg'), 'clone', '-u', branch, repository, package.importdir)
+            package.run(:import, Autobuild.tool('hg'), 'clone', '-u', branch, repository, package.importdir, retry: true)
         end
     end
 

data/lib/autobuild/import/svn.rb

@@ -1,5 +1,6 @@
 require 'autobuild/subcommand'
 require 'autobuild/importer'
+require 'rexml/document'
 
 module Autobuild
     class SVN < Importer
@@ -11,67 +12,209 @@ module Autobuild
 	# This importer uses the 'svn' tool to perform the import. It defaults
 	# to 'svn' and can be configured by doing 
 	#   Autobuild.programs['svn'] = 'my_svn_tool'
-        def initialize(source, options = {})
-
-            @source = [*source].join("/")
+        def initialize(svnroot, options = {})
+            svnroot = [*svnroot].join("/")
             svnopts, common = Kernel.filter_options options,
                 :svnup => [], :svnco => [], :revision => nil,
-                :repository_id => "svn:#{@source}"
-            @program    = Autobuild.tool('svn')
-            @options_up = [*svnopts[:svnup]]
-            @options_co = [*svnopts[:svnco]]
-            if rev = svnopts[:revision]
-                @options_up << "--revision" << rev
-                @options_co << "--revision" << rev
-            end
+                :repository_id => "svn:#{svnroot}"
+            common[:repository_id] = svnopts.delete(:repository_id)
+            relocate(svnroot, svnopts)
             super(common.merge(repository_id: svnopts[:repository_id]))
         end
 
-        private
+        # @deprecated use {svnroot} instead
+        #
+        # @return [String]
+        def source; svnroot end
+
+        # Returns the SVN root
+        #
+        # @return [String]
+        attr_reader :svnroot
 
-        def update(package,only_local=false) # :nodoc:
+        attr_reader :revision
+
+        def relocate(root, options = Hash.new)
+            @svnroot = [*root].join("/")
+            @options_up = [*options[:svnup]]
+            @options_co = [*options[:svnco]]
+            @revision = options[:revision]
+            if revision
+                @options_co << '--revision' << revision
+                # We do not add it to @options_up as the behaviour depends on
+                # the parameters given to {update} and to the state of the
+                # working copy
+            end
+        end
+
+        # Returns the SVN revision of the package
+        #
+        # @param [Package] package
+        # @return [Integer]
+        # @raises ConfigException if 'svn info' did not return a Revision field
+        # @raises (see svn_info)
+        def svn_revision(package)
+            svninfo = svn_info(package)
+            revision = svninfo.grep(/^Revision: /).first
+            if !revision
+                raise ConfigException.new(package, 'import'), "cannot get SVN information for #{package.importdir}"
+            end
+            revision =~ /Revision: (\d+)/
+            Integer($1)
+        end
+
+        # Returns the URL of the remote SVN repository
+        #
+        # @param [Package] package
+        # @return [String]
+        # @raises ConfigException if 'svn info' did not return a URL field
+        # @raises (see svn_info)
+        def svn_url(package)
+            svninfo = svn_info(package)
+            url = svninfo.grep(/^URL: /).first
+            if !url
+                raise ConfigException.new(package, 'import'), "cannot get SVN information for #{package.importdir}"
+            end
+            url.chomp =~ /URL: (.+)/
+            $1
+        end
+
+        # Returns true if the SVN working copy at package.importdir has local
+        # modifications
+        #
+        # @param [Package] package the package we want to test against
+        # @param [Boolean] with_untracked_files if true, the presence of files
+        #   neither ignored nor under version control will count has local
+        #   modification.
+        # @return [Boolean]
+        def has_local_modifications?(package, with_untracked_files = false)
+            status = run_svn(package, 'status', '--xml')
+
+            not_modified = %w{external ignored none normal}
+            if !with_untracked_files
+                not_modified << "unversioned"
+            end
+
+            REXML::Document.new(status.join("")).
+                elements.enum_for(:each, '//wc-status').
+                any? do |status_item|
+                    !not_modified.include?(status_item.attributes['item'].to_s)
+                end
+        end
+
+        # Returns status information for package
+        #
+        # Given that subversion is not a distributed VCS, the only status
+        # returned are either {Status::UP_TO_DATE} or {Status::SIMPLE_UPDATE}.
+        # Moreover, if the status is local-only,
+        # {Package::Status#remote_commits} will not be filled (querying the log
+        # requires accessing the SVN server)
+        #
+        # @return [Package::Status]
+        def status(package, only_local = false)
+            status = Status.new
+            status.uncommitted_code = has_local_modifications?(package)
             if only_local
-                Autobuild.warn "The importer #{self.class} does not support local updates, skipping #{self}"
+                status.status = Status::UP_TO_DATE
+            else
+                log = run_svn(package, 'log', '-r', 'BASE:HEAD', '--xml', '.')
+                log = REXML::Document.new(log.join("\n"))
+                missing_revisions = log.elements.enum_for(:each, 'log/logentry').map do |l|
+                    rev = l.attributes['revision']
+                    date = l.elements['date'].first.to_s
+                    author = l.elements['author'].first.to_s
+                    msg = l.elements['msg'].first.to_s.split("\n").first
+                    "#{rev} #{DateTime.parse(date)} #{author} #{msg}"
+                end
+                status.remote_commits = missing_revisions[1..-1]
+                status.status =
+                    if missing_revisions.empty?
+                        Status::UP_TO_DATE
+                    else
+                        Status::SIMPLE_UPDATE
+                    end
+            end
+            status
+        end
+
+        # Helper method to run a SVN command on a package's working copy
+        def run_svn(package, *args, &block)
+            options = Hash.new
+            if args.last.kind_of?(Hash)
+                options = args.pop
+            end
+            options, other_options = Kernel.filter_options options,
+                working_directory: package.importdir, retry: true
+            options = options.merge(other_options)
+            package.run(:import, Autobuild.tool(:svn), *args, options, &block)
+        end
+
+        def validate_importdir(package)
+            # This upgrades the local SVN filesystem if needed and checks that
+            # it actually is a SVN repository in the first place
+            svn_info(package)
+        end
+
+        # Returns the result of the 'svn info' command
+        #
+        # It automatically runs svn upgrade if needed
+        #
+        # @param [Package] package
+        # @return [Array<String>] the lines returned by svn info, with the
+        #   trailing newline removed
+        # @raises [SubcommandFailed] if svn info failed
+        # @raises [ConfigException] if the working copy is not a subversion
+        #   working copy
+        def svn_info(package)
+            old_lang, ENV['LC_ALL'] = ENV['LC_ALL'], 'C'
+            begin
+                svninfo = run_svn(package, 'info')
+            rescue SubcommandFailed => e
+                if e.output.find { |l| l =~ /svn upgrade/ }
+                    # Try svn upgrade and info again
+                    run_svn(package, 'upgrade', retry: false)
+                    svninfo = run_svn(package, 'info')
+                else raise
+                end
+            end
+
+            if !svninfo.grep(/is not a working copy/).empty?
+                raise ConfigException.new(package, 'import'),
+                    "#{package.importdir} does not appear to be a Subversion working copy"
+            end
+            svninfo
+        ensure
+            ENV['LC_ALL'] = old_lang
+        end
+
+        def update(package, options = Hash.new) # :nodoc:
+            if options[:only_local]
+                package.warn "%s: the svn importer does not support local updates, skipping"
                 return
             end
-            Dir.chdir(package.importdir) do
-		old_lang, ENV['LC_ALL'] = ENV['LC_ALL'], 'C'
-                svninfo = []
-                begin
-                    Subprocess.run(package, :import, @program, 'info') do |line|
-                        svninfo << line
-                    end
-                rescue SubcommandFailed => e
-                    if svninfo.find { |l| l =~ /svn upgrade/ }
-                        # Try svn upgrade and info again
-                        Subprocess.run(package, :import, @program, 'upgrade')
-                        svninfo.clear
-                        Subprocess.run(package, :import, @program, 'info') do |line|
-                            svninfo << line
-                        end
-                    else raise
-                    end
+
+            url = svn_url(package)
+            if url != svnroot
+                raise ConfigException.new(package, 'import'), "current checkout found at #{package.importdir} is from #{url}, was expecting #{svnroot}"
+            end
+
+            options_up = @options_up.dup
+            if revision
+                if options[:reset] || svn_revision(package) < revision
+                    options_up << '--revision' << revision
+                elsif revision
+                    # Don't update if the current revision is greater-or-equal
+                    # than the target revision
+                    return
                 end
-		ENV['LC_ALL'] = old_lang
-		unless url = svninfo.grep(/^URL: /).first
-		    if svninfo.grep(/is not a working copy/).empty?
-			raise ConfigException.new(package, 'import'), "#{package.importdir} is not a Subversion working copy"
-		    else
-			raise ConfigException.new(package, 'import'), "Bug: cannot get SVN information for #{package.importdir}"
-		    end
-		end
-		url.chomp =~ /URL: (.+)/
-		source = $1
-		if source != @source
-		    raise ConfigException.new(package, 'import'), "current checkout found at #{package.importdir} is from #{source}, was expecting #{@source}"
-		end
-                Subprocess.run(package, :import, @program, 'up', "--non-interactive", *@options_up)
             end
+
+            run_svn(package, 'up', "--non-interactive", *options_up)
         end
 
         def checkout(package) # :nodoc:
-            options = [ @program, 'co', "--non-interactive" ] + @options_co + [ @source, package.importdir ]
-            Subprocess.run(package, :import, *options)
+            run_svn(package, 'co', "--non-interactive", *@options_co, svnroot, package.importdir,
+                    working_directory: nil)
         end
     end
 

data/lib/autobuild/importer.rb

@@ -53,8 +53,8 @@ class Importer
         # repository and not in the remote one (would be pushed by an update)
         attr_accessor :local_commits
 
-        def initialize
-            @status = -1
+        def initialize(status = -1)
+            @status = status
             @uncommitted_code = false
             @remote_commits = Array.new
             @local_commits  = Array.new
@@ -141,6 +141,18 @@ class Importer
         end
     end
 
+    def update_retry_count(original_error, retry_count)
+        if !original_error.respond_to?(:retry?) ||
+            !original_error.retry?
+            return
+        end
+
+        retry_count += 1
+        if retry_count <= self.retry_count
+            retry_count
+        end
+    end
+
     def perform_update(package,only_local=false)
         cur_patches    = currently_applied_patches(package)
         needed_patches = self.patches
@@ -176,10 +188,8 @@ class Importer
                 end
             end
 
-            retry_count += 1
-            if retry_count > self.retry_count
-                raise
-            end
+            retry_count = update_retry_count(original_error, retry_count)
+            raise if !retry_count
             package.message "update failed in #{package.importdir}, retrying (#{retry_count}/#{self.retry_count})"
             retry
         ensure
@@ -201,9 +211,9 @@ class Importer
                 checkout(package)
             rescue Interrupt
                 raise
-            rescue ::Exception
-                retry_count += 1
-                if retry_count > self.retry_count
+            rescue ::Exception => original_error
+                retry_count = update_retry_count(original_error, retry_count)
+                if !retry_count
                     raise
                 end
                 package.message "checkout of %s failed, deleting the source directory #{package.importdir} and retrying (#{retry_count}/#{self.retry_count})"
@@ -225,13 +235,43 @@ class Importer
         fallback(e, package, :import, package)
     end
 
-    # Performs the import of +package+
-    def import(package,only_local = false)
+    # Imports the given package
+    #
+    # The importer will checkout or update code in package.importdir. No update
+    # will be done if {update?} returns false.
+    #
+    # @raises ConfigException if package.importdir exists and is not a directory
+    #
+    # @option options [Boolean] :only_local (false) if true, will only perform
+    #   actions that do not require network access. Importers that do not
+    #   support this mode will simply do nothing
+    # @option options [Boolean] :reset (false) if true, the importer's
+    #   configuration is interpreted as a hard state in which it should put the
+    #   working copy. Otherwise, it tries to update the local repository with
+    #   the remote information. For instance, a git importer for which a commit
+    #   ID is given will, in this mode, reset the repository to the requested ID
+    #   (if that does not involve losing commits). Otherwise, it will only
+    #   ensure that the requested commit ID is present in the current HEAD.
+    def import(package, options = Hash.new)
+        # Backward compatibility
+        if !options.kind_of?(Hash)
+            options = !!options
+            Autoproj.warn "calling #import with a boolean as second argument is deprecated, switch to the named argument interface instead"
+            Autoproj.warn "   e.g. call import(package, only_local: #{options})"
+            Autoproj.warn "   #{caller(1).first}"
+            options = Hash[only_local: !!options]
+        end
+
+        options = Kernel.validate_options options,
+            only_local: false,
+            reset: false,
+            checkout_only: false
+
         importdir = package.importdir
         if File.directory?(importdir)
             package.isolate_errors(false) do
-                if package.update?
-                    perform_update(package,only_local)
+                if !options[:checkout_only] && package.update?
+                    perform_update(package, options)
                 else
                     if Autobuild.verbose
                         package.message "%s: not updating"
@@ -240,7 +280,7 @@ class Importer
                 end
             end
 
-        elsif File.exists?(importdir)
+        elsif File.exist?(importdir)
             raise ConfigException.new(package, 'import'), "#{importdir} exists but is not a directory"
         else
             perform_checkout(package)
@@ -275,23 +315,22 @@ class Importer
     end
 
     def call_patch(package, reverse, file, patch_level)
-        patch = Autobuild.tool('patch')
-        Dir.chdir(package.importdir) do
-            Subprocess.run(package, :patch, patch, "-p#{patch_level}", (reverse ? '-R' : nil), '--forward', :input => file)
-        end
+        package.run(:patch, Autobuild.tool('patch'),
+                    "-p#{patch_level}", (reverse ? '-R' : nil), '--forward', input: file,
+                    working_directory: package.importdir)
     end
 
     def apply(package, path, patch_level = 0);   call_patch(package, false, path, patch_level) end
     def unapply(package, path, patch_level = 0); call_patch(package, true, path, patch_level)   end
 
-    def parse_patch_list(patches_file)
+    def parse_patch_list(package, patches_file)
         File.readlines(patches_file).map do |line| 
             line = line.rstrip
             if line =~ /^(.*)\s+(\d+)$/
-                path = $1
+                path = File.expand_path($1, package.srcdir)
                 level = Integer($2)
             else
-                path = line
+                path = File.expand_path(line, package.srcdir)
                 level = 0
             end
             [path, level, File.read(path)]
@@ -300,13 +339,13 @@ class Importer
 
     def currently_applied_patches(package)
         patches_file = patchlist(package)
-        if File.exists?(patches_file)
-            return parse_patch_list(patches_file)
+        if File.exist?(patches_file)
+            return parse_patch_list(package, patches_file)
         end
 
         patches_file = File.join(package.importdir, "patches-autobuild-stamp")
-        if File.exists?(patches_file)
-            cur_patches = parse_patch_list(patches_file)
+        if File.exist?(patches_file)
+            cur_patches = parse_patch_list(package, patches_file)
             save_patch_state(package, cur_patches)
             FileUtils.rm_f patches_file
             return currently_applied_patches(package)
@@ -319,7 +358,9 @@ class Importer
         # Get the list of already applied patches
         cur_patches = currently_applied_patches(package)
 
-        if cur_patches.map(&:last) == patches.map(&:last)
+        cur_patches_state = cur_patches.map { |_, level, content| [level, content] }
+        patches_state     = patches.map { |_, level, content| [level, content] }
+        if cur_patches_state == patches_state
             return false
         end
 
@@ -342,9 +383,9 @@ class Importer
                 cur_patches.pop
             end
 
-            patches.to_a.each do |p, level, content|
-                apply(package, p, level)
-                cur_patches << [p, level, content]
+            patches.to_a.each do |new_patch, new_patch_level, content|
+                apply(package, new_patch, new_patch_level)
+                cur_patches << [new_patch, new_patch_level, content]
 	    end
         ensure
             save_patch_state(package, cur_patches)
@@ -357,14 +398,18 @@ class Importer
         patch_dir = patchdir(package)
         FileUtils.mkdir_p patch_dir
         cur_patches = cur_patches.each_with_index.map do |(path, level, content), idx|
-        path = File.join(patch_dir, idx.to_s)
-        File.open(path, 'w') do |patch_io|
-            patch_io.write content
-        end
-        [path, level]
+            path = File.join(patch_dir, idx.to_s)
+            File.open(path, 'w') do |patch_io|
+                patch_io.write content
+            end
+            [path, level]
         end
         File.open(patchlist(package), 'w') do |f|
-            f.write(cur_patches.map { |p, l| "#{p} #{l}" }.join("\n"))
+            patch_state = cur_patches.map do |path, level|
+                path = Pathname.new(path).relative_path_from( Pathname.new(package.srcdir) ).to_s
+                "#{path} #{level}"
+            end
+            f.write(patch_state.join("\n"))
         end
     end