autoproj 1.12.6 → 1.13.0.b1

data/lib/autoproj/configuration.rb

@@ -210,13 +210,108 @@ module Autoproj
             end
         end
 
+        # Returns true if packages and prefixes should be auto-generated, based
+        # on the SHA of the package names. This is meant to be used for build
+        # services that want to check that dependencies are properly set
+        #
+        # The default is false (disabled)
+        #
+        # @return [Boolean]
         def randomize_layout?
             get('randomize_layout', false)
         end
 
+        # Sets whether the layout should be randomized
+        #
+        # @return [Boolean]
+        # @see randomize_layout?
         def randomize_layout=(value)
             set('randomize_layout', value, true)
         end
+
+        DEFAULT_UTILITY_SETUP = Hash[
+            'doc' => true,
+            'test' => false]
+
+        # The configuration key that should be used to store the utility
+        # enable/disable information 
+        #
+        # @param [String] the utility name
+        # @return [String] the config key
+        def utility_key(utility)
+            "autoproj_#{utility}_utility"
+        end
+
+        # Returns whether a given utility is enabled for the package
+        #
+        # If there is no specific configuration for the package, uses the global
+        # default set with utility_enable_all or utility_disable_all. If none of
+        # these methods has been called, uses the default in
+        # {DEFAULT_UTILITY_SETUP}
+        #
+        # @param [String] utility the utility name (e.g. 'doc' or 'test')
+        # @param [String] package the package name
+        # @return [Boolean] true if the utility should be enabled for the
+        #   requested package and false otherwise
+        def utility_enabled_for?(utility, package)
+            utility_config = get(utility_key(utility), Hash.new)
+            if utility_config.has_key?(package)
+                utility_config[package]
+            else get("#{utility_key(utility)}_default", DEFAULT_UTILITY_SETUP[utility])
+            end
+        end
+
+        # Enables a utility for all packages
+        #
+        # This both sets the default value for all packages and resets all
+        # package-specific values set with {utility_enable_for} and
+        # {utility_disable_for}
+        #
+        # @param [String] utility the utility name (e.g. 'doc' or 'test')
+        # @return [void]
+        def utility_enable_all(utility)
+            reset(utility_key(utility))
+            set("#{utility_key(utility)}_default", true)
+        end
+
+        # Enables a utility for a specific package
+        #
+        # Note that if the default for this utility is to be enabled, this is
+        # essentially a no-op.
+        #
+        # @param [String] utility the utility name (e.g. 'doc' or 'test')
+        # @param [String] package the package name
+        # @return [void]
+        def utility_enable_for(utility, package)
+            utility_config = get(utility_key(utility), Hash.new)
+            set(utility_key(utility), utility_config.merge(package => true))
+        end
+
+        # Disables a utility for all packages
+        #
+        # This both sets the default value for all packages and resets all
+        # package-specific values set with {utility_enable_for} and
+        # {utility_disable_for}
+        #
+        # @param [String] utility the utility name (e.g. 'doc' or 'test')
+        # @return [void]
+        def utility_disable_all(utility)
+            reset(utility_key(utility))
+            set("#{utility_key(utility)}_default", false)
+        end
+
+        # Disables a utility for a specific package
+        #
+        # Note that if the default for this utility is to be disabled, this is
+        # essentially a no-op.
+        #
+        # @param [String] utility the utility name (e.g. 'doc' or 'test')
+        # @param [String] package the package name
+        # @return [void]
+        def utility_disable_for(utility, package)
+            utility_config = get(utility_key(utility), Hash.new)
+            set(utility_key(utility), utility_config.merge(package => false))
+        end
     end
 end
 

data/lib/autoproj/default.osdeps

@@ -195,5 +195,8 @@ pip:
   opensuse: python-pip
   fedora: python-pip
 
+sudo:
+  default: sudo
+
 # vim: expandtab
 

data/lib/autoproj/gitorious.rb

@@ -25,51 +25,35 @@ module Autoproj
             :http_url => "https://git.#{base_url}",
             :ssh_url  => "git@#{base_url}:",
             :fallback_to_http => true,
-            :default => 'http,ssh',
-            :disabled_methods => []
-            
-        disabled_methods = Array(options[:disabled_methods])
+            :default => 'git,ssh'
+
+        gitorious_long_doc = [
+            "How should I interact with #{base_url} (git, http or ssh)",
+            "If you give two values, comma-separated, the first one will be",
+            "used for pulling and the second one for pushing"]
 
         access_methods = Hash[
             'git' => 'git,ssh',
             'ssh' => 'ssh,ssh',
             'http' => 'http,http']
 
-        gitorious_long_doc = [
-            "How should I interact with #{base_url} (#{(access_methods.keys - disabled_methods).sort.join(", ")})",
-            "If you give two values, comma-separated, the first one will be",
-            "used for pulling and the second one for pushing"]
-
-        validator = lambda do |value|
-            if value =~ /,/
-                value.split(',').each do |method|
-                    if !access_methods.has_key?(method)
-                        raise Autoproj::InputError, "#{method} is not a known access method"
-                    elsif disabled_methods.include?(method)
-                        raise Autoproj::InputError, "#{method} is disabled on #{base_url}"
+        configuration_option name, 'string',
+            :default => options[:default],
+            :doc => gitorious_long_doc do |value|
+                if value =~ /,/
+                    value.split(',').each do |method|
+                        if !access_methods.has_key?(method)
+                            raise Autoproj::InputError, "#{method} is not a known access method"
+                        end
                     end
+                elsif !access_methods.has_key?(value)
+                    raise Autoproj::InputError, "#{value} is not a known access method"
                 end
-            elsif !access_methods.has_key?(value)
-                raise Autoproj::InputError, "#{value} is not a known access method"
-            elsif disabled_methods.include?(value)
-                raise Autoproj::InputError, "#{method} is disabled on #{base_url}"
-            end
-
-            value
-        end
 
-        configuration_option name, 'string',
-            :default => options[:default],
-            :doc => gitorious_long_doc, &validator
+                value
+            end
 
-        access_mode = Autoproj.config.get(name)
-        begin
-            validator[access_mode]
-        rescue Autoproj::InputError => e
-            Autoproj.warn e.message
-            Autoproj.config.reset(name)
-            access_mode = Autoproj.config.get(name)
-        end
+        access_mode = Autoproj.user_config(name)
         access_mode = access_methods[access_mode] || access_mode
         pull, push = access_mode.split(',')
         [[pull, "_ROOT"], [push, "_PUSH_ROOT"]].each do |method, var_suffix|
@@ -89,11 +73,15 @@ module Autoproj
             end
             pull_base_url = Autoproj.user_config("#{name}_ROOT")
             push_base_url = Autoproj.user_config("#{name}_PUSH_ROOT")
-            Hash[:type => 'git', :url => "#{pull_base_url}#{url}", :push_to => "#{push_base_url}#{url}", :retry_count => 10].merge(vcs_options)
+            Hash[type: 'git',
+                 url: "#{pull_base_url}#{url}",
+                 push_to: "#{push_base_url}#{url}",
+                 retry_count: 10,
+                 repository_id: "#{name.downcase}:#{url}"].merge(vcs_options)
         end
     end
 end
 
-Autoproj.gitorious_server_configuration('GITORIOUS', 'gitorious.org', :default => 'http,ssh', :disabled_methods => 'git')
+Autoproj.gitorious_server_configuration('GITORIOUS', 'gitorious.org')
 Autoproj.gitorious_server_configuration('GITHUB', 'github.com', :http_url => 'https://github.com', :default => 'http,ssh')
 

data/lib/autoproj/manifest.rb

@@ -40,7 +40,8 @@ module Autoproj
         # @return [Array<PackageSet>]
         attr_writer :package_sets
 
-        # Returns true if +pkg_name+ has been explicitely selected
+        # Returns true if +pkg_name+ has been explicitely selected, either by
+        # the command line or through the layout
         def explicitly_selected_package?(pkg_name)
             explicit_selection && explicit_selection.include?(pkg_name)
         end
@@ -65,8 +66,14 @@ module Autoproj
 
             @file = file
             @data = data
-            @ignored_packages |= (data['ignored_packages'] || Set.new)
-            data['exclude_packages'] ||= Set.new
+            @ignored_packages |= (data['ignored_packages'] || Set.new).to_set
+            @manifest_exclusions |= (data['exclude_packages'] || Set.new).to_set
+
+            @normalized_layout = Hash.new
+            compute_normalized_layout(
+                normalized_layout,
+                '/',
+                data['layout'] || Hash.new)
 
             if data['constants']
                 @constant_definitions = Autoproj.resolve_constant_definitions(data['constants'])
@@ -128,6 +135,7 @@ module Autoproj
             @ignored_os_dependencies = Set.new
             @reused_installations = Array.new
             @ignored_packages = Set.new
+            @manifest_exclusions = Set.new
 
             @constant_definitions = Hash.new
             if Autoproj.has_config_key?('manifest_source')
@@ -139,7 +147,7 @@ module Autoproj
         # Call this method to ignore a specific package. It must not be used in
         # init.rb, as the manifest is not yet loaded then
         def ignore_package(package_name)
-            @ignored_packages << package_name
+            @ignored_packages << package_name.to_str
         end
 
         # True if the given package should not be built, with the packages that
@@ -185,7 +193,7 @@ module Autoproj
         # The set of package names that are listed in the excluded_packages
         # section of the manifest
         def manifest_exclusions
-            data['exclude_packages']
+            @manifest_exclusions
         end
 
         # A package_name => reason map of the exclusions added with #add_exclusion.
@@ -198,6 +206,17 @@ module Autoproj
             automatic_exclusions[package_name] = reason
         end
 
+        # Tests whether the given package is excluded in the manifest
+        def excluded_in_manifest?(package_name)
+            manifest_exclusions.any? do |matcher|
+                if (pkg_set = metapackages[matcher]) && pkg_set.include?(package_name)
+                    true
+                else
+                    Regexp.new(matcher) === package_name
+                end
+            end
+        end
+
         # If +package_name+ is excluded from the build, returns a string that
         # tells why. Otherwise, returns nil
         #
@@ -205,7 +224,7 @@ module Autoproj
         # exclude_packages section of the manifest, or because they are
         # disabled on this particular operating system.
         def exclusion_reason(package_name)
-            if manifest_exclusions.any? { |l| Regexp.new(l) =~ package_name }
+            if excluded_in_manifest?(package_name)
                 "#{package_name} is listed in the exclude_packages section of the manifest"
             else
                 automatic_exclusions[package_name]
@@ -218,7 +237,11 @@ module Autoproj
         # This is useful to avoid building packages that are of no use for the
         # user.
         def excluded?(package_name)
-            if manifest_exclusions.any? { |l| Regexp.new(l) =~ package_name }
+            package_name = package_name.to_str
+
+            if normalized_layout.has_key?(package_name)
+                false
+            elsif excluded_in_manifest?(package_name)
                 true
             elsif automatic_exclusions.any? { |pkg_name, | pkg_name == package_name }
                 true
@@ -375,6 +398,12 @@ module Autoproj
             packages[name]
         end
 
+        def find_autobuild_package(name)
+            if pkg = packages[name]
+                pkg.autobuild
+            end
+        end
+
         def package(name)
             packages[name]
         end
@@ -500,6 +529,10 @@ module Autoproj
             set
         end
 
+        def main_package_set
+            each_package_set.find(&:main?)
+        end
+
         # Exception raised when a caller requires to use an excluded package
         class ExcludedPackage < ConfigError
             attr_reader :name
@@ -813,7 +846,11 @@ module Autoproj
             end
         end
 
-        def normalized_layout(result = Hash.new, layout_level = '/', layout_data = (data['layout'] || Hash.new))
+        # A mapping from names to layout placement, as found in the layout
+        # section of the manifest
+        attr_reader :normalized_layout
+
+        def compute_normalized_layout(result, layout_level, layout_data)
             layout_data.each do |value|
                 if value.kind_of?(Hash)
                     subname, subdef = value.find { true }
@@ -877,7 +914,7 @@ module Autoproj
             pkg.autobuild.description = manifest
             package_manifests[package.name] = manifest
 
-            manifest.each_dependency do |name, is_optional|
+            manifest.each_dependency(pkg.modes) do |name, is_optional|
                 begin
                     if is_optional
                         package.optional_dependency name

data/lib/autoproj/ops/cache.rb

@@ -37,7 +37,7 @@ module Autoproj
                     FileUtils.mkdir_p File.dirname(pkg.importdir)
                     Autobuild::Subprocess.run("autoproj-cache", "import", Autobuild.tool(:git), "--git-dir", pkg.importdir, 'init', "--bare")
                 end
-                pkg.importer.update_remotes_configuration(pkg, 'autoproj-cache')
+                pkg.importer.update_remotes_configuration(pkg)
 
                 with_retry(10) do
                     Autobuild::Subprocess.run('autoproj-cache', :import, Autobuild.tool('git'), '--git-dir', pkg.importdir, 'remote', 'update', 'autobuild')

data/lib/autoproj/ops/configuration.rb

@@ -96,7 +96,7 @@ module Autoproj
                     fake_package.update = false
                 end
             end
-            fake_package.import(only_local)
+            fake_package.import(only_local: only_local)
 
         rescue Autobuild::ConfigException => e
             raise ConfigError.new, "cannot import #{name}: #{e.message}", e.backtrace
@@ -177,9 +177,11 @@ module Autoproj
             pkg_set
         end
 
-        def queue_auto_imports_if_needed(queue, pkg_set)
+        def queue_auto_imports_if_needed(queue, pkg_set, root_set)
             if pkg_set.auto_imports?
                 pkg_set.each_raw_imported_set do |import_vcs, import_options|
+                    repository_id = repository_id_of(import_vcs)
+                    import_vcs = root_set.overrides_for("pkg_set:#{repository_id}", import_vcs)
                     queue << [import_vcs, import_options, pkg_set]
                 end
             end
@@ -191,10 +193,7 @@ module Autoproj
                 return "local:#{vcs.url}"
             end
 
-            name = PackageSet.name_of(manifest, vcs)
-            raw_local_dir = PackageSet.raw_local_dir_of(vcs)
-            fake_package = Tools.create_autobuild_package(vcs, name, raw_local_dir)
-            fake_package.importer.repository_id
+            vcs.create_autobuild_importer.repository_id
         end
 
         # Load the package set information
@@ -210,7 +209,7 @@ module Autoproj
             by_repository_id = Hash.new
             by_name = Hash.new
 
-            queue = queue_auto_imports_if_needed(Array.new, root_pkg_set)
+            queue = queue_auto_imports_if_needed(Array.new, root_pkg_set, root_pkg_set)
             while !queue.empty?
                 vcs, options, imported_from = queue.shift
                 repository_id = repository_id_of(vcs)
@@ -258,7 +257,7 @@ module Autoproj
                 by_name[pkg_set.name] = [pkg_set, vcs, options, imported_from]
 
                 # Finally, queue the imports
-                queue_auto_imports_if_needed(queue, pkg_set)
+                queue_auto_imports_if_needed(queue, pkg_set, root_pkg_set)
             end
 
             cleanup_remotes_dir(package_sets)

data/lib/autoproj/ops/snapshot.rb

@@ -0,0 +1,258 @@
+module Autoproj
+    module Ops
+    class Snapshot
+        # Update version control information with new choices
+        #
+        # The two parameters are formatted as expected in the version_control
+        # and overrides fields in source.yml / overrides.yml, that is (in YAML)
+        #
+        #   - package_name:
+        #     version: '10'
+        #     control: '20'
+        #     info: '30'
+        #
+        # The two parameters are expected to only use full package names, and
+        # not regular expressions
+        #
+        # @param [Array<String=>Hash>] overrides the information that should augment
+        #   the current state
+        # @param [Array<String=>Hash>] state the current state
+        # @param [Hash] the updated information
+        def self.merge_packets( overrides, state )
+            result = overrides.dup
+            overriden = overrides.map { |entry| entry.keys.first }.to_set
+            state.each do |pkg|
+                name, _ = pkg.first
+                if !overriden.include?(name)
+                    result << pkg
+                end
+            end
+            result
+        end
+
+        def sort_versions(versions)
+            pkg_sets, pkgs = versions.partition { |n, _| n =~ /^pkg_set:/ }
+            pkg_sets.sort_by { |n, _| n.keys.first } +
+                pkgs.sort_by { |n, _| n.keys.first }
+        end
+
+        def save_versions( versions, versions_file, options = Hash.new )
+            options = Kernel.validate_options options,
+                replace: false
+
+            existing_versions = Array.new
+            if !options[:replace] && File.exists?(versions_file)
+                existing_versions = YAML.load( File.read( versions_file ) ) ||
+                    Array.new
+            end
+
+            # create direcotry for versions file first
+            FileUtils.mkdir_p(File.dirname( versions_file ))
+
+            # augment the versions file with the updated versions
+            Snapshot.merge_packets( versions, existing_versions )
+
+            versions = sort_versions(versions)
+
+            # write the yaml file
+            File.open(versions_file, 'w') do |io|
+                io.write YAML.dump(versions)
+            end
+        end
+
+        def self.snapshot( packages, target_dir )
+            # todo
+        end
+
+        attr_reader :manifest
+
+        # Control what happens if a package fails to be snapshotted
+        #
+        # If true, the failure to snapshot a package should lead to a warning.
+        # Otherwise (the default), it leads to an error.
+        #
+        # @return [Boolean]
+        # @see initialize error_or_warn
+        def keep_going?; !!@keep_going end
+
+        def initialize(manifest, options = Hash.new)
+            @manifest = manifest
+            options = Kernel.validate_options options,
+                keep_going: false
+            @keep_going = options[:keep_going]
+        end
+
+        def snapshot_package_sets(target_dir = nil)
+            result = Array.new
+            manifest.each_package_set do |pkg_set|
+                next if pkg_set.local?
+
+                if vcs_info = pkg_set.snapshot(target_dir)
+                    result << Hash["pkg_set:#{pkg_set.repository_id}", vcs_info]
+                else
+                    error_or_warn(pkg_set, "cannot snapshot #{package_name}: importer snapshot failed")
+                end
+            end
+            result
+        end
+
+        def error_or_warn(package, error_msg)
+            if keep_going?
+                Autoproj.warn error_msg
+            else
+                raise Autobuild::PackageException.new(package, 'snapshot'), error_msg
+            end
+        end
+
+        def snapshot_packages(packages, target_dir = nil)
+            result = Array.new
+            packages.each do |package_name|
+                package  = manifest.packages[package_name]
+                if !package
+                    raise ArgumentError, "#{package_name} is not a known package"
+                end
+                importer = package.autobuild.importer
+                if !importer
+                    error_or_warn(package, "cannot snapshot #{package_name} as it has no importer")
+                    next
+                elsif !importer.respond_to?(:snapshot)
+                    error_or_warn(package, "cannot snapshot #{package_name} as the #{importer.class} importer does not support it")
+                    next
+                end
+
+                vcs_info = importer.snapshot(package.autobuild, target_dir)
+                if vcs_info
+                    result << Hash[package_name, vcs_info]
+                else
+                    error_or_warn(package, "cannot snapshot #{package_name}: importer snapshot failed")
+                end
+            end
+            result
+        end
+
+        # Returns the list of existing version tags
+        def tags(package)
+            importer = package.importer
+            all_tags = importer.run_git_bare(package, 'tag')
+            all_tags.find_all do |tag_name|
+            end
+        end
+
+        # Returns a package that is used to store this installs import history
+        #
+        # Its importer is guaranteed to be a git importer
+        #
+        # @return [Autobuild::Package] a package whose importer is
+        #   {Autobuild::Git}
+        def import_state_log_package
+            manifest.main_package_set.create_autobuild_package
+        end
+
+        def import_state_log_ref
+            "refs/autoproj"
+        end
+
+        DEFAULT_VERSIONS_FILE_BASENAME = "50-versions.yml"
+
+        def import_state_log_file
+            File.join(OVERRIDES_DIR, DEFAULT_VERSIONS_FILE_BASENAME)
+        end
+
+        def current_import_state
+            main = import_state_log_package
+            # Try to resolve the log ref, and extract the version file from it
+            begin
+                yaml = main.importer.show(main, import_state_log_ref, import_state_log_file)
+                YAML.load(yaml) || Array.new
+            rescue Autobuild::PackageException
+                Array.new
+            end
+        end
+
+        def update_package_import_state(name, packages)
+            current_versions = current_import_state
+            if current_versions.empty?
+                # Do a full snapshot this time only
+                Autoproj.message "  building initial autoproj import log, this may take a while"
+                packages = manifest.all_selected_packages.
+                    find_all { |pkg| File.directory?(manifest.find_package(pkg).autobuild.srcdir) }
+            end
+            versions  = snapshot_package_sets
+            versions += snapshot_packages(packages)
+            versions = Snapshot.merge_packets(versions, current_versions)
+            save_import_state(name, versions)
+        end
+
+        def save_import_state(name, versions)
+            versions = sort_versions(versions)
+            
+            main = import_state_log_package
+            git_dir = main.importer.git_dir(main, false)
+            # Ensure that our ref is being logged
+            FileUtils.touch File.join(git_dir, 'logs', *import_state_log_ref.split("/"))
+            # Create the commit with the versions info
+            commit_id = Snapshot.create_commit(main, import_state_log_file, name) do |io|
+                YAML.dump(versions, io)
+            end
+            # And save it in our reflog
+            main.importer.run_git_bare(main, "update-ref", '-m', name, import_state_log_ref, commit_id)
+        end
+
+        # Create a git commit in which a file contains provided content
+        #
+        # The target git repository's current index and history is left
+        # unmodified. The only modification is the creation of a new dangling
+        # commit.
+        #
+        # It creates a temporary file and gives it to the block so that the file
+        # gets filled with the new content
+        #
+        # @yieldparam [Tempfile] io a temporary file 
+        # @param [Autobuild::Package] a package object whose importer is a git
+        #   importer. The git commit is created in this repository
+        # @param [String] path the file to be created or updated, relative to
+        #   the root of the git repository
+        # @param [String] the commit message
+        # @return [String] the commit ID
+        def self.create_commit(pkg, path, message)
+            importer = pkg.importer
+            object_id = Tempfile.open 'autoproj-versions' do |io|
+                yield(io)
+                io.flush
+                importer.run_git_bare(
+                    pkg, 'hash-object', '-w',
+                    '--path', path, io.path).first
+            end
+
+            cacheinfo = ["100644", object_id, path]
+            if Autobuild::Git.at_least_version(2, 1)
+                cacheinfo = cacheinfo.join(",")
+            end
+
+            # Create the tree using a temporary index in order to not mess with
+            # the user's index state. read-tree initializes the new index and
+            # then we add the overrides file with update-index / write-tree
+            our_index = File.join(importer.git_dir(pkg, false), 'index.autoproj')
+            FileUtils.rm_f our_index
+            begin
+                ENV['GIT_INDEX_FILE'] = our_index
+                importer.run_git_bare(pkg, 'read-tree', 'HEAD')
+                # And add the new file
+                importer.run_git_bare(
+                    pkg, 'update-index',
+                    '--add', '--cacheinfo', *cacheinfo)
+                tree_id = importer.run_git_bare(pkg, 'write-tree').first
+            ensure
+                ENV.delete('GIT_INDEX_FILE')
+                FileUtils.rm_f our_index
+            end
+
+            head_id = importer.rev_parse(pkg, 'HEAD')
+
+            importer.run_git_bare(
+                pkg, 'commit-tree',
+                tree_id, '-p', head_id, input_streams: [message]).first
+        end
+    end
+    end
+end