autoproj 2.4.0 → 2.5.0.pre1

data/lib/autoproj/environment.rb

@@ -53,7 +53,7 @@ def export_env_sh(subdir = nil, options = Hash.new)
             end
 
             if new_content != existing_content
-                File.open(filename, 'w') { |io| io.write new_content }
+                Ops.atomic_write(filename) { |io| io.write new_content }
                 true
             end
         end

data/lib/autoproj/exceptions.rb

@@ -1,3 +1,5 @@
+require 'autobuild/exceptions'
+
 module Autoproj
     class ConfigError < RuntimeError
         attr_accessor :file
@@ -84,6 +86,9 @@ class OutdatedWorkspace < InvalidWorkspace; end
     # Exception raised when initializing on a workspace that is not the current
     # one
     class MismatchingWorkspace < InvalidWorkspace; end
+
+    # Raised by 'which' when an executable cannot be found
+    class ExecutableNotFound < ArgumentError; end
 end
 
 

data/lib/autoproj/installation_manifest.rb

@@ -61,7 +61,7 @@ def load
 
         # Save the installation manifest
         def save(path = self.path)
-            File.open(path, 'w') do |io|
+            Ops.atomic_write(path) do |io|
                 marshalled_package_sets = each_package_set.map do |v|
                     Hash['package_set' => v.name,
                          'vcs' => v.vcs.to_hash,
@@ -79,7 +79,7 @@ def save(path = self.path)
                          'prefix' => v.prefix,
                          'dependencies' => v.dependencies]
                 end
-                YAML.dump(marshalled_package_sets + marshalled_packages, io)
+                io.write YAML.dump(marshalled_package_sets + marshalled_packages)
             end
         end
 

data/lib/autoproj/ops/atomic_write.rb

@@ -0,0 +1,36 @@
+module Autoproj
+    module Ops
+        # Shamelessly stolen from ActiveSupport
+        def self.atomic_write(file_name, temp_dir = Dir.tmpdir)
+            require 'tempfile' unless defined?(Tempfile)
+            require 'fileutils' unless defined?(FileUtils)
+    
+            temp_file = Tempfile.new(File.basename(file_name), temp_dir)
+            yield temp_file
+            temp_file.flush
+            begin temp_file.fsync
+            rescue NotImplementedError
+            end
+            temp_file.close
+    
+            begin
+                # Get original file permissions
+                old_stat = File.stat(file_name)
+            rescue Errno::ENOENT
+                # No old permissions, write a temp file to determine the defaults
+                check_name = File.join(
+                    File.dirname(file_name), ".permissions_check.#{Thread.current.object_id}.#{Process.pid}.#{rand(1000000)}")
+                File.open(check_name, "w") { }
+                old_stat = File.stat(check_name)
+                File.unlink(check_name)
+            end
+
+            # Overwrite original file with temp file
+            FileUtils.mv(temp_file.path, file_name)
+
+            # Set correct permissions on new file
+            File.chown(old_stat.uid, old_stat.gid, file_name)
+            File.chmod(old_stat.mode, file_name)
+        end
+    end
+end

data/lib/autoproj/ops/cached_env.rb

@@ -0,0 +1,36 @@
+require 'autobuild/environment'
+
+module Autoproj
+    module Ops
+        def self.cached_env_path(root_dir)
+            File.join(root_dir, '.autoproj', 'env.yml')
+        end
+
+        def self.load_cached_env(root_dir)
+            path = cached_env_path(root_dir)
+            if File.file?(path)
+                env = YAML.load(File.read(path))
+                Autobuild::Environment::ExportedEnvironment.new(
+                    env['set'], env['unset'], env['update'])
+            end
+        end
+
+        def self.save_cached_env(root_dir, env)
+            env = env.exported_environment
+            path = cached_env_path(root_dir)
+            existing =
+                begin
+                    YAML.load(File.read(path))
+                rescue Exception
+                end
+
+            env = Hash['set' => env.set, 'unset' => env.unset, 'update' => env.update]
+            if env != existing
+                Ops.atomic_write(path) do |io|
+                    io.write YAML.dump(env)
+                end
+                true
+            end
+        end
+    end
+end

data/lib/autoproj/ops/import.rb

@@ -181,7 +181,7 @@ def import_selected_packages(selection,
 
                 # This is used in the ensure block, initialize as early as
                 # possible
-                executor = Concurrent::FixedThreadPool.new(parallel, max_length: 0)
+                executor = Concurrent::FixedThreadPool.new(parallel)
                 manifest = ws.manifest
 
                 selected_packages = selection.each_source_package_name.map do |pkg_name|

data/lib/autoproj/ops/install.rb

@@ -13,7 +13,7 @@ module Ops
         class Install
             class UnexpectedBinstub < RuntimeError; end
 
-            # The directory in which to install autoproj
+            # The created workspace's root directory
             attr_reader :root_dir
             # Content of the Gemfile generated to install autoproj itself
             attr_accessor :gemfile
@@ -306,12 +306,12 @@ def install_autoproj(bundler)
                     exit 1
                 end
             ensure
-                self.class.rewrite_shims(shims_path, ruby_executable, autoproj_gemfile_path, gems_gem_home)
+                self.class.rewrite_shims(shims_path, ruby_executable, root_dir, autoproj_gemfile_path, gems_gem_home)
             end
 
             EXCLUDED_FROM_SHIMS = %w{rake thor}
 
-            def self.rewrite_shims(shim_path, ruby_executable, autoproj_gemfile_path, gems_gem_home)
+            def self.rewrite_shims(shim_path, ruby_executable, root_dir, autoproj_gemfile_path, gems_gem_home)
                 FileUtils.mkdir_p shim_path
                 File.open(File.join(shim_path, 'ruby'), 'w') do |io|
                     io.puts "#! /bin/sh"
@@ -337,7 +337,7 @@ def self.rewrite_shims(shim_path, ruby_executable, autoproj_gemfile_path, gems_g
                             io.puts shim_bundler(ruby_executable, autoproj_gemfile_path, gems_gem_home)
                         else
                             load_line = bin_script_lines.grep(/load Gem.bin_path/).first
-                            io.puts shim_script(ruby_executable, autoproj_gemfile_path, gems_gem_home, load_line)
+                            io.puts shim_script(ruby_executable, root_dir, autoproj_gemfile_path, gems_gem_home, load_line)
                         end
                     end
                     FileUtils.chmod 0755, bin_shim
@@ -361,7 +361,7 @@ def self.shim_bundler(ruby_executable, autoproj_gemfile_path, gems_gem_home)
 load Gem.bin_path('bundler', 'bundler')"
             end
             
-            def self.shim_script(ruby_executable, autoproj_gemfile_path, gems_gem_home, load_line)
+            def self.shim_script(ruby_executable, root_dir, autoproj_gemfile_path, gems_gem_home, load_line)
 "#! #{ruby_executable}
 
 if defined?(Bundler)
@@ -373,6 +373,7 @@ def self.shim_script(ruby_executable, autoproj_gemfile_path, gems_gem_home, load
 end
 
 ENV['BUNDLE_GEMFILE'] = '#{autoproj_gemfile_path}'
+ENV['AUTOPROJ_CURRENT_ROOT'] = '#{root_dir}'
 require 'rubygems'
 Gem.paths = Hash['GEM_HOME' => '#{gems_gem_home}', 'GEM_PATH' => '']
 require 'bundler/setup'
@@ -421,7 +422,7 @@ def save_gemfile
                     "config_path = File.join(__dir__, 'config.yml')",
                     "if File.file?(config_path)",
                     "    require 'yaml'",
-                    "    config = YAML.load(File.read(config_path))",
+                    "    config = YAML.load(File.read(config_path)) || Hash.new",
                     "    (config['plugins'] || Hash.new).each do |plugin_name, (version, options)|",
                     "        gem plugin_name, version, **options",
                     "    end",
@@ -490,6 +491,7 @@ def install
                 self.class.rewrite_shims(
                     File.join(dot_autoproj, 'bin'),
                     ruby_executable,
+                    root_dir,
                     autoproj_gemfile_path,
                     gems_gem_home)
                 env['PATH'].unshift File.join(dot_autoproj, 'bin')
@@ -504,10 +506,10 @@ def load_config
                 
                 config = Hash.new
                 if File.file?(v1_config_path)
-                    config.merge!(YAML.load(File.read(v1_config_path)))
+                    config.merge!(YAML.load(File.read(v1_config_path)) || Hash.new)
                 end
                 if File.file?(autoproj_config_path)
-                    config.merge!(YAML.load(File.read(autoproj_config_path)))
+                    config.merge!(YAML.load(File.read(autoproj_config_path)) || Hash.new)
                 end
 
                 ruby = RbConfig::CONFIG['RUBY_INSTALL_NAME']

data/lib/autoproj/ops/watch.rb

@@ -0,0 +1,37 @@
+module Autoproj
+    module Ops
+        def self.watch_marker_path(root_dir)
+            File.join(root_dir, '.autoproj', 'watch')
+        end
+
+        def self.watch_running?(root_dir)
+            io = File.open(watch_marker_path(root_dir))
+            !io.flock(File::LOCK_EX | File::LOCK_NB)
+        rescue Errno::ENOENT
+            false
+        ensure
+            io.close if io
+        end
+
+        class WatchAlreadyRunning < RuntimeError; end
+
+        def self.watch_create_marker(root_dir)
+            io = File.open(watch_marker_path(root_dir), 'a+')
+            if !io.flock(File::LOCK_EX | File::LOCK_NB)
+                raise WatchAlreadyRunning, "autoproj watch is already running as PID #{io.read.strip}"
+            end
+            io.truncate(0)
+            io.puts Process.pid
+            io.flush
+
+        rescue Exception
+            io.close if io
+            raise
+        end
+
+        def self.watch_cleanup_marker(io)
+            FileUtils.rm_f io.path
+            io.close
+        end
+    end
+end

data/lib/autoproj/ops/which.rb

@@ -0,0 +1,45 @@
+require 'pathname'
+require 'autoproj/exceptions'
+require 'autobuild/environment'
+
+module Autoproj
+    module Ops
+        # Find the given executable file in PATH
+        #
+        # If `cmd` is an absolute path, it will either return it or raise if
+        # `cmd` is not executable. Otherwise, looks for an executable named
+        # `cmd` in PATH and returns it, or raises if it cannot be found. The
+        # exception contains a more detailed reason for failure
+        # 
+        #
+        # @param [String] cmd
+        # @return [String] the resolved program
+        # @raise [ExecutableNotFound] if an executable file named `cmd` cannot
+        #   be found
+        def self.which(cmd, path_entries: nil)
+            path = Pathname.new(cmd)
+            if path.absolute?
+                if path.file? && path.executable?
+                    return cmd
+                elsif path.exist?
+                    raise ExecutableNotFound.new(cmd), "given command `#{cmd}` exists but is not an executable file"
+                else
+                    raise ExecutableNotFound.new(cmd), "given command `#{cmd}` does not exist, an executable file was expected"
+                end
+            else
+                if path_entries.respond_to?(:call)
+                    path_entries = path_entries.call
+                end
+                absolute = Autobuild::Environment.find_executable_in_path(cmd, path_entries)
+
+                if absolute
+                    return absolute
+                elsif file = Autobuild::Environment.find_in_path(cmd, path_entries)
+                    raise ExecutableNotFound.new(cmd), "`#{cmd}` resolves to #{file} which is not executable"
+                else
+                    raise ExecutableNotFound.new(cmd), "cannot resolve `#{cmd}` to an executable in the workspace"
+                end
+            end
+        end
+    end
+end

data/lib/autoproj/package_managers/bundler_manager.rb

@@ -82,7 +82,7 @@ def initialize_environment
                 FileUtils.mkdir_p prefix_gems
                 gemfile = File.join(prefix_gems, 'Gemfile')
                 if !File.exist?(gemfile)
-                    File.open(gemfile, 'w') do |io|
+                    Ops.atomic_write(gemfile) do |io|
                         io.puts "eval_gemfile \"#{File.join(ws.dot_autoproj_dir, 'Gemfile')}\""
                     end
                 end
@@ -314,7 +314,7 @@ def install(gems, filter_uptodate_packages: false, install_only: false)
                 # wrong to avoid leaving bundler in an inconsistent state
                 backup_files(backups)
                 if !File.file?("#{gemfile_path}.orig")
-                    File.open("#{gemfile_path}.orig", 'w') do |io|
+                    Ops.atomic_write("#{gemfile_path}.orig") do |io|
                         io.puts "eval_gemfile \"#{File.join(ws.dot_autoproj_dir, 'Gemfile')}\""
                     end
                 end
@@ -338,7 +338,7 @@ def install(gems, filter_uptodate_packages: false, install_only: false)
 
                 FileUtils.mkdir_p root_dir
                 if updated = (!File.exist?(gemfile_path) || File.read(gemfile_path) != gemfile_contents)
-                    File.open(gemfile_path, 'w') do |io|
+                    Ops.atomic_write(gemfile_path) do |io|
                         io.puts "ruby \"#{RUBY_VERSION}\" if respond_to?(:ruby)"
                         io.puts gemfile_contents
                     end

data/lib/autoproj/package_managers/pip_manager.rb

@@ -43,7 +43,8 @@ def install(pips, filter_uptodate_packages: false, install_only: false)
                         "#{pips.sort.join(", ")}"
 
                     cmdlines.each do |c|
-                        Autobuild::Subprocess.run 'autoproj', 'osdeps', *c
+                        Autobuild::Subprocess.run 'autoproj', 'osdeps', *c,
+                            env: ws.env.resolved_env
                     end
 
                     pips.each do |p|

data/lib/autoproj/test.rb

@@ -271,8 +271,7 @@ def ws_create_os_package_resolver
                 os_package_manager: 'os')
         end
 
-        def ws_create
-            dir = make_tmpdir
+        def ws_create(dir = make_tmpdir)
             require 'autoproj/ops/main_config_switcher'
             FileUtils.cp_r Ops::MainConfigSwitcher::MAIN_CONFIGURATION_TEMPLATE, File.join(dir, 'autoproj')
             FileUtils.mkdir_p File.join(dir, '.autoproj')

data/lib/autoproj/version.rb

@@ -1,3 +1,3 @@
 module Autoproj
-    VERSION = "2.4.0"
+    VERSION = "2.5.0.pre1"
 end

data/lib/autoproj/workspace.rb

@@ -1,6 +1,3 @@
-require 'autoproj/ops/import'
-require 'autoproj/ops/install'
-
 module Autoproj
     class Workspace < Ops::Loader
         attr_reader :root_dir
@@ -273,6 +270,7 @@ def setup
             rewrite_shims
             autodetect_operating_system
             config.validate_ruby_executable
+            Autobuild.programs['ruby'] = config.ruby_executable
             config.apply_autobuild_configuration
             load_autoprojrc
             load_main_initrb
@@ -285,7 +283,7 @@ def setup
 
             Autobuild.prefix = prefix_dir
             FileUtils.mkdir_p File.join(prefix_dir, '.autoproj')
-            File.open(File.join(prefix_dir, '.autoproj', 'config.yml'), 'w') do |io|
+            Ops.atomic_write(File.join(prefix_dir, '.autoproj', 'config.yml')) do |io|
                 io.puts "workspace: \"#{root_dir}\""
             end
 
@@ -308,7 +306,7 @@ def install_ruby_shims
             FileUtils.mkdir_p bindir
             env.add 'PATH', bindir
 
-            File.open(File.join(bindir, 'ruby'), 'w') do |io|
+            Ops.atomic_write(File.join(bindir, 'ruby')) do |io|
                 io.puts "#! /bin/sh"
                 io.puts "exec #{config.ruby_executable} \"$@\""
             end
@@ -318,7 +316,7 @@ def install_ruby_shims
                 # Look for the corresponding gem program
                 prg_name = "#{name}#{install_suffix}"
                 if File.file?(prg_path = File.join(RbConfig::CONFIG['bindir'], prg_name))
-                    File.open(File.join(bindir, name), 'w') do |io|
+                    Ops.atomic_write(File.join(bindir, name)) do |io|
                         io.puts "#! #{config.ruby_executable}"
                         io.puts "exec \"#{prg_path}\", *ARGV"
                     end
@@ -330,7 +328,7 @@ def install_ruby_shims
         def rewrite_shims
             gemfile  = File.join(dot_autoproj_dir, 'Gemfile')
             binstubs = File.join(dot_autoproj_dir, 'bin')
-            Ops::Install.rewrite_shims(binstubs, config.ruby_executable, gemfile, config.gems_gem_home)
+            Ops::Install.rewrite_shims(binstubs, config.ruby_executable, root_dir, gemfile, config.gems_gem_home)
         end
 
         def update_bundler
@@ -361,7 +359,7 @@ def update_autoproj(restart_on_update: true)
                 PackageManagers::BundlerManager.run_bundler_install(
                     self, gemfile, binstubs: binstubs)
             ensure
-                Ops::Install.rewrite_shims(binstubs, config.ruby_executable, gemfile, config.gems_gem_home)
+                rewrite_shims
             end
             if restart_on_update
                 new_autoproj_path = PackageManagers::BundlerManager.bundle_gem_path(
@@ -635,9 +633,19 @@ def full_env
 
         # Export the workspace's env.sh file
         def export_env_sh(package_names = nil, shell_helpers: true)
+            full_env = self.full_env
+            save_cached_env(full_env)
             full_env.export_env_sh(shell_helpers: shell_helpers)
         end
 
+        def save_cached_env(env = self.full_env)
+            Ops.save_cached_env(root_dir, env)
+        end
+
+        def load_cached_env
+            Ops.load_cached_env(root_dir)
+        end
+
         def pristine_os_packages(packages, options = Hash.new)
             os_package_installer.pristine(packages, options)
         end
@@ -702,9 +710,6 @@ def register_package(package, block = nil, package_set = manifest.main_package_s
             pkg
         end
 
-        class ExecutableNotFound < ArgumentError
-        end
-
         # Find the given executable file in PATH
         #
         # If `cmd` is an absolute path, it will either return it or raise if
@@ -717,27 +722,8 @@ class ExecutableNotFound < ArgumentError
         # @return [String] the resolved program
         # @raise [ExecutableNotFound] if an executable file named `cmd` cannot
         #   be found
-        def which(cmd)
-            path = Pathname.new(cmd)
-            if path.absolute?
-                if path.file? && path.executable?
-                    return cmd
-                elsif path.exist?
-                    raise ExecutableNotFound.new(cmd), "given command `#{cmd}` exists but is not an executable file"
-                else
-                    raise ExecutableNotFound.new(cmd), "given command `#{cmd}` does not exist, an executable file was expected"
-                end
-            else
-                env = full_env
-                absolute = env.find_executable_in_path(cmd)
-                if absolute
-                    return absolute
-                elsif file = env.find_in_path(cmd)
-                    raise ExecutableNotFound.new(cmd), "`#{cmd}` resolves to #{file} which is not executable"
-                else
-                    raise ExecutableNotFound.new(cmd), "cannot resolve `#{cmd}` to an executable in the workspace"
-                end
-            end
+        def which(cmd, path_entries: nil)
+            Ops.which(cmd, path_entries: -> { full_env.value('PATH') || Array.new })
         end
     end