autoproj 2.0.3 → 2.1.0.rc1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +196 -10
  3. data/autoproj.gemspec +8 -8
  4. data/bin/alocate +12 -1
  5. data/bin/alog +8 -0
  6. data/bin/amake +12 -1
  7. data/bin/aup +12 -1
  8. data/bin/autoproj_bootstrap +15 -6
  9. data/bin/autoproj_install +15 -6
  10. data/lib/autoproj/cli/base.rb +15 -0
  11. data/lib/autoproj/cli/bootstrap.rb +3 -0
  12. data/lib/autoproj/cli/build.rb +6 -0
  13. data/lib/autoproj/cli/clean.rb +13 -5
  14. data/lib/autoproj/cli/doc.rb +8 -0
  15. data/lib/autoproj/cli/envsh.rb +3 -5
  16. data/lib/autoproj/cli/inspection_tool.rb +0 -16
  17. data/lib/autoproj/cli/main.rb +71 -52
  18. data/lib/autoproj/cli/main_test.rb +4 -0
  19. data/lib/autoproj/cli/osdeps.rb +5 -6
  20. data/lib/autoproj/cli/show.rb +1 -1
  21. data/lib/autoproj/cli/status.rb +77 -19
  22. data/lib/autoproj/cli/update.rb +16 -26
  23. data/lib/autoproj/configuration.rb +5 -0
  24. data/lib/autoproj/environment.rb +11 -1
  25. data/lib/autoproj/local_package_set.rb +2 -0
  26. data/lib/autoproj/manifest.rb +16 -0
  27. data/lib/autoproj/ops/cache.rb +3 -0
  28. data/lib/autoproj/ops/configuration.rb +3 -2
  29. data/lib/autoproj/ops/install.rb +13 -4
  30. data/lib/autoproj/ops/main_config_switcher.rb +3 -3
  31. data/lib/autoproj/ops/snapshot.rb +4 -7
  32. data/lib/autoproj/package_definition.rb +10 -0
  33. data/lib/autoproj/package_managers/bundler_manager.rb +1 -0
  34. data/lib/autoproj/package_managers/pip_manager.rb +1 -0
  35. data/lib/autoproj/package_set.rb +39 -6
  36. data/lib/autoproj/test.rb +8 -0
  37. data/lib/autoproj/version.rb +1 -1
  38. data/lib/autoproj/workspace.rb +13 -1
  39. data/samples/autoproj/README.md +60 -0
  40. data/samples/autoproj/init.rb +17 -18
  41. data/samples/autoproj/manifest +18 -34
  42. data/samples/autoproj/overrides.d/.gitattributes +1 -0
  43. data/samples/manifest.xml +2 -7
  44. metadata +64 -36
  45. data/samples/autoproj/README.txt +0 -52
  46. data/samples/autoproj/overrides.rb +0 -4
  47. data/samples/autoproj/overrides.yml +0 -18
  48. data/samples/osdeps.yml +0 -65
@@ -12,7 +12,11 @@ def validate_options(selection, options)
12
12
  options[:from] = Autoproj::InstallationManifest.from_workspace_root(from)
13
13
  end
14
14
 
15
- if options[:aup] && !options[:all] && selection.empty?
15
+ if options[:no_deps_shortcut]
16
+ options[:deps] = false
17
+ end
18
+
19
+ if options[:aup] && !options[:config] && !options[:all] && selection.empty?
16
20
  if Dir.pwd == ws.root_dir
17
21
  options[:all] = true
18
22
  else
@@ -55,6 +59,7 @@ def validate_options(selection, options)
55
59
  (has_explicit_selection && !selection.empty?) ||
56
60
  (!has_explicit_selection && !options[:config] && !options[:autoproj])
57
61
 
62
+ options[:bundler] = update_autoproj
58
63
  options[:autoproj] = update_autoproj
59
64
  options[:config] = update_config
60
65
  options[:packages] = update_packages
@@ -63,10 +68,11 @@ def validate_options(selection, options)
63
68
 
64
69
  def run(selected_packages, options)
65
70
  ws.manifest.accept_unavailable_osdeps = !options[:osdeps]
66
-
67
71
  ws.setup
68
72
  ws.autodetect_operating_system(force: true)
69
-
73
+ if options[:bundler]
74
+ ws.update_bundler
75
+ end
70
76
  if options[:autoproj]
71
77
  ws.update_autoproj
72
78
  end
@@ -80,15 +86,18 @@ def run(selected_packages, options)
80
86
  keep_going: options[:keep_going],
81
87
  retry_count: options[:retry_count])
82
88
  rescue ImportFailed => configuration_import_failure
83
- if !options[:keep_going] || !options[:packages]
89
+ if !options[:keep_going]
84
90
  raise
85
91
  end
86
92
  ensure
87
93
  ws.config.save
88
94
  end
89
95
 
90
- if !options[:packages]
91
- return [], [], []
96
+ if options[:packages]
97
+ command_line_selection, selected_packages =
98
+ finish_loading_configuration(selected_packages)
99
+ else
100
+ command_line_selection, selected_packages = [], PackageSelection.new
92
101
  end
93
102
 
94
103
  osdeps_options = normalize_osdeps_options(
@@ -97,8 +106,6 @@ def run(selected_packages, options)
97
106
  osdeps: options[:osdeps],
98
107
  osdeps_filter_uptodate: options[:osdeps_filter_uptodate])
99
108
 
100
- command_line_selection, selected_packages =
101
- finish_loading_configuration(selected_packages)
102
109
  source_packages, osdep_packages, import_failure =
103
110
  update_packages(
104
111
  selected_packages,
@@ -119,8 +126,7 @@ def run(selected_packages, options)
119
126
  ws.install_os_packages(osdep_packages, **osdeps_options)
120
127
  end
121
128
 
122
- ws.export_env_sh(source_packages)
123
- Autoproj.message " updated #{ws.root_dir}/#{Autoproj::ENV_FILENAME}", :green
129
+ export_env_sh
124
130
 
125
131
  if import_failure && configuration_import_failure
126
132
  raise ImportFailed.new(configuration_import_failure.original_errors + import_failure.original_errors)
@@ -195,22 +201,6 @@ def update_packages(selected_packages,
195
201
  return import_failure.source_packages, import_failure.osdep_packages, import_failure
196
202
  end
197
203
 
198
- def load_all_available_package_manifests
199
- # Load the manifest for packages that are already present on the
200
- # file system
201
- ws.manifest.each_autobuild_package do |pkg|
202
- if pkg.checked_out?
203
- begin
204
- ws.manifest.load_package_manifest(pkg.name)
205
- rescue Interrupt
206
- raise
207
- rescue Exception => e
208
- Autoproj.warn "cannot load package manifest for #{pkg.name}: #{e.message}"
209
- end
210
- end
211
- end
212
- end
213
-
214
204
  def setup_update_from(other_root)
215
205
  manifest.each_autobuild_package do |pkg|
216
206
  if pkg.importer.respond_to?(:pick_from_autoproj_root)
@@ -467,5 +467,10 @@ def merge(conf)
467
467
  def prefer_indep_over_os_packages?
468
468
  get('prefer_indep_over_os_packages', false)
469
469
  end
470
+
471
+ # The configuration as a key => value map
472
+ def to_hash
473
+ @config.dup
474
+ end
470
475
  end
471
476
  end
@@ -35,7 +35,12 @@ def export_env_sh(subdir = nil, options = Hash.new)
35
35
  source_after(File.join(shell_dir, "autoproj_sh"))
36
36
  end
37
37
 
38
- File.open(filename, "w") do |io|
38
+ existing_content =
39
+ begin File.read(filename)
40
+ rescue SystemCallError
41
+ end
42
+
43
+ StringIO.open(new_content = String.new, 'w') do |io|
39
44
  if inherit?
40
45
  io.write <<-EOF
41
46
  if test -n "$AUTOPROJ_CURRENT_ROOT" && test "$AUTOPROJ_CURRENT_ROOT" != "#{root_dir}"; then
@@ -46,6 +51,11 @@ def export_env_sh(subdir = nil, options = Hash.new)
46
51
  end
47
52
  super(io)
48
53
  end
54
+
55
+ if new_content != existing_content
56
+ File.open(filename, 'w') { |io| io.write new_content }
57
+ true
58
+ end
49
59
  end
50
60
  end
51
61
 
@@ -71,6 +71,8 @@ def raw_description_file
71
71
  overrides_data = Autoproj.in_file(overrides_file_path, Autoproj::YAML_LOAD_ERROR) do
72
72
  YAML.load(File.read(overrides_file_path)) || Hash.new
73
73
  end
74
+ overrides_data = PackageSet.validate_and_normalize_source_file(
75
+ overrides_file_path, overrides_data)
74
76
  description = description.merge(overrides_data)
75
77
  end
76
78
 
@@ -1024,6 +1024,22 @@ def load_package_manifest(pkg)
1024
1024
  manifest
1025
1025
  end
1026
1026
 
1027
+ def load_all_available_package_manifests
1028
+ # Load the manifest for packages that are already present on the
1029
+ # file system
1030
+ each_package_definition do |pkg|
1031
+ if pkg.checked_out?
1032
+ begin
1033
+ load_package_manifest(pkg)
1034
+ rescue Interrupt
1035
+ raise
1036
+ rescue Exception => e
1037
+ Autoproj.warn "cannot load package manifest for #{pkg.name}: #{e.message}"
1038
+ end
1039
+ end
1040
+ end
1041
+ end
1042
+
1027
1043
  # The set of overrides added with #add_osdeps_overrides
1028
1044
  attr_reader :osdeps_overrides
1029
1045
 
@@ -46,6 +46,9 @@ def cache_git(pkg, options = Hash.new)
46
46
  with_retry(10) do
47
47
  Autobuild::Subprocess.run('autoproj-cache', :import, Autobuild.tool('git'), '--git-dir', pkg.importdir, 'remote', 'update', 'autobuild')
48
48
  end
49
+ with_retry(10) do
50
+ Autobuild::Subprocess.run('autoproj-cache', :import, Autobuild.tool('git'), '--git-dir', pkg.importdir, 'fetch', '--tags')
51
+ end
49
52
  Autobuild::Subprocess.run('autoproj-cache', :import, Autobuild.tool('git'), '--git-dir', pkg.importdir, 'gc', '--prune=all')
50
53
  end
51
54
 
@@ -120,13 +120,14 @@ def update_remote_package_set(vcs,
120
120
  reset: false,
121
121
  retry_count: nil)
122
122
 
123
- name = PackageSet.name_of(ws, vcs)
124
123
  raw_local_dir = PackageSet.raw_local_dir_of(ws, vcs)
125
-
126
124
  if checkout_only && File.exist?(raw_local_dir)
127
125
  return
128
126
  end
129
127
 
128
+ # name_of does minimal validation of source.yml, so do it here
129
+ # even though we don't really need it
130
+ name = PackageSet.name_of(ws, vcs, ignore_load_errors: true)
130
131
  ws.install_os_packages([vcs.type], all: nil)
131
132
  update_configuration_repository(
132
133
  vcs, name, raw_local_dir,
@@ -38,6 +38,7 @@ def initialize(root_dir)
38
38
  @autoproj_options = Array.new
39
39
 
40
40
  @env = Hash.new
41
+ env['RUBYOPT'] = []
41
42
  env['RUBYLIB'] = []
42
43
  env['GEM_PATH'] = []
43
44
  env['GEM_HOME'] = []
@@ -182,6 +183,7 @@ def self.guess_gem_program
182
183
  # @return [String]
183
184
  def default_gemfile_contents(autoproj_version = ">= 2.0.0")
184
185
  ["source \"#{gem_source}\"",
186
+ "ruby \"#{RUBY_VERSION}\" if respond_to?(:ruby)",
185
187
  "gem \"autoproj\", \"#{autoproj_version}\"",
186
188
  "gem \"utilrb\", \">= 3.0.1\""].join("\n")
187
189
  end
@@ -250,14 +252,19 @@ def find_bundler(gem_program)
250
252
  end
251
253
  end
252
254
 
253
- def install_bundler(gem_program)
255
+ def install_bundler(gem_program, silent: false)
254
256
  local = ['--local'] if local?
255
257
 
258
+ redirection = Hash.new
259
+ if silent
260
+ redirection = Hash[out: :close]
261
+ end
262
+
256
263
  result = system(
257
- env_for_child,
264
+ env_for_child.merge('GEM_HOME' => gems_gem_home),
258
265
  Gem.ruby, gem_program, 'install', '--env-shebang', '--no-document', '--no-format-executable', '--clear-sources', '--source', gem_source,
259
266
  *local,
260
- "--bindir=#{File.join(gems_gem_home, 'bin')}", 'bundler')
267
+ "--bindir=#{File.join(gems_gem_home, 'bin')}", 'bundler', **redirection)
261
268
 
262
269
  if !result
263
270
  STDERR.puts "FATAL: failed to install bundler in #{gems_gem_home}"
@@ -563,14 +570,16 @@ def call_stage2
563
570
 
564
571
  def stage2(*vars)
565
572
  require 'autobuild'
566
- puts "saving env.sh and .autoproj/env.sh"
573
+ puts "saving temporary env.sh and .autoproj/env.sh"
567
574
  save_env_sh(*vars)
575
+ puts "running 'autoproj envsh' to generate a proper env.sh"
568
576
  if !system(Gem.ruby, autoproj_path, 'envsh', *autoproj_options)
569
577
  STDERR.puts "failed to run autoproj envsh on the newly installed autoproj (#{autoproj_path})"
570
578
  exit 1
571
579
  end
572
580
  # This is really needed on an existing install to install the
573
581
  # gems that were present in the v1 layout
582
+ puts "running 'autoproj osdeps' to re-install missing gems"
574
583
  if !system(Gem.ruby, autoproj_path, 'osdeps')
575
584
  STDERR.puts "failed to run autoproj osdeps on the newly installed autoproj (#{autoproj_path})"
576
585
  exit 1
@@ -14,7 +14,7 @@ def initialize(ws)
14
14
  #
15
15
  # @see check_root_dir_empty
16
16
  EXPECTED_ROOT_ENTRIES = [".", "..", "autoproj_bootstrap",
17
- ".gems", "bootstrap.sh", ENV_FILENAME].to_set
17
+ ".autoproj", "bootstrap.sh", ENV_FILENAME].to_set
18
18
 
19
19
  # Verifies that {#root_dir} contains only expected entries, to make
20
20
  # sure that the user bootstraps into a new directory
@@ -82,7 +82,7 @@ def bootstrap(buildconf_info, options = Hash.new)
82
82
 
83
83
  ws.config.validate_ruby_executable
84
84
  ws.config.set 'reused_autoproj_installations', options[:reuse], true
85
- ws.env.export_env_sh(nil, shell_helpers: ws.config.shell_helpers?)
85
+ ws.env.export_env_sh(shell_helpers: ws.config.shell_helpers?)
86
86
 
87
87
  # If we are not getting the installation setup from a VCS, copy the template
88
88
  # files
@@ -110,7 +110,7 @@ def bootstrap(buildconf_info, options = Hash.new)
110
110
  url = VCSDefinition.to_absolute_url(url, ws.root_dir)
111
111
  do_switch_config(false, type, url, *options)
112
112
  end
113
- ws.env.export_env_sh(nil, shell_helpers: ws.config.shell_helpers?)
113
+ ws.env.export_env_sh(shell_helpers: ws.config.shell_helpers?)
114
114
  ws.config.save
115
115
  end
116
116
 
@@ -19,15 +19,12 @@ class Snapshot
19
19
  # @param [Array<String=>Hash>] state the current state
20
20
  # @param [Hash] the updated information
21
21
  def self.merge_packets( overrides, state )
22
- result = overrides.dup
23
22
  overriden = overrides.map { |entry| entry.keys.first }.to_set
24
- state.each do |pkg|
23
+ filtered_state = state.find_all do |pkg|
25
24
  name, _ = pkg.first
26
- if !overriden.include?(name)
27
- result << pkg
28
- end
25
+ !overriden.include?(name)
29
26
  end
30
- result
27
+ filtered_state + overrides
31
28
  end
32
29
 
33
30
  def self.update_log_available?(manifest)
@@ -54,7 +51,7 @@ def save_versions( versions, versions_file, replace: false)
54
51
  FileUtils.mkdir_p(File.dirname( versions_file ))
55
52
 
56
53
  # augment the versions file with the updated versions
57
- Snapshot.merge_packets( versions, existing_versions )
54
+ versions = Snapshot.merge_packets( versions, existing_versions )
58
55
 
59
56
  versions = sort_versions(versions)
60
57
 
@@ -69,5 +69,15 @@ def add_setup_block(block)
69
69
  block.call(autobuild)
70
70
  end
71
71
  end
72
+
73
+ # Whether this package is already checked out
74
+ def checked_out?
75
+ autobuild.checked_out?
76
+ end
77
+
78
+ # Add another package as a dependency of this one
79
+ def depends_on(pkg)
80
+ autobuild.depends_on(pkg.autobuild)
81
+ end
72
82
  end
73
83
  end
@@ -332,6 +332,7 @@ def install(gems, filter_uptodate_packages: false, install_only: false)
332
332
  FileUtils.mkdir_p root_dir
333
333
  if updated = (!File.exist?(gemfile_path) || File.read(gemfile_path) != gemfile_contents)
334
334
  File.open(gemfile_path, 'w') do |io|
335
+ io.puts "ruby \"#{RUBY_VERSION}\" if respond_to?(:ruby)"
335
336
  io.puts gemfile_contents
336
337
  end
337
338
  end
@@ -6,6 +6,7 @@ class PipManager < Manager
6
6
 
7
7
  def initialize_environment
8
8
  ws.env.set 'PYTHONUSERBASE', pip_home
9
+ ws.env.add_path 'PATH', File.join(pip_home, 'bin')
9
10
  end
10
11
 
11
12
  # Return the directory where python packages are installed to.
@@ -225,12 +225,15 @@ def snapshot(target_dir, options = Hash.new)
225
225
  # checked out, and the vcs (as a string) otherwise
226
226
  #
227
227
  # @return [String]
228
- def self.name_of(ws, vcs, raw_local_dir: raw_local_dir_of(ws, vcs))
228
+ def self.name_of(ws, vcs, raw_local_dir: raw_local_dir_of(ws, vcs), ignore_load_errors: false)
229
229
  if File.directory?(raw_local_dir)
230
- raw_description_file(raw_local_dir, package_set_name: "#{vcs.type}:#{vcs.url}")['name']
231
- else
232
- vcs.to_s
230
+ begin
231
+ return raw_description_file(raw_local_dir, package_set_name: "#{vcs.type}:#{vcs.url}")['name']
232
+ rescue ConfigError
233
+ raise if !ignore_load_errors
234
+ end
233
235
  end
236
+ vcs.to_s
234
237
  end
235
238
 
236
239
  # Returns the local directory in which the given package set should be
@@ -258,7 +261,7 @@ def self.resolve_definition(ws, raw_spec, from: nil, raw: Array.new)
258
261
  spec = VCSDefinition.normalize_vcs_hash(raw_spec, base_dir: ws.config_dir)
259
262
  options, vcs_spec = Kernel.filter_options spec, auto_imports: true
260
263
 
261
- vcs_spec = Autoproj.expand(vcs_spec, ws.manifest.constant_definitions)
264
+ vcs_spec = Autoproj.expand(vcs_spec, ws.config.to_hash.merge(ws.manifest.constant_definitions))
262
265
  return VCSDefinition.from_raw(vcs_spec, from: from, raw: raw), options
263
266
  end
264
267
 
@@ -309,6 +312,34 @@ def local_dir
309
312
  end
310
313
  end
311
314
 
315
+ # @api private
316
+ #
317
+ # Validate and normalizes a raw source file
318
+ def self.validate_and_normalize_source_file(yaml_path, yaml)
319
+ yaml = yaml.dup
320
+ %w{imports version_control}.each do |entry_name|
321
+ yaml[entry_name] ||= Array.new
322
+ if !yaml[entry_name].respond_to?(:to_ary)
323
+ raise ConfigError.new(yaml_path), "expected the '#{entry_name}' entry to be an array"
324
+ end
325
+ end
326
+
327
+ %w{constants}.each do |entry_name|
328
+ yaml[entry_name] ||= Hash.new
329
+ if !yaml[entry_name].respond_to?(:to_h)
330
+ raise ConfigError.new(yaml_path), "expected the '#{entry_name}' entry to be a map"
331
+ end
332
+ end
333
+
334
+ if yaml.has_key?('overrides')
335
+ yaml['overrides'] ||= Array.new
336
+ if !yaml['overrides'].respond_to?(:to_ary)
337
+ raise ConfigError.new(yaml_path), "expected the 'overrides' entry to be an array"
338
+ end
339
+ end
340
+ yaml
341
+ end
342
+
312
343
  # @api private
313
344
  #
314
345
  # Read the description information for a package set in a given
@@ -330,7 +361,8 @@ def self.raw_description_file(raw_local_dir, package_set_name: nil)
330
361
  newdefs = Autoproj.in_file(source_file, Autoproj::YAML_LOAD_ERROR) do
331
362
  YAML.load(File.read(source_file))
332
363
  end
333
- source_definition.merge!(newdefs || Hash.new) do |k, old, new|
364
+ newdefs = validate_and_normalize_source_file(source_file, newdefs || Hash.new)
365
+ source_definition.merge!(newdefs) do |k, old, new|
334
366
  if old.respond_to?(:to_ary)
335
367
  old + new
336
368
  else new
@@ -340,6 +372,7 @@ def self.raw_description_file(raw_local_dir, package_set_name: nil)
340
372
  if !source_definition['name']
341
373
  raise ConfigError.new(master_source_file), "#{master_source_file} does not have a 'name' field"
342
374
  end
375
+
343
376
  source_definition
344
377
  end
345
378
 
data/lib/autoproj/test.rb CHANGED
@@ -246,6 +246,12 @@ def workspace_env(varname)
246
246
  stdout.chomp
247
247
  end
248
248
 
249
+ def in_ws
250
+ Dir.chdir(@ws.root_dir) do
251
+ yield
252
+ end
253
+ end
254
+
249
255
  attr_reader :ws_os_package_resolver
250
256
 
251
257
  def ws_define_package_manager(name, strict: false, call_while_empty: false)
@@ -276,6 +282,8 @@ def ws_create
276
282
  dir, os_package_resolver: ws_os_package_resolver,
277
283
  package_managers: ws_package_managers)
278
284
  ws.config.set 'osdeps_mode', 'all'
285
+ ws.config.set 'GITHUB', 'http,ssh', true
286
+ ws.config.set 'GITORIOUS', 'http,ssh', true
279
287
  ws.config.set 'gems_install_path', File.join(dir, 'gems')
280
288
  ws.config.save
281
289
  ws.prefix_dir = make_tmpdir