autoproj 2.11.0 → 2.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +5 -8
  3. data/.travis.yml +5 -3
  4. data/autoproj.gemspec +6 -6
  5. data/bin/alog +1 -0
  6. data/bin/autoproj +1 -1
  7. data/bin/autoproj_bootstrap +130 -67
  8. data/bin/autoproj_bootstrap.in +9 -7
  9. data/bin/autoproj_install +129 -63
  10. data/bin/autoproj_install.in +8 -3
  11. data/lib/autoproj/autobuild_extensions/dsl.rb +27 -12
  12. data/lib/autoproj/base.rb +18 -0
  13. data/lib/autoproj/cli/base.rb +1 -1
  14. data/lib/autoproj/cli/build.rb +8 -3
  15. data/lib/autoproj/cli/cache.rb +79 -7
  16. data/lib/autoproj/cli/inspection_tool.rb +5 -6
  17. data/lib/autoproj/cli/main.rb +33 -9
  18. data/lib/autoproj/cli/show.rb +12 -18
  19. data/lib/autoproj/cli/status.rb +15 -9
  20. data/lib/autoproj/cli/test.rb +1 -1
  21. data/lib/autoproj/cli/update.rb +72 -17
  22. data/lib/autoproj/cli/utility.rb +25 -28
  23. data/lib/autoproj/configuration.rb +15 -4
  24. data/lib/autoproj/default.osdeps +29 -3
  25. data/lib/autoproj/environment.rb +17 -13
  26. data/lib/autoproj/installation_manifest.rb +7 -5
  27. data/lib/autoproj/manifest.rb +14 -6
  28. data/lib/autoproj/ops/build.rb +23 -21
  29. data/lib/autoproj/ops/cache.rb +151 -33
  30. data/lib/autoproj/ops/cached_env.rb +2 -2
  31. data/lib/autoproj/ops/import.rb +23 -4
  32. data/lib/autoproj/ops/install.rb +121 -60
  33. data/lib/autoproj/ops/phase_reporting.rb +49 -0
  34. data/lib/autoproj/ops/snapshot.rb +2 -1
  35. data/lib/autoproj/ops/tools.rb +2 -2
  36. data/lib/autoproj/os_package_installer.rb +19 -11
  37. data/lib/autoproj/package_definition.rb +1 -1
  38. data/lib/autoproj/package_managers/apt_dpkg_manager.rb +49 -28
  39. data/lib/autoproj/package_managers/bundler_manager.rb +102 -19
  40. data/lib/autoproj/package_managers/homebrew_manager.rb +2 -2
  41. data/lib/autoproj/package_managers/pip_manager.rb +34 -22
  42. data/lib/autoproj/package_managers/shell_script_manager.rb +44 -24
  43. data/lib/autoproj/package_manifest.rb +43 -31
  44. data/lib/autoproj/package_set.rb +2 -2
  45. data/lib/autoproj/python.rb +285 -0
  46. data/lib/autoproj/test.rb +26 -10
  47. data/lib/autoproj/variable_expansion.rb +3 -1
  48. data/lib/autoproj/vcs_definition.rb +25 -12
  49. data/lib/autoproj/version.rb +1 -1
  50. data/lib/autoproj/workspace.rb +60 -16
  51. data/lib/autoproj.rb +3 -0
  52. metadata +17 -28
data/lib/autoproj/base.rb CHANGED
@@ -56,4 +56,22 @@ module Autoproj
56
56
  end
57
57
  end
58
58
  end
59
+
60
+ # Shim for a smooth upgrade path to bundler 2.1+
61
+ def self.bundler_unbundled_system(*args, **options)
62
+ if Bundler.respond_to?(:unbundled_system)
63
+ Bundler.unbundled_system(*args, **options)
64
+ else
65
+ Bundler.clean_system(*args, **options)
66
+ end
67
+ end
68
+
69
+ # Shim for a smooth upgrade path to bundler 2.1+
70
+ def self.bundler_with_unbundled_env(&block)
71
+ if Bundler.respond_to?(:with_unbundled_env)
72
+ Bundler.with_unbundled_env(&block)
73
+ else
74
+ Bundler.with_clean_env(&block)
75
+ end
76
+ end
59
77
  end
@@ -151,7 +151,7 @@ module Autoproj
151
151
  non_imported_packages: non_imported_packages,
152
152
  auto_exclude: auto_exclude)
153
153
 
154
- return source_packages, osdep_packages, resolved_selection
154
+ [source_packages, osdep_packages, resolved_selection]
155
155
  rescue ExcludedSelection => e
156
156
  raise CLIInvalidSelection, e.message, e.backtrace
157
157
  end
@@ -18,12 +18,14 @@ module Autoproj
18
18
  end
19
19
 
20
20
  def run(selected_packages, **options)
21
- build_options, options = filter_options options,
21
+ build_options, options = filter_options(
22
+ options,
22
23
  force: false,
23
24
  rebuild: false,
24
25
  parallel: nil,
25
26
  confirm: true,
26
27
  not: Array.new
28
+ )
27
29
 
28
30
  command_line_selection, source_packages, _osdep_packages =
29
31
  super(selected_packages,
@@ -41,6 +43,7 @@ module Autoproj
41
43
  # Disable all packages that are not selected
42
44
  ws.manifest.each_autobuild_package do |pkg|
43
45
  next if active_packages.include?(pkg.name)
46
+
44
47
  pkg.disable
45
48
  end
46
49
 
@@ -61,11 +64,13 @@ module Autoproj
61
64
  else 'force-build'
62
65
  end
63
66
  if build_options[:confirm] != false
64
- opt = BuildOption.new("", "boolean",
67
+ opt = BuildOption.new(
68
+ "", "boolean",
65
69
  {
66
70
  doc: "this is going to trigger a #{mode_name} "\
67
71
  "of all packages. Is that really what you want ?"
68
- }, nil)
72
+ }, nil
73
+ )
69
74
  raise Interrupt unless opt.ask(false)
70
75
  end
71
76
 
@@ -1,32 +1,104 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require 'autoproj/cli/inspection_tool'
2
4
  require 'autoproj/ops/cache'
3
5
 
4
6
  module Autoproj
5
7
  module CLI
6
8
  class Cache < InspectionTool
9
+ def parse_gem_compile(string)
10
+ scanner = StringScanner.new(string)
11
+ name = scanner.scan(/[^\[]+/)
12
+
13
+ level = 0
14
+ artifacts = []
15
+ artifact_include = nil
16
+ artifact_name = ''.dup
17
+ until scanner.eos?
18
+ c = scanner.getch
19
+ if level == 0
20
+ raise ArgumentError, "expected '[' but got '#{c}'" unless c == '['
21
+
22
+ level = 1
23
+ include_c = scanner.getch
24
+ if %w[+ -].include?(include_c)
25
+ artifact_include = (include_c == '+')
26
+ elsif include_c == ']'
27
+ raise ArgumentError, "empty [] found in '#{string}'"
28
+ else
29
+ raise ArgumentError,
30
+ "expected '+' or '-' but got '#{c}' in '#{string}'"
31
+ end
32
+ next
33
+ end
34
+
35
+ if c == ']'
36
+ level -= 1
37
+ if level == 0
38
+ artifacts << [artifact_include, artifact_name]
39
+ artifact_name = ''.dup
40
+ next
41
+ end
42
+ end
43
+
44
+ artifact_name << c
45
+ end
46
+
47
+ raise ArgumentError, "missing closing ']' in #{string}" if level != 0
48
+
49
+ [name, artifacts: artifacts]
50
+ end
51
+
7
52
  def validate_options(argv, options = Hash.new)
8
53
  argv, options = super
9
54
 
10
55
  if argv.empty?
11
56
  default_cache_dirs = Autobuild::Importer.default_cache_dirs
12
57
  if !default_cache_dirs || default_cache_dirs.empty?
13
- raise CLIInvalidArguments, "no cache directory defined with e.g. the AUTOBUILD_CACHE_DIR environment variable, expected one cache directory as argument"
58
+ raise CLIInvalidArguments,
59
+ "no cache directory defined with e.g. the "\
60
+ "AUTOBUILD_CACHE_DIR environment variable, "\
61
+ "expected one cache directory as argument"
14
62
  end
15
- Autoproj.warn "using cache directory #{default_cache_dirs.first} from the autoproj configuration"
63
+ Autoproj.warn "using cache directory #{default_cache_dirs.first} "\
64
+ "from the autoproj configuration"
16
65
  argv << default_cache_dirs.first
17
66
  end
18
67
 
19
- return File.expand_path(argv.first, ws.root_dir), *argv[1..-1], options
68
+ if (compile = options[:gems_compile])
69
+ options[:gems_compile] = compile.map do |name|
70
+ parse_gem_compile(name)
71
+ end
72
+ end
73
+
74
+ [File.expand_path(argv.first, ws.root_dir), *argv[1..-1], options]
20
75
  end
21
76
 
22
- def run(cache_dir, *package_names, all: true, keep_going: false, checkout_only: false)
77
+ def run(cache_dir, *package_names,
78
+ keep_going: false,
79
+ packages: true, all: true, checkout_only: false,
80
+ gems: false, gems_compile: [], gems_compile_force: false)
23
81
  initialize_and_load
24
82
  finalize_setup
25
83
 
26
- cache_op = Autoproj::Ops::Cache.new(cache_dir, ws.manifest)
27
- cache_op.create_or_update(*package_names, all: all, keep_going: keep_going, checkout_only: checkout_only)
84
+ cache_op = Autoproj::Ops::Cache.new(cache_dir, ws)
85
+ if packages
86
+ cache_op.create_or_update(
87
+ *package_names,
88
+ all: all, keep_going: keep_going,
89
+ checkout_only: checkout_only
90
+ )
91
+ end
92
+
93
+ if gems
94
+ Autoproj.message "caching gems in #{cache_op.gems_cache_dir}"
95
+ cache_op.create_or_update_gems(
96
+ keep_going: keep_going,
97
+ compile: gems_compile,
98
+ compile_force: gems_compile_force
99
+ )
100
+ end
28
101
  end
29
102
  end
30
103
  end
31
104
  end
32
-
@@ -8,9 +8,7 @@ module Autoproj
8
8
  def initialize_and_load(mainline: nil)
9
9
  Autoproj.silent do
10
10
  ws.setup
11
- if mainline == 'mainline' || mainline == 'true'
12
- mainline = true
13
- end
11
+ mainline = true if %w[mainline true].include?(mainline)
14
12
  ws.load_package_sets(mainline: mainline)
15
13
  ws.config.save
16
14
  ws.setup_all_package_directories
@@ -22,7 +20,7 @@ module Autoproj
22
20
  # @param [Array<String>] packages the list of package names
23
21
  # @param [Symbol] non_imported_packages whether packages that are
24
22
  # not yet imported should be ignored (:ignore) or returned
25
- # (:return).
23
+ # (:return).
26
24
  # @option options recursive (true) whether the package resolution
27
25
  # should return the package(s) and their dependencies
28
26
  #
@@ -32,7 +30,8 @@ module Autoproj
32
30
  # the arguments were pointing within the configuration area
33
31
  def finalize_setup(packages = [], non_imported_packages: :ignore, recursive: true, auto_exclude: false)
34
32
  Autoproj.silent do
35
- packages, config_selected = normalize_command_line_package_selection(packages)
33
+ packages, config_selected =
34
+ normalize_command_line_package_selection(packages)
36
35
  # Call resolve_user_selection once to auto-add packages, so
37
36
  # that they're available to e.g. overrides.rb
38
37
  resolve_user_selection(packages)
@@ -41,7 +40,7 @@ module Autoproj
41
40
  resolve_selection(packages, recursive: recursive, non_imported_packages: non_imported_packages, auto_exclude: auto_exclude)
42
41
  ws.finalize_setup
43
42
  ws.export_installation_manifest
44
- return source_packages, osdep_packages, resolved_selection, config_selected
43
+ [source_packages, osdep_packages, resolved_selection, config_selected]
45
44
  end
46
45
  end
47
46
  end
@@ -134,8 +134,9 @@ module Autoproj
134
134
  end
135
135
  cli = CLI.const_get(classname).new
136
136
  begin
137
- run_args = cli.validate_options(args, options.merge(extra_options))
138
- cli.run(*run_args)
137
+ *run_args, kw = cli.validate_options(args, options.merge(extra_options))
138
+ kw = (kw || {}).transform_keys(&:to_sym)
139
+ cli.run(*run_args, **kw)
139
140
  ensure
140
141
  cli.notify_env_sh_updated if cli.respond_to?(:notify_env_sh_updated)
141
142
  end
@@ -228,6 +229,8 @@ module Autoproj
228
229
  desc: "compare to the given baseline. if 'true', the comparison will ignore any override, otherwise it will take into account overrides only up to the given package set"
229
230
  option :auto_exclude, type: :boolean,
230
231
  desc: 'if true, packages that fail to import will be excluded from the build'
232
+ option :ask, type: :boolean, default: false,
233
+ desc: 'ask whether each package should or should not be updated'
231
234
  def update(*packages)
232
235
  report_options = Hash[silent: false, on_package_failures: default_report_on_package_failures]
233
236
  if options[:auto_exclude]
@@ -300,13 +303,34 @@ In this case, the default is false
300
303
  end
301
304
  end
302
305
 
303
- desc 'cache CACHE_DIR', 'create or update a cache directory that can be given to AUTOBUILD_CACHE_DIR'
304
- option :keep_going, aliases: :k,
305
- desc: 'do not stop on errors'
306
- option :checkout_only, aliases: :c, type: :boolean, default: false,
307
- desc: "only checkout packages, do not update already-cached ones"
308
- option :all, type: :boolean, default: true,
309
- desc: "cache all defined packages (the default) or only the selected ones"
306
+ desc 'cache CACHE_DIR', 'create or update a cache directory that '\
307
+ 'can be given to AUTOBUILD_CACHE_DIR'
308
+ option :keep_going,
309
+ aliases: :k,
310
+ desc: 'do not stop on errors'
311
+ option :checkout_only,
312
+ aliases: :c, type: :boolean, default: false,
313
+ desc: 'only checkout packages, do not update already-cached ones'
314
+ option :all,
315
+ type: :boolean, default: true,
316
+ desc: 'cache all defined packages (the default), '\
317
+ ' or only the selected ones'
318
+ option :packages,
319
+ type: :boolean, default: true,
320
+ desc: 'update the package cache'
321
+ option :gems,
322
+ type: :boolean, default: false,
323
+ desc: 'update the gems cache'
324
+ option :gems_compile_force,
325
+ type: :boolean, default: false,
326
+ desc: 'with --gems-compile, recompile existing gems as well'
327
+ option :gems_compile,
328
+ type: :array,
329
+ desc: 'pre-compile the following gems. This requires gem-compiler '\
330
+ 'to be available in the workspace. Use GEM_NAME+ARTIFACT'\
331
+ '[+ARTIFACT] to add files or directories to the precompiled '\
332
+ 'gems beyond what gem-compiler auto-adds (which is mostly '\
333
+ 'dynamic libraries)'
310
334
  def cache(*args)
311
335
  run_autoproj_cli(:cache, :Cache, Hash[], *args)
312
336
  end
@@ -17,7 +17,8 @@ module Autoproj
17
17
  source_packages, osdep_packages, * =
18
18
  finalize_setup(user_selection, recursive: recursive, non_imported_packages: :return)
19
19
  else
20
- source_packages, osdep_packages = Array.new, Array.new
20
+ source_packages = []
21
+ osdep_packages = []
21
22
  end
22
23
 
23
24
  all_matching_osdeps = osdep_packages.map { |pkg| [pkg, true] }
@@ -232,7 +233,7 @@ module Autoproj
232
233
  end
233
234
  end
234
235
 
235
- if !selections.empty?
236
+ unless selections.empty?
236
237
  puts " selected by way of"
237
238
  selections.each do |root_pkg|
238
239
  paths = find_selection_path(root_pkg, pkg_name)
@@ -248,9 +249,7 @@ module Autoproj
248
249
  end
249
250
 
250
251
  def find_selection_path(from, to)
251
- if from == to
252
- return [[from]]
253
- end
252
+ return [[from]] if from == to
254
253
 
255
254
  all_paths = Array.new
256
255
  ws.manifest.resolve_package_name(from).each do |pkg_type, pkg_name|
@@ -262,19 +261,17 @@ module Autoproj
262
261
 
263
262
  pkg = ws.manifest.find_autobuild_package(pkg_name)
264
263
  pkg.dependencies.each do |dep_pkg_name|
265
- if result = find_selection_path(dep_pkg_name, to)
264
+ if (result = find_selection_path(dep_pkg_name, to))
266
265
  all_paths.concat(result.map { |p| path + p })
267
266
  end
268
267
  end
269
- if pkg.os_packages.include?(to)
270
- all_paths << (path + [to])
271
- end
268
+ all_paths << (path + [to]) if pkg.os_packages.include?(to)
272
269
  end
273
270
 
274
271
  # Now filter common trailing subpaths
275
272
  all_paths = all_paths.sort_by(&:size)
276
273
  filtered_paths = Array.new
277
- while !all_paths.empty?
274
+ until all_paths.empty?
278
275
  path = all_paths.shift
279
276
  filtered_paths << path
280
277
  size = path.size
@@ -290,19 +287,15 @@ module Autoproj
290
287
  options = vcs.dup
291
288
  type = options.delete('type')
292
289
  url = options.delete('url')
293
- else
290
+ else
294
291
  options = vcs.options
295
292
  type = vcs.type
296
293
  url = vcs.url
297
294
  end
298
295
 
299
296
  fields = []
300
- if type
301
- fields << ['type', type]
302
- end
303
- if url
304
- fields << ['url', url]
305
- end
297
+ fields << ['type', type] if type
298
+ fields << ['url', url] if url
306
299
  fields = fields.concat(options.to_a.sort_by { |k, _| k.to_s })
307
300
  fields.map do |key, value|
308
301
  if value.respond_to?(:to_str) && File.file?(value) && value =~ /^\//
@@ -315,9 +308,10 @@ module Autoproj
315
308
  def compute_all_revdeps(pkg_revdeps, revdeps)
316
309
  pkg_revdeps = pkg_revdeps.dup
317
310
  all_revdeps = Array.new
318
- while !pkg_revdeps.empty?
311
+ until pkg_revdeps.empty?
319
312
  parent_name = pkg_revdeps.shift
320
313
  next if all_revdeps.include?(parent_name)
314
+
321
315
  all_revdeps << parent_name
322
316
  pkg_revdeps.concat(revdeps[parent_name].to_a)
323
317
  end
@@ -73,7 +73,7 @@ module Autoproj
73
73
  end
74
74
  end
75
75
 
76
- def report_exception(package_status, msg, e)
76
+ def self.report_exception(package_status, msg, e)
77
77
  package_status.msg << Autoproj.color(" #{msg} (#{e})", :red)
78
78
  if Autobuild.debug
79
79
  package_status.msg.concat(e.backtrace.map do |line|
@@ -82,8 +82,8 @@ module Autoproj
82
82
  end
83
83
  end
84
84
 
85
- PackageStatus = Struct.new :msg, :sync, :uncommitted, :local, :remote
86
- def status_of_package(package_description, only_local: false, snapshot: false)
85
+ PackageStatus = Struct.new :msg, :sync, :unexpected, :uncommitted, :local, :remote
86
+ def self.status_of_package(package_description, only_local: false, snapshot: false)
87
87
  pkg = package_description.autobuild
88
88
  importer = pkg.importer
89
89
  package_status = PackageStatus.new(Array.new, false, false, false, false)
@@ -96,7 +96,7 @@ module Autoproj
96
96
  else
97
97
  begin status = importer.status(pkg, only_local: only_local)
98
98
  rescue StandardError => e
99
- report_exception(package_status, "failed to fetch status information", e)
99
+ self.report_exception(package_status, "failed to fetch status information", e)
100
100
  return package_status
101
101
  end
102
102
 
@@ -108,7 +108,7 @@ module Autoproj
108
108
  rescue Autobuild::PackageException
109
109
  Hash.new
110
110
  rescue StandardError => e
111
- report_exception(package_status, "failed to fetch snapshotting information", e)
111
+ self.report_exception(package_status, "failed to fetch snapshotting information", e)
112
112
  return package_status
113
113
  end
114
114
  if snapshot_overrides_vcs?(importer, package_description.vcs, snapshot_version)
@@ -122,6 +122,7 @@ module Autoproj
122
122
  end
123
123
 
124
124
  status.unexpected_working_copy_state.each do |msg|
125
+ package_status.unexpected = true
125
126
  package_status.msg << Autoproj.color(" #{msg}", :red, :bold)
126
127
  end
127
128
 
@@ -173,13 +174,15 @@ module Autoproj
173
174
  end
174
175
  noninteractive = noninteractive.map do |pkg|
175
176
  future = Concurrent::Future.execute(executor: executor) do
176
- status_of_package(pkg, snapshot: snapshot, only_local: only_local)
177
+ Status.status_of_package(
178
+ pkg, snapshot: snapshot, only_local: only_local
179
+ )
177
180
  end
178
181
  [pkg, future]
179
182
  end
180
183
 
181
184
  (noninteractive + interactive).each do |pkg, future|
182
- if future
185
+ if future
183
186
  if progress
184
187
  wait_timeout = 1
185
188
  while true
@@ -196,7 +199,10 @@ module Autoproj
196
199
  if !(status = future.value)
197
200
  raise future.reason
198
201
  end
199
- else status = status_of_package(pkg, snapshot: snapshot, only_local: only_local)
202
+ else
203
+ status = Status.status_of_package(
204
+ pkg, snapshot: snapshot, only_local: only_local
205
+ )
200
206
  end
201
207
 
202
208
  result.uncommitted ||= status.uncommitted
@@ -263,7 +269,7 @@ module Autoproj
263
269
  sync_packages = ""
264
270
  end
265
271
 
266
- STDERR.print
272
+ STDERR.print
267
273
 
268
274
  if status.msg.size == 1
269
275
  Autoproj.message "#{pkg_name}: #{status.msg.first}"