autoproj 2.11.0 → 2.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +5 -8
  3. data/.travis.yml +5 -3
  4. data/autoproj.gemspec +6 -6
  5. data/bin/alog +1 -0
  6. data/bin/autoproj +1 -1
  7. data/bin/autoproj_bootstrap +130 -67
  8. data/bin/autoproj_bootstrap.in +9 -7
  9. data/bin/autoproj_install +129 -63
  10. data/bin/autoproj_install.in +8 -3
  11. data/lib/autoproj/autobuild_extensions/dsl.rb +27 -12
  12. data/lib/autoproj/base.rb +18 -0
  13. data/lib/autoproj/cli/base.rb +1 -1
  14. data/lib/autoproj/cli/build.rb +8 -3
  15. data/lib/autoproj/cli/cache.rb +79 -7
  16. data/lib/autoproj/cli/inspection_tool.rb +5 -6
  17. data/lib/autoproj/cli/main.rb +33 -9
  18. data/lib/autoproj/cli/show.rb +12 -18
  19. data/lib/autoproj/cli/status.rb +15 -9
  20. data/lib/autoproj/cli/test.rb +1 -1
  21. data/lib/autoproj/cli/update.rb +72 -17
  22. data/lib/autoproj/cli/utility.rb +25 -28
  23. data/lib/autoproj/configuration.rb +15 -4
  24. data/lib/autoproj/default.osdeps +29 -3
  25. data/lib/autoproj/environment.rb +17 -13
  26. data/lib/autoproj/installation_manifest.rb +7 -5
  27. data/lib/autoproj/manifest.rb +14 -6
  28. data/lib/autoproj/ops/build.rb +23 -21
  29. data/lib/autoproj/ops/cache.rb +151 -33
  30. data/lib/autoproj/ops/cached_env.rb +2 -2
  31. data/lib/autoproj/ops/import.rb +23 -4
  32. data/lib/autoproj/ops/install.rb +121 -60
  33. data/lib/autoproj/ops/phase_reporting.rb +49 -0
  34. data/lib/autoproj/ops/snapshot.rb +2 -1
  35. data/lib/autoproj/ops/tools.rb +2 -2
  36. data/lib/autoproj/os_package_installer.rb +19 -11
  37. data/lib/autoproj/package_definition.rb +1 -1
  38. data/lib/autoproj/package_managers/apt_dpkg_manager.rb +49 -28
  39. data/lib/autoproj/package_managers/bundler_manager.rb +102 -19
  40. data/lib/autoproj/package_managers/homebrew_manager.rb +2 -2
  41. data/lib/autoproj/package_managers/pip_manager.rb +34 -22
  42. data/lib/autoproj/package_managers/shell_script_manager.rb +44 -24
  43. data/lib/autoproj/package_manifest.rb +43 -31
  44. data/lib/autoproj/package_set.rb +2 -2
  45. data/lib/autoproj/python.rb +285 -0
  46. data/lib/autoproj/test.rb +26 -10
  47. data/lib/autoproj/variable_expansion.rb +3 -1
  48. data/lib/autoproj/vcs_definition.rb +25 -12
  49. data/lib/autoproj/version.rb +1 -1
  50. data/lib/autoproj/workspace.rb +60 -16
  51. data/lib/autoproj.rb +3 -0
  52. metadata +17 -28
@@ -168,7 +168,7 @@ module Autoproj
168
168
  @version_control = Array.new
169
169
  @overrides = Array.new
170
170
  @raw_local_dir = raw_local_dir
171
- @default_importer = VCSDefinition.from_raw(type: 'none')
171
+ @default_importer = VCSDefinition.from_raw({ type: 'none' })
172
172
 
173
173
  @imports = Set.new
174
174
  @imports_vcs = Array.new
@@ -239,7 +239,7 @@ module Autoproj
239
239
  else
240
240
  package = create_autobuild_package
241
241
  if package.importer.respond_to?(:snapshot)
242
- package.importer.snapshot(package, target_dir, options)
242
+ package.importer.snapshot(package, target_dir, **options)
243
243
  end
244
244
  end
245
245
  end
@@ -0,0 +1,285 @@
1
+ require 'open3'
2
+ require 'rubygems'
3
+
4
+ module Autoproj
5
+ module Python
6
+ # Get the python version for a given python executable
7
+ # @return [String] The python version as <major>.<minor>
8
+ def self.get_python_version(python_bin)
9
+ unless File.exist?(python_bin)
10
+ raise ArgumentError, "Autoproj::Python.get_python_version executable "\
11
+ "'#{python_bin}' does not exist"
12
+ end
13
+
14
+ cmd = "#{python_bin} -c \"import sys;"\
15
+ "version=sys.version_info[:3]; "\
16
+ "print('{0}.{1}'.format(*version))\"".strip
17
+
18
+ msg, status = Open3.capture2e(cmd)
19
+ if status.success?
20
+ msg.strip
21
+
22
+ else
23
+ raise "Autoproj::Python.get_python_version identification"\
24
+ " of python version for '#{python_bin}' failed: #{msg}"
25
+ end
26
+ end
27
+
28
+ def self.get_pip_version(pip_bin)
29
+ unless File.exist?(pip_bin)
30
+ raise ArgumentError, "Autoproj::Python.get_pip_version executable "\
31
+ "'#{pip_bin}' does not exist"
32
+ end
33
+
34
+ cmd = "#{pip_bin} --version"
35
+
36
+ msg, status = Open3.capture2e(cmd)
37
+ if status.success?
38
+ msg.split(" ")[1]
39
+
40
+ else
41
+ raise "Autoproj::Python.get_pip_version identification"\
42
+ " of pip version for '#{pip_bin}' failed: #{msg}"
43
+ end
44
+ end
45
+
46
+ def self.validate_version(version, version_constraint)
47
+ if !version_constraint
48
+ true
49
+ else
50
+ dependency = Gem::Dependency.new("python", version_constraint)
51
+ dependency.match?("python", version)
52
+ end
53
+ end
54
+
55
+ # Validate that a given python executable's version fulfills
56
+ # a given version constraint
57
+ # @param [String] python_bin the python executable
58
+ # @param [String] version_constraint version constraint, e.g., <3.8, >= 3.7, 3.6
59
+ # @return [String,Bool] Version and validation result, i.e.,
60
+ # True if binary fulfills the version constraint, false otherwise
61
+ def self.validate_python_version(python_bin, version_constraint)
62
+ version = get_python_version(python_bin)
63
+ [version, validate_version(version, version_constraint)]
64
+ end
65
+
66
+ # Find python given a version constraint
67
+ # @return [String,String] path to python executable and python version
68
+ def self.find_python(ws: Autoproj.workspace,
69
+ version: ws.config.get('python_version', nil))
70
+ finders = [
71
+ -> { Autobuild.programs['python'] },
72
+ -> { `which python3`.strip },
73
+ -> { `which python`.strip }
74
+ ]
75
+
76
+ finders.each do |finder|
77
+ python_bin = finder.call
78
+ if python_bin && !python_bin.empty?
79
+ python_version, valid = validate_python_version(python_bin, version)
80
+ return python_bin, python_version if valid
81
+ end
82
+ end
83
+ raise "Autoproj::Python.find_python_bin: failed to find python" \
84
+ " for version '#{version}'"
85
+ end
86
+
87
+ # Get information about the python executable from autoproj config,
88
+ # but ensure the version constraint matches
89
+ #
90
+ # @return [String, String] Return path and version if the constraints
91
+ # are fulfilled nil otherwise
92
+
93
+ def self.get_python_from_config(ws: Autoproj.workspace, version: nil)
94
+ config_bin = ws.config.get('python_executable', nil)
95
+ return unless config_bin
96
+
97
+ config_version = ws.config.get('python_version', nil)
98
+ config_version ||= get_python_version(config_bin)
99
+
100
+ # If a version constraint is given, ensure fulfillment
101
+ if validate_version(config_version, version)
102
+ [config_bin, config_version]
103
+ else
104
+ raise "python_executable in autoproj config with " \
105
+ "version '#{config_version}' does not match "\
106
+ "version constraints '#{version}'"
107
+ end
108
+ end
109
+
110
+ def self.custom_resolve_python(bin: nil,
111
+ version: nil)
112
+ version, valid = validate_python_version(bin, version)
113
+ if valid
114
+ [bin, version]
115
+ else
116
+ raise "Autoproj::Python.resolve_python: requested python"\
117
+ "executable '#{bin}' does not satisfy version"\
118
+ "constraints '#{version}'"
119
+ end
120
+ end
121
+
122
+ def self.auto_resolve_python(ws: Autoproj.workspace,
123
+ version: nil)
124
+ version_constraint = version
125
+ resolvers = [
126
+ -> { get_python_from_config(ws: ws, version: version_constraint) },
127
+ -> { find_python(ws: ws, version: version_constraint) }
128
+ ]
129
+
130
+ bin = nil
131
+ resolvers.each do |resolver|
132
+ bin, version = resolver.call
133
+ if bin && File.exist?(bin) && version
134
+ Autoproj.debug "Autoproj::Python.resolve_python: " \
135
+ "found python '#{bin}' version '#{version}'"
136
+ break
137
+ end
138
+ rescue RuntimeError => e
139
+ Autoproj.debug "Autoproj::Python.resolve_python: " \
140
+ "resolver failed: #{e}"
141
+ end
142
+
143
+ unless bin
144
+ msg = "Autoproj::Python.resolve_python: " \
145
+ "failed to find a python executable"
146
+ if version_constraint
147
+ msg += " satisfying version constraint '#{version_constraint}'"
148
+ end
149
+ raise msg
150
+ end
151
+ [bin, version]
152
+ end
153
+
154
+ # Resolve the python executable according to a given version constraint
155
+ # @param [Autoproj.workspace] ws Autoproj workspace
156
+ # @param [String] bin Path to the python executable that shall be used,
157
+ # first fallback is the python_executable set in Autoproj's configuration,
158
+ # second fallback is a full search
159
+ # @param [String] version version constraint
160
+ # @return [String,String] python path and python version
161
+ def self.resolve_python(ws: Autoproj.workspace,
162
+ bin: nil,
163
+ version: nil)
164
+ if bin
165
+ custom_resolve_python(bin: bin, version: version)
166
+ else
167
+ auto_resolve_python(ws: ws, version: version)
168
+ end
169
+ end
170
+
171
+ def self.remove_python_shims(root_dir)
172
+ shim_path = File.join(root_dir, "install", "bin", "python")
173
+ FileUtils.rm shim_path if File.exist?(shim_path)
174
+ end
175
+
176
+ def self.remove_pip_shims(root_dir)
177
+ shim_path = File.join(root_dir, "install", "bin", "pip")
178
+ FileUtils.rm shim_path if File.exist?(shim_path)
179
+ end
180
+
181
+ def self.rewrite_python_shims(python_executable, root_dir)
182
+ shim_path = File.join(root_dir, "install", "bin")
183
+ unless File.exist?(shim_path)
184
+ FileUtils.mkdir_p shim_path
185
+ Autoproj.warn "Autoproj::Python.rewrite_python_shims: creating "\
186
+ "#{shim_path} - "\
187
+ "are you operating on a valid autoproj workspace?"
188
+ end
189
+
190
+ python_path = File.join(shim_path, 'python')
191
+ File.open(python_path, 'w') do |io|
192
+ io.puts "#! /bin/sh"
193
+ io.puts "exec #{python_executable} \"$@\""
194
+ end
195
+ FileUtils.chmod 0o755, python_path
196
+ python_path
197
+ end
198
+
199
+ def self.rewrite_pip_shims(python_executable, root_dir)
200
+ shim_path = File.join(root_dir, "install", "bin")
201
+ unless File.exist?(shim_path)
202
+ FileUtils.mkdir_p shim_path
203
+ Autoproj.warn "Autoproj::Python.rewrite_pip_shims: creating "\
204
+ "#{shim_path} - "\
205
+ "are you operating on a valid autoproj workspace?"
206
+ end
207
+ pip_path = File.join(shim_path, 'pip')
208
+ File.open(pip_path, 'w') do |io|
209
+ io.puts "#! /bin/sh"
210
+ io.puts "exec #{python_executable} -m pip \"$@\""
211
+ end
212
+ FileUtils.chmod 0o755, pip_path
213
+ pip_path
214
+ end
215
+
216
+ # Activate configuration for python in the autoproj configuration
217
+ # @return [String,String] python path and python version
218
+ def self.activate_python(ws: Autoproj.workspace,
219
+ bin: nil,
220
+ version: nil)
221
+ bin, version = resolve_python(ws: ws, bin: bin, version: version)
222
+ ws.config.set('python_executable', bin, true)
223
+ ws.config.set('python_version', version, true)
224
+
225
+ ws.osdep_suffixes << "python#{$1}" if version =~ /^([0-9]+)\./
226
+
227
+ rewrite_python_shims(bin, ws.root_dir)
228
+ rewrite_pip_shims(bin, ws.root_dir)
229
+ [bin, version]
230
+ end
231
+
232
+ def self.deactivate_python(ws: Autoproj.workspace)
233
+ remove_python_shims(ws.root_dir)
234
+ remove_pip_shims(ws.root_dir)
235
+ ws.config.reset('python_executable')
236
+ ws.config.reset('python_version')
237
+ end
238
+
239
+ # Allow to update the PYTHONPATH for package if autoproj configuration
240
+ # USE_PYTHON is set to true.
241
+ # Then tries to guess the python binary from Autobuild.programs['python']
242
+ # and system's default setting
243
+ # @param [Autobuild::Package] pkg
244
+ # @param [Autoproj.workspace] ws Autoproj workspace
245
+ # @param [String] bin Path to a custom python version
246
+ # @param [String] version version constraint for python executable
247
+ # @return tuple of [executable, version, site-packages path] if set,
248
+ # otherwise nil
249
+ def self.activate_python_path(pkg,
250
+ ws: Autoproj.workspace,
251
+ bin: nil,
252
+ version: nil)
253
+ return unless ws.config.get('USE_PYTHON', nil)
254
+
255
+ bin, version = resolve_python(ws: ws, bin: bin, version: version)
256
+ path = File.join(pkg.prefix, "lib",
257
+ "python#{version}", "site-packages")
258
+ pkg.env_add_path 'PYTHONPATH', path
259
+
260
+ [bin, version, path]
261
+ end
262
+
263
+ def self.setup_python_configuration_options(ws: Autoproj.workspace)
264
+ ws.config.declare 'USE_PYTHON', 'boolean',
265
+ default: 'no',
266
+ doc: ["Do you want to activate python?"]
267
+
268
+ if ws.config.get("USE_PYTHON")
269
+ unless ws.config.has_value_for?('python_executable')
270
+ remove_python_shims(ws.root_dir)
271
+ remove_pip_shims(ws.root_dir)
272
+ python_bin, = auto_resolve_python(ws: ws)
273
+ end
274
+
275
+ ws.config.declare 'python_executable', 'string',
276
+ default: python_bin.to_s,
277
+ doc: ["Select the path to the python executable"]
278
+
279
+ activate_python(ws: ws)
280
+ else
281
+ deactivate_python(ws: ws)
282
+ end
283
+ end
284
+ end
285
+ end
data/lib/autoproj/test.rb CHANGED
@@ -71,6 +71,7 @@ module Autoproj
71
71
  Autobuild.logdir = make_tmpdir
72
72
  ws_define_package_manager 'os'
73
73
  ws_define_package_manager 'os_indep'
74
+ Autobuild.progress_display_period = 0
74
75
 
75
76
  super
76
77
  end
@@ -80,6 +81,7 @@ module Autoproj
80
81
  @tmpdir.each do |dir|
81
82
  FileUtils.remove_entry_secure dir
82
83
  end
84
+ Rake::Task.clear
83
85
  Autobuild::Package.clear
84
86
  Autoproj.silent = false
85
87
 
@@ -94,6 +96,11 @@ module Autoproj
94
96
  end
95
97
  end
96
98
 
99
+ def data_path(*args)
100
+ File.expand_path(File.join(*args),
101
+ File.join(__dir__, '..', '..', 'test', 'data'))
102
+ end
103
+
97
104
  def create_bootstrap
98
105
  ws_create
99
106
  end
@@ -170,7 +177,7 @@ gem 'autobuild', path: '#{autobuild_dir}'
170
177
  'PACKAGE_BASE_DIR' => package_base_dir,
171
178
  'RUBY' => Gem.ruby
172
179
  ]
173
- result = Bundler.clean_system(
180
+ result = Autoproj.bundler_unbundled_system(
174
181
  default_env.merge(env), script, *arguments,
175
182
  in: :close, **Hash[chdir: dir].merge(system_options))
176
183
  end
@@ -201,7 +208,7 @@ gem 'autobuild', path: '#{autobuild_dir}'
201
208
  cached_bundler_gem = File.join(vendor, bundler_filename)
202
209
  unless File.file?(cached_bundler_gem)
203
210
  FileUtils.mkdir_p vendor
204
- Bundler.clean_system(
211
+ Autoproj.bundler_unbundled_system(
205
212
  Ops::Install.guess_gem_program, 'fetch', '-v',
206
213
  Bundler::VERSION, 'bundler', chdir: vendor)
207
214
 
@@ -211,7 +218,7 @@ gem 'autobuild', path: '#{autobuild_dir}'
211
218
  end
212
219
 
213
220
  capture_subprocess_io do
214
- Bundler.clean_system(
221
+ Autoproj.bundler_unbundled_system(
215
222
  Hash['GEM_HOME' => fixture_gem_home, 'GEM_PATH' => nil],
216
223
  Ops::Install.guess_gem_program, 'install', '--no-document',
217
224
  cached_bundler_gem)
@@ -256,7 +263,7 @@ gem 'autobuild', path: '#{autobuild_dir}'
256
263
 
257
264
  def find_bundled_gem_path(bundler, gem_name, gemfile)
258
265
  out_r, out_w = IO.pipe
259
- result = Bundler.clean_system(
266
+ result = Autoproj.bundler_unbundled_system(
260
267
  bundler, 'show', gem_name,
261
268
  out: out_w,
262
269
  chdir: File.dirname(gemfile))
@@ -326,8 +333,9 @@ gem 'autobuild', path: '#{autobuild_dir}'
326
333
  ws.manifest.clear_layout
327
334
  end
328
335
 
329
- def ws_define_package_set(name, vcs = VCSDefinition.from_raw(type: 'none'),
330
- **options)
336
+ def ws_define_package_set(
337
+ name, vcs = VCSDefinition.from_raw({ type: 'none' }), **options
338
+ )
331
339
  package_set = PackageSet.new(ws, vcs, name: name, **options)
332
340
  ws.manifest.register_package_set(package_set)
333
341
  package_set
@@ -344,8 +352,9 @@ gem 'autobuild', path: '#{autobuild_dir}'
344
352
  package_set
345
353
  end
346
354
 
347
- def ws_add_package_set_to_layout(name, vcs = VCSDefinition.from_raw(type: 'none'),
348
- **options)
355
+ def ws_add_package_set_to_layout(
356
+ name, vcs = VCSDefinition.from_raw({ type: 'none' }), **options
357
+ )
349
358
  package_set = ws_define_package_set(name, vcs, **options)
350
359
  ws.manifest.add_package_set_to_layout(package_set)
351
360
  package_set
@@ -370,9 +379,16 @@ gem 'autobuild', path: '#{autobuild_dir}'
370
379
 
371
380
  def ws_define_package(package_type, package_name,
372
381
  package_set: ws.manifest.main_package_set,
373
- create: true)
382
+ create: true, &block)
374
383
  package = Autobuild.send(package_type, package_name)
375
- package.srcdir = File.join(ws.root_dir, package_name.to_s)
384
+ ws_setup_package(
385
+ package, package_set: package_set, create: create, &block
386
+ )
387
+ end
388
+
389
+ def ws_setup_package(package, package_set: ws.manifest.main_package_set,
390
+ create: true)
391
+ package.srcdir = File.join(ws.root_dir, package.name.to_s)
376
392
  FileUtils.mkdir_p package.srcdir if create
377
393
  autoproj_package = ws.register_package(package, nil, package_set)
378
394
  yield(package) if block_given?
@@ -80,7 +80,9 @@ module Autoproj
80
80
  end
81
81
 
82
82
  # True if the given string contains expansions
83
- def self.contains_expansion?(string); string =~ /\$/ end
83
+ def self.contains_expansion?(string)
84
+ string.respond_to?(:to_str) && string.to_str =~ /\$/
85
+ end
84
86
 
85
87
  def self.resolve_one_constant(name, value, result, definitions)
86
88
  result[name] ||= single_expansion(value, result) do |missing_name|
@@ -33,7 +33,7 @@ module Autoproj
33
33
  HistoryEntry = Struct.new :package_set, :vcs
34
34
  RawEntry = Struct.new :package_set, :file, :vcs
35
35
 
36
- def initialize(type, url, vcs_options, from: nil, raw: Array.new, history: Array.new)
36
+ def initialize(type, url, vcs_options, from: nil, raw: [], history: [])
37
37
  if !raw.respond_to?(:to_ary)
38
38
  raise ArgumentError, "wrong format for the raw field (#{raw.inspect})"
39
39
  end
@@ -49,7 +49,7 @@ module Autoproj
49
49
 
50
50
  # Create a null VCS object
51
51
  def self.none
52
- from_raw(type: 'none')
52
+ from_raw({ type: 'none' })
53
53
  end
54
54
 
55
55
  # Whether there is actually a version control definition
@@ -151,10 +151,14 @@ module Autoproj
151
151
  elsif base_dir
152
152
  File.expand_path(short_url, base_dir)
153
153
  else
154
- raise ArgumentError, "VCS path '#{short_url}' is relative and no base_dir was given"
154
+ raise ArgumentError,
155
+ "VCS path '#{short_url}' is relative and no "\
156
+ "base_dir was given"
155
157
  end
156
158
  if !File.directory?(source_dir)
157
- raise ArgumentError, "'#{short_url}' is neither a remote source specification, nor an existing local directory"
159
+ raise ArgumentError,
160
+ "'#{short_url}' is neither a remote source "\
161
+ 'specification, nor an existing local directory'
158
162
  end
159
163
  spec.merge!(type: 'local', url: source_dir)
160
164
  end
@@ -195,17 +199,26 @@ module Autoproj
195
199
  # @return [VCSDefinition]
196
200
  # @raise ArgumentError if the raw specification does not match an
197
201
  # expected format
198
- def self.from_raw(spec, from: nil, raw: Array.new, history: Array.new)
202
+ def self.from_raw(spec, from: nil, raw: [], history: [])
199
203
  normalized_spec = normalize_vcs_hash(spec)
200
204
  if !(type = normalized_spec.delete(:type))
201
- raise ArgumentError, "the source specification #{raw_spec_to_s(spec)} normalizes into #{raw_spec_to_s(normalized_spec)}, which does not have a VCS type"
202
- elsif !(url = normalized_spec.delete(:url))
203
- if type != 'none'
204
- raise ArgumentError, "the source specification #{raw_spec_to_s(spec)} normalizes into #{raw_spec_to_s(normalized_spec)}, which does not have a URL. Only VCS of type 'none' do not require one"
205
- end
205
+ raise ArgumentError,
206
+ "the source specification #{raw_spec_to_s(spec)} normalizes "\
207
+ "into #{raw_spec_to_s(normalized_spec)}, "\
208
+ 'which does not have a VCS type'
209
+ end
210
+
211
+ if !(url = normalized_spec.delete(:url)) && type != 'none'
212
+ raise ArgumentError,
213
+ "the source specification #{raw_spec_to_s(spec)} normalizes "\
214
+ "into #{raw_spec_to_s(normalized_spec)}, "\
215
+ 'which does not have a URL. '\
216
+ 'Only VCS of type \'none\' do not require one'
206
217
  end
207
218
 
208
- VCSDefinition.new(type, url, normalized_spec, from: from, history: history, raw: raw)
219
+ VCSDefinition.new(
220
+ type, url, normalized_spec, from: from, history: history, raw: raw
221
+ )
209
222
  end
210
223
 
211
224
  def ==(other_vcs)
@@ -307,7 +320,7 @@ module Autoproj
307
320
  if !handler
308
321
  raise ArgumentError, "there is no source handler for #{vcs}"
309
322
  else
310
- return handler.call(url, options)
323
+ handler.call(url, **options)
311
324
  end
312
325
  end
313
326