autoproj 2.0.0.rc37 → 2.0.0.rc38

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +4 -2
  3. data/Rakefile +1 -1
  4. data/bin/autoproj_bootstrap +34 -2
  5. data/bin/autoproj_bootstrap.in +4 -2
  6. data/bin/autoproj_install +34 -2
  7. data/bin/autoproj_install.in +4 -2
  8. data/lib/autoproj.rb +9 -2
  9. data/lib/autoproj/autobuild.rb +13 -742
  10. data/lib/autoproj/autobuild_extensions/archive_importer.rb +44 -0
  11. data/lib/autoproj/autobuild_extensions/dsl.rb +439 -0
  12. data/lib/autoproj/autobuild_extensions/git.rb +116 -0
  13. data/lib/autoproj/autobuild_extensions/package.rb +159 -0
  14. data/lib/autoproj/autobuild_extensions/svn.rb +11 -0
  15. data/lib/autoproj/cli/base.rb +17 -18
  16. data/lib/autoproj/cli/clean.rb +1 -2
  17. data/lib/autoproj/cli/envsh.rb +1 -2
  18. data/lib/autoproj/cli/inspection_tool.rb +12 -21
  19. data/lib/autoproj/cli/locate.rb +130 -73
  20. data/lib/autoproj/cli/main.rb +31 -5
  21. data/lib/autoproj/cli/main_plugin.rb +79 -0
  22. data/lib/autoproj/cli/main_test.rb +19 -5
  23. data/lib/autoproj/cli/osdeps.rb +1 -2
  24. data/lib/autoproj/cli/patcher.rb +21 -0
  25. data/lib/autoproj/cli/query.rb +34 -41
  26. data/lib/autoproj/cli/show.rb +121 -52
  27. data/lib/autoproj/cli/status.rb +4 -5
  28. data/lib/autoproj/cli/tag.rb +1 -1
  29. data/lib/autoproj/cli/test.rb +7 -6
  30. data/lib/autoproj/cli/update.rb +8 -22
  31. data/lib/autoproj/cli/versions.rb +1 -2
  32. data/lib/autoproj/configuration.rb +1 -1
  33. data/lib/autoproj/environment.rb +2 -7
  34. data/lib/autoproj/exceptions.rb +10 -8
  35. data/lib/autoproj/find_workspace.rb +46 -12
  36. data/lib/autoproj/installation_manifest.rb +34 -25
  37. data/lib/autoproj/local_package_set.rb +86 -0
  38. data/lib/autoproj/manifest.rb +448 -503
  39. data/lib/autoproj/metapackage.rb +31 -5
  40. data/lib/autoproj/ops/configuration.rb +46 -45
  41. data/lib/autoproj/ops/import.rb +150 -60
  42. data/lib/autoproj/ops/install.rb +25 -1
  43. data/lib/autoproj/ops/loader.rb +4 -1
  44. data/lib/autoproj/ops/main_config_switcher.rb +4 -4
  45. data/lib/autoproj/ops/snapshot.rb +4 -3
  46. data/lib/autoproj/os_package_installer.rb +105 -46
  47. data/lib/autoproj/os_package_resolver.rb +63 -36
  48. data/lib/autoproj/package_definition.rb +1 -0
  49. data/lib/autoproj/package_managers/apt_dpkg_manager.rb +30 -27
  50. data/lib/autoproj/package_managers/bundler_manager.rb +64 -18
  51. data/lib/autoproj/package_managers/gem_manager.rb +4 -2
  52. data/lib/autoproj/package_managers/manager.rb +26 -7
  53. data/lib/autoproj/package_managers/shell_script_manager.rb +4 -4
  54. data/lib/autoproj/package_managers/zypper_manager.rb +1 -1
  55. data/lib/autoproj/package_manifest.rb +154 -137
  56. data/lib/autoproj/package_selection.rb +16 -2
  57. data/lib/autoproj/package_set.rb +352 -309
  58. data/lib/autoproj/query.rb +13 -1
  59. data/lib/autoproj/system.rb +2 -2
  60. data/lib/autoproj/test.rb +164 -11
  61. data/lib/autoproj/variable_expansion.rb +15 -42
  62. data/lib/autoproj/vcs_definition.rb +93 -76
  63. data/lib/autoproj/version.rb +1 -1
  64. data/lib/autoproj/workspace.rb +116 -80
  65. metadata +10 -2
@@ -4,7 +4,7 @@ class Metapackage
4
4
  # The metapackage name
5
5
  attr_reader :name
6
6
  # The packages listed in this metapackage
7
- attr_reader :packages
7
+ attr_reader :packages_by_name
8
8
  # The normal dependency handling behaviour is to generate an error if a
9
9
  # metapackage is selected for the build but some of its dependencies
10
10
  # cannot be built. This modifies the behaviour to simply ignore the
@@ -20,22 +20,40 @@ def weak_dependencies?
20
20
 
21
21
  def initialize(name)
22
22
  @name = name
23
- @packages = []
23
+ @packages_by_name = Hash.new
24
24
  @weak_dependencies = false
25
25
  end
26
26
 
27
+ def size
28
+ packages_by_name.size
29
+ end
30
+
31
+ # Deprecated, use #each_package instead
32
+ def packages
33
+ Autoproj.warn_deprecated "use #each_package instead"
34
+ each_package.to_a
35
+ end
36
+
27
37
  # Adds a package to this metapackage
28
38
  #
29
39
  # @param [Autobuild::Package] pkg
30
40
  def add(pkg)
31
- @packages << pkg
41
+ packages_by_name[pkg.name] = pkg
42
+ end
43
+
44
+ # Remove a package from this metapackage
45
+ def remove(pkg)
46
+ if pkg.respond_to?(:name)
47
+ pkg = pkg.name
48
+ end
49
+ packages_by_name.delete(pkg)
32
50
  end
33
51
 
34
52
  # Lists the packages contained in this metapackage
35
53
  #
36
54
  # @yieldparam [Autobuild::Package] pkg
37
55
  def each_package(&block)
38
- @packages.each(&block)
56
+ packages_by_name.each_value(&block)
39
57
  end
40
58
 
41
59
  # Tests if the given package is included in this metapackage
@@ -45,7 +63,15 @@ def include?(pkg)
45
63
  if !pkg.respond_to?(:to_str)
46
64
  pkg = pkg.name
47
65
  end
48
- @packages.any? { |p| p.name == pkg }
66
+ packages_by_name.has_key?(pkg)
67
+ end
68
+
69
+ def clear
70
+ packages_by_name.clear
71
+ end
72
+
73
+ def delete_if
74
+ packages_by_name.delete_if { |name, package| yield(package) }
49
75
  end
50
76
  end
51
77
  end
@@ -41,11 +41,9 @@ def manifest_path
41
41
  # @option options [InstallationManifest] :update_from
42
42
  # another autoproj installation from which we
43
43
  # should update (instead of the normal VCS)
44
- def initialize(workspace, options = Hash.new)
45
- options = validate_options options,
46
- update_from: nil
44
+ def initialize(workspace, update_from: nil)
47
45
  @ws = workspace
48
- @update_from = options[:update_from]
46
+ @update_from = update_from
49
47
  @remote_update_message_displayed = false
50
48
  end
51
49
 
@@ -129,8 +127,8 @@ def update_remote_package_set(vcs, options = Hash.new)
129
127
  reset: false,
130
128
  retry_count: nil
131
129
 
132
- name = PackageSet.name_of(ws.manifest, vcs)
133
- raw_local_dir = PackageSet.raw_local_dir_of(vcs)
130
+ name = PackageSet.name_of(ws, vcs)
131
+ raw_local_dir = PackageSet.raw_local_dir_of(ws, vcs)
134
132
 
135
133
  return if options[:checkout_only] && File.exist?(raw_local_dir)
136
134
 
@@ -140,7 +138,7 @@ def update_remote_package_set(vcs, options = Hash.new)
140
138
  Autoproj.message("autoproj: updating remote definitions of package sets", :bold)
141
139
  @remote_update_message_displayed = true
142
140
  end
143
- ws.install_os_packages([vcs.type])
141
+ ws.install_os_packages([vcs.type], all: nil)
144
142
  update_configuration_repository(
145
143
  vcs, name, raw_local_dir, options)
146
144
  end
@@ -150,8 +148,8 @@ def update_remote_package_set(vcs, options = Hash.new)
150
148
  # @param [VCSDefinition] vcs the package set VCS
151
149
  # @return [String] the full path to the created user dir
152
150
  def create_remote_set_user_dir(vcs)
153
- name = PackageSet.name_of(ws.manifest, vcs)
154
- raw_local_dir = PackageSet.raw_local_dir_of(vcs)
151
+ name = PackageSet.name_of(ws, vcs)
152
+ raw_local_dir = PackageSet.raw_local_dir_of(ws, vcs)
155
153
  FileUtils.mkdir_p(remotes_user_dir)
156
154
  symlink_dest = File.join(remotes_user_dir, name)
157
155
 
@@ -172,7 +170,7 @@ def create_remote_set_user_dir(vcs)
172
170
  end
173
171
 
174
172
  def load_package_set(vcs, options, imported_from)
175
- pkg_set = PackageSet.new(ws.manifest, vcs)
173
+ pkg_set = PackageSet.new(ws, vcs)
176
174
  pkg_set.auto_imports = options[:auto_imports]
177
175
  ws.load_if_present(pkg_set, pkg_set.local_dir, 'init.rb')
178
176
  pkg_set.load_description_file
@@ -186,22 +184,14 @@ def load_package_set(vcs, options, imported_from)
186
184
  def queue_auto_imports_if_needed(queue, pkg_set, root_set)
187
185
  if pkg_set.auto_imports?
188
186
  pkg_set.each_raw_imported_set do |import_vcs, import_options|
189
- repository_id = repository_id_of(import_vcs)
190
- import_vcs = root_set.overrides_for("pkg_set:#{repository_id}", import_vcs)
187
+ vcs_overrides_key = import_vcs.overrides_key
188
+ import_vcs = root_set.resolve_overrides("pkg_set:#{vcs_overrides_key}", import_vcs)
191
189
  queue << [import_vcs, import_options, pkg_set]
192
190
  end
193
191
  end
194
192
  queue
195
193
  end
196
194
 
197
- def repository_id_of(vcs)
198
- if vcs.local?
199
- return "local:#{vcs.url}"
200
- end
201
-
202
- vcs.create_autobuild_importer.repository_id
203
- end
204
-
205
195
  # Load the package set information
206
196
  #
207
197
  # It loads the package set information as required by {manifest} and
@@ -230,7 +220,7 @@ def load_and_update_package_sets(root_pkg_set, options = Hash.new)
230
220
  queue = queue_auto_imports_if_needed(Array.new, root_pkg_set, root_pkg_set)
231
221
  while !queue.empty?
232
222
  vcs, import_options, imported_from = queue.shift
233
- repository_id = repository_id_of(vcs)
223
+ repository_id = vcs.overrides_key
234
224
  if already_processed = by_repository_id[repository_id]
235
225
  already_processed_vcs, already_processed_from, pkg_set = *already_processed
236
226
  if (already_processed_from != root_pkg_set) && (already_processed_vcs != vcs)
@@ -249,12 +239,10 @@ def load_and_update_package_sets(root_pkg_set, options = Hash.new)
249
239
  # retrieve the actual name of the package set
250
240
  if !vcs.local?
251
241
  update_remote_package_set(vcs, options)
252
- create_remote_set_user_dir(vcs)
253
- raw_local_dir = PackageSet.raw_local_dir_of(vcs)
254
- required_remotes_dirs << raw_local_dir
242
+ required_remotes_dirs << PackageSet.raw_local_dir_of(ws, vcs)
255
243
  end
256
244
 
257
- name = PackageSet.name_of(ws.manifest, vcs)
245
+ name = PackageSet.name_of(ws, vcs)
258
246
 
259
247
  required_user_dirs = by_name.collect { |k,v| k }
260
248
  Autoproj.debug "Trying to load package_set: #{name} from definition #{repository_id}"
@@ -264,8 +252,8 @@ def load_and_update_package_sets(root_pkg_set, options = Hash.new)
264
252
  already_loaded_pkg_set, already_loaded_vcs = *already_loaded
265
253
  if already_loaded_vcs != vcs
266
254
  if imported_from
267
- Autoproj.warn "redundant auto-import by #{imported_from.name} for package set '#{name}'."
268
- Autoproj.warn " A package set with the same name (#{name}) has already been imported from"
255
+ Autoproj.warn "redundant auto-import of package set '#{name}' by package set '#{imported_from.name}'"
256
+ Autoproj.warn " A package set with the same name has already been imported from"
269
257
  Autoproj.warn " #{already_loaded_vcs}"
270
258
  Autoproj.warn " Skipping the following one: "
271
259
  Autoproj.warn " #{vcs}"
@@ -277,12 +265,14 @@ def load_and_update_package_sets(root_pkg_set, options = Hash.new)
277
265
  if imported_from
278
266
  already_loaded_pkg_set.imported_from << imported_from
279
267
  imported_from.imports << already_loaded_pkg_set
268
+ by_repository_id[repository_id][2] = already_loaded_pkg_set
280
269
  end
281
270
  next
282
- else
283
- create_remote_set_user_dir(vcs)
284
271
  end
285
272
 
273
+ if !vcs.local?
274
+ create_remote_set_user_dir(vcs)
275
+ end
286
276
  pkg_set = load_package_set(vcs, import_options, imported_from)
287
277
  by_repository_id[repository_id][2] = pkg_set
288
278
  package_sets << pkg_set
@@ -335,7 +325,7 @@ def sort_package_sets_by_import_order(package_sets, root_pkg_set)
335
325
  # considered in turn, and added at the earliest place that fits
336
326
  # the dependencies
337
327
  topological = Array.new
338
- queue = package_sets.to_a
328
+ queue = (package_sets.to_a + [root_pkg_set]).uniq
339
329
  while !queue.empty?
340
330
  last_size = queue.size
341
331
  pending = queue.dup
@@ -355,7 +345,7 @@ def sort_package_sets_by_import_order(package_sets, root_pkg_set)
355
345
 
356
346
  result = root_pkg_set.imports.to_a.dup
357
347
  to_insert = topological.dup.
358
- find_all { |pkg_set| !result.include?(pkg_set) }
348
+ find_all { |p| !result.include?(p) }
359
349
  while !to_insert.empty?
360
350
  pkg_set = to_insert.shift
361
351
  dependencies = pkg_set.imports.dup
@@ -369,7 +359,13 @@ def sort_package_sets_by_import_order(package_sets, root_pkg_set)
369
359
  result.insert(i + 1, pkg_set)
370
360
  end
371
361
  end
372
- result << root_pkg_set
362
+
363
+ # Sanity check related to the root package set
364
+ # - it should be last
365
+ # - it should be present only once
366
+ if result.last != root_pkg_set
367
+ raise InternalError, "failed to sort the package sets: the root package set should be last, but is not"
368
+ end
373
369
  result
374
370
  end
375
371
 
@@ -383,34 +379,39 @@ def load_package_sets(options = Hash.new)
383
379
  update_configuration(options)
384
380
  end
385
381
 
386
- def update_configuration(options = Hash.new)
387
- if !options.kind_of?(Hash)
388
- options = Hash[only_local: options]
389
- end
390
- options = validate_options options,
382
+ def update_configuration(
391
383
  only_local: false,
392
384
  checkout_only: !Autobuild.do_update,
393
385
  ignore_errors: false,
394
386
  reset: false,
395
- retry_count: nil
387
+ retry_count: nil)
388
+
389
+ update_options = Hash[
390
+ only_local: only_local,
391
+ checkout_only: checkout_only,
392
+ ignore_errors: ignore_errors,
393
+ reset: reset,
394
+ retry_count: retry_count]
396
395
 
397
- # Load the installation's manifest a first time, to check if we should
398
- # update it ... We assume that the OS dependencies for this VCS is already
399
- # installed (i.e. that the user did not remove it)
400
396
  if ws.manifest.vcs && !ws.manifest.vcs.local?
401
- update_main_configuration(options)
397
+ update_main_configuration(**update_options)
402
398
  end
403
399
  ws.load_main_initrb
404
400
  ws.manifest.load(manifest_path)
405
-
406
- root_pkg_set = ws.manifest.local_package_set
401
+ root_pkg_set = ws.manifest.main_package_set
407
402
  root_pkg_set.load_description_file
408
403
  root_pkg_set.explicit = true
409
- package_sets = load_and_update_package_sets(root_pkg_set, options)
404
+ update_package_sets(**update_options)
405
+ end
406
+
407
+ def update_package_sets(**update_options)
408
+ root_pkg_set = ws.manifest.main_package_set
409
+ package_sets = load_and_update_package_sets(root_pkg_set, **update_options)
410
410
  root_pkg_set.imports.each do |pkg_set|
411
411
  pkg_set.explicit = true
412
412
  end
413
413
  package_sets = sort_package_sets_by_import_order(package_sets, root_pkg_set)
414
+ ws.manifest.reset_package_sets
414
415
  package_sets.each do |pkg_set|
415
416
  ws.manifest.register_package_set(pkg_set)
416
417
  end
@@ -13,9 +13,9 @@ def mark_exclusion_along_revdeps(pkg_name, revdeps, chain = [], reason = nil)
13
13
  reason = ws.manifest.exclusion_reason(pkg_name)
14
14
  else
15
15
  if chain.size == 1
16
- ws.manifest.add_exclusion(pkg_name, "its dependency #{reason}")
16
+ ws.manifest.exclude_package(pkg_name, "its dependency #{reason}")
17
17
  else
18
- ws.manifest.add_exclusion(pkg_name, "#{reason} (dependency chain: #{chain.join(">")})")
18
+ ws.manifest.exclude_package(pkg_name, "#{reason} (dependency chain: #{chain.join(">")})")
19
19
  end
20
20
  end
21
21
 
@@ -34,11 +34,11 @@ def import_next_step(pkg, reverse_dependencies)
34
34
  new_packages = []
35
35
  pkg.dependencies.each do |dep_name|
36
36
  reverse_dependencies[dep_name] << pkg.name
37
- new_packages << ws.manifest.find_autobuild_package(dep_name)
37
+ new_packages << ws.manifest.find_package_definition(dep_name)
38
38
  end
39
39
  pkg_opt_deps, pkg_opt_os_deps = pkg.partition_optional_dependencies
40
40
  pkg_opt_deps.each do |dep_name|
41
- new_packages << ws.manifest.find_autobuild_package(dep_name)
41
+ new_packages << ws.manifest.find_package_definition(dep_name)
42
42
  end
43
43
 
44
44
  # Handle OS dependencies, excluding the package if some
@@ -82,7 +82,9 @@ def pre_package_import(selection, manifest, pkg, reverse_dependencies)
82
82
 
83
83
  def post_package_import(selection, manifest, pkg, reverse_dependencies)
84
84
  Rake::Task["#{pkg.name}-import"].instance_variable_set(:@already_invoked, true)
85
- manifest.load_package_manifest(pkg.name)
85
+ if pkg.checked_out?
86
+ manifest.load_package_manifest(pkg.name)
87
+ end
86
88
 
87
89
  # The package setup mechanisms might have added an exclusion
88
90
  # on this package. Handle this.
@@ -95,35 +97,82 @@ def post_package_import(selection, manifest, pkg, reverse_dependencies)
95
97
  elsif manifest.ignored?(pkg.name)
96
98
  false
97
99
  else
98
- Autoproj.each_post_import_block(pkg) do |block|
99
- block.call(pkg)
100
+ if pkg.checked_out?
101
+ Autoproj.each_post_import_block(pkg) do |block|
102
+ block.call(pkg)
103
+ end
100
104
  end
101
105
  import_next_step(pkg, reverse_dependencies)
102
106
  end
103
107
  end
104
108
 
109
+ # Install the VCS osdep for the given packages
110
+ #
111
+ # @param [Hash] osdeps_options the options that will be passed to
112
+ # {Workspace#install_os_packages}
113
+ # @return [Set] the set of installed OS packages
114
+ def install_vcs_packages_for(*packages, **osdeps_options)
115
+ vcs_to_install = packages.map { |pkg| pkg.vcs.type }.uniq
116
+ # This assumes that the VCS packages do not depend on a
117
+ # 'strict' package mangers such as e.g. BundlerManager
118
+ ws.install_os_packages(vcs_to_install, all: nil, **osdeps_options)
119
+ vcs_to_install
120
+ end
121
+
122
+ # @api private
123
+ #
124
+ # Queue the work necessary to import the given package, making sure
125
+ # that the execution results end up in a given queue
126
+ #
127
+ # @param executor the future executor
128
+ # @param [Queue] completion_queue the queue where the completion
129
+ # results should be pushed, as a (package, time, result,
130
+ # error_reason) tuple
131
+ # @param [Integer] retry_count the number of retries that are
132
+ # allowed. Set to zero for no retry
133
+ # @param [Hash] import_options options passed to {Autobuild::Importer#import}
134
+ def queue_import_work(executor, completion_queue, pkg, retry_count: nil, **import_options)
135
+ import_future = Concurrent::Future.new(executor: executor, args: [pkg]) do |import_pkg|
136
+ ## COMPLETELY BYPASS RAKE HERE
137
+ # The reason is that the ordering of import/prepare between
138
+ # packages is not important BUT the ordering of import vs.
139
+ # prepare in one package IS important: prepare is the method
140
+ # that takes into account dependencies.
141
+ if retry_count
142
+ import_pkg.autobuild.importer.retry_count = retry_count
143
+ end
144
+ import_pkg.autobuild.import(**import_options)
145
+ end
146
+ import_future.add_observer do |time, result, reason|
147
+ completion_queue << [pkg, time, result, reason]
148
+ end
149
+ import_future.execute
150
+ end
151
+
105
152
  class ImportFailed < RuntimeError; end
106
153
 
107
154
  # Import all packages from the given selection, and their
108
155
  # dependencies
109
- def import_selected_packages(selection, updated_packages, options = Hash.new)
110
- parallel_options, options = Kernel.filter_options options,
111
- parallel: ws.config.parallel_import_level
156
+ def import_selected_packages(selection, updated_packages,
157
+ parallel: ws.config.parallel_import_level,
158
+ recursive: true,
159
+ retry_count: nil,
160
+ ignore_errors: false,
161
+ install_vcs_packages: Hash.new,
162
+ non_imported_packages: :checkout,
163
+ **import_options)
164
+
165
+ if ![:checkout, :ignore, :return].include?(non_imported_packages)
166
+ raise ArgumentError, "invalid value for 'non_imported_packages'. Expected one of :checkout, :ignore or :return but got #{non_imported_packages}"
167
+ end
112
168
 
113
169
  # This is used in the ensure block, initialize as early as
114
170
  # possible
115
- executor = Concurrent::FixedThreadPool.new(parallel_options[:parallel], max_length: 0)
116
-
117
- options, import_options = Kernel.filter_options options,
118
- recursive: true,
119
- retry_count: nil
120
-
121
- ignore_errors = options[:ignore_errors]
122
- retry_count = options[:retry_count]
171
+ executor = Concurrent::FixedThreadPool.new(parallel, max_length: 0)
123
172
  manifest = ws.manifest
124
173
 
125
174
  selected_packages = selection.each_source_package_name.map do |pkg_name|
126
- manifest.find_autobuild_package(pkg_name)
175
+ manifest.find_package_definition(pkg_name)
127
176
  end.to_set
128
177
 
129
178
  # The reverse dependencies for the package tree. It is discovered as
@@ -138,69 +187,96 @@ def import_selected_packages(selection, updated_packages, options = Hash.new)
138
187
  pending_packages = Set.new
139
188
  # The set of all packages that are currently selected by +selection+
140
189
  all_processed_packages = Set.new
141
- interactive_imports = Array.new
190
+ main_thread_imports = Array.new
142
191
  package_queue = selected_packages.to_a.sort_by(&:name)
143
192
  failures = Hash.new
193
+ missing_vcs = Array.new
194
+ installed_vcs_packages = Set['none', 'local']
144
195
  while failures.empty? || ignore_errors
145
196
  # Queue work for all packages in the queue
146
197
  package_queue.each do |pkg|
147
198
  # Remove packages that have already been processed
148
199
  next if all_processed_packages.include?(pkg)
200
+ if (non_imported_packages != :checkout) && !File.directory?(pkg.autobuild.srcdir)
201
+ if non_imported_packages == :return
202
+ all_processed_packages << pkg
203
+ completion_queue << [pkg, Time.now, false, nil]
204
+ next
205
+ else
206
+ all_processed_packages << pkg
207
+ ws.manifest.ignore_package(pkg.name)
208
+ next
209
+ end
210
+ elsif install_vcs_packages && !installed_vcs_packages.include?(pkg.vcs.type)
211
+ missing_vcs << pkg
212
+ next
213
+ end
149
214
  all_processed_packages << pkg
150
215
 
151
- if !pre_package_import(selection, manifest, pkg, reverse_dependencies)
216
+ importer = pkg.autobuild.importer
217
+ if !pre_package_import(selection, manifest, pkg.autobuild, reverse_dependencies)
152
218
  next
153
- elsif !pkg.importer
219
+ elsif !importer
220
+ # The validity of this is checked in
221
+ # pre_package_import
154
222
  completion_queue << [pkg, Time.now, false, nil]
155
223
  next
156
- elsif pkg.importer.interactive?
157
- interactive_imports << pkg
224
+ elsif importer.interactive?
225
+ main_thread_imports << pkg
226
+ next
227
+ elsif pkg.autobuild.checked_out? && import_options[:checkout_only]
228
+ main_thread_imports << pkg
158
229
  next
159
230
  end
160
231
 
161
232
  pending_packages << pkg
162
- import_future = Concurrent::Future.new(executor: executor, args: [pkg]) do |import_pkg|
163
- ## COMPLETELY BYPASS RAKE HERE
164
- # The reason is that the ordering of import/prepare between
165
- # packages is not important BUT the ordering of import vs.
166
- # prepare in one package IS important: prepare is the method
167
- # that takes into account dependencies.
168
- if retry_count
169
- import_pkg.importer.retry_count = retry_count
170
- end
171
- import_pkg.import(import_options.merge(allow_interactive: false))
172
- end
173
- import_future.add_observer do |time, result, reason|
174
- completion_queue << [pkg, time, result, reason]
233
+ begin
234
+ queue_import_work(
235
+ executor, completion_queue, pkg, retry_count: retry_count,
236
+ **import_options.merge(allow_interactive: false))
237
+ rescue Exception
238
+ pending_packages.delete(pkg)
239
+ raise
175
240
  end
176
- import_future.execute
241
+ true
177
242
  end
178
243
  package_queue.clear
179
244
 
180
245
  if completion_queue.empty? && pending_packages.empty?
246
+ if !missing_vcs.empty?
247
+ installed_vcs_packages.merge(
248
+ install_vcs_packages_for(*missing_vcs, **install_vcs_packages))
249
+ package_queue.concat(missing_vcs)
250
+ missing_vcs.clear
251
+ next
252
+ end
253
+
181
254
  # We've nothing to process anymore ... process
182
255
  # interactive imports if there are some. Otherwise,
183
256
  # we're done
184
- if interactive_imports.empty?
257
+ if main_thread_imports.empty?
185
258
  break
186
259
  else
187
- interactive_imports.each do |pkg|
260
+ main_thread_imports.delete_if do |pkg|
188
261
  begin
189
- result = pkg.import(import_options.merge(allow_interactive: true))
262
+ if retry_count
263
+ pkg.autobuild.importer.retry_count = retry_count
264
+ end
265
+ result = pkg.autobuild.import(
266
+ **import_options.merge(allow_interactive: true))
190
267
  rescue Exception => reason
191
268
  end
192
269
  completion_queue << [pkg, Time.now, result, reason]
193
270
  end
194
- interactive_imports.clear
195
271
  end
196
272
  end
197
273
 
198
- # And wait one to finish
274
+ # And wait for one to finish
199
275
  pkg, _time, _result, reason = completion_queue.pop
200
276
  pending_packages.delete(pkg)
201
277
  if reason
202
278
  if reason.kind_of?(Autobuild::InteractionRequired)
203
- interactive_imports << pkg
279
+ main_thread_imports << pkg
204
280
  else
205
281
  # One importer failed... terminate
206
282
  Autoproj.error "import of #{pkg.name} failed"
@@ -210,13 +286,13 @@ def import_selected_packages(selection, updated_packages, options = Hash.new)
210
286
  failures[pkg] = reason
211
287
  end
212
288
  else
213
- if new_packages = post_package_import(selection, manifest, pkg, reverse_dependencies)
289
+ if new_packages = post_package_import(selection, manifest, pkg.autobuild, reverse_dependencies)
214
290
  # Excluded dependencies might have caused the package to be
215
291
  # excluded as well ... do not add any dependency to the
216
292
  # processing queue if it is the case
217
293
  if manifest.excluded?(pkg.name)
218
294
  selection.filter_excluded_and_ignored_packages(manifest)
219
- elsif options[:recursive]
295
+ elsif recursive
220
296
  package_queue = new_packages.sort_by(&:name)
221
297
  end
222
298
  end
@@ -240,7 +316,12 @@ def import_selected_packages(selection, updated_packages, options = Hash.new)
240
316
  executor.shutdown
241
317
  executor.wait_for_termination
242
318
  end
243
- updated_packages.concat(all_processed_packages.find_all(&:updated?).map(&:name))
319
+ if all_processed_packages
320
+ all_updated_packages = all_processed_packages.find_all do |processed_pkg|
321
+ processed_pkg.autobuild.updated?
322
+ end
323
+ updated_packages.concat(all_updated_packages.map(&:name))
324
+ end
244
325
  end
245
326
 
246
327
  def finalize_package_load(processed_packages)
@@ -256,8 +337,9 @@ def finalize_package_load(processed_packages)
256
337
 
257
338
  next if manifest.ignored?(pkg_name) || manifest.excluded?(pkg_name)
258
339
 
259
- pkg = manifest.find_autobuild_package(pkg_name)
260
- if !processed_packages.include?(pkg) && File.directory?(pkg.srcdir)
340
+ pkg_definition = manifest.find_package_definition(pkg_name)
341
+ pkg = pkg_definition.autobuild
342
+ if !processed_packages.include?(pkg_definition) && File.directory?(pkg.srcdir)
261
343
  manifest.load_package_manifest(pkg.name)
262
344
  Autoproj.each_post_import_block(pkg) do |block|
263
345
  block.call(pkg)
@@ -286,37 +368,45 @@ def finalize_package_load(processed_packages)
286
368
  all
287
369
  end
288
370
 
289
- def import_packages(selection, options = Hash.new)
371
+ def import_packages(selection,
372
+ non_imported_packages: :checkout,
373
+ warn_about_ignored_packages: true,
374
+ warn_about_excluded_packages: true,
375
+ recursive: true,
376
+ ignore_errors: false,
377
+ install_vcs_packages: Hash.new,
378
+ **import_options)
379
+
290
380
  # Used in the ensure block, initialize as soon as possible
291
381
  updated_packages = Array.new
292
382
 
293
- options, import_options = Kernel.filter_options options,
294
- warn_about_ignored_packages: true,
295
- warn_about_excluded_packages: true,
296
- recursive: true
297
-
298
383
  manifest = ws.manifest
299
384
 
300
385
  all_processed_packages = import_selected_packages(
301
- selection, updated_packages, import_options.merge(recursive: options[:recursive]))
386
+ selection, updated_packages,
387
+ non_imported_packages: non_imported_packages,
388
+ ignore_errors: ignore_errors,
389
+ recursive: recursive,
390
+ install_vcs_packages: install_vcs_packages,
391
+ **import_options)
302
392
  finalize_package_load(all_processed_packages)
303
393
 
304
394
  all_enabled_osdeps = selection.each_osdep_package_name.to_set
305
395
  all_enabled_sources = all_processed_packages.map(&:name)
306
- if options[:recursive]
396
+ if recursive
307
397
  all_processed_packages.each do |pkg|
308
- all_enabled_osdeps.merge(pkg.os_packages)
398
+ all_enabled_osdeps.merge(pkg.autobuild.os_packages)
309
399
  end
310
400
  end
311
401
 
312
- if options[:warn_about_excluded_packages]
402
+ if warn_about_excluded_packages
313
403
  selection.exclusions.each do |sel, pkg_names|
314
404
  pkg_names.sort.each do |pkg_name|
315
405
  Autoproj.warn "#{pkg_name}, which was selected for #{sel}, cannot be built: #{manifest.exclusion_reason(pkg_name)}", :bold
316
406
  end
317
407
  end
318
408
  end
319
- if options[:warn_about_ignored_packages]
409
+ if warn_about_ignored_packages
320
410
  selection.ignores.each do |sel, pkg_names|
321
411
  pkg_names.sort.each do |pkg_name|
322
412
  Autoproj.warn "#{pkg_name}, which was selected for #{sel}, is ignored", :bold