autoproj 2.0.0.rc37 → 2.0.0.rc38

Sign up to get free protection for your applications and to get access to all the features.
Files changed (65) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +4 -2
  3. data/Rakefile +1 -1
  4. data/bin/autoproj_bootstrap +34 -2
  5. data/bin/autoproj_bootstrap.in +4 -2
  6. data/bin/autoproj_install +34 -2
  7. data/bin/autoproj_install.in +4 -2
  8. data/lib/autoproj.rb +9 -2
  9. data/lib/autoproj/autobuild.rb +13 -742
  10. data/lib/autoproj/autobuild_extensions/archive_importer.rb +44 -0
  11. data/lib/autoproj/autobuild_extensions/dsl.rb +439 -0
  12. data/lib/autoproj/autobuild_extensions/git.rb +116 -0
  13. data/lib/autoproj/autobuild_extensions/package.rb +159 -0
  14. data/lib/autoproj/autobuild_extensions/svn.rb +11 -0
  15. data/lib/autoproj/cli/base.rb +17 -18
  16. data/lib/autoproj/cli/clean.rb +1 -2
  17. data/lib/autoproj/cli/envsh.rb +1 -2
  18. data/lib/autoproj/cli/inspection_tool.rb +12 -21
  19. data/lib/autoproj/cli/locate.rb +130 -73
  20. data/lib/autoproj/cli/main.rb +31 -5
  21. data/lib/autoproj/cli/main_plugin.rb +79 -0
  22. data/lib/autoproj/cli/main_test.rb +19 -5
  23. data/lib/autoproj/cli/osdeps.rb +1 -2
  24. data/lib/autoproj/cli/patcher.rb +21 -0
  25. data/lib/autoproj/cli/query.rb +34 -41
  26. data/lib/autoproj/cli/show.rb +121 -52
  27. data/lib/autoproj/cli/status.rb +4 -5
  28. data/lib/autoproj/cli/tag.rb +1 -1
  29. data/lib/autoproj/cli/test.rb +7 -6
  30. data/lib/autoproj/cli/update.rb +8 -22
  31. data/lib/autoproj/cli/versions.rb +1 -2
  32. data/lib/autoproj/configuration.rb +1 -1
  33. data/lib/autoproj/environment.rb +2 -7
  34. data/lib/autoproj/exceptions.rb +10 -8
  35. data/lib/autoproj/find_workspace.rb +46 -12
  36. data/lib/autoproj/installation_manifest.rb +34 -25
  37. data/lib/autoproj/local_package_set.rb +86 -0
  38. data/lib/autoproj/manifest.rb +448 -503
  39. data/lib/autoproj/metapackage.rb +31 -5
  40. data/lib/autoproj/ops/configuration.rb +46 -45
  41. data/lib/autoproj/ops/import.rb +150 -60
  42. data/lib/autoproj/ops/install.rb +25 -1
  43. data/lib/autoproj/ops/loader.rb +4 -1
  44. data/lib/autoproj/ops/main_config_switcher.rb +4 -4
  45. data/lib/autoproj/ops/snapshot.rb +4 -3
  46. data/lib/autoproj/os_package_installer.rb +105 -46
  47. data/lib/autoproj/os_package_resolver.rb +63 -36
  48. data/lib/autoproj/package_definition.rb +1 -0
  49. data/lib/autoproj/package_managers/apt_dpkg_manager.rb +30 -27
  50. data/lib/autoproj/package_managers/bundler_manager.rb +64 -18
  51. data/lib/autoproj/package_managers/gem_manager.rb +4 -2
  52. data/lib/autoproj/package_managers/manager.rb +26 -7
  53. data/lib/autoproj/package_managers/shell_script_manager.rb +4 -4
  54. data/lib/autoproj/package_managers/zypper_manager.rb +1 -1
  55. data/lib/autoproj/package_manifest.rb +154 -137
  56. data/lib/autoproj/package_selection.rb +16 -2
  57. data/lib/autoproj/package_set.rb +352 -309
  58. data/lib/autoproj/query.rb +13 -1
  59. data/lib/autoproj/system.rb +2 -2
  60. data/lib/autoproj/test.rb +164 -11
  61. data/lib/autoproj/variable_expansion.rb +15 -42
  62. data/lib/autoproj/vcs_definition.rb +93 -76
  63. data/lib/autoproj/version.rb +1 -1
  64. data/lib/autoproj/workspace.rb +116 -80
  65. metadata +10 -2
@@ -4,7 +4,7 @@ class Metapackage
4
4
  # The metapackage name
5
5
  attr_reader :name
6
6
  # The packages listed in this metapackage
7
- attr_reader :packages
7
+ attr_reader :packages_by_name
8
8
  # The normal dependency handling behaviour is to generate an error if a
9
9
  # metapackage is selected for the build but some of its dependencies
10
10
  # cannot be built. This modifies the behaviour to simply ignore the
@@ -20,22 +20,40 @@ def weak_dependencies?
20
20
 
21
21
  def initialize(name)
22
22
  @name = name
23
- @packages = []
23
+ @packages_by_name = Hash.new
24
24
  @weak_dependencies = false
25
25
  end
26
26
 
27
+ def size
28
+ packages_by_name.size
29
+ end
30
+
31
+ # Deprecated, use #each_package instead
32
+ def packages
33
+ Autoproj.warn_deprecated "use #each_package instead"
34
+ each_package.to_a
35
+ end
36
+
27
37
  # Adds a package to this metapackage
28
38
  #
29
39
  # @param [Autobuild::Package] pkg
30
40
  def add(pkg)
31
- @packages << pkg
41
+ packages_by_name[pkg.name] = pkg
42
+ end
43
+
44
+ # Remove a package from this metapackage
45
+ def remove(pkg)
46
+ if pkg.respond_to?(:name)
47
+ pkg = pkg.name
48
+ end
49
+ packages_by_name.delete(pkg)
32
50
  end
33
51
 
34
52
  # Lists the packages contained in this metapackage
35
53
  #
36
54
  # @yieldparam [Autobuild::Package] pkg
37
55
  def each_package(&block)
38
- @packages.each(&block)
56
+ packages_by_name.each_value(&block)
39
57
  end
40
58
 
41
59
  # Tests if the given package is included in this metapackage
@@ -45,7 +63,15 @@ def include?(pkg)
45
63
  if !pkg.respond_to?(:to_str)
46
64
  pkg = pkg.name
47
65
  end
48
- @packages.any? { |p| p.name == pkg }
66
+ packages_by_name.has_key?(pkg)
67
+ end
68
+
69
+ def clear
70
+ packages_by_name.clear
71
+ end
72
+
73
+ def delete_if
74
+ packages_by_name.delete_if { |name, package| yield(package) }
49
75
  end
50
76
  end
51
77
  end
@@ -41,11 +41,9 @@ def manifest_path
41
41
  # @option options [InstallationManifest] :update_from
42
42
  # another autoproj installation from which we
43
43
  # should update (instead of the normal VCS)
44
- def initialize(workspace, options = Hash.new)
45
- options = validate_options options,
46
- update_from: nil
44
+ def initialize(workspace, update_from: nil)
47
45
  @ws = workspace
48
- @update_from = options[:update_from]
46
+ @update_from = update_from
49
47
  @remote_update_message_displayed = false
50
48
  end
51
49
 
@@ -129,8 +127,8 @@ def update_remote_package_set(vcs, options = Hash.new)
129
127
  reset: false,
130
128
  retry_count: nil
131
129
 
132
- name = PackageSet.name_of(ws.manifest, vcs)
133
- raw_local_dir = PackageSet.raw_local_dir_of(vcs)
130
+ name = PackageSet.name_of(ws, vcs)
131
+ raw_local_dir = PackageSet.raw_local_dir_of(ws, vcs)
134
132
 
135
133
  return if options[:checkout_only] && File.exist?(raw_local_dir)
136
134
 
@@ -140,7 +138,7 @@ def update_remote_package_set(vcs, options = Hash.new)
140
138
  Autoproj.message("autoproj: updating remote definitions of package sets", :bold)
141
139
  @remote_update_message_displayed = true
142
140
  end
143
- ws.install_os_packages([vcs.type])
141
+ ws.install_os_packages([vcs.type], all: nil)
144
142
  update_configuration_repository(
145
143
  vcs, name, raw_local_dir, options)
146
144
  end
@@ -150,8 +148,8 @@ def update_remote_package_set(vcs, options = Hash.new)
150
148
  # @param [VCSDefinition] vcs the package set VCS
151
149
  # @return [String] the full path to the created user dir
152
150
  def create_remote_set_user_dir(vcs)
153
- name = PackageSet.name_of(ws.manifest, vcs)
154
- raw_local_dir = PackageSet.raw_local_dir_of(vcs)
151
+ name = PackageSet.name_of(ws, vcs)
152
+ raw_local_dir = PackageSet.raw_local_dir_of(ws, vcs)
155
153
  FileUtils.mkdir_p(remotes_user_dir)
156
154
  symlink_dest = File.join(remotes_user_dir, name)
157
155
 
@@ -172,7 +170,7 @@ def create_remote_set_user_dir(vcs)
172
170
  end
173
171
 
174
172
  def load_package_set(vcs, options, imported_from)
175
- pkg_set = PackageSet.new(ws.manifest, vcs)
173
+ pkg_set = PackageSet.new(ws, vcs)
176
174
  pkg_set.auto_imports = options[:auto_imports]
177
175
  ws.load_if_present(pkg_set, pkg_set.local_dir, 'init.rb')
178
176
  pkg_set.load_description_file
@@ -186,22 +184,14 @@ def load_package_set(vcs, options, imported_from)
186
184
  def queue_auto_imports_if_needed(queue, pkg_set, root_set)
187
185
  if pkg_set.auto_imports?
188
186
  pkg_set.each_raw_imported_set do |import_vcs, import_options|
189
- repository_id = repository_id_of(import_vcs)
190
- import_vcs = root_set.overrides_for("pkg_set:#{repository_id}", import_vcs)
187
+ vcs_overrides_key = import_vcs.overrides_key
188
+ import_vcs = root_set.resolve_overrides("pkg_set:#{vcs_overrides_key}", import_vcs)
191
189
  queue << [import_vcs, import_options, pkg_set]
192
190
  end
193
191
  end
194
192
  queue
195
193
  end
196
194
 
197
- def repository_id_of(vcs)
198
- if vcs.local?
199
- return "local:#{vcs.url}"
200
- end
201
-
202
- vcs.create_autobuild_importer.repository_id
203
- end
204
-
205
195
  # Load the package set information
206
196
  #
207
197
  # It loads the package set information as required by {manifest} and
@@ -230,7 +220,7 @@ def load_and_update_package_sets(root_pkg_set, options = Hash.new)
230
220
  queue = queue_auto_imports_if_needed(Array.new, root_pkg_set, root_pkg_set)
231
221
  while !queue.empty?
232
222
  vcs, import_options, imported_from = queue.shift
233
- repository_id = repository_id_of(vcs)
223
+ repository_id = vcs.overrides_key
234
224
  if already_processed = by_repository_id[repository_id]
235
225
  already_processed_vcs, already_processed_from, pkg_set = *already_processed
236
226
  if (already_processed_from != root_pkg_set) && (already_processed_vcs != vcs)
@@ -249,12 +239,10 @@ def load_and_update_package_sets(root_pkg_set, options = Hash.new)
249
239
  # retrieve the actual name of the package set
250
240
  if !vcs.local?
251
241
  update_remote_package_set(vcs, options)
252
- create_remote_set_user_dir(vcs)
253
- raw_local_dir = PackageSet.raw_local_dir_of(vcs)
254
- required_remotes_dirs << raw_local_dir
242
+ required_remotes_dirs << PackageSet.raw_local_dir_of(ws, vcs)
255
243
  end
256
244
 
257
- name = PackageSet.name_of(ws.manifest, vcs)
245
+ name = PackageSet.name_of(ws, vcs)
258
246
 
259
247
  required_user_dirs = by_name.collect { |k,v| k }
260
248
  Autoproj.debug "Trying to load package_set: #{name} from definition #{repository_id}"
@@ -264,8 +252,8 @@ def load_and_update_package_sets(root_pkg_set, options = Hash.new)
264
252
  already_loaded_pkg_set, already_loaded_vcs = *already_loaded
265
253
  if already_loaded_vcs != vcs
266
254
  if imported_from
267
- Autoproj.warn "redundant auto-import by #{imported_from.name} for package set '#{name}'."
268
- Autoproj.warn " A package set with the same name (#{name}) has already been imported from"
255
+ Autoproj.warn "redundant auto-import of package set '#{name}' by package set '#{imported_from.name}'"
256
+ Autoproj.warn " A package set with the same name has already been imported from"
269
257
  Autoproj.warn " #{already_loaded_vcs}"
270
258
  Autoproj.warn " Skipping the following one: "
271
259
  Autoproj.warn " #{vcs}"
@@ -277,12 +265,14 @@ def load_and_update_package_sets(root_pkg_set, options = Hash.new)
277
265
  if imported_from
278
266
  already_loaded_pkg_set.imported_from << imported_from
279
267
  imported_from.imports << already_loaded_pkg_set
268
+ by_repository_id[repository_id][2] = already_loaded_pkg_set
280
269
  end
281
270
  next
282
- else
283
- create_remote_set_user_dir(vcs)
284
271
  end
285
272
 
273
+ if !vcs.local?
274
+ create_remote_set_user_dir(vcs)
275
+ end
286
276
  pkg_set = load_package_set(vcs, import_options, imported_from)
287
277
  by_repository_id[repository_id][2] = pkg_set
288
278
  package_sets << pkg_set
@@ -335,7 +325,7 @@ def sort_package_sets_by_import_order(package_sets, root_pkg_set)
335
325
  # considered in turn, and added at the earliest place that fits
336
326
  # the dependencies
337
327
  topological = Array.new
338
- queue = package_sets.to_a
328
+ queue = (package_sets.to_a + [root_pkg_set]).uniq
339
329
  while !queue.empty?
340
330
  last_size = queue.size
341
331
  pending = queue.dup
@@ -355,7 +345,7 @@ def sort_package_sets_by_import_order(package_sets, root_pkg_set)
355
345
 
356
346
  result = root_pkg_set.imports.to_a.dup
357
347
  to_insert = topological.dup.
358
- find_all { |pkg_set| !result.include?(pkg_set) }
348
+ find_all { |p| !result.include?(p) }
359
349
  while !to_insert.empty?
360
350
  pkg_set = to_insert.shift
361
351
  dependencies = pkg_set.imports.dup
@@ -369,7 +359,13 @@ def sort_package_sets_by_import_order(package_sets, root_pkg_set)
369
359
  result.insert(i + 1, pkg_set)
370
360
  end
371
361
  end
372
- result << root_pkg_set
362
+
363
+ # Sanity check related to the root package set
364
+ # - it should be last
365
+ # - it should be present only once
366
+ if result.last != root_pkg_set
367
+ raise InternalError, "failed to sort the package sets: the root package set should be last, but is not"
368
+ end
373
369
  result
374
370
  end
375
371
 
@@ -383,34 +379,39 @@ def load_package_sets(options = Hash.new)
383
379
  update_configuration(options)
384
380
  end
385
381
 
386
- def update_configuration(options = Hash.new)
387
- if !options.kind_of?(Hash)
388
- options = Hash[only_local: options]
389
- end
390
- options = validate_options options,
382
+ def update_configuration(
391
383
  only_local: false,
392
384
  checkout_only: !Autobuild.do_update,
393
385
  ignore_errors: false,
394
386
  reset: false,
395
- retry_count: nil
387
+ retry_count: nil)
388
+
389
+ update_options = Hash[
390
+ only_local: only_local,
391
+ checkout_only: checkout_only,
392
+ ignore_errors: ignore_errors,
393
+ reset: reset,
394
+ retry_count: retry_count]
396
395
 
397
- # Load the installation's manifest a first time, to check if we should
398
- # update it ... We assume that the OS dependencies for this VCS is already
399
- # installed (i.e. that the user did not remove it)
400
396
  if ws.manifest.vcs && !ws.manifest.vcs.local?
401
- update_main_configuration(options)
397
+ update_main_configuration(**update_options)
402
398
  end
403
399
  ws.load_main_initrb
404
400
  ws.manifest.load(manifest_path)
405
-
406
- root_pkg_set = ws.manifest.local_package_set
401
+ root_pkg_set = ws.manifest.main_package_set
407
402
  root_pkg_set.load_description_file
408
403
  root_pkg_set.explicit = true
409
- package_sets = load_and_update_package_sets(root_pkg_set, options)
404
+ update_package_sets(**update_options)
405
+ end
406
+
407
+ def update_package_sets(**update_options)
408
+ root_pkg_set = ws.manifest.main_package_set
409
+ package_sets = load_and_update_package_sets(root_pkg_set, **update_options)
410
410
  root_pkg_set.imports.each do |pkg_set|
411
411
  pkg_set.explicit = true
412
412
  end
413
413
  package_sets = sort_package_sets_by_import_order(package_sets, root_pkg_set)
414
+ ws.manifest.reset_package_sets
414
415
  package_sets.each do |pkg_set|
415
416
  ws.manifest.register_package_set(pkg_set)
416
417
  end
@@ -13,9 +13,9 @@ def mark_exclusion_along_revdeps(pkg_name, revdeps, chain = [], reason = nil)
13
13
  reason = ws.manifest.exclusion_reason(pkg_name)
14
14
  else
15
15
  if chain.size == 1
16
- ws.manifest.add_exclusion(pkg_name, "its dependency #{reason}")
16
+ ws.manifest.exclude_package(pkg_name, "its dependency #{reason}")
17
17
  else
18
- ws.manifest.add_exclusion(pkg_name, "#{reason} (dependency chain: #{chain.join(">")})")
18
+ ws.manifest.exclude_package(pkg_name, "#{reason} (dependency chain: #{chain.join(">")})")
19
19
  end
20
20
  end
21
21
 
@@ -34,11 +34,11 @@ def import_next_step(pkg, reverse_dependencies)
34
34
  new_packages = []
35
35
  pkg.dependencies.each do |dep_name|
36
36
  reverse_dependencies[dep_name] << pkg.name
37
- new_packages << ws.manifest.find_autobuild_package(dep_name)
37
+ new_packages << ws.manifest.find_package_definition(dep_name)
38
38
  end
39
39
  pkg_opt_deps, pkg_opt_os_deps = pkg.partition_optional_dependencies
40
40
  pkg_opt_deps.each do |dep_name|
41
- new_packages << ws.manifest.find_autobuild_package(dep_name)
41
+ new_packages << ws.manifest.find_package_definition(dep_name)
42
42
  end
43
43
 
44
44
  # Handle OS dependencies, excluding the package if some
@@ -82,7 +82,9 @@ def pre_package_import(selection, manifest, pkg, reverse_dependencies)
82
82
 
83
83
  def post_package_import(selection, manifest, pkg, reverse_dependencies)
84
84
  Rake::Task["#{pkg.name}-import"].instance_variable_set(:@already_invoked, true)
85
- manifest.load_package_manifest(pkg.name)
85
+ if pkg.checked_out?
86
+ manifest.load_package_manifest(pkg.name)
87
+ end
86
88
 
87
89
  # The package setup mechanisms might have added an exclusion
88
90
  # on this package. Handle this.
@@ -95,35 +97,82 @@ def post_package_import(selection, manifest, pkg, reverse_dependencies)
95
97
  elsif manifest.ignored?(pkg.name)
96
98
  false
97
99
  else
98
- Autoproj.each_post_import_block(pkg) do |block|
99
- block.call(pkg)
100
+ if pkg.checked_out?
101
+ Autoproj.each_post_import_block(pkg) do |block|
102
+ block.call(pkg)
103
+ end
100
104
  end
101
105
  import_next_step(pkg, reverse_dependencies)
102
106
  end
103
107
  end
104
108
 
109
+ # Install the VCS osdep for the given packages
110
+ #
111
+ # @param [Hash] osdeps_options the options that will be passed to
112
+ # {Workspace#install_os_packages}
113
+ # @return [Set] the set of installed OS packages
114
+ def install_vcs_packages_for(*packages, **osdeps_options)
115
+ vcs_to_install = packages.map { |pkg| pkg.vcs.type }.uniq
116
+ # This assumes that the VCS packages do not depend on a
117
+ # 'strict' package mangers such as e.g. BundlerManager
118
+ ws.install_os_packages(vcs_to_install, all: nil, **osdeps_options)
119
+ vcs_to_install
120
+ end
121
+
122
+ # @api private
123
+ #
124
+ # Queue the work necessary to import the given package, making sure
125
+ # that the execution results end up in a given queue
126
+ #
127
+ # @param executor the future executor
128
+ # @param [Queue] completion_queue the queue where the completion
129
+ # results should be pushed, as a (package, time, result,
130
+ # error_reason) tuple
131
+ # @param [Integer] retry_count the number of retries that are
132
+ # allowed. Set to zero for no retry
133
+ # @param [Hash] import_options options passed to {Autobuild::Importer#import}
134
+ def queue_import_work(executor, completion_queue, pkg, retry_count: nil, **import_options)
135
+ import_future = Concurrent::Future.new(executor: executor, args: [pkg]) do |import_pkg|
136
+ ## COMPLETELY BYPASS RAKE HERE
137
+ # The reason is that the ordering of import/prepare between
138
+ # packages is not important BUT the ordering of import vs.
139
+ # prepare in one package IS important: prepare is the method
140
+ # that takes into account dependencies.
141
+ if retry_count
142
+ import_pkg.autobuild.importer.retry_count = retry_count
143
+ end
144
+ import_pkg.autobuild.import(**import_options)
145
+ end
146
+ import_future.add_observer do |time, result, reason|
147
+ completion_queue << [pkg, time, result, reason]
148
+ end
149
+ import_future.execute
150
+ end
151
+
105
152
  class ImportFailed < RuntimeError; end
106
153
 
107
154
  # Import all packages from the given selection, and their
108
155
  # dependencies
109
- def import_selected_packages(selection, updated_packages, options = Hash.new)
110
- parallel_options, options = Kernel.filter_options options,
111
- parallel: ws.config.parallel_import_level
156
+ def import_selected_packages(selection, updated_packages,
157
+ parallel: ws.config.parallel_import_level,
158
+ recursive: true,
159
+ retry_count: nil,
160
+ ignore_errors: false,
161
+ install_vcs_packages: Hash.new,
162
+ non_imported_packages: :checkout,
163
+ **import_options)
164
+
165
+ if ![:checkout, :ignore, :return].include?(non_imported_packages)
166
+ raise ArgumentError, "invalid value for 'non_imported_packages'. Expected one of :checkout, :ignore or :return but got #{non_imported_packages}"
167
+ end
112
168
 
113
169
  # This is used in the ensure block, initialize as early as
114
170
  # possible
115
- executor = Concurrent::FixedThreadPool.new(parallel_options[:parallel], max_length: 0)
116
-
117
- options, import_options = Kernel.filter_options options,
118
- recursive: true,
119
- retry_count: nil
120
-
121
- ignore_errors = options[:ignore_errors]
122
- retry_count = options[:retry_count]
171
+ executor = Concurrent::FixedThreadPool.new(parallel, max_length: 0)
123
172
  manifest = ws.manifest
124
173
 
125
174
  selected_packages = selection.each_source_package_name.map do |pkg_name|
126
- manifest.find_autobuild_package(pkg_name)
175
+ manifest.find_package_definition(pkg_name)
127
176
  end.to_set
128
177
 
129
178
  # The reverse dependencies for the package tree. It is discovered as
@@ -138,69 +187,96 @@ def import_selected_packages(selection, updated_packages, options = Hash.new)
138
187
  pending_packages = Set.new
139
188
  # The set of all packages that are currently selected by +selection+
140
189
  all_processed_packages = Set.new
141
- interactive_imports = Array.new
190
+ main_thread_imports = Array.new
142
191
  package_queue = selected_packages.to_a.sort_by(&:name)
143
192
  failures = Hash.new
193
+ missing_vcs = Array.new
194
+ installed_vcs_packages = Set['none', 'local']
144
195
  while failures.empty? || ignore_errors
145
196
  # Queue work for all packages in the queue
146
197
  package_queue.each do |pkg|
147
198
  # Remove packages that have already been processed
148
199
  next if all_processed_packages.include?(pkg)
200
+ if (non_imported_packages != :checkout) && !File.directory?(pkg.autobuild.srcdir)
201
+ if non_imported_packages == :return
202
+ all_processed_packages << pkg
203
+ completion_queue << [pkg, Time.now, false, nil]
204
+ next
205
+ else
206
+ all_processed_packages << pkg
207
+ ws.manifest.ignore_package(pkg.name)
208
+ next
209
+ end
210
+ elsif install_vcs_packages && !installed_vcs_packages.include?(pkg.vcs.type)
211
+ missing_vcs << pkg
212
+ next
213
+ end
149
214
  all_processed_packages << pkg
150
215
 
151
- if !pre_package_import(selection, manifest, pkg, reverse_dependencies)
216
+ importer = pkg.autobuild.importer
217
+ if !pre_package_import(selection, manifest, pkg.autobuild, reverse_dependencies)
152
218
  next
153
- elsif !pkg.importer
219
+ elsif !importer
220
+ # The validity of this is checked in
221
+ # pre_package_import
154
222
  completion_queue << [pkg, Time.now, false, nil]
155
223
  next
156
- elsif pkg.importer.interactive?
157
- interactive_imports << pkg
224
+ elsif importer.interactive?
225
+ main_thread_imports << pkg
226
+ next
227
+ elsif pkg.autobuild.checked_out? && import_options[:checkout_only]
228
+ main_thread_imports << pkg
158
229
  next
159
230
  end
160
231
 
161
232
  pending_packages << pkg
162
- import_future = Concurrent::Future.new(executor: executor, args: [pkg]) do |import_pkg|
163
- ## COMPLETELY BYPASS RAKE HERE
164
- # The reason is that the ordering of import/prepare between
165
- # packages is not important BUT the ordering of import vs.
166
- # prepare in one package IS important: prepare is the method
167
- # that takes into account dependencies.
168
- if retry_count
169
- import_pkg.importer.retry_count = retry_count
170
- end
171
- import_pkg.import(import_options.merge(allow_interactive: false))
172
- end
173
- import_future.add_observer do |time, result, reason|
174
- completion_queue << [pkg, time, result, reason]
233
+ begin
234
+ queue_import_work(
235
+ executor, completion_queue, pkg, retry_count: retry_count,
236
+ **import_options.merge(allow_interactive: false))
237
+ rescue Exception
238
+ pending_packages.delete(pkg)
239
+ raise
175
240
  end
176
- import_future.execute
241
+ true
177
242
  end
178
243
  package_queue.clear
179
244
 
180
245
  if completion_queue.empty? && pending_packages.empty?
246
+ if !missing_vcs.empty?
247
+ installed_vcs_packages.merge(
248
+ install_vcs_packages_for(*missing_vcs, **install_vcs_packages))
249
+ package_queue.concat(missing_vcs)
250
+ missing_vcs.clear
251
+ next
252
+ end
253
+
181
254
  # We've nothing to process anymore ... process
182
255
  # interactive imports if there are some. Otherwise,
183
256
  # we're done
184
- if interactive_imports.empty?
257
+ if main_thread_imports.empty?
185
258
  break
186
259
  else
187
- interactive_imports.each do |pkg|
260
+ main_thread_imports.delete_if do |pkg|
188
261
  begin
189
- result = pkg.import(import_options.merge(allow_interactive: true))
262
+ if retry_count
263
+ pkg.autobuild.importer.retry_count = retry_count
264
+ end
265
+ result = pkg.autobuild.import(
266
+ **import_options.merge(allow_interactive: true))
190
267
  rescue Exception => reason
191
268
  end
192
269
  completion_queue << [pkg, Time.now, result, reason]
193
270
  end
194
- interactive_imports.clear
195
271
  end
196
272
  end
197
273
 
198
- # And wait one to finish
274
+ # And wait for one to finish
199
275
  pkg, _time, _result, reason = completion_queue.pop
200
276
  pending_packages.delete(pkg)
201
277
  if reason
202
278
  if reason.kind_of?(Autobuild::InteractionRequired)
203
- interactive_imports << pkg
279
+ main_thread_imports << pkg
204
280
  else
205
281
  # One importer failed... terminate
206
282
  Autoproj.error "import of #{pkg.name} failed"
@@ -210,13 +286,13 @@ def import_selected_packages(selection, updated_packages, options = Hash.new)
210
286
  failures[pkg] = reason
211
287
  end
212
288
  else
213
- if new_packages = post_package_import(selection, manifest, pkg, reverse_dependencies)
289
+ if new_packages = post_package_import(selection, manifest, pkg.autobuild, reverse_dependencies)
214
290
  # Excluded dependencies might have caused the package to be
215
291
  # excluded as well ... do not add any dependency to the
216
292
  # processing queue if it is the case
217
293
  if manifest.excluded?(pkg.name)
218
294
  selection.filter_excluded_and_ignored_packages(manifest)
219
- elsif options[:recursive]
295
+ elsif recursive
220
296
  package_queue = new_packages.sort_by(&:name)
221
297
  end
222
298
  end
@@ -240,7 +316,12 @@ def import_selected_packages(selection, updated_packages, options = Hash.new)
240
316
  executor.shutdown
241
317
  executor.wait_for_termination
242
318
  end
243
- updated_packages.concat(all_processed_packages.find_all(&:updated?).map(&:name))
319
+ if all_processed_packages
320
+ all_updated_packages = all_processed_packages.find_all do |processed_pkg|
321
+ processed_pkg.autobuild.updated?
322
+ end
323
+ updated_packages.concat(all_updated_packages.map(&:name))
324
+ end
244
325
  end
245
326
 
246
327
  def finalize_package_load(processed_packages)
@@ -256,8 +337,9 @@ def finalize_package_load(processed_packages)
256
337
 
257
338
  next if manifest.ignored?(pkg_name) || manifest.excluded?(pkg_name)
258
339
 
259
- pkg = manifest.find_autobuild_package(pkg_name)
260
- if !processed_packages.include?(pkg) && File.directory?(pkg.srcdir)
340
+ pkg_definition = manifest.find_package_definition(pkg_name)
341
+ pkg = pkg_definition.autobuild
342
+ if !processed_packages.include?(pkg_definition) && File.directory?(pkg.srcdir)
261
343
  manifest.load_package_manifest(pkg.name)
262
344
  Autoproj.each_post_import_block(pkg) do |block|
263
345
  block.call(pkg)
@@ -286,37 +368,45 @@ def finalize_package_load(processed_packages)
286
368
  all
287
369
  end
288
370
 
289
- def import_packages(selection, options = Hash.new)
371
+ def import_packages(selection,
372
+ non_imported_packages: :checkout,
373
+ warn_about_ignored_packages: true,
374
+ warn_about_excluded_packages: true,
375
+ recursive: true,
376
+ ignore_errors: false,
377
+ install_vcs_packages: Hash.new,
378
+ **import_options)
379
+
290
380
  # Used in the ensure block, initialize as soon as possible
291
381
  updated_packages = Array.new
292
382
 
293
- options, import_options = Kernel.filter_options options,
294
- warn_about_ignored_packages: true,
295
- warn_about_excluded_packages: true,
296
- recursive: true
297
-
298
383
  manifest = ws.manifest
299
384
 
300
385
  all_processed_packages = import_selected_packages(
301
- selection, updated_packages, import_options.merge(recursive: options[:recursive]))
386
+ selection, updated_packages,
387
+ non_imported_packages: non_imported_packages,
388
+ ignore_errors: ignore_errors,
389
+ recursive: recursive,
390
+ install_vcs_packages: install_vcs_packages,
391
+ **import_options)
302
392
  finalize_package_load(all_processed_packages)
303
393
 
304
394
  all_enabled_osdeps = selection.each_osdep_package_name.to_set
305
395
  all_enabled_sources = all_processed_packages.map(&:name)
306
- if options[:recursive]
396
+ if recursive
307
397
  all_processed_packages.each do |pkg|
308
- all_enabled_osdeps.merge(pkg.os_packages)
398
+ all_enabled_osdeps.merge(pkg.autobuild.os_packages)
309
399
  end
310
400
  end
311
401
 
312
- if options[:warn_about_excluded_packages]
402
+ if warn_about_excluded_packages
313
403
  selection.exclusions.each do |sel, pkg_names|
314
404
  pkg_names.sort.each do |pkg_name|
315
405
  Autoproj.warn "#{pkg_name}, which was selected for #{sel}, cannot be built: #{manifest.exclusion_reason(pkg_name)}", :bold
316
406
  end
317
407
  end
318
408
  end
319
- if options[:warn_about_ignored_packages]
409
+ if warn_about_ignored_packages
320
410
  selection.ignores.each do |sel, pkg_names|
321
411
  pkg_names.sort.each do |pkg_name|
322
412
  Autoproj.warn "#{pkg_name}, which was selected for #{sel}, is ignored", :bold