autoproj 2.0.0.rc37 → 2.0.0.rc38
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +4 -2
- data/Rakefile +1 -1
- data/bin/autoproj_bootstrap +34 -2
- data/bin/autoproj_bootstrap.in +4 -2
- data/bin/autoproj_install +34 -2
- data/bin/autoproj_install.in +4 -2
- data/lib/autoproj.rb +9 -2
- data/lib/autoproj/autobuild.rb +13 -742
- data/lib/autoproj/autobuild_extensions/archive_importer.rb +44 -0
- data/lib/autoproj/autobuild_extensions/dsl.rb +439 -0
- data/lib/autoproj/autobuild_extensions/git.rb +116 -0
- data/lib/autoproj/autobuild_extensions/package.rb +159 -0
- data/lib/autoproj/autobuild_extensions/svn.rb +11 -0
- data/lib/autoproj/cli/base.rb +17 -18
- data/lib/autoproj/cli/clean.rb +1 -2
- data/lib/autoproj/cli/envsh.rb +1 -2
- data/lib/autoproj/cli/inspection_tool.rb +12 -21
- data/lib/autoproj/cli/locate.rb +130 -73
- data/lib/autoproj/cli/main.rb +31 -5
- data/lib/autoproj/cli/main_plugin.rb +79 -0
- data/lib/autoproj/cli/main_test.rb +19 -5
- data/lib/autoproj/cli/osdeps.rb +1 -2
- data/lib/autoproj/cli/patcher.rb +21 -0
- data/lib/autoproj/cli/query.rb +34 -41
- data/lib/autoproj/cli/show.rb +121 -52
- data/lib/autoproj/cli/status.rb +4 -5
- data/lib/autoproj/cli/tag.rb +1 -1
- data/lib/autoproj/cli/test.rb +7 -6
- data/lib/autoproj/cli/update.rb +8 -22
- data/lib/autoproj/cli/versions.rb +1 -2
- data/lib/autoproj/configuration.rb +1 -1
- data/lib/autoproj/environment.rb +2 -7
- data/lib/autoproj/exceptions.rb +10 -8
- data/lib/autoproj/find_workspace.rb +46 -12
- data/lib/autoproj/installation_manifest.rb +34 -25
- data/lib/autoproj/local_package_set.rb +86 -0
- data/lib/autoproj/manifest.rb +448 -503
- data/lib/autoproj/metapackage.rb +31 -5
- data/lib/autoproj/ops/configuration.rb +46 -45
- data/lib/autoproj/ops/import.rb +150 -60
- data/lib/autoproj/ops/install.rb +25 -1
- data/lib/autoproj/ops/loader.rb +4 -1
- data/lib/autoproj/ops/main_config_switcher.rb +4 -4
- data/lib/autoproj/ops/snapshot.rb +4 -3
- data/lib/autoproj/os_package_installer.rb +105 -46
- data/lib/autoproj/os_package_resolver.rb +63 -36
- data/lib/autoproj/package_definition.rb +1 -0
- data/lib/autoproj/package_managers/apt_dpkg_manager.rb +30 -27
- data/lib/autoproj/package_managers/bundler_manager.rb +64 -18
- data/lib/autoproj/package_managers/gem_manager.rb +4 -2
- data/lib/autoproj/package_managers/manager.rb +26 -7
- data/lib/autoproj/package_managers/shell_script_manager.rb +4 -4
- data/lib/autoproj/package_managers/zypper_manager.rb +1 -1
- data/lib/autoproj/package_manifest.rb +154 -137
- data/lib/autoproj/package_selection.rb +16 -2
- data/lib/autoproj/package_set.rb +352 -309
- data/lib/autoproj/query.rb +13 -1
- data/lib/autoproj/system.rb +2 -2
- data/lib/autoproj/test.rb +164 -11
- data/lib/autoproj/variable_expansion.rb +15 -42
- data/lib/autoproj/vcs_definition.rb +93 -76
- data/lib/autoproj/version.rb +1 -1
- data/lib/autoproj/workspace.rb +116 -80
- metadata +10 -2
data/lib/autoproj/metapackage.rb
CHANGED
@@ -4,7 +4,7 @@ class Metapackage
|
|
4
4
|
# The metapackage name
|
5
5
|
attr_reader :name
|
6
6
|
# The packages listed in this metapackage
|
7
|
-
attr_reader :
|
7
|
+
attr_reader :packages_by_name
|
8
8
|
# The normal dependency handling behaviour is to generate an error if a
|
9
9
|
# metapackage is selected for the build but some of its dependencies
|
10
10
|
# cannot be built. This modifies the behaviour to simply ignore the
|
@@ -20,22 +20,40 @@ def weak_dependencies?
|
|
20
20
|
|
21
21
|
def initialize(name)
|
22
22
|
@name = name
|
23
|
-
@
|
23
|
+
@packages_by_name = Hash.new
|
24
24
|
@weak_dependencies = false
|
25
25
|
end
|
26
26
|
|
27
|
+
def size
|
28
|
+
packages_by_name.size
|
29
|
+
end
|
30
|
+
|
31
|
+
# Deprecated, use #each_package instead
|
32
|
+
def packages
|
33
|
+
Autoproj.warn_deprecated "use #each_package instead"
|
34
|
+
each_package.to_a
|
35
|
+
end
|
36
|
+
|
27
37
|
# Adds a package to this metapackage
|
28
38
|
#
|
29
39
|
# @param [Autobuild::Package] pkg
|
30
40
|
def add(pkg)
|
31
|
-
|
41
|
+
packages_by_name[pkg.name] = pkg
|
42
|
+
end
|
43
|
+
|
44
|
+
# Remove a package from this metapackage
|
45
|
+
def remove(pkg)
|
46
|
+
if pkg.respond_to?(:name)
|
47
|
+
pkg = pkg.name
|
48
|
+
end
|
49
|
+
packages_by_name.delete(pkg)
|
32
50
|
end
|
33
51
|
|
34
52
|
# Lists the packages contained in this metapackage
|
35
53
|
#
|
36
54
|
# @yieldparam [Autobuild::Package] pkg
|
37
55
|
def each_package(&block)
|
38
|
-
|
56
|
+
packages_by_name.each_value(&block)
|
39
57
|
end
|
40
58
|
|
41
59
|
# Tests if the given package is included in this metapackage
|
@@ -45,7 +63,15 @@ def include?(pkg)
|
|
45
63
|
if !pkg.respond_to?(:to_str)
|
46
64
|
pkg = pkg.name
|
47
65
|
end
|
48
|
-
|
66
|
+
packages_by_name.has_key?(pkg)
|
67
|
+
end
|
68
|
+
|
69
|
+
def clear
|
70
|
+
packages_by_name.clear
|
71
|
+
end
|
72
|
+
|
73
|
+
def delete_if
|
74
|
+
packages_by_name.delete_if { |name, package| yield(package) }
|
49
75
|
end
|
50
76
|
end
|
51
77
|
end
|
@@ -41,11 +41,9 @@ def manifest_path
|
|
41
41
|
# @option options [InstallationManifest] :update_from
|
42
42
|
# another autoproj installation from which we
|
43
43
|
# should update (instead of the normal VCS)
|
44
|
-
def initialize(workspace,
|
45
|
-
options = validate_options options,
|
46
|
-
update_from: nil
|
44
|
+
def initialize(workspace, update_from: nil)
|
47
45
|
@ws = workspace
|
48
|
-
@update_from =
|
46
|
+
@update_from = update_from
|
49
47
|
@remote_update_message_displayed = false
|
50
48
|
end
|
51
49
|
|
@@ -129,8 +127,8 @@ def update_remote_package_set(vcs, options = Hash.new)
|
|
129
127
|
reset: false,
|
130
128
|
retry_count: nil
|
131
129
|
|
132
|
-
name = PackageSet.name_of(ws
|
133
|
-
raw_local_dir = PackageSet.raw_local_dir_of(vcs)
|
130
|
+
name = PackageSet.name_of(ws, vcs)
|
131
|
+
raw_local_dir = PackageSet.raw_local_dir_of(ws, vcs)
|
134
132
|
|
135
133
|
return if options[:checkout_only] && File.exist?(raw_local_dir)
|
136
134
|
|
@@ -140,7 +138,7 @@ def update_remote_package_set(vcs, options = Hash.new)
|
|
140
138
|
Autoproj.message("autoproj: updating remote definitions of package sets", :bold)
|
141
139
|
@remote_update_message_displayed = true
|
142
140
|
end
|
143
|
-
ws.install_os_packages([vcs.type])
|
141
|
+
ws.install_os_packages([vcs.type], all: nil)
|
144
142
|
update_configuration_repository(
|
145
143
|
vcs, name, raw_local_dir, options)
|
146
144
|
end
|
@@ -150,8 +148,8 @@ def update_remote_package_set(vcs, options = Hash.new)
|
|
150
148
|
# @param [VCSDefinition] vcs the package set VCS
|
151
149
|
# @return [String] the full path to the created user dir
|
152
150
|
def create_remote_set_user_dir(vcs)
|
153
|
-
name = PackageSet.name_of(ws
|
154
|
-
raw_local_dir = PackageSet.raw_local_dir_of(vcs)
|
151
|
+
name = PackageSet.name_of(ws, vcs)
|
152
|
+
raw_local_dir = PackageSet.raw_local_dir_of(ws, vcs)
|
155
153
|
FileUtils.mkdir_p(remotes_user_dir)
|
156
154
|
symlink_dest = File.join(remotes_user_dir, name)
|
157
155
|
|
@@ -172,7 +170,7 @@ def create_remote_set_user_dir(vcs)
|
|
172
170
|
end
|
173
171
|
|
174
172
|
def load_package_set(vcs, options, imported_from)
|
175
|
-
pkg_set = PackageSet.new(ws
|
173
|
+
pkg_set = PackageSet.new(ws, vcs)
|
176
174
|
pkg_set.auto_imports = options[:auto_imports]
|
177
175
|
ws.load_if_present(pkg_set, pkg_set.local_dir, 'init.rb')
|
178
176
|
pkg_set.load_description_file
|
@@ -186,22 +184,14 @@ def load_package_set(vcs, options, imported_from)
|
|
186
184
|
def queue_auto_imports_if_needed(queue, pkg_set, root_set)
|
187
185
|
if pkg_set.auto_imports?
|
188
186
|
pkg_set.each_raw_imported_set do |import_vcs, import_options|
|
189
|
-
|
190
|
-
import_vcs = root_set.
|
187
|
+
vcs_overrides_key = import_vcs.overrides_key
|
188
|
+
import_vcs = root_set.resolve_overrides("pkg_set:#{vcs_overrides_key}", import_vcs)
|
191
189
|
queue << [import_vcs, import_options, pkg_set]
|
192
190
|
end
|
193
191
|
end
|
194
192
|
queue
|
195
193
|
end
|
196
194
|
|
197
|
-
def repository_id_of(vcs)
|
198
|
-
if vcs.local?
|
199
|
-
return "local:#{vcs.url}"
|
200
|
-
end
|
201
|
-
|
202
|
-
vcs.create_autobuild_importer.repository_id
|
203
|
-
end
|
204
|
-
|
205
195
|
# Load the package set information
|
206
196
|
#
|
207
197
|
# It loads the package set information as required by {manifest} and
|
@@ -230,7 +220,7 @@ def load_and_update_package_sets(root_pkg_set, options = Hash.new)
|
|
230
220
|
queue = queue_auto_imports_if_needed(Array.new, root_pkg_set, root_pkg_set)
|
231
221
|
while !queue.empty?
|
232
222
|
vcs, import_options, imported_from = queue.shift
|
233
|
-
repository_id =
|
223
|
+
repository_id = vcs.overrides_key
|
234
224
|
if already_processed = by_repository_id[repository_id]
|
235
225
|
already_processed_vcs, already_processed_from, pkg_set = *already_processed
|
236
226
|
if (already_processed_from != root_pkg_set) && (already_processed_vcs != vcs)
|
@@ -249,12 +239,10 @@ def load_and_update_package_sets(root_pkg_set, options = Hash.new)
|
|
249
239
|
# retrieve the actual name of the package set
|
250
240
|
if !vcs.local?
|
251
241
|
update_remote_package_set(vcs, options)
|
252
|
-
|
253
|
-
raw_local_dir = PackageSet.raw_local_dir_of(vcs)
|
254
|
-
required_remotes_dirs << raw_local_dir
|
242
|
+
required_remotes_dirs << PackageSet.raw_local_dir_of(ws, vcs)
|
255
243
|
end
|
256
244
|
|
257
|
-
name = PackageSet.name_of(ws
|
245
|
+
name = PackageSet.name_of(ws, vcs)
|
258
246
|
|
259
247
|
required_user_dirs = by_name.collect { |k,v| k }
|
260
248
|
Autoproj.debug "Trying to load package_set: #{name} from definition #{repository_id}"
|
@@ -264,8 +252,8 @@ def load_and_update_package_sets(root_pkg_set, options = Hash.new)
|
|
264
252
|
already_loaded_pkg_set, already_loaded_vcs = *already_loaded
|
265
253
|
if already_loaded_vcs != vcs
|
266
254
|
if imported_from
|
267
|
-
Autoproj.warn "redundant auto-import
|
268
|
-
Autoproj.warn " A package set with the same name
|
255
|
+
Autoproj.warn "redundant auto-import of package set '#{name}' by package set '#{imported_from.name}'"
|
256
|
+
Autoproj.warn " A package set with the same name has already been imported from"
|
269
257
|
Autoproj.warn " #{already_loaded_vcs}"
|
270
258
|
Autoproj.warn " Skipping the following one: "
|
271
259
|
Autoproj.warn " #{vcs}"
|
@@ -277,12 +265,14 @@ def load_and_update_package_sets(root_pkg_set, options = Hash.new)
|
|
277
265
|
if imported_from
|
278
266
|
already_loaded_pkg_set.imported_from << imported_from
|
279
267
|
imported_from.imports << already_loaded_pkg_set
|
268
|
+
by_repository_id[repository_id][2] = already_loaded_pkg_set
|
280
269
|
end
|
281
270
|
next
|
282
|
-
else
|
283
|
-
create_remote_set_user_dir(vcs)
|
284
271
|
end
|
285
272
|
|
273
|
+
if !vcs.local?
|
274
|
+
create_remote_set_user_dir(vcs)
|
275
|
+
end
|
286
276
|
pkg_set = load_package_set(vcs, import_options, imported_from)
|
287
277
|
by_repository_id[repository_id][2] = pkg_set
|
288
278
|
package_sets << pkg_set
|
@@ -335,7 +325,7 @@ def sort_package_sets_by_import_order(package_sets, root_pkg_set)
|
|
335
325
|
# considered in turn, and added at the earliest place that fits
|
336
326
|
# the dependencies
|
337
327
|
topological = Array.new
|
338
|
-
queue = package_sets.to_a
|
328
|
+
queue = (package_sets.to_a + [root_pkg_set]).uniq
|
339
329
|
while !queue.empty?
|
340
330
|
last_size = queue.size
|
341
331
|
pending = queue.dup
|
@@ -355,7 +345,7 @@ def sort_package_sets_by_import_order(package_sets, root_pkg_set)
|
|
355
345
|
|
356
346
|
result = root_pkg_set.imports.to_a.dup
|
357
347
|
to_insert = topological.dup.
|
358
|
-
find_all { |
|
348
|
+
find_all { |p| !result.include?(p) }
|
359
349
|
while !to_insert.empty?
|
360
350
|
pkg_set = to_insert.shift
|
361
351
|
dependencies = pkg_set.imports.dup
|
@@ -369,7 +359,13 @@ def sort_package_sets_by_import_order(package_sets, root_pkg_set)
|
|
369
359
|
result.insert(i + 1, pkg_set)
|
370
360
|
end
|
371
361
|
end
|
372
|
-
|
362
|
+
|
363
|
+
# Sanity check related to the root package set
|
364
|
+
# - it should be last
|
365
|
+
# - it should be present only once
|
366
|
+
if result.last != root_pkg_set
|
367
|
+
raise InternalError, "failed to sort the package sets: the root package set should be last, but is not"
|
368
|
+
end
|
373
369
|
result
|
374
370
|
end
|
375
371
|
|
@@ -383,34 +379,39 @@ def load_package_sets(options = Hash.new)
|
|
383
379
|
update_configuration(options)
|
384
380
|
end
|
385
381
|
|
386
|
-
def update_configuration(
|
387
|
-
if !options.kind_of?(Hash)
|
388
|
-
options = Hash[only_local: options]
|
389
|
-
end
|
390
|
-
options = validate_options options,
|
382
|
+
def update_configuration(
|
391
383
|
only_local: false,
|
392
384
|
checkout_only: !Autobuild.do_update,
|
393
385
|
ignore_errors: false,
|
394
386
|
reset: false,
|
395
|
-
retry_count: nil
|
387
|
+
retry_count: nil)
|
388
|
+
|
389
|
+
update_options = Hash[
|
390
|
+
only_local: only_local,
|
391
|
+
checkout_only: checkout_only,
|
392
|
+
ignore_errors: ignore_errors,
|
393
|
+
reset: reset,
|
394
|
+
retry_count: retry_count]
|
396
395
|
|
397
|
-
# Load the installation's manifest a first time, to check if we should
|
398
|
-
# update it ... We assume that the OS dependencies for this VCS is already
|
399
|
-
# installed (i.e. that the user did not remove it)
|
400
396
|
if ws.manifest.vcs && !ws.manifest.vcs.local?
|
401
|
-
update_main_configuration(
|
397
|
+
update_main_configuration(**update_options)
|
402
398
|
end
|
403
399
|
ws.load_main_initrb
|
404
400
|
ws.manifest.load(manifest_path)
|
405
|
-
|
406
|
-
root_pkg_set = ws.manifest.local_package_set
|
401
|
+
root_pkg_set = ws.manifest.main_package_set
|
407
402
|
root_pkg_set.load_description_file
|
408
403
|
root_pkg_set.explicit = true
|
409
|
-
|
404
|
+
update_package_sets(**update_options)
|
405
|
+
end
|
406
|
+
|
407
|
+
def update_package_sets(**update_options)
|
408
|
+
root_pkg_set = ws.manifest.main_package_set
|
409
|
+
package_sets = load_and_update_package_sets(root_pkg_set, **update_options)
|
410
410
|
root_pkg_set.imports.each do |pkg_set|
|
411
411
|
pkg_set.explicit = true
|
412
412
|
end
|
413
413
|
package_sets = sort_package_sets_by_import_order(package_sets, root_pkg_set)
|
414
|
+
ws.manifest.reset_package_sets
|
414
415
|
package_sets.each do |pkg_set|
|
415
416
|
ws.manifest.register_package_set(pkg_set)
|
416
417
|
end
|
data/lib/autoproj/ops/import.rb
CHANGED
@@ -13,9 +13,9 @@ def mark_exclusion_along_revdeps(pkg_name, revdeps, chain = [], reason = nil)
|
|
13
13
|
reason = ws.manifest.exclusion_reason(pkg_name)
|
14
14
|
else
|
15
15
|
if chain.size == 1
|
16
|
-
ws.manifest.
|
16
|
+
ws.manifest.exclude_package(pkg_name, "its dependency #{reason}")
|
17
17
|
else
|
18
|
-
ws.manifest.
|
18
|
+
ws.manifest.exclude_package(pkg_name, "#{reason} (dependency chain: #{chain.join(">")})")
|
19
19
|
end
|
20
20
|
end
|
21
21
|
|
@@ -34,11 +34,11 @@ def import_next_step(pkg, reverse_dependencies)
|
|
34
34
|
new_packages = []
|
35
35
|
pkg.dependencies.each do |dep_name|
|
36
36
|
reverse_dependencies[dep_name] << pkg.name
|
37
|
-
new_packages << ws.manifest.
|
37
|
+
new_packages << ws.manifest.find_package_definition(dep_name)
|
38
38
|
end
|
39
39
|
pkg_opt_deps, pkg_opt_os_deps = pkg.partition_optional_dependencies
|
40
40
|
pkg_opt_deps.each do |dep_name|
|
41
|
-
new_packages << ws.manifest.
|
41
|
+
new_packages << ws.manifest.find_package_definition(dep_name)
|
42
42
|
end
|
43
43
|
|
44
44
|
# Handle OS dependencies, excluding the package if some
|
@@ -82,7 +82,9 @@ def pre_package_import(selection, manifest, pkg, reverse_dependencies)
|
|
82
82
|
|
83
83
|
def post_package_import(selection, manifest, pkg, reverse_dependencies)
|
84
84
|
Rake::Task["#{pkg.name}-import"].instance_variable_set(:@already_invoked, true)
|
85
|
-
|
85
|
+
if pkg.checked_out?
|
86
|
+
manifest.load_package_manifest(pkg.name)
|
87
|
+
end
|
86
88
|
|
87
89
|
# The package setup mechanisms might have added an exclusion
|
88
90
|
# on this package. Handle this.
|
@@ -95,35 +97,82 @@ def post_package_import(selection, manifest, pkg, reverse_dependencies)
|
|
95
97
|
elsif manifest.ignored?(pkg.name)
|
96
98
|
false
|
97
99
|
else
|
98
|
-
|
99
|
-
|
100
|
+
if pkg.checked_out?
|
101
|
+
Autoproj.each_post_import_block(pkg) do |block|
|
102
|
+
block.call(pkg)
|
103
|
+
end
|
100
104
|
end
|
101
105
|
import_next_step(pkg, reverse_dependencies)
|
102
106
|
end
|
103
107
|
end
|
104
108
|
|
109
|
+
# Install the VCS osdep for the given packages
|
110
|
+
#
|
111
|
+
# @param [Hash] osdeps_options the options that will be passed to
|
112
|
+
# {Workspace#install_os_packages}
|
113
|
+
# @return [Set] the set of installed OS packages
|
114
|
+
def install_vcs_packages_for(*packages, **osdeps_options)
|
115
|
+
vcs_to_install = packages.map { |pkg| pkg.vcs.type }.uniq
|
116
|
+
# This assumes that the VCS packages do not depend on a
|
117
|
+
# 'strict' package mangers such as e.g. BundlerManager
|
118
|
+
ws.install_os_packages(vcs_to_install, all: nil, **osdeps_options)
|
119
|
+
vcs_to_install
|
120
|
+
end
|
121
|
+
|
122
|
+
# @api private
|
123
|
+
#
|
124
|
+
# Queue the work necessary to import the given package, making sure
|
125
|
+
# that the execution results end up in a given queue
|
126
|
+
#
|
127
|
+
# @param executor the future executor
|
128
|
+
# @param [Queue] completion_queue the queue where the completion
|
129
|
+
# results should be pushed, as a (package, time, result,
|
130
|
+
# error_reason) tuple
|
131
|
+
# @param [Integer] retry_count the number of retries that are
|
132
|
+
# allowed. Set to zero for no retry
|
133
|
+
# @param [Hash] import_options options passed to {Autobuild::Importer#import}
|
134
|
+
def queue_import_work(executor, completion_queue, pkg, retry_count: nil, **import_options)
|
135
|
+
import_future = Concurrent::Future.new(executor: executor, args: [pkg]) do |import_pkg|
|
136
|
+
## COMPLETELY BYPASS RAKE HERE
|
137
|
+
# The reason is that the ordering of import/prepare between
|
138
|
+
# packages is not important BUT the ordering of import vs.
|
139
|
+
# prepare in one package IS important: prepare is the method
|
140
|
+
# that takes into account dependencies.
|
141
|
+
if retry_count
|
142
|
+
import_pkg.autobuild.importer.retry_count = retry_count
|
143
|
+
end
|
144
|
+
import_pkg.autobuild.import(**import_options)
|
145
|
+
end
|
146
|
+
import_future.add_observer do |time, result, reason|
|
147
|
+
completion_queue << [pkg, time, result, reason]
|
148
|
+
end
|
149
|
+
import_future.execute
|
150
|
+
end
|
151
|
+
|
105
152
|
class ImportFailed < RuntimeError; end
|
106
153
|
|
107
154
|
# Import all packages from the given selection, and their
|
108
155
|
# dependencies
|
109
|
-
def import_selected_packages(selection, updated_packages,
|
110
|
-
|
111
|
-
|
156
|
+
def import_selected_packages(selection, updated_packages,
|
157
|
+
parallel: ws.config.parallel_import_level,
|
158
|
+
recursive: true,
|
159
|
+
retry_count: nil,
|
160
|
+
ignore_errors: false,
|
161
|
+
install_vcs_packages: Hash.new,
|
162
|
+
non_imported_packages: :checkout,
|
163
|
+
**import_options)
|
164
|
+
|
165
|
+
if ![:checkout, :ignore, :return].include?(non_imported_packages)
|
166
|
+
raise ArgumentError, "invalid value for 'non_imported_packages'. Expected one of :checkout, :ignore or :return but got #{non_imported_packages}"
|
167
|
+
end
|
112
168
|
|
113
169
|
# This is used in the ensure block, initialize as early as
|
114
170
|
# possible
|
115
|
-
executor = Concurrent::FixedThreadPool.new(
|
116
|
-
|
117
|
-
options, import_options = Kernel.filter_options options,
|
118
|
-
recursive: true,
|
119
|
-
retry_count: nil
|
120
|
-
|
121
|
-
ignore_errors = options[:ignore_errors]
|
122
|
-
retry_count = options[:retry_count]
|
171
|
+
executor = Concurrent::FixedThreadPool.new(parallel, max_length: 0)
|
123
172
|
manifest = ws.manifest
|
124
173
|
|
125
174
|
selected_packages = selection.each_source_package_name.map do |pkg_name|
|
126
|
-
manifest.
|
175
|
+
manifest.find_package_definition(pkg_name)
|
127
176
|
end.to_set
|
128
177
|
|
129
178
|
# The reverse dependencies for the package tree. It is discovered as
|
@@ -138,69 +187,96 @@ def import_selected_packages(selection, updated_packages, options = Hash.new)
|
|
138
187
|
pending_packages = Set.new
|
139
188
|
# The set of all packages that are currently selected by +selection+
|
140
189
|
all_processed_packages = Set.new
|
141
|
-
|
190
|
+
main_thread_imports = Array.new
|
142
191
|
package_queue = selected_packages.to_a.sort_by(&:name)
|
143
192
|
failures = Hash.new
|
193
|
+
missing_vcs = Array.new
|
194
|
+
installed_vcs_packages = Set['none', 'local']
|
144
195
|
while failures.empty? || ignore_errors
|
145
196
|
# Queue work for all packages in the queue
|
146
197
|
package_queue.each do |pkg|
|
147
198
|
# Remove packages that have already been processed
|
148
199
|
next if all_processed_packages.include?(pkg)
|
200
|
+
if (non_imported_packages != :checkout) && !File.directory?(pkg.autobuild.srcdir)
|
201
|
+
if non_imported_packages == :return
|
202
|
+
all_processed_packages << pkg
|
203
|
+
completion_queue << [pkg, Time.now, false, nil]
|
204
|
+
next
|
205
|
+
else
|
206
|
+
all_processed_packages << pkg
|
207
|
+
ws.manifest.ignore_package(pkg.name)
|
208
|
+
next
|
209
|
+
end
|
210
|
+
elsif install_vcs_packages && !installed_vcs_packages.include?(pkg.vcs.type)
|
211
|
+
missing_vcs << pkg
|
212
|
+
next
|
213
|
+
end
|
149
214
|
all_processed_packages << pkg
|
150
215
|
|
151
|
-
|
216
|
+
importer = pkg.autobuild.importer
|
217
|
+
if !pre_package_import(selection, manifest, pkg.autobuild, reverse_dependencies)
|
152
218
|
next
|
153
|
-
elsif !
|
219
|
+
elsif !importer
|
220
|
+
# The validity of this is checked in
|
221
|
+
# pre_package_import
|
154
222
|
completion_queue << [pkg, Time.now, false, nil]
|
155
223
|
next
|
156
|
-
elsif
|
157
|
-
|
224
|
+
elsif importer.interactive?
|
225
|
+
main_thread_imports << pkg
|
226
|
+
next
|
227
|
+
elsif pkg.autobuild.checked_out? && import_options[:checkout_only]
|
228
|
+
main_thread_imports << pkg
|
158
229
|
next
|
159
230
|
end
|
160
231
|
|
161
232
|
pending_packages << pkg
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
import_pkg.importer.retry_count = retry_count
|
170
|
-
end
|
171
|
-
import_pkg.import(import_options.merge(allow_interactive: false))
|
172
|
-
end
|
173
|
-
import_future.add_observer do |time, result, reason|
|
174
|
-
completion_queue << [pkg, time, result, reason]
|
233
|
+
begin
|
234
|
+
queue_import_work(
|
235
|
+
executor, completion_queue, pkg, retry_count: retry_count,
|
236
|
+
**import_options.merge(allow_interactive: false))
|
237
|
+
rescue Exception
|
238
|
+
pending_packages.delete(pkg)
|
239
|
+
raise
|
175
240
|
end
|
176
|
-
|
241
|
+
true
|
177
242
|
end
|
178
243
|
package_queue.clear
|
179
244
|
|
180
245
|
if completion_queue.empty? && pending_packages.empty?
|
246
|
+
if !missing_vcs.empty?
|
247
|
+
installed_vcs_packages.merge(
|
248
|
+
install_vcs_packages_for(*missing_vcs, **install_vcs_packages))
|
249
|
+
package_queue.concat(missing_vcs)
|
250
|
+
missing_vcs.clear
|
251
|
+
next
|
252
|
+
end
|
253
|
+
|
181
254
|
# We've nothing to process anymore ... process
|
182
255
|
# interactive imports if there are some. Otherwise,
|
183
256
|
# we're done
|
184
|
-
if
|
257
|
+
if main_thread_imports.empty?
|
185
258
|
break
|
186
259
|
else
|
187
|
-
|
260
|
+
main_thread_imports.delete_if do |pkg|
|
188
261
|
begin
|
189
|
-
|
262
|
+
if retry_count
|
263
|
+
pkg.autobuild.importer.retry_count = retry_count
|
264
|
+
end
|
265
|
+
result = pkg.autobuild.import(
|
266
|
+
**import_options.merge(allow_interactive: true))
|
190
267
|
rescue Exception => reason
|
191
268
|
end
|
192
269
|
completion_queue << [pkg, Time.now, result, reason]
|
193
270
|
end
|
194
|
-
interactive_imports.clear
|
195
271
|
end
|
196
272
|
end
|
197
273
|
|
198
|
-
# And wait one to finish
|
274
|
+
# And wait for one to finish
|
199
275
|
pkg, _time, _result, reason = completion_queue.pop
|
200
276
|
pending_packages.delete(pkg)
|
201
277
|
if reason
|
202
278
|
if reason.kind_of?(Autobuild::InteractionRequired)
|
203
|
-
|
279
|
+
main_thread_imports << pkg
|
204
280
|
else
|
205
281
|
# One importer failed... terminate
|
206
282
|
Autoproj.error "import of #{pkg.name} failed"
|
@@ -210,13 +286,13 @@ def import_selected_packages(selection, updated_packages, options = Hash.new)
|
|
210
286
|
failures[pkg] = reason
|
211
287
|
end
|
212
288
|
else
|
213
|
-
if new_packages = post_package_import(selection, manifest, pkg, reverse_dependencies)
|
289
|
+
if new_packages = post_package_import(selection, manifest, pkg.autobuild, reverse_dependencies)
|
214
290
|
# Excluded dependencies might have caused the package to be
|
215
291
|
# excluded as well ... do not add any dependency to the
|
216
292
|
# processing queue if it is the case
|
217
293
|
if manifest.excluded?(pkg.name)
|
218
294
|
selection.filter_excluded_and_ignored_packages(manifest)
|
219
|
-
elsif
|
295
|
+
elsif recursive
|
220
296
|
package_queue = new_packages.sort_by(&:name)
|
221
297
|
end
|
222
298
|
end
|
@@ -240,7 +316,12 @@ def import_selected_packages(selection, updated_packages, options = Hash.new)
|
|
240
316
|
executor.shutdown
|
241
317
|
executor.wait_for_termination
|
242
318
|
end
|
243
|
-
|
319
|
+
if all_processed_packages
|
320
|
+
all_updated_packages = all_processed_packages.find_all do |processed_pkg|
|
321
|
+
processed_pkg.autobuild.updated?
|
322
|
+
end
|
323
|
+
updated_packages.concat(all_updated_packages.map(&:name))
|
324
|
+
end
|
244
325
|
end
|
245
326
|
|
246
327
|
def finalize_package_load(processed_packages)
|
@@ -256,8 +337,9 @@ def finalize_package_load(processed_packages)
|
|
256
337
|
|
257
338
|
next if manifest.ignored?(pkg_name) || manifest.excluded?(pkg_name)
|
258
339
|
|
259
|
-
|
260
|
-
|
340
|
+
pkg_definition = manifest.find_package_definition(pkg_name)
|
341
|
+
pkg = pkg_definition.autobuild
|
342
|
+
if !processed_packages.include?(pkg_definition) && File.directory?(pkg.srcdir)
|
261
343
|
manifest.load_package_manifest(pkg.name)
|
262
344
|
Autoproj.each_post_import_block(pkg) do |block|
|
263
345
|
block.call(pkg)
|
@@ -286,37 +368,45 @@ def finalize_package_load(processed_packages)
|
|
286
368
|
all
|
287
369
|
end
|
288
370
|
|
289
|
-
def import_packages(selection,
|
371
|
+
def import_packages(selection,
|
372
|
+
non_imported_packages: :checkout,
|
373
|
+
warn_about_ignored_packages: true,
|
374
|
+
warn_about_excluded_packages: true,
|
375
|
+
recursive: true,
|
376
|
+
ignore_errors: false,
|
377
|
+
install_vcs_packages: Hash.new,
|
378
|
+
**import_options)
|
379
|
+
|
290
380
|
# Used in the ensure block, initialize as soon as possible
|
291
381
|
updated_packages = Array.new
|
292
382
|
|
293
|
-
options, import_options = Kernel.filter_options options,
|
294
|
-
warn_about_ignored_packages: true,
|
295
|
-
warn_about_excluded_packages: true,
|
296
|
-
recursive: true
|
297
|
-
|
298
383
|
manifest = ws.manifest
|
299
384
|
|
300
385
|
all_processed_packages = import_selected_packages(
|
301
|
-
selection, updated_packages,
|
386
|
+
selection, updated_packages,
|
387
|
+
non_imported_packages: non_imported_packages,
|
388
|
+
ignore_errors: ignore_errors,
|
389
|
+
recursive: recursive,
|
390
|
+
install_vcs_packages: install_vcs_packages,
|
391
|
+
**import_options)
|
302
392
|
finalize_package_load(all_processed_packages)
|
303
393
|
|
304
394
|
all_enabled_osdeps = selection.each_osdep_package_name.to_set
|
305
395
|
all_enabled_sources = all_processed_packages.map(&:name)
|
306
|
-
if
|
396
|
+
if recursive
|
307
397
|
all_processed_packages.each do |pkg|
|
308
|
-
all_enabled_osdeps.merge(pkg.os_packages)
|
398
|
+
all_enabled_osdeps.merge(pkg.autobuild.os_packages)
|
309
399
|
end
|
310
400
|
end
|
311
401
|
|
312
|
-
if
|
402
|
+
if warn_about_excluded_packages
|
313
403
|
selection.exclusions.each do |sel, pkg_names|
|
314
404
|
pkg_names.sort.each do |pkg_name|
|
315
405
|
Autoproj.warn "#{pkg_name}, which was selected for #{sel}, cannot be built: #{manifest.exclusion_reason(pkg_name)}", :bold
|
316
406
|
end
|
317
407
|
end
|
318
408
|
end
|
319
|
-
if
|
409
|
+
if warn_about_ignored_packages
|
320
410
|
selection.ignores.each do |sel, pkg_names|
|
321
411
|
pkg_names.sort.each do |pkg_name|
|
322
412
|
Autoproj.warn "#{pkg_name}, which was selected for #{sel}, is ignored", :bold
|