autoproj 2.0.0.rc37 → 2.0.0.rc38
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +4 -2
- data/Rakefile +1 -1
- data/bin/autoproj_bootstrap +34 -2
- data/bin/autoproj_bootstrap.in +4 -2
- data/bin/autoproj_install +34 -2
- data/bin/autoproj_install.in +4 -2
- data/lib/autoproj.rb +9 -2
- data/lib/autoproj/autobuild.rb +13 -742
- data/lib/autoproj/autobuild_extensions/archive_importer.rb +44 -0
- data/lib/autoproj/autobuild_extensions/dsl.rb +439 -0
- data/lib/autoproj/autobuild_extensions/git.rb +116 -0
- data/lib/autoproj/autobuild_extensions/package.rb +159 -0
- data/lib/autoproj/autobuild_extensions/svn.rb +11 -0
- data/lib/autoproj/cli/base.rb +17 -18
- data/lib/autoproj/cli/clean.rb +1 -2
- data/lib/autoproj/cli/envsh.rb +1 -2
- data/lib/autoproj/cli/inspection_tool.rb +12 -21
- data/lib/autoproj/cli/locate.rb +130 -73
- data/lib/autoproj/cli/main.rb +31 -5
- data/lib/autoproj/cli/main_plugin.rb +79 -0
- data/lib/autoproj/cli/main_test.rb +19 -5
- data/lib/autoproj/cli/osdeps.rb +1 -2
- data/lib/autoproj/cli/patcher.rb +21 -0
- data/lib/autoproj/cli/query.rb +34 -41
- data/lib/autoproj/cli/show.rb +121 -52
- data/lib/autoproj/cli/status.rb +4 -5
- data/lib/autoproj/cli/tag.rb +1 -1
- data/lib/autoproj/cli/test.rb +7 -6
- data/lib/autoproj/cli/update.rb +8 -22
- data/lib/autoproj/cli/versions.rb +1 -2
- data/lib/autoproj/configuration.rb +1 -1
- data/lib/autoproj/environment.rb +2 -7
- data/lib/autoproj/exceptions.rb +10 -8
- data/lib/autoproj/find_workspace.rb +46 -12
- data/lib/autoproj/installation_manifest.rb +34 -25
- data/lib/autoproj/local_package_set.rb +86 -0
- data/lib/autoproj/manifest.rb +448 -503
- data/lib/autoproj/metapackage.rb +31 -5
- data/lib/autoproj/ops/configuration.rb +46 -45
- data/lib/autoproj/ops/import.rb +150 -60
- data/lib/autoproj/ops/install.rb +25 -1
- data/lib/autoproj/ops/loader.rb +4 -1
- data/lib/autoproj/ops/main_config_switcher.rb +4 -4
- data/lib/autoproj/ops/snapshot.rb +4 -3
- data/lib/autoproj/os_package_installer.rb +105 -46
- data/lib/autoproj/os_package_resolver.rb +63 -36
- data/lib/autoproj/package_definition.rb +1 -0
- data/lib/autoproj/package_managers/apt_dpkg_manager.rb +30 -27
- data/lib/autoproj/package_managers/bundler_manager.rb +64 -18
- data/lib/autoproj/package_managers/gem_manager.rb +4 -2
- data/lib/autoproj/package_managers/manager.rb +26 -7
- data/lib/autoproj/package_managers/shell_script_manager.rb +4 -4
- data/lib/autoproj/package_managers/zypper_manager.rb +1 -1
- data/lib/autoproj/package_manifest.rb +154 -137
- data/lib/autoproj/package_selection.rb +16 -2
- data/lib/autoproj/package_set.rb +352 -309
- data/lib/autoproj/query.rb +13 -1
- data/lib/autoproj/system.rb +2 -2
- data/lib/autoproj/test.rb +164 -11
- data/lib/autoproj/variable_expansion.rb +15 -42
- data/lib/autoproj/vcs_definition.rb +93 -76
- data/lib/autoproj/version.rb +1 -1
- data/lib/autoproj/workspace.rb +116 -80
- metadata +10 -2
data/lib/autoproj/metapackage.rb
CHANGED
@@ -4,7 +4,7 @@ class Metapackage
|
|
4
4
|
# The metapackage name
|
5
5
|
attr_reader :name
|
6
6
|
# The packages listed in this metapackage
|
7
|
-
attr_reader :
|
7
|
+
attr_reader :packages_by_name
|
8
8
|
# The normal dependency handling behaviour is to generate an error if a
|
9
9
|
# metapackage is selected for the build but some of its dependencies
|
10
10
|
# cannot be built. This modifies the behaviour to simply ignore the
|
@@ -20,22 +20,40 @@ def weak_dependencies?
|
|
20
20
|
|
21
21
|
def initialize(name)
|
22
22
|
@name = name
|
23
|
-
@
|
23
|
+
@packages_by_name = Hash.new
|
24
24
|
@weak_dependencies = false
|
25
25
|
end
|
26
26
|
|
27
|
+
def size
|
28
|
+
packages_by_name.size
|
29
|
+
end
|
30
|
+
|
31
|
+
# Deprecated, use #each_package instead
|
32
|
+
def packages
|
33
|
+
Autoproj.warn_deprecated "use #each_package instead"
|
34
|
+
each_package.to_a
|
35
|
+
end
|
36
|
+
|
27
37
|
# Adds a package to this metapackage
|
28
38
|
#
|
29
39
|
# @param [Autobuild::Package] pkg
|
30
40
|
def add(pkg)
|
31
|
-
|
41
|
+
packages_by_name[pkg.name] = pkg
|
42
|
+
end
|
43
|
+
|
44
|
+
# Remove a package from this metapackage
|
45
|
+
def remove(pkg)
|
46
|
+
if pkg.respond_to?(:name)
|
47
|
+
pkg = pkg.name
|
48
|
+
end
|
49
|
+
packages_by_name.delete(pkg)
|
32
50
|
end
|
33
51
|
|
34
52
|
# Lists the packages contained in this metapackage
|
35
53
|
#
|
36
54
|
# @yieldparam [Autobuild::Package] pkg
|
37
55
|
def each_package(&block)
|
38
|
-
|
56
|
+
packages_by_name.each_value(&block)
|
39
57
|
end
|
40
58
|
|
41
59
|
# Tests if the given package is included in this metapackage
|
@@ -45,7 +63,15 @@ def include?(pkg)
|
|
45
63
|
if !pkg.respond_to?(:to_str)
|
46
64
|
pkg = pkg.name
|
47
65
|
end
|
48
|
-
|
66
|
+
packages_by_name.has_key?(pkg)
|
67
|
+
end
|
68
|
+
|
69
|
+
def clear
|
70
|
+
packages_by_name.clear
|
71
|
+
end
|
72
|
+
|
73
|
+
def delete_if
|
74
|
+
packages_by_name.delete_if { |name, package| yield(package) }
|
49
75
|
end
|
50
76
|
end
|
51
77
|
end
|
@@ -41,11 +41,9 @@ def manifest_path
|
|
41
41
|
# @option options [InstallationManifest] :update_from
|
42
42
|
# another autoproj installation from which we
|
43
43
|
# should update (instead of the normal VCS)
|
44
|
-
def initialize(workspace,
|
45
|
-
options = validate_options options,
|
46
|
-
update_from: nil
|
44
|
+
def initialize(workspace, update_from: nil)
|
47
45
|
@ws = workspace
|
48
|
-
@update_from =
|
46
|
+
@update_from = update_from
|
49
47
|
@remote_update_message_displayed = false
|
50
48
|
end
|
51
49
|
|
@@ -129,8 +127,8 @@ def update_remote_package_set(vcs, options = Hash.new)
|
|
129
127
|
reset: false,
|
130
128
|
retry_count: nil
|
131
129
|
|
132
|
-
name = PackageSet.name_of(ws
|
133
|
-
raw_local_dir = PackageSet.raw_local_dir_of(vcs)
|
130
|
+
name = PackageSet.name_of(ws, vcs)
|
131
|
+
raw_local_dir = PackageSet.raw_local_dir_of(ws, vcs)
|
134
132
|
|
135
133
|
return if options[:checkout_only] && File.exist?(raw_local_dir)
|
136
134
|
|
@@ -140,7 +138,7 @@ def update_remote_package_set(vcs, options = Hash.new)
|
|
140
138
|
Autoproj.message("autoproj: updating remote definitions of package sets", :bold)
|
141
139
|
@remote_update_message_displayed = true
|
142
140
|
end
|
143
|
-
ws.install_os_packages([vcs.type])
|
141
|
+
ws.install_os_packages([vcs.type], all: nil)
|
144
142
|
update_configuration_repository(
|
145
143
|
vcs, name, raw_local_dir, options)
|
146
144
|
end
|
@@ -150,8 +148,8 @@ def update_remote_package_set(vcs, options = Hash.new)
|
|
150
148
|
# @param [VCSDefinition] vcs the package set VCS
|
151
149
|
# @return [String] the full path to the created user dir
|
152
150
|
def create_remote_set_user_dir(vcs)
|
153
|
-
name = PackageSet.name_of(ws
|
154
|
-
raw_local_dir = PackageSet.raw_local_dir_of(vcs)
|
151
|
+
name = PackageSet.name_of(ws, vcs)
|
152
|
+
raw_local_dir = PackageSet.raw_local_dir_of(ws, vcs)
|
155
153
|
FileUtils.mkdir_p(remotes_user_dir)
|
156
154
|
symlink_dest = File.join(remotes_user_dir, name)
|
157
155
|
|
@@ -172,7 +170,7 @@ def create_remote_set_user_dir(vcs)
|
|
172
170
|
end
|
173
171
|
|
174
172
|
def load_package_set(vcs, options, imported_from)
|
175
|
-
pkg_set = PackageSet.new(ws
|
173
|
+
pkg_set = PackageSet.new(ws, vcs)
|
176
174
|
pkg_set.auto_imports = options[:auto_imports]
|
177
175
|
ws.load_if_present(pkg_set, pkg_set.local_dir, 'init.rb')
|
178
176
|
pkg_set.load_description_file
|
@@ -186,22 +184,14 @@ def load_package_set(vcs, options, imported_from)
|
|
186
184
|
def queue_auto_imports_if_needed(queue, pkg_set, root_set)
|
187
185
|
if pkg_set.auto_imports?
|
188
186
|
pkg_set.each_raw_imported_set do |import_vcs, import_options|
|
189
|
-
|
190
|
-
import_vcs = root_set.
|
187
|
+
vcs_overrides_key = import_vcs.overrides_key
|
188
|
+
import_vcs = root_set.resolve_overrides("pkg_set:#{vcs_overrides_key}", import_vcs)
|
191
189
|
queue << [import_vcs, import_options, pkg_set]
|
192
190
|
end
|
193
191
|
end
|
194
192
|
queue
|
195
193
|
end
|
196
194
|
|
197
|
-
def repository_id_of(vcs)
|
198
|
-
if vcs.local?
|
199
|
-
return "local:#{vcs.url}"
|
200
|
-
end
|
201
|
-
|
202
|
-
vcs.create_autobuild_importer.repository_id
|
203
|
-
end
|
204
|
-
|
205
195
|
# Load the package set information
|
206
196
|
#
|
207
197
|
# It loads the package set information as required by {manifest} and
|
@@ -230,7 +220,7 @@ def load_and_update_package_sets(root_pkg_set, options = Hash.new)
|
|
230
220
|
queue = queue_auto_imports_if_needed(Array.new, root_pkg_set, root_pkg_set)
|
231
221
|
while !queue.empty?
|
232
222
|
vcs, import_options, imported_from = queue.shift
|
233
|
-
repository_id =
|
223
|
+
repository_id = vcs.overrides_key
|
234
224
|
if already_processed = by_repository_id[repository_id]
|
235
225
|
already_processed_vcs, already_processed_from, pkg_set = *already_processed
|
236
226
|
if (already_processed_from != root_pkg_set) && (already_processed_vcs != vcs)
|
@@ -249,12 +239,10 @@ def load_and_update_package_sets(root_pkg_set, options = Hash.new)
|
|
249
239
|
# retrieve the actual name of the package set
|
250
240
|
if !vcs.local?
|
251
241
|
update_remote_package_set(vcs, options)
|
252
|
-
|
253
|
-
raw_local_dir = PackageSet.raw_local_dir_of(vcs)
|
254
|
-
required_remotes_dirs << raw_local_dir
|
242
|
+
required_remotes_dirs << PackageSet.raw_local_dir_of(ws, vcs)
|
255
243
|
end
|
256
244
|
|
257
|
-
name = PackageSet.name_of(ws
|
245
|
+
name = PackageSet.name_of(ws, vcs)
|
258
246
|
|
259
247
|
required_user_dirs = by_name.collect { |k,v| k }
|
260
248
|
Autoproj.debug "Trying to load package_set: #{name} from definition #{repository_id}"
|
@@ -264,8 +252,8 @@ def load_and_update_package_sets(root_pkg_set, options = Hash.new)
|
|
264
252
|
already_loaded_pkg_set, already_loaded_vcs = *already_loaded
|
265
253
|
if already_loaded_vcs != vcs
|
266
254
|
if imported_from
|
267
|
-
Autoproj.warn "redundant auto-import
|
268
|
-
Autoproj.warn " A package set with the same name
|
255
|
+
Autoproj.warn "redundant auto-import of package set '#{name}' by package set '#{imported_from.name}'"
|
256
|
+
Autoproj.warn " A package set with the same name has already been imported from"
|
269
257
|
Autoproj.warn " #{already_loaded_vcs}"
|
270
258
|
Autoproj.warn " Skipping the following one: "
|
271
259
|
Autoproj.warn " #{vcs}"
|
@@ -277,12 +265,14 @@ def load_and_update_package_sets(root_pkg_set, options = Hash.new)
|
|
277
265
|
if imported_from
|
278
266
|
already_loaded_pkg_set.imported_from << imported_from
|
279
267
|
imported_from.imports << already_loaded_pkg_set
|
268
|
+
by_repository_id[repository_id][2] = already_loaded_pkg_set
|
280
269
|
end
|
281
270
|
next
|
282
|
-
else
|
283
|
-
create_remote_set_user_dir(vcs)
|
284
271
|
end
|
285
272
|
|
273
|
+
if !vcs.local?
|
274
|
+
create_remote_set_user_dir(vcs)
|
275
|
+
end
|
286
276
|
pkg_set = load_package_set(vcs, import_options, imported_from)
|
287
277
|
by_repository_id[repository_id][2] = pkg_set
|
288
278
|
package_sets << pkg_set
|
@@ -335,7 +325,7 @@ def sort_package_sets_by_import_order(package_sets, root_pkg_set)
|
|
335
325
|
# considered in turn, and added at the earliest place that fits
|
336
326
|
# the dependencies
|
337
327
|
topological = Array.new
|
338
|
-
queue = package_sets.to_a
|
328
|
+
queue = (package_sets.to_a + [root_pkg_set]).uniq
|
339
329
|
while !queue.empty?
|
340
330
|
last_size = queue.size
|
341
331
|
pending = queue.dup
|
@@ -355,7 +345,7 @@ def sort_package_sets_by_import_order(package_sets, root_pkg_set)
|
|
355
345
|
|
356
346
|
result = root_pkg_set.imports.to_a.dup
|
357
347
|
to_insert = topological.dup.
|
358
|
-
find_all { |
|
348
|
+
find_all { |p| !result.include?(p) }
|
359
349
|
while !to_insert.empty?
|
360
350
|
pkg_set = to_insert.shift
|
361
351
|
dependencies = pkg_set.imports.dup
|
@@ -369,7 +359,13 @@ def sort_package_sets_by_import_order(package_sets, root_pkg_set)
|
|
369
359
|
result.insert(i + 1, pkg_set)
|
370
360
|
end
|
371
361
|
end
|
372
|
-
|
362
|
+
|
363
|
+
# Sanity check related to the root package set
|
364
|
+
# - it should be last
|
365
|
+
# - it should be present only once
|
366
|
+
if result.last != root_pkg_set
|
367
|
+
raise InternalError, "failed to sort the package sets: the root package set should be last, but is not"
|
368
|
+
end
|
373
369
|
result
|
374
370
|
end
|
375
371
|
|
@@ -383,34 +379,39 @@ def load_package_sets(options = Hash.new)
|
|
383
379
|
update_configuration(options)
|
384
380
|
end
|
385
381
|
|
386
|
-
def update_configuration(
|
387
|
-
if !options.kind_of?(Hash)
|
388
|
-
options = Hash[only_local: options]
|
389
|
-
end
|
390
|
-
options = validate_options options,
|
382
|
+
def update_configuration(
|
391
383
|
only_local: false,
|
392
384
|
checkout_only: !Autobuild.do_update,
|
393
385
|
ignore_errors: false,
|
394
386
|
reset: false,
|
395
|
-
retry_count: nil
|
387
|
+
retry_count: nil)
|
388
|
+
|
389
|
+
update_options = Hash[
|
390
|
+
only_local: only_local,
|
391
|
+
checkout_only: checkout_only,
|
392
|
+
ignore_errors: ignore_errors,
|
393
|
+
reset: reset,
|
394
|
+
retry_count: retry_count]
|
396
395
|
|
397
|
-
# Load the installation's manifest a first time, to check if we should
|
398
|
-
# update it ... We assume that the OS dependencies for this VCS is already
|
399
|
-
# installed (i.e. that the user did not remove it)
|
400
396
|
if ws.manifest.vcs && !ws.manifest.vcs.local?
|
401
|
-
update_main_configuration(
|
397
|
+
update_main_configuration(**update_options)
|
402
398
|
end
|
403
399
|
ws.load_main_initrb
|
404
400
|
ws.manifest.load(manifest_path)
|
405
|
-
|
406
|
-
root_pkg_set = ws.manifest.local_package_set
|
401
|
+
root_pkg_set = ws.manifest.main_package_set
|
407
402
|
root_pkg_set.load_description_file
|
408
403
|
root_pkg_set.explicit = true
|
409
|
-
|
404
|
+
update_package_sets(**update_options)
|
405
|
+
end
|
406
|
+
|
407
|
+
def update_package_sets(**update_options)
|
408
|
+
root_pkg_set = ws.manifest.main_package_set
|
409
|
+
package_sets = load_and_update_package_sets(root_pkg_set, **update_options)
|
410
410
|
root_pkg_set.imports.each do |pkg_set|
|
411
411
|
pkg_set.explicit = true
|
412
412
|
end
|
413
413
|
package_sets = sort_package_sets_by_import_order(package_sets, root_pkg_set)
|
414
|
+
ws.manifest.reset_package_sets
|
414
415
|
package_sets.each do |pkg_set|
|
415
416
|
ws.manifest.register_package_set(pkg_set)
|
416
417
|
end
|
data/lib/autoproj/ops/import.rb
CHANGED
@@ -13,9 +13,9 @@ def mark_exclusion_along_revdeps(pkg_name, revdeps, chain = [], reason = nil)
|
|
13
13
|
reason = ws.manifest.exclusion_reason(pkg_name)
|
14
14
|
else
|
15
15
|
if chain.size == 1
|
16
|
-
ws.manifest.
|
16
|
+
ws.manifest.exclude_package(pkg_name, "its dependency #{reason}")
|
17
17
|
else
|
18
|
-
ws.manifest.
|
18
|
+
ws.manifest.exclude_package(pkg_name, "#{reason} (dependency chain: #{chain.join(">")})")
|
19
19
|
end
|
20
20
|
end
|
21
21
|
|
@@ -34,11 +34,11 @@ def import_next_step(pkg, reverse_dependencies)
|
|
34
34
|
new_packages = []
|
35
35
|
pkg.dependencies.each do |dep_name|
|
36
36
|
reverse_dependencies[dep_name] << pkg.name
|
37
|
-
new_packages << ws.manifest.
|
37
|
+
new_packages << ws.manifest.find_package_definition(dep_name)
|
38
38
|
end
|
39
39
|
pkg_opt_deps, pkg_opt_os_deps = pkg.partition_optional_dependencies
|
40
40
|
pkg_opt_deps.each do |dep_name|
|
41
|
-
new_packages << ws.manifest.
|
41
|
+
new_packages << ws.manifest.find_package_definition(dep_name)
|
42
42
|
end
|
43
43
|
|
44
44
|
# Handle OS dependencies, excluding the package if some
|
@@ -82,7 +82,9 @@ def pre_package_import(selection, manifest, pkg, reverse_dependencies)
|
|
82
82
|
|
83
83
|
def post_package_import(selection, manifest, pkg, reverse_dependencies)
|
84
84
|
Rake::Task["#{pkg.name}-import"].instance_variable_set(:@already_invoked, true)
|
85
|
-
|
85
|
+
if pkg.checked_out?
|
86
|
+
manifest.load_package_manifest(pkg.name)
|
87
|
+
end
|
86
88
|
|
87
89
|
# The package setup mechanisms might have added an exclusion
|
88
90
|
# on this package. Handle this.
|
@@ -95,35 +97,82 @@ def post_package_import(selection, manifest, pkg, reverse_dependencies)
|
|
95
97
|
elsif manifest.ignored?(pkg.name)
|
96
98
|
false
|
97
99
|
else
|
98
|
-
|
99
|
-
|
100
|
+
if pkg.checked_out?
|
101
|
+
Autoproj.each_post_import_block(pkg) do |block|
|
102
|
+
block.call(pkg)
|
103
|
+
end
|
100
104
|
end
|
101
105
|
import_next_step(pkg, reverse_dependencies)
|
102
106
|
end
|
103
107
|
end
|
104
108
|
|
109
|
+
# Install the VCS osdep for the given packages
|
110
|
+
#
|
111
|
+
# @param [Hash] osdeps_options the options that will be passed to
|
112
|
+
# {Workspace#install_os_packages}
|
113
|
+
# @return [Set] the set of installed OS packages
|
114
|
+
def install_vcs_packages_for(*packages, **osdeps_options)
|
115
|
+
vcs_to_install = packages.map { |pkg| pkg.vcs.type }.uniq
|
116
|
+
# This assumes that the VCS packages do not depend on a
|
117
|
+
# 'strict' package mangers such as e.g. BundlerManager
|
118
|
+
ws.install_os_packages(vcs_to_install, all: nil, **osdeps_options)
|
119
|
+
vcs_to_install
|
120
|
+
end
|
121
|
+
|
122
|
+
# @api private
|
123
|
+
#
|
124
|
+
# Queue the work necessary to import the given package, making sure
|
125
|
+
# that the execution results end up in a given queue
|
126
|
+
#
|
127
|
+
# @param executor the future executor
|
128
|
+
# @param [Queue] completion_queue the queue where the completion
|
129
|
+
# results should be pushed, as a (package, time, result,
|
130
|
+
# error_reason) tuple
|
131
|
+
# @param [Integer] retry_count the number of retries that are
|
132
|
+
# allowed. Set to zero for no retry
|
133
|
+
# @param [Hash] import_options options passed to {Autobuild::Importer#import}
|
134
|
+
def queue_import_work(executor, completion_queue, pkg, retry_count: nil, **import_options)
|
135
|
+
import_future = Concurrent::Future.new(executor: executor, args: [pkg]) do |import_pkg|
|
136
|
+
## COMPLETELY BYPASS RAKE HERE
|
137
|
+
# The reason is that the ordering of import/prepare between
|
138
|
+
# packages is not important BUT the ordering of import vs.
|
139
|
+
# prepare in one package IS important: prepare is the method
|
140
|
+
# that takes into account dependencies.
|
141
|
+
if retry_count
|
142
|
+
import_pkg.autobuild.importer.retry_count = retry_count
|
143
|
+
end
|
144
|
+
import_pkg.autobuild.import(**import_options)
|
145
|
+
end
|
146
|
+
import_future.add_observer do |time, result, reason|
|
147
|
+
completion_queue << [pkg, time, result, reason]
|
148
|
+
end
|
149
|
+
import_future.execute
|
150
|
+
end
|
151
|
+
|
105
152
|
class ImportFailed < RuntimeError; end
|
106
153
|
|
107
154
|
# Import all packages from the given selection, and their
|
108
155
|
# dependencies
|
109
|
-
def import_selected_packages(selection, updated_packages,
|
110
|
-
|
111
|
-
|
156
|
+
def import_selected_packages(selection, updated_packages,
|
157
|
+
parallel: ws.config.parallel_import_level,
|
158
|
+
recursive: true,
|
159
|
+
retry_count: nil,
|
160
|
+
ignore_errors: false,
|
161
|
+
install_vcs_packages: Hash.new,
|
162
|
+
non_imported_packages: :checkout,
|
163
|
+
**import_options)
|
164
|
+
|
165
|
+
if ![:checkout, :ignore, :return].include?(non_imported_packages)
|
166
|
+
raise ArgumentError, "invalid value for 'non_imported_packages'. Expected one of :checkout, :ignore or :return but got #{non_imported_packages}"
|
167
|
+
end
|
112
168
|
|
113
169
|
# This is used in the ensure block, initialize as early as
|
114
170
|
# possible
|
115
|
-
executor = Concurrent::FixedThreadPool.new(
|
116
|
-
|
117
|
-
options, import_options = Kernel.filter_options options,
|
118
|
-
recursive: true,
|
119
|
-
retry_count: nil
|
120
|
-
|
121
|
-
ignore_errors = options[:ignore_errors]
|
122
|
-
retry_count = options[:retry_count]
|
171
|
+
executor = Concurrent::FixedThreadPool.new(parallel, max_length: 0)
|
123
172
|
manifest = ws.manifest
|
124
173
|
|
125
174
|
selected_packages = selection.each_source_package_name.map do |pkg_name|
|
126
|
-
manifest.
|
175
|
+
manifest.find_package_definition(pkg_name)
|
127
176
|
end.to_set
|
128
177
|
|
129
178
|
# The reverse dependencies for the package tree. It is discovered as
|
@@ -138,69 +187,96 @@ def import_selected_packages(selection, updated_packages, options = Hash.new)
|
|
138
187
|
pending_packages = Set.new
|
139
188
|
# The set of all packages that are currently selected by +selection+
|
140
189
|
all_processed_packages = Set.new
|
141
|
-
|
190
|
+
main_thread_imports = Array.new
|
142
191
|
package_queue = selected_packages.to_a.sort_by(&:name)
|
143
192
|
failures = Hash.new
|
193
|
+
missing_vcs = Array.new
|
194
|
+
installed_vcs_packages = Set['none', 'local']
|
144
195
|
while failures.empty? || ignore_errors
|
145
196
|
# Queue work for all packages in the queue
|
146
197
|
package_queue.each do |pkg|
|
147
198
|
# Remove packages that have already been processed
|
148
199
|
next if all_processed_packages.include?(pkg)
|
200
|
+
if (non_imported_packages != :checkout) && !File.directory?(pkg.autobuild.srcdir)
|
201
|
+
if non_imported_packages == :return
|
202
|
+
all_processed_packages << pkg
|
203
|
+
completion_queue << [pkg, Time.now, false, nil]
|
204
|
+
next
|
205
|
+
else
|
206
|
+
all_processed_packages << pkg
|
207
|
+
ws.manifest.ignore_package(pkg.name)
|
208
|
+
next
|
209
|
+
end
|
210
|
+
elsif install_vcs_packages && !installed_vcs_packages.include?(pkg.vcs.type)
|
211
|
+
missing_vcs << pkg
|
212
|
+
next
|
213
|
+
end
|
149
214
|
all_processed_packages << pkg
|
150
215
|
|
151
|
-
|
216
|
+
importer = pkg.autobuild.importer
|
217
|
+
if !pre_package_import(selection, manifest, pkg.autobuild, reverse_dependencies)
|
152
218
|
next
|
153
|
-
elsif !
|
219
|
+
elsif !importer
|
220
|
+
# The validity of this is checked in
|
221
|
+
# pre_package_import
|
154
222
|
completion_queue << [pkg, Time.now, false, nil]
|
155
223
|
next
|
156
|
-
elsif
|
157
|
-
|
224
|
+
elsif importer.interactive?
|
225
|
+
main_thread_imports << pkg
|
226
|
+
next
|
227
|
+
elsif pkg.autobuild.checked_out? && import_options[:checkout_only]
|
228
|
+
main_thread_imports << pkg
|
158
229
|
next
|
159
230
|
end
|
160
231
|
|
161
232
|
pending_packages << pkg
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
import_pkg.importer.retry_count = retry_count
|
170
|
-
end
|
171
|
-
import_pkg.import(import_options.merge(allow_interactive: false))
|
172
|
-
end
|
173
|
-
import_future.add_observer do |time, result, reason|
|
174
|
-
completion_queue << [pkg, time, result, reason]
|
233
|
+
begin
|
234
|
+
queue_import_work(
|
235
|
+
executor, completion_queue, pkg, retry_count: retry_count,
|
236
|
+
**import_options.merge(allow_interactive: false))
|
237
|
+
rescue Exception
|
238
|
+
pending_packages.delete(pkg)
|
239
|
+
raise
|
175
240
|
end
|
176
|
-
|
241
|
+
true
|
177
242
|
end
|
178
243
|
package_queue.clear
|
179
244
|
|
180
245
|
if completion_queue.empty? && pending_packages.empty?
|
246
|
+
if !missing_vcs.empty?
|
247
|
+
installed_vcs_packages.merge(
|
248
|
+
install_vcs_packages_for(*missing_vcs, **install_vcs_packages))
|
249
|
+
package_queue.concat(missing_vcs)
|
250
|
+
missing_vcs.clear
|
251
|
+
next
|
252
|
+
end
|
253
|
+
|
181
254
|
# We've nothing to process anymore ... process
|
182
255
|
# interactive imports if there are some. Otherwise,
|
183
256
|
# we're done
|
184
|
-
if
|
257
|
+
if main_thread_imports.empty?
|
185
258
|
break
|
186
259
|
else
|
187
|
-
|
260
|
+
main_thread_imports.delete_if do |pkg|
|
188
261
|
begin
|
189
|
-
|
262
|
+
if retry_count
|
263
|
+
pkg.autobuild.importer.retry_count = retry_count
|
264
|
+
end
|
265
|
+
result = pkg.autobuild.import(
|
266
|
+
**import_options.merge(allow_interactive: true))
|
190
267
|
rescue Exception => reason
|
191
268
|
end
|
192
269
|
completion_queue << [pkg, Time.now, result, reason]
|
193
270
|
end
|
194
|
-
interactive_imports.clear
|
195
271
|
end
|
196
272
|
end
|
197
273
|
|
198
|
-
# And wait one to finish
|
274
|
+
# And wait for one to finish
|
199
275
|
pkg, _time, _result, reason = completion_queue.pop
|
200
276
|
pending_packages.delete(pkg)
|
201
277
|
if reason
|
202
278
|
if reason.kind_of?(Autobuild::InteractionRequired)
|
203
|
-
|
279
|
+
main_thread_imports << pkg
|
204
280
|
else
|
205
281
|
# One importer failed... terminate
|
206
282
|
Autoproj.error "import of #{pkg.name} failed"
|
@@ -210,13 +286,13 @@ def import_selected_packages(selection, updated_packages, options = Hash.new)
|
|
210
286
|
failures[pkg] = reason
|
211
287
|
end
|
212
288
|
else
|
213
|
-
if new_packages = post_package_import(selection, manifest, pkg, reverse_dependencies)
|
289
|
+
if new_packages = post_package_import(selection, manifest, pkg.autobuild, reverse_dependencies)
|
214
290
|
# Excluded dependencies might have caused the package to be
|
215
291
|
# excluded as well ... do not add any dependency to the
|
216
292
|
# processing queue if it is the case
|
217
293
|
if manifest.excluded?(pkg.name)
|
218
294
|
selection.filter_excluded_and_ignored_packages(manifest)
|
219
|
-
elsif
|
295
|
+
elsif recursive
|
220
296
|
package_queue = new_packages.sort_by(&:name)
|
221
297
|
end
|
222
298
|
end
|
@@ -240,7 +316,12 @@ def import_selected_packages(selection, updated_packages, options = Hash.new)
|
|
240
316
|
executor.shutdown
|
241
317
|
executor.wait_for_termination
|
242
318
|
end
|
243
|
-
|
319
|
+
if all_processed_packages
|
320
|
+
all_updated_packages = all_processed_packages.find_all do |processed_pkg|
|
321
|
+
processed_pkg.autobuild.updated?
|
322
|
+
end
|
323
|
+
updated_packages.concat(all_updated_packages.map(&:name))
|
324
|
+
end
|
244
325
|
end
|
245
326
|
|
246
327
|
def finalize_package_load(processed_packages)
|
@@ -256,8 +337,9 @@ def finalize_package_load(processed_packages)
|
|
256
337
|
|
257
338
|
next if manifest.ignored?(pkg_name) || manifest.excluded?(pkg_name)
|
258
339
|
|
259
|
-
|
260
|
-
|
340
|
+
pkg_definition = manifest.find_package_definition(pkg_name)
|
341
|
+
pkg = pkg_definition.autobuild
|
342
|
+
if !processed_packages.include?(pkg_definition) && File.directory?(pkg.srcdir)
|
261
343
|
manifest.load_package_manifest(pkg.name)
|
262
344
|
Autoproj.each_post_import_block(pkg) do |block|
|
263
345
|
block.call(pkg)
|
@@ -286,37 +368,45 @@ def finalize_package_load(processed_packages)
|
|
286
368
|
all
|
287
369
|
end
|
288
370
|
|
289
|
-
def import_packages(selection,
|
371
|
+
def import_packages(selection,
|
372
|
+
non_imported_packages: :checkout,
|
373
|
+
warn_about_ignored_packages: true,
|
374
|
+
warn_about_excluded_packages: true,
|
375
|
+
recursive: true,
|
376
|
+
ignore_errors: false,
|
377
|
+
install_vcs_packages: Hash.new,
|
378
|
+
**import_options)
|
379
|
+
|
290
380
|
# Used in the ensure block, initialize as soon as possible
|
291
381
|
updated_packages = Array.new
|
292
382
|
|
293
|
-
options, import_options = Kernel.filter_options options,
|
294
|
-
warn_about_ignored_packages: true,
|
295
|
-
warn_about_excluded_packages: true,
|
296
|
-
recursive: true
|
297
|
-
|
298
383
|
manifest = ws.manifest
|
299
384
|
|
300
385
|
all_processed_packages = import_selected_packages(
|
301
|
-
selection, updated_packages,
|
386
|
+
selection, updated_packages,
|
387
|
+
non_imported_packages: non_imported_packages,
|
388
|
+
ignore_errors: ignore_errors,
|
389
|
+
recursive: recursive,
|
390
|
+
install_vcs_packages: install_vcs_packages,
|
391
|
+
**import_options)
|
302
392
|
finalize_package_load(all_processed_packages)
|
303
393
|
|
304
394
|
all_enabled_osdeps = selection.each_osdep_package_name.to_set
|
305
395
|
all_enabled_sources = all_processed_packages.map(&:name)
|
306
|
-
if
|
396
|
+
if recursive
|
307
397
|
all_processed_packages.each do |pkg|
|
308
|
-
all_enabled_osdeps.merge(pkg.os_packages)
|
398
|
+
all_enabled_osdeps.merge(pkg.autobuild.os_packages)
|
309
399
|
end
|
310
400
|
end
|
311
401
|
|
312
|
-
if
|
402
|
+
if warn_about_excluded_packages
|
313
403
|
selection.exclusions.each do |sel, pkg_names|
|
314
404
|
pkg_names.sort.each do |pkg_name|
|
315
405
|
Autoproj.warn "#{pkg_name}, which was selected for #{sel}, cannot be built: #{manifest.exclusion_reason(pkg_name)}", :bold
|
316
406
|
end
|
317
407
|
end
|
318
408
|
end
|
319
|
-
if
|
409
|
+
if warn_about_ignored_packages
|
320
410
|
selection.ignores.each do |sel, pkg_names|
|
321
411
|
pkg_names.sort.each do |pkg_name|
|
322
412
|
Autoproj.warn "#{pkg_name}, which was selected for #{sel}, is ignored", :bold
|