autoproj 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,12 @@
1
+ git:
2
+ debian: git-core
3
+
4
+ svn:
5
+ debian: svn
6
+
7
+ cmake:
8
+ debian: cmake
9
+
10
+ autotools:
11
+ debian: [automake1.9, autoconf]
12
+
@@ -0,0 +1,697 @@
1
+ require 'yaml'
2
+ require 'utilrb/kernel/options'
3
+ require 'nokogiri'
4
+ require 'set'
5
+
6
+ module Autobuild
7
+ class Package
8
+ def os_packages
9
+ @os_packages || Array.new
10
+ end
11
+ def depends_on_os_package(name)
12
+ @os_packages ||= Array.new
13
+ @os_packages << name
14
+ end
15
+ end
16
+ end
17
+
18
+ module Autoproj
19
+ @build_system_dependencies = Set.new
20
+ def self.add_build_system_dependency(*names)
21
+ @build_system_dependencies |= names.to_set
22
+ end
23
+ class << self
24
+ attr_reader :build_system_dependencies
25
+ end
26
+
27
+ def self.expand_environment(value)
28
+ # Perform constant expansion on the defined environment variables,
29
+ # including the option set
30
+ options = Autoproj.option_set
31
+ if Autoproj.manifest
32
+ loop do
33
+ new_value = Autoproj.manifest.single_expansion(value, options)
34
+ if new_value == value
35
+ break
36
+ else
37
+ value = new_value
38
+ end
39
+ end
40
+ else
41
+ value
42
+ end
43
+ end
44
+
45
+ @env_inherit = Set.new
46
+ def self.env_inherit?(name)
47
+ @env_inherit.include?(name)
48
+ end
49
+ def self.env_inherit(*names)
50
+ @env_inherit |= names
51
+ end
52
+
53
+ # Set a new environment variable
54
+ def self.env_set(name, *value)
55
+ Autobuild.environment.delete(name)
56
+ env_add(name, *value)
57
+ end
58
+ def self.env_add(name, *value)
59
+ value = value.map { |v| expand_environment(v) }
60
+ Autobuild.env_add(name, *value)
61
+ end
62
+ def self.env_set_path(name, *value)
63
+ Autobuild.environment.delete(name)
64
+ env_add_path(name, *value)
65
+ end
66
+ def self.env_add_path(name, *value)
67
+ value = value.map { |v| expand_environment(v) }
68
+ Autobuild.env_add_path(name, *value)
69
+ end
70
+
71
+ class VCSDefinition
72
+ attr_reader :type
73
+ attr_reader :url
74
+ attr_reader :options
75
+
76
+ def initialize(type, url, options)
77
+ @type, @url, @options = type, url, options
78
+ if type != "local" && !Autobuild.respond_to?(type)
79
+ raise ConfigError, "version control #{type} is unknown to autoproj"
80
+ end
81
+ end
82
+
83
+ def local?
84
+ @type == 'local'
85
+ end
86
+
87
+ def create_autobuild_importer
88
+ url = Autoproj.single_expansion(self.url, 'HOME' => ENV['HOME'])
89
+ if url && url !~ /^(\w+:\/)?\/|^\w+\@/
90
+ url = File.expand_path(url, Autoproj.root_dir)
91
+ end
92
+ Autobuild.send(type, url, options)
93
+ end
94
+
95
+ def to_s; "#{type}:#{url}" end
96
+ end
97
+
98
+ def self.vcs_definition_to_hash(spec)
99
+ if spec.respond_to?(:to_str)
100
+ vcs, *url = spec.to_str.split ':'
101
+ spec = if url.empty?
102
+ source_dir = File.expand_path(File.join(Autoproj.config_dir, spec))
103
+ if !File.directory?(source_dir)
104
+ raise ConfigError, "'#{spec.inspect}' is neither a remote source specification, nor a local source definition"
105
+ end
106
+
107
+ Hash[:type => 'local', :url => source_dir]
108
+ else
109
+ Hash[:type => vcs.to_str, :url => url.join(":").to_str]
110
+ end
111
+ end
112
+
113
+ spec, vcs_options = Kernel.filter_options spec, :type => nil, :url => nil
114
+
115
+ return spec.merge(vcs_options)
116
+ end
117
+
118
+ # Autoproj configuration files accept VCS definitions in three forms:
119
+ # * as a plain string, which is a relative/absolute path
120
+ # * as a plain string, which is a vcs_type:url string
121
+ # * as a hash
122
+ #
123
+ # This method normalizes the three forms into a VCSDefinition object
124
+ def self.normalize_vcs_definition(spec)
125
+ spec = vcs_definition_to_hash(spec)
126
+ if !(spec[:type] && spec[:url])
127
+ raise ConfigError, "the source specification #{spec.inspect} misses either the VCS type or an URL"
128
+ end
129
+
130
+ spec, vcs_options = Kernel.filter_options spec, :type => nil, :url => nil
131
+ return VCSDefinition.new(spec[:type], spec[:url], vcs_options)
132
+ end
133
+
134
+ def self.single_expansion(data, definitions)
135
+ definitions.each do |name, expanded|
136
+ data = data.gsub /\$#{Regexp.quote(name)}\b/, expanded
137
+ end
138
+ data
139
+ end
140
+
141
+ # A source is a version control repository which contains general source
142
+ # information with package version control information (source.yml file),
143
+ # package definitions (.autobuild files), and finally definition of
144
+ # dependencies that are provided by the operating system (.osdeps file).
145
+ class Source
146
+ # The VCSDefinition object that defines the version control holding
147
+ # information for this source. Local sources (i.e. the ones that are not
148
+ # under version control) use the 'local' version control name. For them,
149
+ # local? returns true.
150
+ attr_accessor :vcs
151
+ attr_reader :source_definition
152
+ attr_reader :constants_definitions
153
+
154
+ # Create this source from a VCSDefinition object
155
+ def initialize(vcs)
156
+ @vcs = vcs
157
+ end
158
+
159
+ # True if this source has already been checked out on the local autoproj
160
+ # installation
161
+ def present?; File.directory?(local_dir) end
162
+ # True if this source is local, i.e. is not under a version control
163
+ def local?; vcs.local? end
164
+ # The directory in which data for this source will be checked out
165
+ def local_dir
166
+ if local?
167
+ vcs.url
168
+ else
169
+ File.join(Autoproj.config_dir, "remotes", automatic_name)
170
+ end
171
+ end
172
+
173
+ # A name generated from the VCS url
174
+ def automatic_name
175
+ vcs.to_s.gsub(/[^\w]/, '_')
176
+ end
177
+
178
+ # Returns the source name
179
+ def name
180
+ if @source_definition then
181
+ @source_definition['name'] || automatic_name
182
+ else
183
+ automatic_name
184
+ end
185
+ end
186
+
187
+ def raw_description_file
188
+ if !present?
189
+ raise InternalError, "source #{vcs} has not been fetched yet, cannot load description for it"
190
+ end
191
+
192
+ source_file = File.join(local_dir, "source.yml")
193
+ if !File.exists?(source_file)
194
+ raise ConfigError, "source #{vcs.type}:#{vcs.url} should have a source.yml file, but does not"
195
+ end
196
+
197
+ begin
198
+ source_definition = YAML.load(File.read(source_file))
199
+ rescue ArgumentError => e
200
+ raise ConfigError, "error in #{source_file}: #{e.message}"
201
+ end
202
+
203
+ if !source_definition
204
+ raise ConfigError, "#{source_file} does not have a 'name' field"
205
+ end
206
+
207
+ source_definition
208
+ end
209
+
210
+ # Load the source.yml file that describes this source, and resolve the
211
+ # $BLABLA values that are in there. Use #raw_description_file to avoid
212
+ # resolving those values
213
+ def load_description_file
214
+ @source_definition = raw_description_file
215
+
216
+ # Compute the definition of constants
217
+ begin
218
+ constants = source_definition['constants'] || Hash.new
219
+ constants['HOME'] = ENV['HOME']
220
+
221
+ redo_expansion = true
222
+ @constants_definitions = constants
223
+ while redo_expansion
224
+ redo_expansion = false
225
+ constants.dup.each do |name, url|
226
+ # Extract all expansions in the url
227
+ if url =~ /\$(\w+)/
228
+ expansion_name = $1
229
+
230
+ if constants[expansion_name]
231
+ constants[name] = single_expansion(url)
232
+ else
233
+ begin constants[name] = single_expansion(url,
234
+ expansion_name => Autoproj.user_config(expansion_name))
235
+ rescue ConfigError => e
236
+ raise ConfigError, "constant '#{expansion_name}', used in the definition of '#{name}' is defined nowhere"
237
+ end
238
+ end
239
+ redo_expansion = true
240
+ end
241
+ end
242
+ end
243
+
244
+ rescue ConfigError => e
245
+ raise ConfigError, "#{e.message} in #{File.join(local_dir, "source.yml")}", e.backtrace
246
+ end
247
+ end
248
+
249
+ # True if the given string contains expansions
250
+ def contains_expansion?(string); string =~ /\$/ end
251
+
252
+ def single_expansion(data, additional_expansions = Hash.new)
253
+ Autoproj.single_expansion(data, additional_expansions.merge(constants_definitions))
254
+ end
255
+
256
+ # Expands the given string as much as possible using the expansions
257
+ # listed in the source.yml file, and returns it. Raises if not all
258
+ # variables can be expanded.
259
+ def expand(data, additional_expansions = Hash.new)
260
+ if !source_definition
261
+ load_description_file
262
+ end
263
+
264
+ if data.respond_to?(:to_hash)
265
+ data.dup.each do |name, value|
266
+ data[name] = expand(value, additional_expansions)
267
+ end
268
+ else
269
+ data = single_expansion(data, additional_expansions)
270
+ if contains_expansion?(data)
271
+ raise ConfigError, "some expansions are not defined in #{data.inspect}"
272
+ end
273
+ end
274
+
275
+ data
276
+ end
277
+
278
+ # Returns an importer definition for the given package, if one is
279
+ # available. Otherwise returns nil.
280
+ #
281
+ # The returned value is a VCSDefinition object.
282
+ def importer_definition_for(package_name)
283
+ urls = source_definition['urls'] || Hash.new
284
+ urls['HOME'] = ENV['HOME']
285
+
286
+ all_vcs = source_definition['version_control']
287
+ if all_vcs && !all_vcs.kind_of?(Array)
288
+ raise ConfigError, "wrong format for the version_control field"
289
+ end
290
+
291
+ vcs_spec = Hash.new
292
+
293
+ if all_vcs
294
+ all_vcs.each do |spec|
295
+ name, spec = spec.to_a.first
296
+ if Regexp.new(name) =~ package_name
297
+ vcs_spec = vcs_spec.merge(spec)
298
+ end
299
+ end
300
+ end
301
+
302
+ if !vcs_spec.empty?
303
+ expansions = Hash["PACKAGE" => package_name]
304
+
305
+ vcs_spec = expand(vcs_spec, expansions)
306
+ vcs_spec = Autoproj.vcs_definition_to_hash(vcs_spec)
307
+ vcs_spec.dup.each do |name, value|
308
+ vcs_spec[name] = expand(value, expansions)
309
+ end
310
+ vcs_spec
311
+
312
+ Autoproj.normalize_vcs_definition(vcs_spec)
313
+ end
314
+ rescue ConfigError => e
315
+ raise ConfigError, "#{e.message} in the source.yml file of #{name} (#{File.join(local_dir, "source.yml")})", e.backtrace
316
+ end
317
+ end
318
+
319
+ class Manifest
320
+ FakePackage = Struct.new :name, :srcdir
321
+ def self.load(file)
322
+ begin
323
+ data = YAML.load(File.read(file))
324
+ rescue ArgumentError => e
325
+ raise ConfigError, "error in #{file}: #{e.message}"
326
+ end
327
+ Manifest.new(file, data)
328
+ end
329
+
330
+ # The manifest data as a Hash
331
+ attr_reader :data
332
+
333
+ # The set of packages defined so far as a mapping from package name to
334
+ # [Autobuild::Package, source, file] tuple
335
+ attr_reader :packages
336
+
337
+ # A mapping from package names into PackageManifest objects
338
+ attr_reader :package_manifests
339
+
340
+ attr_reader :file
341
+
342
+ def initialize(file, data)
343
+ @file = file
344
+ @data = data
345
+ @packages = Hash.new
346
+ @package_manifests = Hash.new
347
+ end
348
+
349
+ # Lists the autobuild files that are part of the sources listed in this
350
+ # manifest
351
+ def each_autobuild_file(source_name = nil, &block)
352
+ if !block_given?
353
+ return enum_for(:each_source_file, source_name)
354
+ end
355
+
356
+ # This looks very inefficient, but it is because source names
357
+ # are contained in the source definition file (source.yml) and
358
+ # we must therefore load that file to check the source name ...
359
+ #
360
+ # And honestly I don't think someone will have 20 000 sources
361
+ done_something = false
362
+ each_source do |source|
363
+ next if source_name && source.name != source_name
364
+ done_something = true
365
+
366
+ Dir.glob(File.join(source.local_dir, "*.autobuild")).each do |file|
367
+ yield(source, file)
368
+ end
369
+ end
370
+
371
+ if source_name && !done_something
372
+ raise ConfigError, "source '#{source_name}' does not exist"
373
+ end
374
+ end
375
+
376
+ def each_osdeps_file
377
+ if !block_given?
378
+ return enum_for(:each_source_file)
379
+ end
380
+
381
+ each_source do |source|
382
+ Dir.glob(File.join(source.local_dir, "*.osdeps")).each do |file|
383
+ yield(source, file)
384
+ end
385
+ end
386
+ end
387
+
388
+ def has_remote_sources?
389
+ each_remote_source(false).any? { true }
390
+ end
391
+
392
+ # Like #each_source, but filters out local sources
393
+ def each_remote_source(load_description = true)
394
+ if !block_given?
395
+ enum_for(:each_remote_source, load_description)
396
+ else
397
+ each_source(load_description) do |source|
398
+ if !source.local?
399
+ yield(source)
400
+ end
401
+ end
402
+ end
403
+ end
404
+
405
+ # call-seq:
406
+ # each_source { |source_description| ... }
407
+ #
408
+ # Lists all sources defined in this manifest, by yielding a Source
409
+ # object that describes the source.
410
+ def each_source(load_description = true)
411
+ if !block_given?
412
+ return enum_for(:each_source)
413
+ end
414
+
415
+ return if !data['sources']
416
+
417
+ data['sources'].each do |spec|
418
+ # Look up for short notation (i.e. not an explicit hash). It is
419
+ # either vcs_type:url or just url. In the latter case, we expect
420
+ # 'url' to be a path to a local directory
421
+ vcs_def = begin
422
+ Autoproj.normalize_vcs_definition(spec)
423
+ rescue ConfigError => e
424
+ raise ConfigError, "in #{file}: #{e.message}"
425
+ end
426
+
427
+ source = Source.new(vcs_def)
428
+ if source.present? && load_description
429
+ source.load_description_file
430
+ end
431
+
432
+ yield(source)
433
+ end
434
+ end
435
+
436
+ # Register a new package
437
+ def register_package(package, source, file)
438
+ @packages[package.name] = [package, source, file]
439
+ end
440
+
441
+ def definition_source(package_name)
442
+ @packages[package_name][1]
443
+ end
444
+ def definition_file(package_name)
445
+ @packages[package_name][2]
446
+ end
447
+
448
+ # Lists all defined packages and where they have been defined
449
+ def each_package
450
+ if !block_given?
451
+ return enum_for(:each_package)
452
+ end
453
+ packages.each_value { |package, _| yield(package) }
454
+ end
455
+
456
+ def self.update_remote_source(source)
457
+ importer = source.vcs.create_autobuild_importer
458
+ fake_package = FakePackage.new(source.automatic_name, source.local_dir)
459
+
460
+ importer.import(fake_package)
461
+ end
462
+
463
+ def update_remote_sources
464
+ # Iterate on the remote sources, without loading the source.yml
465
+ # file (we're not ready for that yet)
466
+ each_remote_source(false) do |source|
467
+ Manifest.update_remote_source(source)
468
+ end
469
+ end
470
+
471
+ # Sets up the package importers based on the information listed in
472
+ # the source's source.yml
473
+ #
474
+ # The priority logic is that we take the sources one by one in the order
475
+ # listed in the autoproj main manifest, and first come first used.
476
+ #
477
+ # A source that defines a particular package in its autobuild file
478
+ # *must* provide the corresponding VCS line in its source.yml file.
479
+ # However, it is possible for a source that does *not* define a package
480
+ # to override the VCS
481
+ #
482
+ # In other words: if package P is defined by source S1, and source S0
483
+ # imports S1, then
484
+ # * S1 must have a VCS line for P
485
+ # * S0 can have a VCS line for P, which would override the one defined
486
+ # by S1
487
+ def load_importers
488
+ packages.each_value do |package, package_source, package_source_file|
489
+ vcs = each_source.find do |source|
490
+ vcs = source.importer_definition_for(package.name)
491
+ if vcs
492
+ break(vcs)
493
+ elsif package_source.name == source.name
494
+ break
495
+ end
496
+ end
497
+
498
+ if vcs
499
+ Autoproj.add_build_system_dependency vcs.type
500
+ package.importer = vcs.create_autobuild_importer
501
+ else
502
+ raise ConfigError, "source #{package_source.name} defines #{package.name}, but does not provide a version control definition for it"
503
+ end
504
+ end
505
+ end
506
+
507
+ # +name+ can either be the name of a source or the name of a package. In
508
+ # the first case, we return all packages defined by that source. In the
509
+ # latter case, we return the singleton array [name]
510
+ def resolve_package_set(name)
511
+ if Autobuild::Package[name]
512
+ [name]
513
+ else
514
+ source = each_source.find { |source| source.name == name }
515
+ if !source
516
+ raise ConfigError, "#{name} is neither a package nor a source"
517
+ end
518
+ packages.values.find_all { |pkg, pkg_src, _| pkg_src.name == source.name }.
519
+ map { |pkg, _| pkg.name }
520
+ end
521
+ end
522
+
523
+ # Returns the packages contained in the provided layout definition
524
+ #
525
+ # If recursive is false, yields only the packages at this level.
526
+ # Otherwise, return all packages.
527
+ def layout_packages(layout_def, recursive)
528
+ result = []
529
+ layout_def.each do |value|
530
+ if !value.kind_of?(Hash) # sublayout
531
+ result.concat(resolve_package_set(value))
532
+ end
533
+ end
534
+
535
+ if recursive
536
+ each_sublayout(layout_def) do |sublayout_name, sublayout_def|
537
+ result.concat(layout_packages(sublayout_def, true))
538
+ end
539
+ end
540
+
541
+ result
542
+ end
543
+
544
+ def each_sublayout(layout_def)
545
+ layout_def.each do |value|
546
+ if value.kind_of?(Hash)
547
+ name, layout = value.find { true }
548
+ yield(name, layout)
549
+ end
550
+ end
551
+ end
552
+
553
+ # Looks into the layout setup in the manifest, and yields each layout
554
+ # and sublayout in order
555
+ def each_package_set(selection, layout_name = '/', layout_def = data['layout'], &block)
556
+ if !layout_def
557
+ yield('', default_packages, default_packages)
558
+ return nil
559
+ end
560
+
561
+ selection = selection.to_set
562
+
563
+ # First of all, do the packages at this level
564
+ packages = layout_packages(layout_def, false)
565
+ if selection && !selection.any? { |sel| layout_name =~ /^\/?#{Regexp.new(sel)}\/?/ }
566
+ selected_packages = packages.find_all { |pkg_name| selection.include?(pkg_name) }
567
+ else
568
+ selected_packages = packages.dup
569
+ end
570
+ if !packages.empty?
571
+ yield(layout_name, packages.to_set, selected_packages.to_set)
572
+ end
573
+
574
+ # Now, enumerate the sublayouts
575
+ each_sublayout(layout_def) do |subname, sublayout|
576
+ each_package_set(selection, "#{layout_name}#{subname}/", sublayout, &block)
577
+ end
578
+ end
579
+
580
+ def default_packages
581
+ names = if layout = data['layout']
582
+ layout_packages(layout, true)
583
+ else
584
+ # No layout, all packages are selected
585
+ packages.values.map do |package, source, _|
586
+ package.name
587
+ end
588
+ end
589
+ names.to_set
590
+ end
591
+
592
+ # Loads the package's manifest.xml files, and extracts dependency
593
+ # information from them. The dependency information is then used applied
594
+ # to the autobuild packages.
595
+ #
596
+ # Right now, the absence of a manifest makes autoproj only issue a
597
+ # warning. This will later be changed into an error.
598
+ def load_package_manifests(selected_packages)
599
+ packages.each_value do |package, source, file|
600
+ next unless selected_packages.include?(package.name)
601
+ manifest_path = File.join(package.srcdir, "manifest.xml")
602
+ if !File.file?(manifest_path)
603
+ Autoproj.warn "#{package.name} from #{source.name} does not have a manifest"
604
+ next
605
+ end
606
+
607
+ manifest = PackageManifest.load(package, manifest_path)
608
+ package_manifests[package.name] = manifest
609
+
610
+ manifest.each_package_dependency do |name|
611
+ if Autoproj.verbose
612
+ STDERR.puts " #{package.name} depends on #{name}"
613
+ end
614
+ begin
615
+ package.depends_on name
616
+ rescue Autobuild::ConfigException => e
617
+ raise ConfigError, "manifest of #{package.name} from #{source.name} lists '#{name}' as dependency, but this package does not exist (manifest file: #{manifest_path})"
618
+ end
619
+ end
620
+ end
621
+ end
622
+
623
+ AUTOPROJ_OSDEPS = File.join(File.expand_path(File.dirname(__FILE__)), 'default.osdeps')
624
+ # Returns an OSDependencies instance that defined the known OS packages,
625
+ # as well as how to install them
626
+ def known_os_packages
627
+ osdeps = OSDependencies.load(AUTOPROJ_OSDEPS)
628
+
629
+ each_osdeps_file do |source, file|
630
+ osdeps.merge(OSDependencies.load(file))
631
+ end
632
+ osdeps
633
+ end
634
+
635
+ def install_os_dependencies
636
+ osdeps = known_os_packages
637
+
638
+ all_packages = Set.new
639
+ package_manifests.each_value do |pkg_manifest|
640
+ all_packages |= pkg_manifest.each_os_dependency.to_set
641
+ end
642
+
643
+ osdeps.install(all_packages)
644
+ end
645
+ end
646
+
647
+ # The singleton manifest object on which the current run works
648
+ class << self
649
+ attr_accessor :manifest
650
+ end
651
+
652
+ class PackageManifest
653
+ def self.load(package, file)
654
+ doc = Nokogiri::XML(File.read(file))
655
+ PackageManifest.new(package, doc)
656
+ end
657
+
658
+ # The Autobuild::Package instance this manifest applies on
659
+ attr_reader :package
660
+ # The raw XML data as a Nokogiri document
661
+ attr_reader :xml
662
+
663
+ def initialize(package, doc)
664
+ @package = package
665
+ @xml = doc
666
+ end
667
+
668
+ def each_os_dependency
669
+ if block_given?
670
+ xml.xpath('//rosdep').each do |node|
671
+ yield(node['name'])
672
+ end
673
+ package.os_packages.each do |name|
674
+ yield(name)
675
+ end
676
+ else
677
+ enum_for :each_os_dependency
678
+ end
679
+ end
680
+
681
+ def each_package_dependency
682
+ if block_given?
683
+ xml.xpath('//depend').each do |node|
684
+ dependency = node['package']
685
+ if dependency
686
+ yield(dependency)
687
+ else
688
+ raise ConfigError, "manifest of #{package.name} has a <depend> tag without a 'package' attribute"
689
+ end
690
+ end
691
+ else
692
+ enum_for :each_package_dependency
693
+ end
694
+ end
695
+ end
696
+ end
697
+