autobuild 1.19.0 → 1.22.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (42) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/lint.yml +25 -0
  3. data/.github/workflows/test.yml +30 -0
  4. data/.rubocop.yml +14 -7
  5. data/autobuild.gemspec +8 -6
  6. data/bin/autobuild +1 -1
  7. data/lib/autobuild/build_logfile.rb +1 -2
  8. data/lib/autobuild/config.rb +18 -5
  9. data/lib/autobuild/configurable.rb +3 -1
  10. data/lib/autobuild/environment.rb +28 -45
  11. data/lib/autobuild/exceptions.rb +11 -5
  12. data/lib/autobuild/import/archive.rb +31 -22
  13. data/lib/autobuild/import/cvs.rb +6 -6
  14. data/lib/autobuild/import/darcs.rb +4 -4
  15. data/lib/autobuild/import/git-lfs.rb +4 -4
  16. data/lib/autobuild/import/git.rb +149 -70
  17. data/lib/autobuild/import/hg.rb +7 -7
  18. data/lib/autobuild/import/svn.rb +15 -9
  19. data/lib/autobuild/importer.rb +38 -38
  20. data/lib/autobuild/mail_reporter.rb +5 -2
  21. data/lib/autobuild/package.rb +24 -14
  22. data/lib/autobuild/packages/autotools.rb +4 -9
  23. data/lib/autobuild/packages/cmake.rb +16 -7
  24. data/lib/autobuild/packages/dummy.rb +0 -4
  25. data/lib/autobuild/packages/gnumake.rb +1 -1
  26. data/lib/autobuild/packages/orogen.rb +11 -4
  27. data/lib/autobuild/packages/pkgconfig.rb +2 -2
  28. data/lib/autobuild/packages/python.rb +1 -2
  29. data/lib/autobuild/packages/ruby.rb +5 -5
  30. data/lib/autobuild/parallel.rb +12 -17
  31. data/lib/autobuild/pkgconfig.rb +1 -0
  32. data/lib/autobuild/progress_display.rb +130 -49
  33. data/lib/autobuild/rake_task_extension.rb +6 -0
  34. data/lib/autobuild/reporting.rb +20 -7
  35. data/lib/autobuild/subcommand.rb +24 -23
  36. data/lib/autobuild/test_utility.rb +2 -1
  37. data/lib/autobuild/timestamps.rb +3 -3
  38. data/lib/autobuild/utility.rb +29 -9
  39. data/lib/autobuild/version.rb +1 -1
  40. data/lib/autobuild.rb +0 -3
  41. metadata +42 -26
  42. data/.travis.yml +0 -19
@@ -25,6 +25,7 @@ module Autobuild
25
25
  end
26
26
 
27
27
  attr_writer :full_reconfigures
28
+
28
29
  def full_reconfigures?
29
30
  @full_reconfigures
30
31
  end
@@ -36,8 +37,7 @@ module Autobuild
36
37
  # It can be overriden on a per-package basis with CMake.generator=
37
38
  attr_accessor :generator
38
39
 
39
- attr_reader :prefix_path
40
- attr_reader :module_path
40
+ attr_reader :prefix_path, :module_path
41
41
 
42
42
  # Whether files that are not within CMake's install manifest but are
43
43
  # present in the prefix should be deleted. Note that the contents of
@@ -63,6 +63,7 @@ module Autobuild
63
63
 
64
64
  # a key => value association of defines for CMake
65
65
  attr_reader :defines
66
+
66
67
  # The list of all -D options that should be passed on to CMake
67
68
  def all_defines
68
69
  additional_defines = Hash[
@@ -81,6 +82,7 @@ module Autobuild
81
82
  # Sets a generator explicitely for this component. See #generator and
82
83
  # CMake.generator
83
84
  attr_writer :generator
85
+
84
86
  # The CMake generator to use. You must choose one that generates
85
87
  # Makefiles. If not set for this package explicitely, it is using the
86
88
  # global value CMake.generator.
@@ -267,7 +269,7 @@ module Autobuild
267
269
  run('doc', Autobuild.tool(:doxygen), doxyfile)
268
270
  end
269
271
 
270
- def common_utility_handling(utility, target, start_msg, done_msg)
272
+ def common_utility_handling(utility, target, *args, start_msg, done_msg)
271
273
  utility.source_ref_dir = builddir
272
274
  utility.task do
273
275
  progress_start start_msg, :done_message => done_msg do
@@ -277,7 +279,7 @@ module Autobuild
277
279
  run(utility.name,
278
280
  Autobuild.tool(:make),
279
281
  "-j#{parallel_build_level}",
280
- target,
282
+ target, *args,
281
283
  working_directory: builddir)
282
284
  end
283
285
  yield if block_given?
@@ -295,7 +297,7 @@ module Autobuild
295
297
 
296
298
  def with_tests(target = 'test', &block)
297
299
  common_utility_handling(
298
- test_utility, target,
300
+ test_utility, target, "ARGS=-V",
299
301
  "running tests for %s",
300
302
  "successfully ran tests for %s", &block)
301
303
  end
@@ -320,7 +322,7 @@ module Autobuild
320
322
  end
321
323
  end
322
324
 
323
- def import(options = Hash.new)
325
+ def import(**options)
324
326
  super
325
327
 
326
328
  Dir.glob(File.join(srcdir, "*.pc.in")) do |file|
@@ -410,7 +412,7 @@ module Autobuild
410
412
  in_dir(builddir) do
411
413
  unless File.file?(File.join(srcdir, 'CMakeLists.txt'))
412
414
  raise ConfigException.new(self, 'configure'),
413
- "#{srcdir} contains no CMakeLists.txt file"
415
+ "#{srcdir} contains no CMakeLists.txt file"
414
416
  end
415
417
 
416
418
  command = ["cmake"]
@@ -542,5 +544,12 @@ module Autobuild
542
544
  message "%s: removed #{counter} obsolete files from prefix (cmake)"
543
545
  end
544
546
  end
547
+
548
+ def self_fingerprint
549
+ return unless (base = super)
550
+
551
+ all_defines = self.class.defines.merge(defines).sort_by(&:first)
552
+ Digest::SHA1.hexdigest(base + all_defines.join(""))
553
+ end
545
554
  end
546
555
  end
@@ -8,10 +8,6 @@ module Autobuild
8
8
  "#{srcdir}/#{STAMPFILE}"
9
9
  end
10
10
 
11
- def initialize(*args)
12
- super
13
- end
14
-
15
11
  def import(options = Hash.new); end
16
12
 
17
13
  def prepare
@@ -95,7 +95,7 @@ module Autobuild
95
95
  def self.make_subcommand(pkg, phase, *options, &block)
96
96
  invoke_make_parallel(pkg, Autobuild.tool(:make)) do |*make_parallel_options|
97
97
  pkg.run(phase, Autobuild.tool(:make),
98
- *make_parallel_options, *options, &block)
98
+ *make_parallel_options, *options, &block)
99
99
  end
100
100
  end
101
101
  end
@@ -97,19 +97,19 @@ module Autobuild
97
97
  end
98
98
  end
99
99
 
100
- attr_writer :corba
100
+ attr_writer :corba, :orogen_file
101
+
101
102
  def corba
102
103
  @corba || (@corba.nil? && Orogen.corba)
103
104
  end
104
105
 
105
106
  # Overrides the global Orocos.extended_states for this particular package
106
107
  attr_writer :extended_states
108
+
107
109
  def extended_states
108
110
  @extended_states || (@extended_states.nil? && Orogen.extended_states)
109
111
  end
110
112
 
111
- attr_writer :orogen_file
112
-
113
113
  # Path to the orogen file used for this package
114
114
  #
115
115
  # If not set, the class will look for a .orogen file in the package
@@ -128,7 +128,7 @@ module Autobuild
128
128
  return File.basename(path)
129
129
  end
130
130
  raise ArgumentError,
131
- "cannot find an oroGen specification file in #{srcdir}"
131
+ "cannot find an oroGen specification file in #{srcdir}"
132
132
  end
133
133
  end
134
134
 
@@ -235,6 +235,13 @@ module Autobuild
235
235
  cmdline << "--type-export-policy=#{Orogen.default_type_export_policy}"
236
236
  cmdline << "--transports=#{Orogen.transports.sort.uniq.join(',')}"
237
237
  end
238
+ if version >= "1.2"
239
+ cmdline << "--parallel-codegen=#{parallel_build_level}"
240
+ if (job_server = Autobuild.parallel_task_manager&.job_server)
241
+ fds = "#{job_server.rio.fileno},#{job_server.wio.fileno}"
242
+ cmdline << "--jobserver-auth=#{fds}"
243
+ end
244
+ end
238
245
 
239
246
  # Now, add raw options
240
247
  #
@@ -2,8 +2,7 @@ require 'autobuild/pkgconfig'
2
2
 
3
3
  module Autobuild
4
4
  class InstalledPkgConfig < Package
5
- attr_reader :pkgconfig
6
- attr_reader :prefix
5
+ attr_reader :pkgconfig, :prefix
7
6
 
8
7
  def initialize(name)
9
8
  @pkgconfig = PkgConfig.new(name)
@@ -23,6 +22,7 @@ module Autobuild
23
22
  pcfile
24
23
  end
25
24
  end
25
+
26
26
  def self.installed_pkgconfig(name, &block)
27
27
  InstalledPkgConfig.new(name, &block)
28
28
  end
@@ -9,8 +9,7 @@ module Autobuild
9
9
 
10
10
  # Handler class to build python-based packages
11
11
  class Python < Configurable
12
- attr_accessor :buildflags
13
- attr_accessor :installflags
12
+ attr_accessor :buildflags, :installflags
14
13
 
15
14
  def configurestamp
16
15
  "#{builddir}/configure-autobuild-stamp"
@@ -40,7 +40,7 @@ module Autobuild
40
40
  done_message: 'generated documentation for %s' do
41
41
  run 'doc',
42
42
  Autobuild.tool_in_path('ruby'), '-S',
43
- Autobuild.tool('rake'), rake_doc_task,
43
+ Autobuild.tool('rake'), rake_doc_task,
44
44
  working_directory: srcdir
45
45
  end
46
46
  end
@@ -52,7 +52,7 @@ module Autobuild
52
52
  done_message: 'tests passed for %s' do
53
53
  run 'test',
54
54
  Autobuild.tool_in_path('ruby'), '-S',
55
- Autobuild.tool('rake'), rake_test_task, *rake_test_options,
55
+ Autobuild.tool('rake'), rake_test_task, *rake_test_options,
56
56
  working_directory: srcdir
57
57
  end
58
58
  end
@@ -62,7 +62,7 @@ module Autobuild
62
62
  if setup_task && File.file?(File.join(srcdir, 'Rakefile'))
63
63
  run 'post-install',
64
64
  Autobuild.tool_in_path('ruby'), '-S',
65
- Autobuild.tool('rake'), setup_task,
65
+ Autobuild.tool('rake'), setup_task,
66
66
  working_directory: srcdir
67
67
  end
68
68
  end
@@ -80,7 +80,7 @@ module Autobuild
80
80
  %w[ext tmp].each do |extdir|
81
81
  if File.directory?(extdir)
82
82
  Find.find(extdir) do |file|
83
- next if file !~ /\<Makefile\>|\<CMakeCache.txt\>$/
83
+ next if file !~ /<Makefile>|<CMakeCache.txt>$/
84
84
 
85
85
  FileUtils.rm_rf file
86
86
  end
@@ -94,7 +94,7 @@ module Autobuild
94
94
  begin
95
95
  run 'clean',
96
96
  Autobuild.tool_in_path('ruby'), '-S',
97
- Autobuild.tool('rake'), rake_clean_task,
97
+ Autobuild.tool('rake'), rake_clean_task,
98
98
  working_directory: srcdir
99
99
  rescue Autobuild::SubcommandFailed => e
100
100
  warn "%s: clean failed. If this package does not need a clean target,"
@@ -4,15 +4,10 @@ module Autobuild
4
4
  # Since autobuild does not use task arguments, we don't support them for
5
5
  # simplicity
6
6
  class RakeTaskParallelism
7
- attr_reader :available_workers
8
- attr_reader :finished_workers
9
- attr_reader :workers
10
-
11
- attr_reader :job_server
7
+ attr_reader :available_workers, :finished_workers, :workers, :job_server
12
8
 
13
9
  class JobServer
14
- attr_reader :rio
15
- attr_reader :wio
10
+ attr_reader :rio, :wio
16
11
 
17
12
  def initialize(level)
18
13
  @rio, @wio = IO.pipe
@@ -76,15 +71,12 @@ module Autobuild
76
71
  end
77
72
 
78
73
  class ProcessingState
79
- attr_reader :reverse_dependencies
80
- attr_reader :processed
81
- attr_reader :started_packages
82
- attr_reader :active_tasks
83
- attr_reader :queue
84
- attr_reader :priorities
85
-
86
- def initialize(reverse_dependencies)
74
+ attr_reader :reverse_dependencies, :processed, :started_packages,
75
+ :active_tasks, :queue, :priorities
76
+
77
+ def initialize(reverse_dependencies, completion_callback: proc {})
87
78
  @reverse_dependencies = reverse_dependencies
79
+ @completion_callback = completion_callback
88
80
  @processed = Set.new
89
81
  @active_tasks = Set.new
90
82
  @priorities = Hash.new
@@ -143,6 +135,8 @@ module Autobuild
143
135
  push(candidate, priorities[task])
144
136
  end
145
137
  end
138
+
139
+ @completion_callback.call(task)
146
140
  end
147
141
 
148
142
  def trivial_task?(task)
@@ -153,7 +147,7 @@ module Autobuild
153
147
 
154
148
  # Invokes the provided tasks. Unlike the rake code, this is a toplevel
155
149
  # algorithm that does not use recursion
156
- def invoke_parallel(required_tasks)
150
+ def invoke_parallel(required_tasks, completion_callback: proc {})
157
151
  tasks = Set.new
158
152
  reverse_dependencies = Hash.new { |h, k| h[k] = Set.new }
159
153
  required_tasks.each do |t|
@@ -162,7 +156,8 @@ module Autobuild
162
156
  # The queue is the set of tasks for which all prerequisites have
163
157
  # been successfully executed (or where not needed). I.e. it is the
164
158
  # set of tasks that can be queued for execution.
165
- state = ProcessingState.new(reverse_dependencies)
159
+ state = ProcessingState.new(reverse_dependencies,
160
+ completion_callback: completion_callback)
166
161
  tasks.each do |t|
167
162
  state.push(t) if state.ready?(t)
168
163
  end
@@ -5,6 +5,7 @@ class PkgConfig
5
5
 
6
6
  def initialize(name)
7
7
  @name = name
8
+ super()
8
9
  end
9
10
 
10
11
  def to_s
@@ -1,3 +1,6 @@
1
+ require "concurrent/atomic/atomic_boolean"
2
+ require "concurrent/array"
3
+
1
4
  module Autobuild
2
5
  # Management of the progress display
3
6
  class ProgressDisplay
@@ -5,20 +8,72 @@ module Autobuild
5
8
  @io = io
6
9
  @cursor = TTY::Cursor
7
10
  @last_formatted_progress = []
8
- @progress_messages = []
11
+ @progress_messages = Concurrent::Array.new
9
12
 
10
13
  @silent = false
11
14
  @color = color
12
- @progress_enabled = true
13
15
  @display_lock = Mutex.new
16
+
17
+ @next_progress_display = Time.at(0)
18
+ @progress_mode = :single_line
19
+ @progress_period = 0.1
20
+
21
+ @message_queue = Queue.new
22
+ @forced_progress_display = Concurrent::AtomicBoolean.new(false)
14
23
  end
15
24
 
16
- attr_writer :silent
25
+ def synchronize(&block)
26
+ result = @display_lock.synchronize(&block)
27
+ refresh_display
28
+ result
29
+ end
30
+
31
+ # Set the minimum time between two progress messages
32
+ #
33
+ # @see period
34
+ def progress_period=(period)
35
+ @progress_period = Float(period)
36
+ end
37
+
38
+ # Minimum time between two progress displays
39
+ #
40
+ # This does not affect normal messages
41
+ #
42
+ # @return [Float]
43
+ attr_reader :progress_period
44
+
45
+ # Valid progress modes
46
+ #
47
+ # @see progress_mode=
48
+ PROGRESS_MODES = %I[single_line newline off].freeze
49
+
50
+ # Sets how progress messages will be displayed
51
+ #
52
+ # @param [String] the new mode. Can be either 'single_line', where a
53
+ # progress message replaces the last one, 'newline' which displays
54
+ # each on a new line or 'off' to disable progress messages altogether
55
+ def progress_mode=(mode)
56
+ mode = mode.to_sym
57
+ unless PROGRESS_MODES.include?(mode)
58
+ raise ArgumentError,
59
+ "#{mode} is not a valid mode, expected one of "\
60
+ "#{PROGRESS_MODES.join(', ')}"
61
+ end
62
+ @progress_mode = mode
63
+ end
64
+
65
+ # Return the current display mode
66
+ #
67
+ # @return [Symbol]
68
+ # @see mode=
69
+ attr_reader :progress_mode
17
70
 
18
71
  def silent?
19
72
  @silent
20
73
  end
21
74
 
75
+ attr_writer :silent
76
+
22
77
  def silent
23
78
  silent = @silent
24
79
  @silent = true
@@ -27,24 +82,23 @@ module Autobuild
27
82
  @silent = silent
28
83
  end
29
84
 
30
- attr_writer :progress_enabled
85
+ # @deprecated use progress_mode= instead
86
+ def progress_enabled=(flag)
87
+ self.progress_mode = flag ? :single_line : :off
88
+ end
31
89
 
90
+ # Whether progress messages will be displayed at all
32
91
  def progress_enabled?
33
- !@silent && @progress_enabled
92
+ !@silent && (@progress_mode != :off)
34
93
  end
35
94
 
36
95
  def message(message, *args, io: @io, force: false)
37
96
  return if silent? && !force
38
97
 
39
98
  io = args.pop if args.last.respond_to?(:to_io)
99
+ @message_queue << [message, args, io]
40
100
 
41
- @display_lock.synchronize do
42
- io.print "#{@cursor.column(1)}#{@cursor.clear_screen_down}"\
43
- "#{@color.call(message, *args)}\n"
44
- io.flush if @io != io
45
- display_progress
46
- @io.flush
47
- end
101
+ refresh_display
48
102
  end
49
103
 
50
104
  def progress_start(key, *args, done_message: nil)
@@ -53,13 +107,13 @@ module Autobuild
53
107
  formatted_message = @color.call(*args)
54
108
  @progress_messages << [key, formatted_message]
55
109
  if progress_enabled?
56
- @display_lock.synchronize do
57
- display_progress
58
- end
110
+ @forced_progress_display.make_true
59
111
  else
60
112
  message " #{formatted_message}"
61
113
  end
62
114
 
115
+ refresh_display
116
+
63
117
  if block_given?
64
118
  begin
65
119
  result = yield
@@ -73,72 +127,99 @@ module Autobuild
73
127
  end
74
128
 
75
129
  def progress(key, *args)
76
- @display_lock.synchronize do
77
- found = false
78
- @progress_messages.map! do |msg_key, msg|
79
- if msg_key == key
80
- found = true
81
- [msg_key, @color.call(*args)]
82
- else
83
- [msg_key, msg]
84
- end
130
+ found = false
131
+ @progress_messages.map! do |msg_key, msg|
132
+ if msg_key == key
133
+ found = true
134
+ [msg_key, @color.call(*args)]
135
+ else
136
+ [msg_key, msg]
85
137
  end
86
- @progress_messages << [key, @color.call(*args)] unless found
87
- display_progress
88
138
  end
139
+ @progress_messages << [key, @color.call(*args)] unless found
140
+
141
+ refresh_display
89
142
  end
90
143
 
91
144
  def progress_done(key, display_last = true, message: nil)
92
- changed = @display_lock.synchronize do
93
- current_size = @progress_messages.size
94
- @progress_messages.delete_if do |msg_key, msg|
95
- if msg_key == key
96
- message = msg if display_last && !message
97
- true
98
- end
145
+ current_size = @progress_messages.size
146
+ @progress_messages.delete_if do |msg_key, msg|
147
+ if msg_key == key
148
+ message = msg if display_last && !message
149
+ true
99
150
  end
100
- current_size != @progress_messages.size
101
151
  end
152
+ changed = current_size != @progress_messages.size
102
153
 
103
154
  if changed
104
155
  if message
105
156
  message(" #{message}")
106
- # Note: message calls display_progress already
157
+ # NOTE: message updates the display already
107
158
  else
108
- @display_lock.synchronize do
109
- display_progress
110
- end
159
+ refresh_display
111
160
  end
112
161
  true
113
162
  end
114
163
  end
115
164
 
116
- def display_progress
165
+ def refresh_display
166
+ return unless @display_lock.try_lock
167
+
168
+ begin
169
+ refresh_display_under_lock
170
+ ensure
171
+ @display_lock.unlock
172
+ end
173
+ end
174
+
175
+ def refresh_display_under_lock
176
+ # Display queued messages
177
+ until @message_queue.empty?
178
+ message, args, io = @message_queue.pop
179
+ io.print @cursor.clear_screen_down if @progress_mode == :single_line
180
+ io.puts @color.call(message, *args)
181
+
182
+ io.flush if @io != io
183
+ end
184
+
185
+ # And re-display the progress
186
+ display_progress(consider_period: @forced_progress_display.false?)
187
+ @forced_progress_display.make_false
188
+ @io.flush
189
+ end
190
+
191
+ def display_progress(consider_period: true)
117
192
  return unless progress_enabled?
193
+ return if consider_period && (@next_progress_display > Time.now)
118
194
 
119
- formatted = format_grouped_messages(@progress_messages.map(&:last),
120
- indent: " ")
121
- @io.print @cursor.clear_screen_down
122
- @io.print formatted.join("\n")
123
- if formatted.size > 1
124
- @io.print "#{@cursor.up(formatted.size - 1)}#{@cursor.column(0)}"
195
+ formatted = format_grouped_messages(
196
+ @progress_messages.map(&:last),
197
+ indent: " "
198
+ )
199
+ if @progress_mode == :newline
200
+ @io.print formatted.join("\n")
201
+ @io.print "\n"
125
202
  else
203
+ @io.print @cursor.clear_screen_down
204
+ @io.print formatted.join("\n")
205
+ @io.print @cursor.up(formatted.size - 1) if formatted.size > 1
126
206
  @io.print @cursor.column(0)
127
207
  end
128
208
  @io.flush
209
+ @next_progress_display = Time.now + @progress_period
129
210
  end
130
211
 
131
212
  def find_common_prefix(msg, other_msg)
132
- msg = msg.split(" ")
133
- other_msg = other_msg.split(" ")
213
+ msg = msg.split(' ')
214
+ other_msg = other_msg.split(' ')
134
215
  msg.each_with_index do |token, idx|
135
216
  if other_msg[idx] != token
136
217
  prefix = msg[0..(idx - 1)].join(" ")
137
- prefix << " " unless prefix.empty?
218
+ prefix << ' ' unless prefix.empty?
138
219
  return prefix
139
220
  end
140
221
  end
141
- msg.join(" ")
222
+ msg.join(' ')
142
223
  end
143
224
 
144
225
  def group_messages(messages)