omnibus 5.0.0 → 5.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. checksums.yaml +4 -4
  2. data/.rspec +2 -0
  3. data/.travis.yml +1 -0
  4. data/CHANGELOG.md +26 -0
  5. data/Gemfile +3 -0
  6. data/MAINTAINERS.md +1 -0
  7. data/appveyor.yml +1 -1
  8. data/bin/omnibus +5 -0
  9. data/lib/omnibus/builder.rb +165 -26
  10. data/lib/omnibus/digestable.rb +4 -2
  11. data/lib/omnibus/fetcher.rb +18 -5
  12. data/lib/omnibus/fetchers/git_fetcher.rb +38 -22
  13. data/lib/omnibus/fetchers/net_fetcher.rb +106 -37
  14. data/lib/omnibus/fetchers/path_fetcher.rb +13 -12
  15. data/lib/omnibus/file_syncer.rb +33 -14
  16. data/lib/omnibus/generator_files/README.md.erb +1 -1
  17. data/lib/omnibus/generator_files/package_scripts/postinst.erb +3 -3
  18. data/lib/omnibus/generator_files/package_scripts/postrm.erb +1 -1
  19. data/lib/omnibus/generator_files/package_scripts/preinst.erb +1 -1
  20. data/lib/omnibus/generator_files/package_scripts/prerm.erb +3 -3
  21. data/lib/omnibus/git_cache.rb +20 -7
  22. data/lib/omnibus/health_check.rb +144 -12
  23. data/lib/omnibus/packagers/bff.rb +57 -5
  24. data/lib/omnibus/packagers/deb.rb +2 -2
  25. data/lib/omnibus/packagers/pkg.rb +2 -2
  26. data/lib/omnibus/packagers/solaris.rb +18 -6
  27. data/lib/omnibus/project.rb +1 -1
  28. data/lib/omnibus/s3_cache.rb +8 -2
  29. data/lib/omnibus/software.rb +152 -18
  30. data/lib/omnibus/sugarable.rb +1 -5
  31. data/lib/omnibus/util.rb +1 -1
  32. data/lib/omnibus/version.rb +1 -1
  33. data/omnibus.gemspec +4 -1
  34. data/resources/bff/config.erb +7 -0
  35. data/resources/deb/md5sums.erb +1 -1
  36. data/spec/functional/builder_spec.rb +89 -2
  37. data/spec/functional/fetchers/git_fetcher_spec.rb +44 -37
  38. data/spec/functional/fetchers/net_fetcher_spec.rb +36 -5
  39. data/spec/functional/fetchers/path_fetcher_spec.rb +28 -28
  40. data/spec/unit/builder_spec.rb +143 -11
  41. data/spec/unit/fetchers/git_fetcher_spec.rb +23 -59
  42. data/spec/unit/fetchers/net_fetcher_spec.rb +151 -63
  43. data/spec/unit/fetchers/path_fetcher_spec.rb +4 -35
  44. data/spec/unit/git_cache_spec.rb +13 -14
  45. data/spec/unit/health_check_spec.rb +90 -0
  46. data/spec/unit/library_spec.rb +1 -1
  47. data/spec/unit/packagers/bff_spec.rb +126 -3
  48. data/spec/unit/packagers/deb_spec.rb +8 -3
  49. data/spec/unit/packagers/pkg_spec.rb +19 -19
  50. data/spec/unit/packagers/solaris_spec.rb +13 -1
  51. data/spec/unit/software_spec.rb +242 -38
  52. metadata +7 -6
  53. data/lib/omnibus/generator_files/package_scripts/makeselfinst.erb +0 -0
@@ -1,4 +1,4 @@
1
- #
1
+
2
2
  # Copyright 2012-2014 Chef Software, Inc.
3
3
  #
4
4
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,10 +14,18 @@
14
14
  # limitations under the License.
15
15
  #
16
16
 
17
+ require 'omnibus/sugarable'
18
+ begin
19
+ require 'pedump'
20
+ rescue LoadError
21
+ STDERR.puts "pedump not found - windows health checks disabled"
22
+ end
23
+
17
24
  module Omnibus
18
25
  class HealthCheck
19
26
  include Logging
20
27
  include Util
28
+ include Sugarable
21
29
 
22
30
  WHITELIST_LIBS = [
23
31
  /ld-linux/,
@@ -140,6 +148,7 @@ module Omnibus
140
148
  /libelf\.so/,
141
149
  /libkvm\.so/,
142
150
  /libprocstat\.so/,
151
+ /libmd\.so/,
143
152
  ].freeze
144
153
 
145
154
  class << self
@@ -177,16 +186,18 @@ module Omnibus
177
186
  # if the healthchecks pass
178
187
  #
179
188
  def run!
180
- if Ohai['platform'] == 'windows'
181
- log.warn(log_key) { 'Skipping health check on Windows' }
182
- return true
183
- end
184
189
  log.info(log_key) {"Running health on #{project.name}"}
185
190
  bad_libs = case Ohai['platform']
186
191
  when 'mac_os_x'
187
192
  health_check_otool
188
193
  when 'aix'
189
194
  health_check_aix
195
+ when 'windows'
196
+ # TODO: objdump -p will provided a very limited check of
197
+ # explicit dependencies on windows. Most dependencies are
198
+ # implicit and hence not detected.
199
+ log.warn(log_key) { 'Skipping dependency health checks on Windows.' }
200
+ {}
190
201
  else
191
202
  health_check_ldd
192
203
  end
@@ -277,14 +288,114 @@ module Omnibus
277
288
  raise HealthCheckFailed
278
289
  end
279
290
 
291
+ conflict_map = {}
292
+
293
+ conflict_map = relocation_check if relocation_checkable?
294
+
295
+ if conflict_map.keys.length > 0
296
+ log.warn(log_key) { 'Multiple dlls with overlapping images detected' }
297
+
298
+ conflict_map.each do |lib_name, data|
299
+ base = data[:base]
300
+ size = data[:size]
301
+ next_valid_base = data[:base] + data[:size]
302
+
303
+ log.warn(log_key) do
304
+ out = "Overlapping dll detected:\n"
305
+ out << " #{lib_name} :\n"
306
+ out << " IMAGE BASE: #{hex}\n" % base
307
+ out << " IMAGE SIZE: #{hex} (#{size} bytes)\n" % size
308
+ out << " NEXT VALID BASE: #{hex}\n" % next_valid_base
309
+ out << " CONFLICTS:\n"
310
+
311
+ data[:conflicts].each do |conflict_name|
312
+ cbase = conflict_map[conflict_name][:base]
313
+ csize = conflict_map[conflict_name][:size]
314
+ out << " - #{conflict_name} #{hex} + #{hex}\n" % [cbase, csize]
315
+ end
316
+
317
+ out
318
+ end
319
+ end
320
+
321
+ # Don't raise an error yet. This is only bad for FIPS mode.
322
+ end
323
+
280
324
  true
281
325
  end
282
326
 
327
+ # Ensure the method relocation_check is able to run
328
+ #
329
+ # @return [Boolean]
330
+ #
331
+ def relocation_checkable?
332
+ return false unless windows?
333
+
334
+ begin
335
+ require 'pedump'
336
+ true
337
+ rescue LoadError
338
+ false
339
+ end
340
+ end
341
+
342
+ # Check dll image location overlap/conflicts on windows.
343
+ #
344
+ # @return [Hash<String, Hash<Symbol, ...>>]
345
+ # library_name ->
346
+ # :base -> base address
347
+ # :size -> the total image size in bytes
348
+ # :conflicts -> array of library names that overlap
349
+ #
350
+ def relocation_check
351
+ conflict_map = {}
352
+
353
+ embedded_bin = "#{project.install_dir}/embedded/bin"
354
+ Dir.glob("#{embedded_bin}/*.dll") do |lib_path|
355
+ log.debug(log_key) { "Analyzing dependencies for #{lib_path}" }
356
+
357
+ File.open(lib_path, 'rb') do |f|
358
+ dump = PEdump.new(lib_path)
359
+ pe = dump.pe f
360
+
361
+ # Don't scan dlls for a different architecture.
362
+ next if windows_arch_i386? == pe.x64?
363
+
364
+ lib_name = File.basename(lib_path)
365
+ base = pe.ioh.ImageBase
366
+ size = pe.ioh.SizeOfImage
367
+ conflicts = []
368
+
369
+ # This can be done more smartly but O(n^2) is just fine for n = small
370
+ conflict_map.each do |candidate_name, details|
371
+ unless details[:base] >= base + size ||
372
+ details[:base] + details[:size] <= base
373
+ details[:conflicts] << lib_name
374
+ conflicts << candidate_name
375
+ end
376
+ end
377
+
378
+ conflict_map[lib_name] = {
379
+ base: base,
380
+ size: size,
381
+ conflicts: conflicts,
382
+ }
383
+
384
+ log.debug(log_key) { "Discovered #{lib_name} at #{hex} + #{hex}" % [ base, size ] }
385
+ end
386
+ end
387
+
388
+ # Filter out non-conflicting entries.
389
+ conflict_map.delete_if do |lib_name, details|
390
+ details[:conflicts].empty?
391
+ end
392
+ end
393
+
283
394
  #
284
395
  # Run healthchecks against otool.
285
396
  #
286
- # @return [Array<String>]
287
- # the bad libraries
397
+ # @return [Hash<String, Hash<String, Hash<String, Int>>>]
398
+ # the bad libraries (library_name -> dependency_name -> satisfied_lib_path -> count)
288
399
  #
289
400
  def health_check_otool
290
401
  current_library = nil
@@ -307,8 +418,8 @@ module Omnibus
307
418
  #
308
419
  # Run healthchecks against aix.
309
420
  #
310
- # @return [Array<String>]
311
- # the bad libraries
421
+ # @return [Hash<String, Hash<String, Hash<String, Int>>>]
422
+ # the bad libraries (library_name -> dependency_name -> satisfied_lib_path -> count)
312
423
  #
313
424
  def health_check_aix
314
425
  current_library = nil
@@ -334,9 +445,9 @@ module Omnibus
334
445
 
335
446
  #
336
447
  # Run healthchecks against ldd.
337
- #
338
- # @return [Array<String>]
339
- # the bad libraries
448
+ #
449
+ # @return [Hash<String, Hash<String, Hash<String, Int>>>]
450
+ # the bad libraries (library_name -> dependency_name -> satisfied_lib_path -> count)
340
451
  #
341
452
  def health_check_ldd
342
453
  current_library = nil
@@ -374,6 +485,16 @@ module Omnibus
374
485
 
375
486
  private
376
487
 
488
+ #
489
+ # This is the printf style format string to render a pointer/size_t on the
490
+ # current platform.
491
+ #
492
+ # @return [String]
493
+ #
494
+ def hex
495
+ windows_arch_i386? ? "0x%08x" : "0x%016x"
496
+ end
497
+
377
498
  #
378
499
  # The list of whitelisted (ignored) files from the project and softwares.
379
500
  #
@@ -404,6 +525,17 @@ module Omnibus
404
525
  #
405
526
  # Check the given path and library for "bad" libraries.
406
527
  #
528
+ # @param [Hash<String, Hash<String, Hash<String, Int>>>]
529
+ # the bad libraries (library_name -> dependency_name -> satisfied_lib_path -> count)
530
+ # @param [String]
531
+ # the library being analyzed
532
+ # @param [String]
533
+ # dependency library name
534
+ # @param [String]
535
+ # actual path of library satisfying the dependency
536
+ #
537
+ # @return the modified bad_library hash
538
+ #
407
539
  def check_for_bad_library(bad_libs, current_library, name, linked)
408
540
  safe = nil
409
541
 
@@ -21,6 +21,8 @@ module Omnibus
21
21
  # Default Omnibus naming
22
22
  preinst: 'Pre-installation Script',
23
23
  postinst: 'Post-installation Script',
24
+ config: 'Configuration Script',
25
+ unconfig: 'Unconfiguration Script',
24
26
  prerm: 'Pre_rm Script',
25
27
  postrm: 'Unconfiguration Script',
26
28
  }.freeze
@@ -127,21 +129,51 @@ module Omnibus
127
129
  # Unconfiguration Script: /path/script
128
130
  #
129
131
  def write_gen_template
132
+
133
+ # Get a list of all files
134
+ files = FileSyncer.glob("#{staging_dir}/**/*").map do |path|
135
+
136
+ # If paths have colons or commas, rename them and add them to a post-install,
137
+ # post-sysck renaming script ('config') which is created if needed
138
+ if path.match(/:|,/)
139
+ alt = path.gsub(/(:|,)/, '__')
140
+ log.debug(log_key) { "Renaming #{path} to #{alt}" }
141
+
142
+ File.rename(path, alt) if File.exists?(path)
143
+
144
+ # Create a config script if needed based on resources/bff/config.erb
145
+ config_script_path = File.join(scripts_staging_dir, 'config')
146
+ unless File.exists? config_script_path
147
+ render_template(resource_path('config.erb'),
148
+ destination: "#{scripts_staging_dir}/config",
149
+ variables: {
150
+ name: project.name
151
+ }
152
+ )
153
+ end
154
+
155
+ File.open(File.join(scripts_staging_dir, 'config'), 'a') do |file|
156
+ file.puts "mv '#{alt.gsub(/^#{staging_dir}/, '')}' '#{path.gsub(/^#{staging_dir}/, '')}'"
157
+ end
158
+
159
+ path = alt
160
+ end
161
+
162
+ path.gsub(/^#{staging_dir}/, '')
163
+ end
164
+
130
165
  # Create a map of scripts that exist to inject into the template
131
166
  scripts = SCRIPT_MAP.inject({}) do |hash, (script, installp_key)|
132
167
  staging_path = File.join(scripts_staging_dir, script.to_s)
133
168
 
134
169
  if File.file?(staging_path)
135
170
  hash[installp_key] = staging_path
171
+ log.debug(log_key) { installp_key + ":\n" + File.read(staging_path) }
136
172
  end
137
173
 
138
174
  hash
139
175
  end
140
176
 
141
- # Get a list of all files
142
- files = FileSyncer.glob("#{staging_dir}/**/*")
143
- .map { |path| path.gsub(/^#{staging_dir}/, '') }
144
-
145
177
  render_template(resource_path('gen.template.erb'),
146
178
  destination: File.join(staging_dir, 'gen.template'),
147
179
  variables: {
@@ -154,6 +186,9 @@ module Omnibus
154
186
  scripts: scripts,
155
187
  }
156
188
  )
189
+
190
+ # Print the full contents of the rendered template file for mkinstallp's use
191
+ log.debug(log_key) { "Rendered Template:\n" + File.read(File.join(staging_dir, 'gen.template')) }
157
192
  end
158
193
 
159
194
  #
@@ -173,7 +208,8 @@ module Omnibus
173
208
  # This implies that if we are in /tmp/staging/project/dir/things,
174
209
  # we will chown from 'project' on, rather than 'project/dir', which leaves
175
210
  # project owned by the build user (which is incorrect)
176
- shellout!("sudo chown -R 0:0 #{File.join(staging_dir, project.install_dir.match(/^\/?(\w+)/).to_s)}")
211
+ # First - let's find out who we are.
212
+ shellout!("sudo chown -Rh 0:0 #{File.join(staging_dir, project.install_dir.match(/^\/?(\w+)/).to_s)}")
177
213
  log.info(log_key) { "Creating .bff file" }
178
214
 
179
215
  # Since we want the owner to be root, we need to sudo the mkinstallp
@@ -181,10 +217,26 @@ module Omnibus
181
217
  # directory.
182
218
  shellout!("sudo /usr/sbin/mkinstallp -d #{staging_dir} -T #{File.join(staging_dir, 'gen.template')}")
183
219
 
220
+ # Print the full contents of the inventory file generated by mkinstallp
221
+ # from within the staging_dir's .info folder (where control files for the
222
+ # packaging process are kept.)
223
+ log.debug(log_key) do
224
+ "With .inventory file of:\n" + File.read("#{
225
+ File.join( staging_dir, '.info', "#{safe_base_package_name}.inventory" )
226
+ }")
227
+ end
228
+
184
229
  # Copy the resulting package up to the package_dir
185
230
  FileSyncer.glob(File.join(staging_dir, 'tmp/*.bff')).each do |bff|
186
231
  copy_file(bff, File.join(Config.package_dir, create_bff_file_name))
187
232
  end
233
+
234
+ ensure
235
+ # chown back to original user's uid/gid so cleanup works correctly
236
+ original_uid = shellout!("id -u").stdout.chomp
237
+ original_gid = shellout!("id -g").stdout.chomp
238
+
239
+ shellout!("sudo chown -Rh #{original_uid}:#{original_gid} #{staging_dir}")
188
240
  end
189
241
 
190
242
  #
@@ -263,7 +263,7 @@ module Omnibus
263
263
  def write_md5_sums
264
264
  path = "#{staging_dir}/**/*"
265
265
  hash = FileSyncer.glob(path).inject({}) do |hash, path|
266
- if File.file?(path) && !File.symlink?(path)
266
+ if File.file?(path) && !File.symlink?(path) && !(File.dirname(path) == debian_dir)
267
267
  relative_path = path.gsub("#{staging_dir}/", '')
268
268
  hash[relative_path] = digest(path, :md5)
269
269
  end
@@ -413,7 +413,7 @@ module Omnibus
413
413
  # see https://wiki.debian.org/Arm64Port
414
414
  'arm64'
415
415
  when 'ppc64le'
416
- # Debian prefers to use ppc64el for little endian architecture name
416
+ # Debian prefers to use ppc64el for little endian architecture name
417
417
  # where as others like gnutools/rhel use ppc64le( note the last 2 chars)
418
418
  # see http://linux.debian.ports.powerpc.narkive.com/8eeWSBtZ/switching-ppc64el-port-name-to-ppc64le
419
419
  'ppc64el' #dpkg --print-architecture = ppc64el
@@ -262,14 +262,14 @@ module Omnibus
262
262
  # @return [String]
263
263
  #
264
264
  def safe_base_package_name
265
- if project.package_name =~ /\A[[:alnum:]]+\z/
265
+ if project.package_name =~ /\A[[:alnum:]-]+\z/
266
266
  project.package_name.dup
267
267
  else
268
268
  converted = project.package_name.downcase.gsub(/[^[:alnum:]+]/, '')
269
269
 
270
270
  log.warn(log_key) do
271
271
  "The `name' component of Mac package names can only include " \
272
- "alphabetical characters (a-z, A-Z), and numbers (0-9). Converting " \
272
+ "alphabetical characters (a-z, A-Z), numbers (0-9), and -. Converting " \
273
273
  "`#{project.package_name}' to `#{converted}'."
274
274
  end
275
275
 
@@ -68,11 +68,11 @@ module Omnibus
68
68
  SCRIPT_MAP.each do |source, destination|
69
69
  source_path = File.join(project.package_scripts_path, source.to_s)
70
70
 
71
- if File.file?(source_path)
72
- destination_path = staging_dir_path(destination)
73
- log.debug(log_key) { "Adding script `#{source}' to `#{destination_path}'" }
74
- copy_file(source_path, destination_path)
75
- end
71
+ next unless File.file?(source_path)
72
+
73
+ destination_path = staging_dir_path(destination)
74
+ log.debug(log_key) { "Adding script `#{source}' to `#{destination_path}'" }
75
+ copy_file(source_path, destination_path)
76
76
  end
77
77
  end
78
78
 
@@ -82,6 +82,18 @@ module Omnibus
82
82
  def write_prototype_file
83
83
  shellout! "cd #{install_dirname} && find #{install_basename} -print > #{staging_dir_path('files')}"
84
84
 
85
+ File.open staging_dir_path('files.clean'), 'w+' do |fout|
86
+ File.open staging_dir_path('files') do |fin|
87
+ fin.each_line do |line|
88
+ if line.chomp =~ /\s/
89
+ log.warn(log_key) { "Skipping packaging '#{line}' file due to whitespace in filename" }
90
+ else
91
+ fout.write(line)
92
+ end
93
+ end
94
+ end
95
+ end
96
+
85
97
  # generate list of control files
86
98
  File.open staging_dir_path('Prototype'), 'w+' do |f|
87
99
  f.write <<-EOF.gsub(/^ {10}/, '')
@@ -92,7 +104,7 @@ module Omnibus
92
104
  end
93
105
 
94
106
  # generate the prototype's file list
95
- shellout! "cd #{install_dirname} && pkgproto < #{staging_dir_path('files')} > #{staging_dir_path('Prototype.files')}"
107
+ shellout! "cd #{install_dirname} && pkgproto < #{staging_dir_path('files.clean')} > #{staging_dir_path('Prototype.files')}"
96
108
 
97
109
  # fix up the user and group in the file list to root
98
110
  shellout! "awk '{ $5 = \"root\"; $6 = \"root\"; print }' < #{staging_dir_path('Prototype.files')} >> #{staging_dir_path('Prototype')}"
@@ -199,7 +199,7 @@ module Omnibus
199
199
 
200
200
  #
201
201
  # Path to the +/files+ directory in the omnibus project. This directory can
202
- # contain arbritary files used by the project.
202
+ # contain arbitrary files used by the project.
203
203
  #
204
204
  # @example
205
205
  # patch = File.join(files_path, 'rubygems', 'patch.rb')
@@ -20,6 +20,7 @@ require 'omnibus/s3_helpers'
20
20
  module Omnibus
21
21
  class S3Cache
22
22
  include Logging
23
+ extend Digestable
23
24
 
24
25
  class << self
25
26
  include S3Helpers
@@ -75,9 +76,14 @@ module Omnibus
75
76
  log.info(log_key) do
76
77
  "Caching '#{fetcher.downloaded_file}' to '#{Config.s3_bucket}/#{key}'"
77
78
  end
79
+
80
+ # Fetcher has already verified the downloaded file in software.fetch.
81
+ # Compute the md5 from scratch because the fetcher may have been
82
+ # specified with a different hashing algorithm.
83
+ md5 = digest(fetcher.downloaded_file, :md5)
78
84
 
79
85
  File.open(fetcher.downloaded_file, 'rb') do |file|
80
- store_object(key, file, software.fetcher.checksum, 'public-read')
86
+ store_object(key, file, md5, 'public-read')
81
87
  end
82
88
  end
83
89
 
@@ -101,7 +107,7 @@ module Omnibus
101
107
  # @private
102
108
  #
103
109
  # The key with which to cache the package on S3. This is the name of the
104
- # package, the version of the package, and its checksum.
110
+ # package, the version of the package, and its md5 checksum.
105
111
  #
106
112
  # @example
107
113
  # "zlib-1.2.6-618e944d7c7cd6521551e30b32322f4a"