omnibus 5.0.0 → 5.1.0

data/lib/omnibus/health_check.rb

@@ -1,4 +1,4 @@
-#
+
 # Copyright 2012-2014 Chef Software, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,10 +14,18 @@
 # limitations under the License.
 #
 
+require 'omnibus/sugarable'
+begin
+  require 'pedump'
+rescue LoadError
+  STDERR.puts "pedump not found - windows health checks disabled"
+end
+
 module Omnibus
   class HealthCheck
     include Logging
     include Util
+    include Sugarable
 
     WHITELIST_LIBS = [
       /ld-linux/,
@@ -140,6 +148,7 @@ module Omnibus
       /libelf\.so/,
       /libkvm\.so/,
       /libprocstat\.so/,
+      /libmd\.so/,
     ].freeze
 
     class << self
@@ -177,16 +186,18 @@ module Omnibus
     #   if the healthchecks pass
     #
     def run!
-      if Ohai['platform'] == 'windows'
-        log.warn(log_key) { 'Skipping health check on Windows' }
-        return true
-      end
       log.info(log_key) {"Running health on #{project.name}"}
       bad_libs =  case Ohai['platform']
                   when 'mac_os_x'
                     health_check_otool
                   when 'aix'
                     health_check_aix
+                  when 'windows'
+                    # TODO: objdump -p will provided a very limited check of
+                    # explicit dependencies on windows. Most dependencies are
+                    # implicit and hence not detected.
+                    log.warn(log_key) { 'Skipping dependency health checks on Windows.' }
+                    {}
                   else
                     health_check_ldd
                   end
@@ -277,14 +288,114 @@ module Omnibus
         raise HealthCheckFailed
       end
 
+      conflict_map = {}
+
+      conflict_map = relocation_check if relocation_checkable?
+
+      if conflict_map.keys.length > 0
+        log.warn(log_key) { 'Multiple dlls with overlapping images detected' }
+
+        conflict_map.each do |lib_name, data|
+          base = data[:base]
+          size = data[:size]
+          next_valid_base = data[:base] + data[:size]
+
+          log.warn(log_key) do
+            out =  "Overlapping dll detected:\n"
+            out << "    #{lib_name} :\n"
+            out << "    IMAGE BASE: #{hex}\n" % base
+            out << "    IMAGE SIZE: #{hex} (#{size} bytes)\n" % size
+            out << "    NEXT VALID BASE: #{hex}\n" % next_valid_base
+            out << "    CONFLICTS:\n"
+
+            data[:conflicts].each do |conflict_name|
+              cbase = conflict_map[conflict_name][:base]
+              csize = conflict_map[conflict_name][:size]
+              out << "    - #{conflict_name} #{hex} + #{hex}\n" % [cbase, csize]
+            end
+
+            out
+          end
+        end
+
+        # Don't raise an error yet. This is only bad for FIPS mode.
+      end
+
       true
     end
 
+    # Ensure the method relocation_check is able to run
+    #
+    # @return [Boolean]
+    #
+    def relocation_checkable?
+      return false unless windows?
+
+      begin
+        require 'pedump'
+        true
+      rescue LoadError
+        false
+      end
+    end
+
+    # Check dll image location overlap/conflicts on windows.
+    #
+    # @return [Hash<String, Hash<Symbol, ...>>]
+    #   library_name ->
+    #     :base -> base address
+    #     :size -> the total image size in bytes
+    #     :conflicts -> array of library names that overlap
+    #
+    def relocation_check
+      conflict_map = {}
+
+      embedded_bin = "#{project.install_dir}/embedded/bin"
+      Dir.glob("#{embedded_bin}/*.dll") do |lib_path|
+        log.debug(log_key) { "Analyzing dependencies for #{lib_path}" }
+
+        File.open(lib_path, 'rb') do |f|
+          dump = PEdump.new(lib_path)
+          pe = dump.pe f
+
+          # Don't scan dlls for a different architecture.
+          next if windows_arch_i386? == pe.x64?
+
+          lib_name = File.basename(lib_path)
+          base = pe.ioh.ImageBase
+          size = pe.ioh.SizeOfImage
+          conflicts = []
+
+          # This can be done more smartly but O(n^2) is just fine for n = small
+          conflict_map.each do |candidate_name, details|
+            unless details[:base] >= base + size ||
+                  details[:base] + details[:size] <= base
+              details[:conflicts] << lib_name
+              conflicts << candidate_name
+            end
+          end
+
+          conflict_map[lib_name] = {
+            base: base,
+            size: size,
+            conflicts: conflicts,
+          }
+
+          log.debug(log_key) { "Discovered #{lib_name} at #{hex} + #{hex}" % [ base, size ] }
+        end
+      end
+
+      # Filter out non-conflicting entries.
+      conflict_map.delete_if do |lib_name, details|
+        details[:conflicts].empty?
+      end
+    end
+
     #
     # Run healthchecks against otool.
     #
-    # @return [Array<String>]
-    #   the bad libraries
+    # @return [Hash<String, Hash<String, Hash<String, Int>>>]
+    #   the bad libraries (library_name -> dependency_name -> satisfied_lib_path -> count)
     #
     def health_check_otool
       current_library = nil
@@ -307,8 +418,8 @@ module Omnibus
     #
     # Run healthchecks against aix.
     #
-    # @return [Array<String>]
-    #   the bad libraries
+    # @return [Hash<String, Hash<String, Hash<String, Int>>>]
+    #   the bad libraries (library_name -> dependency_name -> satisfied_lib_path -> count)
     #
     def health_check_aix
       current_library = nil
@@ -334,9 +445,9 @@ module Omnibus
 
     #
     # Run healthchecks against ldd.
-    #
-    # @return [Array<String>]
-    #   the bad libraries
+    # 
+    # @return [Hash<String, Hash<String, Hash<String, Int>>>]
+    #   the bad libraries (library_name -> dependency_name -> satisfied_lib_path -> count)
     #
     def health_check_ldd
       current_library = nil
@@ -374,6 +485,16 @@ module Omnibus
 
     private
 
+    #
+    # This is the printf style format string to render a pointer/size_t on the
+    # current platform.
+    #
+    # @return [String]
+    #
+    def hex
+      windows_arch_i386? ? "0x%08x" : "0x%016x"
+    end
+
     #
     # The list of whitelisted (ignored) files from the project and softwares.
     #
@@ -404,6 +525,17 @@ module Omnibus
     #
     # Check the given path and library for "bad" libraries.
     #
+    # @param [Hash<String, Hash<String, Hash<String, Int>>>]
+    #   the bad libraries (library_name -> dependency_name -> satisfied_lib_path -> count)
+    # @param [String]
+    #   the library being analyzed
+    # @param [String]
+    #   dependency library name
+    # @param [String]
+    #   actual path of library satisfying the dependency
+    #
+    # @return the modified bad_library hash
+    #
     def check_for_bad_library(bad_libs, current_library, name, linked)
       safe = nil
 

data/lib/omnibus/packagers/bff.rb

@@ -21,6 +21,8 @@ module Omnibus
       # Default Omnibus naming
       preinst:  'Pre-installation Script',
       postinst: 'Post-installation Script',
+      config:   'Configuration Script',
+      unconfig: 'Unconfiguration Script',
       prerm:    'Pre_rm Script',
       postrm:   'Unconfiguration Script',
     }.freeze
@@ -127,21 +129,51 @@ module Omnibus
     #   Unconfiguration Script: /path/script
     #
     def write_gen_template
+
+      # Get a list of all files
+      files = FileSyncer.glob("#{staging_dir}/**/*").map do |path|
+
+        # If paths have colons or commas, rename them and add them to a post-install,
+        # post-sysck renaming script ('config') which is created if needed
+        if path.match(/:|,/)
+          alt = path.gsub(/(:|,)/, '__')
+          log.debug(log_key) { "Renaming #{path} to #{alt}" }
+
+          File.rename(path, alt) if File.exists?(path)
+
+          # Create a config script if needed based on resources/bff/config.erb
+          config_script_path = File.join(scripts_staging_dir, 'config')
+          unless File.exists? config_script_path
+            render_template(resource_path('config.erb'),
+              destination: "#{scripts_staging_dir}/config",
+              variables: {
+                name: project.name
+              }
+            )
+          end
+
+          File.open(File.join(scripts_staging_dir, 'config'), 'a') do |file|
+            file.puts "mv '#{alt.gsub(/^#{staging_dir}/, '')}' '#{path.gsub(/^#{staging_dir}/, '')}'"
+          end
+
+          path = alt
+        end
+
+        path.gsub(/^#{staging_dir}/, '')
+      end
+
       # Create a map of scripts that exist to inject into the template
       scripts = SCRIPT_MAP.inject({}) do |hash, (script, installp_key)|
         staging_path =  File.join(scripts_staging_dir, script.to_s)
 
         if File.file?(staging_path)
           hash[installp_key] = staging_path
+          log.debug(log_key) { installp_key + ":\n" + File.read(staging_path) }
         end
 
         hash
       end
 
-      # Get a list of all files
-      files = FileSyncer.glob("#{staging_dir}/**/*")
-                .map { |path| path.gsub(/^#{staging_dir}/, '') }
-
       render_template(resource_path('gen.template.erb'),
         destination: File.join(staging_dir, 'gen.template'),
         variables: {
@@ -154,6 +186,9 @@ module Omnibus
           scripts:        scripts,
         }
       )
+
+      # Print the full contents of the rendered template file for mkinstallp's use
+      log.debug(log_key) { "Rendered Template:\n" + File.read(File.join(staging_dir, 'gen.template')) }
     end
 
     #
@@ -173,7 +208,8 @@ module Omnibus
       # This implies that if we are in /tmp/staging/project/dir/things,
       # we will chown from 'project' on, rather than 'project/dir', which leaves
       # project owned by the build user (which is incorrect)
-      shellout!("sudo chown -R 0:0 #{File.join(staging_dir, project.install_dir.match(/^\/?(\w+)/).to_s)}")
+      # First - let's find out who we are.
+      shellout!("sudo chown -Rh 0:0 #{File.join(staging_dir, project.install_dir.match(/^\/?(\w+)/).to_s)}")
       log.info(log_key) { "Creating .bff file" }
 
       # Since we want the owner to be root, we need to sudo the mkinstallp
@@ -181,10 +217,26 @@ module Omnibus
       # directory.
       shellout!("sudo /usr/sbin/mkinstallp -d #{staging_dir} -T #{File.join(staging_dir, 'gen.template')}")
 
+      # Print the full contents of the inventory file generated by mkinstallp
+      # from within the staging_dir's .info folder (where control files for the
+      # packaging process are kept.)
+      log.debug(log_key) do
+        "With .inventory file of:\n" + File.read("#{
+          File.join( staging_dir, '.info', "#{safe_base_package_name}.inventory" )
+        }")
+      end
+
       # Copy the resulting package up to the package_dir
       FileSyncer.glob(File.join(staging_dir, 'tmp/*.bff')).each do |bff|
         copy_file(bff, File.join(Config.package_dir, create_bff_file_name))
       end
+
+    ensure
+      # chown back to original user's uid/gid so cleanup works correctly
+      original_uid = shellout!("id -u").stdout.chomp
+      original_gid = shellout!("id -g").stdout.chomp
+
+      shellout!("sudo chown -Rh #{original_uid}:#{original_gid} #{staging_dir}")
     end
 
     #

data/lib/omnibus/packagers/deb.rb

@@ -263,7 +263,7 @@ module Omnibus
     def write_md5_sums
       path = "#{staging_dir}/**/*"
       hash = FileSyncer.glob(path).inject({}) do |hash, path|
-        if File.file?(path) && !File.symlink?(path)
+        if File.file?(path) && !File.symlink?(path) && !(File.dirname(path) == debian_dir)
           relative_path = path.gsub("#{staging_dir}/", '')
           hash[relative_path] = digest(path, :md5)
         end
@@ -413,7 +413,7 @@ module Omnibus
         # see https://wiki.debian.org/Arm64Port
         'arm64'
       when 'ppc64le'
-        # Debian prefers to use ppc64el for little endian architecture name 
+        # Debian prefers to use ppc64el for little endian architecture name
         # where as others like gnutools/rhel use ppc64le( note the last 2 chars)
         # see http://linux.debian.ports.powerpc.narkive.com/8eeWSBtZ/switching-ppc64el-port-name-to-ppc64le
         'ppc64el'  #dpkg --print-architecture = ppc64el

data/lib/omnibus/packagers/pkg.rb

@@ -262,14 +262,14 @@ module Omnibus
     # @return [String]
     #
     def safe_base_package_name
-      if project.package_name =~ /\A[[:alnum:]]+\z/
+      if project.package_name =~ /\A[[:alnum:]-]+\z/
         project.package_name.dup
       else
         converted = project.package_name.downcase.gsub(/[^[:alnum:]+]/, '')
 
         log.warn(log_key) do
           "The `name' component of Mac package names can only include " \
-          "alphabetical characters (a-z, A-Z), and numbers (0-9). Converting " \
+          "alphabetical characters (a-z, A-Z), numbers (0-9), and -. Converting " \
           "`#{project.package_name}' to `#{converted}'."
         end
 

data/lib/omnibus/packagers/solaris.rb

@@ -68,11 +68,11 @@ module Omnibus
       SCRIPT_MAP.each do |source, destination|
         source_path = File.join(project.package_scripts_path, source.to_s)
 
-        if File.file?(source_path)
-          destination_path = staging_dir_path(destination)
-          log.debug(log_key) { "Adding script `#{source}' to `#{destination_path}'" }
-          copy_file(source_path, destination_path)
-        end
+        next unless File.file?(source_path)
+
+        destination_path = staging_dir_path(destination)
+        log.debug(log_key) { "Adding script `#{source}' to `#{destination_path}'" }
+        copy_file(source_path, destination_path)
       end
     end
 
@@ -82,6 +82,18 @@ module Omnibus
     def write_prototype_file
       shellout! "cd #{install_dirname} && find #{install_basename} -print > #{staging_dir_path('files')}"
 
+      File.open staging_dir_path('files.clean'), 'w+' do |fout|
+        File.open staging_dir_path('files') do |fin|
+          fin.each_line do |line|
+            if line.chomp =~ /\s/
+              log.warn(log_key) { "Skipping packaging '#{line}' file due to whitespace in filename" }
+            else
+              fout.write(line)
+            end
+          end
+        end
+      end
+
       # generate list of control files
       File.open staging_dir_path('Prototype'), 'w+' do |f|
         f.write <<-EOF.gsub(/^ {10}/, '')
@@ -92,7 +104,7 @@ module Omnibus
       end
 
       # generate the prototype's file list
-      shellout! "cd #{install_dirname} && pkgproto < #{staging_dir_path('files')} > #{staging_dir_path('Prototype.files')}"
+      shellout! "cd #{install_dirname} && pkgproto < #{staging_dir_path('files.clean')} > #{staging_dir_path('Prototype.files')}"
 
       # fix up the user and group in the file list to root
       shellout! "awk '{ $5 = \"root\"; $6 = \"root\"; print }' < #{staging_dir_path('Prototype.files')} >> #{staging_dir_path('Prototype')}"

data/lib/omnibus/project.rb

@@ -199,7 +199,7 @@ module Omnibus
 
     #
     # Path to the +/files+ directory in the omnibus project. This directory can
-    # contain arbritary files used by the project.
+    # contain arbitrary files used by the project.
     #
     # @example
     #   patch = File.join(files_path, 'rubygems', 'patch.rb')

data/lib/omnibus/s3_cache.rb

@@ -20,6 +20,7 @@ require 'omnibus/s3_helpers'
 module Omnibus
   class S3Cache
     include Logging
+    extend Digestable
 
     class << self
       include S3Helpers
@@ -75,9 +76,14 @@ module Omnibus
           log.info(log_key) do
             "Caching '#{fetcher.downloaded_file}' to '#{Config.s3_bucket}/#{key}'"
           end
+          
+          # Fetcher has already verified the downloaded file in software.fetch.
+          # Compute the md5 from scratch because the fetcher may have been
+          # specified with a different hashing algorithm.
+          md5 = digest(fetcher.downloaded_file, :md5)
 
           File.open(fetcher.downloaded_file, 'rb') do |file|
-            store_object(key, file, software.fetcher.checksum, 'public-read')
+            store_object(key, file, md5, 'public-read')
           end
         end
 
@@ -101,7 +107,7 @@ module Omnibus
       # @private
       #
       # The key with which to cache the package on S3. This is the name of the
-      # package, the version of the package, and its checksum.
+      # package, the version of the package, and its md5 checksum.
       #
       # @example
       #   "zlib-1.2.6-618e944d7c7cd6521551e30b32322f4a"