omnibus 5.2.0 → 5.3.0

data/lib/omnibus/fetchers/git_fetcher.rb

@@ -197,13 +197,20 @@ module Omnibus
     #
     # Execute the given git command, inside the +project_dir+.
     #
+    # autcrlf is a hack to help support windows and posix clients using the
+    # same repository but canonicalizing files as they are committed to the
+    # repo but converting line endings when they are actually checked out
+    # into a working tree. We do not want to change the on-disk representation
+    # of our sources regardless of the platform we are building on unless
+    # explicitly asked for. Hence, we disable autocrlf.
+    #
     # @see Util#shellout!
     #
     # @return [Mixlib::ShellOut]
     #   the shellout object
     #
     def git(command)
-      shellout!("git #{command}", cwd: project_dir)
+      shellout!("git -c core.autocrlf=false #{command}", cwd: project_dir)
     end
 
     # Class methods

data/lib/omnibus/fetchers/net_fetcher.rb

@@ -15,11 +15,12 @@
 #
 
 require 'fileutils'
-require 'open-uri'
-require 'ruby-progressbar'
+require 'omnibus/download_helpers'
 
 module Omnibus
   class NetFetcher < Fetcher
+    include DownloadHelpers
+
     # Use 7-zip to extract 7z/zip for Windows
     WIN_7Z_EXTENSIONS = %w(.7z .zip)
 
@@ -159,53 +160,17 @@ module Omnibus
     def download
       log.warn(log_key) { source[:warning] } if source.key?(:warning)
 
-      options = download_headers
+      options = {}
 
       if source[:unsafe]
         log.warn(log_key) { "Permitting unsafe redirects!" }
         options[:allow_unsafe_redirects] = true
       end
 
-      options[:read_timeout] = Omnibus::Config.fetcher_read_timeout
-      fetcher_retries ||= Omnibus::Config.fetcher_retries
-
-      progress_bar = ProgressBar.create(
-        output: $stdout,
-        format: '%e %B %p%% (%r KB/sec)',
-        rate_scale: ->(rate) { rate / 1024 },
-      )
-
-      reported_total = 0
-
-      options[:content_length_proc] = ->(total) {
-        reported_total = total
-        progress_bar.total = total
-      }
-      options[:progress_proc] = ->(step) {
-        downloaded_amount = [step, reported_total].min
-        progress_bar.progress = downloaded_amount
-      }
-
-      file = open(download_url, options)
-      # This is a temporary file. Close and flush it before attempting to copy
-      # it over.
-      file.close
-      FileUtils.cp(file.path, downloaded_file)
-      file.unlink
-    rescue SocketError,
-           Errno::ECONNREFUSED,
-           Errno::ECONNRESET,
-           Errno::ENETUNREACH,
-           Timeout::Error,
-           OpenURI::HTTPError => e
-      if fetcher_retries != 0
-        log.info(log_key) { "Retrying failed download due to #{e} (#{fetcher_retries} retries left)..." }
-        fetcher_retries -= 1
-        retry
-      else
-        log.error(log_key) { "Download failed - #{e.class}!" }
-        raise
-      end
+      # Set the cookie if one was given
+      options['Cookie'] = source[:cookie] if source[:cookie]
+
+      download_file!(download_url, downloaded_file, options)
     end
 
     #
@@ -358,43 +323,5 @@ module Omnibus
     def tar
       Omnibus.which('gtar') ? 'gtar' : 'tar'
     end
-
-    #
-    # The list of headers to pass to the download.
-    #
-    # @return [Hash]
-    #
-    def download_headers
-      {}.tap do |h|
-        # Alright kids, sit down while grandpa tells you a story. Back when the
-        # Internet was just a series of tubes, and you had to "dial in" using
-        # this thing called a "modem", ancient astronaunt theorists (computer
-        # scientists) invented gzip to compress requests sent over said tubes
-        # and make the Internet faster.
-        #
-        # Fast forward to the year of broadband - ungzipping these files was
-        # tedious and hard, so Ruby and other client libraries decided to do it
-        # for you:
-        #
-        #   https://github.com/ruby/ruby/blob/c49ae7/lib/net/http.rb#L1031-L1033
-        #
-        # Meanwhile, software manufacturers began automatically compressing
-        # their software for distribution as a +.tar.gz+, publishing the
-        # appropriate checksums accordingly.
-        #
-        # But consider... If a software manufacturer is publishing the checksum
-        # for a gzipped tarball, and the client is automatically ungzipping its
-        # responses, then checksums can (read: should) never match! Herein lies
-        # the bug that took many hours away from the lives of a once-happy
-        # developer.
-        #
-        # TL;DR - Do not let Ruby ungzip our file
-        #
-        h['Accept-Encoding'] = 'identity'
-
-        # Set the cookie if one was given
-        h['Cookie'] = source[:cookie] if source[:cookie]
-      end
-    end
   end
 end

data/lib/omnibus/health_check.rb

@@ -97,6 +97,12 @@ module Omnibus
       /librt\.so\.1/,
       /libcrypt\.so\.1/,
       /libgdbm\.so\.3/,
+      /libgcc_s\.so\.1/,
+      /libcryptoutil\.so\.1/,
+      /libucrypto\.so\.1/,
+      /libz\.so\.1/, # while we package our own libz, this get dragged along from Solaris 11's libelf library for some reason...
+      /libelf\.so\.1/,
+      /libssp\.so\.0/,
       # solaris 9 libraries:
       /libm\.so\.1/,
       /libc_psr\.so\.1/,
@@ -151,6 +157,49 @@ module Omnibus
       /libmd\.so/,
     ].freeze
 
+    IGNORED_ENDINGS = %w(
+      .[ch]
+      .e*rb
+      .gemspec
+      .gitignore
+      .h*h
+      .java
+      .js
+      .json
+      .lock
+      .log
+      .lua
+      .md
+      .mkd
+      .out
+      .pl
+      .pm
+      .png
+      .py[oc]*
+      .r*html
+      .rdoc
+      .ri
+      .sh
+      .sql
+      .toml
+      .ttf
+      .txt
+      .xml
+      .yml
+      Gemfile
+      LICENSE
+      README
+      Rakefile
+      VERSION
+    ).freeze
+
+    IGNORED_PATTERNS = %w(
+      /share/doc/
+      /share/postgresql/
+      /share/terminfo/
+      /terminfo/
+    ).freeze
+
     class << self
       # @see (HealthCheck#new)
       def run!(project)
@@ -450,10 +499,14 @@ module Omnibus
     #   the bad libraries (library_name -> dependency_name -> satisfied_lib_path -> count)
     #
     def health_check_ldd
+      regexp_ends = '.*(' + IGNORED_ENDINGS.map { |e| e.gsub(/\./, '\.') }.join('|') + ')$'
+      regexp_patterns = IGNORED_PATTERNS.map { |e| '.*' + e.gsub(/\//, '\/') + '.*' }.join('|')
+      regexp = regexp_ends + '|' + regexp_patterns
+
       current_library = nil
       bad_libs = {}
 
-      read_shared_libs("find #{project.install_dir}/ -type f | xargs ldd") do |line|
+      read_shared_libs("find #{project.install_dir}/ -type f -regextype posix-extended ! -regex '#{regexp}' | xargs ldd") do |line|
         case line
         when /^(.+):$/
           current_library = Regexp.last_match[1]

data/lib/omnibus/licensing.rb

@@ -16,10 +16,12 @@
 
 require 'uri'
 require 'fileutils'
+require 'omnibus/download_helpers'
 
 module Omnibus
   class Licensing
     include Logging
+    include DownloadHelpers
 
     OUTPUT_DIRECTORY = "LICENSES".freeze
 
@@ -53,6 +55,7 @@ module Omnibus
     #
     def create!
       prepare
+      validate_license_info
       create_software_license_files
       create_project_license_file
     end
@@ -67,6 +70,49 @@ module Omnibus
       FileUtils.mkdir_p(output_dir)
     end
 
+    #
+    # Inspects the licensing information for the project and the included
+    # software components. Logs the found issues to the log as warning.
+    #
+    # @return [void]
+    #
+    def validate_license_info
+      # First check the project licensing information
+
+      # Check existence of licensing information
+      if project.license == "Unspecified"
+        licensing_warning("Project '#{project.name}' does not contain licensing information.")
+      end
+
+      # Check license file exists
+      if project.license != "Unspecified" && project.license_file.nil?
+        licensing_warning("Project '#{project.name}' does not point to a license file.")
+      end
+
+      # Check used license is a standard license
+      if project.license != "Unspecified" && !STANDARD_LICENSES.include?(project.license)
+        licensing_warning("Project '#{project.name}' is using '#{project.license}' which is not one of the standard licenses identified in https://opensource.org/licenses/alphabetical. Consider using one of the standard licenses.")
+      end
+
+      # Now let's check the licensing info for software components
+      license_map.each do |software_name, license_info|
+        # First check if the software specified a license
+        if license_info[:license] == "Unspecified"
+          licensing_warning("Software '#{software_name}' does not contain licensing information.")
+        end
+
+        # Check if the software specifies any license files
+        if license_info[:license] != "Unspecified" && license_info[:license_files].empty?
+          licensing_warning("Software '#{software_name}' does not point to any license files.")
+        end
+
+        # Check if the software license is one of the standard licenses
+        if license_info[:license] != "Unspecified" && !STANDARD_LICENSES.include?(license_info[:license])
+          licensing_warning("Software '#{software_name}' uses license '#{license_info[:license]}' which is not one of the standard licenses identified in https://opensource.org/licenses/alphabetical. Consider using one of the standard licenses.")
+        end
+      end
+    end
+
     #
     # Creates the top level license file for the project.
     # Top level file is created at #{project.license_file_path}
@@ -97,10 +143,28 @@ module Omnibus
         license_files = values[:license_files]
 
         license_files.each do |license_file|
-          if license_file && local?(license_file)
-            input_file = File.expand_path(license_file, values[:project_dir])
+          if license_file
             output_file = license_package_location(name, license_file)
-            FileUtils.cp(input_file, output_file) if File.exist?(input_file)
+
+            if local?(license_file)
+              input_file = File.expand_path(license_file, values[:project_dir])
+              if File.exist?(input_file)
+                FileUtils.cp(input_file, output_file)
+              else
+                licensing_warning("License file '#{input_file}' does not exist for software '#{name}'.")
+              end
+            else
+              begin
+                download_file!(license_file, output_file, enable_progress_bar: false)
+              rescue SocketError,
+                     Errno::ECONNREFUSED,
+                     Errno::ECONNRESET,
+                     Errno::ENETUNREACH,
+                     Timeout::Error,
+                     OpenURI::HTTPError
+                licensing_warning("Can not download license file '#{license_file}' for software '#{name}'.")
+              end
+            end
           end
         end
       end
@@ -201,7 +265,8 @@ module Omnibus
       if local?(where)
         File.join(output_dir, "#{component_name}-#{File.split(where).last}")
       else
-        where
+        u = URI(where)
+        File.join(output_dir, "#{component_name}-#{File.basename(u.path)}")
       end
     end
 
@@ -223,5 +288,108 @@ module Omnibus
       u = URI(license)
       return u.scheme.nil?
     end
+
+    #
+    # Logs the given message as warning.
+    #
+    # @param [String] message
+    #   message to log as warning
+    def licensing_warning(message)
+      log.warn(log_key) { message }
+    end
+
+    STANDARD_LICENSES = [
+      #
+      # Below licenses are compiled based on https://opensource.org/licenses/alphabetical
+      #
+      "AFL-3.0",       # Academic Free License 3.0
+      "AGPL-3.0",      # Affero General Public License
+      "APL-1.0",       # Adaptive Public License
+      "Apache-2.0",    # Apache License 2.0
+      "APSL-2.0",      # Apple Public Source License
+      "Artistic-2.0",  # Artistic license 2.0
+      "AAL",           # Attribution Assurance Licenses
+      "BSD-3-Clause",  # BSD 3-Clause "New" or "Revised" License
+      "BSD-2-Clause",  # BSD 2-Clause "Simplified" or "FreeBSD" License
+      "BSL-1.0",       # Boost Software License
+      "CECILL-2.1",    # CeCILL License 2.1
+      "CATOSL-1.1",    # Computer Associates Trusted Open Source License 1.1
+      "CDDL-1.0",      # Common Development and Distribution License 1.0
+      "CPAL-1.0",      # Common Public Attribution License 1.0
+      "CUA-OPL-1.0",   # CUA Office Public License Version 1.0
+      "EUDatagrid",    # EU DataGrid Software License
+      "EPL-1.0",       # Eclipse Public License 1.0
+      "eCos-2.0",      # eCos License version 2.0
+      "ECL-2.0",       # Educational Community License, Version 2.0
+      "EFL-2.0",       # Eiffel Forum License V2.0
+      "Entessa",       # Entessa Public License
+      "EUPL-1.1",      # European Union Public License, Version 1.1
+      "Fair",          # Fair License
+      "Frameworx-1.0", # Frameworx License
+      "FPL-1.0.0",     # Free Public License 1.0.0
+      "AGPL-3.0",      # GNU Affero General Public License v3
+      "GPL-2.0",       # GNU General Public License version 2.0
+      "GPL-3.0",       # GNU General Public License version 3.0
+      "LGPL-2.1",      # GNU Library or "Lesser" General Public License version 2.1
+      "LGPL-3.0",      # GNU Library or "Lesser" General Public License version 3.0
+      "HPND",          # Historical Permission Notice and Disclaimer
+      "IPL-1.0",       # IBM Public License 1.0
+      "IPA",           # IPA Font License
+      "ISC",           # ISC License
+      "LPPL-1.3c",     # LaTeX Project Public License 1.3c
+      "LiLiQ-P",       # Licence Libre du Quebec Permissive
+      "LiLiQ-R",       # Licence Libre du Quebec Reciprocite
+      "LiLiQ-R+",      # Licence Libre du Quebec Reciprocite forte
+      "LPL-1.02",      # Lucent Public License Version 1.02
+      "MirOS",         # MirOS Licence
+      "MS-PL",         # Microsoft Public License
+      "MS-RL",         # Microsoft Reciprocal License
+      "MIT",           # MIT license
+      "Motosoto",      # Motosoto License
+      "MPL-2.0",       # Mozilla Public License 2.0
+      "Multics",       # Multics License
+      "NASA-1.3",      # NASA Open Source Agreement 1.3
+      "NTP",           # NTP License
+      "Naumen",        # Naumen Public License
+      "NGPL",          # Nethack General Public License
+      "Nokia",         # Nokia Open Source License
+      "NPOSL-3.0",     # Non-Profit Open Software License 3.0
+      "OCLC-2.0",      # OCLC Research Public License 2.0
+      "OGTSL",         # Open Group Test Suite License
+      "OSL-3.0",       # Open Software License 3.0
+      "OPL-2.1",       # OSET Public License version 2.1
+      "PHP-3.0",       # PHP License 3.0
+      "PostgreSQL",    # The PostgreSQL License
+      "Python-2.0",    # Python License
+      "CNRI-Python",   # CNRI Python license
+      "QPL-1.0",       # Q Public License
+      "RPSL-1.0",      # RealNetworks Public Source License V1.0
+      "RPL-1.5",       # Reciprocal Public License 1.5
+      "RSCPL",         # Ricoh Source Code Public License
+      "OFL-1.1",       # SIL Open Font License 1.1
+      "SimPL-2.0",     # Simple Public License 2.0
+      "Sleepycat",     # Sleepycat License
+      "SPL-1.0",       # Sun Public License 1.0
+      "Watcom-1.0",    # Sybase Open Watcom Public License 1.0
+      "NCSA",          # University of Illinois/NCSA Open Source License
+      "UPL",           # Universal Permissive License
+      "VSL-1.0",       # Vovida Software License v. 1.0
+      "W3C",           # W3C License
+      "WXwindows",     # wxWindows Library License
+      "Xnet",          # X.Net License
+      "0BSD",          # Zero Clause BSD License
+      "ZPL-2.0",       # Zope Public License 2.0
+      "Zlib",          # zlib/libpng license
+      #
+      # In addition to these we would like to add some of the licenses that
+      # are frequently used in our depedencies.
+      #
+      "Public-Domain", # https://opensource.org/faq#public-domain
+      "Ruby",          # http://www.ruby-lang.org/en/LICENSE.txt
+      "Erlang-Public", # http://www.erlang.org/EPLICENSE
+      "Oracle-Binary", # http://www.oracle.com/technetwork/java/javase/terms/license/index.html
+      "OpenSSL",       # https://www.openssl.org/source/license.html
+      "Chef-MLSA",     # https://www.chef.io/online-master-agreement/
+    ].freeze
   end
 end

data/lib/omnibus/packager.rb

@@ -36,6 +36,7 @@ module Omnibus
     PLATFORM_PACKAGER_MAP = {
       'debian'   => DEB,
       'fedora'   => RPM,
+      'suse'     => RPM,
       'rhel'     => RPM,
       'wrlinux'  => RPM,
       'aix'      => BFF,

data/lib/omnibus/packagers/bff.rb

@@ -131,8 +131,14 @@ module Omnibus
     def write_gen_template
 
       # Get a list of all files
-      files = FileSyncer.glob("#{staging_dir}/**/*").map do |path|
-
+      files = FileSyncer.glob("#{staging_dir}/**/*").reject do |path|
+        # remove any files with spaces.
+        if path =~ /[[:space:]]/
+          log.warn(log_key) { "Skipping packaging '#{path}' file due to whitespace in filename" }
+          true
+        end
+      end
+      files.map! do |path|
         # If paths have colons or commas, rename them and add them to a post-install,
         # post-sysck renaming script ('config') which is created if needed
         if path.match(/:|,/)