omnibus 5.5.0 → 5.6.0

data/appveyor.yml

@@ -6,8 +6,8 @@ platform:
 
 environment:
   matrix:
+    - RUBY_VERSION: 23
     - RUBY_VERSION: 22
-    - RUBY_VERSION: 21
 
 clone_folder: c:\projects\omnibus
 clone_depth: 1

data/lib/omnibus.rb

@@ -19,7 +19,11 @@ require "omnibus/core_extensions"
 require "cleanroom"
 require "pathname"
 
+require "omnibus/digestable"
 require "omnibus/exceptions"
+require "omnibus/sugarable"
+require "omnibus/util"
+require "omnibus/fetcher"
 require "omnibus/version"
 
 module Omnibus
@@ -36,9 +40,7 @@ module Omnibus
   autoload :Cleaner,          "omnibus/cleaner"
   autoload :Compressor,       "omnibus/compressor"
   autoload :Config,           "omnibus/config"
-  autoload :Digestable,       "omnibus/digestable"
   autoload :Error,            "omnibus/exceptions"
-  autoload :Fetcher,          "omnibus/fetcher"
   autoload :FileSyncer,       "omnibus/file_syncer"
   autoload :Generator,        "omnibus/generator"
   autoload :GitCache,         "omnibus/git_cache"
@@ -57,10 +59,8 @@ module Omnibus
   autoload :Reports,          "omnibus/reports"
   autoload :S3Cache,          "omnibus/s3_cache"
   autoload :Software,         "omnibus/software"
-  autoload :Sugarable,        "omnibus/sugarable"
   autoload :Templating,       "omnibus/templating"
   autoload :ThreadPool,       "omnibus/thread_pool"
-  autoload :Util,             "omnibus/util"
   autoload :Licensing,        "omnibus/licensing"
 
   autoload :GitFetcher,  "omnibus/fetchers/git_fetcher"

data/lib/omnibus/builder.rb

@@ -18,6 +18,7 @@ require "fileutils"
 require "mixlib/shellout"
 require "ostruct"
 require "pathname"
+require "omnibus/whitelist"
 
 module Omnibus
   class Builder
@@ -275,6 +276,21 @@ module Omnibus
     end
     expose :windows_safe_path
 
+    #
+    # (see Util#compiler_safe_path)
+    #
+    # Some compilers require paths to be formatted in certain ways. This helper
+    # takes in the standard Omnibus-style path and ensures that it is passed
+    # correctly.
+    #
+    # @example
+    #   configure ["--prefix=#{compiler_safe_path(install_dir, "embedded")}"]
+    #
+    def compiler_safe_path(*pieces)
+      super
+    end
+    expose :compiler_safe_path
+
     #
     # @!endgroup
     # --------------------------------------------------
@@ -547,6 +563,29 @@ module Omnibus
     end
     expose :delete
 
+    #
+    # Strip symbols from the given file or directory on the system. This method uses
+    # find and passes the matched files to strip through xargs, ignoring errors.
+    # So one may pass in a specific file/directory or a glob of files.
+    #
+    # @param [String] path
+    #   the path of the file(s) to strip
+    #
+    # @return (see #command)
+    #
+    def strip(path)
+      regexp_ends = ".*(" + IGNORED_ENDINGS.map { |e| e.gsub(/\./, '\.') }.join("|") + ")$"
+      regexp_patterns = IGNORED_PATTERNS.map { |e| ".*" + e.gsub(/\//, '\/') + ".*" }.join("|")
+      regexp = regexp_ends + "|" + regexp_patterns
+
+      # Do not actually care if strip runs on non-strippable file, as its a no-op.  Hence the `|| true` appended.
+      # Do want to avoid stripping files unneccessarily so as not to slow down build process.
+      find_command = "find #{path}/ -type f -regextype posix-extended ! -regex \"#{regexp}\" | xargs strip || true"
+      options = { in_msys_bash: true }
+      command(find_command, options)
+    end
+    expose :strip
+
     #
     # Copy the given source to the destination. This method accepts a single
     # file or a file pattern to match.
@@ -796,14 +835,6 @@ module Omnibus
       # Also make a clone of options so that we can mangle it safely below.
       options = { cwd: software.project_dir }.merge(options)
 
-      if options.delete(:in_msys_bash) && windows?
-        # Mixlib will handle escaping characters for cmd but our command might
-        # contain '. For now, assume that won't happen because I don't know
-        # whether this command is going to be played via cmd or through
-        # ProcessCreate.
-        command_string = "bash -c \'#{command_string}\'"
-      end
-
       # Set the log level to :info so users will see build commands
       options[:log_level] ||= :info
 
@@ -856,7 +887,7 @@ module Omnibus
         if tries <= 0
           raise e
         else
-          delay = delay * 2
+          delay *= 2
 
           log.warn(log_key) do
             label = "#{(Config.build_retries - tries) + 1}/#{Config.build_retries}"
@@ -963,6 +994,8 @@ module Omnibus
         log.warn(log_key) { "Detected command `remove'. Consider using the `delete' DSL method." }
       when /^rsync /i
         log.warn(log_key) { "Detected command `rsync'. Consider using the `sync' DSL method." }
+      when /^strip /i
+        log.warn(log_key) { "Detected command `strip'. Consider using the `strip' DSL method." }
       end
     end
 

data/lib/omnibus/cli.rb

@@ -70,6 +70,10 @@ module Omnibus
       desc: "Use the given manifest when downloading software sources.",
       type: :string,
       default: nil
+    method_option :populate_s3_cache,
+      desc: "Populate the S3 cache.",
+      type: :boolean,
+      default: false
     desc "build PROJECT", "Build the given Omnibus project"
     def build(name)
       manifest = if @options[:use_manifest]
@@ -80,6 +84,7 @@ module Omnibus
 
       project = Project.load(name, manifest)
       say("Building #{project.name} #{project.build_version}...")
+      Omnibus::S3Cache.populate if @options[:populate_s3_cache] && !Omnibus::S3Cache.fetch_missing.empty?
       project.download
       project.build
 

data/lib/omnibus/cli/base.rb

@@ -26,7 +26,7 @@ module Omnibus
         # Handle the case where Thor thinks a trailing --help is actually an
         # argument and blows up...
         if args.length > 1 && !(args & Thor::HELP_MAPPINGS).empty?
-          args = args - Thor::HELP_MAPPINGS
+          args -= Thor::HELP_MAPPINGS
           args.insert(-2, "help")
         end
 

data/lib/omnibus/cli/publish.rb

@@ -76,9 +76,11 @@ module Omnibus
       default: {}
     desc "artifactory REPOSITORY PATTERN", "Publish to an Artifactory instance"
     def artifactory(repository, pattern)
-      Omnibus.logger.deprecated("ArtifactoryPublisher") do
-        "The `--version-manifest' option has been deprecated. Version manifest data is now part of the `*.metadata.json' file"
-      end if options[:version_manifest]
+      if options[:version_manifest]
+        Omnibus.logger.deprecated("ArtifactoryPublisher") do
+          "The `--version-manifest' option has been deprecated. Version manifest data is now part of the `*.metadata.json' file"
+        end
+      end
 
       options[:repository] = repository
       publish(ArtifactoryPublisher, pattern, options)

data/lib/omnibus/compressors/dmg.rb

@@ -118,7 +118,7 @@ module Omnibus
     def clean_disks
       log.info(log_key) { "Cleaning previously mounted disks" }
 
-      existing_disks = shellout!("mount | grep /Volumes/#{volume_name} | awk '{print $1}'")
+      existing_disks = shellout!("mount | grep \"/Volumes/#{volume_name}\" | awk '{print $1}'")
       existing_disks.stdout.lines.each do |existing_disk|
         existing_disk.chomp!
 
@@ -240,7 +240,7 @@ module Omnibus
 
       Dir.chdir(staging_dir) do
         shellout! <<-EOH.gsub(/^ {10}/, "")
-          chmod -Rf go-w /Volumes/#{volume_name}
+          chmod -Rf go-w "/Volumes/#{volume_name}"
           sync
           hdiutil detach "#{@device}"
           hdiutil convert \\

data/lib/omnibus/config.rb

@@ -279,16 +279,29 @@ module Omnibus
     #
     # @return [String]
     default(:s3_access_key) do
-      raise MissingRequiredAttribute.new(self, :s3_access_key, "'ABCD1234'")
+      if s3_profile
+        nil
+      else
+        raise MissingRequiredAttribute.new(self, :s3_access_key, "'ABCD1234'")
+      end
     end
 
     # The S3 secret key to use with S3 caching.
     #
-    # @return [String]
+    # @return [String, nil]
     default(:s3_secret_key) do
-      raise MissingRequiredAttribute.new(self, :s3_secret_key, "'EFGH5678'")
+      if s3_profile
+        nil
+      else
+        raise MissingRequiredAttribute.new(self, :s3_secret_key, "'EFGH5678'")
+      end
     end
 
+    # The AWS credentials profile to use with S3 caching.
+    #
+    # @return [String]
+    default(:s3_profile, nil)
+
     # The region of the S3 bucket you want to cache software artifacts in.
     # Defaults to 'us-east-1'
     #
@@ -297,6 +310,16 @@ module Omnibus
       "us-east-1"
     end
 
+    # The HTTP or HTTPS endpoint to send requests to, when using non-standard endpoint
+    #
+    # @return [String, nil]
+    default(:s3_endpoint, nil)
+
+    # Enable or disable S3 Accelerate support
+    #
+    # @return [true, false]
+    default(:s3_accelerate, false)
+
     # --------------------------------------------------
     # @!endgroup
     #
@@ -390,14 +413,22 @@ module Omnibus
     #
     # @return [String]
     default(:publish_s3_access_key) do
-      raise MissingRequiredAttribute.new(self, :publish_s3_access_key, "'ABCD1234'")
+      if s3_profile
+        nil
+      else
+        raise MissingRequiredAttribute.new(self, :publish_s3_access_key, "'ABCD1234'")
+      end
     end
 
     # The S3 secret key to use for S3 artifact release
     #
     # @return [String]
     default(:publish_s3_secret_key) do
-      raise MissingRequiredAttribute.new(self, :publish_s3_secret_key, "'EFGH5678'")
+      if s3_profile
+        nil
+      else
+        raise MissingRequiredAttribute.new(self, :publish_s3_secret_key, "'EFGH5678'")
+      end
     end
 
     # --------------------------------------------------
@@ -471,6 +502,12 @@ module Omnibus
       :x86
     end
 
+    # Flag specifying whether the project should be built with FIPS
+    # compatability or not.
+    #
+    # @return [true, false]
+    default(:fips_mode, false)
+
     # --------------------------------------------------
     # @!endgroup
     #

data/lib/omnibus/core_extensions/open_uri.rb

@@ -64,6 +64,7 @@ module OpenURI
   #
   # @see http://winstonyw.com/2013/10/02/openuris_open_tempfile_and_stringio/
   #
+  # rubocop:disable Style/ConstantName
   class Buffer
     remove_const :StringMax
     StringMax = 0

data/lib/omnibus/download_helpers.rb

@@ -67,14 +67,14 @@ module Omnibus
             rate_scale: ->(rate) { rate / 1024 }
           )
 
-          options[:content_length_proc] = ->(total) {
+          options[:content_length_proc] = ->(total) do
             reported_total = total
             progress_bar.total = total
-          }
-          options[:progress_proc] = ->(step) {
+          end
+          options[:progress_proc] = ->(step) do
             downloaded_amount = reported_total ? [step, reported_total].min : step
             progress_bar.progress = downloaded_amount
-          }
+          end
         end
 
         file = open(from_url, options)

data/lib/omnibus/fetchers/git_fetcher.rb

@@ -141,7 +141,9 @@ module Omnibus
     # @return [void]
     #
     def git_clone
-      git("clone#{" --recursive" if clone_submodules?} #{source_url} .")
+      retry_block("git clone", [CommandTimeout, CommandFailed]) do
+        git("clone#{" --recursive" if clone_submodules?} #{source_url} .")
+      end
     end
 
     #
@@ -154,7 +156,11 @@ module Omnibus
       # default when a sha1 is provided).  git older than 1.7.5 doesn't
       # support the --detach flag.
       git("checkout #{resolved_version} -f -q")
-      git("submodule update --recursive") if clone_submodules?
+      if clone_submodules?
+        retry_block("git submodule update", [CommandTimeout, CommandFailed]) do
+          git("submodule update --recursive")
+        end
+      end
     end
 
     #
@@ -165,7 +171,9 @@ module Omnibus
     def git_fetch
       fetch_cmd = "fetch #{source_url} #{described_version}"
       fetch_cmd << " --recurse-submodules=on-demand" if clone_submodules?
-      git(fetch_cmd)
+      retry_block("git fetch", [CommandTimeout, CommandFailed]) do
+        git(fetch_cmd)
+      end
     end
 
     #
@@ -214,7 +222,6 @@ module Omnibus
     end
 
     # Class methods
-    public
 
     # Return the SHA1 corresponding to a ref as determined by the remote source.
     #
@@ -260,7 +267,10 @@ module Omnibus
       # allows us to return the SHA of the tagged commit for annotated
       # tags. We take care to only return exact matches in
       # process_remote_list.
-      remote_list = shellout!("git ls-remote \"#{source[:git]}\" \"#{ref}*\"").stdout
+      remote_list = retry_block("git ls-remote", [CommandTimeout, CommandFailed]) do
+        shellout!("git ls-remote \"#{source[:git]}\" \"#{ref}*\"").stdout
+      end
+
       commit_ref = dereference_annotated_tag(remote_list, ref)
 
       unless commit_ref

data/lib/omnibus/fetchers/net_fetcher.rb

@@ -144,7 +144,7 @@ module Omnibus
     #
     def download_url
       if Config.use_s3_caching
-        "http://#{Config.s3_bucket}.s3.amazonaws.com/#{S3Cache.key_for(self)}"
+        S3Cache.url_for(self)
       else
         source[:url]
       end

data/lib/omnibus/fetchers/path_fetcher.rb

@@ -49,7 +49,7 @@ module Omnibus
     #   have already removed anything extraneous.
     #
     def clean
-      return true
+      true
     end
 
     #

data/lib/omnibus/file_syncer.rb

@@ -34,6 +34,7 @@ module Omnibus
     #   the list of all files
     #
     def glob(pattern)
+      pattern = Pathname.new(pattern).cleanpath.to_s
       Dir.glob(pattern, File::FNM_DOTMATCH).sort.reject do |file|
         basename = File.basename(file)
         IGNORED_FILES.include?(basename)
@@ -144,8 +145,7 @@ module Omnibus
             end
           end
         else
-          raise RuntimeError,
-            "Unknown file type: `File.ftype(source_file)' at `#{source_file}'!"
+          raise "Unknown file type: `File.ftype(source_file)' at `#{source_file}'!"
         end
       end
 

data/lib/omnibus/generator_files/omnibus.rb.erb

@@ -33,6 +33,7 @@
 # use_s3_caching true
 # s3_access_key  ENV['AWS_ACCESS_KEY_ID']
 # s3_secret_key  ENV['AWS_SECRET_ACCESS_KEY']
+# s3_profile     ENV['AWS_S3_PROFILE']
 # s3_bucket      ENV['AWS_S3_BUCKET']
 
 # Customize compiler bits

data/lib/omnibus/git_cache.rb

@@ -32,7 +32,7 @@ module Omnibus
     # will not have the generated content, so these snapshots would be
     # incompatible with the current omnibus codebase. Incrementing the serial
     # number ensures these old shapshots will not be used in subsequent builds.
-    SERIAL_NUMBER = 1
+    SERIAL_NUMBER = 3
 
     REQUIRED_GIT_FILES = %w{
 HEAD
@@ -76,6 +76,9 @@ refs}.freeze
       else
         create_directory(File.dirname(cache_path))
         git_cmd("init -q")
+        # On windows, git is very picky about single vs double quotes
+        git_cmd("config --local user.name \"Omnibus Git Cache\"")
+        git_cmd("config --local user.email \"omnibus@localhost\"")
         true
       end
     end
@@ -140,23 +143,26 @@ refs}.freeze
 
       create_cache_path
 
-      restore_me = false
-      cmd = git_cmd(%Q{tag -l "#{tag}"})
-
-      cmd.stdout.each_line do |line|
-        restore_me = true if tag == line.chomp
-      end
-
-      if restore_me
-        log.internal(log_key) { "Detected tag `#{tag}' can be restored, restoring" }
-        git_cmd(%Q{checkout -f "#{tag}"})
+      if has_tag(tag)
+        log.internal(log_key) { "Detected tag `#{tag}' can be restored, marking it for restoration" }
+        git_cmd(%Q{tag -f restore_here "#{tag}"})
         true
+      elsif has_tag("restore_here")
+        log.internal(log_key) { "Could not find tag `#{tag}', restoring previous tag" }
+        restore_from_cache
+        false
       else
-        log.internal(log_key) { "Could not find tag `#{tag}', skipping restore" }
+        log.internal(log_key) { "Could not find marker tag `restore_here', nothing to restore" }
         false
       end
     end
 
+    def restore_from_cache
+      git_cmd("checkout -f restore_here")
+    ensure
+      git_cmd("tag -d restore_here")
+    end
+
     #
     # Git caching will attempt to version embedded git directories, partially
     # versioning them. This causes failures on subsequent runs. This method
@@ -190,7 +196,14 @@ refs}.freeze
     # @return [Mixlib::Shellout] the underlying command object.
     #
     def git_cmd(command)
-      shellout!("git -c core.autocrlf=false --git-dir=#{cache_path} --work-tree=#{install_dir} #{command}")
+      shellout!([
+        "git",
+        "-c core.autocrlf=false",
+        "-c core.ignorecase=false",
+        "--git-dir=\"#{cache_path}\"",
+        "--work-tree=\"#{install_dir}\"",
+        command,
+      ].join(" "))
     end
 
     #
@@ -210,5 +223,10 @@ refs}.freeze
     def log_key
       @log_key ||= "#{super}: #{software.name}"
     end
+
+    def has_tag(tag)
+      cmd = git_cmd(%Q{tag -l "#{tag}"})
+      cmd.stdout.lines.any? { |line| tag == line.chomp }
+    end
   end
 end