omnibus 5.0.0 → 5.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1471f1e4e874c7a6d1146332f02e3a2b01ce2b47
-  data.tar.gz: 15c244b4ee8b8814058f17eecec80ff44b08337f
+  metadata.gz: d6c6004b5090f1bef33eea64eec8e741fef4af4d
+  data.tar.gz: 540e7c85bff51d2e8865325d388e08260be11b98
 SHA512:
-  metadata.gz: 186d0842eb7d493a482f3c661f9248d155db72c58dd8797a160c13900ba01fe7ce893c645bccfe981e922602f023f0e1261114405ddf9e303d169978bccd539f
-  data.tar.gz: 5be67f512343718da2c35bbbc6c18c1e7f0e67fff96a3d3a2b25a7ef5d67706a4b1f9398883e1d28be87529f6a544f63dc3b0ee8d567411135c7663efda38ca6
+  metadata.gz: 3d4e0d40424b6e0e5d2ebbaa0fa411fe9435bde23c508dfa22e84a10337153415e6467c9c40a41cfed138c9cb1ec77660c19a211fe491e8d9839cf83e587a358
+  data.tar.gz: a6d063e1f4772956aa11a0264d5a06611b6de49ea184cc4a89aa0296a1c0cc02be3b21e721270b31730e9ccadff4578dca5edc0df60f5d10e91d77afc24b5efc

data/.rspec

@@ -0,0 +1,2 @@
+--color
+-fd

data/.travis.yml

@@ -8,6 +8,7 @@ branches:
     - 2.0-stable
     - 3.0-stable
     - master
+before_install: gem install bundler
 script: bundle exec rake travis:ci
 notifications:
   slack:

data/CHANGELOG.md

@@ -1,6 +1,32 @@
 Omnibus CHANGELOG
 =================
 
+v5.1.0 (March 10, 2016)
+-----------------------
+### New Features
+- New `configure` dsl method. (#572)
+- New `maintainer` dsl method. (#618)
+- New `update_config_guess` dsl method. (#632)
+- Ability to enable building software components from source on windows. (#572, #583, #584, #586, #612)
+
+### Bug Fixes
+- Default to UTF-8 external encoding globally. (#573)
+- Restore invalid file names on AIX. (#575)
+- Fix bff log loop. (#579)
+- Use 7z.exe instead of tar.exe on windows. (#578)
+- Make generated package scripts old-school Unix friendly. (#582)
+- Fix directory cleanup logic in `git_fetcher`. (#509)
+- Use -O2 when building with standard compiler flags. (#591)
+- Cache software sources under `.../src/<software name>/<package>`. (#597)
+- Add libmd.so.* to freebsd whitelist. (#600)
+- Remove existing links in the destination when syncing files. (#602)
+- Skip adding DEBIAN directory to md5sums file. (#595)
+- Autoprunes files with spaces on Solaris. (#609)
+- Allow assets with non-md5 checksums to be cached in s3. (#611)
+- Print NetFetcher retries at the info level. (#614)
+- Do not modify CRLF when git caching. (#616)
+- Ensure we always swap chown back to default. (#617)
+
 v5.0.0 (November 10, 2015)
 --------------------------
 ### New Features

data/Gemfile

@@ -1,6 +1,9 @@
 source 'https://rubygems.org'
 gemspec
 
+# Fork to allow for a recent version of multipart-post.
+gem 'pedump', git: 'https://github.com/ksubrama/pedump', branch: 'patch-1'
+
 group :docs do
   gem 'yard',          '~> 0.8'
   gem 'redcarpet',     '~> 2.2.2'

data/MAINTAINERS.md

@@ -25,3 +25,4 @@ project lead.
 * [Steven Danna](https://github.com/stevendanna)
 * [Yvonne Lam](http://github.com/yzl)
 * [Thom May](https://github.com/thommay)
+* [Kartik Null Cating-Subramanian](https://github.com/ksubrama)

data/appveyor.yml

@@ -20,7 +20,7 @@ install:
   - echo %PATH%
   - ruby --version
   - gem --version
-  - gem install bundler --quiet --no-ri --no-rdoc
+  - gem install bundler -v 1.10.6 --quiet --no-ri --no-rdoc
   - bundler --version
   - cp C:\Ruby21%ruby_version:~3%\Devkit\mingw\bin\bsdtar.exe C:\Ruby21%ruby_version:~3%\Devkit\mingw\bin\tar.exe
   - appveyor DownloadFile http://curl.haxx.se/ca/cacert.pem -FileName C:\cacert.pem

data/bin/omnibus

@@ -8,4 +8,9 @@ $:.push File.expand_path('../../lib', __FILE__)
 $stdout.sync = true
 
 require 'omnibus/cli'
+
+# Some platforms do not have a UTF-8 locale, so we need to enforce one
+# or else the cacert chain will break among other things
+Encoding.default_external = Encoding::UTF_8
+
 Omnibus::CLI::Runner.new(ARGV.dup).execute!

data/lib/omnibus/builder.rb

@@ -91,6 +91,9 @@ module Omnibus
     # use of +gmake+ over +make+. If applicable, this method will also set
     # the `MAKE=gmake` environment variable when gmake is to be preferred.
     #
+    # On windows you need to have the msys-base package (or some equivalent)
+    # before you can invoke this.
+    #
     # @example With no arguments
     #   make
     #
@@ -105,27 +108,87 @@ module Omnibus
     #
     def make(*args)
       options = args.last.is_a?(Hash) ? args.pop : {}
-      command = args.join(' ')
 
-      make = if makebin = options.delete(:bin)
-               makebin
-             elsif Omnibus.which('gmake')
-               env = options.delete(:env) || {}
-               env = { 'MAKE' => 'gmake' }.merge(env)
-               options[:env] = env
-               'gmake'
-             else
-               'make'
-             end
+      make = options.delete(:bin) ||
+        # Prefer gmake on non-windows environments.
+        if !windows? && Omnibus.which('gmake')
+          env = options.delete(:env) || {}
+          env = { 'MAKE' => 'gmake' }.merge(env)
+          options[:env] = env
+          'gmake'
+        else
+          'make'
+        end
 
-      command("#{make} #{command}".strip, options)
+      options[:in_msys_bash] = true
+      make_cmd = ([make] + args).join(' ').strip
+      command(make_cmd, options)
     end
     expose :make
 
+    # Run a prexisting "./configure" script that was generated by autotools.
+    # On windows, this will run configure within an msys bash shell with
+    # the given arguments. --host is also set on your behalf based on
+    # windows_arch. A default prefix of "#{install_bin}/embedded" is
+    # appended.
+    #
+    # @example With no arguments
+    #   configure
+    # On POSIX systems, this results in:
+    #   ./configure --prefix=/path/to/embedded
+    # On Windows 64-bit, this results in:
+    #   ./configure --host=x86_64-w64-mingw32 --prefix=C:/path/to/embedded
+    # Note that the windows case uses a windows compabile path with forward
+    # slashes - not an msys path. Ensure that the final Makefile is happy
+    # with this and doesn't perform path gymnastics on it. Don't pass
+    # \\ paths unless you particularly enjoy discovering exactly home many
+    # times configure and the Makefile it generates sends your path back
+    # and forth through bash/sh, mingw32 native binaries and msys binaries
+    # and how many backslashes it takes for you to quit software development.
+    #
+    # @example With custom arguments
+    #   configure '--enable-shared'
+    #
+    # @example With custom location of configure bin
+    #   configure '--enable-shared', bin: '../foo/configure'
+    # The path to configure must be a "unix-y" path for both windows and posix
+    # as this path is run under an msys bash shell on windows. Prefer relative
+    # paths lest you incur the wrath of the msys path gods for they are not
+    # kind, just or benevolent.
+    #
+    # @param (see #command)
+    # @return (see #command)
+    #
+    def configure(*args)
+      options = args.last.is_a?(Hash) ? args.pop : {}
+
+      configure = options.delete(:bin) || './configure'
+      configure_cmd = [configure]
+
+      # Pass the host platform as well. msys is configured for 32-bits even
+      # if the actual installed compiler has 64-bit support.
+      configure_cmd << '--host=x86_64-w64-mingw32' if windows? && !windows_arch_i386?
+
+      # Accept a prefix override if provided. Can be set to '' to suppress
+      # this functionality.
+      prefix = options.delete(:prefix) || "#{install_dir}/embedded"
+      configure_cmd << "--prefix=#{prefix}" if prefix && prefix != ''
+
+      configure_cmd.concat args
+      configure_cmd = configure_cmd.join(' ').strip
+
+      options[:in_msys_bash] = true
+      command(configure_cmd, options)
+    end
+    expose :configure
+
     #
     # Apply the patch by the given name. This method will search all possible
     # locations for a patch (such as {Config#software_gems}).
     #
+    # On windows, you must have the the patch package installed before you can
+    # invoke this.
+    #
     # @example
     #   patch source: 'ncurses-clang.patch'
     #
@@ -155,19 +218,26 @@ module Omnibus
         raise MissingPatch.new(source, locations)
       end
 
-      # Apply patches nicely on Windows
-      patch_path = windows_safe_path(patch_path)
+      # Using absolute paths to the patch when invoking patch from within msys
+      # is going to end is tears and table-flips. Use relative paths instead.
+      # It's windows - we don't reasonably expect symlinks to show up any-time
+      # soon and if you're using junction points, you're on your own.
+      clean_patch_path = patch_path
+      if windows?
+        clean_patch_path = Pathname.new(patch_path).relative_path_from(
+          Pathname.new(software.project_dir)).to_s
+      end
 
       if target
-        command = "cat #{patch_path} | patch -p#{plevel} #{target}"
+        patch_cmd = "cat #{clean_patch_path} | patch -p#{plevel} #{target}"
       else
-        command = "patch -d #{software.project_dir} -p#{plevel} -i #{patch_path}"
+        patch_cmd = "patch -p#{plevel} -i #{clean_patch_path}"
       end
 
       patches << patch_path
-
+      options[:in_msys_bash] = true
       build_commands << BuildCommand.new("Apply patch `#{source}'") do
-        shellout!(command, options)
+        shellout!(patch_cmd, options)
       end
     end
     expose :patch
@@ -218,7 +288,7 @@ module Omnibus
     #
     def ruby(command, options = {})
       build_commands << BuildCommand.new("ruby `#{command}'") do
-        bin = windows_safe_path("#{install_dir}/embedded/bin/ruby")
+        bin = embedded_bin('ruby')
         shellout!("#{bin} #{command}", options)
       end
     end
@@ -235,7 +305,7 @@ module Omnibus
     #
     def gem(command, options = {})
       build_commands << BuildCommand.new("gem `#{command}'") do
-        bin = windows_safe_path("#{install_dir}/embedded/bin/gem")
+        bin = embedded_bin('gem')
         shellout!("#{bin} #{command}", options)
       end
     end
@@ -255,7 +325,7 @@ module Omnibus
     #
     def bundle(command, options = {})
       build_commands << BuildCommand.new("bundle `#{command}'") do
-        bin = windows_safe_path("#{install_dir}/embedded/bin/bundle")
+        bin = embedded_bin('bundle')
         shellout!("#{bin} #{command}", options)
       end
     end
@@ -270,18 +340,26 @@ module Omnibus
     # @example
     #   appbundle 'chef'
     #
+    # @param software_name [String]
+    #  The omnibus software definition name that you want to appbundle.  We
+    #  assume that this software definition is a gem that already has `bundle
+    #  install` ran on it so it has a Gemfile.lock to appbundle.
     # @param (see #command)
     # @return (see #command)
     #
-    def appbundle(app_name, options = {})
-      build_commands << BuildCommand.new("appbundle `#{app_name}'") do
+    def appbundle(software_name, options = {})
+      build_commands << BuildCommand.new("appbundle `#{software_name}'") do
+        app_software = project.softwares.find do |p|
+          p.name == software_name
+        end
+
         bin_dir            = "#{install_dir}/bin"
-        appbundler_bin     = windows_safe_path("#{install_dir}/embedded/bin/appbundler")
+        appbundler_bin     = embedded_bin('appbundler')
 
         # Ensure the main bin dir exists
         FileUtils.mkdir_p(bin_dir)
 
-        shellout!("#{appbundler_bin} '#{Omnibus::Config.source_dir}/#{app_name}' '#{bin_dir}'", options)
+        shellout!("#{appbundler_bin} '#{app_software.project_dir}' '#{bin_dir}'", options)
       end
     end
     expose :appbundle
@@ -298,7 +376,7 @@ module Omnibus
     #
     def rake(command, options = {})
       build_commands << BuildCommand.new("rake `#{command}'") do
-        bin = windows_safe_path("#{install_dir}/embedded/bin/rake")
+        bin = embedded_bin('rake')
         shellout!("#{bin} #{command}", options)
       end
     end
@@ -565,6 +643,35 @@ module Omnibus
     end
     expose :sync
 
+    #
+    # Helper method to update config_guess in the software's source
+    # directory.
+    # You should add a dependency on the +config_guess+ software definition if you
+    # want to use this command.
+    # @param [Hash] options
+    #   Supported options are:
+    #     target [String] subdirectory under the software source to copy
+    #       config.guess.to. Default: "."
+    #     install [Array<Symbol>] parts of config.guess to copy.
+    #       Default: [:config_guess, :config_sub]
+    def update_config_guess(target: ".", install: [:config_guess, :config_sub])
+      build_commands << BuildCommand.new("update_config_guess `target: #{target} install: #{install.inspect}'") do
+        config_guess_dir = "#{install_dir}/embedded/lib/config_guess"
+        %w{config.guess config.sub}.each do |c|
+          unless File.exist?(File.join(config_guess_dir, c))
+            raise "Can not find #{c}. Make sure you add a dependency on 'config_guess' in your software definition"
+          end
+        end
+
+        destination = File.join(software.project_dir, target)
+        FileUtils.mkdir_p(destination)
+
+        FileUtils.cp_r("#{config_guess_dir}/config.guess", destination) if install.include? :config_guess
+        FileUtils.cp_r("#{config_guess_dir}/config.sub", destination) if install.include? :config_sub
+      end
+    end
+    expose :update_config_guess
+
     #
     # @!endgroup
     # --------------------------------------------------
@@ -637,6 +744,14 @@ module Omnibus
 
     private
 
+    def embedded_bin(bin)
+      windows_safe_path("#{install_dir}/embedded/bin/#{bin}")
+    end
+
+    def embedded_msys_bin(bin)
+      windows_safe_path("#{install_dir}/embedded/msys/1.0/bin/#{bin}")
+    end
+
     #
     # The **in-order** list of {BuildCommand} for this builder.
     #
@@ -670,12 +785,36 @@ module Omnibus
     # This is a helper method that wraps {Util#shellout!} for the purposes of
     # setting the +:cwd+ value.
     #
+    # It also accepts an :in_msys_bash option which controls whether the
+    # given command is wrapped and run with bash.exe -c on windows.
+    #
     # @see (Util#shellout!)
     #
     def shellout!(command_string, options = {})
       # Make sure the PWD is set to the correct directory
+      # Also make a clone of options so that we can mangle it safely below.
       options = { cwd: software.project_dir }.merge(options)
 
+      if options.delete(:in_msys_bash) && windows?
+        # The command needs to be run within an msys bash environment.
+        # TODO: Eventually, have command search through "build time dependencies"
+        # for bash instead.
+        bash_bin = embedded_msys_bin('bash.exe')
+        unless File.exist?(bash_bin)
+          # Fallback to just looking at the embedded_bin directory in case
+          # we're using devkit or older chef-dk.
+          bash_bin = embedded_bin('bash.exe')
+          unless File.exist?(bash_bin)
+            bash_bin = "bash.exe"
+          end
+        end
+        # Mixlib will handle escaping characters for cmd but our command might
+        # contain '. For now, assume that won't happen because I don't know
+        # whether this command is going to be played via cmd or through
+        # ProcessCreate.
+        command_string = "\"#{bash_bin}\" -c \'#{command_string}\'"
+      end
+
       # Set the log level to :info so users will see build commands
       options[:log_level] ||= :info
 

data/lib/omnibus/digestable.rb

@@ -62,14 +62,16 @@ module Omnibus
     #   the path of the directory to digest
     # @param [Symbol] type
     #   the type of digest to use
+    # @param [Hash] options
+    #   options to pass through to the FileSyncer when scanning for files
     #
     # @return [String]
     #   the hexdigest of the directory
     #
-    def digest_directory(path, type = :md5)
+    def digest_directory(path, type = :md5, options = {})
       digest = digest_from_type(type)
       log.info(log_key) { "Digesting #{path} with #{type}" }
-      FileSyncer.glob("#{path}/**/*").each do |filename|
+      FileSyncer.all_files_under(path, options).each do |filename|
         # Calculate the filename relative to the given path. Since directories
         # are SHAed according to their filepath, two difference directories on
         # disk would have different SHAs even if they had the same content.

data/lib/omnibus/fetcher.rb

@@ -55,7 +55,17 @@ module Omnibus
     attr_reader :described_version
 
     #
-    # The path where extracted software should live.
+    # The path where fetched software should live.
+    #
+    # Only files under this directory are modified. If the source to fetch
+    # is a directory, it is staged rooted here. If it's a file, it's copied
+    # underneath this directory. If it's a tarball, it's extracted here. If
+    # it's a repo, its checkout is rooted here. You get the idea.
+    #
+    # It's named project_dir instead of extract_dir/extract_path because of
+    # legacy reasons. This has nothing to do with project definitions or the
+    # underlying relative_path for a software definition (except for legacy
+    # behavior).
     #
     # @return [String]
     #
@@ -66,11 +76,14 @@ module Omnibus
     #
     # Create a new Fetcher object from the given software.
     #
-    # @param [Software] software
+    # The parameters correspond to the relevant portions of a software
+    # definition that a fetcher needs access to. This avoids strongly coupling
+    # the software object with all fetchers.
+    #
+    # @param [ManifestEntry] manifest_entry
+    # @param [String] project_dir
+    # @param [String] build_dir
     #
-    # To preserve the original interface, this still takes a software-like
-    # argument, but to avoid coupling with the software object, we pull out
-    # what we need and don't touch it again.
     def initialize(manifest_entry, project_dir, build_dir)
       @name    = manifest_entry.name
       @source  = manifest_entry.locked_source

data/lib/omnibus/fetchers/git_fetcher.rb

@@ -23,7 +23,7 @@ module Omnibus
     # @return [true, false]
     #
     def fetch_required?
-      !(cloned? && same_revision?(resolved_version))
+      !(cloned? && contains_revision?(resolved_version))
     end
 
     #
@@ -37,19 +37,19 @@ module Omnibus
     end
 
     #
-    # Clean the project directory by removing the contents from disk.
+    # Clean the project directory by resetting the current working tree to
+    # the required revision.
     #
     # @return [true, false]
-    #   true if the project directory was removed, false otherwise
+    #   true if the project directory was cleaned, false otherwise.
+    #   In our case, we always return true because we always call
+    #   git checkout/clean.
     #
     def clean
-      if cloned?
-        log.info(log_key) { 'Cleaning existing clone' }
-        git('clean -fdx')
-        true
-      else
-        false
-      end
+      log.info(log_key) { 'Cleaning existing clone' }
+      git_checkout
+      git('clean -fdx')
+      true
     end
 
     #
@@ -62,22 +62,30 @@ module Omnibus
       create_required_directories
 
       if cloned?
-        git_fetch unless same_revision?(resolved_version)
+        git_fetch
       else
         force_recreate_project_dir! unless dir_empty?(project_dir)
         git_clone
-        git_checkout
       end
     end
 
     #
-    # The version for this item in the cache. The is the parsed revision of the
-    # item on disk.
+    # The version for this item in the cache.
+    #
+    # This method is called *before* clean but *after* fetch. Do not ever
+    # use the contents of the project_dir here.
+    #
+    # We aren't including the source/repo path here as there could be
+    # multiple branches/tags that all point to the same commit. We're
+    # assuming that we won't realistically ever get two git commits
+    # that are unique but share sha1s.
+    #
+    # TODO: Does this work with submodules?
     #
     # @return [String]
     #
     def version_for_cache
-      "revision:#{current_revision}"
+      "revision:#{resolved_version}"
     end
 
     private
@@ -142,7 +150,10 @@ module Omnibus
     # @return [void]
     #
     def git_checkout
-      git("checkout #{resolved_version}")
+      # We are hoping to perform a checkout with detached HEAD (that's the
+      # default when a sha1 is provided).  git older than 1.7.5 doesn't
+      # support the --detach flag.
+      git("checkout #{resolved_version} -f -q")
     end
 
     #
@@ -154,7 +165,6 @@ module Omnibus
       fetch_cmd = "fetch #{source_url} #{described_version}"
       fetch_cmd << ' --recurse-submodules=on-demand' if clone_submodules?
       git(fetch_cmd)
-      git("reset --hard #{resolved_version}")
     end
 
     #
@@ -163,18 +173,24 @@ module Omnibus
     # @return [String]
     #
     def current_revision
-      git('rev-parse HEAD').stdout.strip
+      cmd = git('rev-parse HEAD')
+      cmd.stdout.strip
     rescue CommandFailed
+      log.debug(log_key) { "unable to determine current revision" }
       nil
     end
 
     #
-    # Determine if the given revision matches the current revision.
+    # Check if the current clone has the requested commit id.
     #
     # @return [true, false]
     #
-    def same_revision?(rev=resolved_version)
-      current_revision == rev
+    def contains_revision?(rev)
+      cmd = git("cat-file -t #{rev}")
+      cmd.stdout.strip == 'commit'
+    rescue CommandFailed
+      log.debug(log_key) { "unable to determine presence of commit #{rev}" }
+      false
     end
 
     #
@@ -236,7 +252,7 @@ module Omnibus
       # allows us to return the SHA of the tagged commit for annotated
       # tags. We take care to only return exact matches in
       # process_remote_list.
-      remote_list = shellout!("git ls-remote \"#{source[:git]}\" #{ref}*").stdout
+      remote_list = shellout!("git ls-remote \"#{source[:git]}\" \"#{ref}*\"").stdout
       commit_ref = dereference_annotated_tag(remote_list, ref)
 
       unless commit_ref