omnibus 5.0.0 → 5.1.0

data/lib/omnibus/fetchers/net_fetcher.rb

@@ -24,7 +24,10 @@ module Omnibus
     WIN_7Z_EXTENSIONS = %w(.7z .zip)
 
     # tar probably has compression scheme linked in, otherwise for tarballs
-    TAR_EXTENSIONS = %w(.tar .tar.gz .tgz .bz2 .tar.xz .txz)
+    COMPRESSED_TAR_EXTENSIONS = %w(.tar.gz .tgz tar.bz2 .tar.xz .txz .tar.lzma)
+    TAR_EXTENSIONS = COMPRESSED_TAR_EXTENSIONS + ['.tar']
+
+    ALL_EXTENSIONS = WIN_7Z_EXTENSIONS + TAR_EXTENSIONS
 
     # Digest types used for verifying file checksums
     DIGESTS = [:sha512, :sha256, :sha1, :md5]
@@ -51,21 +54,21 @@ module Omnibus
     end
 
     #
-    # Clean the project directory by removing the contents from disk.
+    # Clean the project directory if it exists and actually extract
+    # the downloaded file.
     #
     # @return [true, false]
     #   true if the project directory was removed, false otherwise
     #
     def clean
-      if File.exist?(project_dir)
+      needs_cleaning = File.exist?(project_dir)
+      if needs_cleaning
         log.info(log_key) { "Cleaning project directory `#{project_dir}'" }
         FileUtils.rm_rf(project_dir)
-        extract
-        true
-      else
-        extract
-        false
       end
+      create_required_directories
+      deploy
+      needs_cleaning
     end
 
     #
@@ -81,13 +84,15 @@ module Omnibus
       create_required_directories
       download
       verify_checksum!
-      extract
     end
 
     #
     # The version for this item in the cache. This is the digest of downloaded
     # file and the URL where it was downloaded from.
     #
+    # This method is called *before* clean but *after* fetch. Do not ever
+    # use the contents of the project_dir here.
+    #
     # @return [String]
     #
     def version_for_cache
@@ -182,8 +187,11 @@ module Omnibus
       }
 
       file = open(download_url, options)
-      FileUtils.cp(file.path, downloaded_file)
+      # This is a temporary file. Close and flush it before attempting to copy
+      # it over.
       file.close
+      FileUtils.cp(file.path, downloaded_file)
+      file.unlink
     rescue SocketError,
            Errno::ECONNREFUSED,
            Errno::ECONNRESET,
@@ -191,7 +199,7 @@ module Omnibus
            Timeout::Error,
            OpenURI::HTTPError => e
       if fetcher_retries != 0
-        log.debug(log_key) { "Retrying failed download (#{fetcher_retries})..." }
+        log.info(log_key) { "Retrying failed download due to #{e} (#{fetcher_retries} retries left)..." }
         fetcher_retries -= 1
         retry
       else
@@ -205,24 +213,91 @@ module Omnibus
     # ending file extension. In the rare event the file cannot be extracted, it
     # is copied over as a raw file.
     #
-    def extract
-      if command = extract_command
-        log.info(log_key) { "Extracting `#{downloaded_file}' to `#{Config.source_dir}'" }
-        shellout!(command)
+    def deploy
+      if downloaded_file.end_with?(*ALL_EXTENSIONS)
+        log.info(log_key) { "Extracting `#{safe_downloaded_file}' to `#{safe_project_dir}'" }
+        extract
       else
-        log.info(log_key) { "`#{downloaded_file}' is not an archive - copying to `#{project_dir}'" }
+        log.info(log_key) { "`#{safe_downloaded_file}' is not an archive - copying to `#{safe_project_dir}'" }
 
-        if File.directory?(project_dir)
+        if File.directory?(downloaded_file)
           # If the file itself was a directory, copy the whole thing over. This
           # seems unlikely, because I do not think it is a possible to download
           # a folder, but better safe than sorry.
-          FileUtils.cp_r(downloaded_file, project_dir)
+          FileUtils.cp_r("#{downloaded_file}/.", project_dir)
         else
           # In the more likely case that we got a "regular" file, we want that
-          # file to live **inside** the project directory.
-          FileUtils.mkdir_p(project_dir)
-          FileUtils.cp(downloaded_file, "#{project_dir}/")
+          # file to live **inside** the project directory. project_dir should already
+          # exist due to create_required_directories
+          FileUtils.cp(downloaded_file, project_dir)
+        end
+      end
+    end
+
+    #
+    # Extracts the downloaded archive file into project_dir.
+    #
+    # On windows, this is a fuster cluck and we allow users to specify the
+    # preferred extractor to be used. The default is to use tar. User overrides
+    # can be set in source[:extract] as:
+    #   :tar - use tar.exe and fail on errors (default strategy).
+    #   :seven_zip - use 7zip for all tar/compressed tar files on windows.
+    #   :lax_tar - use tar.exe on windows but ignore errors.
+    #
+    # Both 7z and bsdtar have issues on windows.
+    #
+    # 7z cannot extract and untar at the same time. You need to extract to a
+    # temporary location and then extract again into project_dir.
+    #
+    # 7z also doesn't handle symlinks well. A symlink to a non-existent
+    # location simply results in a text file with the target path written in
+    # it. It does this without throwing any errors.
+    #
+    # bsdtar will exit(1) if it is encounters symlinks on windows. So we can't
+    # use shellout! directly.
+    #
+    # bsdtar will also exit(1) and fail to overwrite files at the destination
+    # during extraction if a file already exists at the destination and is
+    # marked read-only. This used to be a problem when we weren't properly
+    # cleaning an existing project_dir. It should be less of a problem now...
+    # but who knows.
+    #
+    def extract
+      # Only used by tar
+      compression_switch = ''
+      compression_switch = 'z'        if downloaded_file.end_with?('gz')
+      compression_switch = '--lzma -' if downloaded_file.end_with?('lzma')
+      compression_switch = 'j'        if downloaded_file.end_with?('bz2')
+      compression_switch = 'J'        if downloaded_file.end_with?('xz')
+
+      if Ohai['platform'] == 'windows'
+        if downloaded_file.end_with?(*TAR_EXTENSIONS) && source[:extract] != :seven_zip
+          returns = [0]
+          returns << 1 if source[:extract] == :lax_tar
+
+          shellout!("tar.exe #{compression_switch}xf #{safe_downloaded_file} -C#{safe_project_dir}", returns: returns)
+        elsif downloaded_file.end_with?(*COMPRESSED_TAR_EXTENSIONS)
+          Dir.mktmpdir do |temp_dir|
+            log.debug(log_key) { "Temporarily extracting `#{safe_downloaded_file}' to `#{temp_dir}'" }
+
+            shellout!("7z.exe x #{safe_downloaded_file} -o#{windows_safe_path(temp_dir)} -r -y")
+
+            fname = File.basename(downloaded_file, File.extname(downloaded_file))
+            fname << ".tar" if downloaded_file.end_with?('tgz', 'txz')
+            next_file = windows_safe_path(File.join(temp_dir, fname))
+
+            log.debug(log_key) { "Temporarily extracting `#{next_file}' to `#{safe_project_dir}'" }
+            shellout!("7z.exe x #{next_file} -o#{safe_project_dir} -r -y")
+          end
+        else
+          shellout!("7z.exe x #{safe_downloaded_file} -o#{safe_project_dir} -r -y")
         end
+      elsif downloaded_file.end_with?('.7z')
+        shellout!("7z x #{safe_downloaded_file} -o#{safe_project_dir} -r -y")
+      elsif downloaded_file.end_with?('.zip')
+        shellout!("unzip #{safe_downloaded_file} -d #{safe_project_dir}")
+      else
+        shellout!("#{tar} #{compression_switch}xf #{safe_downloaded_file} -C#{safe_project_dir}")
       end
     end
 
@@ -242,7 +317,7 @@ module Omnibus
     end
 
     #
-    # Verify the downloaded file has the correct checksum.#
+    # Verify the downloaded file has the correct checksum.
     #
     # @raise [ChecksumMismatch]
     #   if the checksum does not match
@@ -258,26 +333,20 @@ module Omnibus
       end
     end
 
+    def safe_project_dir
+      windows_safe_path(project_dir)
+    end
+
+    def safe_downloaded_file
+      windows_safe_path(downloaded_file)
+    end
+
     #
     # The command to use for extracting this piece of software.
     #
-    # @return [String, nil]
+    # @return [[String]]
     #
     def extract_command
-      if Ohai['platform'] == 'windows' && downloaded_file.end_with?(*WIN_7Z_EXTENSIONS)
-        "7z.exe x #{windows_safe_path(downloaded_file)} -o#{Config.source_dir} -r -y"
-      elsif Ohai['platform'] != 'windows' && downloaded_file.end_with?('.7z')
-        "7z x #{windows_safe_path(downloaded_file)} -o#{Config.source_dir} -r -y"
-      elsif Ohai['platform'] != 'windows' && downloaded_file.end_with?('.zip')
-        "unzip #{windows_safe_path(downloaded_file)} -d #{Config.source_dir}"
-      elsif downloaded_file.end_with?(*TAR_EXTENSIONS)
-        compression_switch = 'z' if downloaded_file.end_with?('gz')
-        compression_switch = 'j' if downloaded_file.end_with?('bz2')
-        compression_switch = 'J' if downloaded_file.end_with?('xz')
-        compression_switch = ''  if downloaded_file.end_with?('tar')
-
-        "#{tar} #{compression_switch}xf #{windows_safe_path(downloaded_file)} -C#{Config.source_dir}"
-      end
     end
 
     #

data/lib/omnibus/fetchers/path_fetcher.rb

@@ -42,17 +42,14 @@ module Omnibus
     # Clean the given path by removing the project directory.
     #
     # @return [true, false]
-    #   true if the directory was cleaned, false otherwise
+    #   true if the directory was cleaned, false otherwise.
+    #   Since we do not currently use the cache to sync files and
+    #   always fetch from source, there is no need to clean anything.
+    #   The fetch step (which needs to be called before clean) would
+    #   have already removed anything extraneous.
     #
     def clean
-      if File.exist?(project_dir)
-        log.info(log_key) { "Cleaning project directory `#{project_dir}'" }
-        FileUtils.rm_rf(project_dir)
-        fetch
-        true
-      else
-        false
-      end
+      return true
     end
 
     #
@@ -74,10 +71,14 @@ module Omnibus
     # The version for this item in the cache. The is the shasum of the directory
     # on disk.
     #
+    # This method is called *before* clean but *after* fetch. Since fetch
+    # automatically cleans, target vs. destination sha doesn't matter. Change this
+    # if that assumption changes.
+    #
     # @return [String]
     #
     def version_for_cache
-      "path:#{source_path}|shasum:#{target_shasum}"
+      "path:#{source_path}|shasum:#{destination_shasum}"
     end
 
     #
@@ -117,7 +118,7 @@ module Omnibus
     # @return [String]
     #
     def target_shasum
-      @target_shasum ||= digest_directory(project_dir, :sha256)
+      @target_shasum ||= digest_directory(project_dir, :sha256, source_options)
     end
 
     #
@@ -126,7 +127,7 @@ module Omnibus
     # @return [String]
     #
     def destination_shasum
-      @destination_shasum ||= digest_directory(source_path, :sha256)
+      @destination_shasum ||= digest_directory(source_path, :sha256, source_options)
     end
   end
 end

data/lib/omnibus/file_syncer.rb

@@ -40,6 +40,31 @@ module Omnibus
       end
     end
 
+    #
+    # Glob for all files under a given path/pattern, removing Ruby's
+    # dumb idea to include +'.'+ and +'..'+ as entries.
+    #
+    # @param [String] source
+    #   the path or glob pattern to get all files from
+    #
+    # @option options [String, Array<String>] :exclude
+    #   a file, folder, or globbing pattern of files to ignore when syncing
+    #
+    # @return [Array<String>]
+    #   the list of all files
+    #
+    def all_files_under(source, options = {})
+      excludes = Array(options[:exclude]).map do |exclude|
+        [exclude, "#{exclude}/*"]
+      end.flatten
+
+      source_files = glob(File.join(source, '**/*'))
+      source_files = source_files.reject do |source_file|
+        basename = relative_path_for(source_file, source)
+        excludes.any? { |exclude| File.fnmatch?(exclude, basename, File::FNM_DOTMATCH) }
+      end
+    end
+
     #
     # Copy the files from +source+ to +destination+, while removing any files
     # in +destination+ that are not present in +source+.
@@ -69,16 +94,7 @@ module Omnibus
           "the `copy' method instead."
       end
 
-      # Reject any files that match the excludes pattern
-      excludes = Array(options[:exclude]).map do |exclude|
-        [exclude, "#{exclude}/*"]
-      end.flatten
-
-      source_files = glob(File.join(source, '**/*'))
-      source_files = source_files.reject do |source_file|
-        basename = relative_path_for(source_file, source)
-        excludes.any? { |exclude| File.fnmatch?(exclude, basename, File::FNM_DOTMATCH) }
-      end
+      source_files = all_files_under(source, options)
 
       # Ensure the destination directory exists
       FileUtils.mkdir_p(destination) unless File.directory?(destination)
@@ -106,9 +122,13 @@ module Omnibus
           # duplicate them, provided their source is in place already
           if hardlink? source_stat
             if existing = hardlink_sources[[source_stat.dev, source_stat.ino]]
-              FileUtils.ln(existing, "#{destination}/#{relative_path}")
+              FileUtils.ln(existing, "#{destination}/#{relative_path}", force: true)
             else
-              FileUtils.cp(source_file, "#{destination}/#{relative_path}")
+              begin
+                FileUtils.cp(source_file, "#{destination}/#{relative_path}")
+              rescue Errno::EACCES
+                FileUtils.cp_r(source_file, "#{destination}/#{relative_path}", remove_destination: true)
+              end
               hardlink_sources.store([source_stat.dev, source_stat.ino], "#{destination}/#{relative_path}")
             end
           else
@@ -120,8 +140,7 @@ module Omnibus
             begin
               FileUtils.cp(source_file, "#{destination}/#{relative_path}")
             rescue Errno::EACCES
-              FileUtils.cp_r(source_file, "#{destination}/#{relative_path}",
-                             :remove_destination => true)
+              FileUtils.cp_r(source_file, "#{destination}/#{relative_path}", remove_destination: true)
             end
           end
         else

data/lib/omnibus/generator_files/README.md.erb

@@ -5,7 +5,7 @@ This project creates full-stack platform-specific packages for
 
 Installation
 ------------
-You must have a sane Ruby 1.9+ environment with Bundler installed. Ensure all
+You must have a sane Ruby 2.0.0+ environment with Bundler installed. Ensure all
 the required gems are installed:
 
 ```shell

data/lib/omnibus/generator_files/package_scripts/postinst.erb

@@ -1,12 +1,12 @@
-#!/bin/bash
+#!/bin/sh
 #
 # Perform necessary <%= config[:name] %> setup steps
 # after package is installed.
 #
 
-PROGNAME=$(basename $0)
+PROGNAME=`basename $0`
 
-function error_exit
+error_exit()
 {
   echo "${PROGNAME}: ${1:-"Unknown Error"}" 1>&2
   exit 1

data/lib/omnibus/generator_files/package_scripts/postrm.erb

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/sh
 #
 # Perform necessary <%= config[:name] %> removal steps
 # after package is uninstalled.

data/lib/omnibus/generator_files/package_scripts/preinst.erb

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/sh
 #
 # Perform necessary <%= config[:name] %> setup steps
 # before package is installed.

data/lib/omnibus/generator_files/package_scripts/prerm.erb

@@ -1,12 +1,12 @@
-#!/bin/bash
+#!/bin/sh
 #
 # Perform necessary <%= config[:name] %> setup steps
 # prior to installing package.
 #
 
-PROGNAME=$(basename $0)
+PROGNAME=`basename $0`
 
-function error_exit
+error_exit()
 {
   echo "${PROGNAME}: ${1:-"Unknown Error"}" 1>&2
   exit 1

data/lib/omnibus/git_cache.rb

@@ -64,7 +64,7 @@ module Omnibus
         false
       else
         create_directory(File.dirname(cache_path))
-        shellout!("git --git-dir=#{cache_path} init -q")
+        git_cmd('init -q')
         true
       end
     end
@@ -113,15 +113,15 @@ module Omnibus
       create_cache_path
       remove_git_dirs
 
-      shellout!(%Q(git --git-dir=#{cache_path} --work-tree=#{install_dir} add -A -f))
+      git_cmd('add -A -f')
 
       begin
-        shellout!(%Q(git --git-dir=#{cache_path} --work-tree=#{install_dir} commit -q -m "Backup of #{tag}"))
+        git_cmd(%Q(commit -q -m "Backup of #{tag}"))
       rescue CommandFailed => e
         raise unless e.message.include?('nothing to commit')
       end
 
-      shellout!(%Q(git --git-dir=#{cache_path} --work-tree=#{install_dir} tag -f "#{tag}"))
+      git_cmd(%Q(tag -f "#{tag}"))
     end
 
     def restore
@@ -129,16 +129,16 @@ module Omnibus
 
       create_cache_path
 
-      cmd = shellout(%Q(git --git-dir=#{cache_path} --work-tree=#{install_dir} tag -l "#{tag}"))
-
       restore_me = false
+      cmd = git_cmd(%Q(tag -l "#{tag}"))
+
       cmd.stdout.each_line do |line|
         restore_me = true if tag == line.chomp
       end
 
       if restore_me
         log.internal(log_key) { "Detected tag `#{tag}' can be restored, restoring" }
-        shellout!(%Q(git --git-dir=#{cache_path} --work-tree=#{install_dir} checkout -f "#{tag}"))
+        git_cmd(%Q(checkout -f "#{tag}"))
         true
       else
         log.internal(log_key) { "Could not find tag `#{tag}', skipping restore" }
@@ -169,6 +169,19 @@ module Omnibus
 
     private
 
+    #
+    # Shell out and invoke a git command in the context of the git cache.
+    #
+    # We explicitly disable autocrlf because we want bit-for-bit storage and
+    # recovery of build output. Hashes calculated on output files will be
+    # invalid if we muck around with files after they have been produced.
+    #
+    # @return [Mixlib::Shellout] the underlying command object.
+    #
+    def git_cmd(command)
+      shellout!("git -c core.autocrlf=false --git-dir=#{cache_path} --work-tree=#{install_dir} #{command}")
+    end
+
     #
     #
     # The installation directory for this software's project. Drive letters are