omnibus 4.0.0 → 4.1.0

data/lib/omnibus/fetchers/net_fetcher.rb

@@ -16,6 +16,7 @@
 
 require 'fileutils'
 require 'open-uri'
+require 'ruby-progressbar'
 
 module Omnibus
   class NetFetcher < Fetcher
@@ -25,6 +26,9 @@ module Omnibus
     # tar probably has compression scheme linked in, otherwise for tarballs
     TAR_EXTENSIONS = %w(.tar .tar.gz .tgz .bz2 .tar.xz .txz)
 
+    # Digest types used for verifying file checksums
+    DIGESTS = [:sha512, :sha256, :sha1, :md5]
+
     #
     # A fetch is required if the downloaded_file (such as a tarball) does not
     # exist on disk, or if the checksum of the downloaded file is different
@@ -33,7 +37,7 @@ module Omnibus
     # @return [true, false]
     #
     def fetch_required?
-      !(File.exist?(downloaded_file) && digest(downloaded_file, :md5) == checksum)
+      !(File.exist?(downloaded_file) && digest(downloaded_file, digest_type) == checksum)
     end
 
     #
@@ -43,7 +47,7 @@ module Omnibus
     # @return [String]
     #
     def version_guid
-      "md5:#{checksum}"
+      "#{digest_type}:#{checksum}"
     end
 
     #
@@ -81,13 +85,24 @@ module Omnibus
     end
 
     #
-    # The version for this item in the cache. The is the md5 of downloaded file
-    # and the URL where it was downloaded from.
+    # The version for this item in the cache. This is the digest of downloaded
+    # file and the URL where it was downloaded from.
     #
     # @return [String]
     #
     def version_for_cache
-      "download_url:#{source[:url]}|md5:#{source[:md5]}"
+      "download_url:#{source[:url]}|#{digest_type}:#{checksum}"
+    end
+
+    #
+    # Returned the resolved version for the manifest.  Since this is a
+    # remote URL, there is no resolution, the version is what we said
+    # it is.
+    #
+    # @return [String]
+    #
+    def self.resolve_version(version, source)
+      version
     end
 
     #
@@ -102,12 +117,12 @@ module Omnibus
     end
 
     #
-    # The checksum (+md5+) as defined by the user in the software definition.
+    # The checksum as defined by the user in the software definition.
     #
     # @return [String]
     #
     def checksum
-      source[:md5]
+      source[digest_type]
     end
 
     private
@@ -123,7 +138,7 @@ module Omnibus
     #
     def download_url
       if Config.use_s3_caching
-        "http://#{Config.s3_bucket}.s3.amazonaws.com/#{S3Cache.key_for(software)}"
+        "http://#{Config.s3_bucket}.s3.amazonaws.com/#{S3Cache.key_for(self)}"
       else
         source[:url]
       end
@@ -147,6 +162,24 @@ module Omnibus
       end
 
       options[:read_timeout] = Omnibus::Config.fetcher_read_timeout
+      fetcher_retries ||= Omnibus::Config.fetcher_retries
+
+      progress_bar = ProgressBar.create(
+        output: $stdout,
+        format: '%e %B %p%% (%r KB/sec)',
+        rate_scale: ->(rate) { rate / 1024 },
+      )
+
+      reported_total = 0
+
+      options[:content_length_proc] = ->(total) {
+        reported_total = total
+        progress_bar.total = total
+      }
+      options[:progress_proc] = ->(step) {
+        downloaded_amount = [step, reported_total].min
+        progress_bar.progress = downloaded_amount
+      }
 
       file = open(download_url, options)
       FileUtils.cp(file.path, downloaded_file)
@@ -155,9 +188,16 @@ module Omnibus
            Errno::ECONNREFUSED,
            Errno::ECONNRESET,
            Errno::ENETUNREACH,
+           Timeout::Error,
            OpenURI::HTTPError => e
-      log.error(log_key) { "Download failed - #{e.class}!" }
-      raise
+      if fetcher_retries != 0
+        log.debug(log_key) { "Retrying failed download (#{fetcher_retries})..." }
+        fetcher_retries -= 1
+        retry
+      else
+        log.error(log_key) { "Download failed - #{e.class}!" }
+        raise
+      end
     end
 
     #
@@ -168,7 +208,7 @@ module Omnibus
     def extract
       if command = extract_command
         log.info(log_key) { "Extracting `#{downloaded_file}' to `#{Config.source_dir}'" }
-        shellout!(extract_command)
+        shellout!(command)
       else
         log.info(log_key) { "`#{downloaded_file}' is not an archive - copying to `#{project_dir}'" }
 
@@ -186,6 +226,17 @@ module Omnibus
       end
     end
 
+    #
+    # The digest type defined in the software definition
+    #
+    # @return [Symbol]
+    #
+    def digest_type
+      DIGESTS.each do |digest|
+        return digest if source.key? digest
+      end
+    end
+
     #
     # Verify the downloaded file has the correct checksum.#
     #
@@ -196,10 +247,10 @@ module Omnibus
       log.info(log_key) { 'Verifying checksum' }
 
       expected = checksum
-      actual   = digest(downloaded_file, :md5)
+      actual   = digest(downloaded_file, digest_type)
 
       if expected != actual
-        raise ChecksumMismatch.new(software, expected, actual)
+        raise ChecksumMismatch.new(self, expected, actual)
       end
     end
 

data/lib/omnibus/fetchers/null_fetcher.rb

@@ -30,6 +30,13 @@ module Omnibus
       nil
     end
 
+    #
+    # @return [String, nil]
+    #
+    def self.resolve_version(version, source)
+      version
+    end
+
     #
     # @return [false]
     #
@@ -41,7 +48,7 @@ module Omnibus
     # @return [void]
     #
     def fetch
-      log.info(log_key) { "Fetching `#{software.name}' (nothing to fetch)" }
+      log.info(log_key) { "Fetching `#{name}' (nothing to fetch)" }
 
       create_required_directories
       nil

data/lib/omnibus/fetchers/path_fetcher.rb

@@ -64,7 +64,10 @@ module Omnibus
       log.info(log_key) { "Copying from `#{source_path}'" }
 
       create_required_directories
-      FileSyncer.sync(source_path, project_dir)
+      FileSyncer.sync(source_path, project_dir, source_options)
+      # Reset target shasum on every fetch
+      @target_shasum = nil
+      target_shasum
     end
 
     #
@@ -77,6 +80,13 @@ module Omnibus
       "path:#{source_path}|shasum:#{target_shasum}"
     end
 
+    #
+    # @return [String, nil]
+    #
+    def self.resolve_version(version, source)
+      version
+    end
+
     private
 
     #
@@ -88,6 +98,19 @@ module Omnibus
       source[:path]
     end
 
+    #
+    # Options to pass to the underlying FileSyncer
+    #
+    # @return [Hash]
+    #
+    def source_options
+      if source[:options] && source[:options].is_a?(Hash)
+        source[:options]
+      else
+        {}
+      end
+    end
+
     #
     # The shasum of the directory **inside** the project.
     #

data/lib/omnibus/file_syncer.rb

@@ -101,7 +101,29 @@ module Omnibus
             FileUtils.ln_sf(target, "#{destination}/#{relative_path}")
           end
         when :file
-          FileUtils.cp(source_file, "#{destination}/#{relative_path}")
+          source_stat = File.stat(source_file)
+          # Detect 'files' which are hard links and use ln instead of cp to
+          # duplicate them, provided their source is in place already
+          if hardlink? source_stat
+            if existing = hardlink_sources[[source_stat.dev, source_stat.ino]]
+              FileUtils.ln(existing, "#{destination}/#{relative_path}")
+            else
+              FileUtils.cp(source_file, "#{destination}/#{relative_path}")
+              hardlink_sources.store([source_stat.dev, source_stat.ino], "#{destination}/#{relative_path}")
+            end
+          else
+            # First attempt a regular copy. If we don't have write
+            # permission on the File, open will probably fail with
+            # EACCES (making it hard to sync files with permission
+            # r--r--r--). Rescue this error and use cp_r's
+            # :remove_destination option.
+            begin
+              FileUtils.cp(source_file, "#{destination}/#{relative_path}")
+            rescue Errno::EACCES
+              FileUtils.cp_r(source_file, "#{destination}/#{relative_path}",
+                             :remove_destination => true)
+            end
+          end
         else
           raise RuntimeError,
             "Unknown file type: `File.ftype(source_file)' at `#{source_file}'!"
@@ -145,5 +167,34 @@ module Omnibus
     def relative_path_for(path, parent)
       Pathname.new(path).relative_path_from(Pathname.new(parent)).to_s
     end
+
+    #
+    # A list of hard link file(s) sources which have already been copied,
+    # indexed by device and inode number.
+    #
+    # @api private
+    #
+    # @return [Hash{Array<FixNum, FixNum> => String}]
+    #
+    def hardlink_sources
+      @hardlink_sources ||= {}
+    end
+
+    #
+    # Determines whether or not a file is a hardlink.
+    #
+    # @param [File::Stat] stat
+    #   the File::Stat object for a file you wand to test
+    #
+    # @return [true, false]
+    #
+    def hardlink?(stat)
+      case stat.ftype.to_sym
+      when :file
+        stat.nlink > 1
+      else
+        false
+      end
+    end
   end
 end

data/lib/omnibus/generator.rb

@@ -106,53 +106,54 @@ module Omnibus
     def create_bff_assets
       return unless options[:bff_assets]
 
-      copy_file(resource_path('bff/gen.template.erb'), "#{target}/resources/bff/gen.template.erb")
+      copy_file(resource_path('bff/gen.template.erb'), "#{target}/resources/#{name}/bff/gen.template.erb")
     end
 
     def create_deb_assets
       return unless options[:deb_assets]
 
-      copy_file(resource_path('deb/conffiles.erb'), "#{target}/resources/deb/conffiles.erb")
-      copy_file(resource_path('deb/control.erb'), "#{target}/resources/deb/control.erb")
-      copy_file(resource_path('deb/md5sums.erb'), "#{target}/resources/deb/md5sums.erb")
+      copy_file(resource_path('deb/conffiles.erb'), "#{target}/resources/#{name}/deb/conffiles.erb")
+      copy_file(resource_path('deb/control.erb'), "#{target}/resources/#{name}/deb/control.erb")
+      copy_file(resource_path('deb/md5sums.erb'), "#{target}/resources/#{name}/deb/md5sums.erb")
     end
 
     def create_dmg_assets
       return unless options[:dmg_assets]
 
-      copy_file(resource_path('dmg/background.png'), "#{target}/resources/dmg/background.png")
-      copy_file(resource_path('dmg/icon.png'), "#{target}/resources/dmg/icon.png")
+      copy_file(resource_path('dmg/background.png'), "#{target}/resources/#{name}/dmg/background.png")
+      copy_file(resource_path('dmg/icon.png'), "#{target}/resources/#{name}/dmg/icon.png")
     end
 
     def create_msi_assets
       return unless options[:msi_assets]
 
-      copy_file(resource_path('msi/localization-en-us.wxl.erb'), "#{target}/resources/msi/localization-en-us.wxl.erb")
-      copy_file(resource_path('msi/parameters.wxi.erb'), "#{target}/resources/msi/parameters.wxi.erb")
-      copy_file(resource_path('msi/source.wxs.erb'), "#{target}/resources/msi/source.wxs.erb")
+      copy_file(resource_path('msi/localization-en-us.wxl.erb'), "#{target}/resources/#{name}/msi/localization-en-us.wxl.erb")
+      copy_file(resource_path('msi/parameters.wxi.erb'), "#{target}/resources/#{name}/msi/parameters.wxi.erb")
+      copy_file(resource_path('msi/source.wxs.erb'), "#{target}/resources/#{name}/msi/source.wxs.erb")
 
-      copy_file(resource_path('msi/assets/LICENSE.rtf'), "#{target}/resources/msi/assets/LICENSE.rtf")
-      copy_file(resource_path('msi/assets/banner_background.bmp'), "#{target}/resources/msi/assets/banner_background.bmp")
-      copy_file(resource_path('msi/assets/dialog_background.bmp'), "#{target}/resources/msi/assets/dialog_background.bmp")
-      copy_file(resource_path('msi/assets/project.ico'), "#{target}/resources/msi/assets/project.ico")
-      copy_file(resource_path('msi/assets/project_16x16.ico'), "#{target}/resources/msi/assets/project_16x16.ico")
-      copy_file(resource_path('msi/assets/project_32x32.ico'), "#{target}/resources/msi/assets/project_32x32.ico")
+      copy_file(resource_path('msi/assets/LICENSE.rtf'), "#{target}/resources/#{name}/msi/assets/LICENSE.rtf")
+      copy_file(resource_path('msi/assets/banner_background.bmp'), "#{target}/resources/#{name}/msi/assets/banner_background.bmp")
+      copy_file(resource_path('msi/assets/dialog_background.bmp'), "#{target}/resources/#{name}/msi/assets/dialog_background.bmp")
+      copy_file(resource_path('msi/assets/project.ico'), "#{target}/resources/#{name}/msi/assets/project.ico")
+      copy_file(resource_path('msi/assets/project_16x16.ico'), "#{target}/resources/#{name}/msi/assets/project_16x16.ico")
+      copy_file(resource_path('msi/assets/project_32x32.ico'), "#{target}/resources/#{name}/msi/assets/project_32x32.ico")
     end
 
     def create_pkg_assets
       return unless options[:pkg_assets]
 
-      copy_file(resource_path('pkg/background.png'), "#{target}/resources/pkg/background.png")
-      copy_file(resource_path('pkg/license.html.erb'), "#{target}/resources/pkg/license.html.erb")
-      copy_file(resource_path('pkg/welcome.html.erb'), "#{target}/resources/pkg/welcome.html.erb")
+      copy_file(resource_path('pkg/background.png'), "#{target}/resources/#{name}/pkg/background.png")
+      copy_file(resource_path('pkg/license.html.erb'), "#{target}/resources/#{name}/pkg/license.html.erb")
+      copy_file(resource_path('pkg/welcome.html.erb'), "#{target}/resources/#{name}/pkg/welcome.html.erb")
+      copy_file(resource_path('pkg/distribution.xml.erb'), "#{target}/resources/#{name}/pkg/distribution.xml.erb")
     end
 
     def create_rpm_assets
       return unless options[:rpm_assets]
 
-      copy_file(resource_path('rpm/rpmmacros.erb'), "#{target}/resources/rpm/rpmmacros.erb")
-      copy_file(resource_path('rpm/signing.erb'), "#{target}/resources/rpm/signing.erb")
-      copy_file(resource_path('rpm/spec.erb'), "#{target}/resources/rpm/spec.erb")
+      copy_file(resource_path('rpm/rpmmacros.erb'), "#{target}/resources/#{name}/rpm/rpmmacros.erb")
+      copy_file(resource_path('rpm/signing.erb'), "#{target}/resources/#{name}/rpm/signing.erb")
+      copy_file(resource_path('rpm/spec.erb'), "#{target}/resources/#{name}/rpm/spec.erb")
     end
     private
 

data/lib/omnibus/generator_files/.kitchen.yml.erb

@@ -9,22 +9,22 @@ driver:
 
 provisioner:
   name: chef_zero
-  require_chef_omnibus: 11.16.4
+  require_chef_omnibus: 12.4.1
 
 platforms:
-  - name: centos-7.0
+  - name: centos-7.1
     run_list: yum-epel::default
-  - name: centos-6.5
+  - name: centos-6.6
     run_list: yum-epel::default
-  - name: centos-5.10
+  - name: centos-5.11
     run_list: yum-epel::default
-  - name: debian-7.2.0
+  - name: debian-7.8
     run_list: apt::default
-  - name: debian-6.0.8
+  - name: debian-6.0.10
     run_list: apt::default
-  - name: freebsd-10.0
+  - name: freebsd-10.1
     run_list: freebsd::portsnap
-  - name: freebsd-9.2
+  - name: freebsd-9.3
     run_list:
       - freebsd::portsnap
       - freebsd::pkgng
@@ -32,10 +32,6 @@ platforms:
     run_list: apt::default
   - name: ubuntu-12.04
     run_list: apt::default
-  - name: ubuntu-11.04
-    run_list: apt::default
-  - name: ubuntu-10.04
-    run_list: apt::default
 
 suites:
   - name: default

data/lib/omnibus/generator_files/Berksfile.erb

@@ -1,4 +1,4 @@
-source 'https://api.berkshelf.com'
+source 'https://supermarket.chef.io'
 
 cookbook 'omnibus'
 
@@ -6,7 +6,7 @@ cookbook 'omnibus'
 # cookbook 'omnibus', github: 'opscode-cookbooks/omnibus'
 
 group :integration do
-  cookbook 'apt',      '~> 2.3'
-  cookbook 'freebsd',  '~> 0.1'
-  cookbook 'yum-epel', '~> 0.3'
+  cookbook 'apt',      '~> 2.8'
+  cookbook 'freebsd',  '~> 0.3'
+  cookbook 'yum-epel', '~> 0.6'
 end

data/lib/omnibus/generator_files/Gemfile.erb

@@ -13,9 +13,9 @@ gem 'omnibus', '~> <%= Omnibus::VERSION.split('.')[0...-1].join('.') %>'
 # by running `bundle install --without development` to speed up build times.
 group :development do
   # Use Berkshelf for resolving cookbook dependencies
-  gem 'berkshelf', '~> 3.0'
+  gem 'berkshelf', '~> 3.3'
 
   # Use Test Kitchen with Vagrant for converging the build environment
-  gem 'test-kitchen',    '~> 1.2'
-  gem 'kitchen-vagrant', '~> 0.14'
+  gem 'test-kitchen',    '~> 1.4'
+  gem 'kitchen-vagrant', '~> 0.18'
 end