omnibus 3.1.1 → 3.2.0.rc.1

data/lib/omnibus/fetcher.rb

@@ -95,7 +95,7 @@ module Omnibus
     # @return [Omnibus::Fetcher]
     def self.for(software)
       if software.source
-        if software.source[:url] && Omnibus.config.use_s3_caching
+        if software.source[:url] && Config.use_s3_caching
           S3CacheFetcher.new(software)
         else
           without_caching_for(software)

data/lib/omnibus/fetchers/net_fetcher.rb

@@ -27,16 +27,20 @@ module Omnibus
     attr_reader :project_file
     attr_reader :source
     attr_reader :source_uri
-    attr_reader :source_dir
     attr_reader :project_dir
 
+    # Use 7-zip to extract 7z/zip for Windows
+    WIN_7Z_EXTENSIONS = %w(.7z .zip)
+
+    # tar probably has compression scheme linked in, otherwise for tarballs
+    TAR_EXTENSIONS = %w(.tar .tar.gz .tgz .bz2 .tar.xz .txz)
+
     def initialize(software)
       @name         = software.name
       @checksum     = software.checksum
       @source       = software.source
       @project_file = software.project_file
       @source_uri   = software.source_uri
-      @source_dir   = software.source_dir
       @project_dir  = software.project_dir
       super
     end
@@ -179,7 +183,7 @@ module Omnibus
 
     def extract
       log.info(log_key) do
-        "Extracting the source in '#{project_file}' to '#{source_dir}'"
+        "Extracting the source in '#{project_file}' to '#{Config.source_dir}'"
       end
 
       cmd = extract_cmd
@@ -197,16 +201,18 @@ module Omnibus
     end
 
     def extract_cmd
-      if project_file.end_with?('.gz') || project_file.end_with?('.tgz')
-        "gzip -dc  #{project_file} | ( cd #{source_dir} && tar -xf - )"
-      elsif project_file.end_with?('.bz2')
-        "bzip2 -dc  #{project_file} | ( cd #{source_dir} && tar -xf - )"
-      elsif project_file.end_with?('.7z')
-        "7z.exe x #{project_file} -o#{source_dir} -r -y"
-      elsif project_file.end_with?('.zip')
-        "unzip #{project_file} -d #{source_dir}"
-      elsif project_file.end_with?('.xz') || project_file.end_with?('.txz')
-        "xz -dc #{project_file} | ( cd #{source_dir} && tar -xf - )"
+      if Ohai['platform'] == 'windows' && project_file.end_with?(*WIN_7Z_EXTENSIONS)
+        "7z.exe x #{project_file} -o#{Config.source_dir} -r -y"
+      elsif Ohai['platform'] != 'windows' && project_file.end_with?('.7z')
+        "7z x #{project_file} -o#{Config.source_dir} -r -y"
+      elsif Ohai['platform'] != 'windows' && project_file.end_with?('.zip')
+        "unzip #{project_file} -d #{Config.source_dir}"
+      elsif project_file.end_with?(*TAR_EXTENSIONS)
+        compression_switch = 'z' if project_file.end_with?('gz')
+        compression_switch = 'j' if project_file.end_with?('bz2')
+        compression_switch = 'J' if project_file.end_with?('xz')
+        compression_switch = '' if project_file.end_with?('tar')
+        "tar #{compression_switch}xf #{project_file} -C#{Config.source_dir}"
       else
         # if we don't recognize the extension, simply copy over the file
         proc do

data/lib/omnibus/fetchers/path_fetcher.rb

@@ -17,6 +17,9 @@
 module Omnibus
   # Fetcher implementation for projects on the filesystem
   class PathFetcher < Fetcher
+
+    include Digestable
+
     def initialize(software)
       @name = software.name
       @source = software.source
@@ -33,7 +36,7 @@ module Omnibus
     end
 
     def rsync
-      if Ohai.platform == 'windows'
+      if Ohai['platform'] == 'windows'
         # Robocopy's return code is 1 if it succesfully copies over the
         # files and 0 if the files are already existing at the destination
         sync_cmd = "robocopy #{@source[:path]}\\ #{@project_dir}\\ /MIR /S"
@@ -53,6 +56,10 @@ module Omnibus
       rsync
     end
 
+    def version_for_cache
+      @version_for_cache ||= digest_directory(@project_dir, :sha256)
+    end
+
     def fetch_required?
       true
     end

data/lib/omnibus/fetchers/s3_cache_fetcher.rb

@@ -16,13 +16,6 @@
 
 module Omnibus
   class S3CacheFetcher < NetFetcher
-    include SoftwareS3URLs
-
-    def initialize(software)
-      @software = software
-      super
-    end
-
     def fetch
       log.info(log_key) do
         "S3 Cache enabled, '#{name}' will be fetched from S3 cache"
@@ -31,8 +24,24 @@ module Omnibus
       super
     end
 
+    #
+    # The source URI for the software definition that is being fetched.
+    #
+    # @return [URI]
+    #
     def source_uri
       URI.parse(url_for(@software))
     end
+
+    private
+
+    #
+    # The URL for the cached software.
+    #
+    # @return [String]
+    #
+    def url_for(software)
+      "http://#{Config.s3_bucket}.s3.amazonaws.com/#{S3Cache.key_for(software)}"
+    end
   end
 end

data/lib/omnibus/generator.rb

@@ -48,7 +48,7 @@ module Omnibus
       template('Gemfile.erb', "#{target}/Gemfile", template_options)
       template('gitignore.erb', "#{target}/.gitignore", template_options)
       template('README.md.erb', "#{target}/README.md", template_options)
-      template('omnibus.rb.example.erb', "#{target}/omnibus.rb.example", template_options)
+      template('omnibus.rb.erb', "#{target}/omnibus.rb", template_options)
     end
 
     def create_project_definition
@@ -124,7 +124,7 @@ module Omnibus
     def template_options
       @template_options ||= {
         name: name,
-        install_path: "/opt/#{name}",
+        install_dir: "/opt/#{name}",
       }
     end
   end

data/lib/omnibus/generator_files/Gemfile.erb

@@ -5,7 +5,10 @@ gem 'berkshelf', '~> 3.0'
 
 # Install omnibus software
 gem 'omnibus', '~> <%= Omnibus::VERSION.split('.')[0...-1].join('.') %>'
-gem 'omnibus-software', github: 'opscode/omnibus-software'
+
+# Use Chef's software definitions. It is recommended that you write your own
+# software definitions, but you can clone/fork Chef's to get you started.
+# gem 'omnibus-software', github: 'opscode/omnibus-software'
 
 # Use Test Kitchen with Vagrant for converging the build environment
 gem 'test-kitchen',    '~> 1.2'

data/lib/omnibus/generator_files/README.md.erb

@@ -44,6 +44,16 @@ the package cache directory (`/var/cache/omnibus/pkg`):
 $ bin/omnibus clean <%= config[:name] %> --purge
 ```
 
+### Publish
+
+Omnibus has a built-in mechanism for releasing to a variety of "backends", such
+as Amazon S3. You must set the proper credentials in your `omnibus.rb` config
+file or specify them via the command line.
+
+```shell
+$ bin/omnibus publish path/to/*.deb --backend s3
+```
+
 ### Help
 
 Full help for the Omnibus command line interface can be accessed with the

data/lib/omnibus/generator_files/{omnibus.rb.example.erb → omnibus.rb.erb}

@@ -1,13 +1,13 @@
 #
 # This file is used to configure the <%= config[:name] %> project. It contains
-# come minimal configuration examples for working with Omnibus. For a full list
+# some minimal configuration examples for working with Omnibus. For a full list
 # of configurable options, please see the documentation for +omnibus/config.rb+.
 #
 
 # Build internally
 # ------------------------------
 # By default, Omnibus uses system folders (like +/var+ and +/opt+) to build and
-# cache compontents. If you would to build everything internally, you can
+# cache components. If you would to build everything internally, you can
 # uncomment the following options. This will prevent the need for root
 # permissions in most cases. You will also need to update the <%= config[:name] %>
 # project configuration to build at +./local/omnibus/build+ instead of
@@ -19,21 +19,30 @@
 #
 # Alternatively you can tune the individual values
 # ------------------------------------------------
-# cache_dir              './local/omnibus/cache'
-# install_path_cache_dir './local/omnibus/cache/install_path'
-# source_dir             './local/omnibus/src'
-# build_dir              './local/omnibus/build'
-# package_dir            './local/omnibus/pkg'
-# package_tmp            './local/omnibus/pkg-tmp'
+# cache_dir     './local/omnibus/cache'
+# git_cache_dir './local/omnibus/cache/install_path'
+# source_dir    './local/omnibus/src'
+# build_dir     './local/omnibus/build'
+# package_dir   './local/omnibus/pkg'
+# package_tmp   './local/omnibus/pkg-tmp'
+
+# Disable git caching
+# ------------------------------
+# use_git_caching false
 
 # Enable S3 asset caching
 # ------------------------------
 # use_s3_caching true
-# s3_access_key  'something'
-# s3_secret_key  'something'
-# s3_bucket      'some-bucket'
+# s3_access_key  ENV['S3_ACCESS_KEY']
+# s3_secret_key  ENV['S3_SECRET_KEY']
+# s3_bucket      ENV['S3_BUCKET']
 
 # Customize compiler bits
 # ------------------------------
 # solaris_compiler 'gcc'
 # build_retries 5
+
+# Load additional software
+# ------------------------------
+# software_gems ['omnibus-software', 'my-company-software']
+# local_software_dirs ['/path/to/local/software']

data/lib/omnibus/generator_files/package_scripts/makeselfinst.erb

@@ -5,7 +5,7 @@
 
 PROGNAME=`basename $0`
 INSTALLER_DIR=`dirname $0`
-DEST_DIR=<%= config[:install_path] %>
+DEST_DIR=<%= config[:install_dir] %>
 CONFIG_DIR=/etc/<%= config[:name] %>
 USAGE="usage: $0"
 

data/lib/omnibus/generator_files/project.rb.erb

@@ -1,9 +1,9 @@
 
 name '<%= config[:name] %>'
 maintainer 'CHANGE ME'
-homepage 'CHANGEME.com'
+homepage 'https://CHANGE-ME.com'
 
-install_path    '<%= config[:install_path] %>'
+install_dir     '<%= config[:install_dir] %>'
 build_version   Omnibus::BuildVersion.semver
 build_iteration 1
 

data/lib/omnibus/generator_files/windows_msi/localization-en-us.wxl.erb

@@ -2,8 +2,8 @@
 <WixLocalization Culture="en-us" xmlns="http://schemas.microsoft.com/wix/2006/localization">
   <!-- http://wix.codeplex.com/SourceControl/changeset/view/792e101c5cf7#src%2fext%2fUIExtension%2fwixlib%2fWixUI_en-us.wxl -->
   <String Id="LANG">1033</String>
-  <String Id="ProductName"><%= friendly_name %></String>
-  <String Id="ManufacturerName"><%= maintainer %></String>
+  <String Id="ProductName"><%= project.friendly_name %></String>
+  <String Id="ManufacturerName"><%= project.maintainer %></String>
   <String Id="WelcomeDlgTitle">{\WixUI_Font_Bigger}Welcome to the [ProductName] Setup Wizard</String>
 
   <String Id="LicenseAgreementDlgTitle">{\WixUI_Font_Title_White}End-User License Agreement</String>
@@ -16,5 +16,5 @@
 
   <String Id="VerifyReadyDlgInstallTitle">{\WixUI_Font_Title_White}Ready to install [ProductName]</String>
 
-  <String Id="FeatureMainName"><%= friendly_name %></String>
+  <String Id="FeatureMainName"><%= project.friendly_name %></String>
 </WixLocalization>

data/lib/omnibus/git_cache.rb

@@ -0,0 +1,192 @@
+#
+# Copyright 2014 Chef Software, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'digest'
+require 'fileutils'
+
+module Omnibus
+  class GitCache
+    include Util
+    include Logging
+
+    REQUIRED_GIT_FILES = [
+      'HEAD',
+      'description',
+      'hooks',
+      'info',
+      'objects',
+      'refs',
+    ].freeze
+
+    #
+    # @return [Software]
+    #
+    attr_reader :software
+
+    #
+    # @param [Software] software
+    #   the software this git cache is for
+    #
+    def initialize(software)
+      @software = software
+    end
+
+    #
+    # The path to the full install_dir cache for the project.
+    #
+    # @return [String]
+    #
+    def cache_path
+      @cache_path ||= File.join(Config.git_cache_dir, install_dir)
+    end
+
+    #
+    # Creates the full path if it does not exist already.
+    #
+    # @return [true, false]
+    #   true if the path was created, false otherwise
+    #
+    def create_cache_path
+      if File.directory?(cache_path)
+        log.info(log_key) { "Cache path `#{cache_path}' exists, skipping creation" }
+        false
+      else
+        log.info(log_key) { "Creating cache path `#{cache_path}'" }
+        FileUtils.mkdir_p(File.dirname(cache_path))
+        shellout!("git --git-dir=#{cache_path} init -q")
+        true
+      end
+    end
+
+    #
+    # Computes the tag for this cache entry.
+    #
+    # @return [String]
+    #
+    def tag
+      return @tag if @tag
+
+      log.info(log_key) { "Calculating tag" }
+
+      # Accumulate an array of all the software projects that come before
+      # the name and version we are tagging. So if you have
+      #
+      # build_order = [ 1, 2, 3, 4, 5 ]
+      #
+      # And we are tagging 3, you would get dep_list = [ 1, 2 ]
+      dep_list = software.project.library.build_order.take_while do |dep|
+        if dep.name == software.name && dep.version == software.version
+          false
+        else
+          true
+        end
+      end
+
+      log.debug(log_key) { "dep_list: #{dep_list.map(&:name).inspect}" }
+
+      # This is the list of all the unqiue shasums of all the software build
+      # dependencies, including the on currently being acted upon.
+      shasums = [dep_list.map(&:shasum), software.shasum].flatten
+      suffix  = Digest::SHA256.hexdigest(shasums.join('|'))
+      @tag    = "#{software.name}-#{suffix}"
+
+      log.debug(log_key) { "tag: #{@tag}" }
+
+      @tag
+    end
+
+    # Create an incremental install path cache for the software step
+    def incremental
+      log.info(log_key) { 'Performing incremental cache' }
+
+      create_cache_path
+      remove_git_dirs
+
+      shellout!(%Q(git --git-dir=#{cache_path} --work-tree=#{install_dir} add -A -f))
+      begin
+        shellout!(%Q(git --git-dir=#{cache_path} --work-tree=#{install_dir} commit -q -m "Backup of #{tag}"))
+      rescue Mixlib::ShellOut::ShellCommandFailed => e
+        if e.message !~ /nothing to commit/
+          raise
+        end
+      end
+      shellout!(%Q(git --git-dir=#{cache_path} --work-tree=#{install_dir} tag -f "#{tag}"))
+    end
+
+    def restore
+      log.info(log_key) { 'Performing cache restoration' }
+
+      create_cache_path
+
+      cmd = shellout(%Q(git --git-dir=#{cache_path} --work-tree=#{install_dir} tag -l "#{tag}"))
+
+      restore_me = false
+      cmd.stdout.each_line do |line|
+        restore_me = true if tag == line.chomp
+      end
+
+      if restore_me
+        log.debug(log_key) { "Detected tag `#{tag}' can be restored, restoring" }
+        shellout!(%Q(git --git-dir=#{cache_path} --work-tree=#{install_dir} checkout -f "#{tag}"))
+        true
+      else
+        log.debug(log_key) { "Could not find tag `#{tag}', skipping restore" }
+        false
+      end
+    end
+
+    #
+    # Git caching will attempt to version embedded git directories, partially
+    # versioning them. This causes failures on subsequent runs. This method
+    # will find git directories and remove them to prevent those errors.
+    #
+    # @return [true]
+    def remove_git_dirs
+      log.info(log_key) { "Removing git directories" }
+
+      Dir.glob("#{install_dir}/**/{,.*}/config").reject do |path|
+        REQUIRED_GIT_FILES.any? do |required_file|
+          !File.exist?(File.join(File.dirname(path), required_file))
+        end
+      end.each do |path|
+        log.info(log_key) { "Removing git dir `#{path}'" }
+        FileUtils.rm_rf(File.dirname(path))
+      end
+
+      true
+    end
+
+    private
+
+    #
+    #
+    # The installation directory for this software's project. Drive letters are
+    # stripped for Windows.
+    #
+    # @return [String]
+    #
+    def install_dir
+      @install_dir ||= software.project.install_dir.sub(/^([A-Za-z]:)/, '')
+    end
+
+    # Override the log_key for this class to include the software name
+    #
+    # @return [String]
+    def log_key
+      @log_key ||= "#{super}: #{software.name}"
+    end
+  end
+end