omnibus 4.0.0 → 4.1.0

data/lib/omnibus/cli/publish.rb

@@ -18,19 +18,27 @@ module Omnibus
   class Command::Publish < Command::Base
     namespace :publish
 
-    # These options are useful for publish packages that were built for a
-    # paticluar platform/version and tested on another platform/version.
+    # This option is useful for publish packages that were built for a
+    # particular platform/version but tested on other platform/versions.
     #
     # For example, one might build on Ubuntu 10.04 and test/publish on
-    # Ubuntu 11.04, 12.04 and Debian 7.
+    # Ubuntu 10.04, 12.04, and 14.04.
     #
-    # If these options are used with the glob pattern support all packages
-    # will be published to the same platform/version.
-    class_option :platform,
-      desc: 'The platform to publish for',
+    # @example JSON
+    #   {
+    #     "ubuntu-10.04": [
+    #       "ubuntu-10.04",
+    #       "ubuntu-12.04",
+    #       "ubuntu-14.04"
+    #     ]
+    #   }
+    #
+    class_option :platform_mappings,
+      desc: 'The optional platform mappings JSON file to publish with',
       type: :string
-    class_option :platform_version,
-      desc: 'The platform version to publish for',
+
+    class_option :version_manifest,
+      desc: 'Path to the version-manifest.json file to publish with',
       type: :string
 
     #
@@ -43,6 +51,10 @@ module Omnibus
       desc: 'The accessibility of the uploaded packages',
       enum: %w(public private),
       default: 'private'
+    method_option :region,
+      type: :string,
+      desc: 'The region in which the bucket is located',
+      default: 'us-east-1'
     desc 's3 BUCKET PATTERN', 'Publish to an S3 bucket'
     def s3(bucket, pattern)
       options[:bucket] = bucket
@@ -54,6 +66,10 @@ module Omnibus
     #
     #   $ omnibus publish artifactory libs-omnibus-local pkg/chef*
     #
+    method_option :build_record,
+      type: :boolean,
+      desc: 'Optionally create an Artifactory build record for the published artifacts',
+      default: true
     desc 'artifactory REPOSITORY PATTERN', 'Publish to an Artifactory instance'
     def artifactory(repository, pattern)
       options[:repository] = repository
@@ -68,8 +84,12 @@ module Omnibus
     # @return [void]
     #
     def publish(klass, pattern, options)
+      if options[:platform_mappings]
+        options[:platform_mappings] = JSON.parse(File.read(File.expand_path(options[:platform_mappings])))
+      end
+
       klass.publish(pattern, options) do |package|
-        say("Uploaded '#{package.name}'", :green)
+        say("Published '#{package.name}' for #{package.metadata[:platform]}-#{package.metadata[:platform_version]}", :green)
       end
     end
   end

data/lib/omnibus/config.rb

@@ -289,6 +289,14 @@ module Omnibus
       raise MissingRequiredAttribute.new(self, :s3_secret_key, "'EFGH5678'")
     end
 
+    # The region of the S3 bucket you want to cache software artifacts in.
+    # Defaults to 'us-east-1'
+    #
+    # @return [String]
+    default(:s3_region) do
+      'us-east-1'
+    end
+
     # --------------------------------------------------
     # @!endgroup
     #
@@ -432,10 +440,36 @@ module Omnibus
       ['omnibus-software']
     end
 
-    # The solaris compiler to use
+    # Solaris linker mapfile to use, if needed
+    # see http://docs.oracle.com/cd/E23824_01/html/819-0690/chapter5-1.html
+    # Path is relative to the 'files' directory in your omnibus project
+    #
+    # For example:
+    #
+    #     /PATH/files/my_map_file 
     #
     # @return [String, nil]
-    default(:solaris_compiler, nil)
+    default(:solaris_linker_mapfile, "files/mapfiles/solaris")
+
+    # Architecture to target when building on windows.  This option
+    # should affect the bit-ness of Ruby and DevKit used, the platform of
+    # any MSIs generated and package dlls being downloaded.
+    #
+    # See the windows_arch_i386? software definition dsl
+    # methods.
+    #
+    # @return [:x86, :x64]
+    default(:windows_arch) do
+      if Ohai['kernel']['machine'] == 'x86_64'
+        Omnibus.logger.deprecated('Config') do
+          "windows_arch is defaulting to :x86. In Omnibus 5, it will " \
+          "default to :x64 if the machine architecture is x86_64. " \
+          "If you would like to continue building 32 bit packages, please "\
+          "manually set windows_arch in your omnibus.rb file to :x86."
+        end
+      end
+      :x86
+    end
 
     # --------------------------------------------------
     # @!endgroup
@@ -488,6 +522,11 @@ module Omnibus
     # @return [Integer]
     default(:fetcher_read_timeout, 60)
 
+    # The number of retries before marking a download as failed
+    #
+    # @return [Integer]
+    default(:fetcher_retries, 5)
+
     # --------------------------------------------------
     # @!endgroup
     #

data/lib/omnibus/digestable.rb

@@ -16,9 +16,14 @@
 
 require 'openssl'
 require 'pathname'
+require 'omnibus/logging'
 
 module Omnibus
   module Digestable
+
+    def self.included(other)
+      other.send(:include, Logging)
+    end
     #
     # Calculate the digest of the file at the given path. Files are read in
     # binary chunks to prevent Ruby from exploding.
@@ -63,7 +68,7 @@ module Omnibus
     #
     def digest_directory(path, type = :md5)
       digest = digest_from_type(type)
-
+      log.info(log_key) { "Digesting #{path} with #{type}" }
       FileSyncer.glob("#{path}/**/*").each do |filename|
         # Calculate the filename relative to the given path. Since directories
         # are SHAed according to their filepath, two difference directories on

data/lib/omnibus/exceptions.rb

@@ -87,7 +87,7 @@ EOH
 
     def to_s
       <<-EOH
-Attempting to evaluate the template `#{@name}', but it was not found at any of
+Attempting to evaluate the template `#{@template}', but it was not found at any of
 the following locations:
 
 #{@search_paths.map { |path| "    #{path}" }.join("\n")}
@@ -309,4 +309,18 @@ Could not resolve `#{ref}' to a valid git SHA-1.
 EOH
     end
   end
+
+  class InvalidVersion < Error
+    def initialize(version)
+      super <<-EOF
+'#{version}' could not be parsed as a valid version.
+EOF
+    end
+  end
+
+  class FailedToTimestampMSI < Error
+    def initialize
+      super("Failed to add timestamp to MSI.")
+    end
+  end
 end

data/lib/omnibus/fetcher.rb

@@ -19,22 +19,65 @@ module Omnibus
     include Digestable
     include Logging
     include Util
+    extend Util
 
     #
-    # The software for this fetcher.
+    # The name of the software this fetcher shall fetch
     #
-    # @return [Software]
+    # @return [String]
+    #
+    attr_reader :name
+
+    #
+    # The source for this fetcher.
+    #
+    # @return [Hash]
+    #
+    attr_reader :source
+
+    #
+    # The exact upstream version that a fetcher should fetch.
+    #
+    # @return [String]
+    #
+    # For sources that allow aliases (branch name, tags, etc). Users
+    # should use the class method resolve_version to determine this
+    # before constructing a fetcher.
+    attr_reader :resolved_version
+
+    #
+    # The upstream version as described before resolution.
+    #
+    # @return [String]
+    #
+    # This will usually be the same as +resolved_version+ but may
+    # refer toa remote ref name or tag for a source such as git.
+    attr_reader :described_version
+
     #
-    attr_reader :software
+    # The path where extracted software should live.
+    #
+    # @return [String]
+    #
+    attr_reader :project_dir
+    attr_reader :build_dir
+
 
     #
     # Create a new Fetcher object from the given software.
     #
     # @param [Software] software
-    #   the software to create this fetcher
     #
-    def initialize(software)
-      @software = software
+    # To preserve the original interface, this still takes a software-like
+    # argument, but to avoid coupling with the software object, we pull out
+    # what we need and don't touch it again.
+    def initialize(manifest_entry, project_dir, build_dir)
+      @name    = manifest_entry.name
+      @source  = manifest_entry.locked_source
+      @resolved_version = manifest_entry.locked_version
+      @described_version = manifest_entry.described_version
+      @project_dir = project_dir
+      @build_dir = build_dir
     end
 
     #
@@ -83,37 +126,20 @@ module Omnibus
     # @!endgroup
     # --------------------------------------------------
 
-    private
-
-    #
-    # The "source" for this software, with applied overrides.
-    #
-    # @return [Hash]
-    #
-    def source
-      software.source
+    def fetcher
+      self
     end
 
     #
-    # The path where extracted software should live.
-    #
-    # @see Software#project_dir
-    #
-    # @return [String]
-    #
-    def project_dir
-      software.project_dir
-    end
-
-    #
-    # The version for this sfotware, with applied overrides.
-    #
-    # @return [String]
+    # All fetchers should prefer resolved_version to version
+    # this is provided for compatibility.
     #
     def version
-      software.version
+      resolved_version
     end
 
+    private
+
     #
     # Override the +log_key+ for this fetcher to include the name of the
     # software during the fetch.
@@ -121,10 +147,9 @@ module Omnibus
     # @return [String]
     #
     def log_key
-      @log_key ||= "#{super}: #{software.name}"
+      @log_key ||= "#{super}: #{name}"
     end
 
-
     #
     # Idempotently create the required directories for building/downloading.
     # Fetchers should call this method before performing any operations that
@@ -136,11 +161,30 @@ module Omnibus
       [
         Config.cache_dir,
         Config.source_dir,
-        software.build_dir,
-        software.project_dir,
+        build_dir,
+        project_dir,
       ].each do |directory|
         FileUtils.mkdir_p(directory) unless File.directory?(directory)
       end
     end
+
+    # Class Methods
+    def self.resolve_version(version, source)
+      fetcher_class_for_source(source).resolve_version(version, source)
+    end
+
+    def self.fetcher_class_for_source(source)
+      if source
+        if source[:url]
+          NetFetcher
+        elsif source[:git]
+          GitFetcher
+        elsif source[:path]
+          PathFetcher
+        end
+      else
+        NullFetcher
+      end
+    end
   end
 end

data/lib/omnibus/fetchers/git_fetcher.rb

@@ -23,7 +23,7 @@ module Omnibus
     # @return [true, false]
     #
     def fetch_required?
-      !(cloned? && same_revision?)
+      !(cloned? && same_revision?(resolved_version))
     end
 
     #
@@ -59,12 +59,12 @@ module Omnibus
     #
     def fetch
       log.info(log_key) { "Fetching from `#{source_url}'" }
-
       create_required_directories
 
       if cloned?
-        git_fetch unless same_revision?
+        git_fetch unless same_revision?(resolved_version)
       else
+        force_recreate_project_dir! unless dir_empty?(project_dir)
         git_clone
         git_checkout
       end
@@ -91,6 +91,33 @@ module Omnibus
       source[:git]
     end
 
+    #
+    # Determine if submodules should be cloned.
+    #
+    # @return [true, false]
+    #
+    def clone_submodules?
+      source[:submodules] || false
+    end
+
+    #
+    # Determine if a directory is empty
+    #
+    # @return [true, false]
+    #
+    def dir_empty?(dir)
+      Dir.entries(dir).reject {|d| ['.', '..'].include?(d) }.empty?
+    end
+
+    #
+    # Forcibly remove and recreate the project directory
+    #
+    def force_recreate_project_dir!
+      log.warn(log_key) { "Removing existing directory #{project_dir} before cloning" }
+      FileUtils.rm_rf(project_dir)
+      Dir.mkdir(project_dir)
+    end
+
     #
     # Determine if the clone exists.
     #
@@ -106,27 +133,28 @@ module Omnibus
     # @return [void]
     #
     def git_clone
-      git("clone #{source_url} .")
+      git("clone#{" --recursive" if clone_submodules?} #{source_url} .")
     end
 
     #
-    # Checkout the +target_revision+.
+    # Checkout the +resolved_version+.
     #
     # @return [void]
     #
     def git_checkout
-      git("fetch --all")
-      git("checkout #{target_revision}")
+      git("checkout #{resolved_version}")
     end
 
     #
-    # Fetch the remote tags and refs, and reset to +target_revision+.
+    # Fetch the remote tags and refs, and reset to +resolved_version+.
     #
     # @return [void]
     #
     def git_fetch
-      git("fetch --all")
-      git("reset --hard #{target_revision}")
+      fetch_cmd = "fetch #{source_url} #{described_version}"
+      fetch_cmd << ' --recurse-submodules=on-demand' if clone_submodules?
+      git(fetch_cmd)
+      git("reset --hard #{resolved_version}")
     end
 
     #
@@ -135,31 +163,55 @@ module Omnibus
     # @return [String]
     #
     def current_revision
-      @current_revision ||= git('rev-parse HEAD').stdout.strip
+      git('rev-parse HEAD').stdout.strip
     rescue CommandFailed
       nil
     end
 
     #
-    # The target revision from the user.
+    # Determine if the given revision matches the current revision.
     #
-    # @return [String]
+    # @return [true, false]
     #
-    def target_revision
-      @target_revision ||= if sha_hash?(version)
-                             version
-                           else
-                             revision_from_remote_reference(version)
-                           end
+    def same_revision?(rev=resolved_version)
+      current_revision == rev
     end
 
     #
-    # Determine if the given revision matches the current revision.
+    # Execute the given git command, inside the +project_dir+.
     #
-    # @return [true, false]
+    # @see Util#shellout!
+    #
+    # @return [Mixlib::ShellOut]
+    #   the shellout object
     #
-    def same_revision?
-      current_revision == target_revision
+    def git(command)
+      shellout!("git #{command}", cwd: project_dir)
+    end
+
+    # Class methods
+    public
+
+    # Return the SHA1 corresponding to a ref as determined by the remote source.
+    #
+    # @return [String]
+    #
+    def self.resolve_version(ref, source)
+      if sha_hash?(ref)
+        # A git server negotiates in terms of refs during the info-refs phase
+        # of a fetch. During upload-pack, the client is not allowed to specify
+        # any sha1s in the "wants" unless the server has publicized them during
+        # info-refs. Hence, the server is allowed to drop requests to fetch
+        # particular sha1s, even if it is an otherwise reachable commit object.
+        # Only when the service is specifically configured with
+        # uploadpack.allowReachableSHA1InWant is there any guarantee that it
+        # considers "naked" wants.
+        log.warn(log_key) { 'git fetch on a sha1 is not guaranteed to work' }
+        log.warn(log_key) { "Specify a ref name instead of #{ref} on #{source}" }
+        ref
+      else
+        revision_from_remote_reference(ref, source)
+      end
     end
 
     #
@@ -167,22 +219,24 @@ module Omnibus
     #
     # @return [true, false]
     #
-    def sha_hash?(rev)
-      rev =~ /^[0-9a-f]{4,40}$/
+    def self.sha_hash?(rev)
+      rev =~ /^[0-9a-f]{4,40}$/i
     end
 
     #
-    # Return the SHA corresponding to ref. If ref is an annotated tag,
-    # return the SHA that was tagged not the SHA of the tag itself.
+    # Return the SHA corresponding to ref.
+    #
+    # If ref is an annotated tag, return the SHA that was tagged not the SHA of
+    # the tag itself.
     #
     # @return [String]
     #
-    def revision_from_remote_reference(ref)
+    def self.revision_from_remote_reference(ref, source)
       # execute `git ls-remote` the trailing '*' does globbing. This
       # allows us to return the SHA of the tagged commit for annotated
       # tags. We take care to only return exact matches in
       # process_remote_list.
-      remote_list = git("ls-remote origin #{ref}*").stdout
+      remote_list = shellout!("git ls-remote \"#{source[:git]}\" #{ref}*").stdout
       commit_ref = dereference_annotated_tag(remote_list, ref)
 
       unless commit_ref
@@ -210,7 +264,7 @@ module Omnibus
     #
     # @return [String]
     #
-    def dereference_annotated_tag(remote_list, ref)
+    def self.dereference_annotated_tag(remote_list, ref)
       # We'll return the SHA corresponding to the ^{} which is the
       # commit pointed to by an annotated tag. If no such commit
       # exists (not an annotated tag) then we return the SHA of the
@@ -231,17 +285,5 @@ module Omnibus
         end
       end
     end
-
-    #
-    # Execute the given git command, inside the +project_dir+.
-    #
-    # @see Util#shellout!
-    #
-    # @return [Mixlib::ShellOut]
-    #   the shellout object
-    #
-    def git(command)
-      shellout!("git #{command}", cwd: project_dir)
-    end
   end
 end