omnibus 4.0.0 → 4.1.0

data/lib/omnibus/publishers/s3_publisher.rb

@@ -14,11 +14,14 @@
 # limitations under the License.
 #
 
+require 'omnibus/s3_helpers'
+
 module Omnibus
   class S3Publisher < Publisher
+    include S3Helpers
+
     def publish(&block)
       log.info(log_key) { 'Starting S3 publisher' }
-      safe_require('uber-s3')
 
       packages.each do |package|
         # Make sure the package is good to go!
@@ -27,16 +30,13 @@ module Omnibus
 
         # Upload the metadata first
         log.debug(log_key) { "Uploading '#{package.metadata.name}'" }
-        client.store(key_for(package, package.metadata.name), package.metadata.to_json,
-          access: access_policy,
-        )
+        store_object(key_for(package, package.metadata.name), package.metadata.to_json,
+                     nil, access_policy)
 
         # Upload the actual package
         log.info(log_key) { "Uploading '#{package.name}'" }
-        client.store(key_for(package, package.name), package.content,
-          access: access_policy,
-          content_md5: package.metadata[:md5],
-        )
+        store_object(key_for(package, package.name), package.content,
+                     package.metadata[:md5], access_policy)
 
         # If a block was given, "yield" the package to the caller
         block.call(package) if block
@@ -45,18 +45,13 @@ module Omnibus
 
     private
 
-    #
-    # The actual S3 client object to communicate with the S3 API.
-    #
-    # @return [UberS3]
-    #
-    def client
-      @client ||= UberS3.new(
-        access_key:        Config.publish_s3_access_key,
+    def s3_configuration
+      {
+        region:            @options[:region],
+        access_key_id:     Config.publish_s3_access_key,
         secret_access_key: Config.publish_s3_secret_key,
-        bucket:            @options[:bucket],
-        adaper:            :net_http,
-      )
+        bucket_name:       @options[:bucket],
+      }
     end
 
     #
@@ -72,8 +67,8 @@ module Omnibus
     #
     def key_for(package, *stuff)
       File.join(
-        publish_platform(package),
-        publish_platform_version(package),
+        package.metadata[:platform],
+        package.metadata[:platform_version],
         package.metadata[:arch],
         package.name,
         *stuff,
@@ -85,14 +80,14 @@ module Omnibus
     # initializer option. Any access control that is not the strict string
     # +"public"+ is assumed to be private.
     #
-    # @return [Symbol]
-    #   the UberS3-ready access policy
+    # @return [String]
+    #   the access policy
     #
     def access_policy
       if @options[:acl].to_s == 'public'
-        :public_read
+        'public-read'
       else
-        :private
+        'private'
       end
     end
   end

data/lib/omnibus/s3_cache.rb

@@ -15,13 +15,14 @@
 #
 
 require 'fileutils'
-require 'uber-s3'
+require 'omnibus/s3_helpers'
 
 module Omnibus
   class S3Cache
     include Logging
 
     class << self
+      include S3Helpers
       #
       # List all software in the cache.
       #
@@ -41,7 +42,7 @@ module Omnibus
       # @return [Array<String>]
       #
       def keys
-        bucket.objects('/').map(&:key)
+        bucket.objects.map(&:key)
       end
 
       #
@@ -70,16 +71,14 @@ module Omnibus
 
           key     = key_for(software)
           fetcher = software.fetcher
-          content = IO.read(fetcher.downloaded_file)
 
           log.info(log_key) do
             "Caching '#{fetcher.downloaded_file}' to '#{Config.s3_bucket}/#{key}'"
           end
 
-          client.store(key, content,
-            access: :public_read,
-            content_md5: software.fetcher.checksum
-          )
+          File.open(fetcher.downloaded_file, 'rb') do |file|
+            store_object(key, file, software.fetcher.checksum, 'public-read')
+          end
         end
 
         true
@@ -129,33 +128,13 @@ module Omnibus
 
       private
 
-      #
-      # The client to connect to S3 with.
-      #
-      # @return [UberS3::Client]
-      #
-      def client
-        @client ||= UberS3.new(
-          access_key:        Config.s3_access_key,
-          secret_access_key: Config.s3_secret_key,
-          bucket:            Config.s3_bucket,
-          adapter:           :net_http,
-        )
-      end
-
-      #
-      # The bucket where the objects live.
-      #
-      # @return [UberS3::Bucket]
-      #
-      def bucket
-        @bucket ||= begin
-          if client.exists?('/')
-            client.bucket
-          else
-            client.connection.put('/')
-          end
-        end
+      def s3_configuration
+        {
+          region:               Config.s3_region,
+          access_key_id:        Config.s3_access_key,
+          secret_access_key:    Config.s3_secret_key,
+          bucket_name:          Config.s3_bucket
+        }
       end
 
       #

data/lib/omnibus/s3_helpers.rb

@@ -0,0 +1,119 @@
+#
+# Copyright 2015 Chef Software, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'aws-sdk'
+require 'base64'
+
+module Omnibus
+  module S3Helpers
+    def self.included(base)
+      base.send(:include, InstanceMethods)
+    end
+
+    module InstanceMethods
+      private
+      #
+      # Returns the configuration for S3. You must provide keys
+      # :region, 
+      #
+      # @example
+      #   {
+      #     region:               'us-east-1',
+      #     access_key_id:        Config.s3_access_key,
+      #     secret_access_key:    Config.s3_secret_key,
+      #     bucket_name:          Config.s3_bucket
+      #   }
+      #
+      # @return [Hash<String, String>]
+      #
+      def s3_configuration
+        raise "You must override s3_configuration"
+      end
+
+      #
+      # The client to connect to S3 with.
+      #
+      # @return [Aws::S3::Resource]
+      #
+      def client
+        @s3_client ||= Aws::S3::Resource.new(
+          region:            s3_configuration[:region],
+          access_key_id:     s3_configuration[:access_key_id],
+          secret_access_key: s3_configuration[:secret_access_key]
+        )
+      end
+
+      #
+      # The bucket where the objects live.
+      #
+      # @return [Aws::S3::Bucket]
+      #
+      def bucket
+        @s3_bucket ||= begin
+                         bucket = client.bucket(s3_configuration[:bucket_name])
+                         unless bucket.exists?
+                           bucket_config = if s3_configuration[:region] == 'us-east-1'
+                                             nil
+                                           else
+                                             {
+                                               location_constraint: s3_configuration[:region]
+                                             }
+                                           end
+                           bucket.create(create_bucket_configuration: bucket_config)
+                         end
+                         bucket
+                       end
+      end
+
+      #
+      # Store an object at the specified key
+      #
+      # @param [String] key
+      # @param [File, String] content
+      # @param [String] content_md5
+      # @param [String] acl
+      #
+      # @return [true]
+      #
+      def store_object(key, content, content_md5, acl)
+        bucket.put_object({
+          key: key,
+          body: content,
+          content_md5: to_base64_digest(content_md5),
+          acl: acl
+        })
+        true
+      end
+
+      #
+      # Convert a hex digest into a base64 hex digest
+      #
+      # For example:
+      # to_base64_digest('c3b5247592ce694f7097873aa07d66fe') => 'w7UkdZLOaU9wl4c6oH1m/g=='
+      #
+      # @param [String] content_md5
+      #
+      # @return [String]
+      #
+      def to_base64_digest(content_md5)
+        if content_md5
+          md5_digest = content_md5.unpack('a2'*16).collect {|i| i.hex.chr }.join
+          Base64.encode64(md5_digest).strip
+        end
+      end
+    end
+  end
+end

data/lib/omnibus/semantic_version.rb

@@ -0,0 +1,57 @@
+#
+# Copyright 2015 Chef Software, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'mixlib/versioning'
+
+module Omnibus
+  class SemanticVersion
+
+    def initialize(version_string)
+      @prefix = if version_string =~ /^v/
+                  "v"
+                else
+                  ""
+                end
+      @version = Mixlib::Versioning.parse(version_string.gsub(/^v/, ""))
+      if @version.nil?
+        raise InvalidVersion, "#{version_string} could not be parsed as a valid version"
+      end
+    end
+
+    def to_s
+      "#{prefix}#{version}"
+    end
+
+    def next_patch
+      s = [version.major, version.minor, version.patch + 1].join(".")
+      self.class.new("#{prefix}#{s}")
+    end
+
+    def next_minor
+      s = [version.major, version.minor + 1, 0].join(".")
+      self.class.new("#{prefix}#{s}")
+    end
+
+    def next_major
+      s = [version.major + 1, 0, 0].join(".")
+      self.class.new("#{prefix}#{s}")
+    end
+
+    private
+
+    attr_reader :prefix, :version
+  end
+end

data/lib/omnibus/software.rb

@@ -16,6 +16,7 @@
 
 require 'fileutils'
 require 'uri'
+require 'omnibus/manifest_entry'
 
 module Omnibus
   class Software
@@ -28,7 +29,7 @@ module Omnibus
       #
       # @return [Software]
       #
-      def load(project, name)
+      def load(project, name, manifest)
         loaded_softwares[name] ||= begin
           filepath = Omnibus.software_path(name)
 
@@ -40,7 +41,7 @@ module Omnibus
             end
           end
 
-          instance = new(project, filepath)
+          instance = new(project, filepath, manifest)
           instance.evaluate_file(filepath)
           instance.load_dependencies
 
@@ -69,6 +70,8 @@ module Omnibus
     include NullArgumentable
     include Sugarable
 
+    attr_reader :manifest
+
     #
     # Create a new software object.
     #
@@ -76,10 +79,12 @@ module Omnibus
     #   the Omnibus project that instantiated this software definition
     # @param [String] filepath
     #   the path to where this software definition lives on disk
+    # @param [String] manifest
+    #   the user-supplied software manifest
     #
     # @return [Software]
     #
-    def initialize(project, filepath = nil)
+    def initialize(project, filepath = nil, manifest=nil)
       unless project.is_a?(Project)
         raise ArgumentError,
           "`project' must be a kind of `Omnibus::Project', but was `#{project.class.inspect}'!"
@@ -88,11 +93,22 @@ module Omnibus
       # Magical methods
       @filepath = filepath
       @project  = project
+      @manifest = manifest
 
       # Overrides
       @overrides = NULL
     end
 
+    def manifest_entry
+      @manifest_entry ||= if manifest
+                            log.info(log_key) {"Using user-supplied manifest entry for #{name}"}
+                            manifest.entry_for(name)
+                          else
+                            log.info(log_key) {"Resolving manifest entry for #{name}"}
+                            to_manifest_entry
+                          end
+    end
+
     #
     # Compare two software projects (by name).
     #
@@ -204,11 +220,21 @@ module Omnibus
     # @option val [String] :path (nil)
     #   a fully-qualified local file system path
     # @option val [String] :md5 (nil)
-    #   the checksum of the downloaded artifact
+    #   the MD5 checksum of the downloaded artifact
+    # @option val [String] :sha1 (nil)
+    #   the SHA1 checksum of the downloaded artifact
+    # @option val [String] :sha256 (nil)
+    #   the SHA256 checksum of the downloaded artifact
+    # @option val [String] :sha512 (nil)
+    #   the SHA512 checksum of the downloaded artifact
     # @option val [String] :cookie (nil)
     #   a cookie to set
     # @option val [String] :warning (nil)
     #   a warning message to print when downloading
+    # @option val [Boolean] :submodules (false)
+    #   clone git submodules
+    #
+    # If multiple checksum types are provided, only the strongest will be used.
     #
     # @return [Hash]
     #
@@ -219,7 +245,8 @@ module Omnibus
             "be a kind of `Hash', but was `#{val.class.inspect}'")
         end
 
-        extra_keys = val.keys - [:git, :path, :url, :md5, :cookie, :warning, :unsafe]
+        extra_keys = val.keys - [:git, :path, :url, :md5, :sha1, :sha256, :sha512,
+                                 :cookie, :warning, :unsafe, :options, :submodules]
         unless extra_keys.empty?
           raise InvalidValue.new(:source,
             "only include valid keys. Invalid keys: #{extra_keys.inspect}")
@@ -240,13 +267,14 @@ module Omnibus
     expose :source
 
     #
-    # Set or retieve the {#default_version} of the software to build.
+    # Set or retrieve the {#default_version} of the software to build.
     #
     # @example
     #   default_version '1.2.3'
     #
     # @param [String] val
-    #   the default version to set for the software
+    #   the default version to set for the software.
+    #   For a git source, the default version may be a git ref (e.g. tag, branch name, or sha).
     #
     # @return [String]
     #
@@ -435,7 +463,6 @@ module Omnibus
             "CC" => "xlc_r -q64",
             "CXX" => "xlC_r -q64",
             "CFLAGS" => "-q64 -I#{install_dir}/embedded/include -D_LARGE_FILES -O",
-            "CXXFLAGS" => "-q64 -I#{install_dir}/embedded/include -D_LARGE_FILES -O",
             "LDFLAGS" => "-q64 -L#{install_dir}/embedded/lib -Wl,-blibpath:#{install_dir}/embedded/lib:/usr/lib:/lib",
             "LD" => "ld -b64",
             "OBJECT_MODE" => "64",
@@ -484,21 +511,36 @@ module Omnibus
       extra_linker_flags = {
         "LD_RUN_PATH" => "#{install_dir}/embedded/lib"
       }
-      # solaris linker can also use LD_OPTIONS, so we throw the kitchen sink against
-      # the linker, to find every way to make it use our rpath.
-      extra_linker_flags.merge!(
-        {
-          "LD_OPTIONS" => "-R#{install_dir}/embedded/lib"
-        }
-      ) if Ohai['platform'] == "solaris2"
+
+      if solaris2?
+        # in order to provide compatibility for earlier versions of libc on solaris 10,
+        # we need to specify a mapfile that restricts the version of system libraries
+        # used. See http://docs.oracle.com/cd/E23824_01/html/819-0690/chapter5-1.html
+        # for more information
+        # use the mapfile if it exists, otherwise ignore it
+        ld_options = "-R#{install_dir}/embedded/lib"
+        mapfile_path = File.expand_path(Config.solaris_linker_mapfile, Config.project_root)
+        ld_options  << " -M #{mapfile_path}" if File.exist?(mapfile_path)
+
+        # solaris linker can also use LD_OPTIONS, so we throw the kitchen sink against
+        # the linker, to find every way to make it use our rpath. This is also required
+        # to use the aforementioned mapfile.
+        extra_linker_flags.merge!(
+          {
+            "LD_OPTIONS" => ld_options
+          }
+        )
+      end
+
       env.merge(compiler_flags).
         merge(extra_linker_flags).
         # always want to favor pkg-config from embedded location to not hose
         # configure scripts which try to be too clever and ignore our explicit
         # CFLAGS and LDFLAGS in favor of pkg-config info
         merge({"PKG_CONFIG_PATH" => "#{install_dir}/embedded/lib/pkgconfig"}).
-        # Set default values for CXXFLAGS.
-        merge('CXXFLAGS' => compiler_flags['CFLAGS'])
+        # Set default values for CXXFLAGS and CPPFLAGS.
+        merge('CXXFLAGS' => compiler_flags['CFLAGS']).
+        merge('CPPFLAGS' => compiler_flags['CFLAGS'])
     end
     expose :with_standard_compiler_flags
 
@@ -566,7 +608,7 @@ module Omnibus
     #
     def load_dependencies
       dependencies.each do |dependency|
-        software = Software.load(project, dependency)
+        Software.load(project, dependency, manifest)
       end
 
       true
@@ -581,6 +623,14 @@ module Omnibus
       @builder ||= Builder.new(self)
     end
 
+    def to_manifest_entry
+      Omnibus::ManifestEntry.new(name, {
+                                   source_type: source_type,
+                                   described_version: version,
+                                   locked_version: Fetcher.resolve_version(version, source),
+                                   locked_source: source})
+    end
+
     #
     # Fetch the software definition using the appropriate fetcher. This may
     # fetch the software from a local path location, git location, or download
@@ -684,25 +734,30 @@ module Omnibus
     end
 
     #
-    # The fetcher for this software, based off of the +source+ attribute.
-    #
-    # - +:url+ - {NetFetcher}
-    # - +:git+ - {GitFetcher}
-    # - +:path+ - {PathFetcher}
+    # The fetcher for this software
     #
     # @return [Fetcher]
     #
     def fetcher
-      @fetcher ||= if source
+      @fetcher ||= Fetcher.fetcher_class_for_source(self.source).new(manifest_entry, project_dir, build_dir)
+    end
+
+    #
+    # The type of source specified for this software defintion.
+    #
+    # @return [String]
+    #
+    def source_type
+      if source
         if source[:url]
-          NetFetcher.new(self)
+          :url
         elsif source[:git]
-          GitFetcher.new(self)
+          :git
         elsif source[:path]
-          PathFetcher.new(self)
+          :path
         end
       else
-        NullFetcher.new(self)
+        :project_local
       end
     end
 
@@ -862,5 +917,9 @@ module Omnibus
     def log_key
       @log_key ||= "#{super}: #{name}"
     end
+
+    def to_s
+      "#{name}[#{filepath}]"
+    end
   end
 end