omnibus 3.1.1 → 3.2.0.rc.1

data/lib/omnibus/publisher.rb

@@ -0,0 +1,76 @@
+#
+# Copyright 2014 Chef Software, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Omnibus
+  class Publisher
+    class << self
+      #
+      # Shortcut class method for creating a new instance of this class and
+      # executing the publishing sequence.
+      #
+      # @param (see Publisher#initialize)
+      #
+      def publish(pattern, options = {}, &block)
+        new(pattern, options).publish(&block)
+      end
+    end
+
+    include Logging
+
+    #
+    # Create a new publisher from the given pattern.
+    #
+    # @param [String] pattern
+    #   the path/pattern of the release artifact(s)
+    # @param [Hash] options
+    #   the list of options passed to the publisher
+    #
+    def initialize(pattern, options = {})
+      @pattern = pattern
+      @options = options.dup
+    end
+
+    #
+    # The list of packages that match the pattern in the initializer.
+    #
+    # @return [Array<String>]
+    #
+    def packages
+      @packages ||= Dir.glob(@pattern).map { |path| Package.new(path) }
+    end
+
+    #
+    # @abstract
+    #
+    # @param [Proc] block
+    #   if given, the block will yield the currently uploading "thing"
+    #
+    # @return [Array<String>]
+    #   the list of uploaded packages
+    #
+    def publish(&_block)
+      raise AbstractMethod.new("#{self.class.name}#publish")
+    end
+
+    private
+
+    def safe_require(name)
+      require name
+    rescue LoadError
+      raise GemNotInstalled.new(name)
+    end
+  end
+end

data/lib/omnibus/publishers/artifactory_publisher.rb

@@ -0,0 +1,168 @@
+#
+# Copyright 2014 Chef Software, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'uri'
+
+module Omnibus
+  class ArtifactoryPublisher < Publisher
+    def publish(&block)
+      log.info(log_key) { 'Starting artifactory publisher' }
+      safe_require('artifactory')
+
+      packages.each do |package|
+        # Make sure the package is good to go!
+        log.debug(log_key) { "Validating '#{package.name}'" }
+        package.validate!
+
+        # Upload the actual package
+        log.info(log_key) { "Uploading '#{package.name}'" }
+        artifact_for(package).upload(
+          repository,
+          remote_path_for(package),
+          metadata_for(package),
+        )
+
+        # If a block was given, "yield" the package to the caller
+        block.call(package) if block
+      end
+    end
+
+    private
+
+    #
+    # The artifact object that corresponds to this package.
+    #
+    # @param [Package] package
+    #   the package to create the artifact from
+    #
+    # @return [Artifactory::Resource::Artifact]
+    #
+    def artifact_for(package)
+      Artifactory::Resource::Artifact.new(
+        local_path: package.path,
+        client:     client,
+        checksums: {
+          'md5'  => package.metadata[:md5],
+          'sha1' => package.metadata[:sha1],
+        }
+      )
+    end
+
+    #
+    # The Artifactory client object to communicate with the Artifactory API.
+    #
+    # @return [Artifactory::Client]
+    #
+    def client
+      @client ||= Artifactory::Client.new(
+        endpoint:       Config.artifactory_endpoint,
+        username:       Config.artifactory_username,
+        password:       Config.artifactory_password,
+        ssl_pem_file:   Config.artifactory_ssl_pem_file,
+        ssl_verify:     Config.artifactory_ssl_verify,
+        proxy_username: Config.artifactory_proxy_username,
+        proxy_password: Config.artifactory_proxy_password,
+        proxy_address:  Config.artifactory_proxy_address,
+        proxy_port:     Config.artifactory_proxy_port,
+      )
+    end
+
+    #
+    # The metadata for this package.
+    #
+    # @param [Package] package
+    #   the package to generate the metadata for
+    #
+    # @return [Hash<String, String>]
+    #
+    def metadata_for(package)
+      {
+        'omnibus.project'          => package.metadata[:name],
+        'omnibus.platform'         => package.metadata[:platform],
+        'omnibus.platform_version' => package.metadata[:platform_version],
+        'omnibus.arch'             => package.metadata[:arch],
+        'omnibus.version'          => package.metadata[:version],
+        'omnibus.md5'              => package.metadata[:md5],
+        'omnibus.sha1'             => package.metadata[:sha1],
+        'omnibus.sha256'           => package.metadata[:sha256],
+        'omnibus.sha512'           => package.metadata[:sha512],
+      }
+    end
+
+    #
+    # The name of the Artifactory repository (as supplied as an option).
+    #
+    # @return [String]
+    #
+    def repository
+      @options[:repository]
+    end
+
+    #
+    # The path where the package will live inside of the Artifactory repository.
+    # This is dynamically computed from the values in the project definition
+    # and the package metadata.
+    #
+    # @example
+    #   com/getchef/chef/11.6.0/chef-11.6.0-1.el6.x86_64.rpm
+    #
+    # @param [Package] package
+    #   the package to generate the remote path for
+    #
+    # @return [String]
+    #
+    def remote_path_for(package)
+      unless package.metadata[:homepage]
+        raise OldMetadata.new(package.metadata.path)
+      end
+
+      domain_parts = parsed_uri_for(package).host.split('.')
+      domain_parts.delete('www')
+      domain_parts.reverse!
+
+      File.join(
+        *domain_parts,
+        package.metadata[:name],
+        package.metadata[:version],
+        package.metadata[:basename],
+      )
+    end
+
+    #
+    # The parsed domain for this package. Ruby's URI parser does not "assume"
+    # a valid protocol, so passing a URI like "CHANGEME.org" will essentially
+    # result in an unsable object that has no useful, extractable information.
+    #
+    # This method will essentially "force" a default protocol on the +homepage+
+    # attribute of the package, so that it can be parsed like a good little URI.
+    #
+    # @param [Package] package
+    #   the package to generate the remote path for
+    #
+    # @return [URI]
+    #   the parsed URI object
+    #
+    def parsed_uri_for(package)
+      raw = package.metadata[:homepage]
+
+      if raw =~ /\Ahttps?:\/\//
+        URI.parse(raw)
+      else
+        URI.parse("http://#{raw}")
+      end
+    end
+  end
+end

data/lib/omnibus/publishers/null_publisher.rb

@@ -0,0 +1,23 @@
+#
+# Copyright 2014 Chef Software, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Omnibus
+  class NullPublisher < Publisher
+    def publish
+      # noop
+    end
+  end
+end

data/lib/omnibus/publishers/s3_publisher.rb

@@ -0,0 +1,99 @@
+#
+# Copyright 2012-2014 Chef Software, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Omnibus
+  class S3Publisher < Publisher
+    def publish(&block)
+      log.info(log_key) { 'Starting S3 publisher' }
+      safe_require('uber-s3')
+
+      packages.each do |package|
+        # Make sure the package is good to go!
+        log.debug(log_key) { "Validating '#{package.name}'" }
+        package.validate!
+
+        # Upload the metadata first
+        log.debug(log_key) { "Uploading '#{package.metadata.name}'" }
+        client.store(key_for(package, package.metadata.name), package.metadata.to_json,
+          access: access_policy,
+        )
+
+        # Upload the actual package
+        log.info(log_key) { "Uploading '#{package.name}'" }
+        client.store(key_for(package, package.name), package.content,
+          access: access_policy,
+          content_md5: package.metadata[:md5],
+        )
+
+        # If a block was given, "yield" the package to the caller
+        block.call(package) if block
+      end
+    end
+
+    private
+
+    #
+    # The actual S3 client object to communicate with the S3 API.
+    #
+    # @return [UberS3]
+    #
+    def client
+      @client ||= UberS3.new(
+        access_key:        Config.publish_s3_access_key,
+        secret_access_key: Config.publish_s3_secret_key,
+        bucket:            @options[:bucket],
+        adaper:            :net_http,
+      )
+    end
+
+    #
+    # The unique upload key for this package. The additional "stuff" is
+    # postfixed to the end of the path.
+    #
+    # @param [Package] package
+    #   the package this key is for
+    # @param [Array<String>] stuff
+    #   the additional things to prepend
+    #
+    # @return [String]
+    #
+    def key_for(package, *stuff)
+      File.join(
+        package.metadata[:platform],
+        package.metadata[:platform_version],
+        package.metadata[:arch],
+        package.name,
+        *stuff,
+      )
+    end
+
+    #
+    # The access policy that corresponds to the +s3_access+ given in the
+    # initializer option. Any access control that is not the strict string
+    # +"public"+ is assumed to be private.
+    #
+    # @return [Symbol]
+    #   the UberS3-ready access policy
+    #
+    def access_policy
+      if @options[:acl].to_s == 'public'
+        :public_read
+      else
+        :private
+      end
+    end
+  end
+end

data/lib/omnibus/s3_cache.rb

@@ -20,90 +20,177 @@ require 'uber-s3'
 module Omnibus
   class S3Cache
     include Logging
-    include SoftwareS3URLs
 
-    def initialize
-      unless config.s3_bucket && config.s3_access_key && config.s3_secret_key
-        raise InvalidS3Configuration.new(config.s3_bucket, config.s3_access_key, config.s3_secret_key)
+    class << self
+      #
+      # List all software in the cache.
+      #
+      # @return [Array<Software>]
+      #
+      def list
+        cached = keys
+        softwares.select do |software|
+          key = key_for(software)
+          cached.include?(key)
+        end
       end
-      @client = UberS3.new(
-        access_key: config.s3_access_key,
-        secret_access_key: config.s3_secret_key,
-        bucket: config.s3_bucket,
-        adapter: :net_http,
-      )
-    end
 
-    def config
-      Omnibus.config
-    end
+      #
+      # The list of objects in the cache, by their key.
+      #
+      # @return [Array<String>]
+      #
+      def keys
+        bucket.objects('/').map(&:key)
+      end
 
-    def list
-      existing_keys = list_by_key
-      tarball_software.select { |s| existing_keys.include?(key_for_package(s)) }
-    end
+      #
+      # List all software missing from the cache.
+      #
+      # @return [Array<Software>]
+      #
+      def missing
+        cached = keys
+        softwares.select do |software|
+          key = key_for(software)
+          !cached.include?(key)
+        end
+      end
 
-    def list_by_key
-      bucket.objects('/').map(&:key)
-    end
+      #
+      # Populate the cache with the all the missing software definitions.
+      #
+      # @return [true]
+      #
+      def populate
+        missing.each do |software|
+          fetch(software)
+
+          key = key_for(software)
+          content = IO.read(software.project_file)
+
+          log.info(log_key) do
+            "Caching '#{software.project_file}' to '#{Config.s3_bucket}/#{key}'"
+          end
+
+          client.store(key, content,
+            access: :public_read,
+            content_md5: software.checksum
+          )
+        end
 
-    def missing
-      already_cached = list_by_key
-      tarball_software.delete_if { |s| already_cached.include?(key_for_package(s)) }
-    end
+        true
+      end
 
-    def tarball_software
-      Omnibus.projects.map do |project|
-        project.library.select { |s| s.source && s.source.key?(:url) }
-      end.flatten
-    end
+      #
+      # Fetch all source tarballs onto the local machine.
+      #
+      # @return [true]
+      #
+      def fetch_missing
+        missing.each do |software|
+          fetch(software)
+        end
+      end
 
-    def populate
-      missing.each do |software|
-        fetch(software)
+      #
+      # @private
+      #
+      # The key with which to cache the package on S3. This is the name of the
+      # package, the version of the package, and its checksum.
+      #
+      # @example
+      #   "zlib-1.2.6-618e944d7c7cd6521551e30b32322f4a"
+      #
+      # @param [Software] software
+      #
+      # @return [String]
+      #
+      def key_for(software)
+        unless software.name
+          raise InsufficientSpecification.new(:name, software)
+        end
 
-        key = key_for_package(software)
-        content = IO.read(software.project_file)
+        unless software.version
+          raise InsufficientSpecification.new(:version, software)
+        end
 
-        log.info(log_key) do
-          "Uploading #{software.project_file} as #{config.s3_bucket}/#{key}"
+        unless software.checksum
+          raise InsufficientSpecification.new(:checksum, software)
         end
 
-        @client.store(key, content, access: :public_read, content_md5: software.checksum)
+        "#{software.name}-#{software.version}-#{software.checksum}"
       end
-    end
 
-    def fetch_missing
-      missing.each do |software|
-        fetch(software)
+      private
+
+      #
+      # The client to connect to S3 with.
+      #
+      # @return [UberS3::Client]
+      #
+      def client
+        @client ||= UberS3.new(
+          access_key:        Config.s3_access_key,
+          secret_access_key: Config.s3_secret_key,
+          bucket:            Config.s3_bucket,
+          adapter:           :net_http,
+        )
       end
-    end
 
-    private
-
-    def ensure_cache_dir
-      FileUtils.mkdir_p(config.cache_dir)
-    end
+      #
+      # The bucket where the objects live.
+      #
+      # @return [UberS3::Bucket]
+      #
+      def bucket
+        @bucket ||= begin
+          if client.exists?('/')
+            client.bucket
+          else
+            client.connection.put('/')
+          end
+        end
+      end
 
-    def fetch(software)
-      log.info(log_key) { "Fetching #{software.name}" }
-      fetcher = Fetcher.without_caching_for(software)
-      if fetcher.fetch_required?
-        log.debug(log_key) { 'Updating cache' }
-        fetcher.download
-        fetcher.verify_checksum!
-      else
-        log.debug(log_key) { 'Cached copy up to date, skipping.' }
+      #
+      # The list of softwares for all Omnibus projects.
+      #
+      # @return [Array<Software>]
+      #
+      def softwares
+        Omnibus.projects.inject({}) do |hash, project|
+          project.library.each do |software|
+            if software.source && software.source.key?(:url)
+              hash[software.name] = software
+            end
+          end
+
+          hash
+        end.values.sort
       end
-    end
 
-    def bucket
-      @bucket ||= begin
-        if @client.exists?('/')
-          @client.bucket
+      #
+      # Fetch the remote software definition onto disk.
+      #
+      # @param [Software] software
+      #   the software to fetch
+      #
+      # @return [true]
+      #
+      def fetch(software)
+        log.info(log_key) { "Fetching #{software.name}" }
+        fetcher = Fetcher.without_caching_for(software)
+
+        if fetcher.fetch_required?
+          log.debug(log_key) { 'Updating cache' }
+          fetcher.download
+          fetcher.verify_checksum!
         else
-          @client.connection.put('/')
+          log.debug(log_key) { 'Cached copy up to date, skipping.' }
         end
+
+        true
       end
     end
   end