dpl 2.0.3.beta.4 → 2.0.3.beta.5

data/lib/dpl/helper/cookbook_site_streaming_uploader.rb

@@ -0,0 +1,249 @@
+#
+# Author:: Stanislav Vitvitskiy
+# Author:: Nuo Yan (nuo@chef.io)
+# Author:: Christopher Walters (<cw@chef.io>)
+# Copyright:: Copyright (c) Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+autoload :URI, "uri"
+module Net
+  autoload :HTTP, "net/http"
+end
+autoload :OpenSSL, "openssl"
+module Mixlib
+  module Authentication
+    autoload :SignedHeaderAuth, "mixlib/authentication/signedheaderauth"
+  end
+end
+require 'chef'
+require "chef-utils/dist" unless defined?(ChefUtils::Dist)
+class Chef
+  class Knife
+    module Core
+      # == Chef::Knife::Core::CookbookSiteStreamingUploader
+      # A streaming multipart HTTP upload implementation. Used to upload cookbooks
+      # (in tarball form) to https://supermarket.chef.io
+      #
+      # inspired by http://stanislavvitvitskiy.blogspot.com/2008/12/multipart-post-in-ruby.html
+      class CookbookSiteStreamingUploader
+
+        DefaultHeaders = { "accept" => "application/json", "x-chef-version" => ::Chef::VERSION }.freeze # rubocop:disable Naming/ConstantName
+
+        class << self
+
+          def create_build_dir(cookbook)
+            tmp_cookbook_path = Tempfile.new("#{ChefUtils::Dist::Infra::SHORT}-#{cookbook.name}-build")
+            tmp_cookbook_path.close
+            tmp_cookbook_dir = tmp_cookbook_path.path
+            File.unlink(tmp_cookbook_dir)
+            FileUtils.mkdir_p(tmp_cookbook_dir)
+            Chef::Log.trace("Staging at #{tmp_cookbook_dir}")
+            checksums_to_on_disk_paths = cookbook.checksums
+            cookbook.each_file do |manifest_record|
+              path_in_cookbook = manifest_record[:path]
+              on_disk_path = checksums_to_on_disk_paths[manifest_record[:checksum]]
+              dest = File.join(tmp_cookbook_dir, cookbook.name.to_s, path_in_cookbook)
+              FileUtils.mkdir_p(File.dirname(dest))
+              Chef::Log.trace("Staging #{on_disk_path} to #{dest}")
+              FileUtils.cp(on_disk_path, dest)
+            end
+
+            # First, generate metadata
+            Chef::Log.trace("Generating metadata")
+            kcm = Chef::Knife::CookbookMetadata.new
+            kcm.config[:cookbook_path] = [ tmp_cookbook_dir ]
+            kcm.name_args = [ cookbook.name.to_s ]
+            kcm.run
+
+            tmp_cookbook_dir
+          end
+
+          def post(to_url, user_id, secret_key_filename, params = {}, headers = {})
+            make_request(:post, to_url, user_id, secret_key_filename, params, headers)
+          end
+
+          def put(to_url, user_id, secret_key_filename, params = {}, headers = {})
+            make_request(:put, to_url, user_id, secret_key_filename, params, headers)
+          end
+
+          def make_request(http_verb, to_url, user_id, secret_key_filename, params = {}, headers = {})
+            boundary = "----RubyMultipartClient" + rand(1000000).to_s + "ZZZZZ"
+            parts = []
+            content_file = nil
+
+            secret_key = OpenSSL::PKey::RSA.new(File.read(secret_key_filename))
+
+            unless params.nil? || params.empty?
+              params.each do |key, value|
+                if value.is_a?(File)
+                  content_file = value
+                  filepath = value.path
+                  filename = File.basename(filepath)
+                  parts << StringPart.new( "--" + boundary + "\r\n" +
+                                          "Content-Disposition: form-data; name=\"" + key.to_s + "\"; filename=\"" + filename + "\"\r\n" +
+                                          "Content-Type: application/octet-stream\r\n\r\n")
+                  parts << StreamPart.new(value, File.size(filepath))
+                  parts << StringPart.new("\r\n")
+                else
+                  parts << StringPart.new( "--" + boundary + "\r\n" +
+                                          "Content-Disposition: form-data; name=\"" + key.to_s + "\"\r\n\r\n")
+                  parts << StringPart.new(value.to_s + "\r\n")
+                end
+              end
+              parts << StringPart.new("--" + boundary + "--\r\n")
+            end
+
+            body_stream = MultipartStream.new(parts)
+
+            timestamp = Time.now.utc.iso8601
+
+            url = URI.parse(to_url)
+
+            Chef::Log.logger.debug("Signing: method: #{http_verb}, url: #{url}, file: #{content_file}, User-id: #{user_id}, Timestamp: #{timestamp}")
+
+            # We use the body for signing the request if the file parameter
+            # wasn't a valid file or wasn't included. Extract the body (with
+            # multi-part delimiters intact) to sign the request.
+            # TODO: tim: 2009-12-28: It'd be nice to remove this special case, and
+            # always hash the entire request body. In the file case it would just be
+            # expanded multipart text - the entire body of the POST.
+            content_body = parts.inject("") { |result, part| result + part.read(0, part.size) }
+            content_file.rewind if content_file # we consumed the file for the above operation, so rewind it.
+
+            signing_options = {
+              http_method: http_verb,
+              path: url.path,
+              user_id: user_id,
+              timestamp: timestamp }
+            (content_file && signing_options[:file] = content_file) || (signing_options[:body] = (content_body || ""))
+
+            headers.merge!(Mixlib::Authentication::SignedHeaderAuth.signing_object(signing_options).sign(secret_key))
+
+            content_file.rewind if content_file
+
+            # net/http doesn't like symbols for header keys, so we'll to_s each one just in case
+            headers = DefaultHeaders.merge(Hash[*headers.map { |k, v| [k.to_s, v] }.flatten])
+
+            req = case http_verb
+                  when :put
+                    Net::HTTP::Put.new(url.path, headers)
+                  when :post
+                    Net::HTTP::Post.new(url.path, headers)
+                  end
+            req.content_length = body_stream.size
+            req.content_type = "multipart/form-data; boundary=" + boundary unless parts.empty?
+            req.body_stream = body_stream
+
+            http = Chef::HTTP::BasicClient.new(url).http_client
+            res = http.request(req)
+
+            # alias status to code and to_s to body for test purposes
+            # TODO: stop the following madness!
+            class << res
+              alias :to_s :body
+
+              # BUG this makes the response compatible with what response_steps expects to test headers (response.headers[] -> response[])
+              def headers # rubocop:disable Lint/NestedMethodDefinition
+                self
+              end
+
+              def status  # rubocop:disable Lint/NestedMethodDefinition
+                code.to_i
+              end
+            end
+            res
+          end
+
+        end
+
+        class StreamPart
+          def initialize(stream, size)
+            @stream, @size = stream, size
+          end
+
+          def size
+            @size
+          end
+
+          # read the specified amount from the stream
+          def read(offset, how_much)
+            @stream.read(how_much)
+          end
+        end
+
+        class StringPart
+          def initialize(str)
+            @str = str
+          end
+
+          def size
+            @str.length
+          end
+
+          # read the specified amount from the string starting at the offset
+          def read(offset, how_much)
+            @str[offset, how_much]
+          end
+        end
+
+        class MultipartStream
+          def initialize(parts)
+            @parts = parts
+            @part_no = 0
+            @part_offset = 0
+          end
+
+          def size
+            @parts.inject(0) { |size, part| size + part.size }
+          end
+
+          def read(how_much, dst_buf = nil)
+            if @part_no >= @parts.size
+              dst_buf.replace("") if dst_buf
+              return dst_buf
+            end
+
+            how_much_current_part = @parts[@part_no].size - @part_offset
+
+            how_much_current_part = if how_much_current_part > how_much
+                                      how_much
+                                    else
+                                      how_much_current_part
+                                    end
+
+            how_much_next_part = how_much - how_much_current_part
+
+            current_part = @parts[@part_no].read(@part_offset, how_much_current_part)
+
+            # recurse into the next part if the current one was not large enough
+            if how_much_next_part > 0
+              @part_no += 1
+              @part_offset = 0
+              next_part = read(how_much_next_part)
+              result = current_part + (next_part || "")
+            else
+              @part_offset += how_much_current_part
+              result = current_part
+            end
+            dst_buf ? dst_buf.replace(result || "") : result
+          end
+        end
+
+      end
+    end
+  end
+end
+

data/lib/dpl/helper/env.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'dpl/helper/memoize'
 
 module Dpl
@@ -21,8 +23,8 @@ module Dpl
       def env(cmd)
         @cmd = cmd
         env = @env.select { |key, _| keys.include?(key) }
-        env = env.map { |key, value| [unprefix(key).downcase.to_sym, value] }.to_h
-        env.map { |key, value| [dealias(key), value] }.to_h
+        env = env.transform_keys { |key| unprefix(key).downcase.to_sym }
+        env.transform_keys { |key| dealias(key) }
       end
 
       def description(cmd)
@@ -33,39 +35,41 @@ module Dpl
       end
 
       def example(cmd)
-        return unless opt = cmd.opts.detect { |opt| opt.secret? }
-        env = self.strs.map { |str| "`#{str}_#{opt.name.upcase}=<#{opt.name}>`" }
-        env += self.strs.map { |str| "`#{str}#{opt.name.upcase}=<#{opt.name}>`" } if opts[:allow_skip_underscore]
+        return unless opt = cmd.opts.detect { |option| option.secret? }
+
+        env = strs.map { |str| "`#{str}_#{opt.name.upcase}=<#{opt.name}>`" }
+        env += strs.map { |str| "`#{str}#{opt.name.upcase}=<#{opt.name}>`" } if opts[:allow_skip_underscore]
         "E.g. the option `--#{opt.name}` can be given as #{sentence(env)}."
       end
 
       def sentence(strs)
         return strs.join if strs.size == 1
+
         [strs[0..-2].join(', '), strs[-1]].join(' or ')
       end
 
       private
 
-        def dealias(key)
-          opt = cmd.opts.detect { |opt| opt.aliases.include?(key) }
-          opt ? opt.name : key
-        end
+      def dealias(key)
+        opt = cmd.opts.detect { |option| option.aliases.include?(key) }
+        opt ? opt.name : key
+      end
 
-        def unprefix(key)
-          strs.inject(key) { |key, str| key.sub(/^#{str}_?/, '') }
-        end
+      def unprefix(key)
+        strs.inject(key) { |key, str| key.sub(/^#{str}_?/, '') }
+      end
 
-        def keys
-          keys = cmd.opts.map(&:name) + cmd.opts.map(&:aliases).flatten
-          strs.map { |str| keys.map { |key| keys_for(str, key) } }.flatten
-        end
-        memoize :keys
+      def keys
+        keys = cmd.opts.map(&:name) + cmd.opts.map(&:aliases).flatten
+        strs.map { |str| keys.map { |key| keys_for(str, key) } }.flatten
+      end
+      memoize :keys
 
-        def keys_for(str, key)
-          keys = [["#{str}_", key.upcase].join]
-          keys << [str, key.upcase].join if opts[:allow_skip_underscore]
-          keys
-        end
+      def keys_for(str, key)
+        keys = [["#{str}_", key.upcase].join]
+        keys << [str, key.upcase].join if opts[:allow_skip_underscore]
+        keys
+      end
     end
 
     # should this sit in Cl?

data/lib/dpl/helper/github.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'dpl/helper/transliterate'
 
 # I18n.load_path << File.expand_path('config/transliterate.yml')

data/lib/dpl/helper/interpolate.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'uri'
 
 module Dpl
@@ -67,15 +69,16 @@ module Dpl
     # length of the original string.
     def obfuscate(str, opts = {})
       return str if opts[:secure] || !str.blacklisted?
+
       keep = (str.length / (4.0 + str.length / 5).round).round
-      keep = 1 if keep == 0
+      keep = 1 if keep.zero?
       str[0, keep] + '*' * (20 - keep)
     end
 
     class Interpolator < Struct.new(:str, :obj, :args, :opts)
       include Interpolate
 
-      MODIFIER = %i(obfuscate escape quote)
+      MODIFIER = %i[obfuscate escape quote].freeze
       PATTERN  = /%\{(\$?[\w]+)\}/
       ENV_VAR  = /^\$[A-Z_]+$/
       UPCASE   = /^[A-Z_]+$/
@@ -93,7 +96,7 @@ module Dpl
         @blacklist_result = false
         str = str.to_s.gsub(PATTERN) do
           @blacklist_result = true
-          normalize(lookup($1.to_sym))
+          normalize(lookup(::Regexp.last_match(1).to_sym))
         end
         @blacklist_result || (args.is_a?(Array) && args.any? { |arg| arg.is_a?(String) && arg.blacklisted? }) ? str.blacklist : str
       end
@@ -108,6 +111,7 @@ module Dpl
 
       def secrets(str)
         return [] unless str.is_a?(String) && str.blacklisted?
+
         opts = obj.class.opts.select(&:secret?)
         secrets = opts.map { |opt| obj.opts[opt.name] }.compact
         secrets.select { |secret| str.include?(secret) }
@@ -117,7 +121,7 @@ module Dpl
         obj.is_a?(Array) ? obj.join(' ') : obj.to_s
       end
 
-      def lookup(key)        
+      def lookup(key)
         if vars? && !var?(key)
           UNKNOWN % key
         elsif mod = modifier(key)

data/lib/dpl/helper/memoize.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Memoize
   class ArgsError < StandardError; end
 
@@ -6,8 +8,9 @@ module Memoize
       ivar = :"@#{name.to_s.sub('?', '_predicate')}"
       prepend Module.new {
         define_method(name) do |*args|
-          raise ArgsError.new('cannot pass arguments to memoized method %p' % name) unless args.empty?
+          raise ArgsError, 'cannot pass arguments to memoized method %p' % name unless args.empty?
           return instance_variable_get(ivar) if instance_variable_defined?(ivar)
+
           instance_variable_set(ivar, super())
         end
       }

data/lib/dpl/helper/squiggle.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Beloved squiggly heredocs did not exist in Ruby 2.1, which we still want to
 # support, so let's give kudos with a method `sq`.
 module Squiggle
@@ -16,7 +18,7 @@ module Squiggle
   #   while this line sits on the same level as the first line.
   # str
   def sq(str)
-    width = str =~ /( *)\S/ && $1.size
+    width = str =~ /( *)\S/ && ::Regexp.last_match(1).size
     str.lines.map { |line| line.gsub(/^ {#{width}}/, '') }.join
   end
 end

data/lib/dpl/helper/transliterate.rb

@@ -1,6 +1,8 @@
+# frozen_string_literal: true
+
 module Dpl
   module Transliterate
-    APPROXIMATIONS = YAML.load(File.read(File.expand_path('../../../../config/transliterate.yml', __FILE__)))
+    APPROXIMATIONS = YAML.load(File.read(File.expand_path('../../../config/transliterate.yml', __dir__)))
 
     def transliterate(string, replacement = '.')
       string.gsub(/[^\x00-\x7f]/u) do |char|

data/lib/dpl/helper/wrap.rb

@@ -1,5 +1,7 @@
+# frozen_string_literal: true
+
 module Wrap
-  extend self
+  module_function
 
   def wrap(str, width = 80)
     str.lines.map do |line|

data/lib/dpl/helper/zip.rb

@@ -1,8 +1,10 @@
+# frozen_string_literal: true
+
 require 'tempfile'
 
 module Dpl
   class Zip < Struct.new(:src, :dest, :opts)
-    ZIP_EXT = %w(.zip .jar)
+    ZIP_EXT = %w[.zip .jar].freeze
 
     def initialize(*)
       require 'zip'

data/lib/dpl/provider/dsl.rb

@@ -1,8 +1,10 @@
+# frozen_string_literal: true
+
 require 'dpl/helper/squiggle'
 require 'dpl/helper/wrap'
 require 'dpl/provider/status'
 
-# TODO figure out how to allow adding domain specific behavior like this to Cl
+# TODO: figure out how to allow adding domain specific behavior like this to Cl
 class Cl::Opt
   OPTS << :interpolate
 
@@ -55,9 +57,10 @@ module Dpl
       def vars(*vars)
         return self.vars.concat(vars) if vars.any?
         return @vars if instance_variable_defined?(:@vars)
+
         vars = superclass.respond_to?(:vars) ? superclass.vars : []
-        reject = %i(flag array internal interpolate secret)
-        opts = reject.inject(self.opts) { |opts, attr| opts.reject(&:"#{attr}?") }
+        reject = %i[flag array internal interpolate secret]
+        opts = reject.inject(self.opts) { |options, attr| options.reject(&:"#{attr}?") }
         @vars = vars.dup.concat(opts.map(&:name)).uniq.sort - [:strategy]
       end
 
@@ -108,6 +111,7 @@ module Dpl
 
       def runtimes(name = nil, requirements = nil)
         return @runtimes ||= [] unless name
+
         runtimes << [name, requirements]
       end
 
@@ -121,6 +125,7 @@ module Dpl
       # @return Previously declared apt packages if no arguments were given.
       def apt(package = nil, cmd = nil)
         return apt << [package, cmd].compact if package
+
         @apt ||= self == Provider ? [] : superclass.apt.dup
       end
 
@@ -144,6 +149,7 @@ module Dpl
       # @return Previously declared gems if no arguments were given
       def gem(name = nil, version = nil, opts = {})
         return gem << [name, version, opts] if name
+
         @gem ||= self == Provider ? [] : superclass.gem.dup
       end
 
@@ -161,6 +167,7 @@ module Dpl
       # @return Previously declared NPM packages if no arguments are given.
       def npm(package = nil, cmd = nil)
         return npm << [package, cmd].compact if package
+
         @npm ||= self == Provider ? [] : superclass.npm.dup
       end
 
@@ -181,6 +188,7 @@ module Dpl
       # @return Previously declared Python packages if no arguments are given.
       def pip(package = nil, cmd = nil, version = nil)
         return pip << [package, cmd, version].compact if package
+
         @pip ||= self == Provider ? [] : superclass.pip.dup
       end
 
@@ -223,6 +231,7 @@ module Dpl
       # commands.
       def cmds(cmds = nil)
         return self.cmds.update(cmds) if cmds
+
         @cmds ||= self == Provider ? {} : superclass.cmds.dup
       end
 
@@ -265,6 +274,7 @@ module Dpl
       # commands.
       def errs(errs = nil)
         return self.errs.update(errs) if errs
+
         @errs ||= self == Provider ? {} : superclass.errs.dup
       end
 
@@ -327,11 +337,13 @@ module Dpl
       # @return Previously declared msgs if no argument is given.
       def msgs(msgs = nil)
         return self.msgs.update(msgs) if msgs
+
         @msgs ||= self == Provider ? {} : superclass.msgs.dup
       end
 
       def strs(strs = nil)
         return self.strs.update(strs) if strs
+
         @strs ||= self == Provider ? {} : superclass.strs.dup
       end
 
@@ -342,6 +354,7 @@ module Dpl
       # @return Previously declared artifacts to keep if no argument is given.
       def keep(*paths)
         return keep.concat(paths) if paths.any?
+
         @keep ||= self == Provider ? [] : superclass.keep.dup
       end
 
@@ -364,6 +377,7 @@ module Dpl
       # @return Previously declared features needed if no argument is given.
       def needs(*features)
         return needs.concat(features) if features.any?
+
         @needs ||= self == Provider ? [] : superclass.needs.dup
       end
 
@@ -380,7 +394,7 @@ module Dpl
       def user_agent(*strs)
         strs.unshift "dpl/#{Dpl::VERSION}"
         strs.unshift 'travis/0.1.0' if ENV['TRAVIS']
-        strs = strs.flat_map { |e| Hash === e ? e.map { |k, v| "#{k}/#{v}" } : e }
+        strs = strs.flat_map { |e| e.is_a?(Hash) ? e.map { |k, v| "#{k}/#{v}" } : e }
         strs.join(' ').gsub(/\s+/, ' ').strip
       end
 

data/lib/dpl/provider/examples.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Dpl
   class Examples < Struct.new(:const)
     def cmds
@@ -58,12 +60,13 @@ module Dpl
     end
 
     def without_required(opts)
-      opts = opts - const.required.flatten.map { |key| const.opts[key] }
+      opts -= const.required.flatten.map { |key| const.opts[key] }
       opts - required_opts.map(&:opts)
     end
 
     def example(opts)
       return unless opts.any?
+
       opts = required_opts.concat(opts).uniq.compact
       Example.new(const, opts)
     end
@@ -117,7 +120,8 @@ module Dpl
       return if opt.type == :flag
       return 1 if opt.type == :integer
       return opt.enum.first if opt.enum?
-      str = opt.strs.detect { |str| str =~ /^--#{opt.name} (.*)$/ } && $1
+
+      str = opt.strs.detect { |str| str =~ /^--#{opt.name} (.*)$/ } && ::Regexp.last_match(1)
       str ? str.downcase : 'str'
     end
 

data/lib/dpl/provider/status.rb

@@ -1,15 +1,17 @@
+# frozen_string_literal: true
+
 module Dpl
   class Provider < Cl::Cmd
     class Status < Struct.new(:provider, :status, :info)
-      STATUS = %i(dev alpha beta stable deprecated)
+      STATUS = %i[dev alpha beta stable deprecated].freeze
 
       MSG = {
-        dev:        'Support for deployments to %s is in **development**',
-        alpha:      'Support for deployments to %s is in **alpha**',
-        beta:       'Support for deployments to %s is in **beta**',
+        dev: 'Support for deployments to %s is in **development**',
+        alpha: 'Support for deployments to %s is in **alpha**',
+        beta: 'Support for deployments to %s is in **beta**',
         deprecated: 'Support for deployments to %s is *deprecated**',
         pre_stable: 'Please see [Maturity Levels](%s) for details.'
-      }
+      }.freeze
 
       URL = 'https://github.com/travis-ci/dpl/#maturity-levels'
 
@@ -23,7 +25,7 @@ module Dpl
       end
 
       def msg
-        msg = "#{MSG[status] % name}"
+        msg = (MSG[status] % name).to_s
         msg << "(#{info})" if info
         msg << ". #{MSG[:pre_stable] % URL}" if pre_stable?
         msg
@@ -31,29 +33,29 @@ module Dpl
 
       private
 
-        def name
-          provider.full_name
-        end
+      def name
+        provider.full_name
+      end
 
-        def pre_stable?
-          STATUS.index(status) < STATUS.index(:stable)
-        end
+      def pre_stable?
+        STATUS.index(status) < STATUS.index(:stable)
+      end
 
-        def stable?
-          status == :stable
-        end
+      def stable?
+        status == :stable
+      end
 
-        def deprecated?
-          status == :deprecated
-        end
+      def deprecated?
+        status == :deprecated
+      end
 
-        def known?(status)
-          STATUS.include?(status)
-        end
+      def known?(status)
+        STATUS.include?(status)
+      end
 
-        def unknown!(status)
-          raise "Unknown status: #{status.inspect}. Known statuses are: #{STATUS.map(&:inspect).join(', ')}"
-        end
+      def unknown!(status)
+        raise "Unknown status: #{status.inspect}. Known statuses are: #{STATUS.map(&:inspect).join(', ')}"
+      end
     end
   end
 end