travis_dpl_test 2.0.3.beta.4.ror

data/lib/dpl/helper/cookbook_site_streaming_uploader.rb

@@ -0,0 +1,249 @@
+#
+# Author:: Stanislav Vitvitskiy
+# Author:: Nuo Yan (nuo@chef.io)
+# Author:: Christopher Walters (<cw@chef.io>)
+# Copyright:: Copyright (c) Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+autoload :URI, "uri"
+module Net
+  autoload :HTTP, "net/http"
+end
+autoload :OpenSSL, "openssl"
+module Mixlib
+  module Authentication
+    autoload :SignedHeaderAuth, "mixlib/authentication/signedheaderauth"
+  end
+end
+require 'chef'
+require "chef-utils/dist" unless defined?(ChefUtils::Dist)
+class Chef
+  class Knife
+    module Core
+      # == Chef::Knife::Core::CookbookSiteStreamingUploader
+      # A streaming multipart HTTP upload implementation. Used to upload cookbooks
+      # (in tarball form) to https://supermarket.chef.io
+      #
+      # inspired by http://stanislavvitvitskiy.blogspot.com/2008/12/multipart-post-in-ruby.html
+      class CookbookSiteStreamingUploader
+
+        DefaultHeaders = { "accept" => "application/json", "x-chef-version" => ::Chef::VERSION }.freeze # rubocop:disable Naming/ConstantName
+
+        class << self
+
+          def create_build_dir(cookbook)
+            tmp_cookbook_path = Tempfile.new("#{ChefUtils::Dist::Infra::SHORT}-#{cookbook.name}-build")
+            tmp_cookbook_path.close
+            tmp_cookbook_dir = tmp_cookbook_path.path
+            File.unlink(tmp_cookbook_dir)
+            FileUtils.mkdir_p(tmp_cookbook_dir)
+            Chef::Log.trace("Staging at #{tmp_cookbook_dir}")
+            checksums_to_on_disk_paths = cookbook.checksums
+            cookbook.each_file do |manifest_record|
+              path_in_cookbook = manifest_record[:path]
+              on_disk_path = checksums_to_on_disk_paths[manifest_record[:checksum]]
+              dest = File.join(tmp_cookbook_dir, cookbook.name.to_s, path_in_cookbook)
+              FileUtils.mkdir_p(File.dirname(dest))
+              Chef::Log.trace("Staging #{on_disk_path} to #{dest}")
+              FileUtils.cp(on_disk_path, dest)
+            end
+
+            # First, generate metadata
+            Chef::Log.trace("Generating metadata")
+            kcm = Chef::Knife::CookbookMetadata.new
+            kcm.config[:cookbook_path] = [ tmp_cookbook_dir ]
+            kcm.name_args = [ cookbook.name.to_s ]
+            kcm.run
+
+            tmp_cookbook_dir
+          end
+
+          def post(to_url, user_id, secret_key_filename, params = {}, headers = {})
+            make_request(:post, to_url, user_id, secret_key_filename, params, headers)
+          end
+
+          def put(to_url, user_id, secret_key_filename, params = {}, headers = {})
+            make_request(:put, to_url, user_id, secret_key_filename, params, headers)
+          end
+
+          def make_request(http_verb, to_url, user_id, secret_key_filename, params = {}, headers = {})
+            boundary = "----RubyMultipartClient" + rand(1000000).to_s + "ZZZZZ"
+            parts = []
+            content_file = nil
+
+            secret_key = OpenSSL::PKey::RSA.new(File.read(secret_key_filename))
+
+            unless params.nil? || params.empty?
+              params.each do |key, value|
+                if value.is_a?(File)
+                  content_file = value
+                  filepath = value.path
+                  filename = File.basename(filepath)
+                  parts << StringPart.new( "--" + boundary + "\r\n" +
+                                          "Content-Disposition: form-data; name=\"" + key.to_s + "\"; filename=\"" + filename + "\"\r\n" +
+                                          "Content-Type: application/octet-stream\r\n\r\n")
+                  parts << StreamPart.new(value, File.size(filepath))
+                  parts << StringPart.new("\r\n")
+                else
+                  parts << StringPart.new( "--" + boundary + "\r\n" +
+                                          "Content-Disposition: form-data; name=\"" + key.to_s + "\"\r\n\r\n")
+                  parts << StringPart.new(value.to_s + "\r\n")
+                end
+              end
+              parts << StringPart.new("--" + boundary + "--\r\n")
+            end
+
+            body_stream = MultipartStream.new(parts)
+
+            timestamp = Time.now.utc.iso8601
+
+            url = URI.parse(to_url)
+
+            Chef::Log.logger.debug("Signing: method: #{http_verb}, url: #{url}, file: #{content_file}, User-id: #{user_id}, Timestamp: #{timestamp}")
+
+            # We use the body for signing the request if the file parameter
+            # wasn't a valid file or wasn't included. Extract the body (with
+            # multi-part delimiters intact) to sign the request.
+            # TODO: tim: 2009-12-28: It'd be nice to remove this special case, and
+            # always hash the entire request body. In the file case it would just be
+            # expanded multipart text - the entire body of the POST.
+            content_body = parts.inject("") { |result, part| result + part.read(0, part.size) }
+            content_file.rewind if content_file # we consumed the file for the above operation, so rewind it.
+
+            signing_options = {
+              http_method: http_verb,
+              path: url.path,
+              user_id: user_id,
+              timestamp: timestamp }
+            (content_file && signing_options[:file] = content_file) || (signing_options[:body] = (content_body || ""))
+
+            headers.merge!(Mixlib::Authentication::SignedHeaderAuth.signing_object(signing_options).sign(secret_key))
+
+            content_file.rewind if content_file
+
+            # net/http doesn't like symbols for header keys, so we'll to_s each one just in case
+            headers = DefaultHeaders.merge(Hash[*headers.map { |k, v| [k.to_s, v] }.flatten])
+
+            req = case http_verb
+                  when :put
+                    Net::HTTP::Put.new(url.path, headers)
+                  when :post
+                    Net::HTTP::Post.new(url.path, headers)
+                  end
+            req.content_length = body_stream.size
+            req.content_type = "multipart/form-data; boundary=" + boundary unless parts.empty?
+            req.body_stream = body_stream
+
+            http = Chef::HTTP::BasicClient.new(url).http_client
+            res = http.request(req)
+
+            # alias status to code and to_s to body for test purposes
+            # TODO: stop the following madness!
+            class << res
+              alias :to_s :body
+
+              # BUG this makes the response compatible with what response_steps expects to test headers (response.headers[] -> response[])
+              def headers # rubocop:disable Lint/NestedMethodDefinition
+                self
+              end
+
+              def status  # rubocop:disable Lint/NestedMethodDefinition
+                code.to_i
+              end
+            end
+            res
+          end
+
+        end
+
+        class StreamPart
+          def initialize(stream, size)
+            @stream, @size = stream, size
+          end
+
+          def size
+            @size
+          end
+
+          # read the specified amount from the stream
+          def read(offset, how_much)
+            @stream.read(how_much)
+          end
+        end
+
+        class StringPart
+          def initialize(str)
+            @str = str
+          end
+
+          def size
+            @str.length
+          end
+
+          # read the specified amount from the string starting at the offset
+          def read(offset, how_much)
+            @str[offset, how_much]
+          end
+        end
+
+        class MultipartStream
+          def initialize(parts)
+            @parts = parts
+            @part_no = 0
+            @part_offset = 0
+          end
+
+          def size
+            @parts.inject(0) { |size, part| size + part.size }
+          end
+
+          def read(how_much, dst_buf = nil)
+            if @part_no >= @parts.size
+              dst_buf.replace("") if dst_buf
+              return dst_buf
+            end
+
+            how_much_current_part = @parts[@part_no].size - @part_offset
+
+            how_much_current_part = if how_much_current_part > how_much
+                                      how_much
+                                    else
+                                      how_much_current_part
+                                    end
+
+            how_much_next_part = how_much - how_much_current_part
+
+            current_part = @parts[@part_no].read(@part_offset, how_much_current_part)
+
+            # recurse into the next part if the current one was not large enough
+            if how_much_next_part > 0
+              @part_no += 1
+              @part_offset = 0
+              next_part = read(how_much_next_part)
+              result = current_part + (next_part || "")
+            else
+              @part_offset += how_much_current_part
+              result = current_part
+            end
+            dst_buf ? dst_buf.replace(result || "") : result
+          end
+        end
+
+      end
+    end
+  end
+end
+

data/lib/dpl/helper/env.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+require 'dpl/helper/memoize'
+
+module Dpl
+  module Env
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+
+    class Env
+      include Memoize
+      # opts[:allow_skip_underscore] allows unconventional ENV vars such as GOOGLECLOUDKEYFILE
+
+      attr_reader :cmd, :env, :strs, :keys, :opts
+
+      def initialize(env, args)
+        @env = env
+        @opts = args.last.is_a?(Hash) ? args.pop : {}
+        @strs = args.map(&:to_s).map(&:upcase)
+      end
+
+      def env(cmd)
+        @cmd = cmd
+        env = @env.select { |key, _| keys.include?(key) }
+        env = env.transform_keys { |key| unprefix(key).downcase.to_sym }
+        env.transform_keys { |key| dealias(key) }
+      end
+
+      def description(cmd)
+        strs = self.strs.map { |str| "#{str}_" }
+        strs += self.strs if opts[:allow_skip_underscore]
+        strs = strs.size > 1 ? "[#{strs.sort.join('|')}]" : strs.join
+        "Options can be given via env vars if prefixed with `#{strs}`. #{example(cmd)}"
+      end
+
+      def example(cmd)
+        return unless opt = cmd.opts.detect { |option| option.secret? }
+
+        env = strs.map { |str| "`#{str}_#{opt.name.upcase}=<#{opt.name}>`" }
+        env += strs.map { |str| "`#{str}#{opt.name.upcase}=<#{opt.name}>`" } if opts[:allow_skip_underscore]
+        "E.g. the option `--#{opt.name}` can be given as #{sentence(env)}."
+      end
+
+      def sentence(strs)
+        return strs.join if strs.size == 1
+
+        [strs[0..-2].join(', '), strs[-1]].join(' or ')
+      end
+
+      private
+
+      def dealias(key)
+        opt = cmd.opts.detect { |option| option.aliases.include?(key) }
+        opt ? opt.name : key
+      end
+
+      def unprefix(key)
+        strs.inject(key) { |key, str| key.sub(/^#{str}_?/, '') }
+      end
+
+      def keys
+        keys = cmd.opts.map(&:name) + cmd.opts.map(&:aliases).flatten
+        strs.map { |str| keys.map { |key| keys_for(str, key) } }.flatten
+      end
+      memoize :keys
+
+      def keys_for(str, key)
+        keys = [["#{str}_", key.upcase].join]
+        keys << [str, key.upcase].join if opts[:allow_skip_underscore]
+        keys
+      end
+    end
+
+    # should this sit in Cl?
+    module ClassMethods
+      def env(*strs)
+        if strs.any?
+          @env = Env.new(ENV, strs)
+        elsif env = @env || superclass.instance_variable_get(:@env)
+          env.env(self)
+        else
+          {}
+        end
+      end
+    end
+
+    def opts
+      @opts ||= self.class.env.merge(super)
+    end
+  end
+end

data/lib/dpl/helper/github.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require 'dpl/helper/transliterate'
+
+# I18n.load_path << File.expand_path('config/transliterate.yml')
+# I18n.eager_load!
+# I18n.config.available_locales_set << :en # seems really wrong, but ¯\_(ツ)_/¯
+
+module Dpl
+  module Github
+    include Transliterate
+
+    def normalize_filename(str)
+      str = File.basename(str)
+      str = str.split(' ').first
+      str = transliterate(str)
+      str.gsub(/[^\w@+\-_]/, '.')
+    end
+
+    extend self
+  end
+end

data/lib/dpl/helper/interpolate.rb

@@ -0,0 +1,160 @@
+# frozen_string_literal: true
+
+require 'uri'
+
+module Dpl
+  module Interpolate
+    # Interpolates variables in the given string.
+    #
+    # Variables can be contained in scripts, shell commands, and messages.
+    # They have the syntax `%{name}` or `%s` (or any other identifier supported
+    # by [Kernel#sprintf](https://ruby-doc.org/core-2.6.3/Kernel.html#method-i-format)).
+    #
+    # This supports two styles of interpolation:
+    #
+    # * Named variables `%{name}` and
+    # * Positional variables.
+    #
+    # Named variable names need to match constants on the provider class, or
+    # methods on the provider instance, which will be called in order to
+    # evaluate the value to be interpolated.
+    #
+    # Positional variables can be used if no corresponding method exists, e.g.
+    # if the value that needs to be interpolated is an argument passed to a
+    # local method.
+    #
+    # For example, using named variables:
+    #
+    #   ```ruby
+    #   def upload_file
+    #     interpolate('Uploading file %{file} to %{target}')
+    #   end
+    #
+    #   def file
+    #     './file_name'
+    #   end
+    #
+    #   def target
+    #     'target host'
+    #   end
+    #   ```
+    #
+    # Using positional variables:
+    #
+    #   ```ruby
+    #   def upload_file(file, target)
+    #     interpolate('Uploading file %s to %s', file, target)
+    #   end
+    #   ```
+    #
+    # Implementors are encouraged to use named variables when possible, but
+    # are free to choose according to their needs.
+    def interpolate(str, args = [], opts = {})
+      args = args.shift if args.is_a?(Array) && args.first.is_a?(Hash)
+      Interpolator.new(str, self, args || {}, opts).apply
+    end
+
+    # Interpolation variables as declared by the provider.
+    #
+    # By default this contains string option names, but additional
+    # methods can be added using Provider::Dsl#vars.
+    def vars
+      self.class.vars
+    end
+
+    # Obfuscates the given string.
+    #
+    # Replaces all but the first N characters with asterisks, and paddes
+    # the string to a standard length of 20 characters. N depends on the
+    # length of the original string.
+    def obfuscate(str, opts = {})
+      return str if opts[:secure] || !str.blacklisted?
+
+      keep = (str.length / (4.0 + str.length / 5).round).round
+      keep = 1 if keep.zero?
+      str[0, keep] + '*' * (20 - keep)
+    end
+
+    class Interpolator < Struct.new(:str, :obj, :args, :opts)
+      include Interpolate
+
+      MODIFIER = %i[obfuscate escape quote].freeze
+      PATTERN  = /%\{(\$?[\w]+)\}/
+      ENV_VAR  = /^\$[A-Z_]+$/
+      UPCASE   = /^[A-Z_]+$/
+      UNKNOWN  = '[unknown variable: %s]'
+
+      def apply
+        str = interpolate(self.str.to_s)
+        str = obfuscate(str) unless opts[:secure]
+        str = str.gsub('  ', ' ') if str.lines.size == 1
+        str
+      end
+
+      def interpolate(str)
+        str = str % args if args.is_a?(Array) && args.any?
+        @blacklist_result = false
+        str = str.to_s.gsub(PATTERN) do
+          @blacklist_result = true
+          normalize(lookup(::Regexp.last_match(1).to_sym))
+        end
+        @blacklist_result || (args.is_a?(Array) && args.any? { |arg| arg.is_a?(String) && arg.blacklisted? }) ? str.blacklist : str
+      end
+
+      def obfuscate(str)
+        secrets(str).inject(str) do |str, secret|
+          secret = secret.dup if secret.frozen?
+          secret.blacklist if str.blacklisted?
+          str.gsub(secret, super(secret))
+        end
+      end
+
+      def secrets(str)
+        return [] unless str.is_a?(String) && str.blacklisted?
+
+        opts = obj.class.opts.select(&:secret?)
+        secrets = opts.map { |opt| obj.opts[opt.name] }.compact
+        secrets.select { |secret| str.include?(secret) }
+      end
+
+      def normalize(obj)
+        obj.is_a?(Array) ? obj.join(' ') : obj.to_s
+      end
+
+      def lookup(key)
+        if vars? && !var?(key)
+          UNKNOWN % key
+        elsif mod = modifier(key)
+          key = key.to_s.sub("#{mod}d_", '')
+          obj.send(mod, lookup(key))
+        elsif key.to_s =~ ENV_VAR
+          ENV[key.to_s.sub('$', '')]
+        elsif key.to_s =~ UPCASE && obj.class.const_defined?(key)
+          obj.class.const_get(key)
+        elsif args.is_a?(Hash) && args.key?(key)
+          args[key]
+        elsif obj.respond_to?(key, true)
+          obj.send(key)
+        else
+          raise KeyError, key
+        end
+      end
+
+      def modifier(key)
+        MODIFIER.detect { |mod| key.to_s.start_with?("#{mod}d_") }
+      end
+
+      def var?(key)
+        vars.include?(key)
+      end
+
+      def vars
+        opts[:vars]
+      end
+
+      def vars?
+        !!vars
+      end
+    end
+  end
+end

data/lib/dpl/helper/memoize.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Memoize
+  class ArgsError < StandardError; end
+
+  module ClassMethods
+    def memoize(name)
+      ivar = :"@#{name.to_s.sub('?', '_predicate')}"
+      prepend Module.new {
+        define_method(name) do |*args|
+          raise ArgsError, 'cannot pass arguments to memoized method %p' % name unless args.empty?
+          return instance_variable_get(ivar) if instance_variable_defined?(ivar)
+
+          instance_variable_set(ivar, super())
+        end
+      }
+    end
+  end
+
+  def self.included(base)
+    base.extend(ClassMethods)
+  end
+end

data/lib/dpl/helper/squiggle.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+# Beloved squiggly heredocs did not exist in Ruby 2.1, which we still want to
+# support, so let's give kudos with a method `sq`.
+module Squiggle
+  # Outdents each line on a multiline string by the number of leading
+  # whitespace characters on the first line.
+  #
+  # This method exists so we can unindet heredoc strings the same way that
+  # Ruby 2.2's squiggly heredocs work, but still support Ruby 2.1 for the
+  # time being.
+  #
+  # For example:
+  #
+  # str = sq(<<-str)
+  #   This multiline string will be outdented by two characters,
+  #     so the extra indentation on this line will be kept,
+  #   while this line sits on the same level as the first line.
+  # str
+  def sq(str)
+    width = str =~ /( *)\S/ && ::Regexp.last_match(1).size
+    str.lines.map { |line| line.gsub(/^ {#{width}}/, '') }.join
+  end
+end

data/lib/dpl/helper/transliterate.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Dpl
+  module Transliterate
+    APPROXIMATIONS = YAML.load(File.read(File.expand_path('../../../config/transliterate.yml', __dir__)))
+
+    def transliterate(string, replacement = '.')
+      string.gsub(/[^\x00-\x7f]/u) do |char|
+        APPROXIMATIONS[char] || replacement
+      end
+    end
+  end
+end

data/lib/dpl/helper/wrap.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Wrap
+  module_function
+
+  def wrap(str, width = 80)
+    str.lines.map do |line|
+      line.size > width ? line.gsub(/(.{1,#{width}})(\s+|$)/, "\\1\n").strip : line
+    end.join("\n")
+  end
+end

data/lib/dpl/helper/zip.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require 'tempfile'
+
+module Dpl
+  class Zip < Struct.new(:src, :dest, :opts)
+    ZIP_EXT = %w[.zip .jar].freeze
+
+    def initialize(*)
+      require 'zip'
+      super
+    end
+
+    def zip
+      if zip_file?
+        File.new(src)
+      elsif dir?
+        zip_dir
+      else
+        zip_file
+      end
+    end
+
+    def zip_dir
+      create(Dir.glob(*glob).reject { |path| dir?(path) })
+    end
+
+    def zip_file
+      create([src])
+    end
+
+    def create(files)
+      ::Zip::File.open(dest, ::Zip::File::CREATE) do |zip|
+        files.each do |file|
+          zip.add(file.sub("#{src}/", ''), file)
+        end
+      end
+      File.new(dest)
+    end
+
+    def zip_file?
+      exts.include?(File.extname(src))
+    end
+
+    def dir?(path = src)
+      File.directory?(path)
+    end
+
+    def copy
+      FileUtils.cp(src, dest)
+    end
+
+    def glob
+      glob = ["#{src}/**/*"]
+      glob << File::FNM_DOTMATCH if dot_match?
+      glob
+    end
+
+    def dot_match?
+      opts[:dot_match]
+    end
+
+    def exts
+      opts[:exts] ||= ZIP_EXT
+    end
+
+    def opts
+      super || {}
+    end
+  end
+end