chef 16.11.7 → 17.2.29

data/lib/chef/chef_fs/parallelizer/flatten_enumerable.rb

@@ -1,35 +0,0 @@
-class Chef
-  module ChefFS
-    class Parallelizer
-      class FlattenEnumerable
-        include Enumerable
-
-        def initialize(enum, levels = nil)
-          @enum = enum
-          @levels = levels
-        end
-
-        attr_reader :enum
-        attr_reader :levels
-
-        def each(&block)
-          enum.each do |value|
-            flatten(value, levels, &block)
-          end
-        end
-
-        private
-
-        def flatten(value, levels, &block)
-          if levels != 0 && value.respond_to?(:each) && !value.is_a?(String)
-            value.each do |child|
-              flatten(child, levels.nil? ? levels : levels - 1, &block)
-            end
-          else
-            yield(value)
-          end
-        end
-      end
-    end
-  end
-end

data/lib/chef/chef_fs/parallelizer/parallel_enumerable.rb

@@ -1,278 +0,0 @@
-require_relative "flatten_enumerable"
-
-class Chef
-  module ChefFS
-    class Parallelizer
-      class ParallelEnumerable
-        include Enumerable
-
-        # options:
-        # :ordered [true|false] - whether the output should stay in the same order
-        #   as the input (even though it may not actually be processed in that
-        #   order). Default: true
-        # :stop_on_exception [true|false] - if true, when an exception occurs in either
-        #   input or output, we wait for any outstanding processing to complete,
-        #   but will not process any new inputs. Default: false
-        # :main_thread_processing [true|false] - whether the main thread pulling
-        #   on each() is allowed to process inputs. Default: true
-        #   NOTE: If you set this to false, parallelizer.kill will stop each()
-        #   in its tracks, so you need to know for sure that won't happen.
-        def initialize(parent_task_queue, input_enumerable, options = {}, &block)
-          @parent_task_queue = parent_task_queue
-          @input_enumerable = input_enumerable
-          @options = options
-          @block = block
-
-          @unconsumed_input = Queue.new
-          @in_process = {}
-          @unconsumed_output = Queue.new
-        end
-
-        attr_reader :parent_task_queue
-        attr_reader :input_enumerable
-        attr_reader :options
-        attr_reader :block
-
-        def each
-          each_with_input do |output, index, input, type|
-            yield output
-          end
-        end
-
-        def each_with_index
-          each_with_input do |output, index, input|
-            yield output, index
-          end
-        end
-
-        def each_with_input
-          exception = nil
-          each_with_exceptions do |output, index, input, type|
-            if type == :exception
-              if @options[:ordered] == false
-                exception ||= output
-              else
-                raise output
-              end
-            else
-              yield output, index, input
-            end
-          end
-          raise exception if exception
-        end
-
-        def each_with_exceptions(&block)
-          if @options[:ordered] == false
-            each_with_exceptions_unordered(&block)
-          else
-            each_with_exceptions_ordered(&block)
-          end
-        end
-
-        def wait
-          exception = nil
-          each_with_exceptions_unordered do |output, index, input, type|
-            exception ||= output if type == :exception
-          end
-          raise exception if exception
-        end
-
-        # Enumerable methods
-        def restricted_copy(enumerable)
-          ParallelEnumerable.new(@parent_task_queue, enumerable, @options, &@block)
-        end
-
-        alias :original_count :count
-
-        def count(*args, &block)
-          if args.size == 0 && block.nil?
-            @input_enumerable.count
-          else
-            original_count(*args, &block)
-          end
-        end
-
-        def first(n = nil)
-          if n
-            restricted_copy(@input_enumerable.first(n)).to_a
-          else
-            first(1)[0]
-          end
-        end
-
-        def drop(n)
-          restricted_copy(@input_enumerable.drop(n)).to_a
-        end
-
-        def flatten(levels = nil)
-          FlattenEnumerable.new(self, levels)
-        end
-
-        def take(n)
-          restricted_copy(@input_enumerable.take(n)).to_a
-        end
-
-        if Enumerable.method_defined?(:lazy)
-          class RestrictedLazy
-            def initialize(parallel_enumerable, actual_lazy)
-              @parallel_enumerable = parallel_enumerable
-              @actual_lazy = actual_lazy
-            end
-
-            def drop(*args, &block)
-              input = @parallel_enumerable.input_enumerable.lazy.drop(*args, &block)
-              @parallel_enumerable.restricted_copy(input)
-            end
-
-            def take(*args, &block)
-              input = @parallel_enumerable.input_enumerable.lazy.take(*args, &block)
-              @parallel_enumerable.restricted_copy(input)
-            end
-
-            def method_missing(method, *args, &block)
-              @actual_lazy.send(:method, *args, &block)
-            end
-          end
-
-          alias :original_lazy :lazy
-
-          def lazy
-            RestrictedLazy.new(self, original_lazy)
-          end
-        end
-
-        private
-
-        def each_with_exceptions_unordered
-          if @each_running
-            raise "each() called on parallel enumerable twice simultaneously! Bad mojo"
-          end
-
-          @each_running = true
-          begin
-            # Grab all the inputs, yielding any responses during enumeration
-            # in case the enumeration itself takes time
-            begin
-              @input_enumerable.each_with_index do |input, index|
-                @unconsumed_input.push([ input, index ])
-                @parent_task_queue.push(method(:process_one))
-
-                stop_processing_input = false
-                until @unconsumed_output.empty?
-                  output, index, input, type = @unconsumed_output.pop
-                  yield output, index, input, type
-                  if type == :exception && @options[:stop_on_exception]
-                    stop_processing_input = true
-                    break
-                  end
-                end
-
-                if stop_processing_input
-                  break
-                end
-              end
-            rescue
-              # We still want to wait for the rest of the outputs to process
-              @unconsumed_output.push([$!, nil, nil, :exception])
-              if @options[:stop_on_exception]
-                @unconsumed_input.clear
-              end
-            end
-
-            until finished?
-              # yield thread to others (for 1.8.7)
-              if @unconsumed_output.empty?
-                sleep(0.01)
-              end
-
-              yield @unconsumed_output.pop until @unconsumed_output.empty?
-
-              # If no one is working on our tasks and we're allowed to
-              # work on them in the main thread, process an input to
-              # move things forward.
-              if @in_process.size == 0 && !(@options[:main_thread_processing] == false)
-                process_one
-              end
-            end
-          rescue
-            # If we exited early, perhaps due to any? finding a result, we want
-            # to make sure and throw away any extra results (gracefully) so that
-            # the next enumerator can start over.
-            unless finished?
-              stop
-            end
-            raise
-          ensure
-            @each_running = false
-          end
-        end
-
-        def each_with_exceptions_ordered
-          next_to_yield = 0
-          unconsumed = {}
-          each_with_exceptions_unordered do |output, index, input, type|
-            unconsumed[index] = [ output, input, type ]
-            while unconsumed[next_to_yield]
-              input_output = unconsumed.delete(next_to_yield)
-              yield input_output[0], next_to_yield, input_output[1], input_output[2]
-              next_to_yield += 1
-            end
-          end
-          input_exception = unconsumed.delete(nil)
-          if input_exception
-            yield input_exception[0], next_to_yield, input_exception[1], input_exception[2]
-          end
-        end
-
-        def stop
-          @unconsumed_input.clear
-          sleep(0.05) while @in_process.size > 0
-          @unconsumed_output.clear
-        end
-
-        #
-        # This is thread safe only if called from the main thread pulling on each().
-        # The order of these checks is important, as well, to be thread safe.
-        # 1. If @unconsumed_input.empty? is true, then we will never have any more
-        # work legitimately picked up.
-        # 2. If @in_process == 0, then there is no work in process, and because of when unconsumed_input is empty, it will never go back up, because
-        # this is called after the input enumerator is finished.  Note that switching #2 and #1
-        # could cause a race, because in_process is incremented *before* consuming input.
-        # 3. If @unconsumed_output.empty? is true, then we are done with outputs.
-        # Thus, 1+2 means no more output will ever show up, and 3 means we've passed all
-        # existing outputs to the user.
-        #
-        def finished?
-          @unconsumed_input.empty? && @in_process.size == 0 && @unconsumed_output.empty?
-        end
-
-        def process_one
-          @in_process[Thread.current] = true
-          begin
-            begin
-              input, index = @unconsumed_input.pop(true)
-              process_input(input, index)
-            rescue ThreadError
-            end
-          ensure
-            @in_process.delete(Thread.current)
-          end
-        end
-
-        def process_input(input, index)
-          begin
-            output = @block.call(input)
-            @unconsumed_output.push([ output, index, input, :result ])
-          rescue StandardError, ScriptError
-            if @options[:stop_on_exception]
-              @unconsumed_input.clear
-            end
-            @unconsumed_output.push([ $!, index, input, :exception ])
-          end
-
-          index
-        end
-      end
-    end
-  end
-end

data/lib/chef/cookbook_site_streaming_uploader.rb

@@ -1,244 +0,0 @@
-#
-# Author:: Stanislav Vitvitskiy
-# Author:: Nuo Yan (nuo@chef.io)
-# Author:: Christopher Walters (<cw@chef.io>)
-# Copyright:: Copyright (c) Chef Software Inc.
-# License:: Apache License, Version 2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-autoload :URI, "uri"
-module Net
-  autoload :HTTP, "net/http"
-end
-autoload :OpenSSL, "openssl"
-module Mixlib
-  module Authentication
-    autoload :SignedHeaderAuth, "mixlib/authentication/signedheaderauth"
-  end
-end
-require "chef-utils/dist" unless defined?(ChefUtils::Dist)
-
-class Chef
-  # == Chef::CookbookSiteStreamingUploader
-  # A streaming multipart HTTP upload implementation. Used to upload cookbooks
-  # (in tarball form) to https://supermarket.chef.io
-  #
-  # inspired by http://stanislavvitvitskiy.blogspot.com/2008/12/multipart-post-in-ruby.html
-  class CookbookSiteStreamingUploader
-
-    DefaultHeaders = { "accept" => "application/json", "x-chef-version" => ::Chef::VERSION }.freeze # rubocop:disable Naming/ConstantName
-
-    class << self
-
-      def create_build_dir(cookbook)
-        tmp_cookbook_path = Tempfile.new("#{ChefUtils::Dist::Infra::SHORT}-#{cookbook.name}-build")
-        tmp_cookbook_path.close
-        tmp_cookbook_dir = tmp_cookbook_path.path
-        File.unlink(tmp_cookbook_dir)
-        FileUtils.mkdir_p(tmp_cookbook_dir)
-        Chef::Log.trace("Staging at #{tmp_cookbook_dir}")
-        checksums_to_on_disk_paths = cookbook.checksums
-        cookbook.each_file do |manifest_record|
-          path_in_cookbook = manifest_record[:path]
-          on_disk_path = checksums_to_on_disk_paths[manifest_record[:checksum]]
-          dest = File.join(tmp_cookbook_dir, cookbook.name.to_s, path_in_cookbook)
-          FileUtils.mkdir_p(File.dirname(dest))
-          Chef::Log.trace("Staging #{on_disk_path} to #{dest}")
-          FileUtils.cp(on_disk_path, dest)
-        end
-
-        # First, generate metadata
-        Chef::Log.trace("Generating metadata")
-        kcm = Chef::Knife::CookbookMetadata.new
-        kcm.config[:cookbook_path] = [ tmp_cookbook_dir ]
-        kcm.name_args = [ cookbook.name.to_s ]
-        kcm.run
-
-        tmp_cookbook_dir
-      end
-
-      def post(to_url, user_id, secret_key_filename, params = {}, headers = {})
-        make_request(:post, to_url, user_id, secret_key_filename, params, headers)
-      end
-
-      def put(to_url, user_id, secret_key_filename, params = {}, headers = {})
-        make_request(:put, to_url, user_id, secret_key_filename, params, headers)
-      end
-
-      def make_request(http_verb, to_url, user_id, secret_key_filename, params = {}, headers = {})
-        boundary = "----RubyMultipartClient" + rand(1000000).to_s + "ZZZZZ"
-        parts = []
-        content_file = nil
-
-        secret_key = OpenSSL::PKey::RSA.new(File.read(secret_key_filename))
-
-        unless params.nil? || params.empty?
-          params.each do |key, value|
-            if value.is_a?(File)
-              content_file = value
-              filepath = value.path
-              filename = File.basename(filepath)
-              parts << StringPart.new( "--" + boundary + "\r\n" +
-                                      "Content-Disposition: form-data; name=\"" + key.to_s + "\"; filename=\"" + filename + "\"\r\n" +
-                                      "Content-Type: application/octet-stream\r\n\r\n")
-              parts << StreamPart.new(value, File.size(filepath))
-              parts << StringPart.new("\r\n")
-            else
-              parts << StringPart.new( "--" + boundary + "\r\n" +
-                                      "Content-Disposition: form-data; name=\"" + key.to_s + "\"\r\n\r\n")
-              parts << StringPart.new(value.to_s + "\r\n")
-            end
-          end
-          parts << StringPart.new("--" + boundary + "--\r\n")
-        end
-
-        body_stream = MultipartStream.new(parts)
-
-        timestamp = Time.now.utc.iso8601
-
-        url = URI.parse(to_url)
-
-        Chef::Log.logger.debug("Signing: method: #{http_verb}, url: #{url}, file: #{content_file}, User-id: #{user_id}, Timestamp: #{timestamp}")
-
-        # We use the body for signing the request if the file parameter
-        # wasn't a valid file or wasn't included. Extract the body (with
-        # multi-part delimiters intact) to sign the request.
-        # TODO: tim: 2009-12-28: It'd be nice to remove this special case, and
-        # always hash the entire request body. In the file case it would just be
-        # expanded multipart text - the entire body of the POST.
-        content_body = parts.inject("") { |result, part| result + part.read(0, part.size) }
-        content_file.rewind if content_file # we consumed the file for the above operation, so rewind it.
-
-        signing_options = {
-          http_method: http_verb,
-          path: url.path,
-          user_id: user_id,
-          timestamp: timestamp }
-        (content_file && signing_options[:file] = content_file) || (signing_options[:body] = (content_body || ""))
-
-        headers.merge!(Mixlib::Authentication::SignedHeaderAuth.signing_object(signing_options).sign(secret_key))
-
-        content_file.rewind if content_file
-
-        # net/http doesn't like symbols for header keys, so we'll to_s each one just in case
-        headers = DefaultHeaders.merge(Hash[*headers.map { |k, v| [k.to_s, v] }.flatten])
-
-        req = case http_verb
-              when :put
-                Net::HTTP::Put.new(url.path, headers)
-              when :post
-                Net::HTTP::Post.new(url.path, headers)
-              end
-        req.content_length = body_stream.size
-        req.content_type = "multipart/form-data; boundary=" + boundary unless parts.empty?
-        req.body_stream = body_stream
-
-        http = Chef::HTTP::BasicClient.new(url).http_client
-        res = http.request(req)
-
-        # alias status to code and to_s to body for test purposes
-        # TODO: stop the following madness!
-        class << res
-          alias :to_s :body
-
-          # BUG this makes the response compatible with what response_steps expects to test headers (response.headers[] -> response[])
-          def headers # rubocop:disable Lint/NestedMethodDefinition
-            self
-          end
-
-          def status  # rubocop:disable Lint/NestedMethodDefinition
-            code.to_i
-          end
-        end
-        res
-      end
-
-    end
-
-    class StreamPart
-      def initialize(stream, size)
-        @stream, @size = stream, size
-      end
-
-      def size
-        @size
-      end
-
-      # read the specified amount from the stream
-      def read(offset, how_much)
-        @stream.read(how_much)
-      end
-    end
-
-    class StringPart
-      def initialize(str)
-        @str = str
-      end
-
-      def size
-        @str.length
-      end
-
-      # read the specified amount from the string starting at the offset
-      def read(offset, how_much)
-        @str[offset, how_much]
-      end
-    end
-
-    class MultipartStream
-      def initialize(parts)
-        @parts = parts
-        @part_no = 0
-        @part_offset = 0
-      end
-
-      def size
-        @parts.inject(0) { |size, part| size + part.size }
-      end
-
-      def read(how_much, dst_buf = nil)
-        if @part_no >= @parts.size
-          dst_buf.replace("") if dst_buf
-          return dst_buf
-        end
-
-        how_much_current_part = @parts[@part_no].size - @part_offset
-
-        how_much_current_part = if how_much_current_part > how_much
-                                  how_much
-                                else
-                                  how_much_current_part
-                                end
-
-        how_much_next_part = how_much - how_much_current_part
-
-        current_part = @parts[@part_no].read(@part_offset, how_much_current_part)
-
-        # recurse into the next part if the current one was not large enough
-        if how_much_next_part > 0
-          @part_no += 1
-          @part_offset = 0
-          next_part = read(how_much_next_part)
-          result = current_part + (next_part || "")
-        else
-          @part_offset += how_much_current_part
-          result = current_part
-        end
-        dst_buf ? dst_buf.replace(result || "") : result
-      end
-    end
-
-  end
-end