chef 16.18.30 → 17.10.163

data/spec/unit/chef_fs/parallelizer_spec.rb

@@ -1,479 +0,0 @@
-require "spec_helper"
-require "chef/chef_fs/parallelizer"
-
-# FIXME: these are disabled on MacOS due to timing issues in our anka build cluster
-# these issues should be fixed and the tests should be re-eenabled.  If we are getting
-# omnibus test phases on mac tests which are reasonable and not ~3 hours long, then the
-# condition to avoid this testing on macs can be deleted
-describe Chef::ChefFS::Parallelizer, :not_supported_on_macos do
-  before :each do
-    @start_time = Time.now
-  end
-
-  def elapsed_time
-    Time.now - @start_time
-  end
-
-  after :each do
-    parallelizer.kill
-  end
-
-  context "With a Parallelizer with 5 threads" do
-    let :parallelizer do
-      Chef::ChefFS::Parallelizer.new(5)
-    end
-
-    def parallelize(inputs, options = {}, &block)
-      parallelizer.parallelize(inputs, { main_thread_processing: false }.merge(options), &block)
-    end
-
-    it "parallel_do creates unordered output as soon as it is available" do
-      outputs = []
-      parallelizer.parallel_do([0.5, 0.3, 0.1]) do |val|
-        sleep val
-        outputs << val
-      end
-      expect(elapsed_time).to be < 0.6
-      expect(outputs).to eq([ 0.1, 0.3, 0.5 ])
-    end
-
-    context "With :ordered => false (unordered output)" do
-      it "An empty input produces an empty output" do
-        expect(parallelize([], ordered: false) do
-          sleep 10
-        end.to_a).to eql([])
-        expect(elapsed_time).to be < 0.1
-      end
-
-      it "10 sleep(0.2)s complete within 0.5 seconds" do
-        expect(parallelize(1.upto(10), ordered: false) do |i|
-          sleep 0.2
-          "x"
-        end.to_a).to eq(%w{x x x x x x x x x x})
-        expect(elapsed_time).to be < 0.5
-      end
-
-      it "The output comes as soon as it is available" do
-        enum = parallelize([0.5, 0.3, 0.1], ordered: false) do |val|
-          sleep val
-          val
-        end
-        expect(enum.map do |value|
-          expect(elapsed_time).to be < value + 0.1
-          value
-        end).to eq([ 0.1, 0.3, 0.5 ])
-      end
-
-      it "An exception in input is passed through but does NOT stop processing" do
-        input = TestEnumerable.new(0.5, 0.3, 0.1) do
-          raise "hi"
-        end
-        enum = parallelize(input, ordered: false) { |x| sleep(x); x }
-        results = []
-        expect { enum.each { |value| results << value } }.to raise_error "hi"
-        expect(results).to eq([ 0.1, 0.3, 0.5 ])
-        expect(elapsed_time).to be < 0.6
-      end
-
-      it "Exceptions in output are raised after all processing is done" do
-        processed = 0
-        enum = parallelize([1, 2, "x", 3], ordered: false) do |x|
-          if x == "x"
-            sleep 0.1
-            raise "hi"
-          end
-          sleep 0.2
-          processed += 1
-          x
-        end
-        results = []
-        expect { enum.each { |value| results << value } }.to raise_error "hi"
-        expect(results.sort).to eq([ 1, 2, 3 ])
-        expect(elapsed_time).to be < 0.3
-        expect(processed).to eq(3)
-      end
-
-      it "Exceptions with :stop_on_exception are raised after all processing is done" do
-        processed = 0
-        parallelized = parallelize([0.3, 0.3, "x", 0.3, 0.3, 0.3, 0.3, 0.3], ordered: false, stop_on_exception: true) do |x|
-          if x == "x"
-            sleep(0.1)
-            raise "hi"
-          end
-          sleep(x)
-          processed += 1
-          x
-        end
-        expect { parallelized.to_a }.to raise_error "hi"
-        expect(processed).to eq(4)
-      end
-    end
-
-    context "With :ordered => true (ordered output)" do
-      it "An empty input produces an empty output" do
-        expect(parallelize([]) do
-          sleep 10
-        end.to_a).to eql([])
-        expect(elapsed_time).to be < 0.1
-      end
-
-      it "10 sleep(0.2)s complete within 0.5 seconds" do
-        expect(parallelize(1.upto(10), ordered: true) do |i|
-          sleep 0.2
-          "x"
-        end.to_a).to eq(%w{x x x x x x x x x x})
-        expect(elapsed_time).to be < 0.5
-      end
-
-      it "Output comes in the order of the input" do
-        enum = parallelize([0.5, 0.3, 0.1]) do |val|
-          sleep val
-          val
-        end.enum_for(:each_with_index)
-        expect(enum.next).to eq([ 0.5, 0 ])
-        expect(enum.next).to eq([ 0.3, 1 ])
-        expect(enum.next).to eq([ 0.1, 2 ])
-        expect(elapsed_time).to be < 0.6
-      end
-
-      it "Exceptions in input are raised in the correct sequence but do NOT stop processing" do
-        input = TestEnumerable.new(0.5, 0.3, 0.1) do
-          raise "hi"
-        end
-        results = []
-        enum = parallelize(input) { |x| sleep(x); x }
-        expect { enum.each { |value| results << value } }.to raise_error "hi"
-        expect(elapsed_time).to be < 0.6
-        expect(results).to eq([ 0.5, 0.3, 0.1 ])
-      end
-
-      it "Exceptions in output are raised in the correct sequence and running processes do NOT stop processing" do
-        processed = 0
-        enum = parallelize([1, 2, "x", 3]) do |x|
-          if x == "x"
-            sleep(0.1)
-            raise "hi"
-          end
-          sleep(0.2)
-          processed += 1
-          x
-        end
-        results = []
-        expect { enum.each { |value| results << value } }.to raise_error "hi"
-        expect(results).to eq([ 1, 2 ])
-        expect(elapsed_time).to be < 0.3
-        expect(processed).to eq(3)
-      end
-
-      it "Exceptions with :stop_on_exception are raised after all processing is done" do
-        processed = 0
-        parallelized = parallelize([0.3, 0.3, "x", 0.3, 0.3, 0.3, 0.3, 0.3], ordered: false, stop_on_exception: true) do |x|
-          if x == "x"
-            sleep(0.1)
-            raise "hi"
-          end
-          sleep(x)
-          processed += 1
-          x
-        end
-        expect { parallelized.to_a }.to raise_error "hi"
-        expect(processed).to eq(4)
-      end
-    end
-
-    it "When the input is slow, output still proceeds" do
-      input = TestEnumerable.new do |&block|
-        block.call(1)
-        sleep 0.1
-        block.call(2)
-        sleep 0.1
-        block.call(3)
-        sleep 0.1
-      end
-      enum = parallelize(input) { |x| x }
-      expect(enum.map do |value|
-        expect(elapsed_time).to be < (value + 1) * 0.1
-        value
-      end).to eq([ 1, 2, 3 ])
-    end
-  end
-
-  context "With a Parallelizer with 1 thread" do
-    let :parallelizer do
-      Chef::ChefFS::Parallelizer.new(1)
-    end
-
-    context "when the thread is occupied with a job" do
-      before :each do
-        parallelizer
-        started = false
-        @occupying_job_finished = occupying_job_finished = [ false ]
-        @thread = Thread.new do
-          parallelizer.parallelize([0], main_thread_processing: false) do |x|
-            started = true
-            sleep(0.3)
-            occupying_job_finished[0] = true
-          end.wait
-        end
-        sleep(0.01) until started
-      end
-
-      after :each do
-        Thread.kill(@thread)
-      end
-
-      it "parallelize with :main_thread_processing = true does not block" do
-        expect(parallelizer.parallelize([1]) do |x|
-          sleep(0.1)
-          x
-        end.to_a).to eq([ 1 ])
-        expect(elapsed_time).to be < 0.2
-      end
-
-      it "parallelize with :main_thread_processing = false waits for the job to finish" do
-        expect(parallelizer.parallelize([1], main_thread_processing: false) do |x|
-          sleep(0.1)
-          x + 1
-        end.to_a).to eq([ 2 ])
-        expect(elapsed_time).to be > 0.3
-      end
-
-      it "resizing the Parallelizer to 0 waits for the job to stop" do
-        expect(elapsed_time).to be < 0.2
-        parallelizer.resize(0)
-        expect(parallelizer.num_threads).to eq(0)
-        expect(elapsed_time).to be > 0.25
-        expect(@occupying_job_finished).to eq([ true ])
-      end
-
-      it "stopping the Parallelizer waits for the job to finish" do
-        expect(elapsed_time).to be < 0.2
-        parallelizer.stop
-        expect(parallelizer.num_threads).to eq(0)
-        expect(elapsed_time).to be > 0.25
-        expect(@occupying_job_finished).to eq([ true ])
-      end
-
-      it "resizing the Parallelizer to 2 does not stop the job" do
-        expect(elapsed_time).to be < 0.2
-        parallelizer.resize(2)
-        expect(parallelizer.num_threads).to eq(2)
-        expect(elapsed_time).to be < 0.2
-        sleep(0.3)
-        expect(@occupying_job_finished).to eq([ true ])
-      end
-    end
-
-    context "enumerable methods should run efficiently" do
-      it ".count does not process anything" do
-        outputs_processed = 0
-        input_mapper = TestEnumerable.new(1, 2, 3, 4, 5, 6)
-        enum = parallelizer.parallelize(input_mapper) do |x|
-          outputs_processed += 1
-          sleep(0.05) # Just enough to yield and get other inputs in the queue
-          x
-        end
-        expect(enum.count).to eq(6)
-        expect(outputs_processed).to eq(0)
-        expect(input_mapper.num_processed).to eq(6)
-      end
-
-      it ".count with arguments works normally" do
-        outputs_processed = 0
-        input_mapper = TestEnumerable.new(1, 1, 1, 1, 2, 2, 2, 3, 3, 4)
-        enum = parallelizer.parallelize(input_mapper) do |x|
-          outputs_processed += 1
-          x
-        end
-        expect(enum.count { |x| x > 1 }).to eq(6)
-        expect(enum.count(2)).to eq(3)
-        expect(outputs_processed).to eq(20)
-        expect(input_mapper.num_processed).to eq(20)
-      end
-
-      it ".first does not enumerate anything other than the first result(s)" do
-        outputs_processed = 0
-        input_mapper = TestEnumerable.new(1, 2, 3, 4, 5, 6)
-        enum = parallelizer.parallelize(input_mapper) do |x|
-          outputs_processed += 1
-          sleep(0.05) # Just enough to yield and get other inputs in the queue
-          x
-        end
-        expect(enum.first).to eq(1)
-        expect(enum.first(2)).to eq([1, 2])
-        expect(outputs_processed).to eq(3)
-        expect(input_mapper.num_processed).to eq(3)
-      end
-
-      it ".take does not enumerate anything other than the first result(s)" do
-        outputs_processed = 0
-        input_mapper = TestEnumerable.new(1, 2, 3, 4, 5, 6)
-        enum = parallelizer.parallelize(input_mapper) do |x|
-          outputs_processed += 1
-          sleep(0.05) # Just enough to yield and get other inputs in the queue
-          x
-        end
-        expect(enum.take(2)).to eq([1, 2])
-        expect(outputs_processed).to eq(2)
-        expect(input_mapper.num_processed).to eq(2)
-      end
-
-      it ".drop does not process anything other than the last result(s)" do
-        outputs_processed = 0
-        input_mapper = TestEnumerable.new(1, 2, 3, 4, 5, 6)
-        enum = parallelizer.parallelize(input_mapper) do |x|
-          outputs_processed += 1
-          sleep(0.05) # Just enough to yield and get other inputs in the queue
-          x
-        end
-        expect(enum.drop(2)).to eq([3, 4, 5, 6])
-        expect(outputs_processed).to eq(4)
-        expect(input_mapper.num_processed).to eq(6)
-      end
-
-      if Enumerable.method_defined?(:lazy)
-        it ".lazy.take does not enumerate anything other than the first result(s)" do
-          outputs_processed = 0
-          input_mapper = TestEnumerable.new(1, 2, 3, 4, 5, 6)
-          enum = parallelizer.parallelize(input_mapper) do |x|
-            outputs_processed += 1
-            sleep(0.05) # Just enough to yield and get other inputs in the queue
-            x
-          end
-          expect(enum.lazy.take(2).to_a).to eq([1, 2])
-          expect(outputs_processed).to eq(2)
-          expect(input_mapper.num_processed).to eq(2)
-        end
-
-        it ".drop does not process anything other than the last result(s)" do
-          outputs_processed = 0
-          input_mapper = TestEnumerable.new(1, 2, 3, 4, 5, 6)
-          enum = parallelizer.parallelize(input_mapper) do |x|
-            outputs_processed += 1
-            sleep(0.05) # Just enough to yield and get other inputs in the queue
-            x
-          end
-          expect(enum.lazy.drop(2).to_a).to eq([3, 4, 5, 6])
-          expect(outputs_processed).to eq(4)
-          expect(input_mapper.num_processed).to eq(6)
-        end
-
-        it "lazy enumerable is actually lazy" do
-          outputs_processed = 0
-          input_mapper = TestEnumerable.new(1, 2, 3, 4, 5, 6)
-          enum = parallelizer.parallelize(input_mapper) do |x|
-            outputs_processed += 1
-            sleep(0.05) # Just enough to yield and get other inputs in the queue
-            x
-          end
-          enum.lazy.take(2)
-          enum.lazy.drop(2)
-          sleep(0.1)
-          expect(outputs_processed).to eq(0)
-          expect(input_mapper.num_processed).to eq(0)
-        end
-      end
-    end
-
-    context "running enumerable multiple times should function correctly" do
-      it ".map twice on the same parallel enumerable returns the correct results and re-processes the input" do
-        outputs_processed = 0
-        input_mapper = TestEnumerable.new(1, 2, 3)
-        enum = parallelizer.parallelize(input_mapper) do |x|
-          outputs_processed += 1
-          x
-        end
-        expect(enum.map { |x| x }).to eq([1, 2, 3])
-        expect(enum.map { |x| x }).to eq([1, 2, 3])
-        expect(outputs_processed).to eq(6)
-        expect(input_mapper.num_processed).to eq(6)
-      end
-
-      it ".first and then .map on the same parallel enumerable returns the correct results and re-processes the input" do
-        outputs_processed = 0
-        input_mapper = TestEnumerable.new(1, 2, 3)
-        enum = parallelizer.parallelize(input_mapper) do |x|
-          outputs_processed += 1
-          x
-        end
-        expect(enum.first).to eq(1)
-        expect(enum.map { |x| x }).to eq([1, 2, 3])
-        expect(outputs_processed).to be >= 4
-        expect(input_mapper.num_processed).to be >= 4
-      end
-
-      it "two simultaneous enumerations throws an exception" do
-        enum = parallelizer.parallelize([1, 2, 3]) { |x| x }
-        a = enum.enum_for(:each)
-        a.next
-        expect do
-          b = enum.enum_for(:each)
-          b.next
-        end.to raise_error(RuntimeError, "each() called on parallel enumerable twice simultaneously! Bad mojo")
-      end
-    end
-  end
-
-  context "With a Parallelizer with 0 threads" do
-    let :parallelizer do
-      Chef::ChefFS::Parallelizer.new(0)
-    end
-
-    context "And main_thread_processing on" do
-      it "succeeds in running" do
-        expect(parallelizer.parallelize([0.5]) { |x| x * 2 }.to_a).to eq([1])
-      end
-    end
-  end
-
-  context "With a Parallelizer with 10 threads" do
-    let :parallelizer do
-      Chef::ChefFS::Parallelizer.new(10)
-    end
-
-    it "does not have contention issues with large numbers of inputs" do
-      expect(parallelizer.parallelize(1.upto(500)) { |x| x + 1 }.to_a).to eq(2.upto(501).to_a)
-    end
-
-    it "does not have contention issues with large numbers of inputs with ordering off" do
-      expect(parallelizer.parallelize(1.upto(500), ordered: false) { |x| x + 1 }.to_a.sort).to eq(2.upto(501).to_a)
-    end
-
-    it "does not have contention issues with large numbers of jobs and inputs with ordering off" do
-      parallelizers = 0.upto(99).map do
-        parallelizer.parallelize(1.upto(500)) { |x| x + 1 }
-      end
-      outputs = []
-      threads = 0.upto(99).map do |i|
-        Thread.new { outputs[i] = parallelizers[i].to_a }
-      end
-      threads.each(&:join)
-      outputs.each { |output| expect(output.sort).to eq(2.upto(501).to_a) }
-    end
-  end
-
-  class TestEnumerable
-    include Enumerable
-
-    def initialize(*values, &block)
-      @values = values
-      @block = block
-      @num_processed = 0
-    end
-
-    attr_reader :num_processed
-
-    def each
-      @values.each do |value|
-        @num_processed += 1
-        yield(value)
-      end
-      if @block
-        @block.call do |value|
-          @num_processed += 1
-          yield(value)
-        end
-      end
-    end
-  end
-end

data/spec/unit/cookbook_site_streaming_uploader_spec.rb

@@ -1,198 +0,0 @@
-#
-# Author:: Xabier de Zuazo (xabier@onddo.com)
-# Copyright:: Copyright 2013-2016, Onddo Labs, SL.
-# License:: Apache License, Version 2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-require "spec_helper"
-
-require "chef/cookbook_site_streaming_uploader"
-
-class FakeTempfile
-  def initialize(basename)
-    @basename = basename
-  end
-
-  def close; end
-
-  def path
-    "#{@basename}.ZZZ"
-  end
-
-end
-
-describe Chef::CookbookSiteStreamingUploader do
-
-  describe "create_build_dir" do
-
-    before(:each) do
-      @cookbook_repo = File.expand_path(File.join(CHEF_SPEC_DATA, "cookbooks"))
-      @loader = Chef::CookbookLoader.new(@cookbook_repo)
-      @loader.load_cookbooks
-      allow(File).to receive(:unlink)
-    end
-
-    it "should create the cookbook tmp dir" do
-      cookbook = @loader[:openldap]
-      files_count = Dir.glob(File.join(@cookbook_repo, cookbook.name.to_s, "**", "*"), File::FNM_DOTMATCH).count { |file| File.file?(file) }
-
-      expect(Tempfile).to receive(:new).with("chef-#{cookbook.name}-build").and_return(FakeTempfile.new("chef-#{cookbook.name}-build"))
-      expect(FileUtils).to receive(:mkdir_p).exactly(files_count + 1).times
-      expect(FileUtils).to receive(:cp).exactly(files_count).times
-      Chef::CookbookSiteStreamingUploader.create_build_dir(cookbook)
-    end
-
-  end # create_build_dir
-
-  describe "make_request" do
-
-    before(:each) do
-      @uri = "http://cookbooks.dummy.com/api/v1/cookbooks"
-      @secret_filename = File.join(CHEF_SPEC_DATA, "ssl/private_key.pem")
-      @rsa_key = File.read(@secret_filename)
-      response = Net::HTTPResponse.new("1.0", "200", "OK")
-      allow_any_instance_of(Net::HTTP).to receive(:request).and_return(response)
-    end
-
-    it "should send an http request" do
-      expect_any_instance_of(Net::HTTP).to receive(:request)
-      Chef::CookbookSiteStreamingUploader.make_request(:post, @uri, "bill", @secret_filename)
-    end
-
-    it "should read the private key file" do
-      expect(File).to receive(:read).with(@secret_filename).and_return(@rsa_key)
-      Chef::CookbookSiteStreamingUploader.make_request(:post, @uri, "bill", @secret_filename)
-    end
-
-    it "should add the authentication signed header" do
-      expect_any_instance_of(Mixlib::Authentication::SigningObject).to receive(:sign).and_return({})
-      Chef::CookbookSiteStreamingUploader.make_request(:post, @uri, "bill", @secret_filename)
-    end
-
-    it "should be able to send post requests" do
-      post = Net::HTTP::Post.new(@uri, {})
-
-      expect(Net::HTTP::Post).to receive(:new).once.and_return(post)
-      expect(Net::HTTP::Put).not_to receive(:new)
-      expect(Net::HTTP::Get).not_to receive(:new)
-      Chef::CookbookSiteStreamingUploader.make_request(:post, @uri, "bill", @secret_filename)
-    end
-
-    it "should be able to send put requests" do
-      put = Net::HTTP::Put.new(@uri, {})
-
-      expect(Net::HTTP::Post).not_to receive(:new)
-      expect(Net::HTTP::Put).to receive(:new).once.and_return(put)
-      expect(Net::HTTP::Get).not_to receive(:new)
-      Chef::CookbookSiteStreamingUploader.make_request(:put, @uri, "bill", @secret_filename)
-    end
-
-    it "should be able to receive files to attach as argument" do
-      Chef::CookbookSiteStreamingUploader.make_request(:put, @uri, "bill", @secret_filename, {
-        myfile: File.new(File.join(CHEF_SPEC_DATA, "config.rb")), # a dummy file
-      })
-    end
-
-    it "should be able to receive strings to attach as argument" do
-      Chef::CookbookSiteStreamingUploader.make_request(:put, @uri, "bill", @secret_filename, {
-        mystring: "Lorem ipsum",
-      })
-    end
-
-    it "should be able to receive strings and files as argument at the same time" do
-      Chef::CookbookSiteStreamingUploader.make_request(:put, @uri, "bill", @secret_filename, {
-        myfile1: File.new(File.join(CHEF_SPEC_DATA, "config.rb")),
-        mystring1: "Lorem ipsum",
-        myfile2: File.new(File.join(CHEF_SPEC_DATA, "config.rb")),
-        mystring2: "Dummy text",
-      })
-    end
-
-  end # make_request
-
-  describe "StreamPart" do
-    before(:each) do
-      @file = File.new(File.join(CHEF_SPEC_DATA, "config.rb"))
-      @stream_part = Chef::CookbookSiteStreamingUploader::StreamPart.new(@file, File.size(@file))
-    end
-
-    it "should create a StreamPart" do
-      expect(@stream_part).to be_instance_of(Chef::CookbookSiteStreamingUploader::StreamPart)
-    end
-
-    it "should expose its size" do
-      expect(@stream_part.size).to eql(File.size(@file))
-    end
-
-    it "should read with offset and how_much" do
-      content = @file.read(4)
-      @file.rewind
-      expect(@stream_part.read(0, 4)).to eql(content)
-    end
-
-  end # StreamPart
-
-  describe "StringPart" do
-    before(:each) do
-      @str = "What a boring string"
-      @string_part = Chef::CookbookSiteStreamingUploader::StringPart.new(@str)
-    end
-
-    it "should create a StringPart" do
-      expect(@string_part).to be_instance_of(Chef::CookbookSiteStreamingUploader::StringPart)
-    end
-
-    it "should expose its size" do
-      expect(@string_part.size).to eql(@str.size)
-    end
-
-    it "should read with offset and how_much" do
-      expect(@string_part.read(2, 4)).to eql(@str[2, 4])
-    end
-
-  end # StringPart
-
-  describe "MultipartStream" do
-    before(:each) do
-      @string1 = "stream1"
-      @string2 = "stream2"
-      @stream1 = Chef::CookbookSiteStreamingUploader::StringPart.new(@string1)
-      @stream2 = Chef::CookbookSiteStreamingUploader::StringPart.new(@string2)
-      @parts = [ @stream1, @stream2 ]
-
-      @multipart_stream = Chef::CookbookSiteStreamingUploader::MultipartStream.new(@parts)
-    end
-
-    it "should create a MultipartStream" do
-      expect(@multipart_stream).to be_instance_of(Chef::CookbookSiteStreamingUploader::MultipartStream)
-    end
-
-    it "should expose its size" do
-      expect(@multipart_stream.size).to eql(@stream1.size + @stream2.size)
-    end
-
-    it "should read with how_much" do
-      expect(@multipart_stream.read(10)).to eql("#{@string1}#{@string2}"[0, 10])
-    end
-
-    it "should read receiving destination buffer as second argument (CHEF-4456: Ruby 2 compat)" do
-      dst_buf = ""
-      @multipart_stream.read(10, dst_buf)
-      expect(dst_buf).to eql("#{@string1}#{@string2}"[0, 10])
-    end
-
-  end # MultipartStream
-
-end