puppet 6.12.0 → 6.17.0

data/spec/integration/application/apply_spec.rb

@@ -1,8 +1,9 @@
 require 'spec_helper'
 require 'puppet_spec/files'
 require 'puppet_spec/compiler'
+require 'puppet_spec/https'
 
-describe "apply" do
+describe "apply", unless: Puppet::Util::Platform.jruby? do
   include PuppetSpec::Files
 
   before :each do
@@ -257,7 +258,7 @@ end
     expect(@logs.map(&:to_s)).to include(/{environment =>.*/)
   end
 
-  it "applies a given file even when an ENC is configured", :unless => Puppet::Util::Platform.windows? || RUBY_PLATFORM == 'java' do
+  it "applies a given file even when an ENC is configured", :unless => Puppet::Util::Platform.windows? || Puppet::Util::Platform.jruby? do
     manifest = file_containing("manifest.pp", "notice('specific manifest applied')")
     enc = script_containing('enc_script',
       :windows => '@echo classes: []' + "\n" + '@echo environment: special',
@@ -378,7 +379,7 @@ end
     # External node script execution will fail, likely due to the tampering
     # with the basic file descriptors.
     # Workaround: Define a log destination and merely inspect logs.
-    context "with an ENC", :unless => RUBY_PLATFORM == 'java' do
+    context "with an ENC" do
       let(:logdest) { tmpfile('logdest') }
       let(:args) { ['-e', execute, '--logdest', logdest ] }
       let(:enc) do
@@ -534,4 +535,132 @@ class amod::bad_type {
       end
     end
   end
+
+  context 'puppet file sources' do
+    let(:env_name) { 'dev' }
+    let(:env_dir) { File.join(Puppet[:environmentpath], env_name) }
+    let(:env) { Puppet::Node::Environment.create(env_name.to_sym, [File.join(env_dir, 'modules')]) }
+    let(:node) { Puppet::Node.new(Puppet[:certname], environment: environment) }
+    let(:apply) { Puppet::Application[:apply] }
+
+    before :each do
+      Puppet[:environment] = env_name
+      Puppet::FileSystem.mkpath(env_dir)
+    end
+
+    it "recursively copies a directory from a module" do
+      dir = File.join(env.full_modulepath, 'amod', 'files', 'dir1', 'dir2')
+      Puppet::FileSystem.mkpath(dir)
+      File.write(File.join(dir, 'file'), 'content from the module')
+
+      base_dir = tmpdir('apply_spec_base')
+      manifest = file_containing("manifest.pp", <<-MANIFEST)
+        file { "#{base_dir}/dir1":
+          ensure  => file,
+          source  => "puppet:///modules/amod/dir1",
+          recurse => true,
+        }
+      MANIFEST
+
+      expect {
+        apply.command_line.args << manifest
+         apply.run
+      }.to exit_with(0)
+       .and output(a_string_matching(
+         /dir1\]\/ensure: created/
+      ).and matching(
+         /dir1\/dir2\]\/ensure: created/
+      ).and matching(
+         /dir1\/dir2\/file\]\/ensure: defined content as '{md5}51f37efb13c3a1e486106f90db6490a5'/
+      )).to_stdout
+
+      dest_file = File.join(base_dir, 'dir1', 'dir2', 'file')
+      expect(File.read(dest_file)).to eq("content from the module")
+    end
+  end
+
+  context 'http report processor' do
+    include_context 'https client'
+
+    before :each do
+      Puppet[:reports] = 'http'
+    end
+
+    let(:apply) { Puppet::Application[:apply] }
+    let(:unknown_server) do
+      unknown_ca_cert = cert_fixture('unknown-ca.pem')
+      PuppetSpec::HTTPSServer.new(
+        ca_cert: unknown_ca_cert,
+        server_cert: cert_fixture('unknown-127.0.0.1.pem'),
+        server_key: key_fixture('unknown-127.0.0.1-key.pem')
+      )
+    end
+
+    it 'submits a report via reporturl' do
+      report = nil
+
+      response_proc = -> (req, res) {
+        report = Puppet::Transaction::Report.convert_from(:yaml, req.body)
+      }
+
+      https = PuppetSpec::HTTPSServer.new
+      https.start_server(response_proc: response_proc) do |https_port|
+        Puppet[:reporturl] = "https://127.0.0.1:#{https_port}/reports/upload"
+
+        expect {
+          apply.command_line.args = ['-e', 'notify { "hi": }']
+          apply.run
+        }.to exit_with(0)
+         .and output(/Applied catalog/).to_stdout
+
+        expect(report).to be_a(Puppet::Transaction::Report)
+        expect(report.resource_statuses['Notify[hi]']).to be_a(Puppet::Resource::Status)
+      end
+    end
+
+    it 'rejects an HTTPS report server whose root cert is not the puppet CA' do
+      unknown_server.start_server do |https_port|
+        Puppet[:reporturl] = "https://127.0.0.1:#{https_port}/reports/upload"
+
+        # processing the report happens after the transaction is finished,
+        # so we expect exit code 0, with a later failure on stderr
+        expect {
+          apply.command_line.args = ['-e', 'notify { "hi": }']
+          apply.run
+        }.to exit_with(0)
+         .and output(/Applied catalog/).to_stdout
+         .and output(/Report processor failed: certificate verify failed \[self signed certificate in certificate chain for CN=Unknown CA\]/).to_stderr
+      end
+    end
+
+    it 'accepts an HTTPS report servers whose cert is in the system CA store' do
+      Puppet[:report_include_system_store] = true
+      report = nil
+
+      response_proc = -> (req, res) {
+        report = Puppet::Transaction::Report.convert_from(:yaml, req.body)
+      }
+
+      # create a temp cacert bundle
+      ssl_file = tmpfile('systemstore')
+      File.write(ssl_file, unknown_server.ca_cert.to_pem)
+
+      unknown_server.start_server(response_proc: response_proc) do |https_port|
+        Puppet[:reporturl] = "https://127.0.0.1:#{https_port}/reports/upload"
+
+        # override path to system cacert bundle, this must be done before
+        # the SSLContext is created and the call to X509::Store.set_default_paths
+        Puppet::Util.withenv("SSL_CERT_FILE" => ssl_file) do
+          expect {
+            apply.command_line.args = ['-e', 'notify { "hi": }']
+            apply.run
+          }.to exit_with(0)
+           .and output(/Applied catalog/).to_stdout
+        end
+
+        expect(report).to be_a(Puppet::Transaction::Report)
+        expect(report.resource_statuses['Notify[hi]']).to be_a(Puppet::Resource::Status)
+      end
+    end
+  end
 end

data/spec/integration/application/filebucket_spec.rb

@@ -0,0 +1,190 @@
+require 'spec_helper'
+require 'puppet/face'
+require 'puppet_spec/puppetserver'
+require 'puppet_spec/files'
+
+describe "puppet filebucket", unless: Puppet::Util::Platform.jruby? do
+  include PuppetSpec::Files
+  include_context "https client"
+
+  let(:server) { PuppetSpec::Puppetserver.new }
+  let(:filebucket) { Puppet::Application[:filebucket] }
+  let(:backup_file) { tmpfile('backup_file') }
+
+  before :each do
+    Puppet[:log_level] = 'debug'
+  end
+
+  it "backs up files to the filebucket server" do
+    File.binwrite(backup_file, 'some random text')
+    md5 = Digest::MD5.file(backup_file).to_s
+
+    server.start_server do |port|
+      Puppet[:masterport] = port
+      expect {
+        filebucket.command_line.args << 'backup'
+        filebucket.command_line.args << backup_file
+        filebucket.run
+      }.to output(a_string_matching(
+        %r{Debug: HTTP HEAD https:\/\/127.0.0.1:#{port}\/puppet\/v3\/file_bucket_file\/md5\/#{md5}\/#{File.realpath(backup_file)}\?environment\=production returned 404 Not Found}
+      ).and matching(
+        %r{Debug: HTTP PUT https:\/\/127.0.0.1:#{port}\/puppet\/v3\/file_bucket_file\/md5\/#{md5}\/#{File.realpath(backup_file)}\?environment\=production returned 200 OK}
+      ).and matching(
+        %r{#{backup_file}: #{md5}}
+      )).to_stdout
+    end
+  end
+
+  it "doesn't attempt to back up file that already exists on the filebucket server" do
+    file_exists_handler = -> (req, res) {
+        res.status = 200
+    }
+
+    File.binwrite(backup_file, 'some random text')
+    md5 = Digest::MD5.file(backup_file).to_s
+
+    server.start_server(mounts: {filebucket: file_exists_handler}) do |port|
+      Puppet[:masterport] = port
+      expect {
+        filebucket.command_line.args << 'backup'
+        filebucket.command_line.args << backup_file
+        filebucket.run
+      }.to output(a_string_matching(
+        %r{Debug: HTTP HEAD https:\/\/127.0.0.1:#{port}\/puppet\/v3\/file_bucket_file\/md5\/#{md5}\/#{File.realpath(backup_file)}\?environment\=production returned 200 OK}
+      ).and matching(
+        %r{#{backup_file}: #{md5}}
+      )).to_stdout
+    end
+  end
+
+  it "downloads files from the filebucket server" do
+    get_handler = -> (req, res) {
+      res['Content-Type'] = 'application/octet-stream'
+      res.body = 'something to store'
+    }
+
+    server.start_server(mounts: {filebucket: get_handler}) do |port|
+      Puppet[:masterport] = port
+      expect {
+        filebucket.command_line.args << 'get'
+        filebucket.command_line.args << 'fac251367c9e083c6b1f0f3181'
+        filebucket.run
+      }.to output(a_string_matching(
+        %r{Debug: HTTP GET https:\/\/127.0.0.1:#{port}\/puppet\/v3\/file_bucket_file\/md5\/fac251367c9e083c6b1f0f3181\?environment\=production returned 200 OK}
+      ).and matching(
+        %r{something to store}
+       )).to_stdout
+    end
+  end
+
+  context 'diff', unless: Puppet::Util::Platform.windows? || Puppet::Util::Platform.jruby? do
+    context 'using a remote bucket' do
+      it 'outputs a diff between a local and remote file' do
+        File.binwrite(backup_file, "bar\nbaz")
+
+        get_handler = -> (req, res) {
+          res['Content-Type'] = 'application/octet-stream'
+          res.body = 'foo'
+        }
+
+        server.start_server(mounts: {filebucket: get_handler}) do |port|
+          Puppet[:masterport] = port
+          expect {
+            filebucket.command_line.args += ['diff', 'fac251367c9e083c6b1f0f3181', backup_file, '--remote']
+            filebucket.run
+          }.to output(a_string_matching(
+            /[-<] ?foo/
+          ).and matching(
+            /[+>] ?bar/
+          ).and matching(
+            /[+>] ?baz/
+          )).to_stdout
+        end
+      end
+
+      it 'outputs a diff between two remote files' do
+        get_handler = -> (req, res) {
+          res['Content-Type'] = 'application/octet-stream'
+          res.body = <<~END
+          --- /opt/puppetlabs/server/data/puppetserver/bucket/d/3/b/0/7/3/8/4/d3b07384d113edec49eaa6238ad5ff00/contents\t2020-04-06 21:25:24.892367570 +0000
+          +++ /opt/puppetlabs/server/data/puppetserver/bucket/9/9/b/9/9/9/2/0/99b999207e287afffc86c053e5693247/contents\t2020-04-06 21:26:13.603398063 +0000
+          @@ -1 +1,2 @@
+          -foo
+          +bar
+          +baz
+          END
+        }
+
+        server.start_server(mounts: {filebucket: get_handler}) do |port|
+          Puppet[:masterport] = port
+          expect {
+            filebucket.command_line.args += ['diff', 'd3b07384d113edec49eaa6238ad5ff00', "99b999207e287afffc86c053e5693247", '--remote']
+            filebucket.run
+          }.to output(a_string_matching(
+            /[-<] ?foo/
+          ).and matching(
+            /[+>] ?bar/
+          ).and matching(
+            /[+>] ?baz/
+          )).to_stdout
+        end
+      end
+    end
+
+    context 'using a local bucket' do
+      let(:filea) {
+        f = tmpfile('filea')
+        File.binwrite(f, 'foo')
+        f
+      }
+      let(:fileb) {
+        f = tmpfile('fileb')
+        File.binwrite(f, "bar\nbaz")
+        f
+      }
+      let(:checksuma) { Digest::MD5.file(filea).to_s }
+      let(:checksumb) { Digest::MD5.file(fileb).to_s }
+
+      it 'compares to files stored in a local bucket' do
+        expect {
+          filebucket.command_line.args = ['backup', filea, '--local']
+          filebucket.run
+        }.to output(/#{filea}: #{checksuma}/).to_stdout
+
+        expect{
+          filebucket.command_line.args = ['backup', fileb, '--local']
+          filebucket.run
+        }.to output(/#{fileb}: #{checksumb}\n/).to_stdout
+
+        expect {
+          filebucket.command_line.args = ['diff', checksuma, checksumb, '--local']
+          filebucket.run
+        }.to output(a_string_matching(
+          /[-<] ?foo/
+        ).and matching(
+          /[+>] ?bar/
+        ).and matching(
+          /[+>] ?baz/
+        )).to_stdout
+      end
+
+      it 'compares a file on the filesystem and a file stored in a local bucket' do
+        expect {
+          filebucket.command_line.args = ['backup', filea, '--local']
+          filebucket.run
+        }.to output(/#{filea}: #{checksuma}/).to_stdout
+
+        expect {
+          filebucket.command_line.args = ['diff', checksuma, fileb, '--local']
+          filebucket.run
+        }.to output(a_string_matching(
+          /[-<] ?foo/
+        ).and matching(
+          /[+>] ?bar/
+        ).and matching(
+          /[+>] ?baz/
+        )).to_stdout
+      end
+    end
+  end
+end

data/spec/integration/application/plugin_spec.rb

@@ -0,0 +1,73 @@
+require 'spec_helper'
+require 'puppet/face'
+require 'puppet_spec/puppetserver'
+
+describe "puppet plugin" do
+  include_context "https client"
+
+  let(:server) { PuppetSpec::Puppetserver.new }
+  let(:plugin) { Puppet::Application[:plugin] }
+  let(:response_body) { "[{\"path\":\"/etc/puppetlabs/code/environments/production/modules\",\"relative_path\":\".\",\"links\":\"follow\",\"owner\":0,\"group\":0,\"mode\":493,\"checksum\":{\"type\":\"ctime\",\"value\":\"{ctime}2020-03-06 20:14:25 UTC\"},\"type\":\"directory\",\"destination\":null}]" }
+
+  it "downloads from plugins, pluginsfacts and locales mounts" do
+    current_version_handler = -> (req, res) {
+      res['X-Puppet-Version'] = Puppet.version
+      res['Content-Type'] = 'application/json'
+      res.body = response_body
+    }
+
+    server.start_server(mounts: {file_metadatas: current_version_handler}) do |port|
+      Puppet[:masterport] = port
+      expect {
+        plugin.command_line.args << 'download'
+        plugin.run
+      }.to exit_with(0)
+       .and output(matching(
+         "Downloaded these plugins: #{Regexp.escape(Puppet[:pluginfactdest])}, #{Regexp.escape(Puppet[:plugindest])}, #{Regexp.escape(Puppet[:localedest])}"
+       )).to_stdout
+    end
+  end
+
+  it "downloads from plugins and pluginsfacts from older puppetservers" do
+    no_locales_handler = -> (req, res) {
+      res['X-Puppet-Version'] = '5.3.3' # locales mount was added in 5.3.4
+      res['Content-Type'] = 'application/json'
+      res.body = response_body
+    }
+
+    server.start_server(mounts: {file_metadatas: no_locales_handler}) do |port|
+      Puppet[:masterport] = port
+      expect {
+        plugin.command_line.args << 'download'
+        plugin.run
+      }.to exit_with(0)
+       .and output(matching(
+         "Downloaded these plugins: #{Regexp.escape(Puppet[:pluginfactdest])}, #{Regexp.escape(Puppet[:plugindest])}"
+       )).to_stdout
+    end
+  end
+
+  it "downloads from an environment that doesn't exist locally" do
+    requested_environment = nil
+
+    current_version_handler = -> (req, res) {
+      res['X-Puppet-Version'] = Puppet.version
+      res['Content-Type'] = 'application/json'
+      res.body = response_body
+      requested_environment = req.query['environment']
+    }
+
+    server.start_server(mounts: {file_metadatas: current_version_handler}) do |port|
+      Puppet[:environment] = 'doesnotexistontheagent'
+      Puppet[:masterport] = port
+      expect {
+        plugin.command_line.args << 'download'
+        plugin.run
+      }.to exit_with(0)
+       .and output(matching("Downloaded these plugins")).to_stdout
+
+      expect(requested_environment).to eq('doesnotexistontheagent')
+    end
+  end
+
+end

data/spec/integration/configurer_spec.rb

@@ -70,18 +70,26 @@ describe Puppet::Configurer do
       end
 
       context 'when resubmit_facts is set to true' do
-        let(:test_facts) { Puppet::Node::Facts.new('configurer.test') }
-        let(:fact_rest_terminus) { Puppet::Node::Facts.indirection.terminus(:rest) }
+        let(:test_facts) { Puppet::Node::Facts.new('configurer.test', {test_fact: 'test value'}) }
 
         before(:each) do
           Puppet[:resubmit_facts] = true
 
           allow(@configurer).to receive(:find_facts).and_return(test_facts)
-          allow(fact_rest_terminus).to receive(:save)
         end
 
-        it 'sends fact data using the rest terminus' do
-          expect(fact_rest_terminus).to receive(:save)
+        it 'uploads facts as application/json' do
+          stub_request(:put, "https://puppet:8140/puppet/v3/facts/configurer.test?environment=production").
+            with(
+              body: hash_including(
+                {
+                  "name" => "configurer.test",
+                  "values" => {"test_fact" => 'test value',},
+                }),
+              headers: {
+                'Accept'=>'application/json, application/x-msgpack, text/pson',
+                'Content-Type'=>'application/json',
+              })
 
           @configurer.run(catalog: @catalog)
         end
@@ -95,8 +103,8 @@ describe Puppet::Configurer do
         end
 
         it 'logs errors that occur during fact submission' do
-          allow(fact_rest_terminus).to receive(:save).and_raise('error sending facts')
-          expect(Puppet).to receive(:log_exception).with(instance_of(RuntimeError),
+          stub_request(:put, "https://puppet:8140/puppet/v3/facts/configurer.test?environment=production").to_return(status: 502)
+          expect(Puppet).to receive(:log_exception).with(Puppet::HTTP::ResponseError,
                                                          /^Failed to submit facts/)
 
           @configurer.run(catalog: @catalog)
@@ -105,6 +113,17 @@ describe Puppet::Configurer do
         it 'records time spent resubmitting facts' do
           report = Puppet::Transaction::Report.new
 
+          stub_request(:put, "https://puppet:8140/puppet/v3/facts/configurer.test?environment=production").
+            with(
+              body: hash_including({
+                "name" => "configurer.test",
+                "values" => {"test_fact": "test value"},
+              }),
+              headers: {
+                'Accept'=>'application/json, application/x-msgpack, text/pson',
+                'Content-Type'=>'application/json',
+              }).to_return(status: 200)
+
           @configurer.run(catalog: @catalog, report: report)
 
           expect(report.metrics['time'].values).to include(["resubmit_facts", anything, Numeric])