puppet 4.10.0 → 5.0.0

data/spec/unit/util_spec.rb

@@ -5,6 +5,12 @@ require 'spec_helper'
 describe Puppet::Util do
   include PuppetSpec::Files
 
+  # Discriminator for tests that attempts to unset HOME since that, for reasons currently unknown,
+  # doesn't work in Ruby >= 2.4.0
+  def self.gte_ruby_2_4
+    @gte_ruby_2_4 ||= SemanticPuppet::Version.parse(RUBY_VERSION) >= SemanticPuppet::Version.parse('2.4.0')
+  end
+
   if Puppet.features.microsoft_windows?
     def set_mode(mode, file)
       Puppet::Util::Windows::Security.set_mode(mode, file)
@@ -297,6 +303,14 @@ describe Puppet::Util do
   end
 
   describe "#path_to_uri" do
+    # different UTF-8 widths
+    # 1-byte A
+    # 2-byte ۿ - http://www.fileformat.info/info/unicode/char/06ff/index.htm - 0xDB 0xBF / 219 191
+    # 3-byte ᚠ - http://www.fileformat.info/info/unicode/char/16A0/index.htm - 0xE1 0x9A 0xA0 / 225 154 160
+    # 4-byte ܎ - http://www.fileformat.info/info/unicode/char/2070E/index.htm - 0xF0 0xA0 0x9C 0x8E / 240 160 156 142
+    let (:mixed_utf8) { "A\u06FF\u16A0\u{2070E}" } # Aۿᚠ܎
+    let (:mixed_utf8_urlencoded) { "A%DB%BF%E1%9A%A0%F0%A0%9C%8E" }
+
     %w[. .. foo foo/bar foo/../bar].each do |path|
       it "should reject relative path: #{path}" do
         expect { Puppet::Util.path_to_uri(path) }.to raise_error(Puppet::Error)
@@ -307,6 +321,42 @@ describe Puppet::Util do
       expect(Puppet::Util.path_to_uri("/foo bar").path).to eq("/foo%20bar")
     end
 
+    it "should properly URI encode + and space in path" do
+      expect(Puppet::Util.path_to_uri("/foo+foo bar").path).to eq("/foo+foo%20bar")
+    end
+
+    # reserved characters are different for each part
+    # https://web.archive.org/web/20151229061347/http://blog.lunatech.com/2009/02/03/what-every-web-developer-must-know-about-url-encoding#Thereservedcharactersaredifferentforeachpart
+    # "?" is allowed unescaped anywhere within a query part,
+    # "/" is allowed unescaped anywhere within a query part,
+    # "=" is allowed unescaped anywhere within a path parameter or query parameter value, and within a path segment,
+    # ":@-._~!$&'()*+,;=" are allowed unescaped anywhere within a path segment part,
+    # "/?:@-._~!$&'()*+,;=" are allowed unescaped anywhere within a fragment part.
+    it "should properly URI encode + and space in path and query" do
+      path = "/foo+foo bar?foo+foo bar"
+      uri = Puppet::Util.path_to_uri(path)
+
+      # Ruby 1.9.3 URI#to_s has a bug that returns ASCII always
+      # despite parts being UTF-8 strings
+      expected_encoding = RUBY_VERSION == '1.9.3' ? Encoding::ASCII : Encoding::UTF_8
+
+      expect(uri.to_s.encoding).to eq(expected_encoding)
+      expect(uri.path).to eq("/foo+foo%20bar")
+      # either + or %20 is correct for an encoded space in query
+      # + is usually used for backward compatibility, but %20 is preferred for compat with Uri.unescape
+      expect(uri.query).to eq("foo%2Bfoo%20bar")
+      # complete roundtrip
+      expect(URI.unescape(uri.to_s)).to eq("file:#{path}")
+      expect(URI.unescape(uri.to_s).encoding).to eq(expected_encoding)
+    end
+
+    it "should perform UTF-8 URI escaping" do
+      uri = Puppet::Util.path_to_uri("/#{mixed_utf8}")
+
+      expect(uri.path.encoding).to eq(Encoding::UTF_8)
+      expect(uri.path).to eq("/#{mixed_utf8_urlencoded}")
+    end
+
     describe "when using platform :posix" do
       before :each do
         Puppet.features.stubs(:posix).returns true
@@ -340,7 +390,172 @@ describe Puppet::Util do
         it "should convert UNC #{path} to absolute URI" do
           uri = Puppet::Util.path_to_uri("\\\\server\\#{path}")
           expect(uri.host).to eq('server')
-          expect(uri.path).to eq('/' + path)
+          expect(uri.path).to eq('/' + Puppet::Util.uri_encode(path))
+        end
+      end
+    end
+  end
+
+  describe "#uri_query_encode" do
+    # different UTF-8 widths
+    # 1-byte A
+    # 2-byte ۿ - http://www.fileformat.info/info/unicode/char/06ff/index.htm - 0xDB 0xBF / 219 191
+    # 3-byte ᚠ - http://www.fileformat.info/info/unicode/char/16A0/index.htm - 0xE1 0x9A 0xA0 / 225 154 160
+    # 4-byte 𠜎 - http://www.fileformat.info/info/unicode/char/2070E/index.htm - 0xF0 0xA0 0x9C 0x8E / 240 160 156 142
+    let (:mixed_utf8) { "A\u06FF\u16A0\u{2070E}" } # Aۿᚠ𠜎
+    let (:mixed_utf8_urlencoded) { "A%DB%BF%E1%9A%A0%F0%A0%9C%8E" }
+
+    it "should perform basic URI escaping that includes space and +" do
+      expect(Puppet::Util.uri_query_encode("foo bar+foo")).to eq("foo%20bar%2Bfoo")
+    end
+
+    it "should URI encode any special characters: = + <space> & * and #" do
+      expect(Puppet::Util.uri_query_encode("foo=bar+foo baz&bar=baz qux&special= *&qux=not fragment#")).to eq("foo%3Dbar%2Bfoo%20baz%26bar%3Dbaz%20qux%26special%3D%20%2A%26qux%3Dnot%20fragment%23")
+    end
+
+    [
+      "A\u06FF\u16A0\u{2070E}",
+      "A\u06FF\u16A0\u{2070E}".force_encoding(Encoding::BINARY)
+    ].each do |uri_string|
+      it "should perform UTF-8 URI escaping, even when input strings are not UTF-8" do
+        uri = Puppet::Util.uri_query_encode(mixed_utf8)
+
+        expect(uri.encoding).to eq(Encoding::UTF_8)
+        expect(uri).to eq(mixed_utf8_urlencoded)
+      end
+    end
+
+    it "should be usable by URI::parse" do
+      uri = URI::parse("puppet://server/path?" + Puppet::Util.uri_query_encode(mixed_utf8))
+
+      expect(uri.scheme).to eq('puppet')
+      expect(uri.host).to eq('server')
+      expect(uri.path).to eq('/path')
+      expect(uri.query).to eq(mixed_utf8_urlencoded)
+    end
+
+    it "should be usable by URI::Generic.build" do
+      params = {
+        :scheme => 'file',
+        :host => 'foobar',
+        :path => '/path/to',
+        :query => Puppet::Util.uri_query_encode(mixed_utf8)
+      }
+
+      uri = URI::Generic.build(params)
+
+      expect(uri.scheme).to eq('file')
+      expect(uri.host).to eq('foobar')
+      expect(uri.path).to eq("/path/to")
+      expect(uri.query).to eq(mixed_utf8_urlencoded)
+    end
+  end
+
+  describe "#uri_encode" do
+    # different UTF-8 widths
+    # 1-byte A
+    # 2-byte ۿ - http://www.fileformat.info/info/unicode/char/06ff/index.htm - 0xDB 0xBF / 219 191
+    # 3-byte ᚠ - http://www.fileformat.info/info/unicode/char/16A0/index.htm - 0xE1 0x9A 0xA0 / 225 154 160
+    # 4-byte ܎ - http://www.fileformat.info/info/unicode/char/2070E/index.htm - 0xF0 0xA0 0x9C 0x8E / 240 160 156 142
+    let (:mixed_utf8) { "A\u06FF\u16A0\u{2070E}" } # Aۿᚠ܎
+    let (:mixed_utf8_urlencoded) { "A%DB%BF%E1%9A%A0%F0%A0%9C%8E" }
+
+    it "should perform URI escaping" do
+      expect(Puppet::Util.uri_encode("/foo bar")).to eq("/foo%20bar")
+    end
+
+    [
+      "A\u06FF\u16A0\u{2070E}",
+      "A\u06FF\u16A0\u{2070E}".force_encoding(Encoding::BINARY)
+    ].each do |uri_string|
+      it "should perform UTF-8 URI escaping, even when input strings are not UTF-8" do
+        uri = Puppet::Util.uri_encode(mixed_utf8)
+
+        expect(uri.encoding).to eq(Encoding::UTF_8)
+        expect(uri).to eq(mixed_utf8_urlencoded)
+      end
+    end
+
+    it "should treat & and = as delimiters in a query string, but URI encode other special characters: + <space> * and #" do
+      input = "http://foo.bar.com/path?foo=bar+foo baz&bar=baz qux&special= *&qux=not fragment#"
+      expected_output = "http://foo.bar.com/path?foo=bar%2Bfoo%20baz&bar=baz%20qux&special=%20%2A&qux=not%20fragment%23"
+      expect(Puppet::Util.uri_encode(input)).to eq(expected_output)
+    end
+
+    it "should be usable by URI::parse" do
+      uri = URI::parse(Puppet::Util.uri_encode("puppet://server/path/to/#{mixed_utf8}"))
+
+      expect(uri.scheme).to eq('puppet')
+      expect(uri.host).to eq('server')
+      expect(uri.path).to eq("/path/to/#{mixed_utf8_urlencoded}")
+    end
+
+    it "should be usable by URI::Generic.build" do
+      params = {
+        :scheme => 'file',
+        :host => 'foobar',
+        :path => Puppet::Util.uri_encode("/path/to/#{mixed_utf8}")
+      }
+
+      uri = URI::Generic.build(params)
+
+      expect(uri.scheme).to eq('file')
+      expect(uri.host).to eq('foobar')
+      expect(uri.path).to eq("/path/to/#{mixed_utf8_urlencoded}")
+    end
+
+    describe "when using platform :posix" do
+      before :each do
+        Puppet.features.stubs(:posix).returns true
+        Puppet.features.stubs(:microsoft_windows?).returns false
+      end
+
+      %w[/ /foo /foo/../bar].each do |path|
+        it "should not replace / in #{path} with %2F" do
+          expect(Puppet::Util.uri_encode(path)).to eq(path)
+        end
+      end
+    end
+
+    describe "with fragment support" do
+      context "disabled by default" do
+        it "should encode # as %23 in path" do
+          encoded = Puppet::Util.uri_encode("/foo bar#fragment")
+          expect(encoded).to eq("/foo%20bar%23fragment")
+        end
+
+        it "should encode # as %23 in query" do
+          encoded = Puppet::Util.uri_encode("/foo bar?baz+qux#fragment")
+          expect(encoded).to eq("/foo%20bar?baz%2Bqux%23fragment")
+        end
+      end
+
+      context "optionally enabled" do
+        it "should leave fragment delimiter # after encoded paths" do
+          encoded = Puppet::Util.uri_encode("/foo bar#fragment", { :allow_fragment => true })
+          expect(encoded).to eq("/foo%20bar#fragment")
+        end
+
+        it "should leave fragment delimiter # after encoded query" do
+          encoded = Puppet::Util.uri_encode("/foo bar?baz+qux#fragment", { :allow_fragment => true })
+          expect(encoded).to eq("/foo%20bar?baz%2Bqux#fragment")
+        end
+      end
+    end
+
+    describe "when using platform :windows" do
+      before :each do
+        Puppet.features.stubs(:posix).returns false
+        Puppet.features.stubs(:microsoft_windows?).returns true
+      end
+
+      it "should url encode \\ as %5C, but not replace : as %3F" do
+        expect(Puppet::Util.uri_encode('c:\\foo\\bar\\baz')).to eq('c:%5Cfoo%5Cbar%5Cbaz')
+      end
+
+      %w[C:/ C:/foo/bar].each do |path|
+        it "should not replace / in #{path} with %2F" do
+          expect(Puppet::Util.uri_encode(path)).to eq(path)
         end
       end
     end
@@ -349,6 +564,13 @@ describe Puppet::Util do
   describe ".uri_to_path" do
     require 'uri'
 
+    # different UTF-8 widths
+    # 1-byte A
+    # 2-byte ۿ - http://www.fileformat.info/info/unicode/char/06ff/index.htm - 0xDB 0xBF / 219 191
+    # 3-byte ᚠ - http://www.fileformat.info/info/unicode/char/16A0/index.htm - 0xE1 0x9A 0xA0 / 225 154 160
+    # 4-byte 𠜎 - http://www.fileformat.info/info/unicode/char/2070E/index.htm - 0xF0 0xA0 0x9C 0x8E / 240 160 156 142
+    let (:mixed_utf8) { "A\u06FF\u16A0\u{2070E}" } # Aۿᚠ𠜎
+
     it "should strip host component" do
       expect(Puppet::Util.uri_to_path(URI.parse('http://foo/bar'))).to eq('/bar')
     end
@@ -361,6 +583,19 @@ describe Puppet::Util do
       expect(Puppet::Util.uri_to_path(URI.parse('http://foo/bar%20baz'))).to eq('/bar baz')
     end
 
+
+    [
+      "http://foo/A%DB%BF%E1%9A%A0%F0%A0%9C%8E",
+      "http://foo/A%DB%BF%E1%9A%A0%F0%A0%9C%8E".force_encoding(Encoding::ASCII)
+    ].each do |uri_string|
+      it "should return paths as UTF-8" do
+        path = Puppet::Util.uri_to_path(URI.parse(uri_string))
+
+        expect(path).to eq("/#{mixed_utf8}")
+        expect(path.encoding).to eq(Encoding::UTF_8)
+      end
+    end
+
     it "should be nil-safe" do
       expect(Puppet::Util.uri_to_path(nil)).to be_nil
     end
@@ -474,7 +709,7 @@ describe Puppet::Util do
       expect(Puppet::Util.which('doesnotexist')).to be_nil
     end
 
-    it "should warn if the user's HOME is not set but their PATH contains a ~" do
+    it "should warn if the user's HOME is not set but their PATH contains a ~", :unless => gte_ruby_2_4 do
       env_path = %w[~/bin /usr/bin /bin].join(File::PATH_SEPARATOR)
 
       env = {:HOME => nil, :PATH => env_path}

data/tasks/benchmark.rake

@@ -94,9 +94,9 @@ namespace :benchmark do
         end
 
         printer = RubyProf::CallTreePrinter.new(result)
-        File.open(File.join("callgrind.#{name}.#{Time.now.to_i}.trace"), "w") do |f|
-          printer.print(f)
-        end
+        printer.print(:profile => name, :path => ENV['TARGET'])
+        path = File.join(ENV['TARGET'], "#{name}.callgrind.out.#{$$}")
+        puts "Generated callgrind file: #{path}"
       end
 
       def to_millis(seconds)

data/tasks/cfpropertylist.rake

@@ -1,6 +1,6 @@
 task 'cfpropertylist' do
   if defined? Pkg::Config and Pkg::Config.project_root
-    cfp_version = "2.2.7"
+    cfp_version = "2.3.5"
     libdir = File.join(Pkg::Config.project_root, "lib")
     source = "https://github.com/ckruse/CFPropertyList/archive/cfpropertylist-#{cfp_version}.tar.gz"
     target_dir = Pkg::Util::File.mktemp

data/tasks/generate_ast_model.rake

@@ -0,0 +1,78 @@
+require 'puppet'
+
+desc "Generate the Pcore model that represents the AST for the Puppet Language"
+task :gen_pcore_ast do
+  Puppet::Pops.generate_ast
+end
+
+module Puppet::Pops
+  def self.generate_ast
+    Puppet.initialize_settings
+    env = Puppet.lookup(:environments).get(Puppet[:environment])
+    loaders = Loaders.new(env)
+    ast_pp = Pathname(__FILE__).parent.parent + 'lib/puppet/pops/model/ast.pp'
+    Puppet.override(:current_environment => env, :loaders => loaders) do
+      ast_factory = Parser::Parser.new.parse_file(ast_pp.expand_path.to_s)
+      ast_model = Types::TypeParser.singleton.interpret(
+        ast_factory.model.body, Loader::PredefinedLoader.new(loaders.find_loader(nil), 'TypeSet loader'))
+
+      ruby = Types::RubyGenerator.new.module_definition_from_typeset(ast_model)
+
+      # Replace ref() constructs to known Pcore types with directly initialized types. ref() cannot be used
+      # since it requires a parser (hen and egg problem)
+      ruby.gsub!(/^module Parser\nmodule Locator\n.*\nend\nend\nmodule Model\n/m, "module Model\n")
+
+      # Remove generated RubyMethod annotations. The ruby methods are there now, no need to also have
+      # the annotations present.
+      ruby.gsub!(/^\s+'annotations' => \{\n\s+ref\('RubyMethod'\) => \{\n.*\n\s+\}\n\s+\},\n/, '')
+
+      ruby.gsub!(/ref\('([A-Za-z]+)'\)/, 'Types::P\1Type::DEFAULT')
+      ruby.gsub!(/ref\('Optional\[([0-9A-Za-z_]+)\]'\)/, 'Types::POptionalType.new(Types::P\1Type::DEFAULT)')
+      ruby.gsub!(/ref\('Array\[([0-9A-Za-z_]+)\]'\)/, 'Types::PArrayType.new(Types::P\1Type::DEFAULT)')
+      ruby.gsub!(/ref\('Optional\[Array\[([0-9A-Za-z_]+)\]\]'\)/,
+          'Types::POptionalType.new(Types::PArrayType.new(Types::P\1Type::DEFAULT))')
+      ruby.gsub!(/ref\("Enum(\[[^\]]+\])"\)/, 'Types::PEnumType.new(\1)')
+
+      # Replace ref() constructs with references to _pcore_type of the types in the module namespace
+      ruby.gsub!(/ref\('Puppet::AST::Locator'\)/, 'Parser::Locator::Locator19._pcore_type')
+      ruby.gsub!(/ref\('Puppet::AST::([0-9A-Za-z_]+)'\)/, '\1._pcore_type')
+      ruby.gsub!(/ref\('Optional\[Puppet::AST::([0-9A-Za-z_]+)\]'\)/, 'Types::POptionalType.new(\1._pcore_type)')
+      ruby.gsub!(/ref\('Array\[Puppet::AST::([0-9A-Za-z_]+)\]'\)/, 'Types::PArrayType.new(\1._pcore_type)')
+      ruby.gsub!(/ref\('Array\[Puppet::AST::([0-9A-Za-z_]+), 1, default\]'\)/,
+          'Types::PArrayType.new(\1._pcore_type, Types::PCollectionType::NOT_EMPTY_SIZE)')
+
+      # Remove the generated ref() method. It's not needed by this model
+      ruby.gsub!(/  def self\.ref\(type_string\)\n.*\n  end\n\n/, '')
+
+      # Replace the generated registration with a registration that uses the static loader. This will
+      # become part of the Puppet bootstrap code and there will be no other loader until we have a
+      # parser.
+      ruby.gsub!(/^Puppet::Pops::Pcore.register_implementations\((\[[^\]]+\])\)/, <<-RUBY)
+module Model
+@@pcore_ast_initialized = false
+def self.register_pcore_types
+  return if @@pcore_ast_initialized
+  @@pcore_ast_initialized = true
+  all_types = \\1
+
+  # Create and register a TypeSet that corresponds to all types in the AST model
+  types_map = {}
+  all_types.each do |type|
+    types_map[type._pcore_type.simple_name] = type._pcore_type
+  end
+  type_set = Types::PTypeSetType.new({
+    'name' => 'Puppet::AST',
+    'pcore_version' => '1.0.0',
+    'types' => types_map
+  })
+  loc = Puppet::Util.path_to_uri("\#{__FILE__}")
+  Loaders.static_loader.set_entry(Loader::TypedName.new(:type, 'puppet::ast', Pcore::RUNTIME_NAME_AUTHORITY), type_set, URI("\#{loc}?line=1"))
+  Loaders.register_static_implementations(all_types)
+end
+end
+RUBY
+      ast_rb = Pathname(__FILE__).parent.parent + 'lib/puppet/pops/model/ast.rb'
+      File.open(ast_rb.to_s, 'w') { |f| f.write(ruby) }
+    end
+  end
+end