puppet 4.1.0 → 4.10.0

data/spec/unit/pops/lookup/lookup_spec.rb

@@ -0,0 +1,262 @@
+require 'spec_helper'
+require 'puppet_spec/files'
+require 'puppet/pops'
+require 'deep_merge/core'
+
+module Puppet::Pops
+module Lookup
+describe 'The lookup API' do
+  include PuppetSpec::Files
+
+  let(:env_name) { 'spec' }
+  let(:code_dir) { Puppet[:environmentpath] }
+  let(:env_dir) { File.join(code_dir, env_name) }
+  let(:env) { Puppet::Node::Environment.create(env_name.to_sym, [File.join(populated_env_dir, 'modules')]) }
+  let(:node) { Puppet::Node.new('test', :environment => env) }
+  let(:compiler) { Puppet::Parser::Compiler.new(node) }
+  let(:scope) { compiler.topscope }
+  let(:invocation) { Invocation.new(scope) }
+
+  let(:code_dir_content) do
+    {
+      'hiera.yaml' => <<-YAML.unindent,
+        version: 5
+        YAML
+      'data' => {
+        'common.yaml' => <<-YAML.unindent
+          a: a (from global)
+          d: d (from global)
+          mod::e: mod::e (from global)
+          YAML
+      }
+    }
+  end
+
+  let(:env_content) do
+    {
+      'hiera.yaml' => <<-YAML.unindent,
+        version: 5
+        YAML
+      'data' => {
+        'common.yaml' => <<-YAML.unindent
+          b: b (from environment)
+          d: d (from environment)
+          mod::f: mod::f (from environment)
+          YAML
+      }
+    }
+  end
+
+  let(:mod_content) do
+    {
+      'hiera.yaml' => <<-YAML.unindent,
+        version: 5
+        YAML
+      'data' => {
+        'common.yaml' => <<-YAML.unindent
+          mod::c: mod::c (from module)
+          mod::e: mod::e (from module)
+          mod::f: mod::f (from module)
+          YAML
+      }
+    }
+  end
+
+  let(:populated_env_dir) do
+    all_content = code_dir_content.merge(env_name => env_content.merge('modules' => { 'mod' => mod_content }))
+    dir_contained_in(code_dir, all_content)
+    all_content.keys.each { |key| PuppetSpec::Files.record_tmp(File.join(code_dir, key)) }
+    env_dir
+  end
+
+  before(:each) do
+    Puppet[:hiera_config] = File.join(code_dir, 'hiera.yaml')
+    Puppet.push_context(:loaders => Puppet::Pops::Loaders.new(env))
+  end
+
+  after(:each) do
+    Puppet.pop_context
+  end
+
+  context 'when doing automatic parameter lookup' do
+
+    let(:mod_content) do
+      {
+        'hiera.yaml' => <<-YAML.unindent,
+          version: 5
+          YAML
+        'data' => {
+          'common.yaml' => <<-YAML.unindent
+            mod::x: mod::x (from module)
+            YAML
+        },
+        'manifests' => {
+           'init.pp' => <<-PUPPET.unindent
+             class mod($x) {
+               notify { $x: }
+             }
+             PUPPET
+        }
+      }
+    end
+    let(:logs) { [] }
+    let(:debugs) { logs.select { |log| log.level == :debug }.map { |log| log.message } }
+
+    it 'includes APL in explain output when debug is enabled' do
+      Puppet[:log_level] = 'debug'
+      Puppet[:code] = 'include mod'
+      Puppet::Util::Log.with_destination(Puppet::Test::LogCollector.new(logs)) do
+        compiler.compile
+      end
+      expect(debugs).to include(/Found key: "mod::x" value: "mod::x \(from module\)"/)
+    end
+  end
+
+  context 'when doing lookup' do
+    it 'finds data in global layer' do
+      expect(Lookup.lookup('a', nil, nil, false, nil, invocation)).to eql('a (from global)')
+    end
+
+    it 'finds data in environment layer' do
+      expect(Lookup.lookup('b', nil, 'not found', true, nil, invocation)).to eql('b (from environment)')
+    end
+
+    it 'global layer wins over environment layer' do
+      expect(Lookup.lookup('d', nil, 'not found', true, nil, invocation)).to eql('d (from global)')
+    end
+
+    it 'finds data in module layer' do
+      expect(Lookup.lookup('mod::c', nil, 'not found', true, nil, invocation)).to eql('mod::c (from module)')
+    end
+
+    it 'global layer wins over module layer' do
+      expect(Lookup.lookup('mod::e', nil, 'not found', true, nil, invocation)).to eql('mod::e (from global)')
+    end
+
+    it 'environment layer wins over module layer' do
+      expect(Lookup.lookup('mod::f', nil, 'not found', true, nil, invocation)).to eql('mod::f (from environment)')
+    end
+
+    context "with 'global_only' set to true in the invocation" do
+      let(:invocation) { Invocation.new(scope).set_global_only }
+
+      it 'finds data in global layer' do
+        expect(Lookup.lookup('a', nil, nil, false, nil, invocation)).to eql('a (from global)')
+      end
+
+      it 'does not find data in environment layer' do
+        expect(Lookup.lookup('b', nil, 'not found', true, nil, invocation)).to eql('not found')
+      end
+
+      it 'does not find data in module layer' do
+        expect(Lookup.lookup('mod::c', nil, 'not found', true, nil, invocation)).to eql('not found')
+      end
+    end
+
+    context "with 'global_only' set to true in the lookup adapter" do
+      it 'finds data in global layer' do
+        invocation.lookup_adapter.set_global_only
+        expect(Lookup.lookup('a', nil, nil, false, nil, invocation)).to eql('a (from global)')
+      end
+
+      it 'does not find data in environment layer' do
+        invocation.lookup_adapter.set_global_only
+        expect(Lookup.lookup('b', nil, 'not found', true, nil, invocation)).to eql('not found')
+      end
+
+      it 'does not find data in module layer' do
+        invocation.lookup_adapter.set_global_only
+        expect(Lookup.lookup('mod::c', nil, 'not found', true, nil, invocation)).to eql('not found')
+      end
+    end
+
+    context 'with subclassed lookup adpater' do
+      let(:other_dir) { tmpdir('other') }
+      let(:other_dir_content) do
+        {
+          'hiera.yaml' => <<-YAML.unindent,
+            version: 5
+            hierarchy:
+              - name: Common
+                path: common.yaml
+              - name: More
+                path: more.yaml
+            YAML
+          'data' => {
+            'common.yaml' => <<-YAML.unindent,
+              a: a (from other global)
+              d: d (from other global)
+              mixed_adapter_hash:
+                a:
+                  ab: value a.ab (from other common global)
+                  ad: value a.ad (from other common global)
+              mod::e: mod::e (from other global)
+              lookup_options:
+                mixed_adapter_hash:
+                  merge: deep
+              YAML
+            'more.yaml' => <<-YAML.unindent
+              mixed_adapter_hash:
+                a:
+                  aa: value a.aa (from other more global)
+                  ac: value a.ac (from other more global)
+              YAML
+          }
+        }
+      end
+
+      let(:populated_other_dir) do
+        dir_contained_in(other_dir, other_dir_content)
+        other_dir
+      end
+
+      before(:each) do
+        eval(<<-RUBY.unindent)
+        class SpecialLookupAdapter < LookupAdapter
+           def initialize(compiler)
+             super
+             set_global_only
+             set_global_hiera_config_path(File.join('#{populated_other_dir}', 'hiera.yaml'))
+           end
+        end
+        RUBY
+      end
+
+      after(:each) do
+        Puppet::Pops::Lookup.send(:remove_const, :SpecialLookupAdapter)
+      end
+
+      let(:other_invocation) { Invocation.new(scope, EMPTY_HASH, EMPTY_HASH, nil, SpecialLookupAdapter) }
+
+      it 'finds different data in global layer' do
+        expect(Lookup.lookup('a', nil, nil, false, nil, other_invocation)).to eql('a (from other global)')
+        expect(Lookup.lookup('a', nil, nil, false, nil, invocation)).to eql('a (from global)')
+      end
+
+      it 'does not find data in environment layer' do
+        expect(Lookup.lookup('b', nil, 'not found', true, nil, other_invocation)).to eql('not found')
+        expect(Lookup.lookup('b', nil, 'not found', true, nil, invocation)).to eql('b (from environment)')
+      end
+
+      it 'does not find data in module layer' do
+        expect(Lookup.lookup('mod::c', nil, 'not found', true, nil, other_invocation)).to eql('not found')
+        expect(Lookup.lookup('mod::c', nil, 'not found', true, nil, invocation)).to eql('mod::c (from module)')
+      end
+
+      it 'resolves lookup options using the custom adapter' do
+        expect(Lookup.lookup('mixed_adapter_hash', nil, 'not found', true, nil, other_invocation)).to eql(
+          {
+            'a' => {
+              'aa' => 'value a.aa (from other more global)',
+              'ab' => 'value a.ab (from other common global)',
+              'ac' => 'value a.ac (from other more global)',
+              'ad' => 'value a.ad (from other common global)'
+            }
+          }
+        )
+      end
+    end
+  end
+end
+end
+end

data/spec/unit/pops/merge_strategy_spec.rb

@@ -0,0 +1,18 @@
+#! /usr/bin/env ruby
+require 'spec_helper'
+require 'puppet/pops'
+
+module Puppet::Pops
+describe 'MergeStrategy' do
+  context 'deep merge' do
+    it 'does not mutate the source of a merge' do
+      a = { 'a' => { 'b' => 'va' }, 'c' => 2 }
+      b = { 'a' => { 'b' => 'vb' }, 'b' => 3}
+      c = MergeStrategy.strategy(:deep).merge(a, b);
+      expect(a).to eql({ 'a' => { 'b' => 'va' }, 'c' => 2 })
+      expect(b).to eql({ 'a' => { 'b' => 'vb' }, 'b' => 3 })
+      expect(c).to eql({ 'a' => { 'b' => 'va' }, 'b' => 3, 'c' => 2 })
+    end
+  end
+end
+end

data/spec/unit/pops/parser/lexer2_spec.rb

@@ -25,7 +25,7 @@ describe 'Lexer2' do
     :RBRACK => ']',
     :LBRACE => '{',
     :RBRACE => '}',
-    :LPAREN => '(',
+    :WSLPAREN => '(', # since it is first on a line it is special (LPAREN handled separately)
     :RPAREN => ')',
     :EQUALS => '=',
     :ISEQUAL => '==',
@@ -91,12 +91,44 @@ describe 'Lexer2' do
     "true"     => :BOOLEAN,
     "in"       => :IN,
     "unless"   => :UNLESS,
+    "private"  => :PRIVATE,
+    "type"     => :TYPE,
+    "attr"     => :ATTR,
   }.each do |string, name|
     it "should lex a keyword from '#{string}'" do
       expect(tokens_scanned_from(string)).to match_tokens2(name)
     end
   end
 
+  context 'when app_management is off (by default)' do
+    {
+      "application"  => :APPLICATION_R,
+      "consumes"     => :CONSUMES_R,
+      "produces"     => :PRODUCES_R,
+      "site"         => :SITE_R,
+    }.each do |string, name|
+      it "should lex a (future reserved) keyword from '#{string}'" do
+        expect(tokens_scanned_from(string)).to match_tokens2(name)
+      end
+    end
+  end
+
+  context 'when app_managment is (turned) on' do
+    before(:each) { Puppet[:app_management] = true }
+    after(:each) { Puppet[:app_management] = false }
+
+    {
+      "application"  => :APPLICATION,
+      "consumes"     => :CONSUMES,
+      "produces"     => :PRODUCES,
+      "site"         => :SITE,
+    }.each do |string, name|
+      it "should lex a keyword from '#{string}'" do
+        expect(tokens_scanned_from(string)).to match_tokens2(name)
+      end
+    end
+  end
+
   # TODO: Complete with all edge cases
   [ 'A', 'A::B', '::A', '::A::B',].each do |string|
     it "should lex a CLASSREF on the form '#{string}'" do
@@ -269,12 +301,32 @@ describe 'Lexer2' do
     expect(tokens_scanned_from("$a [1]")).to match_tokens2(:VARIABLE, :LISTSTART, :NUMBER, :RBRACK)
     expect(tokens_scanned_from("a[1]")).to match_tokens2(:NAME, :LBRACK, :NUMBER, :RBRACK)
     expect(tokens_scanned_from("a [1]")).to match_tokens2(:NAME, :LISTSTART, :NUMBER, :RBRACK)
-    expect(tokens_scanned_from(" if \n\r\t\nif if ")).to match_tokens2(:IF, :IF, :IF)
+  end
+
+  it "differentiates between '(' first on line, and not first on line" do
+    expect(tokens_scanned_from("(")).to match_tokens2(:WSLPAREN)
+    expect(tokens_scanned_from("\n(")).to match_tokens2(:WSLPAREN)
+    expect(tokens_scanned_from("\n\r(")).to match_tokens2(:WSLPAREN)
+    expect(tokens_scanned_from("\n\t(")).to match_tokens2(:WSLPAREN)
+    expect(tokens_scanned_from("\n\r\t(")).to match_tokens2(:WSLPAREN)
+    expect(tokens_scanned_from("\n\u00a0(")).to match_tokens2(:WSLPAREN)
+
+    expect(tokens_scanned_from("x(")).to match_tokens2(:NAME, :LPAREN)
+    expect(tokens_scanned_from("\nx(")).to match_tokens2(:NAME, :LPAREN)
+    expect(tokens_scanned_from("\n\rx(")).to match_tokens2(:NAME, :LPAREN)
+    expect(tokens_scanned_from("\n\tx(")).to match_tokens2(:NAME, :LPAREN)
+    expect(tokens_scanned_from("\n\r\tx(")).to match_tokens2(:NAME, :LPAREN)
+    expect(tokens_scanned_from("\n\u00a0x(")).to match_tokens2(:NAME, :LPAREN)
+
+    expect(tokens_scanned_from("x (")).to match_tokens2(:NAME, :LPAREN)
+    expect(tokens_scanned_from("x\t(")).to match_tokens2(:NAME, :LPAREN)
+    expect(tokens_scanned_from("x\u00a0(")).to match_tokens2(:NAME, :LPAREN)
   end
 
   it "skips whitepsace" do
     expect(tokens_scanned_from(" if if if ")).to match_tokens2(:IF, :IF, :IF)
     expect(tokens_scanned_from(" if \n\r\t\nif if ")).to match_tokens2(:IF, :IF, :IF)
+    expect(tokens_scanned_from(" if \n\r\t\n\u00a0if\u00a0 if ")).to match_tokens2(:IF, :IF, :IF)
   end
 
   it "skips single line comments" do
@@ -290,13 +342,23 @@ describe 'Lexer2' do
     end
   end
 
+  it 'detects unterminated multiline comment' do
+    expect { tokens_scanned_from("/* not terminated\nmultiline\ncomment") }.to raise_error(Puppet::ParseErrorWithIssue) { |e|
+      expect(e.issue_code).to be(Puppet::Pops::Issues::UNCLOSED_MLCOMMENT.issue_code)
+    }
+  end
+
   { "=~" => [:MATCH, "=~ /./"],
     "!~" => [:NOMATCH, "!~ /./"],
     ","  => [:COMMA, ", /./"],
-    "("  => [:LPAREN, "( /./"],
-    "["  => [:LISTSTART, "[ /./"],
-    "["  => [[:NAME, :LBRACK], "a[ /./"],
-    "["  => [[:NAME, :LISTSTART], "a [ /./"],
+
+    "("       => [:WSLPAREN, "( /./"],
+    "x ("     => [[:NAME, :LPAREN], "x ( /./"],
+    "x\\t ("  => [[:NAME, :LPAREN], "x\t ( /./"],
+
+    "[ (liststart)"             => [:LISTSTART, "[ /./"],
+    "[ (LBRACK)"                => [[:NAME, :LBRACK], "a[ /./"],
+    "[ (liststart after name)"  => [[:NAME, :LISTSTART], "a [ /./"],
     "{"  => [:LBRACE, "{ /./"],
     "+"  => [:PLUS, "+ /./"],
     "-"  => [:MINUS, "- /./"],
@@ -350,6 +412,14 @@ describe 'Lexer2' do
     expect(tokens_scanned_from("1 / /./")).to match_tokens2(:NUMBER, :DIV, :REGEX)
   end
 
+  it "should accept newline in a regular expression" do
+    scanned = tokens_scanned_from("/\n.\n/")
+    # Note that strange formatting here is important
+    expect(scanned[0][1][:value]).to eql(/
+.
+/)
+  end
+
   context 'when lexer lexes heredoc' do
     it 'lexes tag, syntax and escapes, margin and right trim' do
       code = <<-CODE
@@ -432,6 +502,18 @@ describe 'Lexer2' do
       end
     end
   end
+  context 'when not given multi byte characters' do
+    it 'produces byte offsets for tokens' do
+      code = <<-"CODE"
+1 2\n3
+      CODE
+      expect(tokens_scanned_from(code)).to match_tokens2(
+        [:NUMBER, '1', {:line => 1, :offset => 0, :length=>1}],
+        [:NUMBER, '2', {:line => 1, :offset => 2, :length=>1}],
+        [:NUMBER, '3', {:line => 2, :offset => 4, :length=>1}]
+      )
+    end
+  end
 
   context 'when dealing with multi byte characters' do
     it 'should support unicode characters' do
@@ -439,8 +521,17 @@ describe 'Lexer2' do
       "x\\u2713y"
       CODE
       # >= Ruby 1.9.3 reports \u
-       expect(tokens_scanned_from(code)).to match_tokens2([:STRING, "x\u2713y"])
+      expect(tokens_scanned_from(code)).to match_tokens2([:STRING, "x\u2713y"])
     end
+
+    it 'should support adjacent short form unicode characters' do
+      code = <<-CODE
+      "x\\u2713\\u2713y"
+      CODE
+      # >= Ruby 1.9.3 reports \u
+      expect(tokens_scanned_from(code)).to match_tokens2([:STRING, "x\u2713\u2713y"])
+    end
+
     it 'should support unicode characters in long form' do
       code = <<-CODE
       "x\\u{1f452}y"
@@ -448,6 +539,30 @@ describe 'Lexer2' do
       expect(tokens_scanned_from(code)).to match_tokens2([:STRING, "x\u{1f452}y"])
     end
 
+    it 'can escape the unicode escape' do
+      code = <<-"CODE"
+      "x\\\\u{1f452}y"
+      CODE
+      expect(tokens_scanned_from(code)).to match_tokens2([:STRING, "x\\u{1f452}y"])
+    end
+
+    it 'produces byte offsets that counts each byte in a comment' do
+      code = <<-"CODE"
+      # \u{0400}\na
+      CODE
+      expect(tokens_scanned_from(code.strip)).to match_tokens2([:NAME, 'a', {:line => 2, :offset => 5, :length=>1}])
+    end
+
+    it 'produces byte offsets that counts each byte in value token' do
+      code = <<-"CODE"
+      '\u{0400}'\na
+      CODE
+      expect(tokens_scanned_from(code.strip)).to match_tokens2(
+        [:STRING, "\u{400}", {:line => 1, :offset => 0, :length=>4}],
+        [:NAME, 'a', {:line => 2, :offset => 5, :length=>1}]
+      )
+    end
+
     it 'should not select LISTSTART token when preceded by multibyte chars' do
       # This test is sensitive to the number of multibyte characters and position of the expressions
       # within the string - it is designed to fail if the position is calculated on the byte offset of the '['
@@ -524,7 +639,7 @@ describe 'Lexer2' do
       [:VARIABLE, "x"],
       :EQUALS,
       [:NUMBER, "10"],
-      [:RENDER_STRING, "just text\n"]
+      [:RENDER_STRING, "      just text\n"]
       )
     end
 
@@ -540,7 +655,39 @@ describe 'Lexer2' do
       [:VARIABLE, "x"],
       :EQUALS,
       [:NUMBER, "10"],
-      [:RENDER_STRING, "just text\n"]
+      [:RENDER_STRING, "      just text\n"]
+      )
+    end
+
+    it 'epp comments strips left whitespace when preceding is right trim' do
+      code = <<-CODE
+      This is <% $x=10 -%>
+      space-before-me-but-not-after   <%# This is an epp comment %>
+      just text
+      CODE
+      expect(epp_tokens_scanned_from(code)).to match_tokens2(
+      :EPP_START,
+      [:RENDER_STRING, "      This is "],
+      [:VARIABLE, "x"],
+      :EQUALS,
+      [:NUMBER, "10"],
+      [:RENDER_STRING, "      space-before-me-but-not-after\n      just text\n"]
+      )
+    end
+
+    it 'epp comments strips left whitespace on same line when preceding is not right trim' do
+      code = <<-CODE
+      This is <% $x=10 %>
+      <%# This is an epp comment -%>
+      just text
+      CODE
+      expect(epp_tokens_scanned_from(code)).to match_tokens2(
+      :EPP_START,
+      [:RENDER_STRING, "      This is "],
+      [:VARIABLE, "x"],
+      :EQUALS,
+      [:NUMBER, "10"],
+      [:RENDER_STRING, "\n      just text\n"]
       )
     end
 
@@ -555,7 +702,7 @@ describe 'Lexer2' do
       [:VARIABLE, "x"],
       :EQUALS,
       [:NUMBER, "10"],
-      [:RENDER_STRING, "<% this is escaped epp %>\n"]
+      [:RENDER_STRING, "      <% this is escaped epp %>\n"]
       )
     end
 
@@ -625,4 +772,83 @@ describe 'Lexer2' do
       expect_issue('"asd', Puppet::Pops::Issues::UNCLOSED_QUOTE)
     end
   end
+
+  context 'when dealing with non UTF-8 and Byte Order Marks (BOMs)' do
+      {
+      'UTF_8'      => [0xEF, 0xBB, 0xBF],
+      'UTF_16_1'   => [0xFE, 0xFF],
+      'UTF_16_2'   => [0xFF, 0xFE],
+      'UTF_32_1'   => [0x00, 0x00, 0xFE, 0xFF],
+      'UTF_32_2'   => [0xFF, 0xFE, 0x00, 0x00],
+      'UTF_1'      => [0xF7, 0x64, 0x4C],
+      'UTF_EBCDIC' => [0xDD, 0x73, 0x66, 0x73],
+      'SCSU'       => [0x0E, 0xFE, 0xFF],
+      'BOCU'       => [0xFB, 0xEE, 0x28],
+      'GB_18030'   => [0x84, 0x31, 0x95, 0x33]
+      }.each do |key, bytes|
+        it "errors on the byte order mark for #{key} '[#{bytes.map() {|b| '%X' % b}.join(' ')}]'" do
+          format_name = key.split('_')[0,2].join('-')
+          bytes_str = "\\[#{bytes.map {|b| '%X' % b}.join(' ')}\\]"
+          fix =  " - remove these from the puppet source"
+          expect {
+            tokens_scanned_from(bytes.pack('C*'))
+          }.to raise_error(Puppet::ParseErrorWithIssue,
+            /Illegal #{format_name} .* at beginning of input: #{bytes_str}#{fix}/)
+        end
+
+       it "can use a possibly 'broken' UTF-16 string without problems for #{key}" do
+         format_name = key.split('_')[0,2].join('-')
+         string = bytes.pack('C*').force_encoding('UTF-16')
+         bytes_str = "\\[#{string.bytes.map {|b| '%X' % b}.join(' ')}\\]"
+         fix =  " - remove these from the puppet source"
+         expect {
+           tokens_scanned_from(string)
+         }.to raise_error(Puppet::ParseErrorWithIssue,
+           /Illegal #{format_name} .* at beginning of input: #{bytes_str}#{fix}/)
+       end
+    end
+  end
+end
+
+describe Puppet::Pops::Parser::Lexer2 do
+
+  include PuppetSpec::Files
+
+  # First line of Rune version of Rune poem at http://www.columbia.edu/~fdc/utf8/
+  # characters chosen since they will not parse on Windows with codepage 437 or 1252
+  # Section 3.2.1.3 of Ruby spec guarantees that \u strings are encoded as UTF-8
+  # Runes (may show up as garbage if font is not available): ᚠᛇᚻ᛫ᛒᛦᚦ᛫ᚠᚱᚩᚠᚢᚱ᛫ᚠᛁᚱᚪ᛫ᚷᛖᚻᚹᛦᛚᚳᚢᛗ
+  let (:rune_utf8) {
+    "\u16A0\u16C7\u16BB\u16EB\u16D2\u16E6\u16A6\u16EB\u16A0\u16B1\u16A9\u16A0\u16A2"
+    "\u16B1\u16EB\u16A0\u16C1\u16B1\u16AA\u16EB\u16B7\u16D6\u16BB\u16B9\u16E6\u16DA"
+    "\u16B3\u16A2\u16D7"
+  }
+
+  context 'when lexing files from disk' do
+    it 'should always read files as UTF-8' do
+      if Puppet.features.microsoft_windows? && Encoding.default_external == Encoding::UTF_8
+        raise 'This test must be run in a codepage other than 65001 to validate behavior'
+      end
+
+      manifest_code = "notify { '#{rune_utf8}': }"
+      manifest = file_containing('manifest.pp', manifest_code)
+      lexed_file = described_class.new.lex_file(manifest)
+
+      expect(lexed_file.string.encoding).to eq(Encoding::UTF_8)
+      expect(lexed_file.string).to eq(manifest_code)
+    end
+
+    it 'currently errors when the UTF-8 BOM (Byte Order Mark) is present when lexing files' do
+      bom = "\uFEFF"
+
+        manifest_code = "#{bom}notify { '#{rune_utf8}': }"
+        manifest = file_containing('manifest.pp', manifest_code)
+
+        expect {
+          lexed_file = described_class.new.lex_file(manifest)
+        }.to raise_error(Puppet::ParseErrorWithIssue,
+          'Illegal UTF-8 Byte Order mark at beginning of input: [EF BB BF] - remove these from the puppet source')
+    end
+  end
+
 end

data/spec/unit/pops/parser/parse_application_spec.rb

@@ -0,0 +1,35 @@
+#! /usr/bin/env ruby
+require 'spec_helper'
+require 'puppet/pops'
+require_relative 'parser_rspec_helper'
+
+describe "egrammar parsing of 'application'" do
+  include ParserRspecHelper
+
+  before(:each) { Puppet[:app_management] = true }
+  after(:each) { Puppet[:app_management] = false }
+
+  it "an empty body" do
+    expect(dump(parse("application foo { }"))).to eq("(application foo () ())")
+  end
+
+  it "an empty body" do
+    prog = <<-EPROG
+application foo {
+  db { one:
+    password => 'secret'
+  }
+}
+EPROG
+    expect(dump(parse(prog))).to eq( [
+"(application foo () (block",
+"  (resource db",
+"    (one",
+"      (password => 'secret')))", "))" ].join("\n"))
+  end
+
+  it "accepts parameters" do
+    s = "application foo($p1 = 'yo', $p2) { }"
+    expect(dump(parse(s))).to eq("(application foo ((= p1 'yo') p2) ())")
+  end
+end