puppet 4.3.1 → 4.3.2

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of puppet might be problematic. Click here for more details.

Files changed (128) hide show
  1. checksums.yaml +4 -4
  2. data/ext/build_defaults.yaml +1 -1
  3. data/lib/hiera/scope.rb +1 -1
  4. data/lib/puppet/application/lookup.rb +41 -43
  5. data/lib/puppet/data_providers/lookup_adapter.rb +73 -26
  6. data/lib/puppet/functions/lookup.rb +126 -150
  7. data/lib/puppet/functions/match.rb +1 -0
  8. data/lib/puppet/indirector/hiera.rb +3 -1
  9. data/lib/puppet/indirector/indirection.rb +6 -2
  10. data/lib/puppet/indirector/json.rb +2 -2
  11. data/lib/puppet/module.rb +3 -2
  12. data/lib/puppet/node.rb +11 -2
  13. data/lib/puppet/parser/compiler.rb +1 -8
  14. data/lib/puppet/parser/functions/lookup.rb +128 -149
  15. data/lib/puppet/parser/functions/match.rb +1 -0
  16. data/lib/puppet/plugins/data_providers/data_provider.rb +3 -2
  17. data/lib/puppet/pops/adapters.rb +43 -0
  18. data/lib/puppet/pops/evaluator/access_operator.rb +3 -3
  19. data/lib/puppet/pops/evaluator/closure.rb +51 -51
  20. data/lib/puppet/pops/evaluator/collector_transformer.rb +16 -0
  21. data/lib/puppet/pops/evaluator/runtime3_support.rb +11 -2
  22. data/lib/puppet/pops/functions/function.rb +6 -2
  23. data/lib/puppet/pops/issues.rb +16 -0
  24. data/lib/puppet/pops/loader/puppet_function_instantiator.rb +3 -2
  25. data/lib/puppet/pops/lookup.rb +3 -0
  26. data/lib/puppet/pops/lookup/explainer.rb +73 -3
  27. data/lib/puppet/pops/lookup/invocation.rb +21 -19
  28. data/lib/puppet/pops/model/factory.rb +153 -155
  29. data/lib/puppet/pops/model/model.rb +9 -0
  30. data/lib/puppet/pops/model/model_label_provider.rb +1 -0
  31. data/lib/puppet/pops/parser/evaluating_parser.rb +3 -3
  32. data/lib/puppet/pops/parser/lexer2.rb +411 -393
  33. data/lib/puppet/pops/parser/slurp_support.rb +5 -1
  34. data/lib/puppet/pops/types/type_calculator.rb +2 -6
  35. data/lib/puppet/pops/types/types.rb +3 -9
  36. data/lib/puppet/pops/validation/checker4_0.rb +36 -12
  37. data/lib/puppet/provider/group/windows_adsi.rb +2 -2
  38. data/lib/puppet/provider/package/pip.rb +11 -1
  39. data/lib/puppet/provider/package/rpm.rb +0 -1
  40. data/lib/puppet/provider/package/yum.rb +1 -1
  41. data/lib/puppet/provider/service/debian.rb +5 -18
  42. data/lib/puppet/provider/service/init.rb +7 -0
  43. data/lib/puppet/provider/service/launchd.rb +6 -0
  44. data/lib/puppet/provider/service/systemd.rb +1 -1
  45. data/lib/puppet/provider/user/windows_adsi.rb +2 -2
  46. data/lib/puppet/provider/yumrepo/inifile.rb +6 -3
  47. data/lib/puppet/resource/type.rb +2 -1
  48. data/lib/puppet/transaction/additional_resource_generator.rb +17 -3
  49. data/lib/puppet/type/group.rb +6 -2
  50. data/lib/puppet/util/windows.rb +4 -0
  51. data/lib/puppet/util/windows/adsi.rb +61 -24
  52. data/lib/puppet/util/windows/principal.rb +181 -0
  53. data/lib/puppet/util/windows/registry.rb +21 -15
  54. data/lib/puppet/util/windows/sid.rb +42 -11
  55. data/lib/puppet/version.rb +1 -1
  56. data/spec/fixtures/unit/application/environments/production/data/common.yaml +4 -0
  57. data/spec/fixtures/unit/application/environments/production/manifests/site.pp +1 -0
  58. data/spec/fixtures/unit/application/environments/puppet_func_provider/environment.conf +1 -0
  59. data/spec/fixtures/unit/application/environments/puppet_func_provider/functions/data.pp +10 -0
  60. data/spec/fixtures/unit/application/environments/puppet_func_provider/manifests/site.pp +1 -0
  61. data/spec/fixtures/unit/data_providers/environments/hiera_module_config/data/common.yaml +4 -0
  62. data/spec/fixtures/unit/data_providers/environments/hiera_module_config/data/specific.yaml +4 -0
  63. data/spec/fixtures/unit/data_providers/environments/hiera_module_config/hiera.yaml +7 -0
  64. data/spec/fixtures/unit/data_providers/environments/hiera_modules/data/common.yaml +4 -0
  65. data/spec/fixtures/unit/data_providers/environments/hiera_modules/data/specific.yaml +4 -0
  66. data/spec/fixtures/unit/data_providers/environments/hiera_modules/environment.conf +2 -0
  67. data/spec/fixtures/unit/data_providers/environments/hiera_modules/hiera.yaml +7 -0
  68. data/spec/fixtures/unit/data_providers/environments/hiera_modules/manifests/site.pp +1 -0
  69. data/spec/fixtures/unit/data_providers/environments/hiera_modules/modules/one/data/common.yaml +6 -0
  70. data/spec/fixtures/unit/data_providers/environments/hiera_modules/modules/one/hiera.yaml +5 -0
  71. data/spec/fixtures/unit/data_providers/environments/hiera_modules/modules/one/manifests/init.pp +2 -0
  72. data/spec/fixtures/unit/data_providers/environments/hiera_modules/modules/one/metadata.json +9 -0
  73. data/spec/fixtures/unit/data_providers/environments/hiera_modules/modules/two/data/common.yaml +4 -0
  74. data/spec/fixtures/unit/data_providers/environments/hiera_modules/modules/two/hiera.yaml +5 -0
  75. data/spec/fixtures/unit/data_providers/environments/hiera_modules/modules/two/manifests/init.pp +3 -0
  76. data/spec/fixtures/unit/data_providers/environments/hiera_modules/modules/two/metadata.json +9 -0
  77. data/spec/fixtures/unit/pops/loaders/loaders/dependent_modules_with_metadata/modules/usee/functions/usee_puppet.pp +3 -0
  78. data/spec/fixtures/unit/pops/loaders/loaders/dependent_modules_with_metadata/{usee → modules/usee}/lib/puppet/functions/usee/callee.rb +0 -0
  79. data/spec/fixtures/unit/pops/loaders/loaders/dependent_modules_with_metadata/modules/usee/lib/puppet/functions/usee/usee_ruby.rb +6 -0
  80. data/spec/fixtures/unit/pops/loaders/loaders/dependent_modules_with_metadata/modules/usee/manifests/init.pp +6 -0
  81. data/spec/fixtures/unit/pops/loaders/loaders/dependent_modules_with_metadata/modules/usee2/lib/puppet/functions/usee2/callee.rb +5 -0
  82. data/spec/fixtures/unit/pops/loaders/loaders/dependent_modules_with_metadata/modules/user/functions/puppet_calling_puppet.pp +5 -0
  83. data/spec/fixtures/unit/pops/loaders/loaders/dependent_modules_with_metadata/modules/user/functions/puppet_calling_puppet_init.pp +5 -0
  84. data/spec/fixtures/unit/pops/loaders/loaders/dependent_modules_with_metadata/modules/user/functions/puppet_calling_ruby.pp +5 -0
  85. data/spec/fixtures/unit/pops/loaders/loaders/dependent_modules_with_metadata/{user → modules/user}/lib/puppet/functions/user/caller.rb +0 -0
  86. data/spec/fixtures/unit/pops/loaders/loaders/dependent_modules_with_metadata/modules/user/lib/puppet/functions/user/caller2.rb +5 -0
  87. data/spec/fixtures/unit/pops/loaders/loaders/dependent_modules_with_metadata/modules/user/lib/puppet/functions/user/ruby_calling_puppet.rb +5 -0
  88. data/spec/fixtures/unit/pops/loaders/loaders/dependent_modules_with_metadata/modules/user/lib/puppet/functions/user/ruby_calling_puppet_init.rb +5 -0
  89. data/spec/fixtures/unit/pops/loaders/loaders/dependent_modules_with_metadata/modules/user/lib/puppet/functions/user/ruby_calling_ruby.rb +5 -0
  90. data/spec/fixtures/unit/pops/loaders/loaders/dependent_modules_with_metadata/modules/user/manifests/init.pp +81 -0
  91. data/spec/fixtures/unit/pops/loaders/loaders/dependent_modules_with_metadata/{user → modules/user}/metadata.json +2 -1
  92. data/spec/integration/parser/collection_spec.rb +8 -0
  93. data/spec/integration/util/windows/principal_spec.rb +115 -0
  94. data/spec/{unit → integration}/util/windows/registry_spec.rb +91 -1
  95. data/spec/integration/util/windows/security_spec.rb +2 -2
  96. data/spec/unit/application/lookup_spec.rb +138 -28
  97. data/spec/unit/data_providers/hiera_data_provider_spec.rb +182 -5
  98. data/spec/unit/face/epp_face_spec.rb +2 -2
  99. data/spec/unit/functions/epp_spec.rb +6 -6
  100. data/spec/unit/functions/inline_epp_spec.rb +4 -4
  101. data/spec/unit/functions/lookup_spec.rb +30 -3
  102. data/spec/unit/functions4_spec.rb +1 -1
  103. data/spec/unit/hiera/scope_spec.rb +5 -2
  104. data/spec/unit/indirector/json_spec.rb +1 -1
  105. data/spec/unit/node_spec.rb +8 -0
  106. data/spec/unit/parser/compiler_spec.rb +0 -18
  107. data/spec/unit/pops/evaluator/access_ops_spec.rb +4 -4
  108. data/spec/unit/pops/evaluator/evaluating_parser_spec.rb +1 -1
  109. data/spec/unit/pops/loaders/loaders_spec.rb +84 -2
  110. data/spec/unit/pops/parser/lexer2_spec.rb +6 -0
  111. data/spec/unit/pops/parser/parser_rspec_helper.rb +5 -0
  112. data/spec/unit/pops/types/type_calculator_spec.rb +0 -17
  113. data/spec/unit/pops/validator/validator_spec.rb +87 -0
  114. data/spec/unit/provider/group/windows_adsi_spec.rb +8 -8
  115. data/spec/unit/provider/package/pip_spec.rb +41 -13
  116. data/spec/unit/provider/package/rpm_spec.rb +2 -25
  117. data/spec/unit/provider/package/yum_spec.rb +1 -1
  118. data/spec/unit/provider/service/debian_spec.rb +6 -24
  119. data/spec/unit/provider/service/init_spec.rb +11 -1
  120. data/spec/unit/provider/service/launchd_spec.rb +11 -0
  121. data/spec/unit/provider/service/systemd_spec.rb +18 -12
  122. data/spec/unit/provider/service/upstart_spec.rb +57 -0
  123. data/spec/unit/provider/user/windows_adsi_spec.rb +5 -5
  124. data/spec/unit/provider/yumrepo/inifile_spec.rb +16 -0
  125. data/spec/unit/resource_spec.rb +12 -2
  126. data/spec/unit/util/windows/adsi_spec.rb +44 -36
  127. data/spec/unit/util/windows/sid_spec.rb +47 -10
  128. metadata +77 -10
@@ -24,6 +24,15 @@ module Puppet::Pops
24
24
  include Puppet::Pops::Containment
25
25
  end
26
26
 
27
+ class Positioned
28
+ module ClassModule
29
+ def set_loc(offset, length)
30
+ @offset = offset
31
+ @length = length
32
+ end
33
+ end
34
+ end
35
+
27
36
  class LocatableExpression
28
37
  module ClassModule
29
38
  # Go through the gymnastics of making either value or pattern settable
@@ -57,6 +57,7 @@ class Puppet::Pops::Model::ModelLabelProvider
57
57
  def label_ConcatenatedString o ; "Double Quoted String" end
58
58
  def label_HeredocExpression o ; "'@(#{o.syntax})' expression" end
59
59
  def label_HostClassDefinition o ; "Host Class Definition" end
60
+ def label_FunctionDefinition o ; "Function Definition" end
60
61
  def label_NodeDefinition o ; "Node Definition" end
61
62
  def label_SiteDefinition o ; "Site Definition" end
62
63
  def label_ResourceTypeDefinition o ; "'define' expression" end
@@ -9,7 +9,7 @@ class Puppet::Pops::Parser::EvaluatingParser
9
9
  @parser = Puppet::Pops::Parser::Parser.new()
10
10
  end
11
11
 
12
- def parse_string(s, file_source = 'unknown')
12
+ def parse_string(s, file_source = nil)
13
13
  @file_source = file_source
14
14
  clear()
15
15
  # Handling of syntax error can be much improved (in general), now it bails out of the parser
@@ -19,7 +19,7 @@ class Puppet::Pops::Parser::EvaluatingParser
19
19
  # Also a possible improvement (if the YAML parser returns positions) is to provide correct output of position.
20
20
  #
21
21
  begin
22
- assert_and_report(parser.parse_string(s))
22
+ assert_and_report(parser.parse_string(s, file_source))
23
23
  rescue Puppet::ParseErrorWithIssue => e
24
24
  raise e
25
25
  rescue Puppet::ParseError => e
@@ -35,7 +35,7 @@ class Puppet::Pops::Parser::EvaluatingParser
35
35
  assert_and_report(parser.parse_file(file))
36
36
  end
37
37
 
38
- def evaluate_string(scope, s, file_source='unknown')
38
+ def evaluate_string(scope, s, file_source = nil)
39
39
  evaluate(scope, parse_string(s, file_source))
40
40
  end
41
41
 
@@ -111,26 +111,26 @@ class Puppet::Pops::Parser::Lexer2
111
111
  # Booleans are pre-calculated (rather than evaluating the strings "false" "true" repeatedly.
112
112
  #
113
113
  KEYWORDS = {
114
- "case" => [:CASE, 'case', 4],
115
- "class" => [:CLASS, 'class', 5],
116
- "default" => [:DEFAULT, 'default', 7],
117
- "define" => [:DEFINE, 'define', 6],
118
- "if" => [:IF, 'if', 2],
119
- "elsif" => [:ELSIF, 'elsif', 5],
120
- "else" => [:ELSE, 'else', 4],
121
- "inherits" => [:INHERITS, 'inherits', 8],
122
- "node" => [:NODE, 'node', 4],
123
- "and" => [:AND, 'and', 3],
124
- "or" => [:OR, 'or', 2],
125
- "undef" => [:UNDEF, 'undef', 5],
126
- "false" => [:BOOLEAN, false, 5],
127
- "true" => [:BOOLEAN, true, 4],
128
- "in" => [:IN, 'in', 2],
129
- "unless" => [:UNLESS, 'unless', 6],
130
- "function" => [:FUNCTION, 'function', 8],
131
- "type" => [:TYPE, 'type', 4],
132
- "attr" => [:ATTR, 'attr', 4],
133
- "private" => [:PRIVATE, 'private', 7],
114
+ 'case' => [:CASE, 'case', 4],
115
+ 'class' => [:CLASS, 'class', 5],
116
+ 'default' => [:DEFAULT, 'default', 7],
117
+ 'define' => [:DEFINE, 'define', 6],
118
+ 'if' => [:IF, 'if', 2],
119
+ 'elsif' => [:ELSIF, 'elsif', 5],
120
+ 'else' => [:ELSE, 'else', 4],
121
+ 'inherits' => [:INHERITS, 'inherits', 8],
122
+ 'node' => [:NODE, 'node', 4],
123
+ 'and' => [:AND, 'and', 3],
124
+ 'or' => [:OR, 'or', 2],
125
+ 'undef' => [:UNDEF, 'undef', 5],
126
+ 'false' => [:BOOLEAN, false, 5],
127
+ 'true' => [:BOOLEAN, true, 4],
128
+ 'in' => [:IN, 'in', 2],
129
+ 'unless' => [:UNLESS, 'unless', 6],
130
+ 'function' => [:FUNCTION, 'function', 8],
131
+ 'type' => [:TYPE, 'type', 4],
132
+ 'attr' => [:ATTR, 'attr', 4],
133
+ 'private' => [:PRIVATE, 'private', 7],
134
134
  }
135
135
 
136
136
  KEYWORDS.each {|k,v| v[1].freeze; v.freeze }
@@ -142,16 +142,16 @@ class Puppet::Pops::Parser::Lexer2
142
142
  # hit the appmgmt-specific code paths
143
143
  APP_MANAGEMENT_TOKENS = {
144
144
  :with_appm => {
145
- "application" => [:APPLICATION, 'application', 11],
146
- "consumes" => [:CONSUMES, 'consumes', 8],
147
- "produces" => [:PRODUCES, 'produces', 8],
148
- "site" => [:SITE, 'site', 4]
145
+ 'application' => [:APPLICATION, 'application', 11],
146
+ 'consumes' => [:CONSUMES, 'consumes', 8],
147
+ 'produces' => [:PRODUCES, 'produces', 8],
148
+ 'site' => [:SITE, 'site', 4]
149
149
  },
150
150
  :without_appm => {
151
- "application" => [:APPLICATION_R, 'application', 11],
152
- "consumes" => [:CONSUMES_R, 'consumes', 8],
153
- "produces" => [:PRODUCES_R, 'produces', 8],
154
- "site" => [:SITE_R, 'site', 4]
151
+ 'application' => [:APPLICATION_R, 'application', 11],
152
+ 'consumes' => [:CONSUMES_R, 'consumes', 8],
153
+ 'produces' => [:PRODUCES_R, 'produces', 8],
154
+ 'site' => [:SITE_R, 'site', 4]
155
155
  }
156
156
  }
157
157
 
@@ -205,6 +205,382 @@ class Puppet::Pops::Parser::Lexer2
205
205
  attr_reader :locator
206
206
 
207
207
  def initialize()
208
+ @selector = {
209
+ '.' => lambda { emit(TOKEN_DOT, @scanner.pos) },
210
+ ',' => lambda { emit(TOKEN_COMMA, @scanner.pos) },
211
+ '[' => lambda do
212
+ before = @scanner.pos
213
+ if (before == 0 || @scanner.string[locator.char_offset(before)-1,1] =~ /[[:blank:]\r\n]+/)
214
+ emit(TOKEN_LISTSTART, before)
215
+ else
216
+ emit(TOKEN_LBRACK, before)
217
+ end
218
+ end,
219
+ ']' => lambda { emit(TOKEN_RBRACK, @scanner.pos) },
220
+ '(' => lambda { emit(TOKEN_LPAREN, @scanner.pos) },
221
+ ')' => lambda { emit(TOKEN_RPAREN, @scanner.pos) },
222
+ ';' => lambda { emit(TOKEN_SEMIC, @scanner.pos) },
223
+ '?' => lambda { emit(TOKEN_QMARK, @scanner.pos) },
224
+ '*' => lambda { emit(TOKEN_TIMES, @scanner.pos) },
225
+ '%' => lambda do
226
+ scn = @scanner
227
+ before = scn.pos
228
+ la = scn.peek(2)
229
+ if la[1] == '>' && @lexing_context[:epp_mode]
230
+ scn.pos += 2
231
+ if @lexing_context[:epp_mode] == :expr
232
+ enqueue_completed(TOKEN_EPPEND, before)
233
+ end
234
+ @lexing_context[:epp_mode] = :text
235
+ interpolate_epp
236
+ else
237
+ emit(TOKEN_MODULO, before)
238
+ end
239
+ end,
240
+ '{' => lambda do
241
+ # The lexer needs to help the parser since the technology used cannot deal with
242
+ # lookahead of same token with different precedence. This is solved by making left brace
243
+ # after ? into a separate token.
244
+ #
245
+ @lexing_context[:brace_count] += 1
246
+ emit(if @lexing_context[:after] == :QMARK
247
+ TOKEN_SELBRACE
248
+ else
249
+ TOKEN_LBRACE
250
+ end, @scanner.pos)
251
+ end,
252
+ '}' => lambda do
253
+ @lexing_context[:brace_count] -= 1
254
+ emit(TOKEN_RBRACE, @scanner.pos)
255
+ end,
256
+
257
+
258
+ # TOKENS @, @@, @(
259
+ '@' => lambda do
260
+ scn = @scanner
261
+ la = scn.peek(2)
262
+ if la[1] == '@'
263
+ emit(TOKEN_ATAT, scn.pos) # TODO; Check if this is good for the grammar
264
+ elsif la[1] == '('
265
+ heredoc
266
+ else
267
+ emit(TOKEN_AT, scn.pos)
268
+ end
269
+ end,
270
+
271
+ # TOKENS |, |>, |>>
272
+ '|' => lambda do
273
+ scn = @scanner
274
+ la = scn.peek(3)
275
+ emit(la[1] == '>' ? (la[2] == '>' ? TOKEN_RRCOLLECT : TOKEN_RCOLLECT) : TOKEN_PIPE, scn.pos)
276
+ end,
277
+
278
+ # TOKENS =, =>, ==, =~
279
+ '=' => lambda do
280
+ scn = @scanner
281
+ la = scn.peek(2)
282
+ emit(case la[1]
283
+ when '='
284
+ TOKEN_ISEQUAL
285
+ when '>'
286
+ TOKEN_FARROW
287
+ when '~'
288
+ TOKEN_MATCH
289
+ else
290
+ TOKEN_EQUALS
291
+ end, scn.pos)
292
+ end,
293
+
294
+ # TOKENS '+', '+=', and '+>'
295
+ '+' => lambda do
296
+ scn = @scanner
297
+ la = scn.peek(2)
298
+ emit(case la[1]
299
+ when '='
300
+ TOKEN_APPENDS
301
+ when '>'
302
+ TOKEN_PARROW
303
+ else
304
+ TOKEN_PLUS
305
+ end, scn.pos)
306
+ end,
307
+
308
+ # TOKENS '-', '->', and epp '-%>' (end of interpolation with trim)
309
+ '-' => lambda do
310
+ scn = @scanner
311
+ la = scn.peek(3)
312
+ before = scn.pos
313
+ if @lexing_context[:epp_mode] && la[1] == '%' && la[2] == '>'
314
+ scn.pos += 3
315
+ if @lexing_context[:epp_mode] == :expr
316
+ enqueue_completed(TOKEN_EPPEND_TRIM, before)
317
+ end
318
+ interpolate_epp(:with_trim)
319
+ else
320
+ emit(case la[1]
321
+ when '>'
322
+ TOKEN_IN_EDGE
323
+ when '='
324
+ TOKEN_DELETES
325
+ else
326
+ TOKEN_MINUS
327
+ end, before)
328
+ end
329
+ end,
330
+
331
+ # TOKENS !, !=, !~
332
+ '!' => lambda do
333
+ scn = @scanner
334
+ la = scn.peek(2)
335
+ emit(case la[1]
336
+ when '='
337
+ TOKEN_NOTEQUAL
338
+ when '~'
339
+ TOKEN_NOMATCH
340
+ else
341
+ TOKEN_NOT
342
+ end, scn.pos)
343
+ end,
344
+
345
+ # TOKENS ~>, ~
346
+ '~' => lambda do
347
+ scn = @scanner
348
+ la = scn.peek(2)
349
+ emit(la[1] == '>' ? TOKEN_IN_EDGE_SUB : TOKEN_TILDE, scn.pos)
350
+ end,
351
+
352
+ '#' => lambda { @scanner.skip(PATTERN_COMMENT); nil },
353
+
354
+ # TOKENS '/', '/*' and '/ regexp /'
355
+ '/' => lambda do
356
+ scn = @scanner
357
+ la = scn.peek(2)
358
+ if la[1] == '*'
359
+ lex_error(Puppet::Pops::Issues::UNCLOSED_MLCOMMENT) if scn.skip(PATTERN_MLCOMMENT).nil?
360
+ nil
361
+ else
362
+ before = scn.pos
363
+ # regexp position is a regexp, else a div
364
+ if regexp_acceptable? && value = scn.scan(PATTERN_REGEX)
365
+ # Ensure an escaped / was not matched
366
+ while value[-2..-2] == STRING_BSLASH_BSLASH # i.e. \\
367
+ value += scn.scan_until(PATTERN_REGEX_END)
368
+ end
369
+ regex = value.sub(PATTERN_REGEX_A, '').sub(PATTERN_REGEX_Z, '').gsub(PATTERN_REGEX_ESC, '/')
370
+ emit_completed([:REGEX, Regexp.new(regex), scn.pos-before], before)
371
+ else
372
+ emit(TOKEN_DIV, before)
373
+ end
374
+ end
375
+ end,
376
+
377
+ # TOKENS <, <=, <|, <<|, <<, <-, <~
378
+ '<' => lambda do
379
+ scn = @scanner
380
+ la = scn.peek(3)
381
+ emit(case la[1]
382
+ when '<'
383
+ if la[2] == '|'
384
+ TOKEN_LLCOLLECT
385
+ else
386
+ TOKEN_LSHIFT
387
+ end
388
+ when '='
389
+ TOKEN_LESSEQUAL
390
+ when '|'
391
+ TOKEN_LCOLLECT
392
+ when '-'
393
+ TOKEN_OUT_EDGE
394
+ when '~'
395
+ TOKEN_OUT_EDGE_SUB
396
+ else
397
+ TOKEN_LESSTHAN
398
+ end, scn.pos)
399
+ end,
400
+
401
+ # TOKENS >, >=, >>
402
+ '>' => lambda do
403
+ scn = @scanner
404
+ la = scn.peek(2)
405
+ emit(case la[1]
406
+ when '>'
407
+ TOKEN_RSHIFT
408
+ when '='
409
+ TOKEN_GREATEREQUAL
410
+ else
411
+ TOKEN_GREATERTHAN
412
+ end, scn.pos)
413
+ end,
414
+
415
+ # TOKENS :, ::CLASSREF, ::NAME
416
+ ':' => lambda do
417
+ scn = @scanner
418
+ la = scn.peek(3)
419
+ before = scn.pos
420
+ if la[1] == ':'
421
+ # PERFORMANCE NOTE: This could potentially be speeded up by using a case/when listing all
422
+ # upper case letters. Alternatively, the 'A', and 'Z' comparisons may be faster if they are
423
+ # frozen.
424
+ #
425
+ la2 = la[2]
426
+ if la2 >= 'A' && la2 <= 'Z'
427
+ # CLASSREF or error
428
+ value = scn.scan(PATTERN_CLASSREF)
429
+ if value && scn.peek(2) != '::'
430
+ after = scn.pos
431
+ emit_completed([:CLASSREF, value.freeze, after-before], before)
432
+ else
433
+ # move to faulty position ('::<uc-letter>' was ok)
434
+ scn.pos = scn.pos + 3
435
+ lex_error(Puppet::Pops::Issues::ILLEGAL_FULLY_QUALIFIED_CLASS_REFERENCE)
436
+ end
437
+ else
438
+ value = scn.scan(PATTERN_BARE_WORD)
439
+ if value
440
+ if value =~ PATTERN_NAME
441
+ emit_completed([:NAME, value.freeze, scn.pos - before], before)
442
+ else
443
+ emit_completed([:WORD, value.freeze, scn.pos - before], before)
444
+ end
445
+ else
446
+ # move to faulty position ('::' was ok)
447
+ scn.pos = scn.pos + 2
448
+ lex_error(Puppet::Pops::Issues::ILLEGAL_FULLY_QUALIFIED_NAME)
449
+ end
450
+ end
451
+ else
452
+ emit(TOKEN_COLON, before)
453
+ end
454
+ end,
455
+
456
+ '$' => lambda do
457
+ scn = @scanner
458
+ before = scn.pos
459
+ if value = scn.scan(PATTERN_DOLLAR_VAR)
460
+ emit_completed([:VARIABLE, value[1..-1].freeze, scn.pos - before], before)
461
+ else
462
+ # consume the $ and let higher layer complain about the error instead of getting a syntax error
463
+ emit(TOKEN_VARIABLE_EMPTY, before)
464
+ end
465
+ end,
466
+
467
+ '"' => lambda do
468
+ # Recursive string interpolation, 'interpolate' either returns a STRING token, or
469
+ # a DQPRE with the rest of the string's tokens placed in the @token_queue
470
+ interpolate_dq
471
+ end,
472
+
473
+ "'" => lambda do
474
+ scn = @scanner
475
+ before = scn.pos
476
+ emit_completed([:STRING, slurp_sqstring.freeze, scn.pos - before], before)
477
+ end,
478
+
479
+ "\n" => lambda do
480
+ # If heredoc_cont is in effect there are heredoc text lines to skip over
481
+ # otherwise just skip the newline.
482
+ #
483
+ ctx = @lexing_context
484
+ if ctx[:newline_jump]
485
+ @scanner.pos = ctx[:newline_jump]
486
+ ctx[:newline_jump] = nil
487
+ else
488
+ @scanner.pos += 1
489
+ end
490
+ nil
491
+ end,
492
+ '' => lambda { nil } # when the peek(1) returns empty
493
+ }
494
+
495
+ [ ' ', "\t", "\r" ].each { |c| @selector[c] = lambda { @scanner.skip(PATTERN_WS); nil } }
496
+
497
+ [ '0', '1', '2', '3', '4', '5', '6', '7', '8', '9'].each do |c|
498
+ @selector[c] = lambda do
499
+ scn = @scanner
500
+ before = scn.pos
501
+ value = scn.scan(PATTERN_NUMBER)
502
+ if value
503
+ length = scn.pos - before
504
+ assert_numeric(value, before)
505
+ emit_completed([:NUMBER, value.freeze, length], before)
506
+ else
507
+ invalid_number = scn.scan_until(PATTERN_NON_WS)
508
+ if before > 1
509
+ after = scn.pos
510
+ scn.pos = before - 1
511
+ if scn.peek(1) == '.'
512
+ # preceded by a dot. Is this a bad decimal number then?
513
+ scn.pos = before - 2
514
+ while scn.peek(1) =~ /^\d$/
515
+ invalid_number = nil
516
+ before = scn.pos
517
+ break if before == 0
518
+ scn.pos = scn.pos - 1
519
+ end
520
+ end
521
+ scn.pos = before
522
+ invalid_number = scn.peek(after - before) unless invalid_number
523
+ end
524
+ length = scn.pos - before
525
+ assert_numeric(invalid_number, before)
526
+ scn.pos = before + 1
527
+ lex_error(Puppet::Pops::Issues::ILLEGAL_NUMBER, {:value => invalid_number})
528
+ end
529
+ end
530
+ end
531
+ ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
532
+ 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '_'].each do |c|
533
+ @selector[c] = lambda do
534
+
535
+ scn = @scanner
536
+ before = scn.pos
537
+ value = scn.scan(PATTERN_BARE_WORD)
538
+ if value && value =~ PATTERN_NAME
539
+ emit_completed(KEYWORDS[value] || @appm_keywords[value] || [:NAME, value.freeze, scn.pos - before], before)
540
+ elsif value
541
+ emit_completed([:WORD, value.freeze, scn.pos - before], before)
542
+ else
543
+ # move to faulty position ([a-z_] was ok)
544
+ scn.pos = scn.pos + 1
545
+ fully_qualified = scn.match?(/::/)
546
+ if fully_qualified
547
+ lex_error(Puppet::Pops::Issues::ILLEGAL_FULLY_QUALIFIED_NAME)
548
+ else
549
+ lex_error(Puppet::Pops::Issues::ILLEGAL_NAME_OR_BARE_WORD)
550
+ end
551
+ end
552
+ end
553
+ end
554
+
555
+ ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M',
556
+ 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z'].each do |c|
557
+ @selector[c] = lambda do
558
+ scn = @scanner
559
+ before = scn.pos
560
+ value = @scanner.scan(PATTERN_CLASSREF)
561
+ if value && @scanner.peek(2) != '::'
562
+ emit_completed([:CLASSREF, value.freeze, scn.pos - before], before)
563
+ else
564
+ # move to faulty position ([A-Z] was ok)
565
+ scn.pos = scn.pos + 1
566
+ lex_error(Puppet::Pops::Issues::ILLEGAL_CLASS_REFERENCE)
567
+ end
568
+ end
569
+ end
570
+
571
+ @selector.default = lambda do
572
+ # In case of unicode spaces of various kinds that are captured by a regexp, but not by the
573
+ # simpler case expression above (not worth handling those special cases with better performance).
574
+ scn = @scanner
575
+ if scn.skip(PATTERN_WS)
576
+ nil
577
+ else
578
+ # "unrecognized char"
579
+ emit([:OTHER, scn.peek(0), 1], scn.pos)
580
+ end
581
+ end
582
+ @selector.each { |k,v| k.freeze }
583
+ @selector.freeze
208
584
  end
209
585
 
210
586
  # Clears the lexer state (it is not required to call this as it will be garbage collected
@@ -264,7 +640,7 @@ class Puppet::Pops::Parser::Lexer2
264
640
  #
265
641
  def lex_file(file)
266
642
  initvars
267
- contents = Puppet::FileSystem.exist?(file) ? Puppet::FileSystem.read(file) : ""
643
+ contents = Puppet::FileSystem.exist?(file) ? Puppet::FileSystem.read(file) : ''
268
644
  @scanner = StringScanner.new(contents.freeze)
269
645
  @locator = Puppet::Pops::Parser::Locator.locator(contents, file)
270
646
  end
@@ -285,7 +661,7 @@ class Puppet::Pops::Parser::Lexer2
285
661
  #
286
662
  def fullscan
287
663
  result = []
288
- scan {|token, value| result.push([token, value]) }
664
+ scan {|token| result.push(token) }
289
665
  result
290
666
  end
291
667
 
@@ -303,16 +679,17 @@ class Puppet::Pops::Parser::Lexer2
303
679
  # every token in the lexed content.
304
680
  #
305
681
  scn = @scanner
682
+ lex_error_without_pos(Puppet::Pops::Issues::NO_INPUT_TO_LEXER) unless scn
683
+
306
684
  ctx = @lexing_context
307
685
  queue = @token_queue
308
-
309
- lex_error_without_pos(Puppet::Pops::Issues::NO_INPUT_TO_LEXER) unless scn
686
+ selector = @selector
310
687
 
311
688
  scn.skip(PATTERN_WS)
312
689
 
313
690
  # This is the lexer's main loop
314
691
  until queue.empty? && scn.eos? do
315
- if token = queue.shift || lex_token
692
+ if token = queue.shift || selector[scn.peek(1)].call
316
693
  ctx[:after] = token[0]
317
694
  yield token
318
695
  end
@@ -326,366 +703,7 @@ class Puppet::Pops::Parser::Lexer2
326
703
  # PERFORMANCE NOTE: Any change to this logic should be performance measured.
327
704
  #
328
705
  def lex_token
329
- # Using three char look ahead (may be faster to do 2 char look ahead since only 2 tokens require a third
330
- scn = @scanner
331
- ctx = @lexing_context
332
- before = @scanner.pos
333
-
334
- # A look ahead of 3 characters is used since the longest operator ambiguity is resolved at that point.
335
- # PERFORMANCE NOTE: It is faster to peek once and use three separate variables for lookahead 0, 1 and 2.
336
- #
337
- la = scn.peek(3)
338
- return nil if la.empty?
339
-
340
- # PERFORMANCE NOTE.
341
- # It is slightly faster to use these local variables than accessing la[0], la[1] etc. in ruby 1.9.3
342
- # But not big enough to warrant two completely different implementations.
343
- #
344
- la0 = la[0]
345
- la1 = la[1]
346
- la2 = la[2]
347
-
348
- # PERFORMANCE NOTE:
349
- # A case when, where all the cases are literal values is the fastest way to map from data to code.
350
- # It is much faster than using a hash with lambdas, hash with symbol used to then invoke send etc.
351
- # This case statement is evaluated for most character positions in puppet source, and great care must
352
- # be taken to not introduce performance regressions.
353
- #
354
- case la0
355
-
356
- when '.'
357
- emit(TOKEN_DOT, before)
358
-
359
- when ','
360
- emit(TOKEN_COMMA, before)
361
-
362
- when '['
363
- if (before == 0 || scn.string[locator.char_offset(before)-1,1] =~ /[[:blank:]\r\n]+/)
364
- emit(TOKEN_LISTSTART, before)
365
- else
366
- emit(TOKEN_LBRACK, before)
367
- end
368
-
369
- when ']'
370
- emit(TOKEN_RBRACK, before)
371
-
372
- when '('
373
- emit(TOKEN_LPAREN, before)
374
-
375
- when ')'
376
- emit(TOKEN_RPAREN, before)
377
-
378
- when ';'
379
- emit(TOKEN_SEMIC, before)
380
-
381
- when '?'
382
- emit(TOKEN_QMARK, before)
383
-
384
- when '*'
385
- emit(TOKEN_TIMES, before)
386
-
387
- when '%'
388
- if la1 == '>' && ctx[:epp_mode]
389
- scn.pos += 2
390
- if ctx[:epp_mode] == :expr
391
- enqueue_completed(TOKEN_EPPEND, before)
392
- end
393
- ctx[:epp_mode] = :text
394
- interpolate_epp
395
- else
396
- emit(TOKEN_MODULO, before)
397
- end
398
-
399
- when '{'
400
- # The lexer needs to help the parser since the technology used cannot deal with
401
- # lookahead of same token with different precedence. This is solved by making left brace
402
- # after ? into a separate token.
403
- #
404
- ctx[:brace_count] += 1
405
- emit(if ctx[:after] == :QMARK
406
- TOKEN_SELBRACE
407
- else
408
- TOKEN_LBRACE
409
- end, before)
410
-
411
- when '}'
412
- ctx[:brace_count] -= 1
413
- emit(TOKEN_RBRACE, before)
414
-
415
- # TOKENS @, @@, @(
416
- when '@'
417
- case la1
418
- when '@'
419
- emit(TOKEN_ATAT, before) # TODO; Check if this is good for the grammar
420
- when '('
421
- heredoc
422
- else
423
- emit(TOKEN_AT, before)
424
- end
425
-
426
- # TOKENS |, |>, |>>
427
- when '|'
428
- emit(case la1
429
- when '>'
430
- la2 == '>' ? TOKEN_RRCOLLECT : TOKEN_RCOLLECT
431
- else
432
- TOKEN_PIPE
433
- end, before)
434
-
435
- # TOKENS =, =>, ==, =~
436
- when '='
437
- emit(case la1
438
- when '='
439
- TOKEN_ISEQUAL
440
- when '>'
441
- TOKEN_FARROW
442
- when '~'
443
- TOKEN_MATCH
444
- else
445
- TOKEN_EQUALS
446
- end, before)
447
-
448
- # TOKENS '+', '+=', and '+>'
449
- when '+'
450
- emit(case la1
451
- when '='
452
- TOKEN_APPENDS
453
- when '>'
454
- TOKEN_PARROW
455
- else
456
- TOKEN_PLUS
457
- end, before)
458
-
459
- # TOKENS '-', '->', and epp '-%>' (end of interpolation with trim)
460
- when '-'
461
- if ctx[:epp_mode] && la1 == '%' && la2 == '>'
462
- scn.pos += 3
463
- if ctx[:epp_mode] == :expr
464
- enqueue_completed(TOKEN_EPPEND_TRIM, before)
465
- end
466
- interpolate_epp(:with_trim)
467
- else
468
- emit(case la1
469
- when '>'
470
- TOKEN_IN_EDGE
471
- when '='
472
- TOKEN_DELETES
473
- else
474
- TOKEN_MINUS
475
- end, before)
476
- end
477
-
478
- # TOKENS !, !=, !~
479
- when '!'
480
- emit(case la1
481
- when '='
482
- TOKEN_NOTEQUAL
483
- when '~'
484
- TOKEN_NOMATCH
485
- else
486
- TOKEN_NOT
487
- end, before)
488
-
489
- # TOKENS ~>, ~
490
- when '~'
491
- emit(la1 == '>' ? TOKEN_IN_EDGE_SUB : TOKEN_TILDE, before)
492
-
493
- when '#'
494
- scn.skip(PATTERN_COMMENT)
495
- nil
496
-
497
- # TOKENS '/', '/*' and '/ regexp /'
498
- when '/'
499
- case la1
500
- when '*'
501
- scn.skip(PATTERN_MLCOMMENT)
502
- nil
503
-
504
- else
505
- # regexp position is a regexp, else a div
506
- if regexp_acceptable? && value = scn.scan(PATTERN_REGEX)
507
- # Ensure an escaped / was not matched
508
- while value[-2..-2] == STRING_BSLASH_BSLASH # i.e. \\
509
- value += scn.scan_until(PATTERN_REGEX_END)
510
- end
511
- regex = value.sub(PATTERN_REGEX_A, '').sub(PATTERN_REGEX_Z, '').gsub(PATTERN_REGEX_ESC, '/')
512
- emit_completed([:REGEX, Regexp.new(regex), scn.pos-before], before)
513
- else
514
- emit(TOKEN_DIV, before)
515
- end
516
- end
517
-
518
- # TOKENS <, <=, <|, <<|, <<, <-, <~
519
- when '<'
520
- emit(case la1
521
- when '<'
522
- if la2 == '|'
523
- TOKEN_LLCOLLECT
524
- else
525
- TOKEN_LSHIFT
526
- end
527
- when '='
528
- TOKEN_LESSEQUAL
529
- when '|'
530
- TOKEN_LCOLLECT
531
- when '-'
532
- TOKEN_OUT_EDGE
533
- when '~'
534
- TOKEN_OUT_EDGE_SUB
535
- else
536
- TOKEN_LESSTHAN
537
- end, before)
538
-
539
- # TOKENS >, >=, >>
540
- when '>'
541
- emit(case la1
542
- when '>'
543
- TOKEN_RSHIFT
544
- when '='
545
- TOKEN_GREATEREQUAL
546
- else
547
- TOKEN_GREATERTHAN
548
- end, before)
549
-
550
- # TOKENS :, ::CLASSREF, ::NAME
551
- when ':'
552
- if la1 == ':'
553
- before = scn.pos
554
- # PERFORMANCE NOTE: This could potentially be speeded up by using a case/when listing all
555
- # upper case letters. Alternatively, the 'A', and 'Z' comparisons may be faster if they are
556
- # frozen.
557
- #
558
- if la2 >= 'A' && la2 <= 'Z'
559
- # CLASSREF or error
560
- value = scn.scan(PATTERN_CLASSREF)
561
- if value && scn.peek(2) != '::'
562
- after = scn.pos
563
- emit_completed([:CLASSREF, value.freeze, after-before], before)
564
- else
565
- # move to faulty position ('::<uc-letter>' was ok)
566
- scn.pos = scn.pos + 3
567
- lex_error(Puppet::Pops::Issues::ILLEGAL_FULLY_QUALIFIED_CLASS_REFERENCE)
568
- end
569
- else
570
- value = scn.scan(PATTERN_BARE_WORD)
571
- if value
572
- if value =~ PATTERN_NAME
573
- emit_completed([:NAME, value.freeze, scn.pos-before], before)
574
- else
575
- emit_completed([:WORD, value.freeze, scn.pos - before], before)
576
- end
577
- else
578
- # move to faulty position ('::' was ok)
579
- scn.pos = scn.pos + 2
580
- lex_error(Puppet::Pops::Issues::ILLEGAL_FULLY_QUALIFIED_NAME)
581
- end
582
- end
583
- else
584
- emit(TOKEN_COLON, before)
585
- end
586
-
587
- when '$'
588
- if value = scn.scan(PATTERN_DOLLAR_VAR)
589
- emit_completed([:VARIABLE, value[1..-1].freeze, scn.pos - before], before)
590
- else
591
- # consume the $ and let higher layer complain about the error instead of getting a syntax error
592
- emit(TOKEN_VARIABLE_EMPTY, before)
593
- end
594
-
595
- when '"'
596
- # Recursive string interpolation, 'interpolate' either returns a STRING token, or
597
- # a DQPRE with the rest of the string's tokens placed in the @token_queue
598
- interpolate_dq
599
-
600
- when "'"
601
- emit_completed([:STRING, slurp_sqstring.freeze, scn.pos - before], before)
602
-
603
- when '0', '1', '2', '3', '4', '5', '6', '7', '8', '9'
604
- value = scn.scan(PATTERN_NUMBER)
605
- if value
606
- length = scn.pos - before
607
- assert_numeric(value, before)
608
- emit_completed([:NUMBER, value.freeze, length], before)
609
- else
610
- invalid_number = scn.scan_until(PATTERN_NON_WS)
611
- if before > 1
612
- after = scn.pos
613
- scn.pos = before - 1
614
- if scn.peek(1) == '.'
615
- # preceded by a dot. Is this a bad decimal number then?
616
- scn.pos = before - 2
617
- while scn.peek(1) =~ /^\d$/
618
- invalid_number = nil
619
- before = scn.pos
620
- break if before == 0
621
- scn.pos = scn.pos - 1
622
- end
623
- end
624
- scn.pos = before
625
- invalid_number = scn.peek(after - before) unless invalid_number
626
- end
627
- length = scn.pos - before
628
- assert_numeric(invalid_number, before)
629
- scn.pos = before + 1
630
- lex_error(Puppet::Pops::Issues::ILLEGAL_NUMBER, {:value => invalid_number})
631
- end
632
-
633
- when 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
634
- 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '_'
635
-
636
- value = scn.scan(PATTERN_BARE_WORD)
637
- if value && value =~ PATTERN_NAME
638
- emit_completed(KEYWORDS[value] || @appm_keywords[value] || [:NAME, value.freeze, scn.pos - before], before)
639
- elsif value
640
- emit_completed([:WORD, value.freeze, scn.pos - before], before)
641
- else
642
- # move to faulty position ([a-z_] was ok)
643
- scn.pos = scn.pos + 1
644
- fully_qualified = scn.match?(/::/)
645
- if fully_qualified
646
- lex_error(Puppet::Pops::Issues::ILLEGAL_FULLY_QUALIFIED_NAME)
647
- else
648
- lex_error(Puppet::Pops::Issues::ILLEGAL_NAME_OR_BARE_WORD)
649
- end
650
- end
651
-
652
- when 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M',
653
- 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z'
654
- value = scn.scan(PATTERN_CLASSREF)
655
- if value && scn.peek(2) != '::'
656
- emit_completed([:CLASSREF, value.freeze, scn.pos - before], before)
657
- else
658
- # move to faulty position ([A-Z] was ok)
659
- scn.pos = scn.pos + 1
660
- lex_error(Puppet::Pops::Issues::ILLEGAL_CLASS_REFERENCE)
661
- end
662
-
663
- when "\n"
664
- # If heredoc_cont is in effect there are heredoc text lines to skip over
665
- # otherwise just skip the newline.
666
- #
667
- if ctx[:newline_jump]
668
- scn.pos = ctx[:newline_jump]
669
- ctx[:newline_jump] = nil
670
- else
671
- scn.pos += 1
672
- end
673
- return nil
674
-
675
- when ' ', "\t", "\r"
676
- scn.skip(PATTERN_WS)
677
- return nil
678
-
679
- else
680
- # In case of unicode spaces of various kinds that are captured by a regexp, but not by the
681
- # simpler case expression above (not worth handling those special cases with better performance).
682
- if scn.skip(PATTERN_WS)
683
- nil
684
- else
685
- # "unrecognized char"
686
- emit([:OTHER, la0, 1], before)
687
- end
688
- end
706
+ @selector[@scanner.peek(1)].call
689
707
  end
690
708
 
691
709
  # Emits (produces) a token [:tokensymbol, TokenValue] and moves the scanner's position past the token