hocon 0.0.7 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. checksums.yaml +7 -0
  2. data/README.md +4 -2
  3. data/lib/hocon.rb +2 -0
  4. data/lib/hocon/config.rb +1010 -0
  5. data/lib/hocon/config_error.rb +32 -2
  6. data/lib/hocon/config_factory.rb +46 -0
  7. data/lib/hocon/config_include_context.rb +49 -0
  8. data/lib/hocon/config_includer_file.rb +27 -0
  9. data/lib/hocon/config_list.rb +49 -0
  10. data/lib/hocon/config_mergeable.rb +74 -0
  11. data/lib/hocon/config_object.rb +144 -1
  12. data/lib/hocon/config_parse_options.rb +33 -9
  13. data/lib/hocon/config_parseable.rb +51 -0
  14. data/lib/hocon/config_render_options.rb +4 -2
  15. data/lib/hocon/config_resolve_options.rb +31 -0
  16. data/lib/hocon/config_syntax.rb +5 -2
  17. data/lib/hocon/config_util.rb +73 -0
  18. data/lib/hocon/config_value.rb +122 -0
  19. data/lib/hocon/config_value_factory.rb +66 -2
  20. data/lib/hocon/config_value_type.rb +5 -2
  21. data/lib/hocon/impl.rb +2 -0
  22. data/lib/hocon/impl/abstract_config_node.rb +29 -0
  23. data/lib/hocon/impl/abstract_config_node_value.rb +11 -0
  24. data/lib/hocon/impl/abstract_config_object.rb +148 -42
  25. data/lib/hocon/impl/abstract_config_value.rb +251 -11
  26. data/lib/hocon/impl/array_iterator.rb +19 -0
  27. data/lib/hocon/impl/config_boolean.rb +7 -1
  28. data/lib/hocon/impl/config_concatenation.rb +177 -28
  29. data/lib/hocon/impl/config_delayed_merge.rb +329 -0
  30. data/lib/hocon/impl/config_delayed_merge_object.rb +274 -0
  31. data/lib/hocon/impl/config_document_parser.rb +647 -0
  32. data/lib/hocon/impl/config_double.rb +44 -0
  33. data/lib/hocon/impl/config_impl.rb +143 -19
  34. data/lib/hocon/impl/config_impl_util.rb +18 -0
  35. data/lib/hocon/impl/config_include_kind.rb +10 -0
  36. data/lib/hocon/impl/config_int.rb +13 -1
  37. data/lib/hocon/impl/config_node_array.rb +11 -0
  38. data/lib/hocon/impl/config_node_comment.rb +19 -0
  39. data/lib/hocon/impl/config_node_complex_value.rb +54 -0
  40. data/lib/hocon/impl/config_node_concatenation.rb +11 -0
  41. data/lib/hocon/impl/config_node_field.rb +81 -0
  42. data/lib/hocon/impl/config_node_include.rb +33 -0
  43. data/lib/hocon/impl/config_node_object.rb +276 -0
  44. data/lib/hocon/impl/config_node_path.rb +48 -0
  45. data/lib/hocon/impl/config_node_root.rb +60 -0
  46. data/lib/hocon/impl/config_node_simple_value.rb +42 -0
  47. data/lib/hocon/impl/config_node_single_token.rb +17 -0
  48. data/lib/hocon/impl/config_null.rb +15 -7
  49. data/lib/hocon/impl/config_number.rb +43 -4
  50. data/lib/hocon/impl/config_parser.rb +403 -0
  51. data/lib/hocon/impl/config_reference.rb +142 -0
  52. data/lib/hocon/impl/config_string.rb +55 -7
  53. data/lib/hocon/impl/container.rb +29 -0
  54. data/lib/hocon/impl/default_transformer.rb +24 -15
  55. data/lib/hocon/impl/from_map_mode.rb +3 -1
  56. data/lib/hocon/impl/full_includer.rb +2 -0
  57. data/lib/hocon/impl/memo_key.rb +42 -0
  58. data/lib/hocon/impl/mergeable_value.rb +8 -0
  59. data/lib/hocon/impl/origin_type.rb +8 -2
  60. data/lib/hocon/impl/parseable.rb +455 -91
  61. data/lib/hocon/impl/path.rb +181 -59
  62. data/lib/hocon/impl/path_builder.rb +24 -3
  63. data/lib/hocon/impl/path_parser.rb +280 -0
  64. data/lib/hocon/impl/replaceable_merge_stack.rb +22 -0
  65. data/lib/hocon/impl/resolve_context.rb +254 -0
  66. data/lib/hocon/impl/resolve_memos.rb +21 -0
  67. data/lib/hocon/impl/resolve_result.rb +39 -0
  68. data/lib/hocon/impl/resolve_source.rb +354 -0
  69. data/lib/hocon/impl/resolve_status.rb +3 -1
  70. data/lib/hocon/impl/simple_config.rb +264 -10
  71. data/lib/hocon/impl/simple_config_document.rb +48 -0
  72. data/lib/hocon/impl/simple_config_list.rb +282 -8
  73. data/lib/hocon/impl/simple_config_object.rb +424 -88
  74. data/lib/hocon/impl/simple_config_origin.rb +263 -71
  75. data/lib/hocon/impl/simple_include_context.rb +31 -1
  76. data/lib/hocon/impl/simple_includer.rb +196 -1
  77. data/lib/hocon/impl/substitution_expression.rb +38 -0
  78. data/lib/hocon/impl/token.rb +17 -4
  79. data/lib/hocon/impl/token_type.rb +6 -2
  80. data/lib/hocon/impl/tokenizer.rb +339 -109
  81. data/lib/hocon/impl/tokens.rb +330 -79
  82. data/lib/hocon/impl/unmergeable.rb +14 -1
  83. data/lib/hocon/impl/unsupported_operation_error.rb +6 -0
  84. data/lib/hocon/impl/url.rb +37 -0
  85. data/lib/hocon/parser.rb +7 -0
  86. data/lib/hocon/parser/config_document.rb +92 -0
  87. data/lib/hocon/parser/config_document_factory.rb +36 -0
  88. data/lib/hocon/parser/config_node.rb +30 -0
  89. metadata +67 -43
  90. data/lib/hocon/impl/config_float.rb +0 -13
  91. data/lib/hocon/impl/parser.rb +0 -977
  92. data/lib/hocon/impl/properties_parser.rb +0 -83
@@ -0,0 +1,38 @@
1
+ require 'hocon/impl'
2
+
3
+
4
+ class Hocon::Impl::SubstitutionExpression
5
+
6
+ def initialize(path, optional)
7
+ @path = path
8
+ @optional = optional
9
+ end
10
+ attr_reader :path, :optional
11
+
12
+ def change_path(new_path)
13
+ if new_path == @path
14
+ self
15
+ else
16
+ Hocon::Impl::SubstitutionExpression.new(new_path, @optional)
17
+ end
18
+ end
19
+
20
+ def to_s
21
+ "${#{@optional ? "?" : ""}#{@path.render}}"
22
+ end
23
+
24
+ def ==(other)
25
+ if other.is_a? Hocon::Impl::SubstitutionExpression
26
+ other.path == @path && other.optional == @optional
27
+ else
28
+ false
29
+ end
30
+ end
31
+
32
+ def hash
33
+ h = 41 * (41 + @path.hash)
34
+ h = 41 * (h + (optional ? 1 : 0))
35
+
36
+ h
37
+ end
38
+ end
@@ -1,14 +1,18 @@
1
+ # encoding: utf-8
2
+
1
3
  require 'hocon/impl'
2
4
  require 'hocon/impl/token_type'
3
5
 
4
6
  class Hocon::Impl::Token
5
- def self.new_without_origin(token_type, debug_string)
6
- Hocon::Impl::Token.new(token_type, nil, debug_string)
7
+ attr_reader :token_type, :token_text
8
+ def self.new_without_origin(token_type, debug_string, token_text)
9
+ Hocon::Impl::Token.new(token_type, nil, token_text, debug_string)
7
10
  end
8
11
 
9
- def initialize(token_type, origin, debug_string = nil)
12
+ def initialize(token_type, origin, token_text = nil, debug_string = nil)
10
13
  @token_type = token_type
11
14
  @origin = origin
15
+ @token_text = token_text
12
16
  @debug_string = debug_string
13
17
  end
14
18
 
@@ -29,4 +33,13 @@ class Hocon::Impl::Token
29
33
  Hocon::Impl::TokenType.name(@token_type)
30
34
  end
31
35
  end
32
- end
36
+
37
+ def ==(other)
38
+ # @origin deliberately left out
39
+ other.is_a?(Hocon::Impl::Token) && @token_type == other.token_type
40
+ end
41
+
42
+ def hash
43
+ @token_type.hash
44
+ end
45
+ end
@@ -1,3 +1,5 @@
1
+ # encoding: utf-8
2
+
1
3
  require 'hocon/impl'
2
4
 
3
5
  class Hocon::Impl::TokenType
@@ -17,6 +19,7 @@ class Hocon::Impl::TokenType
17
19
  PROBLEM = 13
18
20
  COMMENT = 14
19
21
  PLUS_EQUALS = 15
22
+ IGNORED_WHITESPACE = 16
20
23
 
21
24
  def self.name(token_type)
22
25
  case token_type
@@ -36,7 +39,8 @@ class Hocon::Impl::TokenType
36
39
  when PROBLEM then "PROBLEM"
37
40
  when COMMENT then "COMMENT"
38
41
  when PLUS_EQUALS then "PLUS_EQUALS"
39
- else raise ConfigBugError, "Unrecognized token type #{token_type}"
42
+ when IGNORED_WHITESPACE then "IGNORED_WHITESPACE"
43
+ else raise ConfigBugOrBrokenError, "Unrecognized token type #{token_type}"
40
44
  end
41
45
  end
42
- end
46
+ end
@@ -1,3 +1,5 @@
1
+ # encoding: utf-8
2
+
1
3
  require 'hocon/impl'
2
4
  require 'hocon/impl/config_impl_util'
3
5
  require 'hocon/impl/tokens'
@@ -7,6 +9,7 @@ require 'forwardable'
7
9
 
8
10
  class Hocon::Impl::Tokenizer
9
11
  Tokens = Hocon::Impl::Tokens
12
+ ConfigBugOrBrokenError = Hocon::ConfigError::ConfigBugOrBrokenError
10
13
 
11
14
  class TokenizerProblemError < StandardError
12
15
  def initialize(problem)
@@ -18,8 +21,36 @@ class Hocon::Impl::Tokenizer
18
21
  end
19
22
  end
20
23
 
24
+ def self.as_string(codepoint)
25
+ if codepoint == "\n"
26
+ "newline"
27
+ elsif codepoint == "\t"
28
+ "tab"
29
+ elsif codepoint == -1
30
+ "end of file"
31
+ elsif codepoint =~ /[[:cntrl:]]/
32
+ "control character 0x%x" % codepoint
33
+ else
34
+ "%c" % codepoint
35
+ end
36
+ end
37
+
38
+ # Tokenizes a Reader. Does not close the reader; you have to arrange to do
39
+ # that after you're done with the returned iterator.
40
+ def self.tokenize(origin, input, syntax)
41
+ TokenIterator.new(origin, input, syntax != Hocon::ConfigSyntax::JSON)
42
+ end
43
+
44
+ def self.render(tokens)
45
+ rendered_text = ""
46
+ while (t = tokens.next)
47
+ rendered_text << t.token_text
48
+ end
49
+ rendered_text
50
+ end
51
+
21
52
  class TokenIterator
22
- extend Forwardable
53
+
23
54
  class WhitespaceSaver
24
55
  def initialize
25
56
  @whitespace = StringIO.new
@@ -27,17 +58,14 @@ class Hocon::Impl::Tokenizer
27
58
  end
28
59
 
29
60
  def add(c)
30
- if @last_token_was_simple_value
31
- @whitespace << c
32
- end
61
+ @whitespace << c
33
62
  end
34
63
 
35
64
  def check(t, base_origin, line_number)
36
- if Hocon::Impl::Tokenizer::TokenIterator.simple_value?(t)
65
+ if TokenIterator.simple_value?(t)
37
66
  next_is_a_simple_value(base_origin, line_number)
38
67
  else
39
- next_is_not_a_simple_value
40
- nil
68
+ next_is_not_a_simple_value(base_origin, line_number)
41
69
  end
42
70
  end
43
71
 
@@ -45,47 +73,71 @@ class Hocon::Impl::Tokenizer
45
73
  # called if the next token is not a simple value;
46
74
  # discards any whitespace we were saving between
47
75
  # simple values.
48
- def next_is_not_a_simple_value
76
+ def next_is_not_a_simple_value(base_origin, line_number)
49
77
  @last_token_was_simple_value = false
50
- @whitespace.reopen("")
78
+ create_whitespace_token_from_saver(base_origin, line_number)
51
79
  end
52
80
 
53
81
  # called if the next token IS a simple value,
54
82
  # so creates a whitespace token if the previous
55
83
  # token also was.
56
84
  def next_is_a_simple_value(base_origin, line_number)
57
- if @last_token_was_simple_value
58
- # need to save whitespace between the two so
59
- # the parser has the option to concatenate it.
60
- if @whitespace.length > 0
61
- Tokens.new_unquoted_text(
62
- line_origin(base_origin, line_number),
63
- @whitespace.string = ""
64
- )
65
- end
66
- end
85
+ t = create_whitespace_token_from_saver(base_origin, line_number)
86
+ @last_token_was_simple_value = true unless @last_token_was_simple_value
87
+ t
67
88
  end
68
89
 
90
+ def create_whitespace_token_from_saver(base_origin, line_number)
91
+ return nil unless @whitespace.length > 0
92
+ if (@last_token_was_simple_value)
93
+ t = Tokens.new_unquoted_text(
94
+ Hocon::Impl::Tokenizer::TokenIterator.line_origin(base_origin, line_number),
95
+ String.new(@whitespace.string)
96
+ )
97
+ else
98
+ t = Tokens.new_ignored_whitespace(
99
+ Hocon::Impl::Tokenizer::TokenIterator.line_origin(base_origin, line_number),
100
+ String.new(@whitespace.string)
101
+ )
102
+ end
103
+ @whitespace.string = ""
104
+ t
105
+ end
69
106
  end
70
107
 
71
- # chars JSON allows a number to start with
72
- FIRST_NUMBER_CHARS = "0123456789-"
73
- # chars JSON allows to be part of a number
74
- NUMBER_CHARS = "0123456789eE+-."
75
- # chars that stop an unquoted string
76
- NOT_IN_UNQUOTED_TEXT = "$\"{}[]:=,+#`^?!@*&\\"
108
+ def initialize(origin, input, allow_comments)
109
+ @origin = origin
110
+ @input = input
111
+ @allow_comments = allow_comments
112
+ @buffer = []
113
+ @line_number = 1
114
+ @line_origin = @origin.with_line_number(@line_number)
115
+ @tokens = []
116
+ @tokens << Tokens::START
117
+ @whitespace_saver = WhitespaceSaver.new
118
+ end
77
119
 
78
- def self.problem(origin, what, message, suggest_quotes, cause)
79
- if what.nil? || message.nil?
80
- throw Hocon::ConfigError::ConfigBugOrBrokenError.new("internal error, creating bad TokenizerProblemError", nil)
120
+ # this should ONLY be called from nextCharSkippingComments
121
+ # or when inside a quoted string, or when parsing a sequence
122
+ # like ${ or +=, everything else should use
123
+ # nextCharSkippingComments().
124
+ def next_char_raw
125
+ if @buffer.empty?
126
+ begin
127
+ @input.readchar.chr
128
+ rescue EOFError
129
+ -1
130
+ end
131
+ else
132
+ @buffer.pop
81
133
  end
82
- TokenizerProblemError.new(Tokens.new_problem(origin, what, message, suggest_quotes, cause))
83
134
  end
84
135
 
85
- def self.simple_value?(t)
86
- Tokens.substitution?(t) ||
87
- Tokens.unquoted_text?(t) ||
88
- Tokens.value?(t)
136
+ def put_back(c)
137
+ if @buffer.length > 2
138
+ raise ConfigBugOrBrokenError, "bug: putBack() three times, undesirable look-ahead"
139
+ end
140
+ @buffer.push(c)
89
141
  end
90
142
 
91
143
  def self.whitespace?(c)
@@ -96,20 +148,6 @@ class Hocon::Impl::Tokenizer
96
148
  (c != "\n") and (Hocon::Impl::ConfigImplUtil.whitespace?(c))
97
149
  end
98
150
 
99
- def_delegator :@tokens, :each
100
-
101
- def initialize(origin, input, allow_comments)
102
- @origin = origin
103
- @input = input
104
- @allow_comments = allow_comments
105
- @buffer = []
106
- @line_number = 1
107
- @line_origin = @origin.set_line_number(@line_number)
108
- @tokens = []
109
- @tokens << Tokens::START
110
- @whitespace_saver = WhitespaceSaver.new
111
- end
112
-
113
151
  def start_of_comment?(c)
114
152
  if c == -1
115
153
  false
@@ -133,25 +171,7 @@ class Hocon::Impl::Tokenizer
133
171
  end
134
172
  end
135
173
 
136
- def put_back(c)
137
- if @buffer.length > 2
138
- raise ConfigBugError, "bug: putBack() three times, undesirable look-ahead"
139
- end
140
- @buffer.push(c)
141
- end
142
-
143
- def next_char_raw
144
- if @buffer.empty?
145
- begin
146
- @input.readchar.chr
147
- rescue EOFError
148
- -1
149
- end
150
- else
151
- @buffer.pop
152
- end
153
- end
154
-
174
+ # get next char, skipping non-newline whitespace
155
175
  def next_char_after_whitespace(saver)
156
176
  while true
157
177
  c = next_char_raw
@@ -167,6 +187,53 @@ class Hocon::Impl::Tokenizer
167
187
  end
168
188
  end
169
189
 
190
+ def self.problem(origin, what, message, suggest_quotes, cause)
191
+ if what.nil? || message.nil?
192
+ raise ConfigBugOrBrokenError.new("internal error, creating bad TokenizerProblemError")
193
+ end
194
+ TokenizerProblemError.new(Tokens.new_problem(origin, what, message, suggest_quotes, cause))
195
+ end
196
+
197
+ def self.line_origin(base_origin, line_number)
198
+ base_origin.with_line_number(line_number)
199
+ end
200
+
201
+ # ONE char has always been consumed, either the # or the first /, but not
202
+ # both slashes
203
+ def pull_comment(first_char)
204
+ double_slash = false
205
+ if first_char == '/'
206
+ discard = next_char_raw
207
+ if discard != '/'
208
+ raise ConfigBugOrBrokenError, "called pullComment but // not seen"
209
+ end
210
+ double_slash = true
211
+ end
212
+
213
+ io = StringIO.new
214
+ while true
215
+ c = next_char_raw
216
+ if (c == -1) || (c == "\n")
217
+ put_back(c)
218
+ if (double_slash)
219
+ return Tokens.new_comment_double_slash(@line_origin, io.string)
220
+ else
221
+ return Tokens.new_comment_hash(@line_origin, io.string)
222
+ end
223
+ else
224
+ io << c
225
+ end
226
+ end
227
+ end
228
+
229
+ # chars JSON allows a number to start with
230
+ FIRST_NUMBER_CHARS = "0123456789-"
231
+ # chars JSON allows to be part of a number
232
+ NUMBER_CHARS = "0123456789eE+-."
233
+ # chars that stop an unquoted string
234
+ NOT_IN_UNQUOTED_TEXT = "$\"{}[]:=,+#`^?!@*&\\"
235
+
236
+
170
237
  # The rules here are intended to maximize convenience while
171
238
  # avoiding confusion with real valid JSON. Basically anything
172
239
  # that parses as JSON is treated the JSON way and otherwise
@@ -209,27 +276,6 @@ class Hocon::Impl::Tokenizer
209
276
  Tokens.new_unquoted_text(origin, io.string)
210
277
  end
211
278
 
212
-
213
- def pull_comment(first_char)
214
- if first_char == '/'
215
- discard = next_char_raw
216
- if discard != '/'
217
- raise ConfigBugError, "called pullComment but // not seen"
218
- end
219
- end
220
-
221
- io = StringIO.new
222
- while true
223
- c = next_char_raw
224
- if (c == -1) || (c == "\n")
225
- put_back(c)
226
- return Tokens.new_comment(@line_origin, io.string)
227
- else
228
- io << c
229
- end
230
- end
231
- end
232
-
233
279
  def pull_number(first_char)
234
280
  sb = StringIO.new
235
281
  sb << first_char
@@ -251,17 +297,17 @@ class Hocon::Impl::Tokenizer
251
297
  begin
252
298
  if contained_decimal_or_e
253
299
  # force floating point representation
254
- Tokens.new_double(@line_origin, s.to_f, s)
300
+ Tokens.new_double(@line_origin, Float(s), s)
255
301
  else
256
- Tokens.new_long(@line_origin, s.to_i, s)
302
+ Tokens.new_long(@line_origin, Integer(s), s)
257
303
  end
258
304
  rescue ArgumentError => e
259
305
  if e.message =~ /^invalid value for (Float|Integer)\(\)/
260
306
  # not a number after all, see if it's an unquoted string.
261
- s.each do |u|
262
- if NOT_IN_UNQUOTED_TEXT.index
263
- raise self.problem(@line_origin, u, "Reserved character '#{u}'" +
264
- "is not allowed outside quotes", true, nil)
307
+ s.each_char do |u|
308
+ if NOT_IN_UNQUOTED_TEXT.index(u)
309
+ raise self.class.problem(@line_origin, u, "Reserved character '#{u}'" +
310
+ "is not allowed outside quotes", true, nil)
265
311
  end
266
312
  end
267
313
  # no evil chars so we just decide this was a string and
@@ -273,25 +319,125 @@ class Hocon::Impl::Tokenizer
273
319
  end
274
320
  end
275
321
 
322
+ def pull_escape_sequence(sb, sb_orig)
323
+ escaped = next_char_raw
324
+
325
+ if escaped == -1
326
+ error_msg = "End of input but backslash in string had nothing after it"
327
+ raise self.class.problem(@line_origin, "", error_msg, false, nil)
328
+ end
329
+
330
+ # This is needed so we return the unescaped escape characters back out when rendering
331
+ # the token
332
+ sb_orig << "\\" << escaped
333
+
334
+ case escaped
335
+ when "\""
336
+ sb << "\""
337
+ when "\\"
338
+ sb << "\\"
339
+ when "/"
340
+ sb << "/"
341
+ when "b"
342
+ sb << "\b"
343
+ when "f"
344
+ sb << "\f"
345
+ when "n"
346
+ sb << "\n"
347
+ when "r"
348
+ sb << "\r"
349
+ when "t"
350
+ sb << "\t"
351
+ when "u"
352
+ codepoint = ""
353
+
354
+ # Grab the 4 hex chars for the unicode character
355
+ 4.times do
356
+ c = next_char_raw
357
+
358
+ if c == -1
359
+ error_msg = "End of input but expecting 4 hex digits for \\uXXXX escape"
360
+ raise self.class.problem(@line_origin, c, error_msg, false, nil)
361
+ end
362
+
363
+ codepoint << c
364
+ end
365
+ sb_orig << codepoint
366
+ # Convert codepoint to a unicode character
367
+ packed = [codepoint.hex].pack("U")
368
+ if packed == "_"
369
+ raise self.class.problem(@line_origin, codepoint,
370
+ "Malformed hex digits after \\u escape in string: '#{codepoint}'",
371
+ false, nil)
372
+ end
373
+ sb << packed
374
+ else
375
+ error_msg = "backslash followed by '#{escaped}', this is not a valid escape sequence (quoted strings use JSON escaping, so use double-backslash \\ for literal backslash)"
376
+ raise self.class.problem(Hocon::Impl::Tokenizer.as_string(escaped), "", error_msg, false, nil)
377
+ end
378
+ end
379
+
380
+ def append_triple_quoted_string(sb, sb_orig)
381
+ # we are after the opening triple quote and need to consume the
382
+ # close triple
383
+ consecutive_quotes = 0
384
+
385
+ while true
386
+ c = next_char_raw
387
+
388
+ if c == '"'
389
+ consecutive_quotes += 1
390
+ elsif consecutive_quotes >= 3
391
+ # the last three quotes end the string and the other kept.
392
+ sb.string = sb.string[0...-3]
393
+ put_back c
394
+ break
395
+ else
396
+ consecutive_quotes = 0
397
+ if c == -1
398
+ error_msg = "End of input but triple-quoted string was still open"
399
+ raise self.class.problem(@line_origin, c, error_msg, false, nil)
400
+ elsif c == "\n"
401
+ # keep the line number accurate
402
+ @line_number += 1
403
+ @line_origin = @origin.with_line_number(@line_number)
404
+ end
405
+ end
406
+
407
+ sb << c
408
+ sb_orig << c
409
+ end
410
+ end
411
+
276
412
  def pull_quoted_string
277
413
  # the open quote has already been consumed
278
414
  sb = StringIO.new
415
+
416
+ # We need a second StringIO to keep track of escape characters.
417
+ # We want to return them exactly as they appeared in the original text,
418
+ # which means we will need a new StringIO to escape escape characters
419
+ # so we can also keep the actual value of the string. This is gross.
420
+ sb_orig = StringIO.new
421
+ sb_orig << '"'
422
+
279
423
  c = ""
280
424
  while c != '"'
281
425
  c = next_char_raw
282
426
  if c == -1
283
- raise self.problem(@line_origin, c, "End of input but string quote was still open", false, nil)
427
+ raise self.class.problem(@line_origin, c, "End of input but string quote was still open", false, nil)
284
428
  end
285
429
 
286
430
  if c == "\\"
287
- pull_escape_sequence(sb)
431
+ pull_escape_sequence(sb, sb_orig)
288
432
  elsif c == '"'
433
+ sb_orig << c
289
434
  # done!
290
435
  elsif c =~ /[[:cntrl:]]/
291
- raise self.problem(@line_origin, c, "JSON does not allow unescaped #{c}" +
292
- " in quoted strings, use a backslash escape", false, nil)
436
+ raise self.class.problem(@line_origin, c, "JSON does not allow unescaped #{c}" +
437
+ " in quoted strings, use a backslash escape", false, nil)
293
438
  else
294
439
  sb << c
440
+ sb_orig << c
295
441
  end
296
442
  end
297
443
 
@@ -299,13 +445,69 @@ class Hocon::Impl::Tokenizer
299
445
  if sb.length == 0
300
446
  third = next_char_raw
301
447
  if third == '"'
302
- append_triple_quoted_string(sb)
448
+ sb_orig << third
449
+ append_triple_quoted_string(sb, sb_orig)
303
450
  else
304
451
  put_back(third)
305
452
  end
306
453
  end
307
454
 
308
- Tokens.new_string(@line_origin, sb.string)
455
+ Tokens.new_string(@line_origin, sb.string, sb_orig.string)
456
+ end
457
+
458
+ def pull_plus_equals
459
+ # the initial '+' has already been consumed
460
+ c = next_char_raw
461
+
462
+ unless c == '='
463
+ error_msg = "'+' not followed by =, '#{c}' not allowed after '+'"
464
+ raise self.class.problem(@line_origin, c, error_msg, true, nil) # true = suggest quotes
465
+ end
466
+
467
+ Tokens::PLUS_EQUALS
468
+ end
469
+
470
+ def pull_substitution
471
+ # the initial '$' has already been consumed
472
+ c = next_char_raw
473
+ if c != '{'
474
+ error_msg = "'$' not followed by {, '#{c}' not allowed after '$'"
475
+ raise self.class.problem(@line_origin, c, error_msg, true, nil) # true = suggest quotes
476
+ end
477
+
478
+ optional = false
479
+ c = next_char_raw
480
+
481
+ if c == '?'
482
+ optional = true
483
+ else
484
+ put_back(c)
485
+ end
486
+
487
+ saver = WhitespaceSaver.new
488
+ expression = []
489
+
490
+ while true
491
+ t = pull_next_token(saver)
492
+ # note that we avoid validating the allowed tokens inside
493
+ # the substitution here; we even allow nested substitutions
494
+ # in the tokenizer. The parser sorts it out.
495
+
496
+ if t == Tokens::CLOSE_CURLY
497
+ # end the loop, done!
498
+ break
499
+ elsif t == Tokens::EOF
500
+ raise self.class.problem(@line_origin, t, "Substitution ${ was not closed with a }", false, nil)
501
+ else
502
+ whitespace = saver.check(t, @line_origin, @line_number)
503
+ unless whitespace.nil?
504
+ expression << whitespace
505
+ end
506
+ expression << t
507
+ end
508
+ end
509
+
510
+ Tokens.new_substitution(@line_origin, optional, expression)
309
511
  end
310
512
 
311
513
  def pull_next_token(saver)
@@ -316,7 +518,7 @@ class Hocon::Impl::Tokenizer
316
518
  # newline tokens have the just-ended line number
317
519
  line = Tokens.new_line(@line_origin)
318
520
  @line_number += 1
319
- @line_origin = @origin.set_line_number(@line_number)
521
+ @line_origin = @origin.with_line_number(@line_number)
320
522
  line
321
523
  else
322
524
  t = nil
@@ -341,7 +543,7 @@ class Hocon::Impl::Tokenizer
341
543
  if FIRST_NUMBER_CHARS.index(c)
342
544
  t = pull_number(c)
343
545
  elsif NOT_IN_UNQUOTED_TEXT.index(c)
344
- raise Hocon::Impl::Tokenizer::TokenIterator.problem(@line_origin, c, "Reserved character '#{c}' is not allowed outside quotes", true, nil)
546
+ raise self.class.problem(@line_origin, c, "Reserved character '#{c}' is not allowed outside quotes", true, nil)
345
547
  else
346
548
  put_back(c)
347
549
  t = pull_unquoted_text
@@ -350,13 +552,19 @@ class Hocon::Impl::Tokenizer
350
552
  end
351
553
 
352
554
  if t.nil?
353
- raise ConfigBugError, "bug: failed to generate next token"
555
+ raise ConfigBugOrBrokenError, "bug: failed to generate next token"
354
556
  end
355
557
 
356
558
  t
357
559
  end
358
560
  end
359
561
 
562
+ def self.simple_value?(t)
563
+ Tokens.substitution?(t) ||
564
+ Tokens.unquoted_text?(t) ||
565
+ Tokens.value?(t)
566
+ end
567
+
360
568
  def queue_next_token
361
569
  t = pull_next_token(@whitespace_saver)
362
570
  whitespace = @whitespace_saver.check(t, @origin, @line_number)
@@ -366,6 +574,10 @@ class Hocon::Impl::Tokenizer
366
574
  @tokens.push(t)
367
575
  end
368
576
 
577
+ def has_next?
578
+ !@tokens.empty?
579
+ end
580
+
369
581
  def next
370
582
  t = @tokens.shift
371
583
  if (@tokens.empty?) and (t != Tokens::EOF)
@@ -375,19 +587,37 @@ class Hocon::Impl::Tokenizer
375
587
  @tokens.push(e.problem)
376
588
  end
377
589
  if @tokens.empty?
378
- raise ConfigBugError, "bug: tokens queue should not be empty here"
590
+ raise ConfigBugOrBrokenError, "bug: tokens queue should not be empty here"
379
591
  end
380
592
  end
381
593
  t
382
594
  end
383
595
 
384
- def empty?
385
- @tokens.empty?
596
+ def remove
597
+ raise ConfigBugOrBrokenError, "Does not make sense to remove items from token stream"
386
598
  end
387
- end
388
599
 
600
+ def each
601
+ while has_next?
602
+ # Have to use self.next instead of next because next is a reserved word
603
+ yield self.next
604
+ end
605
+ end
606
+
607
+ def map
608
+ token_list = []
609
+ each do |token|
610
+ # yield token to calling method, append whatever is returned from the
611
+ # map block to token_list
612
+ token_list << yield(token)
613
+ end
614
+ token_list
615
+ end
616
+
617
+ def to_list
618
+ # Return array of tokens from the iterator
619
+ self.map { |token| token }
620
+ end
389
621
 
390
- def self.tokenize(origin, input, syntax)
391
- TokenIterator.new(origin, input, syntax != Hocon::ConfigSyntax::JSON)
392
622
  end
393
- end
623
+ end