hocon 0.0.7 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/README.md +4 -2
- data/lib/hocon.rb +2 -0
- data/lib/hocon/config.rb +1010 -0
- data/lib/hocon/config_error.rb +32 -2
- data/lib/hocon/config_factory.rb +46 -0
- data/lib/hocon/config_include_context.rb +49 -0
- data/lib/hocon/config_includer_file.rb +27 -0
- data/lib/hocon/config_list.rb +49 -0
- data/lib/hocon/config_mergeable.rb +74 -0
- data/lib/hocon/config_object.rb +144 -1
- data/lib/hocon/config_parse_options.rb +33 -9
- data/lib/hocon/config_parseable.rb +51 -0
- data/lib/hocon/config_render_options.rb +4 -2
- data/lib/hocon/config_resolve_options.rb +31 -0
- data/lib/hocon/config_syntax.rb +5 -2
- data/lib/hocon/config_util.rb +73 -0
- data/lib/hocon/config_value.rb +122 -0
- data/lib/hocon/config_value_factory.rb +66 -2
- data/lib/hocon/config_value_type.rb +5 -2
- data/lib/hocon/impl.rb +2 -0
- data/lib/hocon/impl/abstract_config_node.rb +29 -0
- data/lib/hocon/impl/abstract_config_node_value.rb +11 -0
- data/lib/hocon/impl/abstract_config_object.rb +148 -42
- data/lib/hocon/impl/abstract_config_value.rb +251 -11
- data/lib/hocon/impl/array_iterator.rb +19 -0
- data/lib/hocon/impl/config_boolean.rb +7 -1
- data/lib/hocon/impl/config_concatenation.rb +177 -28
- data/lib/hocon/impl/config_delayed_merge.rb +329 -0
- data/lib/hocon/impl/config_delayed_merge_object.rb +274 -0
- data/lib/hocon/impl/config_document_parser.rb +647 -0
- data/lib/hocon/impl/config_double.rb +44 -0
- data/lib/hocon/impl/config_impl.rb +143 -19
- data/lib/hocon/impl/config_impl_util.rb +18 -0
- data/lib/hocon/impl/config_include_kind.rb +10 -0
- data/lib/hocon/impl/config_int.rb +13 -1
- data/lib/hocon/impl/config_node_array.rb +11 -0
- data/lib/hocon/impl/config_node_comment.rb +19 -0
- data/lib/hocon/impl/config_node_complex_value.rb +54 -0
- data/lib/hocon/impl/config_node_concatenation.rb +11 -0
- data/lib/hocon/impl/config_node_field.rb +81 -0
- data/lib/hocon/impl/config_node_include.rb +33 -0
- data/lib/hocon/impl/config_node_object.rb +276 -0
- data/lib/hocon/impl/config_node_path.rb +48 -0
- data/lib/hocon/impl/config_node_root.rb +60 -0
- data/lib/hocon/impl/config_node_simple_value.rb +42 -0
- data/lib/hocon/impl/config_node_single_token.rb +17 -0
- data/lib/hocon/impl/config_null.rb +15 -7
- data/lib/hocon/impl/config_number.rb +43 -4
- data/lib/hocon/impl/config_parser.rb +403 -0
- data/lib/hocon/impl/config_reference.rb +142 -0
- data/lib/hocon/impl/config_string.rb +55 -7
- data/lib/hocon/impl/container.rb +29 -0
- data/lib/hocon/impl/default_transformer.rb +24 -15
- data/lib/hocon/impl/from_map_mode.rb +3 -1
- data/lib/hocon/impl/full_includer.rb +2 -0
- data/lib/hocon/impl/memo_key.rb +42 -0
- data/lib/hocon/impl/mergeable_value.rb +8 -0
- data/lib/hocon/impl/origin_type.rb +8 -2
- data/lib/hocon/impl/parseable.rb +455 -91
- data/lib/hocon/impl/path.rb +181 -59
- data/lib/hocon/impl/path_builder.rb +24 -3
- data/lib/hocon/impl/path_parser.rb +280 -0
- data/lib/hocon/impl/replaceable_merge_stack.rb +22 -0
- data/lib/hocon/impl/resolve_context.rb +254 -0
- data/lib/hocon/impl/resolve_memos.rb +21 -0
- data/lib/hocon/impl/resolve_result.rb +39 -0
- data/lib/hocon/impl/resolve_source.rb +354 -0
- data/lib/hocon/impl/resolve_status.rb +3 -1
- data/lib/hocon/impl/simple_config.rb +264 -10
- data/lib/hocon/impl/simple_config_document.rb +48 -0
- data/lib/hocon/impl/simple_config_list.rb +282 -8
- data/lib/hocon/impl/simple_config_object.rb +424 -88
- data/lib/hocon/impl/simple_config_origin.rb +263 -71
- data/lib/hocon/impl/simple_include_context.rb +31 -1
- data/lib/hocon/impl/simple_includer.rb +196 -1
- data/lib/hocon/impl/substitution_expression.rb +38 -0
- data/lib/hocon/impl/token.rb +17 -4
- data/lib/hocon/impl/token_type.rb +6 -2
- data/lib/hocon/impl/tokenizer.rb +339 -109
- data/lib/hocon/impl/tokens.rb +330 -79
- data/lib/hocon/impl/unmergeable.rb +14 -1
- data/lib/hocon/impl/unsupported_operation_error.rb +6 -0
- data/lib/hocon/impl/url.rb +37 -0
- data/lib/hocon/parser.rb +7 -0
- data/lib/hocon/parser/config_document.rb +92 -0
- data/lib/hocon/parser/config_document_factory.rb +36 -0
- data/lib/hocon/parser/config_node.rb +30 -0
- metadata +67 -43
- data/lib/hocon/impl/config_float.rb +0 -13
- data/lib/hocon/impl/parser.rb +0 -977
- data/lib/hocon/impl/properties_parser.rb +0 -83
@@ -0,0 +1,38 @@
|
|
1
|
+
require 'hocon/impl'
|
2
|
+
|
3
|
+
|
4
|
+
class Hocon::Impl::SubstitutionExpression
|
5
|
+
|
6
|
+
def initialize(path, optional)
|
7
|
+
@path = path
|
8
|
+
@optional = optional
|
9
|
+
end
|
10
|
+
attr_reader :path, :optional
|
11
|
+
|
12
|
+
def change_path(new_path)
|
13
|
+
if new_path == @path
|
14
|
+
self
|
15
|
+
else
|
16
|
+
Hocon::Impl::SubstitutionExpression.new(new_path, @optional)
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def to_s
|
21
|
+
"${#{@optional ? "?" : ""}#{@path.render}}"
|
22
|
+
end
|
23
|
+
|
24
|
+
def ==(other)
|
25
|
+
if other.is_a? Hocon::Impl::SubstitutionExpression
|
26
|
+
other.path == @path && other.optional == @optional
|
27
|
+
else
|
28
|
+
false
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def hash
|
33
|
+
h = 41 * (41 + @path.hash)
|
34
|
+
h = 41 * (h + (optional ? 1 : 0))
|
35
|
+
|
36
|
+
h
|
37
|
+
end
|
38
|
+
end
|
data/lib/hocon/impl/token.rb
CHANGED
@@ -1,14 +1,18 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
1
3
|
require 'hocon/impl'
|
2
4
|
require 'hocon/impl/token_type'
|
3
5
|
|
4
6
|
class Hocon::Impl::Token
|
5
|
-
|
6
|
-
|
7
|
+
attr_reader :token_type, :token_text
|
8
|
+
def self.new_without_origin(token_type, debug_string, token_text)
|
9
|
+
Hocon::Impl::Token.new(token_type, nil, token_text, debug_string)
|
7
10
|
end
|
8
11
|
|
9
|
-
def initialize(token_type, origin, debug_string = nil)
|
12
|
+
def initialize(token_type, origin, token_text = nil, debug_string = nil)
|
10
13
|
@token_type = token_type
|
11
14
|
@origin = origin
|
15
|
+
@token_text = token_text
|
12
16
|
@debug_string = debug_string
|
13
17
|
end
|
14
18
|
|
@@ -29,4 +33,13 @@ class Hocon::Impl::Token
|
|
29
33
|
Hocon::Impl::TokenType.name(@token_type)
|
30
34
|
end
|
31
35
|
end
|
32
|
-
|
36
|
+
|
37
|
+
def ==(other)
|
38
|
+
# @origin deliberately left out
|
39
|
+
other.is_a?(Hocon::Impl::Token) && @token_type == other.token_type
|
40
|
+
end
|
41
|
+
|
42
|
+
def hash
|
43
|
+
@token_type.hash
|
44
|
+
end
|
45
|
+
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
1
3
|
require 'hocon/impl'
|
2
4
|
|
3
5
|
class Hocon::Impl::TokenType
|
@@ -17,6 +19,7 @@ class Hocon::Impl::TokenType
|
|
17
19
|
PROBLEM = 13
|
18
20
|
COMMENT = 14
|
19
21
|
PLUS_EQUALS = 15
|
22
|
+
IGNORED_WHITESPACE = 16
|
20
23
|
|
21
24
|
def self.name(token_type)
|
22
25
|
case token_type
|
@@ -36,7 +39,8 @@ class Hocon::Impl::TokenType
|
|
36
39
|
when PROBLEM then "PROBLEM"
|
37
40
|
when COMMENT then "COMMENT"
|
38
41
|
when PLUS_EQUALS then "PLUS_EQUALS"
|
39
|
-
|
42
|
+
when IGNORED_WHITESPACE then "IGNORED_WHITESPACE"
|
43
|
+
else raise ConfigBugOrBrokenError, "Unrecognized token type #{token_type}"
|
40
44
|
end
|
41
45
|
end
|
42
|
-
end
|
46
|
+
end
|
data/lib/hocon/impl/tokenizer.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
1
3
|
require 'hocon/impl'
|
2
4
|
require 'hocon/impl/config_impl_util'
|
3
5
|
require 'hocon/impl/tokens'
|
@@ -7,6 +9,7 @@ require 'forwardable'
|
|
7
9
|
|
8
10
|
class Hocon::Impl::Tokenizer
|
9
11
|
Tokens = Hocon::Impl::Tokens
|
12
|
+
ConfigBugOrBrokenError = Hocon::ConfigError::ConfigBugOrBrokenError
|
10
13
|
|
11
14
|
class TokenizerProblemError < StandardError
|
12
15
|
def initialize(problem)
|
@@ -18,8 +21,36 @@ class Hocon::Impl::Tokenizer
|
|
18
21
|
end
|
19
22
|
end
|
20
23
|
|
24
|
+
def self.as_string(codepoint)
|
25
|
+
if codepoint == "\n"
|
26
|
+
"newline"
|
27
|
+
elsif codepoint == "\t"
|
28
|
+
"tab"
|
29
|
+
elsif codepoint == -1
|
30
|
+
"end of file"
|
31
|
+
elsif codepoint =~ /[[:cntrl:]]/
|
32
|
+
"control character 0x%x" % codepoint
|
33
|
+
else
|
34
|
+
"%c" % codepoint
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
# Tokenizes a Reader. Does not close the reader; you have to arrange to do
|
39
|
+
# that after you're done with the returned iterator.
|
40
|
+
def self.tokenize(origin, input, syntax)
|
41
|
+
TokenIterator.new(origin, input, syntax != Hocon::ConfigSyntax::JSON)
|
42
|
+
end
|
43
|
+
|
44
|
+
def self.render(tokens)
|
45
|
+
rendered_text = ""
|
46
|
+
while (t = tokens.next)
|
47
|
+
rendered_text << t.token_text
|
48
|
+
end
|
49
|
+
rendered_text
|
50
|
+
end
|
51
|
+
|
21
52
|
class TokenIterator
|
22
|
-
|
53
|
+
|
23
54
|
class WhitespaceSaver
|
24
55
|
def initialize
|
25
56
|
@whitespace = StringIO.new
|
@@ -27,17 +58,14 @@ class Hocon::Impl::Tokenizer
|
|
27
58
|
end
|
28
59
|
|
29
60
|
def add(c)
|
30
|
-
|
31
|
-
@whitespace << c
|
32
|
-
end
|
61
|
+
@whitespace << c
|
33
62
|
end
|
34
63
|
|
35
64
|
def check(t, base_origin, line_number)
|
36
|
-
if
|
65
|
+
if TokenIterator.simple_value?(t)
|
37
66
|
next_is_a_simple_value(base_origin, line_number)
|
38
67
|
else
|
39
|
-
next_is_not_a_simple_value
|
40
|
-
nil
|
68
|
+
next_is_not_a_simple_value(base_origin, line_number)
|
41
69
|
end
|
42
70
|
end
|
43
71
|
|
@@ -45,47 +73,71 @@ class Hocon::Impl::Tokenizer
|
|
45
73
|
# called if the next token is not a simple value;
|
46
74
|
# discards any whitespace we were saving between
|
47
75
|
# simple values.
|
48
|
-
def next_is_not_a_simple_value
|
76
|
+
def next_is_not_a_simple_value(base_origin, line_number)
|
49
77
|
@last_token_was_simple_value = false
|
50
|
-
|
78
|
+
create_whitespace_token_from_saver(base_origin, line_number)
|
51
79
|
end
|
52
80
|
|
53
81
|
# called if the next token IS a simple value,
|
54
82
|
# so creates a whitespace token if the previous
|
55
83
|
# token also was.
|
56
84
|
def next_is_a_simple_value(base_origin, line_number)
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
if @whitespace.length > 0
|
61
|
-
Tokens.new_unquoted_text(
|
62
|
-
line_origin(base_origin, line_number),
|
63
|
-
@whitespace.string = ""
|
64
|
-
)
|
65
|
-
end
|
66
|
-
end
|
85
|
+
t = create_whitespace_token_from_saver(base_origin, line_number)
|
86
|
+
@last_token_was_simple_value = true unless @last_token_was_simple_value
|
87
|
+
t
|
67
88
|
end
|
68
89
|
|
90
|
+
def create_whitespace_token_from_saver(base_origin, line_number)
|
91
|
+
return nil unless @whitespace.length > 0
|
92
|
+
if (@last_token_was_simple_value)
|
93
|
+
t = Tokens.new_unquoted_text(
|
94
|
+
Hocon::Impl::Tokenizer::TokenIterator.line_origin(base_origin, line_number),
|
95
|
+
String.new(@whitespace.string)
|
96
|
+
)
|
97
|
+
else
|
98
|
+
t = Tokens.new_ignored_whitespace(
|
99
|
+
Hocon::Impl::Tokenizer::TokenIterator.line_origin(base_origin, line_number),
|
100
|
+
String.new(@whitespace.string)
|
101
|
+
)
|
102
|
+
end
|
103
|
+
@whitespace.string = ""
|
104
|
+
t
|
105
|
+
end
|
69
106
|
end
|
70
107
|
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
108
|
+
def initialize(origin, input, allow_comments)
|
109
|
+
@origin = origin
|
110
|
+
@input = input
|
111
|
+
@allow_comments = allow_comments
|
112
|
+
@buffer = []
|
113
|
+
@line_number = 1
|
114
|
+
@line_origin = @origin.with_line_number(@line_number)
|
115
|
+
@tokens = []
|
116
|
+
@tokens << Tokens::START
|
117
|
+
@whitespace_saver = WhitespaceSaver.new
|
118
|
+
end
|
77
119
|
|
78
|
-
|
79
|
-
|
80
|
-
|
120
|
+
# this should ONLY be called from nextCharSkippingComments
|
121
|
+
# or when inside a quoted string, or when parsing a sequence
|
122
|
+
# like ${ or +=, everything else should use
|
123
|
+
# nextCharSkippingComments().
|
124
|
+
def next_char_raw
|
125
|
+
if @buffer.empty?
|
126
|
+
begin
|
127
|
+
@input.readchar.chr
|
128
|
+
rescue EOFError
|
129
|
+
-1
|
130
|
+
end
|
131
|
+
else
|
132
|
+
@buffer.pop
|
81
133
|
end
|
82
|
-
TokenizerProblemError.new(Tokens.new_problem(origin, what, message, suggest_quotes, cause))
|
83
134
|
end
|
84
135
|
|
85
|
-
def
|
86
|
-
|
87
|
-
|
88
|
-
|
136
|
+
def put_back(c)
|
137
|
+
if @buffer.length > 2
|
138
|
+
raise ConfigBugOrBrokenError, "bug: putBack() three times, undesirable look-ahead"
|
139
|
+
end
|
140
|
+
@buffer.push(c)
|
89
141
|
end
|
90
142
|
|
91
143
|
def self.whitespace?(c)
|
@@ -96,20 +148,6 @@ class Hocon::Impl::Tokenizer
|
|
96
148
|
(c != "\n") and (Hocon::Impl::ConfigImplUtil.whitespace?(c))
|
97
149
|
end
|
98
150
|
|
99
|
-
def_delegator :@tokens, :each
|
100
|
-
|
101
|
-
def initialize(origin, input, allow_comments)
|
102
|
-
@origin = origin
|
103
|
-
@input = input
|
104
|
-
@allow_comments = allow_comments
|
105
|
-
@buffer = []
|
106
|
-
@line_number = 1
|
107
|
-
@line_origin = @origin.set_line_number(@line_number)
|
108
|
-
@tokens = []
|
109
|
-
@tokens << Tokens::START
|
110
|
-
@whitespace_saver = WhitespaceSaver.new
|
111
|
-
end
|
112
|
-
|
113
151
|
def start_of_comment?(c)
|
114
152
|
if c == -1
|
115
153
|
false
|
@@ -133,25 +171,7 @@ class Hocon::Impl::Tokenizer
|
|
133
171
|
end
|
134
172
|
end
|
135
173
|
|
136
|
-
|
137
|
-
if @buffer.length > 2
|
138
|
-
raise ConfigBugError, "bug: putBack() three times, undesirable look-ahead"
|
139
|
-
end
|
140
|
-
@buffer.push(c)
|
141
|
-
end
|
142
|
-
|
143
|
-
def next_char_raw
|
144
|
-
if @buffer.empty?
|
145
|
-
begin
|
146
|
-
@input.readchar.chr
|
147
|
-
rescue EOFError
|
148
|
-
-1
|
149
|
-
end
|
150
|
-
else
|
151
|
-
@buffer.pop
|
152
|
-
end
|
153
|
-
end
|
154
|
-
|
174
|
+
# get next char, skipping non-newline whitespace
|
155
175
|
def next_char_after_whitespace(saver)
|
156
176
|
while true
|
157
177
|
c = next_char_raw
|
@@ -167,6 +187,53 @@ class Hocon::Impl::Tokenizer
|
|
167
187
|
end
|
168
188
|
end
|
169
189
|
|
190
|
+
def self.problem(origin, what, message, suggest_quotes, cause)
|
191
|
+
if what.nil? || message.nil?
|
192
|
+
raise ConfigBugOrBrokenError.new("internal error, creating bad TokenizerProblemError")
|
193
|
+
end
|
194
|
+
TokenizerProblemError.new(Tokens.new_problem(origin, what, message, suggest_quotes, cause))
|
195
|
+
end
|
196
|
+
|
197
|
+
def self.line_origin(base_origin, line_number)
|
198
|
+
base_origin.with_line_number(line_number)
|
199
|
+
end
|
200
|
+
|
201
|
+
# ONE char has always been consumed, either the # or the first /, but not
|
202
|
+
# both slashes
|
203
|
+
def pull_comment(first_char)
|
204
|
+
double_slash = false
|
205
|
+
if first_char == '/'
|
206
|
+
discard = next_char_raw
|
207
|
+
if discard != '/'
|
208
|
+
raise ConfigBugOrBrokenError, "called pullComment but // not seen"
|
209
|
+
end
|
210
|
+
double_slash = true
|
211
|
+
end
|
212
|
+
|
213
|
+
io = StringIO.new
|
214
|
+
while true
|
215
|
+
c = next_char_raw
|
216
|
+
if (c == -1) || (c == "\n")
|
217
|
+
put_back(c)
|
218
|
+
if (double_slash)
|
219
|
+
return Tokens.new_comment_double_slash(@line_origin, io.string)
|
220
|
+
else
|
221
|
+
return Tokens.new_comment_hash(@line_origin, io.string)
|
222
|
+
end
|
223
|
+
else
|
224
|
+
io << c
|
225
|
+
end
|
226
|
+
end
|
227
|
+
end
|
228
|
+
|
229
|
+
# chars JSON allows a number to start with
|
230
|
+
FIRST_NUMBER_CHARS = "0123456789-"
|
231
|
+
# chars JSON allows to be part of a number
|
232
|
+
NUMBER_CHARS = "0123456789eE+-."
|
233
|
+
# chars that stop an unquoted string
|
234
|
+
NOT_IN_UNQUOTED_TEXT = "$\"{}[]:=,+#`^?!@*&\\"
|
235
|
+
|
236
|
+
|
170
237
|
# The rules here are intended to maximize convenience while
|
171
238
|
# avoiding confusion with real valid JSON. Basically anything
|
172
239
|
# that parses as JSON is treated the JSON way and otherwise
|
@@ -209,27 +276,6 @@ class Hocon::Impl::Tokenizer
|
|
209
276
|
Tokens.new_unquoted_text(origin, io.string)
|
210
277
|
end
|
211
278
|
|
212
|
-
|
213
|
-
def pull_comment(first_char)
|
214
|
-
if first_char == '/'
|
215
|
-
discard = next_char_raw
|
216
|
-
if discard != '/'
|
217
|
-
raise ConfigBugError, "called pullComment but // not seen"
|
218
|
-
end
|
219
|
-
end
|
220
|
-
|
221
|
-
io = StringIO.new
|
222
|
-
while true
|
223
|
-
c = next_char_raw
|
224
|
-
if (c == -1) || (c == "\n")
|
225
|
-
put_back(c)
|
226
|
-
return Tokens.new_comment(@line_origin, io.string)
|
227
|
-
else
|
228
|
-
io << c
|
229
|
-
end
|
230
|
-
end
|
231
|
-
end
|
232
|
-
|
233
279
|
def pull_number(first_char)
|
234
280
|
sb = StringIO.new
|
235
281
|
sb << first_char
|
@@ -251,17 +297,17 @@ class Hocon::Impl::Tokenizer
|
|
251
297
|
begin
|
252
298
|
if contained_decimal_or_e
|
253
299
|
# force floating point representation
|
254
|
-
Tokens.new_double(@line_origin, s
|
300
|
+
Tokens.new_double(@line_origin, Float(s), s)
|
255
301
|
else
|
256
|
-
Tokens.new_long(@line_origin, s
|
302
|
+
Tokens.new_long(@line_origin, Integer(s), s)
|
257
303
|
end
|
258
304
|
rescue ArgumentError => e
|
259
305
|
if e.message =~ /^invalid value for (Float|Integer)\(\)/
|
260
306
|
# not a number after all, see if it's an unquoted string.
|
261
|
-
s.
|
262
|
-
if NOT_IN_UNQUOTED_TEXT.index
|
263
|
-
raise self.problem(@line_origin, u, "Reserved character '#{u}'" +
|
264
|
-
|
307
|
+
s.each_char do |u|
|
308
|
+
if NOT_IN_UNQUOTED_TEXT.index(u)
|
309
|
+
raise self.class.problem(@line_origin, u, "Reserved character '#{u}'" +
|
310
|
+
"is not allowed outside quotes", true, nil)
|
265
311
|
end
|
266
312
|
end
|
267
313
|
# no evil chars so we just decide this was a string and
|
@@ -273,25 +319,125 @@ class Hocon::Impl::Tokenizer
|
|
273
319
|
end
|
274
320
|
end
|
275
321
|
|
322
|
+
def pull_escape_sequence(sb, sb_orig)
|
323
|
+
escaped = next_char_raw
|
324
|
+
|
325
|
+
if escaped == -1
|
326
|
+
error_msg = "End of input but backslash in string had nothing after it"
|
327
|
+
raise self.class.problem(@line_origin, "", error_msg, false, nil)
|
328
|
+
end
|
329
|
+
|
330
|
+
# This is needed so we return the unescaped escape characters back out when rendering
|
331
|
+
# the token
|
332
|
+
sb_orig << "\\" << escaped
|
333
|
+
|
334
|
+
case escaped
|
335
|
+
when "\""
|
336
|
+
sb << "\""
|
337
|
+
when "\\"
|
338
|
+
sb << "\\"
|
339
|
+
when "/"
|
340
|
+
sb << "/"
|
341
|
+
when "b"
|
342
|
+
sb << "\b"
|
343
|
+
when "f"
|
344
|
+
sb << "\f"
|
345
|
+
when "n"
|
346
|
+
sb << "\n"
|
347
|
+
when "r"
|
348
|
+
sb << "\r"
|
349
|
+
when "t"
|
350
|
+
sb << "\t"
|
351
|
+
when "u"
|
352
|
+
codepoint = ""
|
353
|
+
|
354
|
+
# Grab the 4 hex chars for the unicode character
|
355
|
+
4.times do
|
356
|
+
c = next_char_raw
|
357
|
+
|
358
|
+
if c == -1
|
359
|
+
error_msg = "End of input but expecting 4 hex digits for \\uXXXX escape"
|
360
|
+
raise self.class.problem(@line_origin, c, error_msg, false, nil)
|
361
|
+
end
|
362
|
+
|
363
|
+
codepoint << c
|
364
|
+
end
|
365
|
+
sb_orig << codepoint
|
366
|
+
# Convert codepoint to a unicode character
|
367
|
+
packed = [codepoint.hex].pack("U")
|
368
|
+
if packed == "_"
|
369
|
+
raise self.class.problem(@line_origin, codepoint,
|
370
|
+
"Malformed hex digits after \\u escape in string: '#{codepoint}'",
|
371
|
+
false, nil)
|
372
|
+
end
|
373
|
+
sb << packed
|
374
|
+
else
|
375
|
+
error_msg = "backslash followed by '#{escaped}', this is not a valid escape sequence (quoted strings use JSON escaping, so use double-backslash \\ for literal backslash)"
|
376
|
+
raise self.class.problem(Hocon::Impl::Tokenizer.as_string(escaped), "", error_msg, false, nil)
|
377
|
+
end
|
378
|
+
end
|
379
|
+
|
380
|
+
def append_triple_quoted_string(sb, sb_orig)
|
381
|
+
# we are after the opening triple quote and need to consume the
|
382
|
+
# close triple
|
383
|
+
consecutive_quotes = 0
|
384
|
+
|
385
|
+
while true
|
386
|
+
c = next_char_raw
|
387
|
+
|
388
|
+
if c == '"'
|
389
|
+
consecutive_quotes += 1
|
390
|
+
elsif consecutive_quotes >= 3
|
391
|
+
# the last three quotes end the string and the other kept.
|
392
|
+
sb.string = sb.string[0...-3]
|
393
|
+
put_back c
|
394
|
+
break
|
395
|
+
else
|
396
|
+
consecutive_quotes = 0
|
397
|
+
if c == -1
|
398
|
+
error_msg = "End of input but triple-quoted string was still open"
|
399
|
+
raise self.class.problem(@line_origin, c, error_msg, false, nil)
|
400
|
+
elsif c == "\n"
|
401
|
+
# keep the line number accurate
|
402
|
+
@line_number += 1
|
403
|
+
@line_origin = @origin.with_line_number(@line_number)
|
404
|
+
end
|
405
|
+
end
|
406
|
+
|
407
|
+
sb << c
|
408
|
+
sb_orig << c
|
409
|
+
end
|
410
|
+
end
|
411
|
+
|
276
412
|
def pull_quoted_string
|
277
413
|
# the open quote has already been consumed
|
278
414
|
sb = StringIO.new
|
415
|
+
|
416
|
+
# We need a second StringIO to keep track of escape characters.
|
417
|
+
# We want to return them exactly as they appeared in the original text,
|
418
|
+
# which means we will need a new StringIO to escape escape characters
|
419
|
+
# so we can also keep the actual value of the string. This is gross.
|
420
|
+
sb_orig = StringIO.new
|
421
|
+
sb_orig << '"'
|
422
|
+
|
279
423
|
c = ""
|
280
424
|
while c != '"'
|
281
425
|
c = next_char_raw
|
282
426
|
if c == -1
|
283
|
-
raise self.problem(@line_origin, c, "End of input but string quote was still open", false, nil)
|
427
|
+
raise self.class.problem(@line_origin, c, "End of input but string quote was still open", false, nil)
|
284
428
|
end
|
285
429
|
|
286
430
|
if c == "\\"
|
287
|
-
pull_escape_sequence(sb)
|
431
|
+
pull_escape_sequence(sb, sb_orig)
|
288
432
|
elsif c == '"'
|
433
|
+
sb_orig << c
|
289
434
|
# done!
|
290
435
|
elsif c =~ /[[:cntrl:]]/
|
291
|
-
raise self.problem(@line_origin, c, "JSON does not allow unescaped #{c}" +
|
292
|
-
|
436
|
+
raise self.class.problem(@line_origin, c, "JSON does not allow unescaped #{c}" +
|
437
|
+
" in quoted strings, use a backslash escape", false, nil)
|
293
438
|
else
|
294
439
|
sb << c
|
440
|
+
sb_orig << c
|
295
441
|
end
|
296
442
|
end
|
297
443
|
|
@@ -299,13 +445,69 @@ class Hocon::Impl::Tokenizer
|
|
299
445
|
if sb.length == 0
|
300
446
|
third = next_char_raw
|
301
447
|
if third == '"'
|
302
|
-
|
448
|
+
sb_orig << third
|
449
|
+
append_triple_quoted_string(sb, sb_orig)
|
303
450
|
else
|
304
451
|
put_back(third)
|
305
452
|
end
|
306
453
|
end
|
307
454
|
|
308
|
-
Tokens.new_string(@line_origin, sb.string)
|
455
|
+
Tokens.new_string(@line_origin, sb.string, sb_orig.string)
|
456
|
+
end
|
457
|
+
|
458
|
+
def pull_plus_equals
|
459
|
+
# the initial '+' has already been consumed
|
460
|
+
c = next_char_raw
|
461
|
+
|
462
|
+
unless c == '='
|
463
|
+
error_msg = "'+' not followed by =, '#{c}' not allowed after '+'"
|
464
|
+
raise self.class.problem(@line_origin, c, error_msg, true, nil) # true = suggest quotes
|
465
|
+
end
|
466
|
+
|
467
|
+
Tokens::PLUS_EQUALS
|
468
|
+
end
|
469
|
+
|
470
|
+
def pull_substitution
|
471
|
+
# the initial '$' has already been consumed
|
472
|
+
c = next_char_raw
|
473
|
+
if c != '{'
|
474
|
+
error_msg = "'$' not followed by {, '#{c}' not allowed after '$'"
|
475
|
+
raise self.class.problem(@line_origin, c, error_msg, true, nil) # true = suggest quotes
|
476
|
+
end
|
477
|
+
|
478
|
+
optional = false
|
479
|
+
c = next_char_raw
|
480
|
+
|
481
|
+
if c == '?'
|
482
|
+
optional = true
|
483
|
+
else
|
484
|
+
put_back(c)
|
485
|
+
end
|
486
|
+
|
487
|
+
saver = WhitespaceSaver.new
|
488
|
+
expression = []
|
489
|
+
|
490
|
+
while true
|
491
|
+
t = pull_next_token(saver)
|
492
|
+
# note that we avoid validating the allowed tokens inside
|
493
|
+
# the substitution here; we even allow nested substitutions
|
494
|
+
# in the tokenizer. The parser sorts it out.
|
495
|
+
|
496
|
+
if t == Tokens::CLOSE_CURLY
|
497
|
+
# end the loop, done!
|
498
|
+
break
|
499
|
+
elsif t == Tokens::EOF
|
500
|
+
raise self.class.problem(@line_origin, t, "Substitution ${ was not closed with a }", false, nil)
|
501
|
+
else
|
502
|
+
whitespace = saver.check(t, @line_origin, @line_number)
|
503
|
+
unless whitespace.nil?
|
504
|
+
expression << whitespace
|
505
|
+
end
|
506
|
+
expression << t
|
507
|
+
end
|
508
|
+
end
|
509
|
+
|
510
|
+
Tokens.new_substitution(@line_origin, optional, expression)
|
309
511
|
end
|
310
512
|
|
311
513
|
def pull_next_token(saver)
|
@@ -316,7 +518,7 @@ class Hocon::Impl::Tokenizer
|
|
316
518
|
# newline tokens have the just-ended line number
|
317
519
|
line = Tokens.new_line(@line_origin)
|
318
520
|
@line_number += 1
|
319
|
-
@line_origin = @origin.
|
521
|
+
@line_origin = @origin.with_line_number(@line_number)
|
320
522
|
line
|
321
523
|
else
|
322
524
|
t = nil
|
@@ -341,7 +543,7 @@ class Hocon::Impl::Tokenizer
|
|
341
543
|
if FIRST_NUMBER_CHARS.index(c)
|
342
544
|
t = pull_number(c)
|
343
545
|
elsif NOT_IN_UNQUOTED_TEXT.index(c)
|
344
|
-
raise
|
546
|
+
raise self.class.problem(@line_origin, c, "Reserved character '#{c}' is not allowed outside quotes", true, nil)
|
345
547
|
else
|
346
548
|
put_back(c)
|
347
549
|
t = pull_unquoted_text
|
@@ -350,13 +552,19 @@ class Hocon::Impl::Tokenizer
|
|
350
552
|
end
|
351
553
|
|
352
554
|
if t.nil?
|
353
|
-
raise
|
555
|
+
raise ConfigBugOrBrokenError, "bug: failed to generate next token"
|
354
556
|
end
|
355
557
|
|
356
558
|
t
|
357
559
|
end
|
358
560
|
end
|
359
561
|
|
562
|
+
def self.simple_value?(t)
|
563
|
+
Tokens.substitution?(t) ||
|
564
|
+
Tokens.unquoted_text?(t) ||
|
565
|
+
Tokens.value?(t)
|
566
|
+
end
|
567
|
+
|
360
568
|
def queue_next_token
|
361
569
|
t = pull_next_token(@whitespace_saver)
|
362
570
|
whitespace = @whitespace_saver.check(t, @origin, @line_number)
|
@@ -366,6 +574,10 @@ class Hocon::Impl::Tokenizer
|
|
366
574
|
@tokens.push(t)
|
367
575
|
end
|
368
576
|
|
577
|
+
def has_next?
|
578
|
+
!@tokens.empty?
|
579
|
+
end
|
580
|
+
|
369
581
|
def next
|
370
582
|
t = @tokens.shift
|
371
583
|
if (@tokens.empty?) and (t != Tokens::EOF)
|
@@ -375,19 +587,37 @@ class Hocon::Impl::Tokenizer
|
|
375
587
|
@tokens.push(e.problem)
|
376
588
|
end
|
377
589
|
if @tokens.empty?
|
378
|
-
raise
|
590
|
+
raise ConfigBugOrBrokenError, "bug: tokens queue should not be empty here"
|
379
591
|
end
|
380
592
|
end
|
381
593
|
t
|
382
594
|
end
|
383
595
|
|
384
|
-
def
|
385
|
-
|
596
|
+
def remove
|
597
|
+
raise ConfigBugOrBrokenError, "Does not make sense to remove items from token stream"
|
386
598
|
end
|
387
|
-
end
|
388
599
|
|
600
|
+
def each
|
601
|
+
while has_next?
|
602
|
+
# Have to use self.next instead of next because next is a reserved word
|
603
|
+
yield self.next
|
604
|
+
end
|
605
|
+
end
|
606
|
+
|
607
|
+
def map
|
608
|
+
token_list = []
|
609
|
+
each do |token|
|
610
|
+
# yield token to calling method, append whatever is returned from the
|
611
|
+
# map block to token_list
|
612
|
+
token_list << yield(token)
|
613
|
+
end
|
614
|
+
token_list
|
615
|
+
end
|
616
|
+
|
617
|
+
def to_list
|
618
|
+
# Return array of tokens from the iterator
|
619
|
+
self.map { |token| token }
|
620
|
+
end
|
389
621
|
|
390
|
-
def self.tokenize(origin, input, syntax)
|
391
|
-
TokenIterator.new(origin, input, syntax != Hocon::ConfigSyntax::JSON)
|
392
622
|
end
|
393
|
-
end
|
623
|
+
end
|