pgsqlarbiter 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,270 +3,357 @@
3
3
  require "strscan"
4
4
 
5
5
  module Pgsqlarbiter
6
- # SQL lexer that converts a query string into an array of {Token} objects.
6
+ # SQL lexer. A rule-for-rule port of PostgreSQL's scanner
7
+ # (src/backend/parser/scan.l) and of the token filter that sits between the
8
+ # scanner and the grammar (base_yylex in src/backend/parser/parser.c).
7
9
  #
8
- # Handles all PostgreSQL token types including keywords, identifiers (plain and
9
- # double-quoted), strings (single-quoted, dollar-quoted, and prefixed), numbers,
10
- # parameters, operators, and punctuation.
10
+ # Token boundaries must match PostgreSQL's exactly: any disagreement lets SQL
11
+ # hide from the analyzer. Where PostgreSQL's behaviour depends on server
12
+ # settings, the lexer assumes the defaults and documents it here:
13
+ #
14
+ # * +standard_conforming_strings+ is +on+ (backslashes are ordinary characters
15
+ # in plain string literals).
16
+ # * The client and server encodings are UTF-8.
17
+ #
18
+ # Unquoted identifiers containing non-ASCII upper-case letters are rejected.
19
+ # PostgreSQL folds those differently depending on the server encoding, so
20
+ # their meaning cannot be known here. Quoting the identifier removes the
21
+ # ambiguity.
11
22
  class Lexer
12
- include TokenType
13
-
14
- # Tokenize a SQL query string.
23
+ # Maximum identifier length in bytes (NAMEDATALEN - 1). PostgreSQL silently
24
+ # truncates longer identifiers.
25
+ MAX_IDENTIFIER_BYTES = 63
26
+
27
+ NON_ASCII = '\u0080-\u{10FFFF}'
28
+ SPACE = /[ \t\n\r\f\v]+/
29
+ LINE_COMMENT = /--[^\n\r]*/
30
+ BLOCK_COMMENT_DELIMITER = %r{/\*|\*/}
31
+ IDENTIFIER = /[A-Za-z_#{NON_ASCII}][A-Za-z_0-9$#{NON_ASCII}]*/
32
+ IDENTIFIER_START = /[A-Za-z_#{NON_ASCII}]/
33
+ DOLLAR_DELIMITER = /\$(?:[A-Za-z_#{NON_ASCII}][A-Za-z_0-9#{NON_ASCII}]*)?\$/
34
+ PARAM = /\$[0-9]+/
35
+ DECIMAL = /[0-9](?:_?[0-9])*/
36
+ RADIX_INTEGER = /0[xX](?:_?[0-9A-Fa-f])+|0[oO](?:_?[0-7])+|0[bB](?:_?[01])+/
37
+ # numeric, real and decinteger. "1..2" is the integer 1 followed by "..".
38
+ NUMBER = /(?:#{DECIMAL}\.(?!\.)(?:#{DECIMAL})?|\.#{DECIMAL}|#{DECIMAL})(?:[eE][-+]?#{DECIMAL})?/
39
+ DECIMAL_INTEGER = /\A#{DECIMAL}\z/
40
+ OPERATOR = %r{[~!@#^&|`?+\-*/%<>=]+}
41
+ OPERATOR_COMMENT_START = %r{/\*|--}
42
+ SELF_CHARS = ",()[].;:+-*/%^<>="
43
+ TWO_CHAR_OPERATORS = {
44
+ "=>" => :EQUALS_GREATER, ">=" => :GREATER_EQUALS, "<=" => :LESS_EQUALS,
45
+ "<>" => :NOT_EQUALS, "!=" => :NOT_EQUALS
46
+ }.freeze
47
+ # After a closing quote, whitespace containing a newline followed by another
48
+ # quote continues the same literal (quotecontinue in scan.l). Comments are
49
+ # allowed in that whitespace. Atomic groups keep this linear.
50
+ QUOTE_CONTINUE = /(?>(?:[ \t\f\v]|--[^\n\r]*)*)[\n\r](?>(?:[ \t\n\r\f\v]|--[^\n\r]*[\n\r])*)'/
51
+ INT32_MAX = 2_147_483_647
52
+
53
+ # Tokenize a SQL string.
15
54
  #
16
- # @param sql [String] the SQL string to tokenize
17
- # @return [Array<Token>] list of tokens ending with an EOF token
18
- # @raise [LexError] on invalid syntax such as unexpected characters or unterminated
19
- # strings/comments
55
+ # @param sql [String] the SQL to tokenize; must be valid UTF-8
56
+ # @return [Array<Token>] tokens in source order, as the grammar expects them
57
+ # @raise [LexError] on anything PostgreSQL's scanner would reject, and on
58
+ # input whose meaning depends on server configuration
20
59
  def tokenize(sql)
21
- @scanner = StringScanner.new(sql)
22
- @tokens = []
23
-
24
- until @scanner.eos?
25
- scan_token
60
+ raise LexError, "SQL must be a String" unless sql.is_a?(String)
61
+ unless (sql.encoding == Encoding::UTF_8 || sql.encoding == Encoding::US_ASCII) && sql.valid_encoding?
62
+ raise LexError, "SQL must be valid UTF-8"
26
63
  end
64
+ raise LexError, "SQL must not contain NUL bytes" if sql.include?("\0")
27
65
 
28
- @tokens << Token.new(type: EOF, value: nil, position: @scanner.pos)
29
- @tokens
66
+ @s = StringScanner.new(sql)
67
+ @tokens = []
68
+ scan_token until @s.eos?
69
+ apply_lookahead_filter(@tokens)
30
70
  end
31
71
 
32
72
  private
33
73
 
34
- def scan_token
35
- pos = @scanner.pos
36
-
37
- # 1. Whitespace — skip
38
- return if @scanner.skip(/\s+/)
39
-
40
- # 2. Line comment — skip
41
- return if @scanner.skip(/--[^\n]*/)
74
+ def emit(type, value, position)
75
+ @tokens << Token.new(type: type, value: value, position: position)
76
+ end
42
77
 
43
- # 3. Block comment — skip (handle nesting)
44
- if @scanner.scan(/\/\*/)
45
- scan_block_comment(pos)
46
- return
78
+ def scan_token
79
+ s = @s
80
+ return if s.skip(SPACE)
81
+ return if s.skip(LINE_COMMENT)
82
+
83
+ pos = s.pos
84
+ return skip_block_comment(pos) if s.skip(%r{/\*})
85
+
86
+ if (text = s.scan(IDENTIFIER))
87
+ scan_identifier(text, pos)
88
+ elsif s.match?(/'/)
89
+ emit(:SCONST, scan_quoted(:xq, pos), pos)
90
+ elsif s.match?(/"/)
91
+ emit(:IDENT, truncate_identifier(scan_double_quoted(pos)), pos)
92
+ elsif (delimiter = s.scan(DOLLAR_DELIMITER))
93
+ finish = s.string.byteindex(delimiter, s.pos)
94
+ raise LexError, "unterminated dollar-quoted string at position #{pos}" unless finish
95
+ emit(:SCONST, s.string.byteslice(s.pos, finish - s.pos), pos)
96
+ s.pos = finish + delimiter.bytesize
97
+ elsif (text = s.scan(PARAM))
98
+ raise LexError, "trailing junk after parameter at position #{pos}" if s.match?(IDENTIFIER_START)
99
+ raise LexError, "parameter number too large at position #{pos}" if text[1..].to_i > INT32_MAX
100
+ emit(:PARAM, text, pos)
101
+ elsif (text = s.scan(RADIX_INTEGER))
102
+ # A following digit or letter would make scan.l's integer_junk rule the
103
+ # longest match.
104
+ raise LexError, "trailing junk after numeric literal at position #{pos}" if s.match?(/[0-9]|#{IDENTIFIER_START}/)
105
+ base = {"x" => 16, "o" => 8, "b" => 2}.fetch(text[1].downcase)
106
+ emit(text[2..].delete("_").to_i(base) <= INT32_MAX ? :ICONST : :FCONST, text, pos)
107
+ elsif (text = s.scan(NUMBER))
108
+ raise LexError, "trailing junk after numeric literal at position #{pos}" if s.match?(IDENTIFIER_START)
109
+ integer = DECIMAL_INTEGER.match?(text) && text.delete("_").to_i <= INT32_MAX
110
+ emit(integer ? :ICONST : :FCONST, text, pos)
111
+ elsif s.skip(/::/) then emit(:TYPECAST, "::", pos)
112
+ elsif s.skip(/\.\./) then emit(:DOT_DOT, "..", pos)
113
+ elsif s.skip(/:=/) then emit(:COLON_EQUALS, ":=", pos)
114
+ elsif (text = s.check(OPERATOR))
115
+ scan_operator(text, pos)
116
+ elsif (text = s.scan(/[,()\[\].;:]/))
117
+ emit(text, text, pos)
118
+ else
119
+ raise LexError, "unexpected character #{s.peek(1).inspect} at position #{pos}"
47
120
  end
121
+ end
48
122
 
49
- # 4. Dollar-quoted string
50
- if @scanner.check(/\$([\p{L}_][\p{L}\d_]*)?\$/)
51
- scan_dollar_string(pos)
52
- return
123
+ # Block comments nest. The opening delimiter has already been consumed.
124
+ def skip_block_comment(pos)
125
+ depth = 1
126
+ while depth > 0
127
+ raise LexError, "unterminated /* comment at position #{pos}" unless @s.skip_until(BLOCK_COMMENT_DELIMITER)
128
+ depth += (@s.matched == "/*") ? 1 : -1
53
129
  end
130
+ end
54
131
 
55
- # 5. Parameter placeholder ($1, $2, ...)
56
- if (m = @scanner.scan(/\$\d+/))
57
- @tokens << Token.new(type: PARAM, value: m, position: pos)
58
- return
132
+ def scan_identifier(text, pos)
133
+ s = @s
134
+ if text.bytesize == 1
135
+ if s.match?(/'/)
136
+ case text
137
+ when "e", "E" then return emit(:SCONST, scan_quoted(:xe, pos), pos)
138
+ when "b", "B" then return emit(:BCONST, scan_quoted(:xb, pos), pos)
139
+ when "x", "X" then return emit(:XCONST, scan_quoted(:xh, pos), pos)
140
+ when "n", "N"
141
+ # N'...' is the keyword NCHAR followed by an ordinary string.
142
+ return emit(:NCHAR, "nchar", pos)
143
+ end
144
+ elsif (text == "u" || text == "U") && s.match?(/&['"]/)
145
+ s.skip(/&/)
146
+ if s.match?(/'/)
147
+ return emit(:USCONST, scan_quoted(:xus, pos), pos)
148
+ else
149
+ return emit(:UIDENT, scan_double_quoted(pos), pos)
150
+ end
151
+ end
59
152
  end
60
153
 
61
- # 6. Single-quoted strings (including E'', B'', X'', N'' prefixes)
62
- # Prefix detection is handled in the identifier branch (step 12)
63
- if @scanner.check(/'/)
64
- scan_single_quoted_string(pos, prefix: nil)
65
- return
154
+ folded = text.downcase(:ascii)
155
+ if (keyword = Keywords::TOKENS[folded])
156
+ return emit(keyword, folded, pos)
66
157
  end
67
158
 
68
- # 7. Double-quoted identifier
69
- if @scanner.check(/"/)
70
- scan_quoted_identifier(pos, unicode: false)
71
- return
159
+ if !folded.ascii_only? && folded != folded.downcase
160
+ raise LexError, "unquoted identifier #{text.inspect} at position #{pos} contains non-ASCII " \
161
+ "upper-case letters, which PostgreSQL folds differently depending on the " \
162
+ "server encoding; quote the identifier"
72
163
  end
164
+ emit(:IDENT, truncate_identifier(folded), pos)
165
+ end
73
166
 
74
- # 8. Numbers
75
- if (m = @scanner.scan(/0[xX][0-9a-fA-F_]+/))
76
- @tokens << Token.new(type: NUMBER, value: m, position: pos)
77
- return
78
- end
79
- if (m = @scanner.scan(/0[oO][0-7_]+/))
80
- @tokens << Token.new(type: NUMBER, value: m, position: pos)
81
- return
82
- end
83
- if (m = @scanner.scan(/0[bB][01_]+/))
84
- @tokens << Token.new(type: NUMBER, value: m, position: pos)
85
- return
86
- end
87
- if (m = @scanner.scan(/\d[\d_]*\.[\d_]+(?:[eE][+-]?\d[\d_]*)?/))
88
- @tokens << Token.new(type: NUMBER, value: m, position: pos)
89
- return
90
- end
91
- if (m = @scanner.scan(/\d[\d_]*[eE][+-]?\d[\d_]*/))
92
- @tokens << Token.new(type: NUMBER, value: m, position: pos)
93
- return
94
- end
95
- if (m = @scanner.scan(/\d[\d_]*/))
96
- # Check this isn't followed by dot+digits (which would be a decimal)
97
- if @scanner.check(/\.[\d_]/)
98
- m += @scanner.scan(/\.[\d_]+(?:[eE][+-]?\d[\d_]*)?/)
167
+ # Scan a quoted literal starting at its opening quote. +mode+ is the scan.l
168
+ # start condition: :xq (plain), :xe (E''), :xus (U&''), :xb (B'') or :xh (X'').
169
+ def scan_quoted(mode, pos)
170
+ s = @s
171
+ s.skip(/'/)
172
+ value = +""
173
+ loop do
174
+ case mode
175
+ when :xe
176
+ if (chunk = s.scan(/[^\\']+/)) then value << chunk
177
+ elsif s.skip(/''/) then value << "'"
178
+ elsif s.skip(/\\/)
179
+ # A backslash always consumes the next character, including a quote.
180
+ # Multi-character escapes (\ooo, \xhh, \uXXXX) never contain a quote,
181
+ # so consuming one character keeps the literal's extent exact.
182
+ escaped = s.getch
183
+ raise LexError, "unterminated quoted string at position #{pos}" unless escaped
184
+ value << "\\" << escaped
185
+ end
186
+ when :xq, :xus
187
+ if (chunk = s.scan(/[^']+/)) then value << chunk
188
+ elsif s.skip(/''/) then value << "'"
189
+ end
190
+ else # :xb, :xh have no escapes at all
191
+ chunk = s.scan(/[^']*/)
192
+ value << chunk
99
193
  end
100
- @tokens << Token.new(type: NUMBER, value: m, position: pos)
101
- return
102
- end
103
- if (m = @scanner.scan(/\.[\d_]+(?:[eE][+-]?\d[\d_]*)?/))
104
- @tokens << Token.new(type: NUMBER, value: m, position: pos)
105
- return
106
- end
107
-
108
- # 9. Typecast ::
109
- if @scanner.scan(/::/)
110
- @tokens << Token.new(type: TYPECAST, value: "::", position: pos)
111
- return
112
- end
113
194
 
114
- # 10. Single-char punctuation
115
- ch = @scanner.peek(1)
116
- case ch
117
- when "("
118
- @scanner.getch
119
- @tokens << Token.new(type: LPAREN, value: "(", position: pos)
120
- return
121
- when ")"
122
- @scanner.getch
123
- @tokens << Token.new(type: RPAREN, value: ")", position: pos)
124
- return
125
- when "["
126
- @scanner.getch
127
- @tokens << Token.new(type: LBRACKET, value: "[", position: pos)
128
- return
129
- when "]"
130
- @scanner.getch
131
- @tokens << Token.new(type: RBRACKET, value: "]", position: pos)
132
- return
133
- when ","
134
- @scanner.getch
135
- @tokens << Token.new(type: COMMA, value: ",", position: pos)
136
- return
137
- when ";"
138
- @scanner.getch
139
- @tokens << Token.new(type: SEMICOLON, value: ";", position: pos)
140
- return
141
- when "*"
142
- @scanner.getch
143
- @tokens << Token.new(type: STAR, value: "*", position: pos)
144
- return
145
- when "."
146
- @scanner.getch
147
- @tokens << Token.new(type: DOT, value: ".", position: pos)
148
- return
149
- end
195
+ next unless s.match?(/'/) || s.eos?
196
+ raise LexError, "unterminated quoted string at position #{pos}" if s.eos?
197
+ next if mode != :xb && mode != :xh && s.match?(/''/)
150
198
 
151
- # 11. Multi-char operators
152
- if (m = @scanner.scan(%r{[+\-/<>=~!@#%^&|`?]+}))
153
- @tokens << Token.new(type: OP, value: m, position: pos)
154
- return
199
+ s.skip(/'/)
200
+ return value unless s.skip(QUOTE_CONTINUE)
155
201
  end
202
+ end
156
203
 
157
- # 12. Unquoted identifier / keyword (with string prefix detection)
158
- if (m = @scanner.scan(/[\p{L}_][\p{L}\d_]*/))
159
- lower = m.downcase
160
-
161
- # Check for U& prefix for unicode strings/identifiers
162
- if lower == "u" && @scanner.check(/&['"]/i)
163
- @scanner.scan(/&/)
164
- if @scanner.check(/'/)
165
- scan_single_quoted_string(pos, prefix: "U&")
166
- else
167
- scan_quoted_identifier(pos, unicode: true)
168
- end
169
- return
170
- end
171
-
172
- # Check for string prefixes: E, B, X, N immediately followed by '
173
- if %w[e b x n].include?(lower) && @scanner.check(/'/)
174
- scan_single_quoted_string(pos, prefix: lower)
175
- return
204
+ # Scan a double-quoted identifier starting at its opening quote.
205
+ def scan_double_quoted(pos)
206
+ s = @s
207
+ s.skip(/"/)
208
+ value = +""
209
+ loop do
210
+ if (chunk = s.scan(/[^"]+/)) then value << chunk
211
+ elsif s.skip(/""/) then value << '"'
212
+ elsif s.skip(/"/) then break
213
+ else raise LexError, "unterminated quoted identifier at position #{pos}"
176
214
  end
215
+ end
216
+ raise LexError, "zero-length delimited identifier at position #{pos}" if value.empty?
217
+ value
218
+ end
177
219
 
178
- upper = m.upcase
179
- if Keywords::ALL.include?(upper)
180
- @tokens << Token.new(type: KEYWORD, value: upper, position: pos)
181
- else
182
- @tokens << Token.new(type: IDENT, value: lower, position: pos)
220
+ # The {operator} rule of scan.l, including its three special cases: an
221
+ # operator stops before an embedded comment start, sheds trailing + and -
222
+ # unless it contains one of ~!@#^&|`?%, and a result that is a single
223
+ # "self" character or a known two-character operator is that token.
224
+ def scan_operator(text, pos)
225
+ length = text.length
226
+ comment = text.index(OPERATOR_COMMENT_START)
227
+ length = comment if comment
228
+
229
+ if length > 1 && (text[length - 1] == "+" || text[length - 1] == "-")
230
+ unless text[0, length - 1].match?(/[~!@#^&|`?%]/)
231
+ length -= 1
232
+ length -= 1 while length > 1 && (text[length - 1] == "+" || text[length - 1] == "-")
183
233
  end
184
- return
185
234
  end
186
235
 
187
- # 13. Single colon (not part of ::)
188
- if @scanner.scan(/:/)
189
- @tokens << Token.new(type: OP, value: ":", position: pos)
190
- return
236
+ text = text[0, length]
237
+ @s.pos += length # operator characters are all single-byte
238
+ if length == 1 && SELF_CHARS.include?(text)
239
+ emit(text, text, pos)
240
+ elsif length == 2 && (type = TWO_CHAR_OPERATORS[text])
241
+ emit(type, text, pos)
242
+ else
243
+ raise LexError, "operator too long at position #{pos}" if length > MAX_IDENTIFIER_BYTES
244
+ emit(:Op, text, pos)
191
245
  end
246
+ end
247
+
248
+ def truncate_identifier(name)
249
+ return name if name.bytesize <= MAX_IDENTIFIER_BYTES
192
250
 
193
- raise LexError, "unexpected character #{@scanner.peek(1).inspect} at position #{pos}"
251
+ name = name.byteslice(0, MAX_IDENTIFIER_BYTES)
252
+ name = name.byteslice(0, name.bytesize - 1) until name.valid_encoding?
253
+ name
194
254
  end
195
255
 
196
- def scan_block_comment(start_pos)
197
- depth = 1
198
- until @scanner.eos?
199
- if @scanner.scan(/\/\*/)
200
- depth += 1
201
- elsif @scanner.scan(/\*\//)
202
- depth -= 1
203
- return if depth == 0
204
- else
205
- @scanner.getch
256
+ # base_yylex: the grammar needs one token of lookahead in a few places, and
257
+ # U& literals are converted to ordinary ones here.
258
+ def apply_lookahead_filter(tokens)
259
+ out = []
260
+ i = 0
261
+ while i < tokens.length
262
+ token = tokens[i]
263
+ following = tokens[i + 1]&.type
264
+ i += 1
265
+ case token.type
266
+ when :FORMAT
267
+ token = token.with(type: :FORMAT_LA) if following == :JSON
268
+ when :NOT
269
+ if following == :BETWEEN || following == :IN_P || following == :LIKE ||
270
+ following == :ILIKE || following == :SIMILAR
271
+ token = token.with(type: :NOT_LA)
272
+ end
273
+ when :NULLS_P
274
+ token = token.with(type: :NULLS_LA) if following == :FIRST_P || following == :LAST_P
275
+ when :WITH
276
+ token = token.with(type: :WITH_LA) if following == :TIME || following == :ORDINALITY
277
+ when :WITHOUT
278
+ token = token.with(type: :WITHOUT_LA) if following == :TIME
279
+ when :UIDENT, :USCONST
280
+ escape = "\\"
281
+ if following == :UESCAPE
282
+ literal = tokens[i + 1]
283
+ unless literal && literal.type == :SCONST
284
+ raise LexError, "UESCAPE must be followed by a simple string literal at position #{token.position}"
285
+ end
286
+ escape = literal.value
287
+ unless escape.bytesize == 1 && !escape.match?(/[0-9A-Fa-f+'" \t\n\r\f\v]/)
288
+ raise LexError, "invalid Unicode escape character at position #{literal.position}"
289
+ end
290
+ i += 2
291
+ end
292
+ value = decode_unicode_escapes(token.value, escape, token.position)
293
+ token = if token.type == :UIDENT
294
+ token.with(type: :IDENT, value: truncate_identifier(value))
295
+ else
296
+ token.with(type: :SCONST, value: value)
297
+ end
206
298
  end
299
+ out << token
207
300
  end
208
- raise LexError, "unterminated block comment starting at position #{start_pos}"
301
+ out
209
302
  end
210
303
 
211
- def scan_dollar_string(start_pos)
212
- @scanner.scan(/\$([\p{L}_][\p{L}\d_]*)?\$/)
213
- tag = @scanner.matched
214
- content = +""
215
- until @scanner.eos?
216
- idx = @scanner.rest.index(tag)
217
- if idx
218
- content << @scanner.rest[0, idx]
219
- @scanner.pos += idx + tag.length
220
- @tokens << Token.new(type: STRING, value: content, position: start_pos)
221
- return
222
- else
223
- content << @scanner.rest
224
- @scanner.terminate
304
+ # str_udeescape: decode \XXXX and \+XXXXXX escapes, with UTF-16 surrogate pairs.
305
+ def decode_unicode_escapes(text, escape, pos)
306
+ out = +""
307
+ pair_first = nil
308
+ chars = text.chars
309
+ i = 0
310
+ invalid_pair = -> { raise LexError, "invalid Unicode surrogate pair at position #{pos}" }
311
+ while i < chars.length
312
+ if chars[i] != escape
313
+ invalid_pair.call if pair_first
314
+ out << chars[i]
315
+ i += 1
316
+ next
225
317
  end
226
- end
227
- raise LexError, "unterminated dollar-quoted string starting at position #{start_pos}"
228
- end
229
318
 
230
- def scan_single_quoted_string(start_pos, prefix:)
231
- @scanner.scan(/'/)
232
- escape_mode = (prefix == "e") # E-strings support backslash escapes
233
- content = +""
319
+ if chars[i + 1] == escape
320
+ invalid_pair.call if pair_first
321
+ out << escape
322
+ i += 2
323
+ next
324
+ end
234
325
 
235
- until @scanner.eos?
236
- if escape_mode && @scanner.scan(/\\/)
237
- if @scanner.eos?
238
- raise LexError, "unterminated string starting at position #{start_pos}"
326
+ digits, width =
327
+ if chars[i + 1] == "+"
328
+ [chars[i + 2, 6], 8]
329
+ else
330
+ [chars[i + 1, 4], 5]
239
331
  end
240
- content << "\\" << @scanner.getch
241
- elsif @scanner.scan(/''/)
242
- content << "''"
243
- elsif @scanner.scan(/'/)
244
- @tokens << Token.new(type: STRING, value: content, position: start_pos)
245
- return
246
- else
247
- content << @scanner.getch
332
+ expected = width - (width == 8 ? 2 : 1)
333
+ unless digits && digits.length == expected && digits.all? { |c| c.match?(/\A[0-9A-Fa-f]\z/) }
334
+ raise LexError, "invalid Unicode escape at position #{pos}"
335
+ end
336
+ code = digits.join.to_i(16)
337
+ raise LexError, "invalid Unicode escape value at position #{pos}" unless code > 0 && code <= 0x10FFFF
338
+
339
+ if pair_first
340
+ invalid_pair.call unless code.between?(0xDC00, 0xDFFF)
341
+ code = ((pair_first & 0x3FF) << 10) + 0x10000 + (code & 0x3FF)
342
+ pair_first = nil
343
+ elsif code.between?(0xDC00, 0xDFFF)
344
+ invalid_pair.call
248
345
  end
249
- end
250
-
251
- raise LexError, "unterminated string starting at position #{start_pos}"
252
- end
253
-
254
- def scan_quoted_identifier(start_pos, unicode:)
255
- @scanner.scan(/"/)
256
- content = +""
257
346
 
258
- until @scanner.eos?
259
- if @scanner.scan(/""/)
260
- content << '"'
261
- elsif @scanner.scan(/"/)
262
- @tokens << Token.new(type: QUOTED_IDENT, value: content, position: start_pos)
263
- return
347
+ if code.between?(0xD800, 0xDBFF)
348
+ pair_first = code
264
349
  else
265
- content << @scanner.getch
350
+ out << [code].pack("U")
266
351
  end
352
+ i += width
267
353
  end
268
-
269
- raise LexError, "unterminated quoted identifier starting at position #{start_pos}"
354
+ invalid_pair.call if pair_first
355
+ raise LexError, "zero-length delimited identifier at position #{pos}" if out.empty? && !text.empty?
356
+ out
270
357
  end
271
358
  end
272
359
  end