pgsqlarbiter 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/pgsqlarbiter/analysis.rb +20 -3
- data/lib/pgsqlarbiter/analyzer.rb +1009 -356
- data/lib/pgsqlarbiter/arbiter.rb +81 -14
- data/lib/pgsqlarbiter/default_query_functions.rb +11 -3
- data/lib/pgsqlarbiter/default_query_types.rb +55 -0
- data/lib/pgsqlarbiter/error.rb +5 -0
- data/lib/pgsqlarbiter/keywords.rb +525 -58
- data/lib/pgsqlarbiter/lexer.rb +303 -216
- data/lib/pgsqlarbiter/parser.rb +55275 -0
- data/lib/pgsqlarbiter/schema.rb +235 -0
- data/lib/pgsqlarbiter/token.rb +14 -23
- data/lib/pgsqlarbiter/verdict.rb +32 -4
- data/lib/pgsqlarbiter/version.rb +1 -1
- data/lib/pgsqlarbiter.rb +34 -11
- metadata +19 -2
data/lib/pgsqlarbiter/lexer.rb
CHANGED
|
@@ -3,270 +3,357 @@
|
|
|
3
3
|
require "strscan"
|
|
4
4
|
|
|
5
5
|
module Pgsqlarbiter
|
|
6
|
-
# SQL lexer
|
|
6
|
+
# SQL lexer. A rule-for-rule port of PostgreSQL's scanner
|
|
7
|
+
# (src/backend/parser/scan.l) and of the token filter that sits between the
|
|
8
|
+
# scanner and the grammar (base_yylex in src/backend/parser/parser.c).
|
|
7
9
|
#
|
|
8
|
-
#
|
|
9
|
-
#
|
|
10
|
-
#
|
|
10
|
+
# Token boundaries must match PostgreSQL's exactly: any disagreement lets SQL
|
|
11
|
+
# hide from the analyzer. Where PostgreSQL's behaviour depends on server
|
|
12
|
+
# settings, the lexer assumes the defaults and documents it here:
|
|
13
|
+
#
|
|
14
|
+
# * +standard_conforming_strings+ is +on+ (backslashes are ordinary characters
|
|
15
|
+
# in plain string literals).
|
|
16
|
+
# * The client and server encodings are UTF-8.
|
|
17
|
+
#
|
|
18
|
+
# Unquoted identifiers containing non-ASCII upper-case letters are rejected.
|
|
19
|
+
# PostgreSQL folds those differently depending on the server encoding, so
|
|
20
|
+
# their meaning cannot be known here. Quoting the identifier removes the
|
|
21
|
+
# ambiguity.
|
|
11
22
|
class Lexer
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
23
|
+
# Maximum identifier length in bytes (NAMEDATALEN - 1). PostgreSQL silently
|
|
24
|
+
# truncates longer identifiers.
|
|
25
|
+
MAX_IDENTIFIER_BYTES = 63
|
|
26
|
+
|
|
27
|
+
NON_ASCII = '\u0080-\u{10FFFF}'
|
|
28
|
+
SPACE = /[ \t\n\r\f\v]+/
|
|
29
|
+
LINE_COMMENT = /--[^\n\r]*/
|
|
30
|
+
BLOCK_COMMENT_DELIMITER = %r{/\*|\*/}
|
|
31
|
+
IDENTIFIER = /[A-Za-z_#{NON_ASCII}][A-Za-z_0-9$#{NON_ASCII}]*/
|
|
32
|
+
IDENTIFIER_START = /[A-Za-z_#{NON_ASCII}]/
|
|
33
|
+
DOLLAR_DELIMITER = /\$(?:[A-Za-z_#{NON_ASCII}][A-Za-z_0-9#{NON_ASCII}]*)?\$/
|
|
34
|
+
PARAM = /\$[0-9]+/
|
|
35
|
+
DECIMAL = /[0-9](?:_?[0-9])*/
|
|
36
|
+
RADIX_INTEGER = /0[xX](?:_?[0-9A-Fa-f])+|0[oO](?:_?[0-7])+|0[bB](?:_?[01])+/
|
|
37
|
+
# numeric, real and decinteger. "1..2" is the integer 1 followed by "..".
|
|
38
|
+
NUMBER = /(?:#{DECIMAL}\.(?!\.)(?:#{DECIMAL})?|\.#{DECIMAL}|#{DECIMAL})(?:[eE][-+]?#{DECIMAL})?/
|
|
39
|
+
DECIMAL_INTEGER = /\A#{DECIMAL}\z/
|
|
40
|
+
OPERATOR = %r{[~!@#^&|`?+\-*/%<>=]+}
|
|
41
|
+
OPERATOR_COMMENT_START = %r{/\*|--}
|
|
42
|
+
SELF_CHARS = ",()[].;:+-*/%^<>="
|
|
43
|
+
TWO_CHAR_OPERATORS = {
|
|
44
|
+
"=>" => :EQUALS_GREATER, ">=" => :GREATER_EQUALS, "<=" => :LESS_EQUALS,
|
|
45
|
+
"<>" => :NOT_EQUALS, "!=" => :NOT_EQUALS
|
|
46
|
+
}.freeze
|
|
47
|
+
# After a closing quote, whitespace containing a newline followed by another
|
|
48
|
+
# quote continues the same literal (quotecontinue in scan.l). Comments are
|
|
49
|
+
# allowed in that whitespace. Atomic groups keep this linear.
|
|
50
|
+
QUOTE_CONTINUE = /(?>(?:[ \t\f\v]|--[^\n\r]*)*)[\n\r](?>(?:[ \t\n\r\f\v]|--[^\n\r]*[\n\r])*)'/
|
|
51
|
+
INT32_MAX = 2_147_483_647
|
|
52
|
+
|
|
53
|
+
# Tokenize a SQL string.
|
|
15
54
|
#
|
|
16
|
-
# @param sql [String] the SQL
|
|
17
|
-
# @return [Array<Token>]
|
|
18
|
-
# @raise [LexError] on
|
|
19
|
-
#
|
|
55
|
+
# @param sql [String] the SQL to tokenize; must be valid UTF-8
|
|
56
|
+
# @return [Array<Token>] tokens in source order, as the grammar expects them
|
|
57
|
+
# @raise [LexError] on anything PostgreSQL's scanner would reject, and on
|
|
58
|
+
# input whose meaning depends on server configuration
|
|
20
59
|
def tokenize(sql)
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
until @scanner.eos?
|
|
25
|
-
scan_token
|
|
60
|
+
raise LexError, "SQL must be a String" unless sql.is_a?(String)
|
|
61
|
+
unless (sql.encoding == Encoding::UTF_8 || sql.encoding == Encoding::US_ASCII) && sql.valid_encoding?
|
|
62
|
+
raise LexError, "SQL must be valid UTF-8"
|
|
26
63
|
end
|
|
64
|
+
raise LexError, "SQL must not contain NUL bytes" if sql.include?("\0")
|
|
27
65
|
|
|
28
|
-
@
|
|
29
|
-
@tokens
|
|
66
|
+
@s = StringScanner.new(sql)
|
|
67
|
+
@tokens = []
|
|
68
|
+
scan_token until @s.eos?
|
|
69
|
+
apply_lookahead_filter(@tokens)
|
|
30
70
|
end
|
|
31
71
|
|
|
32
72
|
private
|
|
33
73
|
|
|
34
|
-
def
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
# 1. Whitespace — skip
|
|
38
|
-
return if @scanner.skip(/\s+/)
|
|
39
|
-
|
|
40
|
-
# 2. Line comment — skip
|
|
41
|
-
return if @scanner.skip(/--[^\n]*/)
|
|
74
|
+
def emit(type, value, position)
|
|
75
|
+
@tokens << Token.new(type: type, value: value, position: position)
|
|
76
|
+
end
|
|
42
77
|
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
78
|
+
def scan_token
|
|
79
|
+
s = @s
|
|
80
|
+
return if s.skip(SPACE)
|
|
81
|
+
return if s.skip(LINE_COMMENT)
|
|
82
|
+
|
|
83
|
+
pos = s.pos
|
|
84
|
+
return skip_block_comment(pos) if s.skip(%r{/\*})
|
|
85
|
+
|
|
86
|
+
if (text = s.scan(IDENTIFIER))
|
|
87
|
+
scan_identifier(text, pos)
|
|
88
|
+
elsif s.match?(/'/)
|
|
89
|
+
emit(:SCONST, scan_quoted(:xq, pos), pos)
|
|
90
|
+
elsif s.match?(/"/)
|
|
91
|
+
emit(:IDENT, truncate_identifier(scan_double_quoted(pos)), pos)
|
|
92
|
+
elsif (delimiter = s.scan(DOLLAR_DELIMITER))
|
|
93
|
+
finish = s.string.byteindex(delimiter, s.pos)
|
|
94
|
+
raise LexError, "unterminated dollar-quoted string at position #{pos}" unless finish
|
|
95
|
+
emit(:SCONST, s.string.byteslice(s.pos, finish - s.pos), pos)
|
|
96
|
+
s.pos = finish + delimiter.bytesize
|
|
97
|
+
elsif (text = s.scan(PARAM))
|
|
98
|
+
raise LexError, "trailing junk after parameter at position #{pos}" if s.match?(IDENTIFIER_START)
|
|
99
|
+
raise LexError, "parameter number too large at position #{pos}" if text[1..].to_i > INT32_MAX
|
|
100
|
+
emit(:PARAM, text, pos)
|
|
101
|
+
elsif (text = s.scan(RADIX_INTEGER))
|
|
102
|
+
# A following digit or letter would make scan.l's integer_junk rule the
|
|
103
|
+
# longest match.
|
|
104
|
+
raise LexError, "trailing junk after numeric literal at position #{pos}" if s.match?(/[0-9]|#{IDENTIFIER_START}/)
|
|
105
|
+
base = {"x" => 16, "o" => 8, "b" => 2}.fetch(text[1].downcase)
|
|
106
|
+
emit(text[2..].delete("_").to_i(base) <= INT32_MAX ? :ICONST : :FCONST, text, pos)
|
|
107
|
+
elsif (text = s.scan(NUMBER))
|
|
108
|
+
raise LexError, "trailing junk after numeric literal at position #{pos}" if s.match?(IDENTIFIER_START)
|
|
109
|
+
integer = DECIMAL_INTEGER.match?(text) && text.delete("_").to_i <= INT32_MAX
|
|
110
|
+
emit(integer ? :ICONST : :FCONST, text, pos)
|
|
111
|
+
elsif s.skip(/::/) then emit(:TYPECAST, "::", pos)
|
|
112
|
+
elsif s.skip(/\.\./) then emit(:DOT_DOT, "..", pos)
|
|
113
|
+
elsif s.skip(/:=/) then emit(:COLON_EQUALS, ":=", pos)
|
|
114
|
+
elsif (text = s.check(OPERATOR))
|
|
115
|
+
scan_operator(text, pos)
|
|
116
|
+
elsif (text = s.scan(/[,()\[\].;:]/))
|
|
117
|
+
emit(text, text, pos)
|
|
118
|
+
else
|
|
119
|
+
raise LexError, "unexpected character #{s.peek(1).inspect} at position #{pos}"
|
|
47
120
|
end
|
|
121
|
+
end
|
|
48
122
|
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
123
|
+
# Block comments nest. The opening delimiter has already been consumed.
|
|
124
|
+
def skip_block_comment(pos)
|
|
125
|
+
depth = 1
|
|
126
|
+
while depth > 0
|
|
127
|
+
raise LexError, "unterminated /* comment at position #{pos}" unless @s.skip_until(BLOCK_COMMENT_DELIMITER)
|
|
128
|
+
depth += (@s.matched == "/*") ? 1 : -1
|
|
53
129
|
end
|
|
130
|
+
end
|
|
54
131
|
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
132
|
+
def scan_identifier(text, pos)
|
|
133
|
+
s = @s
|
|
134
|
+
if text.bytesize == 1
|
|
135
|
+
if s.match?(/'/)
|
|
136
|
+
case text
|
|
137
|
+
when "e", "E" then return emit(:SCONST, scan_quoted(:xe, pos), pos)
|
|
138
|
+
when "b", "B" then return emit(:BCONST, scan_quoted(:xb, pos), pos)
|
|
139
|
+
when "x", "X" then return emit(:XCONST, scan_quoted(:xh, pos), pos)
|
|
140
|
+
when "n", "N"
|
|
141
|
+
# N'...' is the keyword NCHAR followed by an ordinary string.
|
|
142
|
+
return emit(:NCHAR, "nchar", pos)
|
|
143
|
+
end
|
|
144
|
+
elsif (text == "u" || text == "U") && s.match?(/&['"]/)
|
|
145
|
+
s.skip(/&/)
|
|
146
|
+
if s.match?(/'/)
|
|
147
|
+
return emit(:USCONST, scan_quoted(:xus, pos), pos)
|
|
148
|
+
else
|
|
149
|
+
return emit(:UIDENT, scan_double_quoted(pos), pos)
|
|
150
|
+
end
|
|
151
|
+
end
|
|
59
152
|
end
|
|
60
153
|
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
scan_single_quoted_string(pos, prefix: nil)
|
|
65
|
-
return
|
|
154
|
+
folded = text.downcase(:ascii)
|
|
155
|
+
if (keyword = Keywords::TOKENS[folded])
|
|
156
|
+
return emit(keyword, folded, pos)
|
|
66
157
|
end
|
|
67
158
|
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
159
|
+
if !folded.ascii_only? && folded != folded.downcase
|
|
160
|
+
raise LexError, "unquoted identifier #{text.inspect} at position #{pos} contains non-ASCII " \
|
|
161
|
+
"upper-case letters, which PostgreSQL folds differently depending on the " \
|
|
162
|
+
"server encoding; quote the identifier"
|
|
72
163
|
end
|
|
164
|
+
emit(:IDENT, truncate_identifier(folded), pos)
|
|
165
|
+
end
|
|
73
166
|
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
167
|
+
# Scan a quoted literal starting at its opening quote. +mode+ is the scan.l
|
|
168
|
+
# start condition: :xq (plain), :xe (E''), :xus (U&''), :xb (B'') or :xh (X'').
|
|
169
|
+
def scan_quoted(mode, pos)
|
|
170
|
+
s = @s
|
|
171
|
+
s.skip(/'/)
|
|
172
|
+
value = +""
|
|
173
|
+
loop do
|
|
174
|
+
case mode
|
|
175
|
+
when :xe
|
|
176
|
+
if (chunk = s.scan(/[^\\']+/)) then value << chunk
|
|
177
|
+
elsif s.skip(/''/) then value << "'"
|
|
178
|
+
elsif s.skip(/\\/)
|
|
179
|
+
# A backslash always consumes the next character, including a quote.
|
|
180
|
+
# Multi-character escapes (\ooo, \xhh, \uXXXX) never contain a quote,
|
|
181
|
+
# so consuming one character keeps the literal's extent exact.
|
|
182
|
+
escaped = s.getch
|
|
183
|
+
raise LexError, "unterminated quoted string at position #{pos}" unless escaped
|
|
184
|
+
value << "\\" << escaped
|
|
185
|
+
end
|
|
186
|
+
when :xq, :xus
|
|
187
|
+
if (chunk = s.scan(/[^']+/)) then value << chunk
|
|
188
|
+
elsif s.skip(/''/) then value << "'"
|
|
189
|
+
end
|
|
190
|
+
else # :xb, :xh have no escapes at all
|
|
191
|
+
chunk = s.scan(/[^']*/)
|
|
192
|
+
value << chunk
|
|
99
193
|
end
|
|
100
|
-
@tokens << Token.new(type: NUMBER, value: m, position: pos)
|
|
101
|
-
return
|
|
102
|
-
end
|
|
103
|
-
if (m = @scanner.scan(/\.[\d_]+(?:[eE][+-]?\d[\d_]*)?/))
|
|
104
|
-
@tokens << Token.new(type: NUMBER, value: m, position: pos)
|
|
105
|
-
return
|
|
106
|
-
end
|
|
107
|
-
|
|
108
|
-
# 9. Typecast ::
|
|
109
|
-
if @scanner.scan(/::/)
|
|
110
|
-
@tokens << Token.new(type: TYPECAST, value: "::", position: pos)
|
|
111
|
-
return
|
|
112
|
-
end
|
|
113
194
|
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
when "("
|
|
118
|
-
@scanner.getch
|
|
119
|
-
@tokens << Token.new(type: LPAREN, value: "(", position: pos)
|
|
120
|
-
return
|
|
121
|
-
when ")"
|
|
122
|
-
@scanner.getch
|
|
123
|
-
@tokens << Token.new(type: RPAREN, value: ")", position: pos)
|
|
124
|
-
return
|
|
125
|
-
when "["
|
|
126
|
-
@scanner.getch
|
|
127
|
-
@tokens << Token.new(type: LBRACKET, value: "[", position: pos)
|
|
128
|
-
return
|
|
129
|
-
when "]"
|
|
130
|
-
@scanner.getch
|
|
131
|
-
@tokens << Token.new(type: RBRACKET, value: "]", position: pos)
|
|
132
|
-
return
|
|
133
|
-
when ","
|
|
134
|
-
@scanner.getch
|
|
135
|
-
@tokens << Token.new(type: COMMA, value: ",", position: pos)
|
|
136
|
-
return
|
|
137
|
-
when ";"
|
|
138
|
-
@scanner.getch
|
|
139
|
-
@tokens << Token.new(type: SEMICOLON, value: ";", position: pos)
|
|
140
|
-
return
|
|
141
|
-
when "*"
|
|
142
|
-
@scanner.getch
|
|
143
|
-
@tokens << Token.new(type: STAR, value: "*", position: pos)
|
|
144
|
-
return
|
|
145
|
-
when "."
|
|
146
|
-
@scanner.getch
|
|
147
|
-
@tokens << Token.new(type: DOT, value: ".", position: pos)
|
|
148
|
-
return
|
|
149
|
-
end
|
|
195
|
+
next unless s.match?(/'/) || s.eos?
|
|
196
|
+
raise LexError, "unterminated quoted string at position #{pos}" if s.eos?
|
|
197
|
+
next if mode != :xb && mode != :xh && s.match?(/''/)
|
|
150
198
|
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
@tokens << Token.new(type: OP, value: m, position: pos)
|
|
154
|
-
return
|
|
199
|
+
s.skip(/'/)
|
|
200
|
+
return value unless s.skip(QUOTE_CONTINUE)
|
|
155
201
|
end
|
|
202
|
+
end
|
|
156
203
|
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
scan_quoted_identifier(pos, unicode: true)
|
|
168
|
-
end
|
|
169
|
-
return
|
|
170
|
-
end
|
|
171
|
-
|
|
172
|
-
# Check for string prefixes: E, B, X, N immediately followed by '
|
|
173
|
-
if %w[e b x n].include?(lower) && @scanner.check(/'/)
|
|
174
|
-
scan_single_quoted_string(pos, prefix: lower)
|
|
175
|
-
return
|
|
204
|
+
# Scan a double-quoted identifier starting at its opening quote.
|
|
205
|
+
def scan_double_quoted(pos)
|
|
206
|
+
s = @s
|
|
207
|
+
s.skip(/"/)
|
|
208
|
+
value = +""
|
|
209
|
+
loop do
|
|
210
|
+
if (chunk = s.scan(/[^"]+/)) then value << chunk
|
|
211
|
+
elsif s.skip(/""/) then value << '"'
|
|
212
|
+
elsif s.skip(/"/) then break
|
|
213
|
+
else raise LexError, "unterminated quoted identifier at position #{pos}"
|
|
176
214
|
end
|
|
215
|
+
end
|
|
216
|
+
raise LexError, "zero-length delimited identifier at position #{pos}" if value.empty?
|
|
217
|
+
value
|
|
218
|
+
end
|
|
177
219
|
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
220
|
+
# The {operator} rule of scan.l, including its three special cases: an
|
|
221
|
+
# operator stops before an embedded comment start, sheds trailing + and -
|
|
222
|
+
# unless it contains one of ~!@#^&|`?%, and a result that is a single
|
|
223
|
+
# "self" character or a known two-character operator is that token.
|
|
224
|
+
def scan_operator(text, pos)
|
|
225
|
+
length = text.length
|
|
226
|
+
comment = text.index(OPERATOR_COMMENT_START)
|
|
227
|
+
length = comment if comment
|
|
228
|
+
|
|
229
|
+
if length > 1 && (text[length - 1] == "+" || text[length - 1] == "-")
|
|
230
|
+
unless text[0, length - 1].match?(/[~!@#^&|`?%]/)
|
|
231
|
+
length -= 1
|
|
232
|
+
length -= 1 while length > 1 && (text[length - 1] == "+" || text[length - 1] == "-")
|
|
183
233
|
end
|
|
184
|
-
return
|
|
185
234
|
end
|
|
186
235
|
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
236
|
+
text = text[0, length]
|
|
237
|
+
@s.pos += length # operator characters are all single-byte
|
|
238
|
+
if length == 1 && SELF_CHARS.include?(text)
|
|
239
|
+
emit(text, text, pos)
|
|
240
|
+
elsif length == 2 && (type = TWO_CHAR_OPERATORS[text])
|
|
241
|
+
emit(type, text, pos)
|
|
242
|
+
else
|
|
243
|
+
raise LexError, "operator too long at position #{pos}" if length > MAX_IDENTIFIER_BYTES
|
|
244
|
+
emit(:Op, text, pos)
|
|
191
245
|
end
|
|
246
|
+
end
|
|
247
|
+
|
|
248
|
+
def truncate_identifier(name)
|
|
249
|
+
return name if name.bytesize <= MAX_IDENTIFIER_BYTES
|
|
192
250
|
|
|
193
|
-
|
|
251
|
+
name = name.byteslice(0, MAX_IDENTIFIER_BYTES)
|
|
252
|
+
name = name.byteslice(0, name.bytesize - 1) until name.valid_encoding?
|
|
253
|
+
name
|
|
194
254
|
end
|
|
195
255
|
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
256
|
+
# base_yylex: the grammar needs one token of lookahead in a few places, and
|
|
257
|
+
# U& literals are converted to ordinary ones here.
|
|
258
|
+
def apply_lookahead_filter(tokens)
|
|
259
|
+
out = []
|
|
260
|
+
i = 0
|
|
261
|
+
while i < tokens.length
|
|
262
|
+
token = tokens[i]
|
|
263
|
+
following = tokens[i + 1]&.type
|
|
264
|
+
i += 1
|
|
265
|
+
case token.type
|
|
266
|
+
when :FORMAT
|
|
267
|
+
token = token.with(type: :FORMAT_LA) if following == :JSON
|
|
268
|
+
when :NOT
|
|
269
|
+
if following == :BETWEEN || following == :IN_P || following == :LIKE ||
|
|
270
|
+
following == :ILIKE || following == :SIMILAR
|
|
271
|
+
token = token.with(type: :NOT_LA)
|
|
272
|
+
end
|
|
273
|
+
when :NULLS_P
|
|
274
|
+
token = token.with(type: :NULLS_LA) if following == :FIRST_P || following == :LAST_P
|
|
275
|
+
when :WITH
|
|
276
|
+
token = token.with(type: :WITH_LA) if following == :TIME || following == :ORDINALITY
|
|
277
|
+
when :WITHOUT
|
|
278
|
+
token = token.with(type: :WITHOUT_LA) if following == :TIME
|
|
279
|
+
when :UIDENT, :USCONST
|
|
280
|
+
escape = "\\"
|
|
281
|
+
if following == :UESCAPE
|
|
282
|
+
literal = tokens[i + 1]
|
|
283
|
+
unless literal && literal.type == :SCONST
|
|
284
|
+
raise LexError, "UESCAPE must be followed by a simple string literal at position #{token.position}"
|
|
285
|
+
end
|
|
286
|
+
escape = literal.value
|
|
287
|
+
unless escape.bytesize == 1 && !escape.match?(/[0-9A-Fa-f+'" \t\n\r\f\v]/)
|
|
288
|
+
raise LexError, "invalid Unicode escape character at position #{literal.position}"
|
|
289
|
+
end
|
|
290
|
+
i += 2
|
|
291
|
+
end
|
|
292
|
+
value = decode_unicode_escapes(token.value, escape, token.position)
|
|
293
|
+
token = if token.type == :UIDENT
|
|
294
|
+
token.with(type: :IDENT, value: truncate_identifier(value))
|
|
295
|
+
else
|
|
296
|
+
token.with(type: :SCONST, value: value)
|
|
297
|
+
end
|
|
206
298
|
end
|
|
299
|
+
out << token
|
|
207
300
|
end
|
|
208
|
-
|
|
301
|
+
out
|
|
209
302
|
end
|
|
210
303
|
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
@scanner.terminate
|
|
304
|
+
# str_udeescape: decode \XXXX and \+XXXXXX escapes, with UTF-16 surrogate pairs.
|
|
305
|
+
def decode_unicode_escapes(text, escape, pos)
|
|
306
|
+
out = +""
|
|
307
|
+
pair_first = nil
|
|
308
|
+
chars = text.chars
|
|
309
|
+
i = 0
|
|
310
|
+
invalid_pair = -> { raise LexError, "invalid Unicode surrogate pair at position #{pos}" }
|
|
311
|
+
while i < chars.length
|
|
312
|
+
if chars[i] != escape
|
|
313
|
+
invalid_pair.call if pair_first
|
|
314
|
+
out << chars[i]
|
|
315
|
+
i += 1
|
|
316
|
+
next
|
|
225
317
|
end
|
|
226
|
-
end
|
|
227
|
-
raise LexError, "unterminated dollar-quoted string starting at position #{start_pos}"
|
|
228
|
-
end
|
|
229
318
|
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
319
|
+
if chars[i + 1] == escape
|
|
320
|
+
invalid_pair.call if pair_first
|
|
321
|
+
out << escape
|
|
322
|
+
i += 2
|
|
323
|
+
next
|
|
324
|
+
end
|
|
234
325
|
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
326
|
+
digits, width =
|
|
327
|
+
if chars[i + 1] == "+"
|
|
328
|
+
[chars[i + 2, 6], 8]
|
|
329
|
+
else
|
|
330
|
+
[chars[i + 1, 4], 5]
|
|
239
331
|
end
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
332
|
+
expected = width - (width == 8 ? 2 : 1)
|
|
333
|
+
unless digits && digits.length == expected && digits.all? { |c| c.match?(/\A[0-9A-Fa-f]\z/) }
|
|
334
|
+
raise LexError, "invalid Unicode escape at position #{pos}"
|
|
335
|
+
end
|
|
336
|
+
code = digits.join.to_i(16)
|
|
337
|
+
raise LexError, "invalid Unicode escape value at position #{pos}" unless code > 0 && code <= 0x10FFFF
|
|
338
|
+
|
|
339
|
+
if pair_first
|
|
340
|
+
invalid_pair.call unless code.between?(0xDC00, 0xDFFF)
|
|
341
|
+
code = ((pair_first & 0x3FF) << 10) + 0x10000 + (code & 0x3FF)
|
|
342
|
+
pair_first = nil
|
|
343
|
+
elsif code.between?(0xDC00, 0xDFFF)
|
|
344
|
+
invalid_pair.call
|
|
248
345
|
end
|
|
249
|
-
end
|
|
250
|
-
|
|
251
|
-
raise LexError, "unterminated string starting at position #{start_pos}"
|
|
252
|
-
end
|
|
253
|
-
|
|
254
|
-
def scan_quoted_identifier(start_pos, unicode:)
|
|
255
|
-
@scanner.scan(/"/)
|
|
256
|
-
content = +""
|
|
257
346
|
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
content << '"'
|
|
261
|
-
elsif @scanner.scan(/"/)
|
|
262
|
-
@tokens << Token.new(type: QUOTED_IDENT, value: content, position: start_pos)
|
|
263
|
-
return
|
|
347
|
+
if code.between?(0xD800, 0xDBFF)
|
|
348
|
+
pair_first = code
|
|
264
349
|
else
|
|
265
|
-
|
|
350
|
+
out << [code].pack("U")
|
|
266
351
|
end
|
|
352
|
+
i += width
|
|
267
353
|
end
|
|
268
|
-
|
|
269
|
-
raise LexError, "
|
|
354
|
+
invalid_pair.call if pair_first
|
|
355
|
+
raise LexError, "zero-length delimited identifier at position #{pos}" if out.empty? && !text.empty?
|
|
356
|
+
out
|
|
270
357
|
end
|
|
271
358
|
end
|
|
272
359
|
end
|