plume 0.1.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (124) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +506 -11
  3. data/lib/plume/analysis/resolver.rb +797 -0
  4. data/lib/plume/analysis.rb +211 -0
  5. data/lib/plume/ast/alter_table_statement.rb +40 -0
  6. data/lib/plume/ast/analyze_statement.rb +15 -0
  7. data/lib/plume/ast/assignment.rb +18 -0
  8. data/lib/plume/ast/attach_statement.rb +17 -0
  9. data/lib/plume/ast/begin_statement.rb +17 -0
  10. data/lib/plume/ast/binary_expression.rb +71 -0
  11. data/lib/plume/ast/case_condition.rb +29 -0
  12. data/lib/plume/ast/case_expression.rb +22 -0
  13. data/lib/plume/ast/cast_expression.rb +16 -0
  14. data/lib/plume/ast/check_column_constraint.rb +20 -0
  15. data/lib/plume/ast/check_table_constraint.rb +20 -0
  16. data/lib/plume/ast/collate_column_constraint.rb +19 -0
  17. data/lib/plume/ast/collation_expression.rb +26 -0
  18. data/lib/plume/ast/column_constraint.rb +7 -0
  19. data/lib/plume/ast/column_definition.rb +16 -0
  20. data/lib/plume/ast/column_name.rb +22 -0
  21. data/lib/plume/ast/column_type.rb +26 -0
  22. data/lib/plume/ast/commit_statement.rb +13 -0
  23. data/lib/plume/ast/common_table_expression.rb +31 -0
  24. data/lib/plume/ast/compound_select.rb +30 -0
  25. data/lib/plume/ast/conflict_clause.rb +22 -0
  26. data/lib/plume/ast/conflict_resolution.rb +20 -0
  27. data/lib/plume/ast/create_index_statement.rb +29 -0
  28. data/lib/plume/ast/create_table_statement.rb +47 -0
  29. data/lib/plume/ast/create_trigger_statement.rb +43 -0
  30. data/lib/plume/ast/create_view_statement.rb +30 -0
  31. data/lib/plume/ast/create_virtual_table_statement.rb +28 -0
  32. data/lib/plume/ast/default_column_constraint.rb +29 -0
  33. data/lib/plume/ast/deferrable_column_constraint.rb +20 -0
  34. data/lib/plume/ast/delete_statement.rb +28 -0
  35. data/lib/plume/ast/detach_statement.rb +13 -0
  36. data/lib/plume/ast/drop_statement.rb +21 -0
  37. data/lib/plume/ast/empty_function_argument.rb +6 -0
  38. data/lib/plume/ast/exists_expression.rb +15 -0
  39. data/lib/plume/ast/explain_statement.rb +15 -0
  40. data/lib/plume/ast/expression.rb +30 -0
  41. data/lib/plume/ast/foreign_key_action.rb +26 -0
  42. data/lib/plume/ast/foreign_key_clause.rb +50 -0
  43. data/lib/plume/ast/foreign_key_column_constraint.rb +16 -0
  44. data/lib/plume/ast/foreign_key_table_constraint.rb +20 -0
  45. data/lib/plume/ast/frame_boundary.rb +35 -0
  46. data/lib/plume/ast/frame_spec.rb +34 -0
  47. data/lib/plume/ast/function_arguments.rb +18 -0
  48. data/lib/plume/ast/function_reference.rb +29 -0
  49. data/lib/plume/ast/generated_as_column_constraint.rb +24 -0
  50. data/lib/plume/ast/indexed_column.rb +24 -0
  51. data/lib/plume/ast/insert_statement.rb +35 -0
  52. data/lib/plume/ast/join.rb +12 -0
  53. data/lib/plume/ast/join_clause.rb +15 -0
  54. data/lib/plume/ast/join_on_constraint.rb +12 -0
  55. data/lib/plume/ast/join_operator.rb +31 -0
  56. data/lib/plume/ast/join_using_constraint.rb +16 -0
  57. data/lib/plume/ast/like_expression.rb +19 -0
  58. data/lib/plume/ast/literal_expression.rb +29 -0
  59. data/lib/plume/ast/literal_false.rb +7 -0
  60. data/lib/plume/ast/literal_nil.rb +7 -0
  61. data/lib/plume/ast/literal_value.rb +18 -0
  62. data/lib/plume/ast/match_clause.rb +15 -0
  63. data/lib/plume/ast/node.rb +330 -0
  64. data/lib/plume/ast/noop_column_constraint.rb +15 -0
  65. data/lib/plume/ast/noop_table_constraint.rb +15 -0
  66. data/lib/plume/ast/not_like_expression.rb +7 -0
  67. data/lib/plume/ast/not_null_column_constraint.rb +17 -0
  68. data/lib/plume/ast/null_column_constraint.rb +17 -0
  69. data/lib/plume/ast/ordering_term.rb +32 -0
  70. data/lib/plume/ast/over_clause.rb +25 -0
  71. data/lib/plume/ast/parenthetical_expression.rb +37 -0
  72. data/lib/plume/ast/parenthetical_source.rb +19 -0
  73. data/lib/plume/ast/pragma_statement.rb +26 -0
  74. data/lib/plume/ast/primary_key_column_constraint.rb +23 -0
  75. data/lib/plume/ast/primary_key_table_constraint.rb +23 -0
  76. data/lib/plume/ast/raise_expression.rb +23 -0
  77. data/lib/plume/ast/reindex_statement.rb +15 -0
  78. data/lib/plume/ast/release_statement.rb +16 -0
  79. data/lib/plume/ast/result_column.rb +18 -0
  80. data/lib/plume/ast/returning_clause.rb +12 -0
  81. data/lib/plume/ast/rollback_statement.rb +19 -0
  82. data/lib/plume/ast/savepoint_statement.rb +16 -0
  83. data/lib/plume/ast/select_core.rb +27 -0
  84. data/lib/plume/ast/select_statement.rb +34 -0
  85. data/lib/plume/ast/star_function_argument.rb +6 -0
  86. data/lib/plume/ast/star_result_column.rb +16 -0
  87. data/lib/plume/ast/strict_table_option.rb +6 -0
  88. data/lib/plume/ast/subquery_expression.rb +14 -0
  89. data/lib/plume/ast/subquery_source.rb +19 -0
  90. data/lib/plume/ast/table_constraint.rb +7 -0
  91. data/lib/plume/ast/table_function.rb +28 -0
  92. data/lib/plume/ast/table_name.rb +28 -0
  93. data/lib/plume/ast/table_option.rb +7 -0
  94. data/lib/plume/ast/ternary_expression.rb +26 -0
  95. data/lib/plume/ast/token.rb +183 -0
  96. data/lib/plume/ast/unary_expression.rb +36 -0
  97. data/lib/plume/ast/unique_column_constraint.rb +17 -0
  98. data/lib/plume/ast/unique_table_constraint.rb +20 -0
  99. data/lib/plume/ast/update_statement.rb +35 -0
  100. data/lib/plume/ast/upsert_clause.rb +26 -0
  101. data/lib/plume/ast/vacuum_statement.rb +18 -0
  102. data/lib/plume/ast/values_clause.rb +12 -0
  103. data/lib/plume/ast/values_row.rb +13 -0
  104. data/lib/plume/ast/variable.rb +15 -0
  105. data/lib/plume/ast/window_definition.rb +26 -0
  106. data/lib/plume/ast/with_clause.rb +15 -0
  107. data/lib/plume/ast/without_rowid_table_option.rb +6 -0
  108. data/lib/plume/catalog.rb +276 -0
  109. data/lib/plume/comment.rb +73 -0
  110. data/lib/plume/dump.rb +143 -0
  111. data/lib/plume/fingerprint.rb +86 -0
  112. data/lib/plume/lexer.rb +640 -0
  113. data/lib/plume/literal_patches.rb +76 -0
  114. data/lib/plume/normalize.rb +65 -0
  115. data/lib/plume/parameters.rb +52 -0
  116. data/lib/plume/parser.rb +4064 -0
  117. data/lib/plume/pretty_printer.rb +202 -0
  118. data/lib/plume/printer.rb +525 -0
  119. data/lib/plume/sqlite_grammar.rb +86 -0
  120. data/lib/plume/truncate.rb +67 -0
  121. data/lib/plume/union.rb +28 -0
  122. data/lib/plume/version.rb +1 -1
  123. data/lib/plume.rb +274 -3
  124. metadata +176 -12
@@ -0,0 +1,640 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Plume
4
+ # Map ASCII strings to an enumerator of token symbols.
5
+ # adapted from: https://github.com/sqlite/sqlite/blob/master/src/tokenize.c
6
+ class Lexer
7
+ # Character classes for tokenizing
8
+ # adapted from: https://github.com/sqlite/sqlite/blob/master/src/tokenize.c
9
+ TYPES = [
10
+ :X, # The letter 'x', or start of BLOB literal
11
+ :KYWD0, # First letter of a keyword
12
+ :KYWD, # Alphabetics or '_'. Usable in a keyword
13
+ :DIGIT, # Digits
14
+ :DOLLAR, # '$'
15
+ :VARALPHA, # '@', '#', ':'. Alphabetic SQL variables
16
+ :VARNUM, # '?'. Numeric SQL variables
17
+ :SPACE, # Space characters
18
+ :QUOTE, # '"', '\'', or '`'. String literals, quoted ids
19
+ :QUOTE2, # '['. [...] style quoted ids
20
+ :PIPE, # '|'. Bitwise OR or concatenate
21
+ :MINUS, # '-'. Minus or SQL-style comment
22
+ :LT, # '<'. Part of < or <= or <>
23
+ :GT, # '>'. Part of > or >=
24
+ :EQ, # '='. Part of = or ==
25
+ :BANG, # '!'. Part of !=
26
+ :SLASH, # '/'. / or c-style comment
27
+ :LP, # '('
28
+ :RP, # ')'
29
+ :SEMI, # ';'
30
+ :PLUS, # '+'
31
+ :STAR, # '*'
32
+ :PERCENT, # '%'
33
+ :COMMA, # ','
34
+ :AND, # '&'
35
+ :TILDA, # '~'
36
+ :DOT, # '.'
37
+ :ID, # unicode characters usable in IDs
38
+ :ILLEGAL, # Illegal character
39
+ :NUL, # 0x00
40
+ :BOM, # First byte of UTF8 BOM: 0xEF 0xBB 0xBF
41
+ ].freeze
42
+
43
+ # All the keywords of the SQL language in SQLite's dialect
44
+ # adapted from: https://github.com/sqlite/sqlite/blob/master/tool/mkkeywordhash.c
45
+ KEYWORDS = Set[
46
+ :ABORT,
47
+ :ACTION,
48
+ :ADD,
49
+ :AFTER,
50
+ :ALL,
51
+ :ALTER,
52
+ :ALWAYS,
53
+ :ANALYZE,
54
+ :AND,
55
+ :AS,
56
+ :ASC,
57
+ :ATTACH,
58
+ :AUTOINCREMENT,
59
+ :BEFORE,
60
+ :BEGIN,
61
+ :BETWEEN,
62
+ :BY,
63
+ :CASCADE,
64
+ :CASE,
65
+ :CAST,
66
+ :CHECK,
67
+ :COLLATE,
68
+ :COLUMN,
69
+ :COMMIT,
70
+ :CONFLICT,
71
+ :CONSTRAINT,
72
+ :CREATE,
73
+ :CROSS,
74
+ :CURRENT,
75
+ :CURRENT_DATE,
76
+ :CURRENT_TIME,
77
+ :CURRENT_TIMESTAMP,
78
+ :DATABASE,
79
+ :DEFAULT,
80
+ :DEFERRED,
81
+ :DEFERRABLE,
82
+ :DELETE,
83
+ :DESC,
84
+ :DETACH,
85
+ :DISTINCT,
86
+ :DO,
87
+ :DROP,
88
+ :END,
89
+ :EACH,
90
+ :ELSE,
91
+ :ESCAPE,
92
+ :EXCEPT,
93
+ :EXCLUSIVE,
94
+ :EXCLUDE,
95
+ :EXISTS,
96
+ :EXPLAIN,
97
+ :FAIL,
98
+ :FILTER,
99
+ :FIRST,
100
+ :FOLLOWING,
101
+ :FOR,
102
+ :FOREIGN,
103
+ :FROM,
104
+ :FULL,
105
+ :GENERATED,
106
+ :GLOB,
107
+ :GROUP,
108
+ :GROUPS,
109
+ :HAVING,
110
+ :IF,
111
+ :IGNORE,
112
+ :IMMEDIATE,
113
+ :IN,
114
+ :INDEX,
115
+ :INDEXED,
116
+ :INITIALLY,
117
+ :INNER,
118
+ :INSERT,
119
+ :INSTEAD,
120
+ :INTERSECT,
121
+ :INTO,
122
+ :IS,
123
+ :ISNULL,
124
+ :JOIN,
125
+ :KEY,
126
+ :LAST,
127
+ :LEFT,
128
+ :LIKE,
129
+ :LIMIT,
130
+ :MATCH,
131
+ :MATERIALIZED,
132
+ :NATURAL,
133
+ :NO,
134
+ :NOT,
135
+ :NOTHING,
136
+ :NOTNULL,
137
+ :NULL,
138
+ :NULLS,
139
+ :OF,
140
+ :OFFSET,
141
+ :ON,
142
+ :OR,
143
+ :ORDER,
144
+ :OTHERS,
145
+ :OUTER,
146
+ :OVER,
147
+ :PARTITION,
148
+ :PLAN,
149
+ :PRAGMA,
150
+ :PRECEDING,
151
+ :PRIMARY,
152
+ :QUERY,
153
+ :RAISE,
154
+ :RANGE,
155
+ :RECURSIVE,
156
+ :REFERENCES,
157
+ :REGEXP,
158
+ :REINDEX,
159
+ :RELEASE,
160
+ :RENAME,
161
+ :REPLACE,
162
+ :RESTRICT,
163
+ :RETURNING,
164
+ :RIGHT,
165
+ :ROLLBACK,
166
+ :ROW,
167
+ :ROWID,
168
+ :ROWS,
169
+ :SAVEPOINT,
170
+ :SELECT,
171
+ :SET,
172
+ :STORED,
173
+ :STRICT,
174
+ :TABLE,
175
+ :TEMP,
176
+ :TEMPORARY,
177
+ :THEN,
178
+ :TIES,
179
+ :TO,
180
+ :TRANSACTION,
181
+ :TRIGGER,
182
+ :UNBOUNDED,
183
+ :UNION,
184
+ :UNIQUE,
185
+ :UPDATE,
186
+ :USING,
187
+ :VACUUM,
188
+ :VALUES,
189
+ :VIEW,
190
+ :VIRTUAL,
191
+ :WHEN,
192
+ :WHERE,
193
+ :WINDOW,
194
+ :WITH,
195
+ :WITHIN,
196
+ :WITHOUT,
197
+ :TRUE,
198
+ :FALSE,
199
+ ].freeze
200
+
201
+ # Maps ASCII characters to their respective character types
202
+ # Each element's index maps to the byte value of that character
203
+ CHARACTER_TYPES = %i[
204
+ NUL ILLEGAL ILLEGAL ILLEGAL ILLEGAL ILLEGAL ILLEGAL ILLEGAL
205
+ ILLEGAL SPACE SPACE ILLEGAL SPACE SPACE ILLEGAL ILLEGAL
206
+ ILLEGAL ILLEGAL ILLEGAL ILLEGAL ILLEGAL ILLEGAL ILLEGAL ILLEGAL
207
+ ILLEGAL ILLEGAL ILLEGAL ILLEGAL ILLEGAL ILLEGAL ILLEGAL ILLEGAL
208
+ SPACE BANG QUOTE VARALPHA DOLLAR PERCENT AND QUOTE
209
+ LP RP STAR PLUS COMMA MINUS DOT SLASH
210
+ DIGIT DIGIT DIGIT DIGIT DIGIT DIGIT DIGIT DIGIT
211
+ DIGIT DIGIT VARALPHA SEMI LT EQ GT VARNUM
212
+ VARALPHA KYWD0 KYWD0 KYWD0 KYWD0 KYWD0 KYWD0 KYWD0
213
+ KYWD0 KYWD0 KYWD0 KYWD0 KYWD0 KYWD0 KYWD0 KYWD0
214
+ KYWD0 KYWD0 KYWD0 KYWD0 KYWD0 KYWD0 KYWD0 KYWD0
215
+ X KYWD KYWD QUOTE2 ILLEGAL ILLEGAL ILLEGAL KYWD
216
+ QUOTE KYWD0 KYWD0 KYWD0 KYWD0 KYWD0 KYWD0 KYWD0
217
+ KYWD0 KYWD0 KYWD0 KYWD0 KYWD0 KYWD0 KYWD0 KYWD0
218
+ KYWD0 KYWD0 KYWD0 KYWD0 KYWD0 KYWD0 KYWD0 KYWD0
219
+ X KYWD KYWD ILLEGAL PIPE ILLEGAL TILDA ILLEGAL
220
+ ID ID ID ID ID ID ID ID
221
+ ID ID ID ID ID ID ID ID
222
+ ID ID ID ID ID ID ID ID
223
+ ID ID ID ID ID ID ID ID
224
+ ID ID ID ID ID ID ID ID
225
+ ID ID ID ID ID ID ID ID
226
+ ID ID ID ID ID ID ID ID
227
+ ID ID ID ID ID ID ID ID
228
+ ID ID ID ID ID ID ID ID
229
+ ID ID ID ID ID ID ID ID
230
+ ID ID ID ID ID ID ID ID
231
+ ID ID ID ID ID ID ID ID
232
+ ID ID ID ID ID ID ID ID
233
+ ID ID ID ID ID ID ID BOM
234
+ ID ID ID ID ID ID ID ID
235
+ ID ID ID ID ID ID ID ID
236
+ ].freeze
237
+
238
+ # The following 256 element lookup table encodes the characteristics
239
+ # of each ASCII character in a bitmask:
240
+ #
241
+ # space? 0x01
242
+ # alpha? 0x02
243
+ # digit? 0x04
244
+ # alnum? 0x06
245
+ # xdigit? 0x08
246
+ # to_upper 0x20
247
+ # id? 0x40
248
+ # quote? 0x80
249
+ #
250
+ # Bit 0x40 is set if the character is non-alphanumeric and can be used in an
251
+ # SQLite identifier. Identifiers are alphanumerics, "_", "$", and any
252
+ # non-ASCII UTF character. Hence the test for whether or not a character is
253
+ # part of an identifier is 0x46.
254
+ CHARACTER_FLAGS = [
255
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, # 00..07 ........
256
+ 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, # 08..0f ........
257
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, # 10..17 ........
258
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, # 18..1f ........
259
+ 0x01, 0x00, 0x80, 0x00, 0x40, 0x00, 0x00, 0x80, # 20..27 !"#$%&'
260
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, # 28..2f ()*+,-./
261
+ 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, # 30..37 01234567
262
+ 0x0c, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, # 38..3f 89:;<=>?
263
+ 0x00, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x02, # 40..47 @ABCDEFG
264
+ 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, # 48..4f HIJKLMNO
265
+ 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, # 50..57 PQRSTUVW
266
+ 0x02, 0x02, 0x02, 0x80, 0x00, 0x00, 0x00, 0x40, # 58..5f XYZ[\]^_
267
+ 0x80, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x22, # 60..67 `abcdefg
268
+ 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, # 68..6f hijklmno
269
+ 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, # 70..77 pqrstuvw
270
+ 0x22, 0x22, 0x22, 0x00, 0x00, 0x00, 0x00, 0x00, # 78..7f xyz{|}~.
271
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, # 80..87 ........
272
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, # 88..8f ........
273
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, # 90..97 ........
274
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, # 98..9f ........
275
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, # a0..a7 ........
276
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, # a8..af ........
277
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, # b0..b7 ........
278
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, # b8..bf ........
279
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, # c0..c7 ........
280
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, # c8..cf ........
281
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, # d0..d7 ........
282
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, # d8..df ........
283
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, # e0..e7 ........
284
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, # e8..ef ........
285
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, # f0..f7 ........
286
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40 # f8..ff ........
287
+ ].freeze
288
+
289
+ SINGLE_CHAR_TOKENS = [].tap do |it|
290
+ it[0] = :ILLEGAL # \0
291
+ it[37] = :REM # %
292
+ it[38] = :BITAND # &
293
+ it[40] = :LP # (
294
+ it[41] = :RP # )
295
+ it[42] = :STAR # *
296
+ it[43] = :PLUS # +
297
+ it[44] = :COMMA # ,
298
+ it[59] = :SEMI # ;
299
+ it[126] = :BITNOT # ~
300
+ end.freeze
301
+
302
+ attr_reader :sql
303
+ attr_accessor :cursor, :anchor
304
+
305
+ def initialize(sql, skip_spaces: false)
306
+ # Freezing the caller's string would leave them with a frozen string they did not ask for
307
+ @sql =
308
+ case sql.encoding
309
+ when Encoding::UTF_8, Encoding::ASCII_8BIT then sql.frozen? ? sql : sql.dup.freeze
310
+ else sql.encode(Encoding::UTF_8).freeze
311
+ end
312
+ @skip_spaces = skip_spaces
313
+ @cursor, @anchor = 0
314
+ end
315
+
316
+ def next_token
317
+ @anchor = @cursor
318
+ token = lex_next_token
319
+ return next_token if (token == :SPACE || token == :COMMENT) && @skip_spaces
320
+ token
321
+ end
322
+
323
+ def value
324
+ @sql.byteslice(@anchor, (@cursor - @anchor))
325
+ end
326
+
327
+ private
328
+
329
+ def lex_next_token
330
+ current_byte = scan
331
+
332
+ return nil if current_byte.nil?
333
+ if (token = SINGLE_CHAR_TOKENS[current_byte])
334
+ return token
335
+ end
336
+
337
+ character_type = CHARACTER_TYPES[current_byte]
338
+ case character_type
339
+ when :SPACE
340
+ # useless branch conditions to drive branch test coverage results
341
+ nil if current_byte == 32 # " "
342
+ nil if current_byte == 9 # "\t"
343
+ nil if current_byte == 10 # "\n"
344
+ nil if current_byte == 12 # "\f"
345
+ nil if current_byte == 13 # "\r"
346
+
347
+ step until (b = peek).nil? || !space?(b)
348
+
349
+ :SPACE
350
+ when :MINUS
351
+ if peek == 45 # "-"
352
+ step
353
+ step until (b = peek).nil? || b == 10 # "\n"
354
+
355
+ :COMMENT # which SQLite takes for a space [IMP: R-22934-2513]
356
+ elsif peek == 62 # ">"
357
+ # differentiate between different pointers
358
+ if peek(1) == 62 # ">"
359
+ step(2)
360
+ :PTR2
361
+ else
362
+ step
363
+ :PTR1
364
+ end
365
+ else
366
+ :MINUS
367
+ end
368
+ when :SLASH
369
+ return :SLASH if peek.nil? # EOF
370
+ return :SLASH if peek != 42 # "*"
371
+
372
+ step # we know it's a `/*` comment so consume the "*"
373
+ step until (b = peek).nil? || (b == 42 && peek(1) == 47) # consume until EOF or "*/"
374
+ step(2) if peek == 42 && peek(1) == 47 # consume the "*/"
375
+
376
+ :COMMENT # which SQLite takes for a space [IMP: R-22934-2513]
377
+ when :EQ
378
+ step if peek == 61 # "="
379
+ :EQ
380
+ when :LT
381
+ return step && :LE if peek == 61 # "="
382
+ return step && :NE if peek == 62 # ">"
383
+ return step && :LSHIFT if peek == 60 # "<"
384
+
385
+ :LT
386
+ when :GT
387
+ return step && :GE if peek == 61 # "="
388
+ return step && :RSHIFT if peek == 62 # ">"
389
+
390
+ :GT
391
+ when :BANG
392
+ return step && :NE if peek == 61 # "="
393
+
394
+ :ILLEGAL
395
+ when :PIPE
396
+ return step && :CONCAT if peek == 124 # "|"
397
+
398
+ :BITOR
399
+ when :QUOTE
400
+ # useless branch conditions to drive brach test coverage results
401
+ nil if current_byte == 96 # "`"
402
+ nil if current_byte == 39 # "'"
403
+ nil if current_byte == 34 # '"'
404
+
405
+ while (b = scan)
406
+ next unless b == current_byte
407
+ # you can use the same quote to escape itself
408
+ (peek == current_byte) ? step : break
409
+ end
410
+
411
+ return :STRING if b == 39 # "'"
412
+ return :ILLEGAL if b.nil?
413
+
414
+ :ID
415
+ when :DOT, :DIGIT
416
+ # If the current character is a "." and the next character is either EOF or not a digit,
417
+ # then this token is simply the "." character.
418
+ if character_type == :DOT && ((b = peek).nil? || !digit?(b))
419
+ return :DOT
420
+ end
421
+
422
+ # useless branch conditions to drive brach test coverage results
423
+ nil if current_byte == 46 # "."
424
+ nil if current_byte == 48 # "0"
425
+ nil if current_byte == 49 # "1"
426
+ nil if current_byte == 50 # "2"
427
+ nil if current_byte == 51 # "3"
428
+ nil if current_byte == 52 # "4"
429
+ nil if current_byte == 53 # "5"
430
+ nil if current_byte == 54 # "6"
431
+ nil if current_byte == 55 # "7"
432
+ nil if current_byte == 56 # "8"
433
+ nil if current_byte == 57 # "9"
434
+
435
+ token_type = (character_type == :DOT) ? :FLOAT : :INTEGER
436
+ if current_byte == 48 && ((b = peek) == 120 || b == 88) && xdigit?(peek(1)) # "0" and "x" or "X"
437
+ step(2) # consume the "x" or "X" and the first hex digit
438
+ while (b = peek)
439
+ if xdigit?(b)
440
+ step && next
441
+ elsif b == 95 # "_"
442
+ token_type = :QNUMBER
443
+ step && next
444
+ else
445
+ break
446
+ end
447
+ end
448
+
449
+ return token_type
450
+ end
451
+
452
+ while (b = peek)
453
+ if digit?(b)
454
+ step && next
455
+ elsif b == 95 # "_"
456
+ token_type = :QNUMBER
457
+ step && next
458
+ else
459
+ break
460
+ end
461
+ end
462
+
463
+ # A number that began with its "." has had its fraction already, ".5.5" is two tokens
464
+ if character_type != :DOT && peek == 46 # "."
465
+ step
466
+ token_type = :FLOAT if token_type == :INTEGER
467
+
468
+ while (b = peek)
469
+ if digit?(b)
470
+ step && next
471
+ elsif b == 95 # "_"
472
+ token_type = :QNUMBER
473
+ step && next
474
+ else
475
+ break
476
+ end
477
+ end
478
+ end
479
+
480
+ if (peek == 101 || peek == 69) && ( # "e" or "E"
481
+ digit?(peek(1)) || (
482
+ (peek(1) == 43 || peek(1) == 45) && digit?(peek(2)) # "+" or "-"
483
+ )
484
+ )
485
+ step(2)
486
+ token_type = :FLOAT if token_type == :INTEGER
487
+
488
+ while (b = peek)
489
+ if digit?(b)
490
+ step && next
491
+ elsif b == 95 # "_"
492
+ token_type = :QNUMBER
493
+ step && next
494
+ else
495
+ break
496
+ end
497
+ end
498
+ end
499
+
500
+ while id?(peek)
501
+ token_type = :ILLEGAL
502
+ step
503
+ end
504
+
505
+ token_type
506
+ when :QUOTE2
507
+ step until (b = peek).nil? || b == 93 # "]"
508
+
509
+ final = scan
510
+ return :ILLEGAL if final.nil?
511
+
512
+ :ID
513
+ when :VARNUM
514
+ step while digit?(peek)
515
+
516
+ :VARIABLE
517
+ when :DOLLAR, :VARALPHA
518
+ # useless branch conditions to drive brach test coverage results
519
+ nil if current_byte == 35 # "#"
520
+ nil if current_byte == 36 # "$"
521
+ nil if current_byte == 58 # ":"
522
+ nil if current_byte == 64 # "@"
523
+
524
+ n = 0
525
+ token_type = :VARIABLE
526
+ until (b = peek).nil?
527
+ if id?(b)
528
+ n += 1
529
+ elsif n > 0 && b == 40 # "("
530
+ step
531
+ step until (b = peek).nil? || space?(b) || b == 41 # ")"
532
+
533
+ if peek == 41 # ")"
534
+ step
535
+ else
536
+ token_type = :ILLEGAL
537
+ end
538
+
539
+ break
540
+ elsif b == 58 && peek(1) == 58 # ":"
541
+ step
542
+ else
543
+ break
544
+ end
545
+ step
546
+ end
547
+
548
+ if n == 0
549
+ token_type = :ILLEGAL
550
+ end
551
+
552
+ token_type
553
+ when :KYWD0
554
+ # If the next character is not a keyword character, consume all the id characters and return it as an id
555
+ kywd = TYPES.index(:KYWD)
556
+ if (b = peek) && (type = CHARACTER_TYPES[b]) && TYPES.index(type) > kywd
557
+ step while id?(peek)
558
+
559
+ return :ID
560
+ end
561
+
562
+ # Consume all the keyword characters
563
+ step while (b = peek) && (type = CHARACTER_TYPES[b]) && TYPES.index(type) <= kywd
564
+
565
+ # This token started out using characters that can appear in keywords,
566
+ # but now there is a character not allowed within keywords,
567
+ # so this must be an identifier instead
568
+ if (b = peek) && id?(b)
569
+ step while id?(peek)
570
+
571
+ return :ID
572
+ end
573
+
574
+ # Check if the token is a keyword
575
+ token = value.to_sym.upcase
576
+ KEYWORDS.include?(token) ? token : :ID
577
+ when :X
578
+ # useless branch conditions to drive brach test coverage results
579
+ nil if current_byte == 120 # "x"
580
+ nil if current_byte == 88 # "X"
581
+
582
+ if peek == 39 # "'"
583
+ token_type = :BLOB
584
+ step
585
+
586
+ step while xdigit?(peek)
587
+
588
+ if peek != 39 || ((@cursor - @anchor) % 2) != 0
589
+ token_type = :ILLEGAL
590
+ step until (b = peek).nil? || b == 39 # "'"
591
+ end
592
+
593
+ step if !peek.nil?
594
+
595
+ return token_type
596
+ end
597
+
598
+ # If it is not a BLOB literal, then it must be an ID, since no
599
+ # SQL keywords start with the letter 'x'.
600
+ step while id?(peek)
601
+
602
+ :ID
603
+ when :KYWD, :ID
604
+ step while id?(peek)
605
+
606
+ :ID
607
+ when :BOM
608
+ return step(2) && :SPACE if peek == 0xBB && peek(1) == 0xBF
609
+
610
+ step while id?(peek)
611
+ :ID
612
+ else
613
+ :ILLEGAL
614
+ end
615
+ end
616
+
617
+ def peek(n = 0) = @sql.getbyte(@cursor+n)
618
+ def step(n = 1) = @cursor += n
619
+ def scan(n = 0) = peek(n).tap { step(n+1) }
620
+
621
+ def space?(byte) = byte && CHARACTER_FLAGS[byte] & 0x01 != 0
622
+ def alnum?(byte) = byte && CHARACTER_FLAGS[byte] & 0x06 != 0
623
+ def alpha?(byte) = byte && CHARACTER_FLAGS[byte] & 0x02 != 0
624
+ def digit?(byte) = byte && CHARACTER_FLAGS[byte] & 0x04 != 0
625
+ def xdigit?(byte) = byte && CHARACTER_FLAGS[byte] & 0x08 != 0
626
+ def quote?(byte) = byte && CHARACTER_FLAGS[byte] & 0x80 != 0
627
+ def id?(byte) = byte && CHARACTER_FLAGS[byte] & 0x46 != 0
628
+ # If X is a character that can be used in an identifier then
629
+ # IdChar(X) will be true. Otherwise it is false.
630
+ #
631
+ # For ASCII, any character with the high-order bit set is
632
+ # allowed in an identifier. For 7-bit characters,
633
+ # sqlite3IsIdChar[X] must be 1.
634
+ #
635
+ # Ticket #1066. the SQL standard does not allow '$' in the
636
+ # middle of identifiers. But many SQL implementations do.
637
+ # SQLite will allow '$' in identifiers for compatibility.
638
+ # But the feature is undocumented.
639
+ end
640
+ end
@@ -0,0 +1,76 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Plume
4
+ # Stand-ins for what yippee-fun/literal#391 fixes in Literal itself, so that Plume can run
5
+ # in Ractors before that is released. They apply to Plume's own classes only, and each one
6
+ # stays off when the installed Literal does not need it. Once Plume requires a Literal
7
+ # with that fix, delete this file, its require and extend in plume.rb and its prepend in Node
8
+ module LiteralPatches
9
+ # A Mutex can never be shared between Ractors, and Literal keeps one in every schema
10
+ SCHEMA_HOLDS_MUTEX = Literal::Properties::Schema.new.instance_variable_defined?(:@mutex)
11
+
12
+ # Literal 1.9.0 calls the block of a `_Deferred` on every check. Unreleased Literal caches
13
+ # the result instead, but writes to the type on every check while doing so - and a frozen
14
+ # one then raises FrozenError, materialized or not
15
+ DEFERRED_WRITES_ON_CHECK = begin
16
+ probe = Literal::Types::DeferredType.new { Integer }
17
+ probe === 1
18
+ probe.freeze === 1
19
+ false
20
+ rescue FrozenError
21
+ true
22
+ end
23
+
24
+ # A frozen schema with the same readers, which `make_shareable!` swaps in once the props
25
+ # of a node class are all declared
26
+ class ShareableSchema
27
+ include Enumerable
28
+
29
+ attr_reader :properties_index
30
+
31
+ def initialize(schema)
32
+ @properties_index = schema.properties_index.dup
33
+ @sorted_properties = schema.to_a
34
+ end
35
+
36
+ def [](key) = @properties_index[key]
37
+ def each(&) = @sorted_properties.each(&)
38
+ def size = @sorted_properties.size
39
+ def empty? = @sorted_properties.empty?
40
+
41
+ # Literal starts the schema of a subclass from a dup of its parent's
42
+ def dup = Literal::Properties::Schema.new(properties_index: @properties_index.dup, sorted_properties: @sorted_properties.dup)
43
+ end
44
+
45
+ module NodeSchema
46
+ def literal_properties = @shareable_literal_properties || super
47
+
48
+ def make_shareable!
49
+ @shareable_literal_properties = Plume.make_shareable(ShareableSchema.new(literal_properties))
50
+ super
51
+ end
52
+ end
53
+
54
+ # Only ever used when DEFERRED_WRITES_ON_CHECK, so this can rely on the caching version's ivars
55
+ class Deferred < Literal::Types::DeferredType
56
+ def materialize = frozen? ? frozen_type : super
57
+
58
+ # JRuby takes an endless operator ending in "=" for a setter
59
+ def ===(other)
60
+ frozen? ? frozen_type === other : super
61
+ end
62
+
63
+ def >=(other, context: nil)
64
+ frozen? ? Literal.subtype?(other, frozen_type, context:) : super
65
+ end
66
+
67
+ private
68
+
69
+ def frozen_type = UNMATERIALIZED.equal?(@materialized) ? @block.call : @materialized
70
+ end
71
+
72
+ module Types
73
+ def _Deferred(&) = Deferred.new(&)
74
+ end
75
+ end
76
+ end