plume 0.1.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (124) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +506 -11
  3. data/lib/plume/analysis/resolver.rb +797 -0
  4. data/lib/plume/analysis.rb +211 -0
  5. data/lib/plume/ast/alter_table_statement.rb +40 -0
  6. data/lib/plume/ast/analyze_statement.rb +15 -0
  7. data/lib/plume/ast/assignment.rb +18 -0
  8. data/lib/plume/ast/attach_statement.rb +17 -0
  9. data/lib/plume/ast/begin_statement.rb +17 -0
  10. data/lib/plume/ast/binary_expression.rb +71 -0
  11. data/lib/plume/ast/case_condition.rb +29 -0
  12. data/lib/plume/ast/case_expression.rb +22 -0
  13. data/lib/plume/ast/cast_expression.rb +16 -0
  14. data/lib/plume/ast/check_column_constraint.rb +20 -0
  15. data/lib/plume/ast/check_table_constraint.rb +20 -0
  16. data/lib/plume/ast/collate_column_constraint.rb +19 -0
  17. data/lib/plume/ast/collation_expression.rb +26 -0
  18. data/lib/plume/ast/column_constraint.rb +7 -0
  19. data/lib/plume/ast/column_definition.rb +16 -0
  20. data/lib/plume/ast/column_name.rb +22 -0
  21. data/lib/plume/ast/column_type.rb +26 -0
  22. data/lib/plume/ast/commit_statement.rb +13 -0
  23. data/lib/plume/ast/common_table_expression.rb +31 -0
  24. data/lib/plume/ast/compound_select.rb +30 -0
  25. data/lib/plume/ast/conflict_clause.rb +22 -0
  26. data/lib/plume/ast/conflict_resolution.rb +20 -0
  27. data/lib/plume/ast/create_index_statement.rb +29 -0
  28. data/lib/plume/ast/create_table_statement.rb +47 -0
  29. data/lib/plume/ast/create_trigger_statement.rb +43 -0
  30. data/lib/plume/ast/create_view_statement.rb +30 -0
  31. data/lib/plume/ast/create_virtual_table_statement.rb +28 -0
  32. data/lib/plume/ast/default_column_constraint.rb +29 -0
  33. data/lib/plume/ast/deferrable_column_constraint.rb +20 -0
  34. data/lib/plume/ast/delete_statement.rb +28 -0
  35. data/lib/plume/ast/detach_statement.rb +13 -0
  36. data/lib/plume/ast/drop_statement.rb +21 -0
  37. data/lib/plume/ast/empty_function_argument.rb +6 -0
  38. data/lib/plume/ast/exists_expression.rb +15 -0
  39. data/lib/plume/ast/explain_statement.rb +15 -0
  40. data/lib/plume/ast/expression.rb +30 -0
  41. data/lib/plume/ast/foreign_key_action.rb +26 -0
  42. data/lib/plume/ast/foreign_key_clause.rb +50 -0
  43. data/lib/plume/ast/foreign_key_column_constraint.rb +16 -0
  44. data/lib/plume/ast/foreign_key_table_constraint.rb +20 -0
  45. data/lib/plume/ast/frame_boundary.rb +35 -0
  46. data/lib/plume/ast/frame_spec.rb +34 -0
  47. data/lib/plume/ast/function_arguments.rb +18 -0
  48. data/lib/plume/ast/function_reference.rb +29 -0
  49. data/lib/plume/ast/generated_as_column_constraint.rb +24 -0
  50. data/lib/plume/ast/indexed_column.rb +24 -0
  51. data/lib/plume/ast/insert_statement.rb +35 -0
  52. data/lib/plume/ast/join.rb +12 -0
  53. data/lib/plume/ast/join_clause.rb +15 -0
  54. data/lib/plume/ast/join_on_constraint.rb +12 -0
  55. data/lib/plume/ast/join_operator.rb +31 -0
  56. data/lib/plume/ast/join_using_constraint.rb +16 -0
  57. data/lib/plume/ast/like_expression.rb +19 -0
  58. data/lib/plume/ast/literal_expression.rb +29 -0
  59. data/lib/plume/ast/literal_false.rb +7 -0
  60. data/lib/plume/ast/literal_nil.rb +7 -0
  61. data/lib/plume/ast/literal_value.rb +18 -0
  62. data/lib/plume/ast/match_clause.rb +15 -0
  63. data/lib/plume/ast/node.rb +330 -0
  64. data/lib/plume/ast/noop_column_constraint.rb +15 -0
  65. data/lib/plume/ast/noop_table_constraint.rb +15 -0
  66. data/lib/plume/ast/not_like_expression.rb +7 -0
  67. data/lib/plume/ast/not_null_column_constraint.rb +17 -0
  68. data/lib/plume/ast/null_column_constraint.rb +17 -0
  69. data/lib/plume/ast/ordering_term.rb +32 -0
  70. data/lib/plume/ast/over_clause.rb +25 -0
  71. data/lib/plume/ast/parenthetical_expression.rb +37 -0
  72. data/lib/plume/ast/parenthetical_source.rb +19 -0
  73. data/lib/plume/ast/pragma_statement.rb +26 -0
  74. data/lib/plume/ast/primary_key_column_constraint.rb +23 -0
  75. data/lib/plume/ast/primary_key_table_constraint.rb +23 -0
  76. data/lib/plume/ast/raise_expression.rb +23 -0
  77. data/lib/plume/ast/reindex_statement.rb +15 -0
  78. data/lib/plume/ast/release_statement.rb +16 -0
  79. data/lib/plume/ast/result_column.rb +18 -0
  80. data/lib/plume/ast/returning_clause.rb +12 -0
  81. data/lib/plume/ast/rollback_statement.rb +19 -0
  82. data/lib/plume/ast/savepoint_statement.rb +16 -0
  83. data/lib/plume/ast/select_core.rb +27 -0
  84. data/lib/plume/ast/select_statement.rb +34 -0
  85. data/lib/plume/ast/star_function_argument.rb +6 -0
  86. data/lib/plume/ast/star_result_column.rb +16 -0
  87. data/lib/plume/ast/strict_table_option.rb +6 -0
  88. data/lib/plume/ast/subquery_expression.rb +14 -0
  89. data/lib/plume/ast/subquery_source.rb +19 -0
  90. data/lib/plume/ast/table_constraint.rb +7 -0
  91. data/lib/plume/ast/table_function.rb +28 -0
  92. data/lib/plume/ast/table_name.rb +28 -0
  93. data/lib/plume/ast/table_option.rb +7 -0
  94. data/lib/plume/ast/ternary_expression.rb +26 -0
  95. data/lib/plume/ast/token.rb +183 -0
  96. data/lib/plume/ast/unary_expression.rb +36 -0
  97. data/lib/plume/ast/unique_column_constraint.rb +17 -0
  98. data/lib/plume/ast/unique_table_constraint.rb +20 -0
  99. data/lib/plume/ast/update_statement.rb +35 -0
  100. data/lib/plume/ast/upsert_clause.rb +26 -0
  101. data/lib/plume/ast/vacuum_statement.rb +18 -0
  102. data/lib/plume/ast/values_clause.rb +12 -0
  103. data/lib/plume/ast/values_row.rb +13 -0
  104. data/lib/plume/ast/variable.rb +15 -0
  105. data/lib/plume/ast/window_definition.rb +26 -0
  106. data/lib/plume/ast/with_clause.rb +15 -0
  107. data/lib/plume/ast/without_rowid_table_option.rb +6 -0
  108. data/lib/plume/catalog.rb +276 -0
  109. data/lib/plume/comment.rb +73 -0
  110. data/lib/plume/dump.rb +143 -0
  111. data/lib/plume/fingerprint.rb +86 -0
  112. data/lib/plume/lexer.rb +640 -0
  113. data/lib/plume/literal_patches.rb +76 -0
  114. data/lib/plume/normalize.rb +65 -0
  115. data/lib/plume/parameters.rb +52 -0
  116. data/lib/plume/parser.rb +4064 -0
  117. data/lib/plume/pretty_printer.rb +202 -0
  118. data/lib/plume/printer.rb +525 -0
  119. data/lib/plume/sqlite_grammar.rb +86 -0
  120. data/lib/plume/truncate.rb +67 -0
  121. data/lib/plume/union.rb +28 -0
  122. data/lib/plume/version.rb +1 -1
  123. data/lib/plume.rb +274 -3
  124. metadata +176 -12
@@ -0,0 +1,276 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Plume
4
+ # What the read analysis knows about a database: its tables with their columns, its views,
5
+ # and which table each index and trigger belongs to. Names are matched case-insensitively, the way SQLite matches identifiers, and
6
+ # reported in the spelling they were declared with.
7
+ class Catalog
8
+ ROWID_NAMES = %w[rowid oid _rowid_].freeze
9
+
10
+ # `columns` is nil when they cannot be known, like a CREATE TABLE … AS SELECT * from a table
11
+ # the catalog lacks; any column name is then taken as written. `generated_columns` are among
12
+ # `columns` too, since they can be read like any other, but nothing can write to them.
13
+ # `module_name` is set for a virtual table, like "fts5"
14
+ Table = Data.define(:schema, :name, :columns, :hidden_columns, :generated_columns, :rowid_alias, :without_rowid, :module_name) do
15
+ def column(name)
16
+ return name.to_s unless columns
17
+
18
+ needle = name.to_s.downcase(:ascii)
19
+ columns.find { |c| c.downcase(:ascii) == needle } || hidden_columns.find { |c| c.downcase(:ascii) == needle }
20
+ end
21
+
22
+ def rowid? = !without_rowid
23
+ end
24
+
25
+ View = Data.define(:schema, :name, :columns, :select)
26
+ Index = Data.define(:schema, :name, :table)
27
+ Trigger = Data.define(:schema, :name, :table)
28
+
29
+ def self.from_sql(sql) = new.load_sql(sql)
30
+
31
+ # Whether two names are one to SQLite, which folds the case of A-Z only. Compared as bytes, so a
32
+ # name that is not valid UTF-8 compares too - JRuby's casecmp? raises for one
33
+ def self.same_name?(a, b) = !a.nil? && !b.nil? && a.to_s.b.casecmp?(b.to_s.b)
34
+
35
+ # The indexes SQLite makes for a table's UNIQUE and PRIMARY KEY constraints, in the order it
36
+ # makes them - all but the rowid's own primary key, even where two cover the same columns
37
+ def self.autoindexes(statement)
38
+ return [] if statement.select_statement
39
+
40
+ alias_column = new.__send__(:rowid_alias, statement)&.downcase(:ascii)
41
+ constraints = statement.columns.flat_map do |column|
42
+ (column.constraints || []).select { |c| UniqueColumnConstraint === c || (PrimaryKeyColumnConstraint === c && column.name.downcase(:ascii) != alias_column) }
43
+ end
44
+ constraints += (statement.constraints || []).select do |c|
45
+ UniqueTableConstraint === c || (PrimaryKeyTableConstraint === c && !(alias_column && c.columns.size == 1))
46
+ end
47
+ Array.new(constraints.size) { |i| "sqlite_autoindex_#{statement.table.name}_#{i + 1}" }
48
+ end
49
+
50
+ # Takes a sqlite3-ruby `SQLite3::Database` (or anything with a compatible `#execute`)
51
+ def self.from_sqlite(db) = new.load_sqlite(db)
52
+
53
+ def initialize
54
+ @tables = {}
55
+ @views = {}
56
+ @indexes = {}
57
+ @triggers = {}
58
+ end
59
+
60
+ def add_table(name, columns, schema: "main", hidden_columns: [], generated_columns: [], rowid_alias: nil, without_rowid: false, module_name: nil)
61
+ table = Table.new(schema:, name:, columns: columns&.freeze, hidden_columns: hidden_columns.freeze, generated_columns: generated_columns.freeze, rowid_alias:, without_rowid:, module_name:)
62
+ @tables[key(schema, name)] = table
63
+ end
64
+
65
+ # `select` is a `SelectStatement` or the SQL for one
66
+ def add_view(name, select, columns: nil, schema: "main")
67
+ select = Parser.new(select).parse.first if String === select
68
+ @views[key(schema, name)] = View.new(schema:, name:, columns: columns&.freeze, select:)
69
+ end
70
+
71
+ # `table` is the name of the table the index or trigger is on, in the same schema
72
+ def add_index(name, table, schema: "main") = @indexes[key(schema, name)] = Index.new(schema:, name:, table:)
73
+ def add_trigger(name, table, schema: "main") = @triggers[key(schema, name)] = Trigger.new(schema:, name:, table:)
74
+
75
+ # Unqualified names are looked up the way SQLite does: temp first, then main,
76
+ # then attached databases in the order they were added
77
+ def table(name, schema: nil) = lookup(@tables, name, schema)
78
+ def view(name, schema: nil) = lookup(@views, name, schema)
79
+ def index(name, schema: nil) = lookup(@indexes, name, schema)
80
+ def trigger(name, schema: nil) = lookup(@triggers, name, schema)
81
+
82
+ def tables = @tables.values
83
+ def views = @views.values
84
+ def indexes = @indexes.values
85
+ def triggers = @triggers.values
86
+
87
+ # Statements other than CREATE (inserts, pragmas…) do not change the schema, and are skipped
88
+ # without being parsed
89
+ def load_sql(sql)
90
+ Catalog.split_statements(sql).each do |statement_sql|
91
+ leading = Plume.tokenize(statement_sql).first(3) - [:TEMP, :TEMPORARY]
92
+ next unless leading[0] == :CREATE && (leading[1] in :TABLE | :VIEW | :INDEX | :UNIQUE | :TRIGGER | :VIRTUAL)
93
+
94
+ apply(Parser.new(statement_sql).parse.first)
95
+ end
96
+ self
97
+ end
98
+
99
+ # Changes the catalog the way running `statement` would change the schema: CREATE adds,
100
+ # DROP removes, ALTER TABLE renames, adds and drops columns. Other statements change nothing
101
+ def apply(statement)
102
+ case statement
103
+ when CreateTableStatement then add_create_table(statement) unless statement.if_not_exists && table(statement.table.name, schema: statement.table.schema)
104
+ when CreateViewStatement then add_create_view(statement) unless statement.if_not_exists && view(statement.view.name, schema: statement.view.schema)
105
+ when CreateVirtualTableStatement
106
+ add_table(statement.table.name, nil, schema: statement.table.schema || "main", module_name: statement.module_name)
107
+ when CreateIndexStatement
108
+ schema = statement.index.schema || table(statement.table)&.schema || "main"
109
+ add_index(statement.index.name, statement.table, schema:)
110
+ when CreateTriggerStatement
111
+ schema = statement.trigger.schema || (statement.temporary ? "temp" : table(statement.table.table)&.schema || "main")
112
+ add_trigger(statement.trigger.name, statement.table.table, schema:)
113
+ when DropStatement then drop(statement.type, statement.target)
114
+ when AlterTableStatement then alter(statement)
115
+ end
116
+ self
117
+ end
118
+
119
+ def initialize_copy(other)
120
+ super
121
+ @tables = @tables.dup
122
+ @views = @views.dup
123
+ @indexes = @indexes.dup
124
+ @triggers = @triggers.dup
125
+ end
126
+
127
+ def load_sqlite(db)
128
+ db.execute("PRAGMA database_list").each do |(_seq, schema, _file)|
129
+ master = (schema == "temp") ? "sqlite_temp_master" : "#{quote(schema)}.sqlite_master"
130
+ db.execute("SELECT type, name, tbl_name, sql FROM #{master} WHERE type IN ('table', 'view', 'index', 'trigger')").each do |(type, name, table, sql)|
131
+ next if name.start_with?("sqlite_")
132
+
133
+ case type
134
+ when "view"
135
+ statement = Parser.new(sql).parse.first
136
+ add_view(name, statement.select, columns: statement.column_names, schema:)
137
+ when "index" then add_index(name, table, schema:)
138
+ when "trigger" then add_trigger(name, table, schema:)
139
+ else add_sqlite_table(db, schema, name, sql)
140
+ end
141
+ end
142
+ end
143
+ self
144
+ end
145
+
146
+ # Splits SQL into statements, keeping trigger bodies (`BEGIN …; …; END`) in one piece
147
+ def self.split_statements(sql) = Plume.split(sql)
148
+
149
+ private
150
+
151
+ def drop(type, target)
152
+ collection = { TABLE: @tables, VIEW: @views, INDEX: @indexes, TRIGGER: @triggers }.fetch(type)
153
+ object = lookup(collection, target.table, target.schema) or return
154
+ collection.delete(key(object.schema, object.name))
155
+ return unless type == :TABLE || type == :VIEW
156
+
157
+ # A table takes its indexes and triggers with it, a view its triggers
158
+ [@indexes, @triggers].each do |dependents|
159
+ dependents.delete_if { |_, d| Catalog.same_name?(d.schema, object.schema) && Catalog.same_name?(d.table, object.name) }
160
+ end
161
+ end
162
+
163
+ def alter(statement)
164
+ table = table(statement.table.table, schema: statement.table.schema) or return
165
+ columns = table.columns
166
+ rename = -> (list, from, to) { list&.map { |c| Catalog.same_name?(c, from) ? to : c } }
167
+
168
+ changed =
169
+ case statement.action
170
+ when :RENAME_TABLE then return rename_table(table, statement.new_name)
171
+ when :ADD_COLUMN
172
+ definition = statement.column_definition
173
+ generated = definition.constraints&.any?(GeneratedAsColumnConstraint)
174
+ table.with(columns: columns && [*columns, definition.name].freeze, generated_columns: generated ? [*table.generated_columns, definition.name].freeze : table.generated_columns)
175
+ when :RENAME_COLUMN
176
+ from, to = statement.column, statement.new_name
177
+ table.with(
178
+ columns: rename.(columns, from, to)&.freeze, generated_columns: rename.(table.generated_columns, from, to).freeze,
179
+ rowid_alias: Catalog.same_name?(table.rowid_alias, from) ? to : table.rowid_alias,
180
+ )
181
+ when :DROP_COLUMN
182
+ gone = -> (list) { list&.reject { |c| Catalog.same_name?(c, statement.column) }&.freeze }
183
+ table.with(columns: gone.(columns), generated_columns: gone.(table.generated_columns))
184
+ else return
185
+ end
186
+ @tables[key(table.schema, table.name)] = changed
187
+ end
188
+
189
+ def rename_table(table, new_name)
190
+ @tables.delete(key(table.schema, table.name))
191
+ @tables[key(table.schema, new_name)] = table.with(name: new_name)
192
+ old_prefix = "sqlite_autoindex_#{table.name}_"
193
+ @indexes.transform_values! { |i| (Catalog.same_name?(i.schema, table.schema) && Catalog.same_name?(i.table, table.name)) ? i.with(table: new_name) : i }
194
+ @indexes.keys.each do |k|
195
+ index = @indexes[k]
196
+ next unless index.name.start_with?(old_prefix) && index.table == new_name
197
+
198
+ @indexes.delete(k)
199
+ renamed = index.with(name: index.name.sub(old_prefix, "sqlite_autoindex_#{new_name}_"))
200
+ @indexes[key(renamed.schema, renamed.name)] = renamed
201
+ end
202
+ @triggers.transform_values! { |t| (Catalog.same_name?(t.schema, table.schema) && Catalog.same_name?(t.table, table.name)) ? t.with(table: new_name) : t }
203
+ end
204
+
205
+ def key(schema, name) = [schema.to_s.downcase(:ascii), name.to_s.downcase(:ascii)].freeze
206
+
207
+ def lookup(collection, name, schema)
208
+ return collection[key(schema, name)] if schema
209
+
210
+ collection[key("temp", name)] || collection[key("main", name)] ||
211
+ collection.find { |(_s, n), _| n == name.to_s.downcase(:ascii) }&.last
212
+ end
213
+
214
+ def schema_of(name, statement) = name.schema || (statement.temporary ? "temp" : "main")
215
+
216
+ def add_create_table(statement)
217
+ schema = schema_of(statement.table, statement)
218
+ name = statement.table.name
219
+
220
+ if statement.select_statement
221
+ columns = Analysis::Resolver.new(catalog: self).output_columns(statement.select_statement)
222
+ add_table(name, (columns unless columns.include?(nil)), schema:)
223
+ return
224
+ end
225
+
226
+ columns = statement.columns.map(&:name)
227
+ generated = statement.columns.select { |c| c.constraints&.any?(GeneratedAsColumnConstraint) }.map(&:name)
228
+ add_table(name, columns, schema:, generated_columns: generated, rowid_alias: rowid_alias(statement), without_rowid: statement.without_row_id)
229
+ Catalog.autoindexes(statement).each { |index| add_index(index, name, schema:) }
230
+ end
231
+
232
+ def add_create_view(statement)
233
+ schema = schema_of(statement.view, statement)
234
+ add_view(statement.view.name, statement.select, columns: statement.column_names, schema:)
235
+ end
236
+
237
+ # A column becomes another name for the rowid when it is the whole primary key and is
238
+ # declared as exactly INTEGER - not INT, not BIGINT. See https://www.sqlite.org/lang_createtable.html#rowid
239
+ def rowid_alias(statement)
240
+ return if statement.without_row_id
241
+
242
+ candidate = statement.columns.find do |column|
243
+ column.constraints&.any? { |c| PrimaryKeyColumnConstraint === c && c.direction != :DESC }
244
+ end
245
+
246
+ unless candidate
247
+ table_pk = statement.constraints&.find { |c| PrimaryKeyTableConstraint === c }
248
+ if table_pk && table_pk.columns.size == 1
249
+ pk_name = table_pk.columns[0].name.downcase(:ascii)
250
+ candidate = statement.columns.find { |c| c.name.downcase(:ascii) == pk_name }
251
+ end
252
+ end
253
+
254
+ candidate.name if Catalog.same_name?(candidate&.type&.text, "INTEGER")
255
+ end
256
+
257
+ def add_sqlite_table(db, schema, name, sql)
258
+ info = db.execute("PRAGMA #{quote(schema)}.table_xinfo(#{quote(name)})")
259
+ # hidden is 0 for ordinary columns, 1 for hidden virtual table columns, 2 and 3 for generated ones
260
+ columns = info.reject { |row| row[6] == 1 }.map { |row| row[1] }
261
+ hidden = info.select { |row| row[6] == 1 }.map { |row| row[1] }
262
+ generated = info.select { |row| row[6] == 2 || row[6] == 3 }.map { |row| row[1] }
263
+ without_rowid = sql.to_s.match?(/\bWITHOUT\s+ROWID\s*;?\s*\z/i)
264
+ pk_rows = info.select { |row| row[5].to_i > 0 }
265
+ rowid_alias = pk_rows[0][1] if !without_rowid && pk_rows.size == 1 && Catalog.same_name?(pk_rows[0][2].to_s, "INTEGER")
266
+
267
+ statement = Parser.new(sql).parse.first
268
+ module_name = statement.module_name if CreateVirtualTableStatement === statement
269
+ add_table(name, columns, schema:, hidden_columns: hidden, generated_columns: generated, rowid_alias:, without_rowid:, module_name:)
270
+ # The schema has no row for a WITHOUT ROWID table's primary key, though SQLite counts it as an index
271
+ Catalog.autoindexes(statement).each { |index| add_index(index, name, schema:) } if CreateTableStatement === statement
272
+ end
273
+
274
+ def quote(identifier) = %("#{identifier.to_s.gsub('"', '""')}")
275
+ end
276
+ end
@@ -0,0 +1,73 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "uri"
4
+
5
+ module Plume
6
+ # A comment in SQL, `-- like this` or `/* like this */`, with the byte range it takes up
7
+ Comment = Data.define(:text, :location) do
8
+ def block? = text.start_with?("/*")
9
+
10
+ # What is between the markers
11
+ def body
12
+ block? ? text.delete_prefix("/*").delete_suffix("*/") : text.delete_prefix("--")
13
+ end
14
+
15
+ # The tags of a sqlcommenter comment - `/*controller='posts',action='index'*/`, which Rails'
16
+ # query_log_tags writes too - or of its older `key:value` format. Empty for any other comment
17
+ def tags
18
+ parts = body.strip.split(/,(?=[\w.-]+(?:=|:))/)
19
+ return {} if parts.empty?
20
+
21
+ if parts.all? { |part| part.match?(/\A[\w.%-]+='.*'\z/m) }
22
+ parts.to_h do |part|
23
+ key, value = part.split("=", 2)
24
+ [URI.decode_www_form_component(key), URI.decode_www_form_component(value[1..-2].gsub("\\'", "'"))]
25
+ end
26
+ elsif parts.all? { |part| part.match?(/\A[\w.-]+:[^,\s]+\z/) }
27
+ parts.to_h { |part| part.split(":", 2) }
28
+ else
29
+ {}
30
+ end
31
+ end
32
+ end
33
+
34
+ module Comments
35
+ extend self
36
+
37
+ def in(sql)
38
+ lexer = Lexer.new(sql)
39
+ found = []
40
+ while (token = lexer.next_token)
41
+ found << Comment.new(text: lexer.value, location: [lexer.anchor, lexer.cursor].freeze) if token == :COMMENT
42
+ end
43
+ found
44
+ end
45
+
46
+ # The comments right before `node` and right after it, with only spaces between them and it
47
+ def around(node)
48
+ source = node.full_source
49
+ loc = node.location
50
+ return [[], []] unless source && loc
51
+
52
+ tokens = Plume.tokenize(source, with_values: true, with_positions: true, skip_spaces: false)
53
+ first = tokens.index { |_, _, beg, _| beg >= loc[0] }
54
+ last = tokens.rindex { |_, _, _, fin| fin <= loc[1] }
55
+ [gather(tokens, first - 1, -1), gather(tokens, last + 1, 1)]
56
+ end
57
+
58
+ private
59
+
60
+ def gather(tokens, from, step)
61
+ found = []
62
+ i = from
63
+ while i >= 0 && (token = tokens[i])
64
+ type, text, beg, fin = token
65
+ break unless type == :SPACE || type == :COMMENT
66
+
67
+ found << Comment.new(text:, location: [beg, fin].freeze) if type == :COMMENT
68
+ i += step
69
+ end
70
+ (step.negative? ? found.reverse : found)
71
+ end
72
+ end
73
+ end
data/lib/plume/dump.rb ADDED
@@ -0,0 +1,143 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Plume
4
+ # Trees to and from plain hashes that JSON can carry, and back exactly: positions, keywords as
5
+ # written and the source they point into all come through, so `location`, `source_text` and
6
+ # `to_sql` answer as before. A node is its class under "node" and its props under their names,
7
+ # a token is `[type, beg, fin, kind]`, and the source is kept once, at the root.
8
+ # JSON has no symbols, binary strings or infinity, so those go out tagged: {"symbol" => "EQUALS"},
9
+ # {"bytes" => "9e2e", "encoding" => "BINARY"} and {"float" => "Infinity"}
10
+ module Dump
11
+ extend self
12
+
13
+ TOKEN_PROPS = %i[leading trailing].freeze
14
+ SENTINELS = { "LiteralFalse" => LiteralFalse }.freeze
15
+ FLOATS = { "Infinity" => Float::INFINITY, "-Infinity" => -Float::INFINITY, "NaN" => Float::NAN }.freeze
16
+
17
+ # Built without recursing, like `walk`, so the deepest tree SQLite takes can be dumped.
18
+ # Generating JSON that deep needs `max_nesting: false`
19
+ def dump(node)
20
+ return node.map { |n| dump(n) } if Array === node
21
+
22
+ hashes = {}.compare_by_identity
23
+ parents = {}.compare_by_identity
24
+ node.walk do |n, up|
25
+ hashes[n] = { "node" => n.class.name.delete_prefix("Plume::") }
26
+ parents[n] = up
27
+ end
28
+ hashes.each do |n, hash|
29
+ n.class.literal_properties.each do |property|
30
+ name = property.name
31
+ value = n.instance_variable_get(:"@#{name}")
32
+ next if LiteralNil.equal?(value)
33
+ # A node shares its parent's source unless it was built by hand or brought in from elsewhere
34
+ next if name == :full_source && !n.equal?(node) && value == parents[n].full_source
35
+
36
+ hash[name.name] =
37
+ if token?(n.class, name) then dump_token(value)
38
+ elsif Node === value then hashes[value]
39
+ elsif Array === value then value.map { |v| (Node === v) ? hashes[v] : dump_value(v) }
40
+ else
41
+ dump_value(value)
42
+ end
43
+ end
44
+ end
45
+ hashes[node]
46
+ end
47
+
48
+ def load(hash)
49
+ return hash.map { |h| load(h) } if Array === hash
50
+
51
+ # Parents before children, so building them backwards has every child ready
52
+ order = []
53
+ stack = [[hash, nil]]
54
+ while (h, source = stack.pop)
55
+ source = load_value(h["full_source"]) if h.key?("full_source")
56
+ order << [h, source]
57
+ h.each_value do |v|
58
+ if Hash === v && v.key?("node") then stack << [v, source]
59
+ elsif Array === v then v.each { |e| stack << [e, source] if Hash === e && e.key?("node") }
60
+ end
61
+ end
62
+ end
63
+
64
+ built = {}.compare_by_identity
65
+ order.reverse_each { |h, source| built[h] = load_node(h, source, built) }
66
+ built[hash]
67
+ end
68
+
69
+ private
70
+
71
+ def token?(klass, name) = TOKEN_PROPS.include?(name) || klass.token_names.include?(name)
72
+
73
+ def dump_token(tuple)
74
+ return nil if tuple.nil?
75
+
76
+ type, beg, fin, kind = tuple
77
+ [(Array === type) ? type.map(&:name) : type.name, beg, fin, kind.name]
78
+ end
79
+
80
+ def load_token(array)
81
+ return nil if array.nil?
82
+
83
+ type, beg, fin, kind = array
84
+ [(Array === type) ? type.map(&:to_sym) : type.to_sym, beg, fin, kind.to_sym]
85
+ end
86
+
87
+ def dump_value(value)
88
+ case value
89
+ when Symbol then { "symbol" => value.name }
90
+ when String
91
+ if value.encoding == Encoding::BINARY || !value.valid_encoding?
92
+ { "bytes" => value.unpack1("H*"), "encoding" => value.encoding.name }
93
+ else
94
+ value
95
+ end
96
+ when Float then value.finite? ? value : { "float" => value.to_s }
97
+ when Integer, true, false, nil then value
98
+ when Array then value.map { |v| dump_value(v) }
99
+ else
100
+ name = SENTINELS.key(value) or raise ArgumentError, "cannot dump #{value.inspect}"
101
+ { "sentinel" => name }
102
+ end
103
+ end
104
+
105
+ def load_value(value)
106
+ case value
107
+ when Array then value.map { |v| load_value(v) }
108
+ when Hash
109
+ if value.key?("symbol") then value["symbol"].to_sym
110
+ elsif value.key?("bytes") then [value["bytes"]].pack("H*").force_encoding(value["encoding"])
111
+ elsif value.key?("float") then FLOATS.fetch(value["float"])
112
+ elsif value.key?("sentinel") then SENTINELS.fetch(value["sentinel"])
113
+ else
114
+ raise ArgumentError, "cannot load #{value.inspect}"
115
+ end
116
+ else value
117
+ end
118
+ end
119
+
120
+ def load_node(hash, source, built)
121
+ name = hash.fetch("node")
122
+ klass = Plume.const_get(name, false) if name.match?(/\A[A-Z]\w*\z/) && Plume.const_defined?(name, false)
123
+ raise ArgumentError, "#{name.inspect} is not a node" unless Class === klass && klass <= Node
124
+
125
+ props = { full_source: source }
126
+ hash.each do |key, value|
127
+ next if key == "node" || key == "full_source"
128
+
129
+ name = key.to_sym
130
+ props[name] =
131
+ if token?(klass, name) then load_token(value)
132
+ elsif Hash === value && value.key?("node") then built.fetch(value)
133
+ elsif Array === value then value.map { |v| (Hash === v && v.key?("node")) ? built.fetch(v) : load_value(v) }
134
+ else
135
+ load_value(value)
136
+ end
137
+ end
138
+ node = klass.allocate
139
+ node.__send__(:initialize, **props)
140
+ node
141
+ end
142
+ end
143
+ end
@@ -0,0 +1,86 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "digest"
4
+
5
+ module Plume
6
+ # A short, stable hash of what statements mean, which stays the same across the values in them
7
+ # and how they are spelled: constants (as `Normalize` finds them) and parameters all count as one
8
+ # `?`, an IN list or VALUES of nothing but those counts as one item or row, and token positions,
9
+ # keyword and name case and redundant parentheses do not count at all. It is the same in every
10
+ # process and Ruby, unlike `Object#hash`
11
+ module Fingerprint
12
+ extend self
13
+
14
+ SKIPPED = %i[full_source leading trailing].freeze
15
+ Field = Data.define(:name)
16
+
17
+ def of(statements)
18
+ Digest::SHA256.hexdigest(stream(Array(statements)).join)[0, 16]
19
+ end
20
+
21
+ private
22
+
23
+ # The tree walked in order without recursing, each item length-prefixed so none runs into the next
24
+ def stream(statements)
25
+ constants = Set.new.compare_by_identity
26
+ statements.each { |s| constants.merge(Normalize.constants(s)) }
27
+ fields = Hash.new { |h, klass| h[klass] = (klass.literal_properties.map(&:name) - klass.token_names.to_a - SKIPPED) | klass.all_inspectable_props.to_a }
28
+
29
+ out = []
30
+ emit = -> (tag, text) { out << "#{tag}#{text.bytesize}:#{text}" }
31
+ pending = statements.reverse.map { |s| [s, nil] }
32
+ while (item, field = pending.pop)
33
+ case item
34
+ when Variable then emit.("?", "")
35
+ when Node
36
+ next emit.("?", "") if constants.include?(item)
37
+
38
+ if ParentheticalExpression === item && Node === item.contents
39
+ pending << [item.contents, field]
40
+ next
41
+ end
42
+ emit.("n", item.class.name)
43
+ fields[item.class].reverse_each do |name|
44
+ value = node_field(item, name)
45
+ value = collapse(value, item, name, constants)
46
+ pending << [value, name] << [Field.new(name), nil]
47
+ end
48
+ when Array
49
+ emit.("l", item.size.to_s)
50
+ item.reverse_each { |element| pending << [element, field] }
51
+ when Field then emit.("f", item.name.name)
52
+ when String
53
+ # Names are matched without regard to ASCII case, the values a statement keeps (a DEFAULT) are not
54
+ emit.("s", (field == :value) ? item.b : item.b.downcase(:ascii))
55
+ when Symbol then emit.("y", item.name.upcase)
56
+ when LiteralNil, nil then emit.("0", "")
57
+ else emit.("v", item.inspect)
58
+ end
59
+ end
60
+ out
61
+ end
62
+
63
+ def node_field(node, name)
64
+ if ParentheticalExpression === node && name == :value
65
+ node.contents
66
+ else
67
+ node.respond_to?(name) ? node.public_send(name) : node.instance_variable_get(:"@#{name}")
68
+ end
69
+ end
70
+
71
+ # Only the shape of a list of values counts, not how many values it has
72
+ def collapse(value, node, name, constants)
73
+ return value unless Array === value && value.size > 1
74
+
75
+ if ParentheticalExpression === node && name == :value && value.all? { |v| placeholder?(v, constants) }
76
+ value.first(1)
77
+ elsif ValuesClause === node && name == :rows && value.all? { |row| row.values.all? { |v| placeholder?(v, constants) } }
78
+ value.uniq { |row| row.values.size }
79
+ else
80
+ value
81
+ end
82
+ end
83
+
84
+ def placeholder?(node, constants) = Variable === node || constants.include?(node)
85
+ end
86
+ end