plume 0.2.0 → 0.3.0

data/lib/plume/pretty_printer.rb

@@ -0,0 +1,202 @@
+# frozen_string_literal: true
+
+module Plume
+	# The printer laid out over lines: each clause starts a line, a clause too long for
+	# `max_line_length` puts its items one per line, indented by `indent_size`, and a long AND or OR
+	# chain in a condition puts each operand on a line of its own. Subqueries are indented inside
+	# their parentheses. Expressions otherwise print as `Printer` prints them, so what comes out
+	# parses into the same tree
+	class PrettyPrinter < Printer
+		def self.print(node, indent_size: 4, max_line_length: 80, commas_start_of_line: false)
+			new(indent_size:, max_line_length:, commas_start_of_line:).print(node)
+		rescue *STACK_EXHAUSTED
+			raise Error, "statement nested too deeply to print"
+		end
+
+		def initialize(indent_size:, max_line_length:, commas_start_of_line:)
+			super()
+			@pad = " " * indent_size
+			@max = max_line_length
+			@leading_commas = commas_start_of_line
+		end
+
+		private
+
+		def lines(*parts) = parts.flatten.reject { |p| !p || p.empty? }.join("\n")
+
+		# Only the line breaks between tokens: one inside a string or quoted name is part of its value
+		def indent(text)
+			src = text.b
+			breaks = []
+			lexer = Lexer.new(text)
+			while (token = lexer.next_token)
+				next unless token == :SPACE
+
+				((lexer.anchor)...(lexer.cursor)).each { |i| breaks << i if src.getbyte(i) == 10 && i + 1 < src.bytesize && src.getbyte(i + 1) != 10 }
+			end
+
+			out = (src.empty? ? +"" : @pad.b.dup)
+			at = 0
+			breaks.each do |i|
+				out << src.byteslice(at, i + 1 - at) << @pad
+				at = i + 1
+			end
+			(out << src.byteslice(at, src.bytesize - at)).force_encoding(text.encoding)
+		end
+
+		# Lines after the first of a multi-line operand line up under the clause it sits in
+		def hanging(text) = text.include?("\n") ? indent(text).delete_prefix(@pad) : text
+
+		def fits?(text) = !text.include?("\n") && text.length <= @max
+
+		# `KEYWORD a, b, c` on one line if it fits, otherwise the keyword alone and an item per line
+		def clause(keyword, items)
+			items = Array(items).map { |item| (String === item) ? item : print(item) }
+			one_line = "#{keyword} #{items.join(", ")}"
+			return one_line if fits?(one_line)
+
+			body =
+				if @leading_commas
+					items.each_with_index.map { |item, i| i.zero? ? "  #{item}" : ", #{item}" }.join("\n")
+				else
+					items.join(",\n")
+				end
+			"#{keyword}\n#{indent(body)}"
+		end
+
+		# A condition on one line if it fits, otherwise its top-level AND or OR chain an operand per line
+		def condition(keyword, node)
+			one_line = "#{keyword} #{print(node)}"
+			return one_line if fits?(one_line) || !(BinaryExpression === node && (node.operator in :ALL | :ANY))
+
+			chain = [node]
+			chain << chain.last.left while BinaryExpression === chain.last.left && chain.last.left.operator == node.operator
+			word = BINARY_OPERATORS.fetch(node.operator)
+			rest = chain.reverse_each.map { |link| "#{word} #{operand(link.right)}" }
+			lines("#{keyword} #{hanging(operand(chain.last.left))}", rest.map { |line| indent(line) })
+		end
+
+		def parens(inner) = inner.include?("\n") ? "(\n#{indent(inner)}\n)" : "(#{inner})"
+
+		def pretty_order_and_limit(node)
+			limit =
+				if node.offset && node.offset_tk_tok == :COMMA
+					"LIMIT #{print(node.offset)}, #{print(node.limit)}"
+				else
+					join(node.limit && "LIMIT #{print(node.limit)}", node.offset && "OFFSET #{print(node.offset)}")
+				end
+			[node.orderings && clause("ORDER BY", node.orderings), limit]
+		end
+
+		# -- statements
+
+		def print_select_statement(node) = lines(print(node.with_clause), print(node.body), pretty_order_and_limit(node))
+
+		def print_with_clause(node)
+			ctes = node.ctes.map { |cte| print(cte) }
+			head = join("WITH", node.recursive && "RECURSIVE")
+			"#{head} #{ctes.join(",\n")}"
+		end
+
+		def print_common_table_expression(node)
+			materialized = { true => "MATERIALIZED", false => "NOT MATERIALIZED" }[node.materialized]
+			name = identifier(node.name) + (node.columns ? "(#{list(node.columns)})" : "")
+			join(name, "AS", materialized, parens(print(node.select)))
+		end
+
+		def print_compound_select(node) = lines(print(node.left), COMPOUND_OPERATORS.fetch(node.operator), print(node.right))
+
+		def print_select_core(node)
+			lines(
+				clause(node.distinct ? "SELECT DISTINCT" : "SELECT", node.columns),
+				node.source && from(node.source),
+				node.condition && condition("WHERE", node.condition),
+				node.groupings && clause("GROUP BY", node.groupings),
+				node.having && condition("HAVING", node.having),
+				node.windows && clause("WINDOW", node.windows),
+			)
+		end
+
+		def from(source) = (JoinClause === source) ? "FROM #{print(source)}" : clause("FROM", [source])
+
+		def print_values_clause(node) = clause("VALUES", node.rows)
+
+		def print_join_clause(node)
+			node.joins.reduce(print(node.left)) do |sql, each|
+				operator = print(each.operator)
+				right = join(print(each.source), print(each.constraint))
+				(operator == ",") ? "#{sql},\n#{right}" : "#{sql}\n#{operator} #{right}"
+			end
+		end
+
+		def print_subquery_source(node) = join(parens(print(node.select)), alias_of(node))
+		def print_subquery_expression(node) = parens(print(node.select))
+		def print_exists_expression(node) = "EXISTS #{parens(print(node.select))}"
+
+		def print_insert_statement(node)
+			verb = node.or_action ? ((node.insert_kw_tok == :REPLACE) ? "REPLACE" : "INSERT OR #{node.or_action}") : "INSERT"
+			lines(
+				print(node.with_clause),
+				join(verb, "INTO", print(node.table), node.columns && "(#{list(node.columns)})"),
+				node.default_values ? "DEFAULT VALUES" : print(node.select),
+				node.upserts&.map { |u| print(u) },
+				print(node.returning),
+			)
+		end
+
+		def print_upsert_clause(node)
+			target = node.targets && "(#{list(node.targets)})"
+			head = join("ON CONFLICT", target, node.target_condition && "WHERE #{print(node.target_condition)}")
+			return join(head, "DO NOTHING") if node.action == :NOTHING
+
+			lines(head, indent(lines(clause("DO UPDATE SET", node.assignments), node.condition && condition("WHERE", node.condition))))
+		end
+
+		def print_update_statement(node)
+			lines(
+				print(node.with_clause),
+				join(node.or_action ? "UPDATE OR #{node.or_action}" : "UPDATE", print(node.table)),
+				clause("SET", node.assignments),
+				node.source && from(node.source),
+				node.condition && condition("WHERE", node.condition),
+				print(node.returning), pretty_order_and_limit(node),
+			)
+		end
+
+		def print_delete_statement(node)
+			lines(
+				print(node.with_clause), "DELETE FROM #{print(node.table)}",
+				node.condition && condition("WHERE", node.condition),
+				print(node.returning), pretty_order_and_limit(node),
+			)
+		end
+
+		def print_returning_clause(node) = clause("RETURNING", node.columns)
+
+		def print_create_table_statement(node)
+			head = join("CREATE", node.temporary && "TEMP", "TABLE", node.if_not_exists && "IF NOT EXISTS", print(node.table))
+			return lines("#{head} AS", print(node.select_statement)) if node.select_statement
+
+			definitions = [*node.columns, *node.constraints].map { |d| print(d) }
+			join("#{head} (\n#{indent(definitions.join(",\n"))}\n)", node.options && list(node.options))
+		end
+
+		def print_create_view_statement(node)
+			head = join(
+				"CREATE", node.temporary && "TEMP", "VIEW", node.if_not_exists && "IF NOT EXISTS", print(node.view),
+				node.columns && "(#{list(node.columns)})", "AS",
+			)
+			lines(head, print(node.select))
+		end
+
+		def print_create_trigger_statement(node)
+			event = node.columns ? "UPDATE OF #{list(node.columns)}" : node.event.to_s
+			head = join(
+				"CREATE", node.temporary && "TEMP", "TRIGGER", node.if_not_exists && "IF NOT EXISTS", print(node.trigger),
+				node.time&.to_s&.tr("_", " "), event, "ON", print(node.table),
+				node.for_each_row_kw? && "FOR EACH ROW", node.condition && "WHEN #{print(node.condition)}",
+			)
+			lines(head, "BEGIN", node.statements.map { |s| indent("#{print(s)};") }, "END")
+		end
+	end
+end

data/lib/plume/printer.rb

@@ -4,8 +4,9 @@ module Plume
 	# Turns an AST back into SQL. It works from the nodes' readers rather than from the tokens
 	# they were parsed from, so trees built by hand, or changed after parsing, print too. The
 	# output is canonical rather than a copy of the input: keywords come out in upper case,
-	# identifiers are quoted only when they have to be, and every compound operand gets
-	# parentheses so precedence never has to be second-guessed.
+	# identifiers are quoted only when they have to be, and a compound operand gets parentheses
+	# unless it is the left operand of an operator at its own precedence, where SQLite's
+	# left-associativity already groups it.
 	class Printer
 		BARE_IDENTIFIER = /\A[A-Za-z_][A-Za-z0-9_]*\z/
 
@@ -18,6 +19,21 @@ module Plume
 			NOT_REGEXP: "NOT REGEXP", MATCH: "MATCH", NOT_MATCH: "NOT MATCH", IN: "IN", NOT_IN: "NOT IN",
 		}.freeze
 
+		# Names that would read back as the start of an expression
+		EXPRESSION_KEYWORDS = (Parser::LITERAL_KEYWORDS + [:CAST, :RAISE]).freeze
+
+		# Levels Plume's parser and SQLite agree on: SQLite also puts = and IS with IN, LIKE and the rest
+		PRECEDENCE = {
+			ANY: 1, ALL: 2,
+			IN: 3, NOT_IN: 3, MATCH: 3, NOT_MATCH: 3, GLOB: 3, NOT_GLOB: 3, REGEXP: 3, NOT_REGEXP: 3,
+			EQUALS: 4, NOT_EQUALS: 4, IS: 4, IS_NOT: 4,
+			BELOW: 5, ABOVE: 5, ATMOST: 5, ATLEAST: 5,
+			BIT_AND: 6, BIT_OR: 6, BIT_LSHIFT: 6, BIT_RSHIFT: 6,
+			ADD: 7, SUB: 7,
+			MULTIPLY: 8, DIVIDE: 8, MODULO: 8,
+			CONCAT: 9, EXTRACT: 9, RETRIEVE: 9,
+		}.freeze
+
 		COMPOUND_OPERATORS = { UNION: "UNION", UNION_ALL: "UNION ALL", INTERSECT: "INTERSECT", EXCEPT: "EXCEPT" }.freeze
 		JOIN_KINDS = { COMMA: ",", INNER: "JOIN", LEFT: "LEFT JOIN", RIGHT: "RIGHT JOIN", FULL: "FULL JOIN", CROSS: "CROSS JOIN" }.freeze
 		FOREIGN_KEY_ACTIONS = { CASCADE: "CASCADE", RESTRICT: "RESTRICT", SET_NULL: "SET NULL", SET_DEFAULT: "SET DEFAULT", NO_ACTION: "NO ACTION" }.freeze
@@ -25,7 +41,14 @@ module Plume
 		# Operands of these need parentheses to keep their grouping when printed inside another expression
 		COMPOUND_EXPRESSIONS = [BinaryExpression, UnaryExpression, TernaryExpression, LikeExpression, CollationExpression].freeze
 
-		def self.print(node) = new.print(node)
+		Error = Class.new(StandardError)
+
+		def self.print(node)
+			new.print(node)
+		rescue *STACK_EXHAUSTED
+			# As with parsing, how deep is too deep depends on the stack the caller has left
+			raise Error, "statement nested too deeply to print"
+		end
 
 		def print(node)
 			case node
@@ -40,18 +63,22 @@ module Plume
 
 		def identifier(name)
 			name = name.to_s
-			return name if name.match?(BARE_IDENTIFIER) && bare_keyword_ok?(name.upcase.to_sym)
+			return name if name.ascii_only? && name.match?(BARE_IDENTIFIER) && bare_keyword_ok?(name.upcase.to_sym)
 
-			%("#{name.gsub('"', '""')}")
+			%("#{doubled(name, '"')}")
 		end
 
-		# Keywords SQLite falls back to treating as identifiers, like ROWID or KEY, need no quotes
-		def bare_keyword_ok?(word) = !Lexer::KEYWORDS.include?(word) || Parser::TOKEN_FALLBACKS.include?(word)
+		# Keywords SQLite falls back to treating as identifiers, like ROWID or KEY, need no quotes -
+		# except the ones that would read back as the start of an expression
+		def bare_keyword_ok?(word) = !Lexer::KEYWORDS.include?(word) || (Parser::TOKEN_FALLBACKS.include?(word) && !EXPRESSION_KEYWORDS.include?(word))
 
-		def string(value) = "'#{value.to_s.gsub("'", "''")}'"
+		def string(value) = "'#{doubled(value.to_s, "'")}'"
 
 		private
 
+		# Done on the bytes, since TruffleRuby's gsub refuses a string that is not valid UTF-8
+		def doubled(text, quote) = text.b.gsub(quote, quote * 2).force_encoding(text.encoding)
+
 		def underscore(name) = name.gsub(/([a-z\d])([A-Z])/, '\1_\2').downcase
 
 		def join(*parts) = parts.flatten.reject { |p| !p || (p.respond_to?(:empty?) && p.empty?) }.join(" ")
@@ -77,8 +104,12 @@ module Plume
 		def order_and_limit(node)
 			[
 				node.orderings && "ORDER BY #{list(node.orderings)}",
-				node.limit && "LIMIT #{print(node.limit)}",
-				node.offset && "OFFSET #{print(node.offset)}",
+				# `LIMIT offset, limit` keeps its order: swapping the two would renumber their `?` parameters
+				if node.offset && node.offset_tk_tok == :COMMA
+					"LIMIT #{print(node.offset)}, #{print(node.limit)}"
+				else
+					[node.limit && "LIMIT #{print(node.limit)}", node.offset && "OFFSET #{print(node.offset)}"]
+				end,
 			]
 		end
 
@@ -213,6 +244,10 @@ module Plume
 				when :RENAME_COLUMN then "RENAME COLUMN #{identifier(node.column)} TO #{identifier(node.new_name)}"
 				when :ADD_COLUMN then "ADD COLUMN #{print(node.column_definition)}"
 				when :DROP_COLUMN then "DROP COLUMN #{identifier(node.column)}"
+				when :ADD_CONSTRAINT then "ADD #{print(node.constraint)}"
+				when :DROP_CONSTRAINT then "DROP CONSTRAINT #{identifier(node.constraint_name)}"
+				when :SET_NOT_NULL then join("ALTER COLUMN #{identifier(node.column)} SET NOT NULL", print(node.conflict_clause))
+				when :DROP_NOT_NULL then "ALTER COLUMN #{identifier(node.column)} DROP NOT NULL"
 				end
 			join("ALTER TABLE", print(node.table), action)
 		end
@@ -298,8 +333,18 @@ module Plume
 
 		def print_variable(node) = node.name
 
+		# Every binary operator associates to the left, so a left operand at the same precedence needs no
+		# parentheses: `a - b + c`, `a OR b OR c`. A long chain then prints flat rather than nesting a pair
+		# per term, and the left spine is walked in a loop, so its length costs no stack
 		def print_binary_expression(node)
-			"#{operand(node.left)} #{BINARY_OPERATORS.fetch(node.operator)} #{operand(node.right)}"
+			chain = [node]
+			while BinaryExpression === (left = chain.last.left) && PRECEDENCE[left.operator] == PRECEDENCE[node.operator]
+				chain << left
+			end
+
+			chain.reverse_each.reduce(+operand(chain.last.left)) do |sql, link|
+				sql << " " << BINARY_OPERATORS.fetch(link.operator) << " " << operand(link.right)
+			end
 		end
 
 		def print_unary_expression(node)
@@ -334,7 +379,7 @@ module Plume
 
 		def print_parenthetical_expression(node) = "(#{print(node.contents)})"
 
-		def print_cast_expression(node) = "CAST(#{print(node.expression)} AS #{print(node.as)})"
+		def print_cast_expression(node) = "CAST(#{join(print(node.expression), "AS", print(node.as))})"
 
 		def print_case_expression(node)
 			join("CASE", node.predicate && print(node.predicate), list_of(node.conditions), node.else_clause && "ELSE #{print(node.else_clause)}", "END")
@@ -354,7 +399,8 @@ module Plume
 		end
 
 		def print_function_arguments(node)
-			join(node.distinct && "DISTINCT", list(node.expressions), node.order_by && "ORDER BY #{list(node.order_by)}")
+			quantifier = node.distinct ? "DISTINCT" : (node.distinct_kw_tok == :ALL && "ALL")
+			join(quantifier, list(node.expressions), node.order_by && "ORDER BY #{list(node.order_by)}")
 		end
 
 		def print_star_function_argument(_node) = "*"
@@ -442,19 +488,27 @@ module Plume
 		end
 
 		def print_unique_table_constraint(node) = join(constraint_name(node), "UNIQUE (#{list(node.columns)})", print(node.conflict_clause))
-		def print_check_table_constraint(node) = join(constraint_name(node), "CHECK (#{print(node.expression)})")
+		def print_check_table_constraint(node) = join(constraint_name(node), "CHECK (#{print(node.expression)})", print(node.conflict_clause))
 		def print_no_op_table_constraint(node) = constraint_name(node)
 
+		def print_deferrable_column_constraint(node) = join(constraint_name(node), deferrable(node) || "DEFERRABLE")
+
+		# The words as written, or else what `deferred` says
+		def deferrable(node)
+			return Array(node.deferrable_span_tok).join(" ") if node.deferrable_span?
+
+			"DEFERRABLE INITIALLY DEFERRED" if node.deferred
+		end
+
 		def print_foreign_key_table_constraint(node)
 			join(constraint_name(node), "FOREIGN KEY (#{list(node.columns)})", print(node.foreign_key_clause))
 		end
 
 		def print_foreign_key_clause(node)
-			deferrable = node.deferrable_span? ? node.deferrable_span_tok : (node.deferred && "DEFERRABLE INITIALLY DEFERRED")
 			join(
 				"REFERENCES", identifier(node.table), node.columns && "(#{list(node.columns)})",
 				list_of(Array(node.actions)), list_of(Array(node.match_clauses)),
-				deferrable && Array(deferrable).join(" "),
+				deferrable(node),
 			)
 		end
 
@@ -462,7 +516,7 @@ module Plume
 		def print_match_clause(node) = "MATCH #{node.type}"
 
 		def print_indexed_column(node)
-			join(identifier(node.name), node.collation && "COLLATE #{identifier(node.collation)}", node.direction&.to_s)
+			join(identifier(node.name), node.collation && "COLLATE #{identifier(node.collation)}", node.direction&.to_s, node.nulls && "NULLS #{node.nulls.to_s.upcase}")
 		end
 
 		def print_strict_table_option(_node) = "STRICT"

data/lib/plume/truncate.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Plume
+	# SQL shortened to fit a length, for logs and error trackers, by putting `...` where the parts
+	# least needed to recognise the query were: result columns, WHERE conditions, VALUES rows, CTE
+	# bodies, INSERT column lists and UPDATE assignments - the most deeply nested first, the longest
+	# first among those. Everything else stays as written. When that is not enough, the end is cut off
+	module Truncate
+		extend self
+
+		ELLIPSIS = "..."
+
+		def call(sql, max_length)
+			return sql if sql.length <= max_length
+
+			src = sql.b
+			chosen = []
+			result = sql
+			candidates(Parser.new(sql).parse).each do |beg, fin, _depth|
+				next if fin - beg <= ELLIPSIS.bytesize || chosen.any? { |b, f| b <= beg && fin <= f }
+
+				chosen.reject! { |b, f| beg <= b && f <= fin }
+				chosen << [beg, fin]
+				result = render(src, chosen, sql.encoding)
+				return result if result.length <= max_length
+			end
+			"#{result[0, [max_length - ELLIPSIS.length, 0].max]}#{ELLIPSIS}"[0, max_length]
+		end
+
+		# Byte ranges that may be cut, with how deeply they are nested, in the order to cut them
+		def candidates(statements)
+			found = []
+			stack = statements.reverse.map { |s| [s, 0] }
+			while (node, depth = stack.pop)
+				span =
+					case node
+					when SelectCore then [list_span(node.columns), node.condition&.location]
+					when ValuesClause then [list_span(node.rows)]
+					when CommonTableExpression then [node.select&.location]
+					when InsertStatement then [list_span(node.columns)]
+					when UpdateStatement then [list_span(node.assignments)]
+					else []
+					end
+				span.compact.each { |beg, fin| found << [beg, fin, depth] }
+				node.children.reverse_each { |child| stack << [child, depth + 1] }
+			end
+			found.sort_by { |beg, fin, depth| [-depth, beg - fin] }
+		end
+
+		private
+
+		def list_span(nodes)
+			first, last = nodes&.first&.location, nodes&.last&.location
+			[first[0], last[1]] if first && last
+		end
+
+		def render(src, spans, encoding)
+			out = +""
+			at = 0
+			spans.sort.each do |beg, fin|
+				out << src.byteslice(at, beg - at) << ELLIPSIS
+				at = fin
+			end
+			(out << src.byteslice(at, src.bytesize - at)).force_encoding(encoding)
+		end
+	end
+end

data/lib/plume/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Plume
-	VERSION = "0.2.0"
+	VERSION = "0.3.0"
 end

data/lib/plume.rb

@@ -3,10 +3,15 @@
 require "plume/version"
 require "literal"
 require "plume/union"
+require "plume/literal_patches"
 
 module Plume
 	extend Literal::Types
 	extend Union::Types
+	extend LiteralPatches::Types if LiteralPatches::DEFERRED_WRITES_ON_CHECK
+
+	# JRuby and TruffleRuby have no Ractors, so there is nothing to make shareable for
+	def self.make_shareable(obj) = defined?(Ractor) ? Ractor.make_shareable(obj) : obj
 
 	autoload :Lexer, "plume/lexer"
 	autoload :Parser, "plume/parser"
@@ -14,7 +19,15 @@ module Plume
 	autoload :SQLiteGrammar, "plume/sqlite_grammar"
 	autoload :Catalog, "plume/catalog"
 	autoload :Printer, "plume/printer"
+	autoload :PrettyPrinter, "plume/pretty_printer"
 	autoload :Analysis, "plume/analysis"
+	autoload :Parameters, "plume/parameters"
+	autoload :Dump, "plume/dump"
+	autoload :Truncate, "plume/truncate"
+	autoload :Comment, "plume/comment"
+	autoload :Comments, "plume/comment"
+	autoload :Normalize, "plume/normalize"
+	autoload :Fingerprint, "plume/fingerprint"
 
 	autoload :BinaryExpression, "plume/ast/binary_expression"
 	autoload :ParentheticalExpression, "plume/ast/parenthetical_expression"
@@ -33,6 +46,7 @@ module Plume
 	autoload :CreateTableStatement, "plume/ast/create_table_statement"
 	autoload :CreateViewStatement, "plume/ast/create_view_statement"
 	autoload :DefaultColumnConstraint, "plume/ast/default_column_constraint"
+	autoload :DeferrableColumnConstraint, "plume/ast/deferrable_column_constraint"
 	autoload :EmptyFunctionArgument, "plume/ast/empty_function_argument"
 	autoload :Expression, "plume/ast/expression"
 	autoload :ForeignKeyAction, "plume/ast/foreign_key_action"
@@ -118,17 +132,113 @@ module Plume
 	autoload :WindowDefinition, "plume/ast/window_definition"
 	autoload :WithClause, "plume/ast/with_clause"
 
-	SyntaxError = Class.new(StandardError)
+	# Where parsing gave up, for editors and linters that want more than the message:
+	# `location` is the byte range `[beg, fin]` of the offending token, as `Node#location` gives it,
+	# `token` its type (nil at the end of input), `line` counts from 1
+	class SyntaxError < StandardError
+		attr_reader :location, :token, :line, :expected
+
+		def initialize(message = nil, location: nil, token: nil, line: nil, expected: [])
+			@location, @token, @line, @expected = location, token, line, expected
+			super(message)
+		end
+
+		def position = location&.first
+	end
+
+	# What running out of stack raises. JRuby lets a Java StackOverflowError through
+	# when the stack runs out inside its own Java code rather than in Ruby
+	STACK_EXHAUSTED =
+		if defined?(JRUBY_VERSION)
+			require "java"
+			[SystemStackError, Java::JavaLang::StackOverflowError].freeze
+		else
+			[SystemStackError].freeze
+		end
 
-	def self.tokenize(str, with_values: false, skip_spaces: true)
+	# `with_positions` adds each token's byte range `beg, fin`, after the value if there is one
+	def self.tokenize(str, with_values: false, with_positions: false, skip_spaces: true)
 		lex = Lexer.new(str, skip_spaces: skip_spaces)
 		[].tap do |tokens|
 			while (token = lex.next_token)
-				tokens << (with_values ? [token, lex.value] : token)
+				tokens <<
+					if with_positions
+						with_values ? [token, lex.value, lex.anchor, lex.cursor] : [token, lex.anchor, lex.cursor]
+					else
+						with_values ? [token, lex.value] : token
+					end
+			end
+		end
+	end
+
+	# Loads everything that is otherwise autoloaded and freezes what the node classes know
+	# about their props, which is what lets Plume run inside Ractors. Call it from the main
+	# Ractor before starting any. Node classes cannot get new props afterwards
+	def self.eager_load!
+		# SQLiteGrammar is a development tool which pulls in net/http
+		(constants - [:SQLiteGrammar]).each { |name| const_get(name) if autoload?(name) }
+		Analysis::Resolver
+		# Node constructors and type errors reach for these, and Ruby 3.3 cannot autoload outside the main Ractor.
+		# Not every Literal version has all of them
+		%i[Null TypeError ArgumentError].each { |name| Literal.const_get(name) if Literal.const_defined?(name) }
+		# Literal versions that cache a `_Deferred` do so on first use, which a frozen one cannot do.
+		# Each node's schema then shares Expression with the rest of its prop types
+		Expression === nil
+		make_shareable(Token::Tuple)
+		constants.each do |name|
+			klass = const_get(name)
+			klass.make_shareable! if Class === klass && klass <= Node
+		end
+		true
+	end
+
+	# The statements in `sql`, found by their tokens alone, so SQL that does not parse splits too.
+	# Each is its text from its first token to its last, without the semicolon; `with_positions`
+	# gives `[text, beg, fin]` with the byte range. A trigger's body stays in one piece
+	def self.split(sql, with_positions: false)
+		lexer = Lexer.new(sql, skip_spaces: true)
+		statements = []
+		beg = fin = nil
+		leading = [] # the first tokens of the statement, enough to tell a CREATE TRIGGER
+		body = cases = 0
+
+		while (token = lexer.next_token)
+			if token == :SEMI && body.zero?
+				statements << [beg, fin] if beg
+				beg = nil
+				leading.clear
+				cases = 0
+				next
 			end
+
+			beg ||= lexer.anchor
+			fin = lexer.cursor
+			leading << token if leading.size < 3
+			trigger = leading[0] == :CREATE && (leading[1] == :TRIGGER || ((leading[1] in :TEMP | :TEMPORARY) && leading[2] == :TRIGGER))
+			next unless trigger
+
+			# Only END closes the body, and a CASE inside it has an END of its own
+			case token
+			when :BEGIN then body += 1 if body.zero?
+			when :CASE then cases += 1 if body.positive?
+			when :END
+				if cases.positive? then cases -= 1
+				elsif body.positive? then body -= 1
+				end
+			end
+		end
+		statements << [beg, fin] if beg
+
+		src = sql.b
+		statements.map do |b, f|
+			text = src.byteslice(b, f - b).force_encoding(sql.encoding)
+			with_positions ? [text, b, f] : text
 		end
 	end
 
+	# Every comment in `sql`, in order
+	def self.comments(sql) = Comments.in(sql)
+
 	def self.parse(sql)
 		Parser.new(sql).parse
 	end
@@ -143,6 +253,27 @@ module Plume
 		"  #{line}\n  #{caret}"
 	end
 
+	# The bind parameters of one statement, given as SQL or already parsed
+	def self.parameters(sql)
+		statements = (Node === sql) ? [sql] : Array(Parser.new(sql).parse)
+		raise ArgumentError, "expected exactly one statement, got #{statements.size}" unless statements.size == 1
+
+		Parameters.new(statements.first)
+	end
+
+	# A tree from what `Node#dump` gave, or a list of them
+	def self.load(dump) = Dump.load(dump)
+	# At most `max_length` characters of `sql`, with the least telling parts left out first,
+	# see Plume::Truncate. A parsed tree is printed first
+	def self.truncate(sql, max_length) = Truncate.call((Node === sql) ? sql.to_sql : sql, max_length)
+	# The SQL with its constants turned into `?`, see Plume::Normalize. A parsed tree gives a tree back
+	def self.normalize(sql) = (Node === sql) ? Normalize.tree(sql) : Normalize.text(sql)
+
+	# The same for the same query whatever its values and spelling, see Plume::Fingerprint
+	def self.fingerprint(sql) = Fingerprint.of((Node === sql || Array === sql) ? sql : Parser.new(sql).parse)
+
 	def self.reads(...) = Analysis.reads(...)
+	def self.writes(...) = Analysis.writes(...)
 	def self.access(...) = Analysis.access(...)
+	def self.summary(...) = Analysis.summary(...)
 end