plume 0.2.0 → 0.3.0

data/lib/plume/comment.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require "uri"
+
+module Plume
+	# A comment in SQL, `-- like this` or `/* like this */`, with the byte range it takes up
+	Comment = Data.define(:text, :location) do
+		def block? = text.start_with?("/*")
+
+		# What is between the markers
+		def body
+			block? ? text.delete_prefix("/*").delete_suffix("*/") : text.delete_prefix("--")
+		end
+
+		# The tags of a sqlcommenter comment - `/*controller='posts',action='index'*/`, which Rails'
+		# query_log_tags writes too - or of its older `key:value` format. Empty for any other comment
+		def tags
+			parts = body.strip.split(/,(?=[\w.-]+(?:=|:))/)
+			return {} if parts.empty?
+
+			if parts.all? { |part| part.match?(/\A[\w.%-]+='.*'\z/m) }
+				parts.to_h do |part|
+					key, value = part.split("=", 2)
+					[URI.decode_www_form_component(key), URI.decode_www_form_component(value[1..-2].gsub("\\'", "'"))]
+				end
+			elsif parts.all? { |part| part.match?(/\A[\w.-]+:[^,\s]+\z/) }
+				parts.to_h { |part| part.split(":", 2) }
+			else
+				{}
+			end
+		end
+	end
+
+	module Comments
+		extend self
+
+		def in(sql)
+			lexer = Lexer.new(sql)
+			found = []
+			while (token = lexer.next_token)
+				found << Comment.new(text: lexer.value, location: [lexer.anchor, lexer.cursor].freeze) if token == :COMMENT
+			end
+			found
+		end
+
+		# The comments right before `node` and right after it, with only spaces between them and it
+		def around(node)
+			source = node.full_source
+			loc = node.location
+			return [[], []] unless source && loc
+
+			tokens = Plume.tokenize(source, with_values: true, with_positions: true, skip_spaces: false)
+			first = tokens.index { |_, _, beg, _| beg >= loc[0] }
+			last = tokens.rindex { |_, _, _, fin| fin <= loc[1] }
+			[gather(tokens, first - 1, -1), gather(tokens, last + 1, 1)]
+		end
+
+		private
+
+		def gather(tokens, from, step)
+			found = []
+			i = from
+			while i >= 0 && (token = tokens[i])
+				type, text, beg, fin = token
+				break unless type == :SPACE || type == :COMMENT
+
+				found << Comment.new(text:, location: [beg, fin].freeze) if type == :COMMENT
+				i += step
+			end
+			(step.negative? ? found.reverse : found)
+		end
+	end
+end

data/lib/plume/dump.rb

@@ -0,0 +1,143 @@
+# frozen_string_literal: true
+
+module Plume
+	# Trees to and from plain hashes that JSON can carry, and back exactly: positions, keywords as
+	# written and the source they point into all come through, so `location`, `source_text` and
+	# `to_sql` answer as before. A node is its class under "node" and its props under their names,
+	# a token is `[type, beg, fin, kind]`, and the source is kept once, at the root.
+	# JSON has no symbols, binary strings or infinity, so those go out tagged: {"symbol" => "EQUALS"},
+	# {"bytes" => "9e2e", "encoding" => "BINARY"} and {"float" => "Infinity"}
+	module Dump
+		extend self
+
+		TOKEN_PROPS = %i[leading trailing].freeze
+		SENTINELS = { "LiteralFalse" => LiteralFalse }.freeze
+		FLOATS = { "Infinity" => Float::INFINITY, "-Infinity" => -Float::INFINITY, "NaN" => Float::NAN }.freeze
+
+		# Built without recursing, like `walk`, so the deepest tree SQLite takes can be dumped.
+		# Generating JSON that deep needs `max_nesting: false`
+		def dump(node)
+			return node.map { |n| dump(n) } if Array === node
+
+			hashes = {}.compare_by_identity
+			parents = {}.compare_by_identity
+			node.walk do |n, up|
+				hashes[n] = { "node" => n.class.name.delete_prefix("Plume::") }
+				parents[n] = up
+			end
+			hashes.each do |n, hash|
+				n.class.literal_properties.each do |property|
+					name = property.name
+					value = n.instance_variable_get(:"@#{name}")
+					next if LiteralNil.equal?(value)
+					# A node shares its parent's source unless it was built by hand or brought in from elsewhere
+					next if name == :full_source && !n.equal?(node) && value == parents[n].full_source
+
+					hash[name.name] =
+						if token?(n.class, name) then dump_token(value)
+						elsif Node === value then hashes[value]
+						elsif Array === value then value.map { |v| (Node === v) ? hashes[v] : dump_value(v) }
+						else
+							dump_value(value)
+						end
+				end
+			end
+			hashes[node]
+		end
+
+		def load(hash)
+			return hash.map { |h| load(h) } if Array === hash
+
+			# Parents before children, so building them backwards has every child ready
+			order = []
+			stack = [[hash, nil]]
+			while (h, source = stack.pop)
+				source = load_value(h["full_source"]) if h.key?("full_source")
+				order << [h, source]
+				h.each_value do |v|
+					if Hash === v && v.key?("node") then stack << [v, source]
+					elsif Array === v then v.each { |e| stack << [e, source] if Hash === e && e.key?("node") }
+					end
+				end
+			end
+
+			built = {}.compare_by_identity
+			order.reverse_each { |h, source| built[h] = load_node(h, source, built) }
+			built[hash]
+		end
+
+		private
+
+		def token?(klass, name) = TOKEN_PROPS.include?(name) || klass.token_names.include?(name)
+
+		def dump_token(tuple)
+			return nil if tuple.nil?
+
+			type, beg, fin, kind = tuple
+			[(Array === type) ? type.map(&:name) : type.name, beg, fin, kind.name]
+		end
+
+		def load_token(array)
+			return nil if array.nil?
+
+			type, beg, fin, kind = array
+			[(Array === type) ? type.map(&:to_sym) : type.to_sym, beg, fin, kind.to_sym]
+		end
+
+		def dump_value(value)
+			case value
+			when Symbol then { "symbol" => value.name }
+			when String
+				if value.encoding == Encoding::BINARY || !value.valid_encoding?
+					{ "bytes" => value.unpack1("H*"), "encoding" => value.encoding.name }
+				else
+					value
+				end
+			when Float then value.finite? ? value : { "float" => value.to_s }
+			when Integer, true, false, nil then value
+			when Array then value.map { |v| dump_value(v) }
+			else
+				name = SENTINELS.key(value) or raise ArgumentError, "cannot dump #{value.inspect}"
+				{ "sentinel" => name }
+			end
+		end
+
+		def load_value(value)
+			case value
+			when Array then value.map { |v| load_value(v) }
+			when Hash
+				if value.key?("symbol") then value["symbol"].to_sym
+				elsif value.key?("bytes") then [value["bytes"]].pack("H*").force_encoding(value["encoding"])
+				elsif value.key?("float") then FLOATS.fetch(value["float"])
+				elsif value.key?("sentinel") then SENTINELS.fetch(value["sentinel"])
+				else
+					raise ArgumentError, "cannot load #{value.inspect}"
+				end
+			else value
+			end
+		end
+
+		def load_node(hash, source, built)
+			name = hash.fetch("node")
+			klass = Plume.const_get(name, false) if name.match?(/\A[A-Z]\w*\z/) && Plume.const_defined?(name, false)
+			raise ArgumentError, "#{name.inspect} is not a node" unless Class === klass && klass <= Node
+
+			props = { full_source: source }
+			hash.each do |key, value|
+				next if key == "node" || key == "full_source"
+
+				name = key.to_sym
+				props[name] =
+					if token?(klass, name) then load_token(value)
+					elsif Hash === value && value.key?("node") then built.fetch(value)
+					elsif Array === value then value.map { |v| (Hash === v && v.key?("node")) ? built.fetch(v) : load_value(v) }
+					else
+						load_value(value)
+					end
+			end
+			node = klass.allocate
+			node.__send__(:initialize, **props)
+			node
+		end
+	end
+end

data/lib/plume/fingerprint.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require "digest"
+
+module Plume
+	# A short, stable hash of what statements mean, which stays the same across the values in them
+	# and how they are spelled: constants (as `Normalize` finds them) and parameters all count as one
+	# `?`, an IN list or VALUES of nothing but those counts as one item or row, and token positions,
+	# keyword and name case and redundant parentheses do not count at all. It is the same in every
+	# process and Ruby, unlike `Object#hash`
+	module Fingerprint
+		extend self
+
+		SKIPPED = %i[full_source leading trailing].freeze
+		Field = Data.define(:name)
+
+		def of(statements)
+			Digest::SHA256.hexdigest(stream(Array(statements)).join)[0, 16]
+		end
+
+		private
+
+		# The tree walked in order without recursing, each item length-prefixed so none runs into the next
+		def stream(statements)
+			constants = Set.new.compare_by_identity
+			statements.each { |s| constants.merge(Normalize.constants(s)) }
+			fields = Hash.new { |h, klass| h[klass] = (klass.literal_properties.map(&:name) - klass.token_names.to_a - SKIPPED) | klass.all_inspectable_props.to_a }
+
+			out = []
+			emit = -> (tag, text) { out << "#{tag}#{text.bytesize}:#{text}" }
+			pending = statements.reverse.map { |s| [s, nil] }
+			while (item, field = pending.pop)
+				case item
+				when Variable then emit.("?", "")
+				when Node
+					next emit.("?", "") if constants.include?(item)
+
+					if ParentheticalExpression === item && Node === item.contents
+						pending << [item.contents, field]
+						next
+					end
+					emit.("n", item.class.name)
+					fields[item.class].reverse_each do |name|
+						value = node_field(item, name)
+						value = collapse(value, item, name, constants)
+						pending << [value, name] << [Field.new(name), nil]
+					end
+				when Array
+					emit.("l", item.size.to_s)
+					item.reverse_each { |element| pending << [element, field] }
+				when Field then emit.("f", item.name.name)
+				when String
+					# Names are matched without regard to ASCII case, the values a statement keeps (a DEFAULT) are not
+					emit.("s", (field == :value) ? item.b : item.b.downcase(:ascii))
+				when Symbol then emit.("y", item.name.upcase)
+				when LiteralNil, nil then emit.("0", "")
+				else emit.("v", item.inspect)
+				end
+			end
+			out
+		end
+
+		def node_field(node, name)
+			if ParentheticalExpression === node && name == :value
+				node.contents
+			else
+				node.respond_to?(name) ? node.public_send(name) : node.instance_variable_get(:"@#{name}")
+			end
+		end
+
+		# Only the shape of a list of values counts, not how many values it has
+		def collapse(value, node, name, constants)
+			return value unless Array === value && value.size > 1
+
+			if ParentheticalExpression === node && name == :value && value.all? { |v| placeholder?(v, constants) }
+				value.first(1)
+			elsif ValuesClause === node && name == :rows && value.all? { |row| row.values.all? { |v| placeholder?(v, constants) } }
+				value.uniq { |row| row.values.size }
+			else
+				value
+			end
+		end
+
+		def placeholder?(node, constants) = Variable === node || constants.include?(node)
+	end
+end

data/lib/plume/lexer.rb

@@ -201,7 +201,7 @@ module Plume
 		# Maps ASCII characters to their respective character types
 		# Each element's index maps to the byte value of that character
 		CHARACTER_TYPES = %i[
-			NULL			ILLEGAL		ILLEGAL		ILLEGAL		ILLEGAL		ILLEGAL		ILLEGAL		ILLEGAL
+			NUL				ILLEGAL		ILLEGAL		ILLEGAL		ILLEGAL		ILLEGAL		ILLEGAL		ILLEGAL
 			ILLEGAL		SPACE			SPACE			ILLEGAL		SPACE			SPACE			ILLEGAL		ILLEGAL
 			ILLEGAL		ILLEGAL		ILLEGAL		ILLEGAL		ILLEGAL		ILLEGAL		ILLEGAL		ILLEGAL
 			ILLEGAL		ILLEGAL		ILLEGAL		ILLEGAL		ILLEGAL		ILLEGAL		ILLEGAL		ILLEGAL
@@ -297,7 +297,7 @@ module Plume
 			it[44]	=	:COMMA		# ,
 			it[59]	=	:SEMI			# ;
 			it[126]	=	:BITNOT		# ~
-		end
+		end.freeze
 
 		attr_reader :sql
 		attr_accessor :cursor, :anchor
@@ -316,7 +316,7 @@ module Plume
 		def next_token
 			@anchor = @cursor
 			token = lex_next_token
-			return next_token if token == :SPACE && @skip_spaces
+			return next_token if (token == :SPACE || token == :COMMENT) && @skip_spaces
 			token
 		end
 
@@ -352,7 +352,7 @@ module Plume
 					step
 					step until (b = peek).nil? || b == 10 # "\n"
 
-					:SPACE # IMP: R-22934-2513
+					:COMMENT # which SQLite takes for a space [IMP: R-22934-2513]
 				elsif peek == 62 # ">"
 					# differentiate between different pointers
 					if peek(1) == 62 # ">"
@@ -373,7 +373,7 @@ module Plume
 				step until (b = peek).nil? || (b == 42 && peek(1) == 47) # consume until EOF or "*/"
 				step(2) if peek == 42 && peek(1) == 47 # consume the "*/"
 
-				:SPACE # comments are encoded as SPACE tokens [IMP: R-22934-2513]
+				:COMMENT # which SQLite takes for a space [IMP: R-22934-2513]
 			when :EQ
 				step if peek == 61 # "="
 				:EQ
@@ -460,7 +460,8 @@ module Plume
 					end
 				end
 
-				if peek == 46 # "."
+				# A number that began with its "." has had its fraction already, ".5.5" is two tokens
+				if character_type != :DOT && peek == 46 # "."
 					step
 					token_type = :FLOAT if token_type == :INTEGER
 
@@ -604,7 +605,7 @@ module Plume
 
 				:ID
 			when :BOM
-				return step(2) && :SPACE if peek == 0xFE && peek(1) == 0xFF
+				return step(2) && :SPACE if peek == 0xBB && peek(1) == 0xBF
 
 				step while id?(peek)
 				:ID

data/lib/plume/literal_patches.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module Plume
+	# Stand-ins for what yippee-fun/literal#391 fixes in Literal itself, so that Plume can run
+	# in Ractors before that is released. They apply to Plume's own classes only, and each one
+	# stays off when the installed Literal does not need it. Once Plume requires a Literal
+	# with that fix, delete this file, its require and extend in plume.rb and its prepend in Node
+	module LiteralPatches
+		# A Mutex can never be shared between Ractors, and Literal keeps one in every schema
+		SCHEMA_HOLDS_MUTEX = Literal::Properties::Schema.new.instance_variable_defined?(:@mutex)
+
+		# Literal 1.9.0 calls the block of a `_Deferred` on every check. Unreleased Literal caches
+		# the result instead, but writes to the type on every check while doing so - and a frozen
+		# one then raises FrozenError, materialized or not
+		DEFERRED_WRITES_ON_CHECK = begin
+			probe = Literal::Types::DeferredType.new { Integer }
+			probe === 1
+			probe.freeze === 1
+			false
+		rescue FrozenError
+			true
+		end
+
+		# A frozen schema with the same readers, which `make_shareable!` swaps in once the props
+		# of a node class are all declared
+		class ShareableSchema
+			include Enumerable
+
+			attr_reader :properties_index
+
+			def initialize(schema)
+				@properties_index = schema.properties_index.dup
+				@sorted_properties = schema.to_a
+			end
+
+			def [](key) = @properties_index[key]
+			def each(&) = @sorted_properties.each(&)
+			def size = @sorted_properties.size
+			def empty? = @sorted_properties.empty?
+
+			# Literal starts the schema of a subclass from a dup of its parent's
+			def dup = Literal::Properties::Schema.new(properties_index: @properties_index.dup, sorted_properties: @sorted_properties.dup)
+		end
+
+		module NodeSchema
+			def literal_properties = @shareable_literal_properties || super
+
+			def make_shareable!
+				@shareable_literal_properties = Plume.make_shareable(ShareableSchema.new(literal_properties))
+				super
+			end
+		end
+
+		# Only ever used when DEFERRED_WRITES_ON_CHECK, so this can rely on the caching version's ivars
+		class Deferred < Literal::Types::DeferredType
+			def materialize = frozen? ? frozen_type : super
+
+			# JRuby takes an endless operator ending in "=" for a setter
+			def ===(other)
+				frozen? ? frozen_type === other : super
+			end
+
+			def >=(other, context: nil)
+				frozen? ? Literal.subtype?(other, frozen_type, context:) : super
+			end
+
+			private
+
+			def frozen_type = UNMATERIALIZED.equal?(@materialized) ? @block.call : @materialized
+		end
+
+		module Types
+			def _Deferred(&) = Deferred.new(&)
+		end
+	end
+end

data/lib/plume/normalize.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Plume
+	# Constants turned into `?` parameters, so the same query with different values comes out the
+	# same - for grouping a log the way pg_stat_statements does, or for keeping values out of it.
+	# Numbers, strings and blobs are replaced, a sign in front of a number with them. NULL, TRUE,
+	# FALSE and CURRENT_* are not, since `IS ?` would not mean `IS NULL`. Neither is an integer
+	# standing alone in ORDER BY or GROUP BY, which SQLite takes for a column's position.
+	# Only SELECT, INSERT, UPDATE and DELETE (and EXPLAIN of them) are normalized, other statements
+	# are left as they were: their constants are schema, not data
+	module Normalize
+		extend self
+
+		STATEMENTS = [SelectStatement, InsertStatement, UpdateStatement, DeleteStatement].freeze
+
+		# The SQL with each constant swapped for `?` in place, everything else - spacing, case,
+		# comments - kept as written
+		def text(sql)
+			src = sql.b
+			spans = Parser.new(sql).parse.flat_map { |statement| constants(statement).map(&:location) }
+			out = +""
+			at = 0
+			spans.sort_by(&:first).each do |beg, fin|
+				out << src.byteslice(at, beg - at) << "?"
+				at = fin
+			end
+			out << src.byteslice(at, src.bytesize - at)
+			out.force_encoding(sql.encoding)
+		end
+
+		# A copy of the tree with each constant replaced by a `?` Variable
+		def tree(node)
+			targets = Set.new(constants(node)).compare_by_identity
+			node.transform { |n| targets.include?(n) ? Variable.new(name: "?") : n }
+		end
+
+		# The nodes to replace: literals, or a sign with the literal under it
+		def constants(statement)
+			root = (ExplainStatement === statement) ? statement.statement : statement
+			return [] unless STATEMENTS.any? { |klass| klass === root }
+
+			found = []
+			# By identity: hashing a node hashes its whole subtree
+			positional = Set.new.compare_by_identity
+			root.walk_with_slots do |node, parent, prop|
+				case node
+				when OrderingTerm
+					positional << node if SelectStatement === parent && prop == :orderings
+				when UnaryExpression
+					found << node if (node.operator in :NEGATE | :IDENTITY) && value?(node.operand)
+				when LiteralExpression
+					next if UnaryExpression === parent && (parent.operator in :NEGATE | :IDENTITY)
+					next if Integer === node.value && (positional.include?(parent) || (SelectCore === parent && prop == :groupings))
+
+					found << node if value?(node)
+				end
+			end
+			found
+		end
+
+		private
+
+		def value?(node) = LiteralExpression === node && (node.value in Numeric | String)
+	end
+end

data/lib/plume/parameters.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Plume
+	# The bind parameters of one statement, numbered the way SQLite numbers them: `?` takes the
+	# next number, `?N` takes N, and a name keeps the number it first got. `count` and `names`
+	# are what `sqlite3_bind_parameter_count` and `sqlite3_bind_parameter_name` would answer
+	class Parameters
+		include Enumerable
+
+		# One place a parameter appears. `name` is nil for a bare `?`
+		Parameter = Data.define(:number, :name, :node)
+
+		attr_reader :count, :names
+
+		def initialize(statement)
+			slots = {}
+			@count = 0
+			@parameters = variables(statement).map do |variable|
+				name = variable.name
+				number =
+					if name == "?"
+						@count += 1
+					elsif (n = name[/\A\?(\d+)\z/, 1]&.to_i)
+						slots[n] ||= name
+						@count = n if n > @count
+						n
+					elsif (n = slots.key(name))
+						n
+					else
+						slots[@count += 1] = name
+						@count
+					end
+				Parameter.new(number:, name: (name == "?") ? nil : name, node: variable)
+			end.freeze
+			@names = Array.new(@count) { |i| slots[i + 1] }.freeze
+		end
+
+		def each(&) = @parameters.each(&)
+
+		def size = @parameters.size
+
+		# The number a name was given, as `sqlite3_bind_parameter_index` answers it
+		def index(name) = @names.index(name)&.+(1)
+
+		private
+
+		# SQLite numbers them as it reads them, which is not always the order of the tree
+		def variables(statement)
+			statement.each_node.grep(Variable).each_with_index.sort_by { |v, i| [v.location&.first || 0, i] }.map(&:first)
+		end
+	end
+end