usernames 0.1.3

data/lib/usernames/comparison/index_contract.rb

@@ -0,0 +1,241 @@
+# frozen_string_literal: true
+
+require_relative "index_expression"
+
+module Usernames
+  module Comparison
+    # Whether an index read back from the database enforces the comparison
+    # contract for one column — the one question the installer (is the index
+    # already there?), the audit (does the database enforce uniqueness?) and
+    # UniqueViolation (was the violated index the username's?) all ask, so
+    # they can never disagree.
+    #
+    #   contract = IndexContract.new(connection, table: "users", column: :username, mode: :lowercase)
+    #   contract.violation(index)       # => nil, or why not: "only covers rows WHERE (active = 1)"
+    #   contract.satisfied_by?(index)   # => true / false
+    #
+    # It is a recogniser, not a normaliser: the index is read as SQL
+    # (IndexExpression) and has to be one of the few shapes the gem's own
+    # index takes as each database prints it back; anything else does not
+    # count, however much it looks like one.
+    #
+    #                lowercase mode                  preserve mode
+    #   PostgreSQL   username, (username)::text      lower(username COLLATE "C"), lower((username)::text COLLATE "C")
+    #   SQLite       username                        lower(username)
+    #   MySQL        username                        lower(username)
+    #
+    # And it must be unique, valid, over every row and byte-compared:
+    #
+    # * unique over every row: a partial index (`WHERE active`) lets two
+    #   inactive accounts share a name. `WHERE username IS NOT NULL` is the
+    #   one condition that changes nothing, since an absent name is nobody's.
+    # * exactly the key: `(tenant_id, username)` or `substr(username, 1, 3)`
+    #   are unique indexes, but not on the identity.
+    # * byte-wise: a `COLLATE NOCASE` index on SQLite, or a PostgreSQL index
+    #   under a linguistic collation, decides "the same name" differently
+    #   from Ruby. And in every mode but PostgreSQL's preserve (whose
+    #   expression carries its own COLLATE "C"), the queries compare the
+    #   column under the column's collation, so that one has to compare
+    #   bytes too: a BINARY index on a NOCASE column takes `javi` and `JAVI`
+    #   while `WHERE username = 'javi'` finds both.
+    # * valid: PostgreSQL keeps an interrupted CREATE INDEX CONCURRENTLY as
+    #   an invalid index that enforces nothing.
+    class IndexContract
+      # Collations that compare bytes, per adapter (SQLite's BINARY is the default).
+      BYTEWISE = { sqlite: %w[binary], postgresql: %w[c posix], mysql: %w[utf8mb4_bin] }.freeze
+
+      Expression = IndexExpression
+
+      attr_reader :table, :column, :mode, :adapter
+
+      # Whether a column under `collation` is compared the way `mode` needs
+      # on `adapter` — the rule the audit's column check and violation share.
+      # `collation` nil is the column's default: SQLite's BINARY, while
+      # MySQL's default is the server's, which is not utf8mb4_bin.
+      def self.column_collation_ok?(adapter, mode, collation)
+        return true if mode == :preserve && adapter == :postgresql
+
+        bytewise?(adapter, collation)
+      end
+
+      def self.bytewise?(adapter, collation)
+        return adapter == :sqlite if collation.nil?
+
+        BYTEWISE.fetch(adapter).include?(collation.downcase)
+      end
+
+      def initialize(connection, table:, column:, mode:)
+        @connection = connection
+        @table = table.to_s
+        @column = column.to_s
+        @mode = mode
+        @adapter = Comparison.adapter(connection)
+      end
+
+      # The index the gem itself writes: what the installer adds.
+      def definition
+        @definition ||= Comparison.index_definition(adapter: adapter, table: table, column: column, mode: mode)
+      end
+
+      def satisfied_by?(index)
+        violation(index).nil?
+      end
+
+      # nil when `index` enforces the contract, else why it does not.
+      def violation(index)
+        return "isn't unique" unless index.unique
+        return "isn't valid (an interrupted concurrent build enforces nothing)" if index.try(:valid?) == false
+
+        key_violation(index) || predicate_violation(index) || collation_violation(index)
+      end
+
+      # The collation the column itself compares under (nil: SQLite's
+      # default BINARY). On PostgreSQL a column without one has the
+      # database's. SQLite keeps its CREATE TABLE as written, and Active
+      # Record reads a collation back from it only when quoted — to it
+      # `username VARCHAR COLLATE NOCASE` has none — so there the definition
+      # itself is read.
+      def column_collation
+        found = @connection.columns(table).find { |each| each.name == column }&.collation
+        return found if found
+        return sqlite_column_collation if adapter == :sqlite
+        return unless postgresql?
+
+        @connection.select_value("SELECT datcollate FROM pg_database WHERE datname = current_database()")
+      end
+
+      # Whether `index` involves the column at all, so the audit lists it.
+      def mentions?(index)
+        columns = index.columns
+        return Array(columns).map(&:to_s).include?(column) unless columns.is_a?(String)
+
+        Expression.names(columns).any? { |name| name.named?(column) }
+      end
+
+      private
+
+      def expected_key
+        definition[:column].to_s
+      end
+
+      def postgresql?
+        adapter == :postgresql
+      end
+
+      # Active Record reads a plain index back as its columns, and an
+      # expression as the database prints it.
+      def key_violation(index)
+        columns = index.columns
+        unless columns.is_a?(String)
+          return if mode == :lowercase && Array(columns).map(&:to_s) == [column]
+
+          return "is on #{Array(columns).join(', ')}, not #{expected_key}"
+        end
+
+        node = Expression.parse(columns)
+        return "is on an unsupported expression: #{columns}" unless node
+
+        "is on #{columns}, not #{expected_key}" unless expected_key?(node)
+      end
+
+      def expected_key?(node)
+        if mode == :lowercase
+          column?(node, cast: postgresql?)
+        elsif postgresql?
+          node.is_a?(Expression::Lower) && node.operand.is_a?(Expression::Collate) &&
+            node.operand.collation == "C" && column?(node.operand.operand, cast: true)
+        else
+          node.is_a?(Expression::Lower) && column?(node.operand, cast: false)
+        end
+      end
+
+      def column?(node, cast:)
+        node = node.operand if cast && node.is_a?(Expression::Cast)
+        node.is_a?(Expression::Column) && node.named?(column)
+      end
+
+      def predicate_violation(index)
+        return if index.where.blank?
+
+        node = Expression.parse(index.where)
+        return if node.is_a?(Expression::NotNull) && column?(node.operand, cast: false)
+
+        "only covers rows WHERE #{index.where}"
+      end
+
+      # The index's own collations, then the column's: in every mode but
+      # PostgreSQL's preserve the queries compare the column under its own
+      # collation, whatever the index says.
+      def collation_violation(index)
+        own = collations_in(index_sql(index)).find { |name| !self.class.bytewise?(adapter, name) }
+        return "compares under collation #{own.inspect}, not byte by byte" if own
+
+        collation = column_collation
+        return if self.class.column_collation_ok?(adapter, mode, collation)
+
+        "is on a column compared under collation #{collation.inspect}, not byte by byte"
+      end
+
+      # Every name that follows a COLLATE in the index's own SQL.
+      def collations_in(sql)
+        tokens = sql.is_a?(Array) ? sql : Expression.tokens(sql)
+        tokens.each_cons(2).filter_map do |(kind, text), (_, name)|
+          name if kind == :word && text.casecmp?("collate")
+        end
+      end
+
+      def sqlite_column_collation
+        sql = @connection.select_value("SELECT sql FROM sqlite_master WHERE type = 'table' " \
+                                       "AND name = #{@connection.quote(table.split('.').last)}")
+        definitions = column_definitions(Expression.tokens(sql))
+        definition = definitions.find { |tokens| Expression.name(tokens.first)&.named?(column) }
+        collations_in(outside_parentheses(definition.to_a)).last
+      end
+
+      # A column definition without whatever its parentheses hold: only a
+      # COLLATE at its own level is the column's collation. One inside
+      # CHECK (...), DEFAULT (...) or GENERATED ALWAYS AS (...) collates that
+      # expression alone, and a type's VARCHAR(255) holds none. Quoted names
+      # and string literals are single tokens, so a parenthesis inside one
+      # does not count.
+      def outside_parentheses(tokens)
+        depth = 0
+        tokens.select do |token|
+          depth -= 1 if token == [:symbol, ")"]
+          outside = depth.zero? && !token.in?([[:symbol, "("], [:symbol, ")"]])
+          depth += 1 if token == [:symbol, "("]
+          outside
+        end
+      end
+
+      # The comma-separated definitions inside CREATE TABLE's parentheses,
+      # each as its tokens.
+      def column_definitions(tokens)
+        definitions = [[]]
+        depth = 0
+        tokens.each do |token|
+          depth -= 1 if token == [:symbol, ")"]
+          if depth == 1 && token == [:symbol, ","] then definitions << []
+          elsif depth >= 1 then definitions.last << token
+          end
+          depth += 1 if token == [:symbol, "("]
+        end
+        definitions
+      end
+
+      # The index as the database itself prints it: Active Record reads back
+      # neither an index's collation nor whether a column in it has one.
+      def index_sql(index)
+        case adapter
+        when :sqlite
+          @connection.select_value("SELECT sql FROM sqlite_master WHERE type = 'index' " \
+                                   "AND name = #{@connection.quote(index.name)}")
+        when :postgresql
+          qualified = [*table.split(".")[0...-1], index.name].join(".")
+          @connection.select_value("SELECT pg_get_indexdef(#{@connection.quote(@connection.quote_table_name(qualified))}" \
+                                   "::regclass)")
+        end
+      end
+    end
+  end
+end

data/lib/usernames/comparison/index_expression.rb

@@ -0,0 +1,183 @@
+# frozen_string_literal: true
+
+require "strscan"
+
+module Usernames
+  module Comparison
+    # An index key or condition as the database prints it back, read as SQL
+    # rather than as letters: `"user name"` is one identifier and `username`
+    # another, and `"usernameisnotnull"` is a column, not `username IS NOT
+    # NULL` with the spaces taken out.
+    #
+    # It understands only what an index enforcing the comparison contract
+    # can contain, and nothing else parses (another function, an operator, a
+    # literal, a second argument):
+    #
+    #   username  "username"  `username`  [username]   a column
+    #   (username)::text  username::character varying   a PostgreSQL cast
+    #   username COLLATE "C"                            a collation
+    #   lower(username)                                 lower()
+    #   username IS NOT NULL                            a null test
+    #
+    # Parentheses group without changing the meaning, so PostgreSQL's
+    # `lower(((username)::text COLLATE "C"))` reads as
+    # Lower(Collate(Cast(Column), "C")).
+    #
+    #   IndexExpression.parse('lower("user name")')  # => Lower(Column("user name"))
+    #   IndexExpression.parse("substr(username, 1, 3)")  # => nil
+    module IndexExpression
+      # A column, as written: quoted (compared exactly) or bare (compared
+      # ignoring case, as every supported database folds a bare identifier).
+      Column = ::Data.define(:name, :quoted) do
+        def named?(column)
+          quoted ? name == column : name.casecmp?(column)
+        end
+      end
+      Cast = ::Data.define(:operand)
+      Collate = ::Data.define(:operand, :collation)
+      Lower = ::Data.define(:operand)
+      NotNull = ::Data.define(:operand)
+
+      NAMES = %i[identifier word].freeze
+      KEYWORDS = %w[is not null collate].freeze
+      CAST_TYPES = [%w[text], %w[varchar], %w[character varying]].freeze
+
+      class << self
+        # The parsed expression, or nil when it is anything else.
+        def parse(sql)
+          Parser.new(tokens(sql)).parse
+        end
+
+        # Every name in `sql`, keyword or not, as a Column.
+        def names(sql)
+          tokens(sql).filter_map { |token| name(token) }
+        end
+
+        # A token as a Column, when it is a name.
+        def name(token)
+          kind, text = token
+          Column.new(text, kind == :identifier) if NAMES.include?(kind)
+        end
+
+        # [kind, text] pairs: :identifier (quoted, unquoted), :word (bare),
+        # :literal and :symbol. Whitespace only separates.
+        def tokens(sql)
+          scanner = StringScanner.new(sql.to_s)
+          found = []
+          until scanner.eos?
+            next if scanner.skip(/\s+/)
+
+            found << token(scanner)
+          end
+          found
+        end
+
+        private
+
+        def token(scanner)
+          if scanner.scan(/"((?:[^"]|"")*)"/) then [:identifier, scanner[1].gsub('""', '"')]
+          elsif scanner.scan(/`((?:[^`]|``)*)`/) then [:identifier, scanner[1].gsub("``", "`")]
+          elsif scanner.scan(/\[([^\]]*)\]/) then [:identifier, scanner[1]]
+          elsif scanner.scan(/[A-Za-z_][A-Za-z0-9_$]*/) then [:word, scanner.matched]
+          elsif scanner.scan(/'(?:[^']|'')*'/) then [:literal, scanner.matched]
+          else [:symbol, scanner.scan(/::|./m)]
+          end
+        end
+      end
+
+      # Recursive descent over the tokens; nil as soon as one does not fit.
+      class Parser
+        def initialize(tokens)
+          @tokens = tokens
+          @at = 0
+        end
+
+        def parse
+          node = expression
+          node if node && @at == @tokens.size
+        end
+
+        private
+
+        def expression
+          node = postfix
+          return node unless node && word?("is")
+          return unless word?("not", 1) && word?("null", 2)
+
+          advance(3)
+          NotNull.new(node)
+        end
+
+        # A term followed by any casts and collations.
+        def postfix
+          node = primary
+          while node
+            if symbol?("::")
+              advance
+              node = cast_type && Cast.new(node)
+            elsif word?("collate")
+              advance
+              name = collation_name
+              node = name && Collate.new(node, name)
+            else
+              return node
+            end
+          end
+        end
+
+        def primary
+          if symbol?("(")
+            advance
+            closing(expression)
+          elsif word?("lower") && symbol?("(", 1)
+            advance(2)
+            (operand = closing(expression)) && Lower.new(operand)
+          else
+            column
+          end
+        end
+
+        def closing(node)
+          return unless node && symbol?(")")
+
+          advance
+          node
+        end
+
+        def column
+          kind, text = @tokens[@at]
+          return unless kind == :identifier || (kind == :word && !KEYWORDS.include?(text.downcase))
+
+          advance
+          Column.new(text, kind == :identifier)
+        end
+
+        def cast_type
+          type = CAST_TYPES.find { |words| words.each_with_index.all? { |word, offset| word?(word, offset) } }
+          type && advance(type.size)
+        end
+
+        def collation_name
+          kind, text = @tokens[@at]
+          return unless NAMES.include?(kind)
+
+          advance
+          text
+        end
+
+        def word?(word, offset = 0)
+          kind, text = @tokens[@at + offset]
+          kind == :word && text.casecmp?(word)
+        end
+
+        def symbol?(symbol, offset = 0)
+          @tokens[@at + offset] == [:symbol, symbol]
+        end
+
+        def advance(count = 1)
+          @at += count
+        end
+      end
+    end
+  end
+end

data/lib/usernames/comparison.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require_relative "comparison/index_contract"
+
+module Usernames
+  # How the database compares two usernames.
+  #
+  # "The same name" has to mean one thing in Ruby, in the unique index and in
+  # every query, or the validator says a name is free that the index then
+  # refuses (or worse, the other way round). PostgreSQL's `=` and `lower()`
+  # follow the column's collation — under a Turkish collation `I` lowercases
+  # to `ı`, and a nondeterministic collation can ignore punctuation — so the
+  # gem pins the comparison instead of inheriting it:
+  #
+  #                lowercase mode   preserve mode
+  #   PostgreSQL   username         lower(username COLLATE "C")   (ASCII-only, exactly Ruby's downcase)
+  #   SQLite       username         lower(username)               (SQLite's lower() is ASCII-only)
+  #   MySQL        username         lower(username)               (experimental; the column is utf8mb4_bin)
+  #
+  # In lowercase mode the stored value is already canonical, so a plain `=` on
+  # a byte-compared column ("C" / utf8mb4_bin / SQLite's BINARY) is the whole
+  # contract, and an ordinary unique index serves it.
+  #
+  # The same expression builds the unique index, the uniqueness check and
+  # every helper. It is only ever used as a BOUND predicate,
+  #
+  #   where("#{Comparison.sql(User, :username)} = ?", key)
+  #
+  # never as a hash key, which Active Record would quote as a column name.
+  # It holds nothing but gem-written SQL and adapter-quoted identifiers; the
+  # username is always the bind, so the database never lowercases it under
+  # its own locale.
+  module Comparison
+    # The byte-wise collation the column needs, per adapter. SQLite's default
+    # BINARY already is one.
+    COLLATIONS = { postgresql: "C", mysql: "utf8mb4_bin" }.freeze
+
+    class << self
+      # The comparison expression for `column` of `model`, table-qualified so
+      # it stays unambiguous inside a join. The case mode is the one the
+      # model declared with has_username (or the configured one for a model
+      # that only uses the validator).
+      def sql(model, column)
+        model.with_connection do |connection|
+          quoted = "#{connection.quote_table_name(model.table_name)}.#{connection.quote_column_name(column)}"
+          expression(adapter(connection), quoted, mode_for(model, column))
+        end
+      end
+
+      # What the install generator passes to add_index:
+      #
+      #   index_definition(adapter: :postgresql, table: :users, column: :username, mode: :preserve)
+      #   # => { column: 'lower(username COLLATE "C")', name: "index_users_on_lower_username", unique: true }
+      #
+      # The names are the gem's own: UniqueViolation recognises them even
+      # when it cannot read the index back.
+      def index_definition(adapter:, table:, column:, mode:)
+        table = table.to_s.split(".").last
+        if mode == :preserve
+          { column: index_expression(adapter, column), name: "index_#{table}_on_lower_#{column}", unique: true }
+        else
+          { column: column.to_sym, name: "index_#{table}_on_#{column}", unique: true }
+        end
+      end
+
+      # The collation the column itself needs for `mode`: "C" on PostgreSQL,
+      # "utf8mb4_bin" on MySQL, nil on SQLite (whose default BINARY is the
+      # contract). nil for PostgreSQL in preserve mode: the comparison
+      # expression carries its own COLLATE "C", so the column keeps the
+      # database's collation — and `ORDER BY username` keeps sorting the way
+      # people read (a "C" column sorts every uppercase letter first).
+      def column_collation(adapter, mode = :lowercase)
+        adapter = self.adapter(adapter)
+        return nil if mode == :preserve && adapter == :postgresql
+
+        COLLATIONS[adapter]
+      end
+
+      # A table or column name as a migration argument: `:users` when it is
+      # a plain identifier, else a string literal — `"public.users"`, never
+      # the `:public.users` that does not parse.
+      def ruby_name(name)
+        name = name.to_s
+        name.match?(/\A[A-Za-z_][A-Za-z0-9_]*\z/) ? ":#{name}" : name.inspect
+      end
+
+      # :postgresql, :mysql or :sqlite, from a connection or an adapter name
+      # ("PostgreSQL", "postgis", "Mysql2", "Trilogy", "sqlite3").
+      # Any other adapter raises ConfigurationError: the comparison contract
+      # is written for these three, and guessing another database's
+      # collation and lower() is how two people end up with one name.
+      def adapter(connection_or_name)
+        name = connection_or_name.respond_to?(:adapter_name) ? connection_or_name.adapter_name : connection_or_name
+        case name.to_s
+        when /postg/i then :postgresql
+        when /mysql|trilogy|maria/i then :mysql
+        when /sqlite/i then :sqlite
+        else
+          raise ConfigurationError, "usernames supports PostgreSQL, MySQL and SQLite; the #{name.to_s.inspect} adapter " \
+                                    "has no comparison contract"
+        end
+      end
+
+      # The case mode `column` of `model` is compared in.
+      def mode_for(model, column)
+        definition = model.try(:_usernames)&.[](column.to_sym)
+        definition ? definition.mode : Usernames.configuration.case
+      end
+
+      private
+
+      def expression(adapter, quoted, mode)
+        return quoted unless mode == :preserve
+
+        adapter == :postgresql ? %(lower(#{quoted} COLLATE "C")) : "lower(#{quoted})"
+      end
+
+      # As written in a migration: the bare column, and on MySQL the extra
+      # parentheses a functional key part needs (8.0.13+).
+      def index_expression(adapter, column)
+        adapter = self.adapter(adapter)
+        sql = expression(adapter, column.to_s, :preserve)
+        adapter == :mysql ? "(#{sql})" : sql
+      end
+    end
+  end
+end