exwiw 0.9.21 → 0.9.22

data/lib/exwiw/db_introspector/mysql_introspector.rb

@@ -0,0 +1,160 @@
+# frozen_string_literal: true
+
+module Exwiw
+  module DbIntrospector
+    # Reads MySQL's `information_schema`, scoped to `DATABASE()` — the database
+    # the connection was opened against, which is the one the dump would run in.
+    #
+    # Connects through MysqlClient, the same wrapper the dump path uses, so the
+    # driver choice (mysql2 or trilogy) and the "install the gem" error message
+    # are shared rather than reimplemented here.
+    class MysqlIntrospector < Base
+      # MySQL data_type (the type without its length/precision) -> the
+      # ActiveRecord-ish symbol DefaultMask understands. Anything absent maps to
+      # nil, which leaves the column unmasked: binary/blob columns must not
+      # receive a text mask, and enum/set/geometry/bit have no constant that is
+      # valid for every table's declaration.
+      TYPE_MAP = {
+        "char" => :string,
+        "varchar" => :string,
+        "tinytext" => :text,
+        "text" => :text,
+        "mediumtext" => :text,
+        "longtext" => :text,
+        "tinyint" => :integer,
+        "smallint" => :integer,
+        "mediumint" => :integer,
+        "int" => :integer,
+        "integer" => :integer,
+        "bigint" => :integer,
+        "decimal" => :decimal,
+        "numeric" => :decimal,
+        "float" => :float,
+        "double" => :float,
+        "date" => :date,
+        "datetime" => :datetime,
+        "timestamp" => :datetime,
+        "time" => :time,
+        "json" => :json,
+      }.freeze
+
+      # MySQL has no boolean type: `BOOLEAN` is an alias for `TINYINT(1)`, and
+      # the display width is the only trace of the distinction left in the
+      # catalog. Mapping it to :boolean (as every MySQL ORM does) is what makes
+      # a flag column mask to false / to its own default rather than to 0.
+      BOOLEAN_COLUMN_TYPE = "tinyint(1)"
+
+      # A default MySQL evaluates per row rather than storing as a literal.
+      # `extra` carries DEFAULT_GENERATED for an expression default, but only on
+      # servers new enough to support them, so the text is screened as well: a
+      # function call, or a bare keyword such as CURRENT_TIMESTAMP, is not a
+      # value we can mask with. A literal string that happens to contain
+      # parentheses is rejected too — losing a usable default costs nothing more
+      # than falling back to the per-type constant, while accepting an
+      # expression would write a mask the database re-evaluates.
+      EXPRESSION_DEFAULT = /[()]|\Acurrent_(?:timestamp|date|time)\z|\Alocaltime(?:stamp)?\z/i
+
+      def table_names
+        rows(<<~SQL).map { |row| row[0] }.sort
+          SELECT table_name
+          FROM information_schema.tables
+          WHERE table_schema = DATABASE() AND table_type = 'BASE TABLE'
+        SQL
+      end
+
+      def primary_key(table_name)
+        names = rows(<<~SQL).map { |row| row[0] }
+          SELECT column_name
+          FROM information_schema.key_column_usage
+          WHERE table_schema = DATABASE()
+            AND table_name = #{quote(table_name)}
+            AND constraint_name = 'PRIMARY'
+          ORDER BY ordinal_position
+        SQL
+
+        case names.size
+        when 0 then nil
+        when 1 then names.first
+        else names
+        end
+      end
+
+      def columns(table_name)
+        sql = <<~SQL
+          SELECT column_name, data_type, column_type, character_maximum_length, column_default, extra
+          FROM information_schema.columns
+          WHERE table_schema = DATABASE() AND table_name = #{quote(table_name)}
+          ORDER BY ordinal_position
+        SQL
+
+        rows(sql).map do |name, data_type, column_type, character_maximum_length, column_default, extra|
+          type = column_type == BOOLEAN_COLUMN_TYPE ? :boolean : TYPE_MAP[data_type]
+          Column.new(
+            name: name,
+            type: type,
+            limit: character_maximum_length&.to_i,
+            # MySQL has no array column type; a multi-valued column is JSON,
+            # which is masked as JSON rather than as an array.
+            array: false,
+            default: coerce_default(type, literal_default(column_default, extra)),
+          )
+        end
+      end
+
+      def unique_column_names(table_name)
+        rows(<<~SQL).map { |row| row[0] }.to_set
+          SELECT DISTINCT column_name
+          FROM information_schema.statistics
+          WHERE table_schema = DATABASE()
+            AND table_name = #{quote(table_name)}
+            AND non_unique = 0
+        SQL
+      rescue StandardError => e
+        warn_once(
+          :unique_column_names,
+          "exwiw: could not read the indexes of '#{table_name}' (#{e.class}); " \
+          "treating every column as unique-indexed so no constant mask is emitted.",
+        )
+        nil
+      end
+
+      def foreign_keys(table_name)
+        # `referenced_table_name IS NOT NULL` is what distinguishes a foreign
+        # key's rows from the primary/unique key rows sharing this view.
+        build_foreign_keys(table_name, rows(<<~SQL))
+          SELECT constraint_name, column_name, referenced_table_name
+          FROM information_schema.key_column_usage
+          WHERE table_schema = DATABASE()
+            AND table_name = #{quote(table_name)}
+            AND referenced_table_name IS NOT NULL
+          ORDER BY constraint_name, ordinal_position
+        SQL
+      end
+
+      # The catalog's `column_default` as a plain literal, or nil when it is
+      # absent or is an expression (see EXPRESSION_DEFAULT).
+      private def literal_default(column_default, extra)
+        return nil if column_default.nil?
+        return nil if extra.to_s.upcase.include?("DEFAULT_GENERATED")
+        return nil if column_default.match?(EXPRESSION_DEFAULT)
+
+        column_default
+      end
+
+      private def rows(sql)
+        connection.query(sql).rows
+      end
+
+      # Table names reaching this class come from `table_names` (the catalog
+      # itself), so they cannot carry an injection; quoting is defensive, for
+      # the day a caller passes a name from elsewhere.
+      private def quote(value)
+        "'#{value.to_s.gsub("\\", "\\\\\\\\").gsub("'", "''")}'"
+      end
+
+      private def connection
+        @connection ||= Adapter::MysqlClient.new(@connection_config)
+      end
+    end
+  end
+end

data/lib/exwiw/db_introspector/postgresql_introspector.rb

@@ -0,0 +1,215 @@
+# frozen_string_literal: true
+
+module Exwiw
+  module DbIntrospector
+    # Reads PostgreSQL's catalog, scoped to `current_schema()` — the first
+    # schema on the connection's search_path, which is the one an unqualified
+    # table name in the dump resolves to.
+    #
+    # `information_schema` is used where it is unambiguous (tables, columns,
+    # primary keys); unique indexes and foreign keys go through `pg_catalog`
+    # instead. information_schema only lists what a *constraint* declares, so a
+    # bare `CREATE UNIQUE INDEX` would be invisible there, and its
+    # `constraint_column_usage` join multiplies the rows of a composite foreign
+    # key into a cross product that cannot be grouped back.
+    class PostgresqlIntrospector < Base
+      # PostgreSQL data_type (information_schema's spelling) -> the
+      # ActiveRecord-ish symbol DefaultMask understands. An unmapped type stays
+      # nil so no mask is emitted: 'USER-DEFINED' covers every enum and
+      # extension type, where the set of valid values is per-column, and
+      # bytea/uuid/inet/interval have no constant that is safe to write back.
+      TYPE_MAP = {
+        "character" => :string,
+        "character varying" => :string,
+        "text" => :text,
+        "smallint" => :integer,
+        "integer" => :integer,
+        "bigint" => :integer,
+        "numeric" => :decimal,
+        "decimal" => :decimal,
+        "real" => :float,
+        "double precision" => :float,
+        "boolean" => :boolean,
+        "date" => :date,
+        "timestamp without time zone" => :datetime,
+        "timestamp with time zone" => :datetime,
+        "time without time zone" => :time,
+        "time with time zone" => :time,
+        "json" => :json,
+        "jsonb" => :jsonb,
+      }.freeze
+
+      # An ARRAY column reports data_type 'ARRAY' and carries the element type
+      # in udt_name, prefixed with an underscore (`_int4`). The element type is
+      # mapped so the column is still described accurately, even though
+      # DefaultMask emits no mask for an array either way.
+      ARRAY_DATA_TYPE = "ARRAY"
+      ELEMENT_TYPE_MAP = {
+        "bpchar" => :string,
+        "varchar" => :string,
+        "text" => :text,
+        "int2" => :integer,
+        "int4" => :integer,
+        "int8" => :integer,
+        "numeric" => :decimal,
+        "float4" => :float,
+        "float8" => :float,
+        "bool" => :boolean,
+        "date" => :date,
+        "timestamp" => :datetime,
+        "timestamptz" => :datetime,
+        "time" => :time,
+        "timetz" => :time,
+        "json" => :json,
+        "jsonb" => :jsonb,
+      }.freeze
+
+      # PostgreSQL renders a stored default back as the SQL text that produced
+      # it, so a plain literal arrives with its cast attached
+      # (`'member'::user_role`, `0`, `true`) and a computed one as the call that
+      # computes it (`now()`, `nextval('...')`). Only the literal forms are
+      # recognized, and the cast is stripped: matching what a mask may be built
+      # from, rather than trying to exclude every expression, keeps an
+      # unfamiliar expression on the safe side of the line.
+      QUOTED_LITERAL = /\A'((?:[^']|'')*)'(?:::[^']+)?\z/
+      NUMERIC_LITERAL = /\A-?\d+(?:\.\d+)?\z/
+      BOOLEAN_LITERAL = /\A(?:true|false)\z/i
+
+      def table_names
+        rows(<<~SQL).map { |row| row[0] }.sort
+          SELECT table_name
+          FROM information_schema.tables
+          WHERE table_schema = current_schema() AND table_type = 'BASE TABLE'
+        SQL
+      end
+
+      def primary_key(table_name)
+        names = rows(<<~SQL, [table_name]).map { |row| row[0] }
+          SELECT kcu.column_name
+          FROM information_schema.table_constraints tc
+          JOIN information_schema.key_column_usage kcu
+            ON kcu.constraint_name = tc.constraint_name
+           AND kcu.constraint_schema = tc.constraint_schema
+           AND kcu.table_name = tc.table_name
+          WHERE tc.constraint_type = 'PRIMARY KEY'
+            AND tc.table_schema = current_schema()
+            AND tc.table_name = $1
+          ORDER BY kcu.ordinal_position
+        SQL
+
+        case names.size
+        when 0 then nil
+        when 1 then names.first
+        else names
+        end
+      end
+
+      def columns(table_name)
+        sql = <<~SQL
+          SELECT column_name, data_type, udt_name, character_maximum_length, column_default
+          FROM information_schema.columns
+          WHERE table_schema = current_schema() AND table_name = $1
+          ORDER BY ordinal_position
+        SQL
+
+        rows(sql, [table_name]).map do |name, data_type, udt_name, character_maximum_length, column_default|
+          array = data_type == ARRAY_DATA_TYPE
+          type = array ? ELEMENT_TYPE_MAP[udt_name.to_s.delete_prefix("_")] : TYPE_MAP[data_type]
+          Column.new(
+            name: name,
+            type: type,
+            limit: character_maximum_length&.to_i,
+            array: array,
+            default: coerce_default(type, literal_default(column_default)),
+          )
+        end
+      end
+
+      def unique_column_names(table_name)
+        # `attnum = ANY(indkey)` keeps an expression index out of the result on
+        # its own: its entries are recorded as attnum 0, which no real column
+        # has, so the index simply contributes nothing.
+        rows(<<~SQL, [table_name]).map { |row| row[0] }.to_set
+          SELECT att.attname
+          FROM pg_index i
+          JOIN pg_class rel ON rel.oid = i.indrelid
+          JOIN pg_namespace nsp ON nsp.oid = rel.relnamespace
+          JOIN pg_attribute att ON att.attrelid = rel.oid AND att.attnum = ANY(i.indkey)
+          WHERE i.indisunique
+            AND nsp.nspname = current_schema()
+            AND rel.relname = $1
+        SQL
+      rescue StandardError => e
+        warn_once(
+          :unique_column_names,
+          "exwiw: could not read the indexes of '#{table_name}' (#{e.class}); " \
+          "treating every column as unique-indexed so no constant mask is emitted.",
+        )
+        nil
+      end
+
+      def foreign_keys(table_name)
+        # `conkey` lists the constrained columns in key order; unnesting it WITH
+        # ORDINALITY yields the one-row-per-key-column shape build_foreign_keys
+        # groups, so a composite constraint stays recognizable as one.
+        build_foreign_keys(table_name, rows(<<~SQL, [table_name]))
+          SELECT con.conname, att.attname, ref.relname
+          FROM pg_constraint con
+          JOIN pg_class rel ON rel.oid = con.conrelid
+          JOIN pg_namespace nsp ON nsp.oid = rel.relnamespace
+          JOIN pg_class ref ON ref.oid = con.confrelid
+          JOIN unnest(con.conkey) WITH ORDINALITY AS u(attnum, ord) ON TRUE
+          JOIN pg_attribute att ON att.attrelid = con.conrelid AND att.attnum = u.attnum
+          WHERE con.contype = 'f'
+            AND nsp.nspname = current_schema()
+            AND rel.relname = $1
+          ORDER BY con.conname, u.ord
+        SQL
+      end
+
+      # The catalog's `column_default` as a plain literal, or nil when it is
+      # absent or is an expression (see QUOTED_LITERAL and friends).
+      private def literal_default(column_default)
+        return nil if column_default.nil?
+
+        if (match = QUOTED_LITERAL.match(column_default))
+          # Inside a SQL string literal a quote is doubled; undouble it so the
+          # mask is the value the column actually defaults to.
+          return match[1].gsub("''", "'")
+        end
+        return column_default if column_default.match?(NUMERIC_LITERAL)
+        return column_default.downcase if column_default.match?(BOOLEAN_LITERAL)
+
+        nil
+      end
+
+      private def rows(sql, params = nil)
+        params.nil? ? connection.exec(sql).values : connection.exec_params(sql, params).values
+      end
+
+      private def connection
+        @connection ||= begin
+          require_driver!
+          PG.connect(
+            host: @connection_config.host,
+            port: @connection_config.port,
+            user: @connection_config.user,
+            password: @connection_config.password,
+            dbname: @connection_config.database_name,
+          )
+        end
+      end
+
+      # Soft-require the driver, like MysqlClient does, so a host that only ever
+      # runs the MySQL path is not forced to build the pg gem — and so the
+      # failure names the gem to install instead of surfacing a bare LoadError.
+      private def require_driver!
+        require "pg"
+      rescue LoadError
+        raise LoadError,
+              "exwiw needs the 'pg' gem to connect to PostgreSQL. " \
+              "Add `gem \"pg\"` to your Gemfile."
+      end
+    end
+  end
+end

data/lib/exwiw/db_introspector.rb

@@ -0,0 +1,166 @@
+# frozen_string_literal: true
+
+require "set"
+
+module Exwiw
+  # Reads a database's structure — tables, primary keys, columns, unique
+  # indexes, foreign keys — straight from its catalog, with no application code
+  # in the process.
+  #
+  # This is what lets a non-Ruby application keep an exwiw schema config: the
+  # ActiveRecord generator needs the app's models loaded in memory, which only
+  # its own runtime can do, while the database schema is a description every
+  # application shares regardless of the language it is written in. What is
+  # lost by reading the database instead of the models is relations that exist
+  # only in application code (no foreign-key constraint backs them) — which is
+  # why DbSchemaGenerator only ever *adds* belongs_tos and never rewrites the
+  # ones already in the config.
+  #
+  # Everything is scoped to the connection's current database/schema (MySQL:
+  # `DATABASE()`, PostgreSQL: `current_schema()`), i.e. exactly the objects the
+  # extraction itself would see through the same connection. There is no
+  # multi-database grouping equivalent to the ActiveRecord generator's: a
+  # connection points at one database, so one run generates one config
+  # directory, and a second database is a second run.
+  module DbIntrospector
+    # The adapters that can be introspected. sqlite is excluded because its
+    # catalog (PRAGMA-based) is a different shape entirely, and mongodb has no
+    # fixed schema to read — MongoidSchemaGenerator covers that case from the
+    # application side.
+    SUPPORTED_ADAPTERS = %w[mysql postgresql].freeze
+
+    # One column of a table, in the vocabulary DefaultMask.for speaks:
+    #
+    # - `type` is an ActiveRecord-ish symbol (:string, :integer, ...) or nil
+    #   when the database type has no equivalent there. nil is deliberate and
+    #   safe: DefaultMask emits no mask for a type it does not recognize, so an
+    #   exotic column is exported unmasked-but-flagged rather than masked with a
+    #   value it cannot hold.
+    # - `limit` is the character length for text-ish columns, else nil.
+    # - `array` marks a PostgreSQL ARRAY column, where no scalar mask fits.
+    # - `default` is the column's default as a Ruby scalar, or nil when it is
+    #   absent or is an expression the database evaluates per row.
+    Column = Data.define(:name, :type, :limit, :array, :default)
+
+    # Build the introspector for a connection. Takes the same ConnectionConfig
+    # the adapters do, so the CLI can hand over exactly what it already parsed.
+    def self.build(connection_config)
+      case Adapter.normalize_name(connection_config.adapter)
+      when "mysql" then MysqlIntrospector.new(connection_config)
+      when "postgresql" then PostgresqlIntrospector.new(connection_config)
+      else
+        raise ArgumentError,
+              "Schema generation from a database connection supports " \
+              "#{SUPPORTED_ADAPTERS.join(' / ')} only, got #{connection_config.adapter.inspect}."
+      end
+    end
+
+    class Base
+      def initialize(connection_config)
+        @connection_config = connection_config
+      end
+
+      # Sorted names of the tables in the current database/schema. Views are
+      # excluded: they hold no rows of their own, so exporting one would
+      # duplicate data already covered by its underlying tables (and the
+      # restore would fail against a target where the same view exists).
+      def table_names
+        raise NotImplementedError
+      end
+
+      # The table's primary key as a String, an Array for a composite key, or
+      # nil when the table has none. The three cases are what the generator
+      # branches on, so they are kept distinct rather than normalized.
+      def primary_key(_table_name)
+        raise NotImplementedError
+      end
+
+      # The table's columns as Column structs, in ordinal_position order — the
+      # order the config lists them in, which mirrors what a `SELECT *` returns.
+      def columns(_table_name)
+        raise NotImplementedError
+      end
+
+      # The names of columns covered by any unique index or constraint, or nil
+      # when the catalog could not be read. Callers treat nil as "assume every
+      # column is unique", since masking a unique column with a constant makes
+      # every row collide on restore. The primary key is included; it is
+      # unique, and the generator never masks it anyway.
+      def unique_column_names(_table_name)
+        raise NotImplementedError
+      end
+
+      # The table's single-column foreign keys as sorted
+      # `{ table_name:, foreign_key: }` hashes — the belongs_to shape. Composite
+      # foreign keys are skipped with a warning: exwiw joins on one column, so a
+      # multi-column edge cannot be expressed, and silently emitting one of its
+      # columns would produce a join that is quietly wrong.
+      def foreign_keys(_table_name)
+        raise NotImplementedError
+      end
+
+      # Turn the plain default literal the catalog reported into the Ruby
+      # scalar DefaultMask can use as a mask value.
+      #
+      # Deliberately narrow. A default only earns its place as a mask because it
+      # is a value the column provably holds and the application treats as
+      # neutral; a value we had to guess at loses both properties. So a type
+      # whose text form we cannot map back with certainty yields nil and the
+      # per-type constant is used instead. JSON is excluded for that reason:
+      # DefaultMask re-serializes a JSON default with #to_json, which would turn
+      # the catalog's already-serialized text into a doubly-encoded string.
+      private def coerce_default(type, literal)
+        return nil if literal.nil?
+
+        case type
+        when :integer then Integer(literal, exception: false)
+        when :decimal, :float then Float(literal, exception: false)
+        when :boolean then BOOLEAN_DEFAULTS[literal.downcase]
+        when :string, :text, :date, :datetime, :time then literal
+        end
+      end
+
+      # How the two databases spell a boolean default in the catalog: MySQL
+      # stores a TINYINT(1) default as "1"/"0", PostgreSQL a boolean one as
+      # "true"/"false". Anything else is not a boolean literal and falls through
+      # to nil.
+      BOOLEAN_DEFAULTS = {
+        "1" => true, "true" => true,
+        "0" => false, "false" => false,
+      }.freeze
+
+      # Emit `message` to stderr the first time this introspector hits `key`.
+      # A catalog failure is systematic rather than per-table, so repeating it
+      # once per table would bury the rest of the run's output.
+      private def warn_once(key, message)
+        @warned ||= {}
+        return if @warned[key]
+
+        @warned[key] = true
+        warn(message)
+      end
+
+      # Group the catalog's one-row-per-key-column foreign-key listing into
+      # `{ table_name:, foreign_key: }` entries, dropping composite constraints.
+      # Both introspectors read their catalog in that shape (ordered by
+      # constraint name, then key position), so the grouping and the warning
+      # live here rather than being written twice.
+      private def build_foreign_keys(table_name, rows)
+        rows.group_by { |constraint_name, _column_name, _referenced_table| constraint_name }
+            .filter_map do |constraint_name, group|
+              if group.size > 1
+                warn "exwiw: skipping composite foreign key '#{constraint_name}' on '#{table_name}' " \
+                     "(#{group.map { |_c, column_name, _r| column_name }.join(', ')}); " \
+                     "exwiw joins a belongs_to on a single column."
+                next
+              end
+
+              _constraint_name, column_name, referenced_table = group.first
+              { table_name: referenced_table, foreign_key: column_name }
+            end
+            .uniq
+            .sort_by { |entry| [entry[:table_name], entry[:foreign_key]] }
+      end
+    end
+  end
+end