exwiw 0.9.20 → 0.9.22

data/lib/exwiw/db_introspector.rb

@@ -0,0 +1,166 @@
+# frozen_string_literal: true
+
+require "set"
+
+module Exwiw
+  # Reads a database's structure — tables, primary keys, columns, unique
+  # indexes, foreign keys — straight from its catalog, with no application code
+  # in the process.
+  #
+  # This is what lets a non-Ruby application keep an exwiw schema config: the
+  # ActiveRecord generator needs the app's models loaded in memory, which only
+  # its own runtime can do, while the database schema is a description every
+  # application shares regardless of the language it is written in. What is
+  # lost by reading the database instead of the models is relations that exist
+  # only in application code (no foreign-key constraint backs them) — which is
+  # why DbSchemaGenerator only ever *adds* belongs_tos and never rewrites the
+  # ones already in the config.
+  #
+  # Everything is scoped to the connection's current database/schema (MySQL:
+  # `DATABASE()`, PostgreSQL: `current_schema()`), i.e. exactly the objects the
+  # extraction itself would see through the same connection. There is no
+  # multi-database grouping equivalent to the ActiveRecord generator's: a
+  # connection points at one database, so one run generates one config
+  # directory, and a second database is a second run.
+  module DbIntrospector
+    # The adapters that can be introspected. sqlite is excluded because its
+    # catalog (PRAGMA-based) is a different shape entirely, and mongodb has no
+    # fixed schema to read — MongoidSchemaGenerator covers that case from the
+    # application side.
+    SUPPORTED_ADAPTERS = %w[mysql postgresql].freeze
+
+    # One column of a table, in the vocabulary DefaultMask.for speaks:
+    #
+    # - `type` is an ActiveRecord-ish symbol (:string, :integer, ...) or nil
+    #   when the database type has no equivalent there. nil is deliberate and
+    #   safe: DefaultMask emits no mask for a type it does not recognize, so an
+    #   exotic column is exported unmasked-but-flagged rather than masked with a
+    #   value it cannot hold.
+    # - `limit` is the character length for text-ish columns, else nil.
+    # - `array` marks a PostgreSQL ARRAY column, where no scalar mask fits.
+    # - `default` is the column's default as a Ruby scalar, or nil when it is
+    #   absent or is an expression the database evaluates per row.
+    Column = Data.define(:name, :type, :limit, :array, :default)
+
+    # Build the introspector for a connection. Takes the same ConnectionConfig
+    # the adapters do, so the CLI can hand over exactly what it already parsed.
+    def self.build(connection_config)
+      case Adapter.normalize_name(connection_config.adapter)
+      when "mysql" then MysqlIntrospector.new(connection_config)
+      when "postgresql" then PostgresqlIntrospector.new(connection_config)
+      else
+        raise ArgumentError,
+              "Schema generation from a database connection supports " \
+              "#{SUPPORTED_ADAPTERS.join(' / ')} only, got #{connection_config.adapter.inspect}."
+      end
+    end
+
+    class Base
+      def initialize(connection_config)
+        @connection_config = connection_config
+      end
+
+      # Sorted names of the tables in the current database/schema. Views are
+      # excluded: they hold no rows of their own, so exporting one would
+      # duplicate data already covered by its underlying tables (and the
+      # restore would fail against a target where the same view exists).
+      def table_names
+        raise NotImplementedError
+      end
+
+      # The table's primary key as a String, an Array for a composite key, or
+      # nil when the table has none. The three cases are what the generator
+      # branches on, so they are kept distinct rather than normalized.
+      def primary_key(_table_name)
+        raise NotImplementedError
+      end
+
+      # The table's columns as Column structs, in ordinal_position order — the
+      # order the config lists them in, which mirrors what a `SELECT *` returns.
+      def columns(_table_name)
+        raise NotImplementedError
+      end
+
+      # The names of columns covered by any unique index or constraint, or nil
+      # when the catalog could not be read. Callers treat nil as "assume every
+      # column is unique", since masking a unique column with a constant makes
+      # every row collide on restore. The primary key is included; it is
+      # unique, and the generator never masks it anyway.
+      def unique_column_names(_table_name)
+        raise NotImplementedError
+      end
+
+      # The table's single-column foreign keys as sorted
+      # `{ table_name:, foreign_key: }` hashes — the belongs_to shape. Composite
+      # foreign keys are skipped with a warning: exwiw joins on one column, so a
+      # multi-column edge cannot be expressed, and silently emitting one of its
+      # columns would produce a join that is quietly wrong.
+      def foreign_keys(_table_name)
+        raise NotImplementedError
+      end
+
+      # Turn the plain default literal the catalog reported into the Ruby
+      # scalar DefaultMask can use as a mask value.
+      #
+      # Deliberately narrow. A default only earns its place as a mask because it
+      # is a value the column provably holds and the application treats as
+      # neutral; a value we had to guess at loses both properties. So a type
+      # whose text form we cannot map back with certainty yields nil and the
+      # per-type constant is used instead. JSON is excluded for that reason:
+      # DefaultMask re-serializes a JSON default with #to_json, which would turn
+      # the catalog's already-serialized text into a doubly-encoded string.
+      private def coerce_default(type, literal)
+        return nil if literal.nil?
+
+        case type
+        when :integer then Integer(literal, exception: false)
+        when :decimal, :float then Float(literal, exception: false)
+        when :boolean then BOOLEAN_DEFAULTS[literal.downcase]
+        when :string, :text, :date, :datetime, :time then literal
+        end
+      end
+
+      # How the two databases spell a boolean default in the catalog: MySQL
+      # stores a TINYINT(1) default as "1"/"0", PostgreSQL a boolean one as
+      # "true"/"false". Anything else is not a boolean literal and falls through
+      # to nil.
+      BOOLEAN_DEFAULTS = {
+        "1" => true, "true" => true,
+        "0" => false, "false" => false,
+      }.freeze
+
+      # Emit `message` to stderr the first time this introspector hits `key`.
+      # A catalog failure is systematic rather than per-table, so repeating it
+      # once per table would bury the rest of the run's output.
+      private def warn_once(key, message)
+        @warned ||= {}
+        return if @warned[key]
+
+        @warned[key] = true
+        warn(message)
+      end
+
+      # Group the catalog's one-row-per-key-column foreign-key listing into
+      # `{ table_name:, foreign_key: }` entries, dropping composite constraints.
+      # Both introspectors read their catalog in that shape (ordered by
+      # constraint name, then key position), so the grouping and the warning
+      # live here rather than being written twice.
+      private def build_foreign_keys(table_name, rows)
+        rows.group_by { |constraint_name, _column_name, _referenced_table| constraint_name }
+            .filter_map do |constraint_name, group|
+              if group.size > 1
+                warn "exwiw: skipping composite foreign key '#{constraint_name}' on '#{table_name}' " \
+                     "(#{group.map { |_c, column_name, _r| column_name }.join(', ')}); " \
+                     "exwiw joins a belongs_to on a single column."
+                next
+              end
+
+              _constraint_name, column_name, referenced_table = group.first
+              { table_name: referenced_table, foreign_key: column_name }
+            end
+            .uniq
+            .sort_by { |entry| [entry[:table_name], entry[:foreign_key]] }
+      end
+    end
+  end
+end

data/lib/exwiw/db_schema_generator.rb

@@ -0,0 +1,298 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "json"
+require "set"
+
+module Exwiw
+  # Generates (and tidies) a schema config from a live database connection
+  # instead of from an application's models — see DbIntrospector for why an
+  # application exwiw cannot load needs this.
+  #
+  # It mirrors SchemaGenerator table for table and column for column, with one
+  # deliberate difference in how belongs_tos are reconciled (see
+  # #merged_belongs_tos) and one structural simplification: the output directory
+  # is flat, because a connection addresses exactly one database. A second
+  # database is a second run against a second connection, which is also how the
+  # extraction itself treats it.
+  class DbSchemaGenerator
+    # SchemaGenerator::TidyResult plus the belongs_tos this generator can also
+    # remove. Subclassed rather than extended in place so the ActiveRecord
+    # generator's result — and the rake task reporting it — keeps exactly the
+    # contract it has today.
+    class TidyResult < SchemaGenerator::TidyResult
+      attr_reader :removed_belongs_tos
+
+      def initialize
+        super
+        @removed_belongs_tos = {}
+      end
+
+      def add_removed_belongs_to(table_name, target_table_name)
+        (@removed_belongs_tos[table_name] ||= []) << target_table_name
+      end
+
+      def empty?
+        super && @removed_belongs_tos.empty?
+      end
+    end
+
+    # `safe_new_columns` matches SchemaGenerator's: on (the default) every
+    # column is emitted masked as far as its type allows and flagged
+    # `needs_mask_decision: true`, and #merge lets an already-decided entry win,
+    # so in practice only genuinely new columns keep that treatment.
+    def initialize(introspector:, output_dir:, safe_new_columns: true)
+      @introspector = introspector
+      @output_dir = output_dir
+      @safe_new_columns = safe_new_columns
+    end
+
+    # Write one config file per table in the database, merged with whatever is
+    # already on disk, and return the configs as written.
+    #
+    # Unlike SchemaGenerator, the configs are built *inside* the write path
+    # rather than by a separate builder: what the generated belongs_tos are —
+    # and therefore which columns count as structural and must not be masked —
+    # depends on the existing file (see #merged_belongs_tos), so the two cannot
+    # be separated without introspecting the disk twice.
+    def generate!
+      FileUtils.mkdir_p(@output_dir)
+
+      @introspector.table_names.map do |table_name|
+        path = File.join(@output_dir, "#{table_name}.json")
+        existing = read_config(path)
+        generated = build_table(table_name, existing)
+
+        config_to_write = existing ? existing.merge(generated) : generated
+        File.write(path, JSON.pretty_generate(config_to_write.to_hash) + "\n")
+        config_to_write
+      end
+    end
+
+    # Reconcile the config files on disk against the database, removing only
+    # what no longer exists there:
+    #
+    # - a config file whose table is gone is deleted,
+    # - columns a surviving table no longer has are dropped, and
+    # - a belongs_to pointing at a table that is gone is dropped.
+    #
+    # The belongs_to case is this generator's own. Because #generate! only ever
+    # adds relations (never rewrites the list), a relation whose target table
+    # was dropped would otherwise survive every regeneration, and a belongs_to
+    # with no target table crashes dependency resolution at extraction time.
+    # An `ignore: true` entry is kept regardless: those are user tombstones
+    # recording a decision ("this relation is deliberately not extracted"), and
+    # deleting one would invite the next regeneration to add the edge back.
+    #
+    # Like SchemaGenerator#tidy!, nothing is added or regenerated here: every
+    # surviving entry keeps its hand-edited `comment` / `ignore` /
+    # `replace_with` untouched. Returns a TidyResult describing the removals.
+    def tidy!
+      result = TidyResult.new
+      return result unless Dir.exist?(@output_dir)
+
+      # Views are not generated (see DbIntrospector::Base#table_names), so a
+      # config naming one is stale by the same definition as a dropped table.
+      existing_tables = @introspector.table_names.to_set
+
+      Dir[File.join(@output_dir, "*.json")].sort.each do |path|
+        existing = TableConfig.from(JSON.parse(File.read(path)))
+
+        unless existing_tables.include?(existing.name)
+          File.delete(path)
+          result.add_removed_table(existing.name)
+          next
+        end
+
+        changed = false
+        changed |= remove_stale_columns(existing, result)
+        changed |= remove_dangling_belongs_tos(existing, existing_tables, result)
+        File.write(path, JSON.pretty_generate(existing.to_hash) + "\n") if changed
+      end
+
+      result
+    end
+
+    private def read_config(path)
+      return nil unless File.exist?(path)
+
+      TableConfig.from(JSON.parse(File.read(path)))
+    end
+
+    # The config for one table, in the same three shapes SchemaGenerator emits
+    # — with one more case it has to handle: a table with no primary key at
+    # all. ActiveRecord always reports one (a model without it cannot be
+    # queried), but a database is free to have none.
+    private def build_table(table_name, existing)
+      introspected_primary_key = @introspector.primary_key(table_name)
+      primary_key = declared_primary_key(introspected_primary_key, existing)
+      belongs_tos = merged_belongs_tos(table_name, existing)
+      column_names = @introspector.columns(table_name).map(&:name)
+
+      # A composite primary key is not supported yet. The config file is still
+      # generated — with `primary_key` omitted, `ignore: true` and a `type`
+      # marking it unsupported — so it can serve as a signpost for adding
+      # support later, and so a user can wire it up by hand meanwhile.
+      if primary_key.nil? && introspected_primary_key.is_a?(Array)
+        TableConfig.from_symbol_keys(
+          name: table_name,
+          type: TableConfig::UNSUPPORTED_COMPOSITE_PRIMARY_KEY,
+          ignore: true,
+          comment: "exwiw does not support composite primary keys " \
+                   "(#{introspected_primary_key.join(', ')}); data extraction is skipped.",
+          belongs_tos: belongs_tos,
+          columns: column_names.map { |name| { name: name } },
+        )
+      elsif primary_key.nil?
+        # exwiw addresses rows by primary key — it is what an extraction query
+        # filters and joins on — so a table without one cannot be extracted as
+        # it stands. Emitted with `ignore: true` rather than skipped entirely so
+        # the table is visible in the config (and in schema:check) instead of
+        # silently missing, and so opting it in is an edit rather than a
+        # discovery.
+        TableConfig.from_symbol_keys(
+          name: table_name,
+          ignore: true,
+          comment: "This table has no primary key, which exwiw needs to identify and join rows; " \
+                   "data extraction is skipped. To export it, set `primary_key` to a column that " \
+                   "uniquely identifies a row and remove `ignore`.",
+          belongs_tos: belongs_tos,
+          columns: column_names.map { |name| { name: name } },
+        )
+      else
+        TableConfig.from_symbol_keys(
+          name: table_name,
+          primary_key: primary_key,
+          belongs_tos: belongs_tos,
+          columns: build_columns(table_name, primary_key, belongs_tos),
+        )
+      end
+    end
+
+    # The primary key to build a table's config from: the one the database
+    # reports, or — when it reports none exwiw can use — the one the config on
+    # disk declares.
+    #
+    # The fallback is what makes the two signpost shapes below actionable. Both
+    # tell the user to name a primary key by hand ("set `primary_key` to a
+    # column that uniquely identifies a row and remove `ignore`"), and a table
+    # can be perfectly extractable that way — a natural key with a unique index
+    # but no PK constraint, or one column of a composite key that is unique on
+    # its own. Without this, following those instructions would not survive the
+    # next run: `TableConfig#merge` takes `primary_key` from the generated side,
+    # which is still nil because the database is unchanged, so the hand-set key
+    # would be dropped and the table left with nothing to join or filter on —
+    # silently, since the regenerated config is also what `schema check`
+    # compares against.
+    #
+    # A declared key also selects the ordinary table shape, so the `type` /
+    # `comment` signposts are not re-imposed on a table the user has since wired
+    # up (`ignore` is receiver-owned in the merge and already stays removed).
+    private def declared_primary_key(introspected_primary_key, existing)
+      return introspected_primary_key if introspected_primary_key.is_a?(String)
+
+      declared = existing&.primary_key
+      declared.is_a?(String) ? declared : nil
+    end
+
+    # The belongs_tos to generate for a table: everything the existing config
+    # already declares, verbatim and in its on-disk order, plus the
+    # foreign-key-derived relations it does not have yet, appended in sorted
+    # order.
+    #
+    # This union is the one place this generator deliberately departs from the
+    # ActiveRecord one, and it exists because a foreign-key constraint is
+    # strictly weaker evidence than an application model. Plenty of schemas
+    # express a relation only in application code — no constraint backs it —
+    # and the belongs_tos in an existing config are frequently hand-written for
+    # exactly that reason. They are load-bearing: a belongs_to is the path
+    # extraction follows to reach a table, so dropping one silently narrows the
+    # dump. TableConfig#merge rebuilds `belongs_tos` from the generated side,
+    # which is safe when that side saw the models and knows the full set, but
+    # would delete every unbacked relation here.
+    #
+    # So introspection only ever *adds* an edge. Removing one is tidy's job,
+    # where it is driven by the target table actually being gone rather than by
+    # the absence of a constraint (see #tidy!).
+    # Returns plain hashes rather than BelongsTo objects, because that is what
+    # TableConfig.from_symbol_keys consumes (it round-trips the whole table
+    # through JSON) — and because a hash of the existing entry carries its
+    # user-owned `comment` / `ignore` / `ignore_type` / `references` along
+    # without this method having to know they exist.
+    private def merged_belongs_tos(table_name, existing)
+      declared = (existing&.belongs_tos || []).map(&:to_hash)
+      # Identity here is the physical join — target table plus foreign-key
+      # column — rather than BelongsTo#identity, which also distinguishes the
+      # polymorphic type value. A hand-written polymorphic relation already
+      # covers its foreign-key column, so re-adding the bare constraint edge
+      # would emit a second belongs_to joining on the same column.
+      declared_keys = declared.map { |entry| [entry["table_name"], entry["foreign_key"]] }.to_set
+
+      discovered = @introspector.foreign_keys(table_name)
+        .reject { |entry| declared_keys.include?([entry[:table_name], entry[:foreign_key]]) }
+        .map { |entry| { "table_name" => entry[:table_name], "foreign_key" => entry[:foreign_key] } }
+
+      declared + discovered
+    end
+
+    # The `columns` entries for a table: just the name, or — in safe mode — also
+    # a default mask and the `needs_mask_decision` flag, exactly as
+    # SchemaGenerator#build_columns does it. The primary key and the foreign
+    # keys/types the belongs_tos join on are flagged but never masked, since
+    # masking them would break the joins.
+    #
+    # The structural set is computed from the *merged* belongs_tos, so a
+    # relation that exists only in the config — with no constraint behind it —
+    # protects its foreign-key column from a default mask just as a discovered
+    # one does. An `ignore: true` relation counts too: the column is still a
+    # foreign key, and the tombstone says nothing about masking it.
+    private def build_columns(table_name, primary_key, belongs_tos)
+      columns = @introspector.columns(table_name)
+      return columns.map { |column| { name: column.name } } unless @safe_new_columns
+
+      structural = belongs_tos.flat_map { |bt| [bt["foreign_key"], bt["foreign_type"]] }.compact.to_set
+      structural << primary_key
+      unique = @introspector.unique_column_names(table_name)
+
+      columns.map do |column|
+        entry = { name: column.name, needs_mask_decision: true }
+        next entry if structural.include?(column.name)
+
+        mask = DefaultMask.for(
+          name: column.name,
+          type: column.type,
+          limit: column.limit,
+          primary_key: primary_key,
+          array: column.array,
+          unique: unique.nil? || unique.include?(column.name),
+          column_default: column.default,
+        )
+        mask.nil? ? entry : entry.merge(replace_with: mask)
+      end
+    end
+
+    private def remove_stale_columns(existing, result)
+      valid_column_names = @introspector.columns(existing.name).map(&:name).to_set
+      stale = existing.columns.reject { |column| valid_column_names.include?(column.name) }
+      return false if stale.empty?
+
+      existing.columns = existing.columns.select { |column| valid_column_names.include?(column.name) }
+      stale.each { |column| result.add_removed_column(existing.name, column.name) }
+      true
+    end
+
+    private def remove_dangling_belongs_tos(existing, existing_tables, result)
+      dangling = existing.belongs_tos.reject do |belongs_to|
+        # An ignored relation is a user tombstone and is kept whatever its
+        # target is; one with no target at all is already inert (it records a
+        # relation exwiw cannot resolve) and is left alone as well.
+        belongs_to.ignore || belongs_to.table_name.nil? || existing_tables.include?(belongs_to.table_name)
+      end
+      return false if dangling.empty?
+
+      existing.belongs_tos = existing.belongs_tos - dangling
+      dangling.each { |belongs_to| result.add_removed_belongs_to(existing.name, belongs_to.table_name) }
+      true
+    end
+  end
+end

data/lib/exwiw/default_mask.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+require "date"
+
+module Exwiw
+  # The `replace_with` value safe mode attaches to a newly discovered column.
+  #
+  # Only types a constant can safely stand in for are masked: a default the
+  # column cannot hold would fail the restore the dump feeds, which is worse
+  # than exporting the column while its `needs_mask_decision` flag keeps the
+  # change from being merged.
+  module DefaultMask
+    FIXED_DATE = "2000-01-01"
+    FIXED_TIME = "2000-01-01 00:00:00"
+    EMPTY_JSON = "{}"
+    JSON_TYPES = %i[json jsonb].freeze
+
+    # Skip the mask when the column is too short to hold `masked-<primary key>`
+    # for a realistic key; the rendered length is only known per row at dump time.
+    MIN_TEXT_LIMIT = 20
+    MIN_EMAIL_LIMIT = 40
+
+    module_function
+
+    # The default mask for a column, or nil when no safe constant fits it.
+    # `primary_key` is what keeps a text mask unique per row, so without one text
+    # is left unmasked too. Under a unique index (`unique`) only a mask that
+    # varies per row is allowed, or every row would collide on restore.
+    def for(name:, type:, limit:, primary_key:, array: false, unique: false, column_default: nil)
+      return nil if array
+
+      mask =
+        case type
+        when :string, :text then text_mask(name, limit, primary_key)
+        else constant_mask(type, column_default)
+        end
+
+      return nil if mask.nil?
+      return nil if unique && !varies_per_row?(mask, primary_key)
+
+      mask
+    end
+
+    # The column's own default wins over the per-type constant: it is a value the
+    # column provably holds and the one the application treats as neutral, so
+    # masking a `default: true` flag does not turn the feature off for every row.
+    # A default the database computes (`now()`) arrives as nil and falls through.
+    def constant_mask(type, column_default)
+      from_default = default_value(type, column_default)
+      return from_default unless from_default.nil?
+
+      case type
+      when :integer, :decimal, :float then 0
+      when :boolean then false
+      when :date then FIXED_DATE
+      when :datetime, :timestamp, :time then FIXED_TIME
+      when :json, :jsonb then EMPTY_JSON
+      end
+    end
+
+    # The default as a mask value, or nil when it cannot be one. A JSON column's
+    # default is serialized as JSON whatever Ruby class it arrives as, so a string
+    # default keeps its quoting. Anything whose JSON contains an object is
+    # rejected rather than mis-parsed, so an array of objects falls back to the
+    # empty-JSON constant too.
+    def default_value(type, value)
+      return nil if value.nil?
+
+      mask = JSON_TYPES.include?(type) ? value.to_json : scalar_default(value)
+      return nil if mask.is_a?(String) && mask.match?(MaskValue::PLACEHOLDER)
+
+      mask
+    end
+
+    def scalar_default(value)
+      case value
+      when nil then nil
+      when true, false, Integer, Float, String then value
+      when Numeric then value.to_f
+      when Time, DateTime then value.strftime("%Y-%m-%d %H:%M:%S")
+      when Date then value.strftime("%Y-%m-%d")
+      when Hash, Array then value.to_json
+      end
+    end
+
+    def text_mask(name, limit, primary_key)
+      return nil if primary_key.nil?
+
+      if name.to_s.include?("mail")
+        return nil if limit && limit < MIN_EMAIL_LIMIT
+
+        "masked-{#{primary_key}}@example.com"
+      else
+        return nil if limit && limit < MIN_TEXT_LIMIT
+
+        "masked-{#{primary_key}}"
+      end
+    end
+
+    def varies_per_row?(mask, primary_key)
+      mask.is_a?(String) && mask.include?("{#{primary_key}}")
+    end
+  end
+end

data/lib/exwiw/mask_value.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Exwiw
+  # Serdes type for a `replace_with` value: a template String with `{column}`
+  # placeholders, rendered as text, or a non-String JSON scalar used verbatim so
+  # a column that is not text keeps its type (in MongoDB, its BSON type).
+  class MaskValue < Serdes::TypeBase
+    PERMITTED = [String, Integer, Float].freeze
+
+    # What counts as a `{column}` placeholder: an empty brace pair names no
+    # column, which is what lets `{}` be an empty-JSON mask. One definition,
+    # shared by the adapters that parse a template and the generator that must
+    # avoid emitting one — keep it capture-free, since the splitter's `scan`
+    # reads whole matches.
+    PLACEHOLDER = /\{[^{}]+\}/
+
+    def permit?(value)
+      value == true || value == false || PERMITTED.any? { |type| value.is_a?(type) }
+    end
+
+    # Whether `value` is used verbatim rather than rendered as a template.
+    def self.scalar?(value)
+      !value.is_a?(String) && !value.nil?
+    end
+
+    def to_s
+      "mask_value"
+    end
+  end
+end

data/lib/exwiw/mongodb_collection_config.rb

@@ -96,7 +96,8 @@ module Exwiw
     #   bulk_insert_chunk_size, query_timeout_ms, and each field's
     #   `replace_with` / `replace_with_fake_data` masking rule.
     # - generated fields drive the field list (so added/removed fields track the
-    #   model), but a matching receiver field wins to retain its masking.
+    #   model), but for a field the receiver already has, its masking decision
+    #   wins outright — including the parts of it left unset.
     def merge(passed)
       return passed if passed.to_hash == to_hash
 
@@ -133,17 +134,28 @@ module Exwiw
         end
 
         # Take each field from the freshly generated config (so structural facts
-        # like `mongoid_field_name` track the model) but carry over the user's
-        # hand-edited `replace_with`/`comment`/`ignore` when the field still exists.
+        # like `mongoid_field_name` track the model), but once the field already
+        # exists in the receiver, EVERY masking-decision attribute comes from the
+        # receiver — even when it is unset.
+        #
+        # "Even when unset" is the whole point: what these attributes record is a
+        # human's decision about the field, and an absent value is a decision too
+        # ("export this raw", "the flag is resolved"). Keeping the generated value
+        # where the receiver has none was equivalent to "receiver wins" only while
+        # the generator emitted no masks at all; under safe mode
+        # (MongoidSchemaGenerator's `safe_new_columns`) it would put a default mask
+        # back on a field somebody had deliberately unmasked, and silently mask it
+        # again on every regeneration.
         receiver_field_by_name = fields.each_with_object({}) { |f, h| h[f.name] = f }
         merged.fields = passed.fields.map do |pf|
           receiver = receiver_field_by_name[pf.name]
-          if receiver
-            pf.replace_with = receiver.replace_with if receiver.replace_with
-            pf.replace_with_fake_data = receiver.replace_with_fake_data if receiver.replace_with_fake_data
-            pf.comment = receiver.comment if receiver.comment
-            pf.ignore = receiver.ignore unless receiver.ignore.nil?
-          end
+          next pf unless receiver
+
+          pf.replace_with = receiver.replace_with
+          pf.replace_with_fake_data = receiver.replace_with_fake_data
+          pf.comment = receiver.comment
+          pf.ignore = receiver.ignore
+          pf.needs_mask_decision = receiver.needs_mask_decision
           pf
         end
       end
@@ -162,7 +174,7 @@ module Exwiw
         fake_data = field.replace_with_fake_data
         next unless fake_data
 
-        if field.replace_with
+        unless field.replace_with.nil?
           raise ArgumentError,
                 "MongodbCollectionConfig '#{name}' field '#{field.name}': replace_with and " \
                 "replace_with_fake_data cannot be combined; use only one."

data/lib/exwiw/mongodb_field.rb

@@ -5,7 +5,7 @@ module Exwiw
     include Serdes
 
     attribute :name, String
-    attribute :replace_with, optional(String), skip_serializing_if_nil: true
+    attribute :replace_with, Serdes::OptionalType.new(MaskValue.new), skip_serializing_if_nil: true
     # Ruby-process-side masking: replace the value with a deterministic fake
     # value derived from a seed field (see FakeData / RowTransformer). Unlike the
     # SQL adapters — where replace_with runs in the database and fake data needs
@@ -23,6 +23,10 @@ module Exwiw
     # once the config is loaded (see MongodbCollectionConfig#reject_ignored_members!).
     attribute :comment, optional(String), skip_serializing_if_nil: true
     attribute :ignore, Serdes::OptionalType.new(Serdes::ConcreteType.new(Boolean)), skip_serializing_if_nil: true
+    # See TableColumn#needs_mask_decision.
+    attribute :needs_mask_decision,
+              Serdes::OptionalType.new(Serdes::ConcreteType.new(Boolean)),
+              skip_serializing_if_nil: true
 
     def self.from_symbol_keys(hash)
       from(hash.transform_keys(&:to_s))