exwiw 0.9.21 → 0.9.22

data/lib/exwiw/mongoid_schema_generator.rb

@@ -31,21 +31,68 @@ module Exwiw
       end
     end
 
+    # Every generated collection is keyed by the MongoDB document id. Named so
+    # the value the configs carry and the value `DefaultMask` interpolates into a
+    # text mask (`masked-{_id}`) provably come from one place.
+    PRIMARY_KEY = "_id"
+
+    # Mongoid field types (`Model.fields[name].type`, a Ruby class) mapped to the
+    # type symbols `DefaultMask.for` understands, so safe mode can reuse the very
+    # same default masks the ActiveRecord generator emits.
+    #
+    # Keyed by class NAME rather than by the class itself: this constant is
+    # evaluated when exwiw is loaded, which happens in processes that never load
+    # Mongoid (the CLI), so naming `Mongoid::Boolean` here would raise. A name
+    # comparison also sidesteps `DateTime < Date` — an ancestry test would have to
+    # order its branches, while exact names cannot be confused for one another.
+    #
+    # Everything absent from this map (Hash, Array, Object — which is also what a
+    # typeless/dynamic field reports — BSON types, Mongoid::StringifiedSymbol, ...)
+    # deliberately gets NO default mask: a constant that does not fit the field
+    # would be restored as a value the application cannot read, which is worse
+    # than exporting the field while its `needs_mask_decision` flag keeps the
+    # change from being merged.
+    MASKABLE_FIELD_TYPES = {
+      "String" => :string,
+      "Integer" => :integer,
+      "Float" => :float,
+      "BigDecimal" => :decimal,
+      "Mongoid::Boolean" => :boolean,
+      "Time" => :datetime,
+      "DateTime" => :datetime,
+      "ActiveSupport::TimeWithZone" => :datetime,
+      "Date" => :date,
+    }.freeze
+
     # `skip_unsupported`: when true, the generator does not abort on a construct
     # it cannot represent. It skips an unresolvable `belongs_to` (keeping the
     # foreign-key field) and emits an unrepresentable embedded collection as an
     # `ignore: true` top-level config annotated with a `comment`, warning to
     # stderr in both cases. Off by default, so the historical fail-loud behavior
     # is unchanged unless a caller opts in.
-    def self.from_rails_application(output_dir:, skip_unsupported: false)
+    #
+    # `safe_new_columns` mirrors `SchemaGenerator`; see #initialize.
+    def self.from_rails_application(output_dir:, skip_unsupported: false, safe_new_columns: true)
       Rails.application.eager_load!
-      new(models: ::Mongoid.models, output_dir: output_dir, skip_unsupported: skip_unsupported)
+      new(
+        models: ::Mongoid.models,
+        output_dir: output_dir,
+        skip_unsupported: skip_unsupported,
+        safe_new_columns: safe_new_columns,
+      )
     end
 
-    def initialize(models:, output_dir:, skip_unsupported: false)
+    # `safe_new_columns` (the default) emits every field masked — as far as its
+    # Mongoid type allows, see MASKABLE_FIELD_TYPES and DefaultMask — and flagged
+    # `needs_mask_decision: true`. MongodbCollectionConfig#merge lets an existing
+    # entry win, so in practice only fields a model change has just added keep
+    # that treatment. Pass false to bootstrap a config, where every field is new
+    # and flagging all of them at once is noise.
+    def initialize(models:, output_dir:, skip_unsupported: false, safe_new_columns: true)
       @models = models
       @output_dir = output_dir
       @skip_unsupported = skip_unsupported
+      @safe_new_columns = safe_new_columns
     end
 
     def generate!
@@ -68,10 +115,118 @@ module Exwiw
     # on a construct the user has deliberately ignored. Empty (the default) when
     # called directly without an output dir, in which case nothing is honored.
     def build_collections(existing_by_name = {})
-      models = expand_with_descendants(concrete_models)
-      models
-        .group_by { |model| model.collection_name.to_s }
-        .map { |collection_name, group| build_collection_for(collection_name, group, existing_by_name[collection_name]) }
+      models_by_collection_name.map do |collection_name, group|
+        build_collection_for(collection_name, group, existing_by_name[collection_name])
+      end
+    end
+
+    # Reconcile the config files already on disk against the models, deleting the
+    # config of a collection no model stores into any more. This is the
+    # counterpart of `generate!`, which adds and updates config files but can
+    # never delete one: a collection whose model was removed leaves a config that
+    # would otherwise be dumped forever.
+    #
+    # Fields are deliberately NOT touched: `generate!`'s #merge already drives the
+    # field list from the model, so a field the model lost is dropped there.
+    #
+    # Unlike `SchemaGenerator#tidy!`, the source of truth is the model set, not a
+    # live connection. MongoDB has no schema to read: a collection exists only
+    # once a document is written to it, so "still in the database" is not a
+    # question that can be answered before a dump, and introspection here is
+    # class-level (see this class's preamble) and needs no connection at all. The
+    # name set therefore comes from exactly the grouping `generate!` writes files
+    # from (`models_by_collection_name`, descendants expanded and embedded models
+    # included), so the two can never disagree about which files are live.
+    #
+    # A config declaring `embedded_in` is never deleted, whatever its name — see
+    # #hand_written_embedded_config?.
+    #
+    # Returns a `SchemaGenerator::TidyResult` — reused rather than duplicated,
+    # since a removal report is the same shape whatever the ORM — describing the
+    # removals so callers (e.g. the rake task) can report them. Its
+    # `removed_columns` stays empty, for the reason above.
+    def tidy!
+      result = SchemaGenerator::TidyResult.new
+      return result unless @output_dir && File.directory?(@output_dir)
+
+      live_names = models_by_collection_name.keys
+
+      Dir[File.join(@output_dir, "*.json")].sort.each do |path|
+        config = read_raw_config(path)
+
+        # A file that does not parse says nothing about what it describes: its
+        # basename would stand in for the collection name, and for a hand-written
+        # embedded config that basename matches no collection by construction —
+        # so the guard below would be bypassed for exactly the files it exists to
+        # protect, and a stray merge-conflict marker would be enough to delete a
+        # collection's masking rules. Broken JSON already fails loudly wherever
+        # the config is loaded, so leave it in place and say so.
+        if config.nil?
+          warn("exwiw: skipping '#{path}' while tidying: it is not valid JSON, so what it describes cannot be determined.")
+          next
+        end
+
+        name = declared_name(config, path)
+        next if live_names.include?(name)
+        next if hand_written_embedded_config?(config)
+
+        File.delete(path)
+        result.add_removed_table(name)
+      end
+
+      result
+    end
+
+    # Whether a config on disk describes an embedded collection this generator
+    # could not have written, and whose liveness it therefore cannot judge.
+    #
+    # `generate!` writes one config per *collection name*, so when two embedded
+    # classes share a collection name — which they routinely do, since an embedded
+    # class's collection name is derived from the class name alone (`Address`
+    # embedded under two different parents is `addresses` both times) — only one
+    # of them is emitted. The other can only be expressed by hand, under a name
+    # that says where it lives (`orders_deliveries_addresses`) rather than naming
+    # a collection. Such a name matches no model by construction, so "no model
+    # stores into it" is not evidence that it is dead: it is evidence that the
+    # config exists precisely because generation cannot reach it.
+    #
+    # Deleting one is silent and expensive — an embedded config carries the
+    # masking rules applied to those subdocuments, so removing it exports them
+    # raw, and nothing else in the pipeline notices. So an `embedded_in` config
+    # is kept whatever its name; a genuinely dead one is removed by hand, which is
+    # also how it arrived.
+    private def hand_written_embedded_config?(config)
+      config.is_a?(Hash) && !config["embedded_in"].nil?
+    end
+
+    # The collection -> models grouping both `build_collections` and `tidy!` work
+    # from: every model that stores into a collection, keyed by that collection's
+    # name. Models are grouped by `collection_name` so an STI hierarchy collapses
+    # into one entry (see `build_collections`), and `expand_with_descendants`
+    # supplies the subclasses Mongoid does not register.
+    #
+    # Shared so the two operations can never disagree about which collections
+    # exist: `tidy!` deletes exactly the config files `generate!` would not write.
+    private def models_by_collection_name
+      expand_with_descendants(concrete_models).group_by { |model| model.collection_name.to_s }
+    end
+
+    # A config file on disk as plain JSON, rather than through
+    # `MongodbCollectionConfig.from` on purpose: a stale file is exactly the one
+    # that may no longer satisfy the current validations (an unknown key, a
+    # belongs_to whose target is gone), and tidy has to be able to delete such a
+    # file rather than abort on it. Unparseable JSON reads as nil, which the
+    # caller treats as "cannot judge this file" rather than as an empty config.
+    private def read_raw_config(path)
+      JSON.parse(File.read(path))
+    rescue JSON::ParserError
+      nil
+    end
+
+    # The collection a config declares itself to be. Without a readable `name` it
+    # falls back to the file's basename, which is the name `write_files` gives it.
+    private def declared_name(config, path)
+      (config.is_a?(Hash) && config["name"]) || File.basename(path, ".json")
     end
 
     # Loads the configs already on disk so the generator can honor an explicit
@@ -123,7 +278,7 @@ module Exwiw
 
       attrs = {
         name: collection_name,
-        primary_key: "_id",
+        primary_key: PRIMARY_KEY,
         fields: aggregate_fields(ordered),
       }
 
@@ -196,10 +351,13 @@ module Exwiw
     # order. A subclass's `fields` already includes everything it inherits, so
     # the base's fields lead and each subclass appends only its own.
     private def aggregate_fields(models)
+      structural = structural_field_names(models)
+      unique = unique_field_names(models)
+
       seen = {}
       models.each_with_object([]) do |model, fields|
         accessor_by_storage = aliased_field_accessors(model)
-        model.fields.keys.each do |name|
+        model.fields.each do |name, definition|
           next if seen[name]
 
           seen[name] = true
@@ -208,11 +366,131 @@ module Exwiw
           # Ruby accessor so the short key is not cryptic in the config.
           accessor = accessor_by_storage[name]
           field[:mongoid_field_name] = accessor if accessor
-          fields << field
+          fields << field.merge(safe_mode_attributes(name, definition, structural, unique))
         end
       end
     end
 
+    # The safe-mode additions to a field entry: the `needs_mask_decision` flag,
+    # plus a default `replace_with` where a safe one exists. Empty when safe mode
+    # is off, so the emitted config is byte-identical to the historical output.
+    #
+    # A structural field is flagged but never masked: masking it would break the
+    # very lookups the dump is assembled from (see #structural_field_names).
+    private def safe_mode_attributes(name, definition, structural, unique)
+      return {} unless @safe_new_columns
+
+      attrs = { needs_mask_decision: true }
+      return attrs if structural.include?(name)
+
+      mask = default_mask_for(name, definition, unique.include?(name))
+      mask.nil? ? attrs : attrs.merge(replace_with: mask)
+    end
+
+    # The default mask for one field, or nil when no safe constant fits its
+    # Mongoid type (see MASKABLE_FIELD_TYPES).
+    #
+    # `limit: nil` because MongoDB stores no per-field length: the length checks
+    # DefaultMask applies to a `varchar(n)` have nothing to read here, and any
+    # string the mask renders is storable. `unique` comes from the model's index
+    # declarations, so a constant is never emitted for a field a unique index
+    # would then collide on across every restored document.
+    #
+    # NOTE the mask a date/time field gets is DefaultMask's fixed *string*
+    # constant, which replaces a BSON Date with a String and so changes the
+    # field's BSON type. That is deliberate at this stage: the value is only a
+    # proposal, and it rides on `needs_mask_decision` precisely so a human either
+    # keeps it knowingly, replaces it, or drops it before the config is merged.
+    private def default_mask_for(name, definition, unique)
+      type = MASKABLE_FIELD_TYPES[mongoid_type_name(definition)]
+      return nil if type.nil?
+
+      DefaultMask.for(
+        name: name,
+        type: type,
+        limit: nil,
+        primary_key: PRIMARY_KEY,
+        unique: unique,
+        column_default: field_default(definition),
+      )
+    end
+
+    # `Model.fields[name].type` is a Ruby class; an anonymous one (or a field
+    # object that does not expose a type at all) has no name to look up, which
+    # MASKABLE_FIELD_TYPES answers with "no default mask".
+    private def mongoid_type_name(definition)
+      type = definition.type if definition.respond_to?(:type)
+      type.name if type.is_a?(Module)
+    end
+
+    # The field's declared `default:`, handed to DefaultMask so a field with a
+    # default of its own is masked with that value rather than the per-type
+    # constant (see DefaultMask.constant_mask). Mongoid also accepts a *lambda*
+    # default (`default: -> { Time.now }`), which is not a constant at all;
+    # DefaultMask's scalar_default answers such a value with nil, so it falls
+    # through to the per-type constant on its own.
+    private def field_default(definition)
+      definition.default_val if definition.respond_to?(:default_val)
+    end
+
+    # The fields safe mode flags but must never mask, because the dump is
+    # assembled by looking documents up through them:
+    #
+    # - the primary key `_id`, which every belongs_to and every `--ids` filter
+    #   resolves against (and which the text masks interpolate to stay unique),
+    # - the STI discriminator (`Model.discriminator_key`, `_type` by default),
+    #   which is what tells one class's documents from another's inside the
+    #   shared collection, and
+    # - every `belongs_to` foreign key declared by any model of this collection.
+    #
+    # The foreign keys are taken from the MODELS, not from the emitted
+    # `belongs_tos`, so the ones the generator deliberately drops are covered
+    # too: a polymorphic belongs_to (excluded because it has no single target
+    # collection) and a referenced belongs_to on an embedded document (excluded
+    # because an embedded config carries none) still store a reference some other
+    # collection's documents are found by, and masking it would silently rewrite
+    # that reference. A polymorphic relation's type field (`reviewable_type`)
+    # is exempt for the same reason, mirroring how the ActiveRecord generator
+    # treats a belongs_to's `foreign_type`.
+    private def structural_field_names(models)
+      names = [PRIMARY_KEY]
+
+      models.each do |model|
+        names << model.discriminator_key.to_s if model.respond_to?(:discriminator_key)
+        model.relations.each_value do |assoc|
+          next unless assoc.is_a?(::Mongoid::Association::Referenced::BelongsTo)
+
+          names << assoc.foreign_key.to_s
+          names << assoc.inverse_type.to_s if assoc.polymorphic? && assoc.inverse_type
+        end
+      end
+
+      names.uniq
+    end
+
+    # The fields covered by a unique index declared on any model of this
+    # collection (`index({ email: 1 }, unique: true)`), so DefaultMask withholds
+    # any mask that does not vary per document — a constant would collapse every
+    # document onto one value and break the restore with a duplicate key. Every
+    # field of a compound unique index counts, mirroring the ActiveRecord
+    # generator's "any column of a unique index".
+    #
+    # Index declarations are class-level Mongoid metadata (they are what
+    # `db/mongoid.rake`'s create_indexes would build), so this needs no
+    # connection — unlike the ActiveRecord generator, which reads the live
+    # database and has to cope with that failing. An index created out of band
+    # and never declared on the model is therefore invisible here; declaring it
+    # is what makes exwiw (and Mongoid itself) aware of it.
+    private def unique_field_names(models)
+      models.flat_map do |model|
+        next [] unless model.respond_to?(:index_specifications)
+
+        model.index_specifications
+             .select { |spec| spec.options[:unique] }
+             .flat_map { |spec| spec.key.keys.map(&:to_s) }
+      end.uniq
+    end
+
     # Maps a stored document key -> its Mongoid Ruby accessor, but ONLY for
     # genuine `field ..., as:` renames. `Model.aliased_fields` also contains the
     # built-in `id => _id` and one entry per association (e.g. `shop => shop_id`,
@@ -231,7 +509,7 @@ module Exwiw
     end
 
     private def aggregate_belongs_tos(models, existing = nil)
-      ignored_by_fk = ignored_belongs_tos_by_foreign_key(existing)
+      ignored = ignored_belongs_tos(existing)
 
       belongs_to_assocs = models.flat_map do |model|
         model.relations.values.select do |assoc|
@@ -248,20 +526,68 @@ module Exwiw
       # same belongs_to twice, so uniq them.
       belongs_to_assocs
         .reject(&:polymorphic?)
-        .filter_map { |assoc| belongs_to_for(assoc, ignored_by_fk) }
+        .filter_map { |assoc| belongs_to_for(assoc, ignored) }
         .uniq
     end
 
-    # Maps foreign_key -> the on-disk `ignore: true` belongs_to entry, so a
-    # relation the user has explicitly ignored is preserved verbatim instead of
-    # re-resolved (which, for a stale relation whose target class is gone, would
-    # otherwise abort the run).
-    private def ignored_belongs_tos_by_foreign_key(existing)
-      return {} unless existing
+    # The on-disk `ignore: true` belongs_to entries, so a relation the user has
+    # explicitly ignored is preserved verbatim instead of re-resolved (which, for
+    # a stale relation whose target class is gone, would otherwise abort the run).
+    #
+    # Kept as a list rather than indexed by foreign key: a foreign key does not
+    # identify a relation on its own (see #ignored_belongs_to_for).
+    private def ignored_belongs_tos(existing)
+      return [] unless existing
 
-      existing.belongs_tos.select(&:ignore).each_with_object({}) do |bt, acc|
-        acc[bt.foreign_key] = bt
-      end
+      existing.belongs_tos.select(&:ignore)
+    end
+
+    # The on-disk ignored entry that stands for `assoc`, or nil when none does.
+    #
+    # `target_collection` is the association's resolved target, or nil when it
+    # could not be resolved, and that distinction is what the match is keyed on:
+    #
+    # - Resolved: the entry has to agree on BOTH the target collection and the
+    #   foreign key. Two belongs_tos of one collection can legitimately share a
+    #   foreign key — a relation to one collection plus a second relation to
+    #   another declared with `primary_key:`, scoping by the same stored value —
+    #   and matching on the foreign key alone let one ignored entry stand for both
+    #   of them. The second relation was then re-emitted as a copy of that entry,
+    #   `uniq` collapsed the two, and its edge silently vanished from the config.
+    #   An entry naming a different target must therefore not swallow this
+    #   relation; with none matching, the relation is generated normally.
+    # - Unresolved: there is no target to compare, which is precisely the case
+    #   this mechanism exists for (a stale relation whose class is gone, whose
+    #   entry usually carries no `table_name` either), so fall back to matching on
+    #   the foreign key alone.
+    #
+    # An entry with no `table_name` still matches a resolved relation on the
+    # foreign key alone, as a fallback after the exact match: omitting the target
+    # is allowed only on an ignored entry, and dropping the user's decision
+    # because they wrote the minimal form would resurrect an edge they had
+    # deliberately cut. Two foreign-key-sharing relations with one such entry
+    # between them remain genuinely indistinguishable — writing the `table_name`
+    # is what tells exwiw which of them is meant.
+    private def ignored_belongs_to_for(assoc, target_collection, ignored)
+      foreign_key = assoc.foreign_key
+      return ignored.find { |bt| bt.foreign_key == foreign_key } if target_collection.nil?
+
+      ignored.find { |bt| bt.foreign_key == foreign_key && bt.table_name == target_collection } ||
+        ignored.find { |bt| bt.foreign_key == foreign_key && bt.table_name.nil? }
+    end
+
+    # The collection a referenced belongs_to targets, as `[collection_name, nil]`
+    # — or `[nil, error]` when the association's target class no longer exists (a
+    # stale/legacy `belongs_to`, e.g. pointing at a model removed years ago), so
+    # `assoc.klass` raised. The error is carried rather than raised because
+    # resolution is attempted BEFORE the on-disk ignored entries are consulted (to
+    # match one against the right relation), and a relation the user has already
+    # triaged must not abort the run; `belongs_to_for` raises it only once no
+    # entry stands for the relation.
+    private def resolve_target_collection(assoc)
+      [assoc.klass.collection_name.to_s, nil]
+    rescue NameError, ::Mongoid::Errors::MongoidError => e
+      [nil, e]
     end
 
     # Resolves a referenced belongs_to to a `{ table_name, foreign_key }` pair
@@ -272,18 +598,29 @@ module Exwiw
     # its foreign-key column is still tracked as an ordinary field by
     # `aggregate_fields`, mirroring how polymorphic / HABTM relations are dropped.
     #
-    # `ignored_by_fk` carries the on-disk `ignore: true` belongs_to entries: when
-    # this relation's foreign key is among them, the user has explicitly ignored
-    # it, so preserve their entry verbatim (its `ignore_type` / `comment`) without
-    # resolving the — possibly gone — target. The relation is dropped from
-    # extraction at load (`#reject_ignored_members!`) while its FK column stays a
-    # field, and the run never aborts on a relation already triaged.
-    private def belongs_to_for(assoc, ignored_by_fk = {})
-      if (ignored = ignored_by_fk[assoc.foreign_key])
-        return preserve_ignored_belongs_to(ignored)
+    # `ignored` carries the on-disk `ignore: true` belongs_to entries: when one of
+    # them stands for this relation (see #ignored_belongs_to_for), the user has
+    # explicitly ignored it, so preserve their entry verbatim (its `ignore_type` /
+    # `comment`) rather than emitting a freshly derived one. The relation is
+    # dropped from extraction at load (`#reject_ignored_members!`) while its FK
+    # column stays a field, and the run never aborts on a relation already
+    # triaged: resolving the — possibly gone — target is attempted first, but its
+    # failure is only raised once no entry stands for the relation.
+    private def belongs_to_for(assoc, ignored = [])
+      target_collection, resolution_error = resolve_target_collection(assoc)
+
+      if (entry = ignored_belongs_to_for(assoc, target_collection, ignored))
+        return preserve_ignored_belongs_to(entry)
+      end
+
+      if resolution_error
+        raise resolution_error unless @skip_unsupported
+
+        warn("exwiw: skip_unsupported: skipping belongs_to ':#{assoc.name}' that could not be resolved (#{resolution_error.class}: #{resolution_error.message.lines.first&.strip}); its foreign key '#{assoc.foreign_key}' is still kept as a field.")
+        return nil
       end
 
-      result = { table_name: assoc.klass.collection_name.to_s, foreign_key: assoc.foreign_key }
+      result = { table_name: target_collection, foreign_key: assoc.foreign_key }
       # Mongoid's `belongs_to ..., primary_key: :uuid` makes the child's foreign
       # key reference that parent field rather than the parent's `_id`. Surface
       # it as `references` so MongodbAdapter constrains children by the right
@@ -293,11 +630,6 @@ module Exwiw
       reference_field = assoc.primary_key.to_s
       result[:references] = reference_field unless reference_field == "_id"
       result
-    rescue NameError, ::Mongoid::Errors::MongoidError => e
-      raise e unless @skip_unsupported
-
-      warn("exwiw: skip_unsupported: skipping belongs_to ':#{assoc.name}' that could not be resolved (#{e.class}: #{e.message.lines.first&.strip}); its foreign key '#{assoc.foreign_key}' is still kept as a field.")
-      nil
     end
 
     # Re-emits a user's on-disk ignored belongs_to as a symbol-keyed hash (the

data/lib/exwiw/schema_check.rb

@@ -9,7 +9,13 @@ module Exwiw
   # Reports how the committed schema config differs from what the application
   # would generate now, plus the columns still waiting for a masking decision.
   # Regenerating into a copy rather than over the config directory lets it run
-  # on a working tree it must not modify (CI). ActiveRecord only.
+  # on a working tree it must not modify (CI).
+  #
+  # The diff/report side is source-agnostic (it reads JSON config files); what
+  # varies per schema source is only how to regenerate, injected as
+  # `regenerator` — a callable that receives a directory pre-seeded with a copy
+  # of the committed config and rewrites it the way `generate!` + `tidy!`
+  # would. `models:` remains as the ActiveRecord shorthand for it.
   class SchemaCheck
     CATEGORIES = %w[
       added_tables added_columns removed_tables removed_columns changed_tables needs_mask_decision
@@ -20,9 +26,39 @@ module Exwiw
       new(models: ActiveRecord::Base.descendants, schema_dir: schema_dir)
     end
 
-    def initialize(models:, schema_dir:)
-      @models = models
+    # The Mongoid counterpart. `Mongoid.models` registers only the base class of
+    # an inheritance hierarchy, which is exactly what MongoidSchemaGenerator
+    # expects (it expands descendants itself), so the same source the generate
+    # task introspects is used here.
+    def self.from_mongoid_application(schema_dir:)
+      Rails.application.eager_load!
+      new(schema_dir: schema_dir, regenerator: mongoid_regenerator(::Mongoid.models))
+    end
+
+    # The regeneration step `from_mongoid_application` injects, exposed so a
+    # caller (and the specs) can run the check against an explicit model list
+    # without a Rails application — and so what they exercise is the very lambda
+    # production uses.
+    #
+    # `skip_unsupported` is deliberately left off: this runs as a CI gate, and a
+    # newly added construct exwiw cannot represent has to fail the task with the
+    # generator's actionable message rather than quietly become an `ignore: true`
+    # config that reports clean while the collection stops being dumped.
+    def self.mongoid_regenerator(models)
+      lambda do |tmp_dir|
+        # Explicit: safe mode is not optional here, whatever the library default is.
+        MongoidSchemaGenerator.new(models: models, output_dir: tmp_dir, safe_new_columns: true).generate!
+        MongoidSchemaGenerator.new(models: models, output_dir: tmp_dir).tidy!
+      end
+    end
+
+    def initialize(schema_dir:, models: nil, regenerator: nil)
+      if models.nil? && regenerator.nil?
+        raise ArgumentError, "SchemaCheck requires either models: or regenerator:"
+      end
+
       @schema_dir = schema_dir
+      @regenerator = regenerator || active_record_regenerator(models)
     end
 
     # The report as a plain Hash. Every list is sorted, so a given state always
@@ -47,9 +83,15 @@ module Exwiw
 
     private def regenerate_into(tmp_dir)
       FileUtils.cp_r(File.join(@schema_dir, "."), tmp_dir) if Dir.exist?(@schema_dir)
-      # Explicit: safe mode is not optional here, whatever the library default is.
-      SchemaGenerator.new(models: @models, output_dir: tmp_dir, safe_new_columns: true).generate!
-      SchemaGenerator.new(models: @models, output_dir: tmp_dir).tidy!
+      @regenerator.call(tmp_dir)
+    end
+
+    private def active_record_regenerator(models)
+      lambda do |tmp_dir|
+        # Explicit: safe mode is not optional here, whatever the library default is.
+        SchemaGenerator.new(models: models, output_dir: tmp_dir, safe_new_columns: true).generate!
+        SchemaGenerator.new(models: models, output_dir: tmp_dir).tidy!
+      end
     end
 
     # Every config file under `dir`, keyed by its path relative to `dir` so the

data/lib/exwiw/table_config.rb

@@ -170,9 +170,9 @@ module Exwiw
         merged_table.type = passed_table.type
         merged_table.comment = comment
         merged_table.filter = filter
-        merged_table.bulk_insert_chunk_size = passed_table.bulk_insert_chunk_size
         merged_table.ignore = ignore
         # User-owned, never regenerated: carry over from the existing config.
+        merged_table.bulk_insert_chunk_size = bulk_insert_chunk_size
         merged_table.scope_exempt = scope_exempt
         merged_table.scope_column = scope_column
         merged_table.reverse_scope = reverse_scope

data/lib/exwiw/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Exwiw
-  VERSION = "0.9.21"
+  VERSION = "0.9.22"
 end

data/lib/exwiw.rb

@@ -40,6 +40,10 @@ require_relative "exwiw/after_insert_hook"
 require_relative "exwiw/runner"
 require_relative "exwiw/explain_runner"
 require_relative "exwiw/schema_generator"
+require_relative "exwiw/db_introspector"
+require_relative "exwiw/db_introspector/mysql_introspector"
+require_relative "exwiw/db_introspector/postgresql_introspector"
+require_relative "exwiw/db_schema_generator"
 require_relative "exwiw/schema_check"
 require_relative "exwiw/mongoid_schema_generator"
 

data/lib/tasks/exwiw.rake

@@ -100,7 +100,44 @@ namespace :exwiw do
       Exwiw::MongoidSchemaGenerator.from_rails_application(
         output_dir: resolve_schema_dir.call,
         skip_unsupported: ENV["EXWIW_SKIP_UNSUPPORTED"] == "1",
+        # Same convention as `generate`: safe unless EXWIW_NEW_COLUMNS=plain.
+        safe_new_columns: safe_new_columns.call,
       ).generate!
     end
+
+    desc "Remove collections from the schema config that no longer exist in the Mongoid application"
+    task tidy_mongoid: :environment do
+      require "exwiw"
+
+      result = Exwiw::MongoidSchemaGenerator.from_rails_application(
+        output_dir: resolve_schema_dir.call,
+      ).tidy!
+
+      if result.empty?
+        puts "exwiw: schema config is already tidy; nothing to remove."
+      else
+        result.removed_tables.each do |name|
+          puts "exwiw: removed config for collection '#{name}' (no longer exists in the application)."
+        end
+      end
+    end
+
+    desc "Report how the committed schema config differs from the Mongoid application, without changing it"
+    task check_mongoid: :environment do
+      require "exwiw"
+
+      report = Exwiw::SchemaCheck.from_mongoid_application(schema_dir: resolve_schema_dir.call).run
+      json = JSON.pretty_generate(report)
+      puts json
+      # A file too, so a caller need not assume stdout carries only the JSON.
+      File.write(ENV["EXWIW_SCHEMA_CHECK_OUTPUT"], json + "\n") if ENV["EXWIW_SCHEMA_CHECK_OUTPUT"]
+
+      unless Exwiw::SchemaCheck.clean?(report)
+        $stderr.puts "exwiw: the schema config is out of date or has undecided masking; " \
+                     "run `rake exwiw:schema:generate_mongoid exwiw:schema:tidy_mongoid` " \
+                     "and resolve every `needs_mask_decision` field."
+        exit 1
+      end
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: exwiw
 version: !ruby/object:Gem::Version
-  version: 0.9.21
+  version: 0.9.22
 platform: ruby
 authors:
 - Shia
@@ -69,6 +69,10 @@ files:
 - lib/exwiw/belongs_to.rb
 - lib/exwiw/cli.rb
 - lib/exwiw/config_file.rb
+- lib/exwiw/db_introspector.rb
+- lib/exwiw/db_introspector/mysql_introspector.rb
+- lib/exwiw/db_introspector/postgresql_introspector.rb
+- lib/exwiw/db_schema_generator.rb
 - lib/exwiw/ddl_postprocessor.rb
 - lib/exwiw/default_mask.rb
 - lib/exwiw/determine_table_processing_order.rb