exwiw 0.9.20 → 0.9.22

data/lib/exwiw/schema_generator.rb

@@ -43,14 +43,20 @@ module Exwiw
     # belongs_to to it.
     ACTIVE_STORAGE_VARIANT_RECORDS_TABLE = "active_storage_variant_records"
 
-    def self.from_rails_application(output_dir:)
+    def self.from_rails_application(output_dir:, safe_new_columns: true)
       Rails.application.eager_load!
-      new(models: ActiveRecord::Base.descendants, output_dir: output_dir)
+      new(models: ActiveRecord::Base.descendants, output_dir: output_dir, safe_new_columns: safe_new_columns)
     end
 
-    def initialize(models:, output_dir:)
+    # `safe_new_columns` (the default) emits every column masked — as far as its
+    # type allows, see DefaultMask — and flagged `needs_mask_decision: true`.
+    # #merge lets an existing entry win, so in practice only columns a migration
+    # has just added keep that treatment. Pass false to bootstrap a config, where
+    # every column is new and flagging all of them at once is noise.
+    def initialize(models:, output_dir:, safe_new_columns: true)
       @models = models
       @output_dir = output_dir
+      @safe_new_columns = safe_new_columns
     end
 
     def generate!
@@ -239,11 +245,12 @@ module Exwiw
             columns: representative.column_names.map { |name| { name: name } },
           )
         else
+          belongs_tos = aggregate_belongs_tos(model_group)
           TableConfig.from_symbol_keys(
             name: table_name,
             primary_key: primary_key,
-            belongs_tos: aggregate_belongs_tos(model_group),
-            columns: representative.column_names.map { |name| { name: name } },
+            belongs_tos: belongs_tos,
+            columns: build_columns(representative, primary_key, belongs_tos, conn),
           )
         end
       end
@@ -251,6 +258,58 @@ module Exwiw
       tables_from_models + build_rails_managed_tables(conn)
     end
 
+    # The `columns` entries for a table: just the name, or — in safe mode — also
+    # a default mask and the `needs_mask_decision` flag. The primary key and the
+    # foreign keys/types the belongs_tos join on are flagged but never masked,
+    # since masking them would break the joins.
+    private def build_columns(representative, primary_key, belongs_tos, conn)
+      names = representative.column_names
+      return names.map { |name| { name: name } } unless @safe_new_columns
+
+      structural = belongs_tos.flat_map { |bt| [bt[:foreign_key], bt[:foreign_type]] }.compact.to_set
+      structural << primary_key if primary_key
+      columns_by_name = representative.columns.each_with_object({}) { |column, acc| acc[column.name] = column }
+      unique = unique_column_names(conn, representative.table_name)
+      defaults = representative.column_defaults
+
+      names.map do |name|
+        entry = { name: name, needs_mask_decision: true }
+        next entry if structural.include?(name)
+
+        column = columns_by_name[name]
+        mask = column && DefaultMask.for(
+          name: name,
+          type: column.type,
+          limit: column.limit,
+          primary_key: primary_key,
+          # `integer[]` reports `:integer`, so a scalar default would not fit.
+          array: column.respond_to?(:array?) && column.array?,
+          unique: unique.nil? || unique.include?(name),
+          column_default: defaults[name],
+        )
+        mask.nil? ? entry : entry.merge(replace_with: mask)
+      end
+    end
+
+    # Columns covered by a unique index, or nil when they could not be read —
+    # callers treat that as "assume every column is unique", since masking a
+    # unique column with a constant breaks the restore the dump feeds.
+    #
+    # `Array()` because an expression index reports a String (`lower((email)::text)`)
+    # rather than a list; wrapping keeps it from being walked character by
+    # character, and it simply never matches a column name.
+    private def unique_column_names(conn, table_name)
+      conn.indexes(table_name).select(&:unique).flat_map { |index| Array(index.columns) }.to_set
+    rescue StandardError => e
+      # Once per run: a systematic failure would otherwise repeat per table.
+      unless @warned_missing_indexes
+        @warned_missing_indexes = true
+        warn "exwiw: could not read the indexes of '#{table_name}' (#{e.class}); " \
+             "treating every column as unique-indexed so no constant mask is emitted."
+      end
+      nil
+    end
+
     private def concrete_models
       @models.reject(&:abstract_class?).select(&:table_exists?)
     end

data/lib/exwiw/table_column.rb

@@ -5,7 +5,7 @@ module Exwiw
     include Serdes
 
     attribute :name, String
-    attribute :replace_with, optional(String), skip_serializing_if_nil: true
+    attribute :replace_with, Serdes::OptionalType.new(MaskValue.new), skip_serializing_if_nil: true
     attribute :raw_sql, optional(String), skip_serializing_if_nil: true
     # Ruby-process-side masking modes, applied to the fetched rows by
     # RowTransformer (SQL adapters only) — unlike replace_with/raw_sql, which
@@ -17,6 +17,12 @@ module Exwiw
     # INSERT) once the config is loaded (see TableConfig#reject_ignored_members!).
     attribute :comment, optional(String), skip_serializing_if_nil: true
     attribute :ignore, Serdes::OptionalType.new(Serdes::ConcreteType.new(Boolean)), skip_serializing_if_nil: true
+    # Marks a column whose masking nobody has decided on yet. Extraction never
+    # reads it; `schema:generate` emits it and `schema:check` reports it, so CI
+    # can require the decision.
+    attribute :needs_mask_decision,
+              Serdes::OptionalType.new(Serdes::ConcreteType.new(Boolean)),
+              skip_serializing_if_nil: true
 
     def self.from_symbol_keys(hash)
       from(hash.transform_keys(&:to_s))

data/lib/exwiw/table_config.rb

@@ -170,9 +170,9 @@ module Exwiw
         merged_table.type = passed_table.type
         merged_table.comment = comment
         merged_table.filter = filter
-        merged_table.bulk_insert_chunk_size = passed_table.bulk_insert_chunk_size
         merged_table.ignore = ignore
         # User-owned, never regenerated: carry over from the existing config.
+        merged_table.bulk_insert_chunk_size = bulk_insert_chunk_size
         merged_table.scope_exempt = scope_exempt
         merged_table.scope_column = scope_column
         merged_table.reverse_scope = reverse_scope
@@ -259,7 +259,7 @@ module Exwiw
       ruby_side = [column.map && "map", column.replace_with_fake_data && "replace_with_fake_data"].compact
       return if ruby_side.empty?
 
-      sql_side = [column.raw_sql && "raw_sql", column.replace_with && "replace_with"].compact
+      sql_side = [column.raw_sql && "raw_sql", !column.replace_with.nil? && "replace_with"].select { |v| v }
       if ruby_side.size > 1 || sql_side.any?
         raise ArgumentError,
               "Table '#{name}' column '#{column.name}': #{(ruby_side + sql_side).join('/')} cannot be combined; " \

data/lib/exwiw/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Exwiw
-  VERSION = "0.9.20"
+  VERSION = "0.9.22"
 end

data/lib/exwiw.rb

@@ -10,6 +10,8 @@ require_relative "exwiw/config_file"
 require_relative "exwiw/strict_keys"
 require_relative "exwiw/belongs_to"
 require_relative "exwiw/fake_data"
+require_relative "exwiw/mask_value"
+require_relative "exwiw/default_mask"
 require_relative "exwiw/table_column"
 require_relative "exwiw/reverse_scope"
 require_relative "exwiw/batch_scope"
@@ -38,6 +40,11 @@ require_relative "exwiw/after_insert_hook"
 require_relative "exwiw/runner"
 require_relative "exwiw/explain_runner"
 require_relative "exwiw/schema_generator"
+require_relative "exwiw/db_introspector"
+require_relative "exwiw/db_introspector/mysql_introspector"
+require_relative "exwiw/db_introspector/postgresql_introspector"
+require_relative "exwiw/db_schema_generator"
+require_relative "exwiw/schema_check"
 require_relative "exwiw/mongoid_schema_generator"
 
 begin

data/lib/tasks/exwiw.rake

@@ -13,12 +13,18 @@ namespace :exwiw do
       ENV["EXWIW_SCHEMA_DIR_PATH"] || Exwiw::ConfigFile.schema_dir || "exwiw/schema"
     end
 
+    # Safe mode is on unless EXWIW_NEW_COLUMNS=plain, which a first-time
+    # bootstrap wants: there every column is new, so safe mode would flag the
+    # whole config at once. See SchemaGenerator#initialize.
+    safe_new_columns = lambda { ENV["EXWIW_NEW_COLUMNS"] != "plain" }
+
     desc "Generate schema from application"
     task generate: :environment do
       require "exwiw"
 
       groups = Exwiw::SchemaGenerator.from_rails_application(
         output_dir: resolve_schema_dir.call,
+        safe_new_columns: safe_new_columns.call,
       ).generate!
 
       # Surface cross-database belongs_tos the generator auto-ignored: these
@@ -38,6 +44,24 @@ namespace :exwiw do
       end
     end
 
+    desc "Report how the committed schema config differs from the application, without changing it"
+    task check: :environment do
+      require "exwiw"
+
+      report = Exwiw::SchemaCheck.from_rails_application(schema_dir: resolve_schema_dir.call).run
+      json = JSON.pretty_generate(report)
+      puts json
+      # A file too, so a caller need not assume stdout carries only the JSON.
+      File.write(ENV["EXWIW_SCHEMA_CHECK_OUTPUT"], json + "\n") if ENV["EXWIW_SCHEMA_CHECK_OUTPUT"]
+
+      unless Exwiw::SchemaCheck.clean?(report)
+        $stderr.puts "exwiw: the schema config is out of date or has undecided masking; " \
+                     "run `rake exwiw:schema:generate exwiw:schema:tidy` " \
+                     "and resolve every `needs_mask_decision` column."
+        exit 1
+      end
+    end
+
     desc "Remove tables/columns from the schema config that no longer exist in the application"
     task tidy: :environment do
       require "exwiw"
@@ -76,7 +100,44 @@ namespace :exwiw do
       Exwiw::MongoidSchemaGenerator.from_rails_application(
         output_dir: resolve_schema_dir.call,
         skip_unsupported: ENV["EXWIW_SKIP_UNSUPPORTED"] == "1",
+        # Same convention as `generate`: safe unless EXWIW_NEW_COLUMNS=plain.
+        safe_new_columns: safe_new_columns.call,
       ).generate!
     end
+
+    desc "Remove collections from the schema config that no longer exist in the Mongoid application"
+    task tidy_mongoid: :environment do
+      require "exwiw"
+
+      result = Exwiw::MongoidSchemaGenerator.from_rails_application(
+        output_dir: resolve_schema_dir.call,
+      ).tidy!
+
+      if result.empty?
+        puts "exwiw: schema config is already tidy; nothing to remove."
+      else
+        result.removed_tables.each do |name|
+          puts "exwiw: removed config for collection '#{name}' (no longer exists in the application)."
+        end
+      end
+    end
+
+    desc "Report how the committed schema config differs from the Mongoid application, without changing it"
+    task check_mongoid: :environment do
+      require "exwiw"
+
+      report = Exwiw::SchemaCheck.from_mongoid_application(schema_dir: resolve_schema_dir.call).run
+      json = JSON.pretty_generate(report)
+      puts json
+      # A file too, so a caller need not assume stdout carries only the JSON.
+      File.write(ENV["EXWIW_SCHEMA_CHECK_OUTPUT"], json + "\n") if ENV["EXWIW_SCHEMA_CHECK_OUTPUT"]
+
+      unless Exwiw::SchemaCheck.clean?(report)
+        $stderr.puts "exwiw: the schema config is out of date or has undecided masking; " \
+                     "run `rake exwiw:schema:generate_mongoid exwiw:schema:tidy_mongoid` " \
+                     "and resolve every `needs_mask_decision` field."
+        exit 1
+      end
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: exwiw
 version: !ruby/object:Gem::Version
-  version: 0.9.20
+  version: 0.9.22
 platform: ruby
 authors:
 - Shia
@@ -69,13 +69,19 @@ files:
 - lib/exwiw/belongs_to.rb
 - lib/exwiw/cli.rb
 - lib/exwiw/config_file.rb
+- lib/exwiw/db_introspector.rb
+- lib/exwiw/db_introspector/mysql_introspector.rb
+- lib/exwiw/db_introspector/postgresql_introspector.rb
+- lib/exwiw/db_schema_generator.rb
 - lib/exwiw/ddl_postprocessor.rb
+- lib/exwiw/default_mask.rb
 - lib/exwiw/determine_table_processing_order.rb
 - lib/exwiw/embedded_in.rb
 - lib/exwiw/explain_runner.rb
 - lib/exwiw/ext_json.rb
 - lib/exwiw/fake_data.rb
 - lib/exwiw/japanese_names.rb
+- lib/exwiw/mask_value.rb
 - lib/exwiw/mongo_query.rb
 - lib/exwiw/mongodb_collection_config.rb
 - lib/exwiw/mongodb_field.rb
@@ -88,6 +94,7 @@ files:
 - lib/exwiw/reverse_scope.rb
 - lib/exwiw/row_transformer.rb
 - lib/exwiw/runner.rb
+- lib/exwiw/schema_check.rb
 - lib/exwiw/schema_generator.rb
 - lib/exwiw/strict_keys.rb
 - lib/exwiw/table_column.rb