exwiw 0.3.8 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 90a440a039497562ccc9a4ab3045342953ba23faf0fdff48cfd633771a4688a4
-  data.tar.gz: 5f8ee4853cbbe39dedba4176e5db61d91572ee3766d1d76e3eda6977ea373e6b
+  metadata.gz: 568620ee295b1822c2606a42be450bbe5601aabb874606b6773b72619261cbba
+  data.tar.gz: 46f7b6f97c6b77b7046dfb19117734286ecb5d88001214f9d6e1995560f0b50a
 SHA512:
-  metadata.gz: aa34168f0c3b6137eca95bf8ebeea625ea106559a779bd8739c09e5ab8ffeae0a9a84f9ca188809e579ab3d1c3f8c77b68d57d006b156de30b1d01e100435ecb
-  data.tar.gz: c7ad4547852032fcb4c356883232c8a34d6418baf498686ed6f25a80279d1cf61379b566ee29267f989d9de80f4b2490022d03695e686dedf5c506b2c01871f8
+  metadata.gz: 8800c7adfa78ea01fb513eb3dbbe2b3fbfa98b13916d5b4e5a205232851b73e1bbef8cd9d0190abe5a9375d073c753d2f92be583c3a2a96d898cb757cd883a0a
+  data.tar.gz: 02f0ebd17ed423dfffbba3aefe2fbba01b533c23672d680ccce7c21edf2b36dbfc0c66ee72b8f4433af3af643aaea126861233a99d2222a2d347f407c8ab5c27

data/CHANGELOG.md

@@ -2,6 +2,18 @@
 
 ## [Unreleased]
 
+## [0.4.0] - 2026-06-04
+
+### Fixed
+
+- PostgreSQL: extraction no longer fails with `operator does not exist: character varying = uuid` when a `belongs_to` chain crosses a varchar foreign key and a uuid primary key. The adapter now introspects column types via `pg_attribute`/`pg_type` at query compile time and injects `::text` casts on both sides of the comparison when a uuid/varchar mismatch is detected. Covers JOIN ON conditions, WHERE IN subqueries, and bulk-delete IN clauses. ([#73](https://github.com/heyinc/exwiw/pull/73))
+
+## [0.3.9] - 2026-06-03
+
+### Added
+
+- New `exwiw:schema:tidy` rake task. It reconciles the existing schema config against the live database (read through the database connection, not the models) and removes only what no longer exists there: a config file whose table has been dropped is deleted, and columns recorded in a surviving table's config that the table no longer has are dropped from that file. Because it reads the database directly, a table that still exists in the database but has lost (or never had) a model is kept — only a genuinely-dropped table is removed. Unlike `schema:generate` it never adds or regenerates entries, so hand-edited `comment` / `ignore` / `replace_with` on surviving tables/columns are preserved. It honors `OUTPUT_DIR_PATH` and the per-database subdirectory layout, and prints the tables/columns it removed.
+
 ## [0.3.8] - 2026-06-02
 
 ### Added

data/README.md

@@ -144,6 +144,23 @@ By default, the schema files will be saved in the `exwiw` directory. You can spe
 OUTPUT_DIR_PATH=custom_directory bundle exec rake exwiw:schema:generate
 ```
 
+#### Tidying stale config (`schema:tidy`)
+
+`schema:generate` adds and updates config files for the tables it finds, but it never deletes the config file of a table that has been dropped from the application. To reconcile the existing config against the current schema, run:
+
+```bash
+bundle exec rake exwiw:schema:tidy
+```
+
+`schema:tidy` compares the config files already on disk with the **live database** (read through the database connection, not the models) and removes only what no longer exists there:
+
+- a config file whose table has been dropped from the database is **deleted**, and
+- columns recorded in a surviving table's config that the table no longer has are **dropped** from that file.
+
+Because it reads the database directly, a table that still exists in the database but has lost (or never had) an ActiveRecord model is **kept** — only a table that is genuinely gone is removed. (This is the deliberate counterpart to `generate`, which is model-driven and only ever adds what the models know about.)
+
+It respects `OUTPUT_DIR_PATH` and the per-database subdirectory layout in the same way as `schema:generate`. Unlike `generate`, `tidy` never adds or regenerates entries — every surviving table/column (including hand-edited `comment` / `ignore` / `replace_with`) is left untouched, so it is safe to run on a customized config. The task prints which tables and columns it removed (or that the config was already tidy). Stale `belongs_tos` are not pruned by `tidy`; rerun `schema:generate` to refresh those.
+
 #### Multiple databases
 
 If the application uses Rails' multiple-database support (`connects_to`), `schema:generate` buckets models by the database they connect to and writes each database's config files into its own subdirectory of the output directory, named after the database config name (`primary`, `analytics`, ...):

data/lib/exwiw/adapter/postgresql_adapter.rb

@@ -177,8 +177,17 @@ module Exwiw
         end
 
         foreign_key = first_join.foreign_key
-        subquery_sql = compile_ast(subquery_ast)
-        sql += "\nWHERE #{select_query_ast.from_table_name}.#{foreign_key} IN (#{subquery_sql})"
+        outer_table = select_query_ast.from_table_name
+        inner_table = first_join.join_table_name
+        inner_column = first_join.primary_key
+        cast_to = types_need_cast?(
+          column_pg_type(outer_table, foreign_key),
+          column_pg_type(inner_table, inner_column)
+        ) ? 'text' : nil
+        subquery_sql = compile_ast(subquery_ast, select_cast_to: cast_to)
+        outer_expr = "#{outer_table}.#{foreign_key}"
+        outer_expr = "#{outer_expr}::text" if cast_to
+        sql += "\nWHERE #{outer_expr} IN (#{subquery_sql})"
 
         # first_join.base_where_clauses holds conditions on the outer
         # delete-target table (from_table_name), such as a polymorphic type
@@ -195,19 +204,30 @@ module Exwiw
         sql
       end
 
-      def compile_ast(query_ast)
+      def compile_ast(query_ast, select_cast_to: nil)
         raise NotImplementedError unless query_ast.is_a?(Exwiw::QueryAst::Select)
 
         sql = "SELECT "
         sql += if query_ast.select_all
                  "*"
                else
-                 query_ast.columns.map { |col| compile_column_name(query_ast, col) }.join(', ')
+                 cols = query_ast.columns.map { |col| compile_column_name(query_ast, col) }
+                 cols = cols.map { |c| "#{c}::#{select_cast_to}" } if select_cast_to
+                 cols.join(', ')
                end
         sql += " FROM #{query_ast.from_table_name}"
 
         query_ast.join_clauses.each do |join|
-          sql += " JOIN #{join.join_table_name} ON #{join.base_table_name}.#{join.foreign_key} = #{join.join_table_name}.#{join.primary_key}"
+          fk_expr = "#{join.base_table_name}.#{join.foreign_key}"
+          pk_expr = "#{join.join_table_name}.#{join.primary_key}"
+          if types_need_cast?(
+            column_pg_type(join.base_table_name, join.foreign_key),
+            column_pg_type(join.join_table_name, join.primary_key)
+          )
+            fk_expr = "#{fk_expr}::text"
+            pk_expr = "#{pk_expr}::text"
+          end
+          sql += " JOIN #{join.join_table_name} ON #{fk_expr} = #{pk_expr}"
 
           join.where_clauses.each do |where|
             compiled_where_condition = compile_where_condition(where, join.join_table_name)
@@ -246,23 +266,54 @@ module Exwiw
             "#{key} IN (#{values.join(', ')})"
           end
         elsif where_clause.operator == :in_subquery
-          "#{key} IN (#{compile_subquery(where_clause.value)})"
+          subquery_sql = compile_subquery(where_clause.value, outer_table: table_name, outer_column: where_clause.column_name)
+          cast_to = subquery_cast_to(where_clause.value, table_name, where_clause.column_name)
+          outer_key = cast_to ? "#{key}::#{cast_to}" : key
+          "#{outer_key} IN (#{subquery_sql})"
         else
           raise "Unsupported operator: #{where_clause.operator}"
         end
       end
 
-      private def compile_subquery(subquery)
-        # A SelectSubquery wraps a full Select (the referencing table's
-        # extraction query, projected to a foreign key); compile it as-is.
-        return compile_ast(subquery.query) if subquery.is_a?(Exwiw::QueryAst::SelectSubquery)
+      private def compile_subquery(subquery, outer_table: nil, outer_column: nil)
+        cast_to = subquery_cast_to(subquery, outer_table, outer_column)
+
+        if subquery.is_a?(Exwiw::QueryAst::SelectSubquery)
+          return compile_ast(subquery.query, select_cast_to: cast_to)
+        end
 
         inner_values = subquery.where_values.map { |v| escape_value(v) }
-        "SELECT #{subquery.table_name}.#{subquery.select_column} " \
+        select_expr = "#{subquery.table_name}.#{subquery.select_column}"
+        select_expr = "#{select_expr}::#{cast_to}" if cast_to
+        "SELECT #{select_expr} " \
           "FROM #{subquery.table_name} " \
           "WHERE #{subquery.table_name}.#{subquery.where_column} IN (#{inner_values.join(', ')})"
       end
 
+      private def subquery_select_target(subquery)
+        case subquery
+        when Exwiw::QueryAst::SelectSubquery
+          q = subquery.query
+          col = q.columns.first
+          col ? [q.from_table_name, col.name] : [nil, nil]
+        when Exwiw::QueryAst::Subquery
+          [subquery.table_name, subquery.select_column]
+        else
+          [nil, nil]
+        end
+      end
+
+      private def subquery_cast_to(subquery, outer_table, outer_column)
+        return nil if outer_table.nil? || outer_column.nil?
+
+        inner_table, inner_column = subquery_select_target(subquery)
+        return nil if inner_table.nil?
+
+        outer_type = column_pg_type(outer_table, outer_column)
+        inner_type = column_pg_type(inner_table, inner_column)
+        types_need_cast?(outer_type, inner_type) ? 'text' : nil
+      end
+
       private def escape_value(value)
         case value
         when nil
@@ -351,6 +402,36 @@ module Exwiw
         end
       end
 
+      private def column_pg_type(table_name, column_name)
+        @column_type_cache ||= {}
+        cache_key = [table_name, column_name]
+        return @column_type_cache[cache_key] if @column_type_cache.key?(cache_key)
+
+        sql = <<~SQL
+          SELECT t.typname
+          FROM pg_attribute a
+          JOIN pg_class c ON c.oid = a.attrelid
+          JOIN pg_type t  ON t.oid = a.atttypid
+          WHERE c.relname = $1
+            AND a.attname = $2
+            AND a.attnum > 0
+            AND NOT a.attisdropped
+          LIMIT 1
+        SQL
+
+        result = connection.exec_params(sql, [table_name, column_name])
+        @column_type_cache[cache_key] = result.ntuples > 0 ? result.getvalue(0, 0) : nil
+      end
+
+      private def types_need_cast?(type_a, type_b)
+        return false if type_a.nil? || type_b.nil?
+        return false if type_a == type_b
+
+        string_types = %w[varchar text bpchar name].freeze
+        (type_a == 'uuid' && string_types.include?(type_b)) ||
+          (type_b == 'uuid' && string_types.include?(type_a))
+      end
+
       private def connection
         @connection ||=
           begin

data/lib/exwiw/schema_generator.rb

@@ -5,6 +5,30 @@ require "json"
 
 module Exwiw
   class SchemaGenerator
+    # Summary of what `SchemaGenerator#tidy!` removed, returned so callers can
+    # report it. `removed_columns` maps a surviving table's name to the column
+    # names that were dropped from its config.
+    class TidyResult
+      attr_reader :removed_tables, :removed_columns
+
+      def initialize
+        @removed_tables = []
+        @removed_columns = {}
+      end
+
+      def add_removed_table(table_name)
+        @removed_tables << table_name
+      end
+
+      def add_removed_column(table_name, column_name)
+        (@removed_columns[table_name] ||= []) << column_name
+      end
+
+      def empty?
+        @removed_tables.empty? && @removed_columns.empty?
+      end
+    end
+
     # ActiveStorage tracks generated image variants in this table. Its rows are
     # derivative and regenerable — ActiveStorage lazily (re)creates a variant the
     # next time it is requested — so there is little value in exporting them. More
@@ -35,6 +59,60 @@ module Exwiw
       groups
     end
 
+    # Reconcile the config files already on disk against the live database,
+    # removing only what no longer exists there:
+    #
+    # - a config file whose table is no longer present is deleted, and
+    # - columns recorded in a surviving table's config that the table no
+    #   longer has are dropped from that file.
+    #
+    # The source of truth is the database connection (`data_sources` for table
+    # existence — which covers views too — and `columns` for the column list),
+    # NOT `build_table_groups`. `build_table_groups` only knows about tables
+    # that still have an ActiveRecord model, so reconciling against it would
+    # delete the config of a table that is still present in the database but
+    # has merely lost (or never had) a model. Reading the connection directly
+    # avoids that: only a table that is genuinely gone from the database is
+    # removed.
+    #
+    # Unlike `generate!`, tidy never adds or regenerates entries: every
+    # surviving table/column — including its hand-edited `comment` / `ignore` /
+    # `replace_with` — is left untouched, and only the stale entries are
+    # stripped. (Removing a deleted column is something `generate!` already does
+    # incidentally via #merge, but `generate!` can never delete the config file
+    # of a removed table, which is the gap this fills.) Returns a TidyResult
+    # describing the removals so callers (e.g. the rake task) can report them.
+    def tidy!
+      result = TidyResult.new
+
+      model_db_groups.each do |db_name, _group_models, conn|
+        dir = config_dir_for(db_name)
+        next unless Dir.exist?(dir)
+
+        existing_data_sources = conn.data_sources.to_set
+
+        Dir[File.join(dir, "*.json")].sort.each do |path|
+          existing = TableConfig.from(JSON.parse(File.read(path)))
+
+          unless existing_data_sources.include?(existing.name)
+            File.delete(path)
+            result.add_removed_table(existing.name)
+            next
+          end
+
+          valid_column_names = conn.columns(existing.name).map(&:name).to_set
+          stale_columns = existing.columns.reject { |column| valid_column_names.include?(column.name) }
+          next if stale_columns.empty?
+
+          existing.columns = existing.columns.select { |column| valid_column_names.include?(column.name) }
+          File.write(path, JSON.pretty_generate(existing.to_hash) + "\n")
+          stale_columns.each { |column| result.add_removed_column(existing.name, column.name) }
+        end
+      end
+
+      result
+    end
+
     # Returns a Hash keyed by the database name.
     #
     # - Single-database setup: the only key is `nil`, signalling that the table
@@ -44,18 +122,35 @@ module Exwiw
     #   mapping to that database's table configs. They are written into
     #   `output_dir/<db_name>/`.
     def build_table_groups
+      model_db_groups.each_with_object({}) do |(db_name, group_models, conn), result|
+        result[db_name] = build_tables_for(group_models, conn)
+      end
+    end
+
+    # The per-database grouping that both `build_table_groups` and `tidy!` work
+    # from: `[[db_name, models, connection], ...]`.
+    #
+    # - Single-database setup: one entry keyed by `nil` (configs written flat
+    #   into `output_dir`). When there are no models at all, fall back to the
+    #   default connection so callers still have a connection to inspect.
+    # - Multi-database setup (Rails `connects_to`): one entry per database,
+    #   keyed by `connection_db_config.name` ("primary" / "analytics", ...).
+    #
+    # The db_name <-> connection mapping is necessarily model-derived: which
+    # databases the app talks to is only declared on the model side (via
+    # `connects_to`). What each consumer reads *through* that connection differs
+    # — `build_table_groups` builds configs from the models, while `tidy!`
+    # reads the live database's actual tables/columns.
+    private def model_db_groups
       models = concrete_models
       grouped = models.group_by { |model| database_name_for(model) }
 
       if grouped.size <= 1
         conn = models.empty? ? ActiveRecord::Base.connection : models.first.connection
-        return { nil => build_tables_for(models, conn) }
+        return [[nil, models, conn]]
       end
 
-      grouped.each_with_object({}) do |(db_name, group_models), result|
-        conn = group_models.first.connection
-        result[db_name] = build_tables_for(group_models, conn)
-      end
+      grouped.map { |db_name, group_models| [db_name, group_models, group_models.first.connection] }
     end
 
     # Backwards-compatible flat list of all table configs. Only meaningful for
@@ -67,11 +162,18 @@ module Exwiw
 
     def write_groups(groups)
       groups.each do |db_name, tables|
-        dir = db_name.nil? ? @output_dir : File.join(@output_dir, db_name)
-        write_files(dir, tables)
+        write_files(config_dir_for(db_name), tables)
       end
     end
 
+    # The directory a database group's config files live in. A single-database
+    # setup (`db_name` is nil) writes flat into `output_dir`; a multi-database
+    # setup writes into `output_dir/<db_name>/`. Shared by `write_groups` and
+    # `tidy!` so the two operations agree on file locations.
+    private def config_dir_for(db_name)
+      db_name.nil? ? @output_dir : File.join(@output_dir, db_name)
+    end
+
     def write_files(dir, tables)
       FileUtils.mkdir_p(dir)
 

data/lib/exwiw/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Exwiw
-  VERSION = "0.3.8"
+  VERSION = "0.4.0"
 end

data/lib/tasks/exwiw.rake

@@ -11,6 +11,26 @@ namespace :exwiw do
       ).generate!
     end
 
+    desc "Remove tables/columns from the schema config that no longer exist in the application"
+    task tidy: :environment do
+      require "exwiw"
+
+      result = Exwiw::SchemaGenerator.from_rails_application(
+        output_dir: ENV["OUTPUT_DIR_PATH"] || "exwiw",
+      ).tidy!
+
+      if result.empty?
+        puts "exwiw: schema config is already tidy; nothing to remove."
+      else
+        result.removed_tables.each do |name|
+          puts "exwiw: removed config for table '#{name}' (no longer exists in the application)."
+        end
+        result.removed_columns.each do |table_name, columns|
+          puts "exwiw: removed column(s) #{columns.join(', ')} from '#{table_name}' (no longer in the table)."
+        end
+      end
+    end
+
     desc "Generate schema from a Mongoid application"
     task generate_mongoid: :environment do
       require "exwiw"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: exwiw
 version: !ruby/object:Gem::Version
-  version: 0.3.8
+  version: 0.4.0
 platform: ruby
 authors:
 - Shia