exwiw 1.1.6 → 1.1.7

data/docs/mongodb.md

@@ -71,6 +71,10 @@ Where the SQL adapters emit a `UNION` subquery, MongoDB has no cross-collection
 
 Masking (`replace_with`) and `fields` behavior on a reverse-scoped collection are unchanged. Like the SQL key, `reverse_scope` is user-owned: `exwiw:mongoid:schema:generate` never emits it and regeneration preserves a hand-added value.
 
+## Self-referencing `belongs_to`
+
+Unlike the SQL adapters, which keep the ancestors of a narrowed [tree table](../README.md#self-referencing-belongs_to-tree-tables), the MongoDB adapter extracts each collection with a single `find` and does not walk a self-reference. A narrowed collection with a `belongs_to` to itself logs a warning, since a kept document's parent may be missing from the dump.
+
 ## Embedded documents
 
 MongoDB models often store one-to-many relationships as embedded subdocument arrays (e.g. `users` documents with a `posts: [...]` field). To mask fields inside embedded subdocuments, declare a separate config with `embedded_in`:

data/docs/scope-id-set-join-notes.md

@@ -0,0 +1,12 @@
+# Why scope id sets are joined, not `IN (subquery)`
+
+When a table is narrowed by a set of ids that another query produces (referenced-by, `reverse_scope`, the narrowed-parent cascade, polymorphic arms, tree ancestors), the SQL adapters emit a `JOIN` to a `SELECT DISTINCT` derived table instead of `<col> IN (<subquery>)`:
+
+```sql
+… JOIN (SELECT DISTINCT src.<id> AS exwiw_scope_id FROM (<id-set subquery>) AS src) AS ids
+    ON <table>.<col> = ids.exwiw_scope_id
+```
+
+Both forms return the same rows; the `DISTINCT` keeps the join from multiplying them. The plans differ on a large table. MySQL cannot turn `IN (… UNION …)` into a materialized semi-join, so it rewrites it into a correlated `EXISTS` that is evaluated once per outer row: a full scan of the outer table, with the union re-run for every row. The `DISTINCT` derived table cannot be merged into the outer query, so it is materialized once and the outer table is probed by its key. On a large `users` table this is the difference between a full scan and an index lookup. Nested cascades are materialized once per level for the same reason.
+
+PostgreSQL casts both sides to `text` when the key types differ (`uuid` against `varchar`).

data/lib/exwiw/adapter/mongodb_adapter.rb

@@ -171,6 +171,7 @@ module Exwiw
           else
             related_collection_filter(config, config_by_name, dump_target)
           end
+        warn_missing_self_ancestors(config) unless filter.empty?
 
         Exwiw::MongoQuery::Find.new(
           collection: config.name,
@@ -181,6 +182,17 @@ module Exwiw
         )
       end
 
+      # The SQL adapters add the ancestors of a narrowed self-referencing table;
+      # this adapter extracts each collection with a single find, so it cannot.
+      private def warn_missing_self_ancestors(config)
+        return unless config.belongs_tos.any? { |relation| relation.table_name == config.name && !relation.polymorphic? }
+
+        @logger.warn(
+          "  #{config.name} references itself, but the mongodb adapter does not add the ancestors " \
+          "of the documents it keeps; a kept document's parent may be missing."
+        )
+      end
+
       def execute(query)
         @logger.debug("  Executing Mongo find on '#{query.collection}': filter=#{query.filter.inspect} projection=#{query.projection.inspect}")
 

data/lib/exwiw/adapter/mysql_adapter.rb

@@ -353,24 +353,27 @@ module Exwiw
       end
 
       private def compile_subquery(subquery)
-        # A SelectSubquery wraps a full Select (the referencing table's
-        # extraction query, projected to a foreign key); compile it as-is.
-        return compile_ast(subquery.query) if subquery.is_a?(Exwiw::QueryAst::SelectSubquery)
-
-        # A UnionSubquery wraps several such Selects; UNION their compiled forms
-        # into a single id set. Sibling arms can carry the same nested scope
-        # id-set, and MySQL cannot reference one TEMPORARY table twice in a
-        # statement (ER_CANT_REOPEN_TABLE), so nested scopes compile inline here.
-        if subquery.is_a?(Exwiw::QueryAst::UnionSubquery)
-          return without_scope_materialization do
+        case subquery
+        in Exwiw::QueryAst::RecursiveAncestorSubquery
+          compile_ancestor_closure(subquery)
+        in Exwiw::QueryAst::SelectSubquery
+          # A SelectSubquery wraps a full Select (the referencing table's
+          # extraction query, projected to a foreign key); compile it as-is.
+          compile_ast(subquery.query)
+        in Exwiw::QueryAst::UnionSubquery
+          # A UnionSubquery wraps several such Selects; UNION their compiled forms
+          # into a single id set. Sibling arms can carry the same nested scope
+          # id-set, and MySQL cannot reference one TEMPORARY table twice in a
+          # statement (ER_CANT_REOPEN_TABLE), so nested scopes compile inline here.
+          without_scope_materialization do
             subquery.queries.map { |q| compile_ast(q) }.join(' UNION ')
           end
+        in Exwiw::QueryAst::Subquery
+          inner_values = subquery.where_values.map { |v| escape_value(v) }
+          "SELECT #{qualified_name(subquery.table_name, subquery.select_column)} " \
+            "FROM #{quote_table_name(subquery.table_name)} " \
+            "WHERE #{qualified_name(subquery.table_name, subquery.where_column)} IN (#{inner_values.join(', ')})"
         end
-
-        inner_values = subquery.where_values.map { |v| escape_value(v) }
-        "SELECT #{qualified_name(subquery.table_name, subquery.select_column)} " \
-          "FROM #{quote_table_name(subquery.table_name)} " \
-          "WHERE #{qualified_name(subquery.table_name, subquery.where_column)} IN (#{inner_values.join(', ')})"
       end
 
       # Backslash and control-character escapes, matching mysqldump. Escaping

data/lib/exwiw/adapter/postgresql_adapter.rb

@@ -461,25 +461,35 @@ module Exwiw
       private def compile_subquery(subquery, outer_table: nil, outer_column: nil)
         cast_to = subquery_cast_to(subquery, outer_table, outer_column)
 
-        if subquery.is_a?(Exwiw::QueryAst::SelectSubquery)
-          return compile_ast(subquery.query, select_cast_to: cast_to)
+        case subquery
+        in Exwiw::QueryAst::RecursiveAncestorSubquery
+          # The outer key is compared with this table's own primary keys, so it needs no cast.
+          compile_ancestor_closure(subquery)
+        in Exwiw::QueryAst::SelectSubquery
+          compile_ast(subquery.query, select_cast_to: cast_to)
+        in Exwiw::QueryAst::UnionSubquery
+          # A UnionSubquery wraps several projected Selects; UNION their compiled
+          # forms. cast_to is the union-wide decision (see union_cast_to): when any
+          # arm's column type would clash with the outer column or another arm,
+          # every arm's projected column and the outer key are cast to text so the
+          # UNION and the enclosing IN comparison resolve to one type.
+          subquery.queries.map { |q| compile_ast(q, select_cast_to: cast_to) }.join(' UNION ')
+        in Exwiw::QueryAst::Subquery
+          inner_values = subquery.where_values.map { |v| escape_value(v) }
+          select_expr = qualified_name(subquery.table_name, subquery.select_column)
+          select_expr = "#{select_expr}::#{cast_to}" if cast_to
+          "SELECT #{select_expr} " \
+            "FROM #{quote_table_name(subquery.table_name)} " \
+            "WHERE #{qualified_name(subquery.table_name, subquery.where_column)} IN (#{inner_values.join(', ')})"
         end
+      end
 
-        # A UnionSubquery wraps several projected Selects; UNION their compiled
-        # forms. cast_to is the union-wide decision (see union_cast_to): when any
-        # arm's column type would clash with the outer column or another arm,
-        # every arm's projected column and the outer key are cast to text so the
-        # UNION and the enclosing IN comparison resolve to one type.
-        if subquery.is_a?(Exwiw::QueryAst::UnionSubquery)
-          return subquery.queries.map { |q| compile_ast(q, select_cast_to: cast_to) }.join(' UNION ')
-        end
+      # A self-referencing foreign key may be stored as text while the key it
+      # points at is a uuid; reconcile them the same way a JOIN does.
+      private def ancestor_link_condition(table, referenced_column, foreign_key)
+        return super unless types_need_cast?(column_pg_type(table, referenced_column), column_pg_type(table, foreign_key))
 
-        inner_values = subquery.where_values.map { |v| escape_value(v) }
-        select_expr = qualified_name(subquery.table_name, subquery.select_column)
-        select_expr = "#{select_expr}::#{cast_to}" if cast_to
-        "SELECT #{select_expr} " \
-          "FROM #{quote_table_name(subquery.table_name)} " \
-          "WHERE #{qualified_name(subquery.table_name, subquery.where_column)} IN (#{inner_values.join(', ')})"
+        "#{qualified_name(table, referenced_column)}::text = #{ANCESTOR_SET_NAME}.#{quote_identifier(foreign_key)}::text"
       end
 
       private def subquery_select_target(subquery)

data/lib/exwiw/adapter/sqlite_adapter.rb

@@ -232,20 +232,23 @@ module Exwiw
       end
 
       private def compile_subquery(subquery)
-        # A SelectSubquery wraps a full Select (the referencing table's
-        # extraction query, projected to a foreign key); compile it as-is.
-        return compile_ast(subquery.query) if subquery.is_a?(Exwiw::QueryAst::SelectSubquery)
-
-        # A UnionSubquery wraps several such Selects; UNION their compiled forms
-        # into a single id set.
-        if subquery.is_a?(Exwiw::QueryAst::UnionSubquery)
-          return subquery.queries.map { |q| compile_ast(q) }.join(' UNION ')
+        case subquery
+        in Exwiw::QueryAst::RecursiveAncestorSubquery
+          compile_ancestor_closure(subquery)
+        in Exwiw::QueryAst::SelectSubquery
+          # A SelectSubquery wraps a full Select (the referencing table's
+          # extraction query, projected to a foreign key); compile it as-is.
+          compile_ast(subquery.query)
+        in Exwiw::QueryAst::UnionSubquery
+          # A UnionSubquery wraps several such Selects; UNION their compiled forms
+          # into a single id set.
+          subquery.queries.map { |q| compile_ast(q) }.join(' UNION ')
+        in Exwiw::QueryAst::Subquery
+          inner_values = subquery.where_values.map { |v| escape_value(v) }
+          "SELECT #{qualified_name(subquery.table_name, subquery.select_column)} " \
+            "FROM #{quote_table_name(subquery.table_name)} " \
+            "WHERE #{qualified_name(subquery.table_name, subquery.where_column)} IN (#{inner_values.join(', ')})"
         end
-
-        inner_values = subquery.where_values.map { |v| escape_value(v) }
-        "SELECT #{qualified_name(subquery.table_name, subquery.select_column)} " \
-          "FROM #{quote_table_name(subquery.table_name)} " \
-          "WHERE #{qualified_name(subquery.table_name, subquery.where_column)} IN (#{inner_values.join(', ')})"
       end
 
       private def escape_value(value)

data/lib/exwiw/adapter.rb

@@ -307,7 +307,8 @@ module Exwiw
         where_clause.is_a?(Exwiw::QueryAst::WhereClause) &&
           where_clause.operator == :in_subquery &&
           (where_clause.value.is_a?(Exwiw::QueryAst::SelectSubquery) ||
-            where_clause.value.is_a?(Exwiw::QueryAst::UnionSubquery))
+            where_clause.value.is_a?(Exwiw::QueryAst::UnionSubquery) ||
+            where_clause.value.is_a?(Exwiw::QueryAst::RecursiveAncestorSubquery))
       end
 
       # The bare name of the single column a scope subquery projects, used to
@@ -319,8 +320,36 @@ module Exwiw
           subquery.query.columns.first.name
         when Exwiw::QueryAst::UnionSubquery
           subquery.queries.first.columns.first.name
+        when Exwiw::QueryAst::RecursiveAncestorSubquery
+          subquery.primary_key
         end
       end
+
+      ANCESTOR_SET_NAME = "exwiw_ancestors"
+
+      # The WITH clause sits inside the subquery, so the id set stays valid wherever
+      # it is embedded: a derived-table JOIN, a MySQL temporary table, a UNION arm.
+      private def compile_ancestor_closure(subquery)
+        table = subquery.table_name
+        ancestor_columns = [subquery.primary_key, *subquery.links.map(&:foreign_key)].uniq
+        column_list = ancestor_columns.map { |c| quote_identifier(c) }.join(', ')
+        recursive_select = ancestor_columns.map { |c| qualified_name(table, c) }.join(', ')
+        link_conditions = subquery.links.map do |link|
+          ancestor_link_condition(table, link.references || subquery.primary_key, link.foreign_key)
+        end
+
+        "WITH RECURSIVE #{ANCESTOR_SET_NAME} (#{column_list}) AS (" \
+          "#{compile_ast(subquery.base)} " \
+          "UNION " \
+          "SELECT #{recursive_select} FROM #{quote_table_name(table)} " \
+          "JOIN #{ANCESTOR_SET_NAME} ON #{link_conditions.join(' OR ')}" \
+          ") SELECT #{ANCESTOR_SET_NAME}.#{quote_identifier(subquery.primary_key)} FROM #{ANCESTOR_SET_NAME}"
+      end
+
+      # Overridden where a dialect compares the two columns differently.
+      private def ancestor_link_condition(table, referenced_column, foreign_key)
+        "#{qualified_name(table, referenced_column)} = #{ANCESTOR_SET_NAME}.#{quote_identifier(foreign_key)}"
+      end
     end
 
     # @params [Exwiw::QueryAst] query_ast

data/lib/exwiw/query_ast.rb

@@ -20,6 +20,8 @@ module Exwiw
           reads_table?(where_clause.value.query, table_name)
         when UnionSubquery
           where_clause.value.queries.any? { |q| reads_table?(q, table_name) }
+        when RecursiveAncestorSubquery
+          where_clause.value.table_name == table_name || reads_table?(where_clause.value.base, table_name)
         else
           false
         end
@@ -39,6 +41,7 @@ module Exwiw
           case where_clause.value
           when SelectSubquery then id_spaces(where_clause.value.query)
           when UnionSubquery then where_clause.value.queries.flat_map { |q| id_spaces(q) }
+          when RecursiveAncestorSubquery then id_spaces(where_clause.value.base)
           else []
           end
         own + nested
@@ -84,7 +87,7 @@ module Exwiw
         {
           column_name: column_name,
           operator: operator,
-          value: value.is_a?(Subquery) || value.is_a?(SelectSubquery) || value.is_a?(UnionSubquery) ? value.to_h : value,
+          value: SUBQUERY_VALUE_TYPES.any? { |type| value.is_a?(type) } ? value.to_h : value,
         }
       end
     end
@@ -151,6 +154,36 @@ module Exwiw
       end
     end
 
+    # The primary keys of the rows `base` keeps plus every ancestor reachable
+    # through the table's own self-referencing foreign keys (`links`), so a tree
+    # table never keeps a row whose parent was left out:
+    #
+    #   WITH RECURSIVE exwiw_ancestors (<pk>, <fk>...) AS (
+    #     <base projected to pk and every link's fk>
+    #     UNION
+    #     SELECT <table>.<pk>, <table>.<fk>... FROM <table>
+    #       JOIN exwiw_ancestors ON <table>.<ref> = exwiw_ancestors.<fk> OR ...
+    #   ) SELECT <pk> FROM exwiw_ancestors
+    #
+    # UNION (not UNION ALL) stops the walk on cyclic data.
+    RecursiveAncestorSubquery = Struct.new(:base, :table_name, :primary_key, :links, keyword_init: true) do
+      def to_h
+        {
+          ancestors_of: base.to_h,
+          table_name: table_name,
+          primary_key: primary_key,
+          links: links.map(&:to_h),
+        }
+      end
+    end
+
+    # One self-referencing foreign key of a RecursiveAncestorSubquery.
+    # `references` is the column the foreign key points at (the primary key
+    # unless the belongs_to says otherwise).
+    AncestorLink = Struct.new(:foreign_key, :references, keyword_init: true)
+
+    SUBQUERY_VALUE_TYPES = [Subquery, SelectSubquery, UnionSubquery, RecursiveAncestorSubquery].freeze
+
     module ColumnValue
       Base = Struct.new(:name, :value, keyword_init: true)
       Plain = Class.new(Base)

data/lib/exwiw/query_ast_builder.rb

@@ -6,6 +6,20 @@ module Exwiw
       new(table_name, table_by_name, dump_target, logger, allow_reverse: allow_reverse, allow_declared_reverse: allow_declared_reverse, forward_path: forward_path, reverse_path: reverse_path, deep_chain_warned: deep_chain_warned, batch_ids: batch_ids).run
     end
 
+    # The self-referencing foreign keys whose ancestors a narrowed `table` also
+    # keeps. A polymorphic one is not supported: following it would also need its
+    # type column checked at every step.
+    def self.self_ancestor_links(table)
+      return [] if table.primary_key.nil?
+
+      table.belongs_tos.filter_map do |relation|
+        next unless relation.table_name == table.name
+        next if relation.polymorphic?
+
+        QueryAst::AncestorLink.new(foreign_key: relation.foreign_key, references: relation.references)
+      end.uniq
+    end
+
     # Scope-column mode classification for a single table. One of
     # :exempt / :direct / :via_path / :referenced_by / :via_scoped_parent / :unscopable.
     def self.scope_category(table_name, table_by_name, dump_target, logger)
@@ -97,7 +111,9 @@ module Exwiw
       tables.reject(&:ignore).each do |table|
         next unless table.respond_to?(:batch_scope) && table.batch_scope
 
-        new(table.name, table_by_name, dump_target, logger).batch_scope_terminus!
+        builder = new(table.name, table_by_name, dump_target, logger)
+        builder.batch_scope_terminus!
+        builder.warn_batch_scope_ancestor_gaps
       end
     end
 
@@ -216,14 +232,21 @@ module Exwiw
 
     def run
       table = table_by_name.fetch(table_name)
+      ast = scope_mode? ? build_scoped(table) : build_single_target(table)
+      with_self_ancestors(table, ast)
+    end
 
-      return build_scoped(table) if scope_mode?
-
+    private def build_single_target(table)
       arm_queries = target_polymorphic_arm_queries(table)
+      tree_route_clauses = arm_queries ? nil : dump_target_tree_route_clauses(table)
       if arm_queries
         where_clauses = [pk_union_clause(table, arm_queries)]
         where_clauses.push(table.filter) if table.filter
         join_clauses = []
+      elsif tree_route_clauses
+        where_clauses = tree_route_clauses
+        where_clauses.push(table.filter) if table.filter
+        join_clauses = []
       else
         where_clauses = build_where_clauses(table, dump_target)
         join_clauses = build_join_clauses(table, table_by_name, dump_target)
@@ -283,6 +306,13 @@ module Exwiw
         )
       end
 
+      new_extraction_query(table).tap do |ast|
+        join_clauses.each { |join_clause| ast.join(join_clause) }
+        where_clauses.each { |where_clause| ast.where(where_clause) }
+      end
+    end
+
+    private def new_extraction_query(table)
       QueryAst::Select.new.tap do |ast|
         ast.from(table.name)
         if table.rails_managed?
@@ -290,8 +320,28 @@ module Exwiw
         else
           ast.select(table.columns)
         end
-        join_clauses.each { |join_clause| ast.join(join_clause) }
-        where_clauses.each { |where_clause| ast.where(where_clause) }
+      end
+    end
+
+    # Keeps a narrowed tree table's parent ids from dangling. The ancestors are
+    # not held to the table's filter or scope, since valid foreign keys win. A
+    # batched table is left alone: every batch would repeat the ancestors.
+    private def with_self_ancestors(table, ast)
+      return ast unless @batch_ids.nil?
+      return ast if ast.where_clauses.empty? && ast.join_clauses.empty?
+
+      links = self.class.self_ancestor_links(table)
+      return ast if links.empty?
+
+      closure = QueryAst::RecursiveAncestorSubquery.new(
+        base: project_query_to(ast, [table.primary_key, *links.map(&:foreign_key)].uniq),
+        table_name: table.name,
+        primary_key: table.primary_key,
+        links: links
+      )
+
+      new_extraction_query(table).tap do |widened|
+        widened.where(QueryAst::WhereClause.new(column_name: table.primary_key, operator: :in_subquery, value: closure))
       end
     end
 
@@ -552,22 +602,52 @@ module Exwiw
         return nil
       end
 
-      relation, parent, parent_query = candidates.first
-
-      # Project the parent's extraction query down to just its primary key — the
-      # column this table's foreign key points at.
-      pk_column = TableColumn.from_symbol_keys(name: parent.primary_key)
-      projected = QueryAst::Select.new
-      projected.from(parent_query.from_table_name)
-      projected.select([pk_column])
-      parent_query.join_clauses.each { |j| projected.join(j) }
-      parent_query.where_clauses.each { |w| projected.where(w) }
+      relation, _parent, parent_query = candidates.first
+      parent_ids_clause(relation, parent_query)
+    end
 
+    # `<fk> IN (<the parent's extraction query, projected to its primary key>)`.
+    private def parent_ids_clause(relation, parent_query)
+      parent = table_by_name.fetch(relation.table_name)
       QueryAst::WhereClause.new(
         column_name: relation.foreign_key,
         operator: :in_subquery,
-        value: QueryAst::SelectSubquery.new(query: projected)
+        value: QueryAst::SelectSubquery.new(query: project_query_to(parent_query, parent.primary_key))
+      )
+    end
+
+    # A JOIN through a tree table applies that table's own conditions, which its
+    # ancestors are exempt from, so such a route narrows by the first hop's
+    # extraction query instead. A batched table keeps the JOIN its batches slice.
+    private def route_through_tree?(table, route_tables)
+      return false if batched?(table)
+
+      !tree_tables_in(route_tables).empty?
+    end
+
+    private def tree_tables_in(table_names)
+      table_names.select { |name| (hop = table_by_name[name]) && self.class.self_ancestor_links(hop).any? }
+    end
+
+    # nil keeps the JOIN: on a cycle, or when the parent is unconstrained or
+    # scoped through this very table.
+    private def route_parent_query(table, relation)
+      parent = table_by_name[relation.table_name]
+      return nil if parent.nil? || parent.primary_key.nil?
+      return nil if parent.name == table.name || @forward_path.include?(parent.name)
+
+      parent_query = self.class.run(
+        parent.name, table_by_name, dump_target, @logger,
+        forward_path: @forward_path + [table.name], reverse_path: @reverse_path, deep_chain_warned: @deep_chain_warned
       )
+      return nil unless parent_query.where_clauses.any? || parent_query.join_clauses.any?
+      return nil if QueryAst.reads_table?(parent_query, table.name)
+
+      parent_query
+    end
+
+    private def batched?(table)
+      table.respond_to?(:batch_scope) && !table.batch_scope.nil?
     end
 
     private def polymorphic_parent_groups(table)
@@ -695,6 +775,28 @@ module Exwiw
       )
     end
 
+    # Single-target counterpart of #path_or_parent_query_arm; nil keeps the JOIN.
+    private def dump_target_tree_route_clauses(table)
+      return nil if table.name == dump_target.table_name
+
+      relation = table.belongs_to(dump_target.table_name)
+      route = relation ? [dump_target.table_name] : dump_target_route_after_first_hop(table)
+      return nil if route.empty? || !route_through_tree?(table, route)
+
+      relation ||= table.belongs_to(route.first)
+      parent_query = route_parent_query(table, relation)
+      return nil if parent_query.nil?
+
+      [parent_ids_clause(relation, parent_query)].tap do |clauses|
+        clauses.push(polymorphic_type_clause(relation)) if relation.polymorphic?
+      end
+    end
+
+    private def dump_target_route_after_first_hop(table)
+      path = find_path_to_dump_target(table, table_by_name, dump_target)
+      path.size < 2 ? [] : path.drop(1) + [dump_target.table_name]
+    end
+
     # Single-target counterpart of #scoped_arms.
     private def target_polymorphic_arm_queries(table)
       return nil if table.name == dump_target.table_name
@@ -721,18 +823,25 @@ module Exwiw
     end
 
     private def target_arm_query(table, relation)
-      if relation.table_name == dump_target.table_name
+      target = table_by_name[relation.table_name]
+      return nil if target.nil?
+
+      on_target = relation.table_name == dump_target.table_name
+      path = on_target ? [] : find_path_to_dump_target(target, table_by_name, dump_target)
+      joinable = on_target || (path.any? && !path.include?(table.name))
+      route = on_target ? [target.name] : path + [dump_target.table_name]
+      if joinable && route_through_tree?(table, route) && (parent_query = route_parent_query(table, relation))
+        return target_ids_arm_query(table, relation, parent_query)
+      end
+
+      if on_target
         return arm_pk_query(table).tap do |query|
           query.where dump_target_fk_clause(relation.foreign_key)
           query.where polymorphic_type_clause(relation)
         end
       end
 
-      target = table_by_name[relation.table_name]
-      return nil if target.nil?
-
-      path = find_path_to_dump_target(target, table_by_name, dump_target)
-      if path.any? && !path.include?(table.name)
+      if joinable
         return arm_pk_query(table).tap do |query|
           query.join(build_join_clause(table, target, relation))
           build_join_clauses(target, table_by_name, dump_target).each { |join_clause| query.join(join_clause) }
@@ -824,13 +933,7 @@ module Exwiw
     end
 
     private def build_scoped(table)
-      ast = QueryAst::Select.new
-      ast.from(table.name)
-      if table.rails_managed?
-        ast.select_all!
-      else
-        ast.select(table.columns)
-      end
+      ast = new_extraction_query(table)
 
       # Reference/master (or rails-managed) table: export every row.
       return ast if scope_exempt?(table)
@@ -852,6 +955,10 @@ module Exwiw
         build_scoped_join_clauses(arm.path, first_relation: arm.relation).each { |join_clause| ast.join(join_clause) }
         ast.where(table.filter) if table.filter
         return ast
+      elsif arms.size == 1 && !arms.first.relation.polymorphic?
+        ast.where(parent_ids_clause(arms.first.relation, arms.first.target_query))
+        ast.where(table.filter) if table.filter
+        return ast
       elsif arms.size >= 1
         # Several polymorphic arms: INNER JOINing one of them would drop the rows
         # of every other type, so constrain this table to the UNION of the ids the
@@ -1048,6 +1155,30 @@ module Exwiw
       terminus
     end
 
+    # These gaps should reject the batch_scope like the shapes
+    # #batch_scope_terminus! raises on, but only warn so that configs batching a
+    # table that now keeps ancestors still run. Called once from the pre-flight,
+    # after #batch_scope_terminus! accepts the shape; #batch_scope_terminus!
+    # itself also runs mid-extraction.
+    def warn_batch_scope_ancestor_gaps
+      table = table_by_name.fetch(table_name)
+
+      if self.class.self_ancestor_links(table).any?
+        @logger.warn(
+          "  #{table.name} references itself, but a batched table does not add the ancestors " \
+          "of its rows (every batch would repeat them); a kept row's parent may be missing."
+        )
+      end
+
+      route = directly_scoped?(table) ? [] : scoped_arms(table).first.path.drop(1)
+      tree_tables_in(route).each do |tree|
+        @logger.warn(
+          "  #{table.name} is batched through #{tree}, which keeps the ancestors of its rows, but " \
+          "the batches join #{tree} directly; #{table.name} rows of those ancestors are not extracted."
+        )
+      end
+    end
+
     # BFS over belongs_tos to the nearest *directly scoped* ancestor. Unlike the
     # target-mode walk, the returned path INCLUDES that ancestor: the scope column
     # lives on the ancestor itself (not on a foreign key of the child), so the
@@ -1145,7 +1276,7 @@ module Exwiw
         else
           sibling_arms = polymorphic_sibling_arms(table, shortest[1])
           if sibling_arms.nil?
-            [ScopedArm.new(relation: nil, path: shortest)]
+            [path_or_parent_query_arm(table, table.belongs_to(shortest[1]), shortest, pinned: false)]
           else
             arms = sibling_arms.filter_map { |relation| resolve_scoped_arm(table, relation) }
             @logger.debug(
@@ -1170,12 +1301,21 @@ module Exwiw
     # up referenced_by / reverse_scope / the multi-hop cascade for free.
     private def resolve_scoped_arm(table, relation)
       path = find_path_to_scoped(table, first_relation: relation)
-      return ScopedArm.new(relation: relation, path: path) if path.size >= 2
+      return path_or_parent_query_arm(table, relation, path, pinned: true) if path.size >= 2
 
       target_query = scoped_target_query(table, relation, allow_automatic_reverse: true)
       target_query && ScopedArm.new(relation: relation, target_query: target_query)
     end
 
+    # `pinned` keeps the relation on a JOIN arm, which only a polymorphic arm needs.
+    private def path_or_parent_query_arm(table, relation, path, pinned:)
+      if route_through_tree?(table, path.drop(1)) && (parent_query = route_parent_query(table, relation))
+        return ScopedArm.new(relation: relation, target_query: parent_query)
+      end
+
+      ScopedArm.new(relation: pinned ? relation : nil, path: path)
+    end
+
     private def scoped_target_arm_query(table, relation)
       # A parent kept only by the automatic referenced_by exists to keep a child's
       # foreign key valid. Single-target mode extracts only what sits under the
@@ -1307,15 +1447,8 @@ module Exwiw
     # to be constrained: the foreign key alone cannot tell this arm's rows from
     # another arm's (record_id=1 may be any type).
     private def target_ids_arm_query(table, relation, target_query)
-      target = table_by_name.fetch(relation.table_name)
       arm_pk_query(table).tap do |query|
-        query.where QueryAst::WhereClause.new(
-          column_name: relation.foreign_key,
-          operator: :in_subquery,
-          value: QueryAst::SelectSubquery.new(
-            query: project_query_to(target_query, target.primary_key)
-          )
-        )
+        query.where parent_ids_clause(relation, target_query)
         query.where polymorphic_type_clause(relation)
       end
     end
@@ -1328,14 +1461,14 @@ module Exwiw
       )
     end
 
-    # Reduce an extraction query to a single-column id projection, keeping its
-    # joins and filters. The column is forced to a plain TableColumn so any
-    # masking (`replace_with` / `raw_sql`) configured on it cannot corrupt the id
+    # Reduce an extraction query to an id projection, keeping its joins and
+    # filters. The columns are forced to plain TableColumns so any masking
+    # (`replace_with` / `raw_sql`) configured on them cannot corrupt the id
     # comparison.
-    private def project_query_to(query, column_name)
+    private def project_query_to(query, column_names)
       projected = QueryAst::Select.new
       projected.from(query.from_table_name)
-      projected.select([TableColumn.from_symbol_keys(name: column_name)])
+      projected.select(Array(column_names).map { |name| TableColumn.from_symbol_keys(name: name) })
       query.join_clauses.each { |join_clause| projected.join(join_clause) }
       query.where_clauses.each { |where_clause| projected.where(where_clause) }
       projected

data/lib/exwiw/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Exwiw
-  VERSION = "1.1.6"
+  VERSION = "1.1.7"
 end