exwiw 0.9.12 → 0.9.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4be92980b62c26dbb27cef847c3534134cf9e5d27c4676dc2ff28468168e52ed
-  data.tar.gz: 357c2efdbe2e818fa4e5cce3739b5d77bb05b0dda50817dc9a8e622746b3d6ce
+  metadata.gz: bc7d217b16a94d775d54d12c668886e433a44212a92631ea47258c194d71b542
+  data.tar.gz: 8ca5d14a2289315c1595ac249dc6bfc9ecdea1806403321850adadfcb8ae18d4
 SHA512:
-  metadata.gz: 632102dbdeb9b713abf435fff5224565675f7d34843e6a1e447d96b71920018ff2a79f9f27971afc153a27813e6cd68cefc7df790dbc00fe8a371d8d4cb32c5a
-  data.tar.gz: c7dbd4ec63a7b711a276fb4e0c07e5e481621664c5bd07cdc2452b5a60c38c6843e11891cb46d966a969c57e6fe99714d20d5f786a6d9af087d4966180401f58
+  metadata.gz: 134aa36ec540862c384ea51cbab3f1cd04a8ed14710bac097cb6aa9aeaf97dae03b9b74d752da3c3bd594390b70a5238961ad30a8713ca0b5c6403114c4bc666
+  data.tar.gz: 3b7a46ac569963c46faead09ad5f7357e3be1a815e0fa8bcc641a3287e8e1aa07a18af62cb552e8a09b4615e64376892df16e8aae0ad4cba048ee05b6555e6b5

data/CHANGELOG.md

@@ -2,6 +2,12 @@
 
 ## [Unreleased]
 
+## [0.9.13] - 2026-07-28
+
+### Changed
+
+- **The mysql adapter materializes each scope id-set into a session `TEMPORARY TABLE` and reuses it across tables.** Previously the same scope subquery (e.g. the dump target's users id-set, including a multi-arm `reverse_scope` UNION) was embedded — and re-evaluated by the server — in every descendant table's extraction query, which dominated export time on large tenants. During `export`, each distinct id-set is now computed once (`CREATE TEMPORARY TABLE ... AS SELECT DISTINCT ...`, indexed), and descendant queries JOIN the temp table; nested scopes are materialized bottom-up so outer sets build from already-materialized inner ones. The dumped rows are identical. `explain` still compiles the inline form and executes nothing. Read-only replicas are supported: `NO_ENGINE_SUBSTITUTION` is stripped from the session `sql_mode` before the first CREATE, so e.g. Aurora MySQL 3 reader instances (which cannot create InnoDB temp tables) substitute a permitted engine instead of failing with ERROR 3161. If temp table creation still fails (e.g. the DB user lacks `CREATE TEMPORARY TABLES`), materialization is disabled for the run with a warning and extraction falls back to the previous inline subqueries. postgresql/sqlite adapters are unchanged.
+
 ## [0.9.12] - 2026-07-09
 
 ## [0.9.11] - 2026-07-09

data/lib/exwiw/adapter/identifier_quoting.rb

@@ -119,7 +119,7 @@ module Exwiw
         day_minute day_second dec decimal declare default delayed delete
         dense_rank desc describe deterministic distinct distinctrow div double
         drop dual each else elseif empty enclosed escaped except exists exit
-        explain false fetch first_value float float4 float8 for force foreign
+        explain external false fetch first_value float float4 float8 for force foreign
         from fulltext function generated get grant group grouping groups
         having high_priority hour_microsecond hour_minute hour_second if
         ignore in index infile inner inout insensitive insert int int1 int2
@@ -132,13 +132,13 @@ module Exwiw
         minute_second mod modifies natural not no_write_to_binlog nth_value
         ntile null numeric of on optimize optimizer_costs option optionally
         or order out outer outfile over partition percent_rank precision
-        primary procedure purge range rank read reads read_write real
+        primary procedure purge qualify range rank read reads read_write real
         recursive references regexp release rename repeat replace require
         resignal restrict return revoke right rlike row rows row_number
         schema schemas second_microsecond select sensitive separator set show
         signal smallint spatial specific sql sqlexception sqlstate sqlwarning
         sql_big_result sql_calc_found_rows sql_small_result ssl starting
-        stored straight_join system table terminated then tinyblob tinyint
+        stored straight_join system table tablesample terminated then tinyblob tinyint
         tinytext to trailing trigger true undo union unique unlock unsigned
         update usage use using utc_date utc_time utc_timestamp values
         varbinary varchar varcharacter varying virtual when where while

data/lib/exwiw/adapter/mysql_adapter.rb

@@ -59,11 +59,15 @@ module Exwiw
       end
 
       def execute(query_ast)
-        data_sql = commented_sql(query_ast)
+        data_sql = nil
+        count_sql = nil
         # Count via the same FROM/JOIN/WHERE (projection replaced by COUNT(*)) so
         # the Runner can skip empty tables and log the row count without draining
         # the stream. See StreamingResult for why this is not a subquery wrap.
-        count_sql = "#{sql_query_comment(query_ast)} #{compile_ast(query_ast, count_only: true)}"
+        with_scope_materialization do
+          data_sql = commented_sql(query_ast)
+          count_sql = "#{sql_query_comment(query_ast)} #{compile_ast(query_ast, count_only: true)}"
+        end
 
         @logger.debug("  Executing SQL (streaming): \n#{data_sql}")
         StreamingResult.new(client: connection, data_sql: data_sql, count_sql: count_sql)
@@ -292,16 +296,86 @@ module Exwiw
       # to `<col> IN (subquery)`.
       private def compile_scope_join(from_table_name, where_clause, idx)
         subquery = where_clause.value
-        projection = quote_identifier(subquery_projection_name(subquery))
-        src_alias = "exwiw_scope_src_#{idx}"
         ids_alias = "exwiw_scope_ids_#{idx}"
         outer_key = qualified_name(from_table_name, where_clause.column_name)
 
+        if (scope_table = materialized_scope_table(subquery))
+          return "JOIN #{quote_table_name(scope_table)} AS #{ids_alias} " \
+            "ON #{outer_key} = #{ids_alias}.exwiw_scope_id"
+        end
+
+        projection = quote_identifier(subquery_projection_name(subquery))
+        src_alias = "exwiw_scope_src_#{idx}"
+
         "JOIN (SELECT DISTINCT #{src_alias}.#{projection} AS exwiw_scope_id " \
           "FROM (#{compile_subquery(subquery)}) AS #{src_alias}) AS #{ids_alias} " \
           "ON #{outer_key} = #{ids_alias}.exwiw_scope_id"
       end
 
+      # The same scope subquery (e.g. the target-tenant users id-set) is embedded
+      # in every descendant table's extraction query, so the source DB would
+      # re-evaluate it once per table — the dominant cost on large tenants.
+      # During #execute, materialize each distinct id-set once into a session
+      # TEMPORARY TABLE and JOIN that instead. Keyed by the compiled SELECT, so
+      # nested scopes reuse already-materialized parents. #explain and
+      # #describe_query compile without the flag and stay side-effect free.
+      private def with_scope_materialization
+        @materialize_scopes = true
+        yield
+      ensure
+        @materialize_scopes = false
+      end
+
+      private def materialized_scope_table(subquery)
+        return nil unless @materialize_scopes
+        return nil if @scope_materialization_disabled
+
+        select_sql = "SELECT DISTINCT exwiw_scope_src.#{quote_identifier(subquery_projection_name(subquery))} " \
+          "AS exwiw_scope_id FROM (#{compile_subquery(subquery)}) AS exwiw_scope_src"
+
+        @scope_id_tables ||= {}
+        begin
+          prepare_scope_session
+          @scope_id_tables[select_sql] ||= create_scope_id_table(select_sql)
+        rescue StandardError => e
+          # e.g. the DB user lacks CREATE TEMPORARY TABLES; keep extracting with
+          # the inline (per-query) scope subqueries instead of failing the run.
+          @scope_materialization_disabled = true
+          @logger.warn("Disabling scope id-set materialization (#{e.class}: #{e.message}); " \
+            "falling back to inline scope subqueries.")
+          nil
+        end
+      end
+
+      # Read-only replicas (e.g. Aurora MySQL 3 reader instances) cannot create
+      # InnoDB temporary tables: with NO_ENGINE_SUBSTITUTION in sql_mode the
+      # CREATE fails outright (ERROR 3161) instead of substituting a permitted
+      # engine such as MyISAM. Strip that flag once per session — it only
+      # governs DDL engine fallback, not query semantics.
+      private def prepare_scope_session
+        return if @scope_session_prepared
+
+        mode = connection.query("SELECT @@SESSION.sql_mode").rows.dig(0, 0).to_s
+        cleaned = mode.split(',').reject { |part| part == 'NO_ENGINE_SUBSTITUTION' }.join(',')
+        connection.query("SET SESSION sql_mode = '#{cleaned}'") unless cleaned == mode
+        @scope_session_prepared = true
+      end
+
+      # If a step after CREATE fails, the temp table stays in the session until
+      # disconnect, but it is never referenced: the name is only cached (and thus
+      # only joined) after all three statements succeed, and the caller disables
+      # materialization for the rest of the run.
+      private def create_scope_id_table(select_sql)
+        name = "exwiw_scope_id_set_#{@scope_id_tables.size}"
+        connection.query("CREATE TEMPORARY TABLE #{quote_table_name(name)} AS #{select_sql}")
+        # ROW_COUNT() reads the CTAS insert count in O(1); a COUNT(*) would
+        # re-scan the whole id-set just for this log line.
+        count = connection.query("SELECT ROW_COUNT()").rows.dig(0, 0)
+        connection.query("ALTER TABLE #{quote_table_name(name)} ADD INDEX `index_exwiw_scope_id` (exwiw_scope_id)")
+        @logger.info("  Materialized scope id set #{name} (#{count} ids).")
+        name
+      end
+
       private def compile_where_condition(where_clause, table_name)
         # Use as it is if it's a raw query
         return where_clause if where_clause.is_a?(String)

data/lib/exwiw/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Exwiw
-  VERSION = "0.9.12"
+  VERSION = "0.9.13"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: exwiw
 version: !ruby/object:Gem::Version
-  version: 0.9.12
+  version: 0.9.13
 platform: ruby
 authors:
 - Shia