exwiw 0.9.12 → 0.9.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4be92980b62c26dbb27cef847c3534134cf9e5d27c4676dc2ff28468168e52ed
-  data.tar.gz: 357c2efdbe2e818fa4e5cce3739b5d77bb05b0dda50817dc9a8e622746b3d6ce
+  metadata.gz: 1e66762378ba8b4357a2d42fff5fbb1f0c12e96979d0e59f6fdcaae46bd7da4d
+  data.tar.gz: 1c1cd268f516ab60b34915f77fc1ab2ed7e6cdfb8d4cc1e508abf026bb5519fa
 SHA512:
-  metadata.gz: 632102dbdeb9b713abf435fff5224565675f7d34843e6a1e447d96b71920018ff2a79f9f27971afc153a27813e6cd68cefc7df790dbc00fe8a371d8d4cb32c5a
-  data.tar.gz: c7dbd4ec63a7b711a276fb4e0c07e5e481621664c5bd07cdc2452b5a60c38c6843e11891cb46d966a969c57e6fe99714d20d5f786a6d9af087d4966180401f58
+  metadata.gz: dd5dda5f6dcfa4fce06b3e9bd7ef5c8d9d45fb32a2606db0bc2dcf5aa047784036bec61d7a4efe49d92632fb4bdd9bb460445b72587503a05e1bd49f0fa1fd63
+  data.tar.gz: 267f0de9ca7fcca1ee19e4005031c5740493ea9d8550e13af6bc199f90f45377d5f9a840f4379a28fa8fe73ec7e26514a0fa0e2523df4df43143663dbed2dbf0

data/CHANGELOG.md

@@ -2,6 +2,18 @@
 
 ## [Unreleased]
 
+## [0.9.14] - 2026-07-29
+
+### Changed
+
+- **SQL adapters now default `bulk_insert_chunk_size` to 10_000** (previously nil = one INSERT statement per table). A single statement covering a multi-million-row table exceeds the import target's `max_allowed_packet` and aborts the load with "MySQL server has gone away". Output for tables at or below 10,000 rows is byte-identical; larger tables now emit multiple bounded INSERT statements. Set `bulk_insert_chunk_size` in a table config to override.
+
+## [0.9.13] - 2026-07-28
+
+### Changed
+
+- **The mysql adapter materializes each scope id-set into a session `TEMPORARY TABLE` and reuses it across tables.** Previously the same scope subquery (e.g. the dump target's users id-set, including a multi-arm `reverse_scope` UNION) was embedded — and re-evaluated by the server — in every descendant table's extraction query, which dominated export time on large tenants. During `export`, each distinct id-set is now computed once (`CREATE TEMPORARY TABLE ... AS SELECT DISTINCT ...`, indexed), and descendant queries JOIN the temp table; nested scopes are materialized bottom-up so outer sets build from already-materialized inner ones. The dumped rows are identical. `explain` still compiles the inline form and executes nothing. Read-only replicas are supported: `NO_ENGINE_SUBSTITUTION` is stripped from the session `sql_mode` before the first CREATE, so e.g. Aurora MySQL 3 reader instances (which cannot create InnoDB temp tables) substitute a permitted engine instead of failing with ERROR 3161. If temp table creation still fails (e.g. the DB user lacks `CREATE TEMPORARY TABLES`), materialization is disabled for the run with a warning and extraction falls back to the previous inline subqueries. postgresql/sqlite adapters are unchanged.
+
 ## [0.9.12] - 2026-07-09
 
 ## [0.9.11] - 2026-07-09

data/README.md

@@ -753,7 +753,7 @@ Unlike rails-managed entries, `columns` and `belongs_tos` are retained so the en
 
 `bulk_insert_chunk_size` splits the generated `INSERT` statement into multiple statements, each containing at most the specified number of rows. This is useful when the number of records per table is large enough to hit limits like MySQL's `max_allowed_packet`.
 
-If omitted, all records for a table are emitted as a single `INSERT` statement.
+If omitted, the adapter default applies: 10,000 rows per statement for the SQL adapters (1,000 documents per chunk for MongoDB). Tables at or below the chunk size still produce a single `INSERT` statement. To force a single statement regardless of table size, set a value larger than the table's row count.
 
 ### Filter
 

data/docs/sql-dump-optimization-notes.md

@@ -34,7 +34,8 @@ The Runner drives, per table:
    downstream.
 
 2. **to_bulk_insert** — SQL adapters set **no** `default_bulk_insert_chunk_size`
-   (it is `nil`), so the Runner treats the whole table as one chunk and
+   (it was `nil` at measurement time; since then the default is 10_000 —
+   large tables emit multiple bounded INSERT statements), so at measurement time the Runner treated the whole table as one chunk and
    `to_bulk_insert` builds the **entire** `INSERT INTO ... VALUES (...),(...);`
    as one giant String — first an `Array` of N per-row tuple strings, then the
    joined result — held simultaneously with the result set from step 1.

data/lib/exwiw/adapter/identifier_quoting.rb

@@ -119,7 +119,7 @@ module Exwiw
         day_minute day_second dec decimal declare default delayed delete
         dense_rank desc describe deterministic distinct distinctrow div double
         drop dual each else elseif empty enclosed escaped except exists exit
-        explain false fetch first_value float float4 float8 for force foreign
+        explain external false fetch first_value float float4 float8 for force foreign
         from fulltext function generated get grant group grouping groups
         having high_priority hour_microsecond hour_minute hour_second if
         ignore in index infile inner inout insensitive insert int int1 int2
@@ -132,13 +132,13 @@ module Exwiw
         minute_second mod modifies natural not no_write_to_binlog nth_value
         ntile null numeric of on optimize optimizer_costs option optionally
         or order out outer outfile over partition percent_rank precision
-        primary procedure purge range rank read reads read_write real
+        primary procedure purge qualify range rank read reads read_write real
         recursive references regexp release rename repeat replace require
         resignal restrict return revoke right rlike row rows row_number
         schema schemas second_microsecond select sensitive separator set show
         signal smallint spatial specific sql sqlexception sqlstate sqlwarning
         sql_big_result sql_calc_found_rows sql_small_result ssl starting
-        stored straight_join system table terminated then tinyblob tinyint
+        stored straight_join system table tablesample terminated then tinyblob tinyint
         tinytext to trailing trigger true undo union unique unlock unsigned
         update usage use using utc_date utc_time utc_timestamp values
         varbinary varchar varcharacter varying virtual when where while

data/lib/exwiw/adapter/mysql_adapter.rb

@@ -59,11 +59,15 @@ module Exwiw
       end
 
       def execute(query_ast)
-        data_sql = commented_sql(query_ast)
+        data_sql = nil
+        count_sql = nil
         # Count via the same FROM/JOIN/WHERE (projection replaced by COUNT(*)) so
         # the Runner can skip empty tables and log the row count without draining
         # the stream. See StreamingResult for why this is not a subquery wrap.
-        count_sql = "#{sql_query_comment(query_ast)} #{compile_ast(query_ast, count_only: true)}"
+        with_scope_materialization do
+          data_sql = commented_sql(query_ast)
+          count_sql = "#{sql_query_comment(query_ast)} #{compile_ast(query_ast, count_only: true)}"
+        end
 
         @logger.debug("  Executing SQL (streaming): \n#{data_sql}")
         StreamingResult.new(client: connection, data_sql: data_sql, count_sql: count_sql)
@@ -292,16 +296,86 @@ module Exwiw
       # to `<col> IN (subquery)`.
       private def compile_scope_join(from_table_name, where_clause, idx)
         subquery = where_clause.value
-        projection = quote_identifier(subquery_projection_name(subquery))
-        src_alias = "exwiw_scope_src_#{idx}"
         ids_alias = "exwiw_scope_ids_#{idx}"
         outer_key = qualified_name(from_table_name, where_clause.column_name)
 
+        if (scope_table = materialized_scope_table(subquery))
+          return "JOIN #{quote_table_name(scope_table)} AS #{ids_alias} " \
+            "ON #{outer_key} = #{ids_alias}.exwiw_scope_id"
+        end
+
+        projection = quote_identifier(subquery_projection_name(subquery))
+        src_alias = "exwiw_scope_src_#{idx}"
+
         "JOIN (SELECT DISTINCT #{src_alias}.#{projection} AS exwiw_scope_id " \
           "FROM (#{compile_subquery(subquery)}) AS #{src_alias}) AS #{ids_alias} " \
           "ON #{outer_key} = #{ids_alias}.exwiw_scope_id"
       end
 
+      # The same scope subquery (e.g. the target-tenant users id-set) is embedded
+      # in every descendant table's extraction query, so the source DB would
+      # re-evaluate it once per table — the dominant cost on large tenants.
+      # During #execute, materialize each distinct id-set once into a session
+      # TEMPORARY TABLE and JOIN that instead. Keyed by the compiled SELECT, so
+      # nested scopes reuse already-materialized parents. #explain and
+      # #describe_query compile without the flag and stay side-effect free.
+      private def with_scope_materialization
+        @materialize_scopes = true
+        yield
+      ensure
+        @materialize_scopes = false
+      end
+
+      private def materialized_scope_table(subquery)
+        return nil unless @materialize_scopes
+        return nil if @scope_materialization_disabled
+
+        select_sql = "SELECT DISTINCT exwiw_scope_src.#{quote_identifier(subquery_projection_name(subquery))} " \
+          "AS exwiw_scope_id FROM (#{compile_subquery(subquery)}) AS exwiw_scope_src"
+
+        @scope_id_tables ||= {}
+        begin
+          prepare_scope_session
+          @scope_id_tables[select_sql] ||= create_scope_id_table(select_sql)
+        rescue StandardError => e
+          # e.g. the DB user lacks CREATE TEMPORARY TABLES; keep extracting with
+          # the inline (per-query) scope subqueries instead of failing the run.
+          @scope_materialization_disabled = true
+          @logger.warn("Disabling scope id-set materialization (#{e.class}: #{e.message}); " \
+            "falling back to inline scope subqueries.")
+          nil
+        end
+      end
+
+      # Read-only replicas (e.g. Aurora MySQL 3 reader instances) cannot create
+      # InnoDB temporary tables: with NO_ENGINE_SUBSTITUTION in sql_mode the
+      # CREATE fails outright (ERROR 3161) instead of substituting a permitted
+      # engine such as MyISAM. Strip that flag once per session — it only
+      # governs DDL engine fallback, not query semantics.
+      private def prepare_scope_session
+        return if @scope_session_prepared
+
+        mode = connection.query("SELECT @@SESSION.sql_mode").rows.dig(0, 0).to_s
+        cleaned = mode.split(',').reject { |part| part == 'NO_ENGINE_SUBSTITUTION' }.join(',')
+        connection.query("SET SESSION sql_mode = '#{cleaned}'") unless cleaned == mode
+        @scope_session_prepared = true
+      end
+
+      # If a step after CREATE fails, the temp table stays in the session until
+      # disconnect, but it is never referenced: the name is only cached (and thus
+      # only joined) after all three statements succeed, and the caller disables
+      # materialization for the rest of the run.
+      private def create_scope_id_table(select_sql)
+        name = "exwiw_scope_id_set_#{@scope_id_tables.size}"
+        connection.query("CREATE TEMPORARY TABLE #{quote_table_name(name)} AS #{select_sql}")
+        # ROW_COUNT() reads the CTAS insert count in O(1); a COUNT(*) would
+        # re-scan the whole id-set just for this log line.
+        count = connection.query("SELECT ROW_COUNT()").rows.dig(0, 0)
+        connection.query("ALTER TABLE #{quote_table_name(name)} ADD INDEX `index_exwiw_scope_id` (exwiw_scope_id)")
+        @logger.info("  Materialized scope id set #{name} (#{count} ids).")
+        name
+      end
+
       private def compile_where_condition(where_clause, table_name)
         # Use as it is if it's a raw query
         return where_clause if where_clause.is_a?(String)

data/lib/exwiw/adapter/sql_bulk_insert.rb

@@ -18,8 +18,8 @@ module Exwiw
       # Array#map + Array#join (the same C-level path #to_bulk_insert uses) so it
       # stays close to whole-string speed — far faster than a naive row-at-a-time
       # IO#print (see script/bench_sql_dump.rb / docs/sql-dump-optimization-notes.md).
-      # Mirrors MongoDB's default chunk size: bounded work per flush, but the SQL
-      # adapters still emit ONE statement (byte-identical to the un-chunked build).
+      # Bounded work per flush WITHIN a single statement: flush boundaries never
+      # split a statement — statement boundaries come from bulk_insert_chunk_size.
       STREAM_FLUSH_ROWS = 2_000
 
       # Build the whole INSERT statement as a single String. Kept for callers
@@ -37,15 +37,28 @@ module Exwiw
       # resident at a time rather than the entire table's INSERT string. Returns
       # [statement_count, record_count]; record_count is tallied during the single
       # streaming drain so the Runner needs no separate SELECT COUNT(*) pass.
+      # Chunks are buffered off `#each` rather than `results.each_slice(...)`:
+      # each_slice consults the receiver's `#size` (verified on CRuby for both
+      # the block and enumerator forms), which on a streaming result issues a
+      # redundant `SELECT COUNT(*)` — a second full pass over the same filter.
+      # Manual buffering walks the cursor exactly once.
       def write_inserts(io, results, table, chunk_size)
-        chunks = chunk_size ? results.each_slice(chunk_size) : [results]
+        return [1, stream_single_insert(io, results, table)] unless chunk_size
+
         statement_count = 0
         record_count = 0
-        chunks.each do |chunk_rows|
+        buffer = []
+        flush = lambda do
           io.print("\n") if statement_count.positive?
-          record_count += stream_single_insert(io, chunk_rows, table)
+          record_count += stream_single_insert(io, buffer, table)
           statement_count += 1
+          buffer.clear
         end
+        results.each do |row|
+          buffer << row
+          flush.call if buffer.size >= chunk_size
+        end
+        flush.call unless buffer.empty?
         [statement_count, record_count]
       end
 

data/lib/exwiw/adapter.rb

@@ -120,13 +120,14 @@ module Exwiw
       end
 
       # Default bulk-insert chunk size when a table config does not set one.
-      # The Runner streams each chunk straight to the output file, so a non-nil
-      # value here bounds how much serialized output (and how many transient
-      # intermediate objects) live in memory at once. SQL adapters keep nil
-      # (one statement per table, as before); adapters whose output is large
-      # and built per-row (e.g. MongoDB JSONL) override with a positive value.
+      # The Runner streams each chunk straight to the output file, so this
+      # bounds how much serialized output lives in memory at once — and, for
+      # SQL adapters, bounds the size of each INSERT statement: a single
+      # statement covering a multi-million-row table exceeds the target
+      # server's max_allowed_packet at import time ("MySQL server has gone
+      # away"). Table configs can override per table.
       def default_bulk_insert_chunk_size
-        nil
+        10_000
       end
 
       # Write the bulk INSERT/JSONL output for `results` to the open `io`,
@@ -164,9 +165,9 @@ module Exwiw
       # untouched, so the cursor is walked exactly once. The chunk boundaries and
       # "\n" separators reproduce the each_slice output byte-for-byte.
       #
-      # chunk_size is always positive for callers of this default (MongoDB); the
-      # SQL adapters pass nil and override #write_inserts, so the unbounded
-      # nil-branch buffer is never reached here in practice.
+      # chunk_size is always positive for callers of this default (MongoDB,
+      # whose adapter default is 1_000); the SQL adapters override
+      # #write_inserts (SqlBulkInsert) with their own chunked writer.
       def write_inserts(io, results, table, chunk_size)
         statement_count = 0
         record_count = 0

data/lib/exwiw/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Exwiw
-  VERSION = "0.9.12"
+  VERSION = "0.9.14"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: exwiw
 version: !ruby/object:Gem::Version
-  version: 0.9.12
+  version: 0.9.14
 platform: ruby
 authors:
 - Shia