exwiw 0.9.13 → 0.9.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bc7d217b16a94d775d54d12c668886e433a44212a92631ea47258c194d71b542
-  data.tar.gz: 8ca5d14a2289315c1595ac249dc6bfc9ecdea1806403321850adadfcb8ae18d4
+  metadata.gz: 1e66762378ba8b4357a2d42fff5fbb1f0c12e96979d0e59f6fdcaae46bd7da4d
+  data.tar.gz: 1c1cd268f516ab60b34915f77fc1ab2ed7e6cdfb8d4cc1e508abf026bb5519fa
 SHA512:
-  metadata.gz: 134aa36ec540862c384ea51cbab3f1cd04a8ed14710bac097cb6aa9aeaf97dae03b9b74d752da3c3bd594390b70a5238961ad30a8713ca0b5c6403114c4bc666
-  data.tar.gz: 3b7a46ac569963c46faead09ad5f7357e3be1a815e0fa8bcc641a3287e8e1aa07a18af62cb552e8a09b4615e64376892df16e8aae0ad4cba048ee05b6555e6b5
+  metadata.gz: dd5dda5f6dcfa4fce06b3e9bd7ef5c8d9d45fb32a2606db0bc2dcf5aa047784036bec61d7a4efe49d92632fb4bdd9bb460445b72587503a05e1bd49f0fa1fd63
+  data.tar.gz: 267f0de9ca7fcca1ee19e4005031c5740493ea9d8550e13af6bc199f90f45377d5f9a840f4379a28fa8fe73ec7e26514a0fa0e2523df4df43143663dbed2dbf0

data/CHANGELOG.md

@@ -2,6 +2,12 @@
 
 ## [Unreleased]
 
+## [0.9.14] - 2026-07-29
+
+### Changed
+
+- **SQL adapters now default `bulk_insert_chunk_size` to 10_000** (previously nil = one INSERT statement per table). A single statement covering a multi-million-row table exceeds the import target's `max_allowed_packet` and aborts the load with "MySQL server has gone away". Output for tables at or below 10,000 rows is byte-identical; larger tables now emit multiple bounded INSERT statements. Set `bulk_insert_chunk_size` in a table config to override.
+
 ## [0.9.13] - 2026-07-28
 
 ### Changed

data/README.md

@@ -753,7 +753,7 @@ Unlike rails-managed entries, `columns` and `belongs_tos` are retained so the en
 
 `bulk_insert_chunk_size` splits the generated `INSERT` statement into multiple statements, each containing at most the specified number of rows. This is useful when the number of records per table is large enough to hit limits like MySQL's `max_allowed_packet`.
 
-If omitted, all records for a table are emitted as a single `INSERT` statement.
+If omitted, the adapter default applies: 10,000 rows per statement for the SQL adapters (1,000 documents per chunk for MongoDB). Tables at or below the chunk size still produce a single `INSERT` statement. To force a single statement regardless of table size, set a value larger than the table's row count.
 
 ### Filter
 

data/docs/sql-dump-optimization-notes.md

@@ -34,7 +34,8 @@ The Runner drives, per table:
    downstream.
 
 2. **to_bulk_insert** — SQL adapters set **no** `default_bulk_insert_chunk_size`
-   (it is `nil`), so the Runner treats the whole table as one chunk and
+   (it was `nil` at measurement time; since then the default is 10_000 —
+   large tables emit multiple bounded INSERT statements), so at measurement time the Runner treated the whole table as one chunk and
    `to_bulk_insert` builds the **entire** `INSERT INTO ... VALUES (...),(...);`
    as one giant String — first an `Array` of N per-row tuple strings, then the
    joined result — held simultaneously with the result set from step 1.

data/lib/exwiw/adapter/sql_bulk_insert.rb

@@ -18,8 +18,8 @@ module Exwiw
       # Array#map + Array#join (the same C-level path #to_bulk_insert uses) so it
       # stays close to whole-string speed — far faster than a naive row-at-a-time
       # IO#print (see script/bench_sql_dump.rb / docs/sql-dump-optimization-notes.md).
-      # Mirrors MongoDB's default chunk size: bounded work per flush, but the SQL
-      # adapters still emit ONE statement (byte-identical to the un-chunked build).
+      # Bounded work per flush WITHIN a single statement: flush boundaries never
+      # split a statement — statement boundaries come from bulk_insert_chunk_size.
       STREAM_FLUSH_ROWS = 2_000
 
       # Build the whole INSERT statement as a single String. Kept for callers
@@ -37,15 +37,28 @@ module Exwiw
       # resident at a time rather than the entire table's INSERT string. Returns
       # [statement_count, record_count]; record_count is tallied during the single
       # streaming drain so the Runner needs no separate SELECT COUNT(*) pass.
+      # Chunks are buffered off `#each` rather than `results.each_slice(...)`:
+      # each_slice consults the receiver's `#size` (verified on CRuby for both
+      # the block and enumerator forms), which on a streaming result issues a
+      # redundant `SELECT COUNT(*)` — a second full pass over the same filter.
+      # Manual buffering walks the cursor exactly once.
       def write_inserts(io, results, table, chunk_size)
-        chunks = chunk_size ? results.each_slice(chunk_size) : [results]
+        return [1, stream_single_insert(io, results, table)] unless chunk_size
+
         statement_count = 0
         record_count = 0
-        chunks.each do |chunk_rows|
+        buffer = []
+        flush = lambda do
           io.print("\n") if statement_count.positive?
-          record_count += stream_single_insert(io, chunk_rows, table)
+          record_count += stream_single_insert(io, buffer, table)
           statement_count += 1
+          buffer.clear
         end
+        results.each do |row|
+          buffer << row
+          flush.call if buffer.size >= chunk_size
+        end
+        flush.call unless buffer.empty?
         [statement_count, record_count]
       end
 

data/lib/exwiw/adapter.rb

@@ -120,13 +120,14 @@ module Exwiw
       end
 
       # Default bulk-insert chunk size when a table config does not set one.
-      # The Runner streams each chunk straight to the output file, so a non-nil
-      # value here bounds how much serialized output (and how many transient
-      # intermediate objects) live in memory at once. SQL adapters keep nil
-      # (one statement per table, as before); adapters whose output is large
-      # and built per-row (e.g. MongoDB JSONL) override with a positive value.
+      # The Runner streams each chunk straight to the output file, so this
+      # bounds how much serialized output lives in memory at once — and, for
+      # SQL adapters, bounds the size of each INSERT statement: a single
+      # statement covering a multi-million-row table exceeds the target
+      # server's max_allowed_packet at import time ("MySQL server has gone
+      # away"). Table configs can override per table.
       def default_bulk_insert_chunk_size
-        nil
+        10_000
       end
 
       # Write the bulk INSERT/JSONL output for `results` to the open `io`,
@@ -164,9 +165,9 @@ module Exwiw
       # untouched, so the cursor is walked exactly once. The chunk boundaries and
       # "\n" separators reproduce the each_slice output byte-for-byte.
       #
-      # chunk_size is always positive for callers of this default (MongoDB); the
-      # SQL adapters pass nil and override #write_inserts, so the unbounded
-      # nil-branch buffer is never reached here in practice.
+      # chunk_size is always positive for callers of this default (MongoDB,
+      # whose adapter default is 1_000); the SQL adapters override
+      # #write_inserts (SqlBulkInsert) with their own chunked writer.
       def write_inserts(io, results, table, chunk_size)
         statement_count = 0
         record_count = 0

data/lib/exwiw/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Exwiw
-  VERSION = "0.9.13"
+  VERSION = "0.9.14"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: exwiw
 version: !ruby/object:Gem::Version
-  version: 0.9.13
+  version: 0.9.14
 platform: ruby
 authors:
 - Shia