exwiw 0.5.1 → 0.5.3

data/lib/exwiw/runner.rb

@@ -44,7 +44,7 @@ module Exwiw
       QueryAstBuilder.validate_scope!(dumpable_configs, table_by_name, @dump_target, @logger)
 
       @logger.info("Determining table processing order...")
-      ordered_table_names = DetermineTableProcessingOrder.run(dumpable_configs)
+      ordered_table_names = DetermineTableProcessingOrder.run(dumpable_configs, logger: @logger)
 
       clean_output_dir!
 
@@ -97,18 +97,36 @@ module Exwiw
           else
             phase = "generating INSERT statement"
             @logger.debug("  Generate INSERT statement...")
-            chunk_size = table.bulk_insert_chunk_size
-            chunks = chunk_size ? results.each_slice(chunk_size).to_a : [results]
-            insert_sql = chunks.map { |chunk_rows| adapter.to_bulk_insert(chunk_rows, table) }.join("\n")
-
-            @logger.info("  Generated INSERT statement for #{record_num} records (#{chunks.size} statement(s)).")
+            # Stream each chunk straight to the file instead of building the whole
+            # table's INSERT/JSONL output as one string first. This keeps only a
+            # single chunk's serialized text (and its transient intermediate
+            # objects) in memory at a time — important for large MongoDB
+            # collections, whose one-giant-chunk JSONL would otherwise be held in
+            # full alongside the already-large in-memory result set.
+            #
+            # The chunk size falls back to the adapter's default when the table
+            # config does not set one (SQL adapters: nil -> one statement, as
+            # before; MongoDB: a positive default so the output is chunked). The
+            # bytes written are identical to joining the chunks with "\n" and
+            # appending a trailing newline, matching the previous `file.puts`.
+            chunk_size = table.bulk_insert_chunk_size || adapter.default_bulk_insert_chunk_size
+            chunks = chunk_size ? results.each_slice(chunk_size) : [results]
+
+            statement_count = 0
             File.open(File.join(@output_dir, "insert-#{insert_idx}-#{table_name}.#{adapter.output_extension}"), 'w') do |file|
               pre = adapter.pre_insert_sql(table)
               file.puts(pre) if pre
-              file.puts(insert_sql)
+              chunks.each do |chunk_rows|
+                file.print("\n") if statement_count.positive?
+                file.print(adapter.to_bulk_insert(chunk_rows, table))
+                statement_count += 1
+              end
+              file.print("\n")
               post = adapter.post_insert_sql(table)
               file.puts(post) if post
             end
+
+            @logger.info("  Generated INSERT statement for #{record_num} records (#{statement_count} statement(s)).")
           end
 
           if adapter.supports_bulk_delete? && !@insert_only && !(table.respond_to?(:rails_managed?) && table.rails_managed?)

data/lib/exwiw/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Exwiw
-  VERSION = "0.5.1"
+  VERSION = "0.5.3"
 end

data/mise.toml

@@ -1,6 +1,6 @@
 [env]
-# Prepend scenario/bin so `pg_dump` resolves to the wrapper that delegates to
+# Prepend e2e/bin so `pg_dump` resolves to the wrapper that delegates to
 # the postgres container (compose.yml). exwiw's PostgreSQL adapter shells out
 # to pg_dump, which requires a server/client major-version match — the dev DB
 # is postgres:17 while host clients are often older (e.g. Homebrew pg14).
-_.path = ["./scenario/bin"]
+_.path = ["./e2e/bin"]

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: exwiw
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.5.3
 platform: ruby
 authors:
 - Shia
@@ -35,12 +35,15 @@ files:
 - CHANGELOG.md
 - LICENSE.txt
 - README.md
+- docs/optimization-notes.md
+- docs/optimize-mongodb-export-with-native-ext.md
 - docs/plans/2026-05-15-insert-000-schema-file.md
 - docs/plans/2026-05-16-mongodb-from-clean-scenario.md
 - docs/plans/2026-05-22-after-insert-hook.md
 - docs/plans/2026-05-22-postgres-copy-mode-scenario-test.md
 - docs/plans/2026-05-29-rails-managed-tables.md
 - docs/plans/2026-05-31-ids-column-for-sql-adapters.md
+- docs/plans/2026-06-19-mongodb-export-remove-parallelism-native-ext.md
 - exe/exwiw
 - lib/exwiw.rb
 - lib/exwiw/adapter.rb