exwiw 0.5.2 → 0.6.0

data/lib/exwiw/adapter/sqlite_adapter.rb

@@ -3,15 +3,72 @@
 module Exwiw
   module Adapter
     class SqliteAdapter < Base
+      include SqlBulkInsert
+
+      # A lazy, streaming stand-in for the materialized rows #execute used to
+      # return (`connection.execute(sql)`). It walks the result one row at a time
+      # via SQLite's statement cursor (Statement#each -> sqlite3_step) instead of
+      # buffering the whole result set, so the dump's dominant memory cost — a
+      # Ruby array as large as the table — never materializes. The Runner drives
+      # it exactly like the old Array: #size to skip empty tables and log the
+      # count, then a single streaming pass (SqlBulkInsert#write_inserts ->
+      # each_slice) to write the INSERT.
+      #
+      # Mirrors Mysql/PostgresqlAdapter::StreamingResult, with two SQLite
+      # specifics:
+      #   - #size runs a separate `SELECT COUNT(*)` of the same query with the
+      #     projection replaced by COUNT(*) (compile_ast(count_only: true)) —
+      #     exact because exwiw's extraction queries have no DISTINCT/GROUP
+      #     BY/LIMIT, so the row count is independent of the projection. (Unlike
+      #     MySQL, SQLite tolerates a duplicate-column subquery wrap too, but the
+      #     count_only form is shared with MySQL and avoids the extra subquery.)
+      #   - SQLite is an embedded, single-connection engine that allows several
+      #     active prepared statements at once, so the #size COUNT and the data
+      #     cursor do not contend. The statement is closed in an ensure block so
+      #     an abandoned mid-stream iteration still releases the cursor.
+      class StreamingResult
+        include Enumerable
+
+        def initialize(connection:, data_sql:, count_sql:)
+          @connection = connection
+          @data_sql = data_sql
+          @count_sql = count_sql
+        end
+
+        def size
+          @size ||= @connection.execute(@count_sql).dig(0, 0).to_i
+        end
+        alias length size
+
+        # Stream the result set row by row. Each row is an Array of values in
+        # SQLite's native type mapping — byte-identical to what
+        # `connection.execute(sql)` produced, so the generated INSERT is unchanged.
+        def each
+          return enum_for(:each) { size } unless block_given?
+
+          statement = @connection.prepare(@data_sql)
+          begin
+            statement.each { |row| yield row }
+          ensure
+            statement.close
+          end
+          self
+        end
+      end
+
       def build_query(table, dump_target, table_by_name)
         Exwiw::QueryAstBuilder.run(table.name, table_by_name, dump_target, @logger)
       end
 
       def execute(query_ast)
-        sql = commented_sql(query_ast)
-
-        @logger.debug("  Executing SQL: \n#{sql}")
-        connection.execute(sql)
+        data_sql = commented_sql(query_ast)
+        # Count via the same FROM/JOIN/WHERE (projection replaced by COUNT(*)) so
+        # the Runner can skip empty tables and log the row count without draining
+        # the cursor. See StreamingResult for why this is exact.
+        count_sql = "#{sql_query_comment(query_ast)} #{compile_ast(query_ast, count_only: true)}"
+
+        @logger.debug("  Executing SQL (cursor stream): \n#{data_sql}")
+        StreamingResult.new(connection: connection, data_sql: data_sql, count_sql: count_sql)
       end
 
       def explain(query_ast)
@@ -63,22 +120,16 @@ module Exwiw
         stmt.end_with?(';') ? stmt : "#{stmt};"
       end
 
-      def to_bulk_insert(results, table)
+      # The INSERT header for this adapter. SQLite uses bare identifiers.
+      # #to_bulk_insert / #write_inserts (SqlBulkInsert) append the value tuples
+      # and the trailing `;`.
+      private def insert_header(table)
         table_name = table.name
-
-        value_list = results.map do |row|
-          quoted_values = row.map do |value|
-            escape_value(value)
-          end
-          "(" + quoted_values.join(', ') + ")"
-        end
-        values = value_list.join(",\n")
-
         if table.rails_managed?
-          "INSERT INTO #{table_name} VALUES\n#{values};"
+          "INSERT INTO #{table_name} VALUES\n"
         else
           column_names = table.columns.map(&:name).join(', ')
-          "INSERT INTO #{table_name} (#{column_names}) VALUES\n#{values};"
+          "INSERT INTO #{table_name} (#{column_names}) VALUES\n"
         end
       end
 
@@ -140,11 +191,17 @@ module Exwiw
         sql
       end
 
-      def compile_ast(query_ast)
+      # @param count_only [Boolean] emit `SELECT COUNT(*)` instead of the
+      #   projected columns (used by StreamingResult#size). Safe because exwiw's
+      #   extraction queries have no DISTINCT/GROUP BY/LIMIT, so the count does
+      #   not depend on the projection.
+      def compile_ast(query_ast, count_only: false)
         raise NotImplementedError unless query_ast.is_a?(Exwiw::QueryAst::Select)
 
         sql = "SELECT "
-        sql += if query_ast.select_all
+        sql += if count_only
+                 "COUNT(*)"
+               elsif query_ast.select_all
                  "*"
                else
                  query_ast.columns.map { |col| compile_column_name(query_ast, col) }.join(', ')

data/lib/exwiw/adapter.rb

@@ -113,6 +113,44 @@ module Exwiw
         raise NotImplementedError, "COPY format is not supported by #{self.class.name}"
       end
 
+      # Default bulk-insert chunk size when a table config does not set one.
+      # The Runner streams each chunk straight to the output file, so a non-nil
+      # value here bounds how much serialized output (and how many transient
+      # intermediate objects) live in memory at once. SQL adapters keep nil
+      # (one statement per table, as before); adapters whose output is large
+      # and built per-row (e.g. MongoDB JSONL) override with a positive value.
+      def default_bulk_insert_chunk_size
+        nil
+      end
+
+      # Write the bulk INSERT/JSONL output for `results` to the open `io`,
+      # returning the number of statements written. The Runner calls this once
+      # per table for the non-COPY path.
+      #
+      # Default: build each chunk's output as a full string via #to_bulk_insert
+      # and write it, separating statements with "\n" — exactly what the Runner
+      # used to inline. This keeps the dominant memory cost at one chunk's
+      # serialized string (bounded by `chunk_size`), which is why MongoDB sets a
+      # positive default chunk size. Adapters whose output is a single large
+      # statement (the SQL adapters, where chunk_size is nil) override this to
+      # stream the statement to `io` in bounded buffers instead of holding the
+      # whole thing in memory.
+      #
+      # @param io [IO] open output file
+      # @param results [Enumerable] rows/documents from #execute
+      # @param table the table/collection config
+      # @param chunk_size [Integer, nil] rows per statement (nil => one statement)
+      def write_inserts(io, results, table, chunk_size)
+        chunks = chunk_size ? results.each_slice(chunk_size) : [results]
+        statement_count = 0
+        chunks.each do |chunk_rows|
+          io.print("\n") if statement_count.positive?
+          io.print(to_bulk_insert(chunk_rows, table))
+          statement_count += 1
+        end
+        statement_count
+      end
+
       # Run the database-specific EXPLAIN for the given query and return the
       # output as a single string for `explain` subcommand to print.
       # SQL adapters override; MongodbAdapter currently raises.

data/lib/exwiw/ext_json.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Exwiw
+  # MongoDB Relaxed Extended JSON encoder for a single dumped document.
+  #
+  # `encode` is the one entry point. When the optional native extension compiled
+  # (the common case once `gem install exwiw` builds it), it emits the line in a
+  # single C tree-walk; otherwise it falls back to the pure-Ruby path. Both are
+  # byte-for-byte identical — the native path delegates every value it does not
+  # format itself back to `encode_fragment` (see ext/exwiw/ext_json/ext_json.c).
+  module ExtJson
+    module_function
+
+    # Pure-Ruby encoder for one value, identical to the historical
+    # `JSON.generate(doc.as_extended_json(mode: :relaxed))`. Used both as the
+    # whole-document fallback and as the native path's per-value delegate, so the
+    # two paths cannot diverge.
+    def encode_fragment(value)
+      JSON.generate(value.respond_to?(:as_extended_json) ? value.as_extended_json(mode: :relaxed) : value)
+    end
+
+    begin
+      require "exwiw/ext_json_native" # defines Exwiw::ExtJson.encode_native
+      def encode(doc) = encode_native(doc)
+    rescue LoadError
+      # No compiled extension (JRuby/TruffleRuby, or a host where the build
+      # failed): keep exwiw working as a pure-Ruby gem.
+      def encode(doc) = encode_fragment(doc)
+    end
+  end
+end

data/lib/exwiw/runner.rb

@@ -97,18 +97,30 @@ module Exwiw
           else
             phase = "generating INSERT statement"
             @logger.debug("  Generate INSERT statement...")
-            chunk_size = table.bulk_insert_chunk_size
-            chunks = chunk_size ? results.each_slice(chunk_size).to_a : [results]
-            insert_sql = chunks.map { |chunk_rows| adapter.to_bulk_insert(chunk_rows, table) }.join("\n")
-
-            @logger.info("  Generated INSERT statement for #{record_num} records (#{chunks.size} statement(s)).")
+            # Let the adapter write the INSERT/JSONL output straight to the file
+            # instead of building the whole table's output as one string first,
+            # so only a bounded amount of serialized text is resident at a time —
+            # important for large tables/collections whose one-shot output would
+            # otherwise be held in full alongside the already-large result set.
+            #
+            # The chunk size falls back to the adapter's default when the table
+            # config does not set one (SQL adapters: nil -> one statement, but
+            # streamed in bounded buffers; MongoDB: a positive default so the
+            # JSONL is chunked). #write_inserts emits bytes identical to the
+            # previous inline chunk loop and returns the statement count.
+            chunk_size = table.bulk_insert_chunk_size || adapter.default_bulk_insert_chunk_size
+
+            statement_count = 0
             File.open(File.join(@output_dir, "insert-#{insert_idx}-#{table_name}.#{adapter.output_extension}"), 'w') do |file|
               pre = adapter.pre_insert_sql(table)
               file.puts(pre) if pre
-              file.puts(insert_sql)
+              statement_count = adapter.write_inserts(file, results, table, chunk_size)
+              file.print("\n")
               post = adapter.post_insert_sql(table)
               file.puts(post) if post
             end
+
+            @logger.info("  Generated INSERT statement for #{record_num} records (#{statement_count} statement(s)).")
           end
 
           if adapter.supports_bulk_delete? && !@insert_only && !(table.respond_to?(:rails_managed?) && table.rails_managed?)

data/lib/exwiw/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Exwiw
-  VERSION = "0.5.2"
+  VERSION = "0.6.0"
 end

data/lib/exwiw.rb

@@ -5,6 +5,7 @@ require_relative "exwiw/version"
 require "json"
 require "serdes"
 
+require_relative "exwiw/ext_json"
 require_relative "exwiw/belongs_to"
 require_relative "exwiw/table_column"
 require_relative "exwiw/table_config"
@@ -13,6 +14,7 @@ require_relative "exwiw/mongodb_field"
 require_relative "exwiw/mongodb_collection_config"
 require_relative "exwiw/ddl_postprocessor"
 require_relative "exwiw/adapter"
+require_relative "exwiw/adapter/sql_bulk_insert"
 require_relative "exwiw/adapter/sqlite_adapter"
 require_relative "exwiw/adapter/mysql_client"
 require_relative "exwiw/adapter/mysql_adapter"

data/mise.toml

@@ -1,6 +1,6 @@
 [env]
-# Prepend scenario/bin so `pg_dump` resolves to the wrapper that delegates to
+# Prepend e2e/bin so `pg_dump` resolves to the wrapper that delegates to
 # the postgres container (compose.yml). exwiw's PostgreSQL adapter shells out
 # to pg_dump, which requires a server/client major-version match — the dev DB
 # is postgres:17 while host clients are often older (e.g. Homebrew pg14).
-_.path = ["./scenario/bin"]
+_.path = ["./e2e/bin"]

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: exwiw
 version: !ruby/object:Gem::Version
-  version: 0.5.2
+  version: 0.6.0
 platform: ruby
 authors:
 - Shia
@@ -29,25 +29,33 @@ email:
 - rise.shia@gmail.com
 executables:
 - exwiw
-extensions: []
+extensions:
+- ext/exwiw/ext_json/extconf.rb
 extra_rdoc_files: []
 files:
 - CHANGELOG.md
 - LICENSE.txt
 - README.md
+- docs/optimization-notes.md
+- docs/optimize-mongodb-export-with-native-ext.md
 - docs/plans/2026-05-15-insert-000-schema-file.md
 - docs/plans/2026-05-16-mongodb-from-clean-scenario.md
 - docs/plans/2026-05-22-after-insert-hook.md
 - docs/plans/2026-05-22-postgres-copy-mode-scenario-test.md
 - docs/plans/2026-05-29-rails-managed-tables.md
 - docs/plans/2026-05-31-ids-column-for-sql-adapters.md
+- docs/plans/2026-06-19-mongodb-export-remove-parallelism-native-ext.md
+- docs/sql-dump-optimization-notes.md
 - exe/exwiw
+- ext/exwiw/ext_json/ext_json.c
+- ext/exwiw/ext_json/extconf.rb
 - lib/exwiw.rb
 - lib/exwiw/adapter.rb
 - lib/exwiw/adapter/mongodb_adapter.rb
 - lib/exwiw/adapter/mysql_adapter.rb
 - lib/exwiw/adapter/mysql_client.rb
 - lib/exwiw/adapter/postgresql_adapter.rb
+- lib/exwiw/adapter/sql_bulk_insert.rb
 - lib/exwiw/adapter/sqlite_adapter.rb
 - lib/exwiw/after_insert_hook.rb
 - lib/exwiw/belongs_to.rb
@@ -56,6 +64,7 @@ files:
 - lib/exwiw/determine_table_processing_order.rb
 - lib/exwiw/embedded_in.rb
 - lib/exwiw/explain_runner.rb
+- lib/exwiw/ext_json.rb
 - lib/exwiw/mongo_query.rb
 - lib/exwiw/mongodb_collection_config.rb
 - lib/exwiw/mongodb_field.rb