exwiw 0.5.3 → 0.6.1

data/lib/exwiw/adapter/postgresql_adapter.rb

@@ -3,15 +3,91 @@
 module Exwiw
   module Adapter
     class PostgresqlAdapter < Base
+      include SqlBulkInsert
+
+      # A lazy, streaming stand-in for the materialized rows #execute used to
+      # return (`connection.exec(sql).values`). It pulls rows off the wire one
+      # at a time via libpq's single-row mode instead of buffering the whole
+      # result set, so the dump's dominant memory cost — a Ruby array as large
+      # as the table — never materializes. The Runner drives it exactly like the
+      # old Array: #size to skip empty tables and log the count, then a single
+      # streaming pass (SqlBulkInsert#write_inserts -> each_slice) to write the
+      # INSERT.
+      #
+      # Mirrors MongodbAdapter::StreamingResult; two SQL-specific differences:
+      #   - #size cannot be answered cheaply from the cursor, so it runs a
+      #     separate `SELECT COUNT(*)` of the same query. (MongoDB uses
+      #     count_documents, an index-only walk; the SQL COUNT re-runs the query
+      #     plan but transfers no row data — Postgres prunes the unused
+      #     projection of the wrapped subquery.) This keeps the Runner contract
+      #     unchanged, so MongoDB and the other SQL adapters are untouched.
+      #   - the streaming pass ties up the connection until fully drained. The
+      #     Runner always drains it (write_inserts) before issuing any further
+      #     query (post_insert_sql / DELETE) on the same connection, so the
+      #     ordering invariant holds.
+      class StreamingResult
+        include Enumerable
+
+        def initialize(connection:, data_sql:, count_sql:)
+          @connection = connection
+          @data_sql = data_sql
+          @count_sql = count_sql
+        end
+
+        def size
+          @size ||= @connection.exec(@count_sql).getvalue(0, 0).to_i
+        end
+        alias length size
+
+        # Stream the result set row by row. Each row is an Array of String|nil
+        # in libpq's text format — byte-identical to what `#exec(sql).values`
+        # produced, so the generated INSERT is unchanged.
+        def each
+          return enum_for(:each) { size } unless block_given?
+
+          @connection.send_query(@data_sql)
+          @connection.set_single_row_mode
+          begin
+            while (result = @connection.get_result)
+              begin
+                result.check
+                result.each_row { |row| yield row }
+              ensure
+                result.clear
+              end
+            end
+          rescue StandardError
+            # If iteration is abandoned mid-stream (a SQL error surfaced by
+            # #check, or the consumer raised), drain any results still queued so
+            # a later query on this same connection does not fail with "another
+            # command is already in progress".
+            drain
+            raise
+          end
+          self
+        end
+
+        private def drain
+          while (result = @connection.get_result)
+            result.clear
+          end
+        rescue PG::Error
+          # Connection already errored/clean; nothing left to drain.
+        end
+      end
+
       def build_query(table, dump_target, table_by_name)
         Exwiw::QueryAstBuilder.run(table.name, table_by_name, dump_target, @logger)
       end
 
       def execute(query_ast)
-        sql = commented_sql(query_ast)
+        data_sql = commented_sql(query_ast)
+        # Count via the same query (wrapped as a subquery) so the Runner can
+        # skip empty tables and log the row count without draining the stream.
+        count_sql = "#{sql_query_comment(query_ast)} SELECT COUNT(*) FROM (#{compile_ast(query_ast)}) AS exwiw_count_src"
 
-        @logger.debug("  Executing SQL: \n#{sql}")
-        connection.exec(sql).values
+        @logger.debug("  Executing SQL (single-row stream): \n#{data_sql}")
+        StreamingResult.new(connection: connection, data_sql: data_sql, count_sql: count_sql)
       end
 
       def explain(query_ast)
@@ -97,22 +173,16 @@ module Exwiw
         @logger.info("  Wrote schema for #{table_names.size} table(s) to #{output_path}.")
       end
 
-      def to_bulk_insert(results, table)
+      # The INSERT header for this adapter. PostgreSQL uses bare identifiers.
+      # #to_bulk_insert / #write_inserts (SqlBulkInsert) append the value tuples
+      # and the trailing `;`.
+      private def insert_header(table)
         table_name = table.name
-
-        value_list = results.map do |row|
-          quoted_values = row.map do |value|
-            escape_value(value)
-          end
-          "(" + quoted_values.join(', ') + ")"
-        end
-        values = value_list.join(",\n")
-
         if table.rails_managed?
-          "INSERT INTO #{table_name} VALUES\n#{values};"
+          "INSERT INTO #{table_name} VALUES\n"
         else
           column_names = table.columns.map(&:name).join(', ')
-          "INSERT INTO #{table_name} (#{column_names}) VALUES\n#{values};"
+          "INSERT INTO #{table_name} (#{column_names}) VALUES\n"
         end
       end
 

data/lib/exwiw/adapter/sql_bulk_insert.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Exwiw
+  module Adapter
+    # Shared bulk-INSERT construction for the SQL adapters (mysql / postgresql /
+    # sqlite). They produce the same `INSERT INTO ... VALUES (...),(...);` shape
+    # and differ only in the header's identifier quoting (see #insert_header) and
+    # in #escape_value, so both the in-memory builder (#to_bulk_insert) and the
+    # bounded-memory streaming writer (#write_inserts) live here.
+    #
+    # Each including adapter must provide two private methods:
+    #   - insert_header(table) -> the "INSERT INTO ... VALUES\n" prefix
+    #   - escape_value(value)  -> the SQL literal for one column value
+    module SqlBulkInsert
+      # How many rows' tuples to build-and-flush at a time when streaming. Bounds
+      # peak memory to this many tuples (plus their joined string) instead of the
+      # whole table's INSERT string, while keeping each flush a single fast
+      # Array#map + Array#join (the same C-level path #to_bulk_insert uses) so it
+      # stays close to whole-string speed — far faster than a naive row-at-a-time
+      # IO#print (see script/bench_sql_dump.rb / docs/sql-dump-optimization-notes.md).
+      # Mirrors MongoDB's default chunk size: bounded work per flush, but the SQL
+      # adapters still emit ONE statement (byte-identical to the un-chunked build).
+      STREAM_FLUSH_ROWS = 2_000
+
+      # Build the whole INSERT statement as a single String. Kept for callers
+      # that want the string form (and as the readable definition of the exact
+      # bytes #write_inserts streams).
+      def to_bulk_insert(results, table)
+        value_list = results.map { |row| insert_tuple(row) }
+        "#{insert_header(table)}#{value_list.join(",\n")};"
+      end
+
+      # Stream the bulk INSERT(s) straight to `io` instead of materializing the
+      # whole statement string first. Byte-for-byte identical to writing
+      # #to_bulk_insert per chunk joined by "\n" (verified by
+      # insert_output_snapshot_spec), but only one ~STREAM_FLUSH_BYTES buffer is
+      # resident at a time rather than the entire table's INSERT string. Returns
+      # the number of statements written.
+      def write_inserts(io, results, table, chunk_size)
+        chunks = chunk_size ? results.each_slice(chunk_size) : [results]
+        statement_count = 0
+        chunks.each do |chunk_rows|
+          io.print("\n") if statement_count.positive?
+          stream_single_insert(io, chunk_rows, table)
+          statement_count += 1
+        end
+        statement_count
+      end
+
+      # Emit one `INSERT INTO ... VALUES <tuples>;` statement to `io`, building
+      # and flushing the value tuples STREAM_FLUSH_ROWS at a time so the full
+      # statement text is never held in memory at once. Each slice is one fast
+      # map+join; the ",\n" between slices reproduces the same separator
+      # #to_bulk_insert puts between every tuple, so the bytes are identical.
+      private def stream_single_insert(io, rows, table)
+        io.print(insert_header(table))
+        first = true
+        rows.each_slice(STREAM_FLUSH_ROWS) do |slice|
+          io.print(",\n") unless first
+          first = false
+          io.print(slice.map { |row| insert_tuple(row) }.join(",\n"))
+        end
+        io.print(";")
+      end
+
+      private def insert_tuple(row)
+        "(" + row.map { |value| escape_value(value) }.join(', ') + ")"
+      end
+    end
+  end
+end

data/lib/exwiw/adapter/sqlite_adapter.rb

@@ -3,15 +3,72 @@
 module Exwiw
   module Adapter
     class SqliteAdapter < Base
+      include SqlBulkInsert
+
+      # A lazy, streaming stand-in for the materialized rows #execute used to
+      # return (`connection.execute(sql)`). It walks the result one row at a time
+      # via SQLite's statement cursor (Statement#each -> sqlite3_step) instead of
+      # buffering the whole result set, so the dump's dominant memory cost — a
+      # Ruby array as large as the table — never materializes. The Runner drives
+      # it exactly like the old Array: #size to skip empty tables and log the
+      # count, then a single streaming pass (SqlBulkInsert#write_inserts ->
+      # each_slice) to write the INSERT.
+      #
+      # Mirrors Mysql/PostgresqlAdapter::StreamingResult, with two SQLite
+      # specifics:
+      #   - #size runs a separate `SELECT COUNT(*)` of the same query with the
+      #     projection replaced by COUNT(*) (compile_ast(count_only: true)) —
+      #     exact because exwiw's extraction queries have no DISTINCT/GROUP
+      #     BY/LIMIT, so the row count is independent of the projection. (Unlike
+      #     MySQL, SQLite tolerates a duplicate-column subquery wrap too, but the
+      #     count_only form is shared with MySQL and avoids the extra subquery.)
+      #   - SQLite is an embedded, single-connection engine that allows several
+      #     active prepared statements at once, so the #size COUNT and the data
+      #     cursor do not contend. The statement is closed in an ensure block so
+      #     an abandoned mid-stream iteration still releases the cursor.
+      class StreamingResult
+        include Enumerable
+
+        def initialize(connection:, data_sql:, count_sql:)
+          @connection = connection
+          @data_sql = data_sql
+          @count_sql = count_sql
+        end
+
+        def size
+          @size ||= @connection.execute(@count_sql).dig(0, 0).to_i
+        end
+        alias length size
+
+        # Stream the result set row by row. Each row is an Array of values in
+        # SQLite's native type mapping — byte-identical to what
+        # `connection.execute(sql)` produced, so the generated INSERT is unchanged.
+        def each
+          return enum_for(:each) { size } unless block_given?
+
+          statement = @connection.prepare(@data_sql)
+          begin
+            statement.each { |row| yield row }
+          ensure
+            statement.close
+          end
+          self
+        end
+      end
+
       def build_query(table, dump_target, table_by_name)
         Exwiw::QueryAstBuilder.run(table.name, table_by_name, dump_target, @logger)
       end
 
       def execute(query_ast)
-        sql = commented_sql(query_ast)
-
-        @logger.debug("  Executing SQL: \n#{sql}")
-        connection.execute(sql)
+        data_sql = commented_sql(query_ast)
+        # Count via the same FROM/JOIN/WHERE (projection replaced by COUNT(*)) so
+        # the Runner can skip empty tables and log the row count without draining
+        # the cursor. See StreamingResult for why this is exact.
+        count_sql = "#{sql_query_comment(query_ast)} #{compile_ast(query_ast, count_only: true)}"
+
+        @logger.debug("  Executing SQL (cursor stream): \n#{data_sql}")
+        StreamingResult.new(connection: connection, data_sql: data_sql, count_sql: count_sql)
       end
 
       def explain(query_ast)
@@ -63,22 +120,16 @@ module Exwiw
         stmt.end_with?(';') ? stmt : "#{stmt};"
       end
 
-      def to_bulk_insert(results, table)
+      # The INSERT header for this adapter. SQLite uses bare identifiers.
+      # #to_bulk_insert / #write_inserts (SqlBulkInsert) append the value tuples
+      # and the trailing `;`.
+      private def insert_header(table)
         table_name = table.name
-
-        value_list = results.map do |row|
-          quoted_values = row.map do |value|
-            escape_value(value)
-          end
-          "(" + quoted_values.join(', ') + ")"
-        end
-        values = value_list.join(",\n")
-
         if table.rails_managed?
-          "INSERT INTO #{table_name} VALUES\n#{values};"
+          "INSERT INTO #{table_name} VALUES\n"
         else
           column_names = table.columns.map(&:name).join(', ')
-          "INSERT INTO #{table_name} (#{column_names}) VALUES\n#{values};"
+          "INSERT INTO #{table_name} (#{column_names}) VALUES\n"
         end
       end
 
@@ -140,11 +191,17 @@ module Exwiw
         sql
       end
 
-      def compile_ast(query_ast)
+      # @param count_only [Boolean] emit `SELECT COUNT(*)` instead of the
+      #   projected columns (used by StreamingResult#size). Safe because exwiw's
+      #   extraction queries have no DISTINCT/GROUP BY/LIMIT, so the count does
+      #   not depend on the projection.
+      def compile_ast(query_ast, count_only: false)
         raise NotImplementedError unless query_ast.is_a?(Exwiw::QueryAst::Select)
 
         sql = "SELECT "
-        sql += if query_ast.select_all
+        sql += if count_only
+                 "COUNT(*)"
+               elsif query_ast.select_all
                  "*"
                else
                  query_ast.columns.map { |col| compile_column_name(query_ast, col) }.join(', ')

data/lib/exwiw/adapter.rb

@@ -123,6 +123,34 @@ module Exwiw
         nil
       end
 
+      # Write the bulk INSERT/JSONL output for `results` to the open `io`,
+      # returning the number of statements written. The Runner calls this once
+      # per table for the non-COPY path.
+      #
+      # Default: build each chunk's output as a full string via #to_bulk_insert
+      # and write it, separating statements with "\n" — exactly what the Runner
+      # used to inline. This keeps the dominant memory cost at one chunk's
+      # serialized string (bounded by `chunk_size`), which is why MongoDB sets a
+      # positive default chunk size. Adapters whose output is a single large
+      # statement (the SQL adapters, where chunk_size is nil) override this to
+      # stream the statement to `io` in bounded buffers instead of holding the
+      # whole thing in memory.
+      #
+      # @param io [IO] open output file
+      # @param results [Enumerable] rows/documents from #execute
+      # @param table the table/collection config
+      # @param chunk_size [Integer, nil] rows per statement (nil => one statement)
+      def write_inserts(io, results, table, chunk_size)
+        chunks = chunk_size ? results.each_slice(chunk_size) : [results]
+        statement_count = 0
+        chunks.each do |chunk_rows|
+          io.print("\n") if statement_count.positive?
+          io.print(to_bulk_insert(chunk_rows, table))
+          statement_count += 1
+        end
+        statement_count
+      end
+
       # Run the database-specific EXPLAIN for the given query and return the
       # output as a single string for `explain` subcommand to print.
       # SQL adapters override; MongodbAdapter currently raises.

data/lib/exwiw/ext_json.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Exwiw
+  # MongoDB Relaxed Extended JSON encoder for a single dumped document.
+  #
+  # `encode` is the one entry point. When the optional native extension compiled
+  # (the common case once `gem install exwiw` builds it), it emits the line in a
+  # single C tree-walk; otherwise it falls back to the pure-Ruby path. Both are
+  # byte-for-byte identical — the native path delegates every value it does not
+  # format itself back to `encode_fragment` (see ext/exwiw/ext_json/ext_json.c).
+  module ExtJson
+    module_function
+
+    # Pure-Ruby encoder for one value, identical to the historical
+    # `JSON.generate(doc.as_extended_json(mode: :relaxed))`. Used both as the
+    # whole-document fallback and as the native path's per-value delegate, so the
+    # two paths cannot diverge.
+    def encode_fragment(value)
+      JSON.generate(value.respond_to?(:as_extended_json) ? value.as_extended_json(mode: :relaxed) : value)
+    end
+
+    begin
+      require "exwiw/ext_json_native" # defines Exwiw::ExtJson.encode_native
+      def encode(doc) = encode_native(doc)
+    rescue LoadError
+      # No compiled extension (JRuby/TruffleRuby, or a host where the build
+      # failed): keep exwiw working as a pure-Ruby gem.
+      def encode(doc) = encode_fragment(doc)
+    end
+  end
+end

data/lib/exwiw/runner.rb

@@ -97,30 +97,24 @@ module Exwiw
           else
             phase = "generating INSERT statement"
             @logger.debug("  Generate INSERT statement...")
-            # Stream each chunk straight to the file instead of building the whole
-            # table's INSERT/JSONL output as one string first. This keeps only a
-            # single chunk's serialized text (and its transient intermediate
-            # objects) in memory at a time — important for large MongoDB
-            # collections, whose one-giant-chunk JSONL would otherwise be held in
-            # full alongside the already-large in-memory result set.
+            # Let the adapter write the INSERT/JSONL output straight to the file
+            # instead of building the whole table's output as one string first,
+            # so only a bounded amount of serialized text is resident at a time —
+            # important for large tables/collections whose one-shot output would
+            # otherwise be held in full alongside the already-large result set.
             #
             # The chunk size falls back to the adapter's default when the table
-            # config does not set one (SQL adapters: nil -> one statement, as
-            # before; MongoDB: a positive default so the output is chunked). The
-            # bytes written are identical to joining the chunks with "\n" and
-            # appending a trailing newline, matching the previous `file.puts`.
+            # config does not set one (SQL adapters: nil -> one statement, but
+            # streamed in bounded buffers; MongoDB: a positive default so the
+            # JSONL is chunked). #write_inserts emits bytes identical to the
+            # previous inline chunk loop and returns the statement count.
             chunk_size = table.bulk_insert_chunk_size || adapter.default_bulk_insert_chunk_size
-            chunks = chunk_size ? results.each_slice(chunk_size) : [results]
 
             statement_count = 0
             File.open(File.join(@output_dir, "insert-#{insert_idx}-#{table_name}.#{adapter.output_extension}"), 'w') do |file|
               pre = adapter.pre_insert_sql(table)
               file.puts(pre) if pre
-              chunks.each do |chunk_rows|
-                file.print("\n") if statement_count.positive?
-                file.print(adapter.to_bulk_insert(chunk_rows, table))
-                statement_count += 1
-              end
+              statement_count = adapter.write_inserts(file, results, table, chunk_size)
               file.print("\n")
               post = adapter.post_insert_sql(table)
               file.puts(post) if post

data/lib/exwiw/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Exwiw
-  VERSION = "0.5.3"
+  VERSION = "0.6.1"
 end

data/lib/exwiw.rb

@@ -5,6 +5,7 @@ require_relative "exwiw/version"
 require "json"
 require "serdes"
 
+require_relative "exwiw/ext_json"
 require_relative "exwiw/belongs_to"
 require_relative "exwiw/table_column"
 require_relative "exwiw/table_config"
@@ -13,6 +14,7 @@ require_relative "exwiw/mongodb_field"
 require_relative "exwiw/mongodb_collection_config"
 require_relative "exwiw/ddl_postprocessor"
 require_relative "exwiw/adapter"
+require_relative "exwiw/adapter/sql_bulk_insert"
 require_relative "exwiw/adapter/sqlite_adapter"
 require_relative "exwiw/adapter/mysql_client"
 require_relative "exwiw/adapter/mysql_adapter"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: exwiw
 version: !ruby/object:Gem::Version
-  version: 0.5.3
+  version: 0.6.1
 platform: ruby
 authors:
 - Shia
@@ -29,12 +29,14 @@ email:
 - rise.shia@gmail.com
 executables:
 - exwiw
-extensions: []
+extensions:
+- ext/exwiw/ext_json/extconf.rb
 extra_rdoc_files: []
 files:
 - CHANGELOG.md
 - LICENSE.txt
 - README.md
+- docs/mongodb-scoping-fullscan-notes.md
 - docs/optimization-notes.md
 - docs/optimize-mongodb-export-with-native-ext.md
 - docs/plans/2026-05-15-insert-000-schema-file.md
@@ -44,13 +46,17 @@ files:
 - docs/plans/2026-05-29-rails-managed-tables.md
 - docs/plans/2026-05-31-ids-column-for-sql-adapters.md
 - docs/plans/2026-06-19-mongodb-export-remove-parallelism-native-ext.md
+- docs/sql-dump-optimization-notes.md
 - exe/exwiw
+- ext/exwiw/ext_json/ext_json.c
+- ext/exwiw/ext_json/extconf.rb
 - lib/exwiw.rb
 - lib/exwiw/adapter.rb
 - lib/exwiw/adapter/mongodb_adapter.rb
 - lib/exwiw/adapter/mysql_adapter.rb
 - lib/exwiw/adapter/mysql_client.rb
 - lib/exwiw/adapter/postgresql_adapter.rb
+- lib/exwiw/adapter/sql_bulk_insert.rb
 - lib/exwiw/adapter/sqlite_adapter.rb
 - lib/exwiw/after_insert_hook.rb
 - lib/exwiw/belongs_to.rb
@@ -59,6 +65,7 @@ files:
 - lib/exwiw/determine_table_processing_order.rb
 - lib/exwiw/embedded_in.rb
 - lib/exwiw/explain_runner.rb
+- lib/exwiw/ext_json.rb
 - lib/exwiw/mongo_query.rb
 - lib/exwiw/mongodb_collection_config.rb
 - lib/exwiw/mongodb_field.rb