exwiw 0.5.2 → 0.6.0

data/lib/exwiw/adapter/mongodb_adapter.rb

@@ -14,6 +14,57 @@ module Exwiw
         Exwiw::MongodbCollectionConfig
       end
 
+      # A lazy, streaming stand-in for the materialized result array #execute
+      # used to return. Wrapping the Mongo cursor (instead of `.to_a`) keeps the
+      # dump's dominant memory cost — the full result set — off the heap: the
+      # Runner pulls documents through `each_slice`, so at most one chunk of
+      # documents (plus the small propagation-key arrays) is resident at a time,
+      # even for large or embed-heavy collections.
+      #
+      # It satisfies the two things the Runner asks of an execute result:
+      #   - #size: the record count, used to skip empty collections and to log.
+      #     Answered with a `count_documents` query (which only walks index
+      #     entries, far cheaper than fetching every document) rather than by
+      #     draining the cursor.
+      #   - #each (via Enumerable / each_slice): a single streaming pass over the
+      #     cursor. While streaming it captures — per propagation key, BEFORE
+      #     handing the document to the caller's masking — the values downstream
+      #     children will `$in`-match against, publishing them into @state once
+      #     the pass completes.
+      #
+      # Contract note: unlike the old `.to_a` execute, which populated @state
+      # eagerly, this defers state capture until the result is consumed. The
+      # Runner always fully consumes a non-empty result before any child
+      # collection is processed, so propagation is unaffected; a caller that only
+      # needs @state must iterate the result (e.g. `.to_a`).
+      class StreamingResult
+        include Enumerable
+
+        def initialize(view:, collection:, keys:, state:)
+          @view = view
+          @collection = collection
+          @keys = keys
+          @state = state
+        end
+
+        def size
+          @size ||= @view.count_documents
+        end
+        alias length size
+
+        def each
+          return enum_for(:each) { size } unless block_given?
+
+          captured = @keys.each_with_object({}) { |key, acc| acc[key] = [] }
+          @view.each do |doc|
+            @keys.each { |key| captured[key] << doc[key] }
+            yield doc
+          end
+          @state[@collection] = captured
+          self
+        end
+      end
+
       def initialize(connection_config, logger)
         super
         @state = {}
@@ -86,22 +137,24 @@ module Exwiw
       def execute(query)
         @logger.debug("  Executing Mongo find on '#{query.collection}': filter=#{query.filter.inspect} projection=#{query.projection.inspect}")
 
-        docs = db[query.collection]
+        view = db[query.collection]
           .find(query.filter)
           .projection(query.projection)
           .comment(query_comment_text("collection=#{query.collection}"))
-          .to_a
 
-        # Stash, per referenced field, the values children will `$in`-match
-        # against. @propagation_keys is set by the build_query call for this same
+        # Per referenced field, the values children will `$in`-match against.
+        # @propagation_keys is set by the build_query call for this same
         # collection; fall back to the primary key if execute is driven without a
         # preceding build_query (e.g. in isolation from a test).
         keys = @propagation_keys || [query.primary_key]
-        @state[query.collection] = keys.each_with_object({}) do |key, acc|
-          acc[key] = docs.map { |doc| doc[key] }
-        end
 
-        docs
+        # Return a streaming view of the result set rather than `.to_a`-ing the
+        # whole collection into memory. The Runner pulls documents through
+        # `each_slice`, so only one chunk's worth is resident at a time even for
+        # large / embed-heavy collections — the dump's dominant memory cost. The
+        # propagation-key values are captured as the cursor streams and published
+        # into @state once the pass completes (see StreamingResult).
+        StreamingResult.new(view: view, collection: query.collection, keys: keys, state: @state)
       end
 
       # NOTE: relies on @embedded_children_by_parent set by a prior build_query
@@ -110,10 +163,10 @@ module Exwiw
       # to_bulk_insert (SQL adapters don't need it). Safe in Runner, fragile in
       # tests — call build_query first.
       def to_bulk_insert(rows, config)
+        plan = mask_plan(config)
         rows.map do |doc|
-          apply_replace_with!(doc, config)
-          apply_embedded_masking!(doc, config)
-          JSON.generate(extended_json(doc))
+          apply_mask_plan!(doc, plan)
+          Exwiw::ExtJson.encode(doc)
         end.join("\n")
       end
 
@@ -135,6 +188,20 @@ module Exwiw
         'jsonl'
       end
 
+      # Bound how many documents are serialized at once when a collection config
+      # carries no explicit bulk_insert_chunk_size. A MongoDB dump is one JSONL
+      # line per document and, without chunking, the Runner would materialize the
+      # entire collection's output as a single giant string while the full
+      # in-memory result set is still alive — doubling peak memory on large or
+      # embed-heavy collections. Chunking lets the Runner stream each slice to the
+      # file and release its serialized string (and the transient extended-JSON
+      # trees) before building the next.
+      DEFAULT_BULK_INSERT_CHUNK_SIZE = 1_000
+
+      def default_bulk_insert_chunk_size
+        DEFAULT_BULK_INSERT_CHUNK_SIZE
+      end
+
       def schema_output_extension
         'js'
       end
@@ -325,49 +392,101 @@ module Exwiw
         parent_fields[reference_field]
       end
 
-      private def apply_replace_with!(doc, config)
-        config.fields.each do |field|
+      # A masking plan compiled once per collection config and reused for every
+      # document of that collection. `masked_fields` is `[field_name,
+      # template_segments]` for each field carrying a `replace_with`;
+      # `embedded` is one EmbeddedMask per embedded child.
+      MaskPlan = Struct.new(:masked_fields, :embedded)
+
+      # A pre-resolved embedded-child mask: the parent path split once into
+      # `prefix` (the containers to descend into) and `last` (the field holding
+      # the subdocument(s)), plus the child's own MaskPlan.
+      EmbeddedMask = Struct.new(:prefix, :last, :plan)
+
+      # Build (or fetch) the cached MaskPlan for `config`. Masking runs over every
+      # document AND every embedded subdocument, so for an embed-heavy collection
+      # the same per-config decisions — which fields carry a `replace_with`, how
+      # each template splits into segments, where the embedded children live —
+      # were previously recomputed tens of times per document. Compiling them once
+      # per config lets #apply_mask_plan! do nothing but the work that actually
+      # varies per document (rendering templates, descending into subdocuments),
+      # so the saved per-subdocument overhead scales down with embedding count.
+      #
+      # Cached by config name: names are unique within a run and the configs do
+      # not mutate mid-dump. Relies on @embedded_children_by_parent, set by the
+      # build_query call that always precedes to_bulk_insert (see #to_bulk_insert).
+      private def mask_plan(config)
+        (@mask_plans ||= {})[config.name] ||= build_mask_plan(config)
+      end
+
+      private def build_mask_plan(config)
+        masked_fields = config.fields.each_with_object([]) do |field, acc|
           next unless field.replace_with
 
-          doc[field.name] = field.replace_with.gsub(/\{([^{}]+)\}/) do
-            ref = Regexp.last_match(1)
-            (doc.key?(ref) ? doc[ref] : nil).to_s
-          end
+          acc << [field.name, compile_template(field.replace_with)]
+        end
+        embedded = embedded_children_of(config).map do |child|
+          *prefix, last = child.embedded_in.path.split(".")
+          EmbeddedMask.new(prefix, last, build_mask_plan(child))
         end
+        MaskPlan.new(masked_fields, embedded)
       end
 
-      private def apply_embedded_masking!(doc, parent_config)
-        embedded_children_of(parent_config).each do |child|
-          walk(doc, child.embedded_in.path) do |subdoc|
-            apply_replace_with!(subdoc, child)
-            apply_embedded_masking!(subdoc, child)
+      # Apply a precompiled MaskPlan to a document in place: render each masked
+      # field, then descend into each embedded child (recursing into its own
+      # plan). Equivalent to the old apply_replace_with! + apply_embedded_masking!
+      # pair, with all per-config lookups hoisted into the plan.
+      private def apply_mask_plan!(doc, plan)
+        plan.masked_fields.each do |name, segments|
+          doc[name] = render_template(segments, doc)
+        end
+        plan.embedded.each do |child|
+          container = child.prefix.reduce(doc) { |acc, seg| acc.is_a?(Hash) ? acc[seg] : nil }
+          next unless container.is_a?(Hash)
+
+          case (value = container[child.last])
+          when Array then value.each { |sub| apply_mask_plan!(sub, child.plan) if sub.is_a?(Hash) }
+          when Hash  then apply_mask_plan!(value, child.plan)
           end
         end
       end
 
-      private def embedded_children_of(parent_config)
-        @embedded_children_by_parent.fetch(parent_config.name, [])
+      PLACEHOLDER_PATTERN = /\{([^{}]+)\}/
+
+      # Split a `replace_with` template into a flat list of segments (called once
+      # per masked field at plan-build time, see #build_mask_plan). A segment is
+      # either a literal String or a 1-element Array `[ref]` marking a `{ref}`
+      # placeholder. #render_template then concatenates them, skipping the regex
+      # scan / block / `Regexp.last_match` a per-document `gsub` would repeat (~2.5x
+      # faster per field). The segment walk reproduces the old gsub byte-for-byte
+      # (missing keys render as "", literals pass through unchanged).
+      private def compile_template(template)
+        segments = []
+        pos = 0
+        while (md = PLACEHOLDER_PATTERN.match(template, pos))
+          segments << template[pos...md.begin(0)] if md.begin(0) > pos
+          segments << [md[1]]
+          pos = md.end(0)
+        end
+        segments << template[pos..] if pos < template.length
+        segments
       end
 
-      private def walk(doc, dotted_path)
-        segments = dotted_path.split(".")
-        *prefix, last = segments
-        container = prefix.reduce(doc) { |acc, seg| acc.is_a?(Hash) ? acc[seg] : nil }
-        return unless container.is_a?(Hash)
-
-        value = container[last]
-        case value
-        when Array then value.each { |sub| yield sub if sub.is_a?(Hash) }
-        when Hash  then yield value
+      private def render_template(segments, doc)
+        out = +''
+        segments.each do |seg|
+          if seg.is_a?(Array)
+            ref = seg[0]
+            out << (doc.key?(ref) ? doc[ref] : nil).to_s
+          else
+            out << seg
+          end
         end
+        out
       end
 
-      private def extended_json(doc)
-        if doc.respond_to?(:as_extended_json)
-          doc.as_extended_json(mode: :relaxed)
-        else
-          doc
-        end
+      private def embedded_children_of(parent_config)
+        @embedded_children_by_parent.fetch(parent_config.name, [])
       end
 
       private def db

data/lib/exwiw/adapter/mysql_adapter.rb

@@ -5,15 +5,67 @@ require 'open3'
 module Exwiw
   module Adapter
     class MysqlAdapter < Base
+      include SqlBulkInsert
+
+      # A lazy, streaming stand-in for the materialized rows #execute used to
+      # return (`connection.query(sql).rows`). It pulls rows off the wire one at
+      # a time (mysql2 single-row stream) instead of buffering the whole result
+      # set, so the dump's dominant memory cost — a Ruby array as large as the
+      # table — never materializes. The Runner drives it exactly like the old
+      # Array: #size to skip empty tables and log the count, then a single
+      # streaming pass (SqlBulkInsert#write_inserts -> each_slice).
+      #
+      # Mirrors PostgresqlAdapter::StreamingResult, with two MySQL specifics:
+      #   - #size runs a separate `SELECT COUNT(*)` of the same query. Unlike the
+      #     pg path, it does NOT wrap the SELECT in a subquery: MySQL rejects a
+      #     derived table with duplicate column names, which a rails-managed
+      #     `SELECT *` joined to another table produces. Instead the projection
+      #     is replaced by `COUNT(*)` (compile_ast(count_only: true)) — exact
+      #     because exwiw's extraction queries have no DISTINCT/GROUP BY/LIMIT,
+      #     so the row count is independent of the projected columns.
+      #   - the stream ties up the connection until fully drained. The Runner
+      #     always drains it (write_inserts) before any further query
+      #     (post_insert_sql / DELETE), and MysqlClient#stream_rows drains the
+      #     remainder if iteration is abandoned, so the connection stays usable.
+      class StreamingResult
+        include Enumerable
+
+        def initialize(client:, data_sql:, count_sql:)
+          @client = client
+          @data_sql = data_sql
+          @count_sql = count_sql
+        end
+
+        def size
+          @size ||= @client.query(@count_sql).rows.dig(0, 0).to_i
+        end
+        alias length size
+
+        # Stream the result set row by row. Each row is an Array of String|nil
+        # (mysql2 `cast: false` / stringified) — identical to what
+        # `connection.query(sql).rows` produced, so the generated INSERT is
+        # unchanged.
+        def each(&block)
+          return enum_for(:each) { size } unless block_given?
+
+          @client.stream_rows(@data_sql, &block)
+          self
+        end
+      end
+
       def build_query(table, dump_target, table_by_name)
         Exwiw::QueryAstBuilder.run(table.name, table_by_name, dump_target, @logger)
       end
 
       def execute(query_ast)
-        sql = commented_sql(query_ast)
-
-        @logger.debug("  Executing SQL: \n#{sql}")
-        connection.query(sql).rows
+        data_sql = commented_sql(query_ast)
+        # Count via the same FROM/JOIN/WHERE (projection replaced by COUNT(*)) so
+        # the Runner can skip empty tables and log the row count without draining
+        # the stream. See StreamingResult for why this is not a subquery wrap.
+        count_sql = "#{sql_query_comment(query_ast)} #{compile_ast(query_ast, count_only: true)}"
+
+        @logger.debug("  Executing SQL (streaming): \n#{data_sql}")
+        StreamingResult.new(client: connection, data_sql: data_sql, count_sql: count_sql)
       end
 
       def explain(query_ast)
@@ -99,22 +151,16 @@ module Exwiw
         "SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS;"
       end
 
-      def to_bulk_insert(results, table)
+      # The INSERT header for this adapter. MySQL backtick-quotes the table and
+      # column identifiers. #to_bulk_insert / #write_inserts (SqlBulkInsert)
+      # append the value tuples and the trailing `;`.
+      private def insert_header(table)
         table_name = table.name
-
-        value_list = results.map do |row|
-          quoted_values = row.map do |value|
-            escape_value(value)
-          end
-          "(" + quoted_values.join(', ') + ")"
-        end
-        values = value_list.join(",\n")
-
         if table.rails_managed?
-          "INSERT INTO `#{table_name}` VALUES\n#{values};"
+          "INSERT INTO `#{table_name}` VALUES\n"
         else
           column_names = table.columns.map { |c| "`#{c.name}`" }.join(', ')
-          "INSERT INTO `#{table_name}` (#{column_names}) VALUES\n#{values};"
+          "INSERT INTO `#{table_name}` (#{column_names}) VALUES\n"
         end
       end
 
@@ -176,11 +222,17 @@ module Exwiw
         sql
       end
 
-      def compile_ast(query_ast)
+      # @param count_only [Boolean] emit `SELECT COUNT(*)` instead of the
+      #   projected columns (used by StreamingResult#size). Safe because exwiw's
+      #   extraction queries have no DISTINCT/GROUP BY/LIMIT, so the count does
+      #   not depend on the projection.
+      def compile_ast(query_ast, count_only: false)
         raise NotImplementedError unless query_ast.is_a?(Exwiw::QueryAst::Select)
 
         sql = "SELECT "
-        sql += if query_ast.select_all
+        sql += if count_only
+                 "COUNT(*)"
+               elsif query_ast.select_all
                  "*"
                else
                  query_ast.columns.map { |col| compile_column_name(query_ast, col) }.join(', ')

data/lib/exwiw/adapter/mysql_client.rb

@@ -118,6 +118,49 @@ module Exwiw
         end
       end
 
+      # Stream a query's rows one at a time, yielding each as an
+      # Array<String|nil> (the same row shape as #query) instead of buffering
+      # the whole result set. This keeps a large dump's dominant memory cost — a
+      # Ruby array as big as the table — from materializing.
+      #
+      # mysql2 streams server-side (`stream: true` + `cache_rows: false`).
+      # Its contract: a streamed result MUST be fully consumed before the next
+      # query on this connection, or the driver raises "Commands out of sync".
+      # The Runner consumes every row (it writes them all), but if the consumer
+      # block raises mid-stream we drain the remaining rows so the same
+      # connection is still usable for the next table's query.
+      #
+      # trilogy has no streaming cursor (no QUERY_FLAGS_STREAMING), so it buffers
+      # the result and yields from it — parity, but without the memory win (the
+      # same situation as the sqlite adapter). trilogy is a test-only driver;
+      # production connects via mysql2.
+      #
+      # @param sql [String]
+      # @yieldparam row [Array<String|nil>]
+      def stream_rows(sql)
+        return enum_for(:stream_rows, sql) unless block_given?
+
+        case @driver
+        when :mysql2
+          res = raw.query(sql, cast: false, as: :array, stream: true, cache_rows: false)
+          begin
+            res.each { |row| yield row.map { |value| self.class.stringify_value(value) } }
+          rescue StandardError
+            begin
+              res.each { |_row| } # drain the remainder so the connection stays usable
+            rescue StandardError
+              nil
+            end
+            raise
+          end
+        when :trilogy
+          raw.query(sql).rows.each { |row| yield row.map { |value| self.class.stringify_value(value) } }
+        else
+          raise "Unsupported MySQL driver: #{@driver.inspect}"
+        end
+        self
+      end
+
       private def ensure_driver_loaded!
         case @driver
         when :mysql2 then require 'mysql2'

data/lib/exwiw/adapter/postgresql_adapter.rb

@@ -3,15 +3,91 @@
 module Exwiw
   module Adapter
     class PostgresqlAdapter < Base
+      include SqlBulkInsert
+
+      # A lazy, streaming stand-in for the materialized rows #execute used to
+      # return (`connection.exec(sql).values`). It pulls rows off the wire one
+      # at a time via libpq's single-row mode instead of buffering the whole
+      # result set, so the dump's dominant memory cost — a Ruby array as large
+      # as the table — never materializes. The Runner drives it exactly like the
+      # old Array: #size to skip empty tables and log the count, then a single
+      # streaming pass (SqlBulkInsert#write_inserts -> each_slice) to write the
+      # INSERT.
+      #
+      # Mirrors MongodbAdapter::StreamingResult; two SQL-specific differences:
+      #   - #size cannot be answered cheaply from the cursor, so it runs a
+      #     separate `SELECT COUNT(*)` of the same query. (MongoDB uses
+      #     count_documents, an index-only walk; the SQL COUNT re-runs the query
+      #     plan but transfers no row data — Postgres prunes the unused
+      #     projection of the wrapped subquery.) This keeps the Runner contract
+      #     unchanged, so MongoDB and the other SQL adapters are untouched.
+      #   - the streaming pass ties up the connection until fully drained. The
+      #     Runner always drains it (write_inserts) before issuing any further
+      #     query (post_insert_sql / DELETE) on the same connection, so the
+      #     ordering invariant holds.
+      class StreamingResult
+        include Enumerable
+
+        def initialize(connection:, data_sql:, count_sql:)
+          @connection = connection
+          @data_sql = data_sql
+          @count_sql = count_sql
+        end
+
+        def size
+          @size ||= @connection.exec(@count_sql).getvalue(0, 0).to_i
+        end
+        alias length size
+
+        # Stream the result set row by row. Each row is an Array of String|nil
+        # in libpq's text format — byte-identical to what `#exec(sql).values`
+        # produced, so the generated INSERT is unchanged.
+        def each
+          return enum_for(:each) { size } unless block_given?
+
+          @connection.send_query(@data_sql)
+          @connection.set_single_row_mode
+          begin
+            while (result = @connection.get_result)
+              begin
+                result.check
+                result.each_row { |row| yield row }
+              ensure
+                result.clear
+              end
+            end
+          rescue StandardError
+            # If iteration is abandoned mid-stream (a SQL error surfaced by
+            # #check, or the consumer raised), drain any results still queued so
+            # a later query on this same connection does not fail with "another
+            # command is already in progress".
+            drain
+            raise
+          end
+          self
+        end
+
+        private def drain
+          while (result = @connection.get_result)
+            result.clear
+          end
+        rescue PG::Error
+          # Connection already errored/clean; nothing left to drain.
+        end
+      end
+
       def build_query(table, dump_target, table_by_name)
         Exwiw::QueryAstBuilder.run(table.name, table_by_name, dump_target, @logger)
       end
 
       def execute(query_ast)
-        sql = commented_sql(query_ast)
+        data_sql = commented_sql(query_ast)
+        # Count via the same query (wrapped as a subquery) so the Runner can
+        # skip empty tables and log the row count without draining the stream.
+        count_sql = "#{sql_query_comment(query_ast)} SELECT COUNT(*) FROM (#{compile_ast(query_ast)}) AS exwiw_count_src"
 
-        @logger.debug("  Executing SQL: \n#{sql}")
-        connection.exec(sql).values
+        @logger.debug("  Executing SQL (single-row stream): \n#{data_sql}")
+        StreamingResult.new(connection: connection, data_sql: data_sql, count_sql: count_sql)
       end
 
       def explain(query_ast)
@@ -97,22 +173,16 @@ module Exwiw
         @logger.info("  Wrote schema for #{table_names.size} table(s) to #{output_path}.")
       end
 
-      def to_bulk_insert(results, table)
+      # The INSERT header for this adapter. PostgreSQL uses bare identifiers.
+      # #to_bulk_insert / #write_inserts (SqlBulkInsert) append the value tuples
+      # and the trailing `;`.
+      private def insert_header(table)
         table_name = table.name
-
-        value_list = results.map do |row|
-          quoted_values = row.map do |value|
-            escape_value(value)
-          end
-          "(" + quoted_values.join(', ') + ")"
-        end
-        values = value_list.join(",\n")
-
         if table.rails_managed?
-          "INSERT INTO #{table_name} VALUES\n#{values};"
+          "INSERT INTO #{table_name} VALUES\n"
         else
           column_names = table.columns.map(&:name).join(', ')
-          "INSERT INTO #{table_name} (#{column_names}) VALUES\n#{values};"
+          "INSERT INTO #{table_name} (#{column_names}) VALUES\n"
         end
       end
 

data/lib/exwiw/adapter/sql_bulk_insert.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Exwiw
+  module Adapter
+    # Shared bulk-INSERT construction for the SQL adapters (mysql / postgresql /
+    # sqlite). They produce the same `INSERT INTO ... VALUES (...),(...);` shape
+    # and differ only in the header's identifier quoting (see #insert_header) and
+    # in #escape_value, so both the in-memory builder (#to_bulk_insert) and the
+    # bounded-memory streaming writer (#write_inserts) live here.
+    #
+    # Each including adapter must provide two private methods:
+    #   - insert_header(table) -> the "INSERT INTO ... VALUES\n" prefix
+    #   - escape_value(value)  -> the SQL literal for one column value
+    module SqlBulkInsert
+      # How many rows' tuples to build-and-flush at a time when streaming. Bounds
+      # peak memory to this many tuples (plus their joined string) instead of the
+      # whole table's INSERT string, while keeping each flush a single fast
+      # Array#map + Array#join (the same C-level path #to_bulk_insert uses) so it
+      # stays close to whole-string speed — far faster than a naive row-at-a-time
+      # IO#print (see script/bench_sql_dump.rb / docs/sql-dump-optimization-notes.md).
+      # Mirrors MongoDB's default chunk size: bounded work per flush, but the SQL
+      # adapters still emit ONE statement (byte-identical to the un-chunked build).
+      STREAM_FLUSH_ROWS = 2_000
+
+      # Build the whole INSERT statement as a single String. Kept for callers
+      # that want the string form (and as the readable definition of the exact
+      # bytes #write_inserts streams).
+      def to_bulk_insert(results, table)
+        value_list = results.map { |row| insert_tuple(row) }
+        "#{insert_header(table)}#{value_list.join(",\n")};"
+      end
+
+      # Stream the bulk INSERT(s) straight to `io` instead of materializing the
+      # whole statement string first. Byte-for-byte identical to writing
+      # #to_bulk_insert per chunk joined by "\n" (verified by
+      # insert_output_snapshot_spec), but only one ~STREAM_FLUSH_BYTES buffer is
+      # resident at a time rather than the entire table's INSERT string. Returns
+      # the number of statements written.
+      def write_inserts(io, results, table, chunk_size)
+        chunks = chunk_size ? results.each_slice(chunk_size) : [results]
+        statement_count = 0
+        chunks.each do |chunk_rows|
+          io.print("\n") if statement_count.positive?
+          stream_single_insert(io, chunk_rows, table)
+          statement_count += 1
+        end
+        statement_count
+      end
+
+      # Emit one `INSERT INTO ... VALUES <tuples>;` statement to `io`, building
+      # and flushing the value tuples STREAM_FLUSH_ROWS at a time so the full
+      # statement text is never held in memory at once. Each slice is one fast
+      # map+join; the ",\n" between slices reproduces the same separator
+      # #to_bulk_insert puts between every tuple, so the bytes are identical.
+      private def stream_single_insert(io, rows, table)
+        io.print(insert_header(table))
+        first = true
+        rows.each_slice(STREAM_FLUSH_ROWS) do |slice|
+          io.print(",\n") unless first
+          first = false
+          io.print(slice.map { |row| insert_tuple(row) }.join(",\n"))
+        end
+        io.print(";")
+      end
+
+      private def insert_tuple(row)
+        "(" + row.map { |value| escape_value(value) }.join(', ') + ")"
+      end
+    end
+  end
+end