exwiw 0.3.4 → 0.3.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3b68b4eed496cf67cbac408c7f1c233a881b9778b46f0f5a67d0ff50b9321e03
-  data.tar.gz: 521a8cf7d5af0dee4538e407d06d9605d37cb059f5023ee922e11807ff6d1b17
+  metadata.gz: 7c7566e034ea908557d839b53908b3c847444c8097e3b77baa4725603bbf4e52
+  data.tar.gz: 3ca68cf8173d39e3c59f7d7551549ad2a7b6456c1bd16db4f489d11c7bbd9efa
 SHA512:
-  metadata.gz: bdd88c3f13879d23a4ecc7c239f22db2c5c5ddbd3bb06b838ae570d48309ba4f2460a6260d9f848da71717dcf43cae0de3b593e3e6e99f43406d700b39e79a37
-  data.tar.gz: ac2e73630034663ef1ef299fd09925843c086776d10e5998d30758e079a4066f388bdb76672867acacbb57024a5a990336c46e5d8f21d42548862125d1004748
+  metadata.gz: 0b9c68109f7fc4c9abbd814099d38def0aae44fd732de5ec792094bd848dd948b27c17e7ad0139b486eaf658fb5f18422e0ca4cb2b8d26e66098147cbd3a309c
+  data.tar.gz: b3a7ee638ec36b7e9d03bf074b5f35110d7c0fd77db89a079e887c8b2b55bff058cd31d19a4b9479e36cafe59d3b3e55dad175725bdc7c34ed2467c52f20ef08

data/CHANGELOG.md

@@ -2,6 +2,14 @@
 
 ## [Unreleased]
 
+## [0.3.6] - 2026-06-01
+
+## [0.3.5] - 2026-06-01
+
+### Fixed
+
+- MySQL export no longer crashes with `IO#write: "\xE4" from ASCII-8BIT to UTF-8 (Encoding::UndefinedConversionError)` when a value comes from a binary-collation / `VARBINARY` / `BLOB` column but holds UTF-8 text (e.g. Japanese). Both the `mysql2` and `trilogy` drivers tag such values as `ASCII-8BIT`; writing them to the UTF-8 INSERT file failed inside host processes whose `Encoding.default_internal` is UTF-8 (a Rails app, or `RUBYOPT=-EUTF-8`). The driver-returned strings are now re-tagged UTF-8 (bytes unchanged) so the write is a no-op conversion.
+
 ## [0.3.4] - 2026-05-31
 
 ### Changed

data/lib/exwiw/adapter/mongodb_adapter.rb

@@ -120,6 +120,12 @@ module Exwiw
         raise NotImplementedError, "MongodbAdapter does not support explain yet"
       end
 
+      def describe_query(query)
+        "find collection=#{query.collection} filter=#{query.filter.inspect} projection=#{query.projection.inspect}"
+      rescue => e
+        "<unavailable: #{e.class}: #{e.message}>"
+      end
+
       def output_extension
         'jsonl'
       end

data/lib/exwiw/adapter/mysql_client.rb

@@ -57,7 +57,7 @@ module Exwiw
       def self.stringify_value(value)
         case value
         when nil then nil
-        when String then value
+        when String then normalize_encoding(value)
         when Time
           # Emit fractional seconds only when present. A Time can't tell us the
           # column's declared precision, so a zero fraction on a DATETIME(6)
@@ -76,6 +76,21 @@ module Exwiw
         end
       end
 
+      # Re-tag a value string as UTF-8 when it comes back as ASCII-8BIT (BINARY).
+      # Both drivers tag values from binary-collation / VARBINARY / BLOB columns
+      # as ASCII-8BIT even when the bytes are really UTF-8 text. When exwiw runs
+      # inside a host process whose Encoding.default_internal is UTF-8 (e.g. a
+      # Rails app, or RUBYOPT=-EUTF-8), IO#write enables conversion, so writing
+      # such a binary string carrying multi-byte bytes (e.g. Japanese "\xE4...")
+      # to the INSERT file raises "\xE4 from ASCII-8BIT to UTF-8"
+      # (Encoding::UndefinedConversionError). Re-tagging makes that write a
+      # UTF-8 -> UTF-8 no-op; only the tag changes, the bytes pass through.
+      def self.normalize_encoding(str)
+        return str unless str.encoding == Encoding::ASCII_8BIT
+
+        str.dup.force_encoding(Encoding::UTF_8)
+      end
+
       attr_reader :driver
 
       # `driver:` is mainly a test seam to force a specific driver; in normal use
@@ -92,7 +107,8 @@ module Exwiw
         case @driver
         when :mysql2
           res = raw.query(sql, cast: false, as: :array)
-          Result.new(res.fields, res.to_a)
+          rows = res.to_a.map { |row| row.map { |value| self.class.stringify_value(value) } }
+          Result.new(res.fields, rows)
         when :trilogy
           res = raw.query(sql)
           rows = res.rows.map { |row| row.map { |value| self.class.stringify_value(value) } }

data/lib/exwiw/adapter.rb

@@ -139,6 +139,23 @@ module Exwiw
       def commented_sql(query_ast)
         "#{sql_query_comment(query_ast)} #{compile_ast(query_ast)}"
       end
+
+      # One-line, human-readable description of the extraction query, used by the
+      # Runner in error messages so a failure during INSERT/COPY generation (or
+      # query execution) can be traced back to the query that produced the data.
+      # SQL adapters expose the compiled, comment-prefixed SELECT; non-SQL
+      # adapters (e.g. MongodbAdapter) override or fall back to the query object's
+      # own inspect output. Best-effort: never raise from here, since it runs on
+      # an error path.
+      def describe_query(query_ast)
+        if respond_to?(:compile_ast)
+          commented_sql(query_ast)
+        else
+          query_ast.inspect
+        end
+      rescue => e
+        "<unavailable: #{e.class}: #{e.message}>"
+      end
     end
 
     # @params [Exwiw::QueryAst] query_ast

data/lib/exwiw/runner.rb

@@ -60,51 +60,67 @@ module Exwiw
         @logger.info("Processing table '#{table_name}'... (#{idx + 1}/#{total_size})")
 
         query_ast = adapter.build_query(table, @dump_target, table_by_name)
-        results = adapter.execute(query_ast)
-        record_num = results.size
 
-        if record_num.zero?
-          @logger.info("  No records matched. skip this table.")
-          next
-        end
-        insert_idx = (idx + 1).to_s.rjust(3, '0')
-
-        if @output_format == 'copy'
-          @logger.debug("  Generate COPY statement...")
-          copy_sql = adapter.to_copy_from_stdin(results, table)
-          @logger.info("  Generated COPY statement for #{record_num} records.")
-
-          File.open(File.join(@output_dir, "insert-#{insert_idx}-#{table_name}.#{adapter.output_extension}"), 'w') do |file|
-            file.puts(copy_sql)
-            post = adapter.post_insert_sql(table)
-            file.puts(post) if post
+        # Track which phase we are in so that, if an error is raised while
+        # turning the fetched rows into SQL/JSONL, the rescue below can report
+        # both the failing step and the exact extraction query that produced the
+        # data being processed.
+        phase = "executing extraction query"
+        begin
+          results = adapter.execute(query_ast)
+          record_num = results.size
+
+          if record_num.zero?
+            @logger.info("  No records matched. skip this table.")
+            next
           end
-        else
-          @logger.debug("  Generate INSERT statement...")
-          chunk_size = table.bulk_insert_chunk_size
-          chunks = chunk_size ? results.each_slice(chunk_size).to_a : [results]
-          insert_sql = chunks.map { |chunk_rows| adapter.to_bulk_insert(chunk_rows, table) }.join("\n")
-
-          @logger.info("  Generated INSERT statement for #{record_num} records (#{chunks.size} statement(s)).")
-          File.open(File.join(@output_dir, "insert-#{insert_idx}-#{table_name}.#{adapter.output_extension}"), 'w') do |file|
-            file.puts(insert_sql)
-            post = adapter.post_insert_sql(table)
-            file.puts(post) if post
-          end
-        end
-
-        if adapter.supports_bulk_delete? && !@insert_only && !(table.respond_to?(:rails_managed?) && table.rails_managed?)
-          @logger.debug("  Generate DELETE statement...")
-          delete_sql = adapter.to_bulk_delete(query_ast, table)
-          if @logger.debug?
-            @logger.debug("  Generated DELETE statement:\n#{delete_sql}")
+          insert_idx = (idx + 1).to_s.rjust(3, '0')
+
+          if @output_format == 'copy'
+            phase = "generating COPY statement"
+            @logger.debug("  Generate COPY statement...")
+            copy_sql = adapter.to_copy_from_stdin(results, table)
+            @logger.info("  Generated COPY statement for #{record_num} records.")
+
+            File.open(File.join(@output_dir, "insert-#{insert_idx}-#{table_name}.#{adapter.output_extension}"), 'w') do |file|
+              file.puts(copy_sql)
+              post = adapter.post_insert_sql(table)
+              file.puts(post) if post
+            end
           else
-            @logger.info("  Generated DELETE statement.")
+            phase = "generating INSERT statement"
+            @logger.debug("  Generate INSERT statement...")
+            chunk_size = table.bulk_insert_chunk_size
+            chunks = chunk_size ? results.each_slice(chunk_size).to_a : [results]
+            insert_sql = chunks.map { |chunk_rows| adapter.to_bulk_insert(chunk_rows, table) }.join("\n")
+
+            @logger.info("  Generated INSERT statement for #{record_num} records (#{chunks.size} statement(s)).")
+            File.open(File.join(@output_dir, "insert-#{insert_idx}-#{table_name}.#{adapter.output_extension}"), 'w') do |file|
+              file.puts(insert_sql)
+              post = adapter.post_insert_sql(table)
+              file.puts(post) if post
+            end
           end
-          delete_idx = (total_size - idx).to_s.rjust(3, '0')
-          File.open(File.join(@output_dir, "delete-#{delete_idx}-#{table_name}.#{adapter.output_extension}"), 'w') do |file|
-            file.puts(delete_sql)
+
+          if adapter.supports_bulk_delete? && !@insert_only && !(table.respond_to?(:rails_managed?) && table.rails_managed?)
+            phase = "generating DELETE statement"
+            @logger.debug("  Generate DELETE statement...")
+            delete_sql = adapter.to_bulk_delete(query_ast, table)
+            if @logger.debug?
+              @logger.debug("  Generated DELETE statement:\n#{delete_sql}")
+            else
+              @logger.info("  Generated DELETE statement.")
+            end
+            delete_idx = (total_size - idx).to_s.rjust(3, '0')
+            File.open(File.join(@output_dir, "delete-#{delete_idx}-#{table_name}.#{adapter.output_extension}"), 'w') do |file|
+              file.puts(delete_sql)
+            end
           end
+        rescue => e
+          @logger.error("Error while #{phase} for table '#{table_name}' (#{idx + 1}/#{total_size}): #{e.class}: #{e.message}")
+          @logger.error("  Extraction query that produced the data being processed:")
+          @logger.error("    #{adapter.describe_query(query_ast)}")
+          raise
         end
       end
 

data/lib/exwiw/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Exwiw
-  VERSION = "0.3.4"
+  VERSION = "0.3.6"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: exwiw
 version: !ruby/object:Gem::Version
-  version: 0.3.4
+  version: 0.3.6
 platform: ruby
 authors:
 - Shia