exwiw 0.1.3 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 83fdfcea806c556f4eace82d26f2c5fcd08b05c37ee1396b14e3752110e17b73
-  data.tar.gz: 38fc8c299154dab02fd23d1c39d09453ebeaf66be611127453b46908bbe70986
+  metadata.gz: 2f9fb9e57ed24338ac642aa9f0aa200980d8b57fd2d73851249467f9ec9e251b
+  data.tar.gz: 2cb68bd84f67b9a683116d83daacab6a96b65e4802f8b67afb81ef97ebccf3c7
 SHA512:
-  metadata.gz: 2192f38b947c07a5118defa2160316411c5e60f77398b2859d9b31b59079fbabcd2ecb7436071d2ec79629073bdd7339bdc71def16cb709955e8728d363c91dd
-  data.tar.gz: da4fd038b0b21ff9255f5f5d72ad5dbe1b4a42c9d28a4c0e60af19f700c8a82c45226091bad43bdaecda49d5bbc07fb4a7bc8337ffecd793ce4509abd1955b9d
+  metadata.gz: 59bc123c523bf1f97dd97b6c18aa93972926366f121dda9dad9105cf651a783a77b38d7547fad99b7ab1c9509452123bf1c8423203cb0757cf3161d64d296c84
+  data.tar.gz: 6ea7278185d5f92bea6b43d3fb164bb23ae6557ce5d1f3d27e2a86a5147e665a7228ae848e2340c41fd1c5d43ccbdb81decf71804c4d68283de3c61b17be701c

data/CHANGELOG.md

@@ -2,6 +2,28 @@
 
 ## [Unreleased]
 
+### Added
+
+- Add `bulk_insert_chunk_size` table config to split the generated `INSERT` statement into chunks of the specified size. ([#8](https://github.com/riseshia/exwiw/pull/8))
+
+### Changed
+
+- Bump minimum required Ruby version to 3.3.0 and drop Ruby 3.2 from the CI matrix (3.2 reached EOL on 2026-03-31).
+
+### Fixed
+
+- Fix MySQL host access for local rspec runs and switch local dev scripts to inject the password via `MYSQL_PWD` env on `docker compose exec` instead of the `-p` CLI flag. ([#5](https://github.com/riseshia/exwiw/pull/5))
+- Expand `~` in path arguments and validate the existence of `--config-dir`. ([#6](https://github.com/riseshia/exwiw/pull/6))
+- Fix incorrect left-side table in `JOIN ... ON` clause for join chains with 3+ hops, which caused `no such column` / `column does not exist` errors at execute time. ([#7](https://github.com/riseshia/exwiw/pull/7))
+
+## [0.1.4] - 2026-04-04
+
+### Fixed
+
+- Skip models whose table does not exist in `exwiw:schema:generate` task.
+- Add trailing newline to generated schema files.
+- Fixed foreign key constraint errors when exporting child tables with filters on intermediate tables. Filters from intermediate tables are now correctly included in JOIN clauses. ([#3](https://github.com/riseshia/exwiw/pull/3))
+
 ## [0.1.3] - 2025-04-02
 
 ### Fixed

data/README.md

@@ -98,6 +98,7 @@ This is an example of the one table schema:
     "name": "users",
     "primary_key": "id",
     "filter": "users.id > 0",
+    "bulk_insert_chunk_size": 1000,
     "belongs_to": [{
         "name": "companies",
         "foreign_key": "company_id"
@@ -115,6 +116,12 @@ This is an example of the one table schema:
 
 `--config-dir` will use all json files in the specified directory.
 
+### Bulk insert chunk size
+
+`bulk_insert_chunk_size` splits the generated `INSERT` statement into multiple statements, each containing at most the specified number of rows. This is useful when the number of records per table is large enough to hit limits like MySQL's `max_allowed_packet`.
+
+If omitted, all records for a table are emitted as a single `INSERT` statement.
+
 ### Filter
 
 Some case, you don't need full records related to target. e.g. dump user access logs only for the last year.

data/lib/exwiw/adapter/mysql2_adapter.rb

@@ -79,7 +79,7 @@ module Exwiw
         sql += " FROM #{query_ast.from_table_name}"
 
         query_ast.join_clauses.each do |join|
-          sql += " JOIN #{join.join_table_name} ON #{query_ast.from_table_name}.#{join.foreign_key} = #{join.join_table_name}.#{join.primary_key}"
+          sql += " JOIN #{join.join_table_name} ON #{join.base_table_name}.#{join.foreign_key} = #{join.join_table_name}.#{join.primary_key}"
 
           join.where_clauses.each do |where|
             compiled_where_condition = compile_where_condition(where, join.join_table_name)

data/lib/exwiw/adapter/postgresql_adapter.rb

@@ -79,7 +79,7 @@ module Exwiw
         sql += " FROM #{query_ast.from_table_name}"
 
         query_ast.join_clauses.each do |join|
-          sql += " JOIN #{join.join_table_name} ON #{query_ast.from_table_name}.#{join.foreign_key} = #{join.join_table_name}.#{join.primary_key}"
+          sql += " JOIN #{join.join_table_name} ON #{join.base_table_name}.#{join.foreign_key} = #{join.join_table_name}.#{join.primary_key}"
 
           join.where_clauses.each do |where|
             compiled_where_condition = compile_where_condition(where, join.join_table_name)

data/lib/exwiw/adapter/sqlite3_adapter.rb

@@ -79,7 +79,7 @@ module Exwiw
         sql += " FROM #{query_ast.from_table_name}"
 
         query_ast.join_clauses.each do |join|
-          sql += " JOIN #{join.join_table_name} ON #{query_ast.from_table_name}.#{join.foreign_key} = #{join.join_table_name}.#{join.primary_key}"
+          sql += " JOIN #{join.join_table_name} ON #{join.base_table_name}.#{join.foreign_key} = #{join.join_table_name}.#{join.primary_key}"
 
           join.where_clauses.each do |where|
             compiled_where_condition = compile_where_condition(where, join.join_table_name)
@@ -157,7 +157,7 @@ module Exwiw
         @connection ||=
           begin
             require 'sqlite3'
-            SQLite3::Database.new(@connection_config.database_name)
+            SQLite3::Database.new(File.expand_path(@connection_config.database_name))
           end
       end
     end

data/lib/exwiw/cli.rb

@@ -79,10 +79,6 @@ module Exwiw
           end
         end
 
-        if @config_dir.nil?
-          $stderr.puts "Config dir is required"
-        end
-
         if @database_password.nil? || @database_password.empty?
           $stderr.puts "environment variable 'DATABASE_PASSWORD' is required"
           exit 1
@@ -95,6 +91,21 @@ module Exwiw
         exit 1
       end
 
+      if @config_dir.nil?
+        $stderr.puts "Config dir is required"
+        exit 1
+      end
+
+      unless Dir.exist?(@config_dir)
+        $stderr.puts "Config dir does not exist: #{@config_dir}"
+        exit 1
+      end
+
+      if Dir.glob(File.join(@config_dir, "*.json")).empty?
+        $stderr.puts "Config dir contains no .json files: #{@config_dir}"
+        exit 1
+      end
+
       if @target_table_name.nil? || @target_table_name.empty?
         $stderr.puts "Target table is required"
         exit 1
@@ -130,10 +141,12 @@ module Exwiw
         opts.on("-p", "--port=PORT", "Target database port") { |v| @database_port = v }
         opts.on("-u", "--user=USERNAME", "Target database user") { |v| @database_user = v }
         opts.on("-o", "--output-dir=[DUMP_DIR_PATH]", "Output file path. default is dump/") do |v|
-          @output_dir = v.end_with?("/") ? v[0..-2] : v
+          v = v.end_with?("/") ? v[0..-2] : v
+          @output_dir = File.expand_path(v)
         end
         opts.on("-c", "--config-dir=CONFIG_DIR_PATH", "Config dir path.") do |v|
-          @config_dir = v.end_with?("/") ? v[0..-2] : v
+          v = v.end_with?("/") ? v[0..-2] : v
+          @config_dir = File.expand_path(v)
         end
         opts.on("-a", "--adapter=ADAPTER", "Database adapter") { |v| @database_adapter = v }
         opts.on("--database=DATABASE", "Target database name") { |v| @database_name = v }

data/lib/exwiw/query_ast.rb

@@ -20,7 +20,9 @@ module Exwiw
           join_table_name: join_table_name,
           primary_key: primary_key,
         }
-        hash[:where_clauses] = where_clauses.map(&:to_h) if where_clauses.size.positive?
+        if where_clauses.size.positive?
+          hash[:where_clauses] = where_clauses.map { |wc| wc.is_a?(String) ? wc : wc.to_h }
+        end
         hash
       end
     end

data/lib/exwiw/query_ast_builder.rb

@@ -61,6 +61,11 @@ module Exwiw
           )
         end
 
+        # Add filter from intermediate table to join clause
+        if to_table.filter
+          join_clause.where_clauses.push to_table.filter
+        end
+
         join_clauses.push(join_clause)
       end
 

data/lib/exwiw/runner.rb

@@ -46,9 +46,11 @@ module Exwiw
         end
         @logger.debug("  Generate INSERT SQL...")
 
-        insert_sql = adapter.to_bulk_insert(results, table)
+        chunk_size = table.bulk_insert_chunk_size
+        chunks = chunk_size ? results.each_slice(chunk_size).to_a : [results]
+        insert_sql = chunks.map { |chunk_rows| adapter.to_bulk_insert(chunk_rows, table) }.join("\n")
 
-        @logger.info("  Generated INSERT SQL for #{record_num} records.")
+        @logger.info("  Generated INSERT SQL for #{record_num} records (#{chunks.size} statement(s)).")
         insert_idx = (idx + 1).to_s.rjust(3, '0')
         File.open(File.join(@output_dir, "insert-#{insert_idx}-#{table_name}.sql"), 'w') do |file|
           file.puts(insert_sql)

data/lib/exwiw/table_config.rb

@@ -9,6 +9,7 @@ module Exwiw
     attribute :filter, optional(String), skip_serializing_if_nil: true
     attribute :belongs_tos, array(BelongsTo)
     attribute :columns, array(TableColumn)
+    attribute :bulk_insert_chunk_size, optional(Integer), skip_serializing_if_nil: true
 
     def self.from_symbol_keys(hash)
       from(JSON.parse(hash.to_json))
@@ -74,6 +75,7 @@ module Exwiw
         merged_table.primary_key = passed_table.primary_key
         merged_table.filter = filter
         merged_table.belongs_tos = passed_table.belongs_tos
+        merged_table.bulk_insert_chunk_size = passed_table.bulk_insert_chunk_size
 
         receiver_column_by_name = columns.each_with_object({}) { |column, hash| hash[column.name] = column }
 

data/lib/exwiw/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Exwiw
-  VERSION = "0.1.3"
+  VERSION = "0.1.5"
 end

data/lib/tasks/exwiw.rake

@@ -14,6 +14,7 @@ namespace :exwiw do
 
       ActiveRecord::Base.descendants.each do |model|
         next if model.abstract_class?
+        next unless model.table_exists?
         next if table_by_name[model.table_name]
 
         belongs_tos = model.reflect_on_all_associations(:belongs_to).map do |assoc|
@@ -51,9 +52,9 @@ namespace :exwiw do
         if File.exist?(path)
           current_config = Exwiw::TableConfig.from(JSON.parse(File.read(path)))
           merged_config = current_config.merge(table)
-          File.write(path, JSON.pretty_generate(merged_config.to_hash))
+          File.write(path, JSON.pretty_generate(merged_config.to_hash) + "\n")
         else
-          File.write(path, JSON.pretty_generate(table.to_hash))
+          File.write(path, JSON.pretty_generate(table.to_hash) + "\n")
         end
       end
     end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: exwiw
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.5
 platform: ruby
 authors:
 - Shia
 bindir: exe
 cert_chain: []
-date: 2025-04-02 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: serdes
@@ -66,14 +66,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 3.2.0
+      version: 3.3.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.2
+rubygems_version: 3.6.9
 specification_version: 4
 summary: Ruby gem that allows you to export records from a database to a dump file.
 test_files: []