mysql-to-sqlite 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 1ab0d1cf766d8bdd6c149e69adc043a54e43f1c8e260c749004d03a03e50c8e3
+  data.tar.gz: 46b9f74caa805c880d96d48e9ce91560d8143a64b93795a02733e1ec20adf9f0
+SHA512:
+  metadata.gz: 64e7c1ed0fc10ebedeb593cf55aba2ca0630eebaae6b12543476d7f7d93469986e99b40058add4bc966dad96507a5437e84df92ce021bd2fea5efde4f5a3e37e
+  data.tar.gz: da9e356fb107326a0ef4b5ac80b393bd42956cc299c2048563c28dc0978c5af2158cedeb9525636eafaffd814257c862195ed0c76c5eed1170d472b7168cf98b

data/LICENSE.txt

@@ -0,0 +1,15 @@
+MIT License
+
+Copyright (c) 2026 Dean
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this
+software and associated documentation files (the "Software"), to deal in the Software
+without restriction, including without limitation the rights to use, copy, modify,
+merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice shall be included in all copies
+or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED.

data/README.md

@@ -0,0 +1,107 @@
+# mysql-to-sqlite
+
+[![CI](https://github.com/deanpcmad/mysql-to-sqlite/actions/workflows/ci.yml/badge.svg)](https://github.com/deanpcmad/mysql-to-sqlite/actions/workflows/ci.yml)
+
+A standalone, schema-aware copier that copies raw table data from MySQL to an existing
+SQLite database through Active Record, without loading an application's models or
+running callbacks.
+
+The source and destination must have matching table columns and the same maximum
+`schema_migrations.version`. The importer refuses a non-empty destination by default,
+copies in batches, and validates row counts, foreign keys, and integer sequences.
+
+## Install
+
+### Requirements
+
+- Ruby 3.3 or newer
+- MySQL or MariaDB client libraries and development headers, required by `mysql2`
+- SQLite libraries and development headers, required by `sqlite3`
+
+The package names vary by operating system. Install the development packages provided
+by your system package manager before installing the gem.
+
+### Install the command
+
+Install the published gem for standalone use:
+
+```sh
+gem install mysql-to-sqlite
+```
+
+Confirm that the command is available:
+
+```sh
+mysql-to-sqlite --version
+mysql-to-sqlite --help
+```
+
+### Add it to an application
+
+Add the gem to the application's development dependencies:
+
+```ruby
+group :development do
+  gem "mysql-to-sqlite"
+end
+```
+
+Install the bundle and run the command through Bundler:
+
+```sh
+bundle install
+bundle exec mysql-to-sqlite --help
+```
+
+## Use
+
+First create or migrate an empty SQLite database to exactly the schema version recorded
+in MySQL. Then run:
+
+```sh
+mysql-to-sqlite \
+  --source 'mysql2://user:password@127.0.0.1/database' \
+  --destination /absolute/path/to/database.sqlite3 \
+  --yes
+```
+
+The URLs may instead be supplied as `MYSQL_SOURCE_URL` and
+`SQLITE_DATABASE_PATH`. Run `mysql-to-sqlite --help` for all options, or prefix the
+command with `bundle exec` when it is installed through an application's bundle.
+
+`--replace` deletes records from matching destination tables before copying.
+`--ignore-source-only` skips tables found only in MySQL; use it only after inspecting
+why the schemas differ.
+
+Stop all writes during the final import and keep verified backups. This tool copies
+database records only; application uploads and encryption keys must be preserved
+separately.
+
+## Test
+
+The unit tests do not require a database server:
+
+```sh
+bundle exec rake test
+```
+
+The integration suite restores the SQL dumps under `test/fixtures/mysql` into a
+disposable MySQL database and imports them into temporary SQLite files. Start the
+provided MySQL 8.4 service and opt into those tests:
+
+```sh
+docker compose -f compose.test.yml up --detach --wait
+
+MYSQL_TEST_URL='mysql2://root:test-password@127.0.0.1:3307/mysql_to_sqlite_test' \
+  bundle exec rake test
+
+docker compose -f compose.test.yml down
+```
+
+Set `MYSQL_TEST_PORT` on both Compose commands and use the same port in
+`MYSQL_TEST_URL` when port 3307 is unavailable.
+
+The fixtures cover relationships, explicit primary keys and sequences, Unicode,
+binary and null values, replacement mode, source-only tables, schema mismatches, and
+migration-version mismatches. When `MYSQL_TEST_URL` is absent, integration cases are
+reported as skipped.

data/bin/mysql-to-sqlite

@@ -0,0 +1,41 @@
+#!/usr/bin/env ruby
+
+require "optparse"
+
+options = {
+  batch_size: ENV.fetch("IMPORT_BATCH_SIZE", 1_000),
+  destination: ENV["SQLITE_DATABASE_PATH"],
+  source_url: ENV["MYSQL_SOURCE_URL"]
+}
+
+parser = OptionParser.new do |opts|
+  opts.banner = "Usage: mysql-to-sqlite --source MYSQL_URL --destination FILE --yes"
+  opts.on("--source URL", "MySQL URL (or MYSQL_SOURCE_URL)") { |url| options[:source_url] = url }
+  opts.on("--destination FILE", "Existing SQLite database (or SQLITE_DATABASE_PATH)") { |file| options[:destination] = file }
+  opts.on("--batch-size N", Integer, "Rows copied per batch") { |size| options[:batch_size] = size }
+  opts.on("--ignore-source-only", "Skip tables that exist only in MySQL") { options[:ignore_source_only] = true }
+  opts.on("--replace", "Clear destination records before importing") { options[:replace] = true }
+  opts.on("--yes", "Confirm the import") { options[:confirmed] = true }
+  opts.on("-v", "--version", "Show version") do
+    require_relative "../lib/mysql_to_sqlite/version"
+    puts MysqlToSqlite::VERSION
+    exit
+  end
+  opts.on("-h", "--help", "Show this help") do
+    puts opts
+    exit
+  end
+end
+parser.parse!
+
+abort "Pass --source or set MYSQL_SOURCE_URL" if options[:source_url].to_s.empty?
+abort "Pass --destination or set SQLITE_DATABASE_PATH" if options[:destination].to_s.empty?
+abort "Destination does not exist: #{options[:destination]}" unless File.file?(options[:destination])
+abort "Pass --yes after verifying the destination path and backups" unless options[:confirmed]
+
+require_relative "../lib/mysql_to_sqlite"
+
+puts "Importing MySQL data into #{File.expand_path(options[:destination])}"
+MysqlToSqlite::Importer.new(**options.slice(
+  :source_url, :destination, :batch_size, :ignore_source_only, :replace
+)).run!

data/lib/mysql_to_sqlite/importer.rb

@@ -0,0 +1,237 @@
+module MysqlToSqlite
+  class Importer
+    class SourceRecord < ActiveRecord::Base
+      self.abstract_class = true
+    end
+
+    class TargetRecord < ActiveRecord::Base
+      self.abstract_class = true
+    end
+
+    INTERNAL_TABLES = %w[ar_internal_metadata schema_migrations].freeze
+
+    def initialize(source_url:, destination:, batch_size: 1_000, replace: false,
+      ignore_source_only: false, output: $stdout)
+      @source_url = source_url
+      @destination = destination
+      @batch_size = Integer(batch_size)
+      raise ArgumentError, "batch size must be positive" unless @batch_size.positive?
+
+      @replace = replace
+      @ignore_source_only = ignore_source_only
+      @output = output
+    end
+
+    def run!
+      connect!
+      verify_databases!
+      discover_tables!
+      verify_migration_versions!
+      verify_table_sets!
+      verify_table_schemas!
+      report_plan
+
+      without_target_foreign_keys do
+        TargetRecord.transaction do
+          clear_target! if @replace
+          verify_target_is_empty!
+          reset_target_sequences!
+          tables.each { |table| copy_table(table) }
+          validate!
+        end
+      end
+
+      output.puts "Import completed successfully."
+    ensure
+      SourceRecord.connection_pool.disconnect! if SourceRecord.connected?
+      TargetRecord.connection_pool.disconnect! if TargetRecord.connected?
+    end
+
+    private
+
+    attr_reader :batch_size, :output, :source, :tables, :target
+
+    def connect!
+      require "mysql2"
+      require "sqlite3"
+
+      SourceRecord.establish_connection(@source_url)
+      TargetRecord.establish_connection(adapter: "sqlite3", database: @destination)
+      @source = SourceRecord.connection
+      @target = TargetRecord.connection
+    end
+
+    def verify_databases!
+      raise ArgumentError, "source URL must use the mysql2 adapter" unless source.adapter_name == "Mysql2"
+      raise "The destination database must use SQLite" unless target.adapter_name == "SQLite"
+    end
+
+    def discover_tables!
+      @source_tables = source.data_sources - INTERNAL_TABLES
+      @target_tables = target.data_sources - INTERNAL_TABLES
+      @tables = (@source_tables & @target_tables).sort
+    end
+
+    def verify_table_sets!
+      source_only = @source_tables - @target_tables
+      target_only = @target_tables - @source_tables
+
+      if source_only.any? && !@ignore_source_only
+        raise compact(<<~MESSAGE)
+          Source-only tables found: #{source_only.join(", ")}.
+          Prepare the destination at the source migration version, or pass
+          --ignore-source-only after verifying these tables can be skipped.
+        MESSAGE
+      end
+
+      if target_only.any?
+        raise compact(<<~MESSAGE)
+          Destination-only tables found: #{target_only.join(", ")}.
+          Prepare the destination at the same migration version as the source.
+        MESSAGE
+      end
+
+      raise "No application tables were found to import" if tables.empty?
+      output.puts "Skipping source-only tables: #{source_only.join(", ")}" if source_only.any?
+    end
+
+    def verify_table_schemas!
+      mismatches = tables.filter_map do |table|
+        source_columns = source.columns(table).map(&:name)
+        target_columns = target.columns(table).map(&:name)
+        next if source_columns.sort == target_columns.sort
+
+        missing = target_columns - source_columns
+        extra = source_columns - target_columns
+        "#{table} (missing: #{missing.inspect}, extra: #{extra.inspect})"
+      end
+      raise "Schema mismatch: #{mismatches.join("; ")}" if mismatches.any?
+    end
+
+    def verify_migration_versions!
+      source_version = migration_version(source)
+      target_version = migration_version(target)
+      return if source_version == target_version
+
+      raise compact(<<~MESSAGE)
+        Migration versions differ (source: #{source_version || "unknown"},
+        destination: #{target_version || "unknown"}). Prepare the destination at the
+        source version before importing.
+      MESSAGE
+    end
+
+    def report_plan
+      output.puts "Source migration version: #{migration_version(source) || "unknown"}"
+      output.puts "Destination migration version: #{migration_version(target) || "unknown"}"
+      output.puts "Tables to copy: #{tables.join(", ")}"
+    end
+
+    def migration_version(connection)
+      return unless connection.data_source_exists?("schema_migrations")
+
+      connection.select_value("SELECT MAX(version) FROM schema_migrations")
+    end
+
+    def without_target_foreign_keys
+      target.execute("PRAGMA foreign_keys = OFF")
+      yield
+    ensure
+      target.execute("PRAGMA foreign_keys = ON") if target
+    end
+
+    def verify_target_is_empty!
+      populated = tables.filter_map do |table|
+        count = target_count(table)
+        "#{table}=#{count}" if count.positive?
+      end
+      raise "Destination must be empty before import (#{populated.join(", ")})" if populated.any?
+    end
+
+    def clear_target!
+      tables.reverse_each do |table|
+        target.execute("DELETE FROM #{target.quote_table_name(table)}")
+      end
+    end
+
+    def reset_target_sequences!
+      return unless sqlite_sequence_table?
+
+      names = tables.map { |table| target.quote(table) }.join(", ")
+      target.execute("DELETE FROM sqlite_sequence WHERE name IN (#{names})")
+    end
+
+    def copy_table(table)
+      columns = source.columns(table).map(&:name)
+      record_class = Class.new(TargetRecord) do
+        self.table_name = table
+        self.inheritance_column = :_type_disabled
+      end
+      copied = 0
+      order = source.primary_key(table) || columns.first
+
+      loop do
+        rows = source.select_all(compact(<<~SQL)).to_a
+          SELECT #{columns.map { |column| source.quote_column_name(column) }.join(", ")}
+          FROM #{source.quote_table_name(table)}
+          ORDER BY #{source.quote_column_name(order)}
+          LIMIT #{batch_size} OFFSET #{copied}
+        SQL
+        break if rows.empty?
+
+        record_class.insert_all!(rows, record_timestamps: false)
+        copied += rows.length
+        output.puts "#{table}: copied #{copied}" if rows.length == batch_size
+      end
+      output.puts "#{table}: #{copied} row(s)"
+    end
+
+    def validate!
+      failures = tables.filter_map do |table|
+        source_count = source.select_value(
+          "SELECT COUNT(*) FROM #{source.quote_table_name(table)}"
+        ).to_i
+        destination_count = target_count(table)
+        "#{table}: source=#{source_count}, destination=#{destination_count}" unless source_count == destination_count
+      end
+      raise "Row-count validation failed (#{failures.join("; ")})" if failures.any?
+
+      violations = target.select_all("PRAGMA foreign_key_check")
+      raise "Foreign-key validation failed: #{violations.to_a.inspect}" if violations.any?
+
+      validate_sequences!
+    end
+
+    def validate_sequences!
+      return unless sqlite_sequence_table?
+
+      tables.each do |table|
+        primary_key = target.primary_key(table)
+        column = target.columns(table).find { |candidate| candidate.name == primary_key }
+        next unless column&.type == :integer
+
+        max_id = target.select_value(
+          "SELECT MAX(#{target.quote_column_name(primary_key)}) FROM #{target.quote_table_name(table)}"
+        ).to_i
+        sequence = target.select_value(
+          "SELECT seq FROM sqlite_sequence WHERE name = #{target.quote(table)}"
+        ).to_i
+        raise "Sequence validation failed for #{table}" unless max_id == sequence
+      end
+    end
+
+    def target_count(table)
+      target.select_value("SELECT COUNT(*) FROM #{target.quote_table_name(table)}").to_i
+    end
+
+    def sqlite_sequence_table?
+      target.select_value(<<~SQL)&.to_s&.length&.positive?
+        SELECT name FROM sqlite_master
+        WHERE type = 'table' AND name = 'sqlite_sequence'
+      SQL
+    end
+
+    def compact(string)
+      string.lines.map(&:strip).join(" ")
+    end
+  end
+end

data/lib/mysql_to_sqlite/version.rb

@@ -0,0 +1,3 @@
+module MysqlToSqlite
+  VERSION = "0.1.0"
+end

data/lib/mysql_to_sqlite.rb

@@ -0,0 +1,4 @@
+require "active_record"
+
+require_relative "mysql_to_sqlite/importer"
+require_relative "mysql_to_sqlite/version"

metadata

@@ -0,0 +1,94 @@
+--- !ruby/object:Gem::Specification
+name: mysql-to-sqlite
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Dean Perry
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '9'
+- !ruby/object:Gem::Dependency
+  name: mysql2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.1'
+description: A schema-aware MySQL-to-SQLite data importer built on Active Record.
+email:
+- dean@deanpcmad.com
+executables:
+- mysql-to-sqlite
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE.txt
+- README.md
+- bin/mysql-to-sqlite
+- lib/mysql_to_sqlite.rb
+- lib/mysql_to_sqlite/importer.rb
+- lib/mysql_to_sqlite/version.rb
+licenses:
+- MIT
+metadata: {}
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.3'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.16
+specification_version: 4
+summary: Copy a version-matched MySQL database into SQLite
+test_files: []