exwiw 0.1.0

data/lib/exwiw/runner.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+require "fileutils"
+
+module Exwiw
+  class Runner
+    def initialize(
+      connection_config:,
+      output_dir:,
+      config_dir:,
+      dump_target:,
+      logger:
+    )
+      @connection_config = connection_config
+      @output_dir = output_dir
+      @config_dir = config_dir
+      @dump_target = dump_target
+      @logger = logger
+    end
+
+    def run
+      adapter = Adapter.build(@connection_config, @logger)
+      tables = load_table_config
+
+      @logger.info("Determining table processing order...")
+      ordered_table_names = DetermineTableProcessingOrder.run(tables)
+
+      if !Dir.exist?(@output_dir)
+        FileUtils.mkdir_p(@output_dir)
+      end
+
+      table_by_name = tables.each_with_object({}) { |table, hash| hash[table.name] = table }
+
+      total_size = ordered_table_names.size
+      ordered_table_names.each_with_index do |table_name, idx|
+        @logger.info("Processing table '#{table_name}'... (#{idx + 1}/#{total_size})")
+        table = table_by_name.fetch(table_name)
+
+        query_ast = QueryAstBuilder.run(table.name, table_by_name, @dump_target, @logger)
+        results = adapter.execute(query_ast)
+        record_num = results.size
+
+        if record_num.zero?
+          @logger.info("  No records matched. skip this table.")
+          next
+        end
+        @logger.debug("  Generate INSERT SQL...")
+
+        insert_sql = adapter.to_bulk_insert(results, table)
+
+        @logger.info("  Generated INSERT SQL for #{record_num} records.")
+        insert_idx = (idx + 1).to_s.rjust(3, '0')
+        File.open(File.join(@output_dir, "insert-#{insert_idx}-#{table_name}.sql"), 'w') do |file|
+          file.puts(insert_sql)
+        end
+
+        @logger.debug("  Generate DELETE SQL...")
+        delete_sql = adapter.to_bulk_delete(query_ast, table)
+        if @logger.debug?
+          @logger.debug("  Generated DELETE SQL:\n#{delete_sql}")
+        else
+          @logger.info("  Generated DELETE SQL.")
+        end
+        delete_idx = (total_size - idx).to_s.rjust(3, '0')
+        File.open(File.join(@output_dir, "delete-#{delete_idx}-#{table_name}.sql"), 'w') do |file|
+          file.puts(delete_sql)
+        end
+      end
+    end
+
+    private def load_table_config
+      Dir[File.join(@config_dir, "*.json")].map do |file|
+        json = JSON.parse(File.read(file))
+        TableConfig.from(json)
+      end
+    end
+
+    private def build_adapter
+      case @connection_config["adapter"]
+      when "sqlite3"
+        Sqlite3Adapter.new(@connection_config)
+      else
+        raise "Unsupported adapter"
+      end
+    end
+  end
+end

data/lib/exwiw/table_column.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Exwiw
+  class TableColumn
+    include Serdes
+
+    attribute :name, String
+    attribute :replace_with, optional(String)
+    attribute :raw_sql, optional(String)
+
+    def self.from_symbol_keys(hash)
+      from(hash.transform_keys(&:to_s))
+    end
+
+    def to_hash
+      super.compact # drop unusing option
+    end
+  end
+end

data/lib/exwiw/table_config.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module Exwiw
+  class TableConfig
+    include Serdes
+
+    attribute :name, String
+    attribute :primary_key, String
+    attribute :filter, optional(String)
+    attribute :belongs_tos, array(BelongsTo)
+    attribute :columns, array(TableColumn)
+
+    def self.from_symbol_keys(hash)
+      from(hash.transform_keys(&:to_s))
+    end
+
+    def column_names
+      columns.map(&:name)
+    end
+
+    def belongs_to(table_name)
+      belongs_tos.find { |relation| relation.table_name == table_name }
+    end
+
+    def build_extract_query(extract_target_table, extract_target_ids, tables_by_name)
+      # target is itself
+      if name == extract_target_table
+        return [{
+          from: name,
+          where: [{ primary_key => extract_target_ids }],
+          join: [],
+          select: column_names,
+        }]
+      end
+
+      # it is not related to target table
+      if belongs_to.empty?
+        return [{
+          from: name,
+          where: [],
+          join: [],
+          select: column_names,
+        }]
+      end
+
+      belongs_to_extract_target_table = belongs_tos.find { |relation| relation.table_name == extract_target_table }
+      if belongs_to_extract_target_table
+        key = belongs_to_extract_target_table.foreign_key
+        return [{ from: name, where: [{ key => extract_target_ids }], join: [], select: column_names }]
+      end
+
+      ret = compute_dependency_to_table(extract_target_table, tables_by_name)
+
+      if ret.empty?
+        [{
+          from: name,
+          where: [],
+          join: [],
+          select: column_names,
+        }]
+      else
+        last = ret.last
+        last[:where] = [{ last[:foreign_key] => extract_target_ids }]
+        ret
+      end
+    end
+
+    private def compute_dependency_to_table(target_table_name, tables_by_name)
+      return [] if belongs_tos.empty?
+
+      results = belongs_tos.map do |relation|
+        relation_table = tables_by_name[relation.table_name]
+
+        if relation_table.name == target_table_name
+          [{ base_table_name: name, foreign_key: relation.foreign_key,
+             join_table_name: target_table_name, join_key: relation_table.primary_key }]
+        else
+          ret = relation_table.compute_dependency_to_table(target_table_name, tables_by_name)
+          [{ base_table_name: name, foreign_key: relation.foreign_key,
+             join_table_name: relation_table.name, join_key: relation_table.primary_key }] + ret
+        end
+      end.compact
+
+      matched_dependencies = results.select do |dependency|
+        dependency.last[:join_table_name] == target_table_name
+      end
+
+      return [] if matched_dependencies.empty?
+
+      matched_dependencies.min_by(&:size)
+    end
+  end
+end

data/lib/exwiw/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Exwiw
+  VERSION = "0.1.0"
+end

data/lib/exwiw.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require_relative "exwiw/version"
+
+require "json"
+require "serdes"
+
+require_relative "exwiw/adapter"
+require_relative "exwiw/adapter/sqlite3_adapter"
+require_relative "exwiw/adapter/mysql2_adapter"
+require_relative "exwiw/adapter/postgresql_adapter"
+require_relative "exwiw/determine_table_processing_order"
+require_relative "exwiw/query_ast"
+require_relative "exwiw/query_ast_builder"
+require_relative "exwiw/runner"
+require_relative "exwiw/belongs_to"
+require_relative "exwiw/table_column"
+require_relative "exwiw/table_config"
+
+begin
+  require 'rails'
+rescue LoadError
+else
+  require 'exwiw/railtie'
+end
+
+module Exwiw
+  DumpTarget = Struct.new(:table_name, :ids, keyword_init: true)
+  ConnectionConfig = Struct.new(:adapter, :host, :port, :user, :password, :database_name, keyword_init: true)
+end

data/lib/tasks/exwiw.rake

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+namespace :exwiw do
+  namespace :schema do
+    desc "Generate schema from application"
+    task generate: :environment do
+      require "json"
+      require "exwiw"
+      require "fileutils"
+
+      Rails.application.eager_load!
+
+      table_by_name = {}
+
+      ActiveRecord::Base.descendants.each do |model|
+        next if model.abstract_class?
+        next if table_by_name[model.table_name]
+
+        belongs_tos = model.reflect_on_all_associations(:belongs_to).map do |assoc|
+          if assoc.polymorphic?
+            # XXX: Support polymorphic
+            next
+          else
+            Exwiw::BelongsTo.from_symbol_keys({
+              table_name: assoc.table_name,
+              foreign_key: assoc.foreign_key,
+            })
+          end
+        end
+
+        columns = model.column_names.map do |name|
+          Exwiw::TableColumn.from_symbol_keys({ name: name })
+        end
+
+        table = Exwiw::TableConfig.from_symbol_keys({
+          name: model.table_name,
+          primary_key: model.primary_key,
+          belongs_tos: belongs_tos.compact,
+          columns: columns,
+        })
+        table_by_name[table.name] = table
+      end
+
+      tables = table_by_name.values
+
+      FileUtils.mkdir_p("exwiw")
+
+      tables.each do |table|
+        path = File.join("exwiw", "#{table.name}.json")
+        File.write(path, JSON.pretty_generate(table.to_hash))
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,79 @@
+--- !ruby/object:Gem::Specification
+name: exwiw
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Shia
+bindir: exe
+cert_chain: []
+date: 2025-02-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: serdes
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+description: Export What I Want (Exwiw) is a Ruby gem that allows you to export records
+  from a database to a dump file.
+email:
+- rise.shia@gmail.com
+executables:
+- exwiw
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- LICENSE.txt
+- README.md
+- exe/exwiw
+- lib/exwiw.rb
+- lib/exwiw/adapter.rb
+- lib/exwiw/adapter/mysql2_adapter.rb
+- lib/exwiw/adapter/postgresql_adapter.rb
+- lib/exwiw/adapter/sqlite3_adapter.rb
+- lib/exwiw/belongs_to.rb
+- lib/exwiw/cli.rb
+- lib/exwiw/determine_table_processing_order.rb
+- lib/exwiw/query_ast.rb
+- lib/exwiw/query_ast_builder.rb
+- lib/exwiw/railtie.rb
+- lib/exwiw/runner.rb
+- lib/exwiw/table_column.rb
+- lib/exwiw/table_config.rb
+- lib/exwiw/version.rb
+- lib/tasks/exwiw.rake
+homepage: https://github.com/riseshia/exwiw
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/riseshia/exwiw
+  source_code_uri: https://github.com/riseshia/exwiw/tree/main
+  changelog_uri: https://github.com/riseshia/exwiw/blob/main/CHANGELOG.md
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.2.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.2
+specification_version: 4
+summary: Ruby gem that allows you to export records from a database to a dump file.
+test_files: []