exwiw 0.1.0

data/lib/exwiw/adapter/sqlite3_adapter.rb

@@ -0,0 +1,165 @@
+# frozen_string_literal: true
+
+module Exwiw
+  module Adapter
+    class Sqlite3Adapter < Base
+      def execute(query_ast)
+        sql = compile_ast(query_ast)
+
+        @logger.debug("  Executing SQL: \n#{sql}")
+        connection.execute(sql)
+      end
+
+      def to_bulk_insert(results, table)
+        table_name = table.name
+
+        value_list = results.map do |row|
+          quoted_values = row.map do |value|
+            escape_value(value)
+          end
+          "(" + quoted_values.join(', ') + ")"
+        end
+        values = value_list.join(",\n")
+
+        column_names = table.columns.map(&:name).join(', ')
+        "INSERT INTO #{table_name} (#{column_names}) VALUES\n#{values};"
+      end
+
+      def to_bulk_delete(select_query_ast, table)
+        raise NotImplementedError unless select_query_ast.is_a?(Exwiw::QueryAst::Select)
+
+        sql = "DELETE FROM #{select_query_ast.from_table_name}"
+
+        if select_query_ast.join_clauses.empty?
+          # Ignore filter option, because bulk delete is for cleaning before import,
+          # so it should delete all records to avoid foreign key violation & data consistancy.
+          compiled_where_conditions = select_query_ast.
+            where_clauses.
+            select { |where| where.is_a?(Exwiw::QueryAst::WhereClause) }.
+            map do |where|
+            compile_where_condition(where, select_query_ast.from_table_name)
+          end
+
+          if compiled_where_conditions.size > 0
+            sql += "\nWHERE "
+            sql += compiled_where_conditions.join(' AND ')
+          end
+          sql += ";"
+
+          return sql
+        end
+
+        subquery_ast = Exwiw::QueryAst::Select.new
+        first_join = select_query_ast.join_clauses.first.clone
+
+        subquery_ast.from(first_join.join_table_name)
+        primay_key_col = table.columns.find { |col| col.name == table.primary_key }
+        subquery_ast.select([primay_key_col])
+        select_query_ast.join_clauses[1..].each do |join|
+          subquery_ast.join(join)
+        end
+        first_join.where_clauses.each do |where|
+          # Ignore filter option, because bulk delete is for cleaning before import,
+          # so it should delete all records to avoid foreign key violation & data consistancy.
+          subquery_ast.where(where) if where.is_a?(Exwiw::QueryAst::WhereClause)
+        end
+
+        foreign_key = first_join.foreign_key
+        subquery_sql = compile_ast(subquery_ast)
+        sql += "\nWHERE #{select_query_ast.from_table_name}.#{foreign_key} IN (#{subquery_sql});"
+
+        sql
+      end
+
+      def compile_ast(query_ast)
+        raise NotImplementedError unless query_ast.is_a?(Exwiw::QueryAst::Select)
+
+        sql = "SELECT "
+        sql += query_ast.columns.map { |col| compile_column_name(query_ast, col) }.join(', ')
+        sql += " FROM #{query_ast.from_table_name}"
+
+        query_ast.join_clauses.each do |join|
+          sql += " JOIN #{join.join_table_name} ON #{query_ast.from_table_name}.#{join.foreign_key} = #{join.join_table_name}.#{join.primary_key}"
+
+          join.where_clauses.each do |where|
+            compiled_where_condition = compile_where_condition(where, join.join_table_name)
+            sql += " AND #{compiled_where_condition}"
+          end
+        end
+
+        if query_ast.where_clauses.any?
+          sql += " WHERE "
+          sql += query_ast.where_clauses.map { |where| compile_where_condition(where, query_ast.from_table_name) }.join(' AND ')
+        end
+
+        sql
+      end
+
+      private def compile_where_condition(where_clause, table_name)
+        # Use as it is if it's a raw query
+        return where_clause if where_clause.is_a?(String)
+
+        key = "#{table_name}.#{where_clause.column_name}"
+
+        if where_clause.operator == :eq
+          values = where_clause.value.map { |v| escape_value(v) }
+
+          if values.size == 1
+            "#{key} = #{values.first}"
+          else
+            "#{key} IN (#{values.join(', ')})"
+          end
+        else
+          raise "Unsupported operator: #{where_clause.operator}"
+        end
+      end
+
+      private def escape_value(value)
+        case value
+        when nil
+          "NULL"
+        when String
+          qv = escape_single_quote(value)
+          "'#{qv}'"
+        else
+          value
+        end
+      end
+
+      private def escape_single_quote(value)
+        value.gsub("'", "''")
+      end
+
+      private def compile_column_name(ast, column)
+        case column
+        when Exwiw::QueryAst::ColumnValue::Plain
+          "#{ast.from_table_name}.#{column.name}"
+        when Exwiw::QueryAst::ColumnValue::RawSql
+          column.value
+        when Exwiw::QueryAst::ColumnValue::ReplaceWith
+          parts = column.value.scan(/[^{}]+|\{[^{}]*\}/).map do |part|
+            if part.start_with?('{')
+              name = part[1..-2]
+              "#{ast.from_table_name}.#{name}"
+            else
+              "'#{part}'"
+            end
+          end
+
+          replaced = parts.join(" || ")
+          "(#{replaced})"
+        else
+          raise "Unreachable case: #{column.inspect}"
+        end
+      end
+
+      private def connection
+        @connection ||=
+          begin
+            require 'sqlite3'
+            SQLite3::Database.new(@connection_config.database_name)
+          end
+      end
+    end
+  end
+end

data/lib/exwiw/adapter.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Exwiw
+  module Adapter
+    class Base
+      attr_reader :connection_config
+
+      def initialize(connection_config, logger)
+        @connection_config = connection_config
+        @logger = logger
+      end
+    end
+
+    # @params [Exwiw::QueryAst] query_ast
+    def execute(query_ast)
+      raise NotImplementedError
+    end
+
+    # @params [Array<Array<String>>] array of rows
+    # @params [Exwiw::TableConfig] table
+    def to_bulk_insert(results, table)
+      raise NotImplementedError
+    end
+
+    # @params [Exwiw::QueryAst] select_query_ast
+    # @params [Exwiw::TableConfig] table
+    def to_bulk_delete(select_query_ast, table)
+      raise NotImplementedError
+    end
+
+    def self.build(connection_config, logger)
+      case connection_config.adapter
+      when 'sqlite3'
+        Adapter::Sqlite3Adapter.new(connection_config, logger)
+      when 'mysql2'
+        Adapter::Mysql2Adapter.new(connection_config, logger)
+      when 'postgresql'
+        Adapter::PostgresqlAdapter.new(connection_config, logger)
+      else
+        raise 'Unsupported adapter'
+      end
+    end
+  end
+end

data/lib/exwiw/belongs_to.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Exwiw
+  class BelongsTo
+    include Serdes
+
+    attribute :foreign_key, String
+    attribute :table_name, String
+
+    def self.from_symbol_keys(hash)
+      from(hash.transform_keys(&:to_s))
+    end
+  end
+end

data/lib/exwiw/cli.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+require 'logger'
+require 'optparse'
+require 'pathname'
+
+require 'json'
+
+require 'exwiw'
+
+module Exwiw
+  class CLI
+    def self.start(argv)
+      new(argv).run
+    end
+
+    def initialize(argv)
+      @argv = argv.dup
+      @help = argv.empty?
+
+      @database_host = nil
+      @database_port = nil
+      @database_user = nil
+      @database_password = ENV["DATABASE_PASSWORD"]
+      @output_dir = "dump"
+      @config_dir = nil
+      @database_adapter = nil
+      @database_name = nil
+      @target_table_name = nil
+      @ids = []
+      @log_level = :info
+
+      parser.parse!(@argv)
+    end
+
+    def run
+      if @help
+        puts parser.help
+      else
+        validate_options!
+
+        connection_config = ConnectionConfig.new(
+          adapter: @database_adapter,
+          host: @database_host,
+          port: @database_port,
+          user: @database_user,
+          password: @database_password,
+          database_name: @database_name,
+        )
+
+        dump_target = DumpTarget.new(
+          table_name: @target_table_name,
+          ids: @ids,
+        )
+
+        logger = build_logger
+
+        Runner.new(
+          connection_config: connection_config,
+          output_dir: @output_dir,
+          config_dir: @config_dir,
+          dump_target: dump_target,
+          logger: logger,
+        ).run
+      end
+    end
+
+    private def validate_options!
+      if @database_adapter != "sqlite3"
+        {
+          "Target database host" => @database_host,
+          "Target database port" => @database_port,
+          "Database user" => @database_user,
+          "Target database name" => @database_name,
+        }.each do |k, v|
+          if v.nil?
+            $stderr.puts "#{k} is required"
+            exit 1
+          end
+        end
+
+        if @config_dir.nil?
+          $stderr.puts "Config dir is required"
+        end
+
+        if @database_password.nil? || @database_password.empty?
+          $stderr.puts "environment variable 'DATABASE_PASSWORD' is required"
+          exit 1
+        end
+      end
+
+      valid_adapters = ["mysql2", "postgresql", "sqlite3"]
+      unless valid_adapters.include?(@database_adapter)
+        $stderr.puts "Invalid adapter. Available options are: #{valid_adapters.join(', ')}"
+        exit 1
+      end
+
+      if @target_table_name.nil? || @target_table_name.empty?
+        $stderr.puts "Target table is required"
+        exit 1
+      end
+
+      if @ids.empty?
+        $stderr.puts "At least one ID is required"
+        exit 1
+      end
+    end
+
+    private def build_logger
+      formatter = proc do |severity, timestamp, progname, msg|
+        formatted_ts = timestamp.strftime("%Y-%m-%d %H:%M:%S")
+        "#{formatted_ts} [#{progname}]: #{msg}\n"
+      end
+
+      Logger.new(
+        STDOUT,
+        level: @log_level,
+        datetime_format: "%Y-%m-%d %H:%M:%S",
+        progname: "exwiw",
+        formatter: formatter,
+      )
+    end
+
+    private def parser
+      @parser ||= OptionParser.new do |opts|
+        opts.banner = "exwiw #{Exwiw::VERSION}"
+        opts.version = Exwiw::VERSION
+
+        opts.on("-h", "--host=HOST", "Target database host") { |v| @database_host = v }
+        opts.on("-p", "--port=PORT", "Target database port") { |v| @database_port = v }
+        opts.on("-u", "--user=USERNAME", "Target database user") { |v| @database_user = v }
+        opts.on("-o", "--output-dir=[DUMP_DIR_PATH]", "Output file path. default is dump/") do |v|
+          @output_dir = v.end_with?("/") ? v[0..-2] : v
+        end
+        opts.on("-c", "--config-dir=CONFIG_DIR_PATH", "Config dir path.") do |v|
+          @config_dir = v.end_with?("/") ? v[0..-2] : v
+        end
+        opts.on("-a", "--adapter=ADAPTER", "Database adapter") { |v| @database_adapter = v }
+        opts.on("--database=DATABASE", "Target database name") { |v| @database_name = v }
+        opts.on("--target-table=TABLE", "Target table for extraction") { |v| @target_table_name = v }
+        opts.on("--ids=IDS", "Comma-separated list of identifiers") { |v| @ids = v.split(',') }
+        opts.on("--log-level=LEVEL", "Log level (debug, info). default is info") { |v| @log_level = v.to_sym }
+
+        opts.on("--help", "Print this help") do
+          @help = true
+        end
+      end
+    end
+  end
+end

data/lib/exwiw/determine_table_processing_order.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Exwiw
+  module DetermineTableProcessingOrder
+    module_function
+
+    # @param tables [Array<Exwiw::TableConfig>] tables
+    # @return [Array<String>] sorted table names
+    def run(tables)
+      return tables.map(&:name) if tables.size < 2
+
+      ordered_table_names = []
+
+      table_by_name = tables.each_with_object({}) do |table, acc|
+        acc[table.name] = table
+      end
+
+      loop do
+        break if table_by_name.empty?
+
+        tables_with_no_dependencies = table_by_name.values.select do |table|
+          not_resolved_names = compute_table_dependencies(table) - ordered_table_names - [table.name]
+
+          not_resolved_names.empty?
+        end
+
+        tables_with_no_dependencies.each do |table|
+          ordered_table_names << table.name
+          table_by_name.delete(table.name)
+        end
+      end
+
+      ordered_table_names
+    end
+
+    def compute_table_dependencies(table)
+      table.belongs_tos.each_with_object([]) do |relation, acc|
+        acc << relation.table_name
+      end
+    end
+  end
+end

data/lib/exwiw/query_ast.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module Exwiw
+  module QueryAst
+    class JoinClause
+      attr_reader :base_table_name, :foreign_key, :join_table_name, :primary_key, :where_clauses
+
+      def initialize(base_table_name:, foreign_key:, join_table_name:, primary_key:, where_clauses: [])
+        @base_table_name = base_table_name
+        @foreign_key = foreign_key
+        @join_table_name = join_table_name
+        @primary_key = primary_key
+        @where_clauses = where_clauses
+      end
+
+      def to_h
+        hash = {
+          base_table_name: base_table_name,
+          foreign_key: foreign_key,
+          join_table_name: join_table_name,
+          primary_key: primary_key,
+        }
+        hash[:where_clauses] = where_clauses.map(&:to_h) if where_clauses.size.positive?
+        hash
+      end
+    end
+
+    WhereClause = Struct.new(:column_name, :operator, :value, keyword_init: true) do
+      def to_h
+        {
+          column_name: column_name,
+          operator: operator,
+          value: value
+        }
+      end
+    end
+
+    module ColumnValue
+      Base = Struct.new(:name, :value, keyword_init: true)
+      Plain = Class.new(Base)
+      ReplaceWith = Class.new(Base)
+      RawSql = Class.new(Base)
+    end
+
+    class Select
+      attr_reader :from_table_name, :columns, :where_clauses, :join_clauses
+
+      def initialize
+        @from_table_name = nil
+        @columns = []
+        @where_clauses = []
+        @join_clauses = []
+      end
+
+      def from(table)
+        @from_table_name = table
+      end
+
+      def select(columns)
+        @columns = map_column_value(columns)
+      end
+
+      def where(where_clause)
+        @where_clauses << where_clause
+      end
+
+      def join(join_clause)
+        @join_clauses << join_clause
+      end
+
+      private def map_column_value(columns)
+        columns.map do |c|
+          if c.raw_sql
+            QueryAst::ColumnValue::RawSql.new(name: c.name, value: c.raw_sql)
+          elsif c.replace_with
+            QueryAst::ColumnValue::ReplaceWith.new(name: c.name, value: c.replace_with)
+          else
+            QueryAst::ColumnValue::Plain.new(name: c.name, value: c.name)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/exwiw/query_ast_builder.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+module Exwiw
+  class QueryAstBuilder
+    def self.run(table_name, table_by_name, dump_target, logger)
+      new(table_name, table_by_name, dump_target, logger).run
+    end
+
+    attr_reader :table_name, :table_by_name, :dump_target
+
+    def initialize(table_name, table_by_name, dump_target, logger)
+      @table_name = table_name
+      @table_by_name = table_by_name
+      @dump_target = dump_target
+      @logger = logger
+    end
+
+    def run
+      table = table_by_name.fetch(table_name)
+
+      where_clauses = build_where_clauses(table, dump_target)
+      join_clauses = build_join_clauses(table, table_by_name, dump_target)
+
+      QueryAst::Select.new.tap do |ast|
+        ast.from(table.name)
+        ast.select(table.columns)
+        join_clauses.each { |join_clause| ast.join(join_clause) }
+        where_clauses.each { |where_clause| ast.where(where_clause) }
+      end
+    end
+
+    private def build_join_clauses(table, table_by_name, dump_target)
+      path_tables = find_path_to_dump_target(table, table_by_name, dump_target)
+      @logger.debug("  Join path from the table to dump target: #{path_tables}")
+
+      # the path is empty, it means that the table is not related to the dump target
+      # the path is 1, it's impossible case
+      return [] if path_tables.size < 2
+
+      join_clauses = []
+
+      path_tables.each_cons(2) do |from_table_name, to_table_name|
+        from_table = table_by_name[from_table_name]
+        to_table = table_by_name[to_table_name]
+
+        relation = from_table.belongs_to(to_table_name)
+
+        join_clause = QueryAst::JoinClause.new(
+          base_table_name: from_table.name,
+          foreign_key: relation.foreign_key,
+          join_table_name: to_table.name,
+          primary_key: to_table.primary_key,
+          where_clauses: []
+        )
+        relation_to_dump_target = to_table.belongs_to(dump_target.table_name)
+        if relation_to_dump_target
+          join_clause.where_clauses.push QueryAst::WhereClause.new(
+            column_name: relation_to_dump_target.foreign_key,
+            operator: :eq,
+            value: dump_target.ids
+          )
+        end
+
+        join_clauses.push(join_clause)
+      end
+
+      join_clauses
+    end
+
+    private def build_where_clauses(table, dump_target)
+      clauses = []
+
+      if table.name == dump_target.table_name
+        clauses.push Exwiw::QueryAst::WhereClause.new(
+          column_name: 'id',
+          operator: :eq,
+          value: dump_target.ids
+        )
+
+        return clauses
+      end
+
+      belongs_to = table.belongs_to(dump_target.table_name)
+      return clauses if belongs_to.nil?
+
+      clauses.push Exwiw::QueryAst::WhereClause.new(
+        column_name: belongs_to.foreign_key,
+        operator: :eq,
+        value: dump_target.ids
+      )
+
+      if table.filter
+        clauses.push table.filter
+      end
+
+      clauses
+    end
+
+    private def find_path_to_dump_target(table, table_by_name, dump_target)
+      return [] if table.name == dump_target.table_name
+
+      visited = {}
+      queue = [[table.name, []]]
+
+      until queue.empty?
+        current_table_name, path = queue.shift
+        current_table = table_by_name[current_table_name]
+
+        next if visited[current_table_name]
+        visited[current_table_name] = true
+
+        current_table.belongs_tos.each do |relation|
+          next_table_name = relation.table_name
+          next_path = path + [current_table_name]
+
+          return next_path if next_table_name == dump_target.table_name
+
+          queue.push([next_table_name, next_path])
+        end
+      end
+
+      queue
+    end
+  end
+end

data/lib/exwiw/railtie.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Exwiw
+  class Railtie < ::Rails::Railtie
+    rake_tasks do
+      load "tasks/exwiw.rake"
+    end
+  end
+end