gamma_replication 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 2c388f3c2fc5c5b7b1bad0f7b2c6d9f249883979710b2f81b1f405a5557ec201
+  data.tar.gz: 78d48d17c39c15b838c6e2a4a842473dda65711eaa8f6e21ae11de8314c4d945
+SHA512:
+  metadata.gz: 7615b3b2e9cc4fdf11139303436fbdf906c291a477ff8d09263a9773615f2bfeb7a742c093eb262138f79ff93d9aac148ceaa425c105a7653852e10d1dfc8580
+  data.tar.gz: 57bc2df77112be468ed5a07361b2e45230fca972ef71d973a00f8aa4b99adafb968f3095275fb03f5d9be3bd2a8be540141460892f0f9f468a79369e15c2a1cc

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2025 Shinsuke Nishio
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,162 @@
+# GammaReplication
+
+GammaReplication is a tool that reads MySQL binlog using [Maxwell's Daemon](https://github.com/zendesk/maxwell) and replicates data to another MySQL database while masking sensitive information.
+
+## Features
+
+- Real-time replication using MySQL binlog
+- Column-level data masking
+- Flexible hook system for custom data transformation
+- Dry-run mode for operation verification
+
+## Requirements
+
+- Ruby 3.0.0 or higher
+- MySQL 5.7 or higher
+- Maxwell's Daemon
+- Java 8 or higher (for Maxwell's Daemon)
+
+## Directory Structure
+
+The tool expects Maxwell's Daemon to be available in the same directory:
+
+```
+your_project/
+├── maxwell/
+│   └── bin/
+│       └── maxwell
+└── your_application_files
+```
+
+## Installation
+
+```bash
+gem install gamma_replication
+```
+
+Or add this line to your application's Gemfile:
+
+```ruby
+gem 'gamma_replication'
+```
+
+## Setup
+
+1. Set up Maxwell's Daemon:
+```bash
+# Download Maxwell's Daemon
+wget https://github.com/zendesk/maxwell/releases/download/v1.42.2/maxwell-1.42.2.tar.gz
+tar xvf maxwell-1.42.2.tar.gz
+mv maxwell-1.42.2 maxwell
+
+# The maxwell executable will be available at maxwell/bin/maxwell
+```
+
+2. Create configuration files:
+
+```bash
+bin/setup
+```
+
+This command will create the following files:
+- `config.properties`: Maxwell configuration
+- `settings.yml`: Database connection settings
+- `data.yml`: Table and masking configuration
+- `hooks/`: Masking scripts
+
+3. Configure MySQL:
+   - Enable binlog in your MySQL configuration:
+   ```ini
+   [mysqld]
+   server-id=1
+   log-bin=master
+   binlog_format=row
+   ```
+   - Create a user with replication privileges:
+   ```sql
+   CREATE USER 'maxwell'@'%' IDENTIFIED BY 'maxwell';
+   GRANT ALL ON maxwell.* TO 'maxwell'@'%';
+   GRANT SELECT, REPLICATION CLIENT, REPLICATION SLAVE ON *.* TO 'maxwell'@'%';
+   ```
+
+4. Edit configuration files:
+
+### settings.yml
+```yaml
+in_database_config:
+  host: localhost
+  port: 3306
+  username: repl_user
+  password: password
+  database: source_db
+
+out_database_config:
+  host: localhost
+  port: 3306
+  username: root
+  password: password
+  database: target_db
+```
+
+### data.yml
+```yaml
+- data:
+    table: "users"
+    hooks:
+      - column:
+          name:
+            - "email"
+          scripts:
+            - "hooks/mask_email.rb"
+      - column:
+          name:
+            - "phone_number"
+          scripts:
+            - "hooks/mask_phone_number.rb"
+```
+
+## Usage
+
+### Start Replication
+
+```bash
+gamma_replication start -s settings.yml -d data.yml -m config.properties
+```
+
+### Dry Run (Check SQL)
+
+```bash
+gamma_replication dryrun -s settings.yml -d data.yml -m config.properties
+```
+
+## Custom Masking
+
+Create Ruby scripts in the `hooks/` directory to implement custom masking logic:
+
+```ruby
+class MaskEmail
+  def execute(apply, column, value)
+    return value unless apply
+    "masked_#{value}"
+  end
+end
+```
+
+## Development
+
+1. Clone the repository
+2. Run `bin/setup` to install dependencies
+3. Run `rake spec` to run the tests
+4. Run `bin/console` for an interactive prompt
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](LICENSE.txt).
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/bin/console

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "gamma_replication"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+cp config.properties.example config.properties
+cp settings.yml.example settings.yml

data/exe/gamma_replication

@@ -0,0 +1,34 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift File.expand_path("../lib", __dir__)
+require "rubygems"
+require "gamma_replication"
+require "thor"
+
+class GammaReplicationCLI < Thor
+  desc "start", "Start replication using Maxwell"
+  option :settings, aliases: "-s", desc: "Database Settings yaml", required: true
+  option :data, aliases: "-d", desc: "Table Sync Settings yaml", required: true
+  option :hook_dir, aliases: "-h", desc: "Hook script directory", default: "."
+  option :maxwell_config, aliases: "-m", desc: "Maxwell configuration file path", required: true
+  def start
+    GammaReplication::Command::Start.new(options).execute
+  end
+
+  desc "dryrun", "Run replication in dry-run mode (output SQL to STDOUT)"
+  option :settings, aliases: "-s", desc: "Database Settings yaml", required: true
+  option :data, aliases: "-d", desc: "Table Sync Settings yaml", required: true
+  option :hook_dir, aliases: "-h", desc: "Hook script directory", default: "."
+  option :maxwell_config, aliases: "-m", desc: "Maxwell configuration file path", required: true
+  def dryrun
+    GammaReplication::Command::Dryrun.new(options).execute
+  end
+
+  desc "version", "version"
+  def version
+    puts GammaReplication::VERSION
+  end
+end
+
+GammaReplicationCLI.start(ARGV)

data/gamma_replication.gemspec

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require_relative "lib/gamma_replication/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "gamma_replication"
+  spec.version = GammaReplication::VERSION
+  spec.authors = ["Shinsuke Nishio"]
+  spec.email = ["nishio@densan-labs.net"]
+
+  spec.summary = "MySQL replication tool with data masking capability"
+  spec.description = "A tool to replicate MySQL data with the ability to mask sensitive information using Maxwell's Daemon"
+  spec.homepage = "https://github.com/nishio-dens/gamma_replication"
+  spec.license = "MIT"
+  spec.required_ruby_version = ">= 3.0.0"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = spec.homepage
+  spec.metadata["changelog_uri"] = "#{spec.homepage}/blob/main/CHANGELOG.md"
+
+  # Specify which files should be added to the gem when it is released.
+  spec.files = Dir.glob(%w[
+    lib/**/*.rb
+    bin/*
+    *.gemspec
+    README.md
+    LICENSE.txt
+  ])
+  spec.bindir = "exe"
+  spec.executables = ["gamma_replication"]
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "activesupport", ">= 7.0"
+  spec.add_dependency "mysql2", "~> 0.5.5"
+  spec.add_dependency "thor", "~> 1.3"
+
+  spec.add_development_dependency "rake", "~> 13.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "rubocop", "~> 1.21"
+end

data/lib/gamma_replication/command/base_replication.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+module GammaReplication
+  module Command
+    class BaseReplication < Base
+      def initialize(opts)
+        super()
+        setup_database(opts)
+        setup_parser(opts)
+        setup_maxwell(opts)
+      end
+
+      def execute
+        tables = @data_parser.gamma_tables
+        output_setting_warning(tables)
+
+        @table_settings = tables.each_with_object({}) do |table, hash|
+          hash[table.table_name] = table
+        end
+        before_start if respond_to?(:before_start)
+        @maxwell_client.start do |data|
+          process_maxwell_data(data)
+        end
+      end
+
+      private
+
+      def setup_database(opts)
+        @database_settings = GammaReplication::DatabaseSettings.new(opts[:settings])
+        @in_client = GammaReplication::DatabaseConnector::MysqlConnector.new(@database_settings.in_database)
+        @out_client = GammaReplication::DatabaseConnector::MysqlConnector.new(@database_settings.out_database)
+      end
+
+      def setup_parser(opts)
+        @hook_root_dir = opts[:hook_dir] || "."
+        @data_parser = GammaReplication::Parser::DataParser.new(opts[:data], @hook_root_dir, @in_client, @out_client,
+                                                                apply: apply_mode?)
+      end
+
+      def setup_maxwell(opts)
+        @maxwell_client = GammaReplication::MaxwellClient.new(
+          config_path: opts[:maxwell_config] || "config.properties"
+        )
+      end
+
+      def process_maxwell_data(data)
+        return unless should_process_data?(data)
+
+        process_data_by_type(data)
+      rescue StandardError => e
+        logger.error(e)
+      end
+
+      def should_process_data?(data)
+        table_name = data["table"]
+        table_setting = @table_settings[table_name]
+        return false unless table_setting
+        return false if @database_settings.in_database["database"] != data["database"]
+
+        true
+      end
+
+      def process_data_by_type(data)
+        table_setting = @table_settings[data["table"]]
+        case data["type"]
+        when "insert"
+          process_insert(table_setting, data)
+        when "update"
+          process_update(table_setting, data)
+        when "delete"
+          process_delete(table_setting, data)
+        end
+      rescue StandardError => e
+        logger.error(e)
+        # Nothing
+      end
+
+      def process_insert(table_setting, data)
+        record = data["data"]
+        processed_record = apply_hooks(table_setting, record)
+
+        columns = processed_record.keys
+        values = processed_record.values.map { |v| format_value(v) }
+
+        query = "INSERT INTO #{table_setting.table_name} (#{columns.join(",")}) VALUES (#{values.join(",")})"
+        execute_query(query)
+      end
+
+      def process_update(table_setting, data)
+        record = data["data"]
+        old_record = data["old"]
+        processed_record = apply_hooks(table_setting, record)
+
+        set_clause = processed_record.map { |k, v| "#{k} = #{format_value(v)}" }.join(",")
+        where_clause = build_where_clause(old_record, record, table_setting.primary_key)
+
+        query = "UPDATE #{table_setting.table_name} SET #{set_clause} WHERE #{where_clause}"
+        execute_query(query)
+      end
+
+      def process_delete(table_setting, data)
+        old_record = data["old"]
+        where_clause = build_where_clause(old_record, nil, table_setting.primary_key)
+
+        query = "DELETE FROM #{table_setting.table_name} WHERE #{where_clause}"
+        execute_query(query)
+      end
+
+      def apply_hooks(table_setting, record)
+        result = record.dup
+        table_setting.hooks.each do |hook|
+          result = hook.execute_script(result)
+        end
+        result
+      end
+
+      def build_where_clause(old_record, new_record, primary_key)
+        if old_record.present? && old_record[primary_key].present?
+          "#{primary_key} = #{format_value(old_record[primary_key])}"
+        else
+          "#{primary_key} = #{format_value(new_record[primary_key])}"
+        end
+      end
+
+      def format_value(value)
+        case value
+        when nil
+          "NULL"
+        when Numeric
+          value.to_s
+        when Time
+          "'#{value.strftime("%Y-%m-%d %H:%M:%S")}'"
+        else
+          "'#{@out_client.client.escape(value.to_s)}'"
+        end
+      end
+
+      def apply_mode?
+        raise NotImplementedError, "#{self.class} must implement #apply_mode?"
+      end
+
+      def execute_query(query)
+        raise NotImplementedError, "#{self.class} must implement #execute_query"
+      end
+    end
+  end
+end

data/lib/gamma_replication/command/dryrun.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module GammaReplication
+  module Command
+    class Dryrun < BaseReplication
+      def before_start
+        logger.info("Starting DryRun mode...")
+      end
+
+      private
+
+      def apply_mode?
+        false
+      end
+
+      def execute_query(query)
+        logger.info("DryRun: #{query}")
+      end
+    end
+  end
+end

data/lib/gamma_replication/command/start.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module GammaReplication
+  module Command
+    class Start < BaseReplication
+      def initialize(*)
+        super
+        @out_client.client.query("SET FOREIGN_KEY_CHECKS = 0")
+      end
+
+      def apply_mode?
+        true
+      end
+
+      def execute_query(query)
+        logger.info("Executing: #{query}") if ENV["DEBUG"]
+        @out_client.client.query(query)
+      rescue StandardError => e
+        logger.error("Query execution failed: #{e.message}")
+        logger.error(e.backtrace.join("\n"))
+      end
+
+      def finalize
+        @out_client.client.query("SET FOREIGN_KEY_CHECKS = 1")
+        super
+      end
+    end
+  end
+end

data/lib/gamma_replication/command.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module GammaReplication
+  module Command
+    class Base
+      def gamma_tables(_in_client, _out_client, data_parser)
+        data_parser.gamma_tables
+      end
+
+      def output_setting_warning(tables)
+        find_duplicate_tables(tables).each do |table_name|
+          log_duplicate_table_warning(table_name)
+        end
+      end
+
+      private
+
+      def find_duplicate_tables(tables)
+        tables
+          .group_by(&:table_name)
+          .select { |_, group| group.size > 1 }
+          .keys
+      end
+
+      def log_duplicate_table_warning(table_name)
+        logger.warn("Table *#{table_name}* settings are duplicated. Please review your data settings.")
+      end
+
+      def logger
+        @logger ||= Logger.new($stdout)
+      end
+    end
+  end
+end

data/lib/gamma_replication/database_connector/mysql_connector.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require "mysql2"
+
+module GammaReplication
+  module DatabaseConnector
+    class MysqlConnector
+      DEFAULT_PORT = 3306
+
+      attr_reader :config
+
+      def initialize(config)
+        @config = config
+      end
+
+      def client(database_name = @config[:database])
+        @client ||= Mysql2::Client.new(
+          host: @config[:host],
+          port: @config[:port] || DEFAULT_PORT,
+          username: @config[:username],
+          password: @config[:password] || "",
+          database: database_name
+        )
+      end
+
+      def schema_client
+        @schema_client ||= Mysql2::Client.new(
+          host: @config[:host],
+          port: @config[:port] || DEFAULT_PORT,
+          username: @config[:username],
+          password: @config[:password] || "",
+          database: "information_schema"
+        )
+      end
+    end
+  end
+end

data/lib/gamma_replication/database_connector.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module GammaReplication
+  module DatabaseConnector
+  end
+end

data/lib/gamma_replication/database_settings.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module GammaReplication
+  class DatabaseSettings
+    attr_reader :settings, :in_database, :out_database
+
+    def initialize(yaml_path)
+      @settings = YAML.safe_load_file(yaml_path, permitted_classes: [Symbol, Hash],
+                                                 symbolize_names: true).with_indifferent_access
+      @in_database = @settings[:in_database_config]
+      @out_database = @settings[:out_database_config]
+    end
+
+    def in_database_config
+      @in_database
+    end
+
+    def out_database_config
+      @out_database
+    end
+  end
+end

data/lib/gamma_replication/hook.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module GammaReplication
+  class Hook
+    attr_accessor :hook_type, :column_name, :script_path, :root_dir, :apply
+
+    def execute_script(record)
+      validate_script_exists
+      result = record.dup
+      load_script_file
+      execute_hook(result)
+    end
+
+    private
+
+    def validate_script_exists
+      path = script_file_path
+      raise "Hook Scripts Not Found. path: #{path}" unless File.exist?(path)
+    end
+
+    def script_file_path
+      File.join(root_dir, script_path)
+    end
+
+    def load_script_file
+      load script_file_path
+    end
+
+    def execute_hook(record)
+      instance = create_hook_instance
+      process_hook(instance, record)
+    rescue StandardError
+      raise "Invalid Hook Class #{hook_class_name}"
+    end
+
+    def create_hook_instance
+      hook_class_name.constantize.new
+    end
+
+    def hook_class_name
+      File.basename(script_file_path, ".*").camelize
+    end
+
+    def process_hook(instance, record)
+      case hook_type.to_s
+      when "column"
+        process_column_hook(instance, record)
+      when "row"
+        process_row_hook(instance, record)
+      else
+        raise "Invalid hook type: #{hook_type}"
+      end
+    end
+
+    def process_column_hook(instance, record)
+      column = column_name.to_s
+      record[column] = instance.execute(apply, column, record[column])
+      record
+    end
+
+    def process_row_hook(instance, record)
+      instance.execute(apply, record)
+    end
+  end
+end

data/lib/gamma_replication/maxwell_client.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require "json"
+require "open3"
+
+module GammaReplication
+  class MaxwellClient
+    attr_reader :config_path, :maxwell_path
+
+    def initialize(config_path:, maxwell_path: "./maxwell")
+      @config_path = config_path
+      @maxwell_path = maxwell_path
+    end
+
+    def start(&block)
+      cmd = "#{maxwell_path}/bin/maxwell --config #{config_path}"
+
+      IO.popen(cmd) do |io|
+        io.each do |line|
+          data = JSON.parse(line.strip)
+          block.call(data) if block_given?
+        rescue JSON::ParserError => e
+          logger.error("Failed to parse Maxwell output: #{e.message}")
+        end
+      end
+    end
+
+    private
+
+    def logger
+      @logger ||= Logger.new($stdout)
+    end
+  end
+end

data/lib/gamma_replication/parser/data_parser.rb

@@ -0,0 +1,175 @@
+# frozen_string_literal: true
+
+module GammaReplication
+  module Parser
+    class DataParser
+      def initialize(data_yaml_path, hook_root_dir, in_client, out_client, apply: false)
+        @data_settings = YAML.load_file(data_yaml_path).map(&:with_indifferent_access)
+        @hook_root_dir = hook_root_dir
+        @in_client = in_client
+        @out_client = out_client
+        @apply = apply
+      end
+
+      def gamma_tables
+        exist_tables = database_exist_tables
+        @data_settings.flat_map { |d| parse_data_settings(d[:data], exist_tables) }
+      end
+
+      private
+
+      def parse_data_settings(data, exist_tables)
+        tables = find_target_tables(data, exist_tables)
+        tables.map { |t| apply_table_settings(t, data) }
+      end
+
+      def find_target_tables(data, exist_tables)
+        if Array(data[:table]).join == "*"
+          without = Array(data[:table_without]) || []
+          exist_tables.reject { |v| without.include?(v.table_name) }
+        else
+          Array(data[:table]).map do |table_name|
+            exist_tables.find { |t| t.table_name == table_name }
+          end.compact
+        end
+      end
+
+      def apply_table_settings(table, data)
+        table.tap do |t|
+          t.hooks = data[:hooks].present? ? parse_hooks(data[:hooks], t) : []
+        end
+      end
+
+      def database_exist_tables
+        in_tables = select_table_definitions(@in_client)
+        out_tables = select_table_definitions(@out_client)
+
+        (in_tables + out_tables).uniq.map do |table|
+          build_table_info(table, in_tables, out_tables)
+        end
+      end
+
+      def build_table_info(table, in_tables, out_tables)
+        GammaReplication::Table.new.tap do |t|
+          t.table_name = table
+          t.in_exist = in_tables.include?(table)
+          t.out_exist = out_tables.include?(table)
+          t.in_exist_columns = select_column_definitions(@in_client, table)
+          t.out_exist_columns = select_column_definitions(@out_client, table)
+        end
+      end
+
+      def select_table_definitions(client)
+        query = build_table_query(client)
+        client.schema_client.query(query).to_a.map { |v| v["TABLE_NAME"] }
+      end
+
+      def build_table_query(client)
+        database = client.schema_client.escape(client.config[:database])
+        <<~SQL
+          SELECT
+            *
+          FROM
+            TABLES
+          INNER JOIN
+            COLLATION_CHARACTER_SET_APPLICABILITY CCSA
+          ON
+            TABLES.TABLE_COLLATION = CCSA.COLLATION_NAME
+          WHERE
+            TABLE_SCHEMA = '#{database}'
+          ORDER BY
+            TABLE_NAME
+        SQL
+      end
+
+      def select_column_definitions(client, table_name)
+        query = build_column_query(client, table_name)
+        client.schema_client.query(query).to_a.map { |v| v["COLUMN_NAME"] }
+      end
+
+      def build_column_query(client, table_name)
+        database = client.schema_client.escape(client.config[:database])
+        escaped_table = client.schema_client.escape(table_name)
+        <<~SQL
+          SELECT
+            *
+          FROM
+            COLUMNS
+          WHERE
+            TABLE_SCHEMA = '#{database}'
+            AND TABLE_NAME = '#{escaped_table}'
+          ORDER BY
+            TABLE_NAME, ORDINAL_POSITION
+        SQL
+      end
+
+      def parse_hooks(hooks, table)
+        hooks = Array(hooks)
+        hooks.flat_map do |hook|
+          type = determine_hook_type(hook)
+          create_hooks_by_type(type, hook, table)
+        end.compact
+      end
+
+      def determine_hook_type(hook)
+        if hook[:row].present?
+          :row
+        elsif hook[:column].present?
+          :column
+        end
+      end
+
+      def create_hooks_by_type(type, hook, table)
+        case type
+        when :row
+          create_row_hooks(hook[:row], table)
+        when :column
+          create_column_hooks(hook[:column], table)
+        else
+          raise "Unknown Hook Type"
+        end
+      end
+
+      def create_row_hooks(options, table)
+        validate_row_hook_options!(options, table)
+        Array(options[:scripts]).map do |script|
+          build_hook(:row, nil, script)
+        end
+      end
+
+      def create_column_hooks(options, table)
+        validate_column_hook_options!(options, table)
+        column_names = Array(options[:name])
+        scripts = Array(options[:scripts])
+        column_names.product(scripts).map do |column_name, script|
+          build_hook(:column, column_name, script)
+        end
+      end
+
+      def build_hook(type, column_name, script)
+        GammaReplication::Hook.new.tap do |h|
+          h.hook_type = type
+          h.column_name = column_name
+          h.script_path = script
+          h.root_dir = @hook_root_dir
+          h.apply = @apply
+        end
+      end
+
+      def validate_row_hook_options!(options, table)
+        return if options[:scripts].present?
+
+        raise "Required scripts arguments. table: #{table.table_name}, hook_type: row"
+      end
+
+      def validate_column_hook_options!(options, table)
+        unless options[:name].present?
+          raise "Required column name arguments. table: #{table.table_name}, hook_type: column"
+        end
+        return if options[:scripts].present?
+
+        raise "Required scripts arguments. table: #{table.table_name}, hook_type: column"
+      end
+    end
+  end
+end

data/lib/gamma_replication/table.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module GammaReplication
+  class Table
+    DEFAULT_PRIMARY_KEY = "id"
+
+    attr_accessor :table_name, :hooks,
+                  :in_exist, :out_exist, :in_exist_columns, :out_exist_columns
+
+    def initialize
+      @hooks = []
+      @in_exist_columns = []
+      @out_exist_columns = []
+    end
+
+    def record_value(record)
+      result = record.dup
+      hooks.each do |hook|
+        result = hook.execute_script(result)
+      end
+      result
+    end
+
+    def primary_key
+      DEFAULT_PRIMARY_KEY
+    end
+  end
+end

data/lib/gamma_replication/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module GammaReplication
+  VERSION = "0.1.0"
+end

data/lib/gamma_replication.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require "active_support/all"
+require "yaml"
+require_relative "gamma_replication/version"
+require_relative "gamma_replication/database_settings"
+require_relative "gamma_replication/hook"
+require_relative "gamma_replication/table"
+require_relative "gamma_replication/database_connector"
+require_relative "gamma_replication/database_connector/mysql_connector"
+require_relative "gamma_replication/command"
+require_relative "gamma_replication/command/base_replication"
+require_relative "gamma_replication/command/start"
+require_relative "gamma_replication/command/dryrun"
+require_relative "gamma_replication/parser/data_parser"
+require_relative "gamma_replication/maxwell_client"
+
+module GammaReplication
+  class Error < StandardError; end
+end

metadata

@@ -0,0 +1,151 @@
+--- !ruby/object:Gem::Specification
+name: gamma_replication
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Shinsuke Nishio
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2025-01-20 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.0'
+- !ruby/object:Gem::Dependency
+  name: mysql2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.5.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.5.5
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.21'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.21'
+description: A tool to replicate MySQL data with the ability to mask sensitive information
+  using Maxwell's Daemon
+email:
+- nishio@densan-labs.net
+executables:
+- gamma_replication
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE.txt
+- README.md
+- bin/console
+- bin/setup
+- exe/gamma_replication
+- gamma_replication.gemspec
+- lib/gamma_replication.rb
+- lib/gamma_replication/command.rb
+- lib/gamma_replication/command/base_replication.rb
+- lib/gamma_replication/command/dryrun.rb
+- lib/gamma_replication/command/start.rb
+- lib/gamma_replication/database_connector.rb
+- lib/gamma_replication/database_connector/mysql_connector.rb
+- lib/gamma_replication/database_settings.rb
+- lib/gamma_replication/hook.rb
+- lib/gamma_replication/maxwell_client.rb
+- lib/gamma_replication/parser/data_parser.rb
+- lib/gamma_replication/table.rb
+- lib/gamma_replication/version.rb
+homepage: https://github.com/nishio-dens/gamma_replication
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/nishio-dens/gamma_replication
+  source_code_uri: https://github.com/nishio-dens/gamma_replication
+  changelog_uri: https://github.com/nishio-dens/gamma_replication/blob/main/CHANGELOG.md
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.16
+signing_key:
+specification_version: 4
+summary: MySQL replication tool with data masking capability
+test_files: []