gamma 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 257847b37232eb4878c3a62a96994213ea5d8fa6
+  data.tar.gz: 8d856eac239f3646c212473b5b1dd57349c3c2b8
+SHA512:
+  metadata.gz: e624da76ffada8e2b46d724d33fb778ec7d55adb25afeb491c15dd3c8fa22aca140faa6819d3678097eb5c23c5f9c562dce99599600eeb4722318e804bd8cf14
+  data.tar.gz: 55aa8bc74a6fdca5c17095fb5ad5435ccf732fbc18b38b871df2b8a34ebd26396e3630d8e4df7c6f06015d6c234c6627ab7022d5d7ec6782bac3468838fee3c8

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+.DS_Store
+history.json

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.4.2
+before_install: gem install bundler -v 1.16.0

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,74 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at nishio@densan-labs.net. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in gamma.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,46 @@
+PATH
+  remote: .
+  specs:
+    gamma (0.1.0)
+      activesupport
+      colorize (~> 0.8.1)
+      mysql2
+      thor (~> 0.20)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (5.1.5)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (~> 0.7)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+    coderay (1.1.2)
+    colorize (0.8.1)
+    concurrent-ruby (1.0.5)
+    i18n (0.9.5)
+      concurrent-ruby (~> 1.0)
+    method_source (0.9.0)
+    minitest (5.11.3)
+    mysql2 (0.4.10)
+    pry (0.11.3)
+      coderay (~> 1.1.0)
+      method_source (~> 0.9.0)
+    rake (10.5.0)
+    thor (0.20.0)
+    thread_safe (0.3.6)
+    tzinfo (1.2.5)
+      thread_safe (~> 0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.16)
+  gamma!
+  minitest (~> 5.0)
+  pry
+  rake (~> 10.0)
+
+BUNDLED WITH
+   1.16.0

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 Shinsuke Nishio
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,129 @@
+# Gamma
+
+Database Synchronizer. Transfer data from one database to another easily.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'gamma'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install gamma
+
+## Usage
+
+### Dryrun
+
+```
+gamma dryrun --settings ./tmp/settings.yml --data ./tmp/data.yml
+```
+
+### Apply
+
+```
+gamma apply --settings ./tmp/settings.yml --data ./tmp/data.yml
+```
+
+### data.yml Example
+
+```
+- data:
+    table:
+      - "*"
+    table_without:
+      - "users"
+      - "schema_migrations"
+      - "ar_internal_metadata"
+    mode: "replace"
+    delta_column: "updated_at"
+- data:
+    table: "users"
+    mode: "replace"
+    delta_column: "updated_at"
+    hooks:
+      - column:
+          name:
+            - "email"
+          scripts:
+            - "hooks/mask_email.rb"
+      - column:
+          name:
+            - "name"
+          scripts:
+            - "hooks/mask_name.rb"
+      - row:
+          scripts:
+            - "hooks/image.rb"
+```
+
+### Hook Script Example
+
+```
+# $YourDir/hooks/mask_name.rb
+
+class MaskName
+  def execute(apply, column, value)
+    value = "●●#{value[2..-1]}"
+    value
+  end
+end
+
+# $YourDir/hooks/image.rb
+
+class Image
+  def execute(apply, record)
+    #
+    # Copy image data from one storage to another
+    #
+
+    record
+  end
+end
+```
+
+### settings.yml Example
+
+```
+in_database_config:
+  adapter: mysql2
+  encoding: utf8
+  database: real_database
+  pool: 5
+  host: localhost
+  username: root
+  password:
+out_database_config:
+  adapter: mysql2
+  encoding: utf8
+  database: sync_to_database
+  pool: 5
+  host: localhost
+  username: root
+  password:
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/nishio-dens/gamma. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+## Code of Conduct
+
+Everyone interacting in the Gamma project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/nishio-dens/gamma/blob/master/CODE_OF_CONDUCT.md).

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/*_test.rb"]
+end
+
+task :default => :test

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "gamma"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/gamma

@@ -0,0 +1,28 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+require "rubygems"
+require "gamma"
+require "thor"
+
+class GammaCLI < Thor
+
+  desc "apply", "Apply"
+  option :settings, aliases: "-s", desc: "Database Settings yaml", required: true
+  option :data, aliases: "-d", desc: "Table Sync Settings yaml", required: true
+  option :hook_dir, aliases: "-h", desc: "Hook script directory"
+  option :sync_history, desc: "Hook script directory"
+  def apply
+    Gamma::Command::Apply.new(options).execute
+  end
+
+  desc "dryrun", "Dryrun"
+  option :settings, aliases: "-s", desc: "Database Settings yaml", required: true
+  option :data, aliases: "-d", desc: "Table Sync Settings yaml", required: true
+  option :hook_dir, aliases: "-h", desc: "Hook script directory"
+  option :sync_history, desc: "Hook script directory"
+  def dryrun
+    Gamma::Command::Dryrun.new(options).execute
+  end
+end
+
+GammaCLI.start(ARGV)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/gamma.gemspec

@@ -0,0 +1,35 @@
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "gamma/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "gamma"
+  spec.version       = Gamma::VERSION
+  spec.authors       = ["Shinsuke Nishio"]
+  spec.email         = ["nishio@densan-labs.net"]
+
+  spec.summary       = %q{DBSync}
+  spec.description   = %q{DBSync}
+  spec.homepage      = "https://github.com/nishio-dens/gamma"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "mysql2"
+  spec.add_dependency "activesupport"
+  spec.add_dependency "thor", "~> 0.20"
+  spec.add_dependency "colorize", "~> 0.8.1"
+
+  spec.required_ruby_version = ">= 2.3.0"
+
+  spec.add_development_dependency "bundler", "~> 1.16"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "minitest", "~> 5.0"
+  spec.add_development_dependency "pry"
+end

data/lib/gamma.rb

@@ -0,0 +1,31 @@
+require "rubygems"
+require "active_support"
+require "active_support/core_ext"
+require "colorize"
+require "logger"
+require "json"
+
+if ENV["DEBUG"].present?
+  require "pry"
+end
+
+require "gamma/version"
+require "gamma/table"
+require "gamma/hook"
+require "gamma/script"
+require "gamma/script/row_script"
+require "gamma/script/column_script"
+require "gamma/sync_database"
+require "gamma/database_settings"
+require "gamma/database_connector"
+require "gamma/database_connector/mysql_connector"
+require "gamma/parser"
+require "gamma/parser/data_parser"
+require "gamma/importer"
+require "gamma/importer/replace"
+require "gamma/command"
+require "gamma/command/apply"
+require "gamma/command/dryrun"
+
+module Gamma
+end

data/lib/gamma/command.rb

@@ -0,0 +1,16 @@
+class Gamma::Command
+  def gamma_tables(in_client, out_client, data_parser)
+    database_exist_tables = database_tables(in_client, out_client, data_parser)
+  end
+
+  def output_setting_warning(tables)
+    dup_tables = tables.group_by { |t| t.table_name }.select { |k, v| v.size > 1 }.map(&:first)
+    dup_tables.each do |tname|
+      logger.warn("Table *#{tname}* settings are duplicated. Please review your data settings.".red)
+    end
+  end
+
+  def logger
+    @_logger ||= Logger.new(STDOUT)
+  end
+end

data/lib/gamma/command/apply.rb

@@ -0,0 +1,27 @@
+class Gamma::Command::Apply < Gamma::Command
+  def initialize(opts)
+    @database_settings = Gamma::DatabaseSettings.new(opts[:settings])
+    # TODO: support postgres adapter
+    @in_client = Gamma::DatabaseConnector::MysqlConnector.new(@database_settings.in_database)
+    @out_client = Gamma::DatabaseConnector::MysqlConnector.new(@database_settings.out_database)
+    @hook_root_dir = opts[:hook_dir] || "."
+    @syncdb = Gamma::SyncDatabase.new(opts[:sync_history] || "./history.json")
+    @data_parser = Gamma::Parser::DataParser.new(opts[:data], @hook_root_dir, @in_client, @out_client, apply: true)
+  end
+
+  def execute
+    tables = @data_parser.gamma_tables
+    output_setting_warning(tables)
+
+    tables.each do |t|
+      logger.info("[#{t.sync_mode}] Sync Start #{t.table_name}".green)
+
+      case t.sync_mode
+      when "replace"
+        Gamma::Importer::Replace.new(@in_client, @out_client, t, apply: true).execute
+      else
+        logger.info("[#{t.sync_mode}] Sync Failed #{t.table_name}. Unknown Sync mode".red)
+      end
+    end
+  end
+end

data/lib/gamma/command/dryrun.rb

@@ -0,0 +1,27 @@
+class Gamma::Command::Dryrun < Gamma::Command
+  def initialize(opts)
+    @database_settings = Gamma::DatabaseSettings.new(opts[:settings])
+    # TODO: support postgres adapter
+    @in_client = Gamma::DatabaseConnector::MysqlConnector.new(@database_settings.in_database)
+    @out_client = Gamma::DatabaseConnector::MysqlConnector.new(@database_settings.out_database)
+    @hook_root_dir = opts[:hook_dir] || "."
+    @syncdb = Gamma::SyncDatabase.new(opts[:sync_history] || "./history.json")
+    @data_parser = Gamma::Parser::DataParser.new(opts[:data], @hook_root_dir, @in_client, @out_client, apply: false)
+  end
+
+  def execute
+    tables = @data_parser.gamma_tables
+    output_setting_warning(tables)
+
+    tables.each do |t|
+      logger.info("[#{t.sync_mode}] Sync Start #{t.table_name}".green)
+
+      case t.sync_mode
+      when "replace"
+        Gamma::Importer::Replace.new(@in_client, @out_client, t).execute
+      else
+        logger.info("[#{t.sync_mode}] Sync Failed #{t.table_name}. Unknown Sync mode".red)
+      end
+    end
+  end
+end

data/lib/gamma/database_connector.rb

@@ -0,0 +1,2 @@
+class Gamma::DatabaseConnector
+end

data/lib/gamma/database_connector/mysql_connector.rb

@@ -0,0 +1,31 @@
+require 'mysql2'
+
+class Gamma::DatabaseConnector::MysqlConnector < Gamma::DatabaseConnector
+  DEFAULT_PORT = 3306
+
+  attr_reader :config
+
+  def initialize(config)
+    @config = config
+  end
+
+  def client(database_name = @config[:database])
+    @_client ||= Mysql2::Client.new(
+      host: @config[:host],
+      port: @config[:port] || DEFAULT_PORT,
+      username: @config[:username],
+      password: @config[:password] || "",
+      database: database_name
+    )
+  end
+
+  def schema_client
+    @_schema_client ||= Mysql2::Client.new(
+      host: @config[:host],
+      port: @config[:port] || DEFAULT_PORT,
+      username: @config[:username],
+      password: @config[:password] || "",
+      database: "information_schema"
+    )
+  end
+end

data/lib/gamma/database_settings.rb

@@ -0,0 +1,9 @@
+class Gamma::DatabaseSettings
+  attr_reader :settings, :in_database, :out_database
+
+  def initialize(yaml_path)
+    @settings = YAML.load_file(yaml_path).with_indifferent_access
+    @in_database = @settings[:in_database_config]
+    @out_database = @settings[:out_database_config]
+  end
+end

data/lib/gamma/hook.rb

@@ -0,0 +1,30 @@
+class Gamma::Hook
+  attr_accessor :hook_type, :column_name, :script_path, :root_dir, :apply
+
+  def execute_script(record)
+    path = File.join(root_dir, script_path)
+    fail "Hook Scripts Not Found. path: #{path}" unless File.exist?(path)
+
+    result = record
+    scripts = open(File.join(root_dir, script_path)).read
+    eval(scripts)
+
+    begin
+      klass_name = "Gamma::Hook::#{File.basename(path, ".*").camelize}"
+      instance = klass_name.constantize.new
+      case self.hook_type.to_s
+      when "column"
+        r = instance.execute(apply, column_name.to_s, record[column_name.to_s])
+        record[column_name.to_s] = r
+      when "row"
+        record = instance.execute(apply, record)
+      else
+        fail "Error"
+      end
+    rescue => e
+      raise "Invalid Hook Class #{klass_name}"
+    end
+
+    result
+  end
+end

data/lib/gamma/importer.rb

@@ -0,0 +1,13 @@
+class Gamma::Importer
+  def logger
+    @_logger ||= Logger.new(STDOUT)
+  end
+
+  def set_foreign_key_off(client)
+    client.client.query("SET FOREIGN_KEY_CHECKS=0")
+  end
+
+  def set_foreign_key_on(client)
+    client.client.query("SET FOREIGN_KEY_CHECKS=1")
+  end
+end

data/lib/gamma/importer/replace.rb

@@ -0,0 +1,124 @@
+class Gamma::Importer::Replace < Gamma::Importer
+  BATCH_SIZE = 1000
+
+  def initialize(in_client, out_client, table, apply: false)
+    @in_client = in_client
+    @out_client = out_client
+    @table = table
+    @apply = apply
+  end
+
+  def execute
+    set_foreign_key_off(@out_client)
+    batch_sync
+  ensure
+    set_foreign_key_on(@out_client)
+  end
+
+  private
+
+  def batch_sync
+    columns = @table.in_exist_columns & @table.out_exist_columns
+    primary_key = "id" # TODO: Fixme
+
+    current_in_pid = 0
+    current_in_offset = 0
+    while true do
+      select_columns = columns.map { |c| "`#{c}`" }.join(",")
+      break unless select_columns.present?
+
+      in_query = "SELECT #{select_columns} FROM #{@table.table_name} WHERE id > #{current_in_pid} LIMIT #{BATCH_SIZE}"
+      logger.info(in_query) if ENV["DEBUG"]
+      in_records = @in_client.client.query(in_query).to_a
+
+      break unless in_records.present?
+
+      out_records = exist_records(select_columns, primary_key, in_records.map { |v| v[primary_key] })
+
+      in_records.map { |ir| [ir, out_records.find { |v| ir[primary_key] == v[primary_key] }] }.each do |in_record, out_record|
+        if out_record.present?
+          update_out_record(in_record, out_record, primary_key)
+        else
+          insert_out_record(in_record)
+        end
+      end
+
+      current_in_pid = in_records.last[primary_key]
+      current_in_offset = current_in_offset + BATCH_SIZE
+    end
+  rescue => e
+    logger.error("Sync Error #{@table.table_name} \n #{e}\n #{e.backtrace.join("\n")}".red)
+  end
+
+  def exist_records(select_columns, primary_key, in_pids)
+    return [] unless in_pids.present?
+
+    query = "SELECT #{select_columns} FROM #{@table.table_name} WHERE #{primary_key} in (#{in_pids.join(",")})"
+    @out_client.client.query(query).to_a
+  end
+
+  def update_out_record(in_record, out_record, primary_key)
+    need_update = @table.delta_column.blank?
+    need_update ||= @table.delta_column.present? && in_record[@table.delta_column] != out_record[@table.delta_column]
+    if need_update
+      record = @table.record_value(in_record)
+      columns = (@table.in_exist_columns & @table.out_exist_columns).reject { |c| record[c].nil? }
+      values = update_record_values(record, columns)
+
+      query = <<-EOS
+        UPDATE `#{@table.table_name}` SET #{values} WHERE #{primary_key} = #{record[primary_key]}
+      EOS
+      query = query.strip_heredoc
+      logger.info(query) if ENV["DEBUG"]
+
+      if @apply
+        @out_client.client.query(query)
+      else
+        logger.info("DRYRUN: #{query}")
+      end
+    end
+  end
+
+  def insert_out_record(in_record)
+    record = @table.record_value(in_record)
+    columns = (@table.in_exist_columns & @table.out_exist_columns).reject { |c| record[c].nil? }
+    select_columns = columns.map { |c| "`#{c}`" }.join(",")
+    values = insert_record_values(record, columns)
+
+    query = <<-EOS
+      INSERT INTO #{@table.table_name}(#{select_columns}) VALUES (#{values})
+    EOS
+    query = query.strip_heredoc
+    logger.info(query) if ENV["DEBUG"]
+
+    if @apply
+      @out_client.client.query(query)
+    else
+      logger.info("DRYRUN: #{query}")
+    end
+  end
+
+  def insert_record_values(record, columns)
+    r = record
+    columns.map do |v|
+      c = if r[v].is_a?(Time)
+            r[v].strftime("%Y-%m-%d %H:%M:%S")
+          else
+            r[v]
+          end
+      "\"#{c.to_s.gsub('"', '\"')}\""
+    end.join(",")
+  end
+
+  def update_record_values(record, columns)
+    r = record
+    columns.map do |v|
+      c = if r[v].is_a?(Time)
+            r[v].strftime("%Y-%m-%d %H:%M:%S")
+          else
+            r[v]
+          end
+      "#{v} = \"#{c}\""
+    end.join(",")
+  end
+end

data/lib/gamma/parser.rb

@@ -0,0 +1,2 @@
+class Gamma::Parser
+end

data/lib/gamma/parser/data_parser.rb

@@ -0,0 +1,131 @@
+class Gamma::Parser::DataParser < Gamma::Parser
+  DEFAULT_SYNC_MODE = "replace"
+
+  def initialize(data_yaml_path, hook_root_dir, in_client, out_client, apply: false)
+    @data_settings = YAML.load_file(data_yaml_path).map(&:with_indifferent_access)
+    @hook_root_dir = hook_root_dir
+    @in_client = in_client
+    @out_client = out_client
+    @apply = apply
+  end
+
+  def gamma_tables
+    exist_tables = database_exist_tables
+    @data_settings.map do |d|
+      parse_data_settings(d[:data], exist_tables)
+    end.flatten
+  end
+
+  def parse_data_settings(data, exist_tables)
+    tables = if Array(data[:table]).join == "*"
+               without = Array(data[:table_without]) || []
+               exist_tables.reject { |v| without.include?(v.table_name) }
+             else
+               Array(data[:table]).map do |table_name|
+                 exist_tables.find { |t| t.table_name == table_name }
+               end.compact
+             end
+    tables = tables.map do |t|
+      t.sync_mode = data[:mode].presence || DEFAULT_SYNC_MODE
+      t.delta_column = data[:delta_column]
+      t.hooks = data[:hooks].present? ? parse_hooks(data[:hooks], t) : []
+
+      t
+    end
+    tables
+  end
+
+  private
+
+  def database_exist_tables
+    in_tables = select_table_definitions(@in_client)
+    out_tables = select_table_definitions(@out_client)
+
+    (in_tables + out_tables).uniq.map do |table|
+      t = Gamma::Table.new
+      t.table_name = table
+      t.in_exist = in_tables.include?(table)
+      t.out_exist = out_tables.include?(table)
+      t.in_exist_columns = select_column_definitions(@in_client, table)
+      t.out_exist_columns = select_column_definitions(@out_client, table)
+      t
+    end
+  end
+
+  def select_table_definitions(client)
+    query = <<-EOS
+      SELECT
+        *
+      FROM
+        TABLES
+      INNER JOIN
+        COLLATION_CHARACTER_SET_APPLICABILITY CCSA
+      ON
+        TABLES.TABLE_COLLATION = CCSA.COLLATION_NAME
+      WHERE
+        TABLE_SCHEMA = '#{client.schema_client.escape(client.config[:database])}'
+      ORDER BY
+        TABLE_NAME
+    EOS
+    client.schema_client.query(query.strip_heredoc).to_a.map { |v| v["TABLE_NAME"] }
+  end
+
+  def select_column_definitions(client, table_name)
+    query = <<-EOS
+      SELECT
+        *
+      FROM
+        COLUMNS
+      WHERE
+        TABLE_SCHEMA = '#{client.schema_client.escape(client.config[:database])}'
+        AND TABLE_NAME = '#{client.schema_client.escape(table_name)}'
+      ORDER BY
+        TABLE_NAME, ORDINAL_POSITION
+    EOS
+    client.schema_client.query(query.strip_heredoc).to_a.map { |v| v["COLUMN_NAME"] }
+  end
+
+  def parse_hooks(hooks, table)
+    hooks = hooks.is_a?(Array) ? hooks : [hooks]
+    hooks.map do |hook|
+      type = if hook[:row].present?
+               :row
+             elsif hook[:column].present?
+               :column
+             end
+
+      if type == :row
+        options = hook[:row]
+        fail "Required scripts arguments. table: #{table.table_name}, hook_type: #{type}" unless options[:scripts].present?
+
+        Array(options[:scripts]).map do |script|
+          h = Gamma::Hook.new
+          h.hook_type = :row
+          h.column_name = nil
+          h.script_path = script
+          h.root_dir = @hook_root_dir
+          h.apply = @apply
+          h
+        end
+      elsif type == :column
+        options = hook[:column]
+        fail "Required column name arguments. table: #{table.table_name}, hook_type: #{type}" unless options[:name].present?
+        fail "Required scripts arguments. table: #{table.table_name}, hook_type: #{type}" unless options[:scripts].present?
+
+        column_names = Array(options[:name])
+        scripts = Array(options[:scripts])
+        column_names.product(scripts).map do |column_name, script|
+          h = Gamma::Hook.new
+          h.hook_type = :column
+          h.column_name = column_name
+          h.script_path = script
+          h.root_dir = @hook_root_dir
+          h.apply = @apply
+          h
+        end
+      else
+        fail "Unknown Hook Type"
+      end
+    end.flatten.compact
+  end
+end

data/lib/gamma/script.rb

@@ -0,0 +1,2 @@
+class Gamma::Script
+end

data/lib/gamma/script/column_script.rb

@@ -0,0 +1,5 @@
+class Gamma::Script::ColumnScript
+  def execute(apply, column, value)
+    fail "Not Implemented"
+  end
+end

data/lib/gamma/script/row_script.rb

@@ -0,0 +1,5 @@
+class Gamma::Script::RowScript
+  def execute(apply, value)
+    fail "Not Implemented"
+  end
+end

data/lib/gamma/sync_database.rb

@@ -0,0 +1,16 @@
+class Gamma::SyncDatabase
+  def initialize(path)
+    @sync_history_path = path
+    @database = if File.exists?(@sync_history_path)
+                  JSON.parse(open(@sync_history_path).read)
+                else
+                  {}
+                end
+  end
+
+  def save
+    open(@sync_history_path, "w") do |io|
+      JSON.dump(@database, io)
+    end
+  end
+end

data/lib/gamma/table.rb

@@ -0,0 +1,26 @@
+class Gamma::Table
+  attr_accessor :table_name, :in_exist, :out_exist, :in_exist_columns, :out_exist_columns
+  attr_accessor :sync_mode, :delta_column
+  attr_accessor :hooks
+
+  def record_value(record)
+    row_hooks = hooks.select { |h| h.hook_type.to_s == "row" }
+    column_hooks = hooks.select { |h| h.hook_type.to_s == "column" }
+
+    result = record
+    result = row_hooks.reduce(record) { |h, rec| execute_row_hook(rec, h) } if row_hooks.present?
+    result = column_hooks.reduce(record) { |h, rec| execute_row_hook(rec, h) } if column_hooks.present?
+
+    result
+  end
+
+  private
+
+  def execute_row_hook(hook, record)
+    hook.execute_script(record)
+  end
+
+  def execute_column_hook(hook, record)
+    record
+  end
+end

data/lib/gamma/version.rb

@@ -0,0 +1,3 @@
+module Gamma
+  VERSION = "0.1.0"
+end

metadata

@@ -0,0 +1,186 @@
+--- !ruby/object:Gem::Specification
+name: gamma
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Shinsuke Nishio
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2018-03-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: mysql2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.20'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.20'
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.1
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: DBSync
+email:
+- nishio@densan-labs.net
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/gamma
+- bin/setup
+- gamma.gemspec
+- lib/gamma.rb
+- lib/gamma/command.rb
+- lib/gamma/command/apply.rb
+- lib/gamma/command/dryrun.rb
+- lib/gamma/database_connector.rb
+- lib/gamma/database_connector/mysql_connector.rb
+- lib/gamma/database_settings.rb
+- lib/gamma/hook.rb
+- lib/gamma/importer.rb
+- lib/gamma/importer/replace.rb
+- lib/gamma/parser.rb
+- lib/gamma/parser/data_parser.rb
+- lib/gamma/script.rb
+- lib/gamma/script/column_script.rb
+- lib/gamma/script/row_script.rb
+- lib/gamma/sync_database.rb
+- lib/gamma/table.rb
+- lib/gamma/version.rb
+homepage: https://github.com/nishio-dens/gamma
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.3.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.13
+signing_key: 
+specification_version: 4
+summary: DBSync
+test_files: []