rails_sync 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 690c52679acf76eca8c80da03e893888a4937ce7ac85341fdd0f895f4e07f1a9
+  data.tar.gz: 163e59a515c618404bc010ec61dbac2832a0e71b3a23ab86f49fe3d62ac2d06f
+SHA512:
+  metadata.gz: f5ac64f117442021284f15446c2f710e85f2cf66258758e917c8d4f3df078a89160f932e7bd94e1ba1207ad64d1769eaf3b80b6cdbc883749c0ab100046cdfde
+  data.tar.gz: 8a6bc5e78d1092dfdcb5a9fb26b09396100bff48c98f59fae42283bbc1cc0723ec9ad5b3f5873dd2f3c2673b8829d0bf14ccdd9dbd6ff4333de5e59083d6f8dd

data/LICENSE

@@ -0,0 +1,165 @@
+                   GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2018 Mingyuan Qin.
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+
+  This version of the GNU Lesser General Public License incorporates
+the terms and conditions of version 3 of the GNU General Public
+License, supplemented by the additional permissions listed below.
+
+  0. Additional Definitions.
+
+  As used herein, "this License" refers to version 3 of the GNU Lesser
+General Public License, and the "GNU GPL" refers to version 3 of the GNU
+General Public License.
+
+  "The Library" refers to a covered work governed by this License,
+other than an Application or a Combined Work as defined below.
+
+  An "Application" is any work that makes use of an interface provided
+by the Library, but which is not otherwise based on the Library.
+Defining a subclass of a class defined by the Library is deemed a mode
+of using an interface provided by the Library.
+
+  A "Combined Work" is a work produced by combining or linking an
+Application with the Library.  The particular version of the Library
+with which the Combined Work was made is also called the "Linked
+Version".
+
+  The "Minimal Corresponding Source" for a Combined Work means the
+Corresponding Source for the Combined Work, excluding any source code
+for portions of the Combined Work that, considered in isolation, are
+based on the Application, and not on the Linked Version.
+
+  The "Corresponding Application Code" for a Combined Work means the
+object code and/or source code for the Application, including any data
+and utility programs needed for reproducing the Combined Work from the
+Application, but excluding the System Libraries of the Combined Work.
+
+  1. Exception to Section 3 of the GNU GPL.
+
+  You may convey a covered work under sections 3 and 4 of this License
+without being bound by section 3 of the GNU GPL.
+
+  2. Conveying Modified Versions.
+
+  If you modify a copy of the Library, and, in your modifications, a
+facility refers to a function or data to be supplied by an Application
+that uses the facility (other than as an argument passed when the
+facility is invoked), then you may convey a copy of the modified
+version:
+
+   a) under this License, provided that you make a good faith effort to
+   ensure that, in the event an Application does not supply the
+   function or data, the facility still operates, and performs
+   whatever part of its purpose remains meaningful, or
+
+   b) under the GNU GPL, with none of the additional permissions of
+   this License applicable to that copy.
+
+  3. Object Code Incorporating Material from Library Header Files.
+
+  The object code form of an Application may incorporate material from
+a header file that is part of the Library.  You may convey such object
+code under terms of your choice, provided that, if the incorporated
+material is not limited to numerical parameters, data structure
+layouts and accessors, or small macros, inline functions and templates
+(ten or fewer lines in length), you do both of the following:
+
+   a) Give prominent notice with each copy of the object code that the
+   Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the object code with a copy of the GNU GPL and this license
+   document.
+
+  4. Combined Works.
+
+  You may convey a Combined Work under terms of your choice that,
+taken together, effectively do not restrict modification of the
+portions of the Library contained in the Combined Work and reverse
+engineering for debugging such modifications, if you also do each of
+the following:
+
+   a) Give prominent notice with each copy of the Combined Work that
+   the Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the Combined Work with a copy of the GNU GPL and this license
+   document.
+
+   c) For a Combined Work that displays copyright notices during
+   execution, include the copyright notice for the Library among
+   these notices, as well as a reference directing the user to the
+   copies of the GNU GPL and this license document.
+
+   d) Do one of the following:
+
+       0) Convey the Minimal Corresponding Source under the terms of this
+       License, and the Corresponding Application Code in a form
+       suitable for, and under terms that permit, the user to
+       recombine or relink the Application with a modified version of
+       the Linked Version to produce a modified Combined Work, in the
+       manner specified by section 6 of the GNU GPL for conveying
+       Corresponding Source.
+
+       1) Use a suitable shared library mechanism for linking with the
+       Library.  A suitable mechanism is one that (a) uses at run time
+       a copy of the Library already present on the user's computer
+       system, and (b) will operate properly with a modified version
+       of the Library that is interface-compatible with the Linked
+       Version.
+
+   e) Provide Installation Information, but only if you would otherwise
+   be required to provide such information under section 6 of the
+   GNU GPL, and only to the extent that such information is
+   necessary to install and execute a modified version of the
+   Combined Work produced by recombining or relinking the
+   Application with a modified version of the Linked Version. (If
+   you use option 4d0, the Installation Information must accompany
+   the Minimal Corresponding Source and Corresponding Application
+   Code. If you use option 4d1, you must provide the Installation
+   Information in the manner specified by section 6 of the GNU GPL
+   for conveying Corresponding Source.)
+
+  5. Combined Libraries.
+
+  You may place library facilities that are a work based on the
+Library side by side in a single library together with other library
+facilities that are not Applications and are not covered by this
+License, and convey such a combined library under terms of your
+choice, if you do both of the following:
+
+   a) Accompany the combined library with a copy of the same work based
+   on the Library, uncombined with any other library facilities,
+   conveyed under the terms of this License.
+
+   b) Give prominent notice with the combined library that part of it
+   is a work based on the Library, and explaining where to find the
+   accompanying uncombined form of the same work.
+
+  6. Revised Versions of the GNU Lesser General Public License.
+
+  The Free Software Foundation may publish revised and/or new versions
+of the GNU Lesser General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+  Each version is given a distinguishing version number. If the
+Library as you received it specifies that a certain numbered version
+of the GNU Lesser General Public License "or any later version"
+applies to it, you have the option of following the terms and
+conditions either of that published version or of any later version
+published by the Free Software Foundation. If the Library as you
+received it does not specify a version number of the GNU Lesser
+General Public License, you may choose any version of the GNU Lesser
+General Public License ever published by the Free Software Foundation.
+
+  If the Library as you received it specifies that a proxy can decide
+whether future versions of the GNU Lesser General Public License shall
+apply, that proxy's public statement of acceptance of any version is
+permanent authorization for you to choose that version for the
+Library.

data/README.md

@@ -0,0 +1,34 @@
+# RailsSync
+
+* [English](./README.en.md)
+
+
+## 适用场景
+1. 应用之间同步数据；
+2. 新老应用间的数据迁移；
+2. 两个 `table` 字段名称可能不同；
+3. 两个 `table` 基于唯一的id进行同步数据，id可能是主键，也可能不是；
+
+## 注意
+1. 如果同一个表，两边应用都对其同一个字段有内容修改，业务上会比较混乱，建议避免；
+2. 由于是直接数据库层面同步数据，不容易在目标应用内直接触发业务逻辑，可在本应用内基于Audit触发业务逻辑；
+
+## 特性
+1. 绝对靠谱，不依赖binlog，基于主键（支持联合主键）对两个表的各个字段进行比较，找出字段值的变化，新增的记录，删除的记录；
+2. 高性能，10万以内的计算毫秒级；
+3. 兼容性强，可支持各个关系型数据库（依赖 linked DB, mysql FEDERATED引擎等）；
+4. 安全，对数据的改变先记录而不应用改变，可基于记录人工决定是否应用该改变，或者revert相应的改动；
+5. 实时性强，可以通过应用里的数据改变动作触发；
+6. 可以同步多个来源的数据；
+
+## 实现原理
+1. 处理表；
+ a. 对于不同节点，建立临时表，映射远程数据库节点中的对应表（只映射需要比较改变值的字段），在mysql中采用`FEDERATED`引擎；
+2. 通过 inner_join 计算两张表不一样的字段；
+3. 通过 left_join 找出左表多出的记录；
+4. 通过 right_join 找出右表多出的记录；
+
+## Remote Table 
+* postgresql: [dblink](https://www.postgresql.org/docs/10/static/dblink.html)
+* mysql: [FEDERATED](https://dev.mysql.com/doc/refman/5.7/en/federated-storage-engine.html)
+* mariadb: [CONNECT](https://mariadb.com/kb/en/library/connect/)

data/Rakefile

@@ -0,0 +1,32 @@
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+
+require 'rdoc/task'
+
+RDoc::Task.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'RailsSync'
+  rdoc.options << '--line-numbers'
+  rdoc.rdoc_files.include('README.md')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+APP_RAKEFILE = File.expand_path("test/dummy/Rakefile", __dir__)
+load 'rails/tasks/engine.rake'
+
+load 'rails/tasks/statistics.rake'
+
+require 'bundler/gem_tasks'
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = false
+end
+
+task default: :test

data/app/controllers/rails_sync_admin/base_controller.rb

@@ -0,0 +1,4 @@
+class RailsSyncAdmin::BaseController < RailsSync.config.admin_class.constantize
+
+
+end

data/app/controllers/rails_sync_admin/sync_audits_controller.rb

@@ -0,0 +1,40 @@
+class RailsSyncAdmin::SyncAuditsController < RailsSyncAdmin::BaseController
+  before_action :set_sync_audit, only: [:show, :apply, :destroy]
+
+  def index
+    q_params = params.permit(:synchro_type, :state, :operation)
+    @sync_audits = SyncAudit.default_where(q_params).order(id: :desc).page(params[:page])
+    @synchro_types = RailsSync.synchro_types.uniq | SyncAudit.synchro_types
+  end
+
+  def sync
+    @synchro_model = params[:synchro_type].constantize
+    @synchro_model.cache_all_diffs
+
+    redirect_to admin_sync_audits_url, notice: 'Sync Run successfully '
+  end
+
+  def batch
+    AuditApplyJob.perform_later params[:synchro_type]
+    redirect_to admin_sync_audits_url, notice: 'Batch Apply Sync Run successfully in backend'
+  end
+
+  def show
+  end
+
+  def apply
+    @sync_audit.apply_changes
+    redirect_to admin_sync_audits_url, notice: 'Sync audit was successfully applied.'
+  end
+
+  def destroy
+    @sync_audit.destroy
+    redirect_to admin_sync_audits_url, notice: 'Sync audit was successfully destroyed.'
+  end
+
+  private
+  def set_sync_audit
+    @sync_audit = SyncAudit.find(params[:id])
+  end
+
+end

data/app/jobs/audit_apply_job.rb

@@ -0,0 +1,9 @@
+class AuditApplyJob < ActiveJob::Base
+  queue_as :default
+
+  def perform(synchro_type, operation: ['update', 'insert', 'delete'])
+    SyncAudit.apply_synchro(synchro_type, operation: operation)
+    SyncAudit.apply_callback(synchro_type, operation: operation)
+  end
+
+end

data/app/models/mysql/mysql_column.rb

@@ -0,0 +1,5 @@
+class MysqlColumn < ApplicationRecord
+  self.establish_connection connection_config.merge(database: 'information_schema')
+  self.table_name = 'COLUMNS'
+
+end

data/app/models/mysql/mysql_engine.rb

@@ -0,0 +1,5 @@
+class MysqlTable < ApplicationRecord
+  self.establish_connection connection_config.merge(database: 'information_schema')
+  self.table_name = 'TABLES'
+
+end

data/app/models/mysql/mysql_index.rb

@@ -0,0 +1,5 @@
+class MysqlIndex < ApplicationRecord
+  self.establish_connection connection_config.merge(database: 'information_schema')
+  self.table_name = 'STATISTICS'
+
+end

data/app/models/mysql/mysql_server.rb

@@ -0,0 +1,5 @@
+class MysqlServer < ApplicationRecord
+  self.establish_connection connection_config.merge(database: 'mysql')
+  self.table_name = 'servers'
+
+end

data/app/models/mysql/mysql_table.rb

@@ -0,0 +1,5 @@
+class MysqlTable < ApplicationRecord
+  self.establish_connection connection_config.merge(database: 'information_schema')
+  self.table_name = 'ENGINES'
+
+end

data/app/models/sync_audit.rb

@@ -0,0 +1,89 @@
+class SyncAudit < ApplicationRecord
+  serialize :audited_changes, Hash
+  serialize :synchro_params, Hash
+
+  enum state: {
+    init: 'init',
+    applied: 'applied',
+    finished: 'finished'
+  }
+
+  enum operation: {
+    update: 'update',
+    delete: 'delete',
+    insert: 'insert'
+  }, _prefix: true
+
+  belongs_to :synchro, polymorphic: true, optional: true
+  belongs_to :operator, polymorphic: true, optional: true
+
+  after_initialize if: :new_record? do
+    self.state = 'init'
+  end
+
+  def apply_changes
+    if self.operation_update?
+      _synchro = self.synchro || synchro_model.find_by(synchro_params)
+      _synchro.assign_attributes to_apply_params
+      self.state = 'applied'
+      self.synchro_id = _synchro.id
+      self.class.transaction do
+        _synchro.save!
+        self.save!
+      end
+    elsif self.operation_delete? && self.synchro
+      self.class.transaction do
+        self.synchro.destroy!
+        self.update! state: 'applied'
+      end
+    elsif self.operation_insert?
+      _synchro = synchro_model.find_or_initialize_by(synchro_params)
+      _synchro.assign_attributes to_apply_params
+      self.class.transaction do
+        _synchro.save_sneakily!
+        self.update! synchro_id: _synchro.id, state: 'applied'
+      end
+    end
+  end
+
+  def synchro_model
+    @synchro_model ||= self.synchro_type.constantize
+  end
+
+  def to_apply_params
+    x = {}
+    audited_changes.each do |key, v|
+      if synchro_model.columns_hash[key].type == :string
+        x[key] = v[1].to_s
+      else
+        x[key] = v[1]
+      end
+    end
+
+    x
+  end
+
+  def self.synchro_types
+    SyncAudit.select(:synchro_type).distinct.pluck(:synchro_type).compact
+  end
+
+  def self.apply_callback(type, operation: ['update', 'delete', 'insert'])
+    SyncAudit.where(state: :applied, synchro_type: type, operation: operation).find_each do |sync|
+      SyncAudit.transaction do
+        sync.synchro.respond_to?(:after_sync) && sync.synchro.after_sync
+        sync.update! state: 'finished'
+      end
+    end
+  end
+
+  def self.apply_synchro(type, operation: ['update', 'delete', 'insert'])
+    SyncAudit.where(state: 'init', synchro_type: type, operation: operation).find_each do |sync_audit|
+      begin
+        sync_audit.apply_changes
+      rescue SystemStackError, ActiveRecord::ActiveRecordError => e
+        logger.warn e.message
+      end
+    end
+  end
+
+end

data/app/views/rails_sync_admin/sync_audits/_filter.html.erb

@@ -0,0 +1,8 @@
+<div class="ui top attached segment">
+  <% if params[:synchro_type] %>
+    <%= link_to 'Sync', sync_admin_sync_audits_path(synchro_type: params[:synchro_type]), method: :post, class: 'ui blue button' %>
+    <%= link_to 'Batch Apply', batch_admin_sync_audits_path(synchro_type: params[:synchro_type]), method: :post, class: 'ui yellow button' %>
+  <% end %>
+</div>
+
+

data/app/views/rails_sync_admin/sync_audits/_form.html.erb

@@ -0,0 +1,4 @@
+<%= form_with model: @sync_audit, local: true do |f| %>
+  <%= render 'shared/error_messages', target: @sync_audit %>
+  <%= f.submit %>
+<% end %>

data/app/views/rails_sync_admin/sync_audits/_search_form.html.erb

@@ -0,0 +1,35 @@
+<%= search_form_with do |f| %>
+  <div class="fields">
+    <%= f.submit 'Search' %>
+  </div>
+<% end %>
+
+<dl>
+  <dt>Synchro Type:</dt>
+  <dd>
+  <%= link_to 'All', filter_params(except: [:synchro_type]), class: active_params(synchro_type: '', active_class: 'ui basic blue button', item_class: 'ui basic white button') %>
+  <% @synchro_types.each do |synchro_type| %>
+    <%= link_to SyncAudit.enum_i18n(:synchro_type, synchro_type), filter_params(synchro_type: synchro_type), class: active_params(synchro_type: synchro_type, active_class: 'ui basic blue button', item_class: 'ui basic white button') %>
+  <% end %>
+  </dd>
+</dl>
+
+<dl>
+  <dt>State:</dt>
+  <dd>
+  <%= link_to 'All', filter_params(except: [:state]), class: active_params(state: '', active_class: 'ui basic blue button', item_class: 'ui basic white button') %>
+  <% ['init', 'applied', 'finished'].each do |state| %>
+    <%= link_to SyncAudit.enum_i18n(:state, state), filter_params(state: state), class: active_params(state: state, active_class: 'ui basic blue button', item_class: 'ui basic white button') %>
+  <% end %>
+  </dd>
+</dl>
+
+<dl>
+  <dt>Action:</dt>
+  <dd>
+    <%= link_to 'All', filter_params(except: [:operation]), class: active_params(operation: '', active_class: 'ui basic blue button', item_class: 'ui basic white button') %>
+    <% ['update', 'insert', 'delete'].each do |operation| %>
+      <%= link_to SyncAudit.enum_i18n(:operation, operation), filter_params(operation: operation), class: active_params(operation: operation, active_class: 'ui basic blue button', item_class: 'ui basic white button') %>
+    <% end %>
+  </dd>
+</dl>

data/app/views/rails_sync_admin/sync_audits/edit.html.erb

@@ -0,0 +1,9 @@
+<div class="ui segment breadcrumb">
+  <%= link_to 'Back', sync_audits_path, class: 'section' %>
+  <div class="divider"> / </div>
+  <div class="active section">Edit</div>
+</div>
+
+<div class="ui segment">
+  <%= render 'form' %>
+</div>

data/app/views/rails_sync_admin/sync_audits/index.html.erb

@@ -0,0 +1,57 @@
+<div class="ui top attached borderless menu">
+  <div class="header item">Sync Audits</div>
+</div>
+
+<div class="ui attached segment">
+  <%= render 'search_form' %>
+</div>
+
+<%= render 'filter' %>
+<table class="ui bottom attached fixed table">
+  <thead>
+    <tr>
+      <th class="two wide"><%= SyncAudit.human_attribute_name(:id) %></th>
+      <th class="two wide">
+        <p><%= SyncAudit.human_attribute_name(:synchro_type) %></p>
+        <p><%= SyncAudit.human_attribute_name(:synchro_id) %></p>
+      </th>
+      <th class="two wide">
+        <%= SyncAudit.human_attribute_name(:synchro_params) %>
+      </th>
+      <th class="one wide"><%= SyncAudit.human_attribute_name(:action) %></th>
+      <th class="seven wide"><%= SyncAudit.human_attribute_name(:audited_changes) %></th>
+      <th class="two wide">
+        <%= SyncAudit.human_attribute_name(:state) %>
+      </th>
+    </tr>
+  </thead>
+
+  <tbody>
+  <% @sync_audits.each do |sync_audit| %>
+    <tr>
+      <td><%= sync_audit.id %></td>
+      <td>
+        <p><%= sync_audit.synchro_type %></p>
+        <p><%= sync_audit.synchro_id %></p>
+      </td>
+      <td><%= simple_format(sync_audit.synchro_params) %></td>
+      <td><%= sync_audit.operation_i18n %></td>
+      <td><%= simple_format_hash sync_audit.audited_changes %></td>
+      <td class="ui labels">
+        <span class="ui label"><%= sync_audit.state_i18n %></span>
+        <% unless sync_audit.applied? %>
+          <%= link_to 'apply', apply_admin_sync_audit_path(sync_audit), method: :patch, data: {confirm: 'Are you sure?'}, class: 'ui yellow label' %>
+        <% end %>
+        <%= link_to admin_sync_audit_path(sync_audit), class: 'ui mini blue icon button' do %>
+          <i class="pencil alternate icon"></i>
+        <% end %>
+        <%= link_to admin_sync_audit_path(sync_audit), method: :delete, data: {confirm: 'Are you sure?'}, class: 'ui mini red icon button' do %>
+          <i class="times icon"></i>
+        <% end %>
+      </td>
+    </tr>
+  <% end %>
+  </tbody>
+</table>
+
+<%= paginate @sync_audits %>

data/app/views/rails_sync_admin/sync_audits/new.html.erb

@@ -0,0 +1,9 @@
+<div class="ui segment breadcrumb">
+  <%= link_to 'Back', sync_audits_path, class: 'section' %>
+  <div class="divider"> / </div>
+  <div class="active section">Add</div>
+</div>
+
+<div class="ui segment">
+  <%= render 'form' %>
+</div>

data/app/views/rails_sync_admin/sync_audits/synchros.html.erb

@@ -0,0 +1,41 @@
+<div class="ui top attached borderless menu">
+  <div class="header item">Sync Audits</div>
+</div>
+
+<div class="ui attached segment">
+  <%= render 'search_form' %>
+</div>
+
+<table class="ui bottom attached table">
+  <thead>
+    <tr>
+      <th><%= SyncAudit.human_attribute_name(:id) %></th>
+      <th><%= SyncAudit.human_attribute_name(:synchro_type) %></th>
+      <th><%= SyncAudit.human_attribute_name(:synchro_id) %></th>
+      <th><%= SyncAudit.human_attribute_name(:action) %></th>
+      <th><%= SyncAudit.human_attribute_name(:audited_changes) %></th>
+      <th><%= SyncAudit.human_attribute_name(:state) %></th>
+      <th>Actions</th>
+    </tr>
+  </thead>
+
+  <tbody>
+  <% @sync_types.each do |sync_type| %>
+    <tr>
+      <td><%= sync_type %></td>
+      <td><%= sync_audit.synchro_type %></td>
+      <td><%= sync_audit.synchro_id %></td>
+      <td><%= sync_audit.operation_i18n %></td>
+      <td><%= simple_format_hash sync_audit.audited_changes %></td>
+      <td><%= sync_audit.state_i18n %></td>
+      <td>
+        <%= link_to 'apply', apply_admin_sync_audit_path(sync_audit), method: :patch, data: { confirm: 'Are you sure?' }, class: 'ui yellow label' %>
+        <%= link_to 'Show', admin_sync_audit_path(sync_audit), class: 'ui blue label' %>
+        <%= link_to 'Destroy', admin_sync_audit_path(sync_audit), method: :delete, data: { confirm: 'Are you sure?' }, class: 'ui red label' %>
+      </td>
+    </tr>
+  <% end %>
+  </tbody>
+</table>
+
+<%= paginate @sync_audits %>

data/config/locales/en.yml

@@ -0,0 +1,21 @@
+en:
+  the_trade_sync:
+    payments:
+      index:
+        add_new: add
+        import: Import
+        init: 预核销
+        confirmed: 未预核销
+      item:
+        edit: Edit
+        destroy: Destroy
+        show: 支付方式
+        checking: 核销
+        analyze: 分析支付方式
+        adjust: 平账
+  activerecord:
+    attributes:
+      sync_audit/state:
+        init: Pending
+        applied: Applied
+        finished: Finished

data/config/locales/zh.yml

@@ -0,0 +1,8 @@
+zh:
+  rails_sync_admin:
+  activerecord:
+    attributes:
+      sync_audit/state:
+        init: Init
+        applied: 已修改
+        finished: 完成

data/config/routes.rb

@@ -0,0 +1,11 @@
+Rails.application.routes.draw do
+
+  scope :admin, as: 'admin', module: 'rails_sync_admin' do
+    resources :sync_audits do
+      post :sync, on: :collection
+      post :batch, on: :collection
+      patch :apply, on: :member
+    end
+  end
+
+end

data/db/migrate/20180319031059_create_sync_audits.rb

@@ -0,0 +1,16 @@
+class CreateSyncAudits < ActiveRecord::Migration[5.2]
+  def change
+    create_table :sync_audits do |t|
+      t.references :operator, polymorphic: true
+      t.references :synchro, polymorphic: true
+      t.references :destined
+      t.string :synchro_params
+      t.string :operation
+      t.string :audited_changes, limit: 4096
+      t.string :note, limit: 1024
+      t.string :state
+      t.datetime :apply_at
+      t.timestamps
+    end
+  end
+end

data/lib/rails_sync.rb

@@ -0,0 +1,16 @@
+require 'rails_sync/engine'
+require 'rails_sync/config'
+
+require 'rails_sync/active_record'
+require 'rails_sync/adapter'
+
+module RailsSync
+  mattr_accessor :synchro_types do
+    []
+  end
+
+  def self.options
+    @options ||= Rails.application.config_for('rails_sync').with_indifferent_access
+  end
+
+end

data/lib/rails_sync/active_record.rb

@@ -0,0 +1,87 @@
+require 'rails_sync/adapter'
+require 'rails_sync/table'
+require 'rails_sync/analyzer'
+
+module RailsSync::ActiveRecord
+  # source
+  # source_client
+  # dest_table
+  def acts_as_sync(options = {})
+    @syncs ||= []
+
+    options[:dest_table] ||= self.table_name
+    _mappings = Array(options.delete(:mapping)).to_h
+    if options[:only]
+      _filter_columns = self.column_names & Array(options.delete(:only))
+    else
+      _filter_columns = self.column_names - Array(options.delete(:except))
+    end
+    options[:primary_key] = Array(options[:primary_key] || self.primary_key).map { |i| i.to_s }
+    options[:dest_primary_key] = Array(options[:dest_primary_key] || options[:primary_key]).map { |i| i.to_s }
+    options[:dest_conditions] = options[:dest_conditions]
+    options[:full_mappings] = _filter_columns.map { |column_name|
+      next if options[:primary_key].include?(column_name)
+      if _mappings.key?(column_name)
+        [column_name, _mappings[column_name]]
+      else
+        [column_name, column_name]
+      end
+    }.compact
+
+    options[:server_id] = self.server_id
+    options[:analyzer] = RailsSync::Analyzer.new(record: self, **options)
+
+    RailsSync.synchro_types << self.name
+    @syncs << options
+  end
+
+  def server_id
+    begin
+      result = connection.query('select @@server_uuid')
+    rescue
+      result = connection.query('select @@server_id')
+    end
+    _id = result.to_a.flatten.first
+    if _id.is_a?(Hash)
+      _id.values.first
+    else
+      _id
+    end
+  end
+
+  def analyze_diffs(type = 'update')
+    @syncs.flat_map do |options|
+      next if !options[:primary_key].include?(self.primary_key) && type == 'delete'
+      options[:analyzer].analyze_diffs(type)
+    end
+  end
+
+  def cache_diffs(type = 'update')
+    @syncs.flat_map do |options|
+      next if !options[:primary_key].include?(self.primary_key) && type == 'delete'
+      options[:analyzer].cache_diffs(type)
+    end
+  end
+
+  def cache_all_diffs(*types)
+    types = ['update', 'insert', 'delete'] if types.blank?
+    types.each do |type|
+      cache_diffs(type)
+    end
+  end
+
+  def id_insert?
+    @syncs.find { |i| i[:primary_key] == ['id'] }.present?
+  end
+
+  def prepare_sync
+    @syncs.flat_map do |options|
+      options[:analyzer].reset_temp_table unless options[:analyzer].same_server?
+    end
+  end
+
+end
+
+ActiveSupport.on_load :active_record do
+  extend RailsSync::ActiveRecord
+end

data/lib/rails_sync/adapter.rb

@@ -0,0 +1,46 @@
+module RailsSync
+  class Adapter
+    extend ActiveRecord::ConnectionHandling
+    mattr_accessor :connection_handler, instance_writer: false
+    self.connection_handler = ActiveRecord::ConnectionAdapters::ConnectionHandler.new
+
+    def initialize(spec, options = {})
+      @adapter_options = RailsSync.options.fetch(spec, {})
+      @adapter_options[:name] = spec
+
+      begin
+        self.connection
+      rescue
+        client = self.class.connection_handler.establish_connection(@adapter_options)
+        puts "established connection: #{client}"
+      end
+    end
+
+    def retrieve_connection
+      self.class.connection_handler.retrieve_connection(@adapter_options[:name])
+    end
+
+    def server_id
+      begin
+        result = connection.query('select @@server_uuid')
+      rescue
+        result = connection.query('select @@server_id')
+      end
+      _id = result.to_a.flatten.first
+      if _id.is_a?(Hash)
+        _id.values.first
+      else
+        _id
+      end
+    end
+
+    def connection
+      retrieve_connection
+    end
+
+    def url
+      @url ||= "mysql://#{@adapter_options[:username]}:#{@adapter_options[:password]}@#{@adapter_options[:host]}:#{@adapter_options[:port]}/#{@adapter_options[:database]}"
+    end
+
+  end
+end

data/lib/rails_sync/analyzer.rb

@@ -0,0 +1,119 @@
+class RailsSync::Analyzer
+  include RailsSync::Table
+  attr_reader :adapter, :my_arel_table, :dest_arel_table, :synchro_type
+
+  def initialize(options = {})
+    @adapter = RailsSync::Adapter.new(options[:dest])
+    @dest = options[:dest]
+    @record = options[:record]
+    @server_id = options[:server_id]
+
+    @synchro_type = @record.name
+    @table_name = @record.table_name
+    @dest_table = options[:dest_table]
+    @primary_key = options[:primary_key]
+    @dest_primary_key = options[:dest_primary_key]
+    @dest_conditions = Hash(options[:dest_conditions])
+
+    @full_mappings = options[:full_mappings]
+    @my_columns = @primary_key + @full_mappings.map { |col| col[0] }
+    @dest_columns = @dest_primary_key + @full_mappings.map { |col| col[1] }
+
+    instance_table
+    @my_arel_table ||= Arel::Table.new(@table_name)
+    @dest_arel_table ||= Arel::Table.new(@dest_table_name, as: 't1')
+  end
+
+  def connection
+    @record.connection
+  end
+
+  def skip_analyze?(type)
+    ( type == 'delete' && !@primary_key.include?(@record.primary_key) ) ||
+      ( type == 'insert' && @record.id_insert? && !@primary_key.include?(@record.primary_key) )
+  end
+
+  def cache_diffs(type = 'update')
+    analyze_diffs(type).each do |diff|
+      audit = SyncAudit.new synchro_type: synchro_type
+
+      _params = {}
+      @primary_key.each do |primary_key|
+        _params[primary_key] = diff.delete(primary_key).compact.first
+      end
+
+      audit.synchro_params = _params
+      audit.synchro_id = _params['id']
+
+      audit.operation = type
+      audit.audited_changes = diff
+      begin
+        audit.save
+      rescue ActiveRecord::ValueTooLong => e # todo not require active record
+        puts e.message
+      end
+    end
+  end
+
+  def analyze_diffs(type = 'update')
+    return [] if skip_analyze?(type)
+    sql = fetch_diffs(type)
+    results = connection.execute(sql)
+    fields = results.fields.in_groups(2).first
+    results.map do |result|
+      r = result.in_groups(2)
+      hash_value = fields.zip( r[0].zip(r[1]) ).to_h
+      hash_value.select do |key, v|
+        v[0].to_s != v[1].to_s || @primary_key.include?(key)
+      end
+    end
+  end
+
+  def fetch_diffs(type = 'update')
+    if type == 'update'
+      query = analyze_table.join(dest_arel_table).on(on_conditions)
+      query.where(analyze_conditions)
+    elsif type == 'insert'
+      query = analyze_table.join(dest_arel_table, Arel::Nodes::RightOuterJoin).on(on_conditions)
+      query.where(my_arel_table[@primary_key[0]].eq(nil))
+    elsif type == 'delete'
+      query = analyze_table.join(dest_arel_table, Arel::Nodes::OuterJoin).on(on_conditions)
+      query.where(my_arel_table[@primary_key[0]].not_eq(nil).and(dest_arel_table[@dest_primary_key[0]].eq(nil)))
+    else
+      query = analyze_table.join(dest_arel_table, Arel::Nodes::FullOuterJoin).on(on_conditions)
+    end
+
+    query.where(where_conditions) if where_conditions.present?
+    query.to_sql
+  end
+
+  def analyze_table
+    attrs = @my_columns.map { |col| my_arel_table[col] }
+    attrs += @dest_columns.map { |col| dest_arel_table[col] }
+    my_arel_table.project(*attrs)
+  end
+
+  def analyze_conditions
+    mappings = @full_mappings.map do |mapping|
+      my_arel_table[mapping[0]].not_eq(dest_arel_table[mapping[1]])
+    end
+    Arel::Nodes::SqlLiteral.new mappings.map(&:to_sql).join(' OR ')
+  end
+
+  def on_conditions
+    mappings = @primary_key.map.with_index do |left_key, index|
+      my_arel_table[left_key].eq(dest_arel_table[@dest_primary_key[index]])
+    end
+    Arel::Nodes::SqlLiteral.new mappings.map(&:to_sql).join(' AND ')
+  end
+
+  def where_conditions
+    cons = @dest_conditions.map do |key, value|
+      col, meth = key.to_s.split('-')
+      dest_arel_table[col].send meth, value
+    end
+
+    Arel::Nodes::SqlLiteral.new cons.map(&:to_sql).join(' AND ')
+  end
+
+end

data/lib/rails_sync/config.rb

@@ -0,0 +1,10 @@
+require 'active_support/configurable'
+
+module RailsSync
+  include ActiveSupport::Configurable
+
+  configure do |config|
+    config.admin_class = 'Admin::BaseController'
+  end
+
+end

data/lib/rails_sync/engine.rb

@@ -0,0 +1,7 @@
+module RailsSync
+  class Engine < ::Rails::Engine
+
+    config.eager_load_paths += Dir["#{config.root}/app/models/mysql"]
+
+  end
+end

data/lib/rails_sync/table.rb

@@ -0,0 +1,110 @@
+module RailsSync
+  module Table
+    attr_reader :dest_table_name
+
+    def instance_table
+      if same_server?
+        # `source.table_name`
+        @dest_table_name = @adapter.instance_variable_get(:@adapter_options)[:database].to_s + '.' + @dest_table.to_s
+      else
+        # `source_table_name`
+        @dest_table_name = @dest.to_s + '_' + @table_name + '-' + @dest_table.to_s
+      end
+    end
+
+    def same_server?
+      @server_id == adapter.server_id
+    end
+
+    def dest_columns
+      adapter.connection.columns(@dest_table)
+    end
+
+    def dest_indexes
+      results = adapter.connection.indexes(@dest_table)
+      results = results.map { |result| { result.name => result.columns } }
+      results.to_combined_hash  # rails_com core ext
+      results
+    end
+
+    def dest_primary_key
+      adapter.connection.primary_key(@dest_table)
+    end
+
+    def dest_sql_table(only: [], except: [], pure: true)
+      if only.size > 0
+        _columns = dest_columns.select { |column| only.include?(column.name) }
+      else
+        _columns = dest_columns.reject { |column| except.include?(column.name) }
+      end
+
+      if pure
+        sql = ""
+      else
+        sql = "CREATE TABLE `#{@dest_table}` (\n"
+      end
+
+      _columns.each do |column|
+        sql << "  `#{column.name}` #{column.sql_type}"
+        sql << " COLLATE #{column.collation}" if column.collation.present?
+        sql << " NOT NULL" if column.null.is_a?(FalseClass)
+        if column.default
+          sql << " DEFAULT '#{column.default}',\n"
+        elsif column.default.nil? && column.null
+          sql << " DEFAULT NULL,\n"
+        else
+          sql << ",\n"
+        end
+      end
+
+      sql << "  PRIMARY KEY (`#{dest_primary_key}`)"
+
+      _indexes = dest_indexes.reject { |_, value| (Array(value) & _columns.map { |col| col.name }).blank? }
+
+      if _indexes.present?
+        sql << ",\n"
+      else
+        sql << "\n"
+      end
+      _indexes.each do |index, columns|
+        sql << "  KEY `#{index}` ("
+        sql << Array(columns).map { |col| "`#{col}`" }.join(',')
+        sql << "),\n"
+      end
+
+      sql.chomp!(",\n")
+
+      if pure
+        sql
+      else
+        sql << ")"
+      end
+    end
+
+    def reset_temp_table
+      drop_temp_table
+      create_temp_table
+    end
+
+    def create_temp_table
+      unless @dest_columns.include?(dest_primary_key)
+        @dest_columns.unshift dest_primary_key
+      end
+
+      sql = "CREATE TABLE `#{@dest_table_name}` (\n"
+      sql << dest_sql_table(only: @dest_columns)
+      sql << ")"
+      sql << "ENGINE=FEDERATED\n"
+      sql << "CONNECTION='#{adapter.url}/#{@dest_table}';"
+
+      connection.execute(sql)
+    end
+
+    def drop_temp_table
+      sql = "DROP TABLE IF EXISTS `#{@dest_table_name}`"
+
+      connection.execute(sql)
+    end
+
+  end
+end

data/lib/rails_sync/version.rb

@@ -0,0 +1,3 @@
+module RailsSync
+  VERSION = '0.1.0'
+end

data/lib/tasks/the_sync_tasks.rake

@@ -0,0 +1,4 @@
+# desc "Explaining what the task does"
+# task :rails_sync do
+#   # Task goes here
+# end

metadata

@@ -0,0 +1,92 @@
+--- !ruby/object:Gem::Specification
+name: rails_sync
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- qinmingyuan
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-09-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+description: Description of RailsSync.
+email:
+- mingyuan0715@foxmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- Rakefile
+- app/assets/config/rails_sync_manifest.js
+- app/controllers/rails_sync_admin/base_controller.rb
+- app/controllers/rails_sync_admin/sync_audits_controller.rb
+- app/jobs/audit_apply_job.rb
+- app/models/mysql/mysql_column.rb
+- app/models/mysql/mysql_engine.rb
+- app/models/mysql/mysql_index.rb
+- app/models/mysql/mysql_server.rb
+- app/models/mysql/mysql_table.rb
+- app/models/sync_audit.rb
+- app/views/rails_sync_admin/sync_audits/_filter.html.erb
+- app/views/rails_sync_admin/sync_audits/_form.html.erb
+- app/views/rails_sync_admin/sync_audits/_search_form.html.erb
+- app/views/rails_sync_admin/sync_audits/edit.html.erb
+- app/views/rails_sync_admin/sync_audits/index.html.erb
+- app/views/rails_sync_admin/sync_audits/new.html.erb
+- app/views/rails_sync_admin/sync_audits/show.html.erb
+- app/views/rails_sync_admin/sync_audits/synchros.html.erb
+- config/locales/en.yml
+- config/locales/zh.yml
+- config/routes.rb
+- db/migrate/20180319031059_create_sync_audits.rb
+- lib/rails_sync.rb
+- lib/rails_sync/active_record.rb
+- lib/rails_sync/adapter.rb
+- lib/rails_sync/analyzer.rb
+- lib/rails_sync/config.rb
+- lib/rails_sync/engine.rb
+- lib/rails_sync/table.rb
+- lib/rails_sync/version.rb
+- lib/tasks/the_sync_tasks.rake
+homepage: https://github.com/yougexiangfa/rails_sync
+licenses:
+- LGPL-3.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.7
+signing_key: 
+specification_version: 4
+summary: Summary of RailsSync.
+test_files: []