gamma_replication 0.1.0 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2c388f3c2fc5c5b7b1bad0f7b2c6d9f249883979710b2f81b1f405a5557ec201
-  data.tar.gz: 78d48d17c39c15b838c6e2a4a842473dda65711eaa8f6e21ae11de8314c4d945
+  metadata.gz: 39c63ef1a8906600d9df6b25658cbd4a1b8674be75cb6b5d3d870419f9e5f75e
+  data.tar.gz: ad13bc1e0d79976532f368cd555e59cb89300c920743f35d145b7a928a588ae9
 SHA512:
-  metadata.gz: 7615b3b2e9cc4fdf11139303436fbdf906c291a477ff8d09263a9773615f2bfeb7a742c093eb262138f79ff93d9aac148ceaa425c105a7653852e10d1dfc8580
-  data.tar.gz: 57bc2df77112be468ed5a07361b2e45230fca972ef71d973a00f8aa4b99adafb968f3095275fb03f5d9be3bd2a8be540141460892f0f9f468a79369e15c2a1cc
+  metadata.gz: 6a5aa663b77e7354ff500e7fcb3fe936b103edf466442e8e44b9e31297da5c6dd4eec0ba12472f195490f710ab6d3dbd3a351311e2676d1edd104b73b396752f
+  data.tar.gz: bc597b21d63e0135ffa154116db12acf4578678db85ab9ef090df30e7247d078e1907460f377c4baa0111317ae9dde8e0bbc5ab9e57cbcda68a9288074e5ab6f

data/exe/gamma_replication

@@ -12,6 +12,8 @@ class GammaReplicationCLI < Thor
   option :data, aliases: "-d", desc: "Table Sync Settings yaml", required: true
   option :hook_dir, aliases: "-h", desc: "Hook script directory", default: "."
   option :maxwell_config, aliases: "-m", desc: "Maxwell configuration file path", required: true
+  option :enable_stats, type: :boolean, desc: "Enable statistics output (default: true)"
+  option :stats_interval, type: :numeric, desc: "Statistics output interval in seconds (default: 10800)"
   def start
     GammaReplication::Command::Start.new(options).execute
   end
@@ -21,6 +23,8 @@ class GammaReplicationCLI < Thor
   option :data, aliases: "-d", desc: "Table Sync Settings yaml", required: true
   option :hook_dir, aliases: "-h", desc: "Hook script directory", default: "."
   option :maxwell_config, aliases: "-m", desc: "Maxwell configuration file path", required: true
+  option :enable_stats, type: :boolean, desc: "Enable statistics output (default: true)"
+  option :stats_interval, type: :numeric, desc: "Statistics output interval in seconds (default: 10800)"
   def dryrun
     GammaReplication::Command::Dryrun.new(options).execute
   end

data/lib/gamma_replication/command/base_replication.rb

@@ -8,6 +8,9 @@ module GammaReplication
         setup_database(opts)
         setup_parser(opts)
         setup_maxwell(opts)
+        initialize_stats
+        @enable_stats = opts[:enable_stats].nil? ? true : opts[:enable_stats]
+        @stats_interval = (opts[:stats_interval] || 3 * 60 * 60).to_i # Default: 3 hours in seconds
       end
 
       def execute
@@ -18,6 +21,9 @@ module GammaReplication
           hash[table.table_name] = table
         end
         before_start if respond_to?(:before_start)
+
+        setup_stats_reporter if @enable_stats
+
         @maxwell_client.start do |data|
           process_maxwell_data(data)
         end
@@ -25,6 +31,52 @@ module GammaReplication
 
       private
 
+      def initialize_stats
+        @stats = {
+          total: { insert: 0, update: 0, delete: 0 },
+          by_table: {}
+        }
+        @stats_mutex = Mutex.new
+      end
+
+      def setup_stats_reporter
+        @stats_thread = Thread.new do
+          loop do
+            sleep @stats_interval
+            output_stats
+            reset_stats
+          end
+        end
+      end
+
+      def update_stats(table_name, operation)
+        return unless @enable_stats
+
+        @stats_mutex.synchronize do
+          @stats[:total][operation] += 1
+          @stats[:by_table][table_name] ||= { insert: 0, update: 0, delete: 0 }
+          @stats[:by_table][table_name][operation] += 1
+        end
+      end
+
+      def reset_stats
+        @stats_mutex.synchronize do
+          initialize_stats
+        end
+      end
+
+      def output_stats
+        @stats_mutex.synchronize do
+          total_stats = "Total[ins:#{@stats[:total][:insert]} | upd:#{@stats[:total][:update]} | del:#{@stats[:total][:delete]}]"
+
+          table_stats = @stats[:by_table].map do |table, counts|
+            "#{table}[ins:#{counts[:insert]} | upd:#{counts[:update]} | del:#{counts[:delete]}]"
+          end.join("\n")
+
+          logger.info("STATS | #{total_stats}\n#{table_stats}")
+        end
+      end
+
       def setup_database(opts)
         @database_settings = GammaReplication::DatabaseSettings.new(opts[:settings])
         @in_client = GammaReplication::DatabaseConnector::MysqlConnector.new(@database_settings.in_database)
@@ -64,22 +116,30 @@ module GammaReplication
         table_setting = @table_settings[data["table"]]
         case data["type"]
         when "insert"
+          return unless data["data"].present?
+
+          update_stats(data["table"], :insert)
           process_insert(table_setting, data)
         when "update"
+          return unless data["data"].present?
+
+          update_stats(data["table"], :update)
           process_update(table_setting, data)
         when "delete"
+          return unless data["old"].present?
+
+          update_stats(data["table"], :delete)
           process_delete(table_setting, data)
         end
       rescue StandardError => e
-        logger.error(e)
-        # Nothing
+        logger.error("Error processing data: #{data["table"]} - #{e.message}")
       end
 
       def process_insert(table_setting, data)
         record = data["data"]
         processed_record = apply_hooks(table_setting, record)
 
-        columns = processed_record.keys
+        columns = processed_record.keys.map { |k| "`#{k}`" }
         values = processed_record.values.map { |v| format_value(v) }
 
         query = "INSERT INTO #{table_setting.table_name} (#{columns.join(",")}) VALUES (#{values.join(",")})"
@@ -91,7 +151,7 @@ module GammaReplication
         old_record = data["old"]
         processed_record = apply_hooks(table_setting, record)
 
-        set_clause = processed_record.map { |k, v| "#{k} = #{format_value(v)}" }.join(",")
+        set_clause = processed_record.map { |k, v| "`#{k}` = #{format_value(v)}" }.join(",")
         where_clause = build_where_clause(old_record, record, table_setting.primary_key)
 
         query = "UPDATE #{table_setting.table_name} SET #{set_clause} WHERE #{where_clause}"
@@ -100,8 +160,9 @@ module GammaReplication
 
       def process_delete(table_setting, data)
         old_record = data["old"]
-        where_clause = build_where_clause(old_record, nil, table_setting.primary_key)
+        return unless old_record.present? && old_record[table_setting.primary_key].present?
 
+        where_clause = build_where_clause(old_record, nil, table_setting.primary_key)
         query = "DELETE FROM #{table_setting.table_name} WHERE #{where_clause}"
         execute_query(query)
       end
@@ -116,9 +177,12 @@ module GammaReplication
 
       def build_where_clause(old_record, new_record, primary_key)
         if old_record.present? && old_record[primary_key].present?
-          "#{primary_key} = #{format_value(old_record[primary_key])}"
+          "`#{primary_key}` = #{format_value(old_record[primary_key])}"
+        elsif new_record.present? && new_record[primary_key].present?
+          "`#{primary_key}` = #{format_value(new_record[primary_key])}"
         else
-          "#{primary_key} = #{format_value(new_record[primary_key])}"
+          logger.error("Primary key not found. old_record: #{old_record.inspect}, new_record: #{new_record.inspect}")
+          raise "Primary key '#{primary_key}' not found in record"
         end
       end
 
@@ -131,7 +195,34 @@ module GammaReplication
         when Time
           "'#{value.strftime("%Y-%m-%d %H:%M:%S")}'"
         else
-          "'#{@out_client.client.escape(value.to_s)}'"
+          if json_column?(value)
+            sanitized_value = sanitize_json(value)
+            "'#{@out_client.client.escape(sanitized_value)}'"
+          else
+            "'#{@out_client.client.escape(value.to_s)}'"
+          end
+        end
+      end
+
+      def json_column?(value)
+        value.is_a?(String) && (value.start_with?("{") || value.start_with?("["))
+      end
+
+      def sanitize_json(value)
+        JSON.parse(value)
+        value
+      rescue JSON::ParserError => e
+        sanitized = value.gsub(/([{,]\s*)(\w+)(\s*:)/) do
+          "#{::Regexp.last_match(1)}\"#{::Regexp.last_match(2)}\"#{::Regexp.last_match(3)}"
+        end
+        sanitized = sanitized.gsub(/:\s*([^",\s\d\[\]{}-].*?)(,|\}|$)/) do
+          ": \"#{::Regexp.last_match(1)}\"#{::Regexp.last_match(2)}"
+        end
+        begin
+          JSON.parse(sanitized)
+          sanitized
+        rescue JSON::ParserError
+          value.to_json
         end
       end
 

data/lib/gamma_replication/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module GammaReplication
-  VERSION = "0.1.0"
+  VERSION = "0.1.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gamma_replication
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.2
 platform: ruby
 authors:
 - Shinsuke Nishio
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-01-20 00:00:00.000000000 Z
+date: 2025-02-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport