gamma_replication 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b68929290d56de956de9597cfbdb9c6c7e2574f26489b5b55fc0223aa7d3c6ad
-  data.tar.gz: 550aa21f4800a9da12f9d4f7b5bb4565a999321623d700cb74ae360bac461ae5
+  metadata.gz: 39c63ef1a8906600d9df6b25658cbd4a1b8674be75cb6b5d3d870419f9e5f75e
+  data.tar.gz: ad13bc1e0d79976532f368cd555e59cb89300c920743f35d145b7a928a588ae9
 SHA512:
-  metadata.gz: 73720919700ce8c4158bee20a58f375fe5a160ba1e44aa0f4348ed3f7ba0ba60d75c3224563579c65c5616dabd4f764b997c936dee029c12a2bade898f410028
-  data.tar.gz: 10cf80dc382210270080adf30148a7556938bb164633bce1336484f3fdf22be571622ab711e2289aefc7486b8d1c3ea54e12932a117af77540eda7ba73045023
+  metadata.gz: 6a5aa663b77e7354ff500e7fcb3fe936b103edf466442e8e44b9e31297da5c6dd4eec0ba12472f195490f710ab6d3dbd3a351311e2676d1edd104b73b396752f
+  data.tar.gz: bc597b21d63e0135ffa154116db12acf4578678db85ab9ef090df30e7247d078e1907460f377c4baa0111317ae9dde8e0bbc5ab9e57cbcda68a9288074e5ab6f

data/exe/gamma_replication

@@ -12,6 +12,8 @@ class GammaReplicationCLI < Thor
   option :data, aliases: "-d", desc: "Table Sync Settings yaml", required: true
   option :hook_dir, aliases: "-h", desc: "Hook script directory", default: "."
   option :maxwell_config, aliases: "-m", desc: "Maxwell configuration file path", required: true
+  option :enable_stats, type: :boolean, desc: "Enable statistics output (default: true)"
+  option :stats_interval, type: :numeric, desc: "Statistics output interval in seconds (default: 10800)"
   def start
     GammaReplication::Command::Start.new(options).execute
   end
@@ -21,6 +23,8 @@ class GammaReplicationCLI < Thor
   option :data, aliases: "-d", desc: "Table Sync Settings yaml", required: true
   option :hook_dir, aliases: "-h", desc: "Hook script directory", default: "."
   option :maxwell_config, aliases: "-m", desc: "Maxwell configuration file path", required: true
+  option :enable_stats, type: :boolean, desc: "Enable statistics output (default: true)"
+  option :stats_interval, type: :numeric, desc: "Statistics output interval in seconds (default: 10800)"
   def dryrun
     GammaReplication::Command::Dryrun.new(options).execute
   end

data/lib/gamma_replication/command/base_replication.rb

@@ -8,6 +8,9 @@ module GammaReplication
         setup_database(opts)
         setup_parser(opts)
         setup_maxwell(opts)
+        initialize_stats
+        @enable_stats = opts[:enable_stats].nil? ? true : opts[:enable_stats]
+        @stats_interval = (opts[:stats_interval] || 3 * 60 * 60).to_i # Default: 3 hours in seconds
       end
 
       def execute
@@ -18,6 +21,9 @@ module GammaReplication
           hash[table.table_name] = table
         end
         before_start if respond_to?(:before_start)
+
+        setup_stats_reporter if @enable_stats
+
         @maxwell_client.start do |data|
           process_maxwell_data(data)
         end
@@ -25,6 +31,52 @@ module GammaReplication
 
       private
 
+      def initialize_stats
+        @stats = {
+          total: { insert: 0, update: 0, delete: 0 },
+          by_table: {}
+        }
+        @stats_mutex = Mutex.new
+      end
+
+      def setup_stats_reporter
+        @stats_thread = Thread.new do
+          loop do
+            sleep @stats_interval
+            output_stats
+            reset_stats
+          end
+        end
+      end
+
+      def update_stats(table_name, operation)
+        return unless @enable_stats
+
+        @stats_mutex.synchronize do
+          @stats[:total][operation] += 1
+          @stats[:by_table][table_name] ||= { insert: 0, update: 0, delete: 0 }
+          @stats[:by_table][table_name][operation] += 1
+        end
+      end
+
+      def reset_stats
+        @stats_mutex.synchronize do
+          initialize_stats
+        end
+      end
+
+      def output_stats
+        @stats_mutex.synchronize do
+          total_stats = "Total[ins:#{@stats[:total][:insert]} | upd:#{@stats[:total][:update]} | del:#{@stats[:total][:delete]}]"
+
+          table_stats = @stats[:by_table].map do |table, counts|
+            "#{table}[ins:#{counts[:insert]} | upd:#{counts[:update]} | del:#{counts[:delete]}]"
+          end.join("\n")
+
+          logger.info("STATS | #{total_stats}\n#{table_stats}")
+        end
+      end
+
       def setup_database(opts)
         @database_settings = GammaReplication::DatabaseSettings.new(opts[:settings])
         @in_client = GammaReplication::DatabaseConnector::MysqlConnector.new(@database_settings.in_database)
@@ -64,15 +116,23 @@ module GammaReplication
         table_setting = @table_settings[data["table"]]
         case data["type"]
         when "insert"
+          return unless data["data"].present?
+
+          update_stats(data["table"], :insert)
           process_insert(table_setting, data)
         when "update"
+          return unless data["data"].present?
+
+          update_stats(data["table"], :update)
           process_update(table_setting, data)
         when "delete"
+          return unless data["old"].present?
+
+          update_stats(data["table"], :delete)
           process_delete(table_setting, data)
         end
       rescue StandardError => e
-        logger.error(e)
-        # Nothing
+        logger.error("Error processing data: #{data["table"]} - #{e.message}")
       end
 
       def process_insert(table_setting, data)
@@ -100,8 +160,9 @@ module GammaReplication
 
       def process_delete(table_setting, data)
         old_record = data["old"]
-        where_clause = build_where_clause(old_record, nil, table_setting.primary_key)
+        return unless old_record.present? && old_record[table_setting.primary_key].present?
 
+        where_clause = build_where_clause(old_record, nil, table_setting.primary_key)
         query = "DELETE FROM #{table_setting.table_name} WHERE #{where_clause}"
         execute_query(query)
       end
@@ -120,7 +181,8 @@ module GammaReplication
         elsif new_record.present? && new_record[primary_key].present?
           "`#{primary_key}` = #{format_value(new_record[primary_key])}"
         else
-          raise "Primary key not found in record. old_record: #{old_record.inspect}, new_record: #{new_record.inspect}"
+          logger.error("Primary key not found. old_record: #{old_record.inspect}, new_record: #{new_record.inspect}")
+          raise "Primary key '#{primary_key}' not found in record"
         end
       end
 
@@ -133,7 +195,34 @@ module GammaReplication
         when Time
           "'#{value.strftime("%Y-%m-%d %H:%M:%S")}'"
         else
-          "'#{@out_client.client.escape(value.to_s)}'"
+          if json_column?(value)
+            sanitized_value = sanitize_json(value)
+            "'#{@out_client.client.escape(sanitized_value)}'"
+          else
+            "'#{@out_client.client.escape(value.to_s)}'"
+          end
+        end
+      end
+
+      def json_column?(value)
+        value.is_a?(String) && (value.start_with?("{") || value.start_with?("["))
+      end
+
+      def sanitize_json(value)
+        JSON.parse(value)
+        value
+      rescue JSON::ParserError => e
+        sanitized = value.gsub(/([{,]\s*)(\w+)(\s*:)/) do
+          "#{::Regexp.last_match(1)}\"#{::Regexp.last_match(2)}\"#{::Regexp.last_match(3)}"
+        end
+        sanitized = sanitized.gsub(/:\s*([^",\s\d\[\]{}-].*?)(,|\}|$)/) do
+          ": \"#{::Regexp.last_match(1)}\"#{::Regexp.last_match(2)}"
+        end
+        begin
+          JSON.parse(sanitized)
+          sanitized
+        rescue JSON::ParserError
+          value.to_json
         end
       end
 

data/lib/gamma_replication/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module GammaReplication
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gamma_replication
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Shinsuke Nishio
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-01-22 00:00:00.000000000 Z
+date: 2025-02-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport