gamma_replication 0.1.1 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b68929290d56de956de9597cfbdb9c6c7e2574f26489b5b55fc0223aa7d3c6ad
-  data.tar.gz: 550aa21f4800a9da12f9d4f7b5bb4565a999321623d700cb74ae360bac461ae5
+  metadata.gz: 6fcf79da2a605e4984ba168b86117cc087f4da9414fc3075949ec3de0e4150e0
+  data.tar.gz: 369e57afcf359c2f148a0a3c985641ccf5beaa3706aade4c48e210827d5c04ec
 SHA512:
-  metadata.gz: 73720919700ce8c4158bee20a58f375fe5a160ba1e44aa0f4348ed3f7ba0ba60d75c3224563579c65c5616dabd4f764b997c936dee029c12a2bade898f410028
-  data.tar.gz: 10cf80dc382210270080adf30148a7556938bb164633bce1336484f3fdf22be571622ab711e2289aefc7486b8d1c3ea54e12932a117af77540eda7ba73045023
+  metadata.gz: c669c00890ad60fdfe242ad8e472f4f7b63154e4755d22bd798223c47536384f94f51dc8ae6bb2cd8aef83c386f5ac3f72906dcb4b4cccb709beb5282df9a2fa
+  data.tar.gz: f7ff79e999e05d3dabe4e5bd73fc9c0d8bf1939d0faef44391ba212eb5c8516809e65c50c5ee972e044cb6b7d68805c172ad18924f01b53644e2146a68a761ce

data/exe/gamma_replication

@@ -12,6 +12,8 @@ class GammaReplicationCLI < Thor
   option :data, aliases: "-d", desc: "Table Sync Settings yaml", required: true
   option :hook_dir, aliases: "-h", desc: "Hook script directory", default: "."
   option :maxwell_config, aliases: "-m", desc: "Maxwell configuration file path", required: true
+  option :enable_stats, type: :boolean, desc: "Enable statistics output (default: true)"
+  option :stats_interval, type: :numeric, desc: "Statistics output interval in seconds (default: 10800)"
   def start
     GammaReplication::Command::Start.new(options).execute
   end
@@ -21,6 +23,8 @@ class GammaReplicationCLI < Thor
   option :data, aliases: "-d", desc: "Table Sync Settings yaml", required: true
   option :hook_dir, aliases: "-h", desc: "Hook script directory", default: "."
   option :maxwell_config, aliases: "-m", desc: "Maxwell configuration file path", required: true
+  option :enable_stats, type: :boolean, desc: "Enable statistics output (default: true)"
+  option :stats_interval, type: :numeric, desc: "Statistics output interval in seconds (default: 10800)"
   def dryrun
     GammaReplication::Command::Dryrun.new(options).execute
   end

data/lib/gamma_replication/command/base_replication.rb

@@ -8,6 +8,9 @@ module GammaReplication
         setup_database(opts)
         setup_parser(opts)
         setup_maxwell(opts)
+        initialize_stats
+        @enable_stats = opts[:enable_stats].nil? ? true : opts[:enable_stats]
+        @stats_interval = (opts[:stats_interval] || 3 * 60 * 60).to_i # Default: 3 hours in seconds
       end
 
       def execute
@@ -18,6 +21,9 @@ module GammaReplication
           hash[table.table_name] = table
         end
         before_start if respond_to?(:before_start)
+
+        setup_stats_reporter if @enable_stats
+
         @maxwell_client.start do |data|
           process_maxwell_data(data)
         end
@@ -25,6 +31,52 @@ module GammaReplication
 
       private
 
+      def initialize_stats
+        @stats = {
+          total: { insert: 0, update: 0, delete: 0 },
+          by_table: {}
+        }
+        @stats_mutex = Mutex.new
+      end
+
+      def setup_stats_reporter
+        @stats_thread = Thread.new do
+          loop do
+            sleep @stats_interval
+            output_stats
+            reset_stats
+          end
+        end
+      end
+
+      def update_stats(table_name, operation)
+        return unless @enable_stats
+
+        @stats_mutex.synchronize do
+          @stats[:total][operation] += 1
+          @stats[:by_table][table_name] ||= { insert: 0, update: 0, delete: 0 }
+          @stats[:by_table][table_name][operation] += 1
+        end
+      end
+
+      def reset_stats
+        @stats_mutex.synchronize do
+          initialize_stats
+        end
+      end
+
+      def output_stats
+        @stats_mutex.synchronize do
+          total_stats = "Total[ins:#{@stats[:total][:insert]} | upd:#{@stats[:total][:update]} | del:#{@stats[:total][:delete]}]"
+
+          table_stats = @stats[:by_table].map do |table, counts|
+            "#{table}[ins:#{counts[:insert]} | upd:#{counts[:update]} | del:#{counts[:delete]}]"
+          end.join("\n")
+
+          logger.info("STATS | #{total_stats}\n#{table_stats}")
+        end
+      end
+
       def setup_database(opts)
         @database_settings = GammaReplication::DatabaseSettings.new(opts[:settings])
         @in_client = GammaReplication::DatabaseConnector::MysqlConnector.new(@database_settings.in_database)
@@ -48,7 +100,8 @@ module GammaReplication
 
         process_data_by_type(data)
       rescue StandardError => e
-        logger.error(e)
+        error_message = e.message.to_s.split("\n").first
+        logger.error("Error: #{error_message.gsub(/\s+/, " ")}")
       end
 
       def should_process_data?(data)
@@ -64,15 +117,24 @@ module GammaReplication
         table_setting = @table_settings[data["table"]]
         case data["type"]
         when "insert"
+          return unless data["data"].present?
+
+          update_stats(data["table"], :insert)
           process_insert(table_setting, data)
         when "update"
+          return unless data["data"].present?
+
+          update_stats(data["table"], :update)
           process_update(table_setting, data)
         when "delete"
+          return unless data["old"].present?
+
+          update_stats(data["table"], :delete)
           process_delete(table_setting, data)
         end
       rescue StandardError => e
-        logger.error(e)
-        # Nothing
+        error_message = e.message.to_s.split("\n").first
+        logger.error("Error processing #{data["type"]} for table #{data["table"]}: #{error_message.gsub(/\s+/, " ")}")
       end
 
       def process_insert(table_setting, data)
@@ -100,8 +162,9 @@ module GammaReplication
 
       def process_delete(table_setting, data)
         old_record = data["old"]
-        where_clause = build_where_clause(old_record, nil, table_setting.primary_key)
+        return unless old_record.present? && old_record[table_setting.primary_key].present?
 
+        where_clause = build_where_clause(old_record, nil, table_setting.primary_key)
         query = "DELETE FROM #{table_setting.table_name} WHERE #{where_clause}"
         execute_query(query)
       end
@@ -120,7 +183,8 @@ module GammaReplication
         elsif new_record.present? && new_record[primary_key].present?
           "`#{primary_key}` = #{format_value(new_record[primary_key])}"
         else
-          raise "Primary key not found in record. old_record: #{old_record.inspect}, new_record: #{new_record.inspect}"
+          logger.error("Primary key not found. old_record: #{old_record.inspect}, new_record: #{new_record.inspect}")
+          raise "Primary key '#{primary_key}' not found in record"
         end
       end
 
@@ -133,7 +197,34 @@ module GammaReplication
         when Time
           "'#{value.strftime("%Y-%m-%d %H:%M:%S")}'"
         else
-          "'#{@out_client.client.escape(value.to_s)}'"
+          if json_column?(value)
+            sanitized_value = sanitize_json(value)
+            "'#{@out_client.client.escape(sanitized_value)}'"
+          else
+            "'#{@out_client.client.escape(value.to_s)}'"
+          end
+        end
+      end
+
+      def json_column?(value)
+        value.is_a?(String) && (value.start_with?("{") || value.start_with?("["))
+      end
+
+      def sanitize_json(value)
+        JSON.parse(value)
+        value
+      rescue JSON::ParserError => e
+        sanitized = value.gsub(/([{,]\s*)(\w+)(\s*:)/) do
+          "#{::Regexp.last_match(1)}\"#{::Regexp.last_match(2)}\"#{::Regexp.last_match(3)}"
+        end
+        sanitized = sanitized.gsub(/:\s*([^",\s\d\[\]{}-].*?)(,|\}|$)/) do
+          ": \"#{::Regexp.last_match(1)}\"#{::Regexp.last_match(2)}"
+        end
+        begin
+          JSON.parse(sanitized)
+          sanitized
+        rescue JSON::ParserError
+          value.to_json
         end
       end
 

data/lib/gamma_replication/maxwell_client.rb

@@ -19,8 +19,9 @@ module GammaReplication
         io.each do |line|
           data = JSON.parse(line.strip)
           block.call(data) if block_given?
-        rescue JSON::ParserError => e
-          logger.error("Failed to parse Maxwell output: #{e.message}")
+        rescue JSON::ParserError
+          # Ignore Maxwell's non-JSON output (startup messages, etc)
+          next
         end
       end
     end

data/lib/gamma_replication/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module GammaReplication
-  VERSION = "0.1.1"
+  VERSION = "0.1.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gamma_replication
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.3
 platform: ruby
 authors:
 - Shinsuke Nishio
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-01-22 00:00:00.000000000 Z
+date: 2025-02-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport