groonga-delta 1.0.2 → 1.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ccfa632a6c046239e0163f15306f5479a64c220aa9e419bd415376bfeff5eae2
-  data.tar.gz: 3ab279d8f62f66bda8e83fc71c3cb27b5472339e5e373868ccbfde0376759c67
+  metadata.gz: 5837cbbd7824ebf557d24be10007166c476dadf2157a96071522307d0ffbd0d7
+  data.tar.gz: '086fa03eb3a6c9d1a055ec8ffecf0a8b7e9102dcee0fd1d3326ece684f4e5560'
 SHA512:
-  metadata.gz: eaa509b863868cd7fa1944d52603c312dc980707d975db9b52dd8b74131b2eac293cecc71dab48abb0895a22203dc2c59942750186e491209a3f395b481496b0
-  data.tar.gz: 377115552df709288fdff0f1eff36d76e1a99b14e83409f2ab29f5d263b15e94b756d387f357f50423f2015c67f66badb662100d8dac8fdacac67f53d45af0c4
+  metadata.gz: a3db1aedcdbf3bb39312bfb091a778091111eec880488917a4d21598834e08a703181f383f47d06e2646e873d1ea1ac5d8e42ed46d71328ee8ecb8c837aa080d
+  data.tar.gz: 584ecff5ca3bf44fc9f4baea4ea8bd5f6c60227ac0f05367dd0860c74a563b26f4bcac747c74a7ce1c98a77f77100c2c736817ca6daa4af8138aec1372996f1c

data/doc/text/news.md

@@ -1,5 +1,19 @@
 # NEWS
 
+## 1.0.3 - 2022-07-21
+
+### Improvements
+
+  * `import`: Dropped support for `mysqlbinlog`.
+
+  * Changed to use atomic write for `status.yaml`.
+
+  * Changed to keep 100 logs from 7 logs by default.
+
+### Fixes
+
+  * `import`: Fixed a bug that events to be processed can't be detected.
+
 ## 1.0.2 - 2022-06-21
 
 ### Improvements

data/lib/groonga-delta/config.rb

@@ -64,7 +64,7 @@ module GroongaDelta
     end
 
     def log_age
-      @data["log_age"] || 7
+      @data["log_age"] || 100
     end
 
     def log_max_size

data/lib/groonga-delta/import-config.rb

@@ -59,10 +59,6 @@ module GroongaDelta
         resolve_path(@data["binlog_dir"] || "binlog")
       end
 
-      def mysqlbinlog
-        @data["mysqlbinlog"] || "mysqlbinlog"
-      end
-
       def host
         @data["host"] || "localhost"
       end
@@ -183,20 +179,20 @@ module GroongaDelta
         case value
         when String
           case value
-          when /\A(\d+(?:\.\d+))(?:s|sec|second|seconds)?\z/
+          when /\A(\d+(?:\.\d+)?)(?:s|sec|second|seconds)?\z/
             Float($1)
-          when /\A(\d+(?:\.\d+))(?:m|min|minute|minutes)\z/
+          when /\A(\d+(?:\.\d+)?)(?:m|min|minute|minutes)\z/
             Float($1) * 60
-          when /\A(\d+(?:\.\d+))(?:h|hr|hour|hours)\z/
+          when /\A(\d+(?:\.\d+)?)(?:h|hr|hour|hours)\z/
             Float($1) * 60 * 60
-          when /\A(\d+(?:\.\d+))(?:d|day|days)\z/
+          when /\A(\d+(?:\.\d+)?)(?:d|day|days)\z/
             Float($1) * 60 * 60 * 24
-          when /\A(\d+(?:\.\d+))(?:w|week|weeks)\z/
+          when /\A(\d+(?:\.\d+)?)(?:w|week|weeks)\z/
             Float($1) * 60 * 60 * 24 * 7
-          when /\A(\d+(?:\.\d+))(?:month|months)\z/
+          when /\A(\d+(?:\.\d+)?)(?:month|months)\z/
             # Same as systemd. See systemd.time(7)
             Float($1) * 60 * 60 * 24 * 30.44
-          when /\A(\d+(?:\.\d+))(?:y|year|years)\z/
+          when /\A(\d+(?:\.\d+)?)(?:y|year|years)\z/
             # Same as systemd. See systemd.time(7)
             Float($1) * 60 * 60 * 24 * 365.25
           else

data/lib/groonga-delta/import-status.rb

@@ -38,12 +38,16 @@ module GroongaDelta
         @status.update("mysql" => new_data)
       end
 
-      def file
-        self["file"]
+      def last_file
+        self["last_file"] || self["file"] # For backward compatibility
       end
 
-      def position
-        self["position"]
+      def last_position
+        self["last_position"] || self["position"] # For backward compatibility
+      end
+
+      def last_table_map_file
+        self["last_table_map_file"] || self["file"]
       end
 
       def last_table_map_position

data/lib/groonga-delta/mysql-source.rb

@@ -15,6 +15,7 @@
 
 require "arrow"
 require "mysql2"
+require "mysql2-replication"
 
 require_relative "error"
 require_relative "local-writer"
@@ -32,92 +33,7 @@ module GroongaDelta
     end
 
     def import
-      case ENV["GROONGA_DELTA_IMPORT_MYSQL_SOURCE_BACKEND"]
-      when "mysqlbinlog"
-        require "mysql_binlog"
-        import_mysqlbinlog
-      when "mysql2-replication"
-        require "mysql2-replication"
-        import_mysql2_replication
-      else
-        begin
-          require "mysql2-replication"
-        rescue LoadError
-          require "mysql_binlog"
-          import_mysqlbinlog
-        else
-          import_mysql2_replication
-        end
-      end
-    end
-
-    private
-    def import_mysqlbinlog
-      file, position, last_table_map_position = read_current_status
-      FileUtils.mkdir_p(@binlog_dir)
-      local_file = File.join(@binlog_dir, file)
-      unless File.exist?(local_file.succ)
-        command_line = [@config.mysqlbinlog].flatten
-        command_line << "--host=#{@config.host}" if @config.host
-        command_line << "--port=#{@config.port}" if @config.port
-        command_line << "--socket=#{@config.socket}" if @config.socket
-        if @config.replication_slave_user
-          command_line << "--user=#{@config.replication_slave_user}"
-        end
-        if @config.replication_slave_password
-          command_line << "--password=#{@config.replication_slave_password}"
-        end
-        command_line << "--read-from-remote-server"
-        command_line << "--raw"
-        command_line << "--result-file=#{@binlog_dir}/"
-        command_line << file
-        spawn_process(*command_line) do |pid, output_read, error_read|
-        end
-      end
-      reader = MysqlBinlog::BinlogFileReader.new(local_file)
-      binlog = MysqlBinlog::Binlog.new(reader)
-      binlog.checksum = @config.checksum
-      binlog.ignore_rotate = true
-      binlog.each_event do |event|
-        next if event[:position] < last_table_map_position
-        case event[:type]
-        when :rotate_event
-          file = event[:event][:name]
-        when :table_map_event
-          last_table_map_position = event[:position]
-        when :write_rows_event_v1,
-             :write_rows_event_v2,
-             :update_rows_event_v1,
-             :update_rows_event_v2,
-             :delete_rows_event_v1,
-             :delete_rows_event_v2
-          next if event[:position] < position
-          normalized_type = event[:type].to_s.gsub(/_v\d\z/, "").to_sym
-          import_rows_event(normalized_type,
-                            event[:event][:table][:db],
-                            event[:event][:table][:table],
-                            file,
-                            event[:header][:next_position],
-                            last_table_map_position) do
-            case normalized_type
-            when :write_rows_event,
-                 :update_rows_event
-              event[:event][:row_image].collect do |row_image|
-                build_row(row_image[:after][:image])
-              end
-            when :delete_rows_event
-              event[:event][:row_image].collect do |row_image|
-                build_row(row_image[:before][:image])
-              end
-            end
-          end
-          position = event[:header][:next_position]
-        end
-      end
-    end
-
-    def import_mysql2_replication
-      file, position, last_table_map_position = read_current_status
+      current_status = read_current_status
       is_mysql_56_or_later = mysql(@config.select_user,
                                    @config.select_password) do |select_client|
         mysql_version(select_client) >= Gem::Version.new("5.6")
@@ -130,83 +46,85 @@ module GroongaDelta
           replication_client = Mysql2Replication::Client.new(client,
                                                              checksum: "NONE")
         end
-        replication_client.file_name = file
-        current_event_position = last_table_map_position
-        replication_client.start_position = current_event_position
+        current_file = current_status.last_table_map_file
+        replication_client.file_name = current_file
+        current_event_position = current_status.last_table_map_position
+        if current_event_position
+          replication_client.start_position = current_event_position
+        end
         replication_client.open do
           replication_client.each do |event|
             begin
               @logger.debug do
                 event.inspect
               end
-              next if current_event_position < position
               case event
               when Mysql2Replication::RotateEvent
-                file = event.file_name
+                current_file = event.file_name
+                current_status.last_file = current_file
+                current_status.last_position = event.position
               when Mysql2Replication::TableMapEvent
-                last_table_map_event = current_event_position
+                current_status.last_table_map_file = current_file
+                current_status.last_table_map_position = current_event_position
               when Mysql2Replication::RowsEvent
-                event_name = event.class.name.split("::").last
-                normalized_type =
-                  event_name.scan(/[A-Z][a-z]+/).
-                    collect(&:downcase).
-                    join("_").
-                    to_sym
-                import_rows_event(normalized_type,
-                                  event.table_map.database,
-                                  event.table_map.table,
-                                  file,
-                                  event.next_position,
-                                  last_table_map_position) do
-                  case normalized_type
-                  when :update_rows_event
-                    event.updated_rows
-                  else
-                    event.rows
-                  end
-                end
+                next if current_file != current_status.last_file
+                next if current_event_position < current_status.last_position
+                import_rows_event(event, current_status)
               end
             ensure
-              current_event_position = event.next_position
+              # next_position may be 0. For example,
+              # Mysql2Replication::FormatDescriptionEvent#next_position
+              # returns 0.
+              next_position = event.next_position
+              current_event_position = next_position unless next_position.zero?
             end
           end
         end
       end
     end
 
-    def import_rows_event(type,
-                          database_name,
-                          table_name,
-                          file,
-                          next_position,
-                          last_table_map_position,
-                          &block)
+    private
+    def import_rows_event(event, current_status)
+      database_name = event.table_map.database
+      table_name = event.table_map.table
+
       source_table = @mapping[database_name, table_name]
       return if source_table.nil?
 
       table = find_table(database_name, table_name)
       groonga_table = source_table.groonga_table
-      target_rows = block.call
+      case event
+      when Mysql2Replication::UpdateRowsEvent
+        target_rows = event.updated_rows
+      else
+        target_rows = event.rows
+      end
       groonga_records = target_rows.collect do |row|
         record = build_record(table, row)
         groonga_table.generate_record(record)
       end
       return if groonga_records.empty?
 
-      case type
-      when :write_rows_event,
-           :update_rows_event
+      case event
+      when Mysql2Replication::WriteRowsEvent,
+           Mysql2Replication::UpdateRowsEvent
         @writer.write_upserts(groonga_table.name, groonga_records)
-      when :delete_rows_event
+      when Mysql2Replication::DeleteRowsEvent
         groonga_record_keys = groonga_records.collect do |record|
           record[:_key]
         end
         @writer.write_deletes(groonga_table.name,
                               groonga_record_keys)
       end
-      @status.update("file" => file,
-                     "position" => next_position,
-                     "last_table_map_position" => last_table_map_position)
+      current_status.last_position = event.next_position
+      new_status = {
+        "last_file" => current_status.last_file,
+        "last_position" => current_status.last_position,
+        "last_table_map_file" => current_status.last_table_map_file,
+        "last_table_map_position" => current_status.last_table_map_position,
+      }
+      @logger.info("Update status: #{new_status.inspect}")
+      @status.update(new_status)
     end
 
     def wait_process(command_line, pid, output_read, error_read)
@@ -280,17 +198,20 @@ module GroongaDelta
     end
 
     def read_current_status
-      if @status.file
-        [@status.file, @status.position, @status.last_table_map_position]
+      if @status.last_file
+        CurrentStatus.new(@status.last_file,
+                          @status.last_position,
+                          @status.last_table_map_file,
+                          @status.last_table_map_position)
       else
-        file = nil
-        position = 0
+        last_file = nil
+        last_position = 0
         mysql(@config.replication_client_user,
               @config.replication_client_password) do |replication_client|
           replication_client.query("FLUSH TABLES WITH READ LOCK")
           result = replication_client.query("SHOW MASTER STATUS").first
-          file = result["File"]
-          position = result["Position"]
+          last_file = result["File"]
+          last_position = result["Position"]
           mysql(@config.select_user,
                 @config.select_password) do |select_client|
             start_transaction = "START TRANSACTION " +
@@ -304,10 +225,19 @@ module GroongaDelta
             select_client.query("ROLLBACK")
           end
         end
-        @status.update("file" => file,
-                       "position" => position,
-                       "last_table_map_position" => position)
-        [file, position, position]
+        current_status = CurrentStatus.new(last_file,
+                                           last_position,
+                                           last_file,
+                                           last_position)
+        new_status = {
+          "last_file" => current_status.last_file,
+          "last_position" => current_status.last_position,
+          "last_table_map_file" => current_status.last_file,
+          "last_table_map_position" => current_status.last_position,
+        }
+        @logger.info("Update status: #{new_status.inspect}")
+        @status.update(new_status)
+        current_status
       end
     end
 
@@ -405,5 +335,10 @@ module GroongaDelta
       end
       record
     end
+
+    CurrentStatus = Struct.new(:last_file,
+                               :last_position,
+                               :last_table_map_file,
+                               :last_table_map_position)
   end
 end

data/lib/groonga-delta/status.rb

@@ -35,9 +35,11 @@ module GroongaDelta
     def update(data)
       @data.update(data)
       FileUtils.mkdir_p(@dir)
-      File.open(@path, "w") do |output|
+      path_writing = "#{@path}.writing"
+      File.open(path_writing, "w") do |output|
         output.puts(YAML.dump(@data))
       end
+      FileUtils.mv(path_writing, @path)
     end
   end
 end

data/lib/groonga-delta/version.rb

@@ -14,5 +14,5 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 module GroongaDelta
-  VERSION = "1.0.2"
+  VERSION = "1.0.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: groonga-delta
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.0.3
 platform: ruby
 authors:
 - Sutou Kouhei
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-06-21 00:00:00.000000000 Z
+date: 2022-07-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: groonga-client