groonga-delta 1.0.1 → 1.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f518dc76e259b9697b95be025f4e55b1e7fad3daea738a405b3d6a564c65ed76
-  data.tar.gz: ef1be1c1f200ddc94b9460c7216d9e7832993fdbe1ada926f31e87e01e5c5628
+  metadata.gz: ccfa632a6c046239e0163f15306f5479a64c220aa9e419bd415376bfeff5eae2
+  data.tar.gz: 3ab279d8f62f66bda8e83fc71c3cb27b5472339e5e373868ccbfde0376759c67
 SHA512:
-  metadata.gz: 54cb45f40a7d6a676abee8f41c9eb66dd6992f7f0b07b030e9222a75da0435b02ed61361b7bff051e67acfde7cd7d1fc41ad1186c51606051ea921c5cebb674e
-  data.tar.gz: b932e4c42a5f424fb2b6e4f423c9c248bc032acffa6884fa953aeee2f0863022125666d072b18bbe5bc01d4f4eae68fc7156cb8401d9280380e08b57a2e458e0
+  metadata.gz: eaa509b863868cd7fa1944d52603c312dc980707d975db9b52dd8b74131b2eac293cecc71dab48abb0895a22203dc2c59942750186e491209a3f395b481496b0
+  data.tar.gz: 377115552df709288fdff0f1eff36d76e1a99b14e83409f2ab29f5d263b15e94b756d387f357f50423f2015c67f66badb662100d8dac8fdacac67f53d45af0c4

data/doc/text/news.md

@@ -1,5 +1,27 @@
 # NEWS
 
+## 1.0.2 - 2022-06-21
+
+### Improvements
+
+  * `import`: Added support for logging all MySQL replication event
+    details by `debug` log level.
+
+  * `import`: Improved error handling on record generation.
+
+  * `import`: Added support for deleting a record by number/time key.
+
+  * `import`: Added support for vacuuming old delta files.
+
+### Fixes
+
+  * `import`: Fixed a bug that retrying from an error may cause "no
+    table map" error for row events. We need to retry from the last
+    table map event.
+
+  * `apply`: Fixed a bug that delta files not applied yet may not be
+    applied.
+
 ## 1.0.1 - 2022-06-09
 
 ### Improvements

data/lib/groonga-delta/error.rb

@@ -13,6 +13,8 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+require "pp"
+
 module GroongaDelta
   class Error < StandardError
   end
@@ -25,4 +27,22 @@ module GroongaDelta
 
   class ProcessError < Error
   end
+
+  class GenerationError < Error
+    attr_reader :source_record
+    attr_reader :groonga_column
+    attr_reader :detail
+    def initialize(source_record, groonga_column, detail)
+      @source_record = source_record
+      @groonga_column = groonga_column
+      @detail = detail
+      message =
+        "failed to generate a Groonga record:\n" +
+        "source record: #{PP.pp(source_record, '')}" +
+        "Groonga column: #{PP.pp(groonga_column, '')}" +
+        "detail: #{@detail.message}(#{@detail.class})\n" +
+        @detail.backtrace.join("\n")
+      super(message)
+    end
+  end
 end

data/lib/groonga-delta/import-command.rb

@@ -16,6 +16,8 @@
 require_relative "command"
 require_relative "import-config"
 require_relative "import-status"
+require_relative "local-writer"
+require_relative "local-vacuumer"
 
 module GroongaDelta
   class ImportCommand < Command
@@ -23,14 +25,16 @@ module GroongaDelta
     def prepare
       @config = ImportConfig.new(@dir)
       @status = ImportStatus.new(@dir)
+      @writer = LocalWriter.new(@config)
+      @vacuumer = LocalVacuumer.new(@config)
       @sources = []
       if @config.local
         require_relative "local-source"
-        @sources << LocalSource.new(@config, @status)
+        @sources << LocalSource.new(@config, @status, @writer)
       end
       if @config.mysql
         require_relative "mysql-source"
-        @sources << MySQLSource.new(@config, @status)
+        @sources << MySQLSource.new(@config, @status, @writer)
       end
     end
 
@@ -38,6 +42,7 @@ module GroongaDelta
       @sources.each do |source|
         source.import
       end
+      @vacuumer.vacuum
     end
   end
 end

data/lib/groonga-delta/import-config.rb

@@ -38,6 +38,10 @@ module GroongaDelta
       Local.new(@dir, @data["local"])
     end
 
+    def vacuum
+      Vacuum.new(@data["vacuum"] || {})
+    end
+
     def mapping
       Mapping.new(@data["mapping"] || {})
     end
@@ -164,5 +168,44 @@ module GroongaDelta
         @data["initial_max_number"] || Float::INFINITY
       end
     end
+
+    class Vacuum
+      def initialize(data)
+        @data = data
+      end
+
+      def keep_span
+        resolve_span(@data["keep_span"])
+      end
+
+      private
+      def resolve_span(value)
+        case value
+        when String
+          case value
+          when /\A(\d+(?:\.\d+))(?:s|sec|second|seconds)?\z/
+            Float($1)
+          when /\A(\d+(?:\.\d+))(?:m|min|minute|minutes)\z/
+            Float($1) * 60
+          when /\A(\d+(?:\.\d+))(?:h|hr|hour|hours)\z/
+            Float($1) * 60 * 60
+          when /\A(\d+(?:\.\d+))(?:d|day|days)\z/
+            Float($1) * 60 * 60 * 24
+          when /\A(\d+(?:\.\d+))(?:w|week|weeks)\z/
+            Float($1) * 60 * 60 * 24 * 7
+          when /\A(\d+(?:\.\d+))(?:month|months)\z/
+            # Same as systemd. See systemd.time(7)
+            Float($1) * 60 * 60 * 24 * 30.44
+          when /\A(\d+(?:\.\d+))(?:y|year|years)\z/
+            # Same as systemd. See systemd.time(7)
+            Float($1) * 60 * 60 * 24 * 365.25
+          else
+            raise ConfigError, "invalid span value: #{value.inspect}"
+          end
+        else
+          value
+        end
+      end
+    end
   end
 end

data/lib/groonga-delta/import-status.rb

@@ -45,6 +45,10 @@ module GroongaDelta
       def position
         self["position"]
       end
+
+      def last_table_map_position
+        self["last_table_map_position"]
+      end
     end
 
     class Local

data/lib/groonga-delta/local-delta.rb

@@ -13,12 +13,9 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-require "fileutils"
-
 require "groonga/client"
-require "parquet"
 
-require_relative "error"
+require_relative "local-reader"
 
 module GroongaDelta
   class LocalDelta
@@ -30,9 +27,9 @@ module GroongaDelta
     end
 
     def apply
-      start_time = read_current_status
+      reader = LocalReader.new(@logger, @delta_dir)
+      start_time = read_current_status(reader)
       current_time = Time.now.utc
-      targets = list_targets(@delta_dir, start_time, current_time)
       client_options = {
         url: @config.groonga.url,
         read_timeout: @config.groonga.read_timeout,
@@ -44,7 +41,7 @@ module GroongaDelta
                                          target_commands: [],
                                          target_tables: [],
                                          target_columns: [])
-        targets.sort_by(&:timestamp).each do |target|
+        reader.each(start_time, current_time) do |target|
           target.apply(@logger, client, processor)
           @status.update("start_time" => [
                            target.timestamp.to_i,
@@ -55,289 +52,22 @@ module GroongaDelta
     end
 
     private
-    def build_time(year, month, day, hour=0, minute=0, second=0, nanosecond=0)
-      Time.utc(year,
-               month,
-               day,
-               hour,
-               minute,
-               Rational(second * 1_000_000_000 + nanosecond,
-                        1_000_000_000))
-    end
-
-    def read_current_status
+    def read_current_status(reader)
       start_time_unix_time, start_time_nanosecond = @status.start_time
       if start_time_unix_time
         start_time = Time.at(start_time_unix_time).utc
-        build_time(start_time.year,
-                   start_time.month,
-                   start_time.day,
-                   start_time.hour,
-                   start_time.min,
-                   start_time.sec,
-                   start_time_nanosecond)
+        reader.build_time(start_time.year,
+                          start_time.month,
+                          start_time.day,
+                          start_time.hour,
+                          start_time.min,
+                          start_time.sec,
+                          start_time_nanosecond)
       else
         Time.at(0).utc
       end
     end
 
-    def each_target_path(dir,
-                         min_timestamp,
-                         max_timestamp,
-                         accept_directory: true,
-                         &block)
-      Dir.glob("#{dir}/*") do |path|
-        base_name = File.basename(path)
-        if accept_directory and File.directory?(path)
-          timestamp = parse_directory_timestamp(base_name)
-          next if timestamp.nil?
-          next if min_timestamp and timestamp <= min_timestamp
-          next if max_timestamp and timestamp > max_timestamp
-          each_target_path(path,
-                           min_timestamp,
-                           max_timestamp,
-                           accept_directory: false,
-                           &block)
-        elsif File.file?(path)
-          timestamp, action, post_match = parse_file_timestamp(base_name)
-          next if timestamp.nil?
-          next if min_timestamp and timestamp <= min_timestamp
-          next if max_timestamp and timestamp > max_timestamp
-          yield(path, timestamp, action, post_match)
-        end
-      end
-    end
-
-    def each_packed_target_path(dir, min_timestamp, max_timestamp)
-      return unless min_timestamp.to_i.zero?
-      Dir.glob("#{dir}/packed/*") do |path|
-        next unless File.directory?(path)
-        timestamp, action, post_match = parse_file_timestamp(File.basename(path))
-        next if action
-        next unless post_match.empty?
-        yield(path, timestamp)
-      end
-    end
-
-    def list_targets(dir, start_time, current_timestamp)
-      targets = []
-      list_schema_targets(dir, start_time, current_timestamp, targets)
-      Dir.glob("#{dir}/data/*") do |path|
-        next unless File.directory?(path)
-        name = File.basename(path)
-        list_table_targets(path, name, start_time, current_timestamp, targets)
-      end
-      targets
-    end
-
-    def each_schema_target(dir, min_timestamp, max_timestamp)
-      each_target_path(dir,
-                       min_timestamp,
-                       max_timestamp) do |path, timestamp, action, post_match|
-        next if action
-        next unless post_match == ".grn"
-        yield(SchemaTarget.new(path, timestamp))
-      end
-    end
-
-    def list_schema_targets(dir, start_time, current_timestamp, targets)
-      latest_packed_target = nil
-      each_packed_target_path("#{dir}/schema",
-                              start_time,
-                              current_timestamp) do |path, timestamp|
-        if latest_packed_target and latest_packed_target.timestamp > timestamp
-          next
-        end
-        latest_packed_target = PackedSchemaTarget.new(path, timestamp)
-      end
-      if latest_packed_target
-        targets << latest_packed_target
-        each_schema_target(latest_packed_target.path, nil, nil) do |target|
-          latest_packed_target.targets << target
-        end
-      end
-      each_schema_target("#{dir}/schema",
-                         latest_packed_target&.timestamp || start_time,
-                         current_timestamp) do |target|
-        targets << target
-      end
-    end
-
-    TABLE_TARGET_SUFFIXES = [".grn", ".parquet"]
-    def each_table_target(dir, name, min_timestamp, max_timestamp)
-      each_target_path(dir,
-                       min_timestamp,
-                       max_timestamp) do |path, timestamp, action, post_match|
-        next if action.nil?
-        next unless TABLE_TARGET_SUFFIXES.include?(post_match)
-        yield(TableTarget.new(path, timestamp, name, action))
-      end
-    end
-
-    def list_table_targets(dir, name, start_time, current_timestamp, targets)
-      latest_packed_target = nil
-      each_packed_target_path(dir,
-                              start_time,
-                              current_timestamp) do |path, timestamp|
-        if latest_packed_target and latest_packed_target.timestamp > timestamp
-          next
-        end
-        latest_packed_target = PackedTableTarget.new(path, timestamp, name)
-      end
-      if latest_packed_target
-        targets << latest_packed_target
-        each_table_target(latest_packed_target.path, name, nil, nil) do |target|
-          latest_packed_target.targets << target
-        end
-      end
-      each_table_target(dir,
-                        name,
-                        latest_packed_target&.timestamp || start_time,
-                        current_timestamp) do |target|
-        targets << target
-      end
-    end
-
-    def parse_directory_timestamp(base_name)
-      case base_name
-      when /\A(\d{4})-(\d{2})-(\d{2})\z/
-        match = Regexp.last_match
-        year = match[1].to_i
-        month = match[2].to_i
-        day = match[3].to_i
-        build_time(year, month, day)
-      else
-        nil
-      end
-    end
-
-    def parse_file_timestamp(base_name)
-      case base_name
-      when /\A(\d{4})-(\d{2})-(\d{2})-(\d{2})-(\d{2})-(\d{2})-(\d{9})(?:-(\w+))?/
-        match = Regexp.last_match
-        year = match[1].to_i
-        month = match[2].to_i
-        day = match[3].to_i
-        hour = match[4].to_i
-        minute = match[5].to_i
-        second = match[6].to_i
-        nanosecond = match[7].to_i
-        action = match[8]
-        timestamp = build_time(year,
-                               month,
-                               day,
-                               hour,
-                               minute,
-                               second,
-                               nanosecond)
-        [timestamp, action, match.post_match]
-      else
-        nil
-      end
-    end
-
-    module ApplyLoggable
-      private
-      def apply_log(logger, path)
-        logger.info("Start applying: #{path}")
-        result = yield
-        logger.info("Applied: #{path}")
-        result
-      end
-    end
-
-    class SchemaTarget
-      include ApplyLoggable
-
-      attr_reader :path
-      attr_reader :timestamp
-      def initialize(path, timestamp)
-        @path = path
-        @timestamp = timestamp
-      end
-
-      def apply(logger, client, processor)
-        apply_log(logger, @path) do
-          processor.load(@path)
-        end
-      end
-    end
-
-    class PackedSchemaTarget
-      include ApplyLoggable
-
-      attr_reader :path
-      attr_reader :timestamp
-      attr_reader :targets
-      def initialize(path, timestamp)
-        @path = path
-        @timestamp = timestamp
-        @targets = []
-      end
-
-      def apply(logger, client, processor)
-        apply_log(logger, @path) do
-          @targets.sort_by(&:timestamp).each do |target|
-            target.apply(logger, client, processor)
-          end
-        end
-      end
-    end
-
-    class TableTarget
-      include ApplyLoggable
-
-      attr_reader :path
-      attr_reader :timestamp
-      attr_reader :name
-      attr_reader :action
-      def initialize(path, timestamp, name, action)
-        @path = path
-        @timestamp = timestamp
-        @name = name
-        @action = action
-      end
-
-      def apply(logger, client, processor)
-        apply_log(logger, @path) do
-          if @path.end_with?(".grn")
-            processor.load(@path)
-          else
-            # TODO: Add support for @action == "delete"
-            table = Arrow::Table.load(@path)
-            command = Groonga::Command::Load.new(table: @name,
-                                                 values: table,
-                                                 command_version: "3")
-            response = client.load(command.arguments)
-            processor.process_response(response, command)
-          end
-        end
-      end
-    end
-
-    class PackedTableTarget
-      include ApplyLoggable
-
-      attr_reader :path
-      attr_reader :timestamp
-      attr_reader :name
-      attr_reader :targets
-      def initialize(path, timestamp, name)
-        @path = path
-        @timestamp = timestamp
-        @name = name
-        @targets = []
-      end
-
-      def apply(logger, client, processor)
-        apply_log(logger, @path) do
-          @targets.sort_by(&:timestamp).each do |target|
-            target.apply(logger, client, processor)
-          end
-        end
-      end
-    end
-
     class CommandProcessor < Groonga::Client::CommandProcessor
       def initialize(config, *args)
         @config = config

data/lib/groonga-delta/local-reader.rb

@@ -0,0 +1,353 @@
+# Copyright (C) 2021-2022  Sutou Kouhei <kou@clear-code.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require "fileutils"
+
+require "groonga/command"
+require "parquet"
+
+require_relative "error"
+
+module GroongaDelta
+  class LocalReader
+    def initialize(logger, dir)
+      @logger = logger
+      @dir = dir
+    end
+
+    def each(min_timestamp=nil, max_timestamp=nil, &block)
+      unless block_given?
+        return to_enum(__method__, min_timestamp, max_timestamp)
+      end
+
+      targets = list_targets(@dir, min_timestamp, max_timestamp)
+      targets.sort_by(&:timestamp).each(&block)
+    end
+
+    def build_time(year, month, day, hour=0, minute=0, second=0, nanosecond=0)
+      Time.utc(year,
+               month,
+               day,
+               hour,
+               minute,
+               Rational(second * 1_000_000_000 + nanosecond,
+                        1_000_000_000))
+    end
+
+    private
+    def each_target_path(dir,
+                         min_timestamp,
+                         max_timestamp,
+                         accept_directory: true,
+                         &block)
+      if min_timestamp
+        min_timestamp_day = Time.utc(min_timestamp.year,
+                                     min_timestamp.month,
+                                     min_timestamp.day)
+      end
+      if max_timestamp
+        max_timestamp_day = Time.utc(max_timestamp.year,
+                                     max_timestamp.month,
+                                     max_timestamp.day)
+      end
+      Dir.glob("#{dir}/*") do |path|
+        base_name = File.basename(path)
+        if accept_directory and File.directory?(path)
+          timestamp = parse_directory_timestamp(base_name)
+          next if timestamp.nil?
+          next if min_timestamp_day and timestamp < min_timestamp_day
+          next if max_timestamp_day and timestamp > max_timestamp_day
+          each_target_path(path,
+                           min_timestamp,
+                           max_timestamp,
+                           accept_directory: false,
+                           &block)
+        elsif File.file?(path)
+          timestamp, action, post_match = parse_file_timestamp(base_name)
+          next if timestamp.nil?
+          next if min_timestamp and timestamp <= min_timestamp
+          next if max_timestamp and timestamp > max_timestamp
+          yield(path, timestamp, action, post_match)
+        end
+      end
+    end
+
+    def each_packed_target_path(dir, min_timestamp, max_timestamp)
+      return unless min_timestamp.to_i.zero?
+      Dir.glob("#{dir}/packed/*") do |path|
+        next unless File.directory?(path)
+        timestamp, action, post_match = parse_file_timestamp(File.basename(path))
+        next if action
+        next unless post_match.empty?
+        yield(path, timestamp)
+      end
+    end
+
+    def list_targets(dir, min_timestamp, max_timestamp)
+      targets = []
+      list_schema_targets(dir, min_timestamp, max_timestamp, targets)
+      Dir.glob("#{dir}/data/*") do |path|
+        next unless File.directory?(path)
+        name = File.basename(path)
+        list_table_targets(path, name, min_timestamp, max_timestamp, targets)
+      end
+      targets
+    end
+
+    def each_schema_target(dir, min_timestamp, max_timestamp)
+      each_target_path(dir,
+                       min_timestamp,
+                       max_timestamp) do |path, timestamp, action, post_match|
+        next if action
+        next unless post_match == ".grn"
+        yield(SchemaTarget.new(path, timestamp))
+      end
+    end
+
+    def list_schema_targets(dir, min_timestamp, max_timestamp, targets)
+      latest_packed_target = nil
+      each_packed_target_path("#{dir}/schema",
+                              min_timestamp,
+                              max_timestamp) do |path, timestamp|
+        if latest_packed_target and latest_packed_target.timestamp > timestamp
+          next
+        end
+        latest_packed_target = PackedSchemaTarget.new(path, timestamp)
+      end
+      if latest_packed_target
+        targets << latest_packed_target
+        each_schema_target(latest_packed_target.path, nil, nil) do |target|
+          latest_packed_target.targets << target
+        end
+      end
+      each_schema_target("#{dir}/schema",
+                         latest_packed_target&.timestamp || min_timestamp,
+                         max_timestamp) do |target|
+        targets << target
+      end
+    end
+
+    TABLE_TARGET_SUFFIXES = [".grn", ".parquet"]
+    def each_table_target(dir, name, min_timestamp, max_timestamp)
+      each_target_path(dir,
+                       min_timestamp,
+                       max_timestamp) do |path, timestamp, action, post_match|
+        next if action.nil?
+        next unless TABLE_TARGET_SUFFIXES.include?(post_match)
+        yield(TableTarget.new(path, timestamp, name, action))
+      end
+    end
+
+    def list_table_targets(dir, name, min_timestamp, max_timestamp, targets)
+      latest_packed_target = nil
+      each_packed_target_path(dir,
+                              min_timestamp,
+                              max_timestamp) do |path, timestamp|
+        if latest_packed_target and latest_packed_target.timestamp > timestamp
+          next
+        end
+        latest_packed_target = PackedTableTarget.new(path, timestamp, name)
+      end
+      if latest_packed_target
+        targets << latest_packed_target
+        each_table_target(latest_packed_target.path, name, nil, nil) do |target|
+          latest_packed_target.targets << target
+        end
+      end
+      each_table_target(dir,
+                        name,
+                        latest_packed_target&.timestamp || min_timestamp,
+                        max_timestamp) do |target|
+        targets << target
+      end
+    end
+
+    def parse_directory_timestamp(base_name)
+      case base_name
+      when /\A(\d{4})-(\d{2})-(\d{2})\z/
+        match = Regexp.last_match
+        year = match[1].to_i
+        month = match[2].to_i
+        day = match[3].to_i
+        build_time(year, month, day)
+      else
+        nil
+      end
+    end
+
+    def parse_file_timestamp(base_name)
+      case base_name
+      when /\A(\d{4})-(\d{2})-(\d{2})-(\d{2})-(\d{2})-(\d{2})-(\d{9})(?:-(\w+))?/
+        match = Regexp.last_match
+        year = match[1].to_i
+        month = match[2].to_i
+        day = match[3].to_i
+        hour = match[4].to_i
+        minute = match[5].to_i
+        second = match[6].to_i
+        nanosecond = match[7].to_i
+        action = match[8]
+        timestamp = build_time(year,
+                               month,
+                               day,
+                               hour,
+                               minute,
+                               second,
+                               nanosecond)
+        [timestamp, action, match.post_match]
+      else
+        nil
+      end
+    end
+
+    module Loggable
+      private
+      def log(logger, path, before_message, after_message)
+        logger.info("#{before_message}: #{path}")
+        result = yield
+        logger.info("#{after_message}: #{path}")
+        result
+      end
+
+      def apply_log(logger, path, &block)
+        log(logger, path, "Start applying", "Applied", &block)
+      end
+
+      def vacuum_log(logger, path, &block)
+        log(logger, path, "Start vacuuming", "Vacuumed", &block)
+      end
+    end
+
+    class SchemaTarget
+      include Loggable
+
+      attr_reader :path
+      attr_reader :timestamp
+      def initialize(path, timestamp)
+        @path = path
+        @timestamp = timestamp
+      end
+
+      def apply(logger, client, processor)
+        apply_log(logger, @path) do
+          processor.load(@path)
+        end
+      end
+
+      def vacuum(logger)
+        vacuum_log(logger, @path) do
+          FileUtils.rm(@path)
+        end
+      end
+    end
+
+    class PackedSchemaTarget
+      include Loggable
+
+      attr_reader :path
+      attr_reader :timestamp
+      attr_reader :targets
+      def initialize(path, timestamp)
+        @path = path
+        @timestamp = timestamp
+        @targets = []
+      end
+
+      def apply(logger, client, processor)
+        apply_log(logger, @path) do
+          @targets.sort_by(&:timestamp).each do |target|
+            target.apply(logger, client, processor)
+          end
+        end
+      end
+
+      def vacuum(logger)
+        vacuum_log(logger, @path) do
+          @targets.sort_by(&:timestamp).each do |target|
+            target.vacuum(logger)
+          end
+        end
+      end
+    end
+
+    class TableTarget
+      include Loggable
+
+      attr_reader :path
+      attr_reader :timestamp
+      attr_reader :name
+      attr_reader :action
+      def initialize(path, timestamp, name, action)
+        @path = path
+        @timestamp = timestamp
+        @name = name
+        @action = action
+      end
+
+      def apply(logger, client, processor)
+        apply_log(logger, @path) do
+          if @path.end_with?(".grn")
+            processor.load(@path)
+          else
+            # TODO: Add support for @action == "delete"
+            table = Arrow::Table.load(@path)
+            command = Groonga::Command::Load.new(table: @name,
+                                                 values: table,
+                                                 command_version: "3")
+            response = client.load(command.arguments)
+            processor.process_response(response, command)
+          end
+        end
+      end
+
+      def vacuum(logger)
+        vacuum_log(logger, @path) do
+          FileUtils.rm(@path)
+        end
+      end
+    end
+
+    class PackedTableTarget
+      include Loggable
+
+      attr_reader :path
+      attr_reader :timestamp
+      attr_reader :name
+      attr_reader :targets
+      def initialize(path, timestamp, name)
+        @path = path
+        @timestamp = timestamp
+        @name = name
+        @targets = []
+      end
+
+      def apply(logger, client, processor)
+        apply_log(logger, @path) do
+          @targets.sort_by(&:timestamp).each do |target|
+            target.apply(logger, client, processor)
+          end
+        end
+      end
+
+      def vacuum(logger)
+        vacuum_log(logger, @path) do
+          @targets.sort_by(&:timestamp).each do |target|
+            target.vacuum(logger)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/groonga-delta/local-source.rb

@@ -15,15 +15,15 @@
 
 require "groonga/command/parser"
 
-require_relative "writer"
+require_relative "local-writer"
 
 module GroongaDelta
   class LocalSource
-    def initialize(config, status)
+    def initialize(config, status, writer)
       @logger = config.logger
-      @writer = Writer.new(@logger, config.delta_dir)
       @config = config.local
       @status = status.local
+      @writer = writer
     end
 
     def import

data/lib/groonga-delta/local-vacuumer.rb

@@ -0,0 +1,39 @@
+# Copyright (C) 2022  Sutou Kouhei <kou@clear-code.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require "fileutils"
+
+require_relative "local-reader"
+
+module GroongaDelta
+  class LocalVacuumer
+    def initialize(config)
+      @logger = config.logger
+      @delta_dir = config.delta_dir
+      @config = config.vacuum
+    end
+
+    def vacuum
+      keep_span = @config.keep_span
+      return if keep_span.nil?
+      return if keep_span < 0
+      reader = LocalReader.new(@logger, @delta_dir)
+      max_timestamp = Time.now.utc - keep_span
+      reader.each(nil, max_timestamp) do |target|
+        target.vacuum(@logger)
+      end
+    end
+  end
+end

data/lib/groonga-delta/{writer.rb → local-writer.rb}

@@ -20,10 +20,11 @@ require "groonga/command"
 require "parquet"
 
 module GroongaDelta
-  class Writer
-    def initialize(logger, dir)
-      @logger = logger
-      @dir = dir
+  class LocalWriter
+    def initialize(config)
+      @config = config
+      @logger = @config.logger
+      @dir = @config.delta_dir
     end
 
     def write_upserts(table, records, packed: false)
@@ -74,7 +75,7 @@ module GroongaDelta
         delete = Groonga::Command::Delete.new
         delete[:table] = table
         keys.each do |key|
-          delete[:key] = key
+          delete[:key] = format_key(key)
           output.puts(delete.to_command_format)
         end
       end
@@ -131,5 +132,16 @@ module GroongaDelta
                   open_output: open_output,
                   &block)
     end
+
+    def format_key(key)
+      case key
+      when Integer, Float
+        key.to_s
+      when Time
+        key.strftime("%Y-%m-%d %H:%M:%S.%6N")
+      else
+        key
+      end
+    end
   end
 end

data/lib/groonga-delta/mapping.rb

@@ -142,8 +142,12 @@ module GroongaDelta
       def generate_record(source_record)
         record = {}
         @groonga_columns.each do |groonga_column|
-          value = groonga_column.generate_value(source_record)
-          record[groonga_column.name.to_sym] = value
+          begin
+            value = groonga_column.generate_value(source_record)
+            record[groonga_column.name.to_sym] = value
+          rescue => error
+            raise GenerationError.new(source_record, groonga_column, error)
+          end
         end
         record
       end
@@ -255,7 +259,16 @@ module GroongaDelta
       end
 
       def normalize_value(value)
-        case type
+        case @type
+        when nil, "ShortText", "Text", "LongText"
+          encoding = value.encoding
+          if encoding == Encoding::ASCII_8BIT
+            value.force_encoding(Encoding::UTF_8)
+            return value if value.valid_encoding?
+            value.encode(Encoding::UTF_8, encoding)
+          else
+            value.encode(Encoding::UTF_8)
+          end
         when "Time"
           time_max = @restriction.time_max
           time_min = @restriction.time_min

data/lib/groonga-delta/mysql-source.rb

@@ -17,17 +17,17 @@ require "arrow"
 require "mysql2"
 
 require_relative "error"
-require_relative "writer"
+require_relative "local-writer"
 
 module GroongaDelta
   class MySQLSource
-    def initialize(config, status)
+    def initialize(config, status, writer)
       @logger = config.logger
-      @writer = Writer.new(@logger, config.delta_dir)
       @config = config.mysql
       @binlog_dir = @config.binlog_dir
       @mapping = config.mapping
       @status = status.mysql
+      @writer = writer
       @tables = {}
     end
 
@@ -53,7 +53,7 @@ module GroongaDelta
 
     private
     def import_mysqlbinlog
-      file, position = read_current_status
+      file, position, last_table_map_position = read_current_status
       FileUtils.mkdir_p(@binlog_dir)
       local_file = File.join(@binlog_dir, file)
       unless File.exist?(local_file.succ)
@@ -79,23 +79,26 @@ module GroongaDelta
       binlog.checksum = @config.checksum
       binlog.ignore_rotate = true
       binlog.each_event do |event|
-        next if event[:position] < position
+        next if event[:position] < last_table_map_position
         case event[:type]
         when :rotate_event
-          @status.update("file" => event[:event][:name],
-                         "position" => event[:event][:pos])
+          file = event[:event][:name]
+        when :table_map_event
+          last_table_map_position = event[:position]
         when :write_rows_event_v1,
              :write_rows_event_v2,
              :update_rows_event_v1,
              :update_rows_event_v2,
              :delete_rows_event_v1,
              :delete_rows_event_v2
+          next if event[:position] < position
           normalized_type = event[:type].to_s.gsub(/_v\d\z/, "").to_sym
           import_rows_event(normalized_type,
                             event[:event][:table][:db],
                             event[:event][:table][:table],
                             file,
-                            event[:header][:next_position]) do
+                            event[:header][:next_position],
+                            last_table_map_position) do
             case normalized_type
             when :write_rows_event,
                  :update_rows_event
@@ -114,7 +117,7 @@ module GroongaDelta
     end
 
     def import_mysql2_replication
-      file, position = read_current_status
+      file, position, last_table_map_position = read_current_status
       is_mysql_56_or_later = mysql(@config.select_user,
                                    @config.select_password) do |select_client|
         mysql_version(select_client) >= Gem::Version.new("5.6")
@@ -128,31 +131,43 @@ module GroongaDelta
                                                              checksum: "NONE")
         end
         replication_client.file_name = file
-        replication_client.start_position = position
+        current_event_position = last_table_map_position
+        replication_client.start_position = current_event_position
         replication_client.open do
           replication_client.each do |event|
-            case event
-            when Mysql2Replication::RotateEvent
-              file = event.file_name
-            when Mysql2Replication::RowsEvent
-              event_name = event.class.name.split("::").last
-              normalized_type =
-              event_name.scan(/[A-Z][a-z]+/).
-                collect(&:downcase).
-                join("_").
-                to_sym
-              import_rows_event(normalized_type,
-                                event.table_map.database,
-                                event.table_map.table,
-                                file,
-                                event.next_position) do
-                case normalized_type
-                when :update_rows_event
-                  event.updated_rows
-                else
-                  event.rows
+            begin
+              @logger.debug do
+                event.inspect
+              end
+              next if current_event_position < position
+              case event
+              when Mysql2Replication::RotateEvent
+                file = event.file_name
+              when Mysql2Replication::TableMapEvent
+                last_table_map_event = current_event_position
+              when Mysql2Replication::RowsEvent
+                event_name = event.class.name.split("::").last
+                normalized_type =
+                  event_name.scan(/[A-Z][a-z]+/).
+                    collect(&:downcase).
+                    join("_").
+                    to_sym
+                import_rows_event(normalized_type,
+                                  event.table_map.database,
+                                  event.table_map.table,
+                                  file,
+                                  event.next_position,
+                                  last_table_map_position) do
+                  case normalized_type
+                  when :update_rows_event
+                    event.updated_rows
+                  else
+                    event.rows
+                  end
                 end
               end
+            ensure
+              current_event_position = event.next_position
             end
           end
         end
@@ -164,6 +179,7 @@ module GroongaDelta
                           table_name,
                           file,
                           next_position,
+                          last_table_map_position,
                           &block)
       source_table = @mapping[database_name, table_name]
       return if source_table.nil?
@@ -189,7 +205,8 @@ module GroongaDelta
                               groonga_record_keys)
       end
       @status.update("file" => file,
-                     "position" => next_position)
+                     "position" => next_position,
+                     "last_table_map_position" => last_table_map_position)
     end
 
     def wait_process(command_line, pid, output_read, error_read)
@@ -264,7 +281,7 @@ module GroongaDelta
 
     def read_current_status
       if @status.file
-        [@status.file, @status.position]
+        [@status.file, @status.position, @status.last_table_map_position]
       else
         file = nil
         position = 0
@@ -288,8 +305,9 @@ module GroongaDelta
           end
         end
         @status.update("file" => file,
-                       "position" => position)
-        [file, position]
+                       "position" => position,
+                       "last_table_map_position" => position)
+        [file, position, position]
       end
     end
 

data/lib/groonga-delta/version.rb

@@ -14,5 +14,5 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 module GroongaDelta
-  VERSION = "1.0.1"
+  VERSION = "1.0.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: groonga-delta
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.0.2
 platform: ruby
 authors:
 - Sutou Kouhei
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-06-09 00:00:00.000000000 Z
+date: 2022-06-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: groonga-client
@@ -80,13 +80,15 @@ files:
 - lib/groonga-delta/import-config.rb
 - lib/groonga-delta/import-status.rb
 - lib/groonga-delta/local-delta.rb
+- lib/groonga-delta/local-reader.rb
 - lib/groonga-delta/local-source.rb
+- lib/groonga-delta/local-vacuumer.rb
+- lib/groonga-delta/local-writer.rb
 - lib/groonga-delta/ltsv-log-formatter.rb
 - lib/groonga-delta/mapping.rb
 - lib/groonga-delta/mysql-source.rb
 - lib/groonga-delta/status.rb
 - lib/groonga-delta/version.rb
-- lib/groonga-delta/writer.rb
 homepage: https://github.com/groonga/groonga-delta
 licenses:
 - GPL-3.0+