groonga-delta 1.0.0 → 1.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ddcaa199b89bab185924a694bf7e64e1b6a8c2cd63d1ea855f59a5ac456c317e
-  data.tar.gz: 06a1a8e01e343b4fca3faf2cc8139ed8adaadc6aeb55c7c6315e1c1bc7f40962
+  metadata.gz: 5837cbbd7824ebf557d24be10007166c476dadf2157a96071522307d0ffbd0d7
+  data.tar.gz: '086fa03eb3a6c9d1a055ec8ffecf0a8b7e9102dcee0fd1d3326ece684f4e5560'
 SHA512:
-  metadata.gz: 39252c1602f63ff3eeddafdb676bdeef1334bed4df5a0e29fb8d14f0ae1467b98b925829333be865badf7bc3ec9fae332477e4dd6dc78c81cfddfa12e4bd8734
-  data.tar.gz: cc50bce6f1da66d697bf0ce97a191e1d67fa474af9a795c0f709e15d0fbf60f94a551af902b58a2f6eac6fbf2c637524335728269e1e211f3c1f35e84eb232d5
+  metadata.gz: a3db1aedcdbf3bb39312bfb091a778091111eec880488917a4d21598834e08a703181f383f47d06e2646e873d1ea1ac5d8e42ed46d71328ee8ecb8c837aa080d
+  data.tar.gz: 584ecff5ca3bf44fc9f4baea4ea8bd5f6c60227ac0f05367dd0860c74a563b26f4bcac747c74a7ce1c98a77f77100c2c736817ca6daa4af8138aec1372996f1c

data/doc/text/news.md

@@ -1,5 +1,47 @@
 # NEWS
 
+## 1.0.3 - 2022-07-21
+
+### Improvements
+
+  * `import`: Dropped support for `mysqlbinlog`.
+
+  * Changed to use atomic write for `status.yaml`.
+
+  * Changed to keep 100 logs from 7 logs by default.
+
+### Fixes
+
+  * `import`: Fixed a bug that events to be processed can't be detected.
+
+## 1.0.2 - 2022-06-21
+
+### Improvements
+
+  * `import`: Added support for logging all MySQL replication event
+    details by `debug` log level.
+
+  * `import`: Improved error handling on record generation.
+
+  * `import`: Added support for deleting a record by number/time key.
+
+  * `import`: Added support for vacuuming old delta files.
+
+### Fixes
+
+  * `import`: Fixed a bug that retrying from an error may cause "no
+    table map" error for row events. We need to retry from the last
+    table map event.
+
+  * `apply`: Fixed a bug that delta files not applied yet may not be
+    applied.
+
+## 1.0.1 - 2022-06-09
+
+### Improvements
+
+  * Added support for `SIGINT` by `SIGTERM`.
+
 ## 1.0.0 - 2022-03-07
 
 Initial release.

data/lib/groonga-delta/command.rb

@@ -34,6 +34,16 @@ module GroongaDelta
             sleep(@config.polling_interval)
           end
           true
+        rescue Interrupt
+          true
+        rescue SignalException => error
+          case error.signm
+          when "SIGTERM"
+            true
+          else
+            @config.logger.error(error) if @config
+            raise
+          end
         rescue => error
           @config.logger.error(error) if @config
           raise

data/lib/groonga-delta/config.rb

@@ -64,7 +64,7 @@ module GroongaDelta
     end
 
     def log_age
-      @data["log_age"] || 7
+      @data["log_age"] || 100
     end
 
     def log_max_size

data/lib/groonga-delta/error.rb

@@ -13,6 +13,8 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+require "pp"
+
 module GroongaDelta
   class Error < StandardError
   end
@@ -25,4 +27,22 @@ module GroongaDelta
 
   class ProcessError < Error
   end
+
+  class GenerationError < Error
+    attr_reader :source_record
+    attr_reader :groonga_column
+    attr_reader :detail
+    def initialize(source_record, groonga_column, detail)
+      @source_record = source_record
+      @groonga_column = groonga_column
+      @detail = detail
+      message =
+        "failed to generate a Groonga record:\n" +
+        "source record: #{PP.pp(source_record, '')}" +
+        "Groonga column: #{PP.pp(groonga_column, '')}" +
+        "detail: #{@detail.message}(#{@detail.class})\n" +
+        @detail.backtrace.join("\n")
+      super(message)
+    end
+  end
 end

data/lib/groonga-delta/import-command.rb

@@ -16,6 +16,8 @@
 require_relative "command"
 require_relative "import-config"
 require_relative "import-status"
+require_relative "local-writer"
+require_relative "local-vacuumer"
 
 module GroongaDelta
   class ImportCommand < Command
@@ -23,14 +25,16 @@ module GroongaDelta
     def prepare
       @config = ImportConfig.new(@dir)
       @status = ImportStatus.new(@dir)
+      @writer = LocalWriter.new(@config)
+      @vacuumer = LocalVacuumer.new(@config)
       @sources = []
       if @config.local
         require_relative "local-source"
-        @sources << LocalSource.new(@config, @status)
+        @sources << LocalSource.new(@config, @status, @writer)
       end
       if @config.mysql
         require_relative "mysql-source"
-        @sources << MySQLSource.new(@config, @status)
+        @sources << MySQLSource.new(@config, @status, @writer)
       end
     end
 
@@ -38,6 +42,7 @@ module GroongaDelta
       @sources.each do |source|
         source.import
       end
+      @vacuumer.vacuum
     end
   end
 end

data/lib/groonga-delta/import-config.rb

@@ -38,6 +38,10 @@ module GroongaDelta
       Local.new(@dir, @data["local"])
     end
 
+    def vacuum
+      Vacuum.new(@data["vacuum"] || {})
+    end
+
     def mapping
       Mapping.new(@data["mapping"] || {})
     end
@@ -55,10 +59,6 @@ module GroongaDelta
         resolve_path(@data["binlog_dir"] || "binlog")
       end
 
-      def mysqlbinlog
-        @data["mysqlbinlog"] || "mysqlbinlog"
-      end
-
       def host
         @data["host"] || "localhost"
       end
@@ -164,5 +164,44 @@ module GroongaDelta
         @data["initial_max_number"] || Float::INFINITY
       end
     end
+
+    class Vacuum
+      def initialize(data)
+        @data = data
+      end
+
+      def keep_span
+        resolve_span(@data["keep_span"])
+      end
+
+      private
+      def resolve_span(value)
+        case value
+        when String
+          case value
+          when /\A(\d+(?:\.\d+)?)(?:s|sec|second|seconds)?\z/
+            Float($1)
+          when /\A(\d+(?:\.\d+)?)(?:m|min|minute|minutes)\z/
+            Float($1) * 60
+          when /\A(\d+(?:\.\d+)?)(?:h|hr|hour|hours)\z/
+            Float($1) * 60 * 60
+          when /\A(\d+(?:\.\d+)?)(?:d|day|days)\z/
+            Float($1) * 60 * 60 * 24
+          when /\A(\d+(?:\.\d+)?)(?:w|week|weeks)\z/
+            Float($1) * 60 * 60 * 24 * 7
+          when /\A(\d+(?:\.\d+)?)(?:month|months)\z/
+            # Same as systemd. See systemd.time(7)
+            Float($1) * 60 * 60 * 24 * 30.44
+          when /\A(\d+(?:\.\d+)?)(?:y|year|years)\z/
+            # Same as systemd. See systemd.time(7)
+            Float($1) * 60 * 60 * 24 * 365.25
+          else
+            raise ConfigError, "invalid span value: #{value.inspect}"
+          end
+        else
+          value
+        end
+      end
+    end
   end
 end

data/lib/groonga-delta/import-status.rb

@@ -38,12 +38,20 @@ module GroongaDelta
         @status.update("mysql" => new_data)
       end
 
-      def file
-        self["file"]
+      def last_file
+        self["last_file"] || self["file"] # For backward compatibility
       end
 
-      def position
-        self["position"]
+      def last_position
+        self["last_position"] || self["position"] # For backward compatibility
+      end
+
+      def last_table_map_file
+        self["last_table_map_file"] || self["file"]
+      end
+
+      def last_table_map_position
+        self["last_table_map_position"]
       end
     end
 

data/lib/groonga-delta/local-delta.rb

@@ -13,12 +13,9 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-require "fileutils"
-
 require "groonga/client"
-require "parquet"
 
-require_relative "error"
+require_relative "local-reader"
 
 module GroongaDelta
   class LocalDelta
@@ -30,9 +27,9 @@ module GroongaDelta
     end
 
     def apply
-      start_time = read_current_status
+      reader = LocalReader.new(@logger, @delta_dir)
+      start_time = read_current_status(reader)
       current_time = Time.now.utc
-      targets = list_targets(@delta_dir, start_time, current_time)
       client_options = {
         url: @config.groonga.url,
         read_timeout: @config.groonga.read_timeout,
@@ -44,7 +41,7 @@ module GroongaDelta
                                          target_commands: [],
                                          target_tables: [],
                                          target_columns: [])
-        targets.sort_by(&:timestamp).each do |target|
+        reader.each(start_time, current_time) do |target|
           target.apply(@logger, client, processor)
           @status.update("start_time" => [
                            target.timestamp.to_i,
@@ -55,289 +52,22 @@ module GroongaDelta
     end
 
     private
-    def build_time(year, month, day, hour=0, minute=0, second=0, nanosecond=0)
-      Time.utc(year,
-               month,
-               day,
-               hour,
-               minute,
-               Rational(second * 1_000_000_000 + nanosecond,
-                        1_000_000_000))
-    end
-
-    def read_current_status
+    def read_current_status(reader)
       start_time_unix_time, start_time_nanosecond = @status.start_time
       if start_time_unix_time
         start_time = Time.at(start_time_unix_time).utc
-        build_time(start_time.year,
-                   start_time.month,
-                   start_time.day,
-                   start_time.hour,
-                   start_time.min,
-                   start_time.sec,
-                   start_time_nanosecond)
+        reader.build_time(start_time.year,
+                          start_time.month,
+                          start_time.day,
+                          start_time.hour,
+                          start_time.min,
+                          start_time.sec,
+                          start_time_nanosecond)
       else
         Time.at(0).utc
       end
     end
 
-    def each_target_path(dir,
-                         min_timestamp,
-                         max_timestamp,
-                         accept_directory: true,
-                         &block)
-      Dir.glob("#{dir}/*") do |path|
-        base_name = File.basename(path)
-        if accept_directory and File.directory?(path)
-          timestamp = parse_directory_timestamp(base_name)
-          next if timestamp.nil?
-          next if min_timestamp and timestamp <= min_timestamp
-          next if max_timestamp and timestamp > max_timestamp
-          each_target_path(path,
-                           min_timestamp,
-                           max_timestamp,
-                           accept_directory: false,
-                           &block)
-        elsif File.file?(path)
-          timestamp, action, post_match = parse_file_timestamp(base_name)
-          next if timestamp.nil?
-          next if min_timestamp and timestamp <= min_timestamp
-          next if max_timestamp and timestamp > max_timestamp
-          yield(path, timestamp, action, post_match)
-        end
-      end
-    end
-
-    def each_packed_target_path(dir, min_timestamp, max_timestamp)
-      return unless min_timestamp.to_i.zero?
-      Dir.glob("#{dir}/packed/*") do |path|
-        next unless File.directory?(path)
-        timestamp, action, post_match = parse_file_timestamp(File.basename(path))
-        next if action
-        next unless post_match.empty?
-        yield(path, timestamp)
-      end
-    end
-
-    def list_targets(dir, start_time, current_timestamp)
-      targets = []
-      list_schema_targets(dir, start_time, current_timestamp, targets)
-      Dir.glob("#{dir}/data/*") do |path|
-        next unless File.directory?(path)
-        name = File.basename(path)
-        list_table_targets(path, name, start_time, current_timestamp, targets)
-      end
-      targets
-    end
-
-    def each_schema_target(dir, min_timestamp, max_timestamp)
-      each_target_path(dir,
-                       min_timestamp,
-                       max_timestamp) do |path, timestamp, action, post_match|
-        next if action
-        next unless post_match == ".grn"
-        yield(SchemaTarget.new(path, timestamp))
-      end
-    end
-
-    def list_schema_targets(dir, start_time, current_timestamp, targets)
-      latest_packed_target = nil
-      each_packed_target_path("#{dir}/schema",
-                              start_time,
-                              current_timestamp) do |path, timestamp|
-        if latest_packed_target and latest_packed_target.timestamp > timestamp
-          next
-        end
-        latest_packed_target = PackedSchemaTarget.new(path, timestamp)
-      end
-      if latest_packed_target
-        targets << latest_packed_target
-        each_schema_target(latest_packed_target.path, nil, nil) do |target|
-          latest_packed_target.targets << target
-        end
-      end
-      each_schema_target("#{dir}/schema",
-                         latest_packed_target&.timestamp || start_time,
-                         current_timestamp) do |target|
-        targets << target
-      end
-    end
-
-    TABLE_TARGET_SUFFIXES = [".grn", ".parquet"]
-    def each_table_target(dir, name, min_timestamp, max_timestamp)
-      each_target_path(dir,
-                       min_timestamp,
-                       max_timestamp) do |path, timestamp, action, post_match|
-        next if action.nil?
-        next unless TABLE_TARGET_SUFFIXES.include?(post_match)
-        yield(TableTarget.new(path, timestamp, name, action))
-      end
-    end
-
-    def list_table_targets(dir, name, start_time, current_timestamp, targets)
-      latest_packed_target = nil
-      each_packed_target_path(dir,
-                              start_time,
-                              current_timestamp) do |path, timestamp|
-        if latest_packed_target and latest_packed_target.timestamp > timestamp
-          next
-        end
-        latest_packed_target = PackedTableTarget.new(path, timestamp, name)
-      end
-      if latest_packed_target
-        targets << latest_packed_target
-        each_table_target(latest_packed_target.path, name, nil, nil) do |target|
-          latest_packed_target.targets << target
-        end
-      end
-      each_table_target(dir,
-                        name,
-                        latest_packed_target&.timestamp || start_time,
-                        current_timestamp) do |target|
-        targets << target
-      end
-    end
-
-    def parse_directory_timestamp(base_name)
-      case base_name
-      when /\A(\d{4})-(\d{2})-(\d{2})\z/
-        match = Regexp.last_match
-        year = match[1].to_i
-        month = match[2].to_i
-        day = match[3].to_i
-        build_time(year, month, day)
-      else
-        nil
-      end
-    end
-
-    def parse_file_timestamp(base_name)
-      case base_name
-      when /\A(\d{4})-(\d{2})-(\d{2})-(\d{2})-(\d{2})-(\d{2})-(\d{9})(?:-(\w+))?/
-        match = Regexp.last_match
-        year = match[1].to_i
-        month = match[2].to_i
-        day = match[3].to_i
-        hour = match[4].to_i
-        minute = match[5].to_i
-        second = match[6].to_i
-        nanosecond = match[7].to_i
-        action = match[8]
-        timestamp = build_time(year,
-                               month,
-                               day,
-                               hour,
-                               minute,
-                               second,
-                               nanosecond)
-        [timestamp, action, match.post_match]
-      else
-        nil
-      end
-    end
-
-    module ApplyLoggable
-      private
-      def apply_log(logger, path)
-        logger.info("Start applying: #{path}")
-        result = yield
-        logger.info("Applied: #{path}")
-        result
-      end
-    end
-
-    class SchemaTarget
-      include ApplyLoggable
-
-      attr_reader :path
-      attr_reader :timestamp
-      def initialize(path, timestamp)
-        @path = path
-        @timestamp = timestamp
-      end
-
-      def apply(logger, client, processor)
-        apply_log(logger, @path) do
-          processor.load(@path)
-        end
-      end
-    end
-
-    class PackedSchemaTarget
-      include ApplyLoggable
-
-      attr_reader :path
-      attr_reader :timestamp
-      attr_reader :targets
-      def initialize(path, timestamp)
-        @path = path
-        @timestamp = timestamp
-        @targets = []
-      end
-
-      def apply(logger, client, processor)
-        apply_log(logger, @path) do
-          @targets.sort_by(&:timestamp).each do |target|
-            target.apply(logger, client, processor)
-          end
-        end
-      end
-    end
-
-    class TableTarget
-      include ApplyLoggable
-
-      attr_reader :path
-      attr_reader :timestamp
-      attr_reader :name
-      attr_reader :action
-      def initialize(path, timestamp, name, action)
-        @path = path
-        @timestamp = timestamp
-        @name = name
-        @action = action
-      end
-
-      def apply(logger, client, processor)
-        apply_log(logger, @path) do
-          if @path.end_with?(".grn")
-            processor.load(@path)
-          else
-            # TODO: Add support for @action == "delete"
-            table = Arrow::Table.load(@path)
-            command = Groonga::Command::Load.new(table: @name,
-                                                 values: table,
-                                                 command_version: "3")
-            response = client.load(command.arguments)
-            processor.process_response(response, command)
-          end
-        end
-      end
-    end
-
-    class PackedTableTarget
-      include ApplyLoggable
-
-      attr_reader :path
-      attr_reader :timestamp
-      attr_reader :name
-      attr_reader :targets
-      def initialize(path, timestamp, name)
-        @path = path
-        @timestamp = timestamp
-        @name = name
-        @targets = []
-      end
-
-      def apply(logger, client, processor)
-        apply_log(logger, @path) do
-          @targets.sort_by(&:timestamp).each do |target|
-            target.apply(logger, client, processor)
-          end
-        end
-      end
-    end
-
     class CommandProcessor < Groonga::Client::CommandProcessor
       def initialize(config, *args)
         @config = config