groonga-delta 1.0.0

data/lib/groonga-delta/import-config.rb

@@ -0,0 +1,168 @@
+# Copyright (C) 2021-2022  Sutou Kouhei <kou@clear-code.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require_relative "config"
+require_relative "mapping"
+
+module GroongaDelta
+  class ImportConfig < Config
+    def initialize(dir)
+      super("groonga-delta-import", dir)
+    end
+
+    def delta_dir
+      resolve_path(@data["delta_dir"] || "delta")
+    end
+
+    def mysql
+      return nil unless @data["mysql"]
+      MySQL.new(@dir,
+                @data["mysql"],
+                @secret_data["mysql"] || {})
+    end
+
+    def local
+      return nil unless @data["local"]
+      Local.new(@dir, @data["local"])
+    end
+
+    def mapping
+      Mapping.new(@data["mapping"] || {})
+    end
+
+    class MySQL
+      include Config::PathResolvable
+
+      def initialize(dir, data, secret_data)
+        @dir = dir
+        @data = data
+        @secret_data = secret_data
+      end
+
+      def binlog_dir
+        resolve_path(@data["binlog_dir"] || "binlog")
+      end
+
+      def mysqlbinlog
+        @data["mysqlbinlog"] || "mysqlbinlog"
+      end
+
+      def host
+        @data["host"] || "localhost"
+      end
+
+      def port
+        @data["port"] || 3306
+      end
+
+      def socket
+        @data["socket"]
+      end
+
+      def user
+        @data["user"]
+      end
+
+      def password
+        @secret_data["password"] || @data["password"]
+      end
+
+      def replication_client
+        @data["replication_client"] || @data
+      end
+
+      def replication_client_user
+        replication_client["user"]
+      end
+
+      def replication_client_password
+        (@secret_data["replication_client"] || @secret_data)["password"] ||
+          replication_client["password"]
+      end
+
+      def replication_slave
+        @data["replication_slave"] || @data
+      end
+
+      def replication_slave_user
+        replication_slave["user"]
+      end
+
+      def replication_slave_password
+        (@secret_data["replication_slave"] || @secret_data)["password"] ||
+          replication_slave["password"]
+      end
+
+      def select
+        @data["select"] || @data
+      end
+
+      def select_user
+        select["user"]
+      end
+
+      def select_password
+        (@secret_data["select"] || @secret_data)["password"] ||
+          select["password"]
+      end
+
+      def checksum
+        _checksum = @data["checksum"]
+        return nil if _checksum.nil?
+        _checksum.to_sym
+      end
+
+      def initial_import_batch_size
+        resolve_size(@data["initial_import_batch_size"] || 1024 * 1024)
+      end
+
+      private
+      def resolve_size(value)
+        case value
+        when String
+          case value
+          when /\A(\d+)[kK]\z/
+            Integer($1, 10) * 1024
+          when /\A(\d+)[mM]\z/
+            Integer($1, 10) * 1024 * 1024
+          when /\A(\d+)[gG]\z/
+            Integer($1, 10) * 1024 * 1024 * 1024
+          else
+            raise ConfigError, "invalid size value: #{value.inspect}"
+          end
+        else
+          value
+        end
+      end
+    end
+
+    class Local
+      include Config::PathResolvable
+
+      def initialize(dir, data)
+        @dir = dir
+        @data = data
+      end
+
+      def dir
+        resolve_path(@data["dir"] || "local")
+      end
+
+      def initial_max_number
+        @data["initial_max_number"] || Float::INFINITY
+      end
+    end
+  end
+end

data/lib/groonga-delta/import-status.rb

@@ -0,0 +1,68 @@
+# Copyright (C) 2021-2022  Sutou Kouhei <kou@clear-code.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require_relative "status"
+
+module GroongaDelta
+  class ImportStatus < Status
+    def mysql
+      MySQL.new(self)
+    end
+
+    def local
+      Local.new(self)
+    end
+
+    class MySQL
+      def initialize(status)
+        @status = status
+      end
+
+      def [](key)
+        (@status["mysql"] || {})[key]
+      end
+
+      def update(new_data)
+        @status.update("mysql" => new_data)
+      end
+
+      def file
+        self["file"]
+      end
+
+      def position
+        self["position"]
+      end
+    end
+
+    class Local
+      def initialize(status)
+        @status = status
+      end
+
+      def [](key)
+        (@status["local"] || {})[key]
+      end
+
+      def update(new_data)
+        @status.update("local" => new_data)
+      end
+
+      def number
+        self["number"]
+      end
+    end
+  end
+end

data/lib/groonga-delta/local-delta.rb

@@ -0,0 +1,386 @@
+# Copyright (C) 2021-2022  Sutou Kouhei <kou@clear-code.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require "fileutils"
+
+require "groonga/client"
+require "parquet"
+
+require_relative "error"
+
+module GroongaDelta
+  class LocalDelta
+    def initialize(config, status)
+      @config = config
+      @status = status
+      @logger = @config.logger
+      @delta_dir = @config.local.delta_dir
+    end
+
+    def apply
+      start_time = read_current_status
+      current_time = Time.now.utc
+      targets = list_targets(@delta_dir, start_time, current_time)
+      client_options = {
+        url: @config.groonga.url,
+        read_timeout: @config.groonga.read_timeout,
+        backend: :synchronous,
+      }
+      Groonga::Client.open(client_options) do |client|
+        processor = CommandProcessor.new(@config,
+                                         client,
+                                         target_commands: [],
+                                         target_tables: [],
+                                         target_columns: [])
+        targets.sort_by(&:timestamp).each do |target|
+          target.apply(@logger, client, processor)
+          @status.update("start_time" => [
+                           target.timestamp.to_i,
+                           target.timestamp.nsec,
+                         ])
+        end
+      end
+    end
+
+    private
+    def build_time(year, month, day, hour=0, minute=0, second=0, nanosecond=0)
+      Time.utc(year,
+               month,
+               day,
+               hour,
+               minute,
+               Rational(second * 1_000_000_000 + nanosecond,
+                        1_000_000_000))
+    end
+
+    def read_current_status
+      start_time_unix_time, start_time_nanosecond = @status.start_time
+      if start_time_unix_time
+        start_time = Time.at(start_time_unix_time).utc
+        build_time(start_time.year,
+                   start_time.month,
+                   start_time.day,
+                   start_time.hour,
+                   start_time.min,
+                   start_time.sec,
+                   start_time_nanosecond)
+      else
+        Time.at(0).utc
+      end
+    end
+
+    def each_target_path(dir,
+                         min_timestamp,
+                         max_timestamp,
+                         accept_directory: true,
+                         &block)
+      Dir.glob("#{dir}/*") do |path|
+        base_name = File.basename(path)
+        if accept_directory and File.directory?(path)
+          timestamp = parse_directory_timestamp(base_name)
+          next if timestamp.nil?
+          next if min_timestamp and timestamp <= min_timestamp
+          next if max_timestamp and timestamp > max_timestamp
+          each_target_path(path,
+                           min_timestamp,
+                           max_timestamp,
+                           accept_directory: false,
+                           &block)
+        elsif File.file?(path)
+          timestamp, action, post_match = parse_file_timestamp(base_name)
+          next if timestamp.nil?
+          next if min_timestamp and timestamp <= min_timestamp
+          next if max_timestamp and timestamp > max_timestamp
+          yield(path, timestamp, action, post_match)
+        end
+      end
+    end
+
+    def each_packed_target_path(dir, min_timestamp, max_timestamp)
+      return unless min_timestamp.to_i.zero?
+      Dir.glob("#{dir}/packed/*") do |path|
+        next unless File.directory?(path)
+        timestamp, action, post_match = parse_file_timestamp(File.basename(path))
+        next if action
+        next unless post_match.empty?
+        yield(path, timestamp)
+      end
+    end
+
+    def list_targets(dir, start_time, current_timestamp)
+      targets = []
+      list_schema_targets(dir, start_time, current_timestamp, targets)
+      Dir.glob("#{dir}/data/*") do |path|
+        next unless File.directory?(path)
+        name = File.basename(path)
+        list_table_targets(path, name, start_time, current_timestamp, targets)
+      end
+      targets
+    end
+
+    def each_schema_target(dir, min_timestamp, max_timestamp)
+      each_target_path(dir,
+                       min_timestamp,
+                       max_timestamp) do |path, timestamp, action, post_match|
+        next if action
+        next unless post_match == ".grn"
+        yield(SchemaTarget.new(path, timestamp))
+      end
+    end
+
+    def list_schema_targets(dir, start_time, current_timestamp, targets)
+      latest_packed_target = nil
+      each_packed_target_path("#{dir}/schema",
+                              start_time,
+                              current_timestamp) do |path, timestamp|
+        if latest_packed_target and latest_packed_target.timestamp > timestamp
+          next
+        end
+        latest_packed_target = PackedSchemaTarget.new(path, timestamp)
+      end
+      if latest_packed_target
+        targets << latest_packed_target
+        each_schema_target(latest_packed_target.path, nil, nil) do |target|
+          latest_packed_target.targets << target
+        end
+      end
+      each_schema_target("#{dir}/schema",
+                         latest_packed_target&.timestamp || start_time,
+                         current_timestamp) do |target|
+        targets << target
+      end
+    end
+
+    TABLE_TARGET_SUFFIXES = [".grn", ".parquet"]
+    def each_table_target(dir, name, min_timestamp, max_timestamp)
+      each_target_path(dir,
+                       min_timestamp,
+                       max_timestamp) do |path, timestamp, action, post_match|
+        next if action.nil?
+        next unless TABLE_TARGET_SUFFIXES.include?(post_match)
+        yield(TableTarget.new(path, timestamp, name, action))
+      end
+    end
+
+    def list_table_targets(dir, name, start_time, current_timestamp, targets)
+      latest_packed_target = nil
+      each_packed_target_path(dir,
+                              start_time,
+                              current_timestamp) do |path, timestamp|
+        if latest_packed_target and latest_packed_target.timestamp > timestamp
+          next
+        end
+        latest_packed_target = PackedTableTarget.new(path, timestamp, name)
+      end
+      if latest_packed_target
+        targets << latest_packed_target
+        each_table_target(latest_packed_target.path, name, nil, nil) do |target|
+          latest_packed_target.targets << target
+        end
+      end
+      each_table_target(dir,
+                        name,
+                        latest_packed_target&.timestamp || start_time,
+                        current_timestamp) do |target|
+        targets << target
+      end
+    end
+
+    def parse_directory_timestamp(base_name)
+      case base_name
+      when /\A(\d{4})-(\d{2})-(\d{2})\z/
+        match = Regexp.last_match
+        year = match[1].to_i
+        month = match[2].to_i
+        day = match[3].to_i
+        build_time(year, month, day)
+      else
+        nil
+      end
+    end
+
+    def parse_file_timestamp(base_name)
+      case base_name
+      when /\A(\d{4})-(\d{2})-(\d{2})-(\d{2})-(\d{2})-(\d{2})-(\d{9})(?:-(\w+))?/
+        match = Regexp.last_match
+        year = match[1].to_i
+        month = match[2].to_i
+        day = match[3].to_i
+        hour = match[4].to_i
+        minute = match[5].to_i
+        second = match[6].to_i
+        nanosecond = match[7].to_i
+        action = match[8]
+        timestamp = build_time(year,
+                               month,
+                               day,
+                               hour,
+                               minute,
+                               second,
+                               nanosecond)
+        [timestamp, action, match.post_match]
+      else
+        nil
+      end
+    end
+
+    module ApplyLoggable
+      private
+      def apply_log(logger, path)
+        logger.info("Start applying: #{path}")
+        result = yield
+        logger.info("Applied: #{path}")
+        result
+      end
+    end
+
+    class SchemaTarget
+      include ApplyLoggable
+
+      attr_reader :path
+      attr_reader :timestamp
+      def initialize(path, timestamp)
+        @path = path
+        @timestamp = timestamp
+      end
+
+      def apply(logger, client, processor)
+        apply_log(logger, @path) do
+          processor.load(@path)
+        end
+      end
+    end
+
+    class PackedSchemaTarget
+      include ApplyLoggable
+
+      attr_reader :path
+      attr_reader :timestamp
+      attr_reader :targets
+      def initialize(path, timestamp)
+        @path = path
+        @timestamp = timestamp
+        @targets = []
+      end
+
+      def apply(logger, client, processor)
+        apply_log(logger, @path) do
+          @targets.sort_by(&:timestamp).each do |target|
+            target.apply(logger, client, processor)
+          end
+        end
+      end
+    end
+
+    class TableTarget
+      include ApplyLoggable
+
+      attr_reader :path
+      attr_reader :timestamp
+      attr_reader :name
+      attr_reader :action
+      def initialize(path, timestamp, name, action)
+        @path = path
+        @timestamp = timestamp
+        @name = name
+        @action = action
+      end
+
+      def apply(logger, client, processor)
+        apply_log(logger, @path) do
+          if @path.end_with?(".grn")
+            processor.load(@path)
+          else
+            # TODO: Add support for @action == "delete"
+            table = Arrow::Table.load(@path)
+            command = Groonga::Command::Load.new(table: @name,
+                                                 values: table,
+                                                 command_version: "3")
+            response = client.load(command.arguments)
+            processor.process_response(response, command)
+          end
+        end
+      end
+    end
+
+    class PackedTableTarget
+      include ApplyLoggable
+
+      attr_reader :path
+      attr_reader :timestamp
+      attr_reader :name
+      attr_reader :targets
+      def initialize(path, timestamp, name)
+        @path = path
+        @timestamp = timestamp
+        @name = name
+        @targets = []
+      end
+
+      def apply(logger, client, processor)
+        apply_log(logger, @path) do
+          @targets.sort_by(&:timestamp).each do |target|
+            target.apply(logger, client, processor)
+          end
+        end
+      end
+    end
+
+    class CommandProcessor < Groonga::Client::CommandProcessor
+      def initialize(config, *args)
+        @config = config
+        super(*args)
+      end
+
+      def process_response(response, command)
+        message = ""
+        case command.command_name
+        when "load"
+          command.arguments.delete(:values)
+          if response.success?
+            message = "#{response.n_loaded_records}: "
+          else
+            load_response = Groonga::Client::Response::Load.new(command,
+                                                                response.header,
+                                                                response.body)
+            message = "#{load_response.n_loaded_records}: "
+          end
+        end
+        if response.success?
+          @config.logger.info("Processed: " +
+                              "#{response.elapsed_time}: " +
+                              "#{command.command_name}: " +
+                              message +
+                              "#{command.to_command_format}")
+        else
+          failed_message = "Failed to process: " +
+                           "#{response.return_code}: " +
+                           "#{response.elapsed_time}: " +
+                           "#{response.error_message}: " +
+                           "#{command.command_name}: " +
+                           message +
+                           "#{command.to_command_format}"
+          case @config.on_error
+          when "ignore"
+          when "warning"
+            @config.logger.warn(failed_message)
+          when "error"
+            raise ExecutionError, failed_message
+          end
+        end
+      end
+    end
+  end
+end