groonga-delta 1.0.0

data/README.md

@@ -0,0 +1,43 @@
+# Groonga delta
+
+## Description
+
+Groonga delta provides delta based data import tools. They can import data to Groonga from other systems such as local file system, MySQL and PostgreSQL.
+
+## Usage
+
+### Import deltas from other systems
+
+Create configuration file: TODO
+
+Run `groonga-delta-import` with the created configuration file:
+
+```bash
+docker run \
+  --rm \
+  --volume /var/lib/groonga-delta:/var/lib/groonga-delta:z \
+  ghcr.io/groonga/groonga-delta:latest \
+  groonga-delta-import \
+    --server \
+    --dir=/var/lib/groonga-delta/import
+```
+
+### Apply imported deltas
+
+Create configuration file: TODO
+
+Run `groonga-delta-apply` with the created configuration file:
+
+```bash
+docker run \
+  --rm \
+  --volume /var/lib/groonga-delta:/var/lib/groonga-delta:z \
+  ghcr.io/groonga/groonga-delta:latest \
+  groonga-delta-apply \
+    --server \
+    --dir=/var/lib/groonga-delta/apply
+```
+
+## License
+
+GPLv3 or later. See `LICENSE.txt` for details.

data/Rakefile

@@ -0,0 +1,36 @@
+# -*- ruby -*-
+#
+# Copyright (C) 2021-2022  Sutou Kouhei <kou@clear-code.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require "rubygems"
+require "bundler/gem_helper"
+
+base_dir = File.join(__dir__)
+
+helper = Bundler::GemHelper.new(base_dir)
+def helper.version_tag
+  version
+end
+
+helper.install
+spec = helper.gemspec
+
+desc "Run tests"
+task :test do
+  ruby("test/run.rb")
+end
+
+task default: :test

data/bin/groonga-delta-apply

@@ -0,0 +1,21 @@
+#!/usr/bin/env ruby
+#
+# Copyright (C) 2021-2022  Sutou Kouhei <kou@clear-code.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require_relative "../lib/groonga-delta"
+
+apply_command = GroongaDelta::ApplyCommand.new
+exit(apply_command.run(ARGV))

data/bin/groonga-delta-import

@@ -0,0 +1,21 @@
+#!/usr/bin/env ruby
+#
+# Copyright (C) 2021-2022  Sutou Kouhei <kou@clear-code.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require_relative "../lib/groonga-delta"
+
+import_command = GroongaDelta::ImportCommand.new
+exit(import_command.run(ARGV))

data/doc/text/news.md

@@ -0,0 +1,5 @@
+# NEWS
+
+## 1.0.0 - 2022-03-07
+
+Initial release.

data/groonga-delta.gemspec

@@ -0,0 +1,54 @@
+# -*- ruby -*-
+#
+# Copyright (C) 2021-2022  Sutou Kouhei <kou@clear-code.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+clean_white_space = lambda do |entry|
+  entry.gsub(/(\A\n+|\n+\z)/, '') + "\n"
+end
+
+require_relative "lib/groonga-delta/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "groonga-delta"
+  spec.version = GroongaDelta::VERSION
+  spec.homepage = "https://github.com/groonga/groonga-delta"
+  spec.authors = ["Sutou Kouhei"]
+  spec.email = ["kou@clear-code.com"]
+
+  readme = File.read("README.md")
+  readme.force_encoding("UTF-8")
+  entries = readme.split(/^\#\#\s(.*)$/)
+  clean_white_space.call(entries[entries.index("Description") + 1])
+  description = clean_white_space.call(entries[entries.index("Description") + 1])
+  spec.summary, spec.description, = description.split(/\n\n+/, 3)
+  spec.license = "GPL-3.0+"
+  spec.files = [
+    "#{spec.name}.gemspec",
+    "Gemfile",
+    "LICENSE.txt",
+    "README.md",
+    "Rakefile",
+    "doc/text/news.md",
+  ]
+  spec.files += Dir.glob("lib/**/*.rb")
+  Dir.chdir("bin") do
+    spec.executables = Dir.glob("*")
+  end
+
+  spec.add_runtime_dependency("groonga-client")
+  spec.add_runtime_dependency("groonga-command-parser")
+  spec.add_runtime_dependency("red-parquet")
+end

data/lib/groonga-delta/apply-command.rb

@@ -0,0 +1,38 @@
+# Copyright (C) 2021-2022  Sutou Kouhei <kou@clear-code.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require_relative "command"
+require_relative "apply-config"
+require_relative "apply-status"
+
+require_relative "local-delta"
+
+module GroongaDelta
+  class ApplyCommand < Command
+    private
+    def prepare
+      @config = ApplyConfig.new(@dir)
+      @status = ApplyStatus.new(@dir)
+      @deltas = []
+      @deltas << LocalDelta.new(@config, @status)
+    end
+
+    def process
+      @deltas.each do |delta|
+        delta.apply
+      end
+    end
+  end
+end

data/lib/groonga-delta/apply-config.rb

@@ -0,0 +1,78 @@
+# Copyright (C) 2021-2022  Sutou Kouhei <kou@clear-code.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require_relative "config"
+
+module GroongaDelta
+  class ApplyConfig < Config
+    def initialize(dir)
+      super("groonga-delta-apply", dir)
+      validate_on_error(on_error)
+    end
+
+    def on_error
+      @data["on_error"] || "error"
+    end
+
+    def groonga
+      Groonga.new(@dir, @data["groonga"] || {})
+    end
+
+    def local
+      Local.new(@dir, @data["local"] || {})
+    end
+
+    private
+    def validate_on_error(on_error)
+      case on_error
+      when "ignore"
+      when "warning"
+      when "error"
+      else
+        message = "on_error must be ignore, warning or error: " +
+                  on_error.inspect
+        raise ConfigError, message
+      end
+    end
+
+    class Groonga
+      def initialize(dir, data)
+        @dir = dir
+        @data = data
+      end
+
+      def url
+        @data["url"] || "http://127.0.0.1:10041"
+      end
+
+      def read_timeout
+        @data["read_timeout"] || ::Groonga::Client::Default::READ_TIMEOUT
+      end
+    end
+
+    class Local
+      include Config::PathResolvable
+
+      def initialize(dir, data)
+        @dir = dir
+        @data = data
+      end
+
+      def delta_dir
+        resolve_path(@data["delta_dir"] || "delta")
+      end
+    end
+  end
+end

data/lib/groonga-delta/apply-status.rb

@@ -0,0 +1,24 @@
+# Copyright (C) 2021  Sutou Kouhei <kou@clear-code.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require_relative "status"
+
+module GroongaDelta
+  class ApplyStatus < Status
+    def start_time
+      self["start_time"]
+    end
+  end
+end

data/lib/groonga-delta/command.rb

@@ -0,0 +1,75 @@
+# Copyright (C) 2021-2022  Sutou Kouhei <kou@clear-code.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require "optparse"
+
+module GroongaDelta
+  class Command
+    def initialize
+      @dir = "."
+      @server = false
+      @config = nil
+    end
+
+    def run(args)
+      catch do |tag|
+        parse_args(args, tag)
+        begin
+          prepare
+          loop do
+            process
+            break unless @server
+            sleep(@config.polling_interval)
+          end
+          true
+        rescue => error
+          @config.logger.error(error) if @config
+          raise
+        end
+      end
+    end
+
+    private
+    def parse_args(args, tag)
+      parser = OptionParser.new
+      parser.on("--dir=DIR",
+                "Use DIR as directory that has configuration files",
+                "(#{@dir})") do |dir|
+        @dir = dir
+      end
+      parser.on("--server",
+                "Run as a server") do
+        @server = true
+      end
+      parser.on("--version",
+                "Show version and exit") do
+        puts(VERSION)
+        throw(tag, true)
+      end
+      parser.on("--help",
+                "Show this message and exit") do
+        puts(parser.help)
+        throw(tag, true)
+      end
+      begin
+        parser.parse!(args.dup)
+      rescue OptionParser::InvalidOption => error
+        puts(error.message)
+        puts(parser.help)
+        throw(tag, false)
+      end
+    end
+  end
+end

data/lib/groonga-delta/config.rb

@@ -0,0 +1,99 @@
+# Copyright (C) 2021-2022  Sutou Kouhei <kou@clear-code.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require "fileutils"
+require "logger"
+require "yaml"
+
+require_relative "ltsv-log-formatter"
+
+class Logger::LogDevice
+  private
+  def add_log_header(file)
+    # Disable adding log header
+  end
+end
+
+module GroongaDelta
+  class Config
+    module PathResolvable
+      private
+      def resolve_path(path)
+        File.expand_path(path, @dir)
+      end
+    end
+
+    include PathResolvable
+
+    def initialize(name, dir)
+      @name = name
+      @dir = dir
+      @path = File.join(@dir, "config.yaml")
+      if File.exist?(@path)
+        @data = YAML.load(File.read(@path))
+      else
+        @data = {}
+      end
+      @secret_path = File.join(@dir, "secret.yaml")
+      if File.exist?(@secret_path)
+        @secret_data = YAML.load(File.read(@secret_path))
+      else
+        @secret_data = {}
+      end
+    end
+
+    def logger
+      @logger ||= create_logger
+    end
+
+    def log_path
+      resolve_path(File.join(@data["log_dir"] || "log",
+                             "#{@name}.log"))
+    end
+
+    def log_age
+      @data["log_age"] || 7
+    end
+
+    def log_max_size
+      @data["log_max_size"] || (1024 * 1024)
+    end
+
+    def log_period_suffix
+      @data["log_period_suffix"] || "%Y-%m-%d"
+    end
+
+    def log_level
+      @data["log_level"] || "info"
+    end
+
+    def polling_interval
+      Float(@data["polling_interval"] || "60")
+    end
+
+    private
+    def create_logger
+      path = log_path
+      FileUtils.mkdir_p(File.dirname(path))
+      Logger.new(path,
+                 log_age,
+                 log_max_size,
+                 formatter: LTSVLogFormatter.new,
+                 level: log_level,
+                 progname: @name,
+                 shift_period_suffix: log_period_suffix)
+    end
+  end
+end

data/lib/groonga-delta/error.rb

@@ -0,0 +1,28 @@
+# Copyright (C) 2022  Sutou Kouhei <kou@clear-code.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+module GroongaDelta
+  class Error < StandardError
+  end
+
+  class ConfigError < Error
+  end
+
+  class ExecutionError < Error
+  end
+
+  class ProcessError < Error
+  end
+end

data/lib/groonga-delta/import-command.rb

@@ -0,0 +1,43 @@
+# Copyright (C) 2021-2022  Sutou Kouhei <kou@clear-code.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require_relative "command"
+require_relative "import-config"
+require_relative "import-status"
+
+module GroongaDelta
+  class ImportCommand < Command
+    private
+    def prepare
+      @config = ImportConfig.new(@dir)
+      @status = ImportStatus.new(@dir)
+      @sources = []
+      if @config.local
+        require_relative "local-source"
+        @sources << LocalSource.new(@config, @status)
+      end
+      if @config.mysql
+        require_relative "mysql-source"
+        @sources << MySQLSource.new(@config, @status)
+      end
+    end
+
+    def process
+      @sources.each do |source|
+        source.import
+      end
+    end
+  end
+end