fluent-plugin-mysql-replicator 1.0.2 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5c523ddf1d5052d11c6c7a0f7e9172d319c881257d791a0006cd1a6601f5a948
-  data.tar.gz: ca9f218dfa56543f141d8911578250194281bfbad16d8f5ed6b4980120d99c8d
+  metadata.gz: 16c6600b210b85bad55965f707eadd53017558fd8a6a2b55cdf28bc1e86b7be3
+  data.tar.gz: 07d8b7367ee234afb128845fb8ef9169929cb18caf4c1813040efe951e72f68e
 SHA512:
-  metadata.gz: a5266cd4f824e781d53eaa613c37f260c7814badcee87fb6c49a1321e9c9ac8bc8d9505b2ea800f3a8ccde237b2b0eeb808e187463996c2b1aecc93065128869
-  data.tar.gz: 70834ef8629d34bbd53aa1ff9d66ba706d6df5b48cd310d8a554c7d25628682633860123f63855e851482abda385e6288c4fcd01da848cbffc56bc0525553968
+  metadata.gz: 65ec0c92b273597f7f7ff976fc42878e4fa413d8471947125a6873d7209142bec87a59aa37e3c491fd1a81dea4a9d448e3280cbca1eafd381fdee241d723bcae
+  data.tar.gz: 20fb6ffe6e217e5d8ee0108c5243fad7f6abd910f1a2d336f65bcb8ba0388ac22e329d408edaa5ad8f298c959cffc2656932d71302d6b2b38c21c28cab14585d

data/.devcontainer/Dockerfile

@@ -0,0 +1,17 @@
+FROM ruby:3.3-slim
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV BUNDLE_PATH=/workspace/vendor/bundle
+
+RUN apt-get update \
+  && apt-get install -y --no-install-recommends \
+    build-essential \
+    default-libmysqlclient-dev \
+    git \
+  && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /workspace
+
+RUN gem install bundler
+
+CMD ["bash"]

data/.devcontainer/devcontainer.json

@@ -0,0 +1,16 @@
+{
+  "name": "fluent-plugin-mysql-replicator",
+  "build": {
+    "dockerfile": "Dockerfile"
+  },
+  "workspaceFolder": "/workspace",
+  "workspaceMount": "source=${localWorkspaceFolder},target=/workspace,type=bind,consistency=cached",
+  "settings": {
+    "terminal.integrated.shell.linux": "/bin/bash"
+  },
+  "extensions": [
+    "rebornix.Ruby"
+  ],
+  "postCreateCommand": "bundle config set --local path \"$BUNDLE_PATH\" && bundle install --jobs 4 --retry 3",
+  "remoteUser": "root"
+}

data/.dockerignore

@@ -0,0 +1,4 @@
+vendor/bundle
+.bundle
+*.gem
+pkg

data/.github/workflows/ci.yml

@@ -0,0 +1,102 @@
+name: CI
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+
+jobs:
+  unit:
+    name: Unit tests (Ruby ${{ matrix.ruby }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: ['3.2', '3.3', '3.4', '4.0']
+    steps:
+      - uses: actions/checkout@v5
+      - name: Install MySQL client headers
+        run: sudo apt-get update && sudo apt-get install -y default-libmysqlclient-dev
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+      - run: bundle exec rake test
+
+  e2e:
+    name: E2E replication (ES ${{ matrix.es_label }})
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # The plugin auto-detects the version and adjusts "_type" accordingly.
+          - es_label: '6.8'
+            es_image: docker.elastic.co/elasticsearch/elasticsearch:6.8.23
+            es_major: '6'
+          - es_label: '8.x'
+            es_image: docker.elastic.co/elasticsearch/elasticsearch:8.18.0
+            es_major: '8'
+          - es_label: '9.x'
+            es_image: docker.elastic.co/elasticsearch/elasticsearch:9.0.0
+            es_major: '9'
+
+    services:
+      mysql:
+        image: mysql:8.4
+        env:
+          MYSQL_ROOT_PASSWORD: root
+        ports:
+          - 3306:3306
+        options: >-
+          --health-cmd="mysqladmin ping -h localhost -proot"
+          --health-interval=10s
+          --health-timeout=5s
+          --health-retries=10
+
+      elasticsearch:
+        image: ${{ matrix.es_image }}
+        env:
+          discovery.type: single-node
+          xpack.security.enabled: 'false'
+          ES_JAVA_OPTS: -Xms512m -Xmx512m
+        ports:
+          - 9200:9200
+
+    env:
+      MYSQL_HOST: 127.0.0.1
+      MYSQL_PORT: 3306
+      MYSQL_USER: root
+      MYSQL_PASSWORD: root
+      MYSQL_DATABASE: e2e_source
+      ES_HOST: 127.0.0.1
+      ES_PORT: 9200
+      ES_MAJOR_VERSION: ${{ matrix.es_major }}
+
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Install MySQL client headers
+        run: sudo apt-get update && sudo apt-get install -y default-libmysqlclient-dev
+
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.4'
+          bundler-cache: true
+
+      - name: Wait for Elasticsearch
+        run: |
+          for i in $(seq 1 30); do
+            if curl -fs "http://127.0.0.1:9200/_cluster/health?wait_for_status=yellow&timeout=5s" >/dev/null; then
+              echo "Elasticsearch is up"; exit 0
+            fi
+            echo "waiting for Elasticsearch ($i)..."; sleep 5
+          done
+          echo "Elasticsearch did not become ready"; exit 1
+
+      - name: Run single-table replication E2E test
+        run: bundle exec ruby test/e2e/replication_single_e2e_test.rb
+
+      - name: Run multi-table replication E2E test
+        run: bundle exec ruby test/e2e/replication_multi_e2e_test.rb

data/.gitignore

@@ -3,3 +3,4 @@
 Gemfile.lock
 pkg/*
 vendor/*
+test/e2e/*.log

data/README.md

@@ -1,4 +1,4 @@
-# fluent-plugin-mysql-replicator [![Build Status](https://travis-ci.org/y-ken/fluent-plugin-mysql-replicator.png?branch=master)](https://travis-ci.org/y-ken/fluent-plugin-mysql-replicator)
+# fluent-plugin-mysql-replicator [![CI](https://github.com/y-ken/fluent-plugin-mysql-replicator/actions/workflows/ci.yml/badge.svg)](https://github.com/y-ken/fluent-plugin-mysql-replicator/actions/workflows/ci.yml)
 
 ## Overview
 
@@ -31,15 +31,33 @@ install with gem or fluent-gem command as:
 
 `````
 # for system installed fluentd
-$ gem install fluent-plugin-mysql-replicator -v 1.0.2
+$ gem install fluent-plugin-mysql-replicator -v 1.0.3
 
 # for td-agent2
 $ sudo td-agent-gem install fluent-plugin-mysql-replicator -v 0.6.1
 
 # for td-agent3
-$ sudo td-agent-gem install fluent-plugin-mysql-replicator -v 1.0.2
+$ sudo td-agent-gem install fluent-plugin-mysql-replicator -v 1.0.3
 `````
 
+## Development container
+
+This repository includes a VS Code Dev Container configuration under `.devcontainer/`.
+Use Docker and Remote Containers / Dev Containers in VS Code to build and open the workspace inside a container.
+The container installs Ruby, Bundler, and required native build dependencies.
+
+After opening the repository in the Dev Container, run:
+
+```
+bundle install --path vendor/bundle
+```
+
+Then run tests like:
+
+```
+bundle exec ruby -Itest test/plugin/test_out_mysql_replicator_elasticsearch.rb
+```
+
 ## Included plugins
 
 * Input Plugin: mysql_replicator
@@ -47,6 +65,15 @@ $ sudo td-agent-gem install fluent-plugin-mysql-replicator -v 1.0.2
 * Output Plugin: mysql_replicator_elasticsearch
 * Output Plugin: mysql_replicator_solr (experimental)
 
+## Elasticsearch version compatibility
+
+`mysql_replicator_elasticsearch` works with Elasticsearch 6.x through 9.x.
+
+Mapping types were removed in Elasticsearch 8.x (and deprecated in 7.x), so the
+`_type` field can no longer be sent in bulk requests. The plugin detects the
+Elasticsearch version on the first write and automatically omits `_type` for
+7.x and later, so no extra configuration is required.
+
 ## Output example
 
 It is a example when detecting insert/update/delete events.

data/fluent-plugin-mysql-replicator.gemspec

@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-mysql-replicator"
-  s.version     = "1.0.2"
+  s.version     = "1.1.0"
   s.authors     = ["Kentaro Yoshida"]
   s.email       = ["y.ken.studio@gmail.com"]
   s.homepage    = "https://github.com/y-ken/fluent-plugin-mysql-replicator"

data/lib/fluent/plugin/in_mysql_replicator_multi.rb

@@ -85,37 +85,49 @@ module Fluent::Plugin
         while @running
           rows_count = 0
           start_time = Time.now
-          unless config['prepared_query'].nil?
-            nest_db = get_origin_connection(config)
-            config['prepared_query'].strip.split(/;/).each do |query|
-              nest_db.query(query)
+          db = nil
+          nest_db = nil
+          begin
+            unless config['prepared_query'].nil?
+              nest_db = get_origin_connection(config)
+              config['prepared_query'].strip.split(/;/).each do |query|
+                nest_db.query(query)
+              end
             end
-          end
-          db = get_origin_connection(config)
-          db.query(config['query']).each do |row|
-            row.each {|k, v| row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
-            row.select {|k, v| v.to_s.strip.match(/^SELECT[^\$]+\$\{[^\}]+\}/i) }.each do |k, v|
-              row[k] = [] unless row[k].is_a?(Array)
-              nest_db.query(v.gsub(/\$\{([^\}]+)\}/) {|matched| row[$1].to_s}).each do |nest_row|
-                nest_row.each {|k, v| nest_row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
-                row[k] << nest_row
+            db = get_origin_connection(config)
+            db.query(config['query']).each do |row|
+              row.each {|k, v| row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
+              row.select {|k, v| v.to_s.strip.match(/^SELECT[^\$]+\$\{[^\}]+\}/i) }.each do |k, v|
+                row[k] = [] unless row[k].is_a?(Array)
+                nest_db.query(v.gsub(/\$\{([^\}]+)\}/) {|matched| row[$1].to_s}).each do |nest_row|
+                  nest_row.each {|k, v| nest_row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
+                  row[k] << nest_row
+                end
               end
+              current_id = row[primary_key]
+              @mutex.synchronize {
+                if row[primary_key].nil?
+                  log.error "mysql_replicator_multi: missing primary_key. :setting_name=>#{config['name']} :primary_key=>#{primary_key}"
+                  break
+                end
+                detect_insert_update(config, row)
+                detect_delete(config, current_id, previous_id)
+              }
+              previous_id = current_id
+              rows_count += 1
             end
-            current_id = row[primary_key]
+          rescue Mysql2::Error => e
+            raise e unless config['enable_retry'] == 1
+
             @mutex.synchronize {
-              if row[primary_key].nil?
-                log.error "mysql_replicator_multi: missing primary_key. :setting_name=>#{config['name']} :primary_key=>#{primary_key}"
-                break
-              end
-              detect_insert_update(config, row)
-              detect_delete(config, current_id, previous_id)
+              log.error "mysql_replicator_multi: failed due to an error caused by the database. :setting_name=>#{config['name']}"
+              log.error "error: #{e.message}"
+              log.error e.backtrace.join("\n")
             }
-            previous_id = current_id
-            rows_count += 1
-          end
-          db.close
-          unless config['prepared_query'].nil?
-            nest_db.close
+            sleep config['retry_interval']
+          ensure
+            db.close if db
+            nest_db.close if nest_db && !config['prepared_query'].nil?
           end
           elapsed_time = sprintf("%0.02f", Time.now - start_time)
           @mutex.synchronize {
@@ -290,7 +302,11 @@ module Fluent::Plugin
           :cache_rows => false
         )
       rescue Mysql2::Error => e
-        raise "mysql_replicator_multi: #{e}"
+        if config['enable_retry'] == 1
+          raise e
+        else
+          raise "mysql_replicator_multi: #{e}"
+        end
       end
     end
   end

data/lib/fluent/plugin/out_mysql_replicator_elasticsearch.rb

@@ -1,5 +1,6 @@
 require 'net/http'
 require 'date'
+require 'yajl'
 require 'fluent/plugin/output'
 
 class Fluent::Plugin::MysqlReplicatorElasticsearchOutput < Fluent::Plugin::Output
@@ -38,6 +39,8 @@ class Fluent::Plugin::MysqlReplicatorElasticsearchOutput < Fluent::Plugin::Outpu
 
   def start
     super
+    # nil means "not yet detected"; resolved on the first write.
+    @suppress_type = nil
   end
 
   def format(tag, time, record)
@@ -57,6 +60,8 @@ class Fluent::Plugin::MysqlReplicatorElasticsearchOutput < Fluent::Plugin::Outpu
   end
 
   def write(chunk)
+    detect_type_suppression if @suppress_type.nil?
+
     bulk_message = []
 
     chunk.msgpack_each do |tag, time, record|
@@ -66,28 +71,60 @@ class Fluent::Plugin::MysqlReplicatorElasticsearchOutput < Fluent::Plugin::Outpu
       id_key = tag_parts['primary_key']
 
       if tag_parts['event'] == 'delete'
-        meta = { "delete" => {"_index" => target_index, "_type" => target_type, "_id" => record[id_key]} }
+        action = {"_index" => target_index, "_id" => record[id_key]}
+        action['_type'] = target_type unless @suppress_type
+        meta = { "delete" => action }
         bulk_message << Yajl::Encoder.encode(meta)
       else
-        meta = { "index" => {"_index" => target_index, "_type" => target_type} }
+        action = {"_index" => target_index}
+        action['_type'] = target_type unless @suppress_type
         if id_key && record[id_key]
-          meta['index']['_id'] = record[id_key]
+          action['_id'] = record[id_key]
         end
+        meta = { "index" => action }
         bulk_message << Yajl::Encoder.encode(meta)
         bulk_message << Yajl::Encoder.encode(record)
       end
     end
     bulk_message << ""
 
-    http = Net::HTTP.new(@host, @port.to_i)
-    http.use_ssl = @ssl
-
     request = Net::HTTP::Post.new('/_bulk', {'content-type' => 'application/json; charset=utf-8'})
     if @username && @password
       request.basic_auth(@username, @password)
     end
 
     request.body = bulk_message.join("\n")
-    http.request(request).value
+    new_http.request(request).value
+  end
+
+  private
+
+  def new_http
+    http = Net::HTTP.new(@host, @port.to_i)
+    http.use_ssl = @ssl
+    http
+  end
+
+  # Mapping types were removed in Elasticsearch 8.x and deprecated in 7.x.
+  # Detect the major version once and omit "_type" for 7.x and later.
+  def detect_type_suppression
+    major = elasticsearch_major_version
+    @suppress_type = !major.nil? && major >= 7
+    if major
+      log.info "mysql_replicator_elasticsearch: detected Elasticsearch #{major}.x, suppress_type=#{@suppress_type}"
+    else
+      log.warn "mysql_replicator_elasticsearch: could not detect Elasticsearch version, sending '_type' (assuming 6.x)"
+    end
+  end
+
+  def elasticsearch_major_version
+    request = Net::HTTP::Get.new('/')
+    request.basic_auth(@username, @password) if @username && @password
+    response = new_http.request(request)
+    number = Yajl::Parser.parse(response.body).dig('version', 'number')
+    number.to_s.split('.').first.to_i
+  rescue => e
+    log.warn "mysql_replicator_elasticsearch: version detection failed: #{e.message}"
+    nil
   end
 end

data/setup_mysql_replicator_multi.sql

@@ -29,6 +29,10 @@ CREATE TABLE IF NOT EXISTS `settings` (
   `enable_loose_insert` int(11) DEFAULT '0',
   -- On enabling 'enable_loose_delete: 1', turn on speculative delete but performance penalty on non-contiguous primary key.
   `enable_loose_delete` int(11) DEFAULT '0',
+  -- On enabling 'enable_retry: 1', automatically retries when an error occurs due to MySQL.
+  `enable_retry` int(11) DEFAULT '1',
+  -- Additional interval when retrying. If not set, waits for the time set in the regular interval column.
+  `retry_interval` int(11) NOT NULL DEFAULT '30',
   PRIMARY KEY (`id`),
   UNIQUE KEY `name` (`name`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8;

data/test/e2e/e2e_helper.rb

@@ -0,0 +1,94 @@
+# Shared helpers for the end-to-end replication tests.
+#
+# Connection settings come from environment variables (with local defaults):
+#   MYSQL_HOST MYSQL_PORT MYSQL_USER MYSQL_PASSWORD
+#   ES_HOST ES_PORT
+require 'mysql2'
+require 'net/http'
+require 'json'
+
+module E2E
+  module_function
+
+  ROOT = File.expand_path('../..', __dir__)
+
+  def mysql_config(database: nil)
+    cfg = {
+      host:     ENV['MYSQL_HOST'] || '127.0.0.1',
+      port:     (ENV['MYSQL_PORT'] || 3306).to_i,
+      username: ENV['MYSQL_USER'] || 'root',
+      password: ENV['MYSQL_PASSWORD'] || 'root',
+    }
+    cfg[:database] = database if database
+    cfg
+  end
+
+  def es_base
+    "http://#{ENV['ES_HOST'] || '127.0.0.1'}:#{ENV['ES_PORT'] || '9200'}"
+  end
+
+  # Major version of the Elasticsearch under test (defaults to 6).
+  def es_major_version
+    (ENV['ES_MAJOR_VERSION'] || '6').to_i
+  end
+
+  # Realtime GET by id. Returns [http_status, parsed_body].
+  # Elasticsearch 7.x+ dropped custom mapping types, so documents are addressed
+  # via the "_doc" endpoint instead of the original type name.
+  def es_get(index, type, id)
+    type_path = es_major_version >= 7 ? '_doc' : type
+    res = Net::HTTP.get_response(URI("#{es_base}/#{index}/#{type_path}/#{id}"))
+    [res.code.to_i, (JSON.parse(res.body) rescue {})]
+  end
+
+  # Drop an index so a test starts from a clean slate (ignores "not found").
+  def es_delete_index(index)
+    uri = URI("#{es_base}/#{index}")
+    Net::HTTP.start(uri.host, uri.port) { |h| h.request(Net::HTTP::Delete.new(uri)) }
+  rescue StandardError
+    # ignore: the index may not exist yet
+  end
+
+  # Poll a condition until it becomes truthy, otherwise fail with a timeout.
+  def wait_until(description, timeout: 90, log_path: nil)
+    deadline = Time.now + timeout
+    loop do
+      return if yield
+      fail_with("timeout (#{timeout}s) waiting for: #{description}", log_path) if Time.now > deadline
+      sleep 1
+    end
+  end
+
+  def fail_with(message, log_path = nil)
+    warn "\n[E2E] FAILED: #{message}"
+    if log_path && File.exist?(log_path)
+      warn "\n----- fluentd.log (tail) -----"
+      warn File.readlines(log_path).last(60).join
+      warn "------------------------------"
+    end
+    exit 1
+  end
+
+  def step(message)
+    puts "[E2E] #{message}"
+  end
+
+  # Boot fluentd as a child process. Returns the pid; logs go to log_path.
+  def spawn_fluentd(conf, log_path)
+    log = File.open(log_path, 'w')
+    Process.spawn(
+      'bundle', 'exec', 'fluentd',
+      '-c', conf,
+      '-p', File.join(ROOT, 'lib', 'fluent', 'plugin'),
+      '--no-supervisor',
+      out: log, err: log, chdir: ROOT
+    )
+  end
+
+  def stop_fluentd(pid)
+    Process.kill('TERM', pid)
+    Process.wait(pid)
+  rescue Errno::ESRCH, Errno::ECHILD
+    # already gone
+  end
+end

data/test/e2e/fluent_multi.conf

@@ -0,0 +1,30 @@
+# E2E pipeline for the multi-table input plugin.
+# in_mysql_replicator_multi reads replication settings from the management
+# database (replicator_manager.settings) and persists per-row hashes into
+# replicator_manager.hash_tables. See test/e2e/replication_multi_e2e_test.rb.
+#
+# bulk_insert_count is set to 1 so hash_tables is flushed immediately on each
+# insert, which keeps the test timing deterministic.
+<source>
+  @type mysql_replicator_multi
+  manager_host "#{ENV['MYSQL_HOST'] || '127.0.0.1'}"
+  manager_port "#{ENV['MYSQL_PORT'] || '3306'}"
+  manager_username "#{ENV['MYSQL_USER'] || 'root'}"
+  manager_password "#{ENV['MYSQL_PASSWORD'] || 'root'}"
+  manager_database "#{ENV['MYSQL_MANAGER_DATABASE'] || 'replicator_manager'}"
+  tag multiindex.multitype.${event}.${primary_key}
+  bulk_insert_count 1
+  bulk_insert_timeout 2
+</source>
+
+<match multiindex.**>
+  @type mysql_replicator_elasticsearch
+  host "#{ENV['ES_HOST'] || '127.0.0.1'}"
+  port "#{ENV['ES_PORT'] || '9200'}"
+  <buffer>
+    @type memory
+    flush_mode interval
+    flush_interval 1s
+    flush_thread_count 1
+  </buffer>
+</match>

data/test/e2e/fluent_single.conf

@@ -0,0 +1,28 @@
+# E2E pipeline: MySQL (in_mysql_replicator) -> Elasticsearch (out_mysql_replicator_elasticsearch)
+# Connection details are injected via environment variables so the same config
+# works both in CI and locally. See test/e2e/replication_e2e_test.rb.
+<source>
+  @type mysql_replicator
+  host "#{ENV['MYSQL_HOST'] || '127.0.0.1'}"
+  port "#{ENV['MYSQL_PORT'] || '3306'}"
+  username "#{ENV['MYSQL_USER'] || 'root'}"
+  password "#{ENV['MYSQL_PASSWORD'] || 'root'}"
+  database "#{ENV['MYSQL_DATABASE'] || 'e2e_source'}"
+  query SELECT id, name, age FROM users
+  primary_key id
+  interval 2s
+  enable_delete true
+  tag myindex.mytype.${event}.${primary_key}
+</source>
+
+<match myindex.**>
+  @type mysql_replicator_elasticsearch
+  host "#{ENV['ES_HOST'] || '127.0.0.1'}"
+  port "#{ENV['ES_PORT'] || '9200'}"
+  <buffer>
+    @type memory
+    flush_mode interval
+    flush_interval 1s
+    flush_thread_count 1
+  </buffer>
+</match>

data/test/e2e/replication_multi_e2e_test.rb

@@ -0,0 +1,122 @@
+#!/usr/bin/env ruby
+# End-to-end test for the multi-table input plugin (in_mysql_replicator_multi).
+#
+# Unlike the single plugin, this one is driven by a management database:
+#   - replicator_manager.settings   : one row per replication job
+#   - replicator_manager.hash_tables: persisted per-row hashes (delete detection)
+#
+# This test builds that management DB from setup_mysql_replicator_multi.sql,
+# registers a settings row pointing at a source table, boots Fluentd wiring
+# in_mysql_replicator_multi -> out_mysql_replicator_elasticsearch, and asserts
+# that INSERT / UPDATE / DELETE propagate to Elasticsearch AND that the
+# hash_tables state is maintained.
+#
+# Delete detection in the multi plugin is gap-based against hash_tables, so the
+# test seeds three rows and deletes the *middle* one (id=2) to exercise it.
+require_relative 'e2e_helper'
+include E2E
+
+CONF      = File.join(ROOT, 'test', 'e2e', 'fluent_multi.conf')
+LOG_PATH  = File.join(ROOT, 'test', 'e2e', 'fluentd_multi.log')
+SETUP_SQL = File.join(ROOT, 'setup_mysql_replicator_multi.sql')
+
+SOURCE_DB    = ENV['MYSQL_DATABASE'] || 'e2e_source'
+MANAGER_DB   = ENV['MYSQL_MANAGER_DATABASE'] || 'replicator_manager'
+SETTING_NAME = 'users_to_es'
+INDEX = 'multiindex'
+TYPE  = 'multitype'
+
+def hash_table_count(client, pk: nil)
+  where = "setting_name = '#{SETTING_NAME}'"
+  where += " AND setting_query_pk = #{pk}" if pk
+  client.query("SELECT COUNT(*) AS c FROM `#{MANAGER_DB}`.hash_tables WHERE #{where}").first['c']
+end
+
+# --- 1. Seed the source database --------------------------------------------
+step "seeding MySQL source database '#{SOURCE_DB}'"
+es_delete_index(INDEX)
+client = Mysql2::Client.new(mysql_config)
+client.query("DROP DATABASE IF EXISTS `#{SOURCE_DB}`")
+client.query("CREATE DATABASE `#{SOURCE_DB}`")
+client.query("USE `#{SOURCE_DB}`")
+client.query(<<~SQL)
+  CREATE TABLE users (
+    id   INT NOT NULL AUTO_INCREMENT,
+    name VARCHAR(255) NOT NULL,
+    age  INT NOT NULL,
+    PRIMARY KEY (id)
+  ) ENGINE=InnoDB DEFAULT CHARSET=utf8
+SQL
+client.query("INSERT INTO users (name, age) VALUES ('alice', 20), ('bob', 30), ('carol', 40)")
+
+# --- 2. Build the management database and register a replication setting ----
+step "building management database '#{MANAGER_DB}' from setup SQL"
+client.query("DROP DATABASE IF EXISTS `#{MANAGER_DB}`")
+File.read(SETUP_SQL).split(/;\s*$/).map(&:strip).reject(&:empty?).each do |stmt|
+  client.query(stmt)
+end
+
+step "registering settings row '#{SETTING_NAME}'"
+cfg = mysql_config
+client.query(<<~SQL)
+  INSERT INTO `#{MANAGER_DB}`.settings
+    (is_active, name, host, port, username, password, `database`,
+     query, prepared_query, `interval`, primary_key, enable_delete)
+  VALUES
+    (1, '#{SETTING_NAME}', '#{cfg[:host]}', #{cfg[:port]},
+     '#{cfg[:username]}', '#{client.escape(cfg[:password].to_s)}', '#{SOURCE_DB}',
+     'SELECT id, name, age FROM users ORDER BY id', '', 2, 'id', 1)
+SQL
+
+# --- 3. Boot Fluentd --------------------------------------------------------
+step "starting Fluentd (#{CONF})"
+fluentd_pid = spawn_fluentd(CONF, LOG_PATH)
+
+begin
+  # --- 4. INSERT is replicated to Elasticsearch -----------------------------
+  step "asserting INSERT replication"
+  {1 => 'alice', 2 => 'bob', 3 => 'carol'}.each do |id, name|
+    wait_until("user #{id} (#{name}) indexed in Elasticsearch", log_path: LOG_PATH) do
+      code, body = es_get(INDEX, TYPE, id)
+      code == 200 && body.dig('_source', 'name') == name
+    end
+  end
+  step "  INSERT OK"
+
+  # --- 5. hash_tables state is persisted ------------------------------------
+  step "asserting hash_tables persistence"
+  wait_until("3 rows recorded in hash_tables for '#{SETTING_NAME}'", log_path: LOG_PATH) do
+    hash_table_count(client) == 3
+  end
+  step "  hash_tables OK"
+
+  # --- 6. UPDATE is replicated ----------------------------------------------
+  step "asserting UPDATE replication"
+  client.query("UPDATE `#{SOURCE_DB}`.users SET age = 21 WHERE id = 1")
+  wait_until("user 1 age updated to 21 in Elasticsearch", log_path: LOG_PATH) do
+    code, body = es_get(INDEX, TYPE, 1)
+    code == 200 && body.dig('_source', 'age') == 21
+  end
+  step "  UPDATE OK"
+
+  # --- 7. DELETE (middle id) is replicated ----------------------------------
+  step "asserting DELETE replication (middle id=2)"
+  client.query("DELETE FROM `#{SOURCE_DB}`.users WHERE id = 2")
+  wait_until("user 2 removed from Elasticsearch", log_path: LOG_PATH) do
+    code, _ = es_get(INDEX, TYPE, 2)
+    code == 404
+  end
+  wait_until("hash_tables entry for id=2 removed", log_path: LOG_PATH) do
+    hash_table_count(client, pk: 2) == 0
+  end
+  # Surviving rows must remain.
+  code1, = es_get(INDEX, TYPE, 1)
+  code3, = es_get(INDEX, TYPE, 3)
+  fail_with("surviving rows were unexpectedly removed (id1=#{code1}, id3=#{code3})", LOG_PATH) unless code1 == 200 && code3 == 200
+  step "  DELETE OK"
+
+  puts "\n[E2E] PASSED (multi): insert/update/delete replicated and hash_tables maintained"
+ensure
+  step "stopping Fluentd"
+  stop_fluentd(fluentd_pid)
+end

data/test/e2e/replication_single_e2e_test.rb

@@ -0,0 +1,74 @@
+#!/usr/bin/env ruby
+# End-to-end test for the single-table input plugin (in_mysql_replicator).
+#
+# It seeds a source table in MySQL, boots a real Fluentd process wiring
+# in_mysql_replicator -> out_mysql_replicator_elasticsearch (fluent_single.conf),
+# and asserts that INSERT / UPDATE / DELETE on MySQL are replicated to
+# Elasticsearch.
+require_relative 'e2e_helper'
+include E2E
+
+CONF     = File.join(ROOT, 'test', 'e2e', 'fluent_single.conf')
+LOG_PATH = File.join(ROOT, 'test', 'e2e', 'fluentd_single.log')
+
+DB    = ENV['MYSQL_DATABASE'] || 'e2e_source'
+INDEX = 'myindex'
+TYPE  = 'mytype'
+
+# --- 1. Seed the source database --------------------------------------------
+step "seeding MySQL source database '#{DB}'"
+es_delete_index(INDEX)
+client = Mysql2::Client.new(mysql_config)
+client.query("DROP DATABASE IF EXISTS `#{DB}`")
+client.query("CREATE DATABASE `#{DB}`")
+client.query("USE `#{DB}`")
+client.query(<<~SQL)
+  CREATE TABLE users (
+    id   INT NOT NULL AUTO_INCREMENT,
+    name VARCHAR(255) NOT NULL,
+    age  INT NOT NULL,
+    PRIMARY KEY (id)
+  ) ENGINE=InnoDB DEFAULT CHARSET=utf8
+SQL
+client.query("INSERT INTO users (name, age) VALUES ('alice', 20), ('bob', 30)")
+
+# --- 2. Boot Fluentd --------------------------------------------------------
+step "starting Fluentd (#{CONF})"
+fluentd_pid = spawn_fluentd(CONF, LOG_PATH)
+
+begin
+  # --- 3. INSERT is replicated ----------------------------------------------
+  step "asserting INSERT replication"
+  wait_until("user 1 (alice) indexed in Elasticsearch", log_path: LOG_PATH) do
+    code, body = es_get(INDEX, TYPE, 1)
+    code == 200 && body.dig('_source', 'name') == 'alice'
+  end
+  wait_until("user 2 (bob) indexed in Elasticsearch", log_path: LOG_PATH) do
+    code, body = es_get(INDEX, TYPE, 2)
+    code == 200 && body.dig('_source', 'name') == 'bob'
+  end
+  step "  INSERT OK"
+
+  # --- 4. UPDATE is replicated ----------------------------------------------
+  step "asserting UPDATE replication"
+  client.query("UPDATE users SET age = 21 WHERE id = 1")
+  wait_until("user 1 age updated to 21 in Elasticsearch", log_path: LOG_PATH) do
+    code, body = es_get(INDEX, TYPE, 1)
+    code == 200 && body.dig('_source', 'age') == 21
+  end
+  step "  UPDATE OK"
+
+  # --- 5. DELETE is replicated ----------------------------------------------
+  step "asserting DELETE replication"
+  client.query("DELETE FROM users WHERE id = 2")
+  wait_until("user 2 removed from Elasticsearch", log_path: LOG_PATH) do
+    code, _ = es_get(INDEX, TYPE, 2)
+    code == 404
+  end
+  step "  DELETE OK"
+
+  puts "\n[E2E] PASSED (single): insert/update/delete replicated MySQL -> Elasticsearch"
+ensure
+  step "stopping Fluentd"
+  stop_fluentd(fluentd_pid)
+end

data/test/plugin/test_out_mysql_replicator_elasticsearch.rb

@@ -21,7 +21,18 @@ class MysqlReplicatorElasticsearchOutput < Test::Unit::TestCase
     {'age' => 26, 'request_id' => '42'}
   end
 
+  # The plugin detects the Elasticsearch version on the first write via GET /,
+  # so stub that endpoint (defaulting to 6.x, which keeps "_type").
+  def stub_elastic_version(url, version="6.8.23")
+    stub_request(:get, url).to_return(
+      :status => 200,
+      :headers => {"Content-Type" => "application/json"},
+      :body => %({"version":{"number":"#{version}"}})
+    )
+  end
+
   def stub_elastic(url="http://localhost:9200/_bulk")
+    stub_elastic_version(url.sub('/_bulk', '/'))
     stub_request(:post, url).with do |req|
       @content_type = req.headers["Content-Type"]
       @index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
@@ -29,6 +40,7 @@ class MysqlReplicatorElasticsearchOutput < Test::Unit::TestCase
   end
 
   def stub_elastic_unavailable(url="http://localhost:9200/_bulk")
+    stub_elastic_version(url.sub('/_bulk', '/'))
     stub_request(:post, url).to_return(:status => [503, "Service Unavailable"])
   end
 
@@ -58,6 +70,16 @@ class MysqlReplicatorElasticsearchOutput < Test::Unit::TestCase
     assert_equal('mytype', index_cmds.first['index']['_type'])
   end
 
+  def test_auto_detects_es8_and_omits_type
+    stub_elastic
+    # Override the version endpoint to report Elasticsearch 8.x.
+    stub_elastic_version("http://localhost:9200/", "8.18.0")
+    driver.run(default_tag: @tag) do
+      driver.feed(sample_record)
+    end
+    assert(!index_cmds.first['index'].has_key?('_type'))
+  end
+
   def test_writes_to_speficied_host
     driver.configure("host 192.168.33.50\n")
     elastic_request = stub_elastic("http://192.168.33.50:9200/_bulk")

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mysql-replicator
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.1.0
 platform: ruby
 authors:
 - Kentaro Yoshida
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-10-27 00:00:00.000000000 Z
+date: 2026-06-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -107,8 +107,11 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".devcontainer/Dockerfile"
+- ".devcontainer/devcontainer.json"
+- ".dockerignore"
+- ".github/workflows/ci.yml"
 - ".gitignore"
-- ".travis.yml"
 - Gemfile
 - LICENSE
 - README.md
@@ -125,6 +128,11 @@ files:
 - lib/fluent/plugin/out_mysql_replicator_elasticsearch.rb
 - lib/fluent/plugin/out_mysql_replicator_solr.rb
 - setup_mysql_replicator_multi.sql
+- test/e2e/e2e_helper.rb
+- test/e2e/fluent_multi.conf
+- test/e2e/fluent_single.conf
+- test/e2e/replication_multi_e2e_test.rb
+- test/e2e/replication_single_e2e_test.rb
 - test/helper.rb
 - test/plugin/test_in_mysql_replicator.rb
 - test/plugin/test_in_mysql_replicator_multi.rb
@@ -149,13 +157,18 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.0.3.1
 signing_key: 
 specification_version: 4
 summary: Fluentd input plugin to track insert/update/delete event from MySQL database
   server. Not only that, it could multiple table replication and generate nested document
   for Elasticsearch/Solr. It's comming support replicate to another RDB/noSQL.
 test_files:
+- test/e2e/e2e_helper.rb
+- test/e2e/fluent_multi.conf
+- test/e2e/fluent_single.conf
+- test/e2e/replication_multi_e2e_test.rb
+- test/e2e/replication_single_e2e_test.rb
 - test/helper.rb
 - test/plugin/test_in_mysql_replicator.rb
 - test/plugin/test_in_mysql_replicator_multi.rb

data/.travis.yml

@@ -1,18 +0,0 @@
-language: ruby
-
-rvm:
-  - 2.4.0
-  - 2.3.3
-  - 2.2
-  - 2.1
-
-services:
-  - mysql
-  - elasticsearch
-
-before_install:
-  - gem update bundler
-
-before_script:
-  - mysql < setup_mysql_replicator_multi.sql
-  - curl https://raw.github.com/moliware/travis-solr/master/travis-solr.sh | SOLR_VERSION=4.6.1 bash