fluent-plugin-mysql-replicator 1.0.3 → 1.2.0

data/lib/fluent/plugin/out_mysql_replicator_elasticsearch.rb

@@ -1,5 +1,6 @@
 require 'net/http'
 require 'date'
+require 'yajl'
 require 'fluent/plugin/output'
 
 class Fluent::Plugin::MysqlReplicatorElasticsearchOutput < Fluent::Plugin::Output
@@ -38,6 +39,8 @@ class Fluent::Plugin::MysqlReplicatorElasticsearchOutput < Fluent::Plugin::Outpu
 
   def start
     super
+    # nil means "not yet detected"; resolved on the first write.
+    @suppress_type = nil
   end
 
   def format(tag, time, record)
@@ -57,6 +60,8 @@ class Fluent::Plugin::MysqlReplicatorElasticsearchOutput < Fluent::Plugin::Outpu
   end
 
   def write(chunk)
+    detect_type_suppression if @suppress_type.nil?
+
     bulk_message = []
 
     chunk.msgpack_each do |tag, time, record|
@@ -66,28 +71,60 @@ class Fluent::Plugin::MysqlReplicatorElasticsearchOutput < Fluent::Plugin::Outpu
       id_key = tag_parts['primary_key']
 
       if tag_parts['event'] == 'delete'
-        meta = { "delete" => {"_index" => target_index, "_type" => target_type, "_id" => record[id_key]} }
+        action = {"_index" => target_index, "_id" => record[id_key]}
+        action['_type'] = target_type unless @suppress_type
+        meta = { "delete" => action }
         bulk_message << Yajl::Encoder.encode(meta)
       else
-        meta = { "index" => {"_index" => target_index, "_type" => target_type} }
+        action = {"_index" => target_index}
+        action['_type'] = target_type unless @suppress_type
         if id_key && record[id_key]
-          meta['index']['_id'] = record[id_key]
+          action['_id'] = record[id_key]
         end
+        meta = { "index" => action }
         bulk_message << Yajl::Encoder.encode(meta)
         bulk_message << Yajl::Encoder.encode(record)
       end
     end
     bulk_message << ""
 
-    http = Net::HTTP.new(@host, @port.to_i)
-    http.use_ssl = @ssl
-
     request = Net::HTTP::Post.new('/_bulk', {'content-type' => 'application/json; charset=utf-8'})
     if @username && @password
       request.basic_auth(@username, @password)
     end
 
     request.body = bulk_message.join("\n")
-    http.request(request).value
+    new_http.request(request).value
+  end
+
+  private
+
+  def new_http
+    http = Net::HTTP.new(@host, @port.to_i)
+    http.use_ssl = @ssl
+    http
+  end
+
+  # Mapping types were removed in Elasticsearch 8.x and deprecated in 7.x.
+  # Detect the major version once and omit "_type" for 7.x and later.
+  def detect_type_suppression
+    major = elasticsearch_major_version
+    @suppress_type = !major.nil? && major >= 7
+    if major
+      log.info "mysql_replicator_elasticsearch: detected Elasticsearch #{major}.x, suppress_type=#{@suppress_type}"
+    else
+      log.warn "mysql_replicator_elasticsearch: could not detect Elasticsearch version, sending '_type' (assuming 6.x)"
+    end
+  end
+
+  def elasticsearch_major_version
+    request = Net::HTTP::Get.new('/')
+    request.basic_auth(@username, @password) if @username && @password
+    response = new_http.request(request)
+    number = Yajl::Parser.parse(response.body).dig('version', 'number')
+    number.to_s.split('.').first.to_i
+  rescue => e
+    log.warn "mysql_replicator_elasticsearch: version detection failed: #{e.message}"
+    nil
   end
 end

data/setup_mysql_replicator_multi.sql

@@ -24,6 +24,9 @@ CREATE TABLE IF NOT EXISTS `settings` (
   `prepared_query` TEXT NOT NULL,
   `interval` int(11) NOT NULL,
   `primary_key` varchar(255) DEFAULT 'id',
+  -- Comma-separated column names whose MySQL JSON values should be parsed into nested objects.
+  -- Intended for Elasticsearch; leave empty for Solr or destinations that cannot store JSON objects.
+  `json_columns` varchar(255) DEFAULT NULL,
   `enable_delete` int(11) DEFAULT '1',
   -- On enabling 'enable_loose_insert: 1', make it faster synchronization to skip checking hash_tables.
   `enable_loose_insert` int(11) DEFAULT '0',

data/test/e2e/e2e_helper.rb

@@ -0,0 +1,94 @@
+# Shared helpers for the end-to-end replication tests.
+#
+# Connection settings come from environment variables (with local defaults):
+#   MYSQL_HOST MYSQL_PORT MYSQL_USER MYSQL_PASSWORD
+#   ES_HOST ES_PORT
+require 'mysql2'
+require 'net/http'
+require 'json'
+
+module E2E
+  module_function
+
+  ROOT = File.expand_path('../..', __dir__)
+
+  def mysql_config(database: nil)
+    cfg = {
+      host:     ENV['MYSQL_HOST'] || '127.0.0.1',
+      port:     (ENV['MYSQL_PORT'] || 3306).to_i,
+      username: ENV['MYSQL_USER'] || 'root',
+      password: ENV['MYSQL_PASSWORD'] || 'root',
+    }
+    cfg[:database] = database if database
+    cfg
+  end
+
+  def es_base
+    "http://#{ENV['ES_HOST'] || '127.0.0.1'}:#{ENV['ES_PORT'] || '9200'}"
+  end
+
+  # Major version of the Elasticsearch under test (defaults to 6).
+  def es_major_version
+    (ENV['ES_MAJOR_VERSION'] || '6').to_i
+  end
+
+  # Realtime GET by id. Returns [http_status, parsed_body].
+  # Elasticsearch 7.x+ dropped custom mapping types, so documents are addressed
+  # via the "_doc" endpoint instead of the original type name.
+  def es_get(index, type, id)
+    type_path = es_major_version >= 7 ? '_doc' : type
+    res = Net::HTTP.get_response(URI("#{es_base}/#{index}/#{type_path}/#{id}"))
+    [res.code.to_i, (JSON.parse(res.body) rescue {})]
+  end
+
+  # Drop an index so a test starts from a clean slate (ignores "not found").
+  def es_delete_index(index)
+    uri = URI("#{es_base}/#{index}")
+    Net::HTTP.start(uri.host, uri.port) { |h| h.request(Net::HTTP::Delete.new(uri)) }
+  rescue StandardError
+    # ignore: the index may not exist yet
+  end
+
+  # Poll a condition until it becomes truthy, otherwise fail with a timeout.
+  def wait_until(description, timeout: 90, log_path: nil)
+    deadline = Time.now + timeout
+    loop do
+      return if yield
+      fail_with("timeout (#{timeout}s) waiting for: #{description}", log_path) if Time.now > deadline
+      sleep 1
+    end
+  end
+
+  def fail_with(message, log_path = nil)
+    warn "\n[E2E] FAILED: #{message}"
+    if log_path && File.exist?(log_path)
+      warn "\n----- fluentd.log (tail) -----"
+      warn File.readlines(log_path).last(60).join
+      warn "------------------------------"
+    end
+    exit 1
+  end
+
+  def step(message)
+    puts "[E2E] #{message}"
+  end
+
+  # Boot fluentd as a child process. Returns the pid; logs go to log_path.
+  def spawn_fluentd(conf, log_path)
+    log = File.open(log_path, 'w')
+    Process.spawn(
+      'bundle', 'exec', 'fluentd',
+      '-c', conf,
+      '-p', File.join(ROOT, 'lib', 'fluent', 'plugin'),
+      '--no-supervisor',
+      out: log, err: log, chdir: ROOT
+    )
+  end
+
+  def stop_fluentd(pid)
+    Process.kill('TERM', pid)
+    Process.wait(pid)
+  rescue Errno::ESRCH, Errno::ECHILD
+    # already gone
+  end
+end

data/test/e2e/fluent_multi.conf

@@ -0,0 +1,30 @@
+# E2E pipeline for the multi-table input plugin.
+# in_mysql_replicator_multi reads replication settings from the management
+# database (replicator_manager.settings) and persists per-row hashes into
+# replicator_manager.hash_tables. See test/e2e/replication_multi_e2e_test.rb.
+#
+# bulk_insert_count is set to 1 so hash_tables is flushed immediately on each
+# insert, which keeps the test timing deterministic.
+<source>
+  @type mysql_replicator_multi
+  manager_host "#{ENV['MYSQL_HOST'] || '127.0.0.1'}"
+  manager_port "#{ENV['MYSQL_PORT'] || '3306'}"
+  manager_username "#{ENV['MYSQL_USER'] || 'root'}"
+  manager_password "#{ENV['MYSQL_PASSWORD'] || 'root'}"
+  manager_database "#{ENV['MYSQL_MANAGER_DATABASE'] || 'replicator_manager'}"
+  tag multiindex.multitype.${event}.${primary_key}
+  bulk_insert_count 1
+  bulk_insert_timeout 2
+</source>
+
+<match multiindex.**>
+  @type mysql_replicator_elasticsearch
+  host "#{ENV['ES_HOST'] || '127.0.0.1'}"
+  port "#{ENV['ES_PORT'] || '9200'}"
+  <buffer>
+    @type memory
+    flush_mode interval
+    flush_interval 1s
+    flush_thread_count 1
+  </buffer>
+</match>

data/test/e2e/fluent_single.conf

@@ -0,0 +1,29 @@
+# E2E pipeline: MySQL (in_mysql_replicator) -> Elasticsearch (out_mysql_replicator_elasticsearch)
+# Connection details are injected via environment variables so the same config
+# works both in CI and locally. See test/e2e/replication_e2e_test.rb.
+<source>
+  @type mysql_replicator
+  host "#{ENV['MYSQL_HOST'] || '127.0.0.1'}"
+  port "#{ENV['MYSQL_PORT'] || '3306'}"
+  username "#{ENV['MYSQL_USER'] || 'root'}"
+  password "#{ENV['MYSQL_PASSWORD'] || 'root'}"
+  database "#{ENV['MYSQL_DATABASE'] || 'e2e_source'}"
+  query SELECT id, name, age, profile FROM users
+  primary_key id
+  json_columns profile
+  interval 2s
+  enable_delete true
+  tag myindex.mytype.${event}.${primary_key}
+</source>
+
+<match myindex.**>
+  @type mysql_replicator_elasticsearch
+  host "#{ENV['ES_HOST'] || '127.0.0.1'}"
+  port "#{ENV['ES_PORT'] || '9200'}"
+  <buffer>
+    @type memory
+    flush_mode interval
+    flush_interval 1s
+    flush_thread_count 1
+  </buffer>
+</match>

data/test/e2e/replication_multi_e2e_test.rb

@@ -0,0 +1,140 @@
+#!/usr/bin/env ruby
+# End-to-end test for the multi-table input plugin (in_mysql_replicator_multi).
+#
+# Unlike the single plugin, this one is driven by a management database:
+#   - replicator_manager.settings   : one row per replication job
+#   - replicator_manager.hash_tables: persisted per-row hashes (delete detection)
+#
+# This test builds that management DB from setup_mysql_replicator_multi.sql,
+# registers a settings row pointing at a source table, boots Fluentd wiring
+# in_mysql_replicator_multi -> out_mysql_replicator_elasticsearch, and asserts
+# that INSERT / UPDATE / DELETE propagate to Elasticsearch AND that the
+# hash_tables state is maintained.
+#
+# Delete detection in the multi plugin is gap-based against hash_tables, so the
+# test seeds three rows and deletes the *middle* one (id=2) to exercise it.
+require_relative 'e2e_helper'
+include E2E
+
+CONF      = File.join(ROOT, 'test', 'e2e', 'fluent_multi.conf')
+LOG_PATH  = File.join(ROOT, 'test', 'e2e', 'fluentd_multi.log')
+SETUP_SQL = File.join(ROOT, 'setup_mysql_replicator_multi.sql')
+
+SOURCE_DB    = ENV['MYSQL_DATABASE'] || 'e2e_source'
+MANAGER_DB   = ENV['MYSQL_MANAGER_DATABASE'] || 'replicator_manager'
+SETTING_NAME = 'users_to_es'
+INDEX = 'multiindex'
+TYPE  = 'multitype'
+
+def hash_table_count(client, pk: nil)
+  where = "setting_name = '#{SETTING_NAME}'"
+  where += " AND setting_query_pk = #{pk}" if pk
+  client.query("SELECT COUNT(*) AS c FROM `#{MANAGER_DB}`.hash_tables WHERE #{where}").first['c']
+end
+
+# --- 1. Seed the source database --------------------------------------------
+step "seeding MySQL source database '#{SOURCE_DB}'"
+es_delete_index(INDEX)
+client = Mysql2::Client.new(mysql_config)
+client.query("DROP DATABASE IF EXISTS `#{SOURCE_DB}`")
+client.query("CREATE DATABASE `#{SOURCE_DB}`")
+client.query("USE `#{SOURCE_DB}`")
+client.query(<<~SQL)
+  CREATE TABLE users (
+    id      INT NOT NULL AUTO_INCREMENT,
+    name    VARCHAR(255) NOT NULL,
+    age     INT NOT NULL,
+    profile JSON,
+    PRIMARY KEY (id)
+  ) ENGINE=InnoDB DEFAULT CHARSET=utf8
+SQL
+client.query(<<~SQL)
+  INSERT INTO users (name, age, profile) VALUES
+    ('alice', 20, '{"city":"Tokyo","tags":["a","b"]}'),
+    ('bob',   30, '{"city":"Osaka","tags":["c"]}'),
+    ('carol', 40, '{"city":"Nagoya","tags":[]}')
+SQL
+
+# --- 2. Build the management database and register a replication setting ----
+step "building management database '#{MANAGER_DB}' from setup SQL"
+client.query("DROP DATABASE IF EXISTS `#{MANAGER_DB}`")
+File.read(SETUP_SQL).split(/;\s*$/).map(&:strip).reject(&:empty?).each do |stmt|
+  client.query(stmt)
+end
+
+step "registering settings row '#{SETTING_NAME}'"
+cfg = mysql_config
+client.query(<<~SQL)
+  INSERT INTO `#{MANAGER_DB}`.settings
+    (is_active, name, host, port, username, password, `database`,
+     query, prepared_query, `interval`, primary_key, enable_delete, json_columns)
+  VALUES
+    (1, '#{SETTING_NAME}', '#{cfg[:host]}', #{cfg[:port]},
+     '#{cfg[:username]}', '#{client.escape(cfg[:password].to_s)}', '#{SOURCE_DB}',
+     'SELECT id, name, age, profile FROM users ORDER BY id', '', 2, 'id', 1, 'profile')
+SQL
+
+# --- 3. Boot Fluentd --------------------------------------------------------
+step "starting Fluentd (#{CONF})"
+fluentd_pid = spawn_fluentd(CONF, LOG_PATH)
+
+begin
+  # --- 4. INSERT is replicated to Elasticsearch -----------------------------
+  step "asserting INSERT replication"
+  {1 => 'alice', 2 => 'bob', 3 => 'carol'}.each do |id, name|
+    wait_until("user #{id} (#{name}) indexed in Elasticsearch", log_path: LOG_PATH) do
+      code, body = es_get(INDEX, TYPE, id)
+      code == 200 && body.dig('_source', 'name') == name
+    end
+  end
+  step "  INSERT OK"
+
+  # --- 4b. JSON column is indexed as a nested object (not an escaped string) -
+  step "asserting JSON column is replicated as a nested object"
+  wait_until("user 1 profile.city == Tokyo in Elasticsearch", log_path: LOG_PATH) do
+    code, body = es_get(INDEX, TYPE, 1)
+    code == 200 && body.dig('_source', 'profile', 'city') == 'Tokyo'
+  end
+  _, body = es_get(INDEX, TYPE, 1)
+  unless body.dig('_source', 'profile', 'tags') == ['a', 'b']
+    fail_with("profile was not indexed as a nested object: #{body.dig('_source', 'profile').inspect}", LOG_PATH)
+  end
+  step "  JSON OK"
+
+  # --- 5. hash_tables state is persisted ------------------------------------
+  step "asserting hash_tables persistence"
+  wait_until("3 rows recorded in hash_tables for '#{SETTING_NAME}'", log_path: LOG_PATH) do
+    hash_table_count(client) == 3
+  end
+  step "  hash_tables OK"
+
+  # --- 6. UPDATE is replicated ----------------------------------------------
+  step "asserting UPDATE replication"
+  client.query("UPDATE `#{SOURCE_DB}`.users SET age = 21 WHERE id = 1")
+  wait_until("user 1 age updated to 21 in Elasticsearch", log_path: LOG_PATH) do
+    code, body = es_get(INDEX, TYPE, 1)
+    code == 200 && body.dig('_source', 'age') == 21
+  end
+  step "  UPDATE OK"
+
+  # --- 7. DELETE (middle id) is replicated ----------------------------------
+  step "asserting DELETE replication (middle id=2)"
+  client.query("DELETE FROM `#{SOURCE_DB}`.users WHERE id = 2")
+  wait_until("user 2 removed from Elasticsearch", log_path: LOG_PATH) do
+    code, _ = es_get(INDEX, TYPE, 2)
+    code == 404
+  end
+  wait_until("hash_tables entry for id=2 removed", log_path: LOG_PATH) do
+    hash_table_count(client, pk: 2) == 0
+  end
+  # Surviving rows must remain.
+  code1, = es_get(INDEX, TYPE, 1)
+  code3, = es_get(INDEX, TYPE, 3)
+  fail_with("surviving rows were unexpectedly removed (id1=#{code1}, id3=#{code3})", LOG_PATH) unless code1 == 200 && code3 == 200
+  step "  DELETE OK"
+
+  puts "\n[E2E] PASSED (multi): insert/update/delete replicated and hash_tables maintained"
+ensure
+  step "stopping Fluentd"
+  stop_fluentd(fluentd_pid)
+end

data/test/e2e/replication_single_e2e_test.rb

@@ -0,0 +1,91 @@
+#!/usr/bin/env ruby
+# End-to-end test for the single-table input plugin (in_mysql_replicator).
+#
+# It seeds a source table in MySQL, boots a real Fluentd process wiring
+# in_mysql_replicator -> out_mysql_replicator_elasticsearch (fluent_single.conf),
+# and asserts that INSERT / UPDATE / DELETE on MySQL are replicated to
+# Elasticsearch.
+require_relative 'e2e_helper'
+include E2E
+
+CONF     = File.join(ROOT, 'test', 'e2e', 'fluent_single.conf')
+LOG_PATH = File.join(ROOT, 'test', 'e2e', 'fluentd_single.log')
+
+DB    = ENV['MYSQL_DATABASE'] || 'e2e_source'
+INDEX = 'myindex'
+TYPE  = 'mytype'
+
+# --- 1. Seed the source database --------------------------------------------
+step "seeding MySQL source database '#{DB}'"
+es_delete_index(INDEX)
+client = Mysql2::Client.new(mysql_config)
+client.query("DROP DATABASE IF EXISTS `#{DB}`")
+client.query("CREATE DATABASE `#{DB}`")
+client.query("USE `#{DB}`")
+client.query(<<~SQL)
+  CREATE TABLE users (
+    id      INT NOT NULL AUTO_INCREMENT,
+    name    VARCHAR(255) NOT NULL,
+    age     INT NOT NULL,
+    profile JSON,
+    PRIMARY KEY (id)
+  ) ENGINE=InnoDB DEFAULT CHARSET=utf8
+SQL
+client.query(<<~SQL)
+  INSERT INTO users (name, age, profile) VALUES
+    ('alice', 20, '{"city":"Tokyo","tags":["a","b"]}'),
+    ('bob',   30, '{"city":"Osaka","tags":["c"]}')
+SQL
+
+# --- 2. Boot Fluentd --------------------------------------------------------
+step "starting Fluentd (#{CONF})"
+fluentd_pid = spawn_fluentd(CONF, LOG_PATH)
+
+begin
+  # --- 3. INSERT is replicated ----------------------------------------------
+  step "asserting INSERT replication"
+  wait_until("user 1 (alice) indexed in Elasticsearch", log_path: LOG_PATH) do
+    code, body = es_get(INDEX, TYPE, 1)
+    code == 200 && body.dig('_source', 'name') == 'alice'
+  end
+  wait_until("user 2 (bob) indexed in Elasticsearch", log_path: LOG_PATH) do
+    code, body = es_get(INDEX, TYPE, 2)
+    code == 200 && body.dig('_source', 'name') == 'bob'
+  end
+  step "  INSERT OK"
+
+  # --- 3b. JSON column is indexed as a nested object (not an escaped string) -
+  step "asserting JSON column is replicated as a nested object"
+  wait_until("user 1 profile.city == Tokyo in Elasticsearch", log_path: LOG_PATH) do
+    code, body = es_get(INDEX, TYPE, 1)
+    code == 200 && body.dig('_source', 'profile', 'city') == 'Tokyo'
+  end
+  _, body = es_get(INDEX, TYPE, 1)
+  unless body.dig('_source', 'profile', 'tags') == ['a', 'b']
+    fail_with("profile was not indexed as a nested object: #{body.dig('_source', 'profile').inspect}", LOG_PATH)
+  end
+  step "  JSON OK"
+
+  # --- 4. UPDATE is replicated ----------------------------------------------
+  step "asserting UPDATE replication"
+  client.query("UPDATE users SET age = 21 WHERE id = 1")
+  wait_until("user 1 age updated to 21 in Elasticsearch", log_path: LOG_PATH) do
+    code, body = es_get(INDEX, TYPE, 1)
+    code == 200 && body.dig('_source', 'age') == 21
+  end
+  step "  UPDATE OK"
+
+  # --- 5. DELETE is replicated ----------------------------------------------
+  step "asserting DELETE replication"
+  client.query("DELETE FROM users WHERE id = 2")
+  wait_until("user 2 removed from Elasticsearch", log_path: LOG_PATH) do
+    code, _ = es_get(INDEX, TYPE, 2)
+    code == 404
+  end
+  step "  DELETE OK"
+
+  puts "\n[E2E] PASSED (single): insert/update/delete replicated MySQL -> Elasticsearch"
+ensure
+  step "stopping Fluentd"
+  stop_fluentd(fluentd_pid)
+end

data/test/plugin/test_in_mysql_replicator.rb

@@ -30,11 +30,111 @@ class MysqlReplicatorInputTest < Test::Unit::TestCase
       tag             input.mysql
       query           SELECT id, text from search_text
       enable_delete   no
+      json_columns    geometry,attrs
     ]
     assert_equal 'localhost', d.instance.host
     assert_equal 3306, d.instance.port
     assert_equal 30, d.instance.interval
     assert_equal 'input.mysql', d.instance.tag
     assert_equal false, d.instance.enable_delete
+    assert_equal ['geometry', 'attrs'], d.instance.json_columns
+  end
+
+  def test_json_columns_defaults_to_empty
+    d = create_driver
+    assert_equal [], d.instance.json_columns
+  end
+
+  def test_parse_json_columns
+    d = create_driver
+    row = {
+      'id'       => 1,
+      'geometry' => '{"type":"Polygon","coordinates":[[136.8,35.1]]}',
+      'tags'     => '[1,2,3]',
+      'broken'   => 'not json {',
+      'plain'    => 'hello',
+      'number'   => 5,
+    }
+    d.instance.parse_json_columns!(row, ['geometry', 'tags', 'broken', 'number', 'missing'])
+
+    assert_equal({'type' => 'Polygon', 'coordinates' => [[136.8, 35.1]]}, row['geometry'])
+    assert_equal([1, 2, 3], row['tags'])
+    assert_equal('not json {', row['broken'])  # malformed JSON stays as the original string
+    assert_equal(5, row['number'])             # non-string values are untouched
+    assert_equal('hello', row['plain'])        # columns not listed are untouched
+  end
+
+  def test_parse_json_columns_noop_when_empty
+    d = create_driver
+    row = {'geometry' => '{"a":1}'}
+    d.instance.parse_json_columns!(row, [])
+    assert_equal('{"a":1}', row['geometry'])
+  end
+
+  # --- #42: delete detection must not build an integer Range from primary keys ---
+
+  def deleted_ids(previous_ids, current_ids)
+    conf = %[
+      tag   input.mysql
+      query SELECT id, text from search_test
+    ]
+    create_driver(conf).instance.detect_deleted_ids(previous_ids, current_ids)
+  end
+
+  def test_detect_deleted_ids_first_poll_reports_no_deletes
+    assert_equal [], deleted_ids([], [1, 2, 3])
+  end
+
+  def test_detect_deleted_ids_first_poll_with_string_keys_does_not_raise
+    # Regression for #42: the old `[*1...'c']` raised "bad value for range".
+    assert_nothing_raised do
+      assert_equal [], deleted_ids([], %w[a b c])
+    end
+  end
+
+  def test_detect_deleted_ids_first_poll_with_sparse_ids_has_no_phantom_deletes
+    # The old code returned `[*1...99] - [1, 50, 99]`, emitting deletes for ids
+    # that never existed. The first poll must only establish a baseline.
+    assert_equal [], deleted_ids([], [1, 50, 99])
+  end
+
+  def test_detect_deleted_ids_diffs_against_previous_snapshot
+    assert_equal [2], deleted_ids([1, 2, 3], [1, 3])
+  end
+
+  def test_detect_deleted_ids_empty_current_does_not_mass_delete
+    assert_equal [], deleted_ids([1, 2, 3], [])
+  end
+
+  # --- #4: a nested sub-query fires only for a SELECT template with ${...} ---
+
+  def nested?(value)
+    conf = %[
+      tag   input.mysql
+      query SELECT id, text from search_test
+    ]
+    create_driver(conf).instance.nested_query_value?(value)
+  end
+
+  def test_nested_query_value_true_for_select_with_placeholder
+    assert_true nested?("SELECT * FROM child WHERE parent_id = ${id}")
+  end
+
+  def test_nested_query_value_false_for_plain_text_starting_with_select
+    # Regression for #4: a data value beginning with "SELECT" must not run as SQL.
+    assert_false nested?("SELECT YOUR PLAN")
+  end
+
+  def test_nested_query_value_false_for_word_select
+    assert_false nested?("Selecting the best option")
+  end
+
+  def test_nested_query_value_false_for_select_without_placeholder
+    assert_false nested?("select count(*) from search_test")
+  end
+
+  def test_nested_query_value_false_for_non_string_values
+    assert_false nested?(12345)
+    assert_false nested?(nil)
   end
 end

data/test/plugin/test_out_mysql_replicator_elasticsearch.rb

@@ -21,7 +21,18 @@ class MysqlReplicatorElasticsearchOutput < Test::Unit::TestCase
     {'age' => 26, 'request_id' => '42'}
   end
 
+  # The plugin detects the Elasticsearch version on the first write via GET /,
+  # so stub that endpoint (defaulting to 6.x, which keeps "_type").
+  def stub_elastic_version(url, version="6.8.23")
+    stub_request(:get, url).to_return(
+      :status => 200,
+      :headers => {"Content-Type" => "application/json"},
+      :body => %({"version":{"number":"#{version}"}})
+    )
+  end
+
   def stub_elastic(url="http://localhost:9200/_bulk")
+    stub_elastic_version(url.sub('/_bulk', '/'))
     stub_request(:post, url).with do |req|
       @content_type = req.headers["Content-Type"]
       @index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
@@ -29,6 +40,7 @@ class MysqlReplicatorElasticsearchOutput < Test::Unit::TestCase
   end
 
   def stub_elastic_unavailable(url="http://localhost:9200/_bulk")
+    stub_elastic_version(url.sub('/_bulk', '/'))
     stub_request(:post, url).to_return(:status => [503, "Service Unavailable"])
   end
 
@@ -58,6 +70,16 @@ class MysqlReplicatorElasticsearchOutput < Test::Unit::TestCase
     assert_equal('mytype', index_cmds.first['index']['_type'])
   end
 
+  def test_auto_detects_es8_and_omits_type
+    stub_elastic
+    # Override the version endpoint to report Elasticsearch 8.x.
+    stub_elastic_version("http://localhost:9200/", "8.18.0")
+    driver.run(default_tag: @tag) do
+      driver.feed(sample_record)
+    end
+    assert(!index_cmds.first['index'].has_key?('_type'))
+  end
+
   def test_writes_to_speficied_host
     driver.configure("host 192.168.33.50\n")
     elastic_request = stub_elastic("http://192.168.33.50:9200/_bulk")