fluent-plugin-mysql-replicator 1.1.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 16c6600b210b85bad55965f707eadd53017558fd8a6a2b55cdf28bc1e86b7be3
-  data.tar.gz: 07d8b7367ee234afb128845fb8ef9169929cb18caf4c1813040efe951e72f68e
+  metadata.gz: ea8d93c147df39ff2c34ac573a5e981c1e0a46da8445395eaf38cdc44abf7c0e
+  data.tar.gz: f190b88e93c4318b9c2ec42863186f7dc1f0a3a911bb60e2b689a84687bffdcd
 SHA512:
-  metadata.gz: 65ec0c92b273597f7f7ff976fc42878e4fa413d8471947125a6873d7209142bec87a59aa37e3c491fd1a81dea4a9d448e3280cbca1eafd381fdee241d723bcae
-  data.tar.gz: 20fb6ffe6e217e5d8ee0108c5243fad7f6abd910f1a2d336f65bcb8ba0388ac22e329d408edaa5ad8f298c959cffc2656932d71302d6b2b38c21c28cab14585d
+  metadata.gz: bfbf04041525c768d56c7b0bb3d2cd69ca23076eca57d162277d81a06c1cb4290e4b0d402e46e4b136ebf2d45b8fdca59555c1406c9763336b67f3cf2b4be35d
+  data.tar.gz: 2872f2e0d15084d3680a77498411aa5165acb000b2b5744be19686a2dcd89b41ccee8169956203bb3b9c9a004b0aba6d16d741147215b518e0690889b220ab98

data/.devcontainer/Dockerfile

@@ -1,4 +1,4 @@
-FROM ruby:3.3-slim
+FROM ruby:3.4-slim
 
 ENV DEBIAN_FRONTEND=noninteractive
 ENV BUNDLE_PATH=/workspace/vendor/bundle

data/.github/workflows/ci.yml

@@ -100,3 +100,66 @@ jobs:
 
       - name: Run multi-table replication E2E test
         run: bundle exec ruby test/e2e/replication_multi_e2e_test.rb
+
+  # Container path: the official fluent/fluentd Docker image (gem-based).
+  install-smoke-docker:
+    name: Install smoke (Docker fluent/fluentd ${{ matrix.fluentd_tag }})
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        # "edge" tracks the latest Fluentd release to catch drift early. The
+        # packaged-distribution path is covered by install-smoke-fluent-package.
+        fluentd_tag: ['edge-debian']
+
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.4'
+
+      - name: Build gem
+        run: |
+          mkdir -p pkg
+          gem build fluent-plugin-mysql-replicator.gemspec -o pkg/fluent-plugin-mysql-replicator.gem
+
+      - name: Install on official fluentd image (reproduces README steps)
+        run: |
+          docker run --rm --user root --entrypoint sh \
+            -v "${{ github.workspace }}:/workspace:ro" \
+            "fluent/fluentd:${{ matrix.fluentd_tag }}" \
+            /workspace/ci/install_smoke.sh
+
+  # Package path: fluent-package, the LTS distribution that replaced td-agent
+  # and is what most production users run today. v6 LTS bundles Fluentd v1.19.
+  install-smoke-fluent-package:
+    name: Install smoke (fluent-package v6 LTS)
+    runs-on: ubuntu-latest  # GitHub's ubuntu-latest is Ubuntu 24.04 (Noble)
+
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.4'
+
+      - name: Build gem
+        run: |
+          mkdir -p pkg
+          gem build fluent-plugin-mysql-replicator.gemspec -o pkg/fluent-plugin-mysql-replicator.gem
+
+      - name: Install fluent-package v6 LTS
+        run: |
+          curl -fsSL https://fluentd.cdn.cncf.io/sh/install-ubuntu-noble-fluent-package6-lts.sh | sh
+          /opt/fluent/bin/fluentd --version
+
+      - name: Install MySQL client headers
+        run: sudo apt-get update && sudo apt-get install -y build-essential default-libmysqlclient-dev
+
+      - name: Install plugin via fluent-gem and verify it loads
+        run: |
+          sudo /opt/fluent/bin/fluent-gem install pkg/fluent-plugin-mysql-replicator.gem
+          sudo /opt/fluent/bin/fluent-gem list fluent-plugin-mysql-replicator
+          /opt/fluent/bin/ruby ci/verify_plugins_load.rb

data/CHANGELOG.md

@@ -0,0 +1,90 @@
+# Changelog
+
+All notable changes to this project are documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [1.2.0] - 2026-06-16
+
+### Added
+- JSON column support for nested object indexing. The new opt-in `json_columns`
+  option parses MySQL `JSON` string columns into nested objects before they are
+  emitted, so Elasticsearch indexes them as real nested JSON instead of escaped
+  strings. Available on both `mysql_replicator` (config option) and
+  `mysql_replicator_multi` (new `json_columns` column on the `settings` table).
+  Malformed JSON and non-string values are left untouched. ([#48], implements [#18])
+
+### Fixed
+- Delete detection no longer raises `bad value for range` (and no longer
+  allocates a huge array or emits phantom delete events) when the `primary_key`
+  is non-integer (e.g. a UUID `CHAR(36)`) or a large/sparse integer. The first
+  poll now only establishes a baseline. ([#42])
+- In `mysql_replicator`, a nested sub-query is now triggered only when a column
+  value is an actual query template containing a `${placeholder}`. Previously any
+  value that merely began with the word `SELECT` was executed as SQL. This aligns
+  the single plugin with the fix already present in `mysql_replicator_multi`. ([#4])
+
+### Documentation
+- Document that the `mysql2` native extension requires the MySQL client
+  development headers (`default-libmysqlclient-dev` / `mysql-devel`), add a
+  "Failed to build gem native extension" troubleshooting note, and clarify that
+  change detection uses an in-memory hash of every row (not an `updated_at`
+  column). ([#43], [#40])
+
+### CI
+- Add install-smoke jobs that build and install the gem on **fluent-package v6
+  LTS** (the td-agent successor) and on the official `fluent/fluentd` image,
+  guarding against native-extension build regressions.
+
+## [1.1.0] - 2026-06-15
+
+### Added
+- Auto-detect the Elasticsearch version on first write and support Elasticsearch
+  6.x through 9.x. The `_type` field is omitted automatically for 7.x and later,
+  where mapping types were removed.
+- Dev Container configuration and end-to-end (E2E) replication tests, including
+  multi-table replication.
+
+### Changed
+- Migrated CI to GitHub Actions (removed Travis CI) and run the unit-test matrix
+  on Ruby 3.2–4.0, aligned with supported Fluentd versions.
+
+## [1.0.3] - 2024-02-28
+
+### Added
+- Reconnection option to recover from database connection errors. ([#45])
+
+## [1.0.2] - 2021-10-27
+
+### Fixed
+- Manage threads correctly in `mysql_replicator_multi`. ([#39])
+
+## [1.0.1] - 2019-11-19
+
+### Fixed
+- Ensure the nested-query database connection (`nest_db`) is always closed.
+
+### Documentation
+- Documentation updates.
+
+## [1.0.0] - 2019-11-13
+
+- First 1.0 release, targeting the Fluentd v0.14+ plugin API. Earlier 0.x
+  history is available in the git log.
+
+[1.2.0]: https://github.com/y-ken/fluent-plugin-mysql-replicator/compare/v1.1.0...v1.2.0
+[1.1.0]: https://github.com/y-ken/fluent-plugin-mysql-replicator/compare/v1.0.3...v1.1.0
+[1.0.3]: https://github.com/y-ken/fluent-plugin-mysql-replicator/compare/v1.0.2...v1.0.3
+[1.0.2]: https://github.com/y-ken/fluent-plugin-mysql-replicator/compare/v1.0.1...v1.0.2
+[1.0.1]: https://github.com/y-ken/fluent-plugin-mysql-replicator/compare/v1.0.0...v1.0.1
+[1.0.0]: https://github.com/y-ken/fluent-plugin-mysql-replicator/releases/tag/v1.0.0
+
+[#4]: https://github.com/y-ken/fluent-plugin-mysql-replicator/issues/4
+[#18]: https://github.com/y-ken/fluent-plugin-mysql-replicator/pull/18
+[#39]: https://github.com/y-ken/fluent-plugin-mysql-replicator/pull/39
+[#40]: https://github.com/y-ken/fluent-plugin-mysql-replicator/issues/40
+[#42]: https://github.com/y-ken/fluent-plugin-mysql-replicator/issues/42
+[#43]: https://github.com/y-ken/fluent-plugin-mysql-replicator/issues/43
+[#45]: https://github.com/y-ken/fluent-plugin-mysql-replicator/pull/45
+[#48]: https://github.com/y-ken/fluent-plugin-mysql-replicator/pull/48

data/README.md

@@ -15,14 +15,28 @@ It's comming support replicate to another RDB/noSQL.
 
 ## Dependency
 
-before use, install dependent library as:
+This plugin depends on the [mysql2](https://rubygems.org/gems/mysql2) gem, which
+builds a native extension at install time. You need both a C compiler toolchain
+**and the MySQL client development headers** before installing.
 
 ```bash
-# for RHEL/CentOS
+# for RHEL/CentOS/Fedora
 $ sudo yum group install "Development Tools"
+$ sudo yum install mysql-devel        # or mariadb-devel
 
-# for Ubuntu/Debian
-$ sudo apt-get install build-essential
+# for Ubuntu/Debian (including the official fluent/fluentd Docker image)
+$ sudo apt-get install build-essential default-libmysqlclient-dev
+```
+
+### Troubleshooting: "Failed to build gem native extension"
+
+If `gem install` (or `td-agent-gem install`) fails while building the `mysql2`
+native extension, the MySQL client headers are almost always missing. Install
+the development package for your platform shown above, then retry the
+installation. On the official `fluent/fluentd:*-debian` images, run:
+
+```bash
+$ apt-get update && apt-get install -y build-essential default-libmysqlclient-dev
 ```
 
 ## Installation
@@ -74,6 +88,40 @@ Mapping types were removed in Elasticsearch 8.x (and deprecated in 7.x), so the
 Elasticsearch version on the first write and automatically omits `_type` for
 7.x and later, so no extra configuration is required.
 
+## JSON column support
+
+MySQL `JSON` columns (MySQL 5.7.8+ / 8.x) are returned by the driver as plain
+strings, so by default they reach Elasticsearch as escaped strings rather than
+nested objects. List such columns in `json_columns` to have their values parsed
+into nested objects before they are emitted.
+
+* `mysql_replicator` (single): set the `json_columns` option (comma-separated).
+
+  ```
+  <source>
+    @type        mysql_replicator
+    # ...
+    query        SELECT id, name, geometry FROM places
+    json_columns geometry,attrs
+  </source>
+  ```
+
+* `mysql_replicator_multi`: set the `json_columns` column (comma-separated) on
+  the relevant row of the `settings` management table. Existing installs can add
+  the column with:
+
+  ```sql
+  ALTER TABLE settings ADD COLUMN `json_columns` varchar(255) DEFAULT NULL AFTER `primary_key`;
+  ```
+
+Notes:
+
+* This option is intended for Elasticsearch. **Do not set it when the destination
+  cannot store JSON objects (e.g. the Solr output)** — leave it empty there.
+* Only top-level columns are parsed (columns inside nested documents are not).
+* Malformed JSON and non-string values are left untouched, so enabling the option
+  never corrupts non-JSON data.
+
 ## Output example
 
 It is a example when detecting insert/update/delete events.

data/Tutorial-mysql_replicator.md

@@ -8,6 +8,26 @@ It is useful for these purpose.
 **Note:**  
 On syncing 300 million rows table, it will consume around 800MB of memory with ruby 1.9.3 environment.
 
+### How it works
+
+`mysql_replicator` does **not** rely on any `updated_at`/timestamp column to
+detect changes. On every `interval`, it re-runs the configured `query`, computes
+a hash for each returned row, and keeps an **in-memory hash table of every row**
+(keyed by `primary_key`) to compare against the previous run:
+
+* a row whose `primary_key` was not seen before → `insert` event
+* a row whose hash changed since the previous run → `update` event
+* a `primary_key` that disappeared from the result set → `delete` event (when `enable_delete yes`)
+
+Because the whole result set is held in memory, this plugin is best suited for
+small-to-medium tables. For millions of rows or multiple tables, prefer
+[`mysql_replicator_multi`](Tutorial-mysql_replicator_multi.md), which stores the
+hash table in a MySQL management table instead of Ruby memory.
+
+The `updated_at` column is therefore not required. It only becomes useful if you
+intentionally narrow the `query` to recently changed rows together with
+`enable_delete no`, as shown in the `enable_delete` comment below.
+
 ### configuration
 
 `````
@@ -21,7 +41,7 @@ On syncing 300 million rows table, it will consume around 800MB of memory with r
   database myweb
 
   # Set replicate query configuration.
-  query SELECT id, text, updated_at from search_test;
+  query SELECT id, text from search_test;
   primary_key id # specify unique key (default: id)
   interval 10s  # execute query interval (default: 1m)
 

data/ci/install_smoke.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+# Install smoke test: reproduces the README install steps inside the official
+# fluent/fluentd Debian image to verify the mysql2 native extension actually
+# builds and the plugins load. Guards against the "Failed to build gem native
+# extension" class of breakage reported in #43 / #16 / #35.
+set -eux
+
+GEM_FILE="${GEM_FILE:-/workspace/pkg/fluent-plugin-mysql-replicator.gem}"
+
+# Build toolchain + MySQL client development headers (the bit users forget).
+apt-get update
+apt-get install -y build-essential default-libmysqlclient-dev
+
+# Installing the gem compiles the mysql2 native extension; this fails loudly
+# if the client headers are missing, which is exactly what we want to catch.
+gem install "$GEM_FILE"
+
+# Confirm the installed plugins (and the mysql2 native extension) load at runtime.
+ruby "$(dirname "$0")/verify_plugins_load.rb"

data/ci/verify_plugins_load.rb

@@ -0,0 +1,5 @@
+# Loads the plugins to confirm they (and the mysql2 native extension) are
+# usable at runtime after a fresh install. Shared by both install-smoke jobs.
+require "fluent/plugin/in_mysql_replicator"
+require "fluent/plugin/in_mysql_replicator_multi"
+puts "input plugins loaded; mysql2 native extension OK"

data/fluent-plugin-mysql-replicator.gemspec

@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-mysql-replicator"
-  s.version     = "1.1.0"
+  s.version     = "1.2.0"
   s.authors     = ["Kentaro Yoshida"]
   s.email       = ["y.ken.studio@gmail.com"]
   s.homepage    = "https://github.com/y-ken/fluent-plugin-mysql-replicator"

data/lib/fluent/plugin/in_mysql_replicator.rb

@@ -1,5 +1,6 @@
 require 'mysql2'
 require 'digest/sha1'
+require 'json'
 require 'fluent/plugin/input'
 
 module Fluent::Plugin
@@ -19,6 +20,10 @@ module Fluent::Plugin
     config_param :primary_key, :string, :default => 'id'
     config_param :interval, :string, :default => '1m'
     config_param :enable_delete, :bool, :default => true
+    # Comma-separated column names whose MySQL JSON values should be parsed into
+    # nested objects before emitting. Intended for Elasticsearch; do not set it
+    # when the destination cannot store JSON objects (e.g. Solr).
+    config_param :json_columns, :array, :default => []
     config_param :tag, :string, :default => nil
 
     def configure(conf)
@@ -29,7 +34,7 @@ module Fluent::Plugin
         raise Fluent::ConfigError, "mysql_replicator: missing 'tag' parameter. Please add following line into config like 'tag replicator.mydatabase.mytable.${event}.${primary_key}'"
       end
 
-      log.info "adding mysql_replicator worker. :tag=>#{tag} :query=>#{@query} :prepared_query=>#{@prepared_query} :interval=>#{@interval}sec :enable_delete=>#{enable_delete}"
+      log.info "adding mysql_replicator worker. :tag=>#{tag} :query=>#{@query} :prepared_query=>#{@prepared_query} :interval=>#{@interval}sec :enable_delete=>#{enable_delete} :json_columns=>#{@json_columns}"
     end
 
     def start
@@ -71,7 +76,8 @@ module Fluent::Plugin
           current_ids << row[@primary_key]
           current_hash = Digest::SHA1.hexdigest(row.flatten.join)
           row.each {|k, v| row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
-          row.select {|k, v| v.to_s.strip.match(/^SELECT(\s+)/i) }.each do |k, v|
+          parse_json_columns!(row, @json_columns)
+          row.select {|k, v| nested_query_value?(v) }.each do |k, v|
             row[k] = [] unless row[k].is_a?(Array)
             nest_rows, prepared_con = query(v.gsub(/\$\{([^\}]+)\}/, row[$1].to_s), prepared_con)
             nest_rows.each do |nest_row|
@@ -97,13 +103,7 @@ module Fluent::Plugin
         con.close
         ids = current_ids
         if @enable_delete
-          if current_ids.empty?
-            deleted_ids = Array.new
-          elsif previous_ids.empty?
-            deleted_ids = [*1...current_ids.max] - current_ids
-          else
-            deleted_ids = previous_ids - current_ids
-          end
+          deleted_ids = detect_deleted_ids(previous_ids, current_ids)
           if deleted_ids.count > 0
             hash_delete_by_list(table_hash, deleted_ids)
             deleted_ids.each do |id|
@@ -122,6 +122,35 @@ module Fluent::Plugin
       deleted_keys.each{|k| hash.delete(k)}
     end
 
+    # Returns the primary keys that disappeared since the previous poll.
+    #
+    # The first poll only establishes a baseline: there is no previous snapshot
+    # to diff against, so nothing is reported as deleted yet. This also avoids
+    # the old `[*1...current_ids.max]` range, which raised "bad value for range"
+    # for non-integer primary keys and allocated a huge array (and emitted
+    # phantom deletes) for large / sparse integer ids. (#42)
+    def detect_deleted_ids(previous_ids, current_ids)
+      return [] if previous_ids.empty?
+      return [] if current_ids.empty?
+      previous_ids - current_ids
+    end
+
+    # Parse the given columns' JSON string values into nested objects in place.
+    # Non-string values, missing columns, and malformed JSON are left untouched
+    # so enabling this never corrupts non-JSON data.
+    def parse_json_columns!(row, columns)
+      return if columns.empty?
+      columns.each do |col|
+        v = row[col]
+        next unless v.is_a?(String)
+        begin
+          row[col] = JSON.parse(v)
+        rescue JSON::ParserError
+          # leave the original string as-is on malformed JSON
+        end
+      end
+    end
+
     def format_tag(tag, param)
       pattern = {'${event}' => param[:event].to_s, '${primary_key}' => @primary_key}
       tag.gsub(/(\${[a-z_]+})/) do
@@ -130,6 +159,15 @@ module Fluent::Plugin
       end
     end
 
+    # A column value triggers a nested sub-query only when it is a query
+    # template containing a `${placeholder}` (e.g. "SELECT ... WHERE x = ${id}").
+    # Requiring the placeholder prevents ordinary text values that merely begin
+    # with the word "SELECT" from being executed as SQL. (#4; mirrors the fix
+    # already applied to mysql_replicator_multi in #6.)
+    def nested_query_value?(value)
+      value.to_s.strip.match?(/^SELECT[^\$]+\$\{[^\}]+\}/i)
+    end
+
     def emit_record(tag, record)
       router.emit(tag, Fluent::Engine.now, record)
     end

data/lib/fluent/plugin/in_mysql_replicator_multi.rb

@@ -9,6 +9,7 @@ module Fluent::Plugin
     def initialize
       require 'mysql2'
       require 'digest/sha1'
+      require 'json'
       super
     end
 
@@ -81,6 +82,7 @@ module Fluent::Plugin
           log.info "mysql_replicator_multi: polling start. :config=>#{masked_config}"
         }
         primary_key = config['primary_key']
+        json_columns = (config['json_columns'] || '').split(',').map(&:strip).reject(&:empty?)
         previous_id = current_id = nil
         while @running
           rows_count = 0
@@ -97,6 +99,7 @@ module Fluent::Plugin
             db = get_origin_connection(config)
             db.query(config['query']).each do |row|
               row.each {|k, v| row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
+              parse_json_columns!(row, json_columns)
               row.select {|k, v| v.to_s.strip.match(/^SELECT[^\$]+\$\{[^\}]+\}/i) }.each do |k, v|
                 row[k] = [] unless row[k].is_a?(Array)
                 nest_db.query(v.gsub(/\$\{([^\}]+)\}/) {|matched| row[$1].to_s}).each do |nest_row|
@@ -266,6 +269,22 @@ module Fluent::Plugin
       @hash_table_bulk_insert_last_time = Time.now
     end
 
+    # Parse the given columns' JSON string values into nested objects in place.
+    # Non-string values, missing columns, and malformed JSON are left untouched
+    # so enabling this never corrupts non-JSON data.
+    def parse_json_columns!(row, columns)
+      return if columns.empty?
+      columns.each do |col|
+        v = row[col]
+        next unless v.is_a?(String)
+        begin
+          row[col] = JSON.parse(v)
+        rescue JSON::ParserError
+          # leave the original string as-is on malformed JSON
+        end
+      end
+    end
+
     def emit_record(tag, record)
       router.emit(tag, Fluent::Engine.now, record)
     end

data/setup_mysql_replicator_multi.sql

@@ -24,6 +24,9 @@ CREATE TABLE IF NOT EXISTS `settings` (
   `prepared_query` TEXT NOT NULL,
   `interval` int(11) NOT NULL,
   `primary_key` varchar(255) DEFAULT 'id',
+  -- Comma-separated column names whose MySQL JSON values should be parsed into nested objects.
+  -- Intended for Elasticsearch; leave empty for Solr or destinations that cannot store JSON objects.
+  `json_columns` varchar(255) DEFAULT NULL,
   `enable_delete` int(11) DEFAULT '1',
   -- On enabling 'enable_loose_insert: 1', make it faster synchronization to skip checking hash_tables.
   `enable_loose_insert` int(11) DEFAULT '0',

data/test/e2e/fluent_single.conf

@@ -8,8 +8,9 @@
   username "#{ENV['MYSQL_USER'] || 'root'}"
   password "#{ENV['MYSQL_PASSWORD'] || 'root'}"
   database "#{ENV['MYSQL_DATABASE'] || 'e2e_source'}"
-  query SELECT id, name, age FROM users
+  query SELECT id, name, age, profile FROM users
   primary_key id
+  json_columns profile
   interval 2s
   enable_delete true
   tag myindex.mytype.${event}.${primary_key}

data/test/e2e/replication_multi_e2e_test.rb

@@ -41,13 +41,19 @@ client.query("CREATE DATABASE `#{SOURCE_DB}`")
 client.query("USE `#{SOURCE_DB}`")
 client.query(<<~SQL)
   CREATE TABLE users (
-    id   INT NOT NULL AUTO_INCREMENT,
-    name VARCHAR(255) NOT NULL,
-    age  INT NOT NULL,
+    id      INT NOT NULL AUTO_INCREMENT,
+    name    VARCHAR(255) NOT NULL,
+    age     INT NOT NULL,
+    profile JSON,
     PRIMARY KEY (id)
   ) ENGINE=InnoDB DEFAULT CHARSET=utf8
 SQL
-client.query("INSERT INTO users (name, age) VALUES ('alice', 20), ('bob', 30), ('carol', 40)")
+client.query(<<~SQL)
+  INSERT INTO users (name, age, profile) VALUES
+    ('alice', 20, '{"city":"Tokyo","tags":["a","b"]}'),
+    ('bob',   30, '{"city":"Osaka","tags":["c"]}'),
+    ('carol', 40, '{"city":"Nagoya","tags":[]}')
+SQL
 
 # --- 2. Build the management database and register a replication setting ----
 step "building management database '#{MANAGER_DB}' from setup SQL"
@@ -61,11 +67,11 @@ cfg = mysql_config
 client.query(<<~SQL)
   INSERT INTO `#{MANAGER_DB}`.settings
     (is_active, name, host, port, username, password, `database`,
-     query, prepared_query, `interval`, primary_key, enable_delete)
+     query, prepared_query, `interval`, primary_key, enable_delete, json_columns)
   VALUES
     (1, '#{SETTING_NAME}', '#{cfg[:host]}', #{cfg[:port]},
      '#{cfg[:username]}', '#{client.escape(cfg[:password].to_s)}', '#{SOURCE_DB}',
-     'SELECT id, name, age FROM users ORDER BY id', '', 2, 'id', 1)
+     'SELECT id, name, age, profile FROM users ORDER BY id', '', 2, 'id', 1, 'profile')
 SQL
 
 # --- 3. Boot Fluentd --------------------------------------------------------
@@ -83,6 +89,18 @@ begin
   end
   step "  INSERT OK"
 
+  # --- 4b. JSON column is indexed as a nested object (not an escaped string) -
+  step "asserting JSON column is replicated as a nested object"
+  wait_until("user 1 profile.city == Tokyo in Elasticsearch", log_path: LOG_PATH) do
+    code, body = es_get(INDEX, TYPE, 1)
+    code == 200 && body.dig('_source', 'profile', 'city') == 'Tokyo'
+  end
+  _, body = es_get(INDEX, TYPE, 1)
+  unless body.dig('_source', 'profile', 'tags') == ['a', 'b']
+    fail_with("profile was not indexed as a nested object: #{body.dig('_source', 'profile').inspect}", LOG_PATH)
+  end
+  step "  JSON OK"
+
   # --- 5. hash_tables state is persisted ------------------------------------
   step "asserting hash_tables persistence"
   wait_until("3 rows recorded in hash_tables for '#{SETTING_NAME}'", log_path: LOG_PATH) do

data/test/e2e/replication_single_e2e_test.rb

@@ -24,13 +24,18 @@ client.query("CREATE DATABASE `#{DB}`")
 client.query("USE `#{DB}`")
 client.query(<<~SQL)
   CREATE TABLE users (
-    id   INT NOT NULL AUTO_INCREMENT,
-    name VARCHAR(255) NOT NULL,
-    age  INT NOT NULL,
+    id      INT NOT NULL AUTO_INCREMENT,
+    name    VARCHAR(255) NOT NULL,
+    age     INT NOT NULL,
+    profile JSON,
     PRIMARY KEY (id)
   ) ENGINE=InnoDB DEFAULT CHARSET=utf8
 SQL
-client.query("INSERT INTO users (name, age) VALUES ('alice', 20), ('bob', 30)")
+client.query(<<~SQL)
+  INSERT INTO users (name, age, profile) VALUES
+    ('alice', 20, '{"city":"Tokyo","tags":["a","b"]}'),
+    ('bob',   30, '{"city":"Osaka","tags":["c"]}')
+SQL
 
 # --- 2. Boot Fluentd --------------------------------------------------------
 step "starting Fluentd (#{CONF})"
@@ -49,6 +54,18 @@ begin
   end
   step "  INSERT OK"
 
+  # --- 3b. JSON column is indexed as a nested object (not an escaped string) -
+  step "asserting JSON column is replicated as a nested object"
+  wait_until("user 1 profile.city == Tokyo in Elasticsearch", log_path: LOG_PATH) do
+    code, body = es_get(INDEX, TYPE, 1)
+    code == 200 && body.dig('_source', 'profile', 'city') == 'Tokyo'
+  end
+  _, body = es_get(INDEX, TYPE, 1)
+  unless body.dig('_source', 'profile', 'tags') == ['a', 'b']
+    fail_with("profile was not indexed as a nested object: #{body.dig('_source', 'profile').inspect}", LOG_PATH)
+  end
+  step "  JSON OK"
+
   # --- 4. UPDATE is replicated ----------------------------------------------
   step "asserting UPDATE replication"
   client.query("UPDATE users SET age = 21 WHERE id = 1")

data/test/plugin/test_in_mysql_replicator.rb

@@ -30,11 +30,111 @@ class MysqlReplicatorInputTest < Test::Unit::TestCase
       tag             input.mysql
       query           SELECT id, text from search_text
       enable_delete   no
+      json_columns    geometry,attrs
     ]
     assert_equal 'localhost', d.instance.host
     assert_equal 3306, d.instance.port
     assert_equal 30, d.instance.interval
     assert_equal 'input.mysql', d.instance.tag
     assert_equal false, d.instance.enable_delete
+    assert_equal ['geometry', 'attrs'], d.instance.json_columns
+  end
+
+  def test_json_columns_defaults_to_empty
+    d = create_driver
+    assert_equal [], d.instance.json_columns
+  end
+
+  def test_parse_json_columns
+    d = create_driver
+    row = {
+      'id'       => 1,
+      'geometry' => '{"type":"Polygon","coordinates":[[136.8,35.1]]}',
+      'tags'     => '[1,2,3]',
+      'broken'   => 'not json {',
+      'plain'    => 'hello',
+      'number'   => 5,
+    }
+    d.instance.parse_json_columns!(row, ['geometry', 'tags', 'broken', 'number', 'missing'])
+
+    assert_equal({'type' => 'Polygon', 'coordinates' => [[136.8, 35.1]]}, row['geometry'])
+    assert_equal([1, 2, 3], row['tags'])
+    assert_equal('not json {', row['broken'])  # malformed JSON stays as the original string
+    assert_equal(5, row['number'])             # non-string values are untouched
+    assert_equal('hello', row['plain'])        # columns not listed are untouched
+  end
+
+  def test_parse_json_columns_noop_when_empty
+    d = create_driver
+    row = {'geometry' => '{"a":1}'}
+    d.instance.parse_json_columns!(row, [])
+    assert_equal('{"a":1}', row['geometry'])
+  end
+
+  # --- #42: delete detection must not build an integer Range from primary keys ---
+
+  def deleted_ids(previous_ids, current_ids)
+    conf = %[
+      tag   input.mysql
+      query SELECT id, text from search_test
+    ]
+    create_driver(conf).instance.detect_deleted_ids(previous_ids, current_ids)
+  end
+
+  def test_detect_deleted_ids_first_poll_reports_no_deletes
+    assert_equal [], deleted_ids([], [1, 2, 3])
+  end
+
+  def test_detect_deleted_ids_first_poll_with_string_keys_does_not_raise
+    # Regression for #42: the old `[*1...'c']` raised "bad value for range".
+    assert_nothing_raised do
+      assert_equal [], deleted_ids([], %w[a b c])
+    end
+  end
+
+  def test_detect_deleted_ids_first_poll_with_sparse_ids_has_no_phantom_deletes
+    # The old code returned `[*1...99] - [1, 50, 99]`, emitting deletes for ids
+    # that never existed. The first poll must only establish a baseline.
+    assert_equal [], deleted_ids([], [1, 50, 99])
+  end
+
+  def test_detect_deleted_ids_diffs_against_previous_snapshot
+    assert_equal [2], deleted_ids([1, 2, 3], [1, 3])
+  end
+
+  def test_detect_deleted_ids_empty_current_does_not_mass_delete
+    assert_equal [], deleted_ids([1, 2, 3], [])
+  end
+
+  # --- #4: a nested sub-query fires only for a SELECT template with ${...} ---
+
+  def nested?(value)
+    conf = %[
+      tag   input.mysql
+      query SELECT id, text from search_test
+    ]
+    create_driver(conf).instance.nested_query_value?(value)
+  end
+
+  def test_nested_query_value_true_for_select_with_placeholder
+    assert_true nested?("SELECT * FROM child WHERE parent_id = ${id}")
+  end
+
+  def test_nested_query_value_false_for_plain_text_starting_with_select
+    # Regression for #4: a data value beginning with "SELECT" must not run as SQL.
+    assert_false nested?("SELECT YOUR PLAN")
+  end
+
+  def test_nested_query_value_false_for_word_select
+    assert_false nested?("Selecting the best option")
+  end
+
+  def test_nested_query_value_false_for_select_without_placeholder
+    assert_false nested?("select count(*) from search_test")
+  end
+
+  def test_nested_query_value_false_for_non_string_values
+    assert_false nested?(12345)
+    assert_false nested?(nil)
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mysql-replicator
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.2.0
 platform: ruby
 authors:
 - Kentaro Yoshida
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2026-06-15 00:00:00.000000000 Z
+date: 2026-06-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -112,12 +112,15 @@ files:
 - ".dockerignore"
 - ".github/workflows/ci.yml"
 - ".gitignore"
+- CHANGELOG.md
 - Gemfile
 - LICENSE
 - README.md
 - Rakefile
 - Tutorial-mysql_replicator.md
 - Tutorial-mysql_replicator_multi.md
+- ci/install_smoke.sh
+- ci/verify_plugins_load.rb
 - example/mysql_multi_table_to_elasticsearch.md
 - example/mysql_multi_table_to_solr.md
 - example/mysql_single_table_to_elasticsearch.md