fluent-plugin-mysql-replicator 1.1.0 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 16c6600b210b85bad55965f707eadd53017558fd8a6a2b55cdf28bc1e86b7be3
-  data.tar.gz: 07d8b7367ee234afb128845fb8ef9169929cb18caf4c1813040efe951e72f68e
+  metadata.gz: 50b2ac9d97e49b3febf1c20fc98c815b3d6a919d299a24511a72b8b1637d5585
+  data.tar.gz: f39b735a34afd31a77f5e82339c449265dd4ef455456a85e2902dab91db79130
 SHA512:
-  metadata.gz: 65ec0c92b273597f7f7ff976fc42878e4fa413d8471947125a6873d7209142bec87a59aa37e3c491fd1a81dea4a9d448e3280cbca1eafd381fdee241d723bcae
-  data.tar.gz: 20fb6ffe6e217e5d8ee0108c5243fad7f6abd910f1a2d336f65bcb8ba0388ac22e329d408edaa5ad8f298c959cffc2656932d71302d6b2b38c21c28cab14585d
+  metadata.gz: d57b725ee936f2ffc8ae55631e75759c68de863022cb5e5740be87eae6a756b9ddc5a531825bb603a02f6a7f57752ce75d389ff3fd777ba86e5126f83f392710
+  data.tar.gz: 8b644659039eae41a70dbe27f206673fb1420372321c00af802925c52b7d8fe3b5abb880d0aaf059521ceb80fbd414c1a41a7b4fef6c6dd8fd2a05ef746fdf05

data/.devcontainer/Dockerfile

@@ -1,4 +1,4 @@
-FROM ruby:3.3-slim
+FROM ruby:3.4-slim
 
 ENV DEBIAN_FRONTEND=noninteractive
 ENV BUNDLE_PATH=/workspace/vendor/bundle

data/.github/workflows/ci.yml

@@ -100,3 +100,66 @@ jobs:
 
       - name: Run multi-table replication E2E test
         run: bundle exec ruby test/e2e/replication_multi_e2e_test.rb
+
+  # Container path: the official fluent/fluentd Docker image (gem-based).
+  install-smoke-docker:
+    name: Install smoke (Docker fluent/fluentd ${{ matrix.fluentd_tag }})
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        # "edge" tracks the latest Fluentd release to catch drift early. The
+        # packaged-distribution path is covered by install-smoke-fluent-package.
+        fluentd_tag: ['edge-debian']
+
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.4'
+
+      - name: Build gem
+        run: |
+          mkdir -p pkg
+          gem build fluent-plugin-mysql-replicator.gemspec -o pkg/fluent-plugin-mysql-replicator.gem
+
+      - name: Install on official fluentd image (reproduces README steps)
+        run: |
+          docker run --rm --user root --entrypoint sh \
+            -v "${{ github.workspace }}:/workspace:ro" \
+            "fluent/fluentd:${{ matrix.fluentd_tag }}" \
+            /workspace/ci/install_smoke.sh
+
+  # Package path: fluent-package, the LTS distribution that replaced td-agent
+  # and is what most production users run today. v6 LTS bundles Fluentd v1.19.
+  install-smoke-fluent-package:
+    name: Install smoke (fluent-package v6 LTS)
+    runs-on: ubuntu-latest  # GitHub's ubuntu-latest is Ubuntu 24.04 (Noble)
+
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.4'
+
+      - name: Build gem
+        run: |
+          mkdir -p pkg
+          gem build fluent-plugin-mysql-replicator.gemspec -o pkg/fluent-plugin-mysql-replicator.gem
+
+      - name: Install fluent-package v6 LTS
+        run: |
+          curl -fsSL https://fluentd.cdn.cncf.io/sh/install-ubuntu-noble-fluent-package6-lts.sh | sh
+          /opt/fluent/bin/fluentd --version
+
+      - name: Install MySQL client headers
+        run: sudo apt-get update && sudo apt-get install -y build-essential default-libmysqlclient-dev
+
+      - name: Install plugin via fluent-gem and verify it loads
+        run: |
+          sudo /opt/fluent/bin/fluent-gem install pkg/fluent-plugin-mysql-replicator.gem
+          sudo /opt/fluent/bin/fluent-gem list fluent-plugin-mysql-replicator
+          /opt/fluent/bin/ruby ci/verify_plugins_load.rb

data/CHANGELOG.md

@@ -0,0 +1,105 @@
+# Changelog
+
+All notable changes to this project are documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [1.3.0] - 2026-06-16
+
+### Added
+- Date-based Elasticsearch index names. If the index-name segment of the tag
+  contains `strftime` tokens (e.g. `%Y%m%d`), they are expanded using the
+  record's event time, enabling Logstash-style dated indices such as
+  `myindex-20180831`. Index names without a `%` are unchanged. ([#27])
+- Composite primary key support in `mysql_replicator`. `primary_key` now accepts
+  a comma-separated list of columns; the combination is used for change
+  detection and as the Elasticsearch document `_id` (values joined by `,`). A
+  single-column key behaves exactly as before. ([#7])
+
+## [1.2.0] - 2026-06-16
+
+### Added
+- JSON column support for nested object indexing. The new opt-in `json_columns`
+  option parses MySQL `JSON` string columns into nested objects before they are
+  emitted, so Elasticsearch indexes them as real nested JSON instead of escaped
+  strings. Available on both `mysql_replicator` (config option) and
+  `mysql_replicator_multi` (new `json_columns` column on the `settings` table).
+  Malformed JSON and non-string values are left untouched. ([#48], implements [#18])
+
+### Fixed
+- Delete detection no longer raises `bad value for range` (and no longer
+  allocates a huge array or emits phantom delete events) when the `primary_key`
+  is non-integer (e.g. a UUID `CHAR(36)`) or a large/sparse integer. The first
+  poll now only establishes a baseline. ([#42])
+- In `mysql_replicator`, a nested sub-query is now triggered only when a column
+  value is an actual query template containing a `${placeholder}`. Previously any
+  value that merely began with the word `SELECT` was executed as SQL. This aligns
+  the single plugin with the fix already present in `mysql_replicator_multi`. ([#4])
+
+### Documentation
+- Document that the `mysql2` native extension requires the MySQL client
+  development headers (`default-libmysqlclient-dev` / `mysql-devel`), add a
+  "Failed to build gem native extension" troubleshooting note, and clarify that
+  change detection uses an in-memory hash of every row (not an `updated_at`
+  column). ([#43], [#40])
+
+### CI
+- Add install-smoke jobs that build and install the gem on **fluent-package v6
+  LTS** (the td-agent successor) and on the official `fluent/fluentd` image,
+  guarding against native-extension build regressions.
+
+## [1.1.0] - 2026-06-15
+
+### Added
+- Auto-detect the Elasticsearch version on first write and support Elasticsearch
+  6.x through 9.x. The `_type` field is omitted automatically for 7.x and later,
+  where mapping types were removed.
+- Dev Container configuration and end-to-end (E2E) replication tests, including
+  multi-table replication.
+
+### Changed
+- Migrated CI to GitHub Actions (removed Travis CI) and run the unit-test matrix
+  on Ruby 3.2–4.0, aligned with supported Fluentd versions.
+
+## [1.0.3] - 2024-02-28
+
+### Added
+- Reconnection option to recover from database connection errors. ([#45])
+
+## [1.0.2] - 2021-10-27
+
+### Fixed
+- Manage threads correctly in `mysql_replicator_multi`. ([#39])
+
+## [1.0.1] - 2019-11-19
+
+### Fixed
+- Ensure the nested-query database connection (`nest_db`) is always closed.
+
+### Documentation
+- Documentation updates.
+
+## [1.0.0] - 2019-11-13
+
+- First 1.0 release, targeting the Fluentd v0.14+ plugin API. Earlier 0.x
+  history is available in the git log.
+
+[1.3.0]: https://github.com/y-ken/fluent-plugin-mysql-replicator/compare/v1.2.0...v1.3.0
+[1.2.0]: https://github.com/y-ken/fluent-plugin-mysql-replicator/compare/v1.1.0...v1.2.0
+[1.1.0]: https://github.com/y-ken/fluent-plugin-mysql-replicator/compare/v1.0.3...v1.1.0
+[1.0.3]: https://github.com/y-ken/fluent-plugin-mysql-replicator/compare/v1.0.2...v1.0.3
+[1.0.2]: https://github.com/y-ken/fluent-plugin-mysql-replicator/compare/v1.0.1...v1.0.2
+[1.0.1]: https://github.com/y-ken/fluent-plugin-mysql-replicator/compare/v1.0.0...v1.0.1
+[1.0.0]: https://github.com/y-ken/fluent-plugin-mysql-replicator/releases/tag/v1.0.0
+
+[#4]: https://github.com/y-ken/fluent-plugin-mysql-replicator/issues/4
+[#7]: https://github.com/y-ken/fluent-plugin-mysql-replicator/pull/7
+[#18]: https://github.com/y-ken/fluent-plugin-mysql-replicator/pull/18
+[#27]: https://github.com/y-ken/fluent-plugin-mysql-replicator/issues/27
+[#39]: https://github.com/y-ken/fluent-plugin-mysql-replicator/pull/39
+[#40]: https://github.com/y-ken/fluent-plugin-mysql-replicator/issues/40
+[#42]: https://github.com/y-ken/fluent-plugin-mysql-replicator/issues/42
+[#43]: https://github.com/y-ken/fluent-plugin-mysql-replicator/issues/43
+[#45]: https://github.com/y-ken/fluent-plugin-mysql-replicator/pull/45
+[#48]: https://github.com/y-ken/fluent-plugin-mysql-replicator/pull/48

data/README.md

@@ -15,14 +15,28 @@ It's comming support replicate to another RDB/noSQL.
 
 ## Dependency
 
-before use, install dependent library as:
+This plugin depends on the [mysql2](https://rubygems.org/gems/mysql2) gem, which
+builds a native extension at install time. You need both a C compiler toolchain
+**and the MySQL client development headers** before installing.
 
 ```bash
-# for RHEL/CentOS
+# for RHEL/CentOS/Fedora
 $ sudo yum group install "Development Tools"
+$ sudo yum install mysql-devel        # or mariadb-devel
 
-# for Ubuntu/Debian
-$ sudo apt-get install build-essential
+# for Ubuntu/Debian (including the official fluent/fluentd Docker image)
+$ sudo apt-get install build-essential default-libmysqlclient-dev
+```
+
+### Troubleshooting: "Failed to build gem native extension"
+
+If `gem install` (or `td-agent-gem install`) fails while building the `mysql2`
+native extension, the MySQL client headers are almost always missing. Install
+the development package for your platform shown above, then retry the
+installation. On the official `fluent/fluentd:*-debian` images, run:
+
+```bash
+$ apt-get update && apt-get install -y build-essential default-libmysqlclient-dev
 ```
 
 ## Installation
@@ -74,6 +88,87 @@ Mapping types were removed in Elasticsearch 8.x (and deprecated in 7.x), so the
 Elasticsearch version on the first write and automatically omits `_type` for
 7.x and later, so no extra configuration is required.
 
+## JSON column support
+
+MySQL `JSON` columns (MySQL 5.7.8+ / 8.x) are returned by the driver as plain
+strings, so by default they reach Elasticsearch as escaped strings rather than
+nested objects. List such columns in `json_columns` to have their values parsed
+into nested objects before they are emitted.
+
+* `mysql_replicator` (single): set the `json_columns` option (comma-separated).
+
+  ```
+  <source>
+    @type        mysql_replicator
+    # ...
+    query        SELECT id, name, geometry FROM places
+    json_columns geometry,attrs
+  </source>
+  ```
+
+* `mysql_replicator_multi`: set the `json_columns` column (comma-separated) on
+  the relevant row of the `settings` management table. Existing installs can add
+  the column with:
+
+  ```sql
+  ALTER TABLE settings ADD COLUMN `json_columns` varchar(255) DEFAULT NULL AFTER `primary_key`;
+  ```
+
+Notes:
+
+* This option is intended for Elasticsearch. **Do not set it when the destination
+  cannot store JSON objects (e.g. the Solr output)** — leave it empty there.
+* Only top-level columns are parsed (columns inside nested documents are not).
+* Malformed JSON and non-string values are left untouched, so enabling the option
+  never corrupts non-JSON data.
+
+## Date-based index names
+
+`mysql_replicator_elasticsearch` resolves the target index name from the tag
+(via `tag_format`). If that index-name segment contains `strftime` tokens such
+as `%Y%m%d`, they are expanded using the record's event time, so you can create
+Logstash-style dated indices like `myindex-20180831`.
+
+Put the tokens in the index-name part of the input plugin's `tag` (the segment
+must not contain `.`, so use `%Y%m%d` or `%Y-%m-%d`):
+
+```
+<source>
+  @type mysql_replicator
+  # ...
+  tag   myindex-%Y%m%d.mytype.${event}.${primary_key}
+</source>
+```
+
+Index names that contain no `%` are left unchanged, so this is fully backward
+compatible.
+
+> **Note on deletions:** delete events target the index computed from the delete
+> event's own time, so date-rotated indices are best suited to insert-only data
+> (a record inserted on a previous day lives in that day's index).
+
+## Composite primary keys
+
+`primary_key` accepts a comma-separated list of columns, so tables keyed by more
+than one column are supported:
+
+```
+<source>
+  @type        mysql_replicator
+  # ...
+  query        SELECT tenant_id, id, name FROM items
+  primary_key  tenant_id,id
+</source>
+```
+
+Change detection (insert/update/delete) then keys on the combination of those
+columns, and the Elasticsearch document `_id` becomes their values joined by `,`
+(e.g. `10,7`). A single-column `primary_key` (the default `id`) behaves exactly
+as before.
+
+This applies to `mysql_replicator`; `mysql_replicator_multi` still expects a
+single-column primary key.
+
 ## Output example
 
 It is a example when detecting insert/update/delete events.

data/Tutorial-mysql_replicator.md

@@ -8,6 +8,26 @@ It is useful for these purpose.
 **Note:**  
 On syncing 300 million rows table, it will consume around 800MB of memory with ruby 1.9.3 environment.
 
+### How it works
+
+`mysql_replicator` does **not** rely on any `updated_at`/timestamp column to
+detect changes. On every `interval`, it re-runs the configured `query`, computes
+a hash for each returned row, and keeps an **in-memory hash table of every row**
+(keyed by `primary_key`) to compare against the previous run:
+
+* a row whose `primary_key` was not seen before → `insert` event
+* a row whose hash changed since the previous run → `update` event
+* a `primary_key` that disappeared from the result set → `delete` event (when `enable_delete yes`)
+
+Because the whole result set is held in memory, this plugin is best suited for
+small-to-medium tables. For millions of rows or multiple tables, prefer
+[`mysql_replicator_multi`](Tutorial-mysql_replicator_multi.md), which stores the
+hash table in a MySQL management table instead of Ruby memory.
+
+The `updated_at` column is therefore not required. It only becomes useful if you
+intentionally narrow the `query` to recently changed rows together with
+`enable_delete no`, as shown in the `enable_delete` comment below.
+
 ### configuration
 
 `````
@@ -21,7 +41,7 @@ On syncing 300 million rows table, it will consume around 800MB of memory with r
   database myweb
 
   # Set replicate query configuration.
-  query SELECT id, text, updated_at from search_test;
+  query SELECT id, text from search_test;
   primary_key id # specify unique key (default: id)
   interval 10s  # execute query interval (default: 1m)
 

data/ci/install_smoke.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+# Install smoke test: reproduces the README install steps inside the official
+# fluent/fluentd Debian image to verify the mysql2 native extension actually
+# builds and the plugins load. Guards against the "Failed to build gem native
+# extension" class of breakage reported in #43 / #16 / #35.
+set -eux
+
+GEM_FILE="${GEM_FILE:-/workspace/pkg/fluent-plugin-mysql-replicator.gem}"
+
+# Build toolchain + MySQL client development headers (the bit users forget).
+apt-get update
+apt-get install -y build-essential default-libmysqlclient-dev
+
+# Installing the gem compiles the mysql2 native extension; this fails loudly
+# if the client headers are missing, which is exactly what we want to catch.
+gem install "$GEM_FILE"
+
+# Confirm the installed plugins (and the mysql2 native extension) load at runtime.
+ruby "$(dirname "$0")/verify_plugins_load.rb"

data/ci/verify_plugins_load.rb

@@ -0,0 +1,5 @@
+# Loads the plugins to confirm they (and the mysql2 native extension) are
+# usable at runtime after a fresh install. Shared by both install-smoke jobs.
+require "fluent/plugin/in_mysql_replicator"
+require "fluent/plugin/in_mysql_replicator_multi"
+puts "input plugins loaded; mysql2 native extension OK"

data/fluent-plugin-mysql-replicator.gemspec

@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-mysql-replicator"
-  s.version     = "1.1.0"
+  s.version     = "1.3.0"
   s.authors     = ["Kentaro Yoshida"]
   s.email       = ["y.ken.studio@gmail.com"]
   s.homepage    = "https://github.com/y-ken/fluent-plugin-mysql-replicator"

data/lib/fluent/plugin/in_mysql_replicator.rb

@@ -1,5 +1,6 @@
 require 'mysql2'
 require 'digest/sha1'
+require 'json'
 require 'fluent/plugin/input'
 
 module Fluent::Plugin
@@ -16,9 +17,16 @@ module Fluent::Plugin
     config_param :encoding, :string, :default => 'utf8'
     config_param :query, :string
     config_param :prepared_query, :string, :default => nil
-    config_param :primary_key, :string, :default => 'id'
+    # A single column name, or a comma-separated list for a composite key
+    # (e.g. "tenant_id,id"). The id used for change detection and the
+    # Elasticsearch document _id is the combination of these columns.
+    config_param :primary_key, :array, :default => ['id']
     config_param :interval, :string, :default => '1m'
     config_param :enable_delete, :bool, :default => true
+    # Comma-separated column names whose MySQL JSON values should be parsed into
+    # nested objects before emitting. Intended for Elasticsearch; do not set it
+    # when the destination cannot store JSON objects (e.g. Solr).
+    config_param :json_columns, :array, :default => []
     config_param :tag, :string, :default => nil
 
     def configure(conf)
@@ -29,7 +37,7 @@ module Fluent::Plugin
         raise Fluent::ConfigError, "mysql_replicator: missing 'tag' parameter. Please add following line into config like 'tag replicator.mydatabase.mytable.${event}.${primary_key}'"
       end
 
-      log.info "adding mysql_replicator worker. :tag=>#{tag} :query=>#{@query} :prepared_query=>#{@prepared_query} :interval=>#{@interval}sec :enable_delete=>#{enable_delete}"
+      log.info "adding mysql_replicator worker. :tag=>#{tag} :query=>#{@query} :prepared_query=>#{@prepared_query} :interval=>#{@interval}sec :enable_delete=>#{enable_delete} :json_columns=>#{@json_columns}"
     end
 
     def start
@@ -68,10 +76,12 @@ module Fluent::Plugin
         end
         rows, con = query(@query, con)
         rows.each do |row|
-          current_ids << row[@primary_key]
+          id = extract_id(row)
+          current_ids << id
           current_hash = Digest::SHA1.hexdigest(row.flatten.join)
           row.each {|k, v| row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
-          row.select {|k, v| v.to_s.strip.match(/^SELECT(\s+)/i) }.each do |k, v|
+          parse_json_columns!(row, @json_columns)
+          row.select {|k, v| nested_query_value?(v) }.each do |k, v|
             row[k] = [] unless row[k].is_a?(Array)
             nest_rows, prepared_con = query(v.gsub(/\$\{([^\}]+)\}/, row[$1].to_s), prepared_con)
             nest_rows.each do |nest_row|
@@ -80,35 +90,29 @@ module Fluent::Plugin
             end
             prepared_con.close
           end
-          if row[@primary_key].nil?
-            log.error "mysql_replicator: missing primary_key. :tag=>#{tag} :primary_key=>#{primary_key}"
+          if id.any?(&:nil?)
+            log.error "mysql_replicator: missing primary_key. :tag=>#{tag} :primary_key=>#{@primary_key.join(',')} :id=>#{id}"
             break
           end
-          if !table_hash.include?(row[@primary_key])
+          if !table_hash.include?(id)
             tag = format_tag(@tag, {:event => :insert})
             emit_record(tag, row)
-          elsif table_hash[row[@primary_key]] != current_hash
+          elsif table_hash[id] != current_hash
             tag = format_tag(@tag, {:event => :update})
             emit_record(tag, row)
           end
-          table_hash[row[@primary_key]] = current_hash
+          table_hash[id] = current_hash
           rows_count += 1
         end
         con.close
         ids = current_ids
         if @enable_delete
-          if current_ids.empty?
-            deleted_ids = Array.new
-          elsif previous_ids.empty?
-            deleted_ids = [*1...current_ids.max] - current_ids
-          else
-            deleted_ids = previous_ids - current_ids
-          end
+          deleted_ids = detect_deleted_ids(previous_ids, current_ids)
           if deleted_ids.count > 0
             hash_delete_by_list(table_hash, deleted_ids)
             deleted_ids.each do |id|
               tag = format_tag(@tag, {:event => :delete})
-              emit_record(tag, {@primary_key => id})
+              emit_record(tag, Hash[@primary_key.zip(id)])
             end
           end
         end
@@ -122,14 +126,58 @@ module Fluent::Plugin
       deleted_keys.each{|k| hash.delete(k)}
     end
 
+    # A row's id is the array of its primary-key column values, supporting
+    # composite keys. It is a single-element array for a single-column key.
+    def extract_id(row)
+      @primary_key.map {|col| row[col] }
+    end
+
+    # Returns the primary keys that disappeared since the previous poll.
+    #
+    # The first poll only establishes a baseline: there is no previous snapshot
+    # to diff against, so nothing is reported as deleted yet. This also avoids
+    # the old `[*1...current_ids.max]` range, which raised "bad value for range"
+    # for non-integer primary keys and allocated a huge array (and emitted
+    # phantom deletes) for large / sparse integer ids. (#42)
+    def detect_deleted_ids(previous_ids, current_ids)
+      return [] if previous_ids.empty?
+      return [] if current_ids.empty?
+      previous_ids - current_ids
+    end
+
+    # Parse the given columns' JSON string values into nested objects in place.
+    # Non-string values, missing columns, and malformed JSON are left untouched
+    # so enabling this never corrupts non-JSON data.
+    def parse_json_columns!(row, columns)
+      return if columns.empty?
+      columns.each do |col|
+        v = row[col]
+        next unless v.is_a?(String)
+        begin
+          row[col] = JSON.parse(v)
+        rescue JSON::ParserError
+          # leave the original string as-is on malformed JSON
+        end
+      end
+    end
+
     def format_tag(tag, param)
-      pattern = {'${event}' => param[:event].to_s, '${primary_key}' => @primary_key}
+      pattern = {'${event}' => param[:event].to_s, '${primary_key}' => @primary_key.join(',')}
       tag.gsub(/(\${[a-z_]+})/) do
         log.warn "mysql_replicator: missing placeholder. :tag=>#{tag} :placeholder=>#{$1}" unless pattern.include?($1)
         pattern[$1]
       end
     end
 
+    # A column value triggers a nested sub-query only when it is a query
+    # template containing a `${placeholder}` (e.g. "SELECT ... WHERE x = ${id}").
+    # Requiring the placeholder prevents ordinary text values that merely begin
+    # with the word "SELECT" from being executed as SQL. (#4; mirrors the fix
+    # already applied to mysql_replicator_multi in #6.)
+    def nested_query_value?(value)
+      value.to_s.strip.match?(/^SELECT[^\$]+\$\{[^\}]+\}/i)
+    end
+
     def emit_record(tag, record)
       router.emit(tag, Fluent::Engine.now, record)
     end

data/lib/fluent/plugin/in_mysql_replicator_multi.rb

@@ -9,6 +9,7 @@ module Fluent::Plugin
     def initialize
       require 'mysql2'
       require 'digest/sha1'
+      require 'json'
       super
     end
 
@@ -81,6 +82,7 @@ module Fluent::Plugin
           log.info "mysql_replicator_multi: polling start. :config=>#{masked_config}"
         }
         primary_key = config['primary_key']
+        json_columns = (config['json_columns'] || '').split(',').map(&:strip).reject(&:empty?)
         previous_id = current_id = nil
         while @running
           rows_count = 0
@@ -97,6 +99,7 @@ module Fluent::Plugin
             db = get_origin_connection(config)
             db.query(config['query']).each do |row|
               row.each {|k, v| row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
+              parse_json_columns!(row, json_columns)
               row.select {|k, v| v.to_s.strip.match(/^SELECT[^\$]+\$\{[^\}]+\}/i) }.each do |k, v|
                 row[k] = [] unless row[k].is_a?(Array)
                 nest_db.query(v.gsub(/\$\{([^\}]+)\}/) {|matched| row[$1].to_s}).each do |nest_row|
@@ -266,6 +269,22 @@ module Fluent::Plugin
       @hash_table_bulk_insert_last_time = Time.now
     end
 
+    # Parse the given columns' JSON string values into nested objects in place.
+    # Non-string values, missing columns, and malformed JSON are left untouched
+    # so enabling this never corrupts non-JSON data.
+    def parse_json_columns!(row, columns)
+      return if columns.empty?
+      columns.each do |col|
+        v = row[col]
+        next unless v.is_a?(String)
+        begin
+          row[col] = JSON.parse(v)
+        rescue JSON::ParserError
+          # leave the original string as-is on malformed JSON
+        end
+      end
+    end
+
     def emit_record(tag, record)
       router.emit(tag, Fluent::Engine.now, record)
     end

data/lib/fluent/plugin/out_mysql_replicator_elasticsearch.rb

@@ -66,20 +66,20 @@ class Fluent::Plugin::MysqlReplicatorElasticsearchOutput < Fluent::Plugin::Outpu
 
     chunk.msgpack_each do |tag, time, record|
       tag_parts = tag.match(@tag_format)
-      target_index = tag_parts['index_name']
+      target_index = resolve_index_name(tag_parts['index_name'], time)
       target_type = tag_parts['type_name']
-      id_key = tag_parts['primary_key']
+      id_keys = tag_parts['primary_key'].to_s.split(',')
 
       if tag_parts['event'] == 'delete'
-        action = {"_index" => target_index, "_id" => record[id_key]}
+        action = {"_index" => target_index, "_id" => join_id(record, id_keys)}
         action['_type'] = target_type unless @suppress_type
         meta = { "delete" => action }
         bulk_message << Yajl::Encoder.encode(meta)
       else
         action = {"_index" => target_index}
         action['_type'] = target_type unless @suppress_type
-        if id_key && record[id_key]
-          action['_id'] = record[id_key]
+        if !id_keys.empty? && id_keys.all? {|k| !record[k].nil? }
+          action['_id'] = join_id(record, id_keys)
         end
         meta = { "index" => action }
         bulk_message << Yajl::Encoder.encode(meta)
@@ -105,6 +105,23 @@ class Fluent::Plugin::MysqlReplicatorElasticsearchOutput < Fluent::Plugin::Outpu
     http
   end
 
+  # Build the document _id from one or more primary-key columns. A single key
+  # yields its value; a composite key yields the values joined by ",".
+  def join_id(record, id_keys)
+    id_keys.map {|k| record[k] }.join(',')
+  end
+
+  # Expand strftime tokens (e.g. "%Y%m%d") in the index name using the record's
+  # event time, enabling date-based indices such as "myindex-20180831". Index
+  # names without a "%" are returned unchanged.
+  def resolve_index_name(index_name, time)
+    return index_name unless index_name && index_name.include?('%')
+    Time.at(time.to_i).strftime(index_name)
+  rescue => e
+    log.warn "mysql_replicator_elasticsearch: failed to expand index name '#{index_name}': #{e.message}"
+    index_name
+  end
+
   # Mapping types were removed in Elasticsearch 8.x and deprecated in 7.x.
   # Detect the major version once and omit "_type" for 7.x and later.
   def detect_type_suppression

data/setup_mysql_replicator_multi.sql

@@ -24,6 +24,9 @@ CREATE TABLE IF NOT EXISTS `settings` (
   `prepared_query` TEXT NOT NULL,
   `interval` int(11) NOT NULL,
   `primary_key` varchar(255) DEFAULT 'id',
+  -- Comma-separated column names whose MySQL JSON values should be parsed into nested objects.
+  -- Intended for Elasticsearch; leave empty for Solr or destinations that cannot store JSON objects.
+  `json_columns` varchar(255) DEFAULT NULL,
   `enable_delete` int(11) DEFAULT '1',
   -- On enabling 'enable_loose_insert: 1', make it faster synchronization to skip checking hash_tables.
   `enable_loose_insert` int(11) DEFAULT '0',

data/test/e2e/fluent_single.conf

@@ -8,8 +8,9 @@
   username "#{ENV['MYSQL_USER'] || 'root'}"
   password "#{ENV['MYSQL_PASSWORD'] || 'root'}"
   database "#{ENV['MYSQL_DATABASE'] || 'e2e_source'}"
-  query SELECT id, name, age FROM users
+  query SELECT id, name, age, profile FROM users
   primary_key id
+  json_columns profile
   interval 2s
   enable_delete true
   tag myindex.mytype.${event}.${primary_key}

data/test/e2e/replication_multi_e2e_test.rb

@@ -41,13 +41,19 @@ client.query("CREATE DATABASE `#{SOURCE_DB}`")
 client.query("USE `#{SOURCE_DB}`")
 client.query(<<~SQL)
   CREATE TABLE users (
-    id   INT NOT NULL AUTO_INCREMENT,
-    name VARCHAR(255) NOT NULL,
-    age  INT NOT NULL,
+    id      INT NOT NULL AUTO_INCREMENT,
+    name    VARCHAR(255) NOT NULL,
+    age     INT NOT NULL,
+    profile JSON,
     PRIMARY KEY (id)
   ) ENGINE=InnoDB DEFAULT CHARSET=utf8
 SQL
-client.query("INSERT INTO users (name, age) VALUES ('alice', 20), ('bob', 30), ('carol', 40)")
+client.query(<<~SQL)
+  INSERT INTO users (name, age, profile) VALUES
+    ('alice', 20, '{"city":"Tokyo","tags":["a","b"]}'),
+    ('bob',   30, '{"city":"Osaka","tags":["c"]}'),
+    ('carol', 40, '{"city":"Nagoya","tags":[]}')
+SQL
 
 # --- 2. Build the management database and register a replication setting ----
 step "building management database '#{MANAGER_DB}' from setup SQL"
@@ -61,11 +67,11 @@ cfg = mysql_config
 client.query(<<~SQL)
   INSERT INTO `#{MANAGER_DB}`.settings
     (is_active, name, host, port, username, password, `database`,
-     query, prepared_query, `interval`, primary_key, enable_delete)
+     query, prepared_query, `interval`, primary_key, enable_delete, json_columns)
   VALUES
     (1, '#{SETTING_NAME}', '#{cfg[:host]}', #{cfg[:port]},
      '#{cfg[:username]}', '#{client.escape(cfg[:password].to_s)}', '#{SOURCE_DB}',
-     'SELECT id, name, age FROM users ORDER BY id', '', 2, 'id', 1)
+     'SELECT id, name, age, profile FROM users ORDER BY id', '', 2, 'id', 1, 'profile')
 SQL
 
 # --- 3. Boot Fluentd --------------------------------------------------------
@@ -83,6 +89,18 @@ begin
   end
   step "  INSERT OK"
 
+  # --- 4b. JSON column is indexed as a nested object (not an escaped string) -
+  step "asserting JSON column is replicated as a nested object"
+  wait_until("user 1 profile.city == Tokyo in Elasticsearch", log_path: LOG_PATH) do
+    code, body = es_get(INDEX, TYPE, 1)
+    code == 200 && body.dig('_source', 'profile', 'city') == 'Tokyo'
+  end
+  _, body = es_get(INDEX, TYPE, 1)
+  unless body.dig('_source', 'profile', 'tags') == ['a', 'b']
+    fail_with("profile was not indexed as a nested object: #{body.dig('_source', 'profile').inspect}", LOG_PATH)
+  end
+  step "  JSON OK"
+
   # --- 5. hash_tables state is persisted ------------------------------------
   step "asserting hash_tables persistence"
   wait_until("3 rows recorded in hash_tables for '#{SETTING_NAME}'", log_path: LOG_PATH) do

data/test/e2e/replication_single_e2e_test.rb

@@ -24,13 +24,18 @@ client.query("CREATE DATABASE `#{DB}`")
 client.query("USE `#{DB}`")
 client.query(<<~SQL)
   CREATE TABLE users (
-    id   INT NOT NULL AUTO_INCREMENT,
-    name VARCHAR(255) NOT NULL,
-    age  INT NOT NULL,
+    id      INT NOT NULL AUTO_INCREMENT,
+    name    VARCHAR(255) NOT NULL,
+    age     INT NOT NULL,
+    profile JSON,
     PRIMARY KEY (id)
   ) ENGINE=InnoDB DEFAULT CHARSET=utf8
 SQL
-client.query("INSERT INTO users (name, age) VALUES ('alice', 20), ('bob', 30)")
+client.query(<<~SQL)
+  INSERT INTO users (name, age, profile) VALUES
+    ('alice', 20, '{"city":"Tokyo","tags":["a","b"]}'),
+    ('bob',   30, '{"city":"Osaka","tags":["c"]}')
+SQL
 
 # --- 2. Boot Fluentd --------------------------------------------------------
 step "starting Fluentd (#{CONF})"
@@ -49,6 +54,18 @@ begin
   end
   step "  INSERT OK"
 
+  # --- 3b. JSON column is indexed as a nested object (not an escaped string) -
+  step "asserting JSON column is replicated as a nested object"
+  wait_until("user 1 profile.city == Tokyo in Elasticsearch", log_path: LOG_PATH) do
+    code, body = es_get(INDEX, TYPE, 1)
+    code == 200 && body.dig('_source', 'profile', 'city') == 'Tokyo'
+  end
+  _, body = es_get(INDEX, TYPE, 1)
+  unless body.dig('_source', 'profile', 'tags') == ['a', 'b']
+    fail_with("profile was not indexed as a nested object: #{body.dig('_source', 'profile').inspect}", LOG_PATH)
+  end
+  step "  JSON OK"
+
   # --- 4. UPDATE is replicated ----------------------------------------------
   step "asserting UPDATE replication"
   client.query("UPDATE users SET age = 21 WHERE id = 1")

data/test/plugin/test_in_mysql_replicator.rb

@@ -30,11 +30,137 @@ class MysqlReplicatorInputTest < Test::Unit::TestCase
       tag             input.mysql
       query           SELECT id, text from search_text
       enable_delete   no
+      json_columns    geometry,attrs
     ]
     assert_equal 'localhost', d.instance.host
     assert_equal 3306, d.instance.port
     assert_equal 30, d.instance.interval
     assert_equal 'input.mysql', d.instance.tag
     assert_equal false, d.instance.enable_delete
+    assert_equal ['geometry', 'attrs'], d.instance.json_columns
+  end
+
+  def test_json_columns_defaults_to_empty
+    d = create_driver
+    assert_equal [], d.instance.json_columns
+  end
+
+  def test_parse_json_columns
+    d = create_driver
+    row = {
+      'id'       => 1,
+      'geometry' => '{"type":"Polygon","coordinates":[[136.8,35.1]]}',
+      'tags'     => '[1,2,3]',
+      'broken'   => 'not json {',
+      'plain'    => 'hello',
+      'number'   => 5,
+    }
+    d.instance.parse_json_columns!(row, ['geometry', 'tags', 'broken', 'number', 'missing'])
+
+    assert_equal({'type' => 'Polygon', 'coordinates' => [[136.8, 35.1]]}, row['geometry'])
+    assert_equal([1, 2, 3], row['tags'])
+    assert_equal('not json {', row['broken'])  # malformed JSON stays as the original string
+    assert_equal(5, row['number'])             # non-string values are untouched
+    assert_equal('hello', row['plain'])        # columns not listed are untouched
+  end
+
+  def test_parse_json_columns_noop_when_empty
+    d = create_driver
+    row = {'geometry' => '{"a":1}'}
+    d.instance.parse_json_columns!(row, [])
+    assert_equal('{"a":1}', row['geometry'])
+  end
+
+  # --- #42: delete detection must not build an integer Range from primary keys ---
+
+  def deleted_ids(previous_ids, current_ids)
+    conf = %[
+      tag   input.mysql
+      query SELECT id, text from search_test
+    ]
+    create_driver(conf).instance.detect_deleted_ids(previous_ids, current_ids)
+  end
+
+  def test_detect_deleted_ids_first_poll_reports_no_deletes
+    assert_equal [], deleted_ids([], [1, 2, 3])
+  end
+
+  def test_detect_deleted_ids_first_poll_with_string_keys_does_not_raise
+    # Regression for #42: the old `[*1...'c']` raised "bad value for range".
+    assert_nothing_raised do
+      assert_equal [], deleted_ids([], %w[a b c])
+    end
+  end
+
+  def test_detect_deleted_ids_first_poll_with_sparse_ids_has_no_phantom_deletes
+    # The old code returned `[*1...99] - [1, 50, 99]`, emitting deletes for ids
+    # that never existed. The first poll must only establish a baseline.
+    assert_equal [], deleted_ids([], [1, 50, 99])
+  end
+
+  def test_detect_deleted_ids_diffs_against_previous_snapshot
+    assert_equal [2], deleted_ids([1, 2, 3], [1, 3])
+  end
+
+  def test_detect_deleted_ids_empty_current_does_not_mass_delete
+    assert_equal [], deleted_ids([1, 2, 3], [])
+  end
+
+  # --- #4: a nested sub-query fires only for a SELECT template with ${...} ---
+
+  def nested?(value)
+    conf = %[
+      tag   input.mysql
+      query SELECT id, text from search_test
+    ]
+    create_driver(conf).instance.nested_query_value?(value)
+  end
+
+  def test_nested_query_value_true_for_select_with_placeholder
+    assert_true nested?("SELECT * FROM child WHERE parent_id = ${id}")
+  end
+
+  def test_nested_query_value_false_for_plain_text_starting_with_select
+    # Regression for #4: a data value beginning with "SELECT" must not run as SQL.
+    assert_false nested?("SELECT YOUR PLAN")
+  end
+
+  def test_nested_query_value_false_for_word_select
+    assert_false nested?("Selecting the best option")
+  end
+
+  def test_nested_query_value_false_for_select_without_placeholder
+    assert_false nested?("select count(*) from search_test")
+  end
+
+  def test_nested_query_value_false_for_non_string_values
+    assert_false nested?(12345)
+    assert_false nested?(nil)
+  end
+
+  # --- #7: composite primary key support ---
+
+  def composite_driver
+    create_driver(%[
+      tag         input.mysql
+      query       SELECT tenant_id, id, text from t
+      primary_key tenant_id,id
+    ])
+  end
+
+  def test_primary_key_defaults_to_id_array
+    assert_equal ['id'], create_driver.instance.primary_key
+  end
+
+  def test_primary_key_parses_composite_list
+    assert_equal ['tenant_id', 'id'], composite_driver.instance.primary_key
+  end
+
+  def test_extract_id_single_key_is_one_element_array
+    assert_equal [7], create_driver.instance.extract_id({'id' => 7, 'text' => 'x'})
+  end
+
+  def test_extract_id_returns_composite_values
+    assert_equal [10, 7], composite_driver.instance.extract_id({'tenant_id' => 10, 'id' => 7, 'text' => 'x'})
   end
 end

data/test/plugin/test_out_mysql_replicator_elasticsearch.rb

@@ -61,6 +61,32 @@ class MysqlReplicatorElasticsearchOutput < Test::Unit::TestCase
     assert_equal('myindex', index_cmds.first['index']['_index'])
   end
 
+  def test_expands_strftime_tokens_in_index_name
+    stub_elastic
+    time = Time.utc(2018, 8, 31, 12, 0, 0).to_i
+    driver.run(default_tag: 'myindex-%Y%m%d.mytype.insert.id') do
+      driver.feed(time, sample_record)
+    end
+    expected = "myindex-#{Time.at(time).strftime('%Y%m%d')}"
+    assert_equal(expected, index_cmds.first['index']['_index'])
+  end
+
+  def test_index_name_without_token_is_unchanged
+    stub_elastic
+    driver.run(default_tag: 'plainindex.mytype.insert.id') do
+      driver.feed(sample_record)
+    end
+    assert_equal('plainindex', index_cmds.first['index']['_index'])
+  end
+
+  def test_composite_primary_key_builds_joined_id
+    stub_elastic
+    driver.run(default_tag: 'myindex.mytype.insert.tenant_id,id') do
+      driver.feed({'tenant_id' => 10, 'id' => 7, 'text' => 'x'})
+    end
+    assert_equal('10,7', index_cmds.first['index']['_id'])
+  end
+
   def test_writes_to_speficied_type
     driver.configure("type_name mytype\n")
     stub_elastic

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mysql-replicator
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.3.0
 platform: ruby
 authors:
 - Kentaro Yoshida
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2026-06-15 00:00:00.000000000 Z
+date: 2026-06-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -112,12 +112,15 @@ files:
 - ".dockerignore"
 - ".github/workflows/ci.yml"
 - ".gitignore"
+- CHANGELOG.md
 - Gemfile
 - LICENSE
 - README.md
 - Rakefile
 - Tutorial-mysql_replicator.md
 - Tutorial-mysql_replicator_multi.md
+- ci/install_smoke.sh
+- ci/verify_plugins_load.rb
 - example/mysql_multi_table_to_elasticsearch.md
 - example/mysql_multi_table_to_solr.md
 - example/mysql_single_table_to_elasticsearch.md