fluent-plugin-mysql-replicator 1.3.0 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 50b2ac9d97e49b3febf1c20fc98c815b3d6a919d299a24511a72b8b1637d5585
-  data.tar.gz: f39b735a34afd31a77f5e82339c449265dd4ef455456a85e2902dab91db79130
+  metadata.gz: 26f611dc962db43660ca6004ce2b634e9a863d554ff25328d7d39e1bc4c5a14e
+  data.tar.gz: dd4834e5c827564da74189882a605381fbe9be31b1755cc772ceb9bed0c09937
 SHA512:
-  metadata.gz: d57b725ee936f2ffc8ae55631e75759c68de863022cb5e5740be87eae6a756b9ddc5a531825bb603a02f6a7f57752ce75d389ff3fd777ba86e5126f83f392710
-  data.tar.gz: 8b644659039eae41a70dbe27f206673fb1420372321c00af802925c52b7d8fe3b5abb880d0aaf059521ceb80fbd414c1a41a7b4fef6c6dd8fd2a05ef746fdf05
+  metadata.gz: efc707ef5d9f6606baa7d573d974701bd1cedb4e34bf6a67ebda681c313b190d88ca700c505857526694b216759093cf5f834486c065074a99adb3705d5b076f
+  data.tar.gz: 17bec8dd052fb658430c1cd4d15c7731c5f7afaf27a62add1e56b6e6134f928bc5934d5db825a2918a894b0ff2f3504fe9b120ff19eaa9ca89f3c8f93d1d84b5

data/CHANGELOG.md

@@ -5,6 +5,25 @@ All notable changes to this project are documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.4.0] - 2026-06-17
+
+### Added
+- Plugin-managed Elasticsearch index templates. New `template_name`,
+  `template_file`, `template_overwrite`, and `use_legacy_template` options on
+  `mysql_replicator_elasticsearch` install an index template on startup, so newly
+  created indices — including future date-rolled ones — get the desired mapping
+  before the first document locks in dynamic mapping. This enables mappings that
+  cannot be changed after the fact: `keyword` / non-analyzed fields ([#20]) and
+  `geo_point` ([#36]). `use_legacy_template` defaults to `false` (composable
+  `PUT /_index_template`, Elasticsearch >= 7.8); set it to `true` for the legacy
+  `PUT /_template` API (Elasticsearch 6.x+).
+
+### Documentation
+- Document that MySQL `DECIMAL` columns are emitted as strings (because
+  `BigDecimal` cannot cross Fluentd's msgpack buffer, which also preserves exact
+  precision) and should be mapped as `double` or `scaled_float` in an index
+  template, where Elasticsearch coerces the numeric string at index time. ([#36])
+
 ## [1.3.0] - 2026-06-16
 
 ### Added
@@ -85,6 +104,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - First 1.0 release, targeting the Fluentd v0.14+ plugin API. Earlier 0.x
   history is available in the git log.
 
+[1.4.0]: https://github.com/y-ken/fluent-plugin-mysql-replicator/compare/v1.3.0...v1.4.0
 [1.3.0]: https://github.com/y-ken/fluent-plugin-mysql-replicator/compare/v1.2.0...v1.3.0
 [1.2.0]: https://github.com/y-ken/fluent-plugin-mysql-replicator/compare/v1.1.0...v1.2.0
 [1.1.0]: https://github.com/y-ken/fluent-plugin-mysql-replicator/compare/v1.0.3...v1.1.0
@@ -96,7 +116,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 [#4]: https://github.com/y-ken/fluent-plugin-mysql-replicator/issues/4
 [#7]: https://github.com/y-ken/fluent-plugin-mysql-replicator/pull/7
 [#18]: https://github.com/y-ken/fluent-plugin-mysql-replicator/pull/18
+[#20]: https://github.com/y-ken/fluent-plugin-mysql-replicator/issues/20
 [#27]: https://github.com/y-ken/fluent-plugin-mysql-replicator/issues/27
+[#36]: https://github.com/y-ken/fluent-plugin-mysql-replicator/issues/36
 [#39]: https://github.com/y-ken/fluent-plugin-mysql-replicator/pull/39
 [#40]: https://github.com/y-ken/fluent-plugin-mysql-replicator/issues/40
 [#42]: https://github.com/y-ken/fluent-plugin-mysql-replicator/issues/42

data/README.md

@@ -169,6 +169,70 @@ as before.
 This applies to `mysql_replicator`; `mysql_replicator_multi` still expects a
 single-column primary key.
 
+## Index templates (mappings)
+
+By default Elasticsearch infers field types from the first document (dynamic
+mapping), and an existing field's mapping cannot be changed afterwards. To
+control a field's mapping — e.g. `keyword` for a non-analyzed field, or
+`geo_point`, which dynamic mapping never infers — install an **index template**
+so it is applied to indices *before* the first document is written.
+
+`mysql_replicator_elasticsearch` can install a template on startup. The option
+names mirror `fluent-plugin-elasticsearch` (the default here uses the modern
+composable API, whereas fluent-plugin-elasticsearch defaults to legacy):
+
+```
+<match replicator.**>
+  @type mysql_replicator_elasticsearch
+  # ...
+  template_name        myindex_template
+  template_file        /etc/fluent/myindex_template.json
+  template_overwrite   false   # set true to replace an existing template
+  use_legacy_template  false   # default. false: PUT /_index_template (ES >= 7.8); true: PUT /_template (legacy, ES 6.x+)
+</match>
+```
+
+`template_name` and `template_file` must be set together. The template is a
+**server-side rule**, so once installed it is applied automatically to every new
+index whose name matches its `index_patterns` — **including future date-rolled
+indices** (see *Date-based index names* above) — with no per-write work.
+
+Example `template_file` (composable format, the default) mapping a non-analyzed
+field as `keyword`, a coordinate field as `geo_point`, and a `DECIMAL` column as
+a numeric (see below):
+
+```json
+{
+  "index_patterns": ["myindex-*"],
+  "template": {
+    "mappings": {
+      "properties": {
+        "message":  { "type": "keyword" },
+        "location": { "type": "geo_point" },
+        "price":    { "type": "scaled_float", "scaling_factor": 100 }
+      }
+    }
+  }
+}
+```
+
+For Elasticsearch 6.x — or to reuse an existing `fluent-plugin-elasticsearch`
+legacy template — set `use_legacy_template true` and put `settings`/`mappings`
+at the top level (not under `template`).
+
+### Numeric columns (DECIMAL)
+
+MySQL `DECIMAL` columns are sent to Elasticsearch **as strings** — not because of
+a type-inference issue, but because `BigDecimal` cannot cross Fluentd's msgpack
+buffer between the input and output plugins, so the value is stringified (which
+also preserves its exact precision). Under the default dynamic mapping such a
+field is therefore indexed as text.
+
+To index it as a number, map the field in your template as `double`, or — to keep
+exact fixed-point precision — `scaled_float` (as `price` above). Elasticsearch's
+`coerce` (enabled by default for numeric types) converts the numeric string to a
+number at index time, so no value conversion is needed in the plugin.
+
 ## Output example
 
 It is a example when detecting insert/update/delete events.

data/fluent-plugin-mysql-replicator.gemspec

@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-mysql-replicator"
-  s.version     = "1.3.0"
+  s.version     = "1.4.0"
   s.authors     = ["Kentaro Yoshida"]
   s.email       = ["y.ken.studio@gmail.com"]
   s.homepage    = "https://github.com/y-ken/fluent-plugin-mysql-replicator"

data/lib/fluent/plugin/out_mysql_replicator_elasticsearch.rb

@@ -17,6 +17,20 @@ class Fluent::Plugin::MysqlReplicatorElasticsearchOutput < Fluent::Plugin::Outpu
   config_param :username, :string, :default => nil
   config_param :password, :string, :default => nil, :secret => true
 
+  # Optional: install an Elasticsearch index template on startup so that newly
+  # created indices (including future date-rolled ones) get the desired mapping
+  # (e.g. geo_point or keyword fields) before the first document locks in
+  # dynamic mapping. template_name and template_file must be set together.
+  #
+  # Parameter names mirror fluent-plugin-elasticsearch. The default uses the
+  # modern composable API (fluent-plugin-elasticsearch defaults to legacy):
+  #   use_legacy_template false (default) -> PUT /_index_template/<name>  (composable, ES >= 7.8)
+  #   use_legacy_template true            -> PUT /_template/<name>        (legacy, ES 6.x+)
+  config_param :template_name, :string, :default => nil
+  config_param :template_file, :string, :default => nil
+  config_param :template_overwrite, :bool, :default => false
+  config_param :use_legacy_template, :bool, :default => false
+
   config_section :buffer do
     config_set_default :@type, DEFAULT_BUFFER_TYPE
   end
@@ -35,12 +49,21 @@ class Fluent::Plugin::MysqlReplicatorElasticsearchOutput < Fluent::Plugin::Outpu
     else
       @tag_format = Regexp.new(conf['tag_format'])
     end
+
+    if @template_name.nil? != @template_file.nil?
+      raise Fluent::ConfigError, "mysql_replicator_elasticsearch: 'template_name' and 'template_file' must be set together"
+    end
+    if @template_file && !File.exist?(@template_file)
+      raise Fluent::ConfigError, "mysql_replicator_elasticsearch: template_file not found: #{@template_file}"
+    end
   end
 
   def start
     super
     # nil means "not yet detected"; resolved on the first write.
     @suppress_type = nil
+    @es_version = nil
+    @template_installed = false
   end
 
   def format(tag, time, record)
@@ -61,6 +84,7 @@ class Fluent::Plugin::MysqlReplicatorElasticsearchOutput < Fluent::Plugin::Outpu
 
   def write(chunk)
     detect_type_suppression if @suppress_type.nil?
+    install_template_once if @template_name
 
     bulk_message = []
 
@@ -123,25 +147,76 @@ class Fluent::Plugin::MysqlReplicatorElasticsearchOutput < Fluent::Plugin::Outpu
   end
 
   # Mapping types were removed in Elasticsearch 8.x and deprecated in 7.x.
-  # Detect the major version once and omit "_type" for 7.x and later.
+  # Detect the version once and omit "_type" for 7.x and later.
   def detect_type_suppression
-    major = elasticsearch_major_version
+    @es_version = elasticsearch_version
+    major = @es_version&.first
     @suppress_type = !major.nil? && major >= 7
     if major
-      log.info "mysql_replicator_elasticsearch: detected Elasticsearch #{major}.x, suppress_type=#{@suppress_type}"
+      log.info "mysql_replicator_elasticsearch: detected Elasticsearch #{@es_version.join('.')}, suppress_type=#{@suppress_type}"
     else
       log.warn "mysql_replicator_elasticsearch: could not detect Elasticsearch version, sending '_type' (assuming 6.x)"
     end
   end
 
-  def elasticsearch_major_version
+  # Returns the Elasticsearch version as [major, minor], or nil if undetectable.
+  def elasticsearch_version
     request = Net::HTTP::Get.new('/')
     request.basic_auth(@username, @password) if @username && @password
     response = new_http.request(request)
     number = Yajl::Parser.parse(response.body).dig('version', 'number')
-    number.to_s.split('.').first.to_i
+    return nil if number.nil?
+    number.to_s.split('.').first(2).map(&:to_i)
   rescue => e
     log.warn "mysql_replicator_elasticsearch: version detection failed: #{e.message}"
     nil
   end
+
+  # Install the configured index template once, on the first write. The template
+  # is a server-side rule, so Elasticsearch applies it to every new index whose
+  # name matches its index_patterns -- including future date-rolled indices --
+  # without any further action here. Failures are logged but never abort indexing.
+  def install_template_once
+    return if @template_installed
+    @template_installed = true
+    install_template
+  rescue => e
+    log.warn "mysql_replicator_elasticsearch: failed to install index template '#{@template_name}': #{e.message}"
+  end
+
+  def install_template
+    if !@use_legacy_template && !composable_templates_supported?
+      log.warn "mysql_replicator_elasticsearch: composable index templates require Elasticsearch >= 7.8; skipping template '#{@template_name}' (detected #{@es_version&.join('.') || 'unknown'}). Set 'use_legacy_template true' for older Elasticsearch."
+      return
+    end
+    if !@template_overwrite && template_exists?
+      log.info "mysql_replicator_elasticsearch: index template '#{@template_name}' already exists; skipping (set 'template_overwrite true' to replace)"
+      return
+    end
+    put_template(File.read(@template_file))
+    log.info "mysql_replicator_elasticsearch: installed index template '#{@template_name}' (#{@use_legacy_template ? 'legacy' : 'composable'})"
+  end
+
+  def composable_templates_supported?
+    return false if @es_version.nil?
+    major, minor = @es_version
+    major > 7 || (major == 7 && minor >= 8)
+  end
+
+  def template_path
+    @use_legacy_template ? "/_template/#{@template_name}" : "/_index_template/#{@template_name}"
+  end
+
+  def template_exists?
+    request = Net::HTTP::Get.new(template_path)
+    request.basic_auth(@username, @password) if @username && @password
+    new_http.request(request).code.to_i == 200
+  end
+
+  def put_template(body)
+    request = Net::HTTP::Put.new(template_path, {'content-type' => 'application/json; charset=utf-8'})
+    request.basic_auth(@username, @password) if @username && @password
+    request.body = body
+    new_http.request(request).value
+  end
 end

data/test/plugin/test_out_mysql_replicator_elasticsearch.rb

@@ -1,6 +1,7 @@
 require 'helper'
 require 'fluent/test/driver/output'
 require 'webmock/test_unit'
+require 'tempfile'
 
 WebMock.disable_net_connect!
 
@@ -13,6 +14,19 @@ class MysqlReplicatorElasticsearchOutput < Test::Unit::TestCase
     @tag = 'myindex.mytype.insert.id'
   end
 
+  def teardown
+    WebMock.reset!
+    (@tmpfiles || []).each {|f| f.close! rescue nil }
+  end
+
+  def write_template_file(body = '{"index_patterns":["myindex-*"],"mappings":{"properties":{"loc":{"type":"geo_point"}}}}')
+    file = Tempfile.new(['tmpl', '.json'])
+    file.write(body)
+    file.flush
+    (@tmpfiles ||= []) << file
+    file.path
+  end
+
   def driver(conf='')
     @driver ||= Fluent::Test::Driver::Output.new(Fluent::Plugin::MysqlReplicatorElasticsearchOutput).configure(conf)
   end
@@ -187,6 +201,49 @@ class MysqlReplicatorElasticsearchOutput < Test::Unit::TestCase
     }
   end
 
+  def test_installs_composable_template_by_default
+    stub_elastic
+    stub_elastic_version("http://localhost:9200/", "8.18.0")
+    stub_request(:get, "http://localhost:9200/_index_template/my_tmpl").to_return(:status => 404)
+    put_tmpl = stub_request(:put, "http://localhost:9200/_index_template/my_tmpl").to_return(:status => 200, :body => "{}")
+    driver.configure("template_name my_tmpl\ntemplate_file #{write_template_file}\n")
+    driver.run(default_tag: @tag) { driver.feed(sample_record) }
+    assert_requested(put_tmpl)
+  end
+
+  def test_installs_legacy_template_when_enabled
+    stub_elastic # version 6.8.23
+    stub_request(:get, "http://localhost:9200/_template/my_tmpl").to_return(:status => 404)
+    put_tmpl = stub_request(:put, "http://localhost:9200/_template/my_tmpl").to_return(:status => 200, :body => "{}")
+    driver.configure("template_name my_tmpl\ntemplate_file #{write_template_file}\nuse_legacy_template true\n")
+    driver.run(default_tag: @tag) { driver.feed(sample_record) }
+    assert_requested(put_tmpl)
+  end
+
+  def test_skips_composable_template_on_old_elasticsearch
+    stub_elastic # version 6.8.23 (< 7.8), default composable
+    put_tmpl = stub_request(:put, "http://localhost:9200/_index_template/my_tmpl")
+    driver.configure("template_name my_tmpl\ntemplate_file #{write_template_file}\n")
+    driver.run(default_tag: @tag) { driver.feed(sample_record) }
+    assert_not_requested(put_tmpl)
+  end
+
+  def test_skips_template_when_it_exists_and_no_overwrite
+    stub_elastic
+    stub_elastic_version("http://localhost:9200/", "8.18.0")
+    stub_request(:get, "http://localhost:9200/_index_template/my_tmpl").to_return(:status => 200, :body => "{}")
+    put_tmpl = stub_request(:put, "http://localhost:9200/_index_template/my_tmpl")
+    driver.configure("template_name my_tmpl\ntemplate_file #{write_template_file}\n")
+    driver.run(default_tag: @tag) { driver.feed(sample_record) }
+    assert_not_requested(put_tmpl)
+  end
+
+  def test_template_name_without_file_raises
+    assert_raise(Fluent::ConfigError) do
+      driver.configure("template_name my_tmpl\n")
+    end
+  end
+
   def test_writes_to_https_host
     driver.configure("ssl true\n")
     elastic_request = stub_elastic("https://localhost:9200/_bulk")

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mysql-replicator
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.4.0
 platform: ruby
 authors:
 - Kentaro Yoshida
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2026-06-16 00:00:00.000000000 Z
+date: 2026-06-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake