fluent-plugin-mysql-replicator 1.2.0 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ea8d93c147df39ff2c34ac573a5e981c1e0a46da8445395eaf38cdc44abf7c0e
-  data.tar.gz: f190b88e93c4318b9c2ec42863186f7dc1f0a3a911bb60e2b689a84687bffdcd
+  metadata.gz: 50b2ac9d97e49b3febf1c20fc98c815b3d6a919d299a24511a72b8b1637d5585
+  data.tar.gz: f39b735a34afd31a77f5e82339c449265dd4ef455456a85e2902dab91db79130
 SHA512:
-  metadata.gz: bfbf04041525c768d56c7b0bb3d2cd69ca23076eca57d162277d81a06c1cb4290e4b0d402e46e4b136ebf2d45b8fdca59555c1406c9763336b67f3cf2b4be35d
-  data.tar.gz: 2872f2e0d15084d3680a77498411aa5165acb000b2b5744be19686a2dcd89b41ccee8169956203bb3b9c9a004b0aba6d16d741147215b518e0690889b220ab98
+  metadata.gz: d57b725ee936f2ffc8ae55631e75759c68de863022cb5e5740be87eae6a756b9ddc5a531825bb603a02f6a7f57752ce75d389ff3fd777ba86e5126f83f392710
+  data.tar.gz: 8b644659039eae41a70dbe27f206673fb1420372321c00af802925c52b7d8fe3b5abb880d0aaf059521ceb80fbd414c1a41a7b4fef6c6dd8fd2a05ef746fdf05

data/CHANGELOG.md

@@ -5,6 +5,18 @@ All notable changes to this project are documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.3.0] - 2026-06-16
+
+### Added
+- Date-based Elasticsearch index names. If the index-name segment of the tag
+  contains `strftime` tokens (e.g. `%Y%m%d`), they are expanded using the
+  record's event time, enabling Logstash-style dated indices such as
+  `myindex-20180831`. Index names without a `%` are unchanged. ([#27])
+- Composite primary key support in `mysql_replicator`. `primary_key` now accepts
+  a comma-separated list of columns; the combination is used for change
+  detection and as the Elasticsearch document `_id` (values joined by `,`). A
+  single-column key behaves exactly as before. ([#7])
+
 ## [1.2.0] - 2026-06-16
 
 ### Added
@@ -73,6 +85,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - First 1.0 release, targeting the Fluentd v0.14+ plugin API. Earlier 0.x
   history is available in the git log.
 
+[1.3.0]: https://github.com/y-ken/fluent-plugin-mysql-replicator/compare/v1.2.0...v1.3.0
 [1.2.0]: https://github.com/y-ken/fluent-plugin-mysql-replicator/compare/v1.1.0...v1.2.0
 [1.1.0]: https://github.com/y-ken/fluent-plugin-mysql-replicator/compare/v1.0.3...v1.1.0
 [1.0.3]: https://github.com/y-ken/fluent-plugin-mysql-replicator/compare/v1.0.2...v1.0.3
@@ -81,7 +94,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 [1.0.0]: https://github.com/y-ken/fluent-plugin-mysql-replicator/releases/tag/v1.0.0
 
 [#4]: https://github.com/y-ken/fluent-plugin-mysql-replicator/issues/4
+[#7]: https://github.com/y-ken/fluent-plugin-mysql-replicator/pull/7
 [#18]: https://github.com/y-ken/fluent-plugin-mysql-replicator/pull/18
+[#27]: https://github.com/y-ken/fluent-plugin-mysql-replicator/issues/27
 [#39]: https://github.com/y-ken/fluent-plugin-mysql-replicator/pull/39
 [#40]: https://github.com/y-ken/fluent-plugin-mysql-replicator/issues/40
 [#42]: https://github.com/y-ken/fluent-plugin-mysql-replicator/issues/42

data/README.md

@@ -122,6 +122,53 @@ Notes:
 * Malformed JSON and non-string values are left untouched, so enabling the option
   never corrupts non-JSON data.
 
+## Date-based index names
+
+`mysql_replicator_elasticsearch` resolves the target index name from the tag
+(via `tag_format`). If that index-name segment contains `strftime` tokens such
+as `%Y%m%d`, they are expanded using the record's event time, so you can create
+Logstash-style dated indices like `myindex-20180831`.
+
+Put the tokens in the index-name part of the input plugin's `tag` (the segment
+must not contain `.`, so use `%Y%m%d` or `%Y-%m-%d`):
+
+```
+<source>
+  @type mysql_replicator
+  # ...
+  tag   myindex-%Y%m%d.mytype.${event}.${primary_key}
+</source>
+```
+
+Index names that contain no `%` are left unchanged, so this is fully backward
+compatible.
+
+> **Note on deletions:** delete events target the index computed from the delete
+> event's own time, so date-rotated indices are best suited to insert-only data
+> (a record inserted on a previous day lives in that day's index).
+
+## Composite primary keys
+
+`primary_key` accepts a comma-separated list of columns, so tables keyed by more
+than one column are supported:
+
+```
+<source>
+  @type        mysql_replicator
+  # ...
+  query        SELECT tenant_id, id, name FROM items
+  primary_key  tenant_id,id
+</source>
+```
+
+Change detection (insert/update/delete) then keys on the combination of those
+columns, and the Elasticsearch document `_id` becomes their values joined by `,`
+(e.g. `10,7`). A single-column `primary_key` (the default `id`) behaves exactly
+as before.
+
+This applies to `mysql_replicator`; `mysql_replicator_multi` still expects a
+single-column primary key.
+
 ## Output example
 
 It is a example when detecting insert/update/delete events.

data/fluent-plugin-mysql-replicator.gemspec

@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-mysql-replicator"
-  s.version     = "1.2.0"
+  s.version     = "1.3.0"
   s.authors     = ["Kentaro Yoshida"]
   s.email       = ["y.ken.studio@gmail.com"]
   s.homepage    = "https://github.com/y-ken/fluent-plugin-mysql-replicator"

data/lib/fluent/plugin/in_mysql_replicator.rb

@@ -17,7 +17,10 @@ module Fluent::Plugin
     config_param :encoding, :string, :default => 'utf8'
     config_param :query, :string
     config_param :prepared_query, :string, :default => nil
-    config_param :primary_key, :string, :default => 'id'
+    # A single column name, or a comma-separated list for a composite key
+    # (e.g. "tenant_id,id"). The id used for change detection and the
+    # Elasticsearch document _id is the combination of these columns.
+    config_param :primary_key, :array, :default => ['id']
     config_param :interval, :string, :default => '1m'
     config_param :enable_delete, :bool, :default => true
     # Comma-separated column names whose MySQL JSON values should be parsed into
@@ -73,7 +76,8 @@ module Fluent::Plugin
         end
         rows, con = query(@query, con)
         rows.each do |row|
-          current_ids << row[@primary_key]
+          id = extract_id(row)
+          current_ids << id
           current_hash = Digest::SHA1.hexdigest(row.flatten.join)
           row.each {|k, v| row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
           parse_json_columns!(row, @json_columns)
@@ -86,18 +90,18 @@ module Fluent::Plugin
             end
             prepared_con.close
           end
-          if row[@primary_key].nil?
-            log.error "mysql_replicator: missing primary_key. :tag=>#{tag} :primary_key=>#{primary_key}"
+          if id.any?(&:nil?)
+            log.error "mysql_replicator: missing primary_key. :tag=>#{tag} :primary_key=>#{@primary_key.join(',')} :id=>#{id}"
             break
           end
-          if !table_hash.include?(row[@primary_key])
+          if !table_hash.include?(id)
             tag = format_tag(@tag, {:event => :insert})
             emit_record(tag, row)
-          elsif table_hash[row[@primary_key]] != current_hash
+          elsif table_hash[id] != current_hash
             tag = format_tag(@tag, {:event => :update})
             emit_record(tag, row)
           end
-          table_hash[row[@primary_key]] = current_hash
+          table_hash[id] = current_hash
           rows_count += 1
         end
         con.close
@@ -108,7 +112,7 @@ module Fluent::Plugin
             hash_delete_by_list(table_hash, deleted_ids)
             deleted_ids.each do |id|
               tag = format_tag(@tag, {:event => :delete})
-              emit_record(tag, {@primary_key => id})
+              emit_record(tag, Hash[@primary_key.zip(id)])
             end
           end
         end
@@ -122,6 +126,12 @@ module Fluent::Plugin
       deleted_keys.each{|k| hash.delete(k)}
     end
 
+    # A row's id is the array of its primary-key column values, supporting
+    # composite keys. It is a single-element array for a single-column key.
+    def extract_id(row)
+      @primary_key.map {|col| row[col] }
+    end
+
     # Returns the primary keys that disappeared since the previous poll.
     #
     # The first poll only establishes a baseline: there is no previous snapshot
@@ -152,7 +162,7 @@ module Fluent::Plugin
     end
 
     def format_tag(tag, param)
-      pattern = {'${event}' => param[:event].to_s, '${primary_key}' => @primary_key}
+      pattern = {'${event}' => param[:event].to_s, '${primary_key}' => @primary_key.join(',')}
       tag.gsub(/(\${[a-z_]+})/) do
         log.warn "mysql_replicator: missing placeholder. :tag=>#{tag} :placeholder=>#{$1}" unless pattern.include?($1)
         pattern[$1]

data/lib/fluent/plugin/out_mysql_replicator_elasticsearch.rb

@@ -66,20 +66,20 @@ class Fluent::Plugin::MysqlReplicatorElasticsearchOutput < Fluent::Plugin::Outpu
 
     chunk.msgpack_each do |tag, time, record|
       tag_parts = tag.match(@tag_format)
-      target_index = tag_parts['index_name']
+      target_index = resolve_index_name(tag_parts['index_name'], time)
       target_type = tag_parts['type_name']
-      id_key = tag_parts['primary_key']
+      id_keys = tag_parts['primary_key'].to_s.split(',')
 
       if tag_parts['event'] == 'delete'
-        action = {"_index" => target_index, "_id" => record[id_key]}
+        action = {"_index" => target_index, "_id" => join_id(record, id_keys)}
         action['_type'] = target_type unless @suppress_type
         meta = { "delete" => action }
         bulk_message << Yajl::Encoder.encode(meta)
       else
         action = {"_index" => target_index}
         action['_type'] = target_type unless @suppress_type
-        if id_key && record[id_key]
-          action['_id'] = record[id_key]
+        if !id_keys.empty? && id_keys.all? {|k| !record[k].nil? }
+          action['_id'] = join_id(record, id_keys)
         end
         meta = { "index" => action }
         bulk_message << Yajl::Encoder.encode(meta)
@@ -105,6 +105,23 @@ class Fluent::Plugin::MysqlReplicatorElasticsearchOutput < Fluent::Plugin::Outpu
     http
   end
 
+  # Build the document _id from one or more primary-key columns. A single key
+  # yields its value; a composite key yields the values joined by ",".
+  def join_id(record, id_keys)
+    id_keys.map {|k| record[k] }.join(',')
+  end
+
+  # Expand strftime tokens (e.g. "%Y%m%d") in the index name using the record's
+  # event time, enabling date-based indices such as "myindex-20180831". Index
+  # names without a "%" are returned unchanged.
+  def resolve_index_name(index_name, time)
+    return index_name unless index_name && index_name.include?('%')
+    Time.at(time.to_i).strftime(index_name)
+  rescue => e
+    log.warn "mysql_replicator_elasticsearch: failed to expand index name '#{index_name}': #{e.message}"
+    index_name
+  end
+
   # Mapping types were removed in Elasticsearch 8.x and deprecated in 7.x.
   # Detect the major version once and omit "_type" for 7.x and later.
   def detect_type_suppression

data/test/plugin/test_in_mysql_replicator.rb

@@ -137,4 +137,30 @@ class MysqlReplicatorInputTest < Test::Unit::TestCase
     assert_false nested?(12345)
     assert_false nested?(nil)
   end
+
+  # --- #7: composite primary key support ---
+
+  def composite_driver
+    create_driver(%[
+      tag         input.mysql
+      query       SELECT tenant_id, id, text from t
+      primary_key tenant_id,id
+    ])
+  end
+
+  def test_primary_key_defaults_to_id_array
+    assert_equal ['id'], create_driver.instance.primary_key
+  end
+
+  def test_primary_key_parses_composite_list
+    assert_equal ['tenant_id', 'id'], composite_driver.instance.primary_key
+  end
+
+  def test_extract_id_single_key_is_one_element_array
+    assert_equal [7], create_driver.instance.extract_id({'id' => 7, 'text' => 'x'})
+  end
+
+  def test_extract_id_returns_composite_values
+    assert_equal [10, 7], composite_driver.instance.extract_id({'tenant_id' => 10, 'id' => 7, 'text' => 'x'})
+  end
 end

data/test/plugin/test_out_mysql_replicator_elasticsearch.rb

@@ -61,6 +61,32 @@ class MysqlReplicatorElasticsearchOutput < Test::Unit::TestCase
     assert_equal('myindex', index_cmds.first['index']['_index'])
   end
 
+  def test_expands_strftime_tokens_in_index_name
+    stub_elastic
+    time = Time.utc(2018, 8, 31, 12, 0, 0).to_i
+    driver.run(default_tag: 'myindex-%Y%m%d.mytype.insert.id') do
+      driver.feed(time, sample_record)
+    end
+    expected = "myindex-#{Time.at(time).strftime('%Y%m%d')}"
+    assert_equal(expected, index_cmds.first['index']['_index'])
+  end
+
+  def test_index_name_without_token_is_unchanged
+    stub_elastic
+    driver.run(default_tag: 'plainindex.mytype.insert.id') do
+      driver.feed(sample_record)
+    end
+    assert_equal('plainindex', index_cmds.first['index']['_index'])
+  end
+
+  def test_composite_primary_key_builds_joined_id
+    stub_elastic
+    driver.run(default_tag: 'myindex.mytype.insert.tenant_id,id') do
+      driver.feed({'tenant_id' => 10, 'id' => 7, 'text' => 'x'})
+    end
+    assert_equal('10,7', index_cmds.first['index']['_id'])
+  end
+
   def test_writes_to_speficied_type
     driver.configure("type_name mytype\n")
     stub_elastic

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mysql-replicator
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.3.0
 platform: ruby
 authors:
 - Kentaro Yoshida