fluent-plugin-record-modifier 2.1.1 → 2.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5223ce121e04b41b661a2d575a7067f69295ea5d665656cb37a23acbd71d3624
-  data.tar.gz: 00e6c78cf07e354ab0a0d77d42a5bc00f782a1bcb9ca56899440ac75dbd6f4b1
+  metadata.gz: 0162d793dd9d7c24bb4e3ec50aabfd2aa1e094b389d7da5236a9c99a60360209
+  data.tar.gz: 0e4a332894d537696733f3b0cfd824d689a187187ec86987a8bf533ef4d4fff9
 SHA512:
-  metadata.gz: 79597158486f6da9646479758d981afc494da860653006d93428abf9d5c211008f60339ca39d238a04b77f2aadd38f459f70b4dccb46c23618614cbc75795894
-  data.tar.gz: 8912416c324b79f0d4e4e6d29f8625105e1c983492e509a56484a81d5cb0c72b6270a915f087b4946d5d522b1817133a99341b9a91c241de01c4f7c501f3310b
+  metadata.gz: 21a8f63aba701b8ffa5c77cf4b1bc6fc373b015a601e658abe77de4a1763de485e5e416b8958a6e9d0ace9b4dfc3ac401fc2ab9f07189b8452daea21153a0816
+  data.tar.gz: c5450f8d89ca5f9578947e49e2723620641658db0607a69ac8cceb19d1bcb3903562c380f0bc5f831c7fe5baeaa7a9c44a54dd485dd00719d3d05f31cedaee00

data/.github/workflows/linux.yml

@@ -5,12 +5,19 @@ on:
 jobs:
   build:
     runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ matrix.experimental }}
     strategy:
       fail-fast: false
       matrix:
-        ruby: ['2.6', '2.7', '3.0', '3.1']
+        ruby: ['3.1', '3.2', '3.3', '3.4']
         os:
           - ubuntu-latest
+        experimental: [false]
+        include:
+          - ruby: head
+            os: ubuntu-latest
+            experimental: true
+
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v2

data/ChangeLog

@@ -1,3 +1,12 @@
+Release 2.2.1 - 2025/08/24
+
+  * Add `use_nil` support
+
+
+Release 2.2.0 - 2023/12/08
+
+  * `remove_keys` parameter supports nested delete
+
 Release 2.1.1 - 2022/08/25
 
   * Improve whitelist_keys performance

data/README.md

@@ -124,6 +124,8 @@ then you got new record like below:
 {"key3":"bar"}
 ```
 
+Since v2.2.0, `remove_keys` supports nested key delete via [`record_accessor` syntax](https://docs.fluentd.org/plugin-helper-overview/api-plugin-helper-record_accessor).
+
 ### whitelist_keys
 
 If you want to handle the set of explicitly specified keys, you can use `whitelist_keys` of this plugin. It's exclusive with `remove_keys`.
@@ -202,7 +204,7 @@ If you need own complex logic in filter, writing filter plugin is better. But if
   <record>
     _dummy_ ${if record.has_key?('foo'); record['bar'] = 'Hi!'; end; nil}
   </record>
-</filter>}
+</filter>
 ```
 
 ### record_modifier output

data/VERSION

@@ -1 +1 @@
-2.1.1
+2.2.1

data/fluent-plugin-record-modifier.gemspec

@@ -16,7 +16,7 @@ Gem::Specification.new do |gem|
   gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   gem.require_paths = ['lib']
 
-  gem.add_dependency "fluentd", [">= 1.0", "< 2"]
+  gem.add_dependency "fluentd", [">= 1.1", "< 2"]
   gem.add_development_dependency "rake", ">= 0.9.2"
-  gem.add_development_dependency("test-unit", ["~> 3.3"])
+  gem.add_development_dependency("test-unit", ["~> 3.4.0"])
 end

data/lib/fluent/plugin/filter_record_modifier.rb

@@ -4,6 +4,8 @@ module Fluent
   class Plugin::RecordModifierFilter < Plugin::Filter
     Fluent::Plugin.register_filter('record_modifier', self)
 
+    helpers :record_accessor
+
     config_param :prepare_value, :string, default: nil,
                  desc: <<-DESC
 Prepare values for filtering in configure phase. Prepared values can be used in <record>.
@@ -17,7 +19,7 @@ e.g. handling char encoding correctly.
 In more detail, please refer this section:
 https://github.com/repeatedly/fluent-plugin-record-modifier#char_encoding.
 DESC
-    config_param :remove_keys, :string, default: nil,
+    config_param :remove_keys, :array, default: nil,
                  desc: <<-DESC
 The logs include needless record keys in some cases.
 You can remove it by using `remove_keys` parameter.
@@ -72,7 +74,7 @@ DESC
       conf.elements.select { |element| element.name == 'record' }.each do |element|
         element.each_pair do |k, v|
           element.has_key?(k) # to suppress unread configuration warning
-          @has_tag_parts = true if v.include?('tag_parts')
+          @has_tag_parts = true if v && v.include?('tag_parts')
           @map[k] = DynamicExpander.new(k, v, @prepare_value)
         end
       end
@@ -80,7 +82,9 @@ DESC
       if @remove_keys and @whitelist_keys
         raise Fluent::ConfigError, "remove_keys and whitelist_keys are exclusive with each other."
       elsif @remove_keys
-        @remove_keys = @remove_keys.split(',').map(&:strip)
+        @remove_keys = @remove_keys.map { |key|
+          record_accessor_create(key)
+        }
       elsif @whitelist_keys
         @whitelist_keys = @whitelist_keys.split(',').map(&:strip)
         @whitelist_keys.concat(@map.keys).uniq!
@@ -98,8 +102,8 @@ DESC
       }
 
       if @remove_keys
-        @remove_keys.each { |v|
-          record.delete(v)
+        @remove_keys.each { |ra|
+          ra.delete(record)
         }
       elsif @whitelist_keys
         modified = {}
@@ -163,7 +167,7 @@ DESC
 
     class DynamicExpander
       def initialize(param_key, param_value, prepare_value)
-        if param_value.include?('${')
+        if param_value && param_value.include?('${')
           __str_eval_code__ = parse_parameter(param_value)
 
           # Use class_eval with string instead of define_method for performance.

data/lib/fluent/plugin/out_record_modifier.rb

@@ -77,7 +77,7 @@ DESC
       conf.elements.select { |element| element.name == 'record' }.each do |element|
         element.each_pair do |k, v|
           element.has_key?(k) # to suppress unread configuration warning
-          @has_tag_parts = true if v.include?('tag_parts')
+          @has_tag_parts = true if v && v.include?('tag_parts')
           @map[k] = DynamicExpander.new(k, v, @prepare_value)
         end
       end
@@ -198,7 +198,7 @@ DESC
       attr_reader :param_value
 
       def initialize(param_key, param_value, prepare_value)
-        if param_value.include?('${')
+        if param_value && param_value.include?('${')
           __str_eval_code__ = parse_parameter(param_value)
 
           # Use class_eval with string instead of define_method for performance.

data/test/test_filter_record_modifier.rb

@@ -114,6 +114,18 @@ class RecordModifierFilterTest < Test::Unit::TestCase
     assert_equal [{"k4" => 'v'}], d.filtered.map { |e| e.last }
   end
 
+  def test_remove_nested_keys
+    d = create_driver %[
+      remove_keys k1, $.kubernetes.test
+    ]
+
+    d.run(default_tag: @tag) do
+      d.feed("k1" => 'v', "kubernetes" => {"test" => 'v', "prod" => 'v'}, "k2" => 'v')
+    end
+
+    assert_equal [{"kubernetes" => {"prod" => 'v'}, "k2" => 'v'}], d.filtered.map { |e| e.last }
+  end
+
   def test_remove_non_whitelist_keys
     d = create_driver %[
       <record>
@@ -214,4 +226,23 @@ CONFIG
       ], d.filtered.map { |e| e.last }
     end
   end
+
+  def test_use_nil
+    require "fluent/version"
+    if Gem::Version.new(Fluent::VERSION) < Gem::Version.new("1.8.0")
+      omit "use_nil is only available in Fluentd 1.8.0 and higher"
+    end
+
+    d = create_driver %q[
+      <record>
+        test_key "#{use_nil}"
+      </record>
+    ]
+
+    d.run(default_tag: @tag) do
+      d.feed("k" => "v")
+    end
+
+    assert_equal [{"k" => "v", "test_key" => nil}], d.filtered.map(&:last)
+  end
 end

data/test/test_out_record_modifier.rb

@@ -1,6 +1,7 @@
+require 'fluent/test'
 require 'fluent/test/driver/output'
 require 'fluent/plugin/out_record_modifier'
-
+require 'test/unit'
 
 class RecordModifierOutputTest < Test::Unit::TestCase
   def setup
@@ -139,4 +140,24 @@ class RecordModifierOutputTest < Test::Unit::TestCase
 
     assert_equal [{"k1" => 'v', "k2" => 'v', 'foo' => 'bar'}], d.events.map { |e| e.last }
   end
+
+  def test_use_nil
+    require "fluent/version"
+    if Gem::Version.new(Fluent::VERSION) < Gem::Version.new("1.8.0")
+      omit "use_nil is only available in Fluentd 1.8.0 and higher"
+    end
+
+    d = create_driver %q[
+      tag foo.filtered
+      <record>
+        test_key "#{use_nil}"
+      </record>
+    ]
+
+    d.run(default_tag: "test_tags") do
+      d.feed("k" => "v")
+    end
+
+    assert_equal [{"k" => "v", "test_key" => nil}], d.events.map(&:last)
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-record-modifier
 version: !ruby/object:Gem::Version
-  version: 2.1.1
+  version: 2.2.1
 platform: ruby
 authors:
 - Masahiro Nakagawa
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-08-24 00:00:00.000000000 Z
+date: 2025-08-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -16,7 +16,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.1'
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.1'
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -50,14 +50,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.3'
+        version: 3.4.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.3'
+        version: 3.4.0
 description: Filter plugin for modifying event record
 email: repeatedly@gmail.com
 executables: []
@@ -95,7 +95,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
+rubygems_version: 3.4.1
 signing_key:
 specification_version: 4
 summary: Filter plugin for modifying event record