fluent-plugin-record-modifier 2.1.0 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b7d9232d881589bfd365d2669fb26e8e57faadbcf3a7ced27dff2637bf017002
-  data.tar.gz: c185b00755209331f684d868af17d6ce382294c321b7405c424182d9f4b28928
+  metadata.gz: 929d54c71c0ef86c544ed2797b68f8cb77439ae848ba10d1b95139a5141fa532
+  data.tar.gz: ff1966ae5435bdc0a013e375f5c6a7e25d05c3a0c31966637befe0edda4d2f4d
 SHA512:
-  metadata.gz: 2b48f455cf36f6ed57492e77344462d4b9253b4c46068e6314c9da88f3b1b20a268c0c637306a28bc93d063f77427896f0bce2fd32568a9851458cc89a7d01c0
-  data.tar.gz: dedbdf338fc5260fca9271c09af8a06ed00ab8866a7a65b12cd6aa400500272e570f4829b9700e893cc26223a5223e02d4081c27802b3d2cf9591b332b887d49
+  metadata.gz: efa42ddd9e8e6bbc02d69a81377356e47a44a02bdbcc18cfbbfa10cc043dd85f90127d92f359f827b6f72b5f5b27a4fb805723e17a55b89f9c11eef9c4ca2f33
+  data.tar.gz: ff225acaaa97faa6223d1fe266a1be717a457eb9941db840a080c1c0a89f57deceb50808351dee4a9028c1a7630997a680743a4041508987ed114c36ec8dad6f

data/.github/workflows/linux.yml

@@ -0,0 +1,33 @@
+name: linux
+on:
+  - push
+  - pull_request
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ matrix.experimental }}
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: ['2.7', '3.0', '3.1', '3.2']
+        os:
+          - ubuntu-latest
+        experimental: [false]
+        include:
+          - ruby-version: head
+            os: ubuntu-latest
+            experimental: true
+
+    name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v2
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+    - name: unit testing
+      env:
+        CI: true
+      run: |
+        gem install bundler rake
+        bundle install --jobs 4 --retry 3
+        bundle exec rake test

data/ChangeLog

@@ -1,3 +1,8 @@
+Release 2.1.1 - 2022/08/25
+
+  * Improve whitelist_keys performance
+  * Fix documentation for named capture references
+
 Release 2.1.0 - 2019/12/18
 
   * Add <record> keys to whitelist_keys

data/README.md

@@ -124,6 +124,8 @@ then you got new record like below:
 {"key3":"bar"}
 ```
 
+Since v2.2.0, `remove_keys` supports nested key delete via [`record_accessor` syntax](https://docs.fluentd.org/plugin-helper-overview/api-plugin-helper-record_accessor).
+
 ### whitelist_keys
 
 If you want to handle the set of explicitly specified keys, you can use `whitelist_keys` of this plugin. It's exclusive with `remove_keys`.
@@ -164,7 +166,7 @@ If you want to replace specific value for keys you can use `replace` section.
     # your regexp
     expression /^(?<start>.+).{2}(?<end>.+)$/
     # replace string
-    replace \\k<start>ors\\k<end>
+    replace \k<start>ors\k<end>
   </replace>
   # replace key key2
   <replace>
@@ -173,7 +175,7 @@ If you want to replace specific value for keys you can use `replace` section.
     # your regexp
     expression /^(.{1}).{2}(.{1})$/
     # replace string
-    replace \\1ors\\2
+    replace \1ors\2
   </replace>
 </filter>
 ```
@@ -202,7 +204,7 @@ If you need own complex logic in filter, writing filter plugin is better. But if
   <record>
     _dummy_ ${if record.has_key?('foo'); record['bar'] = 'Hi!'; end; nil}
   </record>
-</filter>}
+</filter>
 ```
 
 ### record_modifier output

data/VERSION

@@ -1 +1 @@
-2.1.0
+2.2.0

data/fluent-plugin-record-modifier.gemspec

@@ -16,7 +16,7 @@ Gem::Specification.new do |gem|
   gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   gem.require_paths = ['lib']
 
-  gem.add_dependency "fluentd", [">= 1.0", "< 2"]
+  gem.add_dependency "fluentd", [">= 1.1", "< 2"]
   gem.add_development_dependency "rake", ">= 0.9.2"
-  gem.add_development_dependency("test-unit", ["~> 3.1.4"])
+  gem.add_development_dependency("test-unit", ["~> 3.4.0"])
 end

data/lib/fluent/plugin/filter_record_modifier.rb

@@ -4,6 +4,8 @@ module Fluent
   class Plugin::RecordModifierFilter < Plugin::Filter
     Fluent::Plugin.register_filter('record_modifier', self)
 
+    helpers :record_accessor
+
     config_param :prepare_value, :string, default: nil,
                  desc: <<-DESC
 Prepare values for filtering in configure phase. Prepared values can be used in <record>.
@@ -17,7 +19,7 @@ e.g. handling char encoding correctly.
 In more detail, please refer this section:
 https://github.com/repeatedly/fluent-plugin-record-modifier#char_encoding.
 DESC
-    config_param :remove_keys, :string, default: nil,
+    config_param :remove_keys, :array, default: nil,
                  desc: <<-DESC
 The logs include needless record keys in some cases.
 You can remove it by using `remove_keys` parameter.
@@ -80,7 +82,9 @@ DESC
       if @remove_keys and @whitelist_keys
         raise Fluent::ConfigError, "remove_keys and whitelist_keys are exclusive with each other."
       elsif @remove_keys
-        @remove_keys = @remove_keys.split(',').map(&:strip)
+        @remove_keys = @remove_keys.map { |key|
+          record_accessor_create(key)
+        }
       elsif @whitelist_keys
         @whitelist_keys = @whitelist_keys.split(',').map(&:strip)
         @whitelist_keys.concat(@map.keys).uniq!
@@ -98,13 +102,13 @@ DESC
       }
 
       if @remove_keys
-        @remove_keys.each { |v|
-          record.delete(v)
+        @remove_keys.each { |ra|
+          ra.delete(record)
         }
       elsif @whitelist_keys
         modified = {}
-        record.each do |k, v|
-          modified[k] = v if @whitelist_keys.include?(k)
+        @whitelist_keys.each do |key|
+          modified[key] = record[key] if record.has_key?(key)
         end
         record = modified
       end

data/lib/fluent/plugin/out_record_modifier.rb

@@ -138,8 +138,8 @@ DESC
         }
       elsif @whitelist_keys
         modified = {}
-        record.each do |k, v|
-          modified[k] = v if @whitelist_keys.include?(k)
+        @whitelist_keys.each do |key|
+          modified[key] = record[key] if record.has_key?(key)
         end
         record = modified
       end

data/test/test_filter_record_modifier.rb

@@ -114,6 +114,18 @@ class RecordModifierFilterTest < Test::Unit::TestCase
     assert_equal [{"k4" => 'v'}], d.filtered.map { |e| e.last }
   end
 
+  def test_remove_nested_keys
+    d = create_driver %[
+      remove_keys k1, $.kubernetes.test
+    ]
+
+    d.run(default_tag: @tag) do
+      d.feed("k1" => 'v', "kubernetes" => {"test" => 'v', "prod" => 'v'}, "k2" => 'v')
+    end
+
+    assert_equal [{"kubernetes" => {"prod" => 'v'}, "k2" => 'v'}], d.filtered.map { |e| e.last }
+  end
+
   def test_remove_non_whitelist_keys
     d = create_driver %[
       <record>
@@ -145,18 +157,18 @@ class RecordModifierFilterTest < Test::Unit::TestCase
   end
 
   def test_replace_values
-    d = create_driver %[
+    d = create_driver <<'CONFIG'
         <replace>
           key k1
           expression /^(?<start>.+).{2}(?<end>.+)$/
-          replace \\k<start>ors\\k<end>
+          replace \k<start>ors\k<end>
         </replace>
         <replace>
           key k2
           expression /^(.{1}).{2}(.{1})$/
-          replace \\1ors\\2
+          replace \1ors\2
         </replace>
-    ]
+CONFIG
 
     d.run(default_tag: @tag) do
       d.feed("k1" => 'hoge', "k2" => 'hoge', "k3" => 'bar')
@@ -166,13 +178,13 @@ class RecordModifierFilterTest < Test::Unit::TestCase
   end
 
   def test_does_not_replace
-    d = create_driver %[
+    d = create_driver <<'CONFIG'
         <replace>
           key k1
           expression /^(?<start>.+).{2}(?<end>.+)$/
-          replace \\k<start>ors\\k<end>
+          replace \k<start>ors\k<end>
         </replace>
-    ]
+CONFIG
 
     d.run(default_tag: @tag) do
       d.feed("k1" => 'hog')

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-record-modifier
 version: !ruby/object:Gem::Version
-  version: 2.1.0
+  version: 2.2.0
 platform: ruby
 authors:
 - Masahiro Nakagawa
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-12-18 00:00:00.000000000 Z
+date: 2023-12-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -16,7 +16,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.1'
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.1'
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -50,20 +50,21 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.1.4
+        version: 3.4.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.1.4
+        version: 3.4.0
 description: Filter plugin for modifying event record
 email: repeatedly@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/linux.yml"
 - ".travis.yml"
 - ChangeLog
 - Gemfile
@@ -79,7 +80,7 @@ homepage: https://github.com/repeatedly/fluent-plugin-record-modifier
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -94,8 +95,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.4.1
+signing_key:
 specification_version: 4
 summary: Filter plugin for modifying event record
 test_files: