fluent-plugin-filter-list 0.6.0 → 0.7.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a6ecc0716888d8def1ffa0bdc4906c27aa610b6892c12d81e04fbf5f6dacd49d
-  data.tar.gz: 682c3cc3a8b27a86bb0f4cf40cc2e994444e9369d49470c611d0e47cef057425
+  metadata.gz: 43e0a813287433d553675fa915bf85f371e693a06284f9b6b459de6541009798
+  data.tar.gz: 2d0774d211c5d2ea449dd25be10ea238a0208bc3ed051923ab43bd2bc0c14e85
 SHA512:
-  metadata.gz: c52e32c0fbf6c82410adab0de450957aa7fd2567d6d69e167c8e047c3e4f64ed464b8090134dc07c9e29e81c89cd1917db18039574d6d2e5e0453962d2ed2e1e
-  data.tar.gz: 82bea7aa160a072d49356ce8e4840345d27bd547b2993d49efcae588acc4042dd24097bb43787f05a3ac93e5a9e0f04e1c6f5b02b06dfb45e91047636717355e
+  metadata.gz: 63650ac66d20e8d8df80e27c8e3e677111bab9201dd375aa96ea57088e6fb32cc66a176b2f2d468630ab0f553dcddca8188dee724ca87ded717cedbf7cb5efb2
+  data.tar.gz: 26fe217237f2ea7c2406894ff4991646560234c62073efb6fbd0ab048cac41499917c557377de3a7f8a435792854d4e6b445437e777998373f7c2248414ffb73

data/.github/dependabot.yml

@@ -0,0 +1,8 @@
+version: 2
+updates:
+- package-ecosystem: bundler
+  directory: "/"
+  schedule:
+    interval: daily
+    time: '20:00'
+  open-pull-requests-limit: 10

data/.rubocop.yml

@@ -1,6 +1,11 @@
-Naming/UncommunicativeMethodParamName:
-  Enabled: false
-Metrics/LineLength:
+AllCops:
+  TargetRubyVersion: 2.5
+  NewCops: enable
+
+
+Naming/MethodParameterName:
+  Enabled: false
+Layout/LineLength:
   Max: 160
 Style/FormatString:
   EnforcedStyle: sprintf
@@ -35,7 +40,7 @@ Metrics/PerceivedComplexity:
 Style/GlobalVars:
   Exclude:
     - 'test/test_helper.rb'
-Style/MethodMissingSuper:
+Lint/MissingSuper:
   Exclude:
     - 'test/test_helper.rb'
 Style/MissingRespondToMissing:
@@ -45,9 +50,7 @@ Style/TernaryParentheses:
   Enabled: false
 Style/BlockDelimiters:
   AutoCorrect: false
-Style/BlockDelimiters:
-  AutoCorrect: false
-Style/BracesAroundHashParameters:
-  Enabled: false
-Layout/IndentHash:
+Layout/FirstHashElementIndentation:
   EnforcedStyle: consistent
+Style/RedundantSelfAssignment:
+  Enabled: false

data/.travis.yml

@@ -1,12 +1,11 @@
 language: ruby
 rvm:
-  - 2.3
-  - 2.4
   - 2.5
   - 2.6
+  - 2.7
 before_install:
-  - gem update --remote bundler
   - gem update --system
-  - gem install rubocop bundler
+  - gem update --remote bundler
 script:
+  - bundle exec rubocop -d
   - bundle exec rake

data/Gemfile

@@ -4,4 +4,4 @@ source 'https://rubygems.org'
 gemspec
 
 # Added at 2017-08-14 19:47:59 +0900 by 5hun:
-gem 'rubocop', '~> 0.62.0', require: false, group: [:development]
+gem 'rubocop', '~> 1.16.1', require: false, group: [:development]

data/README.md

@@ -29,6 +29,9 @@ This repository contains two plugins: _Filter_ and _Output_, and expects two mai
 Use the `filter_list` filter. Configure fluentd as follows.
 
 #### ACMatcher
+
+_ACMatcher_ is a matcher using [Aho–Corasick algorithm](https://en.wikipedia.org/wiki/Aho%E2%80%93Corasick_algorithm) to enable faster multiple-pattern matching.
+
 ```
 <filter pattern>
   @type filter_list
@@ -66,7 +69,7 @@ While the following message is passed through as the target field specified in t
 }
 ```
 
-Additionally, the following message is also omitted since `filter_empty` is `true`. The value is determined to be empty when the trimed value is empty.
+Additionally, the following message is also omitted since `filter_empty` is `true`. The value is determined to be empty when the trimmed value is empty.
 
 ```json
 {
@@ -75,6 +78,19 @@ Additionally, the following message is also omitted since `filter_empty` is `tru
 }
 ```
 
+All these examples are _blacklisting_. That is, a text matched to a pattern will be determined to be filtered. The plugin provides the other type of filtering: _whitelisting_. With the type you can filter records that don't match any pattern. You can enable _whitelisting_ by specifying the `action` (the default value is _blacklist_) explicitly as follows.
+
+```
+<filter>
+  @type filter_list
+
+  filter AC
+  key_to_filter foo
+  pattern_file_paths blacklist.txt
+  action whitelist
+</filter>
+```
+
 #### IPMatcher
 ```
 <filter pattern>
@@ -132,6 +148,7 @@ The other use case is to filter messages likewise, but process the filtered mess
   key_to_filter field_name_you_want_to_filter
   pattern_file_paths ["file_including_patterns_separated_by_new_line"]
   filter_empty true
+  action blacklist
 
   <retag>
     add_prefix x # retag non-filtered messages whose tag will be "x.your_tag"

data/fluent-plugin-out_filter_list.gemspec

@@ -8,6 +8,7 @@ Gem::Specification.new do |spec|
   spec.version       = Fluent::Plugin::FilterList::VERSION
   spec.authors       = ['Shun Yanaura']
   spec.email         = ['metroplexity@gmail.com']
+  spec.required_ruby_version = '>= 2.5.0'
 
   spec.summary       = 'A fluentd output plugin to filter keywords from messages'
   spec.description   = 'A fluentd output plugin to filter keywords from messages'
@@ -21,9 +22,9 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_development_dependency 'bundler', '~> 2.0'
-  spec.add_development_dependency 'minitest', '~> 5.11'
-  spec.add_development_dependency 'rake', '~> 12.3'
-  spec.add_development_dependency 'test-unit', '~> 3.2'
+  spec.add_development_dependency 'bundler', '~> 2.1'
+  spec.add_development_dependency 'minitest', '~> 5.13'
+  spec.add_development_dependency 'rake', '~> 13.0'
+  spec.add_development_dependency 'test-unit', '~> 3.3'
   spec.add_runtime_dependency 'fluentd', '>= 0.14.0', '< 2.0.0'
 end

data/lib/base.rb

@@ -0,0 +1,8 @@
+module BaseFilter
+  def should_filter?(target)
+    return false if target.nil?
+    return true if @filter_empty && target.strip.empty?
+
+    @action == (@matcher.matches?(target) ? :blacklist : :whitelist)
+  end
+end

data/lib/fluent/plugin/filter_filter_list.rb

@@ -2,12 +2,14 @@ require 'fluent/plugin/out_filter_list/version'
 require 'matcher'
 require 'fluent/plugin/filter'
 require 'ip'
+require 'base'
 
 module Fluent
   module Plugin
     class FilterListFilter < Filter
       include Matchers
       include IP
+      include BaseFilter
 
       Plugin.register_filter('filter_list', self)
 
@@ -15,6 +17,7 @@ module Fluent
       config_param :key_to_filter, :string, default: nil
       config_param :pattern_file_paths, :array, default: [], value_type: :string
       config_param :filter_empty, :bool, default: false
+      config_param :action, :enum, list: %i[blacklist whitelist], default: :blacklist
 
       def configure(conf)
         super
@@ -24,7 +27,8 @@ module Fluent
 
       def filter(_tag, _time, record)
         target = record[@key_to_filter]
-        return nil if target && (@matcher.matches?(target) || (@filter_empty && target.strip.empty?))
+
+        return nil if should_filter?(target)
 
         record
       end

data/lib/fluent/plugin/out_filter_list.rb

@@ -2,12 +2,14 @@ require 'fluent/plugin/output'
 require 'fluent/plugin/out_filter_list/version'
 require 'matcher'
 require 'ip'
+require 'base'
 
 module Fluent
   module Plugin
     class FilterListOutput < Output
       include Matchers
       include IP
+      include BaseFilter
 
       Plugin.register_output('filter_list', self)
 
@@ -17,6 +19,7 @@ module Fluent
       config_param :key_to_filter, :string, default: nil
       config_param :pattern_file_paths, :array, default: [], value_type: :string
       config_param :filter_empty, :bool, default: false
+      config_param :action, :enum, list: %i[blacklist whitelist], default: :blacklist
 
       config_section :retag, required: true, multi: false do
         config_param :tag, :string, default: nil
@@ -28,10 +31,6 @@ module Fluent
         config_param :add_prefix, :string, default: nil
       end
 
-      def initialize
-        super
-      end
-
       def validate(retag)
         return unless retag
         raise Fluent::ConfigError, "missing tag and add_prefix" unless retag.tag || retag.add_prefix
@@ -39,15 +38,15 @@ module Fluent
       end
 
       def configure_prefixes
-        @prefix_for_filtered_tag = @retag_for_filtered.add_prefix + '.' if @retag_for_filtered && @retag_for_filtered.add_prefix
-        @prefix_for_filtered_tag = @retag_for_filtered && @retag_for_filtered.add_prefix ? @retag_for_filtered.add_prefix + '.' : ''
-        @prefix = @retag && @retag.add_prefix ? @retag.add_prefix + '.' : ''
+        @prefix_for_filtered_tag = "#{@retag_for_filtered.add_prefix}." if @retag_for_filtered&.add_prefix
+        @prefix_for_filtered_tag = @retag_for_filtered&.add_prefix ? "#{@retag_for_filtered.add_prefix}." : ''
+        @prefix = @retag&.add_prefix ? "#{@retag.add_prefix}." : ''
       end
 
       def configure(conf)
         super
         [@retag, @retag_for_filtered].each { |c| validate c }
-        patterns = @pattern_file_paths.flat_map { |p| File.readlines(p).map(&:chomp).reject(&:empty?) }
+        patterns = @pattern_file_paths.flat_map { |p| File.readlines(p).map(&:strip).reject(&:empty?) }
         @matcher = (@filter == 'IP') ? IPMatcher.new(patterns) : ACMatcher.new(patterns)
         configure_prefixes
       end
@@ -72,7 +71,7 @@ module Fluent
           target = record[@key_to_filter]
           log.debug "target: #{target}"
           # Do filter
-          if target && (@matcher.matches?(target) || (@filter_empty && target.strip.empty?))
+          if should_filter?(target)
             if @retag_for_filtered
               t = @retag_for_filtered.tag || ((tag && !tag.empty?) ? @prefix_for_filtered_tag + tag : @retag_for_filtered.add_prefix)
               log.debug "re-emit with the tag: '#{t}', originally: '#{tag}'"

data/lib/fluent/plugin/out_filter_list/version.rb

@@ -1,7 +1,7 @@
 module Fluent
   module Plugin
     module FilterList
-      VERSION = "0.6.0"
+      VERSION = "0.7.3"
     end
   end
 end

data/lib/matcher.rb

@@ -13,16 +13,26 @@ module Matchers
     def matches?(text)
       node = @trie.root
       text.to_s.chars.each do |char|
-        node = node.failure while node.children[char].nil? # Follow failure if it exists in case pattern doesn't match
+        failure = node.failure
         node = node.children[char]
-        return true unless node.output.nil?
+
+        return true unless node.nil? || node.output.nil?
+        return true unless failure.nil? || failure.output.nil?
+
+        # Follow failure if it exists in case pattern doesn't match
+        node = failure.children[char] if node.nil?
       end
-      false
+
+      return false if node.nil?
+      return true unless node.failure.nil? || node.failure.output.nil?
+
+      !node.output.nil?
     end
   end
 
   class IPMatcher
     attr_reader :trie
+
     include IP
 
     def initialize(patterns)
@@ -40,6 +50,7 @@ module Matchers
 
   class Trie
     attr_reader :root
+
     def initialize(patterns)
       @root = Node.new
       @root.children.default = @root
@@ -59,7 +70,7 @@ module Matchers
 
     def new_queue
       q = Queue.new
-      @root.children.values.each do |child|
+      @root.children.each_value do |child|
         q.push(child)
         child.failure = @root # set root on root's children's failure
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-filter-list
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.7.3
 platform: ruby
 authors:
 - Shun Yanaura
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-01-10 00:00:00.000000000 Z
+date: 2021-06-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -16,56 +16,56 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '2.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '2.1'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.11'
+        version: '5.13'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.11'
+        version: '5.13'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.3'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.3'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.2'
+        version: '3.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.2'
+        version: '3.3'
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
@@ -95,6 +95,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/dependabot.yml"
 - ".gitignore"
 - ".rubocop.yml"
 - ".travis.yml"
@@ -106,6 +107,7 @@ files:
 - bin/console
 - bin/setup
 - fluent-plugin-out_filter_list.gemspec
+- lib/base.rb
 - lib/fluent/plugin/filter_filter_list.rb
 - lib/fluent/plugin/out_filter_list.rb
 - lib/fluent/plugin/out_filter_list/version.rb
@@ -115,7 +117,7 @@ homepage: https://github.com/yanana/fluent-plugin-filter-list
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -123,15 +125,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 2.5.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.2
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
 summary: A fluentd output plugin to filter keywords from messages
 test_files: []