fluent-plugin-filter-list 0.5.0 → 0.7.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 17bee9e79c216bfbdf746e8d7d089d229b2997bccf376d24c5e7311f1931f2b4
-  data.tar.gz: b00d3238b9a8d775f5f1dc2e7fd03beedde536bc6894d38f22091395d3dde1bf
+  metadata.gz: d213a158b64257afde99c26284bf0215c1a5d208362f0f00012580852654244e
+  data.tar.gz: 37c5a0219f4bab0de18b6aac6c75d6d90f63bd60f5eddb5399bfb0522c563706
 SHA512:
-  metadata.gz: c578ea80823b2c26e70a37cf63b3df24832bef79013fcd7e1c28cbf937d220130581bd2a382a6ef06b5c9d1b96838a12d154b1263512920214041cfdfe467ee5
-  data.tar.gz: 513142ca73ebf949f94d3f22fea2deff6e5147ffea92d65687ce33c80c6217e9d70d47dc3ec53b47858a5edb725c1b43ba70334e0feece51971978ca9fe886f4
+  metadata.gz: 776d9222ea8997eda4506348282ed67415296da63c4dcd16569f2a7db35863a5844b3e3869cff5d9dcd9a077d4738a9733e4cf3b3436bc7ec08a26b8d4d84503
+  data.tar.gz: b73678d3bd2ba2a415499256cf6b8b7ba1c8c0a88800745d6668aaea1ba049d59343ca2665063cb4a11c1d099ce9ce825fffc57465dde845ee62c34b39f3a49a

data/.github/dependabot.yml

@@ -0,0 +1,8 @@
+version: 2
+updates:
+- package-ecosystem: bundler
+  directory: "/"
+  schedule:
+    interval: daily
+    time: '20:00'
+  open-pull-requests-limit: 10

data/.rubocop.yml

@@ -1,6 +1,11 @@
-Naming/UncommunicativeMethodParamName:
-  Enabled: false
-Metrics/LineLength:
+AllCops:
+  TargetRubyVersion: 2.5
+  NewCops: enable
+
+
+Naming/MethodParameterName:
+  Enabled: false
+Layout/LineLength:
   Max: 160
 Style/FormatString:
   EnforcedStyle: sprintf
@@ -35,16 +40,17 @@ Metrics/PerceivedComplexity:
 Style/GlobalVars:
   Exclude:
     - 'test/test_helper.rb'
-Style/MethodMissing:
+Lint/MissingSuper:
+  Exclude:
+    - 'test/test_helper.rb'
+Style/MissingRespondToMissing:
   Exclude:
     - 'test/test_helper.rb'
 Style/TernaryParentheses:
   Enabled: false
 Style/BlockDelimiters:
   AutoCorrect: false
-Style/BlockDelimiters:
-  AutoCorrect: false
-Style/BracesAroundHashParameters:
-  Enabled: false
-Layout/IndentHash:
+Layout/FirstHashElementIndentation:
   EnforcedStyle: consistent
+Style/RedundantSelfAssignment:
+  Enabled: false

data/.travis.yml

@@ -1,12 +1,11 @@
 language: ruby
 rvm:
-  - 2.2.10
-  - 2.3.7
-  - 2.4.4
-  - 2.5.1
+  - 2.5
+  - 2.6
+  - 2.7
 before_install:
-  - gem update --remote bundler
   - gem update --system
-  - gem install rubocop
+  - gem update --remote bundler
 script:
+  - bundle exec rubocop -d
   - bundle exec rake

data/Gemfile

@@ -4,4 +4,4 @@ source 'https://rubygems.org'
 gemspec
 
 # Added at 2017-08-14 19:47:59 +0900 by 5hun:
-gem 'rubocop', '~> 0.54.0', group: [:development]
+gem 'rubocop', '~> 0.92.0', require: false, group: [:development]

data/README.md

@@ -29,13 +29,16 @@ This repository contains two plugins: _Filter_ and _Output_, and expects two mai
 Use the `filter_list` filter. Configure fluentd as follows.
 
 #### ACMatcher
+
+_ACMatcher_ is a matcher using [Aho–Corasick algorithm](https://en.wikipedia.org/wiki/Aho%E2%80%93Corasick_algorithm) to enable faster multiple-pattern matching.
+
 ```
 <filter pattern>
   @type filter_list
 
   filter AC
   key_to_filter x
-  patterns_file_path blacklist.txt
+  pattern_file_paths ["blacklist_1.txt", "blacklist_2.txt"]
   filter_empty true
 </filter>
 ```
@@ -66,7 +69,7 @@ While the following message is passed through as the target field specified in t
 }
 ```
 
-Additionally, the following message is also omitted since `filter_empty` is `true`. The value is determined to be empty when the trimed value is empty.
+Additionally, the following message is also omitted since `filter_empty` is `true`. The value is determined to be empty when the trimmed value is empty.
 
 ```json
 {
@@ -75,6 +78,19 @@ Additionally, the following message is also omitted since `filter_empty` is `tru
 }
 ```
 
+All these examples are _blacklisting_. That is, a text matched to a pattern will be determined to be filtered. The plugin provides the other type of filtering: _whitelisting_. With the type you can filter records that don't match any pattern. You can enable _whitelisting_ by specifying the `action` (the default value is _blacklist_) explicitly as follows.
+
+```
+<filter>
+  @type filter_list
+
+  filter AC
+  key_to_filter foo
+  pattern_file_paths blacklist.txt
+  action whitelist
+</filter>
+```
+
 #### IPMatcher
 ```
 <filter pattern>
@@ -82,7 +98,7 @@ Additionally, the following message is also omitted since `filter_empty` is `tru
 
   filter IP
   key_to_filter ip
-  patterns_file_path blacklist.txt
+  pattern_file_paths blacklist.txt
 </filter>
 ```
 
@@ -130,8 +146,9 @@ The other use case is to filter messages likewise, but process the filtered mess
   @type filter_list
 
   key_to_filter field_name_you_want_to_filter
-  patterns_file_path file_including_patterns_separated_by_new_line
+  pattern_file_paths ["file_including_patterns_separated_by_new_line"]
   filter_empty true
+  action blacklist
 
   <retag>
     add_prefix x # retag non-filtered messages whose tag will be "x.your_tag"

data/fluent-plugin-out_filter_list.gemspec

@@ -8,6 +8,7 @@ Gem::Specification.new do |spec|
   spec.version       = Fluent::Plugin::FilterList::VERSION
   spec.authors       = ['Shun Yanaura']
   spec.email         = ['metroplexity@gmail.com']
+  spec.required_ruby_version = '>= 2.5.0'
 
   spec.summary       = 'A fluentd output plugin to filter keywords from messages'
   spec.description   = 'A fluentd output plugin to filter keywords from messages'
@@ -21,9 +22,9 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_development_dependency 'bundler', '~> 1.16'
-  spec.add_development_dependency 'minitest', '~> 5.11'
-  spec.add_development_dependency 'rake', '~> 12.3'
-  spec.add_development_dependency 'test-unit', '~> 3.2'
+  spec.add_development_dependency 'bundler', '~> 2.1'
+  spec.add_development_dependency 'minitest', '~> 5.13'
+  spec.add_development_dependency 'rake', '~> 13.0'
+  spec.add_development_dependency 'test-unit', '~> 3.3'
   spec.add_runtime_dependency 'fluentd', '>= 0.14.0', '< 2.0.0'
 end

data/lib/base.rb

@@ -0,0 +1,8 @@
+module BaseFilter
+  def should_filter?(target)
+    return false if target.nil?
+    return true if @filter_empty && target.strip.empty?
+
+    @action == (@matcher.matches?(target) ? :blacklist : :whitelist)
+  end
+end

data/lib/fluent/plugin/filter_filter_list.rb

@@ -2,29 +2,34 @@ require 'fluent/plugin/out_filter_list/version'
 require 'matcher'
 require 'fluent/plugin/filter'
 require 'ip'
+require 'base'
 
 module Fluent
   module Plugin
     class FilterListFilter < Filter
       include Matchers
       include IP
+      include BaseFilter
 
       Plugin.register_filter('filter_list', self)
 
       config_param :filter, :string, default: 'AC'
       config_param :key_to_filter, :string, default: nil
-      config_param :patterns_file_path, :string, default: ''
+      config_param :pattern_file_paths, :array, default: [], value_type: :string
       config_param :filter_empty, :bool, default: false
+      config_param :action, :enum, list: %i[blacklist whitelist], default: :blacklist
 
       def configure(conf)
         super
-        patterns = @patterns_file_path.empty? ? [] : File.readlines(@patterns_file_path).map(&:chomp).reject(&:empty?)
+        patterns = @pattern_file_paths.flat_map { |p| File.readlines(p).map(&:chomp).reject(&:empty?) }
         @matcher = (@filter == 'IP') ? IPMatcher.new(patterns) : ACMatcher.new(patterns)
       end
 
       def filter(_tag, _time, record)
         target = record[@key_to_filter]
-        return nil if target && (@matcher.matches?(target) || (@filter_empty && target.strip.empty?))
+
+        return nil if should_filter?(target)
+
         record
       end
     end

data/lib/fluent/plugin/out_filter_list.rb

@@ -2,12 +2,14 @@ require 'fluent/plugin/output'
 require 'fluent/plugin/out_filter_list/version'
 require 'matcher'
 require 'ip'
+require 'base'
 
 module Fluent
   module Plugin
     class FilterListOutput < Output
       include Matchers
       include IP
+      include BaseFilter
 
       Plugin.register_output('filter_list', self)
 
@@ -15,8 +17,9 @@ module Fluent
 
       config_param :filter, :string, default: 'AC'
       config_param :key_to_filter, :string, default: nil
-      config_param :patterns_file_path, :string, default: ''
+      config_param :pattern_file_paths, :array, default: [], value_type: :string
       config_param :filter_empty, :bool, default: false
+      config_param :action, :enum, list: %i[blacklist whitelist], default: :blacklist
 
       config_section :retag, required: true, multi: false do
         config_param :tag, :string, default: nil
@@ -28,10 +31,6 @@ module Fluent
         config_param :add_prefix, :string, default: nil
       end
 
-      def initialize
-        super
-      end
-
       def validate(retag)
         return unless retag
         raise Fluent::ConfigError, "missing tag and add_prefix" unless retag.tag || retag.add_prefix
@@ -39,15 +38,15 @@ module Fluent
       end
 
       def configure_prefixes
-        @prefix_for_filtered_tag = @retag_for_filtered.add_prefix + '.' if @retag_for_filtered && @retag_for_filtered.add_prefix
-        @prefix_for_filtered_tag = @retag_for_filtered && @retag_for_filtered.add_prefix ? @retag_for_filtered.add_prefix + '.' : ''
-        @prefix = @retag && @retag.add_prefix ? @retag.add_prefix + '.' : ''
+        @prefix_for_filtered_tag = "#{@retag_for_filtered.add_prefix}." if @retag_for_filtered&.add_prefix
+        @prefix_for_filtered_tag = @retag_for_filtered&.add_prefix ? "#{@retag_for_filtered.add_prefix}." : ''
+        @prefix = @retag&.add_prefix ? "#{@retag.add_prefix}." : ''
       end
 
       def configure(conf)
         super
         [@retag, @retag_for_filtered].each { |c| validate c }
-        patterns = @patterns_file_path.empty? ? [] : File.readlines(@patterns_file_path).map(&:chomp).reject(&:empty?)
+        patterns = @pattern_file_paths.flat_map { |p| File.readlines(p).map(&:strip).reject(&:empty?) }
         @matcher = (@filter == 'IP') ? IPMatcher.new(patterns) : ACMatcher.new(patterns)
         configure_prefixes
       end
@@ -72,7 +71,7 @@ module Fluent
           target = record[@key_to_filter]
           log.debug "target: #{target}"
           # Do filter
-          if target && (@matcher.matches?(target) || (@filter_empty && target.strip.empty?))
+          if should_filter?(target)
             if @retag_for_filtered
               t = @retag_for_filtered.tag || ((tag && !tag.empty?) ? @prefix_for_filtered_tag + tag : @retag_for_filtered.add_prefix)
               log.debug "re-emit with the tag: '#{t}', originally: '#{tag}'"

data/lib/fluent/plugin/out_filter_list/version.rb

@@ -1,7 +1,7 @@
 module Fluent
   module Plugin
     module FilterList
-      VERSION = "0.5.0"
+      VERSION = "0.7.2"
     end
   end
 end

data/lib/matcher.rb

@@ -13,16 +13,25 @@ module Matchers
     def matches?(text)
       node = @trie.root
       text.to_s.chars.each do |char|
-        node = node.failure while node.children[char].nil? # Follow failure if it exists in case pattern doesn't match
+        failure = node.failure
         node = node.children[char]
-        return true unless node.output.nil?
+
+        return true unless node.nil? || node.output.nil?
+        return true unless failure.nil? || failure.output.nil?
+
+        # Follow failure if it exists in case pattern doesn't match
+        node = failure if node.nil?
       end
-      false
+
+      return false if node.failure.nil?
+
+      !node.failure.output.nil?
     end
   end
 
   class IPMatcher
     attr_reader :trie
+
     include IP
 
     def initialize(patterns)
@@ -32,6 +41,7 @@ module Matchers
 
     def matches?(text)
       return false if text.nil?
+
       ip = IPAddr.new(text).to_i.to_s(2).rjust(32, '0')
       trie.forward_match(ip)
     end
@@ -39,6 +49,7 @@ module Matchers
 
   class Trie
     attr_reader :root
+
     def initialize(patterns)
       @root = Node.new
       @root.children.default = @root
@@ -58,7 +69,7 @@ module Matchers
 
     def new_queue
       q = Queue.new
-      @root.children.values.each do |child|
+      @root.children.each_value do |child|
         q.push(child)
         child.failure = @root # set root on root's children's failure
       end
@@ -82,10 +93,12 @@ module Matchers
 
     def forward_match(pattern)
       return false if @root.children.empty?
+
       cur_node = @root
       pattern.chars.each do |char|
         return true if cur_node.children.empty?
         return false unless cur_node.children.key?(char)
+
         cur_node = cur_node.children[char]
       end
       true

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-filter-list
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.7.2
 platform: ruby
 authors:
 - Shun Yanaura
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-04-26 00:00:00.000000000 Z
+date: 2020-10-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -16,56 +16,56 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: '2.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: '2.1'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.11'
+        version: '5.13'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.11'
+        version: '5.13'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.3'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.3'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.2'
+        version: '3.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.2'
+        version: '3.3'
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
@@ -95,6 +95,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/dependabot.yml"
 - ".gitignore"
 - ".rubocop.yml"
 - ".travis.yml"
@@ -106,6 +107,7 @@ files:
 - bin/console
 - bin/setup
 - fluent-plugin-out_filter_list.gemspec
+- lib/base.rb
 - lib/fluent/plugin/filter_filter_list.rb
 - lib/fluent/plugin/out_filter_list.rb
 - lib/fluent/plugin/out_filter_list/version.rb
@@ -115,7 +117,7 @@ homepage: https://github.com/yanana/fluent-plugin-filter-list
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -123,16 +125,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 2.5.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
 summary: A fluentd output plugin to filter keywords from messages
 test_files: []