fluent-plugin-filter-list 0.2.4 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d59e0c9511bee9b07d913940698e10474ff94f71
-  data.tar.gz: b40e7e4c5831ccdce088917b3bd3609cbbf83f2b
+  metadata.gz: 9ea3dab5080825c71ef4c5809764f0a677d46b2f
+  data.tar.gz: 1d3bee1125fba97deee08dff1f7337ed268b1a3d
 SHA512:
-  metadata.gz: d83ac5416fd197ea9c556174eebc306441edc3ebe45d4b3b156c3eabe4b3db6da5efd8f3d1f284ed5c84c97980bb40e38be62448e1e013907514bbedc1d6719e
-  data.tar.gz: 6b8fd6b97f2f30fbc98e5d95056fe56e9bb2558f353039af335cba9f090183fa7ef05c756406992f8f3ebe79d40578ce4309c35ce51754b67ce9a893d9ab246a
+  metadata.gz: c1325ec3086b46309c0e183b6059753ac9b298e9efb2a3c3ca55dcded59d6d0783f7b97c1f842a2116751c2e991b8a25ee92af8df1f2ffa5b22c30dfadc5a8de
+  data.tar.gz: bef22cada3e0ee5c7fc909c0e05b004fb7f2a724c5a7f1b6120fc86dcee2fb9a1e4f747caca1b0e20a070c889180ef6626c6f1e94525a9da2bada360d9df760c

data/README.md

@@ -2,7 +2,7 @@
 
 [![Build Status](https://travis-ci.org/yanana/fluent-plugin-filter-list.svg?branch=master)](https://travis-ci.org/yanana/fluent-plugin-filter-list)
 
-[Fluentd](http://fluentd.org/) output plugin that filters messages whose value of specified field matches values in a list of patterns (like agrep). This plugin not only filters messages and discards them but also provides the feature to retag filtered records.
+Want to filter fluentd messages containing black-listed words in the list effectively? Use the _fluent-plugin-filter-list_ plugin. The plugin enables you to filter messages in the list of words you provide. You can either discard such messages simply, or process them in a different flow by retagging them.
 
 ## Installation
 
@@ -22,13 +22,52 @@ Or install it yourself as:
 
 ## Usage
 
+This repository contains two plugins: _Filter_ and _Output_, and expects two main use cases.
+
+### Filter plugin
+
+Use the `filter_list` filter. Configure fluentd as follows.
+
+```
+<filter pattern>
+  @type filter_list
+
+  key_to_filter xyz
+  patterns_file_path blacklist.txt
+</filter>
+```
+
+Given the `blacklist.txt` is as follows.
+
+```
+foo
+bar
+buzz
+```
+
+The following message is discarded since its `x` field contains the sequence of characters _bar_, contained in the list.
+
+```json
+{"x":"halbart","y":1}
+```
+
+While the following message is passed through as the target field specified in the config is not _y_ but _x_ .
+
+```json
+{"x":1,"y":"halbart"}
 ```
-<match your_tag>
+
+### Output plugin
+
+The other use case is to filter messages likewise, but process the filtered messages in a different tag. You need to configure the plugin to tell it how to retag both non-filtered messages and filtered messages. We provide two mutually-exclusive parameters: `tag` and `add_prefix`. THe `tag` parameter tells the plugin to retag the message with the value exactly provided by the parameter. The `add_prefix` parameter tells the plugin to retag the messages with the original tag prepended with the value you provide. So if the original message had a tag _foo_ and you set the `add_prefix` parameter _filtered_, then the processed message would have the tag _filtered.foo_ (note that the period before the original tag value is also prepended).
+
+```
+<match pattern>
   @type filter_list
-  
+
   key_to_filter field_name_you_want_to_filter
   patterns_file_path file_including_patterns_separated_by_new_line
-  
+
   <retag>
     add_prefix x # retag non-filtered messages whose tag will be "x.your_tag"
   </retag>

data/fluent-plugin-out_filter_list.gemspec

@@ -7,7 +7,7 @@ require 'fluent/plugin/out_filter_list/version'
 
 Gem::Specification.new do |spec|
   spec.name          = 'fluent-plugin-filter-list'
-  spec.version       = Fluent::OutFilterList::VERSION
+  spec.version       = Fluent::Plugin::FilterList::VERSION
   spec.authors       = ['Shun Yanaura']
   spec.email         = ['metroplexity@gmail.com']
 
@@ -27,5 +27,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rake', '~> 12.0'
   spec.add_development_dependency 'minitest', '~> 5.0'
   spec.add_development_dependency 'test-unit', '~> 3.2'
-  spec.add_runtime_dependency 'fluentd', '~> 0.12', '>= 0.12.0'
+  spec.add_runtime_dependency 'fluentd', '~> 0.14', '>= 0.14.0'
 end

data/lib/fluent/plugin/filter_filter_list.rb

@@ -2,24 +2,26 @@ require 'fluent/plugin/out_filter_list/version'
 require 'aho_corasick'
 
 module Fluent
-  class FilterListFilter < Filter
-    include Matchers
+  module Plugin
+    class FilterListFilter < Filter
+      include Matchers
 
-    Plugin.register_filter('filter_list', self)
+      Plugin.register_filter('filter_list', self)
 
-    config_param :key_to_filter, :string, default: nil
-    config_param :patterns_file_path, :string, default: ''
+      config_param :key_to_filter, :string, default: nil
+      config_param :patterns_file_path, :string, default: ''
 
-    def configure(conf)
-      super
-      patterns = @patterns_file_path.empty? ? [] : File.readlines(@patterns_file_path).map(&:chomp).reject(&:empty?)
-      @matcher = ACMatcher.new(patterns)
-    end
+      def configure(conf)
+        super
+        patterns = @patterns_file_path.empty? ? [] : File.readlines(@patterns_file_path).map(&:chomp).reject(&:empty?)
+        @matcher = ACMatcher.new(patterns)
+      end
 
-    def filter(_tag, _time, record)
-      target = record[@key_to_filter]
-      return nil if target && @matcher.matches?(target)
-      record
+      def filter(_tag, _time, record)
+        target = record[@key_to_filter]
+        return nil if target && @matcher.matches?(target)
+        record
+      end
     end
   end
 end

data/lib/fluent/plugin/out_filter_list.rb

@@ -1,66 +1,75 @@
+require 'fluent/plugin/output'
 require 'fluent/plugin/out_filter_list/version'
 require 'aho_corasick'
 
 module Fluent
-  class FilterListOutput < Output
-    include Matchers
+  module Plugin
+    class FilterListOutput < Output
+      include Matchers
 
-    Plugin.register_output('filter_list', self)
+      Plugin.register_output('filter_list', self)
 
-    config_param :key_to_filter, :string, default: nil
-    config_param :patterns_file_path, :string, default: ''
+      helpers :event_emitter
 
-    config_section :retag, required: true, multi: false do
-      config_param :tag, :string, default: nil
-      config_param :add_prefix, :string, default: nil
-    end
+      config_param :key_to_filter, :string, default: nil
+      config_param :patterns_file_path, :string, default: ''
 
-    config_section :retag_filtered, param_name: :retag_for_filtered, required: false, multi: false do
-      config_param :tag, :string, default: nil
-      config_param :add_prefix, :string, default: nil
-    end
+      config_section :retag, required: true, multi: false do
+        config_param :tag, :string, default: nil
+        config_param :add_prefix, :string, default: nil
+      end
 
-    def initialize
-      super
-    end
+      config_section :retag_filtered, param_name: :retag_for_filtered, required: false, multi: false do
+        config_param :tag, :string, default: nil
+        config_param :add_prefix, :string, default: nil
+      end
 
-    def validate(retag)
-      return unless retag
-      raise Fluent::ConfigError, "missing tag and add_prefix" unless retag.tag || retag.add_prefix
-      raise Fluent::ConfigError, "tag and add_prefix are mutually exclusive" if retag.tag && retag.add_prefix
-    end
+      def initialize
+        super
+      end
 
-    def configure_prefixes
-      @prefix_for_filtered_tag = @retag_for_filtered.add_prefix + '.' if @retag_for_filtered && @retag_for_filtered.add_prefix
-      @prefix_for_filtered_tag = @retag_for_filtered && @retag_for_filtered.add_prefix ? @retag_for_filtered.add_prefix + '.' : ''
-      @prefix = @retag && @retag.add_prefix ? @retag.add_prefix + '.' : ''
-    end
+      def validate(retag)
+        return unless retag
+        raise Fluent::ConfigError, "missing tag and add_prefix" unless retag.tag || retag.add_prefix
+        raise Fluent::ConfigError, "tag and add_prefix are mutually exclusive" if retag.tag && retag.add_prefix
+      end
 
-    def configure(conf)
-      super
-      [@retag, @retag_for_filtered].each { |c| validate c }
-      patterns = @patterns_file_path.empty? ? [] : File.readlines(@patterns_file_path).map(&:chomp).reject(&:empty?)
-      @matcher = ACMatcher.new(patterns)
-      configure_prefixes
-      log.debug "prefix: #{@prefix}, prefix_for_filtered_tag: #{@prefix_for_filtered_tag || ''}"
-    end
+      def configure_prefixes
+        @prefix_for_filtered_tag = @retag_for_filtered.add_prefix + '.' if @retag_for_filtered && @retag_for_filtered.add_prefix
+        @prefix_for_filtered_tag = @retag_for_filtered && @retag_for_filtered.add_prefix ? @retag_for_filtered.add_prefix + '.' : ''
+        @prefix = @retag && @retag.add_prefix ? @retag.add_prefix + '.' : ''
+      end
+
+      def configure(conf)
+        super
+        [@retag, @retag_for_filtered].each { |c| validate c }
+        patterns = @patterns_file_path.empty? ? [] : File.readlines(@patterns_file_path).map(&:chomp).reject(&:empty?)
+        @matcher = ACMatcher.new(patterns)
+        configure_prefixes
+        log.debug "prefix: #{@prefix}, prefix_for_filtered_tag: #{@prefix_for_filtered_tag || ''}"
+      end
+
+      def multi_workers_ready?
+        true
+      end
 
-    def emit(tag, es, _chain)
-      es.each do |time, record|
-        target = record[@key_to_filter]
-        log.debug "target: #{target}"
-        # Do filter
-        if target && @matcher.matches?(target)
-          if @retag_for_filtered
-            tag = @retag_for_filtered.tag || ((tag && !tag.empty?) ? @prefix_for_filtered_tag + tag : @retag_for_filtered.add_prefix)
-            log.debug "re-emit with tag: #{tag}"
-            router.emit(tag, time, record)
+      def process(tag, es)
+        es.each do |time, record|
+          target = record[@key_to_filter]
+          log.debug "target: #{target}"
+          # Do filter
+          if target && @matcher.matches?(target)
+            if @retag_for_filtered
+              tag = @retag_for_filtered.tag || ((tag && !tag.empty?) ? @prefix_for_filtered_tag + tag : @retag_for_filtered.add_prefix)
+              log.debug "re-emit with tag: #{tag}"
+              router.emit(tag, time, record)
+            end
+            next
           end
-          next
+          tag = @retag.tag || ((tag && !tag.empty?) ? @prefix + tag : @retag.add_prefix)
+          log.debug "re-emit with tag: #{tag}"
+          router.emit(tag, time, record)
         end
-        tag = @retag.tag || ((tag && !tag.empty?) ? @prefix + tag : @retag.add_prefix)
-        log.debug "re-emit with tag: #{tag}"
-        router.emit(tag, time, record)
       end
     end
   end

data/lib/fluent/plugin/out_filter_list/version.rb

@@ -1,5 +1,7 @@
 module Fluent
-  module OutFilterList
-    VERSION = "0.2.4"
+  module Plugin
+    module FilterList
+      VERSION = "0.3.0"
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-filter-list
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.3.0
 platform: ruby
 authors:
 - Shun Yanaura
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-08-14 00:00:00.000000000 Z
+date: 2017-08-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -72,20 +72,20 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.12'
+        version: '0.14'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.12.0
+        version: 0.14.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.12'
+        version: '0.14'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.12.0
+        version: 0.14.0
 description: A fluentd output plugin to filter keywords from messages
 email:
 - metroplexity@gmail.com