fluent-plugin-filter-list 0.3.6 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fc2eaf47d5a020a5503509718a07bcb8517c5f51f8c39945f5a8fd43f744bd60
-  data.tar.gz: 344a564835d25eae279e841b6981277d04a4765d8c114c949f0f38d6eec1124f
+  metadata.gz: e2c3677869c408f56ed9c9aefc31cedc9a152e34872e1ae9e2ec0e6d8a138af2
+  data.tar.gz: 92986f0067646a44c583bc46dee304320ae128035680175d2c1de43decde02c4
 SHA512:
-  metadata.gz: 60ee5a756156e77e040f8d418be1e0efb534c70a26506057a0ec8ad6a0db99c0fb79d13e07c4591216e477642dd4984f035f9b14a0ecb0d88eec9a824311c6d6
-  data.tar.gz: 14dee243e22ebed510eae9b3daac978b428e9b3f70037e5dbf9bca953a504e4393d6f5a0671bc282d47aa6434332d8bfa5a92743cb09e8e638cf3f4d06fbfad9
+  metadata.gz: d084ca17a78eac9cc3c8faa0be7f2e4a0e49a2f0bfea39d8de8a0a3d6bb52a7335ded2d9bc8104ee6a8adde964db140d1df10c1365c064caf7fd7e1ded586a21
+  data.tar.gz: 3d9541d21edc8a1a1ea84759e53459d5b8d99be6d979986e537c5be8660c3077db3d580ebbac622f69193d18fbfa17fdcf98bc1416d3d309936533ac2fb1f3dc

data/.rubocop.yml

@@ -1,7 +1,11 @@
+Naming/UncommunicativeMethodParamName:
+  Enabled: false
 Metrics/LineLength:
   Max: 160
 Style/FormatString:
   EnforcedStyle: sprintf
+Style/FormatStringToken:
+  Enabled: false
 Style/StringLiterals:
   Enabled: false
 Metrics/MethodLength:

data/.travis.yml

@@ -1,8 +1,9 @@
 language: ruby
 rvm:
-  - 2.2.4
-  - 2.3.3
-  - 2.4.1
+  - 2.2.10
+  - 2.3.7
+  - 2.4.4
+  - 2.5.1
 before_install:
   - gem update --remote bundler
   - gem update --system

data/Gemfile

@@ -4,4 +4,4 @@ source 'https://rubygems.org'
 gemspec
 
 # Added at 2017-08-14 19:47:59 +0900 by 5hun:
-gem 'rubocop', '~> 0.49.1', group: [:development]
+gem 'rubocop', '~> 0.54.0', group: [:development]

data/README.md

@@ -28,10 +28,12 @@ This repository contains two plugins: _Filter_ and _Output_, and expects two mai
 
 Use the `filter_list` filter. Configure fluentd as follows.
 
+#### ACMatcher
 ```
 <filter pattern>
   @type filter_list
 
+  filter AC
   key_to_filter xyz
   patterns_file_path blacklist.txt
 </filter>
@@ -57,6 +59,43 @@ While the following message is passed through as the target field specified in t
 {"x":1,"y":"halbart"}
 ```
 
+#### IPMatcher
+```
+<filter pattern>
+  @type filter_list
+
+  filter IP
+  key_to_filter ip
+  patterns_file_path blacklist.txt
+</filter>
+```
+
+Given the `blacklist.txt` is as follows.
+
+```
+192.168.1.0/24
+127.0.0.1/24
+255.255.0.0
+```
+
+The following message is discarded since its `ip` field is the IP address in the list (exact IP).
+
+```json
+{"ip":"255.255.0.0","y":1}
+```
+
+Also the following message is discarded since its `ip` field is the IP address in the list (CIDR-notated IP).
+
+```json
+{"ip":"192.168.1.255","y":1}
+```
+
+While the following message is passed through.
+
+```json
+{"ip":"192.168.2.0","y":1}
+```
+
 ### Output plugin
 
 The other use case is to filter messages likewise, but process the filtered messages in a different tag. You need to configure the plugin to tell it how to retag both non-filtered messages and filtered messages. We provide two mutually-exclusive parameters: `tag` and `add_prefix`. THe `tag` parameter tells the plugin to retag the message with the value exactly provided by the parameter. The `add_prefix` parameter tells the plugin to retag the messages with the original tag prepended with the value you provide. So if the original message had a tag _foo_ and you set the `add_prefix` parameter _filtered_, then the processed message would have the tag _filtered.foo_ (note that the period before the original tag value is also prepended).

data/fluent-plugin-out_filter_list.gemspec

@@ -1,6 +1,4 @@
-# coding: utf-8
-
-lib = File.expand_path('../lib', __FILE__)
+lib = File.expand_path('lib', __dir__)
 
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'fluent/plugin/out_filter_list/version'
@@ -19,13 +17,13 @@ Gem::Specification.new do |spec|
   spec.files         = `git ls-files -z`.split("\x0").reject do |f|
     f.match(%r{^(test|spec|features)/})
   end
-  spec.bindir        = 'exe'
-  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.bindir        = 'bin'
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
   spec.add_development_dependency 'bundler', '~> 1.16'
+  spec.add_development_dependency 'minitest', '~> 5.11'
   spec.add_development_dependency 'rake', '~> 12.3'
-  spec.add_development_dependency 'minitest', '~> 5.0'
   spec.add_development_dependency 'test-unit', '~> 3.2'
   spec.add_runtime_dependency 'fluentd', '>= 0.14.0', '< 2.0.0'
 end

data/lib/fluent/plugin/filter_filter_list.rb

@@ -1,21 +1,24 @@
 require 'fluent/plugin/out_filter_list/version'
-require 'aho_corasick'
+require 'matcher'
 require 'fluent/plugin/filter'
+require 'ip'
 
 module Fluent
   module Plugin
     class FilterListFilter < Filter
       include Matchers
+      include IP
 
       Plugin.register_filter('filter_list', self)
 
+      config_param :filter, :string, default: 'AC'
       config_param :key_to_filter, :string, default: nil
       config_param :patterns_file_path, :string, default: ''
 
       def configure(conf)
         super
         patterns = @patterns_file_path.empty? ? [] : File.readlines(@patterns_file_path).map(&:chomp).reject(&:empty?)
-        @matcher = ACMatcher.new(patterns)
+        @matcher = (@filter == 'IP') ? IPMatcher.new(patterns) : ACMatcher.new(patterns)
       end
 
       def filter(_tag, _time, record)

data/lib/fluent/plugin/out_filter_list.rb

@@ -1,16 +1,19 @@
 require 'fluent/plugin/output'
 require 'fluent/plugin/out_filter_list/version'
-require 'aho_corasick'
+require 'matcher'
+require 'ip'
 
 module Fluent
   module Plugin
     class FilterListOutput < Output
       include Matchers
+      include IP
 
       Plugin.register_output('filter_list', self)
 
       helpers :event_emitter
 
+      config_param :filter, :string, default: 'AC'
       config_param :key_to_filter, :string, default: nil
       config_param :patterns_file_path, :string, default: ''
 
@@ -44,7 +47,7 @@ module Fluent
         super
         [@retag, @retag_for_filtered].each { |c| validate c }
         patterns = @patterns_file_path.empty? ? [] : File.readlines(@patterns_file_path).map(&:chomp).reject(&:empty?)
-        @matcher = ACMatcher.new(patterns)
+        @matcher = (@filter == 'IP') ? IPMatcher.new(patterns) : ACMatcher.new(patterns)
         configure_prefixes
       end
 

data/lib/fluent/plugin/out_filter_list/version.rb

@@ -1,7 +1,7 @@
 module Fluent
   module Plugin
     module FilterList
-      VERSION = "0.3.6"
+      VERSION = "0.4.1"
     end
   end
 end

data/lib/ip.rb

@@ -0,0 +1,17 @@
+module IP
+  require 'ipaddr'
+  class IP
+    attr_reader :subnet
+
+    # This constructor accepts both CIDR-notated IP and also exact IP.
+    def initialize(ip)
+      @address = IPAddr.new(ip)
+      @subnet = 32 - Math.log2(@address.to_range.to_a.size).round
+    end
+
+    def to_binary
+      binary = @address.to_i.to_s(2)
+      binary.slice(0, @subnet)
+    end
+  end
+end

data/lib/{aho_corasick.rb → matcher.rb}

@@ -1,3 +1,6 @@
+require 'ipaddr'
+require 'ip'
+
 module Matchers
   class ACMatcher
     attr_reader :trie
@@ -18,6 +21,22 @@ module Matchers
     end
   end
 
+  class IPMatcher
+    attr_reader :trie
+    include IP
+
+    def initialize(patterns)
+      patterns = (patterns || []).compact.reject(&:empty?).map { |ip| IP.new(ip) }.map(&:to_binary)
+      @trie = Trie.new patterns
+    end
+
+    def matches?(text)
+      return false if text.nil?
+      ip = IPAddr.new(text).to_i.to_s(2)
+      trie.forward_match(ip)
+    end
+  end
+
   class Trie
     attr_reader :root
     def initialize(patterns)
@@ -55,13 +74,22 @@ module Matchers
         cur_node.children.each do |char, child|
           q.push(child)
           detect_node = cur_node.failure || @root
-          while detect_node.children[char].nil?
-            detect_node = detect_node.failure
-          end
+          detect_node = detect_node.failure while detect_node.children[char].nil?
           child.failure = detect_node.children[char]
         end
       end
     end
+
+    def forward_match(pattern)
+      return false if @root.children.empty?
+      cur_node = @root
+      pattern.split('').each do |char|
+        return true if cur_node.children.empty?
+        return false unless cur_node.children.key?(char)
+        cur_node = cur_node.children[char]
+      end
+      true
+    end
   end
 
   class Node

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-filter-list
 version: !ruby/object:Gem::Version
-  version: 0.3.6
+  version: 0.4.1
 platform: ruby
 authors:
 - Shun Yanaura
 autorequire: 
-bindir: exe
+bindir: bin
 cert_chain: []
-date: 2017-12-11 00:00:00.000000000 Z
+date: 2018-04-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -25,33 +25,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.16'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.3'
+        version: '5.11'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.3'
+        version: '5.11'
 - !ruby/object:Gem::Dependency
-  name: minitest
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '12.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '12.3'
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
@@ -89,7 +89,9 @@ dependencies:
 description: A fluentd output plugin to filter keywords from messages
 email:
 - metroplexity@gmail.com
-executables: []
+executables:
+- console
+- setup
 extensions: []
 extra_rdoc_files: []
 files:
@@ -104,10 +106,11 @@ files:
 - bin/console
 - bin/setup
 - fluent-plugin-out_filter_list.gemspec
-- lib/aho_corasick.rb
 - lib/fluent/plugin/filter_filter_list.rb
 - lib/fluent/plugin/out_filter_list.rb
 - lib/fluent/plugin/out_filter_list/version.rb
+- lib/ip.rb
+- lib/matcher.rb
 homepage: https://github.com/yanana/fluent-plugin-filter-list
 licenses:
 - MIT
@@ -128,7 +131,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.3
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: A fluentd output plugin to filter keywords from messages