fluent-plugin-filter-list 0.7.4 → 0.7.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6025312b2ef8ab49eba3c259f54a37d835ccac19e74bb781210cfc18e661791a
-  data.tar.gz: e646c57736caf246e975543f5f0dafdeda23b4a62c5e3053e0316cc73477d3a4
+  metadata.gz: 2e248391649cea2b7fa1ef396a5ae5bcdae45959775296b58dd845d334709993
+  data.tar.gz: 7ca0d518e0aae01a31a218f20613952202ec07eace82152393f3f186d896d9cc
 SHA512:
-  metadata.gz: 28746a246fc18ab753febab8b7af0647b6e7e3fe661401d10b783668c6341cd42ecca70ac3522091c447c26af8c0862de56f5bf9567c316fd25d7d9755435df8
-  data.tar.gz: 757804606fb9536d34a9f96fe003af127b850c8bcb48b2680e93ed5acaa1cc843d8545bbbf7d6d5e3d0263ecf5396231be7a82a457f20a4c95446cb03ae82565
+  metadata.gz: 386303ab0eee0b66080c20db661f8498baafd99fd5f1c76fab156c001dd32228a096bbe6e3caa74d0ece4889f2a23390b4bb407c2850ba6c48d0118871cfeff9
+  data.tar.gz: af54b9cb60bfa64207e8281a4b8c2219dc6c63f8e83413b9707f68555036165fe17eb4aa43fa0c92994faf8ed8022808c9674a1994f12a3aed15693e6c8700b0

data/.rubocop.yml

@@ -14,7 +14,7 @@ Style/FormatStringToken:
 Style/StringLiterals:
   Enabled: false
 Metrics/MethodLength:
-  Max: 15
+  Max: 25
 Style/Documentation:
   Enabled: false
 Style/FrozenStringLiteralComment:

data/Gemfile

@@ -4,4 +4,4 @@ source 'https://rubygems.org'
 gemspec
 
 # Added at 2017-08-14 19:47:59 +0900 by 5hun:
-gem 'rubocop', '~> 1.16.1', require: false, group: [:development]
+gem 'rubocop', '~> 1.17.0', require: false, group: [:development]

data/lib/fluent/plugin/out_filter_list/version.rb

@@ -1,7 +1,7 @@
 module Fluent
   module Plugin
     module FilterList
-      VERSION = "0.7.4"
+      VERSION = "0.7.5"
     end
   end
 end

data/lib/matcher.rb

@@ -7,27 +7,19 @@ module Matchers
 
     def initialize(patterns)
       patterns = (patterns || []).compact.reject(&:empty?)
-      @trie = Trie.new patterns
+      @machine = ACAutomaton.new patterns
     end
 
     def matches?(text)
-      node = @trie.root
-      text.to_s.chars.each do |char|
-        failure = node.failure
-        node = node.children[char]
-
-        return true unless node.nil? || node.output.nil?
-        return true unless failure.nil? || failure.output.nil?
+      return false if text.nil? || text == ''
 
-        # Follow failure if it exists in case pattern doesn't match
-        node = failure.children[char] if node.nil?
-        node = failure if node.nil?
-      end
+      @machine.matches? text.to_s
+    end
 
-      return false if node.nil?
-      return true unless node.failure.nil? || node.failure.output.nil?
+    def find(text)
+      return false if text.nil? || text == ''
 
-      !node.output.nil?
+      @machine.find(text)
     end
   end
 
@@ -50,6 +42,30 @@ module Matchers
   end
 
   class Trie
+    class Node
+      attr_reader :children
+
+      def initialize
+        @children = {}
+        @children.default = nil
+      end
+
+      def insert(char)
+        @children[char] = Node.new unless @children.key?(char)
+        @children[char]
+      end
+
+      def forward(str)
+        children = @children
+        child = nil
+        str.chars.each do |char|
+          child = children[char]
+          children = child.children
+        end
+        child
+      end
+    end
+
     attr_reader :root
 
     def initialize(patterns)
@@ -58,38 +74,12 @@ module Matchers
       patterns.each do |pattern|
         insert(pattern)
       end
-      build
     end
 
     def insert(pattern = '')
       current_node = @root
-      pattern.chars.each_with_index do |char, i|
+      pattern.chars.each do |char|
         current_node = current_node.insert(char)
-        current_node.output = pattern if i == pattern.length - 1
-      end
-    end
-
-    def new_queue
-      q = Queue.new
-      @root.children.each_value do |child|
-        q.push(child)
-        child.failure = @root # set root on root's children's failure
-      end
-      q
-    end
-
-    def build
-      # Update failure on each node.
-      # Search longest matching suffix (which becomes failure) by BFS. In case no matching suffix, root becomes failure.
-      q = new_queue
-      until q.empty?
-        cur_node = q.pop
-        cur_node.children.each do |char, child|
-          q.push(child)
-          detect_node = cur_node.failure || @root
-          detect_node = detect_node.failure while detect_node.children[char].nil?
-          child.failure = detect_node.children[char]
-        end
       end
     end
 
@@ -107,20 +97,162 @@ module Matchers
     end
   end
 
-  class Node
-    attr_reader :children
-    attr_accessor :failure, :output
+  # An AC automaton.
+  # Based on https://www.cs.uku.fi/~kilpelai/BSA05/lectures/slides04.pdf.
+  class ACAutomaton
+    class Node
+      # Manages the goto as a character -> node ID mapping.
+      attr_reader :goto
+      # Uniquely (in an automaton) assigned ID of the Node.
+      attr_reader :id
+      attr_accessor :failure
+      # Stores out of AC, that is the index of the patterns.
+      attr_accessor :out
+
+      def initialize(id: 0, goto: {}, failure: 0, out: [])
+        @id = id
+        @goto = goto
+        @failure = failure
+        @out = out
+      end
+
+      def root?
+        @id.zero?
+      end
+
+      def g(char)
+        if (next_node = @goto[char])
+          return next_node
+        end
+        return 0 if root?
+
+        nil
+      end
 
-    def initialize
-      @children = {}
-      @children.default = nil
-      @output = nil
-      @failure = nil
+      def to_s
+        "id: #{@id}, goto: #{@goto}, failure: #{@failure}, out: #{@out}"
+      end
+
+      def ==(other)
+        @id == other.id && @goto == other.goto && @failure == other.failure && @out == other.out
+      end
     end
+    # Nodes are managed in an array. The indices are
+    attr_reader :nodes, :patterns
+
+    def initialize(patterns)
+      @nodes = []
+      @patterns = patterns || []
+      build(@patterns)
+    end
+
+    # Creates a new node and returns the id.
+    # This method is not thread safe.
+    def new_node
+      id = @nodes.size
+      node = Node.new(id: id)
+      @nodes.push(node)
+      id
+    end
+
+    def build(patterns)
+      build_goto(patterns)
+      build_failure
+    end
+
+    def build_goto(patterns)
+      root = new_node
+      patterns.each_with_index do |pattern, i|
+        q = root
+        pattern.chars.each do |char|
+          next_q = @nodes[q].goto[char]
+          if next_q
+            q = next_q
+          else
+            new_q = new_node
+            @nodes[q].goto[char] = new_q
+            q = new_q
+          end
+        end
+
+        @nodes[q].out.push(i)
+      end
+    end
+
+    def build_failure
+      queue = [0]
+      until queue.empty?
+        n = queue.shift
+        node = @nodes[n]
+        @nodes[n].goto.each do |c, next_node|
+          queue.push(next_node)
+
+          next if n.zero?
+
+          failure = node.failure
+          failure = @nodes[failure].failure while @nodes[failure].g(c).nil?
+          @nodes[next_node].failure = @nodes[failure].g(c)
+          @nodes[next_node].out.concat(@nodes[@nodes[next_node].failure].out)
+        end
+      end
+    end
+
+    def find(text)
+      return [] if text.nil? || text == ''
+
+      find_id(text).map do |id|
+        @patterns[id]
+      end
+    end
+
+    # Finds and retuns matched pattens' indices.
+    def find_id(text)
+      return [] if text.nil? || text == ''
+
+      q = 0
+      result = []
+      text.chars.each_with_index do |c, _i|
+        loop do
+          node = @nodes[q]
+          if (to_go_next = node.goto[c])
+            q = to_go_next
+            break
+          end
+          break if q.zero?
+
+          q = node.failure
+        end
+
+        out = @nodes[q].out
+        result.concat(out) unless out.empty?
+      end
+
+      result
+    end
+
+    # Returns true if the text matches any pattern, otherwise false.
+    def matches?(text)
+      return false if text.nil?
+
+      q = 0
+      text.chars.each do |c|
+        loop do
+          node = @nodes[q]
+          if (to_go_next = node.goto[c])
+            q = to_go_next
+            break
+          end
+          break if q.zero?
+
+          q = node.failure
+        end
+
+        out = @nodes[q].out
+
+        return true unless out.empty?
+      end
 
-    def insert(char)
-      @children[char] = Node.new unless @children.key?(char)
-      @children[char]
+      false
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-filter-list
 version: !ruby/object:Gem::Version
-  version: 0.7.4
+  version: 0.7.5
 platform: ruby
 authors:
 - Shun Yanaura
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-06-21 00:00:00.000000000 Z
+date: 2021-09-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -132,7 +132,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: A fluentd output plugin to filter keywords from messages