fluent-plugin-filter_where 1.0.0

data/lib/fluent/plugin/filter_where/parser/exp.rb

@@ -0,0 +1,178 @@
+# Operation Node of AST (Abstract Syntax Tree)
+require_relative 'literal'
+
+module Fluent
+  module FilterWhere
+    class Parser
+      class  Exp
+        def eval(record)
+          raise NotImplementedError
+        end
+      end
+
+      class BinaryOpExp < Exp
+        attr_reader :left, :right, :operator
+
+        # @param [ParserLiteral] left
+        # @param [ParserLiteral] right
+        # @param [Symbol] operator
+        def initialize(left, right, operator)
+          @left = left
+          @right = right
+          @operator = operator
+        end
+      end
+
+      class BooleanOpExp < BinaryOpExp
+        # @return [Boolean]
+        def eval(record)
+          l = left.get(record)
+          r = right.get(record)
+          case operator
+          when :EQ
+            l == r
+          when :NEQ
+            l != r
+          else
+            assert(false)
+            false
+          end
+        end
+      end
+
+      class NumberOpExp < BinaryOpExp
+        # @return [Boolean]
+        def eval(record)
+          l = left.get(record)
+          r = right.get(record)
+          case operator
+          when :EQ
+            l == r
+          when :NEQ
+            l != r
+          when :GT
+            l > r
+          when :GE
+            l >= r
+          when :LT
+            l < r
+          when :LE
+            l <= r
+          else
+            assert(false)
+            false
+          end
+        end
+      end
+
+      class StringOpExp < BinaryOpExp
+        def eval(record)
+          l = left.get(record)
+          r = right.get(record)
+          case operator
+          when :EQ
+            l == r
+          when :NEQ
+            l != r
+          # when :GT
+          #   l > r
+          # when :GE
+          #   l >= r
+          # when :LT
+          #   l < r
+          # when :LE
+          #   l <= r
+          when :START_WITH
+            l.start_with?(r)
+          when :END_WITH
+            l.end_with?(r)
+          when :INCLUDE
+            l.include?(r)
+          else
+            assert(false)
+            false
+          end
+        end
+      end
+
+      class RegexpOpExp < BinaryOpExp
+        attr_reader :pattern
+
+        def initialize(left, right, operator)
+          super(left, right, operator)
+          @pattern = Regexp.compile(right.val)
+        end
+
+        def eval(record)
+          l = left.get(record)
+          !!@pattern.match(l)
+        end
+      end
+
+      class NullOpExp < Exp
+        attr_reader :literal, :operator
+
+        # @param [ParserLiteral] literal
+        # @param [Symbol] operator
+        def initialize(literal, operator)
+          @literal = literal
+          @operator = operator
+        end
+
+        # @return [Boolean]
+        def eval(record)
+          is_null = literal.get(record).nil?
+          case operator
+          when :EQ
+            is_null
+          when :NEQ
+            ! is_null
+          else
+            assert(false)
+            false
+          end
+        end
+      end
+
+      class LogicalOpExp < Exp
+        attr_reader :left, :right, :operator
+
+        # @param [ParserLiteral] left
+        # @param [ParserLiteral] right
+        # @param [Symbol] operator
+        def initialize(left, right, operator)
+          @left = left
+          @right = right
+          @operator = operator
+        end
+
+        def eval(record)
+          l = left.eval(record)
+          r = right.eval(record)
+          case operator
+          when :OR
+            return l || r
+          when :AND
+            l && r
+          else
+            assert(false)
+            false
+          end
+        end
+      end
+
+      class NegateOpExp < Exp
+        attr_reader :exp
+
+        # @param [Exp] exp
+        def initialize(exp)
+          @exp = exp
+        end
+
+        def eval(record)
+          ! exp.eval(record)
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/filter_where/parser/literal.rb

@@ -0,0 +1,56 @@
+# Literal Node of AST (Abstract Syntax Tree)
+require 'time'
+
+module Fluent
+  module FilterWhere
+    class Parser
+      class Literal
+        attr_reader :text
+        attr_reader :val
+
+        def get(record)
+          @val
+        end
+      end
+
+      class BooleanLiteral < Literal
+        def initialize(text)
+          @text = text
+          if text.downcase == 'true'.freeze
+            @val = true
+          elsif text.downcase == 'false'.freeze
+            @val = false
+          else
+            raise ConfigError.new("\"%s\" is not a Boolean literal" % text)
+          end
+        end
+      end
+
+      class NumberLiteral < Literal
+        def initialize(text)
+          @text = text
+          @val = Float(text)
+        end
+      end
+
+      class StringLiteral < Literal
+        def initialize(text)
+          @text = text
+          @val = text
+        end
+      end
+
+      class IdentifierLiteral < Literal
+        attr_reader :name
+
+        def initialize(name)
+          @name = name
+        end
+
+        def get(record)
+          record[@name]
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/filter_where/v12.rb

@@ -0,0 +1,26 @@
+require_relative 'core'
+
+module Fluent
+  class FilterWhere < Filter
+    Fluent::Plugin.register_filter('where', self)
+
+    include ::Fluent::FilterWhere::Core
+
+    def initialize
+      super
+    end
+
+    # To support log_level option implemented by Fluentd v0.10.43
+    unless method_defined?(:log)
+      define_method("log") { $log }
+    end
+
+    def configure(conf)
+      super
+    end
+
+    def filter(tag, time, record)
+      super
+    end
+  end
+end

data/lib/fluent/plugin/filter_where/v14.rb

@@ -0,0 +1,21 @@
+require_relative 'core'
+
+module Fluent
+  class Plugin::FilterWhere < Plugin::Filter
+    Fluent::Plugin.register_filter('where', self)
+
+    include ::Fluent::FilterWhere::Core
+
+    def initialize
+      super
+    end
+    
+    def configure(conf)
+      super
+    end
+
+    def filter(tag, time, record)
+      super
+    end
+  end
+end

data/test/bench_filter_where.rb

@@ -0,0 +1,23 @@
+# encoding: UTF-8
+require_relative 'helper'
+require 'fluent/plugin/filter_where'
+require 'benchmark'
+Fluent::Test.setup
+
+# setup
+message = {'message' => "2013/01/13T07:02:11.124202 INFO GET /ping"}
+time = event_time
+
+string_eq = create_driver(%[where string = 'string'])
+string_regexp = create_driver(%[where string REGEXP '.*string.*'])
+
+# bench
+n = 10000
+Benchmark.bm(7) do |x|
+  x.report("string_eq")  { string_eq.run  { n.times { string_eq.feed(time, message)  } } }
+  x.report("string_regexp")  { string_regexp.run  { n.times { string_regexp.feed(time, message)  } } }
+end
+
+#               user     system      total        real
+# string_eq  0.030000   0.000000   0.030000 (  0.036840)
+# string_regexp  0.040000   0.000000   0.040000 (  0.043812)

data/test/helper.rb

@@ -0,0 +1,98 @@
+require 'test/unit'
+require 'test/unit/rr'
+require 'timecop'
+require 'fluent/log'
+require 'fluent/test'
+
+ROOT = File.dirname(__dir__)
+
+unless defined?(Test::Unit::AssertionFailedError)
+  class Test::Unit::AssertionFailedError < StandardError
+  end
+end
+
+# Reduce sleep period at
+# https://github.com/fluent/fluentd/blob/a271b3ec76ab7cf89ebe4012aa5b3912333dbdb7/lib/fluent/test/base.rb#L81
+module Fluent
+  module Test
+    class TestDriver
+      def run(num_waits = 10, &block)
+        @instance.start
+        begin
+          # wait until thread starts
+          # num_waits.times { sleep 0.05 }
+          sleep 0.05
+          return yield
+        ensure
+          @instance.shutdown
+        end
+      end
+    end
+  end
+end
+
+## v0.14
+# d = create_driver(conf, syntax: :v1)
+# d.run(default_tag: default_tag) do
+#   d.feed(time, record)
+# end
+# d.filtered_records
+#
+## v0.12
+# d = create_driver(conf, use_v1, default_tag: tag)
+# d.run do
+#   d.emit(record, time)
+# end
+# d.filtered # es
+#
+## Ours
+# d = create_driver(conf, syntax: :v1, default_tag: tag)
+# d.run do
+#   d.feed(time, record)
+# end
+# d.filtered_records
+
+require 'fluent/version'
+major, minor, patch = Fluent::VERSION.split('.').map(&:to_i)
+if major > 0 || (major == 0 && minor >= 14)
+  require 'fluent/test/driver/filter'
+  require 'fluent/test/helpers'
+  include Fluent::Test::Helpers
+
+  class FilterWhereTestDriver < Fluent::Test::Driver::Filter
+    def initialize(klass, tag)
+      super(klass)
+      @tag = tag
+    end
+
+    def run(&block)
+      super(default_tag: @tag, &block)
+    end
+  end
+
+  def create_driver(conf, syntax: :v1, default_tag: 'tag')
+    FilterWhereTestDriver.new(Fluent::Plugin::FilterWhere, default_tag).configure(conf, syntax: syntax)
+  end
+else # <= v0.12
+  def event_time(str)
+    Time.parse(str)
+  end
+
+  class FilterWhereTestDriver < Fluent::Test::FilterTestDriver
+    def configure(conf, syntax: :v1)
+      syntax == :v1 ? super(conf, true) : super(conf, false)
+    end
+
+    def feed(time, record)
+      emit(record, time)
+    end
+
+    def filtered_records
+      @filtered.map {|_time, record| record }
+    end
+  end
+
+  def create_driver(conf, syntax: :v1, default_tag: 'tag')
+    FilterWhereTestDriver.new(Fluent::Plugin::FilterWhere, default_tag).configure(conf, syntax: syntax)
+  end
+end

data/test/test_filter_where.rb

@@ -0,0 +1,37 @@
+require_relative 'helper'
+require 'time'
+require 'fluent/plugin/filter_where'
+
+Fluent::Test.setup
+
+class FilterWhereTest < Test::Unit::TestCase
+  setup do
+    @time = event_time("2010-05-04 03:02:01")
+    Timecop.freeze(@time)
+  end
+
+  teardown do
+    Timecop.return
+  end
+
+  sub_test_case '#configure' do
+    test 'check default' do
+      assert_raise(Fluent::ConfigError) { create_driver(%[]) }
+    end
+
+    test "where" do
+      assert_nothing_raised { create_driver(%[where foo = 'foo']) }
+    end
+  end
+
+  sub_test_case '#filter' do
+    test "where" do
+      d = create_driver(%[where foo = 'foo'])
+      d.run do
+        d.feed(@time, {'foo' => 'foo'})
+        d.feed(@time, {'foo' => 'bar'})
+      end
+      assert { d.filtered_records == [{'foo' => 'foo'}] }
+    end
+  end
+end