logstash-filter-math 1.0 → 1.1.0

data/lib/logstash/filters/math_calculation_elements.rb

@@ -0,0 +1,128 @@
+# encoding: utf-8
+require "logstash/util/loggable"
+
+module LogStash module Filters
+  module MathCalculationElements
+    REGISTER_REFERENCE_RE = /^MEM\[(\d+)]$/
+
+    def self.build(reference, position, register)
+      case reference
+      when Numeric
+        if position == 3
+          # literal reference for result element
+          nil
+        else
+          LiteralElement.new(reference, position)
+        end
+      when String
+        match = REGISTER_REFERENCE_RE.match(reference)
+        if match
+          RegisterElement.new(reference, position, match[1].to_i, register)
+        else
+          FieldElement.new(reference, position)
+        end
+      else
+        nil
+      end
+    end
+
+    class RegisterElement
+      # supports `get` and `set`
+      def initialize(reference, position, index, register)
+        @reference = reference
+        @position = position
+        @index = index
+        @register = register
+        @description = (position == 3 ? "#{@index}" : "operand #{@position}").prepend("register ").concat(": '#{@reference}'")
+      end
+
+      def literal?
+        false
+      end
+
+      def set(value, event)
+        # raise usage error if called when position != 3 ??
+        @register[@index] = value
+      end
+
+      def get(event)
+        @register[@index] #log warning if nil
+      end
+
+      def inspect
+        "\"#{@description}\""
+      end
+
+      def to_s
+        @description
+      end
+    end
+
+    class FieldElement
+      include LogStash::Util::Loggable
+      # supports `get` and `set`
+      def initialize(field, position)
+        @field = field
+        @position = position
+        @description = (position == 3 ? "result" : "operand #{@position}").prepend("event ").concat(": '#{@field}'")
+      end
+
+      def literal?
+        false
+      end
+
+      def set(value, event)
+        event.set(@field, value)
+      end
+
+      def get(event)
+        value = event.get(@field)
+        if value.nil?
+          logger.warn("field not found", "field" => @field, "event" => event.to_hash)
+          return nil
+        end
+        case value
+        when Numeric
+          value
+        when LogStash::Timestamp, Time
+          value.to_f
+        else
+          logger.warn("field value is not numeric or time", "field" => @field, "value" => value, "event" => event.to_hash)
+          nil
+        end
+      end
+
+      def inspect
+        "\"#{@description}\""
+      end
+
+      def to_s
+        @description
+      end
+    end
+
+    class LiteralElement
+      # does not support `set`
+      def initialize(literal, position)
+        @literal = literal
+        @position = position
+      end
+
+      def literal?
+        true
+      end
+
+      def get(event = nil)
+        @literal
+      end
+
+      def inspect
+        "\"operand #{@position}: #{@literal.inspect}\""
+      end
+
+      def to_s
+        inspect
+      end
+    end
+  end
+end end

data/lib/logstash/filters/math_functions.rb

@@ -0,0 +1,142 @@
+# encoding: utf-8
+require "logstash/util/loggable"
+
+module LogStash module Filters
+  module MathFunctions
+    module DivByZeroValidityCheck
+      def invalid?(op1, op2, event = nil)
+        if op2.zero?
+          warning = "a divisor of zero is not permitted"
+          if event
+            # called from filter so log
+            logger.warn(warning, "operand 1" => op1, "operand 2" => op2, "event" => event.to_hash)
+            return true
+          else
+            # called from register don't log return warning for the error that is raised
+            return warning
+          end
+        end
+        nil
+      end
+    end
+
+    module NoValidityCheckNeeded
+      def invalid?(op1, op2, event = nil)
+        nil
+      end
+    end
+
+    class Add
+      include NoValidityCheckNeeded
+
+      def name
+        "add"
+      end
+
+      def call(op1, op2)
+        op1 + op2
+      end
+    end
+
+    class Subtract
+      include NoValidityCheckNeeded
+
+      def name
+        "subtract"
+      end
+      def call(op1, op2)
+        op1 - op2
+      end
+    end
+
+    class Multiply
+      include NoValidityCheckNeeded
+
+      def name
+        "multiply"
+      end
+
+      def call(op1, op2)
+        op1 * op2
+      end
+    end
+
+    class Round
+      include NoValidityCheckNeeded
+
+      def name
+        "round"
+      end
+
+      def call(op1, op2)
+        op1.round(op2)
+      end
+    end
+
+    class Power
+      include LogStash::Util::Loggable
+
+      def name
+        "power"
+      end
+
+      def call(op1, op2)
+        op1 ** op2
+      end
+
+      def invalid?(op1, op2, event = nil)
+        if op1.is_a?(Numeric) && op1 < 0 && !op2.integer?
+          warning = "raising a negative number to a fractional exponent results in a complex number that cannot be stored in an event"
+          if event
+            # called from filter so log
+            logger.warn(warning, "operand 1" => op1, "operand 2" => op2, "event" => event.to_hash)
+            return true
+          else
+            # called from register don't log return warning for the error that is raised
+            return warning
+          end
+        end
+        nil
+      end
+    end
+
+    class Divide
+      include LogStash::Util::Loggable
+      include DivByZeroValidityCheck
+
+      def name
+        "divide"
+      end
+
+      def call(op1, op2)
+        op1 / op2
+      end
+    end
+
+    class FloatDivide
+      include LogStash::Util::Loggable
+      include DivByZeroValidityCheck
+
+      def name
+        "float_divide"
+      end
+
+      def call(op1, op2)
+        op1.fdiv(op2)
+      end
+    end
+
+    class Modulo
+      include LogStash::Util::Loggable
+      include DivByZeroValidityCheck
+
+      def name
+        "modulo"
+      end
+
+      def call(op1, op2)
+        op1 % op2
+      end
+    end
+  end
+end end

data/logstash-filter-math.gemspec

@@ -1,24 +1,24 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-filter-math'
-  s.version         = '1.0'
+  s.version         = '1.1.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Do simple math functions on numeric fields."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
-  s.authors         = ["Robin Clarke"]
-  s.email           = 'robin@robinclarke.net'
-  s.homepage        = "http://www.elastics.co/guide/en/logstash/current/index.html"
+  s.authors         = ["Elastic"]
+  s.email           = 'info@elastic.co'
+  s.homepage        = "http://www.elastic.co/guide/en/logstash/current/index.html"
   s.require_paths = ["lib"]
 
-
   # Files
-  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
-   # Tests
+  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "docs/**/*"]
+
+  # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})
 
   # Special flag to let us know this is actually a logstash plugin
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
+  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_development_dependency 'logstash-devutils'
 end

data/spec/filters/math_spec.rb

@@ -1,27 +1,26 @@
 # encoding: utf-8
-require "logstash/devutils/rspec/spec_helper"
+require_relative "../spec_helper"
 require "logstash/filters/math"
 
 describe LogStash::Filters::Math do
-   
   describe "Additions" do
     # The logstash config.
     config <<-CONFIG
       filter {  math { calculate => [ [ "add", "var1", "var2", "result" ] ] } }
     CONFIG
-  
+
     describe "should add two integers" do
       sample( "var1" => -2,  "var2" => 7 ) do
         expect( subject.get("result") ).to eq( 5 )
       end
     end
-    
+
     describe "should add two floats" do
       sample( "var1" => -2.4,  "var2" => 7.8 ) do
         expect( subject.get("result") ).to eq( 5.4  )
       end
     end
-    
+
     describe "two huge numbers should add to infinity" do
       sample( "var1" => 1.79769313486232e+308,  "var2" => 1e+308 ) do
         expect( subject.get("result") ).to eq( Float::INFINITY )
@@ -39,7 +38,7 @@ describe LogStash::Filters::Math do
         expect( subject.get("result") ).to be_nil
       end
     end
-  
+
     describe "Second value missing should result in nil" do
       sample( "var1" => 3 ) do
         expect( subject.get("result") ).to be_nil
@@ -50,27 +49,27 @@ describe LogStash::Filters::Math do
   describe "Subtractions" do
     # The logstash config.
     config <<-CONFIG
-      filter {  math { calculate => [ [ "sub", "var1", "var2", "result" ] ] } }
+      filter {  math { calculate => [ [ "-", "var1", "var2", "result" ] ] } }
     CONFIG
-  
+
     describe "should subtract two integers" do
       sample( "var1" => -2,  "var2" => 7 ) do
         expect( subject.get("result") ).to eq( -9 )
       end
     end
-    
+
     describe "should subtract two floats" do
       sample( "var1" => -2.4,  "var2" => 7.8 ) do
         expect( subject.get("result") ).to eq( -10.2  )
       end
     end
-    
+
     describe "two huge negative numbers should subtract to negative infinity" do
       sample( "var1" => -1.79769313486232e+308,  "var2" => -1e+308 ) do
         expect( subject.get("result") ).to eq( -Float::INFINITY )
       end
     end
-    
+
     describe "one value being 0 should work" do
       sample( "var1" => 0,  "var2" => 7.8 ) do
         expect( subject.get("result") ).to eq( -7.8 )
@@ -82,7 +81,7 @@ describe LogStash::Filters::Math do
         expect( subject.get("result") ).to be_nil
       end
     end
-    
+
     describe "Second value missing should result in nil" do
       sample( "var1" => 3 ) do
         expect( subject.get("result") ).to be_nil
@@ -90,30 +89,77 @@ describe LogStash::Filters::Math do
     end
   end
 
+  describe "Rounding" do
+    describe "when using a field as the right hand operand" do
+      config <<-CONFIG
+        filter {  math { calculate => [ [ "round", "var1", "var2", "result" ] ] } }
+      CONFIG
+
+      describe "should round a float field" do
+        sample( "var1" => 0.42424242, "var2" => 2 ) do
+          expect( subject.get("result") ).to eq( 0.42 )
+        end
+      end
+
+      describe "should round an integer field" do
+        sample( "var1" => 42, "var2" => 2 ) do
+          expect( subject.get("result") ).to eq( 42.0 )
+        end
+      end
+    end
+
+    describe "when using a literal as the right hand operand" do
+      config <<-CONFIG
+        filter {  math { calculate => [ [ "round", "var1", 3, "result" ] ] } }
+      CONFIG
+
+      describe "should round a float field" do
+        sample( "var1" => 0.42424242 ) do
+          expect( subject.get("result") ).to eq( 0.424 )
+        end
+      end
+
+      describe "should convert an integer to a float" do
+        sample( "var1" => 42 ) do
+          expect( subject.get("result") ).to eq( 42.0 )
+        end
+      end
+
+      describe "should convert a float to an integer" do
+        config <<-CONFIG
+          filter {  math { calculate => [ [ "round", "var1", 0, "result" ] ] } }
+        CONFIG
+        sample( "var1" => 42.0 ) do
+          expect( subject.get("result") ).to eq( 42 )
+        end
+      end
+    end
+  end
+
   describe "Multiplication" do
     # The logstash config.
     config <<-CONFIG
-      filter {  math { calculate => [ [ "mpx", "var1", "var2", "result" ] ] } }
+      filter {  math { calculate => [ [ "*", "var1", "var2", "result" ] ] } }
     CONFIG
-  
+
     describe "should multiply two integers" do
       sample( "var1" => -2,  "var2" => 7 ) do
         expect( subject.get("result") ).to eq( -14 )
       end
     end
-    
+
     describe "should multiply two floats" do
       sample( "var1" => -2.4,  "var2" => 7.8 ) do
         expect( subject.get("result") ).to eq( -18.72 )
       end
     end
-    
+
     describe "two huge numbers should multiply to infinity" do
       sample( "var1" => 1.79769313486232e+300,  "var2" => 1e+300 ) do
         expect( subject.get("result") ).to eq( Float::INFINITY )
       end
     end
-    
+
     describe "one value being 0 should result in 0" do
       sample( "var1" => 0,  "var2" => 7.8 ) do
         expect( subject.get("result") ).to eq( 0 )
@@ -125,7 +171,7 @@ describe LogStash::Filters::Math do
         expect( subject.get("result") ).to be_nil
       end
     end
-    
+
     describe "Second value missing should result in nil" do
       sample( "var1" => 3 ) do
         expect( subject.get("result") ).to be_nil
@@ -133,36 +179,148 @@ describe LogStash::Filters::Math do
     end
   end
 
+  describe "Exponentiation" do
+    # The logstash config.
+    describe "verbose operation name" do
+      config <<-CONFIG
+        filter {  math { calculate => [ [ "to the power of", "var1", "var2", "result" ] ] } }
+      CONFIG
+
+      describe "should exponentiate two integers" do
+        sample( "var1" => -2,  "var2" => 5 ) do
+          expect( subject.get("result") ).to eq( -32 )
+        end
+      end
+
+      describe "should exponentiate two floats" do
+        sample( "var1" => 2.2,  "var2" => 1.2 ) do
+          expect( subject.get("result") ).to eq( 2.5757708085227633 )
+        end
+      end
+
+      describe "two huge numbers should exponentiate to infinity" do
+        sample( "var1" => 1.79769313486232e+300,  "var2" => 1e+300 ) do
+          expect( subject.get("result") ).to eq( Float::INFINITY )
+        end
+      end
+    end
+
+    describe "terse operation name" do
+      config <<-CONFIG
+        filter {  math { calculate => [ [ "**", "var1", "var2", "result" ] ] } }
+      CONFIG
+
+      describe "base value being negative and the exponent being fractional should result in nil" do
+        sample( "var1" => -2.2,  "var2" => 7.8 ) do
+          expect( subject.get("result") ).to eq( nil )
+        end
+      end
+
+      describe "base value being 0 should result in 0" do
+        sample( "var1" => 0,  "var2" => 7.8 ) do
+          expect( subject.get("result") ).to eq( 0 )
+        end
+      end
+
+      describe "exponent value being 0 should result in 1" do
+        sample( "var1" => 7.8,  "var2" => 0 ) do
+          expect( subject.get("result") ).to eq( 1 )
+        end
+      end
+
+      describe "first value missing should result in nil" do
+        sample( "var2" => 3 ) do
+          expect( subject.get("result") ).to be_nil
+        end
+      end
+
+      describe "Second value missing should result in nil" do
+        sample( "var1" => 3 ) do
+          expect( subject.get("result") ).to be_nil
+        end
+      end
+    end
+  end
+
   describe "Division" do
     # The logstash config.
     config <<-CONFIG
-      filter {  math { calculate => [ [ "div", "var1", "var2", "result" ] ] } }
+      filter {  math { calculate => [ [ "/", "var1", "var2", "result" ] ] } }
     CONFIG
-  
+
+    describe "should divide two integers" do
+      sample( "var1" => -2,  "var2" => 7 ) do
+        expect( subject.get("result") ).to eq( -1 )
+      end
+    end
+
+    describe "should divide two floats" do
+      sample( "var1" => -2.4,  "var2" => 7.8 ) do
+        expect( subject.get("result") ).to eq( -0.3076923076923077 )
+      end
+    end
+
+    describe "1 divided by a huge number should give zero" do
+      sample( "var1" => 1, "var2" => 1.79769313486232e+308 ) do
+        expect( subject.get("result") ).to eq( 0 )
+      end
+    end
+
+    describe "First variable being zero should result in 0" do
+      sample( "var1" => 0,  "var2" => 7.8 ) do
+        expect( subject.get("result") ).to eq( 0 )
+      end
+    end
+
+    describe "Second variable being zero should result in nil (would be infinity, but this can't be represented in JSON)" do
+      sample( "var1" => 2,  "var2" => 0 ) do
+        expect( subject.get("result") ).to be_nil
+      end
+    end
+
+    describe "first value missing should result in nil" do
+      sample( "var2" => 3 ) do
+        expect( subject.get("result") ).to be_nil
+      end
+    end
+
+    describe "Second value missing should result in nil" do
+      sample( "var1" => 3 ) do
+        expect( subject.get("result") ).to be_nil
+      end
+    end
+  end
+
+  describe "FloatDiv" do
+    # The logstash config.
+    config <<-CONFIG
+      filter {  math { calculate => [ [ "fdiv", "var1", "var2", "result" ] ] } }
+    CONFIG
+
     describe "should divide two integers" do
       sample( "var1" => -2,  "var2" => 7 ) do
         expect( subject.get("result") ).to eq( -0.2857142857142857 )
       end
     end
-    
+
     describe "should divide two floats" do
       sample( "var1" => -2.4,  "var2" => 7.8 ) do
         expect( subject.get("result") ).to eq( -0.3076923076923077 )
       end
     end
-    
+
     describe "1 divided by a huge number should give zero" do
       sample( "var1" => 1, "var2" => 1.79769313486232e+308 ) do
         expect( subject.get("result") ).to eq( 0 )
       end
     end
-    
+
     describe "First variable being zero should result in 0" do
       sample( "var1" => 0,  "var2" => 7.8 ) do
         expect( subject.get("result") ).to eq( 0 )
       end
     end
-    
+
     describe "Second variable being zero should result in nil (would be infinity, but this can't be represented in JSON)" do
       sample( "var1" => 2,  "var2" => 0 ) do
         expect( subject.get("result") ).to be_nil
@@ -174,29 +332,155 @@ describe LogStash::Filters::Math do
         expect( subject.get("result") ).to be_nil
       end
     end
-    
+
     describe "Second value missing should result in nil" do
       sample( "var1" => 3 ) do
         expect( subject.get("result") ).to be_nil
       end
     end
   end
-  
+
+  describe "Modulo" do
+    # The logstash config.
+    config <<-CONFIG
+      filter {  math { calculate => [ [ "mod", "var1", "var2", "result" ] ] } }
+    CONFIG
+
+    describe "should get modulo of two integers" do
+      sample( "var1" => 53,  "var2" => 13 ) do
+        expect( subject.get("result") ).to eq( 1 )
+      end
+    end
+
+    describe "should get modulo of two floats" do
+      sample( "var1" => 53.4,  "var2" => 13.1 ) do
+        expect( subject.get("result") ).to eq( 1.0 )
+      end
+    end
+
+    describe "1 modulo a huge number should give 1.0" do
+      sample( "var1" => 1, "var2" => 1.79769313486232e+308 ) do
+        expect( subject.get("result") ).to eq( 1.0 )
+      end
+    end
+
+    describe "First variable being zero should result in 0.0" do
+      sample( "var1" => 0,  "var2" => 7.8 ) do
+        expect( subject.get("result") ).to eq( 0.0 )
+      end
+    end
+
+    describe "Second variable being zero should result in nil (would be infinity, but this can't be represented in JSON)" do
+      sample( "var1" => 2,  "var2" => 0 ) do
+        expect( subject.get("result") ).to be_nil
+      end
+    end
+
+    describe "first value missing should result in nil" do
+      sample( "var2" => 3 ) do
+        expect( subject.get("result") ).to be_nil
+      end
+    end
+
+    describe "Second value missing should result in nil" do
+      sample( "var1" => 3 ) do
+        expect( subject.get("result") ).to be_nil
+      end
+    end
+  end
+
+  describe "Literals" do
+    context "how much smaller is one number than another in percent" do
+      config <<-CONFIG
+        filter {  math { calculate => [ [ "float divide", "var1", "var2", "result" ],[ "times", "result", 100, "percent_difference" ] ] } }
+      CONFIG
+      sample( "var1" => 13, "var2" => 104 ) do
+        expect( subject.get("percent_difference") ).to eq(12.5)
+      end
+    end
+
+    context "what is the reciprocal of a value" do
+      config <<-CONFIG
+        filter {  math { calculate => [ [ "float divide", 1, "var1", "result" ] ] } }
+      CONFIG
+      sample( "var1" => 8 ) do
+        expect( subject.get("result") ).to eq(0.125)
+      end
+    end
+
+    context "when specifying a zero literal, a divide by zero error is detected at plugin register" do
+      it "raises a validation error" do
+        pipeline = new_pipeline_from_string('filter {  math { calculate => [ [ "divide", "var1", 0, "result" ] ] } }')
+        expect { pipeline.instance_eval{ @filters.each(&:register) } }.to raise_exception(LogStash::ConfigurationError, /Numeric literals are specified as in the calculation but the function invalidates with 'a divisor of zero is not permitted'/)
+      end
+    end
+
+    context "when specifying literals: -2 and 1.2 with the Power function, an error is detected at plugin register" do
+      it "raises a validation error" do
+        pipeline = new_pipeline_from_string('filter {  math { calculate => [ [ "**", -2, 1.2, "result" ] ] } }')
+        expect { pipeline.instance_eval{ @filters.each(&:register) } }.to raise_exception(LogStash::ConfigurationError, /Numeric literals are specified as in the calculation but the function invalidates with 'raising a negative number to a fractional exponent results in a complex number that cannot be stored in an event'/)
+      end
+    end
+  end
+
+  describe "Timestamps" do
+    t1 = Time.new(2018, 06, 8, 11, 0, 0,"+00:00")
+    t2 = Time.new(2018, 06, 8, 11, 0, 30,"+00:00")
+    config <<-CONFIG
+      filter {  math { calculate => [ [ "sub", "[var2]", "[var1]", "result" ] ] } }
+    CONFIG
+
+    context "subtracting 2 LogStash::Timestamps" do
+      sample( "var1" => LogStash::Timestamp.at(t1.to_f),  "var2" =>  LogStash::Timestamp.at(t2.to_f)) do
+        expect( subject.get("result") ).to eq(30.0)
+      end
+    end
+
+    context "subtracting a Time from a Timestamp" do
+      sample( "var1" => t1,  "var2" =>  LogStash::Timestamp.at(t2.to_f)) do
+        expect( subject.get("result") ).to eq(30.0)
+      end
+    end
+  end
+
   describe "Sequence" do
     # The logstash config.
     config <<-CONFIG
-      filter {  math { calculate => [ 
-          [ "add", "var1", "var2", "r1" ],
-          [ "sub", "var3", "var4", "r2" ],
-          [ "mpx", "r1", "r2", "result" ]
+      filter {  math { calculate => [
+            [ "add", "var1", "var2", "r1" ],
+            [ "sub", "var3", "var4", "r2" ],
+            [ "mpx", "r1", "r2", "result" ]
           ] } }
     CONFIG
-  
-    describe "results of one calculation can be used in the next calculation"
+
+    describe "results of one calculation can be used in the next calculation" do
+      sample( "var1" => 3.4,  "var2" => 6.6, "var3" => 4.4, "var4" => 2.4 ) do
+        # I would really expect 20.0 here... what kind of floating point error is this?!
+        expect( subject.get("result") ).to eq( 20.000000000000004 )
+      end
+    end
+  end
+
+  describe "Sequence with registers" do
+    describe "results of one calculation can be used in the next calculation" do
+      config <<-CONFIG
+        filter {  math { calculate => [
+              [ "+", "[var1]", "[var2]", "MEM[0]" ],
+              [ "-", "[var3]", "[var4]", "MEM[1]" ],
+              [ "*", "MEM[0]", "MEM[1]", "[result]" ]
+            ] } }
+      CONFIG
       sample( "var1" => 3.4,  "var2" => 6.6, "var3" => 4.4, "var4" => 2.4 ) do
         # I would really expect 20.0 here... what kind of floating point error is this?!
         expect( subject.get("result") ).to eq( 20.000000000000004 )
       end
     end
 
+    describe "when a register is used as the very last target, a configuration error is raised" do
+      it "raises a validation error" do
+        pipeline = new_pipeline_from_string('filter {  math { calculate => [ [ "**", "[var1]", "[var2]", "MEM[0]" ] ] } }')
+        expect { pipeline.instance_eval{ @filters.each(&:register) } }.to raise_exception(LogStash::ConfigurationError, /The final target is a Register, the overall calculation result will not be set in the event/)
+      end
+    end
+  end
 end