logstash-filter-math 1.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: d4dbbbc58d14856fcd8e53f57c6897c2a2eb7221
-  data.tar.gz: c939090de8e97cfd20622fb6c84064f59c2e67ae
+SHA256:
+  metadata.gz: bcecb6c0a2a2be1670c7c579a48b0565b796609329d51ef1f12f2a6603ff25eb
+  data.tar.gz: 2ff472b3df566011a47c83bad03066c05ad7028f0a9be7aa4b29b11ae4f86bd6
 SHA512:
-  metadata.gz: 4f9340a61978c42ef4c56436a4b29f15d38f1fffa433bb27bae632888b7b0e9d16232ae785752521cfa157fe4f05b0f6f5c7e47b525fda6cdbdd8ddec0090334
-  data.tar.gz: 4df67712a02104a781721991bf36221004056efad23a1e50c745fd8048691a8d126b9e8727001743c8b7ed93304869dc081f60b4cc857f431521e2a9f3663a37
+  metadata.gz: af24bac7497bfd33eaf330480cabbee1598896c939cccafbfd55bb170cb41e566a8cd98552c7e1d1eb49e143f60ce678f1849cc5a9a7fd5b26b1bd68db859eb1
+  data.tar.gz: 7b34a8f23971a80328ca1bc33f909a618d01ebddf5b0d47e24d531ed45a350b5e0fea820e0f08d9a7beb93b27646dbd0c18d8e03a5c3a6dc27c9e444d0cfa538

data/CHANGELOG.md

@@ -0,0 +1,15 @@
+## 1.1.0
+  - Bumping to this version as this plugin was published to rubygems at v1.0
+  - Fixed add backward compatible Operator reference aliases `'sub', 'mpx'` [math filter #8](https://github.com/logstash-plugins/logstash-filter-math/pull/8)
+
+## 1.0.0
+  - This plugin is now in the logstash-plugins GH org
+  - Added Memory registers
+  - Added More functions Power, Round, Modulo, FloatDivide
+  - Added Literals
+  - Added Operator reference aliases, e.g. `'*', 'times', 'multiply'` all refer to Multiply.
+  - Added asciidoc documentation
+  - Above features provided by [math filter #5](https://github.com/logstash-plugins/logstash-filter-math/pull/5)
+
+## 1.0
+  - Published to rubygems by Robin Clarke

data/CONTRIBUTORS

@@ -0,0 +1,11 @@
+The following is a list of people who have contributed ideas, code, bug
+reports, or in general have helped this plugin along its way.
+
+Contributors:
+* Robin Clarke (robin13)
+* Guy Boertje (guyboertje)
+
+Note: If you've sent us patches, bug reports, or otherwise contributed to
+this plugin, and you aren't on the list above and want to be, please let us know
+and we'll make sure you're here. Contributions from folks like you are what make
+open source awesome.

data/Gemfile

@@ -1,2 +1,10 @@
 source 'https://rubygems.org'
 gemspec
+
+logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
+use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
+
+if Dir.exist?(logstash_path) && use_logstash_source
+  gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
+  gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
+end

data/README.md

@@ -1,24 +1,25 @@
 # logstash-filter-math
 
-This plugin provides the ability to do various simple math operations (addition, subtraction, 
-multiplication and division) on document fields.
+This plugin provides the ability to do various simple math operations (addition, subtraction, multiplication and division) on document fields
 
 # Logstash Plugin
 
+[![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-filter-math.svg)](https://travis-ci.org/logstash-plugins/logstash-filter-math)
+
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
 It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
 
 ## Documentation
 
-Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elastic.org/guide/en/logstash/current/).
+Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elastic.co/guide/en/logstash/current/).
 
 - For formatting code or config example, you can use the asciidoc `[source,ruby]` directive
 - For more asciidoc formatting tips, see the excellent reference here https://github.com/elastic/docs#asciidoc-guide
 
 ## Need Help?
 
-Need help? Try #logstash on freenode IRC or the logstash-users@googlegroups.com mailing list.
+Need help? Try #logstash on freenode IRC or the https://discuss.elastic.co/c/logstash discussion forum.
 
 ## Developing
 
@@ -58,7 +59,12 @@ gem "logstash-filter-awesome", :path => "/your/local/logstash-filter-awesome"
 ```
 - Install plugin
 ```sh
+# Logstash 2.3 and higher
+bin/logstash-plugin install --no-verify
+
+# Prior to Logstash 2.3
 bin/plugin install --no-verify
+
 ```
 - Run Logstash with your plugin
 ```sh
@@ -76,7 +82,12 @@ gem build logstash-filter-awesome.gemspec
 ```
 - Install the plugin from the Logstash home
 ```sh
-bin/plugin install /your/local/plugin/logstash-filter-awesome.gem
+# Logstash 2.3 and higher
+bin/logstash-plugin install --no-verify
+
+# Prior to Logstash 2.3
+bin/plugin install --no-verify
+
 ```
 - Start Logstash and proceed to test the plugin
 
@@ -89,3 +100,4 @@ Programming is not a required skill. Whatever you've seen about open source and
 It is more important to the community that you are able to contribute.
 
 For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file.
+

data/docs/index.asciidoc

@@ -0,0 +1,207 @@
+:plugin: math
+:type: filter
+
+///////////////////////////////////////////
+START - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+:version: %VERSION%
+:release_date: %RELEASE_DATE%
+:changelog_url: %CHANGELOG_URL%
+:include_path: ../../../../logstash/docs/include
+///////////////////////////////////////////
+END - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+
+[id="plugins-{type}s-{plugin}"]
+
+=== Math filter plugin
+
+include::{include_path}/plugin_header.asciidoc[]
+
+==== Description
+
+This filter performs simple arithmetic calculations on Numeric or Logstash Timestamp values taken
+from fields in an event.
+
+The following example shows how multiple steps can be calculated.
+Imagine you have two fields that represent distances in kilometers and you need
+to add them together and multiply the result by 1000 to get total
+distance in meters.
+
+["source","json",subs="callouts"]
+-----
+filter {
+  math {
+    calculate => [
+      [ "add", "[walk1_distance]", "[walk2_distance]", "MEM[0]" ], <1>
+      [ "multiply", MEM[0], 1000, "[total_distance_m]" ] <2>
+    ]
+  }
+}
+-----
+<1> This calculation adds the two field's values together and stores the result in a memory register.
+<2> This calculation multiplies the value in the memory register with a literal numeric value.
+
+Here's a full example that takes outside and inside air temperatures in degrees
+Fahrenheit and calculates the difference in Celsius. The arithmetic equivalent is:
+
+`delta_in_c = round( ((inside - 32) * 5 / 9) - ((outside - 32) * 5 / 9) )`
+
+[source,json]
+-----
+input {
+  generator {
+    message => '{"sensor":"temperature-1", "inside": 71.24, "outside": 61.7 }'
+    count => 1
+  }
+}
+
+filter {
+  json {
+    source => "message"
+  }
+  if "_jsonparsefailure" not in [tags]  {
+    math {
+      calculate => [
+        [ "fdiv", 5, 9, "MEM[0]" ],
+        [ "subtract", "[outside]", 32, "MEM[1]" ],
+        [ "multiply", "MEM[1]", "MEM[0]", "MEM[1]" ],
+        [ "subtract", "[inside]", 32, "MEM[2]" ],
+        [ "multiply", "MEM[2]", "MEM[0]", "MEM[2]" ],
+        [ "subtract", "MEM[2]", "MEM[1]", "MEM[3]" ],
+        [ "round", "MEM[3]", 1, "[delta_in_c]" ]
+      ]
+    }
+  }
+}
+
+output {
+  stdout {
+    codec => rubydebug
+  }
+}
+-----
+
+The resulting event looks like this.
+
+[source,shell]
+{
+        "inside" => 71.24,
+    "@timestamp" => 2018-06-23T13:25:22.298Z,
+        "sensor" => "temperature-1",
+    "delta_in_c" => 5.3,
+       "message" => "{\"sensor\":\"temperature-1\", \"inside\": 71.24, \"outside\": 61.7 }",
+      "@version" => "1",
+      "sequence" => 0,
+          "host" => "Elastics-MacBook-Pro.local",
+       "outside" => 61.7
+}
+
+[id="plugins-{type}s-{plugin}-options"]
+==== Math Filter Configuration Options
+
+This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
+
+[cols="<,<,<",options="header",]
+|=======================================================================
+|Setting |Input type|Required
+| <<plugins-{type}s-{plugin}-calculate>> |<<array,array>> of <<array,array>>|Yes
+|=======================================================================
+
+Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
+filter plugins.
+
+&nbsp;
+
+[id="plugins-{type}s-{plugin}-calculate"]
+===== `calculate`
+
+  * This is a required setting.
+  * Value type is <<array,array>> of <<array,array>>s
+  * There is no default value for this setting.
+
+The calculation to be performed. As can be seen from the example above by using
+multiple inner arrays one can perform calculation with multiple steps or multiple
+distinct calculations on a single event.
+
+Each inner array *must have 4 elements*
+
+The first element must be the operator. Valid operators are:
+
+[cols="<,<",options="header",]
+|=======================================================================
+| Operation | Representations
+| Add | '+', 'add', 'plus'
+| Subtract | '-', 'sub', 'subtract'
+| Multiply | '*', 'mpx', 'times', 'multiply'
+| Round | 'round'
+| Power | '**', '^', 'to the power of'
+| Divide | '/', 'div', 'divide'
+| Modulo | 'mod', 'modulo'
+| FloatDivide | 'fdiv', 'float divide'
+|=======================================================================
+
+[TIP]
+You may use any of the representations to refer to an operator. For instance, long hand representation
+maybe more clear to read when you or a co-worker looks at your config some months later.
+
+The second element is the left hand side operand. It can be a field, a literal or a memory register.
+If referring to a memory register, ensure that it has been set to a value in a previous calculation.
+[NOTE]
+A literal can be a float or an integer. Exponent expressions, e.g. `10e+3` are not supported.
+
+The third element is the right hand side operand. It can be a field, a literal or a memory register.
+
+The fourth element is the "target". It is where the result is stored and can be a
+memory register or a field in your event. It cannot be a literal.
+[NOTE]
+You will get a warning if the target of the last calculation is a memory register because
+the final result will not be added to the event.
+
+===== Valid Values
+For a calculation to continue, any operand taken from a field in the event must be
+`Numeric` or a `Logstash Timestamp` and not nil. Timestamps are converted to
+floating point seconds since the UNIX epoch (Jan, 1 1970 00:00:00 UTC) before being operated on.
+
+===== Operators
+The operators `Divide`, `FloatDivide` and `Modulo` have a divide by zero check before the operation is executed. +
+The `Power` operator has a check for a negative number being raised to a fractional power as this results in
+a Complex number that can't be stored in an event or serialized to JSON. +
+The `Round` operator can convert integers to floats `round(42, 1) -> 42.0`
+and floats to integers (rounding up or down) `round(0.75, 0) -> 1`.
+
+===== Memory Registers
+You can choose not to use memory registers and store intermediate results in fields instead but then you may
+need to remove the fields later. +
+Memory registers are implemented as a sparse array and the integer between the square brackets is a zero based
+direct index into the array. For example, `MEM[5]`, 5 is a reference to the 6th element in the array.
+The array is cleared for each event, this means that you can't leave a value behind for a later event to use.
+Each math filter will have its own memory register array so you can't share values between math filters
+in the same pipeline or across pipelines.
+
+[TIP]
+Use the bracketed notation e.g. `[fieldname]` to better distinguish fields from
+memory register references.
+
+===== Debugging
+There is some logging of the calculation progress at the debug logging level. +
+This is an excerpt from the full example at the top of this page:
+[source,shell]
+[DEBUG][logstash.filters.math    ] executing {"function"=>"float_divide", "left_field"=>"operand 1: 5", "right_field"=>"operand 2: 9", "target"=>"register 0: 'MEM[0]'"}
+[DEBUG][logstash.filters.math    ] calculation result stored {"function"=>"float_divide", "target"=>"register 0: 'MEM[0]'", "result"=>0.5555555555555556}
+[DEBUG][logstash.filters.math    ] executing {"function"=>"subtract", "left_field"=>"event operand 1: '[outside]'", "right_field"=>"operand 2: 32", "target"=>"register 1: 'MEM[1]'"}
+[DEBUG][logstash.filters.math    ] calculation result stored {"function"=>"subtract", "target"=>"register 1: 'MEM[1]'", "result"=>#<BigDecimal:2c16adee,'0.297E2',3(4)>}
+[DEBUG][logstash.filters.math    ] executing {"function"=>"multiply", "left_field"=>"register operand 1: 'MEM[1]'", "right_field"=>"register operand 2: 'MEM[0]'", "target"=>"register 1: 'MEM[1]'"}
+[DEBUG][logstash.filters.math    ] calculation result stored {"function"=>"multiply", "target"=>"register 1: 'MEM[1]'", "result"=>#<BigDecimal:76bc6a4c,'0.1650000000000000132E2',19(20)>}
+[DEBUG][logstash.filters.math    ] executing {"function"=>"subtract", "left_field"=>"event operand 1: '[inside]'", "right_field"=>"operand 2: 32", "target"=>"register 2: 'MEM[2]'"}
+[DEBUG][logstash.filters.math    ] calculation result stored {"function"=>"subtract", "target"=>"register 2: 'MEM[2]'", "result"=>#<BigDecimal:6761cedc,'0.3924E2',4(8)>}
+[DEBUG][logstash.filters.math    ] executing {"function"=>"multiply", "left_field"=>"register operand 1: 'MEM[2]'", "right_field"=>"register operand 2: 'MEM[0]'", "target"=>"register 2: 'MEM[2]'"}
+[DEBUG][logstash.filters.math    ] calculation result stored {"function"=>"multiply", "target"=>"register 2: 'MEM[2]'", "result"=>#<BigDecimal:5b03b20a,'0.21800000000000001744E2',20(24)>}
+[DEBUG][logstash.filters.math    ] executing {"function"=>"subtract", "left_field"=>"register operand 1: 'MEM[2]'", "right_field"=>"register operand 2: 'MEM[1]'", "target"=>"register 3: 'MEM[3]'"}
+[DEBUG][logstash.filters.math    ] calculation result stored {"function"=>"subtract", "target"=>"register 3: 'MEM[3]'", "result"=>#<BigDecimal:2e0e043,'0.5300000000000000424E1',19(20)>}
+[DEBUG][logstash.filters.math    ] executing {"function"=>"round", "left_field"=>"register operand 1: 'MEM[3]'", "right_field"=>"operand 2: 1", "target"=>"event result: '[delta_in_c]'"}
+[DEBUG][logstash.filters.math    ] calculation result stored {"function"=>"round", "target"=>"event result: '[delta_in_c]'", "result"=>#<BigDecimal:5415cc0,'0.53E1',2(4)>}
+
+
+[id="plugins-{type}s-{plugin}-common-options"]
+include::{include_path}/{type}.asciidoc[]

data/lib/logstash/filters/math.rb

@@ -1,14 +1,20 @@
-require "logstash/filters/base"
+# encoding: utf-8
+
 require "logstash/namespace"
-# Do various simple math operations
+require "logstash/filters/base"
+
+require_relative "math_functions"
+require_relative "math_calculation_elements"
+
+# Do various simple math functions
 # Configuration:
 # filter {
 #   math {
 #     calculate => [
-#         [ "add", "a_field1", "a_field2", "a_target" ],   # a + b => target
-#         [ "sub", "b_field1", "b_field2", "b_target" ],   # a - b => target
-#         [ "div", "c_field1", "c_field2", "c_target" ],   # a / b => target
-#         [ "mpx", "d_field1", "d_field2", "d_target" ]    # a * b => target
+#         [ "+", "a_field1", "a_field2", "a_target" ],   # a + b => target
+#         [ "-", "b_field1", "b_field2", "b_target" ],   # a - b => target
+#         [ "/", "c_field1", "c_field2", "c_target" ],   # a / b => target
+#         [ "*", "d_field1", "d_field2", "d_target" ]    # a * b => target
 #     ]
 #   }
 # }
@@ -19,50 +25,107 @@ require "logstash/namespace"
 #
 # Works with float and integer values
 
-class LogStash::Filters::Math < LogStash::Filters::Base
+module LogStash module Filters class Math < LogStash::Filters::Base
+  # TODO: Add support for unitary functions like abs and negation (-),
+  #       these would have one operand and may or may not have a target.
+  #       Add support for constants as either operand, e.g. seconds to millis, 100 / N
+  #       Add support for conversions: distance(mi <-> km)
+  #       Add support for percent change
   config_name "math"
 
   # fields - second subtracted from the first
   config :calculate, :validate => :array, :required => true
 
   public
+
   def register
-    # Do some sanity checks that calculate is actually an array-of-arrays, and that each calculation (sub-array)
-    # is exactly 4 fields and the first field is a valid calculation opperator name.
-    for calculation in calculate do
-      if calculation.length % 4 != 0
-        abort("Each calculation must have 4 elements, this one had " + calculation.length + " " + calculation.to_s )
-      end # end calculaction.length is 4
-      if ! calculation[0].match('^(add|sub|div|mpx)$' )
-        abort("First element of a calculation must be add|sub|div|mpx, but is: " + calculation[0] )
-      end # if calculation[0] valid
-    end # for each calculate
-  end # def register
+    functions = {}
+    [
+      [MathFunctions::Add.new, '+', 'add', 'plus'],
+      [MathFunctions::Subtract.new, '-', 'sub', 'subtract'],
+      [MathFunctions::Multiply.new, '*', 'mpx', 'times', 'multiply'],
+      [MathFunctions::Round.new, 'round'],
+      [MathFunctions::Power.new, '**', '^', 'to the power of'],
+      [MathFunctions::Divide.new, '/', 'div', 'divide'],
+      [MathFunctions::Modulo.new, 'mod', 'modulo'],
+      [MathFunctions::FloatDivide.new, 'fdiv', 'float divide']
+    ].each do |list|
+      value = list.shift
+      list.each{|key| functions[key] = value}
+    end
+
+    # Do some sanity checks that calculate is actually an array of arrays, and that each calculation (inner array)
+    # is exactly 4 fields and the first field is a valid calculation operator name.
+    @calculate_copy = []
+    all_function_keys = functions.keys
+    @register = []
+    calculate.each do |calculation|
+      if calculation.size != 4
+        raise LogStash::ConfigurationError, I18n.t(
+          "logstash.runner.configuration.invalid_plugin_register",
+          :plugin => "filter",
+          :type => "math",
+          :error => "Invalid number of elements in a calculation setting: expected 4, got: #{calculation.size}. You specified: #{calculation}"
+        )
+      end
+      function_key, operand1, operand2, target = calculation
+      if !all_function_keys.include?(function_key)
+        raise LogStash::ConfigurationError, I18n.t(
+          "logstash.runner.configuration.invalid_plugin_register",
+          :plugin => "filter",
+          :type => "math",
+          :error => "Invalid first element of a calculation: expected one of #{all_function_keys.join(',')}, got: #{function_key}. You specified: #{calculation.join(',')}"
+        )
+      end
+      function = functions[function_key]
+
+      left_element = MathCalculationElements.build(operand1, 1, @register)
+      right_element = MathCalculationElements.build(operand2, 2, @register)
+      if right_element.literal?
+        lhs = left_element.literal? ? left_element.get : 1
+        warning = function.invalid?(lhs, right_element.get)
+        unless warning.nil?
+          raise LogStash::ConfigurationError, I18n.t(
+            "logstash.runner.configuration.invalid_plugin_register",
+            :plugin => "filter",
+            :type => "math",
+            :error => "Numeric literals are specified as in the calculation but the function invalidates with '#{warning}'. You specified: #{calculation.join(',')}"
+          )
+        end
+      end
+      result_element = MathCalculationElements.build(target, 3, @register)
+      @calculate_copy << [function, left_element, right_element, result_element]
+    end
+    if @calculate_copy.last.last.is_a?(MathCalculationElements::RegisterElement)
+      raise LogStash::ConfigurationError, I18n.t(
+        "logstash.runner.configuration.invalid_plugin_register",
+        :plugin => "filter",
+        :type => "math",
+        :error => "The final target is a Register, the overall calculation result will not be set in the event"
+      )
+    end
+  end
 
-  public
   def filter(event)
-    return unless filter?(event)
-    for calculation in calculate do
-      # Check that all the fields exist and are numeric
-      next unless event.include?(calculation[1])
-      next unless event.include?(calculation[2])
-      next unless event.get(calculation[1]) == 0 or event.get(calculation[1]).is_a? Float or event.get(calculation[1]).is_a? Integer
-      next unless event.get(calculation[2]) == 0 or event.get(calculation[2]).is_a? Float or event.get(calculation[2]).is_a? Integer
-      case calculation[0]
-      when "add"
-        event.set( calculation[3], event.get(calculation[1]) + event.get(calculation[2]) )
-      when "sub"
-        event.set(calculation[3], event.get(calculation[1]) - event.get(calculation[2]) )
-      when "div"
-        # Avoid division by zero
-        next if event.get( calculation[2] ) == 0
-        event.set( calculation[3], event.get( calculation[1]).to_f / event.get(calculation[2]) )
-      when "mpx"
-        event.set( calculation[3], event.get( calculation[1]) * event.get( calculation[2] ) )
-      end # case calculation[0]
-    end # for each calculate
+    event_changed = false # can exit if none of the calculations are are suitable
+    @register.clear # don't carry over register results from one event to the next.
+    @calculate_copy.each do |function, left_element, right_element, result_element|
+      logger.debug("executing", "function" => function.name, "left_field" => left_element, "right_field" => right_element, "target" => result_element)
+      # TODO add support for automatic conversion to Numeric if String
+      operand1 = left_element.get(event)
+      operand2 = right_element.get(event)
+      # allow all the validation warnings to be logged before we skip to next
+      next if operand1.nil? || operand2.nil?
+      next if function.invalid?(operand1, operand2, event)
+
+      result = function.call(operand1, operand2)
+      result_element.set(result, event)
+      logger.debug("calculation result stored", "function" => function.name, "target" => result_element, "result" => result)
+      event_changed = true
+    end
+    return unless event_changed
     filter_matched(event)
-  end # def filter
-end # class LogStash::Filters::Math
+  end
+end end end