logstash-filter-math 1.1.0 → 1.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bcecb6c0a2a2be1670c7c579a48b0565b796609329d51ef1f12f2a6603ff25eb
-  data.tar.gz: 2ff472b3df566011a47c83bad03066c05ad7028f0a9be7aa4b29b11ae4f86bd6
+  metadata.gz: 7acec65c8e0272d6a835d61e803de1b244cf838d649930dfaf8649e45c3388d0
+  data.tar.gz: 4ca406f2e0403317e20aaf74463c61b834399aa4840ef6a69d81357a24373d6d
 SHA512:
-  metadata.gz: af24bac7497bfd33eaf330480cabbee1598896c939cccafbfd55bb170cb41e566a8cd98552c7e1d1eb49e143f60ce678f1849cc5a9a7fd5b26b1bd68db859eb1
-  data.tar.gz: 7b34a8f23971a80328ca1bc33f909a618d01ebddf5b0d47e24d531ed45a350b5e0fea820e0f08d9a7beb93b27646dbd0c18d8e03a5c3a6dc27c9e444d0cfa538
+  metadata.gz: a005e6bf35f50b0e18207c875cab6d4878ad49088b8fe90bc1210946610cee915f281f427851407f5927a81e7ebea1dce4d942ec2078da501ded4a70a85e0ce5
+  data.tar.gz: 4dc84a9bd0a94b1c4ae727485f0ef6ba26e4d4c4ff96dbbb14edd59dbde45a101b6ba32da9ba77c389893a188e13c0642ae6a9a9a8a4498784d30068700caafa

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 1.1.1
+  - Fix to make registers threadsafe. [math filter #10](https://github.com/logstash-plugins/logstash-filter-math/issues/10)
+
 ## 1.1.0
   - Bumping to this version as this plugin was published to rubygems at v1.0
   - Fixed add backward compatible Operator reference aliases `'sub', 'mpx'` [math filter #8](https://github.com/logstash-plugins/logstash-filter-math/pull/8)

data/lib/logstash/filters/event_register_context.rb

@@ -0,0 +1,31 @@
+# encoding: utf-8
+
+module LogStash module Filters
+  class EventRegisterContext
+
+    attr_reader :event, :register
+
+    def initialize(event)
+      @event = event
+      @register = []
+    end
+
+    def get(element)
+      case element
+      when MathCalculationElements::RegisterElement
+        @register[element.key]
+      when MathCalculationElements::FieldElement
+        @event.get(element.key)
+      end
+    end
+
+    def set(element, value)
+      case element
+      when MathCalculationElements::RegisterElement
+        @register[element.key] = value
+      when MathCalculationElements::FieldElement
+        @event.set(element.key, value)
+      end
+    end
+  end
+end end

data/lib/logstash/filters/math.rb

@@ -3,6 +3,7 @@
 require "logstash/namespace"
 require "logstash/filters/base"
 
+require_relative "event_register_context"
 require_relative "math_functions"
 require_relative "math_calculation_elements"
 
@@ -58,7 +59,6 @@ module LogStash module Filters class Math < LogStash::Filters::Base
     # is exactly 4 fields and the first field is a valid calculation operator name.
     @calculate_copy = []
     all_function_keys = functions.keys
-    @register = []
     calculate.each do |calculation|
       if calculation.size != 4
         raise LogStash::ConfigurationError, I18n.t(
@@ -79,8 +79,8 @@ module LogStash module Filters class Math < LogStash::Filters::Base
       end
       function = functions[function_key]
 
-      left_element = MathCalculationElements.build(operand1, 1, @register)
-      right_element = MathCalculationElements.build(operand2, 2, @register)
+      left_element = MathCalculationElements.build(operand1, 1)
+      right_element = MathCalculationElements.build(operand2, 2)
       if right_element.literal?
         lhs = left_element.literal? ? left_element.get : 1
         warning = function.invalid?(lhs, right_element.get)
@@ -93,7 +93,7 @@ module LogStash module Filters class Math < LogStash::Filters::Base
           )
         end
       end
-      result_element = MathCalculationElements.build(target, 3, @register)
+      result_element = MathCalculationElements.build(target, 3)
       @calculate_copy << [function, left_element, right_element, result_element]
     end
     if @calculate_copy.last.last.is_a?(MathCalculationElements::RegisterElement)
@@ -108,18 +108,18 @@ module LogStash module Filters class Math < LogStash::Filters::Base
 
   def filter(event)
     event_changed = false # can exit if none of the calculations are are suitable
-    @register.clear # don't carry over register results from one event to the next.
+    context = EventRegisterContext.new(event) # creates a new registers array, each element can use the event or register from the store
     @calculate_copy.each do |function, left_element, right_element, result_element|
       logger.debug("executing", "function" => function.name, "left_field" => left_element, "right_field" => right_element, "target" => result_element)
       # TODO add support for automatic conversion to Numeric if String
-      operand1 = left_element.get(event)
-      operand2 = right_element.get(event)
+      operand1 = left_element.get(context)
+      operand2 = right_element.get(context)
       # allow all the validation warnings to be logged before we skip to next
       next if operand1.nil? || operand2.nil?
       next if function.invalid?(operand1, operand2, event)
 
       result = function.call(operand1, operand2)
-      result_element.set(result, event)
+      result_element.set(result, context)
       logger.debug("calculation result stored", "function" => function.name, "target" => result_element, "result" => result)
       event_changed = true
     end

data/lib/logstash/filters/math_calculation_elements.rb

@@ -5,7 +5,7 @@ module LogStash module Filters
   module MathCalculationElements
     REGISTER_REFERENCE_RE = /^MEM\[(\d+)]$/
 
-    def self.build(reference, position, register)
+    def self.build(reference, position)
       case reference
       when Numeric
         if position == 3
@@ -17,7 +17,7 @@ module LogStash module Filters
       when String
         match = REGISTER_REFERENCE_RE.match(reference)
         if match
-          RegisterElement.new(reference, position, match[1].to_i, register)
+          RegisterElement.new(reference, position, match[1].to_i)
         else
           FieldElement.new(reference, position)
         end
@@ -28,25 +28,28 @@ module LogStash module Filters
 
     class RegisterElement
       # supports `get` and `set`
-      def initialize(reference, position, index, register)
+      def initialize(reference, position, index)
         @reference = reference
         @position = position
         @index = index
-        @register = register
         @description = (position == 3 ? "#{@index}" : "operand #{@position}").prepend("register ").concat(": '#{@reference}'")
       end
 
+      def key
+        @index
+      end
+
       def literal?
         false
       end
 
-      def set(value, event)
+      def set(value, event_register_context)
         # raise usage error if called when position != 3 ??
-        @register[@index] = value
+        event_register_context.set(self, value)
       end
 
-      def get(event)
-        @register[@index] #log warning if nil
+      def get(event_register_context)
+        event_register_context.get(self) #log warning if nil
       end
 
       def inspect
@@ -60,6 +63,7 @@ module LogStash module Filters
 
     class FieldElement
       include LogStash::Util::Loggable
+
       # supports `get` and `set`
       def initialize(field, position)
         @field = field
@@ -67,18 +71,22 @@ module LogStash module Filters
         @description = (position == 3 ? "result" : "operand #{@position}").prepend("event ").concat(": '#{@field}'")
       end
 
+      def key
+        @field
+      end
+
       def literal?
         false
       end
 
-      def set(value, event)
-        event.set(@field, value)
+      def set(value, event_register_context)
+        event_register_context.set(self, value)
       end
 
-      def get(event)
-        value = event.get(@field)
+      def get(event_register_context)
+        value = event_register_context.get(self)
         if value.nil?
-          logger.warn("field not found", "field" => @field, "event" => event.to_hash)
+          logger.warn("field not found", "field" => @field, "event" => event_register_context.event.to_hash)
           return nil
         end
         case value
@@ -87,7 +95,7 @@ module LogStash module Filters
         when LogStash::Timestamp, Time
           value.to_f
         else
-          logger.warn("field value is not numeric or time", "field" => @field, "value" => value, "event" => event.to_hash)
+          logger.warn("field value is not numeric or time", "field" => @field, "value" => value, "event" => event_register_context.event.to_hash)
           nil
         end
       end
@@ -108,11 +116,15 @@ module LogStash module Filters
         @position = position
       end
 
+      def key
+        nil
+      end
+
       def literal?
         true
       end
 
-      def get(event = nil)
+      def get(event_register_context = nil)
         @literal
       end
 

data/logstash-filter-math.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-filter-math'
-  s.version         = '1.1.0'
+  s.version         = '1.1.1'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Do simple math functions on numeric fields."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/math_spec.rb

@@ -443,6 +443,43 @@ describe LogStash::Filters::Math do
     end
   end
 
+  describe "multithreading" do
+    it "should still calculate correctly" do
+      array = [
+        [ "+", "[var1]", "[var2]", "MEM[0]" ],
+        [ "-", "[var3]", "[var4]", "MEM[1]" ],
+        [ "*", "MEM[0]", "MEM[1]", "[result]" ]
+      ]
+
+      math_hash = {"calculate" => array}
+
+      event1 = LogStash::Event.new("var1" => 3.4,  "var2" => 6.6, "var3" => 4.4, "var4" => 2.4)
+      event2 = LogStash::Event.new("var1" => 6.8,  "var2" => 13.2, "var3" => 8.8, "var4" => 4.8)
+      plugin = described_class.new(math_hash)
+      plugin.register
+      expect do
+        thread1 = Thread.new(plugin, event1) do |plugin, event|
+          100.times do
+            plugin.filter(event)
+            result = event.get("result")
+            raise "Thread 1 failed, result is: #{result}" if result != 20.000000000000004
+            sleep 0.011
+          end
+        end
+        thread2 = Thread.new(plugin, event2) do |plugin, event|
+          100.times do
+            plugin.filter(event)
+            result = event.get("result")
+            raise "Thread 2 failed, result is: #{result}" if result != 80.00000000000001
+            sleep 0.01
+          end
+        end
+        thread1.join
+        thread2.join
+      end.not_to raise_exception
+    end
+  end
+
   describe "Sequence" do
     # The logstash config.
     config <<-CONFIG

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-math
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.1.1
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-06-29 00:00:00.000000000 Z
+date: 2018-09-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -58,6 +58,7 @@ files:
 - LICENSE
 - README.md
 - docs/index.asciidoc
+- lib/logstash/filters/event_register_context.rb
 - lib/logstash/filters/math.rb
 - lib/logstash/filters/math_calculation_elements.rb
 - lib/logstash/filters/math_functions.rb