logstash-filter-transaction_time 1.0.3 → 1.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fd604885e2f901a408bb03385458f31c1671003a
-  data.tar.gz: 7694726b5fb532e2cc6b81a5b5bbf7ea23d0ec70
+  metadata.gz: '03089c0ec29302fde4807201a27eedabd29f9dc5'
+  data.tar.gz: aadea35e5b5cdd92321e1bbe7310faa0252c1a02
 SHA512:
-  metadata.gz: c2a6efc82adfa6debb09aeece93b06e22743c0ac07151b632f20fbd7a9633bf0c74fab6f1c0d846fa98938ec6b15ac66b4dd0c1527f39c03c4265b1d8a9be2dc
-  data.tar.gz: 9a1f33409749d7adb38befec19d2dd9b5ce0bfb59f513c96fba99b911a4f44b6f3efee4055cd5135f58836490f8d85fc4dc881554ccd335e46e1a4883b6ebcaa
+  metadata.gz: f8b2558058d717b92ba30c3a4b0172997d138702169b12cf3cbc46af193d922d22fe11dd8c9fd8bb8493c3e808e3a85ff5a4a30d32fabc00544b83adf0dc3feb
+  data.tar.gz: 9a1e6678888c794ce841ad2d9a1dbbe6ba086ca72d2cd03ce6eea4c043a5c8ab1a88412996064d54c7d680dca26cd1d26e47196864e2f4d5b6b91d46759344ad

data/lib/logstash/filters/transaction_time.rb

@@ -19,6 +19,10 @@ require "logstash/namespace"
 #         replace_timestamp => ['keep', 'oldest', 'newest']
 #         filter_tag => "transaction tag"
 #         attach_event => ['first','last','oldest','newest','none']
+#         release_expired => [true,false]
+#         store_data_oldest => []
+#         store_data_newest => []
+#         periodic_flush => [true,false]
 #       }
 #     }
 #
@@ -71,11 +75,42 @@ require "logstash/namespace"
 # The attach_event parameter can be used to append information from one of the events to the
 # new transaction_time event. The default is to not attach anything. 
 # The memory footprint is kept to a minimum by using the default value.
+#
+# The release_expired parameter determines if the first event in an expired transactions 
+# should be released or not. Defaults to true
+#
+# The parameters store_data_oldest and store_data_newest are both used in order to attach 
+# specific fields from oldest respectively newest event. An example of this could be:
+# 
+#    store_data_oldest => ["@timestamp", "work_unit", "work_center", "message_type"]
+#    store_data_newest => ["@timestamp", "work_unit", "work_center", "message_type"]
+# 
+# Which will result in the genereated transaction event inluding the specified fields from 
+# oldest and newest events in a hashmap named oldest/newest under the hash named "transaction_data"
+# Example of output data:
+# "transaction_data" => {
+#        "oldest" => {
+#            "message_type" => "MaterialIdentified",
+#              "@timestamp" => 2018-10-31T07:36:23.072Z,
+#               "work_unit" => "WT000743",
+#             "work_center" => "WR000046"
+#        },
+#        "newest" => {
+#            "message_type" => "Recipe",
+#              "@timestamp" => 2018-10-31T07:36:28.188Z,
+#               "work_unit" => "WT000743",
+#             "work_center" => "WR000046"
+#        }
+#    }
+
+
 
 class LogStash::Filters::TransactionTime < LogStash::Filters::Base
 
   HOST_FIELD = "host"
   TRANSACTION_TIME_TAG = "TransactionTime"
+  TRANSACTION_TIME_DATA = "transaction_data"
+  TRANSACTION_TIME_EXPIRED_TAG = "TransactionTimeExpired"
   TRANSACTION_TIME_FIELD = "transaction_time"
   TRANSACTION_UID_FIELD = "transaction_uid"
   TIMESTAMP_START_FIELD = "timestamp_start"
@@ -96,14 +131,28 @@ class LogStash::Filters::TransactionTime < LogStash::Filters::Base
   # Whether or not to attach one or none of the events in a transaction to the output event. 
   # Defaults to 'none' - which reduces memory footprint by not adding the event to the transactionlist.
   config :attach_event, :validate => ['first','last','oldest','newest','none'], :default => 'none'
+  # Wheter or not to release the first event in expired transactions
+  config :release_expired, :validate => :boolean, :default => true
+
+  # Store data from the oldest message. Specify array of keys to store.
+  config :store_data_oldest, :validate => :array, :default => []
+  # Store data from the newest message. Specify array of keys to store
+  config :store_data_newest, :validate => :array, :default => []
+
+  # This filter must have its flush function called periodically to be able to purge
+  # expired stored start events.
+  config :periodic_flush, :validate => :boolean, :default => true
 
   public
   def register
     # Add instance variables 
     @transactions = Hash.new
     @mutex = Mutex.new
-    @storeEvent = !(@attach_event.eql?"none")
+    @attachData = (!@store_data_oldest.nil? && @store_data_oldest.any?) || (!@store_data_newest.nil? && @store_data_newest.any?)
+    @storeEvent = (!(@attach_event.eql?"none") || @attachData)
     @@timestampTag = @timestamp_tag
+    @logger.info("Setting up INFO")
+    @logger.debug("Setting up DEBUG")
   end # def register
 
   def transactions
@@ -121,24 +170,30 @@ class LogStash::Filters::TransactionTime < LogStash::Filters::Base
 
     @logger.debug("Received UID", uid: uid)
 
+
     #Dont use filter-plugin on events created by this filter-plugin
     #Dont use filter on anything else but events with the filter_tag if specified
-    if (!uid.nil? && (event.get("tags").nil? || !event.get("tags").include?(TRANSACTION_TIME_TAG)) &&
+    if (!uid.nil? && (event.get("tags").nil? || !event.get("tags").include?(TRANSACTION_TIME_TAG)) && 
+      (event.get("tags").nil? || !event.get("tags").include?(TRANSACTION_TIME_EXPIRED_TAG)) &&
       (@filter_tag.nil? || (!event.get("tags").nil? && event.get("tags").include?(@filter_tag))))
-      filter_matched(event) 
+      
+
       @mutex.synchronize do
         if(!@transactions.has_key?(uid))
           @transactions[uid] = LogStash::Filters::TransactionTime::Transaction.new(event, uid, @storeEvent)
 
         else #End of transaction
           @transactions[uid].addSecond(event,@storeEvent)
-          transaction_event = new_transactiontime_event(@transactions[uid])
+          transaction_event = new_transactiontime_event(@transactions[uid], @attachData)
           filter_matched(transaction_event)
           yield transaction_event if block_given?
           @transactions.delete(uid)
         end
       end
     end
+    #Always forward original event
+    filter_matched(event) 
+    return event
 
   end # def filter
 
@@ -146,12 +201,17 @@ class LogStash::Filters::TransactionTime < LogStash::Filters::Base
   # The method is invoked by LogStash every 5 seconds.
   def flush(options = {})
     expired_elements = []
-
+    #@logger.info("FLUSH")
     @mutex.synchronize do
       increment_age_by(5)
       expired_elements = remove_expired_elements()
     end
 
+    expired_elements.each do |element|
+      filter_matched(element)
+    end
+    return expired_elements
+    #yield expired_elements if block_given?
     #return create_expired_events_from(expired_elements)
   end
 
@@ -162,13 +222,15 @@ class LogStash::Filters::TransactionTime < LogStash::Filters::Base
     end
   end
 
-  # Remove the expired "start events" from the internal
+  # Remove the expired "events" from the internal
   # buffer and return them.
   def remove_expired_elements()
     expired = []
     @transactions.delete_if do |key, transaction|
       if(transaction.age >= @timeout)
-        expired << transaction
+        #print("Deleting expired_elements")
+        transaction.tag(TRANSACTION_TIME_EXPIRED_TAG)
+        (expired << transaction.getEvents()).flatten!
         next true
       end
       next false
@@ -176,7 +238,7 @@ class LogStash::Filters::TransactionTime < LogStash::Filters::Base
     return expired
   end
 
-  def new_transactiontime_event(transaction)
+  def new_transactiontime_event(transaction, attachData)
 
 
       case @attach_event
@@ -199,12 +261,17 @@ class LogStash::Filters::TransactionTime < LogStash::Filters::Base
       transaction_event.set(TRANSACTION_UID_FIELD, transaction.uid)
       transaction_event.set(TIMESTAMP_START_FIELD, transaction.getOldestTimestamp())
 
+      #Attach transaction data if any
+      if(attachData)
+        transaction_data = transaction.getData(store_data_oldest,store_data_newest)
+        transaction_event.set(TRANSACTION_TIME_DATA,transaction_data)
+      end
+
       if(@replace_timestamp.eql?'oldest')
         transaction_event.set("@timestamp", transaction.getOldestTimestamp())
       elsif (@replace_timestamp.eql?'newest')
         transaction_event.set("@timestamp", transaction.getNewestTimestamp())
       end
-          
 
       return transaction_event
   end
@@ -219,7 +286,7 @@ end # class LogStash::Filters::TransactionTime
 
 
 class LogStash::Filters::TransactionTime::Transaction
-  attr_accessor :firstEvent, :lastEvent,:firstTimestamp, :secondTimestamp, :uid, :age, :diff
+  attr_accessor :firstEvent, :lastEvent,:firstTimestamp, :secondTimestamp, :uid, :age, :diff, :data
 
   def initialize(firstEvent, uid, storeEvent = false)
     if(storeEvent)
@@ -230,6 +297,33 @@ class LogStash::Filters::TransactionTime::Transaction
     @age = 0
   end
 
+  def getData(oldestKeys, newestKeys)
+    if(oldestKeys.any?)
+      storeData("oldest",oldestKeys,getOldestEvent())
+    end
+    if(newestKeys.any?)
+      storeData("newest",newestKeys,getNewestEvent())
+    end
+    return @data
+  end
+
+  def storeData(subDataName, dataKeys, dataEvent)
+    if(@data.nil?)
+      @data = Hash.new
+    end
+
+    if(@data[subDataName].nil?)
+      hashData = Hash.new
+    else
+      hashData = @data.get(subDataName)
+    end
+
+    dataKeys.each do |dataKey|
+      hashData[dataKey] = dataEvent.get(dataKey)
+      @data[subDataName] = hashData
+    end
+  end
+
   def addSecond(lastEvent,storeEvent = false)
     if(storeEvent)
       @lastEvent = lastEvent
@@ -282,4 +376,24 @@ class LogStash::Filters::TransactionTime::Transaction
 
     return getNewestTimestamp() - getOldestTimestamp()
   end
+
+  def tag(value)
+    if(!firstEvent.nil?)
+      firstEvent.tag(value)
+    end
+    if(!lastEvent.nil?)
+      lastEvent.tag(value)
+    end
+  end
+
+  def getEvents()
+    events = []
+    if(!firstEvent.nil?)
+      events << firstEvent
+    end
+    if(!lastEvent.nil?)
+      events << lastEvent
+    end
+    return events
+  end
 end

data/logstash-filter-transaction_time.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-filter-transaction_time'
-  s.version       = '1.0.3'
+  s.version       = '1.0.4'
   s.licenses      = ['Apache-2.0','Apache License (2.0)']
   s.summary       = 'Writes the time difference between two events in a transaction to a new event'
   s.description   = 'This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program. Source-code and documentation available at github: https://github.com/AddinITAB/logstash-filter-transaction_time'

data/spec/filters/transaction_time_spec.rb

@@ -5,6 +5,7 @@ require "logstash/filters/transaction_time"
 
 describe LogStash::Filters::TransactionTime do
   UID_FIELD = "uniqueIdField"
+
   TIMEOUT = 30
 
 
@@ -147,6 +148,21 @@ describe LogStash::Filters::TransactionTime do
         @filter.flush()
         insist { @filter.transactions.size } == 0
       end
+      it "releases the transactions to output" do
+        @filter.filter(event("message" => "Log message", UID_FIELD => uid, "@timestamp" => "2018-04-22T09:46:22.000+0100"))
+        insist { @filter.transactions.size } == 1
+        @filter.filter(event("message" => "Log message", UID_FIELD => uid2, "@timestamp" => "2018-04-22T09:46:22.000+0100"))
+        insist { @filter.transactions.size } == 2
+        ((TIMEOUT/5)-1).times do
+          @filter.flush()
+        end 
+        counter = 0
+        @filter.flush().each do |event|
+          counter+=1
+          #print(event)
+        end
+        insist { counter } == 2
+      end
       it "does not flush newer transactions" do
         @filter.filter(event("message" => "Log message", UID_FIELD => uid, "@timestamp" => "2018-04-22T09:46:22.000+0100"))
         insist { @filter.transactions.size } == 1
@@ -225,7 +241,6 @@ describe LogStash::Filters::TransactionTime do
     end
   end
 
-
   context "Testing attach_event." do
     uid = "9ACCA7B7-D0E9-4E52-A023-9D588E5BE42C"
     describe "Config attach_event" do
@@ -316,4 +331,50 @@ describe LogStash::Filters::TransactionTime do
     end
   end
 
+  context "Testing store_data." do
+    uid = "9ACCA7B7-D0E9-4E52-A023-9D588E5BE42C"
+    describe "Config store_data_oldest" do
+      it "attaches data from oldest event into transaction" do
+        config = {"store_data_oldest" => ["datafield1"]}
+        @config.merge!(config)
+
+        @filter = LogStash::Filters::TransactionTime.new(@config)
+        @filter.register
+
+        @filter.filter(event("message" => "first", UID_FIELD => uid, "@timestamp" => "2018-04-22T09:46:22.000+0100", "datafield1" => "OldestData"))
+        @filter.filter(event("message" => "last", UID_FIELD => uid, "@timestamp" => "2018-04-22T09:46:22.100+0100", "datafield1" => "NewestData")) do | new_event |
+          insist { new_event } != nil
+          insist { new_event.get("tags").include?("TransactionTime") }
+          insist { new_event.get("message") } != nil
+          insist { new_event.get("message") } == "first"
+          insist { new_event.get("transaction_data") } != nil
+          insist { new_event.get("transaction_data")["oldest"]["datafield1"] } != nil
+          insist { new_event.get("transaction_data")["oldest"]["datafield1"] } == "OldestData"
+        end
+      end
+
+      describe "and store_data_newest" do
+        it "attaches data from both newest and oldest event into transaction" do
+          config = {"store_data_newest" => ["datafield1"], "store_data_oldest" => ["datafield1"]}
+          @config.merge!(config)
+
+          @filter = LogStash::Filters::TransactionTime.new(@config)
+          @filter.register
+
+          @filter.filter(event("message" => "first", UID_FIELD => uid, "@timestamp" => "2018-04-22T09:46:22.000+0100", "datafield1" => "OldestData"))
+          @filter.filter(event("message" => "last", UID_FIELD => uid, "@timestamp" => "2018-04-22T09:46:22.100+0100", "datafield1" => "NewestData")) do | new_event |
+            insist { new_event } != nil
+            insist { new_event.get("tags").include?("TransactionTime") }
+            insist { new_event.get("message") } != nil
+            insist { new_event.get("message") } == "first"
+            insist { new_event.get("transaction_data") } != nil
+            insist { new_event.get("transaction_data")["newest"]["datafield1"] } != nil
+            insist { new_event.get("transaction_data")["newest"]["datafield1"] } == "NewestData"
+            insist { new_event.get("transaction_data")["oldest"]["datafield1"] } != nil
+            insist { new_event.get("transaction_data")["oldest"]["datafield1"] } == "OldestData"
+          end
+        end
+      end
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-transaction_time
 version: !ruby/object:Gem::Version
-  version: 1.0.3
+  version: 1.0.4
 platform: ruby
 authors:
 - Tommy Welleby
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-04-27 00:00:00.000000000 Z
+date: 2018-10-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement