logstash-filter-transaction_time 1.0.4 → 1.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: '03089c0ec29302fde4807201a27eedabd29f9dc5'
-  data.tar.gz: aadea35e5b5cdd92321e1bbe7310faa0252c1a02
+  metadata.gz: 074c50d41cd992c872b665bb63497ceee820cf5a
+  data.tar.gz: eeab3271345ab338bddcce72683b744a04a6cf9c
 SHA512:
-  metadata.gz: f8b2558058d717b92ba30c3a4b0172997d138702169b12cf3cbc46af193d922d22fe11dd8c9fd8bb8493c3e808e3a85ff5a4a30d32fabc00544b83adf0dc3feb
-  data.tar.gz: 9a1e6678888c794ce841ad2d9a1dbbe6ba086ca72d2cd03ce6eea4c043a5c8ab1a88412996064d54c7d680dca26cd1d26e47196864e2f4d5b6b91d46759344ad
+  metadata.gz: f244fe40c2e89b9a7a9f7e95441b7ce6b5a9fe2006d3d1fc6ab77d892b095969888552aa40003a776fda376b244a2f8dd9980a9e5b5a1423c92574e3f2354537
+  data.tar.gz: e9c27b093beac560791750525774e74feac4fded750ac4c3a804efc9d99e69a7473a90f707129122a2585146970091a4f3970074af1fdcdf50b12d121c6fae06

data/README.md

@@ -8,7 +8,122 @@ But instead of defining a start and an end for a transaction - only the unique i
 Per default the transaction time is stored together with the unique identifier in a new event, which may be stored in the same or another index.
 The information from the first, last, oldest or newest event may be attached with the new transaction_time event.
 
-
+# Usage
+
+ The TransactionTime filter measures the time between two events in a transaction
+
+ This filter is supposed to be used instead of logstash-filters-elapsed
+ when you know that the order of a transaction cannot be guaranteed.
+ Which is most likely the case if you are using multiple workers and 
+ a big amount of events are entering the pipeline in a rapid manner.
+
+ ## The configuration:
+ ```ruby
+     filter {
+       transaction_time {
+         uid_field => "Transaction-unique field"
+         ignore_uid => []
+         timeout => seconds
+         timestamp_tag => "name of timestamp"
+         replace_timestamp => ['keep', 'oldest', 'newest']
+         filter_tag => "transaction tag"
+         attach_event => ['first','last','oldest','newest','none']
+         release_expired => [true,false]
+         store_data_oldest => []
+         store_data_newest => []
+         periodic_flush => [true,false]
+       }
+     }
+```
+- `uid_field`
+ The only required parameter is "uid_field" which is used to identify
+ the events in a transaction. A transaction is concidered complete 
+ when two events with the same UID has been captured. 
+ It is when a transaction completes that the transaction time is calculated.
+
+- `ignore_uid`
+ The ignore_uid field takes an array of strings. These strings represent specific UIDs
+ that should be ignored. This can be useful for ignoring parsing errors.
+ Example: 
+   ```ruby 
+   ignore_uid => ["%{[transactionUID][0]}", ""] 
+   ``` 
+  Will ignore events having empty string or "%{[transactionUID][0]}" in the uid_field.
+ 
+ - `timeout`
+ The timeout parameter determines the maximum length of a transaction. 
+ It is set to 300 (5 minutes) by default. 
+ The transaction will not be recorded if timeout duration is exceeded.
+ The value of this parameter will have an impact on the memory footprint of the plugin.
+
+- `timestamp_tag`
+ The timestamp_tag parameter may be used to select a specific field in the events to use
+ when calculating the transaction time. The default field is @timestamp.
+ 
+- `replace_timestamp`
+  The new event created when a transaction completes may set its own timestamp 
+ to when it completes  (default) or it may use the timestamp of one of the events in the transaction.
+ The parameter replace_timestamp is used to specify this behaviour.
+
+- `filter_tag`
+ Since this plugin exclusivly calculates the time between events in a transaction,
+ it may be wise to filter out the events that are infact not transactions.
+ This will help reduce both the memory footprint and processing time of this plugin, 
+ especially if the pipeline receives a lot of non-transactional events.
+ You could use grok and/or mutate to apply this filter like this:
+```ruby
+     filter {
+       grok{
+         match => { "message" => "(?<message_type>.*)\t(?<msgbody>.*)\t+UID:%{UUID:uid}" }
+       }
+       if [message_type] in ["MaterialIdentified","Recipe","Result"."ReleaseMaterial"]{
+         mutate {
+           add_tag => "Transaction"
+         }
+       }
+       transaction_time {
+         uid_field => "UID"
+         filter_tag => "Transaction"
+       }
+     }
+```
+  In the example, grok is used to identify the message_type and then the tag "transaction" is added for a specific set of messages. This tag is then used in the transaction_time as filter_tag.  Only the messages with this tag will be evaluated.
+  > **Note**: Do not use reserved name "_TransactionTime_" which is added to all events created by this plugin
+
+- `attach_event`
+ The attach_event parameter can be used to append information from one of the events to the
+ new transaction_time event. The default is to not attach anything. 
+ The memory footprint is kept to a minimum by using the default value.
+
+- `release_expired`
+ The release_expired parameter determines if the first event in an expired transactions 
+ should be released or not. Defaults to true
+
+- `store_data_oldest/store_data_newest`
+The parameters store_data_oldest and store_data_newest are both used in order to attach 
+specific fields from oldest respectively newest event. An example of this could be:
+ ```ruby
+    store_data_oldest => ["@timestamp", "work_unit", "work_center", "message_type"]
+    store_data_newest => ["@timestamp", "work_unit", "work_center", "message_type"]
+ ```
+  Which will result in the genereated transaction event inluding the specified fields from oldest and newest events in a hashmap named oldest/newest under the hash named "transaction_data"
+  Example of output data:
+ ```
+ "transaction_data" => {
+        "oldest" => {
+            "message_type" => "MaterialIdentified",
+              "@timestamp" => 2018-10-31T07:36:23.072Z,
+               "work_unit" => "WT000743",
+             "work_center" => "WR000046"
+        },
+        "newest" => {
+            "message_type" => "Recipe",
+              "@timestamp" => 2018-10-31T07:36:28.188Z,
+               "work_unit" => "WT000743",
+             "work_center" => "WR000046"
+        }
+    }
+```
 
 # Logstash Plugin
 

data/lib/logstash/filters/transaction_time.rb

@@ -14,6 +14,7 @@ require "logstash/namespace"
 #     filter {
 #       transaction_time {
 #         uid_field => "Transaction-unique field"
+#         ignore_uid => []
 #         timeout => seconds
 #         timestamp_tag => "name of timestamp"
 #         replace_timestamp => ['keep', 'oldest', 'newest']
@@ -31,6 +32,12 @@ require "logstash/namespace"
 # the events in a transaction. A transaction is concidered complete 
 # when two events with the same UID has been captured. 
 # It is when a transaction completes that the transaction time is calculated.
+#
+# The ignore_uid field takes an array of strings. These strings represent specific UIDs
+# that should be ignored. This can be useful for ignoring parsing errors.
+# Example: 
+#   ignore_uid => ["%{[transactionUID][0]}", ""]
+# Will ignore events having empty string or "%{[transactionUID][0]}" in the uid_field.
 # 
 # The timeout parameter determines the maximum length of a transaction. 
 # It is set to 300 (5 minutes) by default. 
@@ -119,6 +126,8 @@ class LogStash::Filters::TransactionTime < LogStash::Filters::Base
   
   # The name of the UID-field used to identify transaction-pairs
   config :uid_field, :validate => :string, :required => true
+  # Array of UIDs to ignore (useful for ignoring parse-errors).
+  config :ignore_uids, :validate => :array, :default => []
   # The amount of time (in seconds) before a transaction is dropped. Defaults to 5 minutes
   config :timeout, :validate => :number, :default => 300
   # What tag to use as timestamp when calculating the elapsed transaction time. Defaults to @timestamp
@@ -177,17 +186,18 @@ class LogStash::Filters::TransactionTime < LogStash::Filters::Base
       (event.get("tags").nil? || !event.get("tags").include?(TRANSACTION_TIME_EXPIRED_TAG)) &&
       (@filter_tag.nil? || (!event.get("tags").nil? && event.get("tags").include?(@filter_tag))))
       
-
-      @mutex.synchronize do
-        if(!@transactions.has_key?(uid))
-          @transactions[uid] = LogStash::Filters::TransactionTime::Transaction.new(event, uid, @storeEvent)
-
-        else #End of transaction
-          @transactions[uid].addSecond(event,@storeEvent)
-          transaction_event = new_transactiontime_event(@transactions[uid], @attachData)
-          filter_matched(transaction_event)
-          yield transaction_event if block_given?
-          @transactions.delete(uid)
+      if not @ignore_uids.include?(uid)
+        @mutex.synchronize do
+          if(!@transactions.has_key?(uid))
+            @transactions[uid] = LogStash::Filters::TransactionTime::Transaction.new(event, uid, @storeEvent)
+
+          else #End of transaction
+            @transactions[uid].addSecond(event,@storeEvent)
+            transaction_event = new_transactiontime_event(@transactions[uid], @attachData)
+            filter_matched(transaction_event)
+            yield transaction_event if block_given?
+            @transactions.delete(uid)
+          end
         end
       end
     end
@@ -207,10 +217,14 @@ class LogStash::Filters::TransactionTime < LogStash::Filters::Base
       expired_elements = remove_expired_elements()
     end
 
-    expired_elements.each do |element|
-      filter_matched(element)
+    if @release_expired
+      expired_elements.each do |element|
+        filter_matched(element)
+      end
+      #print("Exp" + options.to_s + expired_elements.to_s)
+      return expired_elements
     end
-    return expired_elements
+    return []
     #yield expired_elements if block_given?
     #return create_expired_events_from(expired_elements)
   end

data/logstash-filter-transaction_time.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-filter-transaction_time'
-  s.version       = '1.0.4'
+  s.version       = '1.0.5'
   s.licenses      = ['Apache-2.0','Apache License (2.0)']
   s.summary       = 'Writes the time difference between two events in a transaction to a new event'
   s.description   = 'This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program. Source-code and documentation available at github: https://github.com/AddinITAB/logstash-filter-transaction_time'

data/spec/filters/transaction_time_spec.rb

@@ -175,6 +175,26 @@ describe LogStash::Filters::TransactionTime do
         insist { @filter.transactions.size } == 1
       end
     end
+    describe "Setup release_expired = false" do
+      it "never releases any expired events when flush is called" do
+        config = {"release_expired" => false}
+        @config.merge!(config)
+
+        @filter.filter(event("message" => "Log message", UID_FIELD => uid, "@timestamp" => "2018-04-22T09:46:22.000+0100"))
+        insist { @filter.transactions.size } == 1
+        @filter.filter(event("message" => "Log message", UID_FIELD => uid2, "@timestamp" => "2018-04-22T09:46:22.000+0100"))
+        insist { @filter.transactions.size } == 2
+
+        #Looks like flush doesn't have config-scope. Setting release_expired hard instead of by config. Will it work like intended when using only config?
+        @filter.release_expired = false
+        ((TIMEOUT/5)+1).times do
+          flushRes = @filter.flush({"from" => "test" })
+          insist { (flushRes.any?) } == false
+          #insist { @filter.flush().nil? }
+        end 
+        insist { @filter.transactions.size } == 0
+      end
+    end
   end
 
   context "Testing Timestamp Override." do
@@ -185,6 +205,7 @@ describe LogStash::Filters::TransactionTime do
           config = {"replace_timestamp" => 'oldest'}
           @config.merge!(config)
 
+
           @filter = LogStash::Filters::TransactionTime.new(@config)
           @filter.register
 
@@ -377,4 +398,34 @@ describe LogStash::Filters::TransactionTime do
       end
     end
   end
+  context "Testing ignore_uids." do
+    nokUid = "Erroneous UID"
+    uid = "9ACCA7B7-D0E9-4E52-A023-9D588E5BE42C"
+    describe "Config ignore_uids set" do
+      it "will not accept events with specified uid as transactions" do
+        config = {"ignore_uids" => ["Erroneous UID"]}
+        @config.merge!(config)
+
+        @filter = LogStash::Filters::TransactionTime.new(@config)
+        @filter.register
+
+        @filter.filter(event("message" => "first", UID_FIELD => nokUid, "@timestamp" => "2018-04-22T09:46:22.000+0100"))
+        @filter.filter(event("message" => "last", UID_FIELD => nokUid, "@timestamp" => "2018-04-22T09:46:22.100+0100")) do | new_event |
+          insist { new_event } == nil
+        end
+      end
+      it "will accept other events as transactions" do
+        config = {"ignore_uids" => ["Erroneous UID"]}
+        @config.merge!(config)
+
+        @filter = LogStash::Filters::TransactionTime.new(@config)
+        @filter.register
+
+        @filter.filter(event("message" => "first", UID_FIELD => uid, "@timestamp" => "2018-04-22T09:46:22.000+0100"))
+        @filter.filter(event("message" => "last", UID_FIELD => uid, "@timestamp" => "2018-04-22T09:46:22.100+0100")) do | new_event |
+          insist { new_event } != nil
+        end
+      end
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-transaction_time
 version: !ruby/object:Gem::Version
-  version: 1.0.4
+  version: 1.0.5
 platform: ruby
 authors:
 - Tommy Welleby
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-10-31 00:00:00.000000000 Z
+date: 2018-11-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement