logstash-filter-transaction_time 1.0.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: df5ee7635060bbbad794fd0e0f833f461ba721b1
-  data.tar.gz: 7b6a091cf7af7fb18c113f7f51589c70d56c8536
+  metadata.gz: 791175d5aab745baae8183abe08c091593f0d0ca
+  data.tar.gz: d6d2326111fb55b7b36816dbc068b3f63f8a58c3
 SHA512:
-  metadata.gz: 85dfd46602f89ac8cfefdc04cf0884689df48521f1d36cfde35a9762881c7d399525128c5ab5b3eb2cfd725029ab1c3d201a789d0eea93bcfa5c1beb83b5a184
-  data.tar.gz: faa6f933e23971346e19b4724354f494c0bb66f0e23da2c10917d21cb5c8353ba217b19f3bb0b94b730eab54b060e58a9fc3947bde8e187ced12252f19ff336f
+  metadata.gz: 3c6ec8cda7e6da7c3dd9b32c0a347a5bfd811d4ecf182341b8dd054bf3b94bcaa8b40c76ef3ad5447dcb073a83ccfaf51102c865a08d8f72182343740508e801
+  data.tar.gz: 9733638d214930bebae4a08f0aa22046a32bed7f12394df2ff6f7446e76004684d6e65a9bb2d44636b46c14427ac68ca21d792fcbd929fac1f528e1207710aa2

data/README.md

@@ -1,12 +1,13 @@
 # About
-This plugin is a substitute for the logstash-filter-elapsed plugin. 
+This plugin is a substitute for the [logstash-filter-elapsed](https://www.elastic.co/guide/en/logstash/current/plugins-filters-elapsed.html) plugin. 
 The elapsed-plugin requires a transaction to be executed in a specified order and then decorates the last part of the transaction (or creates a new event) with the elapsed time.
 The order of which the parts of a transaction is received cannot always be predicted when using multiple workers for a pipeline.
 Hence the need for this plugin.
 This plugin, like elapsed, uses a unique identifier to pair events in a transaction.
 But instead of defining a start and an end for a transaction - only the unique identifier is used. 
-This of course has some implications. The biggest one not being able to decorate the last part of the transaction since it may or may not be the same type of event.
-Instead the transaction time is stored together with the unique identifier. Either in the same or another index.
+Per default the transaction time is stored together with the unique identifier in a new event, which may be stored in the same or another index.
+The information from the first, last, oldest or newest event may be attached with the new transaction_time event.
+
 
 
 # Logstash Plugin

data/lib/logstash/filters/transaction_time.rb

@@ -2,10 +2,74 @@
 require "logstash/filters/base"
 require "logstash/namespace"
 
-# This  filter will replace the contents of the default 
-# message field with whatever you specify in the configuration.
+# The TransactionTime filter measures the time between two events in a transaction
 #
-# It is only intended to be used as an .
+# This filter is supposed to be used instead of logstash-filters-elapsed
+# when you know that the order of a transaction cannot be guaranteed.
+# Which is most likely the case if you are using multiple workers and 
+# a big amount of events are entering the pipeline in a rapid manner.
+#
+# # The configuration looks like this:
+# [source,ruby]
+#     filter {
+#       transaction_time {
+#         uid_field => "Transaction-unique field"
+#         timeout => seconds
+#         timestamp_tag => "name of timestamp"
+#         replace_timestamp => ['keep', 'oldest', 'newest']
+#         filter_tag => "transaction tag"
+#         attach_event => ['first','last','oldest','newest','none']
+#       }
+#     }
+#
+#
+# The only required parameter is "uid_field" which is used to identify
+# the events in a transaction. A transaction is concidered complete 
+# when two events with the same UID has been captured. 
+# It is when a transaction completes that the transaction time is calculated.
+# 
+# The timeout parameter determines the maximum length of a transaction. 
+# It is set to 300 (5 minutes) by default. 
+# The transaction will not be recorded if timeout duration is exceeded.
+# The value of this parameter will have an impact on the memory footprint of the plugin.
+#
+# The timestamp_tag parameter may be used to select a specific field in the events to use
+# when calculating the transaction time. The default field is @timestamp.
+# 
+# The new event created when a transaction completes may set its own timestamp (default)
+# to when it completes or it may use the timestamp of one of the events in the transaction.
+# The parameter replace_timestamp is used specify this behaviour.
+#
+# Since this plugin exclusivly calculates the time between events in a transaction,
+# it may be wise to filter out the events that are infact not transactions.
+# This will help reduce both the memory footprint and processing time of this plugin, 
+# especially if the pipeline receives a lot of non-transactional events.
+# You could use grok and/or mutate to apply this filter like this:
+# [source,ruby]
+#     filter {
+#       grok{
+#         match => { "message" => "(?<message_type>.*)\t(?<msgbody>.*)\t+UID:%{UUID:uid}" }
+#       }
+#       if [message_type] in ["MaterialIdentified","Recipe","Result"."ReleaseMaterial"]{
+#         mutate {
+#           add_tag => "Transaction"
+#         }
+#       }
+#       transaction_time {
+#         uid_field => "Transaction-unique field"
+#         filter_tag => "transaction tag"
+#       }
+#     }
+#
+# In the example, grok is used to identify the message_type and then the tag "transaction"
+# is added for a specific set of messages. 
+# This tag is then used in the transaction_time as filter_tag. 
+# Only the messages with this tag will be evaluated.
+#
+# The attach_event parameter can be used to append information from one of the events to the
+# new transaction_time event. The default is to not attach anything. 
+# The memory footprint is kept to a minimum by using the default value.
+
 class LogStash::Filters::TransactionTime < LogStash::Filters::Base
 
   HOST_FIELD = "host"

data/logstash-filter-transaction_time.gemspec

@@ -1,9 +1,9 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-filter-transaction_time'
-  s.version       = '1.0.0'
+  s.version       = '1.0.1'
   s.licenses      = ['Apache-2.0','Apache License (2.0)']
   s.summary       = 'Writes the time difference between two events in a transaction to a new event'
-  s.description   = 'This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program'
+  s.description   = 'This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program. Source-code and documentation available at github: https://github.com/AddinITAB/logstash-filter-transaction_time'
   s.homepage      = 'http://addinit.se/'
   s.authors       = ['Tommy Welleby']
   s.email         = 'tommy.welleby@addinit.se'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-transaction_time
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
 platform: ruby
 authors:
 - Tommy Welleby
@@ -38,9 +38,10 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: This gem is a Logstash plugin required to be installed on top of the
+description: 'This gem is a Logstash plugin required to be installed on top of the
   Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
-  gem is not a stand-alone program
+  gem is not a stand-alone program. Source-code and documentation available at github:
+  https://github.com/AddinITAB/logstash-filter-transaction_time'
 email: tommy.welleby@addinit.se
 executables: []
 extensions: []