logstash-filter-kafka_time_machine 3.0.0 → 3.0.2.pre

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f8ebc093ec36312aa92be6f7b158cd840021cad43bc1cacda8a8e9b816c9f2cf
-  data.tar.gz: 821bbb0f5b29ba23c4e650366da3eeb0ebaa94b6f55eb49f33d2b05c13571fbb
+  metadata.gz: f0493db2db2981e58eb360408a6ca29529f267e3b2f3206275329bc8c05ac473
+  data.tar.gz: 02266346ba816a9fef2b12f4aba8bc283847fa7f55ebbaae507ca107798458f8
 SHA512:
-  metadata.gz: 8c341477b352621f6eb708b63637b1caa1f39aeb9c9e2e399ec2d28c8c8aeb96e813cdbaa0f2f409811f75bb42fdc03cf741ac9acd22c861ff1727f49f523c78
-  data.tar.gz: bca9c8ef6a474f761403e298ad9e494316a50b109e3aa666e85390db556dc749b732eee30a4399a430c4d81854fe1561b8c13f47fe97c0eabd44bac38e1a0776
+  metadata.gz: bea392a66af7f726c13d953ca1f03bfaebfe0782c5df11944cb8b8db845650f5619e0f1d8dab31e22de50372929dd1c68288678bcf5d91931688593887249bec
+  data.tar.gz: 48ba960f4285a64b5c5f81be76e0380c39581e9adcdfa62c46062e7b77ff6b4d422524a4cafeeebae43d5c17511ef665d84e3a842a6e0919c4ad031a1ec94624

data/lib/logstash/filters/kafka_time_machine.rb

@@ -38,6 +38,9 @@ class LogStash::Filters::KafkaTimeMachine < LogStash::Filters::Base
   # Owner of the event currenty being process.
   config :event_owner, :validate => :string, :required => true
 
+  # tag of the event currenty being process.
+  config :event_tag, :validate => :string, :required => true
+
   # Current time since EPOCH in ms that should be set in the generated metric
   config :event_time_ms, :validate => :string, :required => true
 
@@ -49,7 +52,7 @@ class LogStash::Filters::KafkaTimeMachine < LogStash::Filters::Base
 
   public
   def register
-  
+
   end
 
   public
@@ -57,7 +60,7 @@ class LogStash::Filters::KafkaTimeMachine < LogStash::Filters::Base
 
     @logger.debug("Starting filter calculations")
 
-    # Note - It was considered to error check for strings that are invalid, i.e. "%{[@metadata][ktm][kafka_datacenter_shipper]}".  However, this string being present is a good way to identify 
+    # Note - It was considered to error check for strings that are invalid, i.e. "%{[@metadata][ktm][kafka_datacenter_shipper]}".  However, this string being present is a good way to identify
     # shipper/indexer logstash configs that are wrong so its allowed to pass through unaltered.
     #
     # Extract all string values to local variables.
@@ -78,9 +81,12 @@ class LogStash::Filters::KafkaTimeMachine < LogStash::Filters::Base
     shipper_logstash_kafka_read_time = get_numeric(event.sprintf(@logstash_kafka_read_time_shipper))
     indexer_kafka_append_time        = get_numeric(event.sprintf(@kafka_append_time_indexer))
     indexer_logstash_kafka_read_time = get_numeric(event.sprintf(@logstash_kafka_read_time_indexer))
-    
+
+    # Extract tags
+    event_tag = event.sprintf(@event_tag)
+
     # Validate the shipper data
-    shipper_kafka_array = Array[shipper_kafka_datacenter, shipper_kafka_topic, shipper_kafka_consumer_group, shipper_kafka_append_time, shipper_logstash_kafka_read_time, event_owner, event_time_ms, elasticsearch_cluster, elasticsearch_cluster_index]
+    shipper_kafka_array = Array[shipper_kafka_datacenter, shipper_kafka_topic, shipper_kafka_consumer_group, shipper_kafka_append_time, shipper_logstash_kafka_read_time, event_owner, event_tag, event_time_ms, elasticsearch_cluster, elasticsearch_cluster_index]
     if (shipper_kafka_array.any? { |text| text.nil? || text.to_s.empty? })
       @logger.debug("shipper_kafka_array invalid: Found null")
       error_string_shipper = sprintf("Error in shipper data: %s", shipper_kafka_array)
@@ -95,7 +101,7 @@ class LogStash::Filters::KafkaTimeMachine < LogStash::Filters::Base
     end
 
     # Validate the indexer data
-    indexer_kafka_array = Array[shipper_kafka_datacenter, indexer_kafka_topic, indexer_kafka_consumer_group, indexer_kafka_append_time, indexer_logstash_kafka_read_time, event_owner, event_time_ms, elasticsearch_cluster, elasticsearch_cluster_index]
+    indexer_kafka_array = Array[shipper_kafka_datacenter, indexer_kafka_topic, indexer_kafka_consumer_group, indexer_kafka_append_time, indexer_logstash_kafka_read_time, event_owner, event_tag, event_time_ms, elasticsearch_cluster, elasticsearch_cluster_index]
     if (indexer_kafka_array.any? { |text| text.nil? || text.to_s.empty? })
       @logger.debug("indexer_kafka_array invalid: Found null")
       error_string_indexer = sprintf("Error in indexer data: %s", indexer_kafka_array)
@@ -109,10 +115,18 @@ class LogStash::Filters::KafkaTimeMachine < LogStash::Filters::Base
       indexer_kafka_lag_ms             = indexer_logstash_kafka_read_time - indexer_kafka_append_time
     end
 
-    # Add in the size of the payload field
-    payload_size_bytes = 0
+    # Add in the size of the payload field if it exist
     if event.get("[payload]")
-      payload_size_bytes = event.get("[payload]").bytesize
+      # pipeline gets random poison pill that crashes KTM with error: "error=>"(NoMethodError) undefined method `bytesize' for #Hash:0x38fe76d4", :exception=>Java::OrgJrubyExceptions::NoMethodError"
+      # This should prevent the crash and log the error
+      begin
+        payload_size_bytes = event.get("[payload]").bytesize
+      rescue NoMethodError => e
+        payload_size_bytes = 0
+        @logger.fatal("payload bytesize operation failed: #{e.message}")
+      end
+    else
+      payload_size_bytes = 0
     end
 
     # Set time (nanoseconds) for event that is generated
@@ -128,34 +142,34 @@ class LogStash::Filters::KafkaTimeMachine < LogStash::Filters::Base
     if (shipper_valid == true && indexer_valid == true && epoch_time_ns != nil)
       total_kafka_lag_ms = indexer_logstash_kafka_read_time - shipper_kafka_append_time
 
-      point_ktm = create_point_ktm(shipper_kafka_datacenter, event_owner, payload_size_bytes, "total", total_kafka_lag_ms, epoch_time_ns, elasticsearch_cluster, elasticsearch_cluster_index)
+      point_ktm = create_point_ktm(shipper_kafka_datacenter, event_owner, event_tag, payload_size_bytes, "total", total_kafka_lag_ms, epoch_time_ns, elasticsearch_cluster, elasticsearch_cluster_index)
       ktm_metric_event_array.push point_ktm
 
     elsif (shipper_valid == true && indexer_valid == false && epoch_time_ns != nil)
-      point_ktm = create_point_ktm(shipper_kafka_datacenter, event_owner, payload_size_bytes, "shipper", shipper_kafka_lag_ms, epoch_time_ns, elasticsearch_cluster, elasticsearch_cluster_index)
+      point_ktm = create_point_ktm(shipper_kafka_datacenter, event_owner, event_tag, payload_size_bytes, "shipper", shipper_kafka_lag_ms, epoch_time_ns, elasticsearch_cluster, elasticsearch_cluster_index)
       ktm_metric_event_array.push point_ktm
 
-      point_ktm = create_point_ktm_error(shipper_kafka_datacenter, event_owner, epoch_time_ns, "indexer", elasticsearch_cluster, elasticsearch_cluster_index)
+      point_ktm = create_point_ktm_error(shipper_kafka_datacenter, event_owner, event_tag, epoch_time_ns, "indexer", elasticsearch_cluster, elasticsearch_cluster_index)
       ktm_metric_event_array.push point_ktm
 
     elsif (indexer_valid == true && shipper_valid == false && epoch_time_ns != nil)
-      point_ktm = create_point_ktm(shipper_kafka_datacenter, event_owner, payload_size_bytes, "indexer", indexer_kafka_lag_ms, epoch_time_ns, elasticsearch_cluster, elasticsearch_cluster_index)
+      point_ktm = create_point_ktm(shipper_kafka_datacenter, event_owner, event_tag, payload_size_bytes, "indexer", indexer_kafka_lag_ms, epoch_time_ns, elasticsearch_cluster, elasticsearch_cluster_index)
       ktm_metric_event_array.push point_ktm
 
-      point_ktm = create_point_ktm_error(shipper_kafka_datacenter, event_owner, epoch_time_ns, "shipper", elasticsearch_cluster, elasticsearch_cluster_index)
+      point_ktm = create_point_ktm_error(shipper_kafka_datacenter, event_owner, event_tag, epoch_time_ns, "shipper", elasticsearch_cluster, elasticsearch_cluster_index)
       ktm_metric_event_array.push point_ktm
 
     elsif (indexer_valid == false && shipper_valid == false)
 
-      point_ktm = create_point_ktm_error(shipper_kafka_datacenter, event_owner, epoch_time_ns, "insufficient_data", elasticsearch_cluster, elasticsearch_cluster_index)
+      point_ktm = create_point_ktm_error(shipper_kafka_datacenter, event_owner, event_tag, epoch_time_ns, "insufficient_data", elasticsearch_cluster, elasticsearch_cluster_index)
       ktm_metric_event_array.push point_ktm
 
       error_string = sprintf("Error kafka_time_machine: Could not build valid response --> %s, %s", error_string_shipper, error_string_indexer)
       @logger.debug(error_string)
-    
+
     else
 
-      point_ktm = create_point_ktm_error(shipper_kafka_datacenter, event_owner, epoch_time_ns, "unknown", elasticsearch_cluster, elasticsearch_cluster_index)
+      point_ktm = create_point_ktm_error(shipper_kafka_datacenter, event_owner, event_tag, epoch_time_ns, "unknown", elasticsearch_cluster, elasticsearch_cluster_index)
       ktm_metric_event_array.push point_ktm
 
       error_string = "Unknown error encountered"
@@ -165,22 +179,22 @@ class LogStash::Filters::KafkaTimeMachine < LogStash::Filters::Base
 
     # Publish even event in our array
     ktm_metric_event_array.each do |metric_event|
-      
+
       # Create new event for KTM metric
       event_ktm = LogStash::Event.new(metric_event)
       event_ktm.set("[@metadata][ktm_tags][ktm_metric]", "true")
 
       filter_matched(event_ktm)
       yield event_ktm
-    
+
     end
 
   end # def filter
 
   # Creates hash with ktm data point to return
   public
-  def create_point_ktm(datacenter, event_owner, payload_size_bytes, lag_type, lag_ms, epoch_time_ns, elasticsearch_cluster, elasticsearch_cluster_index)
-    
+  def create_point_ktm(datacenter, event_owner, event_tag, payload_size_bytes, lag_type, lag_ms, epoch_time_ns, elasticsearch_cluster, elasticsearch_cluster_index)
+
     point = Hash.new
 
     # Name of point and time created
@@ -190,6 +204,7 @@ class LogStash::Filters::KafkaTimeMachine < LogStash::Filters::Base
     # tags
     point["datacenter"]       = datacenter
     point["owner"]            = event_owner
+    point["tag"]              = event_tag
     point["lag_type"]         = lag_type
     point["es_cluster"]       = elasticsearch_cluster
     point["es_cluster_index"] = elasticsearch_cluster_index
@@ -204,7 +219,7 @@ class LogStash::Filters::KafkaTimeMachine < LogStash::Filters::Base
 
   # Creates hash with ktm data point to return
   public
-  def create_point_ktm_error(datacenter, event_owner, epoch_time_ns, type, elasticsearch_cluster, elasticsearch_cluster_index)
+  def create_point_ktm_error(datacenter, event_owner, event_tag, epoch_time_ns, type, elasticsearch_cluster, elasticsearch_cluster_index)
 
     # Check for nil values
     if (nil == datacenter)
@@ -215,6 +230,10 @@ class LogStash::Filters::KafkaTimeMachine < LogStash::Filters::Base
       event_owner = "unknown"
     end
 
+    if (nil == event_tag)
+      event_tag = "unknown"
+    end
+
     # set time if we didn't recieve it
     if (nil == epoch_time_ns)
       epoch_time_ns = ((Time.now.to_f * 1000).to_i)*1000000
@@ -225,33 +244,34 @@ class LogStash::Filters::KafkaTimeMachine < LogStash::Filters::Base
     # Name of point and time created
     point["name"]           = "ktm_error"
     point["epoch_time_ns"] = epoch_time_ns
-    
+
     # tags
     point["datacenter"]       = datacenter
     point["owner"]            = event_owner
+    point["tag"]              = event_tag
     point["source"]           = type
     point["es_cluster"]       = elasticsearch_cluster
     point["es_cluster_index"] = elasticsearch_cluster_index
-    
+
     # fields
     point["count"] = 1
-    
+
     return point
 
   end # def create_point_ktm_error
 
   # Ensures the provided value is numeric; if not returns 'nil'
   public
-  def get_numeric(input_str) 
+  def get_numeric(input_str)
 
     # @logger.debug("Aggregate timeout for '#{@task_id}' pattern: #{@timeout} seconds")
     @logger.debug("get_numeric operating on: #{input_str} ")
-    
+
     is_numeric = input_str.to_s.match(/\A[+-]?\d+?(\.\d+)?\Z/) == nil ? false : true
     if (true == is_numeric)
       @logger.debug("get_numeric - valid value provided")
       num_value = Float(sprintf(input_str))
-      
+
       if (false == num_value.positive?)
         @logger.debug("get_numeric - negative value provided")
         num_value = nil
@@ -268,4 +288,4 @@ class LogStash::Filters::KafkaTimeMachine < LogStash::Filters::Base
 
   end # def get_numberic
 
-end # class LogStash::Filters::KafkaTimeMachine
+end # class LogStash::Filters::KafkaTimeMachine

data/logstash-filter-kafka_time_machine.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
     s.name = 'logstash-filter-kafka_time_machine'
-    s.version = '3.0.0'
+    s.version = '3.0.2.pre'
     s.licenses = ['Apache-2.0']
     s.summary = "Calculate total time of logstash event that traversed 2 Kafka queues from a shipper site to an indexer site"
     s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -11,7 +11,7 @@ Gem::Specification.new do |s|
 
     # Files
     s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
-    
+
     # Tests
     s.test_files = s.files.grep(%r{^(test|spec|features)/})
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-kafka_time_machine
 version: !ruby/object:Gem::Version
-  version: 3.0.0
+  version: 3.0.2.pre
 platform: ruby
 authors:
 - Chris Foster
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-11-10 00:00:00.000000000 Z
+date: 2025-01-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core-plugin-api
@@ -73,9 +73,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubygems_version: 3.0.3.1
 signing_key: