logstash-filter-kafka_time_machine 3.0.0 → 3.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f8ebc093ec36312aa92be6f7b158cd840021cad43bc1cacda8a8e9b816c9f2cf
-  data.tar.gz: 821bbb0f5b29ba23c4e650366da3eeb0ebaa94b6f55eb49f33d2b05c13571fbb
+  metadata.gz: 61eed3c10584ae85f6fed99b68b4a217e628d37a61471cf6d36fe3a1abcbf74d
+  data.tar.gz: 9e73985ed0bc52b621c9fc68539c8eac0bc9f6fadfa66f429ec8011d86c67891
 SHA512:
-  metadata.gz: 8c341477b352621f6eb708b63637b1caa1f39aeb9c9e2e399ec2d28c8c8aeb96e813cdbaa0f2f409811f75bb42fdc03cf741ac9acd22c861ff1727f49f523c78
-  data.tar.gz: bca9c8ef6a474f761403e298ad9e494316a50b109e3aa666e85390db556dc749b732eee30a4399a430c4d81854fe1561b8c13f47fe97c0eabd44bac38e1a0776
+  metadata.gz: b8e9551aa515878242d9528048ba731e6a1c59bc4bbadffd6f916a736685dd6749cdeed9cad1b55498eb1519366e8c07acf7eeb2e74ccf2f71686fa08835c456
+  data.tar.gz: 332d6d9f0756db77f0176337a370430cb622e6968e20f470727cd3d6f9979509b54472f398701b500651f0f6f30dcdeb4e0ff54db65d34c0c687c76fc661b307

data/lib/logstash/filters/kafka_time_machine.rb

@@ -49,7 +49,7 @@ class LogStash::Filters::KafkaTimeMachine < LogStash::Filters::Base
 
   public
   def register
-  
+
   end
 
   public
@@ -57,7 +57,7 @@ class LogStash::Filters::KafkaTimeMachine < LogStash::Filters::Base
 
     @logger.debug("Starting filter calculations")
 
-    # Note - It was considered to error check for strings that are invalid, i.e. "%{[@metadata][ktm][kafka_datacenter_shipper]}".  However, this string being present is a good way to identify 
+    # Note - It was considered to error check for strings that are invalid, i.e. "%{[@metadata][ktm][kafka_datacenter_shipper]}".  However, this string being present is a good way to identify
     # shipper/indexer logstash configs that are wrong so its allowed to pass through unaltered.
     #
     # Extract all string values to local variables.
@@ -78,7 +78,7 @@ class LogStash::Filters::KafkaTimeMachine < LogStash::Filters::Base
     shipper_logstash_kafka_read_time = get_numeric(event.sprintf(@logstash_kafka_read_time_shipper))
     indexer_kafka_append_time        = get_numeric(event.sprintf(@kafka_append_time_indexer))
     indexer_logstash_kafka_read_time = get_numeric(event.sprintf(@logstash_kafka_read_time_indexer))
-    
+
     # Validate the shipper data
     shipper_kafka_array = Array[shipper_kafka_datacenter, shipper_kafka_topic, shipper_kafka_consumer_group, shipper_kafka_append_time, shipper_logstash_kafka_read_time, event_owner, event_time_ms, elasticsearch_cluster, elasticsearch_cluster_index]
     if (shipper_kafka_array.any? { |text| text.nil? || text.to_s.empty? })
@@ -109,10 +109,18 @@ class LogStash::Filters::KafkaTimeMachine < LogStash::Filters::Base
       indexer_kafka_lag_ms             = indexer_logstash_kafka_read_time - indexer_kafka_append_time
     end
 
-    # Add in the size of the payload field
-    payload_size_bytes = 0
+    # Add in the size of the payload field if it exist
     if event.get("[payload]")
-      payload_size_bytes = event.get("[payload]").bytesize
+      # pipeline gets random poison pill that crashes KTM with error: "error=>"(NoMethodError) undefined method `bytesize' for #Hash:0x38fe76d4", :exception=>Java::OrgJrubyExceptions::NoMethodError"
+      # This should prevent the crash and log the error
+      begin
+        payload_size_bytes = event.get("[payload]").bytesize
+      rescue NoMethodError => e
+        payload_size_bytes = 0
+        @logger.fatal("payload bytesize operation failed: #{e.message}")
+      end
+    else
+      payload_size_bytes = 0
     end
 
     # Set time (nanoseconds) for event that is generated
@@ -152,7 +160,7 @@ class LogStash::Filters::KafkaTimeMachine < LogStash::Filters::Base
 
       error_string = sprintf("Error kafka_time_machine: Could not build valid response --> %s, %s", error_string_shipper, error_string_indexer)
       @logger.debug(error_string)
-    
+
     else
 
       point_ktm = create_point_ktm_error(shipper_kafka_datacenter, event_owner, epoch_time_ns, "unknown", elasticsearch_cluster, elasticsearch_cluster_index)
@@ -165,14 +173,14 @@ class LogStash::Filters::KafkaTimeMachine < LogStash::Filters::Base
 
     # Publish even event in our array
     ktm_metric_event_array.each do |metric_event|
-      
+
       # Create new event for KTM metric
       event_ktm = LogStash::Event.new(metric_event)
       event_ktm.set("[@metadata][ktm_tags][ktm_metric]", "true")
 
       filter_matched(event_ktm)
       yield event_ktm
-    
+
     end
 
   end # def filter
@@ -180,7 +188,7 @@ class LogStash::Filters::KafkaTimeMachine < LogStash::Filters::Base
   # Creates hash with ktm data point to return
   public
   def create_point_ktm(datacenter, event_owner, payload_size_bytes, lag_type, lag_ms, epoch_time_ns, elasticsearch_cluster, elasticsearch_cluster_index)
-    
+
     point = Hash.new
 
     # Name of point and time created
@@ -225,33 +233,33 @@ class LogStash::Filters::KafkaTimeMachine < LogStash::Filters::Base
     # Name of point and time created
     point["name"]           = "ktm_error"
     point["epoch_time_ns"] = epoch_time_ns
-    
+
     # tags
     point["datacenter"]       = datacenter
     point["owner"]            = event_owner
     point["source"]           = type
     point["es_cluster"]       = elasticsearch_cluster
     point["es_cluster_index"] = elasticsearch_cluster_index
-    
+
     # fields
     point["count"] = 1
-    
+
     return point
 
   end # def create_point_ktm_error
 
   # Ensures the provided value is numeric; if not returns 'nil'
   public
-  def get_numeric(input_str) 
+  def get_numeric(input_str)
 
     # @logger.debug("Aggregate timeout for '#{@task_id}' pattern: #{@timeout} seconds")
     @logger.debug("get_numeric operating on: #{input_str} ")
-    
+
     is_numeric = input_str.to_s.match(/\A[+-]?\d+?(\.\d+)?\Z/) == nil ? false : true
     if (true == is_numeric)
       @logger.debug("get_numeric - valid value provided")
       num_value = Float(sprintf(input_str))
-      
+
       if (false == num_value.positive?)
         @logger.debug("get_numeric - negative value provided")
         num_value = nil

data/logstash-filter-kafka_time_machine.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
     s.name = 'logstash-filter-kafka_time_machine'
-    s.version = '3.0.0'
+    s.version = '3.0.1'
     s.licenses = ['Apache-2.0']
     s.summary = "Calculate total time of logstash event that traversed 2 Kafka queues from a shipper site to an indexer site"
     s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -11,7 +11,7 @@ Gem::Specification.new do |s|
 
     # Files
     s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
-    
+
     # Tests
     s.test_files = s.files.grep(%r{^(test|spec|features)/})
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-kafka_time_machine
 version: !ruby/object:Gem::Version
-  version: 3.0.0
+  version: 3.0.1
 platform: ruby
 authors:
 - Chris Foster
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-11-10 00:00:00.000000000 Z
+date: 2024-02-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core-plugin-api