logstash-filter-kafka_time_machine 3.0.1 → 3.0.2.pre

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 61eed3c10584ae85f6fed99b68b4a217e628d37a61471cf6d36fe3a1abcbf74d
-  data.tar.gz: 9e73985ed0bc52b621c9fc68539c8eac0bc9f6fadfa66f429ec8011d86c67891
+  metadata.gz: f0493db2db2981e58eb360408a6ca29529f267e3b2f3206275329bc8c05ac473
+  data.tar.gz: 02266346ba816a9fef2b12f4aba8bc283847fa7f55ebbaae507ca107798458f8
 SHA512:
-  metadata.gz: b8e9551aa515878242d9528048ba731e6a1c59bc4bbadffd6f916a736685dd6749cdeed9cad1b55498eb1519366e8c07acf7eeb2e74ccf2f71686fa08835c456
-  data.tar.gz: 332d6d9f0756db77f0176337a370430cb622e6968e20f470727cd3d6f9979509b54472f398701b500651f0f6f30dcdeb4e0ff54db65d34c0c687c76fc661b307
+  metadata.gz: bea392a66af7f726c13d953ca1f03bfaebfe0782c5df11944cb8b8db845650f5619e0f1d8dab31e22de50372929dd1c68288678bcf5d91931688593887249bec
+  data.tar.gz: 48ba960f4285a64b5c5f81be76e0380c39581e9adcdfa62c46062e7b77ff6b4d422524a4cafeeebae43d5c17511ef665d84e3a842a6e0919c4ad031a1ec94624

data/lib/logstash/filters/kafka_time_machine.rb

@@ -38,6 +38,9 @@ class LogStash::Filters::KafkaTimeMachine < LogStash::Filters::Base
   # Owner of the event currenty being process.
   config :event_owner, :validate => :string, :required => true
 
+  # tag of the event currenty being process.
+  config :event_tag, :validate => :string, :required => true
+
   # Current time since EPOCH in ms that should be set in the generated metric
   config :event_time_ms, :validate => :string, :required => true
 
@@ -79,8 +82,11 @@ class LogStash::Filters::KafkaTimeMachine < LogStash::Filters::Base
     indexer_kafka_append_time        = get_numeric(event.sprintf(@kafka_append_time_indexer))
     indexer_logstash_kafka_read_time = get_numeric(event.sprintf(@logstash_kafka_read_time_indexer))
 
+    # Extract tags
+    event_tag = event.sprintf(@event_tag)
+
     # Validate the shipper data
-    shipper_kafka_array = Array[shipper_kafka_datacenter, shipper_kafka_topic, shipper_kafka_consumer_group, shipper_kafka_append_time, shipper_logstash_kafka_read_time, event_owner, event_time_ms, elasticsearch_cluster, elasticsearch_cluster_index]
+    shipper_kafka_array = Array[shipper_kafka_datacenter, shipper_kafka_topic, shipper_kafka_consumer_group, shipper_kafka_append_time, shipper_logstash_kafka_read_time, event_owner, event_tag, event_time_ms, elasticsearch_cluster, elasticsearch_cluster_index]
     if (shipper_kafka_array.any? { |text| text.nil? || text.to_s.empty? })
       @logger.debug("shipper_kafka_array invalid: Found null")
       error_string_shipper = sprintf("Error in shipper data: %s", shipper_kafka_array)
@@ -95,7 +101,7 @@ class LogStash::Filters::KafkaTimeMachine < LogStash::Filters::Base
     end
 
     # Validate the indexer data
-    indexer_kafka_array = Array[shipper_kafka_datacenter, indexer_kafka_topic, indexer_kafka_consumer_group, indexer_kafka_append_time, indexer_logstash_kafka_read_time, event_owner, event_time_ms, elasticsearch_cluster, elasticsearch_cluster_index]
+    indexer_kafka_array = Array[shipper_kafka_datacenter, indexer_kafka_topic, indexer_kafka_consumer_group, indexer_kafka_append_time, indexer_logstash_kafka_read_time, event_owner, event_tag, event_time_ms, elasticsearch_cluster, elasticsearch_cluster_index]
     if (indexer_kafka_array.any? { |text| text.nil? || text.to_s.empty? })
       @logger.debug("indexer_kafka_array invalid: Found null")
       error_string_indexer = sprintf("Error in indexer data: %s", indexer_kafka_array)
@@ -136,26 +142,26 @@ class LogStash::Filters::KafkaTimeMachine < LogStash::Filters::Base
     if (shipper_valid == true && indexer_valid == true && epoch_time_ns != nil)
       total_kafka_lag_ms = indexer_logstash_kafka_read_time - shipper_kafka_append_time
 
-      point_ktm = create_point_ktm(shipper_kafka_datacenter, event_owner, payload_size_bytes, "total", total_kafka_lag_ms, epoch_time_ns, elasticsearch_cluster, elasticsearch_cluster_index)
+      point_ktm = create_point_ktm(shipper_kafka_datacenter, event_owner, event_tag, payload_size_bytes, "total", total_kafka_lag_ms, epoch_time_ns, elasticsearch_cluster, elasticsearch_cluster_index)
       ktm_metric_event_array.push point_ktm
 
     elsif (shipper_valid == true && indexer_valid == false && epoch_time_ns != nil)
-      point_ktm = create_point_ktm(shipper_kafka_datacenter, event_owner, payload_size_bytes, "shipper", shipper_kafka_lag_ms, epoch_time_ns, elasticsearch_cluster, elasticsearch_cluster_index)
+      point_ktm = create_point_ktm(shipper_kafka_datacenter, event_owner, event_tag, payload_size_bytes, "shipper", shipper_kafka_lag_ms, epoch_time_ns, elasticsearch_cluster, elasticsearch_cluster_index)
       ktm_metric_event_array.push point_ktm
 
-      point_ktm = create_point_ktm_error(shipper_kafka_datacenter, event_owner, epoch_time_ns, "indexer", elasticsearch_cluster, elasticsearch_cluster_index)
+      point_ktm = create_point_ktm_error(shipper_kafka_datacenter, event_owner, event_tag, epoch_time_ns, "indexer", elasticsearch_cluster, elasticsearch_cluster_index)
       ktm_metric_event_array.push point_ktm
 
     elsif (indexer_valid == true && shipper_valid == false && epoch_time_ns != nil)
-      point_ktm = create_point_ktm(shipper_kafka_datacenter, event_owner, payload_size_bytes, "indexer", indexer_kafka_lag_ms, epoch_time_ns, elasticsearch_cluster, elasticsearch_cluster_index)
+      point_ktm = create_point_ktm(shipper_kafka_datacenter, event_owner, event_tag, payload_size_bytes, "indexer", indexer_kafka_lag_ms, epoch_time_ns, elasticsearch_cluster, elasticsearch_cluster_index)
       ktm_metric_event_array.push point_ktm
 
-      point_ktm = create_point_ktm_error(shipper_kafka_datacenter, event_owner, epoch_time_ns, "shipper", elasticsearch_cluster, elasticsearch_cluster_index)
+      point_ktm = create_point_ktm_error(shipper_kafka_datacenter, event_owner, event_tag, epoch_time_ns, "shipper", elasticsearch_cluster, elasticsearch_cluster_index)
       ktm_metric_event_array.push point_ktm
 
     elsif (indexer_valid == false && shipper_valid == false)
 
-      point_ktm = create_point_ktm_error(shipper_kafka_datacenter, event_owner, epoch_time_ns, "insufficient_data", elasticsearch_cluster, elasticsearch_cluster_index)
+      point_ktm = create_point_ktm_error(shipper_kafka_datacenter, event_owner, event_tag, epoch_time_ns, "insufficient_data", elasticsearch_cluster, elasticsearch_cluster_index)
       ktm_metric_event_array.push point_ktm
 
       error_string = sprintf("Error kafka_time_machine: Could not build valid response --> %s, %s", error_string_shipper, error_string_indexer)
@@ -163,7 +169,7 @@ class LogStash::Filters::KafkaTimeMachine < LogStash::Filters::Base
 
     else
 
-      point_ktm = create_point_ktm_error(shipper_kafka_datacenter, event_owner, epoch_time_ns, "unknown", elasticsearch_cluster, elasticsearch_cluster_index)
+      point_ktm = create_point_ktm_error(shipper_kafka_datacenter, event_owner, event_tag, epoch_time_ns, "unknown", elasticsearch_cluster, elasticsearch_cluster_index)
       ktm_metric_event_array.push point_ktm
 
       error_string = "Unknown error encountered"
@@ -187,7 +193,7 @@ class LogStash::Filters::KafkaTimeMachine < LogStash::Filters::Base
 
   # Creates hash with ktm data point to return
   public
-  def create_point_ktm(datacenter, event_owner, payload_size_bytes, lag_type, lag_ms, epoch_time_ns, elasticsearch_cluster, elasticsearch_cluster_index)
+  def create_point_ktm(datacenter, event_owner, event_tag, payload_size_bytes, lag_type, lag_ms, epoch_time_ns, elasticsearch_cluster, elasticsearch_cluster_index)
 
     point = Hash.new
 
@@ -198,6 +204,7 @@ class LogStash::Filters::KafkaTimeMachine < LogStash::Filters::Base
     # tags
     point["datacenter"]       = datacenter
     point["owner"]            = event_owner
+    point["tag"]              = event_tag
     point["lag_type"]         = lag_type
     point["es_cluster"]       = elasticsearch_cluster
     point["es_cluster_index"] = elasticsearch_cluster_index
@@ -212,7 +219,7 @@ class LogStash::Filters::KafkaTimeMachine < LogStash::Filters::Base
 
   # Creates hash with ktm data point to return
   public
-  def create_point_ktm_error(datacenter, event_owner, epoch_time_ns, type, elasticsearch_cluster, elasticsearch_cluster_index)
+  def create_point_ktm_error(datacenter, event_owner, event_tag, epoch_time_ns, type, elasticsearch_cluster, elasticsearch_cluster_index)
 
     # Check for nil values
     if (nil == datacenter)
@@ -223,6 +230,10 @@ class LogStash::Filters::KafkaTimeMachine < LogStash::Filters::Base
       event_owner = "unknown"
     end
 
+    if (nil == event_tag)
+      event_tag = "unknown"
+    end
+
     # set time if we didn't recieve it
     if (nil == epoch_time_ns)
       epoch_time_ns = ((Time.now.to_f * 1000).to_i)*1000000
@@ -237,6 +248,7 @@ class LogStash::Filters::KafkaTimeMachine < LogStash::Filters::Base
     # tags
     point["datacenter"]       = datacenter
     point["owner"]            = event_owner
+    point["tag"]              = event_tag
     point["source"]           = type
     point["es_cluster"]       = elasticsearch_cluster
     point["es_cluster_index"] = elasticsearch_cluster_index
@@ -276,4 +288,4 @@ class LogStash::Filters::KafkaTimeMachine < LogStash::Filters::Base
 
   end # def get_numberic
 
-end # class LogStash::Filters::KafkaTimeMachine
+end # class LogStash::Filters::KafkaTimeMachine

data/logstash-filter-kafka_time_machine.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
     s.name = 'logstash-filter-kafka_time_machine'
-    s.version = '3.0.1'
+    s.version = '3.0.2.pre'
     s.licenses = ['Apache-2.0']
     s.summary = "Calculate total time of logstash event that traversed 2 Kafka queues from a shipper site to an indexer site"
     s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-kafka_time_machine
 version: !ruby/object:Gem::Version
-  version: 3.0.1
+  version: 3.0.2.pre
 platform: ruby
 authors:
 - Chris Foster
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-02-27 00:00:00.000000000 Z
+date: 2025-01-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core-plugin-api
@@ -73,9 +73,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubygems_version: 3.0.3.1
 signing_key: