logstash-filter-kafka_time_machine 2.0.0 → 2.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a0ab8be37285b43b8785aea650cc6eb6d456d701ba36aeb0d5c2cbcb020d470d
-  data.tar.gz: d6ccd08fa8024d7cced6c529708ad5cd5f15ef683c043206966c58b989d28f42
+  metadata.gz: dcb429711e99220eb57f095154d68f2b3f477956587762c4ba03ef7bc434ba30
+  data.tar.gz: 3c66923274a218187bef908747091baa540523729c6d961f02ab002eb6701fce
 SHA512:
-  metadata.gz: 73b806dbef6c52765e674dc7acec647c90427ff40270c9665eaf07c29818ecca558c564cb812aa9cd81601a9b3009ccdd5b7d465ef18352a4f146c195f9daf16
-  data.tar.gz: 3f318fbafd7bd1283599ac68b4d7b29ea9187f7772d8353235d0d4d2cc06be34afc2bf440d226600ba125ad4a9a34d058d77f05c87c94c45d9f3339cbdc79e31
+  metadata.gz: 840b0fbdef1e7096c2e51cdaac3a4ef38e5e7830fb8263e8511d8e09116a13a92a90dcf13ffb72ffef51da4f66aaf898235d30ee50c6f7adddab3c79a428bb80
+  data.tar.gz: 6ffbb0731c74f2b7168dccd79f4a78f00b168baab2d5d111ac9a872b39c24f3aca8fa5b92e67053a7f5abef71a0fc44561604f56f669c40812a10c237e5b65e9

data/lib/logstash/filters/kafka_time_machine.rb

@@ -41,6 +41,12 @@ class LogStash::Filters::KafkaTimeMachine < LogStash::Filters::Base
   # Current time since EPOCH in ms that should be set in the influxdb generated metric
   config :event_time_ms, :validate => :string, :required => true
 
+  # Current time since EPOCH in ms that should be set in the influxdb generated metric
+  config :elasticsearch_cluster, :validate => :string, :required => true
+
+  # Current time since EPOCH in ms that should be set in the influxdb generated metric
+  config :elasticsearch_cluster_index, :validate => :string, :required => true
+
   public
   def register
   
@@ -61,6 +67,8 @@ class LogStash::Filters::KafkaTimeMachine < LogStash::Filters::Base
     shipper_kafka_consumer_group     = event.sprintf(@kafka_consumer_group_shipper)
     indexer_kafka_topic              = event.sprintf(@kafka_topic_indexer)
     indexer_kafka_consumer_group     = event.sprintf(@kafka_consumer_group_indexer)
+    elasticsearch_cluster            = event.sprintf(@elasticsearch_cluster)
+    elasticsearch_cluster_index      = event.sprintf(@elasticsearch_cluster_index)
 
     # Extract all the "time" related values to local variables.  This need special handling due to the Float() operation.
     #
@@ -72,10 +80,11 @@ class LogStash::Filters::KafkaTimeMachine < LogStash::Filters::Base
     indexer_logstash_kafka_read_time = get_numeric(event.sprintf(@logstash_kafka_read_time_indexer))
     
     # Validate the shipper data
-    shipper_kafka_array = Array[shipper_kafka_datacenter, shipper_kafka_topic, shipper_kafka_consumer_group, shipper_kafka_append_time, shipper_logstash_kafka_read_time, event_owner, event_time_ms]
+    shipper_kafka_array = Array[shipper_kafka_datacenter, shipper_kafka_topic, shipper_kafka_consumer_group, shipper_kafka_append_time, shipper_logstash_kafka_read_time, event_owner, event_time_ms, elasticsearch_cluster, elasticsearch_cluster_index]
     if (shipper_kafka_array.any? { |text| text.nil? || text.to_s.empty? })
       @logger.debug("shipper_kafka_array invalid: Found null")
       error_string_shipper = sprintf("Error in shipper data: %s", shipper_kafka_array)
+      @logger.debug(error_string_shipper)
       shipper_valid = false
     else
       @logger.debug("shipper_kafka_array valid")
@@ -86,10 +95,11 @@ class LogStash::Filters::KafkaTimeMachine < LogStash::Filters::Base
     end
 
     # Validate the indexer data
-    indexer_kafka_array = Array[shipper_kafka_datacenter, indexer_kafka_topic, indexer_kafka_consumer_group, indexer_kafka_append_time, indexer_logstash_kafka_read_time, event_owner, event_time_ms]
+    indexer_kafka_array = Array[shipper_kafka_datacenter, indexer_kafka_topic, indexer_kafka_consumer_group, indexer_kafka_append_time, indexer_logstash_kafka_read_time, event_owner, event_time_ms, elasticsearch_cluster, elasticsearch_cluster_index]
     if (indexer_kafka_array.any? { |text| text.nil? || text.to_s.empty? })
       @logger.debug("indexer_kafka_array invalid: Found null")
       error_string_indexer = sprintf("Error in indexer data: %s", indexer_kafka_array)
+      @logger.debug(error_string_indexer)
       indexer_valid = false
     else
       @logger.debug("indexer_kafka_array valid")
@@ -118,26 +128,26 @@ class LogStash::Filters::KafkaTimeMachine < LogStash::Filters::Base
     if (shipper_valid == true && indexer_valid == true && epoch_time_ns != nil)
       total_kafka_lag_ms = indexer_logstash_kafka_read_time - shipper_kafka_append_time
 
-      point_influxdb = create_influxdb_point_ktm(shipper_kafka_datacenter, event_owner, payload_bytesize, "total", total_kafka_lag_ms, epoch_time_ns)
+      point_influxdb = create_influxdb_point_ktm(shipper_kafka_datacenter, event_owner, payload_bytesize, "total", total_kafka_lag_ms, epoch_time_ns, elasticsearch_cluster, elasticsearch_cluster_index)
       ktm_metric_event_array.push point_influxdb
 
     elsif (shipper_valid == true && indexer_valid == false && epoch_time_ns != nil)
-      point_influxdb = create_influxdb_point_ktm(shipper_kafka_datacenter, event_owner, payload_bytesize, "shipper", shipper_kafka_lag_ms, epoch_time_ns)
+      point_influxdb = create_influxdb_point_ktm(shipper_kafka_datacenter, event_owner, payload_bytesize, "shipper", shipper_kafka_lag_ms, epoch_time_ns, elasticsearch_cluster, elasticsearch_cluster_index)
       ktm_metric_event_array.push point_influxdb
 
-      point_influxdb = create_influxdb_point_ktm_error(shipper_kafka_datacenter, event_owner, epoch_time_ns, "indexer")
+      point_influxdb = create_influxdb_point_ktm_error(shipper_kafka_datacenter, event_owner, epoch_time_ns, "indexer", elasticsearch_cluster, elasticsearch_cluster_index)
       ktm_metric_event_array.push point_influxdb
 
     elsif (indexer_valid == true && shipper_valid == false && epoch_time_ns != nil)
-      point_influxdb = create_influxdb_point_ktm(shipper_kafka_datacenter, event_owner, payload_bytesize, "indexer", indexer_kafka_lag_ms, epoch_time_ns)
+      point_influxdb = create_influxdb_point_ktm(shipper_kafka_datacenter, event_owner, payload_bytesize, "indexer", indexer_kafka_lag_ms, epoch_time_ns, elasticsearch_cluster, elasticsearch_cluster_index)
       ktm_metric_event_array.push point_influxdb
 
-      point_influxdb = create_influxdb_point_ktm_error(shipper_kafka_datacenter, event_owner, epoch_time_ns, "shipper")
+      point_influxdb = create_influxdb_point_ktm_error(shipper_kafka_datacenter, event_owner, epoch_time_ns, "shipper", elasticsearch_cluster, elasticsearch_cluster_index)
       ktm_metric_event_array.push point_influxdb
 
     elsif (indexer_valid == false && shipper_valid == false)
 
-      point_influxdb = create_influxdb_point_ktm_error(shipper_kafka_datacenter, event_owner, epoch_time_ns, "insufficient_data")
+      point_influxdb = create_influxdb_point_ktm_error(shipper_kafka_datacenter, event_owner, epoch_time_ns, "insufficient_data", elasticsearch_cluster, elasticsearch_cluster_index)
       ktm_metric_event_array.push point_influxdb
 
       error_string = sprintf("Error kafka_time_machine: Could not build valid response --> %s, %s", error_string_shipper, error_string_indexer)
@@ -145,7 +155,7 @@ class LogStash::Filters::KafkaTimeMachine < LogStash::Filters::Base
     
     else
 
-      point_influxdb = create_influxdb_point_ktm_error(shipper_kafka_datacenter, event_owner, epoch_time_ns, "unknown")
+      point_influxdb = create_influxdb_point_ktm_error(shipper_kafka_datacenter, event_owner, epoch_time_ns, "unknown", elasticsearch_cluster, elasticsearch_cluster_index)
       ktm_metric_event_array.push point_influxdb
 
       error_string = "Unknown error encountered"
@@ -171,10 +181,10 @@ class LogStash::Filters::KafkaTimeMachine < LogStash::Filters::Base
 
   # Creates an Influx DB line-protocol data point to return
   public
-  def create_influxdb_point_ktm(datacenter, event_owner, payload_size_bytes, lag_type, lag_ms, epoch_time_ns)
+  def create_influxdb_point_ktm(datacenter, event_owner, payload_size_bytes, lag_type, lag_ms, epoch_time_ns, elasticsearch_cluster, elasticsearch_cluster_index)
 
     point = InfluxDB2::Point.new( name: "ktm",
-                                  tags: {datacenter: datacenter, owner: event_owner, lag_type: lag_type},
+                                  tags: {datacenter: datacenter, owner: event_owner, lag_type: lag_type, es_cluster: elasticsearch_cluster, es_cluster_index: elasticsearch_cluster_index},
                                   fields: {payload_size_bytes: payload_size_bytes, lag_ms: lag_ms},
                                   time: epoch_time_ns)
 
@@ -185,7 +195,7 @@ class LogStash::Filters::KafkaTimeMachine < LogStash::Filters::Base
 
   # Creates an Influx DB line-protocol data point to return
   public
-  def create_influxdb_point_ktm_error(datacenter, event_owner, epoch_time_ns, type)
+  def create_influxdb_point_ktm_error(datacenter, event_owner, epoch_time_ns, type, elasticsearch_cluster, elasticsearch_cluster_index)
 
     # Check for nil values
     if (nil == datacenter)
@@ -202,7 +212,7 @@ class LogStash::Filters::KafkaTimeMachine < LogStash::Filters::Base
     end
 
     point = InfluxDB2::Point.new( name: "ktm_error",
-                                  tags: {datacenter: datacenter, owner: event_owner, source: type},
+                                  tags: {datacenter: datacenter, owner: event_owner, source: type, es_cluster: elasticsearch_cluster, es_cluster_index: elasticsearch_cluster_index},
                                   fields: {count: 1},
                                   time: epoch_time_ns)
 

data/logstash-filter-kafka_time_machine.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
     s.name = 'logstash-filter-kafka_time_machine'
-    s.version = '2.0.0'
+    s.version = '2.0.1'
     s.licenses = ['Apache-2.0']
     s.summary = "Calculate total time of logstash event that traversed 2 Kafka queues from a shipper site to an indexer site"
     s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-kafka_time_machine
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.0.1
 platform: ruby
 authors:
 - Chris Foster
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-10-21 00:00:00.000000000 Z
+date: 2021-10-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core-plugin-api