logstash-filter-kafka_time_machine 0.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 4afd292b610638a04cf07a7c272c59819c561d43941f3784a3db4d67a91d4e40
+  data.tar.gz: 55b7b510230ff6549506b45a47fdd7457d7129a51188fd5820411bc76d1ccb5b
+SHA512:
+  metadata.gz: 511df4078b94243f61877b2d2391692ab4623b1dbff23f064cda7a14323ef5a84482d96755a75fb7f592834c9762cb4d3c9071596e88ffe1d850fd5b18a666ae
+  data.tar.gz: e0c80cceba1e9f214cf481ab3d6aa65874007e5c80696401c610d1a519c79d6d7a97392f179b35461cef6b0b0b25c8ab59a030879af9280c29c10ddc57aa4bc1

data/Gemfile

@@ -0,0 +1,10 @@
+source 'https://rubygems.org'
+gemspec
+
+logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
+use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
+
+if Dir.exist?(logstash_path) && use_logstash_source
+  gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
+  gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
+end

data/README.md

@@ -0,0 +1,3 @@
+# logstash-filter-kafka_time_machine
+
+TBD

data/lib/logstash/filters/kafkatimemachine.rb

@@ -0,0 +1,78 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+require "logstash/event"
+
+class LogStash::Filters::KafkaTimeMachine < LogStash::Filters::Base
+
+  config_name "kafkatimemachine"
+
+  public
+  def register
+  
+  end
+
+  public
+  def filter(event)
+
+    # Extract shipper data and check for validity; note that kafka_datacenter_shipper is used for both shipper and indexer arrays
+    kafka_datacenter_shipper = event.get("[@metadata][kafka_datacenter_shipper]")
+    kafka_topic_shipper = event.get("[@metadata][kafka_topic_shipper]")
+    kafka_consumer_group_shipper = event.get("[@metadata][kafka_consumer_group_shipper]")
+    kafka_append_time_shipper = Float(event.get("[@metadata][kafka_append_time_shipper]")) rescue nil 
+    logstash_kafka_read_time_shipper = Float(event.get("[@metadata][logstash_kafka_read_time_shipper]")) rescue nil
+
+    kafka_shipper_array = Array[kafka_datacenter_shipper, kafka_topic_shipper, kafka_consumer_group_shipper, kafka_append_time_shipper, logstash_kafka_read_time_shipper]
+    @logger.debug("kafka_shipper_array: #{kafka_shipper_array}")
+
+    if (kafka_shipper_array.any? { |text| text.nil? || text.to_s.empty? })
+      @logger.debug("kafka_shipper_array invalid: Found null")
+      error_string_shipper = "Error in shipper data: #{kafka_shipper_array}"
+      shipper_valid = false
+    else
+      @logger.debug("kafka_shipper_array valid")
+      shipper_valid = true
+      logstash_kafka_read_time_shipper = logstash_kafka_read_time_shipper.to_i
+      kafka_append_time_shipper = kafka_append_time_shipper.to_i
+      kafka_shipper_lag_ms = logstash_kafka_read_time_shipper - kafka_append_time_shipper
+    end
+
+    # Extract indexer data and check for validity
+    kafka_topic_indexer = event.get("[@metadata][kafka_topic_indexer]")
+    kafka_consumer_group_indexer = event.get("[@metadata][kafka_consumer_group_indexer]")
+    kafka_append_time_indexer = Float(event.get("[@metadata][kafka_append_time_indexer]")) rescue nil
+    logstash_kafka_read_time_indexer = Float(event.get("[@metadata][logstash_kafka_read_time_indexer]")) rescue nil
+
+    kafka_indexer_array = Array[kafka_datacenter_shipper, kafka_topic_indexer, kafka_consumer_group_indexer, kafka_append_time_indexer, logstash_kafka_read_time_indexer]
+    @logger.debug("kafka_indexer_array: #{kafka_indexer_array}")
+
+    if (kafka_indexer_array.any? { |text| text.nil? || text.to_s.empty? })
+      @logger.debug("kafka_indexer_array invalid: Found null")
+      error_string_indexer = "Error in indexer data: #{kafka_indexer_array}"
+      indexer_valid = false
+    else
+      @logger.debug("kafka_indexer_array valid")
+      indexer_valid = true
+      logstash_kafka_read_time_indexer = logstash_kafka_read_time_indexer.to_i
+      kafka_append_time_indexer = kafka_append_time_indexer.to_i
+      kafka_indexer_lag_ms = logstash_kafka_read_time_indexer - kafka_append_time_indexer
+    end
+
+    if (shipper_valid == true && indexer_valid == true)
+      kafka_total_lag_ms = logstash_kafka_read_time_indexer - kafka_append_time_shipper
+      event.set("[_ktm]", {"lag_total" => kafka_total_lag_ms, "lag_indexer" => kafka_indexer_lag_ms, "lag_shipper" => kafka_shipper_lag_ms, "datacenter_shipper" => kafka_datacenter_shipper, "kafka_topic_indexer" => kafka_topic_indexer, "kafka_consumer_group_indexer" => kafka_consumer_group_indexer, "kafka_topic_shipper" => kafka_topic_shipper, "kafka_consumer_group_shipper" => kafka_consumer_group_shipper, "tags" => ["ktm_lag_complete"] })
+    elsif (shipper_valid == true && indexer_valid == false)
+      event.set("[_ktm]", {"lag_shipper" => kafka_shipper_lag_ms, "datacenter_shipper" => kafka_datacenter_shipper, "kafka_topic_shipper" => kafka_topic_shipper, "kafka_consumer_group_shipper" => kafka_consumer_group_shipper, "tags" => ["ktm_lag_shipper"] })
+    elsif (indexer_valid == true && shipper_valid == false)
+      event.set("[_ktm]", {"lag_indexer" => kafka_indexer_lag_ms, "datacenter_shipper" => kafka_datacenter_shipper, "kafka_topic_indexer" => kafka_topic_indexer, "kafka_consumer_group_indexer" => kafka_consumer_group_indexer, "tags" => ["ktm_lag_indexer"] })
+    elsif (indexer_valid == false && shipper_valid == false)
+      @logger.error("Error kafkatimemachine: Could not build valid response --> #{error_string_shipper}, #{error_string_indexer}")
+      # event.set("[_ktm]", {"error_shipper" => error_string_shipper, "error_indexer" => error_string_indexer, "datacenter_shipper" => kafka_datacenter_shipper, "tags" => ["ktm_error"]  })
+    end
+
+    # filter_matched should go in the last line of our successful code
+    filter_matched(event)
+
+  end # def filter
+
+end # class LogStash::Filters::KafkaTimeMachine

data/logstash-filter-kafka_time_machine.gemspec

@@ -0,0 +1,24 @@
+Gem::Specification.new do |s|
+    s.name = 'logstash-filter-kafka_time_machine'
+    s.version = '0.2.0'
+    s.licenses = ['Apache-2.0']
+    s.summary = "Calculate total time of logstash event that traversed 2 Kafka queues from a shipper site to an indexer site"
+    s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
+    s.authors = ["Chris Foster"]
+    s.email = 'chrifost@cisco.com'
+    s.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html"
+    s.require_paths = ["lib"]
+
+    # Files
+    s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+    
+    # Tests
+    s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+    # Special flag to let us know this is actually a logstash plugin
+    s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
+
+    # Gem dependencies
+    s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+    s.add_development_dependency 'logstash-devutils', '~> 0'
+  end

metadata

@@ -0,0 +1,85 @@
+--- !ruby/object:Gem::Specification
+name: logstash-filter-kafka_time_machine
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+platform: ruby
+authors:
+- Chris Foster
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2021-06-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: logstash-core-plugin-api
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
+  gem is not a stand-alone program
+email: chrifost@cisco.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- README.md
+- lib/logstash/filters/kafkatimemachine.rb
+- logstash-filter-kafka_time_machine.gemspec
+homepage: http://www.elastic.co/guide/en/logstash/current/index.html
+licenses:
+- Apache-2.0
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: filter
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: Calculate total time of logstash event that traversed 2 Kafka queues from
+  a shipper site to an indexer site
+test_files: []