logstash-codec-kafka_time_machine 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 6a00a02e273acd58d4a7e6a06573f02e100202d2
+  data.tar.gz: 6c0dfc8295cb4d4cc769c26beb3deca3220b5fb1
+SHA512:
+  metadata.gz: 3163b97371f86c32c2725e5813781c2e20c4bd458ba5090d0dad0cc8fccf116b1a7c499cbdee940cf20fcdae98d8aa1c2d3a31423af2b233d17df85bfc71595a
+  data.tar.gz: f61e0581276644f12d0d301145f9ac9be37a78e198329ade28bcfc92ddc1850b25c99b68bfa78aeaeeef80413df85e622a7431d2e1ec0bd7df2b916414a81257

data/Gemfile

@@ -0,0 +1,10 @@
+source 'https://rubygems.org'
+gemspec
+
+logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
+use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
+
+if Dir.exist?(logstash_path) && use_logstash_source
+  gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
+  gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
+end

data/lib/logstash/codecs/kafkatimemachine.rb

@@ -0,0 +1,109 @@
+# encoding: utf-8
+require "logstash/codecs/base"
+require "logstash/namespace"
+require "logstash/event"
+
+class LogStash::Codecs::KafkaTimeMachine < LogStash::Codecs::Base
+
+  config_name "kafkatimemachine"
+
+  @write_end_of_log = true
+
+  # Enable debug log file writes
+  config :enable_log, :validate => :boolean, :default => false
+
+  def file_output( output_line )
+    
+    # Limit max file size to 5MB to protect integrity of host
+    max_file_size = 5242880
+
+    # Open file and append until max size reached
+    File.open("/tmp/kafkatimemachine.txt", "a") do |f|
+      if (f.size <= max_file_size)
+        f.puts(output_line)
+        @write_end_of_log = true
+      elsif (f.size > max_file_size && @write_end_of_log == true)
+        f.puts("Maximum file size of #{max_file_size} bytes reached; delete /tmp/kafkatimemachine.txt to resume writing")
+        @write_end_of_log = false
+      end
+    end
+
+  end
+
+  public
+  def register
+  
+  end
+
+  public
+  def decode(data)
+    raise "Not implemented"
+  end # def decode
+
+  public
+  def encode(event)
+
+    # Extract producer data and check for validity
+    kafka_datacenter_producer = event.get("[@metadata][kafka_datacenter_producer]")
+    kafka_topic_producer = event.get("[@metadata][kafka_topic_producer]")
+    kafka_consumer_group_producer = event.get("[@metadata][kafka_consumer_group_producer]")
+    kafka_append_time_producer = Integer(event.get("[@metadata][kafka_append_time_producer]")) rescue nil 
+    logstash_kafka_read_time_producer = Integer(event.get("[@metadata][logstash_kafka_read_time_producer]")) rescue nil
+
+    kafka_producer_array = Array[kafka_datacenter_producer, kafka_topic_producer, kafka_consumer_group_producer, kafka_append_time_producer, logstash_kafka_read_time_producer]
+    @logger.debug("kafka_producer_array: #{kafka_producer_array}")
+
+    if (kafka_producer_array.any? { |text| text.nil? || text.to_s.empty? })
+      @logger.debug("kafka_producer_array invalid: Found null")
+      error_string_producer = "Error in producer data: #{kafka_producer_array}"
+      producer_valid = false
+    else
+      @logger.debug("kafka_producer_array valid")
+      producer_valid = true
+      kafka_producer_lag_ms = logstash_kafka_read_time_producer - kafka_append_time_producer
+    end
+
+    # Extract aggregate data and check for validity
+    kafka_datacenter_aggregate = event.get("[@metadata][kafka_datacenter_aggregate]")
+    kafka_topic_aggregate = event.get("[@metadata][kafka_topic_aggregate]")
+    kafka_consumer_group_aggregate = event.get("[@metadata][kafka_consumer_group_aggregate]")
+    kafka_append_time_aggregate = Integer(event.get("[@metadata][kafka_append_time_aggregate]")) rescue nil
+    logstash_kafka_read_time_aggregate = Integer(event.get("[@metadata][logstash_kafka_read_time_aggregate]")) rescue nil
+
+    kafka_aggregate_array = Array[kafka_datacenter_aggregate, kafka_topic_aggregate, kafka_consumer_group_aggregate, kafka_append_time_aggregate, logstash_kafka_read_time_aggregate]
+    @logger.debug("kafka_aggregate_array: #{kafka_aggregate_array}")
+
+    if (kafka_aggregate_array.any? { |text| text.nil? || text.to_s.empty? })
+      @logger.debug("kafka_aggregate_array invalid: Found null")
+      error_string_aggregate = "Error in aggregate data: #{kafka_aggregate_array}"
+      aggregate_valid = false
+    else
+      @logger.debug("kafka_aggregate_array valid")
+      aggregate_valid = true
+      kafka_aggregate_lag_ms = logstash_kafka_read_time_aggregate - kafka_append_time_aggregate
+    end
+
+    # Get current time for influxdb timestamp
+    kafka_logstash_influx_metric_time = (Time.now.to_f * (1000*1000*1000)).to_i
+
+    if (producer_valid == true && aggregate_valid == true)
+      kafka_total_lag_ms = logstash_kafka_read_time_aggregate - kafka_append_time_producer
+      influx_line_protocol = "kafka_lag_time,meta_source=lma,meta_type=ktm,meta_datacenter=#{kafka_datacenter_producer},ktm_lag_type=complete,kafka_topic_aggregate=#{kafka_topic_aggregate},kafka_consumer_group_aggregate=#{kafka_consumer_group_aggregate},kafka_topic_producer=#{kafka_topic_producer},kafka_consumer_group_producer=#{kafka_consumer_group_producer} kafka_total_lag_ms=#{kafka_total_lag_ms},kafka_aggregate_lag_ms=#{kafka_aggregate_lag_ms},kafka_producer_lag_ms=#{kafka_producer_lag_ms} #{kafka_logstash_influx_metric_time}"
+    elsif (producer_valid == true && aggregate_valid == false)
+      influx_line_protocol = "kafka_lag_time,meta_source=lma,meta_type=ktm,meta_datacenter=#{kafka_datacenter_producer},ktm_lag_type=producer,kafka_topic_producer=#{kafka_topic_producer},kafka_consumer_group_producer=#{kafka_consumer_group_producer} kafka_producer_lag_ms=#{kafka_producer_lag_ms} #{kafka_logstash_influx_metric_time}"
+    elsif (aggregate_valid == true && producer_valid == false)
+      influx_line_protocol = "kafka_lag_time,meta_source=lma,meta_type=ktm,meta_datacenter=#{kafka_datacenter_aggregate},ktm_lag_type=aggregate,kafka_topic_aggregate=#{kafka_topic_aggregate},kafka_consumer_group_aggregate=#{kafka_consumer_group_aggregate} kafka_aggregate_lag_ms=#{kafka_aggregate_lag_ms} #{kafka_logstash_influx_metric_time}"
+    elsif (aggregate_valid == false && producer_valid == false)
+      @logger.error("Error kafkatimemachine: Could not build valid response --> #{error_string_producer}, #{error_string_aggregate}")
+      influx_line_protocol = nil
+    end
+
+    if (!influx_line_protocol.nil? && @enable_log == true)
+      file_output(influx_line_protocol)
+    end
+
+    @on_event.call(event, event.sprintf(influx_line_protocol))
+
+  end # def encode
+
+end # class LogStash::Codecs::KafkaTimeMachine

data/logstash-codec-kafka_time_machine.gemspec

@@ -0,0 +1,24 @@
+Gem::Specification.new do |s|
+    s.name = 'logstash-codec-kafka_time_machine'
+    s.version = '0.1.0'
+    s.licenses = ['Apache-2.0']
+    s.summary = "Calculate total time of logstash event that traversed 2 Kafka queues from a producer site to an aggregate site"
+    s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
+    s.authors = ["Chris Foster"]
+    s.email = 'chrifost@cisco.com'
+    s.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html"
+    s.require_paths = ["lib"]
+
+    # Files
+    s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+    
+    # Tests
+    s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+    # Special flag to let us know this is actually a logstash plugin
+    s.metadata = { "logstash_plugin" => "true", "logstash_group" => "codec" }
+
+    # Gem dependencies
+    s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+    s.add_development_dependency 'logstash-devutils', '~> 0'
+  end

metadata

@@ -0,0 +1,85 @@
+--- !ruby/object:Gem::Specification
+name: logstash-codec-kafka_time_machine
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Chris Foster
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-05-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: logstash-core-plugin-api
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
+  gem is not a stand-alone program
+email: chrifost@cisco.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- lib/logstash/codecs/kafkatimemachine.rb
+- logstash-codec-kafka_time_machine.gemspec
+homepage: http://www.elastic.co/guide/en/logstash/current/index.html
+licenses:
+- Apache-2.0
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: codec
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2.3
+signing_key: 
+specification_version: 4
+summary: Calculate total time of logstash event that traversed 2 Kafka queues from
+  a producer site to an aggregate site
+test_files: []