logstash-codec-kafka_time_machine 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 6a00a02e273acd58d4a7e6a06573f02e100202d2
4
+ data.tar.gz: 6c0dfc8295cb4d4cc769c26beb3deca3220b5fb1
5
+ SHA512:
6
+ metadata.gz: 3163b97371f86c32c2725e5813781c2e20c4bd458ba5090d0dad0cc8fccf116b1a7c499cbdee940cf20fcdae98d8aa1c2d3a31423af2b233d17df85bfc71595a
7
+ data.tar.gz: f61e0581276644f12d0d301145f9ac9be37a78e198329ade28bcfc92ddc1850b25c99b68bfa78aeaeeef80413df85e622a7431d2e1ec0bd7df2b916414a81257
data/Gemfile ADDED
@@ -0,0 +1,10 @@
1
+ source 'https://rubygems.org'
2
+ gemspec
3
+
4
+ logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
5
+ use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
6
+
7
+ if Dir.exist?(logstash_path) && use_logstash_source
8
+ gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
9
+ gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
10
+ end
@@ -0,0 +1,109 @@
1
+ # encoding: utf-8
2
+ require "logstash/codecs/base"
3
+ require "logstash/namespace"
4
+ require "logstash/event"
5
+
6
+ class LogStash::Codecs::KafkaTimeMachine < LogStash::Codecs::Base
7
+
8
+ config_name "kafkatimemachine"
9
+
10
+ @write_end_of_log = true
11
+
12
+ # Enable debug log file writes
13
+ config :enable_log, :validate => :boolean, :default => false
14
+
15
+ def file_output( output_line )
16
+
17
+ # Limit max file size to 5MB to protect integrity of host
18
+ max_file_size = 5242880
19
+
20
+ # Open file and append until max size reached
21
+ File.open("/tmp/kafkatimemachine.txt", "a") do |f|
22
+ if (f.size <= max_file_size)
23
+ f.puts(output_line)
24
+ @write_end_of_log = true
25
+ elsif (f.size > max_file_size && @write_end_of_log == true)
26
+ f.puts("Maximum file size of #{max_file_size} bytes reached; delete /tmp/kafkatimemachine.txt to resume writing")
27
+ @write_end_of_log = false
28
+ end
29
+ end
30
+
31
+ end
32
+
33
+ public
34
+ def register
35
+
36
+ end
37
+
38
+ public
39
+ def decode(data)
40
+ raise "Not implemented"
41
+ end # def decode
42
+
43
+ public
44
+ def encode(event)
45
+
46
+ # Extract producer data and check for validity
47
+ kafka_datacenter_producer = event.get("[@metadata][kafka_datacenter_producer]")
48
+ kafka_topic_producer = event.get("[@metadata][kafka_topic_producer]")
49
+ kafka_consumer_group_producer = event.get("[@metadata][kafka_consumer_group_producer]")
50
+ kafka_append_time_producer = Integer(event.get("[@metadata][kafka_append_time_producer]")) rescue nil
51
+ logstash_kafka_read_time_producer = Integer(event.get("[@metadata][logstash_kafka_read_time_producer]")) rescue nil
52
+
53
+ kafka_producer_array = Array[kafka_datacenter_producer, kafka_topic_producer, kafka_consumer_group_producer, kafka_append_time_producer, logstash_kafka_read_time_producer]
54
+ @logger.debug("kafka_producer_array: #{kafka_producer_array}")
55
+
56
+ if (kafka_producer_array.any? { |text| text.nil? || text.to_s.empty? })
57
+ @logger.debug("kafka_producer_array invalid: Found null")
58
+ error_string_producer = "Error in producer data: #{kafka_producer_array}"
59
+ producer_valid = false
60
+ else
61
+ @logger.debug("kafka_producer_array valid")
62
+ producer_valid = true
63
+ kafka_producer_lag_ms = logstash_kafka_read_time_producer - kafka_append_time_producer
64
+ end
65
+
66
+ # Extract aggregate data and check for validity
67
+ kafka_datacenter_aggregate = event.get("[@metadata][kafka_datacenter_aggregate]")
68
+ kafka_topic_aggregate = event.get("[@metadata][kafka_topic_aggregate]")
69
+ kafka_consumer_group_aggregate = event.get("[@metadata][kafka_consumer_group_aggregate]")
70
+ kafka_append_time_aggregate = Integer(event.get("[@metadata][kafka_append_time_aggregate]")) rescue nil
71
+ logstash_kafka_read_time_aggregate = Integer(event.get("[@metadata][logstash_kafka_read_time_aggregate]")) rescue nil
72
+
73
+ kafka_aggregate_array = Array[kafka_datacenter_aggregate, kafka_topic_aggregate, kafka_consumer_group_aggregate, kafka_append_time_aggregate, logstash_kafka_read_time_aggregate]
74
+ @logger.debug("kafka_aggregate_array: #{kafka_aggregate_array}")
75
+
76
+ if (kafka_aggregate_array.any? { |text| text.nil? || text.to_s.empty? })
77
+ @logger.debug("kafka_aggregate_array invalid: Found null")
78
+ error_string_aggregate = "Error in aggregate data: #{kafka_aggregate_array}"
79
+ aggregate_valid = false
80
+ else
81
+ @logger.debug("kafka_aggregate_array valid")
82
+ aggregate_valid = true
83
+ kafka_aggregate_lag_ms = logstash_kafka_read_time_aggregate - kafka_append_time_aggregate
84
+ end
85
+
86
+ # Get current time for influxdb timestamp
87
+ kafka_logstash_influx_metric_time = (Time.now.to_f * (1000*1000*1000)).to_i
88
+
89
+ if (producer_valid == true && aggregate_valid == true)
90
+ kafka_total_lag_ms = logstash_kafka_read_time_aggregate - kafka_append_time_producer
91
+ influx_line_protocol = "kafka_lag_time,meta_source=lma,meta_type=ktm,meta_datacenter=#{kafka_datacenter_producer},ktm_lag_type=complete,kafka_topic_aggregate=#{kafka_topic_aggregate},kafka_consumer_group_aggregate=#{kafka_consumer_group_aggregate},kafka_topic_producer=#{kafka_topic_producer},kafka_consumer_group_producer=#{kafka_consumer_group_producer} kafka_total_lag_ms=#{kafka_total_lag_ms},kafka_aggregate_lag_ms=#{kafka_aggregate_lag_ms},kafka_producer_lag_ms=#{kafka_producer_lag_ms} #{kafka_logstash_influx_metric_time}"
92
+ elsif (producer_valid == true && aggregate_valid == false)
93
+ influx_line_protocol = "kafka_lag_time,meta_source=lma,meta_type=ktm,meta_datacenter=#{kafka_datacenter_producer},ktm_lag_type=producer,kafka_topic_producer=#{kafka_topic_producer},kafka_consumer_group_producer=#{kafka_consumer_group_producer} kafka_producer_lag_ms=#{kafka_producer_lag_ms} #{kafka_logstash_influx_metric_time}"
94
+ elsif (aggregate_valid == true && producer_valid == false)
95
+ influx_line_protocol = "kafka_lag_time,meta_source=lma,meta_type=ktm,meta_datacenter=#{kafka_datacenter_aggregate},ktm_lag_type=aggregate,kafka_topic_aggregate=#{kafka_topic_aggregate},kafka_consumer_group_aggregate=#{kafka_consumer_group_aggregate} kafka_aggregate_lag_ms=#{kafka_aggregate_lag_ms} #{kafka_logstash_influx_metric_time}"
96
+ elsif (aggregate_valid == false && producer_valid == false)
97
+ @logger.error("Error kafkatimemachine: Could not build valid response --> #{error_string_producer}, #{error_string_aggregate}")
98
+ influx_line_protocol = nil
99
+ end
100
+
101
+ if (!influx_line_protocol.nil? && @enable_log == true)
102
+ file_output(influx_line_protocol)
103
+ end
104
+
105
+ @on_event.call(event, event.sprintf(influx_line_protocol))
106
+
107
+ end # def encode
108
+
109
+ end # class LogStash::Codecs::KafkaTimeMachine
@@ -0,0 +1,24 @@
1
+ Gem::Specification.new do |s|
2
+ s.name = 'logstash-codec-kafka_time_machine'
3
+ s.version = '0.1.0'
4
+ s.licenses = ['Apache-2.0']
5
+ s.summary = "Calculate total time of logstash event that traversed 2 Kafka queues from a producer site to an aggregate site"
6
+ s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
7
+ s.authors = ["Chris Foster"]
8
+ s.email = 'chrifost@cisco.com'
9
+ s.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html"
10
+ s.require_paths = ["lib"]
11
+
12
+ # Files
13
+ s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
14
+
15
+ # Tests
16
+ s.test_files = s.files.grep(%r{^(test|spec|features)/})
17
+
18
+ # Special flag to let us know this is actually a logstash plugin
19
+ s.metadata = { "logstash_plugin" => "true", "logstash_group" => "codec" }
20
+
21
+ # Gem dependencies
22
+ s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
23
+ s.add_development_dependency 'logstash-devutils', '~> 0'
24
+ end
metadata ADDED
@@ -0,0 +1,85 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: logstash-codec-kafka_time_machine
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Chris Foster
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2020-05-11 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: logstash-core-plugin-api
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: '1.60'
20
+ - - "<="
21
+ - !ruby/object:Gem::Version
22
+ version: '2.99'
23
+ type: :runtime
24
+ prerelease: false
25
+ version_requirements: !ruby/object:Gem::Requirement
26
+ requirements:
27
+ - - ">="
28
+ - !ruby/object:Gem::Version
29
+ version: '1.60'
30
+ - - "<="
31
+ - !ruby/object:Gem::Version
32
+ version: '2.99'
33
+ - !ruby/object:Gem::Dependency
34
+ name: logstash-devutils
35
+ requirement: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - "~>"
38
+ - !ruby/object:Gem::Version
39
+ version: '0'
40
+ type: :development
41
+ prerelease: false
42
+ version_requirements: !ruby/object:Gem::Requirement
43
+ requirements:
44
+ - - "~>"
45
+ - !ruby/object:Gem::Version
46
+ version: '0'
47
+ description: This gem is a logstash plugin required to be installed on top of the
48
+ Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
49
+ gem is not a stand-alone program
50
+ email: chrifost@cisco.com
51
+ executables: []
52
+ extensions: []
53
+ extra_rdoc_files: []
54
+ files:
55
+ - Gemfile
56
+ - lib/logstash/codecs/kafkatimemachine.rb
57
+ - logstash-codec-kafka_time_machine.gemspec
58
+ homepage: http://www.elastic.co/guide/en/logstash/current/index.html
59
+ licenses:
60
+ - Apache-2.0
61
+ metadata:
62
+ logstash_plugin: 'true'
63
+ logstash_group: codec
64
+ post_install_message:
65
+ rdoc_options: []
66
+ require_paths:
67
+ - lib
68
+ required_ruby_version: !ruby/object:Gem::Requirement
69
+ requirements:
70
+ - - ">="
71
+ - !ruby/object:Gem::Version
72
+ version: '0'
73
+ required_rubygems_version: !ruby/object:Gem::Requirement
74
+ requirements:
75
+ - - ">="
76
+ - !ruby/object:Gem::Version
77
+ version: '0'
78
+ requirements: []
79
+ rubyforge_project:
80
+ rubygems_version: 2.5.2.3
81
+ signing_key:
82
+ specification_version: 4
83
+ summary: Calculate total time of logstash event that traversed 2 Kafka queues from
84
+ a producer site to an aggregate site
85
+ test_files: []