logstash-filter-kafka_time_machine 0.4.0 → 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: f737fcbdc55c777f9ff5000e778c8fd26ceda00fdb87ee48ecb6c65d7dafaf90
|
|
4
|
+
data.tar.gz: 5c4f1cd0dc80f63cd97a93701c2d12d79e0c97a93f913187c6dcaa1e19ca306a
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: e0549b7084ce70c4af5e94d249ebf244d215f4bffbdc4db5c9b0356041cc50b736e0e2dcf0680b0d049b0f38402a9f3931aa2eae6ace9b41cd3628ae04c73fe0
|
|
7
|
+
data.tar.gz: '04338e54be18ff407221ed442d62d641cbcf4391adb5cd85d919efa0c37fe158f18bcfa88cace01c69f8ed44b7156a1ac96e9e6b42124fed87a1873d4e49eee1'
|
|
@@ -60,15 +60,21 @@ class LogStash::Filters::KafkaTimeMachine < LogStash::Filters::Base
|
|
|
60
60
|
|
|
61
61
|
if (shipper_valid == true && indexer_valid == true)
|
|
62
62
|
kafka_total_lag_ms = logstash_kafka_read_time_indexer - kafka_append_time_shipper
|
|
63
|
-
event.set("[ktm]", {"
|
|
63
|
+
event.set("[ktm]", {"lag_total_ms" => kafka_total_lag_ms, "lag_indexer" => kafka_indexer_lag_ms, "lag_shipper" => kafka_shipper_lag_ms, "datacenter_shipper" => kafka_datacenter_shipper, "kafka_topic_indexer" => kafka_topic_indexer, "kafka_consumer_group_indexer" => kafka_consumer_group_indexer, "kafka_topic_shipper" => kafka_topic_shipper, "kafka_consumer_group_shipper" => kafka_consumer_group_shipper, "tags" => ["ktm_lag_total"] })
|
|
64
64
|
elsif (shipper_valid == true && indexer_valid == false)
|
|
65
|
-
event.set("[ktm]", {"
|
|
65
|
+
event.set("[ktm]", {"lag_shipper_ms" => kafka_shipper_lag_ms, "datacenter_shipper" => kafka_datacenter_shipper, "kafka_topic_shipper" => kafka_topic_shipper, "kafka_consumer_group_shipper" => kafka_consumer_group_shipper, "tags" => ["ktm_lag_shipper"] })
|
|
66
66
|
elsif (indexer_valid == true && shipper_valid == false)
|
|
67
|
-
event.set("[ktm]", {"
|
|
67
|
+
event.set("[ktm]", {"lag_indexer_ms" => kafka_indexer_lag_ms, "datacenter_shipper" => kafka_datacenter_shipper, "kafka_topic_indexer" => kafka_topic_indexer, "kafka_consumer_group_indexer" => kafka_consumer_group_indexer, "tags" => ["ktm_lag_indexer"] })
|
|
68
68
|
elsif (indexer_valid == false && shipper_valid == false)
|
|
69
69
|
@logger.debug("Error kafkatimemachine: Could not build valid response --> #{error_string_shipper}, #{error_string_indexer}")
|
|
70
70
|
end
|
|
71
71
|
|
|
72
|
+
# Add in the size of the payload field
|
|
73
|
+
if event.get("[payload]")
|
|
74
|
+
payload_bytesize = event.get("[payload]").bytesize
|
|
75
|
+
event.set("[ktm][payload_size_bytes]", payload_bytesize)
|
|
76
|
+
end
|
|
77
|
+
|
|
72
78
|
# filter_matched should go in the last line of our successful code
|
|
73
79
|
filter_matched(event)
|
|
74
80
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Gem::Specification.new do |s|
|
|
2
2
|
s.name = 'logstash-filter-kafka_time_machine'
|
|
3
|
-
s.version = '0.
|
|
3
|
+
s.version = '0.5.0'
|
|
4
4
|
s.licenses = ['Apache-2.0']
|
|
5
5
|
s.summary = "Calculate total time of logstash event that traversed 2 Kafka queues from a shipper site to an indexer site"
|
|
6
6
|
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: logstash-filter-kafka_time_machine
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.5.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Chris Foster
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2021-06-
|
|
11
|
+
date: 2021-06-17 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: logstash-core-plugin-api
|