logstash-filter-kafka_time_machine 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: f737fcbdc55c777f9ff5000e778c8fd26ceda00fdb87ee48ecb6c65d7dafaf90
|
|
4
|
+
data.tar.gz: 5c4f1cd0dc80f63cd97a93701c2d12d79e0c97a93f913187c6dcaa1e19ca306a
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: e0549b7084ce70c4af5e94d249ebf244d215f4bffbdc4db5c9b0356041cc50b736e0e2dcf0680b0d049b0f38402a9f3931aa2eae6ace9b41cd3628ae04c73fe0
|
|
7
|
+
data.tar.gz: '04338e54be18ff407221ed442d62d641cbcf4391adb5cd85d919efa0c37fe158f18bcfa88cace01c69f8ed44b7156a1ac96e9e6b42124fed87a1873d4e49eee1'
|
|
@@ -60,15 +60,21 @@ class LogStash::Filters::KafkaTimeMachine < LogStash::Filters::Base
|
|
|
60
60
|
|
|
61
61
|
if (shipper_valid == true && indexer_valid == true)
|
|
62
62
|
kafka_total_lag_ms = logstash_kafka_read_time_indexer - kafka_append_time_shipper
|
|
63
|
-
event.set("[ktm]", {"
|
|
63
|
+
event.set("[ktm]", {"lag_total_ms" => kafka_total_lag_ms, "lag_indexer" => kafka_indexer_lag_ms, "lag_shipper" => kafka_shipper_lag_ms, "datacenter_shipper" => kafka_datacenter_shipper, "kafka_topic_indexer" => kafka_topic_indexer, "kafka_consumer_group_indexer" => kafka_consumer_group_indexer, "kafka_topic_shipper" => kafka_topic_shipper, "kafka_consumer_group_shipper" => kafka_consumer_group_shipper, "tags" => ["ktm_lag_total"] })
|
|
64
64
|
elsif (shipper_valid == true && indexer_valid == false)
|
|
65
|
-
event.set("[ktm]", {"
|
|
65
|
+
event.set("[ktm]", {"lag_shipper_ms" => kafka_shipper_lag_ms, "datacenter_shipper" => kafka_datacenter_shipper, "kafka_topic_shipper" => kafka_topic_shipper, "kafka_consumer_group_shipper" => kafka_consumer_group_shipper, "tags" => ["ktm_lag_shipper"] })
|
|
66
66
|
elsif (indexer_valid == true && shipper_valid == false)
|
|
67
|
-
event.set("[ktm]", {"
|
|
67
|
+
event.set("[ktm]", {"lag_indexer_ms" => kafka_indexer_lag_ms, "datacenter_shipper" => kafka_datacenter_shipper, "kafka_topic_indexer" => kafka_topic_indexer, "kafka_consumer_group_indexer" => kafka_consumer_group_indexer, "tags" => ["ktm_lag_indexer"] })
|
|
68
68
|
elsif (indexer_valid == false && shipper_valid == false)
|
|
69
69
|
@logger.debug("Error kafkatimemachine: Could not build valid response --> #{error_string_shipper}, #{error_string_indexer}")
|
|
70
70
|
end
|
|
71
71
|
|
|
72
|
+
# Add in the size of the payload field
|
|
73
|
+
if event.get("[payload]")
|
|
74
|
+
payload_bytesize = event.get("[payload]").bytesize
|
|
75
|
+
event.set("[ktm][payload_size_bytes]", payload_bytesize)
|
|
76
|
+
end
|
|
77
|
+
|
|
72
78
|
# filter_matched should go in the last line of our successful code
|
|
73
79
|
filter_matched(event)
|
|
74
80
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Gem::Specification.new do |s|
|
|
2
2
|
s.name = 'logstash-filter-kafka_time_machine'
|
|
3
|
-
s.version = '0.
|
|
3
|
+
s.version = '0.5.0'
|
|
4
4
|
s.licenses = ['Apache-2.0']
|
|
5
5
|
s.summary = "Calculate total time of logstash event that traversed 2 Kafka queues from a shipper site to an indexer site"
|
|
6
6
|
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: logstash-filter-kafka_time_machine
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.5.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Chris Foster
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2021-06-
|
|
11
|
+
date: 2021-06-17 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: logstash-core-plugin-api
|