fluent-plugin-elasticsearch-stats 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +1 -1
- data/.ruby-version +1 -1
- data/Gemfile.lock +1 -1
- data/README.md +33 -29
- data/fluent-plugin-elasticsearch-stats.gemspec +3 -3
- data/lib/fluent/plugin/elasticsearch_stats/base_data.rb +49 -25
- data/lib/fluent/plugin/elasticsearch_stats/client.rb +4 -4
- data/lib/fluent/plugin/elasticsearch_stats/collector.rb +2 -1
- data/lib/fluent/plugin/elasticsearch_stats/dangling_data.rb +1 -1
- data/lib/fluent/plugin/elasticsearch_stats/metric.rb +11 -2
- data/lib/fluent/plugin/in_elasticsearch_stats.rb +17 -1
- metadata +5 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b85d9b62807f0710d7c7b55242271b90deda0eb3aff6cbaba56e079cda20261a
|
4
|
+
data.tar.gz: 8c6c8ff1e8c784ad6ab377805be083aff4b27463404df765b77d578a934a19c6
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d783330de75059628def68f20cdcd8f9ba2e3e9b8f99e2d1990a655be837440cf56b337b32ef5561bc8f0a27d21685456cb5bc29117ca23a3e327ab8b9f653ca
|
7
|
+
data.tar.gz: 69d6a1a09c83aa98754e04d9d162877d330496a8e5ea899a648b02ad698471380844e8cd10d251683f7a4626f274d4bed695784ab8f1d1a791964d4ea97f670d
|
data/.rubocop.yml
CHANGED
data/.ruby-version
CHANGED
@@ -1 +1 @@
|
|
1
|
-
3.3.
|
1
|
+
3.3.5
|
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
@@ -34,26 +34,29 @@ Example of config
|
|
34
34
|
|
35
35
|
#### global options
|
36
36
|
|
37
|
-
| setting
|
38
|
-
|
39
|
-
| tag
|
40
|
-
| urls
|
41
|
-
| timeout
|
42
|
-
| username
|
43
|
-
| password
|
44
|
-
| user_agent
|
45
|
-
| ca_file
|
46
|
-
| verify_ssl
|
47
|
-
|
|
48
|
-
| interval
|
49
|
-
|
|
50
|
-
| metric_prefix
|
51
|
-
| metadata_prefix
|
52
|
-
| timestamp_format
|
53
|
-
| event_name_separator
|
54
|
-
|
|
55
|
-
| index_base_pattern
|
56
|
-
| index_base_replacement
|
37
|
+
| setting | type | default | description |
|
38
|
+
|-------------------------------|--------------------|-----------------------------------|---------------------------------------------------------------------|
|
39
|
+
| tag | | elasticsearch_stats | tag to emit events on |
|
40
|
+
| urls | | ["http://localhost:9200"] | list of urls to poll |
|
41
|
+
| timeout | second | 10 | timeout for each call |
|
42
|
+
| username | | | username for basic authentication |
|
43
|
+
| password | | | password for basic authentication |
|
44
|
+
| user_agent | | fluent-plugin-elasticsearch-stats | user agent for http request |
|
45
|
+
| ca_file | file | | CA cert file to use for request |
|
46
|
+
| verify_ssl | bool | true | option to verify certificate/host |
|
47
|
+
| | | | |
|
48
|
+
| interval | second | 300 | interval for probe execution |
|
49
|
+
| | | | |
|
50
|
+
| metric_prefix | | | prefix for metric fields |
|
51
|
+
| metadata_prefix | | | prefix for metadata fields |
|
52
|
+
| timestamp_format | iso\|epochmillis | iso | event timestamp format |
|
53
|
+
| event_name_separator | | / | event name separator |
|
54
|
+
| | | | |
|
55
|
+
| index_base_pattern | regexp | | base index pattern to generate aggregated index metrics |
|
56
|
+
| index_base_replacement | regexp replacement | \1 | base index pattern replacement to generate aggregated index metrics |
|
57
|
+
| | | | |
|
58
|
+
| aggregated_index_metrics_only | bool | false | for index matrics, only generate aggregated metrics |
|
59
|
+
| aggregated_index_metrics | array | [sum] | select in [avg, count, min, max, sum] |
|
57
60
|
|
58
61
|
#### cluster health options
|
59
62
|
|
@@ -67,7 +70,7 @@ Events from **[/_cluster/health?level=indices](https://www.elastic.co/guide/en/e
|
|
67
70
|
|
68
71
|
#### cluster stats options
|
69
72
|
|
70
|
-
Events from
|
73
|
+
Events from **[/_cluster/stats](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-stats.html)**.
|
71
74
|
|
72
75
|
| setting | type | default | description |
|
73
76
|
|--------------------------------|-------------|---------|--------------------------------------------|
|
@@ -75,7 +78,7 @@ Events from **/_cluster/stats**.
|
|
75
78
|
|
76
79
|
#### nodes stats options
|
77
80
|
|
78
|
-
Events from
|
81
|
+
Events from **[/_nodes/stats](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-nodes-stats.html)**.
|
79
82
|
|
80
83
|
| setting | type | default | description |
|
81
84
|
|---------------------|----------------------------------------------------------------------|---------|-----------------------------------|
|
@@ -85,17 +88,18 @@ Events from **/_nodes/stats**.
|
|
85
88
|
|
86
89
|
#### indices stats options
|
87
90
|
|
88
|
-
Events from
|
91
|
+
Events from **[/_all/_stats](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-stats.html)**.
|
89
92
|
|
90
|
-
| setting
|
91
|
-
|
92
|
-
| indices_stats
|
93
|
-
| indices_stats_level
|
94
|
-
| indices
|
93
|
+
| setting | type | default | description |
|
94
|
+
|-----------------------|---------------------------|----------|-------------------------------------|
|
95
|
+
| indices_stats | true\|false | true | enable indices stats events collect |
|
96
|
+
| indices_stats_level | cluster\|indices\|shards | indices | indices_stats details level |
|
97
|
+
| indices | | ["_all"] | indices to collect stats on |
|
98
|
+
| indices_stats_metrics | completion,docs,store,... | [] | |
|
95
99
|
|
96
100
|
#### dangling options
|
97
101
|
|
98
|
-
Events from
|
102
|
+
Events from **[/_dangling](https://www.elastic.co/guide/en/elasticsearch/reference/current/dangling-indices-list.html)**.
|
99
103
|
|
100
104
|
| setting | type | default | description |
|
101
105
|
|----------|-------------|---------|--------------------------------|
|
@@ -5,15 +5,15 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |spec|
|
7
7
|
spec.name = 'fluent-plugin-elasticsearch-stats'
|
8
|
-
spec.version = '0.
|
8
|
+
spec.version = '0.3.0'
|
9
9
|
spec.authors = ['Thomas Tych']
|
10
10
|
spec.email = ['thomas.tych@gmail.com']
|
11
11
|
|
12
12
|
spec.summary = 'fluentd plugin to generate elasticsearch cluster stats events.'
|
13
|
-
spec.homepage = 'https://gitlab.com/ttych/fluent-plugin-elasticsearch-
|
13
|
+
spec.homepage = 'https://gitlab.com/ttych/fluent-plugin-elasticsearch-stats'
|
14
14
|
spec.license = 'Apache-2.0'
|
15
15
|
|
16
|
-
spec.required_ruby_version = '>= 2.
|
16
|
+
spec.required_ruby_version = '>= 2.7.0'
|
17
17
|
|
18
18
|
spec.metadata['rubygems_mfa_required'] = 'true'
|
19
19
|
|
@@ -40,6 +40,8 @@ module Fluent
|
|
40
40
|
base_metrics = extract_indices_base_metrics
|
41
41
|
aggregated_metrics = extract_indices_aggregated_metrics(base_metrics)
|
42
42
|
|
43
|
+
return aggregated_metrics if metric.aggregated_index_metrics_only
|
44
|
+
|
43
45
|
base_metrics + aggregated_metrics
|
44
46
|
end
|
45
47
|
|
@@ -76,35 +78,57 @@ module Fluent
|
|
76
78
|
.set(label: 'index', value: index_base)
|
77
79
|
.set(label: 'aggregated', value: true)
|
78
80
|
|
81
|
+
index_base_count = 0
|
82
|
+
|
79
83
|
index_base_metrics.each do |metric_name, metric_name_metrics|
|
80
84
|
metric_values = metric_name_metrics.map { |a_metric| a_metric[metric.value_label] }
|
81
85
|
count = metric_values.size
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
86
|
+
index_base_count = count if count > index_base_count
|
87
|
+
|
88
|
+
if metric.aggregated_index_metrics.include?('count')
|
89
|
+
metrics << metric.format(name: [metric_name, 'count'],
|
90
|
+
value: count,
|
91
|
+
family: family,
|
92
|
+
metadata: local_metadata)
|
93
|
+
end
|
94
|
+
|
95
|
+
if metric.aggregated_index_metrics.include?('min')
|
96
|
+
min = metric_values.min
|
97
|
+
metrics << metric.format(name: [metric_name, 'min'],
|
98
|
+
value: min,
|
99
|
+
family: family,
|
100
|
+
metadata: local_metadata)
|
101
|
+
end
|
102
|
+
|
103
|
+
if metric.aggregated_index_metrics.include?('max')
|
104
|
+
max = metric_values.max
|
105
|
+
metrics << metric.format(name: [metric_name, 'max'],
|
106
|
+
value: max,
|
107
|
+
family: family,
|
108
|
+
metadata: local_metadata)
|
109
|
+
end
|
110
|
+
|
111
|
+
if metric.aggregated_index_metrics.include?('sum')
|
112
|
+
sum = metric_values.sum
|
113
|
+
metrics << metric.format(name: [metric_name, 'sum'],
|
114
|
+
value: sum,
|
115
|
+
family: family,
|
116
|
+
metadata: local_metadata)
|
117
|
+
end
|
118
|
+
|
119
|
+
if metric.aggregated_index_metrics.include?('avg')
|
120
|
+
avg = sum / count.to_f
|
121
|
+
metrics << metric.format(name: [metric_name, 'avg'],
|
122
|
+
value: avg,
|
123
|
+
family: family,
|
124
|
+
metadata: local_metadata)
|
125
|
+
end
|
107
126
|
end
|
127
|
+
|
128
|
+
metrics << metric.format(name: %w[index count],
|
129
|
+
value: index_base_count,
|
130
|
+
family: family,
|
131
|
+
metadata: local_metadata)
|
108
132
|
end
|
109
133
|
|
110
134
|
metrics.compact
|
@@ -16,6 +16,7 @@ module Fluent
|
|
16
16
|
CLUSTER_HEALTH_LEVEL = 'cluster'
|
17
17
|
NODES_STATS_LEVEL = 'cluster'
|
18
18
|
INDICES_STATS_LEVEL = 'indices'
|
19
|
+
INDICES = [:_all]
|
19
20
|
|
20
21
|
ALLOWED_CLUSTER_HEALTH_LEVELS = %i[cluster indices shards].freeze
|
21
22
|
ALLOWED_NODES_STATS_LEVELS = %i[nodes indices shards].freeze
|
@@ -93,13 +94,12 @@ module Fluent
|
|
93
94
|
end
|
94
95
|
|
95
96
|
# https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-stats.html
|
96
|
-
def indices_stats(indices:
|
97
|
+
def indices_stats(indices: INDICES, level: INDICES_STATS_LEVEL, metrics: nil)
|
98
|
+
indices ||= INDICES
|
97
99
|
endpoint = "/_stats"
|
98
100
|
endpoint = "/#{indices.join(',')}#{endpoint}" if !indices.nil? && !indices.empty?
|
99
101
|
endpoint += "/#{metrics.join(',')}" if metrics&.any?
|
100
|
-
params = {
|
101
|
-
level: level
|
102
|
-
}
|
102
|
+
params = { level: level }
|
103
103
|
get(endpoint, params)
|
104
104
|
end
|
105
105
|
|
@@ -63,7 +63,8 @@ module Fluent
|
|
63
63
|
without_error(rescue_return: []) do
|
64
64
|
data = client.indices_stats(
|
65
65
|
indices: stats_config.indices,
|
66
|
-
level: stats_config.indices_stats_level
|
66
|
+
level: stats_config.indices_stats_level,
|
67
|
+
metrics: stats_config.indices_stats_metrics
|
67
68
|
)
|
68
69
|
IndicesStatsData.new(data).extract_metrics
|
69
70
|
end
|
@@ -14,7 +14,7 @@ module Fluent
|
|
14
14
|
|
15
15
|
def generate_dangling_indices_count
|
16
16
|
metrics = []
|
17
|
-
metrics << metric.format(name: %w[
|
17
|
+
metrics << metric.format(name: %w[dangling_indices count],
|
18
18
|
value: dangling_indices.size,
|
19
19
|
family: family,
|
20
20
|
metadata: metadata)
|
@@ -7,8 +7,12 @@ module Fluent
|
|
7
7
|
DEFAULT_NAME_SEPARATOR = '/'
|
8
8
|
DEFAULT_TIMESTAMP_FORMAT = :iso
|
9
9
|
|
10
|
+
ALLOWED_AGGREGATED_INDEX_METRICS = %w[count min max avg sum].freeze
|
11
|
+
DEFAULT_AGGREGATED_INDEX_METRICS = ['sum'].freeze
|
12
|
+
|
10
13
|
class << self
|
11
|
-
attr_accessor :metric_prefix, :index_base_pattern, :index_base_replacement
|
14
|
+
attr_accessor :metric_prefix, :index_base_pattern, :index_base_replacement,
|
15
|
+
:aggregated_index_metrics_only, :aggregated_index_metrics
|
12
16
|
|
13
17
|
def name_separator
|
14
18
|
@name_separator ||= DEFAULT_NAME_SEPARATOR
|
@@ -21,14 +25,19 @@ module Fluent
|
|
21
25
|
end
|
22
26
|
end
|
23
27
|
|
24
|
-
attr_reader :metric_prefix, :timestamp_format, :index_base_pattern, :index_base_replacement,
|
28
|
+
attr_reader :metric_prefix, :timestamp_format, :index_base_pattern, :index_base_replacement,
|
29
|
+
:aggregated_index_metrics_only, :aggregated_index_metrics,
|
30
|
+
:name_separator
|
25
31
|
|
26
32
|
def initialize(metric_prefix: nil, timestamp_format: nil, index_base_pattern: nil, index_base_replacement: nil,
|
33
|
+
aggregated_index_metrics_only: nil, aggregated_index_metrics: DEFAULT_AGGREGATED_INDEX_METRICS,
|
27
34
|
name_separator: nil)
|
28
35
|
@metric_prefix = metric_prefix || self.class.metric_prefix
|
29
36
|
@timestamp_format = timestamp_format || self.class.timestamp_format
|
30
37
|
@index_base_pattern = index_base_pattern || self.class.index_base_pattern
|
31
38
|
@index_base_replacement = index_base_replacement || self.class.index_base_replacement
|
39
|
+
@aggregated_index_metrics_only = aggregated_index_metrics_only || self.class.aggregated_index_metrics_only
|
40
|
+
@aggregated_index_metrics = aggregated_index_metrics || self.class.aggregated_index_metrics
|
32
41
|
@name_separator = name_separator || self.class.name_separator
|
33
42
|
end
|
34
43
|
|
@@ -45,17 +45,21 @@ module Fluent
|
|
45
45
|
DEFAULT_NODES_STATS_METRICS = nil
|
46
46
|
DEFAULT_INDICES_STATS = true
|
47
47
|
DEFAULT_INDICES_STATS_LEVEL = :indices
|
48
|
-
DEFAULT_INDICES =
|
48
|
+
DEFAULT_INDICES = [:_all]
|
49
|
+
DEFAULT_INDICES_STATS_METRICS = nil
|
49
50
|
DEFAULT_SHARDS_STATS = true
|
50
51
|
DEFAULT_DANGLING = false
|
51
52
|
DEFAULT_INDEX_BASE_PATTERN = nil # '/(.*)/'
|
52
53
|
DEFAULT_INDEX_BASE_REPLACEMENT = '\1'
|
53
54
|
DEFAULT_EVENT_NAME_SEPARATOR = '/'
|
54
55
|
DEFAULT_SKIP_SYSTEM_INDICES = true
|
56
|
+
DEFAULT_AGGREGATED_INDEX_METRICS_ONLY = false
|
57
|
+
DEFAULT_AGGREGATED_INDEX_METRICS = ['sum']
|
55
58
|
|
56
59
|
ALLOWED_CLUSTER_HEALTH_LEVELS = Fluent::Plugin::ElasticsearchStats::Client::ALLOWED_CLUSTER_HEALTH_LEVELS
|
57
60
|
ALLOWED_NODES_STATS_LEVELS = Fluent::Plugin::ElasticsearchStats::Client::ALLOWED_NODES_STATS_LEVELS
|
58
61
|
ALLOWED_INDICES_STATS_LEVELS = Fluent::Plugin::ElasticsearchStats::Client::ALLOWED_INDICES_LEVELS
|
62
|
+
ALLOWED_AGGREGATED_INDEX_METRICS = Fluent::Plugin::ElasticsearchStats::Metric::ALLOWED_AGGREGATED_INDEX_METRICS
|
59
63
|
|
60
64
|
desc 'tag to emit events on'
|
61
65
|
config_param :tag, :string, default: DEFAULT_TAG
|
@@ -110,6 +114,8 @@ module Fluent
|
|
110
114
|
config_param :indices, :array, value_type: :string, default: DEFAULT_INDICES
|
111
115
|
desc 'indices_stats details level'
|
112
116
|
config_param :indices_stats_level, :enum, list: ALLOWED_INDICES_STATS_LEVELS, default: DEFAULT_INDICES_STATS_LEVEL
|
117
|
+
desc 'lmits information to specific metrics for indices stats'
|
118
|
+
config_param :indices_stats_metrics, :array, value_type: :string, default: DEFAULT_INDICES_STATS_METRICS
|
113
119
|
|
114
120
|
desc 'collect dangling events'
|
115
121
|
config_param :dangling, :bool, default: DEFAULT_DANGLING
|
@@ -119,6 +125,11 @@ module Fluent
|
|
119
125
|
desc 'base index pattern replacement to generate aggregated index metrics'
|
120
126
|
config_param :index_base_replacement, :string, default: DEFAULT_INDEX_BASE_REPLACEMENT
|
121
127
|
|
128
|
+
desc 'generate only index aggregated metrics and discard index base metrics'
|
129
|
+
config_param :aggregated_index_metrics_only, :bool, default: DEFAULT_AGGREGATED_INDEX_METRICS_ONLY
|
130
|
+
desc 'list of aggregated index metrics to generate'
|
131
|
+
config_param :aggregated_index_metrics, :array, value_type: :string, default: DEFAULT_AGGREGATED_INDEX_METRICS
|
132
|
+
|
122
133
|
# desc 'skip system indices'
|
123
134
|
# config_param :skip_system_indices, :bool, default: DEFAULT_SKIP_SYSTEM_INDICES
|
124
135
|
|
@@ -130,11 +141,16 @@ module Fluent
|
|
130
141
|
|
131
142
|
@mutex_emit = Mutex.new
|
132
143
|
|
144
|
+
wrong_fields = aggregated_index_metrics.select { |item| ! ALLOWED_AGGREGATED_INDEX_METRICS.include?(item) }
|
145
|
+
raise Fluent::ConfigError, "aggregated_index_metrics contains unexpected values: #{wrong_fields}" if wrong_fields.size > 0
|
146
|
+
|
133
147
|
ElasticsearchStats::Metadata.metadata_prefix = metadata_prefix
|
134
148
|
ElasticsearchStats::Metric.metric_prefix = metric_prefix
|
135
149
|
ElasticsearchStats::Metric.timestamp_format = timestamp_format
|
136
150
|
ElasticsearchStats::Metric.index_base_pattern = index_base_pattern
|
137
151
|
ElasticsearchStats::Metric.index_base_replacement = index_base_replacement
|
152
|
+
ElasticsearchStats::Metric.aggregated_index_metrics_only = aggregated_index_metrics_only
|
153
|
+
ElasticsearchStats::Metric.aggregated_index_metrics = aggregated_index_metrics
|
138
154
|
ElasticsearchStats::Metric.name_separator = event_name_separator
|
139
155
|
|
140
156
|
configure_elasticsearchs
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-elasticsearch-stats
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Thomas Tych
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2024-
|
11
|
+
date: 2024-10-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bump
|
@@ -226,7 +226,7 @@ files:
|
|
226
226
|
- lib/fluent/plugin/elasticsearch_stats/nodes_stats_data.rb
|
227
227
|
- lib/fluent/plugin/elasticsearch_stats/utils.rb
|
228
228
|
- lib/fluent/plugin/in_elasticsearch_stats.rb
|
229
|
-
homepage: https://gitlab.com/ttych/fluent-plugin-elasticsearch-
|
229
|
+
homepage: https://gitlab.com/ttych/fluent-plugin-elasticsearch-stats
|
230
230
|
licenses:
|
231
231
|
- Apache-2.0
|
232
232
|
metadata:
|
@@ -239,14 +239,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
239
239
|
requirements:
|
240
240
|
- - ">="
|
241
241
|
- !ruby/object:Gem::Version
|
242
|
-
version: 2.
|
242
|
+
version: 2.7.0
|
243
243
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
244
244
|
requirements:
|
245
245
|
- - ">="
|
246
246
|
- !ruby/object:Gem::Version
|
247
247
|
version: '0'
|
248
248
|
requirements: []
|
249
|
-
rubygems_version: 3.5.
|
249
|
+
rubygems_version: 3.5.18
|
250
250
|
signing_key:
|
251
251
|
specification_version: 4
|
252
252
|
summary: fluentd plugin to generate elasticsearch cluster stats events.
|