fluent-plugin-elasticsearch-stats 0.1.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 0dd73194ef81df48fccec9c76d2b3e7815fbd83736d5a545ad27c58d0a764e5e
4
- data.tar.gz: 806fbd6806d9b6c952e1c9807b528fedd64e27b612a41cd4ffe3b82c09cf9b82
3
+ metadata.gz: b85d9b62807f0710d7c7b55242271b90deda0eb3aff6cbaba56e079cda20261a
4
+ data.tar.gz: 8c6c8ff1e8c784ad6ab377805be083aff4b27463404df765b77d578a934a19c6
5
5
  SHA512:
6
- metadata.gz: d3d4f1ce520a02d3e85e1d1c58289556891321da7f27e6f7898dc65d828b99426d8b2a97867830d367cecffcb8fd01f555a6ded4bf725007382cf294dc48e0a3
7
- data.tar.gz: 6ad75d6fa0c3476bde72ccf8bd5e6526c3080464491c57e5742d8b9f222091fe4c4cf3b09bd873ea9a4180f9b4b2548e149152ad8c6445b533ff4e616a71455a
6
+ metadata.gz: d783330de75059628def68f20cdcd8f9ba2e3e9b8f99e2d1990a655be837440cf56b337b32ef5561bc8f0a27d21685456cb5bc29117ca23a3e327ab8b9f653ca
7
+ data.tar.gz: 69d6a1a09c83aa98754e04d9d162877d330496a8e5ea899a648b02ad698471380844e8cd10d251683f7a4626f274d4bed695784ab8f1d1a791964d4ea97f670d
data/.rubocop.yml CHANGED
@@ -2,7 +2,7 @@ require:
2
2
  - rubocop-rake
3
3
 
4
4
  AllCops:
5
- TargetRubyVersion: 2.4
5
+ TargetRubyVersion: 2.7
6
6
  NewCops: enable
7
7
 
8
8
  Gemspec/DevelopmentDependencies:
data/.ruby-version CHANGED
@@ -1 +1 @@
1
- 3.3.0
1
+ 3.3.5
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- fluent-plugin-elasticsearch-stats (0.1.0)
4
+ fluent-plugin-elasticsearch-stats (0.3.0)
5
5
  faraday (~> 2.9)
6
6
  fluentd (>= 0.14.10, < 2)
7
7
 
data/README.md CHANGED
@@ -34,26 +34,29 @@ Example of config
34
34
 
35
35
  #### global options
36
36
 
37
- | setting | type | default | description |
38
- |------------------------|--------------------|-----------------------------------|---------------------------------------------------------------------|
39
- | tag | | elasticsearch_stats | tag to emit events on |
40
- | urls | | ["http://localhost:9200"] | list of urls to poll |
41
- | timeout | second | 10 | timeout for each call |
42
- | username | | | username for basic authentication |
43
- | password | | | password for basic authentication |
44
- | user_agent | | fluent-plugin-elasticsearch-stats | user agent for http request |
45
- | ca_file | file | | CA cert file to use for request |
46
- | verify_ssl | true\|false | true | option to verify certificate/host |
47
- | | | | |
48
- | interval | second | 300 | interval for probe execution |
49
- | | | | |
50
- | metric_prefix | | | prefix for metric fields |
51
- | metadata_prefix | | | prefix for metadata fields |
52
- | timestamp_format | iso\|epochmillis | iso | event timestamp format |
53
- | event_name_separator | | / | event name separator |
54
- | | | | |
55
- | index_base_pattern | regexp | | base index pattern to generate aggregated index metrics |
56
- | index_base_replacement | regexp replacement | \1 | base index pattern replacement to generate aggregated index metrics |
37
+ | setting | type | default | description |
38
+ |-------------------------------|--------------------|-----------------------------------|---------------------------------------------------------------------|
39
+ | tag | | elasticsearch_stats | tag to emit events on |
40
+ | urls | | ["http://localhost:9200"] | list of urls to poll |
41
+ | timeout | second | 10 | timeout for each call |
42
+ | username | | | username for basic authentication |
43
+ | password | | | password for basic authentication |
44
+ | user_agent | | fluent-plugin-elasticsearch-stats | user agent for http request |
45
+ | ca_file | file | | CA cert file to use for request |
46
+ | verify_ssl | bool | true | option to verify certificate/host |
47
+ | | | | |
48
+ | interval | second | 300 | interval for probe execution |
49
+ | | | | |
50
+ | metric_prefix | | | prefix for metric fields |
51
+ | metadata_prefix | | | prefix for metadata fields |
52
+ | timestamp_format | iso\|epochmillis | iso | event timestamp format |
53
+ | event_name_separator | | / | event name separator |
54
+ | | | | |
55
+ | index_base_pattern | regexp | | base index pattern to generate aggregated index metrics |
56
+ | index_base_replacement | regexp replacement | \1 | base index pattern replacement to generate aggregated index metrics |
57
+ | | | | |
58
+ | aggregated_index_metrics_only | bool | false | for index matrics, only generate aggregated metrics |
59
+ | aggregated_index_metrics | array | [sum] | select in [avg, count, min, max, sum] |
57
60
 
58
61
  #### cluster health options
59
62
 
@@ -67,7 +70,7 @@ Events from **[/_cluster/health?level=indices](https://www.elastic.co/guide/en/e
67
70
 
68
71
  #### cluster stats options
69
72
 
70
- Events from **/_cluster/stats**.
73
+ Events from **[/_cluster/stats](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-stats.html)**.
71
74
 
72
75
  | setting | type | default | description |
73
76
  |--------------------------------|-------------|---------|--------------------------------------------|
@@ -75,7 +78,7 @@ Events from **/_cluster/stats**.
75
78
 
76
79
  #### nodes stats options
77
80
 
78
- Events from **/_nodes/stats**.
81
+ Events from **[/_nodes/stats](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-nodes-stats.html)**.
79
82
 
80
83
  | setting | type | default | description |
81
84
  |---------------------|----------------------------------------------------------------------|---------|-----------------------------------|
@@ -85,17 +88,18 @@ Events from **/_nodes/stats**.
85
88
 
86
89
  #### indices stats options
87
90
 
88
- Events from **/_all/_stats**.
91
+ Events from **[/_all/_stats](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-stats.html)**.
89
92
 
90
- | setting | type | default | description |
91
- |---------------------------------|--------------------------|----------|-------------------------------------|
92
- | indices_stats | true\|false | true | enable indices stats events collect |
93
- | indices_stats_level | cluster\|indices\|shards | indices | indices_stats details level |
94
- | indices | | ["_all"] | indices to collect stats on |
93
+ | setting | type | default | description |
94
+ |-----------------------|---------------------------|----------|-------------------------------------|
95
+ | indices_stats | true\|false | true | enable indices stats events collect |
96
+ | indices_stats_level | cluster\|indices\|shards | indices | indices_stats details level |
97
+ | indices | | ["_all"] | indices to collect stats on |
98
+ | indices_stats_metrics | completion,docs,store,... | [] | |
95
99
 
96
100
  #### dangling options
97
101
 
98
- Events from **/_dangling**.
102
+ Events from **[/_dangling](https://www.elastic.co/guide/en/elasticsearch/reference/current/dangling-indices-list.html)**.
99
103
 
100
104
  | setting | type | default | description |
101
105
  |----------|-------------|---------|--------------------------------|
@@ -5,15 +5,15 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
5
5
 
6
6
  Gem::Specification.new do |spec|
7
7
  spec.name = 'fluent-plugin-elasticsearch-stats'
8
- spec.version = '0.1.0'
8
+ spec.version = '0.3.0'
9
9
  spec.authors = ['Thomas Tych']
10
10
  spec.email = ['thomas.tych@gmail.com']
11
11
 
12
12
  spec.summary = 'fluentd plugin to generate elasticsearch cluster stats events.'
13
- spec.homepage = 'https://gitlab.com/ttych/fluent-plugin-elasticsearch-status'
13
+ spec.homepage = 'https://gitlab.com/ttych/fluent-plugin-elasticsearch-stats'
14
14
  spec.license = 'Apache-2.0'
15
15
 
16
- spec.required_ruby_version = '>= 2.4.0'
16
+ spec.required_ruby_version = '>= 2.7.0'
17
17
 
18
18
  spec.metadata['rubygems_mfa_required'] = 'true'
19
19
 
@@ -40,6 +40,8 @@ module Fluent
40
40
  base_metrics = extract_indices_base_metrics
41
41
  aggregated_metrics = extract_indices_aggregated_metrics(base_metrics)
42
42
 
43
+ return aggregated_metrics if metric.aggregated_index_metrics_only
44
+
43
45
  base_metrics + aggregated_metrics
44
46
  end
45
47
 
@@ -76,35 +78,57 @@ module Fluent
76
78
  .set(label: 'index', value: index_base)
77
79
  .set(label: 'aggregated', value: true)
78
80
 
81
+ index_base_count = 0
82
+
79
83
  index_base_metrics.each do |metric_name, metric_name_metrics|
80
84
  metric_values = metric_name_metrics.map { |a_metric| a_metric[metric.value_label] }
81
85
  count = metric_values.size
82
- min = metric_values.min
83
- max = metric_values.max
84
- sum = metric_values.sum
85
- avg = sum / count.to_f
86
-
87
- metrics << metric.format(name: [metric_name, 'count'],
88
- value: count,
89
- family: family,
90
- metadata: local_metadata)
91
- metrics << metric.format(name: [metric_name, 'min'],
92
- value: min,
93
- family: family,
94
- metadata: local_metadata)
95
- metrics << metric.format(name: [metric_name, 'max'],
96
- value: max,
97
- family: family,
98
- metadata: local_metadata)
99
- metrics << metric.format(name: [metric_name, 'sum'],
100
- value: sum,
101
- family: family,
102
- metadata: local_metadata)
103
- metrics << metric.format(name: [metric_name, 'avg'],
104
- value: avg,
105
- family: family,
106
- metadata: local_metadata)
86
+ index_base_count = count if count > index_base_count
87
+
88
+ if metric.aggregated_index_metrics.include?('count')
89
+ metrics << metric.format(name: [metric_name, 'count'],
90
+ value: count,
91
+ family: family,
92
+ metadata: local_metadata)
93
+ end
94
+
95
+ if metric.aggregated_index_metrics.include?('min')
96
+ min = metric_values.min
97
+ metrics << metric.format(name: [metric_name, 'min'],
98
+ value: min,
99
+ family: family,
100
+ metadata: local_metadata)
101
+ end
102
+
103
+ if metric.aggregated_index_metrics.include?('max')
104
+ max = metric_values.max
105
+ metrics << metric.format(name: [metric_name, 'max'],
106
+ value: max,
107
+ family: family,
108
+ metadata: local_metadata)
109
+ end
110
+
111
+ if metric.aggregated_index_metrics.include?('sum')
112
+ sum = metric_values.sum
113
+ metrics << metric.format(name: [metric_name, 'sum'],
114
+ value: sum,
115
+ family: family,
116
+ metadata: local_metadata)
117
+ end
118
+
119
+ if metric.aggregated_index_metrics.include?('avg')
120
+ avg = sum / count.to_f
121
+ metrics << metric.format(name: [metric_name, 'avg'],
122
+ value: avg,
123
+ family: family,
124
+ metadata: local_metadata)
125
+ end
107
126
  end
127
+
128
+ metrics << metric.format(name: %w[index count],
129
+ value: index_base_count,
130
+ family: family,
131
+ metadata: local_metadata)
108
132
  end
109
133
 
110
134
  metrics.compact
@@ -16,6 +16,7 @@ module Fluent
16
16
  CLUSTER_HEALTH_LEVEL = 'cluster'
17
17
  NODES_STATS_LEVEL = 'cluster'
18
18
  INDICES_STATS_LEVEL = 'indices'
19
+ INDICES = [:_all]
19
20
 
20
21
  ALLOWED_CLUSTER_HEALTH_LEVELS = %i[cluster indices shards].freeze
21
22
  ALLOWED_NODES_STATS_LEVELS = %i[nodes indices shards].freeze
@@ -93,13 +94,12 @@ module Fluent
93
94
  end
94
95
 
95
96
  # https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-stats.html
96
- def indices_stats(indices: [:_all], level: INDICES_STATS_LEVEL, metrics: nil)
97
+ def indices_stats(indices: INDICES, level: INDICES_STATS_LEVEL, metrics: nil)
98
+ indices ||= INDICES
97
99
  endpoint = "/_stats"
98
100
  endpoint = "/#{indices.join(',')}#{endpoint}" if !indices.nil? && !indices.empty?
99
101
  endpoint += "/#{metrics.join(',')}" if metrics&.any?
100
- params = {
101
- level: level
102
- }
102
+ params = { level: level }
103
103
  get(endpoint, params)
104
104
  end
105
105
 
@@ -63,7 +63,8 @@ module Fluent
63
63
  without_error(rescue_return: []) do
64
64
  data = client.indices_stats(
65
65
  indices: stats_config.indices,
66
- level: stats_config.indices_stats_level
66
+ level: stats_config.indices_stats_level,
67
+ metrics: stats_config.indices_stats_metrics
67
68
  )
68
69
  IndicesStatsData.new(data).extract_metrics
69
70
  end
@@ -14,7 +14,7 @@ module Fluent
14
14
 
15
15
  def generate_dangling_indices_count
16
16
  metrics = []
17
- metrics << metric.format(name: %w[dangling all count],
17
+ metrics << metric.format(name: %w[dangling_indices count],
18
18
  value: dangling_indices.size,
19
19
  family: family,
20
20
  metadata: metadata)
@@ -7,8 +7,12 @@ module Fluent
7
7
  DEFAULT_NAME_SEPARATOR = '/'
8
8
  DEFAULT_TIMESTAMP_FORMAT = :iso
9
9
 
10
+ ALLOWED_AGGREGATED_INDEX_METRICS = %w[count min max avg sum].freeze
11
+ DEFAULT_AGGREGATED_INDEX_METRICS = ['sum'].freeze
12
+
10
13
  class << self
11
- attr_accessor :metric_prefix, :index_base_pattern, :index_base_replacement
14
+ attr_accessor :metric_prefix, :index_base_pattern, :index_base_replacement,
15
+ :aggregated_index_metrics_only, :aggregated_index_metrics
12
16
 
13
17
  def name_separator
14
18
  @name_separator ||= DEFAULT_NAME_SEPARATOR
@@ -21,14 +25,19 @@ module Fluent
21
25
  end
22
26
  end
23
27
 
24
- attr_reader :metric_prefix, :timestamp_format, :index_base_pattern, :index_base_replacement, :name_separator
28
+ attr_reader :metric_prefix, :timestamp_format, :index_base_pattern, :index_base_replacement,
29
+ :aggregated_index_metrics_only, :aggregated_index_metrics,
30
+ :name_separator
25
31
 
26
32
  def initialize(metric_prefix: nil, timestamp_format: nil, index_base_pattern: nil, index_base_replacement: nil,
33
+ aggregated_index_metrics_only: nil, aggregated_index_metrics: DEFAULT_AGGREGATED_INDEX_METRICS,
27
34
  name_separator: nil)
28
35
  @metric_prefix = metric_prefix || self.class.metric_prefix
29
36
  @timestamp_format = timestamp_format || self.class.timestamp_format
30
37
  @index_base_pattern = index_base_pattern || self.class.index_base_pattern
31
38
  @index_base_replacement = index_base_replacement || self.class.index_base_replacement
39
+ @aggregated_index_metrics_only = aggregated_index_metrics_only || self.class.aggregated_index_metrics_only
40
+ @aggregated_index_metrics = aggregated_index_metrics || self.class.aggregated_index_metrics
32
41
  @name_separator = name_separator || self.class.name_separator
33
42
  end
34
43
 
@@ -45,17 +45,21 @@ module Fluent
45
45
  DEFAULT_NODES_STATS_METRICS = nil
46
46
  DEFAULT_INDICES_STATS = true
47
47
  DEFAULT_INDICES_STATS_LEVEL = :indices
48
- DEFAULT_INDICES = nil
48
+ DEFAULT_INDICES = [:_all]
49
+ DEFAULT_INDICES_STATS_METRICS = nil
49
50
  DEFAULT_SHARDS_STATS = true
50
51
  DEFAULT_DANGLING = false
51
52
  DEFAULT_INDEX_BASE_PATTERN = nil # '/(.*)/'
52
53
  DEFAULT_INDEX_BASE_REPLACEMENT = '\1'
53
54
  DEFAULT_EVENT_NAME_SEPARATOR = '/'
54
55
  DEFAULT_SKIP_SYSTEM_INDICES = true
56
+ DEFAULT_AGGREGATED_INDEX_METRICS_ONLY = false
57
+ DEFAULT_AGGREGATED_INDEX_METRICS = ['sum']
55
58
 
56
59
  ALLOWED_CLUSTER_HEALTH_LEVELS = Fluent::Plugin::ElasticsearchStats::Client::ALLOWED_CLUSTER_HEALTH_LEVELS
57
60
  ALLOWED_NODES_STATS_LEVELS = Fluent::Plugin::ElasticsearchStats::Client::ALLOWED_NODES_STATS_LEVELS
58
61
  ALLOWED_INDICES_STATS_LEVELS = Fluent::Plugin::ElasticsearchStats::Client::ALLOWED_INDICES_LEVELS
62
+ ALLOWED_AGGREGATED_INDEX_METRICS = Fluent::Plugin::ElasticsearchStats::Metric::ALLOWED_AGGREGATED_INDEX_METRICS
59
63
 
60
64
  desc 'tag to emit events on'
61
65
  config_param :tag, :string, default: DEFAULT_TAG
@@ -110,6 +114,8 @@ module Fluent
110
114
  config_param :indices, :array, value_type: :string, default: DEFAULT_INDICES
111
115
  desc 'indices_stats details level'
112
116
  config_param :indices_stats_level, :enum, list: ALLOWED_INDICES_STATS_LEVELS, default: DEFAULT_INDICES_STATS_LEVEL
117
+ desc 'lmits information to specific metrics for indices stats'
118
+ config_param :indices_stats_metrics, :array, value_type: :string, default: DEFAULT_INDICES_STATS_METRICS
113
119
 
114
120
  desc 'collect dangling events'
115
121
  config_param :dangling, :bool, default: DEFAULT_DANGLING
@@ -119,6 +125,11 @@ module Fluent
119
125
  desc 'base index pattern replacement to generate aggregated index metrics'
120
126
  config_param :index_base_replacement, :string, default: DEFAULT_INDEX_BASE_REPLACEMENT
121
127
 
128
+ desc 'generate only index aggregated metrics and discard index base metrics'
129
+ config_param :aggregated_index_metrics_only, :bool, default: DEFAULT_AGGREGATED_INDEX_METRICS_ONLY
130
+ desc 'list of aggregated index metrics to generate'
131
+ config_param :aggregated_index_metrics, :array, value_type: :string, default: DEFAULT_AGGREGATED_INDEX_METRICS
132
+
122
133
  # desc 'skip system indices'
123
134
  # config_param :skip_system_indices, :bool, default: DEFAULT_SKIP_SYSTEM_INDICES
124
135
 
@@ -130,11 +141,16 @@ module Fluent
130
141
 
131
142
  @mutex_emit = Mutex.new
132
143
 
144
+ wrong_fields = aggregated_index_metrics.select { |item| ! ALLOWED_AGGREGATED_INDEX_METRICS.include?(item) }
145
+ raise Fluent::ConfigError, "aggregated_index_metrics contains unexpected values: #{wrong_fields}" if wrong_fields.size > 0
146
+
133
147
  ElasticsearchStats::Metadata.metadata_prefix = metadata_prefix
134
148
  ElasticsearchStats::Metric.metric_prefix = metric_prefix
135
149
  ElasticsearchStats::Metric.timestamp_format = timestamp_format
136
150
  ElasticsearchStats::Metric.index_base_pattern = index_base_pattern
137
151
  ElasticsearchStats::Metric.index_base_replacement = index_base_replacement
152
+ ElasticsearchStats::Metric.aggregated_index_metrics_only = aggregated_index_metrics_only
153
+ ElasticsearchStats::Metric.aggregated_index_metrics = aggregated_index_metrics
138
154
  ElasticsearchStats::Metric.name_separator = event_name_separator
139
155
 
140
156
  configure_elasticsearchs
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-elasticsearch-stats
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0
4
+ version: 0.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Thomas Tych
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2024-03-14 00:00:00.000000000 Z
11
+ date: 2024-10-24 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bump
@@ -226,7 +226,7 @@ files:
226
226
  - lib/fluent/plugin/elasticsearch_stats/nodes_stats_data.rb
227
227
  - lib/fluent/plugin/elasticsearch_stats/utils.rb
228
228
  - lib/fluent/plugin/in_elasticsearch_stats.rb
229
- homepage: https://gitlab.com/ttych/fluent-plugin-elasticsearch-status
229
+ homepage: https://gitlab.com/ttych/fluent-plugin-elasticsearch-stats
230
230
  licenses:
231
231
  - Apache-2.0
232
232
  metadata:
@@ -239,14 +239,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
239
239
  requirements:
240
240
  - - ">="
241
241
  - !ruby/object:Gem::Version
242
- version: 2.4.0
242
+ version: 2.7.0
243
243
  required_rubygems_version: !ruby/object:Gem::Requirement
244
244
  requirements:
245
245
  - - ">="
246
246
  - !ruby/object:Gem::Version
247
247
  version: '0'
248
248
  requirements: []
249
- rubygems_version: 3.5.6
249
+ rubygems_version: 3.5.18
250
250
  signing_key:
251
251
  specification_version: 4
252
252
  summary: fluentd plugin to generate elasticsearch cluster stats events.