fluent-plugin-elasticsearch-stats 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (10) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile.lock +1 -1
  3. data/README.md +11 -9
  4. data/fluent-plugin-elasticsearch-stats.gemspec +1 -1
  5. data/lib/fluent/plugin/elasticsearch_stats/base_data.rb +48 -25
  6. data/lib/fluent/plugin/elasticsearch_stats/client.rb +4 -4
  7. data/lib/fluent/plugin/elasticsearch_stats/collector.rb +2 -1
  8. data/lib/fluent/plugin/elasticsearch_stats/metric.rb +10 -3
  9. data/lib/fluent/plugin/in_elasticsearch_stats.rb +13 -2
  10. metadata +2 -2
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: d776ffcbc115b404d3da96d04215181f1983fcf9c5c731841082c0790f44201d
4
- data.tar.gz: 1c4f4245168f90a8ba7c3880737d34aeb42d2b270d3bf1956c3eb91c7a427415
3
+ metadata.gz: b85d9b62807f0710d7c7b55242271b90deda0eb3aff6cbaba56e079cda20261a
4
+ data.tar.gz: 8c6c8ff1e8c784ad6ab377805be083aff4b27463404df765b77d578a934a19c6
5
5
  SHA512:
6
- metadata.gz: d78dd62a39fcbb39122cedf10e25a39a37a6c0af023c34c7e08015fdc036c3b10c6d8331ef5c7a6c01de7f7ff1d6adb5901a1d4fce4258d01442ad31ac73460e
7
- data.tar.gz: 2841073645e617520f71e4d71f9382d98ed5ee09338681d9b69f7943761b2039068292c28d949cc88e08d8e03c40eb4624915a49205a207ef5a0308d1331710b
6
+ metadata.gz: d783330de75059628def68f20cdcd8f9ba2e3e9b8f99e2d1990a655be837440cf56b337b32ef5561bc8f0a27d21685456cb5bc29117ca23a3e327ab8b9f653ca
7
+ data.tar.gz: 69d6a1a09c83aa98754e04d9d162877d330496a8e5ea899a648b02ad698471380844e8cd10d251683f7a4626f274d4bed695784ab8f1d1a791964d4ea97f670d
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- fluent-plugin-elasticsearch-stats (0.2.0)
4
+ fluent-plugin-elasticsearch-stats (0.3.0)
5
5
  faraday (~> 2.9)
6
6
  fluentd (>= 0.14.10, < 2)
7
7
 
data/README.md CHANGED
@@ -56,6 +56,7 @@ Example of config
56
56
  | index_base_replacement | regexp replacement | \1 | base index pattern replacement to generate aggregated index metrics |
57
57
  | | | | |
58
58
  | aggregated_index_metrics_only | bool | false | for index matrics, only generate aggregated metrics |
59
+ | aggregated_index_metrics | array | [sum] | select in [avg, count, min, max, sum] |
59
60
 
60
61
  #### cluster health options
61
62
 
@@ -69,7 +70,7 @@ Events from **[/_cluster/health?level=indices](https://www.elastic.co/guide/en/e
69
70
 
70
71
  #### cluster stats options
71
72
 
72
- Events from **/_cluster/stats**.
73
+ Events from **[/_cluster/stats](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-stats.html)**.
73
74
 
74
75
  | setting | type | default | description |
75
76
  |--------------------------------|-------------|---------|--------------------------------------------|
@@ -77,7 +78,7 @@ Events from **/_cluster/stats**.
77
78
 
78
79
  #### nodes stats options
79
80
 
80
- Events from **/_nodes/stats**.
81
+ Events from **[/_nodes/stats](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-nodes-stats.html)**.
81
82
 
82
83
  | setting | type | default | description |
83
84
  |---------------------|----------------------------------------------------------------------|---------|-----------------------------------|
@@ -87,17 +88,18 @@ Events from **/_nodes/stats**.
87
88
 
88
89
  #### indices stats options
89
90
 
90
- Events from **/_all/_stats**.
91
+ Events from **[/_all/_stats](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-stats.html)**.
91
92
 
92
- | setting | type | default | description |
93
- |---------------------------------|--------------------------|----------|-------------------------------------|
94
- | indices_stats | true\|false | true | enable indices stats events collect |
95
- | indices_stats_level | cluster\|indices\|shards | indices | indices_stats details level |
96
- | indices | | ["_all"] | indices to collect stats on |
93
+ | setting | type | default | description |
94
+ |-----------------------|---------------------------|----------|-------------------------------------|
95
+ | indices_stats | true\|false | true | enable indices stats events collect |
96
+ | indices_stats_level | cluster\|indices\|shards | indices | indices_stats details level |
97
+ | indices | | ["_all"] | indices to collect stats on |
98
+ | indices_stats_metrics | completion,docs,store,... | [] | |
97
99
 
98
100
  #### dangling options
99
101
 
100
- Events from **/_dangling**.
102
+ Events from **[/_dangling](https://www.elastic.co/guide/en/elasticsearch/reference/current/dangling-indices-list.html)**.
101
103
 
102
104
  | setting | type | default | description |
103
105
  |----------|-------------|---------|--------------------------------|
data/fluent-plugin-elasticsearch-stats.gemspec CHANGED
@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
5
5
 
6
6
  Gem::Specification.new do |spec|
7
7
  spec.name = 'fluent-plugin-elasticsearch-stats'
8
- spec.version = '0.2.0'
8
+ spec.version = '0.3.0'
9
9
  spec.authors = ['Thomas Tych']
10
10
  spec.email = ['thomas.tych@gmail.com']
11
11
 
data/lib/fluent/plugin/elasticsearch_stats/base_data.rb CHANGED
@@ -41,6 +41,7 @@ module Fluent
41
41
  aggregated_metrics = extract_indices_aggregated_metrics(base_metrics)
42
42
 
43
43
  return aggregated_metrics if metric.aggregated_index_metrics_only
44
+
44
45
  base_metrics + aggregated_metrics
45
46
  end
46
47
 
@@ -77,35 +78,57 @@ module Fluent
77
78
  .set(label: 'index', value: index_base)
78
79
  .set(label: 'aggregated', value: true)
79
80
 
81
+ index_base_count = 0
82
+
80
83
  index_base_metrics.each do |metric_name, metric_name_metrics|
81
84
  metric_values = metric_name_metrics.map { |a_metric| a_metric[metric.value_label] }
82
85
  count = metric_values.size
83
- min = metric_values.min
84
- max = metric_values.max
85
- sum = metric_values.sum
86
- avg = sum / count.to_f
87
-
88
- metrics << metric.format(name: [metric_name, 'count'],
89
- value: count,
90
- family: family,
91
- metadata: local_metadata)
92
- metrics << metric.format(name: [metric_name, 'min'],
93
- value: min,
94
- family: family,
95
- metadata: local_metadata)
96
- metrics << metric.format(name: [metric_name, 'max'],
97
- value: max,
98
- family: family,
99
- metadata: local_metadata)
100
- metrics << metric.format(name: [metric_name, 'sum'],
101
- value: sum,
102
- family: family,
103
- metadata: local_metadata)
104
- metrics << metric.format(name: [metric_name, 'avg'],
105
- value: avg,
106
- family: family,
107
- metadata: local_metadata)
86
+ index_base_count = count if count > index_base_count
87
+
88
+ if metric.aggregated_index_metrics.include?('count')
89
+ metrics << metric.format(name: [metric_name, 'count'],
90
+ value: count,
91
+ family: family,
92
+ metadata: local_metadata)
93
+ end
94
+
95
+ if metric.aggregated_index_metrics.include?('min')
96
+ min = metric_values.min
97
+ metrics << metric.format(name: [metric_name, 'min'],
98
+ value: min,
99
+ family: family,
100
+ metadata: local_metadata)
101
+ end
102
+
103
+ if metric.aggregated_index_metrics.include?('max')
104
+ max = metric_values.max
105
+ metrics << metric.format(name: [metric_name, 'max'],
106
+ value: max,
107
+ family: family,
108
+ metadata: local_metadata)
109
+ end
110
+
111
+ if metric.aggregated_index_metrics.include?('sum')
112
+ sum = metric_values.sum
113
+ metrics << metric.format(name: [metric_name, 'sum'],
114
+ value: sum,
115
+ family: family,
116
+ metadata: local_metadata)
117
+ end
118
+
119
+ if metric.aggregated_index_metrics.include?('avg')
120
+ avg = sum / count.to_f
121
+ metrics << metric.format(name: [metric_name, 'avg'],
122
+ value: avg,
123
+ family: family,
124
+ metadata: local_metadata)
125
+ end
108
126
  end
127
+
128
+ metrics << metric.format(name: %w[index count],
129
+ value: index_base_count,
130
+ family: family,
131
+ metadata: local_metadata)
109
132
  end
110
133
 
111
134
  metrics.compact
data/lib/fluent/plugin/elasticsearch_stats/client.rb CHANGED
@@ -16,6 +16,7 @@ module Fluent
16
16
  CLUSTER_HEALTH_LEVEL = 'cluster'
17
17
  NODES_STATS_LEVEL = 'cluster'
18
18
  INDICES_STATS_LEVEL = 'indices'
19
+ INDICES = [:_all]
19
20
 
20
21
  ALLOWED_CLUSTER_HEALTH_LEVELS = %i[cluster indices shards].freeze
21
22
  ALLOWED_NODES_STATS_LEVELS = %i[nodes indices shards].freeze
@@ -93,13 +94,12 @@ module Fluent
93
94
  end
94
95
 
95
96
  # https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-stats.html
96
- def indices_stats(indices: [:_all], level: INDICES_STATS_LEVEL, metrics: nil)
97
+ def indices_stats(indices: INDICES, level: INDICES_STATS_LEVEL, metrics: nil)
98
+ indices ||= INDICES
97
99
  endpoint = "/_stats"
98
100
  endpoint = "/#{indices.join(',')}#{endpoint}" if !indices.nil? && !indices.empty?
99
101
  endpoint += "/#{metrics.join(',')}" if metrics&.any?
100
- params = {
101
- level: level
102
- }
102
+ params = { level: level }
103
103
  get(endpoint, params)
104
104
  end
105
105
 
data/lib/fluent/plugin/elasticsearch_stats/collector.rb CHANGED
@@ -63,7 +63,8 @@ module Fluent
63
63
  without_error(rescue_return: []) do
64
64
  data = client.indices_stats(
65
65
  indices: stats_config.indices,
66
- level: stats_config.indices_stats_level
66
+ level: stats_config.indices_stats_level,
67
+ metrics: stats_config.indices_stats_metrics
67
68
  )
68
69
  IndicesStatsData.new(data).extract_metrics
69
70
  end
data/lib/fluent/plugin/elasticsearch_stats/metric.rb CHANGED
@@ -7,8 +7,12 @@ module Fluent
7
7
  DEFAULT_NAME_SEPARATOR = '/'
8
8
  DEFAULT_TIMESTAMP_FORMAT = :iso
9
9
 
10
+ ALLOWED_AGGREGATED_INDEX_METRICS = %w[count min max avg sum].freeze
11
+ DEFAULT_AGGREGATED_INDEX_METRICS = ['sum'].freeze
12
+
10
13
  class << self
11
- attr_accessor :metric_prefix, :index_base_pattern, :index_base_replacement, :aggregated_index_metrics_only
14
+ attr_accessor :metric_prefix, :index_base_pattern, :index_base_replacement,
15
+ :aggregated_index_metrics_only, :aggregated_index_metrics
12
16
 
13
17
  def name_separator
14
18
  @name_separator ||= DEFAULT_NAME_SEPARATOR
@@ -22,15 +26,18 @@ module Fluent
22
26
  end
23
27
 
24
28
  attr_reader :metric_prefix, :timestamp_format, :index_base_pattern, :index_base_replacement,
25
- :aggregated_index_metrics_only, :name_separator
29
+ :aggregated_index_metrics_only, :aggregated_index_metrics,
30
+ :name_separator
26
31
 
27
- def initialize(metric_prefix: nil, timestamp_format: nil, index_base_pattern: nil, index_base_replacement: nil, aggregated_index_metrics_only: nil,
32
+ def initialize(metric_prefix: nil, timestamp_format: nil, index_base_pattern: nil, index_base_replacement: nil,
33
+ aggregated_index_metrics_only: nil, aggregated_index_metrics: DEFAULT_AGGREGATED_INDEX_METRICS,
28
34
  name_separator: nil)
29
35
  @metric_prefix = metric_prefix || self.class.metric_prefix
30
36
  @timestamp_format = timestamp_format || self.class.timestamp_format
31
37
  @index_base_pattern = index_base_pattern || self.class.index_base_pattern
32
38
  @index_base_replacement = index_base_replacement || self.class.index_base_replacement
33
39
  @aggregated_index_metrics_only = aggregated_index_metrics_only || self.class.aggregated_index_metrics_only
40
+ @aggregated_index_metrics = aggregated_index_metrics || self.class.aggregated_index_metrics
34
41
  @name_separator = name_separator || self.class.name_separator
35
42
  end
36
43
 
data/lib/fluent/plugin/in_elasticsearch_stats.rb CHANGED
@@ -45,7 +45,8 @@ module Fluent
45
45
  DEFAULT_NODES_STATS_METRICS = nil
46
46
  DEFAULT_INDICES_STATS = true
47
47
  DEFAULT_INDICES_STATS_LEVEL = :indices
48
- DEFAULT_INDICES = nil
48
+ DEFAULT_INDICES = [:_all]
49
+ DEFAULT_INDICES_STATS_METRICS = nil
49
50
  DEFAULT_SHARDS_STATS = true
50
51
  DEFAULT_DANGLING = false
51
52
  DEFAULT_INDEX_BASE_PATTERN = nil # '/(.*)/'
@@ -53,10 +54,12 @@ module Fluent
53
54
  DEFAULT_EVENT_NAME_SEPARATOR = '/'
54
55
  DEFAULT_SKIP_SYSTEM_INDICES = true
55
56
  DEFAULT_AGGREGATED_INDEX_METRICS_ONLY = false
57
+ DEFAULT_AGGREGATED_INDEX_METRICS = ['sum']
56
58
 
57
59
  ALLOWED_CLUSTER_HEALTH_LEVELS = Fluent::Plugin::ElasticsearchStats::Client::ALLOWED_CLUSTER_HEALTH_LEVELS
58
60
  ALLOWED_NODES_STATS_LEVELS = Fluent::Plugin::ElasticsearchStats::Client::ALLOWED_NODES_STATS_LEVELS
59
61
  ALLOWED_INDICES_STATS_LEVELS = Fluent::Plugin::ElasticsearchStats::Client::ALLOWED_INDICES_LEVELS
62
+ ALLOWED_AGGREGATED_INDEX_METRICS = Fluent::Plugin::ElasticsearchStats::Metric::ALLOWED_AGGREGATED_INDEX_METRICS
60
63
 
61
64
  desc 'tag to emit events on'
62
65
  config_param :tag, :string, default: DEFAULT_TAG
@@ -111,6 +114,8 @@ module Fluent
111
114
  config_param :indices, :array, value_type: :string, default: DEFAULT_INDICES
112
115
  desc 'indices_stats details level'
113
116
  config_param :indices_stats_level, :enum, list: ALLOWED_INDICES_STATS_LEVELS, default: DEFAULT_INDICES_STATS_LEVEL
117
+ desc 'lmits information to specific metrics for indices stats'
118
+ config_param :indices_stats_metrics, :array, value_type: :string, default: DEFAULT_INDICES_STATS_METRICS
114
119
 
115
120
  desc 'collect dangling events'
116
121
  config_param :dangling, :bool, default: DEFAULT_DANGLING
@@ -120,8 +125,10 @@ module Fluent
120
125
  desc 'base index pattern replacement to generate aggregated index metrics'
121
126
  config_param :index_base_replacement, :string, default: DEFAULT_INDEX_BASE_REPLACEMENT
122
127
 
123
- desc 'base index pattern to generate aggregated index metrics'
128
+ desc 'generate only index aggregated metrics and discard index base metrics'
124
129
  config_param :aggregated_index_metrics_only, :bool, default: DEFAULT_AGGREGATED_INDEX_METRICS_ONLY
130
+ desc 'list of aggregated index metrics to generate'
131
+ config_param :aggregated_index_metrics, :array, value_type: :string, default: DEFAULT_AGGREGATED_INDEX_METRICS
125
132
 
126
133
  # desc 'skip system indices'
127
134
  # config_param :skip_system_indices, :bool, default: DEFAULT_SKIP_SYSTEM_INDICES
@@ -134,12 +141,16 @@ module Fluent
134
141
 
135
142
  @mutex_emit = Mutex.new
136
143
 
144
+ wrong_fields = aggregated_index_metrics.select { |item| ! ALLOWED_AGGREGATED_INDEX_METRICS.include?(item) }
145
+ raise Fluent::ConfigError, "aggregated_index_metrics contains unexpected values: #{wrong_fields}" if wrong_fields.size > 0
146
+
137
147
  ElasticsearchStats::Metadata.metadata_prefix = metadata_prefix
138
148
  ElasticsearchStats::Metric.metric_prefix = metric_prefix
139
149
  ElasticsearchStats::Metric.timestamp_format = timestamp_format
140
150
  ElasticsearchStats::Metric.index_base_pattern = index_base_pattern
141
151
  ElasticsearchStats::Metric.index_base_replacement = index_base_replacement
142
152
  ElasticsearchStats::Metric.aggregated_index_metrics_only = aggregated_index_metrics_only
153
+ ElasticsearchStats::Metric.aggregated_index_metrics = aggregated_index_metrics
143
154
  ElasticsearchStats::Metric.name_separator = event_name_separator
144
155
 
145
156
  configure_elasticsearchs
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-elasticsearch-stats
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.0
4
+ version: 0.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Thomas Tych
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2024-10-23 00:00:00.000000000 Z
11
+ date: 2024-10-24 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bump