fluent-plugin-elasticsearch-stats 0.3.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b85d9b62807f0710d7c7b55242271b90deda0eb3aff6cbaba56e079cda20261a
-  data.tar.gz: 8c6c8ff1e8c784ad6ab377805be083aff4b27463404df765b77d578a934a19c6
+  metadata.gz: f4186ce2d0c16afc286ac07121d65e1169b184ea2b09fb35300dd30e360e936a
+  data.tar.gz: 07c3fd6f782a8c5a006bd9d901d6014a78b8df910954741c2ed6f350f7ed1fc9
 SHA512:
-  metadata.gz: d783330de75059628def68f20cdcd8f9ba2e3e9b8f99e2d1990a655be837440cf56b337b32ef5561bc8f0a27d21685456cb5bc29117ca23a3e327ab8b9f653ca
-  data.tar.gz: 69d6a1a09c83aa98754e04d9d162877d330496a8e5ea899a648b02ad698471380844e8cd10d251683f7a4626f274d4bed695784ab8f1d1a791964d4ea97f670d
+  metadata.gz: ab34eabaf609eba4eacd8b148656968b5fc175e571b6a4531ba1b52e93a654219c5d292297c93893ce2315b410e155a15949dfbd1a6dfa98039c76b7a4caf816
+  data.tar.gz: 661eedf6ac5f362437ccdd447e43ca09a93d439811b1fc9998971e36463c688520f5d89dd35e150e0a2e9b0a9ca34810ebc70e37498144cd5d0d1cf400e71f5b

data/Gemfile.lock

@@ -1,8 +1,9 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-elasticsearch-stats (0.3.0)
+    fluent-plugin-elasticsearch-stats (0.5.0)
       faraday (~> 2.9)
+      faraday-retry (~> 2.2, >= 2.2.1)
       fluentd (>= 0.14.10, < 2)
 
 GEM
@@ -46,6 +47,8 @@ GEM
       faraday-net_http (>= 2.0, < 3.2)
     faraday-net_http (3.1.0)
       net-http
+    faraday-retry (2.2.1)
+      faraday (~> 2.0)
     fluentd (1.16.3)
       bundler
       cool.io (>= 1.4.5, < 2.0.0)

data/fluent-plugin-elasticsearch-stats.gemspec

@@ -5,16 +5,19 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = 'fluent-plugin-elasticsearch-stats'
-  spec.version = '0.3.0'
+  spec.version = '0.5.0'
   spec.authors = ['Thomas Tych']
   spec.email   = ['thomas.tych@gmail.com']
 
   spec.summary       = 'fluentd plugin to generate elasticsearch cluster stats events.'
-  spec.homepage      = 'https://gitlab.com/ttych/fluent-plugin-elasticsearch-stats'
+  spec.homepage      = "https://gitlab.com/ttych/#{spec.name}"
   spec.license       = 'Apache-2.0'
 
   spec.required_ruby_version = '>= 2.7.0'
 
+  spec.metadata['homepage_uri'] = spec.homepage
+  spec.metadata['source_code_uri'] = spec.homepage
+  spec.metadata['documentation_uri'] = "http://www.rubydoc.info/gems/#{spec.name}/#{spec.version}"
   spec.metadata['rubygems_mfa_required'] = 'true'
 
   _, files = `git ls-files -z`.split("\x0").partition do |f|
@@ -36,5 +39,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'timecop', '~> 0.9.6'
 
   spec.add_runtime_dependency 'faraday', '~> 2.9'
+  spec.add_runtime_dependency 'faraday-retry', '~> 2.2', '>= 2.2.1'
   spec.add_runtime_dependency 'fluentd', ['>= 0.14.10', '< 2']
 end

data/lib/fluent/plugin/elasticsearch_stats/base_data.rb

@@ -85,8 +85,10 @@ module Fluent
               count = metric_values.size
               index_base_count = count if count > index_base_count
 
+              aggregated_metric_name = pluralize_metric_name(metric_name)
+
               if metric.aggregated_index_metrics.include?('count')
-                metrics << metric.format(name: [metric_name, 'count'],
+                metrics << metric.format(name: [aggregated_metric_name, 'count'],
                                          value: count,
                                          family: family,
                                          metadata: local_metadata)
@@ -94,7 +96,7 @@ module Fluent
 
               if metric.aggregated_index_metrics.include?('min')
                 min = metric_values.min
-                metrics << metric.format(name: [metric_name, 'min'],
+                metrics << metric.format(name: [aggregated_metric_name, 'min'],
                                          value: min,
                                          family: family,
                                          metadata: local_metadata)
@@ -102,7 +104,7 @@ module Fluent
 
               if metric.aggregated_index_metrics.include?('max')
                 max = metric_values.max
-                metrics << metric.format(name: [metric_name, 'max'],
+                metrics << metric.format(name: [aggregated_metric_name, 'max'],
                                          value: max,
                                          family: family,
                                          metadata: local_metadata)
@@ -110,7 +112,7 @@ module Fluent
 
               if metric.aggregated_index_metrics.include?('sum')
                 sum = metric_values.sum
-                metrics << metric.format(name: [metric_name, 'sum'],
+                metrics << metric.format(name: [aggregated_metric_name, 'sum'],
                                          value: sum,
                                          family: family,
                                          metadata: local_metadata)
@@ -118,14 +120,14 @@ module Fluent
 
               if metric.aggregated_index_metrics.include?('avg')
                 avg = sum / count.to_f
-                metrics << metric.format(name: [metric_name, 'avg'],
+                metrics << metric.format(name: [aggregated_metric_name, 'avg'],
                                          value: avg,
                                          family: family,
                                          metadata: local_metadata)
               end
             end
 
-            metrics << metric.format(name: %w[index count],
+            metrics << metric.format(name: %w[indices count],
                                      value: index_base_count,
                                      family: family,
                                      metadata: local_metadata)
@@ -141,6 +143,11 @@ module Fluent
         def indices
           data['indices']
         end
+
+        def pluralize_metric_name(metric_name)
+          metric_name.sub(/^index#{metric.name_separator}/,
+                          "indices#{metric.name_separator}")
+        end
       end
     end
   end

data/lib/fluent/plugin/elasticsearch_stats/client.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'faraday'
+require 'faraday/retry'
 
 module Fluent
   module Plugin
@@ -9,6 +10,9 @@ module Fluent
         class Error < StandardError
         end
 
+        RETRY_COUNT = 2
+        RETRY_DELAY = 5
+
         TIMEOUT = 10
         USER_AGENT = 'elasticsearch_stats'
 
@@ -16,7 +20,7 @@ module Fluent
         CLUSTER_HEALTH_LEVEL = 'cluster'
         NODES_STATS_LEVEL = 'cluster'
         INDICES_STATS_LEVEL = 'indices'
-        INDICES = [:_all]
+        INDICES = [:_all].freeze
 
         ALLOWED_CLUSTER_HEALTH_LEVELS = %i[cluster indices shards].freeze
         ALLOWED_NODES_STATS_LEVELS = %i[nodes indices shards].freeze
@@ -71,6 +75,11 @@ module Fluent
           @log = log
         end
 
+        def cluster_info
+          endpoint = '/'
+          get(endpoint)
+        end
+
         # https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-health.html
         def cluster_health(level: CLUSTER_HEALTH_LEVEL, local: LOCAL)
           endpoint = '/_cluster/health'
@@ -96,7 +105,7 @@ module Fluent
         # https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-stats.html
         def indices_stats(indices: INDICES, level: INDICES_STATS_LEVEL, metrics: nil)
           indices ||= INDICES
-          endpoint = "/_stats"
+          endpoint = '/_stats'
           endpoint = "/#{indices.join(',')}#{endpoint}" if !indices.nil? && !indices.empty?
           endpoint += "/#{metrics.join(',')}" if metrics&.any?
           params = { level: level }
@@ -141,6 +150,8 @@ module Fluent
             config.response :json
             config.response :raise_error
             config.response :logger, log, headers: false, bodies: false, log_level: :debug if log
+
+            config.request :retry, max: RETRY_COUNT, interval: RETRY_DELAY
             config.adapter :net_http
           end
         end

data/lib/fluent/plugin/elasticsearch_stats/collector.rb

@@ -4,6 +4,8 @@ module Fluent
   module Plugin
     module ElasticsearchStats
       class Collector
+        CLUSTER_INFO_TTL = 12 * 3600
+
         attr_reader :client, :stats_config, :log
 
         def initialize(client:,
@@ -32,7 +34,7 @@ module Fluent
               level: stats_config.cluster_health_level,
               local: stats_config.cluster_health_local
             )
-            ClusterHealthData.new(data).extract_metrics
+            ClusterHealthData.new(data, metadata: metadata).extract_metrics
           end
         end
 
@@ -41,7 +43,7 @@ module Fluent
 
           without_error(rescue_return: []) do
             data = client.cluster_stats
-            ClusterStatsData.new(data).extract_metrics
+            ClusterStatsData.new(data, metadata: metadata).extract_metrics
           end
         end
 
@@ -53,7 +55,7 @@ module Fluent
               level: stats_config.nodes_stats_level,
               metrics: stats_config.nodes_stats_metrics
             )
-            NodesStatsData.new(data).extract_metrics
+            NodesStatsData.new(data, metadata: metadata).extract_metrics
           end
         end
 
@@ -66,7 +68,7 @@ module Fluent
               level: stats_config.indices_stats_level,
               metrics: stats_config.indices_stats_metrics
             )
-            IndicesStatsData.new(data).extract_metrics
+            IndicesStatsData.new(data, metadata: metadata).extract_metrics
           end
         end
 
@@ -75,15 +77,28 @@ module Fluent
 
           without_error(rescue_return: []) do
             data = client.dangling
-            DanglingData.new(data).extract_metrics
+            DanglingData.new(data, metadata: metadata).extract_metrics
           end
         end
 
-        # FIXME: inject metadata !
-        #        cluster metadata / info ?
         def metadata
           Metadata.new
-                  .set(label: 'cluster_url', value: client.url)
+                  .set(label: 'cluster_name', value: cluster_info['cluster_name'])
+          # .set(label: 'cluster_url', value: client.url)
+        end
+
+        def cluster_info(ttl: CLUSTER_INFO_TTL)
+          cluster_info_prev = nil
+          if @cluster_info_timestamp && (Time.now - @cluster_info_timestamp > ttl)
+            cluster_info_prev = @cluster_info
+            @cluster_info = nil
+          end
+
+          without_error do
+            @cluster_info ||= client.cluster_info
+            @cluster_info_timestamp = Time.now
+          end
+          @cluster_info ||= cluster_info_prev
         end
 
         def without_error(error: StandardError, rescue_return: nil)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch-stats
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Thomas Tych
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-10-24 00:00:00.000000000 Z
+date: 2024-10-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bump
@@ -176,6 +176,26 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.9'
+- !ruby/object:Gem::Dependency
+  name: faraday-retry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.2.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.2.1
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
@@ -230,6 +250,9 @@ homepage: https://gitlab.com/ttych/fluent-plugin-elasticsearch-stats
 licenses:
 - Apache-2.0
 metadata:
+  homepage_uri: https://gitlab.com/ttych/fluent-plugin-elasticsearch-stats
+  source_code_uri: https://gitlab.com/ttych/fluent-plugin-elasticsearch-stats
+  documentation_uri: http://www.rubydoc.info/gems/fluent-plugin-elasticsearch-stats/0.5.0
   rubygems_mfa_required: 'true'
 post_install_message:
 rdoc_options: []