fluent-plugin-elasticsearch-stats 0.1.0

data/lib/fluent/plugin/elasticsearch_stats/base_data.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+require_relative 'metadata'
+require_relative 'metric'
+
+module Fluent
+  module Plugin
+    module ElasticsearchStats
+      class BaseData
+        attr_reader :data
+
+        # FIXME
+        #   skip_system_indices
+        def initialize(data, family: self.class::NAME, metadata: Metadata.new, metric: Metric.new)
+          @data = data
+          @family = family
+          @metadata = metadata
+          @metric = metric
+
+          initialize_metadata
+        end
+
+        def extract_metrics
+          []
+        end
+
+        private
+
+        attr_reader :family, :metadata, :metric
+
+        def initialize_metadata
+          metadata.set(label: 'cluster_name', value: cluster_name) if cluster_name
+        end
+
+        def cluster_name
+          data['cluster_name']
+        end
+
+        def extract_indices_metrics
+          base_metrics = extract_indices_base_metrics
+          aggregated_metrics = extract_indices_aggregated_metrics(base_metrics)
+
+          base_metrics + aggregated_metrics
+        end
+
+        def extract_indices_base_metrics
+          return [] if !indices || indices.empty?
+
+          metrics = []
+          indices.each do |index_name, index_stats|
+            local_metadata = metadata.dup.set(label: 'index', value: index_name)
+            flattened_stats = Utils.hash_flatten_keys(index_stats, separator: metric.name_separator)
+            flattened_stats.each do |k, v|
+              metrics << metric.format(name: ['index', k], value: v, family: family, metadata: local_metadata)
+            end
+          end
+          metrics.compact
+        end
+
+        def extract_indices_aggregated_metrics(base_metrics)
+          metrics = []
+
+          grouped_metrics = {}
+          base_metrics.each do |base_metric|
+            index_base = base_metric[metadata.label_for('index_base')]
+            next unless index_base
+
+            base_metric_name = base_metric[metric.name_label]
+            grouped_metrics[index_base] ||= {}
+            grouped_metrics[index_base][base_metric_name] ||= []
+            grouped_metrics[index_base][base_metric_name] << base_metric
+          end
+
+          grouped_metrics.each do |index_base, index_base_metrics|
+            local_metadata = metadata.dup
+                                     .set(label: 'index', value: index_base)
+                                     .set(label: 'aggregated', value: true)
+
+            index_base_metrics.each do |metric_name, metric_name_metrics|
+              metric_values = metric_name_metrics.map { |a_metric| a_metric[metric.value_label] }
+              count = metric_values.size
+              min = metric_values.min
+              max = metric_values.max
+              sum = metric_values.sum
+              avg = sum / count.to_f
+
+              metrics <<  metric.format(name: [metric_name, 'count'],
+                                        value: count,
+                                        family: family,
+                                        metadata: local_metadata)
+              metrics <<  metric.format(name: [metric_name, 'min'],
+                                        value: min,
+                                        family: family,
+                                        metadata: local_metadata)
+              metrics <<  metric.format(name: [metric_name, 'max'],
+                                        value: max,
+                                        family: family,
+                                        metadata: local_metadata)
+              metrics <<  metric.format(name: [metric_name, 'sum'],
+                                        value: sum,
+                                        family: family,
+                                        metadata: local_metadata)
+              metrics <<  metric.format(name: [metric_name, 'avg'],
+                                        value: avg,
+                                        family: family,
+                                        metadata: local_metadata)
+            end
+          end
+
+          metrics.compact
+        end
+
+        def status
+          data['status']
+        end
+
+        def indices
+          data['indices']
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/elasticsearch_stats/client.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+require 'faraday'
+
+module Fluent
+  module Plugin
+    module ElasticsearchStats
+      class Client
+        class Error < StandardError
+        end
+
+        TIMEOUT = 10
+        USER_AGENT = 'elasticsearch_stats'
+
+        LOCAL = false
+        CLUSTER_HEALTH_LEVEL = 'cluster'
+        NODES_STATS_LEVEL = 'cluster'
+        INDICES_STATS_LEVEL = 'indices'
+
+        ALLOWED_CLUSTER_HEALTH_LEVELS = %i[cluster indices shards].freeze
+        ALLOWED_NODES_STATS_LEVELS = %i[nodes indices shards].freeze
+        ALLOWED_NODES_STATS_METRICS = %i[
+          adaptive_selection
+          breaker
+          discovery
+          fs
+          http
+          indexing_pressure
+          indices
+          ingest
+          jvm
+          os
+          process
+          repositories
+          thread_pool
+          transport
+        ].freeze
+        ALLOWED_INDICES_LEVELS = %i[cluster indices shards].freeze
+        ALLOWED_INDICES_METRICS = %i[
+          _all
+          completion
+          docs
+          fielddata
+          flush
+          get
+          indexing
+          merge
+          query_cache
+          refresh
+          request_cache
+          search
+          segments
+          store
+          translog
+        ].freeze
+
+        attr_reader :url, :timeout, :username, :password, :user_agent,
+                    :ca_file, :verify_ssl, :log, :client
+
+        def initialize(url:, timeout: TIMEOUT, username: nil, password: nil,
+                       user_agent: USER_AGENT, ca_file: nil, verify_ssl: true,
+                       log: nil)
+          @url = url
+          @timeout = timeout
+          @username = username
+          @password = password
+          @user_agent = user_agent
+          @ca_file = ca_file
+          @verify_ssl = verify_ssl
+          @log = log
+        end
+
+        # https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-health.html
+        def cluster_health(level: CLUSTER_HEALTH_LEVEL, local: LOCAL)
+          endpoint = '/_cluster/health'
+          params = { level: level, local: local, timeout: "#{timeout}s" }
+          get(endpoint, params)
+        end
+
+        # https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-stats.html
+        def cluster_stats
+          endpoint = '/_cluster/stats'
+          params = { timeout: "#{timeout}s" }
+          get(endpoint, params)
+        end
+
+        # https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-nodes-stats.html
+        def nodes_stats(level: NODES_STATS_LEVEL, metrics: nil)
+          endpoint = '/_nodes/stats'
+          endpoint += "/#{metrics.join(',')}" if metrics&.any?
+          params = { level: level, timeout: "#{timeout}s" }
+          get(endpoint, params)
+        end
+
+        # https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-stats.html
+        def indices_stats(indices: [:_all], level: INDICES_STATS_LEVEL, metrics: nil)
+          endpoint = "/_stats"
+          endpoint = "/#{indices.join(',')}#{endpoint}" if !indices.nil? && !indices.empty?
+          endpoint += "/#{metrics.join(',')}" if metrics&.any?
+          params = {
+            level: level
+          }
+          get(endpoint, params)
+        end
+
+        # https://www.elastic.co/guide/en/elasticsearch/reference/current/dangling-indices-list.html
+        def dangling
+          endpoint = '/_dangling'
+          get(endpoint)
+        end
+
+        private
+
+        def get(endpoint, params = nil, header = nil)
+          response = conn.get(endpoint, params, header)
+          response.body
+        rescue Faraday::Error => e
+          log.error("while get #{endpoint}: #{e}")
+          raise Error, "error on get #{endpoint}", e
+        end
+
+        def conn
+          faraday_options = {
+            request: {
+              open_timeout: timeout + 1,
+              read_timeout: timeout + 1,
+              write_timeout: timeout + 1
+            },
+            ssl: {
+              verify: verify_ssl,
+              ca_file: ca_file
+            },
+            headers: {
+              'User-Agent' => user_agent
+            }
+          }
+
+          @conn ||= Faraday.new(url: url, **faraday_options) do |config|
+            config.request :authorization, :basic, username, password if username && password
+            config.request :json
+            config.response :json
+            config.response :raise_error
+            config.response :logger, log, headers: false, bodies: false, log_level: :debug if log
+            config.adapter :net_http
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/elasticsearch_stats/cluster_health_data.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require_relative 'base_data'
+
+module Fluent
+  module Plugin
+    module ElasticsearchStats
+      class ClusterHealthData < BaseData
+        NAME = 'cluster_health'
+
+        def extract_metrics
+          extract_cluster_metrics + extract_indices_metrics
+        end
+
+        private
+
+        def extract_cluster_metrics
+          data.each_with_object([]) do |(k, v), metrics|
+            metrics << metric.format(name: ['cluster', k], value: v, family: family, metadata: metadata)
+          end.compact
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/elasticsearch_stats/cluster_stats_data.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require_relative 'base_data'
+
+module Fluent
+  module Plugin
+    module ElasticsearchStats
+      class ClusterStatsData < BaseData
+        NAME = 'cluster_stats'
+
+        def extract_metrics
+          extract_base_metrics + extract_indices_metrics + extract_nodes_metrics
+        end
+
+        private
+
+        def extract_base_metrics
+          status_metric = metric.format(
+            name: %w[cluster status], value: status, family: family, metadata: metadata
+          )
+          [status_metric]
+        end
+
+        def extract_indices_metrics
+          metrics = []
+
+          flattened_data = Utils.hash_flatten_keys(indices, separator: metric.name_separator)
+          flattened_data.each do |k, v|
+            metrics << metric.format(name: ['indices', k], value: v, family: family, metadata: metadata)
+          end
+          metrics.compact
+        end
+
+        def extract_nodes_metrics
+          metrics = []
+          flattened_data = Utils.hash_flatten_keys(nodes, separator: metric.name_separator)
+          flattened_data.each do |k, v|
+            metrics << metric.format(name: ['nodes', k], value: v, family: family, metadata: metadata)
+          end
+          metrics.compact
+        end
+
+        def nodes
+          data['nodes']
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/elasticsearch_stats/collector.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+module Fluent
+  module Plugin
+    module ElasticsearchStats
+      class Collector
+        attr_reader :client, :stats_config, :log
+
+        def initialize(client:,
+                       stats_config: {},
+                       log: nil)
+          @client = client
+          @stats_config = stats_config
+          @log = log
+        end
+
+        def collect_stats_metrics
+          metrics = []
+          metrics += collect_cluster_health_metrics
+          metrics += collect_cluster_stats_metrics
+          metrics += collect_nodes_stats_metrics
+          metrics += collect_indices_stats_metrics
+          metrics += collect_dangling_metrics
+          metrics.compact
+        end
+
+        def collect_cluster_health_metrics
+          return [] unless stats_config.cluster_health
+
+          without_error(rescue_return: []) do
+            data = client.cluster_health(
+              level: stats_config.cluster_health_level,
+              local: stats_config.cluster_health_local
+            )
+            ClusterHealthData.new(data).extract_metrics
+          end
+        end
+
+        def collect_cluster_stats_metrics
+          return [] unless stats_config.cluster_stats
+
+          without_error(rescue_return: []) do
+            data = client.cluster_stats
+            ClusterStatsData.new(data).extract_metrics
+          end
+        end
+
+        def collect_nodes_stats_metrics
+          return [] unless stats_config.nodes_stats
+
+          without_error(rescue_return: []) do
+            data = client.nodes_stats(
+              level: stats_config.nodes_stats_level,
+              metrics: stats_config.nodes_stats_metrics
+            )
+            NodesStatsData.new(data).extract_metrics
+          end
+        end
+
+        def collect_indices_stats_metrics
+          return [] unless stats_config.indices_stats
+
+          without_error(rescue_return: []) do
+            data = client.indices_stats(
+              indices: stats_config.indices,
+              level: stats_config.indices_stats_level
+            )
+            IndicesStatsData.new(data).extract_metrics
+          end
+        end
+
+        def collect_dangling_metrics
+          return [] unless stats_config.dangling
+
+          without_error(rescue_return: []) do
+            data = client.dangling
+            DanglingData.new(data).extract_metrics
+          end
+        end
+
+        # FIXME: inject metadata !
+        #        cluster metadata / info ?
+        def metadata
+          Metadata.new
+                  .set(label: 'cluster_url', value: client.url)
+        end
+
+        def without_error(error: StandardError, rescue_return: nil)
+          yield
+        rescue error => e
+          log.error("while collecting metrics: #{e}")
+          rescue_return
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/elasticsearch_stats/dangling_data.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require_relative 'base_data'
+
+module Fluent
+  module Plugin
+    module ElasticsearchStats
+      class DanglingData < BaseData
+        NAME = 'dangling'
+
+        def extract_metrics
+          generate_dangling_indices_count
+        end
+
+        def generate_dangling_indices_count
+          metrics = []
+          metrics << metric.format(name: %w[dangling all count],
+                                   value: dangling_indices.size,
+                                   family: family,
+                                   metadata: metadata)
+          metrics
+        end
+
+        private
+
+        def dangling_indices
+          data.fetch('dangling_indices', [])
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/elasticsearch_stats/indices_stats_data.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require_relative 'base_data'
+
+module Fluent
+  module Plugin
+    module ElasticsearchStats
+      class IndicesStatsData < BaseData
+        NAME = 'indices_stats'
+
+        def extract_metrics
+          extract_all_metrics + extract_indices_metrics
+        end
+
+        def extract_all_metrics
+          return [] if !_all || _all.empty?
+
+          metrics = []
+          flattened = Utils.hash_flatten_keys(_all, separator: metric.name_separator)
+          flattened.each do |k, v|
+            metrics << metric.format(name: ['all_indices', k], value: v, family: family, metadata: metadata)
+          end
+          metrics.compact
+        end
+
+        private
+
+        def _all
+          data['_all']
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/elasticsearch_stats/metadata.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Fluent
+  module Plugin
+    module ElasticsearchStats
+      class Metadata
+        class << self
+          attr_accessor :metadata_prefix
+        end
+
+        attr_reader :metadata, :metadata_prefix
+
+        def initialize(metadata = {}, metadata_prefix: self.class.metadata_prefix)
+          @metadata = {}.update(metadata)
+          @metadata_prefix = metadata_prefix
+        end
+
+        def set(label:, value:)
+          return self if label.nil? || label.to_s.empty?
+
+          @metadata[label_for(label)] = value
+
+          self
+        end
+
+        def get(label)
+          metadata[label_for(label)]
+        end
+
+        def dup
+          self.class.new(
+            metadata.clone,
+            metadata_prefix: metadata_prefix.clone
+          )
+        end
+
+        def label_for(label)
+          "#{metadata_prefix}#{label}"
+        end
+
+        def to_h
+          metadata.clone
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/elasticsearch_stats/metric.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module Fluent
+  module Plugin
+    module ElasticsearchStats
+      class Metric
+        DEFAULT_NAME_SEPARATOR = '/'
+        DEFAULT_TIMESTAMP_FORMAT = :iso
+
+        class << self
+          attr_accessor :metric_prefix, :index_base_pattern, :index_base_replacement
+
+          def name_separator
+            @name_separator ||= DEFAULT_NAME_SEPARATOR
+          end
+
+          attr_writer :name_separator, :timestamp_format
+
+          def timestamp_format
+            @timestamp_format ||= DEFAULT_TIMESTAMP_FORMAT
+          end
+        end
+
+        attr_reader :metric_prefix, :timestamp_format, :index_base_pattern, :index_base_replacement, :name_separator
+
+        def initialize(metric_prefix: nil, timestamp_format: nil, index_base_pattern: nil, index_base_replacement: nil,
+                       name_separator: nil)
+          @metric_prefix = metric_prefix || self.class.metric_prefix
+          @timestamp_format = timestamp_format || self.class.timestamp_format
+          @index_base_pattern = index_base_pattern || self.class.index_base_pattern
+          @index_base_replacement = index_base_replacement || self.class.index_base_replacement
+          @name_separator = name_separator || self.class.name_separator
+        end
+
+        def convert_status(status)
+          case status.downcase
+          when 'green' then 1
+          when 'yellow' then 2
+          when 'red' then 3
+          else 0
+          end
+        end
+
+        def timenow_epochmillis
+          (Time.now.utc.to_f * 1000).to_i
+        end
+
+        def timenow_iso
+          Time.now.utc.iso8601(3)
+        end
+
+        def timenow
+          send("timenow_#{timestamp_format}")
+        end
+
+        def format(name:, value:, family: nil, metadata: nil)
+          return if name.nil? || name.empty? || value.nil?
+
+          value = convert_status(value) if name == 'status' || name[-1] == 'status'
+          name = name.join(name_separator) if name.respond_to?(:join)
+          family = family.join(name_separator) if name.respond_to?(:join)
+
+          return unless value.is_a?(Numeric)
+
+          index_base = compute_index_base(metadata.get('index'))
+          local_metadata = metadata.dup
+          local_metadata.set(label: 'index_base', value: index_base) if index_base
+
+          metric = {}
+          metric.update(local_metadata.to_h)
+          metric['timestamp'] = timenow
+          metric[name_label] = name.to_s
+          metric[value_label] = value
+          metric[family_label] = family.to_s if family
+          metric
+        end
+
+        def name_label
+          "#{metric_prefix}name"
+        end
+
+        def value_label
+          "#{metric_prefix}value"
+        end
+
+        def family_label
+          "#{metric_prefix}family"
+        end
+
+        def compute_index_base(index_name)
+          return nil unless index_name
+          return nil if !index_base_pattern || !index_base_replacement
+
+          index_name.gsub(index_base_pattern, index_base_replacement)
+        rescue StandardError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/elasticsearch_stats/nodes_stats_data.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require_relative 'base_data'
+
+module Fluent
+  module Plugin
+    module ElasticsearchStats
+      class NodesStatsData < BaseData
+        NAME = 'nodes_stats'
+
+        def extract_metrics
+          metrics = []
+          nodes.each_value do |node_stats|
+            metadata.dup
+                    .set(label: 'hostname', value: node_stats['name'])
+                    .set(label: 'host', value: node_stats['host'])
+
+            node_stats.each do |stat_family, stats|
+              next unless stats.is_a?(Hash)
+
+              stats.delete('timestamp')
+              stats.delete('indices') if stat_family == 'indices'
+              # FIXME: indices stats to extract ?
+
+              flattened_stats = Utils.hash_flatten_keys(stats, separator: metric.name_separator)
+              flattened_stats.each do |k, v|
+                metrics << metric.format(name: ['node', stat_family, k], value: v, family: family, metadata: metadata)
+              end
+            end
+          end
+          metrics.compact
+        end
+
+        private
+
+        def nodes
+          data['nodes']
+        end
+      end
+    end
+  end
+end