phihos-fluent-plugin-prometheus 2.0.3.pre.1

data/lib/fluent/plugin/out_prometheus.rb

@@ -0,0 +1,49 @@
+require 'fluent/plugin/output'
+require 'fluent/plugin/prometheus'
+
+module Fluent::Plugin
+  class PrometheusOutput < Fluent::Plugin::Output
+    Fluent::Plugin.register_output('prometheus', self)
+    include Fluent::Plugin::PrometheusLabelParser
+    include Fluent::Plugin::Prometheus
+
+    helpers :thread
+
+    def initialize
+      super
+      @registry = ::Prometheus::Client.registry
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    def configure(conf)
+      super
+      labels = parse_labels_elements(conf)
+      @metrics = Fluent::Plugin::Prometheus.parse_metrics_elements(conf, @registry, labels)
+    end
+
+    def start
+      super
+      Fluent::Plugin::Prometheus.start_retention_threads(
+        @metrics,
+        @registry,
+        method(:thread_create),
+        method(:thread_current_running?),
+        @log
+      )
+      Fluent::Plugin::Prometheus.start_reset_threads(
+        @metrics,
+        @registry,
+        method(:thread_create),
+        method(:thread_current_running?),
+        @log
+      )
+    end
+
+    def process(tag, es)
+      instrument(tag, es, @metrics)
+    end
+  end
+end

data/lib/fluent/plugin/prometheus/data_store.rb

@@ -0,0 +1,103 @@
+# The default Prometheus client data store has no means of removing values.
+# For the "retention" feature we need to be able to remove metrics with specific labels after some time of inactivity.
+# By patching the Metric class and using our own DataStore we implement that missing feature.
+module Prometheus
+  module Client
+    class Metric
+      def remove(labels)
+        label_set = label_set_for(labels)
+        @store.remove(labels: label_set)
+      end
+
+      def reset_values
+        @store.reset_values
+      end
+    end
+  end
+end
+
+module Fluent
+  module Plugin
+    module Prometheus
+      # Stores all the data in simple hashes, one per metric. Each of these metrics
+      # synchronizes access to their hash, but multiple metrics can run observations
+      # concurrently.
+      class DataStore
+        class InvalidStoreSettingsError < StandardError; end
+        DEFAULT_METRIC_SETTINGS = { topk: 0 }
+
+        def for_metric(metric_name, metric_type:, metric_settings: {})
+          settings = DEFAULT_METRIC_SETTINGS.merge(metric_settings)
+          validate_metric_settings(metric_settings: settings)
+          MetricStore.new(metric_settings: settings)
+        end
+
+        private
+
+        def validate_metric_settings(metric_settings:)
+          unless metric_settings.has_key?(:topk) &&
+            (metric_settings[:topk].is_a? Integer) &&
+            metric_settings[:topk] >= 0
+            raise InvalidStoreSettingsError,
+                  "Metrics need a valid :topk key"
+          end
+        end
+
+        class MetricStore
+          def initialize(metric_settings:)
+            @internal_store = Hash.new { |hash, key| hash[key] = 0.0 }
+            @topk = metric_settings[:topk]
+            @lock = Monitor.new
+          end
+
+          def synchronize
+            @lock.synchronize { yield }
+          end
+
+          def set(labels:, val:)
+            synchronize do
+              @internal_store[labels] = val.to_f
+            end
+          end
+
+          def increment(labels:, by: 1)
+            synchronize do
+              @internal_store[labels] += by
+            end
+          end
+
+          def get(labels:)
+            synchronize do
+              @internal_store[labels]
+            end
+          end
+
+          def remove(labels:)
+            synchronize do
+              @internal_store.delete(labels)
+            end
+          end
+
+          def all_values
+            synchronize do
+              store = @internal_store.dup
+              if @topk > 0
+                store.sort_by { |_, value| -value }.first(@topk).to_h
+              else
+                store
+              end
+            end
+          end
+
+          def reset_values
+            synchronize do
+              @internal_store.transform_values! { |_| 0.0 }
+            end
+          end
+        end
+
+        private_constant :MetricStore
+      end
+    end
+  end
+end

data/lib/fluent/plugin/prometheus/placeholder_expander.rb

@@ -0,0 +1,132 @@
+module Fluent
+  module Plugin
+    module Prometheus
+      class ExpandBuilder
+        def self.build(placeholder, log:)
+          new(log: log).build(placeholder)
+        end
+
+        def initialize(log:)
+          @log = log
+        end
+
+        def build(placeholder_values)
+          placeholders = {}
+          placeholder_values.each do |key, value|
+            case value
+            when Array
+              size = value.size
+              value.each_with_index do |v, i|
+                placeholders["${#{key}[#{i}]}"] = v
+                placeholders["${#{key}[#{i - size}]}"] = v
+              end
+            when Hash
+              value.each do |k, v|
+                placeholders[%(${#{key}["#{k}"]})] = v
+              end
+            else
+              if key == 'tag'
+                placeholders.merge!(build_tag(value))
+              else
+                placeholders["${#{key}}"] = value
+              end
+            end
+          end
+
+          Fluent::Plugin::Prometheus::ExpandBuilder::PlaceholderExpander.new(@log, placeholders)
+        end
+
+        private
+
+        def build_tag(tag)
+          tags = tag.split('.')
+
+          placeholders = { '${tag}' => tag }
+
+          size = tags.size
+
+          tags.each_with_index do |v, i|
+            placeholders["${tag_parts[#{i}]}"] = v
+            placeholders["${tag_parts[#{i - size}]}"] = v
+          end
+
+          tag_prefix(tags).each_with_index do |v, i|
+            placeholders["${tag_prefix[#{i}]}"] = v
+          end
+
+          tag_suffix(tags).each_with_index do |v, i|
+            placeholders["${tag_suffix[#{i}]}"] = v
+          end
+
+          placeholders
+        end
+
+        def tag_prefix(tags)
+          tags = tags.dup
+          return [] if tags.empty?
+
+          ret = [tags.shift]
+          tags.each.with_index(1) do |tag, i|
+            ret[i] = "#{ret[i-1]}.#{tag}"
+          end
+          ret
+        end
+
+        def tag_suffix(tags)
+          return [] if tags.empty?
+
+          tags = tags.dup.reverse
+          ret = [tags.shift]
+          tags.each.with_index(1) do |tag, i|
+            ret[i] = "#{tag}.#{ret[i-1]}"
+          end
+          ret
+        end
+
+        class PlaceholderExpander
+          PLACEHOLDER_REGEX = /(\${[^\[}]+(\[[^\]]+\])?})/.freeze
+
+          attr_reader :placeholder
+
+          def initialize(log, placeholder)
+            @placeholder = placeholder
+            @log = log
+            @expander_cache = {}
+          end
+
+          def merge_placeholder(placeholder)
+            @placeholder.merge!(placeholder)
+          end
+
+          def expand(str, dynamic_placeholders: nil)
+            expander = if dynamic_placeholders
+                         if @expander_cache[dynamic_placeholders]
+                           @expander_cache[dynamic_placeholders]
+                         else
+                           e = ExpandBuilder.build(dynamic_placeholders, log: @log)
+                           e.merge_placeholder(@placeholder)
+                           @expander_cache[dynamic_placeholders] = e
+                           e
+                         end
+                       else
+                         self
+                       end
+
+            expander.expand!(str)
+          end
+
+          protected
+
+          def expand!(str)
+            str.gsub(PLACEHOLDER_REGEX) { |value|
+              @placeholder.fetch(value) do
+                @log.warn("unknown placeholder `#{value}` found")
+                value # return as it is
+              end
+            }
+          end
+        end
+      end
+    end
+  end
+end