fluent-plugin-prometheus-pull 0.2.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a562bd7285238974a4973917bee9a6505a808e19bfa84bb4a65629787949ec67
-  data.tar.gz: e9f4a4248f57c35039594cb4bab37c434fa4251f06806c0904fd46adeae44f87
+  metadata.gz: 4bf56d6a19e8eb5e8e032434ebc12267e845cb539ee6b0ceeedd29f8c83e8e1f
+  data.tar.gz: df7deb6aee0408f7f8200d852b2ad63ce969cb9f74fba75ac9c3dc7eb5907693
 SHA512:
-  metadata.gz: cbace9eae7cae1ec8239c74c1d2972615479cafc6b786412644496f7bdc663124c1998d87c66798aaf20106952d0daa17a142082c768b69d0ce1705000fb4ca9
-  data.tar.gz: 87e0325325009159d46cf59863510121ae340b2b958653ad62288a957663e620f6d1c007788ebe6e181b11d9742419126a4b7d03a6a49f07c2238be903fb843e
+  metadata.gz: ab1ee690706221a9b3943cea0218da2e529f103c309f3dfed01f1de5f879de20c6b196a63dd7628b20c215b100430c36dc583b25262b66b83ae2d6c652f7512f
+  data.tar.gz: 72bb075b1a1e15f441a1b2d0d4ef82e89072c5227ff35a337c9c0fe9af4799a1fb3f4bf41d0898d466bd6118552f08881edf3df724702852d5f28ddbe9787381

data/.ruby-version

@@ -1 +1 @@
-2.7.7
+2.7.8

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-prometheus-pull (0.2.1)
+    fluent-plugin-prometheus-pull (0.4.0)
       fluentd (>= 0.14.10, < 2)
 
 GEM
@@ -27,6 +27,8 @@ GEM
     http_parser.rb (0.8.0)
     json (2.6.3)
     kwalify (0.7.2)
+    mocha (2.4.5)
+      ruby2_keywords (>= 0.0.5)
     msgpack (1.6.0)
     parallel (1.22.1)
     parser (3.2.0.0)
@@ -55,6 +57,7 @@ GEM
     rubocop-rake (0.6.0)
       rubocop (~> 1.0)
     ruby-progressbar (1.11.0)
+    ruby2_keywords (0.0.5)
     serverengine (2.3.1)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
@@ -77,6 +80,7 @@ DEPENDENCIES
   bundler (~> 2.3)
   byebug (~> 11.1, >= 11.1.3)
   fluent-plugin-prometheus-pull!
+  mocha (~> 2.4, >= 2.4.5)
   rake (~> 13.0.6)
   reek (~> 6.1, >= 6.1.4)
   rubocop (~> 1.44, >= 1.44.1)
@@ -84,4 +88,4 @@ DEPENDENCIES
   test-unit (~> 3.5.3)
 
 BUNDLED WITH
-   2.4.6
+   2.4.22

data/README.md

@@ -1,43 +1,79 @@
 # fluent-plugin-prometheus-pull
 
-[Fluentd](https://fluentd.org/) input plugin to do something.
+[Fluentd](https://fluentd.org/) input plugin to pull prometheus http endpoint.
 
-TODO: write description for you plugin.
 
-## Installation
+## plugins
+
+### input - prometheus_pull
+
+Pull http prometheus metric endpoint.
+
+with options:
 
-### RubyGems
+| options             | default | usage                                                |
+|---------------------|---------|------------------------------------------------------|
+|                     |         |                                                      |
+| event_url_key       | nil     | define the key that will store the fetched url       |
+| event_url_label_key | nil     | define the key that will store the fetched url label |
+
+Example:
 
 ```
-$ gem install fluent-plugin-prometheus-pull
+<source>
+  @type prometheus_pull
+  urls http://app/metrics,http://app2/metrics
+  interval 300s
+
+  tag metric
+
+  <parse>
+    @type prometheus_text
+    label_prefix  tags_
+    add_type false
+  </parse>
+</source>
 ```
 
-### Bundler
+### parser - prometheus_text
 
-Add following line to your Gemfile:
+Take a "text" (string) of prometheus content, fetched by any mechanism you want,
+then it parses it, transforming each line in 1 event.
 
-```ruby
-gem "fluent-plugin-prometheus-pull"
-```
+with options:
+* ...
+* ...
 
-And then execute:
+Example:
 
 ```
-$ bundle
+<parse>
+  @type prometheus_text
+  label_prefix tags_
+  add_type false
+</parse>
 ```
 
-## Configuration
+## Installation
+
+Manual install, by executing:
 
-You can generate configuration template:
+    $ gem install fluent-plugin-prometheus-pull
 
-```
-$ fluent-plugin-config-format input prometheus-pull
-```
+Add to Gemfile with:
+
+    $ bundle add fluent-plugin-prometheus-pull
+
+
+## Compatibility
+
+plugin will work with:
+- ruby >= 2.7.7
+- td-agent >= 4.0.0
 
-You can copy and paste generated documents here.
 
 ## Copyright
 
-* Copyright(c) 2023- Thomas Tych
+* Copyright(c) 2023-2024 Thomas Tych
 * License
   * Apache License, Version 2.0

data/fluent-plugin-prometheus-pull.gemspec

@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = 'fluent-plugin-prometheus-pull'
-  spec.version = '0.2.1'
+  spec.version = '0.4.0'
   spec.authors = ['Thomas Tych']
   spec.email   = ['thomas.tych@gmail.com']
 
@@ -34,6 +34,8 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rubocop', '~> 1.44', '>= 1.44.1'
   spec.add_development_dependency 'rubocop-rake', '~> 0.6.0'
   spec.add_development_dependency 'test-unit', '~> 3.5.3'
+  spec.add_development_dependency 'mocha', '~> 2.4', '>= 2.4.5'
+
 
   spec.add_runtime_dependency 'fluentd', ['>= 0.14.10', '< 2']
 end

data/lib/fluent/plugin/in_prometheus_pull.rb

@@ -20,6 +20,8 @@ require 'uri'
 
 require 'fluent/plugin/input'
 
+require_relative 'prometheus_pull/labeled_url'
+
 module Fluent
   module Plugin
     # input / source
@@ -27,6 +29,8 @@ module Fluent
     class PrometheusPullInput < Fluent::Plugin::Input
       Fluent::Plugin.register_input('prometheus_pull', self)
 
+      attr_reader :labeled_urls
+
       helpers :timer, :parser, :compat_parameters
 
       desc 'The tag of the event'
@@ -63,31 +67,44 @@ module Fluent
       desc 'CA file path'
       config_param :ca_file, :string, default: nil
 
+      desc 'Event URL key'
+      config_param :event_url_key, :string, default: nil
+      desc 'Event URL label key'
+      config_param :event_url_label_key, :string, default: nil
+
       def configure(conf)
         compat_parameters_convert(conf, :parser)
 
         parser_config = conf.elements('parse').first
         # raise Fluent::ConfigError, '<parse> section is required.' unless parser_config
-        parser_config ||= Fluent::Config::Element.new('parse', '', [], { '@type' => 'prometheus_text' })
+        unless parser_config
+          parser_config ||= Fluent::Config::Element.new('parse', '', { '@type' => 'prometheus_text' }, [])
+          conf.elements.append(parser_config)
+        end
 
         super
 
+        @labeled_urls = urls.map { |url| parse_url(url) }
+
         @parser = parser_create(conf: parser_config)
       end
 
       def start
         super
 
-        timer_execute(:in_prometheus_pull, @interval, &method(:pull))
+        timer_execute(:in_prometheus_pull_timer, @interval, &method(:pull))
       end
 
       def pull
-        urls.each do |url|
+        labeled_urls.each do |url|
           pull_time = Fluent::EventTime.now
-          raw_metrics = fetch(url)
+          raw_metrics = fetch(url.url)
           parser.parse(raw_metrics) do |time, record|
             begin
               time ||= pull_time
+              record[event_url_key] = url.url if event_url_key
+              record[event_url_label_key] = url.label if event_url_label_key && url.label
+
               router.emit(tag, time, record)
             rescue StandardError => e
               error("error #{e}, while emitting #{record}")
@@ -127,8 +144,8 @@ module Fluent
       def fetch(url, redirect_limit = 5)
         raise 'Max number of redirects reached' if redirect_limit <= 0
 
-        uri = URI.parse(url)
-        http(uri) do |http, uri|
+        fetch_uri = URI.parse(url)
+        http(fetch_uri) do |http, uri|
           req = request(uri)
           response = http.request(req)
 
@@ -150,6 +167,10 @@ module Fluent
 
         log.error(message)
       end
+
+      def parse_url(labeled_url)
+        PrometheusPull::LabeledUrl.parse_labeled_url(labeled_url)
+      end
     end
   end
 end

data/lib/fluent/plugin/prometheus_pull/labeled_url.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Fluent
+  module Plugin
+    module PrometheusPull
+      class LabeledUrl
+        LABELED_URL_RE = /^(?:@(?<label>[^@]*)@)?(?<url>.*)$/.freeze
+
+        attr_reader :url, :label
+
+        def initialize(url:, label: nil)
+          @url = url
+          @label = label
+        end
+
+        def to_s
+          url
+        end
+
+        def self.parse_labeled_url(url)
+          match_data = LABELED_URL_RE.match(url)
+
+          raise Fluent::ConfigError, "unable to use url '#{url}'" unless match_data
+
+          new(url: match_data[:url],
+              label: match_data[:label])
+        end
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-prometheus-pull
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.4.0
 platform: ruby
 authors:
 - Thomas Tych
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-02-10 00:00:00.000000000 Z
+date: 2024-10-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bump
@@ -140,6 +140,26 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 3.5.3
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.4.5
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.4.5
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
@@ -179,6 +199,7 @@ files:
 - fluent-plugin-prometheus-pull.gemspec
 - lib/fluent/plugin/in_prometheus_pull.rb
 - lib/fluent/plugin/parser_prometheus_text.rb
+- lib/fluent/plugin/prometheus_pull/labeled_url.rb
 homepage: https://gitlab.com/ttych/fluent-plugin-prometheus-pull.git
 licenses:
 - Apache-2.0