fluent-plugin-jfrog-metrics 0.2.8 → 0.2.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: be1f993ce6a17cbbd55addf9bd1c2c987861373b86d0c9e5bf57fba2391b89c6
-  data.tar.gz: 42caf47f85ef0005eae2b35b35c6c4f7cf9c0cb39bd8f8b361cdf282f8e5ea2c
+  metadata.gz: 4858e796a287f5c256acde626e8932c805f3276403fadd70c9a2300681b3c056
+  data.tar.gz: 9ef7a4f3e8a4db027658f16ef2444cdb2c41fc9e6831a6cbd4cfd37d55c85a23
 SHA512:
-  metadata.gz: 459aa4c98d67051b19e5c3947277ade652eaaea9fb868bd41ac9ba1d0126bc15247d9b1f4076c21abe526ae74e9e0f68d565578916077ffcf3385717354f7b64
-  data.tar.gz: 41008a0e14481e8b1b0a9881ada4559cc4d2edbc13d5af4d43d294b26ebebab58710cd0dfcda55815541b77a129cc68f5fc3b668abcb460a0b8069d0465f5091
+  metadata.gz: ffbaa211813a9c1b7f24204608745d1088213d7140bff0188a2ed9f6aaea7d07296877b9e02809d89ba6ba151124028d578c58b4d1a63244a39328ea452b7015
+  data.tar.gz: 506953ce84e5ec6f2b3510e9485870ccd90ee92fc87d4dad234ff635bbdd80e7731ee0349fd5bbfc78e34e892108a76c94c6a326360413b5efc3c081f9880dc3

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-jfrog-metrics (0.2.8)
+    fluent-plugin-jfrog-metrics (0.2.9)
       fluentd (>= 0.14.10, < 2)
       rest-client (~> 2.1.0)
 

data/README.md

@@ -86,15 +86,21 @@ Integration is done by setting up Xray. Obtain JPD url and access token for API.
 - **apikey** (string) (required if token is not used, do refer Note section for specifics): API Key is the [Artifactory API Key](https://www.jfrog.com/confluence/display/JFROG/User+Profile#UserProfile-APIKey) for authentication
 - **token** (string) (required if apikey is not used, do refer Note section for specifics): Admin token is the [Artifactory Scoped Tokens](https://www.jfrog.com/confluence/display/JFROG/Access+Tokens#AccessTokens-GeneratingAdminTokens) for authentication
 - **metric_prefix** (string) (required): This values pulls the specific metrics. Values can be - jfrog.artifactory, jfrog.xray
-- **interval** (integer) (optional): Wait interval between pulling new events
+- **execution_interval** (integer) (optional): Wait interval between pulling new events (scheduler)
   - Default value: `60`
+- **timeout_interval** (integer) (optional): Timeout interval for pulling new events (scheduler)
+  - Default value: `60`
+- **request_timeout** (integer) (optional): Http request timeout when calling Artifactory/Xray to pull new events (http client)
+  - Default value: `20`
 - **common_jpd** (true / false) (optional): This flag should be set as true only for non-kubernetes installations or installations where JPD base URL is same to access both Artifactory and Xray,
   - ex: https://sample_base_url/artifactory or https://sample_base_url/xray
   - Default value: false
-- **target_platform** (string) (optional): Output format of target platform allowed values SPLUNK and ELASTIC
+- **target_platform** (string) (optional): The target log-vendor ("SPLUNK", "NEWRELIC" or "DATADOG")
   - Default value: `SPLUNK`
 - **verify_ssl** (true / false) (optional): This flag should be set as false in order to bypass client's ssl certificate verification. When false, sets ssl_opts['verify_ssl'] to OpenSSL::SSL::VERIFY_NONE. Otherwise, sets ssl_opts['verify_ssl'] to OpenSSL::SSL::VERIFY_PEER
   - Default value: true
+- **gzip_compression** (true / false) (optional): This flag should be set as true for compressing (gzip) the metrics payload on outbound posts. This parameter is set to false by default for backwards compatibility.
+  - Default value: false
 
 Note:
 
@@ -104,6 +110,7 @@ Note:
 
 ## Copyright
 
-- Copyright(c) 2020 - JFrog
-- License
+* Copyright(c) 2024 - JFrog
+* Maintainers - MahithaB, BenHarosh
+* License
   - Apache License, Version 2.0

data/fluent-plugin-jfrog-metrics.gemspec

@@ -3,12 +3,12 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = 'fluent-plugin-jfrog-metrics'
-  spec.version = '0.2.8'
+  spec.version = '0.2.9'
   spec.authors = ['MahithaB, BenHarosh']
   spec.email   = ['cpe-support@jfrog.com']
 
-  spec.summary       = %q{Fluentd Plugin for converting JFrog Artifactory, Xray generated metrics (Prometheus Exposition Format) to target observability platform format (Splunk HEC, New Relic, Elastic)}
-  spec.description   = %q{Fluentd Plugin for converting JFrog Artifactory, Xray generated metrics (Prometheus Exposition Format) to target observability platform format (Splunk HEC, New Relic, Elastic)}
+  spec.summary       = %q{Fluentd Plugin for converting JFrog Artifactory, Xray generated metrics (Prometheus Exposition Format) to target observability platform format (Splunk HEC, New Relic, DataDog)}
+  spec.description   = %q{Fluentd Plugin for converting JFrog Artifactory, Xray generated metrics (Prometheus Exposition Format) to target observability platform format (Splunk HEC, New Relic, DataDog)}
 
   spec.homepage      = 'https://github.com/jfrog/jfrog-fluentd-plugins/tree/main/fluent-plugin-jfrog-metrics'
   spec.license       = 'Apache-2.0'

data/lib/fluent/plugin/in_jfrog_metrics.rb

@@ -21,8 +21,8 @@ require_relative 'base_metrics_parser'
 require_relative 'metrics_helper'
 require_relative 'newrelic_metrics_parser'
 require_relative 'splunk_metrics_parser'
-require_relative 'elastic_metrics_parser'
 require_relative 'datadog_metrics_parser'
+require_relative 'utility'
 
 module Fluent
   module Plugin
@@ -37,8 +37,9 @@ module Fluent
       config_param :username, :string, default: ''
       config_param :apikey, :string, default: '', :secret => true
       config_param :token, :string, default: '', :secret => true
-      config_param :execution_interval, :time, default: 10
-      config_param :timeout_interval, :time, default: 30
+      config_param :execution_interval, :time, default: 60
+      config_param :timeout_interval, :time, default: 60
+      config_param :request_timeout, :time, default: 20
       config_param :metric_prefix, :string, default: ''
       config_param :target_platform, :string, default: 'SPLUNK'
       config_param :common_jpd, :bool, default: false
@@ -59,7 +60,9 @@ module Fluent
 
         raise Fluent::ConfigError, 'Must define the metric_prefix to use for getting the metrics.' if @metric_prefix == ''
 
-        raise Fluent::ConfigError, 'Must define the vendor to use for getting the metrics.' if @target_platform == ''
+        raise Fluent::ConfigError, 'Must define the target_platform to use for getting the metrics.' if @target_platform == ''
+
+        raise Fluent::ConfigError, 'Must define the target_platform to be fone of the following (DATADOG, NEWRELIC, SPLUNK).' if !(['DATADOG', 'NEWRELIC', 'SPLUNK'].include?(@target_platform))
       end
 
       # `start` is called when starting and after `configure` is successfully completed.
@@ -76,9 +79,9 @@ module Fluent
       end
 
       def run
-        puts('Preparing metrics collection, creating timer task')
+        puts "#{Utility.get_time} Preparing metrics collection, creating timer task"
         timer_task = Concurrent::TimerTask.new(execution_interval: @execution_interval, timeout_interval: @timeout_interval, run_now: true) do
-          puts('Timer task execution')
+          puts "#{Utility.get_time} Timer task execution"
           do_execute
         end
         timer_task.execute
@@ -86,25 +89,39 @@ module Fluent
       end
 
       def do_execute
-        puts('Metrics collection started')
-        metrics_helper = MetricsHelper.new(@metric_prefix, @jpd_url, @username, @apikey, @token, @common_jpd, @verify_ssl)
-        platform_metrics = metrics_helper.get_metrics
+        begin
+          puts "#{Utility.get_time} Metrics collection started"
+          metrics_helper = MetricsHelper.new(@metric_prefix, @jpd_url, @username, @apikey, @token, @common_jpd, @verify_ssl, @request_timeout)
+          platform_metrics = metrics_helper.get_metrics
 
-        additional_metrics = metrics_helper.get_additional_metrics
-        if !additional_metrics.nil? && additional_metrics != ''
-          platform_metrics += additional_metrics.to_s
-        end
-        if @target_platform == 'SPLUNK'
-          parser = SplunkMetricsParser.new(@metric_prefix, router, @tag)
-        elsif @target_platform == 'NEWRELIC'
-          parser = NewRelicMetricsParser.new(@metric_prefix, router, @tag)
-        elsif @target_platform == 'DATADOG'
-          parser = DatadogMetricsParser.new(@metric_prefix, router, @tag)
-        else
-          raise 'Parser Type is not valid.Should be SPLUNK or NEWRELIC or DATADOG'
-        end
-        parser.emit_parsed_metrics(platform_metrics)
-        puts('Metrics collection finished')
+          additional_metrics = metrics_helper.get_additional_metrics
+          if !additional_metrics.nil? && additional_metrics != ''
+            platform_metrics += additional_metrics.to_s
+          end
+          puts "#{Utility.get_time} Metrics collection finished"
+
+          if @target_platform == 'SPLUNK'
+            parser = SplunkMetricsParser.new(@metric_prefix, router, @tag)
+          elsif @target_platform == 'NEWRELIC'
+            parser = NewRelicMetricsParser.new(@metric_prefix, router, @tag)
+          elsif @target_platform == 'DATADOG'
+            parser = DatadogMetricsParser.new(@metric_prefix, router, @tag)
+          else
+            raise 'Parser Type is not valid. target_platform Should be SPLUNK or NEWRELIC or DATADOG'
+          end
+          parser.emit_parsed_metrics(platform_metrics)
+
+        rescue RestClient::Exceptions::OpenTimeout
+          puts "#{Utility.get_time} The request timed out while trying to open a connection. The configured request timeout is: #{@request_timeout}"
+        rescue RestClient::Exceptions::ReadTimeout
+          puts "#{Utility.get_time} The request timed out while waiting for a response. The configured request timeout is: #{@request_timeout}"
+        rescue  RestClient::Exceptions::RequestTimeout
+          puts "#{Utility.get_time} The request timed out. The configured request timeout is: #{@request_timeout}"
+        rescue RestClient::ExceptionWithResponse => e
+          puts "#{Utility.get_time} HTTP request failed: #{e.response}"
+        rescue StandardError => e
+          puts "#{Utility.get_time} An unexpected error occurred: #{e.message}"
+        end  
       end
     end
   end

data/lib/fluent/plugin/metrics_helper.rb

@@ -1,11 +1,12 @@
 # frozen_string_literal: true
 
 require 'rest-client'
+require_relative 'utility'
 
 class MetricsHelper
   @@obs_endpoint_exists = false
 
-  def initialize(metric_prefix, jpd_url, username, apikey, token, common_jpd, verify_ssl)
+  def initialize(metric_prefix, jpd_url, username, apikey, token, common_jpd, verify_ssl, request_timeout)
     @metric_prefix = metric_prefix
     @jpd_url = jpd_url
     @username = username
@@ -13,6 +14,7 @@ class MetricsHelper
     @token = token
     @common_jpd = common_jpd
     @verify_ssl = verify_ssl
+    @request_timeout = request_timeout
   end
 
   def get_metrics
@@ -26,62 +28,73 @@ class MetricsHelper
             "#{@jpd_url}/artifactory/api/v1/metrics"
           end
 
-    puts "Executing #{@metric_prefix} metrics collection from: #{url}"
+    puts "#{Utility.get_time} Executing #{@metric_prefix} metrics collection from: #{url}"
     if !@token.nil? && @token != ''
-      execute_rest_call(url, @username, nil, @token, true, @verify_ssl)
+      execute_rest_call(url, @username, nil, @token, true, @verify_ssl, @request_timeout)
     elsif !@apikey.nil? && @apikey != ''
-      execute_rest_call(url, @username, @apikey, nil, false, @verify_ssl)
+      execute_rest_call(url, @username, @apikey, nil, false, @verify_ssl, @request_timeout)
     end
 
   end
 
   def get_additional_metrics
+    puts "#{Utility.get_time} Aadditional metrics collection started"
     if (@metric_prefix == 'jfrog.artifactory' || @common_jpd == false) && !@token.nil? && @token != ''
       url = "#{@jpd_url}/observability/api/v1/metrics"
-      puts "Executing additional metrics collection from: #{url}"
-      check_endpoint(url, @token, @verify_ssl) if @@obs_endpoint_exists == nil? || !@@obs_endpoint_exists
-      execute_rest_call(url, @username, nil, @token, true, @verify_ssl) if @@obs_endpoint_exists
+      puts "#{Utility.get_time} Collecting additional metrics from: #{url}"
+      check_endpoint(url, @token, @verify_ssl, @request_timeout) if @@obs_endpoint_exists == nil? || !@@obs_endpoint_exists
+      execute_rest_call(url, @username, nil, @token, true, @verify_ssl, @request_timeout) if @@obs_endpoint_exists
     end
   end
 
-  def check_endpoint(url, token, verify_ssl)
+  def check_endpoint(url, token, verify_ssl, request_timeout)
+    puts "#{Utility.get_time} Checking connectivity to endpoint: #{url}"
     response = RestClient::Request.new(
       method: :get,
       url: url,
       headers: { Authorization: "Bearer #{token}"},
-      verify_ssl: verify_ssl
+      verify_ssl: verify_ssl,
+      timeout: request_timeout
     ).execute do |response, request, result|
-      @@obs_endpoint_exists = true if response.code == 200
-      puts "#{url} exists? -> #{@@obs_endpoint_exists}, storing the result for next executions"
+      if response.code == 200
+        @@obs_endpoint_exists = true
+        puts "#{Utility.get_time} #{url} exists: #{@@obs_endpoint_exists}. Storing the result for next executions"
+      else
+        @@obs_endpoint_exists = false 
+        puts "#{Utility.get_time} Cannot verify #{url} endpoint. Received response code: #{response.code}, Response body:\n#{response.body}"
+        raise "Unexpected response code: #{response.code} when calling #{url}" 
+      end
     end
   end
 
-  def execute_rest_call(url, user, password, token, use_token, verify_ssl)
+  def execute_rest_call(url, user, password, token, use_token, verify_ssl, request_timeout)
     request = if use_token == true
-                RestClient::Request.new(
-                  method: :get,
-                  url: url,
-                  headers: { Authorization: "Bearer #{token}" },
-                  verify_ssl: verify_ssl
-                )
-              else
-                RestClient::Request.new(
-                  method: :get,
-                  url: url,
-                  user: user,
-                  password: password,
-                  verify_ssl: verify_ssl
-                )
-              end
-
+      RestClient::Request.new(
+        method: :get,
+        url: url,
+        headers: { Authorization: "Bearer #{token}" },
+        verify_ssl: verify_ssl,
+        timeout: request_timeout     
+      )
+    else
+      RestClient::Request.new(
+        method: :get,
+        url: url,
+        user: user,
+        password: password,
+        verify_ssl: verify_ssl,
+        timeout: request_timeout,      
+      )
+    end
+  
     request.execute do |response, request, result|
       case response.code
       when 200
-        puts "#{@metric_prefix} metrics were successfully collected from url: #{url}"
+        puts "#{Utility.get_time} #{@metric_prefix} metrics were successfully collected from url: #{url}"
         return response.body
       else
-        puts "Cannot fetch #{@metric_prefix} metrics from url: #{url}. Received response code: #{response.code}, Response body:\n#{response.body}"
-        raise Fluent::ConfigError, 'Cannot fetch #{@metric_prefix} metrics'
+        puts "#{Utility.get_time} Cannot fetch #{@metric_prefix} metrics from url: #{url}. Received response code: #{response.code}, Response body:\n#{response.body}"
+        raise "Unexpected response code: #{response.code} when calling #{url}"
       end
     end
   end

data/lib/fluent/plugin/newrelic_metrics_parser.rb

@@ -13,14 +13,12 @@ class NewRelicMetricsParser < BaseMetricsParser
     hash_data_array = []
     data_hash = {}
     data_array = []
-    puts cleaned_data
     cleaned_data.each do |interim_data|
       metrics_hash = {}
       if interim_data =~ /{/ && interim_data =~ /}/
         attributes = {}
         metric_name, additional_dims, metric_val_and_time = interim_data.match(/(.*){(.*)}(.*)/i).captures
         additional_dims.split("\",").each do |interim_data|
-          puts interim_data
           pair_data = interim_data.gsub("\"", "").gsub("{", "").gsub("}", "")
           interim_data_value = pair_data.split("=", 2)[1]
           interim_data_key = pair_data.split("=", 2)[0]

data/lib/fluent/plugin/utility.rb

@@ -0,0 +1,7 @@
+class Utility  
+    # one place to control time format for logginggit 
+    def self.get_time()
+        return Time.now.strftime("%Y-%m-%d %H:%M:%S")
+    end
+
+  end

data/spec/spec_helper.rb

@@ -25,7 +25,6 @@ end
 require 'metrics_helper'
 require 'splunk_metrics_parser'
 require 'newrelic_metrics_parser'
-require 'elastic_metrics_parser'
 
 RSpec.configure do |config|
   # rspec-expectations config goes here. You can use an alternate

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-jfrog-metrics
 version: !ruby/object:Gem::Version
-  version: 0.2.8
+  version: 0.2.9
 platform: ruby
 authors:
 - MahithaB, BenHarosh
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-09-12 00:00:00.000000000 Z
+date: 2024-10-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -102,7 +102,7 @@ dependencies:
         version: 2.1.0
 description: Fluentd Plugin for converting JFrog Artifactory, Xray generated metrics
   (Prometheus Exposition Format) to target observability platform format (Splunk HEC,
-  New Relic, Elastic)
+  New Relic, DataDog)
 email:
 - cpe-support@jfrog.com
 executables: []
@@ -119,15 +119,14 @@ files:
 - fluent-plugin-jfrog-metrics.gemspec
 - lib/fluent/plugin/base_metrics_parser.rb
 - lib/fluent/plugin/datadog_metrics_parser.rb
-- lib/fluent/plugin/elastic_metrics_parser.rb
 - lib/fluent/plugin/in_jfrog_metrics.rb
 - lib/fluent/plugin/metrics_helper.rb
 - lib/fluent/plugin/newrelic_metrics_parser.rb
 - lib/fluent/plugin/splunk_metrics_parser.rb
+- lib/fluent/plugin/utility.rb
 - spec/fixtures/files/creds.rb
 - spec/fixtures/files/sample_artifactory_metrics.txt
 - spec/fixtures/files/sample_xray_metrics.txt
-- spec/lib/elastic_metrics_parser_spec.rb
 - spec/lib/metrics_helper_spec.rb
 - spec/lib/newrelic_metrics_parser_spec.rb
 - spec/lib/splunk_metrics_parser_spec.rb
@@ -158,12 +157,11 @@ signing_key:
 specification_version: 4
 summary: Fluentd Plugin for converting JFrog Artifactory, Xray generated metrics (Prometheus
   Exposition Format) to target observability platform format (Splunk HEC, New Relic,
-  Elastic)
+  DataDog)
 test_files:
 - spec/fixtures/files/creds.rb
 - spec/fixtures/files/sample_artifactory_metrics.txt
 - spec/fixtures/files/sample_xray_metrics.txt
-- spec/lib/elastic_metrics_parser_spec.rb
 - spec/lib/metrics_helper_spec.rb
 - spec/lib/newrelic_metrics_parser_spec.rb
 - spec/lib/splunk_metrics_parser_spec.rb

data/lib/fluent/plugin/elastic_metrics_parser.rb

@@ -1,33 +0,0 @@
-# frozen_string_literal: true
-require 'json'
-require_relative 'base_metrics_parser'
-
-class ElasticMetricsParser < BaseMetricsParser
-  def initialize(metric_prefix, router, tag)
-    @metric_prefix = metric_prefix
-    @router = router
-    @tag = tag
-  end
-
-  def format_data(cleaned_data = [], prefix = '', separator = '')
-    hash_data_array = []
-    hash_data = {}
-    hash_data_with_labels = []
-    cleaned_data.each do |interim_data|
-      if interim_data =~ /{/ && interim_data =~ /}/
-        hash_data_with_dim_fields = {}
-        metric_name, additional_dims, metric_val_and_time = interim_data.match(/(.*){(.*)}(.*)/i).captures
-        additional_dims.split(/,/).map do |interim_dim_data|
-          hash_data_with_dim_fields[prefix + separator + "label" + separator + interim_dim_data.split(/=/)[0]] = interim_dim_data.split(/=/)[1].gsub(/"/, '') if interim_dim_data =~ /=/
-        end
-        hash_data_with_dim_fields[prefix + separator + metric_name] = metric_val_and_time.strip.split[0].to_f
-        hash_data_with_labels << hash_data_with_dim_fields
-      else
-        metric_name, value, = interim_data.split
-        hash_data[prefix + separator + metric_name] = value.to_f
-      end
-    end
-    hash_data_array << hash_data
-    (hash_data_array << hash_data_with_labels).flatten!
-  end
-end

data/spec/lib/elastic_metrics_parser_spec.rb

@@ -1,55 +0,0 @@
-# frozen_string_literal: true
-[
-  File.join(File.dirname(__FILE__), '..'),
-  File.join(File.dirname(__FILE__), '..', 'lib/fluent/plugin'),
-  File.join(File.dirname(__FILE__), '..', 'spec'),
-].each do |dir|
-  $LOAD_PATH.unshift(dir) unless $LOAD_PATH.include?(dir)
-end
-
-require 'elastic_metrics_parser'
-require 'date'
-require 'rspec'
-
-
-RSpec.describe ElasticMetricsParser do
-  describe "#emit_parsed_metrics" do
-    it 'should read sample Artifactory metrics data and verify the size of parsed data > 1' do
-      platform_metrics = File.read('./spec/fixtures/files/sample_artifactory_metrics.txt')
-      expect(platform_metrics.size).to be > 1
-
-      parser = ElasticMetricsParser.new('jfrog.artifactory', '', 'jfrog.artifactory.metrics')
-
-      normalized_data = parser.normalise_data(platform_metrics)
-      expect(normalized_data.size).to be > 1
-
-      cleaned_data = parser.clean_data(normalized_data)
-      expect(cleaned_data.size).to be > 1
-
-      hash_data_array = parser.extract_metrics_in_hash(cleaned_data, 'jfrog.artifactory', '.')
-      expect(hash_data_array.size).to be > 1
-
-      serialized_data = parser.serialize_data(hash_data_array)
-      expect(serialized_data.size).to be > 1
-    end
-
-    it 'should read sample Xray metrics data and verify the size of parsed data > 1' do
-      platform_metrics = File.read('./spec/fixtures/files/sample_xray_metrics.txt')
-      expect(platform_metrics.size).to be > 1
-
-      parser = ElasticMetricsParser.new('jfrog.xray', '', 'jfrog.xray.metrics')
-
-      normalized_data = parser.normalise_data(platform_metrics)
-      expect(normalized_data.size).to be > 1
-
-      cleaned_data = parser.clean_data(normalized_data)
-      expect(cleaned_data.size).to be > 1
-
-      hash_data_array = parser.extract_metrics_in_hash(cleaned_data, 'jfrog.xray', '.')
-      expect(hash_data_array.size).to be > 1
-
-      serialized_data = parser.serialize_data(hash_data_array)
-      expect(serialized_data.size).to be > 1
-    end
-  end
-end