fluent-plugin-jfrog-send-metrics 0.1.7 → 0.1.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 156657150eee6dad8aa4df6dd50ac2e3faf3b881b9e2e21d56bb1a976b36ce67
-  data.tar.gz: 5d544ba0a9cd7ec28a195486a4b9c33e417b1856507e2623bbed556cf8dca305
+  metadata.gz: 898feff8ab8f4a975854370c16b2d400e16918afe77dbc09e76b930ea62a2c05
+  data.tar.gz: 98102867a939e246ba8a688c1afd65853dacb62f0e4df23970ecd0bb5387628a
 SHA512:
-  metadata.gz: ba4a3cacd19a5e5745b7daf3f6e5d99764295f803bd6535c130e74ef78afd67934c3e40cdabf753532143d6ff595089629254947424838372942f13d08b7563e
-  data.tar.gz: 0b95d910ab9bee0581411d0c883ad63d6de201bbd09b45a3703a73b66b0c9806b193dd65951f5708b5b2a71c5c6ccce7f4c4b7a0d2f19a9806ba2af009b86872
+  metadata.gz: efc92c68d8d5eac16199f54d129ad2786356c8b81a30236a7af5aaff559446e8239f9fb378506d4fa9f1d8b8997067f7b8fe2572ce70c6c4b564c7b0882b2265
+  data.tar.gz: 97dad1d675ccb149eaaf3a4d6232e51558221323245eb43acde64147451f056179490d178defd53d223557ce4e3521bc7de12f3a53b5be03c106ee38ed854c81

data/Gemfile.lock

@@ -1,9 +1,9 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-jfrog-send-metrics (0.1.7)
+    fluent-plugin-jfrog-send-metrics (0.1.9)
       fluentd (>= 0.14.10, < 2)
-      rest-client (~> 2.1.0)
+      rest-client (>= 2.1.0)
 
 GEM
   remote: https://rubygems.org/
@@ -49,7 +49,7 @@ GEM
     netrc (0.11.0)
     power_assert (2.0.1)
     rake (12.3.3)
-    rest-client (2.1.0)
+    rest-client (>= 2.1.0)
       http-accept (>= 1.7.0, < 2.0)
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 4.0)

data/README.md

@@ -81,9 +81,14 @@ Obtain respective authentication credentials for log-vendors
 * **http_proxy**(string)(optional): Proxy server URL - which will proxy http/s calls to your observability vendor (DataDog/NewRelic)
 * **verify_ssl** (true / false) (optional): This flag should be set as false in order to bypass client's ssl certificate verification. When false, sets ssl_opts['verify_ssl'] to OpenSSL::SSL::VERIFY_NONE. Otherwise, sets ssl_opts['verify_ssl'] to OpenSSL::SSL::VERIFY_PEER
   * Default value: true
+* **gzip_compression** (true / false) (optional): This flag should be set as true for compressing (gzip) the metrics payload on outbound posts. This parameter is set to false by default for backwards compatibility.
+  * Default value: false
+* **request_timeout** (integer) (optional): Http request timeout when calling Artifactory/Xray to pull new events (http client)
+  * Default value: `20`
 
 ## Copyright
 
-* Copyright(c) 2024- MahithaB, BenH
+* Copyright(c) 2024 - JFrog
+* Maintainers - MahithaB, BenHarosh
 * License
   * Apache License, Version 2.0

data/fluent-plugin-jfrog-send-metrics.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-jfrog-send-metrics"
-  spec.version = "0.1.7"
+  spec.version = "0.1.9"
   spec.authors = ["MahithaB", "BenH"]
   spec.email   = ["partner_support@jfrog.com"]
 

data/lib/fluent/plugin/datadog_metrics_sender.rb

@@ -2,6 +2,8 @@
 require 'json'
 require 'rest-client'
 require 'uri'
+require 'stringio'
+require_relative 'utility'
 
 class DatadogMetrics
   def initialize(apikey, url)
@@ -24,47 +26,54 @@ class DatadogMetrics
     return record
   end
 
-  def send_metrics(ddtags, record, http_proxy, verify_ssl)
-    puts "Additional tags to be added to metrics are:", ddtags
+  def send_metrics(ddtags, record, http_proxy, verify_ssl, request_timeout, gzip_compression, logger)
+    # Get the current local timestamp
+    logger.info("#{Utility.get_time} Additional tags to be added to metrics are:", ddtags)
     metrics_data = add_custom_data(ddtags, record)
-    puts "Sending received metrics data"
+    logger.info("#{Utility.get_time} Sending received metrics data")
 
     if http_proxy
       RestClient.proxy = URI.parse(http_proxy)
-      puts "Using http_proxy param to set proxy for request. Proxy url: #{RestClient.proxy}"
+      logger.info("#{Utility.get_time} Using http_proxy param to set proxy for request. Proxy url: #{RestClient.proxy}")
     elsif ENV['HTTP_PROXY']
       RestClient.proxy = ENV['HTTP_PROXY']
-      puts "Using 'HTTP_PROXY' environment variable to set proxy for request. Proxy url: #{RestClient.proxy}"
+      logger.info("#{Utility.get_time} Using 'HTTP_PROXY' environment variable to set proxy for request. Proxy url: #{RestClient.proxy}")
     elsif ENV['http_proxy']
       RestClient.proxy = ENV['http_proxy']
-      puts "Using 'http_proxy' environment variable to set proxy for request. Proxy url: #{RestClient.proxy}"
+      logger.info("#{Utility.get_time} Using 'http_proxy' environment variable to set proxy for request. Proxy url: #{RestClient.proxy}")
     elsif ENV['https_proxy']
       RestClient.proxy = ENV['https_proxy']
-      puts "Using 'https_proxy' environment variable to set proxy for request. Proxy url: #{RestClient.proxy}"
+      logger.info("#{Utility.get_time} Using 'https_proxy' environment variable to set proxy for request. Proxy url: #{RestClient.proxy}")
     end
 
-    begin
-    response = RestClient::Request.new(
+    headers = {
+      'DD-API-KEY': @apikey, 
+      content_type: :json
+    }
+
+    payload = metrics_data.to_json
+    if gzip_compression
+      payload = Utility.compress_payload(metrics_data)
+      headers[:'Content-Encoding'] = 'gzip'
+    end
+  
+    request = RestClient::Request.new(
         method: :post,
         url: @url,
-        payload:  metrics_data.to_json,
-        headers: {'DD-API-KEY': @apikey, content_type: :json },
-        verify_ssl: verify_ssl
-    ).execute do |response, request, result|
-        case response.code
-        when 202
-          puts 'Metrics were successfully sent to DataDog'
-          return response.body
-        else
-          puts "Cannot send metrics to DataDog url: %s. Received response code: %d, Response body: %s"  % [@url, response.code, response.body]
-        end
+        payload: payload,
+        headers: headers,
+        verify_ssl: verify_ssl,
+        timeout: request_timeout
+    )
+
+    request.execute do |response, request, result|
+      case response.code
+      when 202
+        logger.info("#{Utility.get_time} Metrics were successfully sent to DataDog")
+        return response.body
+      else
+        logger.info("#{Utility.get_time} Cannot send metrics to DataDog url: %s. Received response code: %d, Response body: %s"  % [@url, response.code, response.body])
       end
-    rescue Net::HTTPClientException => e
-      # Handle the HTTP client exception
-      puts "An HTTP client error occurred when sending metrics to DataDog: #{e.message}"
-    rescue StandardError => e
-      # Handle any other exceptions
-      puts "An error occurred when sending metrics to DataDog: #{e.message}"
     end
   end
 end

data/lib/fluent/plugin/newrelic_metrics_sender.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 require 'json'
 require 'rest-client'
+require 'stringio'
+require_relative 'utility'
 
 class NewRelicMetrics
   def initialize(apikey, url)
@@ -8,47 +10,52 @@ class NewRelicMetrics
     @url = url
   end
 
-  def send_metrics(metrics_data, http_proxy, verify_ssl)
-    puts "Sending received metrics data"
+  def send_metrics(metrics_data, http_proxy, verify_ssl, request_timeout, gzip_compression, logger)
+    logger.info("#{Utility.get_time} Sending received metrics data")
     metrics_payload = []
     metrics_payload.push(JSON.parse(metrics_data.to_json))
 
     if http_proxy
       RestClient.proxy = URI.parse(http_proxy)
-      puts "Using http_proxy param to set proxy for request. Proxy url: #{RestClient.proxy}"
+      logger.info("#{Utility.get_time} Using http_proxy param to set proxy for request. Proxy url: #{RestClient.proxy}")
     elsif ENV['HTTP_PROXY']
       RestClient.proxy = ENV['HTTP_PROXY']
-      puts "Using 'HTTP_PROXY' environment variable to set proxy for request. Proxy url: #{RestClient.proxy}"
+      logger.info("#{Utility.get_time} Using 'HTTP_PROXY' environment variable to set proxy for request. Proxy url: #{RestClient.proxy}")
     elsif ENV['http_proxy']
       RestClient.proxy = ENV['http_proxy']
-      puts "Using 'http_proxy' environment variable to set proxy for request. Proxy url: #{RestClient.proxy}"
+      logger.info("#{Utility.get_time} Using 'http_proxy' environment variable to set proxy for request. Proxy url: #{RestClient.proxy}")
     elsif ENV['https_proxy']
       RestClient.proxy = ENV['https_proxy']
-      puts "Using 'https_proxy' environment variable to set proxy for request. Proxy url: #{RestClient.proxy}"
+      logger.info("#{Utility.get_time} Using 'https_proxy' environment variable to set proxy for request. Proxy url: #{RestClient.proxy}")
     end
 
-    begin
-      response = RestClient::Request.new(
-          method: :post,
-          url: @url,
-          payload:  metrics_payload.to_json,
-          headers: {params: {'Api-Key' => @apikey}},
-          verify_ssl: verify_ssl
-      ).execute do |response, request, result|
-        case response.code
-        when 202
-          puts 'Metrics were successfully sent to NewRelic'
-          return response.body
-        else
-          puts "Cannot send metrics to NewRelic url: %s. Received response code: %d, Response body: %s"  % [@url, response.code, response.body]
-        end
+    headers = {
+      params: {'Api-Key' => @apikey}
+    }
+    payload = metrics_payload.to_json
+    if gzip_compression
+      payload = Utility.compress_payload(metrics_payload)
+      headers[:'Content-Encoding'] = 'gzip'
+    end
+
+    request = RestClient::Request.new(
+        method: :post,
+        url: @url,
+        payload:  payload,
+        headers: headers,
+        verify_ssl: verify_ssl,
+        timeout: request_timeout
+    )
+
+    request.execute do |response, request, result|
+      case response.code
+      when 202
+        logger.info("#{Utility.get_time} Metrics were successfully sent to NewRelic")
+        return response.body
+      else
+        logger.info("#{Utility.get_time} Cannot send metrics to NewRelic url: %s. Received response code: %d, Response body: %s"  % [@url, response.code, response.body])
       end
-    rescue Net::HTTPClientException => e
-      # Handle the HTTP client exception
-      puts "An HTTP client error occurred when sending metrics to NewRelic: #{e.message}"
-    rescue StandardError => e
-      # Handle any other exceptions
-      puts "An error occurred when sending metrics to NewRelic: #{e.message}"
     end
   end
 end
+

data/lib/fluent/plugin/out_jfrog_send_metrics.rb

@@ -31,25 +31,45 @@ module Fluent
       config_param :ddtags, :array, default: []
       config_param :http_proxy, :string, :default => nil
       config_param :verify_ssl, :bool, default: true
+      config_param :request_timeout, :time, default: 20
+      config_param :gzip_compression, :bool, default: false
 
       # `configure` is called before `start`.
       # 'conf' is a `Hash` that includes the configuration parameters.
       # If the configuration is invalid, raise `Fluent::ConfigError`.
       def configure(conf)
         super
-        raise Fluent::ConfigError, 'Must define the vendor to use for getting the metrics.' if @target_platform == ''
+        
+        raise Fluent::ConfigError, 'Must define the target_platform to be one of the following (DATADOG, NEWRELIC).' if @target_platform == '' || !(['DATADOG', 'NEWRELIC'].include?(@target_platform))
 
         raise Fluent::ConfigError, 'Must define the apikey to use for authentication.' if @apikey == ''
       end
 
       def process(tag, es)
+        logger = log
         es.each do |time, record|
-          if @target_platform == 'NEWRELIC'
-            vendor = NewRelicMetrics.new(@apikey, @url)
-            vendor.send_metrics(record, @http_proxy, @verify_ssl)
-          elsif @target_platform == 'DATADOG'
-            vendor = DatadogMetrics.new(@apikey, @url)
-            vendor.send_metrics(@ddtags, record, @http_proxy, @verify_ssl)
+          begin
+            logger.info("Sending metrics to target platform: #{@target_platform} started")
+            if @target_platform == 'NEWRELIC'
+              vendor = NewRelicMetrics.new(@apikey, @url)
+              vendor.send_metrics(record, @http_proxy, @verify_ssl, @request_timeout, @gzip_compression, logger)
+            elsif @target_platform == 'DATADOG'
+              vendor = DatadogMetrics.new(@apikey, @url)
+              vendor.send_metrics(@ddtags, record, @http_proxy, @verify_ssl, @request_timeout, @gzip_compression, logger)
+            end
+            logger.info("Sending metrics to target platform: #{@target_platform} finished")
+          rescue RestClient::Exceptions::OpenTimeout
+            logger.info("#{Utility.get_time} The request timed out while trying to open a connection. The configured request timeout is: #{@request_timeout}")
+          rescue RestClient::Exceptions::ReadTimeout
+            logger.info("#{Utility.get_time} The request timed out while waiting for a response. The configured request timeout is: #{@request_timeout}")
+          rescue  RestClient::Exceptions::RequestTimeout
+            logger.info("#{Utility.get_time} The request timed out. The configured request timeout is: #{@request_timeout}")
+          rescue RestClient::ExceptionWithResponse => e
+            logger.info("#{Utility.get_time} HTTP request failed: #{e.response}")
+          rescue Net::HTTPClientException => e
+            logger.info("#{Utility.get_time} An HTTP client error occurred when sending metrics to #{@target_platform}: #{e.message}")
+          rescue StandardError => e
+            logger.info("#{Utility.get_time} An error occurred when sending metrics to #{@target_platform}: #{e.message}")
           end
         end
       end

data/lib/fluent/plugin/utility.rb

@@ -0,0 +1,39 @@
+require 'zlib'
+
+class Utility
+  def self.compress_payload(payload)
+      puts "#{self.get_time} Compressing metrics payload"
+      json_payload = payload.to_json
+  
+      # Compress the JSON string using zlib's GzipWriter
+      compressed_payload = StringIO.new    
+      Zlib::GzipWriter.wrap(compressed_payload) do |gz|
+        begin
+          gz.write(json_payload)
+        ensure
+          gz.close
+        end
+      end
+  
+      # convert the compressed data to a string
+      compressed_data = compressed_payload.string
+  
+      puts "#{self.get_time} Original metrics payload size: #{json_payload.bytesize} bytes"
+      puts "#{self.get_time} Compressed metrics payload size: #{compressed_data.bytesize} bytes"
+  
+      return compressed_data
+  end
+
+  def self.get_time() 
+      return Time.now.strftime("%Y-%m-%d %H:%M:%S")
+  end
+
+  def self.enable_http_client_logs(http_client_logs)
+    if http_client_logs
+      RestClient.log = STDOUT
+    else
+      RestClient.log = nil
+    end
+  end 
+  
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-jfrog-send-metrics
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 0.1.9
 platform: ruby
 authors:
 - MahithaB
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-09-12 00:00:00.000000000 Z
+date: 2024-11-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -118,6 +118,7 @@ files:
 - lib/fluent/plugin/datadog_metrics_sender.rb
 - lib/fluent/plugin/newrelic_metrics_sender.rb
 - lib/fluent/plugin/out_jfrog_send_metrics.rb
+- lib/fluent/plugin/utility.rb
 - spec/fixtures/files/creds.rb
 - spec/fixtures/files/sample_artifactory_newrelic_metrics.txt
 - spec/fixtures/files/sample_xray_newrelic_metrics.txt