fluent-plugin-jfrog-send-metrics 0.1.7 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 156657150eee6dad8aa4df6dd50ac2e3faf3b881b9e2e21d56bb1a976b36ce67
-  data.tar.gz: 5d544ba0a9cd7ec28a195486a4b9c33e417b1856507e2623bbed556cf8dca305
+  metadata.gz: e81703836698da47723aac87029c37beb200bf98a54302c1c9a9112f3256d393
+  data.tar.gz: 410485c8e3f8ee9e8181ea31d92fedbd43c50b27d3ac58816c7a3aeef85a4d77
 SHA512:
-  metadata.gz: ba4a3cacd19a5e5745b7daf3f6e5d99764295f803bd6535c130e74ef78afd67934c3e40cdabf753532143d6ff595089629254947424838372942f13d08b7563e
-  data.tar.gz: 0b95d910ab9bee0581411d0c883ad63d6de201bbd09b45a3703a73b66b0c9806b193dd65951f5708b5b2a71c5c6ccce7f4c4b7a0d2f19a9806ba2af009b86872
+  metadata.gz: 2457370d7894ec11e2191c6a1820f9da8c7933f7be8287682c30f117bf89b38c65c1312b38efea4872c1675d0f5599ebc0b619a6f3b0e2ba65b3b9b4aacade03
+  data.tar.gz: 9d77b96e6443969640c545dbedc239f31ef32214f06e3748bfe2084dc1eb93de0cce6d8a2003593a4c3a44ae28988390d6031eff11905ba9bd024a83ee96ca1c

data/Gemfile.lock

@@ -1,9 +1,9 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-jfrog-send-metrics (0.1.7)
+    fluent-plugin-jfrog-send-metrics (0.1.8)
       fluentd (>= 0.14.10, < 2)
-      rest-client (~> 2.1.0)
+      rest-client (>= 2.1.0)
 
 GEM
   remote: https://rubygems.org/
@@ -49,7 +49,7 @@ GEM
     netrc (0.11.0)
     power_assert (2.0.1)
     rake (12.3.3)
-    rest-client (2.1.0)
+    rest-client (>= 2.1.0)
       http-accept (>= 1.7.0, < 2.0)
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 4.0)

data/README.md

@@ -81,9 +81,14 @@ Obtain respective authentication credentials for log-vendors
 * **http_proxy**(string)(optional): Proxy server URL - which will proxy http/s calls to your observability vendor (DataDog/NewRelic)
 * **verify_ssl** (true / false) (optional): This flag should be set as false in order to bypass client's ssl certificate verification. When false, sets ssl_opts['verify_ssl'] to OpenSSL::SSL::VERIFY_NONE. Otherwise, sets ssl_opts['verify_ssl'] to OpenSSL::SSL::VERIFY_PEER
   * Default value: true
+* **gzip_compression** (true / false) (optional): This flag should be set as true for compressing (gzip) the metrics payload on outbound posts. This parameter is set to false by default for backwards compatibility.
+  * Default value: false
+* **request_timeout** (integer) (optional): Http request timeout when calling Artifactory/Xray to pull new events (http client)
+  * Default value: `20`
 
 ## Copyright
 
-* Copyright(c) 2024- MahithaB, BenH
+* Copyright(c) 2024 - JFrog
+* Maintainers - MahithaB, BenHarosh
 * License
   * Apache License, Version 2.0

data/fluent-plugin-jfrog-send-metrics.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-jfrog-send-metrics"
-  spec.version = "0.1.7"
+  spec.version = "0.1.8"
   spec.authors = ["MahithaB", "BenH"]
   spec.email   = ["partner_support@jfrog.com"]
 

data/lib/fluent/plugin/datadog_metrics_sender.rb

@@ -2,6 +2,8 @@
 require 'json'
 require 'rest-client'
 require 'uri'
+require 'stringio'
+require_relative 'utility'
 
 class DatadogMetrics
   def initialize(apikey, url)
@@ -24,48 +26,54 @@ class DatadogMetrics
     return record
   end
 
-  def send_metrics(ddtags, record, http_proxy, verify_ssl)
-    puts "Additional tags to be added to metrics are:", ddtags
+  def send_metrics(ddtags, record, http_proxy, verify_ssl, request_timeout, gzip_compression)
+    # Get the current local timestamp
+    puts "#{Utility.get_time} Additional tags to be added to metrics are:", ddtags
     metrics_data = add_custom_data(ddtags, record)
-    puts "Sending received metrics data"
+    puts "#{Utility.get_time} Sending received metrics data"
 
     if http_proxy
       RestClient.proxy = URI.parse(http_proxy)
-      puts "Using http_proxy param to set proxy for request. Proxy url: #{RestClient.proxy}"
+      puts "#{Utility.get_time} Using http_proxy param to set proxy for request. Proxy url: #{RestClient.proxy}"
     elsif ENV['HTTP_PROXY']
       RestClient.proxy = ENV['HTTP_PROXY']
-      puts "Using 'HTTP_PROXY' environment variable to set proxy for request. Proxy url: #{RestClient.proxy}"
+      puts "#{Utility.get_time} Using 'HTTP_PROXY' environment variable to set proxy for request. Proxy url: #{RestClient.proxy}"
     elsif ENV['http_proxy']
       RestClient.proxy = ENV['http_proxy']
-      puts "Using 'http_proxy' environment variable to set proxy for request. Proxy url: #{RestClient.proxy}"
+      puts "#{Utility.get_time} Using 'http_proxy' environment variable to set proxy for request. Proxy url: #{RestClient.proxy}"
     elsif ENV['https_proxy']
       RestClient.proxy = ENV['https_proxy']
-      puts "Using 'https_proxy' environment variable to set proxy for request. Proxy url: #{RestClient.proxy}"
+      puts "#{Utility.get_time} Using 'https_proxy' environment variable to set proxy for request. Proxy url: #{RestClient.proxy}"
     end
 
-    begin
+    headers = {
+      'DD-API-KEY': @apikey, 
+      content_type: :json
+    }
+
+    payload = metrics_data.to_json
+    if gzip_compression
+      payload = Utility.compress_payload(metrics_data)
+      headers[:'Content-Encoding'] = 'gzip'
+    end
+  
     response = RestClient::Request.new(
         method: :post,
         url: @url,
-        payload:  metrics_data.to_json,
-        headers: {'DD-API-KEY': @apikey, content_type: :json },
-        verify_ssl: verify_ssl
+        payload: payload,
+        headers: headers,
+        verify_ssl: verify_ssl,
+        timeout: request_timeout
     ).execute do |response, request, result|
-        case response.code
-        when 202
-          puts 'Metrics were successfully sent to DataDog'
-          return response.body
-        else
-          puts "Cannot send metrics to DataDog url: %s. Received response code: %d, Response body: %s"  % [@url, response.code, response.body]
-        end
+      case response.code
+      when 202
+        puts "#{Utility.get_time} Metrics were successfully sent to DataDog"
+        return response.body
+      else
+        puts "#{Utility.get_time} Cannot send metrics to DataDog url: %s. Received response code: %d, Response body: %s"  % [@url, response.code, response.body]
       end
-    rescue Net::HTTPClientException => e
-      # Handle the HTTP client exception
-      puts "An HTTP client error occurred when sending metrics to DataDog: #{e.message}"
-    rescue StandardError => e
-      # Handle any other exceptions
-      puts "An error occurred when sending metrics to DataDog: #{e.message}"
     end
   end
+
 end
 

data/lib/fluent/plugin/newrelic_metrics_sender.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 require 'json'
 require 'rest-client'
+require 'stringio'
+require_relative 'utility'
 
 class NewRelicMetrics
   def initialize(apikey, url)
@@ -8,47 +10,50 @@ class NewRelicMetrics
     @url = url
   end
 
-  def send_metrics(metrics_data, http_proxy, verify_ssl)
-    puts "Sending received metrics data"
+  def send_metrics(metrics_data, http_proxy, verify_ssl, request_timeout, gzip_compression)
+    puts "#{Utility.get_time} Sending received metrics data"
     metrics_payload = []
     metrics_payload.push(JSON.parse(metrics_data.to_json))
 
     if http_proxy
       RestClient.proxy = URI.parse(http_proxy)
-      puts "Using http_proxy param to set proxy for request. Proxy url: #{RestClient.proxy}"
+      puts "#{Utility.get_time} Using http_proxy param to set proxy for request. Proxy url: #{RestClient.proxy}"
     elsif ENV['HTTP_PROXY']
       RestClient.proxy = ENV['HTTP_PROXY']
-      puts "Using 'HTTP_PROXY' environment variable to set proxy for request. Proxy url: #{RestClient.proxy}"
+      puts "#{Utility.get_time} Using 'HTTP_PROXY' environment variable to set proxy for request. Proxy url: #{RestClient.proxy}"
     elsif ENV['http_proxy']
       RestClient.proxy = ENV['http_proxy']
-      puts "Using 'http_proxy' environment variable to set proxy for request. Proxy url: #{RestClient.proxy}"
+      puts "#{Utility.get_time} Using 'http_proxy' environment variable to set proxy for request. Proxy url: #{RestClient.proxy}"
     elsif ENV['https_proxy']
       RestClient.proxy = ENV['https_proxy']
-      puts "Using 'https_proxy' environment variable to set proxy for request. Proxy url: #{RestClient.proxy}"
+      puts "#{Utility.get_time} Using 'https_proxy' environment variable to set proxy for request. Proxy url: #{RestClient.proxy}"
     end
 
-    begin
-      response = RestClient::Request.new(
-          method: :post,
-          url: @url,
-          payload:  metrics_payload.to_json,
-          headers: {params: {'Api-Key' => @apikey}},
-          verify_ssl: verify_ssl
-      ).execute do |response, request, result|
-        case response.code
-        when 202
-          puts 'Metrics were successfully sent to NewRelic'
-          return response.body
-        else
-          puts "Cannot send metrics to NewRelic url: %s. Received response code: %d, Response body: %s"  % [@url, response.code, response.body]
-        end
+    headers = {
+      params: {'Api-Key' => @apikey}
+    }
+    payload = metrics_payload.to_json
+    if gzip_compression
+      payload = Utility.compress_payload(metrics_payload)
+      headers[:'Content-Encoding'] = 'gzip'
+    end
+    
+    response = RestClient::Request.new(
+        method: :post,
+        url: @url,
+        payload:  payload,
+        headers: headers,
+        verify_ssl: verify_ssl,
+        timeout: request_timeout
+    ).execute do |response, request, result|
+      case response.code
+      when 202
+        puts "#{Utility.get_time} Metrics were successfully sent to NewRelic"
+        return response.body
+      else
+        puts "#{Utility.get_time} Cannot send metrics to NewRelic url: %s. Received response code: %d, Response body: %s"  % [@url, response.code, response.body]
       end
-    rescue Net::HTTPClientException => e
-      # Handle the HTTP client exception
-      puts "An HTTP client error occurred when sending metrics to NewRelic: #{e.message}"
-    rescue StandardError => e
-      # Handle any other exceptions
-      puts "An error occurred when sending metrics to NewRelic: #{e.message}"
     end
   end
 end
+

data/lib/fluent/plugin/out_jfrog_send_metrics.rb

@@ -31,25 +31,42 @@ module Fluent
       config_param :ddtags, :array, default: []
       config_param :http_proxy, :string, :default => nil
       config_param :verify_ssl, :bool, default: true
+      config_param :request_timeout, :time, default: 20
+      config_param :gzip_compression, :bool, default: false
 
       # `configure` is called before `start`.
       # 'conf' is a `Hash` that includes the configuration parameters.
       # If the configuration is invalid, raise `Fluent::ConfigError`.
       def configure(conf)
         super
-        raise Fluent::ConfigError, 'Must define the vendor to use for getting the metrics.' if @target_platform == ''
+        
+        raise Fluent::ConfigError, 'Must define the target_platform to be one of the following (DATADOG, NEWRELIC, SPLUNK).' if @target_platform == '' || !(['DATADOG', 'NEWRELIC'].include?(@target_platform))
 
         raise Fluent::ConfigError, 'Must define the apikey to use for authentication.' if @apikey == ''
       end
 
       def process(tag, es)
         es.each do |time, record|
-          if @target_platform == 'NEWRELIC'
-            vendor = NewRelicMetrics.new(@apikey, @url)
-            vendor.send_metrics(record, @http_proxy, @verify_ssl)
-          elsif @target_platform == 'DATADOG'
-            vendor = DatadogMetrics.new(@apikey, @url)
-            vendor.send_metrics(@ddtags, record, @http_proxy, @verify_ssl)
+          begin
+            if @target_platform == 'NEWRELIC'
+              vendor = NewRelicMetrics.new(@apikey, @url)
+              vendor.send_metrics(record, @http_proxy, @verify_ssl, @request_timeout, @gzip_compression)
+            elsif @target_platform == 'DATADOG'
+              vendor = DatadogMetrics.new(@apikey, @url)
+              vendor.send_metrics(@ddtags, record, @http_proxy, @verify_ssl, @request_timeout, @gzip_compression)
+            end
+          rescue RestClient::Exceptions::OpenTimeout
+            puts "#{Utility.get_time} The request timed out while trying to open a connection. The configured request timeout is: #{@request_timeout}"
+          rescue RestClient::Exceptions::ReadTimeout
+            puts "#{Utility.get_time} The request timed out while waiting for a response. The configured request timeout is: #{@request_timeout}"
+          rescue  RestClient::Exceptions::RequestTimeout
+            puts "#{Utility.get_time} The request timed out. The configured request timeout is: #{@request_timeout}"
+          rescue RestClient::ExceptionWithResponse => e
+            puts "#{Utility.get_time} HTTP request failed: #{e.response}"
+          rescue Net::HTTPClientException => e
+            puts "#{Utility.get_time} An HTTP client error occurred when sending metrics to #{@target_platform}: #{e.message}"
+          rescue StandardError => e
+            puts "#{Utility.get_time} An error occurred when sending metrics to #{@target_platform}: #{e.message}"
           end
         end
       end

data/lib/fluent/plugin/utility.rb

@@ -0,0 +1,26 @@
+require 'zlib'
+
+class Utility
+  def self.compress_payload(payload)
+      puts "#{self.get_time} Compressing metrics payload"
+      json_payload = payload.to_json
+  
+      # Compress the JSON string using zlib's GzipWriter
+      compressed_payload = StringIO.new    
+      Zlib::GzipWriter.wrap(compressed_payload) do |gz|
+        gz.write(json_payload)
+      end
+  
+      # convert the compressed data to a string
+      compressed_data = compressed_payload.string
+  
+      puts "#{self.get_time} Original metrics payload size: #{json_payload.bytesize} bytes"
+      puts "#{self.get_time} Compressed metrics payload size: #{compressed_data.bytesize} bytes"
+  
+      return compressed_data
+  end
+
+  def self.get_time() 
+      return Time.now.strftime("%Y-%m-%d %H:%M:%S")
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-jfrog-send-metrics
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 0.1.8
 platform: ruby
 authors:
 - MahithaB
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-09-12 00:00:00.000000000 Z
+date: 2024-10-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -118,6 +118,7 @@ files:
 - lib/fluent/plugin/datadog_metrics_sender.rb
 - lib/fluent/plugin/newrelic_metrics_sender.rb
 - lib/fluent/plugin/out_jfrog_send_metrics.rb
+- lib/fluent/plugin/utility.rb
 - spec/fixtures/files/creds.rb
 - spec/fixtures/files/sample_artifactory_newrelic_metrics.txt
 - spec/fixtures/files/sample_xray_newrelic_metrics.txt