fluent-plugin-jfrog-send-metrics 0.1.7 → 0.1.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 156657150eee6dad8aa4df6dd50ac2e3faf3b881b9e2e21d56bb1a976b36ce67
4
- data.tar.gz: 5d544ba0a9cd7ec28a195486a4b9c33e417b1856507e2623bbed556cf8dca305
3
+ metadata.gz: 898feff8ab8f4a975854370c16b2d400e16918afe77dbc09e76b930ea62a2c05
4
+ data.tar.gz: 98102867a939e246ba8a688c1afd65853dacb62f0e4df23970ecd0bb5387628a
5
5
  SHA512:
6
- metadata.gz: ba4a3cacd19a5e5745b7daf3f6e5d99764295f803bd6535c130e74ef78afd67934c3e40cdabf753532143d6ff595089629254947424838372942f13d08b7563e
7
- data.tar.gz: 0b95d910ab9bee0581411d0c883ad63d6de201bbd09b45a3703a73b66b0c9806b193dd65951f5708b5b2a71c5c6ccce7f4c4b7a0d2f19a9806ba2af009b86872
6
+ metadata.gz: efc92c68d8d5eac16199f54d129ad2786356c8b81a30236a7af5aaff559446e8239f9fb378506d4fa9f1d8b8997067f7b8fe2572ce70c6c4b564c7b0882b2265
7
+ data.tar.gz: 97dad1d675ccb149eaaf3a4d6232e51558221323245eb43acde64147451f056179490d178defd53d223557ce4e3521bc7de12f3a53b5be03c106ee38ed854c81
data/Gemfile.lock CHANGED
@@ -1,9 +1,9 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- fluent-plugin-jfrog-send-metrics (0.1.7)
4
+ fluent-plugin-jfrog-send-metrics (0.1.9)
5
5
  fluentd (>= 0.14.10, < 2)
6
- rest-client (~> 2.1.0)
6
+ rest-client (>= 2.1.0)
7
7
 
8
8
  GEM
9
9
  remote: https://rubygems.org/
@@ -49,7 +49,7 @@ GEM
49
49
  netrc (0.11.0)
50
50
  power_assert (2.0.1)
51
51
  rake (12.3.3)
52
- rest-client (2.1.0)
52
+ rest-client (>= 2.1.0)
53
53
  http-accept (>= 1.7.0, < 2.0)
54
54
  http-cookie (>= 1.0.2, < 2.0)
55
55
  mime-types (>= 1.16, < 4.0)
data/README.md CHANGED
@@ -81,9 +81,14 @@ Obtain respective authentication credentials for log-vendors
81
81
  * **http_proxy**(string)(optional): Proxy server URL - which will proxy http/s calls to your observability vendor (DataDog/NewRelic)
82
82
  * **verify_ssl** (true / false) (optional): This flag should be set as false in order to bypass client's ssl certificate verification. When false, sets ssl_opts['verify_ssl'] to OpenSSL::SSL::VERIFY_NONE. Otherwise, sets ssl_opts['verify_ssl'] to OpenSSL::SSL::VERIFY_PEER
83
83
  * Default value: true
84
+ * **gzip_compression** (true / false) (optional): This flag should be set as true for compressing (gzip) the metrics payload on outbound posts. This parameter is set to false by default for backwards compatibility.
85
+ * Default value: false
86
+ * **request_timeout** (integer) (optional): Http request timeout when calling Artifactory/Xray to pull new events (http client)
87
+ * Default value: `20`
84
88
 
85
89
  ## Copyright
86
90
 
87
- * Copyright(c) 2024- MahithaB, BenH
91
+ * Copyright(c) 2024 - JFrog
92
+ * Maintainers - MahithaB, BenHarosh
88
93
  * License
89
94
  * Apache License, Version 2.0
@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
3
3
 
4
4
  Gem::Specification.new do |spec|
5
5
  spec.name = "fluent-plugin-jfrog-send-metrics"
6
- spec.version = "0.1.7"
6
+ spec.version = "0.1.9"
7
7
  spec.authors = ["MahithaB", "BenH"]
8
8
  spec.email = ["partner_support@jfrog.com"]
9
9
 
@@ -2,6 +2,8 @@
2
2
  require 'json'
3
3
  require 'rest-client'
4
4
  require 'uri'
5
+ require 'stringio'
6
+ require_relative 'utility'
5
7
 
6
8
  class DatadogMetrics
7
9
  def initialize(apikey, url)
@@ -24,47 +26,54 @@ class DatadogMetrics
24
26
  return record
25
27
  end
26
28
 
27
- def send_metrics(ddtags, record, http_proxy, verify_ssl)
28
- puts "Additional tags to be added to metrics are:", ddtags
29
+ def send_metrics(ddtags, record, http_proxy, verify_ssl, request_timeout, gzip_compression, logger)
30
+ # Get the current local timestamp
31
+ logger.info("#{Utility.get_time} Additional tags to be added to metrics are:", ddtags)
29
32
  metrics_data = add_custom_data(ddtags, record)
30
- puts "Sending received metrics data"
33
+ logger.info("#{Utility.get_time} Sending received metrics data")
31
34
 
32
35
  if http_proxy
33
36
  RestClient.proxy = URI.parse(http_proxy)
34
- puts "Using http_proxy param to set proxy for request. Proxy url: #{RestClient.proxy}"
37
+ logger.info("#{Utility.get_time} Using http_proxy param to set proxy for request. Proxy url: #{RestClient.proxy}")
35
38
  elsif ENV['HTTP_PROXY']
36
39
  RestClient.proxy = ENV['HTTP_PROXY']
37
- puts "Using 'HTTP_PROXY' environment variable to set proxy for request. Proxy url: #{RestClient.proxy}"
40
+ logger.info("#{Utility.get_time} Using 'HTTP_PROXY' environment variable to set proxy for request. Proxy url: #{RestClient.proxy}")
38
41
  elsif ENV['http_proxy']
39
42
  RestClient.proxy = ENV['http_proxy']
40
- puts "Using 'http_proxy' environment variable to set proxy for request. Proxy url: #{RestClient.proxy}"
43
+ logger.info("#{Utility.get_time} Using 'http_proxy' environment variable to set proxy for request. Proxy url: #{RestClient.proxy}")
41
44
  elsif ENV['https_proxy']
42
45
  RestClient.proxy = ENV['https_proxy']
43
- puts "Using 'https_proxy' environment variable to set proxy for request. Proxy url: #{RestClient.proxy}"
46
+ logger.info("#{Utility.get_time} Using 'https_proxy' environment variable to set proxy for request. Proxy url: #{RestClient.proxy}")
44
47
  end
45
48
 
46
- begin
47
- response = RestClient::Request.new(
49
+ headers = {
50
+ 'DD-API-KEY': @apikey,
51
+ content_type: :json
52
+ }
53
+
54
+ payload = metrics_data.to_json
55
+ if gzip_compression
56
+ payload = Utility.compress_payload(metrics_data)
57
+ headers[:'Content-Encoding'] = 'gzip'
58
+ end
59
+
60
+ request = RestClient::Request.new(
48
61
  method: :post,
49
62
  url: @url,
50
- payload: metrics_data.to_json,
51
- headers: {'DD-API-KEY': @apikey, content_type: :json },
52
- verify_ssl: verify_ssl
53
- ).execute do |response, request, result|
54
- case response.code
55
- when 202
56
- puts 'Metrics were successfully sent to DataDog'
57
- return response.body
58
- else
59
- puts "Cannot send metrics to DataDog url: %s. Received response code: %d, Response body: %s" % [@url, response.code, response.body]
60
- end
63
+ payload: payload,
64
+ headers: headers,
65
+ verify_ssl: verify_ssl,
66
+ timeout: request_timeout
67
+ )
68
+
69
+ request.execute do |response, request, result|
70
+ case response.code
71
+ when 202
72
+ logger.info("#{Utility.get_time} Metrics were successfully sent to DataDog")
73
+ return response.body
74
+ else
75
+ logger.info("#{Utility.get_time} Cannot send metrics to DataDog url: %s. Received response code: %d, Response body: %s" % [@url, response.code, response.body])
61
76
  end
62
- rescue Net::HTTPClientException => e
63
- # Handle the HTTP client exception
64
- puts "An HTTP client error occurred when sending metrics to DataDog: #{e.message}"
65
- rescue StandardError => e
66
- # Handle any other exceptions
67
- puts "An error occurred when sending metrics to DataDog: #{e.message}"
68
77
  end
69
78
  end
70
79
  end
@@ -1,6 +1,8 @@
1
1
  # frozen_string_literal: true
2
2
  require 'json'
3
3
  require 'rest-client'
4
+ require 'stringio'
5
+ require_relative 'utility'
4
6
 
5
7
  class NewRelicMetrics
6
8
  def initialize(apikey, url)
@@ -8,47 +10,52 @@ class NewRelicMetrics
8
10
  @url = url
9
11
  end
10
12
 
11
- def send_metrics(metrics_data, http_proxy, verify_ssl)
12
- puts "Sending received metrics data"
13
+ def send_metrics(metrics_data, http_proxy, verify_ssl, request_timeout, gzip_compression, logger)
14
+ logger.info("#{Utility.get_time} Sending received metrics data")
13
15
  metrics_payload = []
14
16
  metrics_payload.push(JSON.parse(metrics_data.to_json))
15
17
 
16
18
  if http_proxy
17
19
  RestClient.proxy = URI.parse(http_proxy)
18
- puts "Using http_proxy param to set proxy for request. Proxy url: #{RestClient.proxy}"
20
+ logger.info("#{Utility.get_time} Using http_proxy param to set proxy for request. Proxy url: #{RestClient.proxy}")
19
21
  elsif ENV['HTTP_PROXY']
20
22
  RestClient.proxy = ENV['HTTP_PROXY']
21
- puts "Using 'HTTP_PROXY' environment variable to set proxy for request. Proxy url: #{RestClient.proxy}"
23
+ logger.info("#{Utility.get_time} Using 'HTTP_PROXY' environment variable to set proxy for request. Proxy url: #{RestClient.proxy}")
22
24
  elsif ENV['http_proxy']
23
25
  RestClient.proxy = ENV['http_proxy']
24
- puts "Using 'http_proxy' environment variable to set proxy for request. Proxy url: #{RestClient.proxy}"
26
+ logger.info("#{Utility.get_time} Using 'http_proxy' environment variable to set proxy for request. Proxy url: #{RestClient.proxy}")
25
27
  elsif ENV['https_proxy']
26
28
  RestClient.proxy = ENV['https_proxy']
27
- puts "Using 'https_proxy' environment variable to set proxy for request. Proxy url: #{RestClient.proxy}"
29
+ logger.info("#{Utility.get_time} Using 'https_proxy' environment variable to set proxy for request. Proxy url: #{RestClient.proxy}")
28
30
  end
29
31
 
30
- begin
31
- response = RestClient::Request.new(
32
- method: :post,
33
- url: @url,
34
- payload: metrics_payload.to_json,
35
- headers: {params: {'Api-Key' => @apikey}},
36
- verify_ssl: verify_ssl
37
- ).execute do |response, request, result|
38
- case response.code
39
- when 202
40
- puts 'Metrics were successfully sent to NewRelic'
41
- return response.body
42
- else
43
- puts "Cannot send metrics to NewRelic url: %s. Received response code: %d, Response body: %s" % [@url, response.code, response.body]
44
- end
32
+ headers = {
33
+ params: {'Api-Key' => @apikey}
34
+ }
35
+ payload = metrics_payload.to_json
36
+ if gzip_compression
37
+ payload = Utility.compress_payload(metrics_payload)
38
+ headers[:'Content-Encoding'] = 'gzip'
39
+ end
40
+
41
+ request = RestClient::Request.new(
42
+ method: :post,
43
+ url: @url,
44
+ payload: payload,
45
+ headers: headers,
46
+ verify_ssl: verify_ssl,
47
+ timeout: request_timeout
48
+ )
49
+
50
+ request.execute do |response, request, result|
51
+ case response.code
52
+ when 202
53
+ logger.info("#{Utility.get_time} Metrics were successfully sent to NewRelic")
54
+ return response.body
55
+ else
56
+ logger.info("#{Utility.get_time} Cannot send metrics to NewRelic url: %s. Received response code: %d, Response body: %s" % [@url, response.code, response.body])
45
57
  end
46
- rescue Net::HTTPClientException => e
47
- # Handle the HTTP client exception
48
- puts "An HTTP client error occurred when sending metrics to NewRelic: #{e.message}"
49
- rescue StandardError => e
50
- # Handle any other exceptions
51
- puts "An error occurred when sending metrics to NewRelic: #{e.message}"
52
58
  end
53
59
  end
54
60
  end
61
+
@@ -31,25 +31,45 @@ module Fluent
31
31
  config_param :ddtags, :array, default: []
32
32
  config_param :http_proxy, :string, :default => nil
33
33
  config_param :verify_ssl, :bool, default: true
34
+ config_param :request_timeout, :time, default: 20
35
+ config_param :gzip_compression, :bool, default: false
34
36
 
35
37
  # `configure` is called before `start`.
36
38
  # 'conf' is a `Hash` that includes the configuration parameters.
37
39
  # If the configuration is invalid, raise `Fluent::ConfigError`.
38
40
  def configure(conf)
39
41
  super
40
- raise Fluent::ConfigError, 'Must define the vendor to use for getting the metrics.' if @target_platform == ''
42
+
43
+ raise Fluent::ConfigError, 'Must define the target_platform to be one of the following (DATADOG, NEWRELIC).' if @target_platform == '' || !(['DATADOG', 'NEWRELIC'].include?(@target_platform))
41
44
 
42
45
  raise Fluent::ConfigError, 'Must define the apikey to use for authentication.' if @apikey == ''
43
46
  end
44
47
 
45
48
  def process(tag, es)
49
+ logger = log
46
50
  es.each do |time, record|
47
- if @target_platform == 'NEWRELIC'
48
- vendor = NewRelicMetrics.new(@apikey, @url)
49
- vendor.send_metrics(record, @http_proxy, @verify_ssl)
50
- elsif @target_platform == 'DATADOG'
51
- vendor = DatadogMetrics.new(@apikey, @url)
52
- vendor.send_metrics(@ddtags, record, @http_proxy, @verify_ssl)
51
+ begin
52
+ logger.info("Sending metrics to target platform: #{@target_platform} started")
53
+ if @target_platform == 'NEWRELIC'
54
+ vendor = NewRelicMetrics.new(@apikey, @url)
55
+ vendor.send_metrics(record, @http_proxy, @verify_ssl, @request_timeout, @gzip_compression, logger)
56
+ elsif @target_platform == 'DATADOG'
57
+ vendor = DatadogMetrics.new(@apikey, @url)
58
+ vendor.send_metrics(@ddtags, record, @http_proxy, @verify_ssl, @request_timeout, @gzip_compression, logger)
59
+ end
60
+ logger.info("Sending metrics to target platform: #{@target_platform} finished")
61
+ rescue RestClient::Exceptions::OpenTimeout
62
+ logger.info("#{Utility.get_time} The request timed out while trying to open a connection. The configured request timeout is: #{@request_timeout}")
63
+ rescue RestClient::Exceptions::ReadTimeout
64
+ logger.info("#{Utility.get_time} The request timed out while waiting for a response. The configured request timeout is: #{@request_timeout}")
65
+ rescue RestClient::Exceptions::RequestTimeout
66
+ logger.info("#{Utility.get_time} The request timed out. The configured request timeout is: #{@request_timeout}")
67
+ rescue RestClient::ExceptionWithResponse => e
68
+ logger.info("#{Utility.get_time} HTTP request failed: #{e.response}")
69
+ rescue Net::HTTPClientException => e
70
+ logger.info("#{Utility.get_time} An HTTP client error occurred when sending metrics to #{@target_platform}: #{e.message}")
71
+ rescue StandardError => e
72
+ logger.info("#{Utility.get_time} An error occurred when sending metrics to #{@target_platform}: #{e.message}")
53
73
  end
54
74
  end
55
75
  end
@@ -0,0 +1,39 @@
1
+ require 'zlib'
2
+
3
+ class Utility
4
+ def self.compress_payload(payload)
5
+ puts "#{self.get_time} Compressing metrics payload"
6
+ json_payload = payload.to_json
7
+
8
+ # Compress the JSON string using zlib's GzipWriter
9
+ compressed_payload = StringIO.new
10
+ Zlib::GzipWriter.wrap(compressed_payload) do |gz|
11
+ begin
12
+ gz.write(json_payload)
13
+ ensure
14
+ gz.close
15
+ end
16
+ end
17
+
18
+ # convert the compressed data to a string
19
+ compressed_data = compressed_payload.string
20
+
21
+ puts "#{self.get_time} Original metrics payload size: #{json_payload.bytesize} bytes"
22
+ puts "#{self.get_time} Compressed metrics payload size: #{compressed_data.bytesize} bytes"
23
+
24
+ return compressed_data
25
+ end
26
+
27
+ def self.get_time()
28
+ return Time.now.strftime("%Y-%m-%d %H:%M:%S")
29
+ end
30
+
31
+ def self.enable_http_client_logs(http_client_logs)
32
+ if http_client_logs
33
+ RestClient.log = STDOUT
34
+ else
35
+ RestClient.log = nil
36
+ end
37
+ end
38
+
39
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-jfrog-send-metrics
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.7
4
+ version: 0.1.9
5
5
  platform: ruby
6
6
  authors:
7
7
  - MahithaB
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2024-09-12 00:00:00.000000000 Z
12
+ date: 2024-11-15 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: bundler
@@ -118,6 +118,7 @@ files:
118
118
  - lib/fluent/plugin/datadog_metrics_sender.rb
119
119
  - lib/fluent/plugin/newrelic_metrics_sender.rb
120
120
  - lib/fluent/plugin/out_jfrog_send_metrics.rb
121
+ - lib/fluent/plugin/utility.rb
121
122
  - spec/fixtures/files/creds.rb
122
123
  - spec/fixtures/files/sample_artifactory_newrelic_metrics.txt
123
124
  - spec/fixtures/files/sample_xray_newrelic_metrics.txt