fluent-plugin-jfrog-metrics 0.1.0 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 880965fb77886a036bc1de886a64b1a28a81be51303981359e801d44bd425d32
-  data.tar.gz: 895972cbf8078c299a777b31e286644200e2716514c3fc22697efa00fa2013d6
+  metadata.gz: 35d89c89f714cfeab9228b18f16d1933f8b38074b7df8806a4bc195125c3c69b
+  data.tar.gz: f10912f41f6bfc0afd9fa0834a89d2773019c9d2ba556d2867d5197240181569
 SHA512:
-  metadata.gz: 40e81bba201bfec407c34e556843501ad9011490b6a2c55dc179fb5e2948948e05df86e2654af3c96f1ea6498f94a80771947340d850a03798b7fdc306a40fa1
-  data.tar.gz: da1ead96440264cf9e12180fd4fca0cbffd9e838a8b8cf98f9e35dfe0876e32fa52dfa659fd7b9975175387595a007080f76ad662395a11cb70a972beb182f6c
+  metadata.gz: 3b0a8f44d52302a02a854626901a925da9f0b94d1b594b9c3fb1fd933d20fd68b34a8275628699f196718f219c681d6bcfaf78639861ae80c8d70001a3cf209b
+  data.tar.gz: a3d33863e73d03fe8658e17fd380c5635827b740bac955ede041560c85d879c6662f188c03555962a1e6e879b660c54a1d17687682587f3652b72b7c2a48d6c9

data/Gemfile

@@ -1,8 +1,11 @@
-source "https://rubygems.org"
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
 
 gemspec
 
 group :test do
-  gem "minitest"
-  gem "minitest-reporters", '>= 0.5.0'
-end
+  gem 'minitest'
+  gem 'minitest-reporters', '>= 0.5.0'
+  gem 'rest-client', '>= 2.1.0'
+end

data/Gemfile.lock

@@ -11,30 +11,44 @@ GEM
     builder (3.2.4)
     concurrent-ruby (1.1.9)
     cool.io (1.7.1)
-    fluentd (1.14.2)
+    domain_name (0.5.20190701)
+      unf (>= 0.0.5, < 1.0.0)
+    fluentd (1.14.5)
       bundler
       cool.io (>= 1.4.5, < 2.0.0)
-      http_parser.rb (>= 0.5.1, < 0.8.0)
+      http_parser.rb (>= 0.5.1, < 0.9.0)
       msgpack (>= 1.3.1, < 2.0.0)
-      serverengine (>= 2.2.2, < 3.0.0)
+      serverengine (>= 2.2.5, < 3.0.0)
       sigdump (~> 0.2.2)
       strptime (>= 0.2.4, < 1.0.0)
       tzinfo (>= 1.0, < 3.0)
       tzinfo-data (~> 1.0)
       webrick (>= 1.4.2, < 1.8.0)
       yajl-ruby (~> 1.0)
-    http_parser.rb (0.7.0)
+    http-accept (1.7.0)
+    http-cookie (1.0.4)
+      domain_name (~> 0.5)
+    http_parser.rb (0.8.0)
+    mime-types (3.4.1)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2021.1115)
     minitest (5.14.4)
     minitest-reporters (1.4.3)
       ansi
       builder
       minitest (>= 5.0)
       ruby-progressbar
-    msgpack (1.4.2)
+    msgpack (1.4.5)
+    netrc (0.11.0)
     power_assert (2.0.1)
     rake (12.3.3)
+    rest-client (2.1.0)
+      http-accept (>= 1.7.0, < 2.0)
+      http-cookie (>= 1.0.2, < 2.0)
+      mime-types (>= 1.16, < 4.0)
+      netrc (~> 0.8)
     ruby-progressbar (1.11.0)
-    serverengine (2.2.4)
+    serverengine (2.2.5)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
     strptime (0.2.5)
@@ -44,6 +58,9 @@ GEM
       concurrent-ruby (~> 1.0)
     tzinfo-data (1.2021.5)
       tzinfo (>= 1.0.0)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.8)
     webrick (1.7.0)
     yajl-ruby (1.4.1)
 
@@ -56,6 +73,7 @@ DEPENDENCIES
   minitest
   minitest-reporters (>= 0.5.0)
   rake (~> 12.0)
+  rest-client (>= 2.1.0)
   test-unit (~> 3.0)
 
 BUNDLED WITH

data/README.md

@@ -73,14 +73,24 @@ wget https://raw.githubusercontent.com/jfrog/metrics/master/splunk/splunk_metric
 
 #### Configuration parameters
 Integration is done by setting up Xray. Obtain JPD url and access token for API. Configure the source directive parameters specified below
-* **tag** (string) (required): The value is the tag assigned to the generated events.
-* **jpd_url** (string) (required): JPD url required to pull Xray SIEM violations
-* **apikey** (string) (required): API Key is the [Artifactory API Key](https://www.jfrog.com/confluence/display/JFROG/User+Profile#UserProfile-APIKey) for authentication
+* **tag** (string) (required): The value is the tag assigned to the generated metrics.
+* **jpd_url** (string) (required): JPD url required to pull metrics, (note - if deployed on K8s use the localhost and port number combination per sidecar)
 * **username** (string) (required): USER is the Artifactory username for authentication
+* **apikey** (string) (required if token is not used, do refer Note section for specifics): API Key is the [Artifactory API Key](https://www.jfrog.com/confluence/display/JFROG/User+Profile#UserProfile-APIKey) for authentication
+* **token** (string) (required if apikey is not used, do refer Note section for specifics): Admin token is the [Artifactory Scoped Tokens](https://www.jfrog.com/confluence/display/JFROG/Access+Tokens#AccessTokens-GeneratingAdminTokens) for authentication
 * **metric_prefix** (string) (required): This values pulls the specific metrics. Values can be - jfrog.artifactory, jfrog.xray
 * **interval** (integer) (optional): Wait interval between pulling new events
     * Default value: `60`
-    
+* **common_jpd** (true / false) (optional): This flag should be set as true only for non-kubernetes installations or installations where JPD base URL is same to access both Artifactory and Xray, 
+     * ex: https://sample_base_url/artifactory or https://sample_base_url/xray 
+     * Default value: false
+  
+Note: 
+
+* For Artifactory v7.4 and below only API Key must be used,
+* For Artifactory v7.4 to 7.29 either Token or API Key can be used,
+* For Artifactory v7.30 and above token only must be used. 
+ 
 ## Copyright
 * Copyright(c) 2020 - JFrog
 * License

data/fluent-plugin-jfrog-metrics.gemspec

@@ -3,12 +3,12 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = 'fluent-plugin-jfrog-metrics'
-  spec.version = '0.1.0'
-  spec.authors = ['MahithaB, VasukiN']
-  spec.email   = ['60710901+MahithaB@users.noreply.github.com']
+  spec.version = '0.1.2'
+  spec.authors = ['MahithaB, VasukiN, GiriR']
+  spec.email   = ['cpe-support@jfrog.com']
 
-  spec.summary       = %q{Fluentd Plugin for converting one metrics form of data to another, this is from Prometheus format to Splunk HEC required format}
-  spec.description   = %q{Fluentd Plugin for converting one metrics form of data to another, this is from Prometheus format to Splunk HEC required format}
+  spec.summary       = %q{Fluentd Plugin for converting JFrog Artifactory, Xray generated metrics (Prometheus Exposition Format) to target observability platform format (Splunk HEC)}
+  spec.description   = %q{Fluentd Plugin for converting JFrog Artifactory, Xray generated metrics (Prometheus Exposition Format) to target observability platform format (Splunk HEC)}
   spec.homepage      = 'https://github.com/jfrog/jfrog-fluentd-plugins/tree/main/fluent-plugin-jfrog-metrics'
   spec.license       = 'Apache-2.0'
 

data/lib/fluent/plugin/in_jfrog_metrics.rb

@@ -32,8 +32,10 @@ module Fluent
       config_param :jpd_url, :string, default: ''
       config_param :username, :string, default: ''
       config_param :apikey, :string, default: ''
+      config_param :token, :string, default: ''
       config_param :interval, :time, default: 10
       config_param :metric_prefix, :string, default: ''
+      config_param :common_jpd, :bool, default: false
 
       # `configure` is called before `start`.
       # 'conf' is a `Hash` that includes the configuration parameters.
@@ -42,11 +44,11 @@ module Fluent
         super
         raise Fluent::ConfigError, 'Must define the tag for metrics data.' if @tag == ''
 
-        raise Fluent::ConfigError, 'Must define the jpd_url to scrape metrics' if @jpd_url == ''
+        raise Fluent::ConfigError, 'Must define the jpd_url to scrape metrics.' if @jpd_url == ''
 
-        raise Fluent::ConfigError, 'Must define the username for authentication' if @username == ''
+        raise Fluent::ConfigError, 'Must define the username for authentication.' if @username == ''
 
-        raise Fluent::ConfigError, 'Must define the apikey to use for authentication.' if @apikey == ''
+        raise Fluent::ConfigError, 'Must define the apikey or token for authentication.' if @token == '' &&  @apikey == ''
 
         raise Fluent::ConfigError, 'Must define the interval to use for gathering the metrics.' if @interval == ''
 
@@ -77,8 +79,12 @@ module Fluent
 
       def do_execute
         puts('Executing metrics collection')
-        metrics_helper = MetricsHelper.new(@metric_prefix, @jpd_url, @username, @apikey)
+        metrics_helper = MetricsHelper.new(@metric_prefix, @jpd_url, @username, @apikey, @token, @common_jpd)
         platform_metrics = metrics_helper.get_metrics
+        additional_metrics = metrics_helper.get_additional_metrics
+        if !additional_metrics.nil? && additional_metrics != ''
+          platform_metrics += additional_metrics.to_s
+        end
         parser = MetricsParser.new(@metric_prefix, router, @tag)
         parser.emit_parsed_metrics(platform_metrics)
       end

data/lib/fluent/plugin/metrics_helper.rb

@@ -1,12 +1,17 @@
 # frozen_string_literal: true
+
 require 'rest-client'
 
 class MetricsHelper
-  def initialize(metric_prefix, jpd_url, username, apikey)
+  @@obs_endpoint_exists = false
+
+  def initialize(metric_prefix, jpd_url, username, apikey, token, common_jpd)
     @metric_prefix = metric_prefix
     @jpd_url = jpd_url
     @username = username
     @apikey = apikey
+    @token = token
+    @common_jpd = common_jpd
   end
 
   def get_metrics
@@ -19,22 +24,62 @@ class MetricsHelper
           else
             "#{@jpd_url}/artifactory/api/v1/metrics"
           end
-    execute_rest_call(url, @username, @apikey)
+    if !@token.nil? && @token != ''
+      execute_rest_call(url, @username, nil, @token, false, true)
+    elsif !@apikey.nil? && @apikey != ''
+      execute_rest_call(url, @username, @apikey, nil, false, false)
+    end
+
+  end
+
+  def get_additional_metrics
+    if (@metric_prefix == 'jfrog.artifactory' || @common_jpd == false) && !@token.nil? && @token != ''
+      puts 'Executing additional metrics collection'
+      url = "#{@jpd_url}/observability/api/v1/metrics"
+      check_endpoint(url, @token) if @@obs_endpoint_exists == nil? || !@@obs_endpoint_exists
+      execute_rest_call(url, @username, nil, @token, true, true) if @@obs_endpoint_exists
+    end
   end
 
-  def execute_rest_call(url, user, password)
+  def check_endpoint(url, token)
     response = RestClient::Request.new(
       method: :get,
       url: url,
-      user: user,
-      password: password
+      headers: { Authorization: "Bearer #{token}" }
     ).execute do |response, request, result|
+      @@obs_endpoint_exists = true if response.code == 200
+      puts "#{url} exists? -> #{@@obs_endpoint_exists}, storing the result for next executions"
+    end
+  end
+
+  def execute_rest_call(url, user, password, token, ignore_exception, use_token)
+    request = if use_token == true
+                RestClient::Request.new(
+                  method: :get,
+                  url: url,
+                  headers: { Authorization: "Bearer #{token}" }
+                )
+              else
+                RestClient::Request.new(
+                  method: :get,
+                  url: url,
+                  user: user,
+                  password: password
+                )
+              end
+
+    request.execute do |response, request, result|
       case response.code
       when 200
         return response.body
       else
-        raise Fluent::ConfigError, 'Cannot fetch #{@metric_prefix} metrics'
+        if ignore_exception == true
+          return ''
+        else
+          raise Fluent::ConfigError, 'Cannot fetch #{@metric_prefix} metrics'
+        end
       end
     end
   end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-jfrog-metrics
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.2
 platform: ruby
 authors:
-- MahithaB, VasukiN
-autorequire: 
+- MahithaB, VasukiN, GiriR
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-12-14 00:00:00.000000000 Z
+date: 2022-03-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -72,10 +72,10 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
-description: Fluentd Plugin for converting one metrics form of data to another, this
-  is from Prometheus format to Splunk HEC required format
+description: Fluentd Plugin for converting JFrog Artifactory, Xray generated metrics
+  (Prometheus Exposition Format) to target observability platform format (Splunk HEC)
 email:
-- 60710901+MahithaB@users.noreply.github.com
+- cpe-support@jfrog.com
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -103,7 +103,7 @@ homepage: https://github.com/jfrog/jfrog-fluentd-plugins/tree/main/fluent-plugin
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -118,11 +118,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.1.6
+signing_key:
 specification_version: 4
-summary: Fluentd Plugin for converting one metrics form of data to another, this is
-  from Prometheus format to Splunk HEC required format
+summary: Fluentd Plugin for converting JFrog Artifactory, Xray generated metrics (Prometheus
+  Exposition Format) to target observability platform format (Splunk HEC)
 test_files:
 - spec/fixtures/files/creds.rb
 - spec/fixtures/files/sample_artifactory_metrics.txt