fluent-plugin-jfrog-metrics 0.1.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 880965fb77886a036bc1de886a64b1a28a81be51303981359e801d44bd425d32
-  data.tar.gz: 895972cbf8078c299a777b31e286644200e2716514c3fc22697efa00fa2013d6
+  metadata.gz: ba1a5b8b11e87e7309ca6831f129fc0a5cffb8dbbf0a89bf4fa753f3b21850e9
+  data.tar.gz: 474f79299a90b246ba24e00f4e204a2929643b50afa43df976bfc50664729663
 SHA512:
-  metadata.gz: 40e81bba201bfec407c34e556843501ad9011490b6a2c55dc179fb5e2948948e05df86e2654af3c96f1ea6498f94a80771947340d850a03798b7fdc306a40fa1
-  data.tar.gz: da1ead96440264cf9e12180fd4fca0cbffd9e838a8b8cf98f9e35dfe0876e32fa52dfa659fd7b9975175387595a007080f76ad662395a11cb70a972beb182f6c
+  metadata.gz: 2c10e6dd3dd51fbb7ba9c195df17bf459e1500d53c50b1b607d066c63dfe0703d34525dd1c0f3764913e3d236f0edd3d673476efb21001561eff7f4d4e5b8204
+  data.tar.gz: 435cc105038ce6dc74183a1933aefffbd572da45460589d229326bc4b5b3bba61f7a7dabf2441c825aafbe999a1477580a8aeff4e3b73e15829b4c9c7112723e

data/.gitignore

@@ -1 +1,3 @@
 .DS_Store
+fluent-plugin-jfrog-metrics-*.gem
+fluent-plugin-jfrog-metrics.iml

data/Gemfile

@@ -1,8 +1,11 @@
-source "https://rubygems.org"
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
 
 gemspec
 
 group :test do
-  gem "minitest"
-  gem "minitest-reporters", '>= 0.5.0'
-end
+  gem 'minitest'
+  gem 'minitest-reporters', '>= 0.5.0'
+  gem 'rest-client', '>= 2.1.0'
+end

data/Gemfile.lock

@@ -1,8 +1,9 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-jfrog-metrics (0.1.2)
+    fluent-plugin-jfrog-metrics (0.2.1)
       fluentd (>= 0.14.10, < 2)
+      rest-client (~> 2.0)
 
 GEM
   remote: https://rubygems.org/
@@ -11,30 +12,44 @@ GEM
     builder (3.2.4)
     concurrent-ruby (1.1.9)
     cool.io (1.7.1)
-    fluentd (1.14.2)
+    domain_name (0.5.20190701)
+      unf (>= 0.0.5, < 1.0.0)
+    fluentd (1.14.5)
       bundler
       cool.io (>= 1.4.5, < 2.0.0)
-      http_parser.rb (>= 0.5.1, < 0.8.0)
+      http_parser.rb (>= 0.5.1, < 0.9.0)
       msgpack (>= 1.3.1, < 2.0.0)
-      serverengine (>= 2.2.2, < 3.0.0)
+      serverengine (>= 2.2.5, < 3.0.0)
       sigdump (~> 0.2.2)
       strptime (>= 0.2.4, < 1.0.0)
       tzinfo (>= 1.0, < 3.0)
       tzinfo-data (~> 1.0)
       webrick (>= 1.4.2, < 1.8.0)
       yajl-ruby (~> 1.0)
-    http_parser.rb (0.7.0)
+    http-accept (1.7.0)
+    http-cookie (1.0.4)
+      domain_name (~> 0.5)
+    http_parser.rb (0.8.0)
+    mime-types (3.4.1)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2021.1115)
     minitest (5.14.4)
     minitest-reporters (1.4.3)
       ansi
       builder
       minitest (>= 5.0)
       ruby-progressbar
-    msgpack (1.4.2)
+    msgpack (1.4.5)
+    netrc (0.11.0)
     power_assert (2.0.1)
     rake (12.3.3)
+    rest-client (2.1.0)
+      http-accept (>= 1.7.0, < 2.0)
+      http-cookie (>= 1.0.2, < 2.0)
+      mime-types (>= 1.16, < 4.0)
+      netrc (~> 0.8)
     ruby-progressbar (1.11.0)
-    serverengine (2.2.4)
+    serverengine (2.2.5)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
     strptime (0.2.5)
@@ -44,6 +59,9 @@ GEM
       concurrent-ruby (~> 1.0)
     tzinfo-data (1.2021.5)
       tzinfo (>= 1.0.0)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.8)
     webrick (1.7.0)
     yajl-ruby (1.4.1)
 
@@ -56,6 +74,7 @@ DEPENDENCIES
   minitest
   minitest-reporters (>= 0.5.0)
   rake (~> 12.0)
+  rest-client (>= 2.1.0)
   test-unit (~> 3.0)
 
 BUNDLED WITH

data/README.md

@@ -6,19 +6,18 @@
 
 To build / test locally use rake:
 
-``` 
+```
 rake
 ```
 
 To build install locally use bundler:
 
-``` 
+```
 bundle install
 ```
 
 This will install the gem shown below from source.
 
-
 ## Development
 
 ### Bundler
@@ -45,44 +44,64 @@ $ fluent-plugin-config-format input jfrog-metrics
 
 You can copy and paste generated documents here.
 
-## Installation 
+## Installation
 
 ### RubyGems
+
 ```
 $ gem install rest-client
 ```
+
 ```
 $ gem install thread
 ```
+
 ```
 $ gem install fluent-plugin-jfrog-metrics
 ```
 
 ### Setup & configuration
-Fluentd is the supported log collector for this integration. 
+
+Fluentd is the supported log collector for this integration.
 For Fluentd setup and information, read the JFrog log analytics repository's [README.](https://github.com/jfrog/log-analytics/blob/master/README.md)
 
 #### Fluentd Output
+
 Download fluentd conf for different log-vendors. For example
-Splunk: 
+Splunk:
 
 Splunk setup can be found at [README.](https://github.com/jfrog/metrics/blob/main/splunk/README.md)
-````text
+
+```text
 wget https://raw.githubusercontent.com/jfrog/metrics/master/splunk/splunk_metrics.conf
-````
+```
 
 #### Configuration parameters
+
 Integration is done by setting up Xray. Obtain JPD url and access token for API. Configure the source directive parameters specified below
-* **tag** (string) (required): The value is the tag assigned to the generated events.
-* **jpd_url** (string) (required): JPD url required to pull Xray SIEM violations
-* **apikey** (string) (required): API Key is the [Artifactory API Key](https://www.jfrog.com/confluence/display/JFROG/User+Profile#UserProfile-APIKey) for authentication
-* **username** (string) (required): USER is the Artifactory username for authentication
-* **metric_prefix** (string) (required): This values pulls the specific metrics. Values can be - jfrog.artifactory, jfrog.xray
-* **interval** (integer) (optional): Wait interval between pulling new events
-    * Default value: `60`
-    
+
+- **tag** (string) (required): The value is the tag assigned to the generated metrics.
+- **jpd_url** (string) (required): JPD url required to pull metrics, (note - if deployed on K8s use the localhost and port number combination per sidecar)
+- **username** (string) (required): USER is the Artifactory username for authentication
+- **apikey** (string) (required if token is not used, do refer Note section for specifics): API Key is the [Artifactory API Key](https://www.jfrog.com/confluence/display/JFROG/User+Profile#UserProfile-APIKey) for authentication
+- **token** (string) (required if apikey is not used, do refer Note section for specifics): Admin token is the [Artifactory Scoped Tokens](https://www.jfrog.com/confluence/display/JFROG/Access+Tokens#AccessTokens-GeneratingAdminTokens) for authentication
+- **metric_prefix** (string) (required): This values pulls the specific metrics. Values can be - jfrog.artifactory, jfrog.xray
+- **interval** (integer) (optional): Wait interval between pulling new events
+  - Default value: `60`
+- **common_jpd** (true / false) (optional): This flag should be set as true only for non-kubernetes installations or installations where JPD base URL is same to access both Artifactory and Xray,
+  - ex: https://sample_base_url/artifactory or https://sample_base_url/xray
+  - Default value: false
+- **target_platform** (string) (optional): Output format of target platform allowed values SPLUNK and ELASTIC
+  - Default value: `SPLUNK`
+
+Note:
+
+- For Artifactory v7.4 and below only API Key must be used,
+- For Artifactory v7.4 to 7.29 either Token or API Key can be used,
+- For Artifactory v7.30 and above token only must be used.
+
 ## Copyright
-* Copyright(c) 2020 - JFrog
-* License
-  * Apache License, Version 2.0
 
+- Copyright(c) 2020 - JFrog
+- License
+  - Apache License, Version 2.0

data/fluent-plugin-jfrog-metrics.gemspec

@@ -3,12 +3,13 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = 'fluent-plugin-jfrog-metrics'
-  spec.version = '0.1.0'
-  spec.authors = ['MahithaB, VasukiN']
-  spec.email   = ['60710901+MahithaB@users.noreply.github.com']
+  spec.version = '0.2.1'
+  spec.authors = ['MahithaB, VasukiN, giri-vsr']
+  spec.email   = ['cpe-support@jfrog.com']
+
+  spec.summary       = %q{Fluentd Plugin for converting JFrog Artifactory, Xray generated metrics (Prometheus Exposition Format) to target observability platform format (Splunk HEC, New Relic, Elastic)}
+  spec.description   = %q{Fluentd Plugin for converting JFrog Artifactory, Xray generated metrics (Prometheus Exposition Format) to target observability platform format (Splunk HEC, New Relic, Elastic)}
 
-  spec.summary       = %q{Fluentd Plugin for converting one metrics form of data to another, this is from Prometheus format to Splunk HEC required format}
-  spec.description   = %q{Fluentd Plugin for converting one metrics form of data to another, this is from Prometheus format to Splunk HEC required format}
   spec.homepage      = 'https://github.com/jfrog/jfrog-fluentd-plugins/tree/main/fluent-plugin-jfrog-metrics'
   spec.license       = 'Apache-2.0'
 
@@ -23,5 +24,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'bundler', '~> 1.14'
   spec.add_development_dependency 'rake', '~> 12.0'
   spec.add_development_dependency 'test-unit', '~> 3.0'
+  spec.add_development_dependency "rest-client", "~> 2.0"
   spec.add_runtime_dependency 'fluentd', ['>= 0.14.10', '< 2']
+  spec.add_runtime_dependency "rest-client", "~> 2.0"
 end

data/lib/fluent/plugin/base_metrics_parser.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+require 'json'
+
+class BaseMetricsParser
+
+  def normalise_data(data_to_normalize = [])
+    normalized_data = []
+    if data_to_normalize.length == 1
+      data_to_normalize.each do |interim_data|
+        normalized_data = interim_data.split(/\\n+|\\r+/).reject(&:empty?)
+      end
+    else
+      data_to_normalize.each_line do |interim_data|
+        normalized_data << interim_data.strip unless interim_data == "\n"
+      end
+    end
+    normalized_data
+  end
+
+  def clean_data(data_to_clean = [])
+    cleaned_data = []
+    data_to_clean.each do |interim_data|
+      cleaned_data << interim_data unless interim_data.include? '#'
+    end
+    cleaned_data
+  end
+
+  def serialize_data(hashed_data_array = [])
+    serialised_data = []
+    hashed_data_array.each do |interim_data|
+      serialised_data << JSON.parse(interim_data.to_json)
+    end
+    serialised_data
+  end
+
+  def emit_parsed_metrics(platform_metrics)
+    normalized_data = normalise_data(platform_metrics)
+    cleaned_data = clean_data(normalized_data)
+    hash_data_array = format_data(cleaned_data, @metric_prefix, '.')
+    serialized_data = serialize_data(hash_data_array)
+    serialized_data.each do |interim_data|
+      @router.emit(@tag, Fluent::Engine.now, interim_data)
+    end
+  end
+end

data/lib/fluent/plugin/elastic_metrics_parser.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+require 'json'
+require_relative 'base_metrics_parser'
+
+class ElasticMetricsParser < BaseMetricsParser
+  def initialize(metric_prefix, router, tag)
+    @metric_prefix = metric_prefix
+    @router = router
+    @tag = tag
+  end
+
+  def format_data(cleaned_data = [], prefix = '', separator = '')
+    hash_data_array = []
+    hash_data = {}
+    cleaned_data.each do |interim_data|
+      if interim_data =~ /{/ && interim_data =~ /}/
+        metric_name, additional_dims, metric_val_and_time = interim_data.match(/(.*){(.*)}(.*)/i).captures
+        additional_dim_name = ''
+        if additional_dims =~ /,/
+          additional_dims.split(/,/).map do |interim_dim_data|
+            additional_dim_name += '_' unless additional_dim_name.nil? || additional_dim_name.empty?
+            additional_dim_name += interim_dim_data.gsub(/"/, '').gsub(/=/, '_').gsub(/-/, '_') if interim_data =~ /=/
+          end
+        else
+          additional_dim_name = additional_dims.gsub(/"/, '').gsub(/=/, '_').gsub(/-/, '_') if interim_data =~ /=/
+        end
+        if metric_val_and_time =~ / /
+          hash_data["#{prefix}#{separator}#{metric_name}_#{additional_dim_name}"] =
+              metric_val_and_time.strip.split[0] =~ /^\S*\.\S*$/ ? metric_val_and_time.strip.split[0].to_f : metric_val_and_time.strip.split[0].to_i
+        end
+      else
+        metric_name, value, = interim_data.split
+        hash_data[prefix + separator + metric_name] =
+            value =~ /^\S*\.\S*$/ ? value.to_f : value.to_i
+      end
+    end
+    hash_data_array << hash_data
+  end
+end

data/lib/fluent/plugin/in_jfrog_metrics.rb

@@ -17,8 +17,11 @@
 require 'concurrent'
 require 'rest-client'
 require 'fluent/plugin/input'
-require_relative 'metrics_parser'
+require_relative 'base_metrics_parser'
 require_relative 'metrics_helper'
+require_relative 'newrelic_metrics_parser'
+require_relative 'splunk_metrics_parser'
+require_relative 'elastic_metrics_parser'
 
 module Fluent
   module Plugin
@@ -31,9 +34,12 @@ module Fluent
       config_param :tag, :string, default: ''
       config_param :jpd_url, :string, default: ''
       config_param :username, :string, default: ''
-      config_param :apikey, :string, default: ''
+      config_param :apikey, :string, default: '', :secret => true
+      config_param :token, :string, default: '', :secret => true
       config_param :interval, :time, default: 10
       config_param :metric_prefix, :string, default: ''
+      config_param :target_platform, :string, default: 'SPLUNK'
+      config_param :common_jpd, :bool, default: false
 
       # `configure` is called before `start`.
       # 'conf' is a `Hash` that includes the configuration parameters.
@@ -42,15 +48,17 @@ module Fluent
         super
         raise Fluent::ConfigError, 'Must define the tag for metrics data.' if @tag == ''
 
-        raise Fluent::ConfigError, 'Must define the jpd_url to scrape metrics' if @jpd_url == ''
+        raise Fluent::ConfigError, 'Must define the jpd_url to scrape metrics.' if @jpd_url == ''
 
-        raise Fluent::ConfigError, 'Must define the username for authentication' if @username == ''
+        raise Fluent::ConfigError, 'Must define the username for authentication.' if @username == ''
 
-        raise Fluent::ConfigError, 'Must define the apikey to use for authentication.' if @apikey == ''
+        raise Fluent::ConfigError, 'Must define the apikey or token for authentication.' if @token == '' &&  @apikey == ''
 
         raise Fluent::ConfigError, 'Must define the interval to use for gathering the metrics.' if @interval == ''
 
         raise Fluent::ConfigError, 'Must define the metric_prefix to use for getting the metrics.' if @metric_prefix == ''
+
+        raise Fluent::ConfigError, 'Must define the vendor to use for getting the metrics.' if @target_platform == ''
       end
 
       # `start` is called when starting and after `configure` is successfully completed.
@@ -77,9 +85,21 @@ module Fluent
 
       def do_execute
         puts('Executing metrics collection')
-        metrics_helper = MetricsHelper.new(@metric_prefix, @jpd_url, @username, @apikey)
+        metrics_helper = MetricsHelper.new(@metric_prefix, @jpd_url, @username, @apikey, @token, @common_jpd)
         platform_metrics = metrics_helper.get_metrics
-        parser = MetricsParser.new(@metric_prefix, router, @tag)
+        additional_metrics = metrics_helper.get_additional_metrics
+        if !additional_metrics.nil? && additional_metrics != ''
+          platform_metrics += additional_metrics.to_s
+        end
+        if @target_platform == 'SPLUNK'
+          parser = SplunkMetricsParser.new(@metric_prefix, router, @tag)
+        elsif @target_platform == 'ELASTIC'
+          parser = ElasticMetricsParser.new(@metric_prefix, router, @tag)
+        elsif @target_platform == 'NEWRELIC'
+          parser = NewRelicMetricsParser.new(@metric_prefix, router, @tag)
+        else
+          raise 'Parser Type is not valid.Should be SPLUNK or ELASTIC or NEWRELIC'
+        end
         parser.emit_parsed_metrics(platform_metrics)
       end
     end

data/lib/fluent/plugin/metrics_helper.rb

@@ -1,12 +1,17 @@
 # frozen_string_literal: true
+
 require 'rest-client'
 
 class MetricsHelper
-  def initialize(metric_prefix, jpd_url, username, apikey)
+  @@obs_endpoint_exists = false
+
+  def initialize(metric_prefix, jpd_url, username, apikey, token, common_jpd)
     @metric_prefix = metric_prefix
     @jpd_url = jpd_url
     @username = username
     @apikey = apikey
+    @token = token
+    @common_jpd = common_jpd
   end
 
   def get_metrics
@@ -19,22 +24,62 @@ class MetricsHelper
           else
             "#{@jpd_url}/artifactory/api/v1/metrics"
           end
-    execute_rest_call(url, @username, @apikey)
+    if !@token.nil? && @token != ''
+      execute_rest_call(url, @username, nil, @token, false, true)
+    elsif !@apikey.nil? && @apikey != ''
+      execute_rest_call(url, @username, @apikey, nil, false, false)
+    end
+
+  end
+
+  def get_additional_metrics
+    if (@metric_prefix == 'jfrog.artifactory' || @common_jpd == false) && !@token.nil? && @token != ''
+      puts 'Executing additional metrics collection'
+      url = "#{@jpd_url}/observability/api/v1/metrics"
+      check_endpoint(url, @token) if @@obs_endpoint_exists == nil? || !@@obs_endpoint_exists
+      execute_rest_call(url, @username, nil, @token, true, true) if @@obs_endpoint_exists
+    end
   end
 
-  def execute_rest_call(url, user, password)
+  def check_endpoint(url, token)
     response = RestClient::Request.new(
       method: :get,
       url: url,
-      user: user,
-      password: password
+      headers: { Authorization: "Bearer #{token}" }
     ).execute do |response, request, result|
+      @@obs_endpoint_exists = true if response.code == 200
+      puts "#{url} exists? -> #{@@obs_endpoint_exists}, storing the result for next executions"
+    end
+  end
+
+  def execute_rest_call(url, user, password, token, ignore_exception, use_token)
+    request = if use_token == true
+                RestClient::Request.new(
+                  method: :get,
+                  url: url,
+                  headers: { Authorization: "Bearer #{token}" }
+                )
+              else
+                RestClient::Request.new(
+                  method: :get,
+                  url: url,
+                  user: user,
+                  password: password
+                )
+              end
+
+    request.execute do |response, request, result|
       case response.code
       when 200
         return response.body
       else
-        raise Fluent::ConfigError, 'Cannot fetch #{@metric_prefix} metrics'
+        if ignore_exception == true
+          return ''
+        else
+          raise Fluent::ConfigError, 'Cannot fetch #{@metric_prefix} metrics'
+        end
       end
     end
   end
+
 end

data/lib/fluent/plugin/newrelic_metrics_parser.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+require 'json'
+require_relative 'base_metrics_parser'
+
+class NewRelicMetricsParser < BaseMetricsParser
+  def initialize(metric_prefix, router, tag)
+    @metric_prefix = metric_prefix
+    @router = router
+    @tag = tag
+  end
+
+  def format_data(cleaned_data = [], prefix = '', separator = '')
+    hash_data_array = []
+    data_hash = {}
+    data_array = []
+    cleaned_data.each do |interim_data|
+      metrics_hash = {}
+      if interim_data =~ /{/ && interim_data =~ /}/
+        attributes = {}
+        metric_name, additional_dims, metric_val_and_time = interim_data.match(/(.*){(.*)}(.*)/i).captures
+        if additional_dims =~ /,/
+          additional_dims.split(/,/).map do |interim_data|
+            attributes[interim_data.split(/=/)[0]] = interim_data.split(/=/)[1].gsub(/"/, '') if interim_data =~ /=/
+          end
+        else
+          attribute_name, attribute_value = additional_dims.split('=')
+          attributes[attribute_name] = attribute_value.delete_prefix('"').delete_suffix('"')
+        end
+        if metric_val_and_time =~ / /
+          metrics_hash['name'] = prefix + separator + metric_name
+          metrics_hash['value'] =
+              metric_val_and_time.strip.split[0] =~ /^\S*\.\S*$/ ? metric_val_and_time.strip.split[0].to_f : metric_val_and_time.strip.split[0].to_i
+          metrics_hash['timestamp'] = metric_val_and_time.strip.split[1].to_i
+          metrics_hash['attributes'] = attributes
+        end
+      else
+        metrics_hash['name'], metrics_hash['value'], metrics_hash['timestamp'] = interim_data.split
+        metrics_hash['name'] = prefix + separator + metrics_hash['name']
+        metrics_hash['value'] =
+            metrics_hash['value'] =~ /^\S*\.\S*$/ ? metrics_hash['value'].to_f : metrics_hash['value'].to_i
+        metrics_hash['timestamp'] = metrics_hash['timestamp'].to_i
+      end
+      data_array << metrics_hash
+    end
+    data_hash["metrics"] = data_array
+    hash_data_array.push(data_hash)
+    hash_data_array
+  end
+
+end

data/lib/fluent/plugin/splunk_metrics_parser.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+require 'json'
+require_relative 'base_metrics_parser'
+
+class SplunkMetricsParser < BaseMetricsParser
+  def initialize(metric_prefix, router, tag)
+    @metric_prefix = metric_prefix
+    @router = router
+    @tag = tag
+  end
+
+  def format_data(cleaned_data = [], prefix = '', separator = '')
+    hash_data_array = []
+    cleaned_data.each do |interim_data|
+      hash_data_array << generate_hash_from_data(interim_data, prefix, separator)
+    end
+    hash_data_array
+  end
+
+  def generate_hash_from_data(data = '', prefix = '', separator = '')
+    metrics_hash = {}
+    if data =~ /{/ && data =~ /}/
+      metric_name, additional_dims, metric_val_and_time = data.match(/(.*){(.*)}(.*)/i).captures
+      if metric_val_and_time =~ / /
+        metrics_hash['metric_name'] = prefix + separator + metric_name
+        metrics_hash['value'] =
+            metric_val_and_time.strip.split[0] =~ /^\S*\.\S*$/ ? metric_val_and_time.strip.split[0].to_f : metric_val_and_time.strip.split[0].to_i
+        metrics_hash['time'] = metric_val_and_time.strip.split[1].to_i
+        if additional_dims =~ /,/
+          additional_dims.split(/,/).map do |interim_data|
+            metrics_hash[interim_data.split(/=/)[0]] = interim_data.split(/=/)[1].gsub(/"/, '') if interim_data =~ /=/
+          end
+        end
+      end
+    else
+      metrics_hash['metric_name'], metrics_hash['value'], metrics_hash['time'] = data.split
+      metrics_hash['metric_name'] = prefix + separator + metrics_hash['metric_name']
+      metrics_hash['value'] =
+          metrics_hash['value'] =~ /^\S*\.\S*$/ ? metrics_hash['value'].to_f : metrics_hash['value'].to_i
+      metrics_hash['time'] =  metrics_hash['time'].to_i
+    end
+    metrics_hash
+  end
+end

data/spec/fixtures/files/creds.rb

@@ -2,7 +2,7 @@
 $jpd_url = ''
 $username = ''
 $apikey = ''
-
+$token = ''
 def get_credentials
-  [$jpd_url, $username, $apikey]
+  [$jpd_url, $username, $apikey, $token]
 end

data/spec/fixtures/files/sample_artifactory_metrics.txt

@@ -1,64 +1,64 @@
 # HELP sys_memory_used_bytes Host used virtual memory
-# UPDATED sys_memory_used_bytes 1636577501390
+# UPDATED sys_memory_used_bytes 1645738619452
 # TYPE sys_memory_used_bytes gauge
-sys_memory_used_bytes 3836522496 1636577501390
+sys_memory_used_bytes 3836522496 1645738619452
 # HELP sys_memory_free_bytes Host free virtual memory
-# UPDATED sys_memory_free_bytes 1636577501390
+# UPDATED sys_memory_free_bytes 1645738619452
 # TYPE sys_memory_free_bytes gauge
-sys_memory_free_bytes 8396328960 1636577501390
+sys_memory_free_bytes 8396328960 1645738619452
 # HELP jfrt_runtime_heap_freememory_bytes Free Memory
-# UPDATED jfrt_runtime_heap_freememory_bytes 1636577501390
+# UPDATED jfrt_runtime_heap_freememory_bytes 1645738619452
 # TYPE jfrt_runtime_heap_freememory_bytes gauge
-jfrt_runtime_heap_freememory_bytes 464534120 1636577501390
+jfrt_runtime_heap_freememory_bytes 464534120 1645738619452
 # HELP jfrt_runtime_heap_maxmemory_bytes Max Memory
-# UPDATED jfrt_runtime_heap_maxmemory_bytes 1636577501390
+# UPDATED jfrt_runtime_heap_maxmemory_bytes 1645738619452
 # TYPE jfrt_runtime_heap_maxmemory_bytes gauge
-jfrt_runtime_heap_maxmemory_bytes 2147483648 1636577501390
+jfrt_runtime_heap_maxmemory_bytes 2147483648 1645738619452
 # HELP jfrt_runtime_heap_totalmemory_bytes Total Memory
-# UPDATED jfrt_runtime_heap_totalmemory_bytes 1636577501390
+# UPDATED jfrt_runtime_heap_totalmemory_bytes 1645738619452
 # TYPE jfrt_runtime_heap_totalmemory_bytes gauge
-jfrt_runtime_heap_totalmemory_bytes 1358954496 1636577501390
+jfrt_runtime_heap_totalmemory_bytes 1358954496 1645738619452
 # HELP jfrt_runtime_heap_processors_total Available Processors
-# UPDATED jfrt_runtime_heap_processors_total 1636577501390
+# UPDATED jfrt_runtime_heap_processors_total 1645738619452
 # TYPE jfrt_runtime_heap_processors_total counter
-jfrt_runtime_heap_processors_total 1 1636577501390
+jfrt_runtime_heap_processors_total 1 1645738619452
 # HELP jfrt_db_connections_active_total Total Active Connections
 # UPDATED jfrt_db_connections_active_total 1636577471195
 # TYPE jfrt_db_connections_active_total gauge
-jfrt_db_connections_active_total 0 1636577501390
+jfrt_db_connections_active_total 0 1645738619452
 # HELP jfrt_db_connections_idle_total Total Idle Connections
 # UPDATED jfrt_db_connections_idle_total 1636577471195
 # TYPE jfrt_db_connections_idle_total gauge
-jfrt_db_connections_idle_total 1 1636577501390
+jfrt_db_connections_idle_total 1 1645738619452
 # HELP jfrt_db_connections_max_active_total Total Max Active Connections
 # UPDATED jfrt_db_connections_max_active_total 1636577471195
 # TYPE jfrt_db_connections_max_active_total gauge
-jfrt_db_connections_max_active_total 80 1636577501390
+jfrt_db_connections_max_active_total 80 1645738619452
 # HELP jfrt_db_connections_min_idle_total Total Min Idle Connections
 # UPDATED jfrt_db_connections_min_idle_total 1636577471195
 # TYPE jfrt_db_connections_min_idle_total gauge
-jfrt_db_connections_min_idle_total 1 1636577501390
+jfrt_db_connections_min_idle_total 1 1645738619452
 # HELP sys_cpu_ratio Total cpu load ratio
-# UPDATED sys_cpu_ratio 1636577501390
+# UPDATED sys_cpu_ratio 1645738619452
 # TYPE sys_cpu_ratio gauge
-sys_cpu_ratio 0.16 1636577501390
+sys_cpu_ratio 0.16 1645738619452
 # HELP jfrt_projects_active_total Projects Amount
-# UPDATED jfrt_projects_active_total 1636577501390
+# UPDATED jfrt_projects_active_total 1645738619452
 # TYPE jfrt_projects_active_total counter
-jfrt_projects_active_total 0 1636577501390
+jfrt_projects_active_total 0 1645738619452
 # HELP jfrt_storage_current_total_size_bytes Used Storage
-# UPDATED jfrt_storage_current_total_size_bytes 1636577501390
+# UPDATED jfrt_storage_current_total_size_bytes 1645738619452
 # TYPE jfrt_storage_current_total_size_bytes gauge
-jfrt_storage_current_total_size_bytes 0 1636577501390
+jfrt_storage_current_total_size_bytes 0 1645738619452
 # HELP app_disk_used_bytes Used bytes for app home directory disk device
-# UPDATED app_disk_used_bytes 1636577501390
+# UPDATED app_disk_used_bytes 1645738619452
 # TYPE app_disk_used_bytes gauge
-app_disk_used_bytes 730750976 1636577501390
+app_disk_used_bytes 730750976 1645738619452
 # HELP app_disk_free_bytes Free bytes for app home directory disk device
-# UPDATED app_disk_free_bytes 1636577501390
+# UPDATED app_disk_free_bytes 1645738619452
 # TYPE app_disk_free_bytes gauge
-app_disk_free_bytes 209510809600 1636577501390
+app_disk_free_bytes 209510809600 1645738619452
 # HELP jfrt_artifacts_gc_next_run_seconds Next GC Run
 # UPDATED jfrt_artifacts_gc_next_run_seconds 1636574411092
 # TYPE jfrt_artifacts_gc_next_run_seconds gauge
-jfrt_artifacts_gc_next_run_seconds 14388 1636577501390
+jfrt_artifacts_gc_next_run_seconds 14388 1645738619452

data/spec/fixtures/files/sample_xray_metrics.txt

@@ -1,138 +1,138 @@
 # HELP app_disk_used_bytes Used bytes for app home directory disk device
 # TYPE app_disk_used_bytes gauge
-app_disk_used_bytes 1.48081664e+10 1636577556068
+app_disk_used_bytes 1.48081664e+10 1645738679875
 # HELP app_disk_free_bytes Free bytes for app home directory disk device
 # TYPE app_disk_free_bytes gauge
-app_disk_free_bytes 3.356854272e+10 1636577556068
+app_disk_free_bytes 3.356854272e+10 1645738679875
 # HELP app_io_counters_write_bytes Process io total write bytes
 # TYPE app_io_counters_write_bytes gauge
-app_io_counters_write_bytes 1.217306624e+09 1636577556068
+app_io_counters_write_bytes 1.217306624e+09 1645738679875
 # HELP app_io_counters_read_bytes Process io total read bytes
 # TYPE app_io_counters_read_bytes gauge
-app_io_counters_read_bytes 2.10030592e+08 1636577556068
+app_io_counters_read_bytes 2.10030592e+08 1645738679875
 # HELP app_self_metrics_calc_seconds Total time to collect all metrics
 # TYPE app_self_metrics_calc_seconds gauge
-app_self_metrics_calc_seconds 0.050925766 1636577556068
+app_self_metrics_calc_seconds 0.050925766 1645738679875
 # HELP app_self_metrics_total Count of collected metrics
 # TYPE app_self_metrics_total gauge
-app_self_metrics_total 31 1636577556068
+app_self_metrics_total 31 1645738679875
 # HELP db_connection_pool_in_use_total The number of connections currently in use
 # TYPE db_connection_pool_in_use_total gauge
-db_connection_pool_in_use_total 0 1636577556068
+db_connection_pool_in_use_total 0 1645738679875
 # HELP db_connection_pool_idle_total The number of idle connections
 # TYPE db_connection_pool_idle_total gauge
-db_connection_pool_idle_total 5 1636577556068
+db_connection_pool_idle_total 5 1645738679875
 # HELP db_connection_pool_max_open_total The maximum number of open connections
 # TYPE db_connection_pool_max_open_total gauge
-db_connection_pool_max_open_total 60 1636577556068
+db_connection_pool_max_open_total 60 1645738679875
 # HELP go_memstats_heap_in_use_bytes Process go heap bytes in use
 # TYPE go_memstats_heap_in_use_bytes gauge
-go_memstats_heap_in_use_bytes 1.98139904e+08 1636577556068
+go_memstats_heap_in_use_bytes 1.98139904e+08 1645738679875
 # HELP go_memstats_heap_allocated_bytes Process go heap allocated bytes
 # TYPE go_memstats_heap_allocated_bytes gauge
-go_memstats_heap_allocated_bytes 1.85429088e+08 1636577556068
+go_memstats_heap_allocated_bytes 1.85429088e+08 1645738679875
 # HELP go_memstats_heap_idle_bytes Process go heap idle bytes
 # TYPE go_memstats_heap_idle_bytes gauge
-go_memstats_heap_idle_bytes 1.35733248e+08 1636577556068
+go_memstats_heap_idle_bytes 1.35733248e+08 1645738679875
 # HELP go_memstats_heap_objects_total Process go heap number of objects
 # TYPE go_memstats_heap_objects_total gauge
-go_memstats_heap_objects_total 1.93186e+06 1636577556068
+go_memstats_heap_objects_total 1.93186e+06 1645738679875
 # HELP go_memstats_heap_reserved_bytes Process go heap reserved bytes
 # TYPE go_memstats_heap_reserved_bytes gauge
-go_memstats_heap_reserved_bytes 3.33873152e+08 1636577556068
+go_memstats_heap_reserved_bytes 3.33873152e+08 1645738679875
 # HELP go_memstats_gc_cpu_fraction_ratio Process go cpu used by gc. value is between 0 and 1
 # TYPE go_memstats_gc_cpu_fraction_ratio gauge
-go_memstats_gc_cpu_fraction_ratio 0.00024063137131169772 1636577556068
+go_memstats_gc_cpu_fraction_ratio 0.00024063137131169772 1645738679875
 # HELP go_routines_total Number of goroutines that currently exist
 # TYPE go_routines_total gauge
-go_routines_total 169 1636577556068
+go_routines_total 169 1645738679875
 # HELP jfxr_data_artifacts_total Artifacts of pkg type generic count in Xray
 # UPDATED jfxr_data_artifacts_total 1636513309792
 # TYPE jfxr_data_artifacts_total counter
-jfxr_data_artifacts_total{package_type="generic"} 1 1636577556068
+jfxr_data_artifacts_total{package_type="generic"} 1 1645738679875
 # HELP jfxr_data_components_total Components of pkg type generic count in Xray
 # UPDATED jfxr_data_components_total 1636513309792
 # TYPE jfxr_data_components_total counter
-jfxr_data_components_total{package_type="generic"} 1 1636577556068
+jfxr_data_components_total{package_type="generic"} 1 1645738679875
 # HELP jfxr_db_sync_running_total Is dbsync running
 # UPDATED jfxr_db_sync_running_total 1636577439791
 # TYPE jfxr_db_sync_running_total gauge
-jfxr_db_sync_running_total 0 1636577556068
+jfxr_db_sync_running_total 0 1645738679875
 # HELP jfxr_jira_last_ticket_creation_time_seconds Last ticket creation time
 # UPDATED jfxr_jira_last_ticket_creation_time_seconds 1636577309791
 # TYPE jfxr_jira_last_ticket_creation_time_seconds gauge
-jfxr_jira_last_ticket_creation_time_seconds 0 1636577556068
+jfxr_jira_last_ticket_creation_time_seconds 0 1645738679875
 # HELP jfxr_jira_no_of_errors_in_last_hour_total Total no of errors in last one hour
 # UPDATED jfxr_jira_no_of_errors_in_last_hour_total 1636577309791
 # TYPE jfxr_jira_no_of_errors_in_last_hour_total counter
-jfxr_jira_no_of_errors_in_last_hour_total 0 1636577556068
+jfxr_jira_no_of_errors_in_last_hour_total 0 1645738679875
 # HELP jfxr_jira_last_error_time_seconds Last error occurred time
 # UPDATED jfxr_jira_last_error_time_seconds 1636577309791
 # TYPE jfxr_jira_last_error_time_seconds gauge
-jfxr_jira_last_error_time_seconds 0 1636577556068
+jfxr_jira_last_error_time_seconds 0 1645738679875
 # HELP jfxr_jira_no_of_integrations_total Total no of jira integrations
 # UPDATED jfxr_jira_no_of_integrations_total 1636577309791
 # TYPE jfxr_jira_no_of_integrations_total counter
-jfxr_jira_no_of_integrations_total 0 1636577556068
+jfxr_jira_no_of_integrations_total 0 1645738679875
 # HELP jfxr_jira_no_of_profiles_total Total no of profiles created
 # UPDATED jfxr_jira_no_of_profiles_total 1636577309791
 # TYPE jfxr_jira_no_of_profiles_total counter
-jfxr_jira_no_of_profiles_total 0 1636577556068
+jfxr_jira_no_of_profiles_total 0 1645738679875
 # HELP jfxr_jira_no_of_tickets_created_in_last_one_hour_total Total no of jira tickets created in past one hour
 # UPDATED jfxr_jira_no_of_tickets_created_in_last_one_hour_total 1636577309791
 # TYPE jfxr_jira_no_of_tickets_created_in_last_one_hour_total counter
-jfxr_jira_no_of_tickets_created_in_last_one_hour_total 0 1636577556068
+jfxr_jira_no_of_tickets_created_in_last_one_hour_total 0 1645738679875
 # HELP jfxr_performance_server_up_time_seconds Xray server up time
 # TYPE jfxr_performance_server_up_time_seconds gauge
-jfxr_performance_server_up_time_seconds 928277.850744045 1636577556068
+jfxr_performance_server_up_time_seconds 928277.850744045 1645738679875
 # HELP queue_messages_total The number of messages currently in queue
 # UPDATED queue_messages_total 1636577509792
 # TYPE queue_messages_total gauge
-queue_messages_total{queue_name="alert"} 0 1636577556068
-queue_messages_total{queue_name="alertImpactAnalysis"} 0 1636577556068
-queue_messages_total{queue_name="alertImpactAnalysisRetry"} 0 1636577556068
-queue_messages_total{queue_name="alertRetry"} 0 1636577556068
-queue_messages_total{queue_name="analysis"} 0 1636577556068
-queue_messages_total{queue_name="analysisExistingContent"} 0 1636577556068
-queue_messages_total{queue_name="analysisExistingContentRetry"} 0 1636577556068
-queue_messages_total{queue_name="analysisRetry"} 0 1636577556068
-queue_messages_total{queue_name="buildReport_xray-0"} 0 1636577556068
-queue_messages_total{queue_name="failure"} 0 1636577556068
-queue_messages_total{queue_name="gcSyncMaster"} 0 1636577556068
-queue_messages_total{queue_name="impactAnalysis"} 0 1636577556068
-queue_messages_total{queue_name="impactAnalysisRetry"} 0 1636577556068
-queue_messages_total{queue_name="impactPathRecovery"} 0 1636577556068
-queue_messages_total{queue_name="impactPathRecoveryRetry"} 0 1636577556068
-queue_messages_total{queue_name="index"} 0 1636577556068
-queue_messages_total{queue_name="indexExistingContentRetry"} 0 1636577556068
-queue_messages_total{queue_name="indexExistsContent"} 0 1636577556068
-queue_messages_total{queue_name="indexRetry"} 0 1636577556068
-queue_messages_total{queue_name="job"} 0 1636577556068
-queue_messages_total{queue_name="mdsUpdate"} 0 1636577556068
-queue_messages_total{queue_name="mdsUpdateExistingContent"} 0 1636577556068
-queue_messages_total{queue_name="mdsUpdateExistingContentRetry"} 0 1636577556068
-queue_messages_total{queue_name="mdsUpdateRetry"} 0 1636577556068
-queue_messages_total{queue_name="notification"} 0 1636577556068
-queue_messages_total{queue_name="notificationRetry"} 0 1636577556068
-queue_messages_total{queue_name="persist"} 0 1636577556068
-queue_messages_total{queue_name="persistExistingContent"} 0 1636577556068
-queue_messages_total{queue_name="persistExistingContentRetry"} 0 1636577556068
-queue_messages_total{queue_name="persistRetry"} 0 1636577556068
-queue_messages_total{queue_name="report"} 0 1636577556068
-queue_messages_total{queue_name="reportRetry"} 0 1636577556068
-queue_messages_total{queue_name="ticketing"} 0 1636577556068
-queue_messages_total{queue_name="ticketingRetry"} 0 1636577556068
+queue_messages_total{queue_name="alert"} 0 1645738679875
+queue_messages_total{queue_name="alertImpactAnalysis"} 0 1645738679875
+queue_messages_total{queue_name="alertImpactAnalysisRetry"} 0 1645738679875
+queue_messages_total{queue_name="alertRetry"} 0 1645738679875
+queue_messages_total{queue_name="analysis"} 0 1645738679875
+queue_messages_total{queue_name="analysisExistingContent"} 0 1645738679875
+queue_messages_total{queue_name="analysisExistingContentRetry"} 0 1645738679875
+queue_messages_total{queue_name="analysisRetry"} 0 1645738679875
+queue_messages_total{queue_name="buildReport_xray-0"} 0 1645738679875
+queue_messages_total{queue_name="failure"} 0 1645738679875
+queue_messages_total{queue_name="gcSyncMaster"} 0 1645738679875
+queue_messages_total{queue_name="impactAnalysis"} 0 1645738679875
+queue_messages_total{queue_name="impactAnalysisRetry"} 0 1645738679875
+queue_messages_total{queue_name="impactPathRecovery"} 0 1645738679875
+queue_messages_total{queue_name="impactPathRecoveryRetry"} 0 1645738679875
+queue_messages_total{queue_name="index"} 0 1645738679875
+queue_messages_total{queue_name="indexExistingContentRetry"} 0 1645738679875
+queue_messages_total{queue_name="indexExistsContent"} 0 1645738679875
+queue_messages_total{queue_name="indexRetry"} 0 1645738679875
+queue_messages_total{queue_name="job"} 0 1645738679875
+queue_messages_total{queue_name="mdsUpdate"} 0 1645738679875
+queue_messages_total{queue_name="mdsUpdateExistingContent"} 0 1645738679875
+queue_messages_total{queue_name="mdsUpdateExistingContentRetry"} 0 1645738679875
+queue_messages_total{queue_name="mdsUpdateRetry"} 0 1645738679875
+queue_messages_total{queue_name="notification"} 0 1645738679875
+queue_messages_total{queue_name="notificationRetry"} 0 1645738679875
+queue_messages_total{queue_name="persist"} 0 1645738679875
+queue_messages_total{queue_name="persistExistingContent"} 0 1645738679875
+queue_messages_total{queue_name="persistExistingContentRetry"} 0 1645738679875
+queue_messages_total{queue_name="persistRetry"} 0 1645738679875
+queue_messages_total{queue_name="report"} 0 1645738679875
+queue_messages_total{queue_name="reportRetry"} 0 1645738679875
+queue_messages_total{queue_name="ticketing"} 0 1645738679875
+queue_messages_total{queue_name="ticketingRetry",new_queue_name="ticketingRetry2"} 0 1645738679875
 # TYPE sys_cpu_ratio gauge
-sys_cpu_ratio 0.3333333337029058 1636577556068
+sys_cpu_ratio 0.3333333337029058 1645738679875
 # HELP sys_load_1 Host load average in the last minute
-sys_load_1 2 1636577556068
+sys_load_1 2 1645738679875
 # HELP sys_load_5 Host load average in the last 5 minutes
-sys_load_5 1.82 1636577556068
+sys_load_5 1.82 1645738679875
 # HELP sys_load_15 Host load average in the last 15 minutes
-sys_load_15 1.73 1636577556068
+sys_load_15 1.73 1645738679875
 # HELP sys_memory_used_bytes Host used virtual memory
 # TYPE sys_memory_used_bytes gauge
-sys_memory_used_bytes 6.9718016e+09 1636577556068
+sys_memory_used_bytes 6.9718016e+09 1645738679875
 # HELP sys_memory_free_bytes Host free virtual memory
 # TYPE sys_memory_free_bytes gauge
-sys_memory_free_bytes 3.17022208e+08 1636577556068
+sys_memory_free_bytes 3.17022208e+08 1645738679875

data/spec/lib/{metrics_parser_spec.rb → elastic_metrics_parser_spec.rb}

@@ -7,18 +7,18 @@
   $LOAD_PATH.unshift(dir) unless $LOAD_PATH.include?(dir)
 end
 
-require 'metrics_parser'
+require 'elastic_metrics_parser'
 require 'date'
 require 'rspec'
 
 
-RSpec.describe MetricsParser do
+RSpec.describe ElasticMetricsParser do
   describe "#emit_parsed_metrics" do
     it 'should read sample Artifactory metrics data and verify the size of parsed data > 1' do
       platform_metrics = File.read('./spec/fixtures/files/sample_artifactory_metrics.txt')
       expect(platform_metrics.size).to be > 1
 
-      parser = MetricsParser.new('jfrog.artifactory', '', 'jfrog.artifactory.metrics')
+      parser = ElasticMetricsParser.new('jfrog.artifactory', '', 'jfrog.artifactory.metrics')
 
       normalized_data = parser.normalise_data(platform_metrics)
       expect(normalized_data.size).to be > 1
@@ -37,7 +37,7 @@ RSpec.describe MetricsParser do
       platform_metrics = File.read('./spec/fixtures/files/sample_xray_metrics.txt')
       expect(platform_metrics.size).to be > 1
 
-      parser = MetricsParser.new('jfrog.xray', '', 'jfrog.xray.metrics')
+      parser = ElasticMetricsParser.new('jfrog.xray', '', 'jfrog.xray.metrics')
 
       normalized_data = parser.normalise_data(platform_metrics)
       expect(normalized_data.size).to be > 1

data/spec/lib/metrics_helper_spec.rb

@@ -15,7 +15,7 @@ require './spec/fixtures/files/creds'
 
 
 RSpec.describe MetricsHelper do
-  jpd_url, username, apikey = get_credentials
+  jpd_url, username, apikey, token = get_credentials
   describe '#get_metrics' do
     it 'should return response code 200 and response body > 1' do
       response = RestClient::Request.new(
@@ -28,5 +28,12 @@ RSpec.describe MetricsHelper do
         expect(response.size).to be > 1
       end
     end
+
+    it 'should return response code 200 and response body > 1' do
+      helper = MetricsHelper.new('jfrog.artifactory', jpd_url, username, apikey, token)
+      url = "#{jpd_url}/observability/api/v1/metrics"
+      helper.check_endpoint(url, token)
+    end
+
   end
 end

data/spec/lib/newrelic_metrics_parser_spec.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+[
+    File.join(File.dirname(__FILE__), '..'),
+    File.join(File.dirname(__FILE__), '..', 'lib/fluent/plugin'),
+    File.join(File.dirname(__FILE__), '..', 'spec'),
+].each do |dir|
+  $LOAD_PATH.unshift(dir) unless $LOAD_PATH.include?(dir)
+end
+
+require 'newrelic_metrics_parser'
+require 'date'
+require 'rspec'
+
+
+RSpec.describe NewRelicMetricsParser do
+  describe "#emit_parsed_metrics" do
+    it 'should read sample Artifactory metrics data and verify the size of parsed data > 1' do
+      platform_metrics = File.read('./spec/fixtures/files/sample_artifactory_metrics.txt')
+      expect(platform_metrics.size).to be > 1
+
+      parser = NewRelicMetricsParser.new('jfrog.artifactory', '', 'jfrog.artifactory.metrics')
+
+      normalized_data = parser.normalise_data(platform_metrics)
+      expect(normalized_data.size).to be > 1
+
+      cleaned_data = parser.clean_data(normalized_data)
+      expect(cleaned_data.size).to be > 1
+
+      hash_data_array = parser.format_data(cleaned_data, 'jfrog.artifactory', '.')
+      expect(hash_data_array.size).to be 1
+
+      serialized_data = parser.serialize_data(hash_data_array)
+      expect(serialized_data.size).to be 1
+    end
+
+    it 'should read sample Xray metrics data and verify the size of parsed data > 1' do
+      platform_metrics = File.read('./spec/fixtures/files/sample_xray_metrics.txt')
+      expect(platform_metrics.size).to be > 1
+
+      parser = NewRelicMetricsParser.new('jfrog.xray', '', 'jfrog.xray.metrics')
+
+      normalized_data = parser.normalise_data(platform_metrics)
+      expect(normalized_data.size).to be > 1
+
+      cleaned_data = parser.clean_data(normalized_data)
+      expect(cleaned_data.size).to be > 1
+
+      hash_data_array = parser.format_data(cleaned_data, 'jfrog.xray', '.')
+      expect(hash_data_array.size).to be  1
+
+      serialized_data = parser.serialize_data(hash_data_array)
+      expect(serialized_data.size).to be  1
+    end
+  end
+end

data/spec/lib/splunk_metrics_parser_spec.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+[
+  File.join(File.dirname(__FILE__), '..'),
+  File.join(File.dirname(__FILE__), '..', 'lib/fluent/plugin'),
+  File.join(File.dirname(__FILE__), '..', 'spec'),
+].each do |dir|
+  $LOAD_PATH.unshift(dir) unless $LOAD_PATH.include?(dir)
+end
+
+require 'splunk_metrics_parser'
+require 'date'
+require 'rspec'
+
+
+RSpec.describe SplunkMetricsParser do
+  describe "#emit_parsed_metrics" do
+    it 'should read sample Artifactory metrics data and verify the size of parsed data > 1' do
+      platform_metrics = File.read('./spec/fixtures/files/sample_artifactory_metrics.txt')
+      expect(platform_metrics.size).to be > 1
+
+      parser = SplunkMetricsParser.new('jfrog.artifactory', '', 'jfrog.artifactory.metrics')
+
+      normalized_data = parser.normalise_data(platform_metrics)
+      expect(normalized_data.size).to be > 1
+
+      cleaned_data = parser.clean_data(normalized_data)
+      expect(cleaned_data.size).to be > 1
+
+      hash_data_array = parser.format_data(cleaned_data, 'jfrog.artifactory', '.')
+      expect(hash_data_array.size).to be > 1
+
+      serialized_data = parser.serialize_data(hash_data_array)
+      expect(serialized_data.size).to be > 1
+    end
+
+    it 'should read sample Xray metrics data and verify the size of parsed data > 1' do
+      platform_metrics = File.read('./spec/fixtures/files/sample_xray_metrics.txt')
+      expect(platform_metrics.size).to be > 1
+
+      parser = SplunkMetricsParser.new('jfrog.xray', '', 'jfrog.xray.metrics')
+
+      normalized_data = parser.normalise_data(platform_metrics)
+      expect(normalized_data.size).to be > 1
+
+      cleaned_data = parser.clean_data(normalized_data)
+      expect(cleaned_data.size).to be > 1
+
+      hash_data_array = parser.format_data(cleaned_data, 'jfrog.xray', '.')
+      expect(hash_data_array.size).to be > 1
+
+      serialized_data = parser.serialize_data(hash_data_array)
+      expect(serialized_data.size).to be > 1
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -23,7 +23,9 @@
 end
 
 require 'metrics_helper'
-require 'metrics_parser'
+require 'splunk_metrics_parser'
+require 'newrelic_metrics_parser'
+require 'elastic_metrics_parser'
 
 RSpec.configure do |config|
   # rspec-expectations config goes here. You can use an alternate

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-jfrog-metrics
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.1
 platform: ruby
 authors:
-- MahithaB, VasukiN
-autorequire: 
+- MahithaB, VasukiN, giri-vsr
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-12-14 00:00:00.000000000 Z
+date: 2022-03-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -52,6 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
@@ -72,10 +86,25 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
-description: Fluentd Plugin for converting one metrics form of data to another, this
-  is from Prometheus format to Splunk HEC required format
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+description: Fluentd Plugin for converting JFrog Artifactory, Xray generated metrics
+  (Prometheus Exposition Format) to target observability platform format (Splunk HEC,
+  New Relic, Elastic)
 email:
-- 60710901+MahithaB@users.noreply.github.com
+- cpe-support@jfrog.com
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -88,14 +117,19 @@ files:
 - README.md
 - Rakefile
 - fluent-plugin-jfrog-metrics.gemspec
+- lib/fluent/plugin/base_metrics_parser.rb
+- lib/fluent/plugin/elastic_metrics_parser.rb
 - lib/fluent/plugin/in_jfrog_metrics.rb
 - lib/fluent/plugin/metrics_helper.rb
-- lib/fluent/plugin/metrics_parser.rb
+- lib/fluent/plugin/newrelic_metrics_parser.rb
+- lib/fluent/plugin/splunk_metrics_parser.rb
 - spec/fixtures/files/creds.rb
 - spec/fixtures/files/sample_artifactory_metrics.txt
 - spec/fixtures/files/sample_xray_metrics.txt
+- spec/lib/elastic_metrics_parser_spec.rb
 - spec/lib/metrics_helper_spec.rb
-- spec/lib/metrics_parser_spec.rb
+- spec/lib/newrelic_metrics_parser_spec.rb
+- spec/lib/splunk_metrics_parser_spec.rb
 - spec/spec_helper.rb
 - test/helper.rb
 - test/plugin/test_in_jfrog_metrics.rb
@@ -103,7 +137,7 @@ homepage: https://github.com/jfrog/jfrog-fluentd-plugins/tree/main/fluent-plugin
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -118,17 +152,20 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.1.6
+signing_key:
 specification_version: 4
-summary: Fluentd Plugin for converting one metrics form of data to another, this is
-  from Prometheus format to Splunk HEC required format
+summary: Fluentd Plugin for converting JFrog Artifactory, Xray generated metrics (Prometheus
+  Exposition Format) to target observability platform format (Splunk HEC, New Relic,
+  Elastic)
 test_files:
 - spec/fixtures/files/creds.rb
 - spec/fixtures/files/sample_artifactory_metrics.txt
 - spec/fixtures/files/sample_xray_metrics.txt
+- spec/lib/elastic_metrics_parser_spec.rb
 - spec/lib/metrics_helper_spec.rb
-- spec/lib/metrics_parser_spec.rb
+- spec/lib/newrelic_metrics_parser_spec.rb
+- spec/lib/splunk_metrics_parser_spec.rb
 - spec/spec_helper.rb
 - test/helper.rb
 - test/plugin/test_in_jfrog_metrics.rb

data/lib/fluent/plugin/metrics_parser.rb

@@ -1,83 +0,0 @@
-# frozen_string_literal: true
-require 'json'
-
-class MetricsParser
-  def initialize(metric_prefix, router, tag)
-    @metric_prefix = metric_prefix
-    @router = router
-    @tag = tag
-  end
-
-  def normalise_data(data_to_normalize = [])
-    normalized_data = []
-    if data_to_normalize.length == 1
-      data_to_normalize.each do |interim_data|
-        normalized_data = interim_data.split(/\\n+|\\r+/).reject(&:empty?)
-      end
-    else
-      data_to_normalize.each_line do |interim_data|
-        normalized_data << interim_data.strip unless interim_data == "\n"
-      end
-    end
-    normalized_data
-  end
-
-  def clean_data(data_to_clean = [])
-    cleaned_data = []
-    data_to_clean.each do |interim_data|
-      cleaned_data << interim_data unless interim_data.include? '#'
-    end
-    cleaned_data
-  end
-
-  def extract_metrics_in_hash(cleaned_data = [], prefix = '', separator = '')
-    hash_data_array = []
-    cleaned_data.each do |interim_data|
-      hash_data_array << generate_hash_from_data(interim_data, prefix, separator)
-    end
-    hash_data_array
-  end
-
-  def generate_hash_from_data(data = '', prefix = '', separator = '')
-    metrics_hash = {}
-    if data =~ /{/ && data =~ /}/
-      metric_name, additional_dims, metric_val_and_time = data.match(/(.*){(.*)}(.*)/i).captures
-      if metric_val_and_time =~ / /
-        metrics_hash['metric_name'] = prefix + separator + metric_name
-        metrics_hash['value'] =
-          metric_val_and_time.strip.split[0] =~ /^\S*\.\S*$/ ? metric_val_and_time.strip.split[0].to_f : metric_val_and_time.strip.split[0].to_i
-        metrics_hash['time'] = metric_val_and_time.strip.split[1].to_i
-        if additional_dims =~ /,/
-          additional_dims.split(/,/).map do |interim_data|
-            metrics_hash[interim_data.split(/=/)[0]] = interim_data.split(/=/)[1].gsub(/"/, '') if interim_data =~ /=/
-          end
-        end
-      end
-    else
-      metrics_hash['metric_name'], metrics_hash['value'], metrics_hash['time'] = data.split
-      metrics_hash['metric_name'] = prefix + separator + metrics_hash['metric_name']
-      metrics_hash['value'] =
-        metrics_hash['value'] =~ /^\S*\.\S*$/ ? metrics_hash['value'].to_f : metrics_hash['value'].to_i
-      metrics_hash['time'] =  metrics_hash['time'].to_i
-    end
-    metrics_hash
-  end
-
-  def serialize_data(hashed_data_array = [])
-    serialised_data = []
-    hashed_data_array.each do |interim_data|
-      serialised_data << JSON.parse(interim_data.to_json)
-    end
-    serialised_data
-  end
-
-  def emit_parsed_metrics(platform_metrics)
-    normalized_data = normalise_data(platform_metrics)
-    cleaned_data = clean_data(normalized_data)
-    hash_data_array = extract_metrics_in_hash(cleaned_data, @metric_prefix, '.')
-    serialized_data = serialize_data(hash_data_array)
-    serialized_data.each do |interim_data|
-      @router.emit(@tag, Fluent::Engine.now, interim_data)
-    end
-  end
-end