fluent-plugin-jfrog-metrics 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +1 -0
- data/.rspec +1 -0
- data/Gemfile +8 -0
- data/Gemfile.lock +62 -0
- data/LICENSE +202 -0
- data/README.md +88 -0
- data/Rakefile +13 -0
- data/fluent-plugin-jfrog-metrics.gemspec +27 -0
- data/lib/fluent/plugin/in_jfrog_metrics.rb +87 -0
- data/lib/fluent/plugin/metrics_helper.rb +40 -0
- data/lib/fluent/plugin/metrics_parser.rb +83 -0
- data/spec/fixtures/files/creds.rb +8 -0
- data/spec/fixtures/files/sample_artifactory_metrics.txt +64 -0
- data/spec/fixtures/files/sample_xray_metrics.txt +138 -0
- data/spec/lib/metrics_helper_spec.rb +32 -0
- data/spec/lib/metrics_parser_spec.rb +55 -0
- data/spec/spec_helper.rb +112 -0
- data/test/helper.rb +8 -0
- data/test/plugin/test_in_jfrog_metrics.rb +39 -0
- metadata +134 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 880965fb77886a036bc1de886a64b1a28a81be51303981359e801d44bd425d32
|
4
|
+
data.tar.gz: 895972cbf8078c299a777b31e286644200e2716514c3fc22697efa00fa2013d6
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 40e81bba201bfec407c34e556843501ad9011490b6a2c55dc179fb5e2948948e05df86e2654af3c96f1ea6498f94a80771947340d850a03798b7fdc306a40fa1
|
7
|
+
data.tar.gz: da1ead96440264cf9e12180fd4fca0cbffd9e838a8b8cf98f9e35dfe0876e32fa52dfa659fd7b9975175387595a007080f76ad662395a11cb70a972beb182f6c
|
data/.gitignore
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
.DS_Store
|
data/.rspec
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
--require spec_helper
|
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,62 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
fluent-plugin-jfrog-metrics (0.1.2)
|
5
|
+
fluentd (>= 0.14.10, < 2)
|
6
|
+
|
7
|
+
GEM
|
8
|
+
remote: https://rubygems.org/
|
9
|
+
specs:
|
10
|
+
ansi (1.5.0)
|
11
|
+
builder (3.2.4)
|
12
|
+
concurrent-ruby (1.1.9)
|
13
|
+
cool.io (1.7.1)
|
14
|
+
fluentd (1.14.2)
|
15
|
+
bundler
|
16
|
+
cool.io (>= 1.4.5, < 2.0.0)
|
17
|
+
http_parser.rb (>= 0.5.1, < 0.8.0)
|
18
|
+
msgpack (>= 1.3.1, < 2.0.0)
|
19
|
+
serverengine (>= 2.2.2, < 3.0.0)
|
20
|
+
sigdump (~> 0.2.2)
|
21
|
+
strptime (>= 0.2.4, < 1.0.0)
|
22
|
+
tzinfo (>= 1.0, < 3.0)
|
23
|
+
tzinfo-data (~> 1.0)
|
24
|
+
webrick (>= 1.4.2, < 1.8.0)
|
25
|
+
yajl-ruby (~> 1.0)
|
26
|
+
http_parser.rb (0.7.0)
|
27
|
+
minitest (5.14.4)
|
28
|
+
minitest-reporters (1.4.3)
|
29
|
+
ansi
|
30
|
+
builder
|
31
|
+
minitest (>= 5.0)
|
32
|
+
ruby-progressbar
|
33
|
+
msgpack (1.4.2)
|
34
|
+
power_assert (2.0.1)
|
35
|
+
rake (12.3.3)
|
36
|
+
ruby-progressbar (1.11.0)
|
37
|
+
serverengine (2.2.4)
|
38
|
+
sigdump (~> 0.2.2)
|
39
|
+
sigdump (0.2.4)
|
40
|
+
strptime (0.2.5)
|
41
|
+
test-unit (3.5.0)
|
42
|
+
power_assert
|
43
|
+
tzinfo (2.0.4)
|
44
|
+
concurrent-ruby (~> 1.0)
|
45
|
+
tzinfo-data (1.2021.5)
|
46
|
+
tzinfo (>= 1.0.0)
|
47
|
+
webrick (1.7.0)
|
48
|
+
yajl-ruby (1.4.1)
|
49
|
+
|
50
|
+
PLATFORMS
|
51
|
+
ruby
|
52
|
+
|
53
|
+
DEPENDENCIES
|
54
|
+
bundler (~> 1.14)
|
55
|
+
fluent-plugin-jfrog-metrics!
|
56
|
+
minitest
|
57
|
+
minitest-reporters (>= 0.5.0)
|
58
|
+
rake (~> 12.0)
|
59
|
+
test-unit (~> 3.0)
|
60
|
+
|
61
|
+
BUNDLED WITH
|
62
|
+
1.17.3
|
data/LICENSE
ADDED
@@ -0,0 +1,202 @@
|
|
1
|
+
|
2
|
+
Apache License
|
3
|
+
Version 2.0, January 2004
|
4
|
+
http://www.apache.org/licenses/
|
5
|
+
|
6
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
7
|
+
|
8
|
+
1. Definitions.
|
9
|
+
|
10
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
11
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
12
|
+
|
13
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
14
|
+
the copyright owner that is granting the License.
|
15
|
+
|
16
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
17
|
+
other entities that control, are controlled by, or are under common
|
18
|
+
control with that entity. For the purposes of this definition,
|
19
|
+
"control" means (i) the power, direct or indirect, to cause the
|
20
|
+
direction or management of such entity, whether by contract or
|
21
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
22
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
23
|
+
|
24
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
25
|
+
exercising permissions granted by this License.
|
26
|
+
|
27
|
+
"Source" form shall mean the preferred form for making modifications,
|
28
|
+
including but not limited to software source code, documentation
|
29
|
+
source, and configuration files.
|
30
|
+
|
31
|
+
"Object" form shall mean any form resulting from mechanical
|
32
|
+
transformation or translation of a Source form, including but
|
33
|
+
not limited to compiled object code, generated documentation,
|
34
|
+
and conversions to other media types.
|
35
|
+
|
36
|
+
"Work" shall mean the work of authorship, whether in Source or
|
37
|
+
Object form, made available under the License, as indicated by a
|
38
|
+
copyright notice that is included in or attached to the work
|
39
|
+
(an example is provided in the Appendix below).
|
40
|
+
|
41
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
42
|
+
form, that is based on (or derived from) the Work and for which the
|
43
|
+
editorial revisions, annotations, elaborations, or other modifications
|
44
|
+
represent, as a whole, an original work of authorship. For the purposes
|
45
|
+
of this License, Derivative Works shall not include works that remain
|
46
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
47
|
+
the Work and Derivative Works thereof.
|
48
|
+
|
49
|
+
"Contribution" shall mean any work of authorship, including
|
50
|
+
the original version of the Work and any modifications or additions
|
51
|
+
to that Work or Derivative Works thereof, that is intentionally
|
52
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
53
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
54
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
55
|
+
means any form of electronic, verbal, or written communication sent
|
56
|
+
to the Licensor or its representatives, including but not limited to
|
57
|
+
communication on electronic mailing lists, source code control systems,
|
58
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
59
|
+
Licensor for the purpose of discussing and improving the Work, but
|
60
|
+
excluding communication that is conspicuously marked or otherwise
|
61
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
62
|
+
|
63
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
64
|
+
on behalf of whom a Contribution has been received by Licensor and
|
65
|
+
subsequently incorporated within the Work.
|
66
|
+
|
67
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
68
|
+
this License, each Contributor hereby grants to You a perpetual,
|
69
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
70
|
+
copyright license to reproduce, prepare Derivative Works of,
|
71
|
+
publicly display, publicly perform, sublicense, and distribute the
|
72
|
+
Work and such Derivative Works in Source or Object form.
|
73
|
+
|
74
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
75
|
+
this License, each Contributor hereby grants to You a perpetual,
|
76
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
77
|
+
(except as stated in this section) patent license to make, have made,
|
78
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
79
|
+
where such license applies only to those patent claims licensable
|
80
|
+
by such Contributor that are necessarily infringed by their
|
81
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
82
|
+
with the Work to which such Contribution(s) was submitted. If You
|
83
|
+
institute patent litigation against any entity (including a
|
84
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
85
|
+
or a Contribution incorporated within the Work constitutes direct
|
86
|
+
or contributory patent infringement, then any patent licenses
|
87
|
+
granted to You under this License for that Work shall terminate
|
88
|
+
as of the date such litigation is filed.
|
89
|
+
|
90
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
91
|
+
Work or Derivative Works thereof in any medium, with or without
|
92
|
+
modifications, and in Source or Object form, provided that You
|
93
|
+
meet the following conditions:
|
94
|
+
|
95
|
+
(a) You must give any other recipients of the Work or
|
96
|
+
Derivative Works a copy of this License; and
|
97
|
+
|
98
|
+
(b) You must cause any modified files to carry prominent notices
|
99
|
+
stating that You changed the files; and
|
100
|
+
|
101
|
+
(c) You must retain, in the Source form of any Derivative Works
|
102
|
+
that You distribute, all copyright, patent, trademark, and
|
103
|
+
attribution notices from the Source form of the Work,
|
104
|
+
excluding those notices that do not pertain to any part of
|
105
|
+
the Derivative Works; and
|
106
|
+
|
107
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
108
|
+
distribution, then any Derivative Works that You distribute must
|
109
|
+
include a readable copy of the attribution notices contained
|
110
|
+
within such NOTICE file, excluding those notices that do not
|
111
|
+
pertain to any part of the Derivative Works, in at least one
|
112
|
+
of the following places: within a NOTICE text file distributed
|
113
|
+
as part of the Derivative Works; within the Source form or
|
114
|
+
documentation, if provided along with the Derivative Works; or,
|
115
|
+
within a display generated by the Derivative Works, if and
|
116
|
+
wherever such third-party notices normally appear. The contents
|
117
|
+
of the NOTICE file are for informational purposes only and
|
118
|
+
do not modify the License. You may add Your own attribution
|
119
|
+
notices within Derivative Works that You distribute, alongside
|
120
|
+
or as an addendum to the NOTICE text from the Work, provided
|
121
|
+
that such additional attribution notices cannot be construed
|
122
|
+
as modifying the License.
|
123
|
+
|
124
|
+
You may add Your own copyright statement to Your modifications and
|
125
|
+
may provide additional or different license terms and conditions
|
126
|
+
for use, reproduction, or distribution of Your modifications, or
|
127
|
+
for any such Derivative Works as a whole, provided Your use,
|
128
|
+
reproduction, and distribution of the Work otherwise complies with
|
129
|
+
the conditions stated in this License.
|
130
|
+
|
131
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
132
|
+
any Contribution intentionally submitted for inclusion in the Work
|
133
|
+
by You to the Licensor shall be under the terms and conditions of
|
134
|
+
this License, without any additional terms or conditions.
|
135
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
136
|
+
the terms of any separate license agreement you may have executed
|
137
|
+
with Licensor regarding such Contributions.
|
138
|
+
|
139
|
+
6. Trademarks. This License does not grant permission to use the trade
|
140
|
+
names, trademarks, service marks, or product names of the Licensor,
|
141
|
+
except as required for reasonable and customary use in describing the
|
142
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
143
|
+
|
144
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
145
|
+
agreed to in writing, Licensor provides the Work (and each
|
146
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
147
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
148
|
+
implied, including, without limitation, any warranties or conditions
|
149
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
150
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
151
|
+
appropriateness of using or redistributing the Work and assume any
|
152
|
+
risks associated with Your exercise of permissions under this License.
|
153
|
+
|
154
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
155
|
+
whether in tort (including negligence), contract, or otherwise,
|
156
|
+
unless required by applicable law (such as deliberate and grossly
|
157
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
158
|
+
liable to You for damages, including any direct, indirect, special,
|
159
|
+
incidental, or consequential damages of any character arising as a
|
160
|
+
result of this License or out of the use or inability to use the
|
161
|
+
Work (including but not limited to damages for loss of goodwill,
|
162
|
+
work stoppage, computer failure or malfunction, or any and all
|
163
|
+
other commercial damages or losses), even if such Contributor
|
164
|
+
has been advised of the possibility of such damages.
|
165
|
+
|
166
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
167
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
168
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
169
|
+
or other liability obligations and/or rights consistent with this
|
170
|
+
License. However, in accepting such obligations, You may act only
|
171
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
172
|
+
of any other Contributor, and only if You agree to indemnify,
|
173
|
+
defend, and hold each Contributor harmless for any liability
|
174
|
+
incurred by, or claims asserted against, such Contributor by reason
|
175
|
+
of your accepting any such warranty or additional liability.
|
176
|
+
|
177
|
+
END OF TERMS AND CONDITIONS
|
178
|
+
|
179
|
+
APPENDIX: How to apply the Apache License to your work.
|
180
|
+
|
181
|
+
To apply the Apache License to your work, attach the following
|
182
|
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
183
|
+
replaced with your own identifying information. (Don't include
|
184
|
+
the brackets!) The text should be enclosed in the appropriate
|
185
|
+
comment syntax for the file format. We also recommend that a
|
186
|
+
file or class name and description of purpose be included on the
|
187
|
+
same "printed page" as the copyright notice for easier
|
188
|
+
identification within third-party archives.
|
189
|
+
|
190
|
+
Copyright [yyyy] [name of copyright owner]
|
191
|
+
|
192
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
193
|
+
you may not use this file except in compliance with the License.
|
194
|
+
You may obtain a copy of the License at
|
195
|
+
|
196
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
197
|
+
|
198
|
+
Unless required by applicable law or agreed to in writing, software
|
199
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
200
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
201
|
+
See the License for the specific language governing permissions and
|
202
|
+
limitations under the License.
|
data/README.md
ADDED
@@ -0,0 +1,88 @@
|
|
1
|
+
# fluent-plugin-jfrog-metrics
|
2
|
+
|
3
|
+
[Fluentd](https://fluentd.org/) input plugin to scrape metrics from JFrog Artifactory and Xray openmetrics API endpoints and export them to Fluentd to process into various outputs
|
4
|
+
|
5
|
+
## Building
|
6
|
+
|
7
|
+
To build / test locally use rake:
|
8
|
+
|
9
|
+
```
|
10
|
+
rake
|
11
|
+
```
|
12
|
+
|
13
|
+
To build install locally use bundler:
|
14
|
+
|
15
|
+
```
|
16
|
+
bundle install
|
17
|
+
```
|
18
|
+
|
19
|
+
This will install the gem shown below from source.
|
20
|
+
|
21
|
+
|
22
|
+
## Development
|
23
|
+
|
24
|
+
### Bundler
|
25
|
+
|
26
|
+
Add following line to your Gemfile:
|
27
|
+
|
28
|
+
```ruby
|
29
|
+
gem "fluent-plugin-jfrog-metrics"
|
30
|
+
```
|
31
|
+
|
32
|
+
And then execute:
|
33
|
+
|
34
|
+
```
|
35
|
+
$ bundle
|
36
|
+
```
|
37
|
+
|
38
|
+
### Configuration
|
39
|
+
|
40
|
+
You can generate configuration template:
|
41
|
+
|
42
|
+
```
|
43
|
+
$ fluent-plugin-config-format input jfrog-metrics
|
44
|
+
```
|
45
|
+
|
46
|
+
You can copy and paste generated documents here.
|
47
|
+
|
48
|
+
## Installation
|
49
|
+
|
50
|
+
### RubyGems
|
51
|
+
```
|
52
|
+
$ gem install rest-client
|
53
|
+
```
|
54
|
+
```
|
55
|
+
$ gem install thread
|
56
|
+
```
|
57
|
+
```
|
58
|
+
$ gem install fluent-plugin-jfrog-metrics
|
59
|
+
```
|
60
|
+
|
61
|
+
### Setup & configuration
|
62
|
+
Fluentd is the supported log collector for this integration.
|
63
|
+
For Fluentd setup and information, read the JFrog log analytics repository's [README.](https://github.com/jfrog/log-analytics/blob/master/README.md)
|
64
|
+
|
65
|
+
#### Fluentd Output
|
66
|
+
Download fluentd conf for different log-vendors. For example
|
67
|
+
Splunk:
|
68
|
+
|
69
|
+
Splunk setup can be found at [README.](https://github.com/jfrog/metrics/blob/main/splunk/README.md)
|
70
|
+
````text
|
71
|
+
wget https://raw.githubusercontent.com/jfrog/metrics/master/splunk/splunk_metrics.conf
|
72
|
+
````
|
73
|
+
|
74
|
+
#### Configuration parameters
|
75
|
+
Integration is done by setting up Xray. Obtain JPD url and access token for API. Configure the source directive parameters specified below
|
76
|
+
* **tag** (string) (required): The value is the tag assigned to the generated events.
|
77
|
+
* **jpd_url** (string) (required): JPD url required to pull Xray SIEM violations
|
78
|
+
* **apikey** (string) (required): API Key is the [Artifactory API Key](https://www.jfrog.com/confluence/display/JFROG/User+Profile#UserProfile-APIKey) for authentication
|
79
|
+
* **username** (string) (required): USER is the Artifactory username for authentication
|
80
|
+
* **metric_prefix** (string) (required): This values pulls the specific metrics. Values can be - jfrog.artifactory, jfrog.xray
|
81
|
+
* **interval** (integer) (optional): Wait interval between pulling new events
|
82
|
+
* Default value: `60`
|
83
|
+
|
84
|
+
## Copyright
|
85
|
+
* Copyright(c) 2020 - JFrog
|
86
|
+
* License
|
87
|
+
* Apache License, Version 2.0
|
88
|
+
|
data/Rakefile
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
require "bundler"
|
2
|
+
Bundler::GemHelper.install_tasks
|
3
|
+
|
4
|
+
require "rake/testtask"
|
5
|
+
|
6
|
+
Rake::TestTask.new(:test) do |t|
|
7
|
+
t.libs.push("lib", "test")
|
8
|
+
t.test_files = FileList["test/**/test_*.rb"]
|
9
|
+
t.verbose = true
|
10
|
+
t.warning = true
|
11
|
+
end
|
12
|
+
|
13
|
+
task default: [:test]
|
@@ -0,0 +1,27 @@
|
|
1
|
+
lib = File.expand_path('../lib', __FILE__)
|
2
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
3
|
+
|
4
|
+
Gem::Specification.new do |spec|
|
5
|
+
spec.name = 'fluent-plugin-jfrog-metrics'
|
6
|
+
spec.version = '0.1.0'
|
7
|
+
spec.authors = ['MahithaB, VasukiN']
|
8
|
+
spec.email = ['60710901+MahithaB@users.noreply.github.com']
|
9
|
+
|
10
|
+
spec.summary = %q{Fluentd Plugin for converting one metrics form of data to another, this is from Prometheus format to Splunk HEC required format}
|
11
|
+
spec.description = %q{Fluentd Plugin for converting one metrics form of data to another, this is from Prometheus format to Splunk HEC required format}
|
12
|
+
spec.homepage = 'https://github.com/jfrog/jfrog-fluentd-plugins/tree/main/fluent-plugin-jfrog-metrics'
|
13
|
+
spec.license = 'Apache-2.0'
|
14
|
+
|
15
|
+
test_files, files = `git ls-files -z`.split("\x0").partition do |f|
|
16
|
+
f.match(%r{^(test|spec|features)/})
|
17
|
+
end
|
18
|
+
spec.files = files
|
19
|
+
spec.executables = files.grep(%r{^bin/}) { |f| File.basename(f) }
|
20
|
+
spec.test_files = test_files
|
21
|
+
spec.require_paths = ['lib']
|
22
|
+
|
23
|
+
spec.add_development_dependency 'bundler', '~> 1.14'
|
24
|
+
spec.add_development_dependency 'rake', '~> 12.0'
|
25
|
+
spec.add_development_dependency 'test-unit', '~> 3.0'
|
26
|
+
spec.add_runtime_dependency 'fluentd', ['>= 0.14.10', '< 2']
|
27
|
+
end
|
@@ -0,0 +1,87 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
#
|
3
|
+
# Copyright 2021- MahithaB
|
4
|
+
#
|
5
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
6
|
+
# you may not use this file except in compliance with the License.
|
7
|
+
# You may obtain a copy of the License at
|
8
|
+
#
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
10
|
+
#
|
11
|
+
# Unless required by applicable law or agreed to in writing, software
|
12
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
13
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
14
|
+
# See the License for the specific language governing permissions and
|
15
|
+
# limitations under the License.
|
16
|
+
|
17
|
+
require 'concurrent'
|
18
|
+
require 'rest-client'
|
19
|
+
require 'fluent/plugin/input'
|
20
|
+
require_relative 'metrics_parser'
|
21
|
+
require_relative 'metrics_helper'
|
22
|
+
|
23
|
+
module Fluent
|
24
|
+
module Plugin
|
25
|
+
class JfrogMetricsInput < Fluent::Plugin::Input
|
26
|
+
Fluent::Plugin.register_input('jfrog_metrics', self)
|
27
|
+
helpers :timer
|
28
|
+
# `config_param` defines a parameter.
|
29
|
+
# You can refer to a parameter like an instance variable e.g. @port.
|
30
|
+
# `:default` means that the parameter is optional.
|
31
|
+
config_param :tag, :string, default: ''
|
32
|
+
config_param :jpd_url, :string, default: ''
|
33
|
+
config_param :username, :string, default: ''
|
34
|
+
config_param :apikey, :string, default: ''
|
35
|
+
config_param :interval, :time, default: 10
|
36
|
+
config_param :metric_prefix, :string, default: ''
|
37
|
+
|
38
|
+
# `configure` is called before `start`.
|
39
|
+
# 'conf' is a `Hash` that includes the configuration parameters.
|
40
|
+
# If the configuration is invalid, raise `Fluent::ConfigError`.
|
41
|
+
def configure(conf)
|
42
|
+
super
|
43
|
+
raise Fluent::ConfigError, 'Must define the tag for metrics data.' if @tag == ''
|
44
|
+
|
45
|
+
raise Fluent::ConfigError, 'Must define the jpd_url to scrape metrics' if @jpd_url == ''
|
46
|
+
|
47
|
+
raise Fluent::ConfigError, 'Must define the username for authentication' if @username == ''
|
48
|
+
|
49
|
+
raise Fluent::ConfigError, 'Must define the apikey to use for authentication.' if @apikey == ''
|
50
|
+
|
51
|
+
raise Fluent::ConfigError, 'Must define the interval to use for gathering the metrics.' if @interval == ''
|
52
|
+
|
53
|
+
raise Fluent::ConfigError, 'Must define the metric_prefix to use for getting the metrics.' if @metric_prefix == ''
|
54
|
+
end
|
55
|
+
|
56
|
+
# `start` is called when starting and after `configure` is successfully completed.
|
57
|
+
def start
|
58
|
+
super
|
59
|
+
@running = true
|
60
|
+
@thread = Thread.new(&method(:run))
|
61
|
+
end
|
62
|
+
|
63
|
+
def shutdown
|
64
|
+
@running = false
|
65
|
+
@thread.join
|
66
|
+
super
|
67
|
+
end
|
68
|
+
|
69
|
+
def run
|
70
|
+
puts('Preparing metrics collection, creating timer task')
|
71
|
+
timer_task = Concurrent::TimerTask.new(execution_interval: @interval, timeout_interval: 30, run_now: true) do
|
72
|
+
puts('Timer task execution')
|
73
|
+
do_execute
|
74
|
+
end
|
75
|
+
timer_task.execute
|
76
|
+
end
|
77
|
+
|
78
|
+
def do_execute
|
79
|
+
puts('Executing metrics collection')
|
80
|
+
metrics_helper = MetricsHelper.new(@metric_prefix, @jpd_url, @username, @apikey)
|
81
|
+
platform_metrics = metrics_helper.get_metrics
|
82
|
+
parser = MetricsParser.new(@metric_prefix, router, @tag)
|
83
|
+
parser.emit_parsed_metrics(platform_metrics)
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require 'rest-client'
|
3
|
+
|
4
|
+
class MetricsHelper
|
5
|
+
def initialize(metric_prefix, jpd_url, username, apikey)
|
6
|
+
@metric_prefix = metric_prefix
|
7
|
+
@jpd_url = jpd_url
|
8
|
+
@username = username
|
9
|
+
@apikey = apikey
|
10
|
+
end
|
11
|
+
|
12
|
+
def get_metrics
|
13
|
+
url = nil
|
14
|
+
url = case @metric_prefix
|
15
|
+
when 'jfrog.artifactory'
|
16
|
+
"#{@jpd_url}/artifactory/api/v1/metrics"
|
17
|
+
when 'jfrog.xray'
|
18
|
+
"#{@jpd_url}/xray/api/v1/metrics"
|
19
|
+
else
|
20
|
+
"#{@jpd_url}/artifactory/api/v1/metrics"
|
21
|
+
end
|
22
|
+
execute_rest_call(url, @username, @apikey)
|
23
|
+
end
|
24
|
+
|
25
|
+
def execute_rest_call(url, user, password)
|
26
|
+
response = RestClient::Request.new(
|
27
|
+
method: :get,
|
28
|
+
url: url,
|
29
|
+
user: user,
|
30
|
+
password: password
|
31
|
+
).execute do |response, request, result|
|
32
|
+
case response.code
|
33
|
+
when 200
|
34
|
+
return response.body
|
35
|
+
else
|
36
|
+
raise Fluent::ConfigError, 'Cannot fetch #{@metric_prefix} metrics'
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,83 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require 'json'
|
3
|
+
|
4
|
+
class MetricsParser
|
5
|
+
def initialize(metric_prefix, router, tag)
|
6
|
+
@metric_prefix = metric_prefix
|
7
|
+
@router = router
|
8
|
+
@tag = tag
|
9
|
+
end
|
10
|
+
|
11
|
+
def normalise_data(data_to_normalize = [])
|
12
|
+
normalized_data = []
|
13
|
+
if data_to_normalize.length == 1
|
14
|
+
data_to_normalize.each do |interim_data|
|
15
|
+
normalized_data = interim_data.split(/\\n+|\\r+/).reject(&:empty?)
|
16
|
+
end
|
17
|
+
else
|
18
|
+
data_to_normalize.each_line do |interim_data|
|
19
|
+
normalized_data << interim_data.strip unless interim_data == "\n"
|
20
|
+
end
|
21
|
+
end
|
22
|
+
normalized_data
|
23
|
+
end
|
24
|
+
|
25
|
+
def clean_data(data_to_clean = [])
|
26
|
+
cleaned_data = []
|
27
|
+
data_to_clean.each do |interim_data|
|
28
|
+
cleaned_data << interim_data unless interim_data.include? '#'
|
29
|
+
end
|
30
|
+
cleaned_data
|
31
|
+
end
|
32
|
+
|
33
|
+
def extract_metrics_in_hash(cleaned_data = [], prefix = '', separator = '')
|
34
|
+
hash_data_array = []
|
35
|
+
cleaned_data.each do |interim_data|
|
36
|
+
hash_data_array << generate_hash_from_data(interim_data, prefix, separator)
|
37
|
+
end
|
38
|
+
hash_data_array
|
39
|
+
end
|
40
|
+
|
41
|
+
def generate_hash_from_data(data = '', prefix = '', separator = '')
|
42
|
+
metrics_hash = {}
|
43
|
+
if data =~ /{/ && data =~ /}/
|
44
|
+
metric_name, additional_dims, metric_val_and_time = data.match(/(.*){(.*)}(.*)/i).captures
|
45
|
+
if metric_val_and_time =~ / /
|
46
|
+
metrics_hash['metric_name'] = prefix + separator + metric_name
|
47
|
+
metrics_hash['value'] =
|
48
|
+
metric_val_and_time.strip.split[0] =~ /^\S*\.\S*$/ ? metric_val_and_time.strip.split[0].to_f : metric_val_and_time.strip.split[0].to_i
|
49
|
+
metrics_hash['time'] = metric_val_and_time.strip.split[1].to_i
|
50
|
+
if additional_dims =~ /,/
|
51
|
+
additional_dims.split(/,/).map do |interim_data|
|
52
|
+
metrics_hash[interim_data.split(/=/)[0]] = interim_data.split(/=/)[1].gsub(/"/, '') if interim_data =~ /=/
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
else
|
57
|
+
metrics_hash['metric_name'], metrics_hash['value'], metrics_hash['time'] = data.split
|
58
|
+
metrics_hash['metric_name'] = prefix + separator + metrics_hash['metric_name']
|
59
|
+
metrics_hash['value'] =
|
60
|
+
metrics_hash['value'] =~ /^\S*\.\S*$/ ? metrics_hash['value'].to_f : metrics_hash['value'].to_i
|
61
|
+
metrics_hash['time'] = metrics_hash['time'].to_i
|
62
|
+
end
|
63
|
+
metrics_hash
|
64
|
+
end
|
65
|
+
|
66
|
+
def serialize_data(hashed_data_array = [])
|
67
|
+
serialised_data = []
|
68
|
+
hashed_data_array.each do |interim_data|
|
69
|
+
serialised_data << JSON.parse(interim_data.to_json)
|
70
|
+
end
|
71
|
+
serialised_data
|
72
|
+
end
|
73
|
+
|
74
|
+
def emit_parsed_metrics(platform_metrics)
|
75
|
+
normalized_data = normalise_data(platform_metrics)
|
76
|
+
cleaned_data = clean_data(normalized_data)
|
77
|
+
hash_data_array = extract_metrics_in_hash(cleaned_data, @metric_prefix, '.')
|
78
|
+
serialized_data = serialize_data(hash_data_array)
|
79
|
+
serialized_data.each do |interim_data|
|
80
|
+
@router.emit(@tag, Fluent::Engine.now, interim_data)
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
@@ -0,0 +1,64 @@
|
|
1
|
+
# HELP sys_memory_used_bytes Host used virtual memory
|
2
|
+
# UPDATED sys_memory_used_bytes 1636577501390
|
3
|
+
# TYPE sys_memory_used_bytes gauge
|
4
|
+
sys_memory_used_bytes 3836522496 1636577501390
|
5
|
+
# HELP sys_memory_free_bytes Host free virtual memory
|
6
|
+
# UPDATED sys_memory_free_bytes 1636577501390
|
7
|
+
# TYPE sys_memory_free_bytes gauge
|
8
|
+
sys_memory_free_bytes 8396328960 1636577501390
|
9
|
+
# HELP jfrt_runtime_heap_freememory_bytes Free Memory
|
10
|
+
# UPDATED jfrt_runtime_heap_freememory_bytes 1636577501390
|
11
|
+
# TYPE jfrt_runtime_heap_freememory_bytes gauge
|
12
|
+
jfrt_runtime_heap_freememory_bytes 464534120 1636577501390
|
13
|
+
# HELP jfrt_runtime_heap_maxmemory_bytes Max Memory
|
14
|
+
# UPDATED jfrt_runtime_heap_maxmemory_bytes 1636577501390
|
15
|
+
# TYPE jfrt_runtime_heap_maxmemory_bytes gauge
|
16
|
+
jfrt_runtime_heap_maxmemory_bytes 2147483648 1636577501390
|
17
|
+
# HELP jfrt_runtime_heap_totalmemory_bytes Total Memory
|
18
|
+
# UPDATED jfrt_runtime_heap_totalmemory_bytes 1636577501390
|
19
|
+
# TYPE jfrt_runtime_heap_totalmemory_bytes gauge
|
20
|
+
jfrt_runtime_heap_totalmemory_bytes 1358954496 1636577501390
|
21
|
+
# HELP jfrt_runtime_heap_processors_total Available Processors
|
22
|
+
# UPDATED jfrt_runtime_heap_processors_total 1636577501390
|
23
|
+
# TYPE jfrt_runtime_heap_processors_total counter
|
24
|
+
jfrt_runtime_heap_processors_total 1 1636577501390
|
25
|
+
# HELP jfrt_db_connections_active_total Total Active Connections
|
26
|
+
# UPDATED jfrt_db_connections_active_total 1636577471195
|
27
|
+
# TYPE jfrt_db_connections_active_total gauge
|
28
|
+
jfrt_db_connections_active_total 0 1636577501390
|
29
|
+
# HELP jfrt_db_connections_idle_total Total Idle Connections
|
30
|
+
# UPDATED jfrt_db_connections_idle_total 1636577471195
|
31
|
+
# TYPE jfrt_db_connections_idle_total gauge
|
32
|
+
jfrt_db_connections_idle_total 1 1636577501390
|
33
|
+
# HELP jfrt_db_connections_max_active_total Total Max Active Connections
|
34
|
+
# UPDATED jfrt_db_connections_max_active_total 1636577471195
|
35
|
+
# TYPE jfrt_db_connections_max_active_total gauge
|
36
|
+
jfrt_db_connections_max_active_total 80 1636577501390
|
37
|
+
# HELP jfrt_db_connections_min_idle_total Total Min Idle Connections
|
38
|
+
# UPDATED jfrt_db_connections_min_idle_total 1636577471195
|
39
|
+
# TYPE jfrt_db_connections_min_idle_total gauge
|
40
|
+
jfrt_db_connections_min_idle_total 1 1636577501390
|
41
|
+
# HELP sys_cpu_ratio Total cpu load ratio
|
42
|
+
# UPDATED sys_cpu_ratio 1636577501390
|
43
|
+
# TYPE sys_cpu_ratio gauge
|
44
|
+
sys_cpu_ratio 0.16 1636577501390
|
45
|
+
# HELP jfrt_projects_active_total Projects Amount
|
46
|
+
# UPDATED jfrt_projects_active_total 1636577501390
|
47
|
+
# TYPE jfrt_projects_active_total counter
|
48
|
+
jfrt_projects_active_total 0 1636577501390
|
49
|
+
# HELP jfrt_storage_current_total_size_bytes Used Storage
|
50
|
+
# UPDATED jfrt_storage_current_total_size_bytes 1636577501390
|
51
|
+
# TYPE jfrt_storage_current_total_size_bytes gauge
|
52
|
+
jfrt_storage_current_total_size_bytes 0 1636577501390
|
53
|
+
# HELP app_disk_used_bytes Used bytes for app home directory disk device
|
54
|
+
# UPDATED app_disk_used_bytes 1636577501390
|
55
|
+
# TYPE app_disk_used_bytes gauge
|
56
|
+
app_disk_used_bytes 730750976 1636577501390
|
57
|
+
# HELP app_disk_free_bytes Free bytes for app home directory disk device
|
58
|
+
# UPDATED app_disk_free_bytes 1636577501390
|
59
|
+
# TYPE app_disk_free_bytes gauge
|
60
|
+
app_disk_free_bytes 209510809600 1636577501390
|
61
|
+
# HELP jfrt_artifacts_gc_next_run_seconds Next GC Run
|
62
|
+
# UPDATED jfrt_artifacts_gc_next_run_seconds 1636574411092
|
63
|
+
# TYPE jfrt_artifacts_gc_next_run_seconds gauge
|
64
|
+
jfrt_artifacts_gc_next_run_seconds 14388 1636577501390
|
@@ -0,0 +1,138 @@
|
|
1
|
+
# HELP app_disk_used_bytes Used bytes for app home directory disk device
|
2
|
+
# TYPE app_disk_used_bytes gauge
|
3
|
+
app_disk_used_bytes 1.48081664e+10 1636577556068
|
4
|
+
# HELP app_disk_free_bytes Free bytes for app home directory disk device
|
5
|
+
# TYPE app_disk_free_bytes gauge
|
6
|
+
app_disk_free_bytes 3.356854272e+10 1636577556068
|
7
|
+
# HELP app_io_counters_write_bytes Process io total write bytes
|
8
|
+
# TYPE app_io_counters_write_bytes gauge
|
9
|
+
app_io_counters_write_bytes 1.217306624e+09 1636577556068
|
10
|
+
# HELP app_io_counters_read_bytes Process io total read bytes
|
11
|
+
# TYPE app_io_counters_read_bytes gauge
|
12
|
+
app_io_counters_read_bytes 2.10030592e+08 1636577556068
|
13
|
+
# HELP app_self_metrics_calc_seconds Total time to collect all metrics
|
14
|
+
# TYPE app_self_metrics_calc_seconds gauge
|
15
|
+
app_self_metrics_calc_seconds 0.050925766 1636577556068
|
16
|
+
# HELP app_self_metrics_total Count of collected metrics
|
17
|
+
# TYPE app_self_metrics_total gauge
|
18
|
+
app_self_metrics_total 31 1636577556068
|
19
|
+
# HELP db_connection_pool_in_use_total The number of connections currently in use
|
20
|
+
# TYPE db_connection_pool_in_use_total gauge
|
21
|
+
db_connection_pool_in_use_total 0 1636577556068
|
22
|
+
# HELP db_connection_pool_idle_total The number of idle connections
|
23
|
+
# TYPE db_connection_pool_idle_total gauge
|
24
|
+
db_connection_pool_idle_total 5 1636577556068
|
25
|
+
# HELP db_connection_pool_max_open_total The maximum number of open connections
|
26
|
+
# TYPE db_connection_pool_max_open_total gauge
|
27
|
+
db_connection_pool_max_open_total 60 1636577556068
|
28
|
+
# HELP go_memstats_heap_in_use_bytes Process go heap bytes in use
|
29
|
+
# TYPE go_memstats_heap_in_use_bytes gauge
|
30
|
+
go_memstats_heap_in_use_bytes 1.98139904e+08 1636577556068
|
31
|
+
# HELP go_memstats_heap_allocated_bytes Process go heap allocated bytes
|
32
|
+
# TYPE go_memstats_heap_allocated_bytes gauge
|
33
|
+
go_memstats_heap_allocated_bytes 1.85429088e+08 1636577556068
|
34
|
+
# HELP go_memstats_heap_idle_bytes Process go heap idle bytes
|
35
|
+
# TYPE go_memstats_heap_idle_bytes gauge
|
36
|
+
go_memstats_heap_idle_bytes 1.35733248e+08 1636577556068
|
37
|
+
# HELP go_memstats_heap_objects_total Process go heap number of objects
|
38
|
+
# TYPE go_memstats_heap_objects_total gauge
|
39
|
+
go_memstats_heap_objects_total 1.93186e+06 1636577556068
|
40
|
+
# HELP go_memstats_heap_reserved_bytes Process go heap reserved bytes
|
41
|
+
# TYPE go_memstats_heap_reserved_bytes gauge
|
42
|
+
go_memstats_heap_reserved_bytes 3.33873152e+08 1636577556068
|
43
|
+
# HELP go_memstats_gc_cpu_fraction_ratio Process go cpu used by gc. value is between 0 and 1
|
44
|
+
# TYPE go_memstats_gc_cpu_fraction_ratio gauge
|
45
|
+
go_memstats_gc_cpu_fraction_ratio 0.00024063137131169772 1636577556068
|
46
|
+
# HELP go_routines_total Number of goroutines that currently exist
|
47
|
+
# TYPE go_routines_total gauge
|
48
|
+
go_routines_total 169 1636577556068
|
49
|
+
# HELP jfxr_data_artifacts_total Artifacts of pkg type generic count in Xray
|
50
|
+
# UPDATED jfxr_data_artifacts_total 1636513309792
|
51
|
+
# TYPE jfxr_data_artifacts_total counter
|
52
|
+
jfxr_data_artifacts_total{package_type="generic"} 1 1636577556068
|
53
|
+
# HELP jfxr_data_components_total Components of pkg type generic count in Xray
|
54
|
+
# UPDATED jfxr_data_components_total 1636513309792
|
55
|
+
# TYPE jfxr_data_components_total counter
|
56
|
+
jfxr_data_components_total{package_type="generic"} 1 1636577556068
|
57
|
+
# HELP jfxr_db_sync_running_total Is dbsync running
|
58
|
+
# UPDATED jfxr_db_sync_running_total 1636577439791
|
59
|
+
# TYPE jfxr_db_sync_running_total gauge
|
60
|
+
jfxr_db_sync_running_total 0 1636577556068
|
61
|
+
# HELP jfxr_jira_last_ticket_creation_time_seconds Last ticket creation time
|
62
|
+
# UPDATED jfxr_jira_last_ticket_creation_time_seconds 1636577309791
|
63
|
+
# TYPE jfxr_jira_last_ticket_creation_time_seconds gauge
|
64
|
+
jfxr_jira_last_ticket_creation_time_seconds 0 1636577556068
|
65
|
+
# HELP jfxr_jira_no_of_errors_in_last_hour_total Total no of errors in last one hour
|
66
|
+
# UPDATED jfxr_jira_no_of_errors_in_last_hour_total 1636577309791
|
67
|
+
# TYPE jfxr_jira_no_of_errors_in_last_hour_total counter
|
68
|
+
jfxr_jira_no_of_errors_in_last_hour_total 0 1636577556068
|
69
|
+
# HELP jfxr_jira_last_error_time_seconds Last error occurred time
|
70
|
+
# UPDATED jfxr_jira_last_error_time_seconds 1636577309791
|
71
|
+
# TYPE jfxr_jira_last_error_time_seconds gauge
|
72
|
+
jfxr_jira_last_error_time_seconds 0 1636577556068
|
73
|
+
# HELP jfxr_jira_no_of_integrations_total Total no of jira integrations
|
74
|
+
# UPDATED jfxr_jira_no_of_integrations_total 1636577309791
|
75
|
+
# TYPE jfxr_jira_no_of_integrations_total counter
|
76
|
+
jfxr_jira_no_of_integrations_total 0 1636577556068
|
77
|
+
# HELP jfxr_jira_no_of_profiles_total Total no of profiles created
|
78
|
+
# UPDATED jfxr_jira_no_of_profiles_total 1636577309791
|
79
|
+
# TYPE jfxr_jira_no_of_profiles_total counter
|
80
|
+
jfxr_jira_no_of_profiles_total 0 1636577556068
|
81
|
+
# HELP jfxr_jira_no_of_tickets_created_in_last_one_hour_total Total no of jira tickets created in past one hour
|
82
|
+
# UPDATED jfxr_jira_no_of_tickets_created_in_last_one_hour_total 1636577309791
|
83
|
+
# TYPE jfxr_jira_no_of_tickets_created_in_last_one_hour_total counter
|
84
|
+
jfxr_jira_no_of_tickets_created_in_last_one_hour_total 0 1636577556068
|
85
|
+
# HELP jfxr_performance_server_up_time_seconds Xray server up time
|
86
|
+
# TYPE jfxr_performance_server_up_time_seconds gauge
|
87
|
+
jfxr_performance_server_up_time_seconds 928277.850744045 1636577556068
|
88
|
+
# HELP queue_messages_total The number of messages currently in queue
|
89
|
+
# UPDATED queue_messages_total 1636577509792
|
90
|
+
# TYPE queue_messages_total gauge
|
91
|
+
queue_messages_total{queue_name="alert"} 0 1636577556068
|
92
|
+
queue_messages_total{queue_name="alertImpactAnalysis"} 0 1636577556068
|
93
|
+
queue_messages_total{queue_name="alertImpactAnalysisRetry"} 0 1636577556068
|
94
|
+
queue_messages_total{queue_name="alertRetry"} 0 1636577556068
|
95
|
+
queue_messages_total{queue_name="analysis"} 0 1636577556068
|
96
|
+
queue_messages_total{queue_name="analysisExistingContent"} 0 1636577556068
|
97
|
+
queue_messages_total{queue_name="analysisExistingContentRetry"} 0 1636577556068
|
98
|
+
queue_messages_total{queue_name="analysisRetry"} 0 1636577556068
|
99
|
+
queue_messages_total{queue_name="buildReport_xray-0"} 0 1636577556068
|
100
|
+
queue_messages_total{queue_name="failure"} 0 1636577556068
|
101
|
+
queue_messages_total{queue_name="gcSyncMaster"} 0 1636577556068
|
102
|
+
queue_messages_total{queue_name="impactAnalysis"} 0 1636577556068
|
103
|
+
queue_messages_total{queue_name="impactAnalysisRetry"} 0 1636577556068
|
104
|
+
queue_messages_total{queue_name="impactPathRecovery"} 0 1636577556068
|
105
|
+
queue_messages_total{queue_name="impactPathRecoveryRetry"} 0 1636577556068
|
106
|
+
queue_messages_total{queue_name="index"} 0 1636577556068
|
107
|
+
queue_messages_total{queue_name="indexExistingContentRetry"} 0 1636577556068
|
108
|
+
queue_messages_total{queue_name="indexExistsContent"} 0 1636577556068
|
109
|
+
queue_messages_total{queue_name="indexRetry"} 0 1636577556068
|
110
|
+
queue_messages_total{queue_name="job"} 0 1636577556068
|
111
|
+
queue_messages_total{queue_name="mdsUpdate"} 0 1636577556068
|
112
|
+
queue_messages_total{queue_name="mdsUpdateExistingContent"} 0 1636577556068
|
113
|
+
queue_messages_total{queue_name="mdsUpdateExistingContentRetry"} 0 1636577556068
|
114
|
+
queue_messages_total{queue_name="mdsUpdateRetry"} 0 1636577556068
|
115
|
+
queue_messages_total{queue_name="notification"} 0 1636577556068
|
116
|
+
queue_messages_total{queue_name="notificationRetry"} 0 1636577556068
|
117
|
+
queue_messages_total{queue_name="persist"} 0 1636577556068
|
118
|
+
queue_messages_total{queue_name="persistExistingContent"} 0 1636577556068
|
119
|
+
queue_messages_total{queue_name="persistExistingContentRetry"} 0 1636577556068
|
120
|
+
queue_messages_total{queue_name="persistRetry"} 0 1636577556068
|
121
|
+
queue_messages_total{queue_name="report"} 0 1636577556068
|
122
|
+
queue_messages_total{queue_name="reportRetry"} 0 1636577556068
|
123
|
+
queue_messages_total{queue_name="ticketing"} 0 1636577556068
|
124
|
+
queue_messages_total{queue_name="ticketingRetry"} 0 1636577556068
|
125
|
+
# TYPE sys_cpu_ratio gauge
|
126
|
+
sys_cpu_ratio 0.3333333337029058 1636577556068
|
127
|
+
# HELP sys_load_1 Host load average in the last minute
|
128
|
+
sys_load_1 2 1636577556068
|
129
|
+
# HELP sys_load_5 Host load average in the last 5 minutes
|
130
|
+
sys_load_5 1.82 1636577556068
|
131
|
+
# HELP sys_load_15 Host load average in the last 15 minutes
|
132
|
+
sys_load_15 1.73 1636577556068
|
133
|
+
# HELP sys_memory_used_bytes Host used virtual memory
|
134
|
+
# TYPE sys_memory_used_bytes gauge
|
135
|
+
sys_memory_used_bytes 6.9718016e+09 1636577556068
|
136
|
+
# HELP sys_memory_free_bytes Host free virtual memory
|
137
|
+
# TYPE sys_memory_free_bytes gauge
|
138
|
+
sys_memory_free_bytes 3.17022208e+08 1636577556068
|
@@ -0,0 +1,32 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
[
|
3
|
+
File.join(File.dirname(__FILE__), '..'),
|
4
|
+
File.join(File.dirname(__FILE__), '..', 'lib/fluent/plugin'),
|
5
|
+
File.join(File.dirname(__FILE__), '..', 'spec')
|
6
|
+
].each do |dir|
|
7
|
+
$LOAD_PATH.unshift(dir) unless $LOAD_PATH.include?(dir)
|
8
|
+
end
|
9
|
+
|
10
|
+
require 'metrics_helper'
|
11
|
+
require 'date'
|
12
|
+
require 'rspec'
|
13
|
+
require 'rest-client'
|
14
|
+
require './spec/fixtures/files/creds'
|
15
|
+
|
16
|
+
|
17
|
+
RSpec.describe MetricsHelper do
|
18
|
+
jpd_url, username, apikey = get_credentials
|
19
|
+
describe '#get_metrics' do
|
20
|
+
it 'should return response code 200 and response body > 1' do
|
21
|
+
response = RestClient::Request.new(
|
22
|
+
method: :get,
|
23
|
+
url: "#{jpd_url}/artifactory/api/v1/metrics",
|
24
|
+
user: username.to_s,
|
25
|
+
password: apikey.to_s
|
26
|
+
).execute do |response, request, result|
|
27
|
+
expect(response.code).to eq(200)
|
28
|
+
expect(response.size).to be > 1
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,55 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
[
|
3
|
+
File.join(File.dirname(__FILE__), '..'),
|
4
|
+
File.join(File.dirname(__FILE__), '..', 'lib/fluent/plugin'),
|
5
|
+
File.join(File.dirname(__FILE__), '..', 'spec'),
|
6
|
+
].each do |dir|
|
7
|
+
$LOAD_PATH.unshift(dir) unless $LOAD_PATH.include?(dir)
|
8
|
+
end
|
9
|
+
|
10
|
+
require 'metrics_parser'
|
11
|
+
require 'date'
|
12
|
+
require 'rspec'
|
13
|
+
|
14
|
+
|
15
|
+
RSpec.describe MetricsParser do
|
16
|
+
describe "#emit_parsed_metrics" do
|
17
|
+
it 'should read sample Artifactory metrics data and verify the size of parsed data > 1' do
|
18
|
+
platform_metrics = File.read('./spec/fixtures/files/sample_artifactory_metrics.txt')
|
19
|
+
expect(platform_metrics.size).to be > 1
|
20
|
+
|
21
|
+
parser = MetricsParser.new('jfrog.artifactory', '', 'jfrog.artifactory.metrics')
|
22
|
+
|
23
|
+
normalized_data = parser.normalise_data(platform_metrics)
|
24
|
+
expect(normalized_data.size).to be > 1
|
25
|
+
|
26
|
+
cleaned_data = parser.clean_data(normalized_data)
|
27
|
+
expect(cleaned_data.size).to be > 1
|
28
|
+
|
29
|
+
hash_data_array = parser.extract_metrics_in_hash(cleaned_data, 'jfrog.artifactory', '.')
|
30
|
+
expect(hash_data_array.size).to be > 1
|
31
|
+
|
32
|
+
serialized_data = parser.serialize_data(hash_data_array)
|
33
|
+
expect(serialized_data.size).to be > 1
|
34
|
+
end
|
35
|
+
|
36
|
+
it 'should read sample Xray metrics data and verify the size of parsed data > 1' do
|
37
|
+
platform_metrics = File.read('./spec/fixtures/files/sample_xray_metrics.txt')
|
38
|
+
expect(platform_metrics.size).to be > 1
|
39
|
+
|
40
|
+
parser = MetricsParser.new('jfrog.xray', '', 'jfrog.xray.metrics')
|
41
|
+
|
42
|
+
normalized_data = parser.normalise_data(platform_metrics)
|
43
|
+
expect(normalized_data.size).to be > 1
|
44
|
+
|
45
|
+
cleaned_data = parser.clean_data(normalized_data)
|
46
|
+
expect(cleaned_data.size).to be > 1
|
47
|
+
|
48
|
+
hash_data_array = parser.extract_metrics_in_hash(cleaned_data, 'jfrog.xray', '.')
|
49
|
+
expect(hash_data_array.size).to be > 1
|
50
|
+
|
51
|
+
serialized_data = parser.serialize_data(hash_data_array)
|
52
|
+
expect(serialized_data.size).to be > 1
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,112 @@
|
|
1
|
+
# This file was generated by the `rspec --init` command. Conventionally, all
|
2
|
+
# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
|
3
|
+
# The generated `.rspec` file contains `--require spec_helper` which will cause
|
4
|
+
# this file to always be loaded, without a need to explicitly require it in any
|
5
|
+
# files.
|
6
|
+
#
|
7
|
+
# Given that it is always loaded, you are encouraged to keep this file as
|
8
|
+
# light-weight as possible. Requiring heavyweight dependencies from this file
|
9
|
+
# will add to the boot time of your test suite on EVERY test run, even for an
|
10
|
+
# individual file that may not need all of that loaded. Instead, consider making
|
11
|
+
# a separate helper file that requires the additional dependencies and performs
|
12
|
+
# the additional setup, and require it from the spec files that actually need
|
13
|
+
# it.
|
14
|
+
#
|
15
|
+
# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
|
16
|
+
|
17
|
+
[
|
18
|
+
File.join(File.dirname(__FILE__), '..'),
|
19
|
+
File.join(File.dirname(__FILE__), '..', 'lib/fluent/plugin'),
|
20
|
+
File.join(File.dirname(__FILE__), '..', 'spec')
|
21
|
+
].each do |dir|
|
22
|
+
$LOAD_PATH.unshift(dir) unless $LOAD_PATH.include?(dir)
|
23
|
+
end
|
24
|
+
|
25
|
+
require 'metrics_helper'
|
26
|
+
require 'metrics_parser'
|
27
|
+
|
28
|
+
RSpec.configure do |config|
|
29
|
+
# rspec-expectations config goes here. You can use an alternate
|
30
|
+
# assertion/expectation library such as wrong or the stdlib/minitest
|
31
|
+
# assertions if you prefer.
|
32
|
+
config.expect_with :rspec do |expectations|
|
33
|
+
# This option will default to `true` in RSpec 4. It makes the `description`
|
34
|
+
# and `failure_message` of custom matchers include text for helper methods
|
35
|
+
# defined using `chain`, e.g.:
|
36
|
+
# be_bigger_than(2).and_smaller_than(4).description
|
37
|
+
# # => "be bigger than 2 and smaller than 4"
|
38
|
+
# ...rather than:
|
39
|
+
# # => "be bigger than 2"
|
40
|
+
expectations.include_chain_clauses_in_custom_matcher_descriptions = true
|
41
|
+
end
|
42
|
+
|
43
|
+
# rspec-mocks config goes here. You can use an alternate test double
|
44
|
+
# library (such as bogus or mocha) by changing the `mock_with` option here.
|
45
|
+
config.mock_with :rspec do |mocks|
|
46
|
+
# Prevents you from mocking or stubbing a method that does not exist on
|
47
|
+
# a real object. This is generally recommended, and will default to
|
48
|
+
# `true` in RSpec 4.
|
49
|
+
mocks.verify_partial_doubles = true
|
50
|
+
end
|
51
|
+
|
52
|
+
# This option will default to `:apply_to_host_groups` in RSpec 4 (and will
|
53
|
+
# have no way to turn it off -- the option exists only for backwards
|
54
|
+
# compatibility in RSpec 3). It causes shared context metadata to be
|
55
|
+
# inherited by the metadata hash of host groups and examples, rather than
|
56
|
+
# triggering implicit auto-inclusion in groups with matching metadata.
|
57
|
+
config.shared_context_metadata_behavior = :apply_to_host_groups
|
58
|
+
|
59
|
+
# The settings below are suggested to provide a good initial experience
|
60
|
+
# with RSpec, but feel free to customize to your heart's content.
|
61
|
+
=begin
|
62
|
+
# This allows you to limit a spec run to individual examples or groups
|
63
|
+
# you care about by tagging them with `:focus` metadata. When nothing
|
64
|
+
# is tagged with `:focus`, all examples get run. RSpec also provides
|
65
|
+
# aliases for `it`, `describe`, and `context` that include `:focus`
|
66
|
+
# metadata: `fit`, `fdescribe` and `fcontext`, respectively.
|
67
|
+
config.filter_run_when_matching :focus
|
68
|
+
|
69
|
+
# Allows RSpec to persist some state between runs in order to support
|
70
|
+
# the `--only-failures` and `--next-failure` CLI options. We recommend
|
71
|
+
# you configure your source control system to ignore this file.
|
72
|
+
config.example_status_persistence_file_path = "spec/examples.txt"
|
73
|
+
|
74
|
+
# Limits the available syntax to the non-monkey patched syntax that is
|
75
|
+
# recommended. For more details, see:
|
76
|
+
# - http://rspec.info/blog/2012/06/rspecs-new-expectation-syntax/
|
77
|
+
# - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
|
78
|
+
# - http://rspec.info/blog/2014/05/notable-changes-in-rspec-3/#zero-monkey-patching-mode
|
79
|
+
config.disable_monkey_patching!
|
80
|
+
|
81
|
+
# This setting enables warnings. It's recommended, but in some cases may
|
82
|
+
# be too noisy due to issues in dependencies.
|
83
|
+
config.warnings = true
|
84
|
+
|
85
|
+
# Many RSpec users commonly either run the entire suite or an individual
|
86
|
+
# file, and it's useful to allow more verbose output when running an
|
87
|
+
# individual spec file.
|
88
|
+
if config.files_to_run.one?
|
89
|
+
# Use the documentation formatter for detailed output,
|
90
|
+
# unless a formatter has already been configured
|
91
|
+
# (e.g. via a command-line flag).
|
92
|
+
config.default_formatter = "doc"
|
93
|
+
end
|
94
|
+
|
95
|
+
# Print the 10 slowest examples and example groups at the
|
96
|
+
# end of the spec run, to help surface which specs are running
|
97
|
+
# particularly slow.
|
98
|
+
config.profile_examples = 10
|
99
|
+
|
100
|
+
# Run specs in random order to surface order dependencies. If you find an
|
101
|
+
# order dependency and want to debug it, you can fix the order by providing
|
102
|
+
# the seed, which is printed after each run.
|
103
|
+
# --seed 1234
|
104
|
+
config.order = :random
|
105
|
+
|
106
|
+
# Seed global randomization in this process using the `--seed` CLI option.
|
107
|
+
# Setting this allows you to use `--seed` to deterministically reproduce
|
108
|
+
# test failures related to randomization by passing the same `--seed` value
|
109
|
+
# as the one that triggered the failure.
|
110
|
+
Kernel.srand config.seed
|
111
|
+
=end
|
112
|
+
end
|
data/test/helper.rb
ADDED
@@ -0,0 +1,8 @@
|
|
1
|
+
$LOAD_PATH.unshift(File.expand_path("../../", __FILE__))
|
2
|
+
require "test-unit"
|
3
|
+
require "fluent/test"
|
4
|
+
require "fluent/test/driver/input"
|
5
|
+
require "fluent/test/helpers"
|
6
|
+
|
7
|
+
Test::Unit::TestCase.include(Fluent::Test::Helpers)
|
8
|
+
Test::Unit::TestCase.extend(Fluent::Test::Helpers)
|
@@ -0,0 +1,39 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require_relative '../helper'
|
3
|
+
require 'fluent/plugin/in_jfrog_metrics'
|
4
|
+
|
5
|
+
class JfrogMetricsInputTest < Test::Unit::TestCase
|
6
|
+
setup do
|
7
|
+
Fluent::Test.setup
|
8
|
+
end
|
9
|
+
|
10
|
+
test 'failure' do
|
11
|
+
#flunk
|
12
|
+
end
|
13
|
+
|
14
|
+
# Default configuration for tests
|
15
|
+
CONFIG = %(
|
16
|
+
tag "jfrog.artifactory.metrics"
|
17
|
+
jpd_url "http://localhost:8081"
|
18
|
+
username "admin"
|
19
|
+
apikey "AKCp8k8PbyomdfXsVzoQWaYQZmq9wk49JXQLaP1wXJpa6LdzpFkuJ4VcWd61U6fYWwDJymLDR"
|
20
|
+
metric_prefix "jfrog.artifactory"
|
21
|
+
)
|
22
|
+
|
23
|
+
private
|
24
|
+
|
25
|
+
def create_driver(conf)
|
26
|
+
Fluent::Test::Driver::Input.new(Fluent::Plugin::JfrogMetricsInput).configure(conf)
|
27
|
+
end
|
28
|
+
|
29
|
+
sub_test_case 'Testing' do
|
30
|
+
test 'Testing plugin in_jfrog_metrics' do
|
31
|
+
d = create_driver(CONFIG)
|
32
|
+
begin
|
33
|
+
d.run
|
34
|
+
rescue StandardError => e
|
35
|
+
raise "Test failed due to #{e}"
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
metadata
ADDED
@@ -0,0 +1,134 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-jfrog-metrics
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- MahithaB, VasukiN
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2021-12-14 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.14'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.14'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rake
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '12.0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '12.0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: test-unit
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '3.0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '3.0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: fluentd
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 0.14.10
|
62
|
+
- - "<"
|
63
|
+
- !ruby/object:Gem::Version
|
64
|
+
version: '2'
|
65
|
+
type: :runtime
|
66
|
+
prerelease: false
|
67
|
+
version_requirements: !ruby/object:Gem::Requirement
|
68
|
+
requirements:
|
69
|
+
- - ">="
|
70
|
+
- !ruby/object:Gem::Version
|
71
|
+
version: 0.14.10
|
72
|
+
- - "<"
|
73
|
+
- !ruby/object:Gem::Version
|
74
|
+
version: '2'
|
75
|
+
description: Fluentd Plugin for converting one metrics form of data to another, this
|
76
|
+
is from Prometheus format to Splunk HEC required format
|
77
|
+
email:
|
78
|
+
- 60710901+MahithaB@users.noreply.github.com
|
79
|
+
executables: []
|
80
|
+
extensions: []
|
81
|
+
extra_rdoc_files: []
|
82
|
+
files:
|
83
|
+
- ".gitignore"
|
84
|
+
- ".rspec"
|
85
|
+
- Gemfile
|
86
|
+
- Gemfile.lock
|
87
|
+
- LICENSE
|
88
|
+
- README.md
|
89
|
+
- Rakefile
|
90
|
+
- fluent-plugin-jfrog-metrics.gemspec
|
91
|
+
- lib/fluent/plugin/in_jfrog_metrics.rb
|
92
|
+
- lib/fluent/plugin/metrics_helper.rb
|
93
|
+
- lib/fluent/plugin/metrics_parser.rb
|
94
|
+
- spec/fixtures/files/creds.rb
|
95
|
+
- spec/fixtures/files/sample_artifactory_metrics.txt
|
96
|
+
- spec/fixtures/files/sample_xray_metrics.txt
|
97
|
+
- spec/lib/metrics_helper_spec.rb
|
98
|
+
- spec/lib/metrics_parser_spec.rb
|
99
|
+
- spec/spec_helper.rb
|
100
|
+
- test/helper.rb
|
101
|
+
- test/plugin/test_in_jfrog_metrics.rb
|
102
|
+
homepage: https://github.com/jfrog/jfrog-fluentd-plugins/tree/main/fluent-plugin-jfrog-metrics
|
103
|
+
licenses:
|
104
|
+
- Apache-2.0
|
105
|
+
metadata: {}
|
106
|
+
post_install_message:
|
107
|
+
rdoc_options: []
|
108
|
+
require_paths:
|
109
|
+
- lib
|
110
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
111
|
+
requirements:
|
112
|
+
- - ">="
|
113
|
+
- !ruby/object:Gem::Version
|
114
|
+
version: '0'
|
115
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
116
|
+
requirements:
|
117
|
+
- - ">="
|
118
|
+
- !ruby/object:Gem::Version
|
119
|
+
version: '0'
|
120
|
+
requirements: []
|
121
|
+
rubygems_version: 3.0.3
|
122
|
+
signing_key:
|
123
|
+
specification_version: 4
|
124
|
+
summary: Fluentd Plugin for converting one metrics form of data to another, this is
|
125
|
+
from Prometheus format to Splunk HEC required format
|
126
|
+
test_files:
|
127
|
+
- spec/fixtures/files/creds.rb
|
128
|
+
- spec/fixtures/files/sample_artifactory_metrics.txt
|
129
|
+
- spec/fixtures/files/sample_xray_metrics.txt
|
130
|
+
- spec/lib/metrics_helper_spec.rb
|
131
|
+
- spec/lib/metrics_parser_spec.rb
|
132
|
+
- spec/spec_helper.rb
|
133
|
+
- test/helper.rb
|
134
|
+
- test/plugin/test_in_jfrog_metrics.rb
|