logstash-output-edge_prom 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/Gemfile +19 -0
- data/README.md +88 -0
- data/lib/logstash/outputs/Edge/TimeSeries.rb +67 -0
- data/lib/logstash/outputs/Edge/batch.rb +63 -0
- data/lib/logstash/outputs/Edge/entry.rb +26 -0
- data/lib/logstash/outputs/edge_prom.rb +303 -0
- data/logstash-output-edge_prom.gemspec +33 -0
- metadata +165 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 4ae9aa1cf6be8b8357d308f3957d3d855a0d20cc5a52624e0fd6a03e4dafd58b
|
4
|
+
data.tar.gz: 64bab6b47d181438f7d669327540f9300f5aa0a34702ed7570f715e772c7d23c
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 883f3a85b391f146f8f932640e7c14005c90498a7fa845aa0fa006c1b935fa128f98cf4f4485ae10e9900aeaeb064f9638d889e4d482f27a14f7cbdcb22044b0
|
7
|
+
data.tar.gz: 473d4715f229775c1d68af285eccbd3c2c15e94807a5befea08ca6fdce4add9b27d33ec931bec68bcc1d0fe16832e5a05970cd1b3360bf80e5dea5611a1010f8
|
data/Gemfile
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
source 'https://rubygems.org'
|
2
|
+
|
3
|
+
gemspec
|
4
|
+
|
5
|
+
logstash_path = ENV["LOGSTASH_PATH"] || "./logstash"
|
6
|
+
|
7
|
+
if Dir.exist?(logstash_path)
|
8
|
+
gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
|
9
|
+
gem 'logstash-mixin-http_client', :path => "#{logstash_path}/vendor/bundle/jruby/2.6.0/gems/logstash-mixin-http_client"
|
10
|
+
else
|
11
|
+
raise 'missing logstash vendoring'
|
12
|
+
end
|
13
|
+
|
14
|
+
gem "oauth2", "~> 2.0.9"
|
15
|
+
gem "webmock", "~> 3.8"
|
16
|
+
gem 'logstash-devutils', "~> 2"
|
17
|
+
gem 'sinatra'
|
18
|
+
gem 'webrick'
|
19
|
+
gem "prometheus-client", "~> 1.0.0"
|
data/README.md
ADDED
@@ -0,0 +1,88 @@
|
|
1
|
+
# Contributing to Loki Logstash Output Plugin
|
2
|
+
|
3
|
+
For information about how to use this plugin see this [documentation](../../docs/sources/clients/logstash/_index.md).
|
4
|
+
|
5
|
+
## Install dependencies
|
6
|
+
|
7
|
+
First, make sure you have JDK version `8` or `11` installed and you have set the `JAVA_HOME` environment variable.
|
8
|
+
|
9
|
+
You need to setup JRuby environment to build this plugin. Refer https://github.com/rbenv/rbenv for setting up your rbenv environment.
|
10
|
+
|
11
|
+
After setting up `rbenv`. Install JRuby
|
12
|
+
|
13
|
+
```bash
|
14
|
+
rbenv install jruby-9.2.10.0
|
15
|
+
rbenv local jruby-9.2.10.0
|
16
|
+
```
|
17
|
+
|
18
|
+
Check that the environment is configured
|
19
|
+
|
20
|
+
```bash
|
21
|
+
ruby --version
|
22
|
+
jruby 9.2.10
|
23
|
+
```
|
24
|
+
|
25
|
+
You should make sure you are running `jruby` and not `ruby`. If the command `ruby --version` still shows `ruby` and not `jruby`, check that PATH contains `$HOME/.rbenv/shims` and `$HOME/.rbenv/bin`. Also verify that you have this in your bash profile:
|
26
|
+
|
27
|
+
```bash
|
28
|
+
export PATH="$HOME/.rbenv/bin:$PATH"
|
29
|
+
eval "$(rbenv init -)"
|
30
|
+
```
|
31
|
+
|
32
|
+
Then install bundler:
|
33
|
+
|
34
|
+
```bash
|
35
|
+
gem install bundler:2.1.4
|
36
|
+
```
|
37
|
+
|
38
|
+
Follow those instructions to [install logstash](https://www.elastic.co/guide/en/logstash/current/installing-logstash.html) before moving to the next section.
|
39
|
+
|
40
|
+
## Build and test the plugin
|
41
|
+
|
42
|
+
### Install required packages
|
43
|
+
|
44
|
+
```bash
|
45
|
+
git clone git@github.com:elastic/logstash.git
|
46
|
+
cd logstash
|
47
|
+
git checkout tags/v7.16.1
|
48
|
+
export LOGSTASH_PATH="$(pwd)"
|
49
|
+
export GEM_PATH="$LOGSTASH_PATH/vendor/bundle/jruby/2.5.0"
|
50
|
+
export GEM_HOME="$LOGSTASH_PATH/vendor/bundle/jruby/2.5.0"
|
51
|
+
./gradlew assemble
|
52
|
+
cd ..
|
53
|
+
ruby -S bundle /config set --local path "$LOGSTASH_PATH/vendor/bundle"
|
54
|
+
ruby -S bundle install
|
55
|
+
ruby -S bundle exec rake vendor
|
56
|
+
```
|
57
|
+
|
58
|
+
### Build the plugin
|
59
|
+
|
60
|
+
```bash
|
61
|
+
gem build logstash-output-loki.gemspec
|
62
|
+
```
|
63
|
+
|
64
|
+
### Test
|
65
|
+
|
66
|
+
```bash
|
67
|
+
ruby -S bundle exec rspec
|
68
|
+
```
|
69
|
+
|
70
|
+
Alternatively if you don't want to install JRuby. Enter inside logstash-loki container.
|
71
|
+
|
72
|
+
```bash
|
73
|
+
docker build -t logstash-loki ./
|
74
|
+
docker run -v $(pwd)/spec:/home/logstash/spec -it --rm --entrypoint /bin/sh logstash-loki
|
75
|
+
bundle exec rspec
|
76
|
+
```
|
77
|
+
|
78
|
+
## Install plugin to local logstash
|
79
|
+
|
80
|
+
```bash
|
81
|
+
bin/logstash-plugin install --no-verify --local logstash-output-loki-1.0.0.gem
|
82
|
+
```
|
83
|
+
|
84
|
+
## Send sample event and check plugin is working
|
85
|
+
|
86
|
+
```bash
|
87
|
+
bin/logstash -f loki.conf
|
88
|
+
```
|
@@ -0,0 +1,67 @@
|
|
1
|
+
|
2
|
+
module Edge
|
3
|
+
class TimeSeries
|
4
|
+
attr_reader :labels, :samples
|
5
|
+
|
6
|
+
Label[] labels
|
7
|
+
Sample samples
|
8
|
+
|
9
|
+
def initialize()
|
10
|
+
end
|
11
|
+
|
12
|
+
def label_set_for(labels)
|
13
|
+
#@logger.info("label_set_for: started method")
|
14
|
+
return stringify_values(labels) if !labels.empty?
|
15
|
+
end
|
16
|
+
|
17
|
+
def stringify_values(labels)
|
18
|
+
#@logger.info("stringify_values: started method")
|
19
|
+
stringified = {}
|
20
|
+
labels.each { |k,v| stringified[k] = v.to_s }
|
21
|
+
|
22
|
+
stringified
|
23
|
+
end
|
24
|
+
|
25
|
+
def setLabels=(labels_param)
|
26
|
+
labels_param.each { |k,v| labels.append(Labels.new(k, v)) }
|
27
|
+
end
|
28
|
+
|
29
|
+
def setSamples=(samples_param)
|
30
|
+
samples = Samples.new(samples_param)
|
31
|
+
end
|
32
|
+
|
33
|
+
def WriteRequest(uri, token)
|
34
|
+
self.setLabels(label_set_for({ '__name__':'vsphere_vm', app: 'apmt_metric_test', env: 'test', product_id: 'apmt_test', provider: 'onprem', region: 'westeurope', site_code: 'USTIS01'}))
|
35
|
+
self.setSamples([Time.now.to_f(), 10.785]})
|
36
|
+
payload = Marshal.dump(self)
|
37
|
+
response = token.post(uri, {:body => payload, :headers => {'Content-Type' => 'application/x-protobuf', 'Content-Encoding' => 'snappy', 'User-Agent' => 'mop-edge-1.0.0', 'X-Prometheus-Remote-Write-Version' => '0.1.0'}})
|
38
|
+
return response
|
39
|
+
end
|
40
|
+
|
41
|
+
def Send(uri, token)
|
42
|
+
return WriteRequest(uri, token)
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
class Label
|
47
|
+
String name
|
48
|
+
String value
|
49
|
+
|
50
|
+
def initialize(name_param, value_param)
|
51
|
+
name = name_param
|
52
|
+
value = value_param
|
53
|
+
end
|
54
|
+
|
55
|
+
end
|
56
|
+
|
57
|
+
class Sample
|
58
|
+
Double value
|
59
|
+
Double timestamp
|
60
|
+
|
61
|
+
def initialize(value_param, timestamp_param)
|
62
|
+
timestamp = timestamp_param
|
63
|
+
value = value_param
|
64
|
+
end
|
65
|
+
|
66
|
+
end
|
67
|
+
end
|
@@ -0,0 +1,63 @@
|
|
1
|
+
require 'time'
|
2
|
+
|
3
|
+
module Edge
|
4
|
+
class Batch
|
5
|
+
attr_reader :streams
|
6
|
+
def initialize(e)
|
7
|
+
@bytes = 0
|
8
|
+
@createdAt = Time.now
|
9
|
+
@streams = {}
|
10
|
+
add(e)
|
11
|
+
end
|
12
|
+
|
13
|
+
def size_bytes
|
14
|
+
return @bytes
|
15
|
+
end
|
16
|
+
|
17
|
+
def add(e)
|
18
|
+
@bytes = @bytes + e.entry['line'].length
|
19
|
+
|
20
|
+
# Append the entry to an already existing stream (if any)
|
21
|
+
labels = e.labels.sort.to_h
|
22
|
+
labelkey = labels.to_s
|
23
|
+
if @streams.has_key?(labelkey)
|
24
|
+
stream = @streams[labelkey]
|
25
|
+
stream['entries'].append(e.entry)
|
26
|
+
return
|
27
|
+
else
|
28
|
+
# Add the entry as a new stream
|
29
|
+
@streams[labelkey] = {
|
30
|
+
"labels" => labels,
|
31
|
+
"entries" => [e.entry],
|
32
|
+
}
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def size_bytes_after(line)
|
37
|
+
return @bytes + line.length
|
38
|
+
end
|
39
|
+
|
40
|
+
def age()
|
41
|
+
return Time.now - @createdAt
|
42
|
+
end
|
43
|
+
|
44
|
+
def to_json
|
45
|
+
streams = []
|
46
|
+
@streams.each { |_ , stream|
|
47
|
+
streams.append(build_stream(stream))
|
48
|
+
}
|
49
|
+
return {"streams"=>streams}.to_json
|
50
|
+
end
|
51
|
+
|
52
|
+
def build_stream(stream)
|
53
|
+
values = []
|
54
|
+
stream['entries'].each { |entry|
|
55
|
+
values.append([entry['ts'].to_s, entry['line']])
|
56
|
+
}
|
57
|
+
return {
|
58
|
+
'stream'=>stream['labels'],
|
59
|
+
'values' => values
|
60
|
+
}
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
module Edge
|
2
|
+
def to_ns(s)
|
3
|
+
(s.to_f * (10**9)).to_i
|
4
|
+
end
|
5
|
+
class Entry
|
6
|
+
include Edge
|
7
|
+
attr_reader :labels, :entry
|
8
|
+
def initialize(event,message_field,include_fields)
|
9
|
+
@entry = {
|
10
|
+
"ts" => to_ns(event.get("@timestamp")),
|
11
|
+
"line" => event.get(message_field).to_s
|
12
|
+
}
|
13
|
+
event = event.clone()
|
14
|
+
event.remove(message_field)
|
15
|
+
event.remove("@timestamp")
|
16
|
+
|
17
|
+
@labels = {}
|
18
|
+
event.to_hash.each { |key,value|
|
19
|
+
next if key.start_with?('@')
|
20
|
+
next if value.is_a?(Hash)
|
21
|
+
next if include_fields.length() > 0 and not include_fields.include?(key)
|
22
|
+
@labels[key] = value.to_s
|
23
|
+
}
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,303 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/outputs/base"
|
3
|
+
require "logstash/outputs/edge/entry"
|
4
|
+
require "logstash/outputs/edge/batch"
|
5
|
+
require "logstash/namespace"
|
6
|
+
require 'net/http'
|
7
|
+
require 'time'
|
8
|
+
require 'uri'
|
9
|
+
require 'json'
|
10
|
+
require 'prometheus/client'
|
11
|
+
require 'prometheus/client/push'
|
12
|
+
require 'oauth2'
|
13
|
+
|
14
|
+
class LogStash::Outputs::Edge_Prom < LogStash::Outputs::Base
|
15
|
+
include Edge
|
16
|
+
config_name "edge_prom"
|
17
|
+
|
18
|
+
## 'A single instance of the Output will be shared among the pipeline worker threads'
|
19
|
+
concurrency :single
|
20
|
+
|
21
|
+
## 'Prom URL'
|
22
|
+
config :url, :validate => :string, :required => true
|
23
|
+
|
24
|
+
## 'BasicAuth credentials'
|
25
|
+
config :client_id, :validate => :string, :required => true
|
26
|
+
config :client_secret, :validate => :string, secret: true, :required => true
|
27
|
+
|
28
|
+
## 'Prom Token URL Domain'
|
29
|
+
config :tokenurl_domain, :validate => :string, :required => true
|
30
|
+
|
31
|
+
## 'Prom Token URL Endpoint'
|
32
|
+
config :tokenurl_endpoint, :validate => :string, :default => "oauth2/v2.0/token", :required => false
|
33
|
+
|
34
|
+
## 'Scopes'
|
35
|
+
config :scopes, :validate => :string, :required => true
|
36
|
+
|
37
|
+
## 'Proxy URL'
|
38
|
+
config :proxy_url, :validate => :string, :required => false
|
39
|
+
|
40
|
+
## 'Disable server certificate verification'
|
41
|
+
config :insecure_skip_verify, :validate => :boolean, :default => false, :required => false
|
42
|
+
|
43
|
+
## 'Client certificate'
|
44
|
+
config :cert, :validate => :path, :required => false
|
45
|
+
config :key, :validate => :path, :required => false
|
46
|
+
|
47
|
+
## 'TLS'
|
48
|
+
config :ca_cert, :validate => :path, :required => false
|
49
|
+
|
50
|
+
## 'Maximum batch size to accrue before pushing to loki. Defaults to 102400 bytes'
|
51
|
+
config :batch_size, :validate => :number, :default => 102400, :required => false
|
52
|
+
|
53
|
+
## 'Interval in seconds to wait before pushing a batch of records to loki. Defaults to 1 second'
|
54
|
+
config :batch_wait, :validate => :number, :default => 1, :required => false
|
55
|
+
|
56
|
+
## 'Log line field to pick from logstash. Defaults to "message"'
|
57
|
+
config :message_field, :validate => :string, :default => "message", :required => false
|
58
|
+
|
59
|
+
## 'Backoff configuration. Initial backoff time between retries. Default 1s'
|
60
|
+
config :min_delay, :validate => :number, :default => 1, :required => false
|
61
|
+
|
62
|
+
## 'An array of fields to map to labels, if defined only fields in this list will be mapped.'
|
63
|
+
config :include_fields, :validate => :array, :default => [], :required => false
|
64
|
+
|
65
|
+
## 'Backoff configuration. Maximum backoff time between retries. Default 300s'
|
66
|
+
config :max_delay, :validate => :number, :default => 300, :required => false
|
67
|
+
|
68
|
+
## 'Backoff configuration. Maximum number of retries to do'
|
69
|
+
config :retries, :validate => :number, :default => 10, :required => false
|
70
|
+
|
71
|
+
attr_reader :batch
|
72
|
+
public
|
73
|
+
def register
|
74
|
+
puts "1"
|
75
|
+
@uri = URI.parse(@url)
|
76
|
+
unless @uri.is_a?(URI::HTTP) || @uri.is_a?(URI::HTTPS)
|
77
|
+
raise LogStash::ConfigurationError, "url parameter must be valid HTTP, currently '#{@url}'"
|
78
|
+
end
|
79
|
+
|
80
|
+
if @min_delay > @max_delay
|
81
|
+
raise LogStash::ConfigurationError, "Min delay should be less than Max delay, currently 'Min delay is #{@min_delay} and Max delay is #{@max_delay}'"
|
82
|
+
end
|
83
|
+
|
84
|
+
@logger.info("Prom output plugin", :class => self.class.name)
|
85
|
+
|
86
|
+
# initialize Queue and Mutex
|
87
|
+
@entries = Queue.new
|
88
|
+
@mutex = Mutex.new
|
89
|
+
@stop = false
|
90
|
+
|
91
|
+
# create nil batch object.
|
92
|
+
@batch = nil
|
93
|
+
|
94
|
+
# validate certs
|
95
|
+
if ssl_cert?
|
96
|
+
load_ssl
|
97
|
+
validate_ssl_key
|
98
|
+
end
|
99
|
+
|
100
|
+
# start batch_max_wait and batch_max_size threads
|
101
|
+
@batch_wait_thread = Thread.new{max_batch_wait()}
|
102
|
+
@batch_size_thread = Thread.new{max_batch_size()}
|
103
|
+
end
|
104
|
+
|
105
|
+
def max_batch_size
|
106
|
+
#@logger.info("max_batch_size: started method")
|
107
|
+
loop do
|
108
|
+
@mutex.synchronize do
|
109
|
+
return if @stop
|
110
|
+
end
|
111
|
+
|
112
|
+
e = @entries.deq
|
113
|
+
return if e.nil?
|
114
|
+
|
115
|
+
@mutex.synchronize do
|
116
|
+
if !add_entry_to_batch(e)
|
117
|
+
@logger.debug("Max batch_size is reached. Sending batch to edge-loki")
|
118
|
+
send(@batch)
|
119
|
+
@batch = Batch.new(e)
|
120
|
+
end
|
121
|
+
end
|
122
|
+
end
|
123
|
+
end
|
124
|
+
|
125
|
+
def max_batch_wait
|
126
|
+
# minimum wait frequency is 10 milliseconds
|
127
|
+
min_wait_checkfrequency = 1/100
|
128
|
+
max_wait_checkfrequency = @batch_wait
|
129
|
+
if max_wait_checkfrequency < min_wait_checkfrequency
|
130
|
+
max_wait_checkfrequency = min_wait_checkfrequency
|
131
|
+
end
|
132
|
+
|
133
|
+
loop do
|
134
|
+
@mutex.synchronize do
|
135
|
+
return if @stop
|
136
|
+
end
|
137
|
+
|
138
|
+
sleep(max_wait_checkfrequency)
|
139
|
+
if is_batch_expired
|
140
|
+
@mutex.synchronize do
|
141
|
+
@logger.debug("Max batch_wait time is reached. Sending batch to loki")
|
142
|
+
send(@batch)
|
143
|
+
@batch = nil
|
144
|
+
end
|
145
|
+
end
|
146
|
+
end
|
147
|
+
end
|
148
|
+
|
149
|
+
def ssl_cert?
|
150
|
+
!@key.nil? && !@cert.nil?
|
151
|
+
end
|
152
|
+
|
153
|
+
def load_ssl
|
154
|
+
@cert = OpenSSL::X509::Certificate.new(File.read(@cert)) if @cert
|
155
|
+
@key = OpenSSL::PKey.read(File.read(@key)) if @key
|
156
|
+
end
|
157
|
+
|
158
|
+
def validate_ssl_key
|
159
|
+
if !@key.is_a?(OpenSSL::PKey::RSA) && !@key.is_a?(OpenSSL::PKey::DSA)
|
160
|
+
raise LogStash::ConfigurationError, "Unsupported private key type '#{@key.class}''"
|
161
|
+
end
|
162
|
+
end
|
163
|
+
|
164
|
+
def ssl_opts(uri)
|
165
|
+
opts = {
|
166
|
+
use_ssl: uri.scheme == 'https'
|
167
|
+
}
|
168
|
+
|
169
|
+
# disable server certificate verification
|
170
|
+
if @insecure_skip_verify
|
171
|
+
opts = opts.merge(
|
172
|
+
verify_mode: OpenSSL::SSL::VERIFY_NONE
|
173
|
+
)
|
174
|
+
end
|
175
|
+
|
176
|
+
if !@cert.nil? && !@key.nil?
|
177
|
+
opts = opts.merge(
|
178
|
+
verify_mode: OpenSSL::SSL::VERIFY_PEER,
|
179
|
+
cert: @cert,
|
180
|
+
key: @key
|
181
|
+
)
|
182
|
+
end
|
183
|
+
|
184
|
+
unless @ca_cert.nil?
|
185
|
+
opts = opts.merge(
|
186
|
+
ca_file: @ca_cert
|
187
|
+
)
|
188
|
+
end
|
189
|
+
opts
|
190
|
+
end
|
191
|
+
|
192
|
+
# Add an entry to the current batch returns false if the batch is full and the entry can't be added.
|
193
|
+
def add_entry_to_batch(e)
|
194
|
+
line = e.entry['line']
|
195
|
+
# we don't want to send empty lines.
|
196
|
+
return true if line.to_s.strip.empty?
|
197
|
+
|
198
|
+
if @batch.nil?
|
199
|
+
@batch = Batch.new(e)
|
200
|
+
return true
|
201
|
+
end
|
202
|
+
|
203
|
+
if @batch.size_bytes_after(line) > @batch_size
|
204
|
+
return false
|
205
|
+
end
|
206
|
+
@batch.add(e)
|
207
|
+
return true
|
208
|
+
end
|
209
|
+
|
210
|
+
def is_batch_expired
|
211
|
+
return !@batch.nil? && @batch.age() >= @batch_wait
|
212
|
+
end
|
213
|
+
|
214
|
+
## Receives logstash events
|
215
|
+
public
|
216
|
+
def receive(event)
|
217
|
+
@entries << Entry.new(event, @message_field, @include_fields)
|
218
|
+
end
|
219
|
+
|
220
|
+
def close
|
221
|
+
@entries.close
|
222
|
+
@mutex.synchronize do
|
223
|
+
@stop = true
|
224
|
+
end
|
225
|
+
@batch_wait_thread.join
|
226
|
+
@batch_size_thread.join
|
227
|
+
|
228
|
+
# if by any chance we still have a forming batch, we need to send it.
|
229
|
+
send(@batch) if !@batch.nil?
|
230
|
+
@batch = nil
|
231
|
+
end
|
232
|
+
|
233
|
+
def send(batch)
|
234
|
+
@logger.info("send: started method")
|
235
|
+
payload = batch.to_json
|
236
|
+
client = createClient()
|
237
|
+
access = getToken(client)
|
238
|
+
client = Edge::TimeSeries.new()
|
239
|
+
client.Send(@uri, access)
|
240
|
+
end
|
241
|
+
|
242
|
+
def createClient()
|
243
|
+
@logger.info("createClient: started method")
|
244
|
+
client = OAuth2::Client.new(@client_id, @client_secret, auth_scheme: "basic_auth", site: @tokenurl_domain,:token_url => @tokenurl_endpoint)
|
245
|
+
return client
|
246
|
+
end
|
247
|
+
|
248
|
+
def createClientwithProxy()
|
249
|
+
#@logger.info("createClient: started method")
|
250
|
+
conn_opts = {}
|
251
|
+
conn_opts = conn_opts.merge('ssl' => {verify: false})
|
252
|
+
conn_opts = conn_opts.merge('proxy' => @proxy_url)
|
253
|
+
client = OAuth2::Client.new(@client_id, @client_secret, auth_scheme: "basic_auth", site: @tokenurl_domain,:token_url => @tokenurl_endpoint, :connection_opts => conn_opts)
|
254
|
+
return client
|
255
|
+
end
|
256
|
+
|
257
|
+
def getToken(client)
|
258
|
+
params = {}
|
259
|
+
opts = {}
|
260
|
+
params = params.merge('scope' => @scopes)
|
261
|
+
access = client.client_credentials.get_token(params, opts)
|
262
|
+
return access
|
263
|
+
end
|
264
|
+
|
265
|
+
def edge_http_request(token, payload)
|
266
|
+
@logger.info("edge_http_request: started method")
|
267
|
+
cntLines = JSON.parse(payload)["streams"][0]["values"].size
|
268
|
+
@logger.info("processing #{cntLines} lines to edge-prom")
|
269
|
+
retry_count = 0
|
270
|
+
delay = @min_delay
|
271
|
+
begin
|
272
|
+
res = token.post(@uri, {:body => payload1, :headers => params})
|
273
|
+
status_code = "#{res.status}"
|
274
|
+
@logger.info("send: status_code", :status_code => status_code)
|
275
|
+
begin
|
276
|
+
@logger.info("send: res_body", :res_body => res)
|
277
|
+
#JSON.parse(res.body)
|
278
|
+
#@logger.info("send res parsed", :res.parsed => res.parsed)
|
279
|
+
rescue StandardError => err
|
280
|
+
@logger.warn("Failed to send batch, attempt: ", :error_inspect => err.inspect, :error => err)
|
281
|
+
end
|
282
|
+
return status_code if !status_code.nil? && status_code.to_i != 429 && status_code.to_i.div(100) != 5
|
283
|
+
return res
|
284
|
+
raise StandardError.new res
|
285
|
+
rescue StandardError => e
|
286
|
+
retry_count += 1
|
287
|
+
@logger.warn("Failed to send batch, attempt: #{retry_count}/#{@retries}", :error_inspect => e.inspect, :error => e)
|
288
|
+
if retry_count < @retries
|
289
|
+
sleep delay
|
290
|
+
if delay * 2 <= @max_delay
|
291
|
+
delay = delay * 2
|
292
|
+
else
|
293
|
+
delay = @max_delay
|
294
|
+
end
|
295
|
+
retry
|
296
|
+
else
|
297
|
+
@logger.error("Failed to send batch", :error_inspect => e.inspect, :error => e)
|
298
|
+
return res
|
299
|
+
end
|
300
|
+
end
|
301
|
+
end
|
302
|
+
|
303
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
Gem::Specification.new do |s|
|
2
|
+
s.name = 'logstash-output-edge_prom'
|
3
|
+
s.version = '1.0.0'
|
4
|
+
s.authors = ['Britto Prabhu']
|
5
|
+
s.email = ['britto.prabhu@apmterminals.com']
|
6
|
+
|
7
|
+
s.summary = 'Output plugin to ship logs to Loki server with oauth2 token'
|
8
|
+
s.description = 'This Output plugin to ship logs to Loki server with oauth2 token and proxy support'
|
9
|
+
s.summary = "Sends events to a generic HTTP or HTTPS endpoint with additional oauth2 token support"
|
10
|
+
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program. This plugin supports oauth2 token."
|
11
|
+
s.homepage = 'https://github.com/Maersk-Global/apmt-observability-deployment'
|
12
|
+
s.license = 'Apache-2.0'
|
13
|
+
s.require_paths = ["lib"]
|
14
|
+
|
15
|
+
# Files
|
16
|
+
s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
|
17
|
+
|
18
|
+
# Tests
|
19
|
+
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
20
|
+
|
21
|
+
# Special flag to let us know this is actually a logstash plugin
|
22
|
+
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
|
23
|
+
|
24
|
+
# Gem dependencies
|
25
|
+
s.add_runtime_dependency "oauth2", "~> 2.0.9"
|
26
|
+
s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
|
27
|
+
s.add_runtime_dependency "logstash-mixin-http_client", ">= 7.2.0", "< 8.0.0"
|
28
|
+
s.add_runtime_dependency "prometheus-client", "1.0.0"
|
29
|
+
|
30
|
+
s.add_development_dependency 'logstash-devutils'
|
31
|
+
s.add_development_dependency 'sinatra'
|
32
|
+
s.add_development_dependency 'webrick'
|
33
|
+
end
|
metadata
ADDED
@@ -0,0 +1,165 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: logstash-output-edge_prom
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 1.0.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Britto Prabhu
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2023-09-23 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
requirement: !ruby/object:Gem::Requirement
|
15
|
+
requirements:
|
16
|
+
- - "~>"
|
17
|
+
- !ruby/object:Gem::Version
|
18
|
+
version: 2.0.9
|
19
|
+
name: oauth2
|
20
|
+
prerelease: false
|
21
|
+
type: :runtime
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 2.0.9
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
requirement: !ruby/object:Gem::Requirement
|
29
|
+
requirements:
|
30
|
+
- - ">="
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '1.60'
|
33
|
+
- - "<="
|
34
|
+
- !ruby/object:Gem::Version
|
35
|
+
version: '2.99'
|
36
|
+
name: logstash-core-plugin-api
|
37
|
+
prerelease: false
|
38
|
+
type: :runtime
|
39
|
+
version_requirements: !ruby/object:Gem::Requirement
|
40
|
+
requirements:
|
41
|
+
- - ">="
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
version: '1.60'
|
44
|
+
- - "<="
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: '2.99'
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
49
|
+
requirements:
|
50
|
+
- - ">="
|
51
|
+
- !ruby/object:Gem::Version
|
52
|
+
version: 7.2.0
|
53
|
+
- - "<"
|
54
|
+
- !ruby/object:Gem::Version
|
55
|
+
version: 8.0.0
|
56
|
+
name: logstash-mixin-http_client
|
57
|
+
prerelease: false
|
58
|
+
type: :runtime
|
59
|
+
version_requirements: !ruby/object:Gem::Requirement
|
60
|
+
requirements:
|
61
|
+
- - ">="
|
62
|
+
- !ruby/object:Gem::Version
|
63
|
+
version: 7.2.0
|
64
|
+
- - "<"
|
65
|
+
- !ruby/object:Gem::Version
|
66
|
+
version: 8.0.0
|
67
|
+
- !ruby/object:Gem::Dependency
|
68
|
+
requirement: !ruby/object:Gem::Requirement
|
69
|
+
requirements:
|
70
|
+
- - '='
|
71
|
+
- !ruby/object:Gem::Version
|
72
|
+
version: 1.0.0
|
73
|
+
name: prometheus-client
|
74
|
+
prerelease: false
|
75
|
+
type: :runtime
|
76
|
+
version_requirements: !ruby/object:Gem::Requirement
|
77
|
+
requirements:
|
78
|
+
- - '='
|
79
|
+
- !ruby/object:Gem::Version
|
80
|
+
version: 1.0.0
|
81
|
+
- !ruby/object:Gem::Dependency
|
82
|
+
requirement: !ruby/object:Gem::Requirement
|
83
|
+
requirements:
|
84
|
+
- - ">="
|
85
|
+
- !ruby/object:Gem::Version
|
86
|
+
version: '0'
|
87
|
+
name: logstash-devutils
|
88
|
+
prerelease: false
|
89
|
+
type: :development
|
90
|
+
version_requirements: !ruby/object:Gem::Requirement
|
91
|
+
requirements:
|
92
|
+
- - ">="
|
93
|
+
- !ruby/object:Gem::Version
|
94
|
+
version: '0'
|
95
|
+
- !ruby/object:Gem::Dependency
|
96
|
+
requirement: !ruby/object:Gem::Requirement
|
97
|
+
requirements:
|
98
|
+
- - ">="
|
99
|
+
- !ruby/object:Gem::Version
|
100
|
+
version: '0'
|
101
|
+
name: sinatra
|
102
|
+
prerelease: false
|
103
|
+
type: :development
|
104
|
+
version_requirements: !ruby/object:Gem::Requirement
|
105
|
+
requirements:
|
106
|
+
- - ">="
|
107
|
+
- !ruby/object:Gem::Version
|
108
|
+
version: '0'
|
109
|
+
- !ruby/object:Gem::Dependency
|
110
|
+
requirement: !ruby/object:Gem::Requirement
|
111
|
+
requirements:
|
112
|
+
- - ">="
|
113
|
+
- !ruby/object:Gem::Version
|
114
|
+
version: '0'
|
115
|
+
name: webrick
|
116
|
+
prerelease: false
|
117
|
+
type: :development
|
118
|
+
version_requirements: !ruby/object:Gem::Requirement
|
119
|
+
requirements:
|
120
|
+
- - ">="
|
121
|
+
- !ruby/object:Gem::Version
|
122
|
+
version: '0'
|
123
|
+
description: This gem is a Logstash plugin required to be installed on top of the
|
124
|
+
Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
|
125
|
+
gem is not a stand-alone program. This plugin supports oauth2 token.
|
126
|
+
email:
|
127
|
+
- britto.prabhu@apmterminals.com
|
128
|
+
executables: []
|
129
|
+
extensions: []
|
130
|
+
extra_rdoc_files: []
|
131
|
+
files:
|
132
|
+
- Gemfile
|
133
|
+
- README.md
|
134
|
+
- lib/logstash/outputs/Edge/TimeSeries.rb
|
135
|
+
- lib/logstash/outputs/Edge/batch.rb
|
136
|
+
- lib/logstash/outputs/Edge/entry.rb
|
137
|
+
- lib/logstash/outputs/edge_prom.rb
|
138
|
+
- logstash-output-edge_prom.gemspec
|
139
|
+
homepage: https://github.com/Maersk-Global/apmt-observability-deployment
|
140
|
+
licenses:
|
141
|
+
- Apache-2.0
|
142
|
+
metadata:
|
143
|
+
logstash_plugin: 'true'
|
144
|
+
logstash_group: output
|
145
|
+
post_install_message:
|
146
|
+
rdoc_options: []
|
147
|
+
require_paths:
|
148
|
+
- lib
|
149
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
150
|
+
requirements:
|
151
|
+
- - ">="
|
152
|
+
- !ruby/object:Gem::Version
|
153
|
+
version: '0'
|
154
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
155
|
+
requirements:
|
156
|
+
- - ">="
|
157
|
+
- !ruby/object:Gem::Version
|
158
|
+
version: '0'
|
159
|
+
requirements: []
|
160
|
+
rubygems_version: 3.2.33
|
161
|
+
signing_key:
|
162
|
+
specification_version: 4
|
163
|
+
summary: Sends events to a generic HTTP or HTTPS endpoint with additional oauth2 token
|
164
|
+
support
|
165
|
+
test_files: []
|