logstash-output-edge_loki 1.0.8 → 1.0.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +0 -0
- data/README.md +0 -0
- data/lib/logstash/outputs/edge_loki.rb +87 -72
- data/lib/logstash/outputs/edge_loki_20240722.rb +303 -0
- data/lib/logstash/outputs/loki/batch.rb +0 -0
- data/lib/logstash/outputs/loki/entry.rb +0 -0
- data/logstash-output-edge_loki.gemspec +3 -3
- data/spec/outputs/loki/entry_spec.rb +0 -0
- data/spec/outputs/loki_spec.rb +0 -0
- metadata +17 -16
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5853b6587938a56259061c3bb90d770d233f3a204efe9867d5d35f761d85fb21
|
4
|
+
data.tar.gz: da0b9595f03cd605c6aa28fc66864bcf4375466879636385cef8816491be6494
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7fb2656179619773e32aa46625819b2809c31da737e229033f483d5429d63f585a9b9dffd22a1fcd4daed1971137cc189badaee468c32a83a53f029caffcc99e
|
7
|
+
data.tar.gz: 19cb39bc4d5d9e6b79bf2014f3106615ea788e8abdb3d2abb2d90856234a766853ef859bc7ed0934cf2298c5ce8664d93ca2a7cd9d50725de7c56f8aa3cbc442
|
data/Gemfile
CHANGED
File without changes
|
data/README.md
CHANGED
File without changes
|
@@ -8,6 +8,7 @@ require 'time'
|
|
8
8
|
require 'uri'
|
9
9
|
require 'json'
|
10
10
|
require 'oauth2'
|
11
|
+
require "zlib"
|
11
12
|
|
12
13
|
class LogStash::Outputs::EDGE_Loki < LogStash::Outputs::Base
|
13
14
|
include Loki
|
@@ -16,31 +17,34 @@ class LogStash::Outputs::EDGE_Loki < LogStash::Outputs::Base
|
|
16
17
|
## 'A single instance of the Output will be shared among the pipeline worker threads'
|
17
18
|
concurrency :single
|
18
19
|
|
19
|
-
## 'Loki
|
20
|
-
config :
|
21
|
-
|
22
|
-
## 'Loki Endpoint'
|
23
|
-
config :endpoint, :validate => :string, :required => true
|
20
|
+
## 'Loki URL'
|
21
|
+
config :url, :validate => :string, :required => true
|
24
22
|
|
25
23
|
## 'BasicAuth credentials'
|
26
|
-
config :client_id, :validate => :string, :required =>
|
27
|
-
config :client_secret, :validate => :string, secret: true, :required =>
|
24
|
+
config :client_id, :validate => :string, :required => true
|
25
|
+
config :client_secret, :validate => :string, secret: true, :required => true
|
28
26
|
|
29
|
-
## '
|
30
|
-
config :
|
31
|
-
config :key, :validate => :path, :required => false
|
27
|
+
## 'Loki Token URL Domain'
|
28
|
+
config :tokenurl_domain, :validate => :string, :required => true
|
32
29
|
|
33
|
-
## '
|
34
|
-
config :
|
30
|
+
## 'Loki Token URL Endpoint'
|
31
|
+
config :tokenurl_endpoint, :validate => :string, :default => "oauth2/v2.0/token", :required => false
|
32
|
+
|
33
|
+
## 'Scopes'
|
34
|
+
config :scopes, :validate => :string, :required => true
|
35
|
+
|
36
|
+
## 'Proxy URL'
|
37
|
+
config :proxy_url, :validate => :string, :required => false
|
35
38
|
|
36
39
|
## 'Disable server certificate verification'
|
37
40
|
config :insecure_skip_verify, :validate => :boolean, :default => false, :required => false
|
38
41
|
|
39
|
-
## '
|
40
|
-
config :
|
42
|
+
## 'Client certificate'
|
43
|
+
config :cert, :validate => :path, :required => false
|
44
|
+
config :key, :validate => :path, :required => false
|
41
45
|
|
42
|
-
## '
|
43
|
-
config :
|
46
|
+
## 'TLS'
|
47
|
+
config :ca_cert, :validate => :path, :required => false
|
44
48
|
|
45
49
|
## 'Maximum batch size to accrue before pushing to loki. Defaults to 102400 bytes'
|
46
50
|
config :batch_size, :validate => :number, :default => 102400, :required => false
|
@@ -48,6 +52,9 @@ class LogStash::Outputs::EDGE_Loki < LogStash::Outputs::Base
|
|
48
52
|
## 'Interval in seconds to wait before pushing a batch of records to loki. Defaults to 1 second'
|
49
53
|
config :batch_wait, :validate => :number, :default => 1, :required => false
|
50
54
|
|
55
|
+
# Set this to true if you want to enable gzip compression for your http requests
|
56
|
+
config :http_compression, :validate => :boolean, :default => false
|
57
|
+
|
51
58
|
## 'Log line field to pick from logstash. Defaults to "message"'
|
52
59
|
config :message_field, :validate => :string, :default => "message", :required => false
|
53
60
|
|
@@ -97,7 +104,7 @@ class LogStash::Outputs::EDGE_Loki < LogStash::Outputs::Base
|
|
97
104
|
end
|
98
105
|
|
99
106
|
def max_batch_size
|
100
|
-
|
107
|
+
#@logger.info("max_batch_size: started method")
|
101
108
|
loop do
|
102
109
|
@mutex.synchronize do
|
103
110
|
return if @stop
|
@@ -108,7 +115,7 @@ class LogStash::Outputs::EDGE_Loki < LogStash::Outputs::Base
|
|
108
115
|
|
109
116
|
@mutex.synchronize do
|
110
117
|
if !add_entry_to_batch(e)
|
111
|
-
@logger.debug("Max batch_size is reached. Sending batch to loki")
|
118
|
+
@logger.debug("Max batch_size is reached. Sending batch to edge-loki")
|
112
119
|
send(@batch)
|
113
120
|
@batch = Batch.new(e)
|
114
121
|
end
|
@@ -183,8 +190,7 @@ class LogStash::Outputs::EDGE_Loki < LogStash::Outputs::Base
|
|
183
190
|
opts
|
184
191
|
end
|
185
192
|
|
186
|
-
# Add an entry to the current batch returns false if the batch is full
|
187
|
-
# and the entry can't be added.
|
193
|
+
# Add an entry to the current batch returns false if the batch is full and the entry can't be added.
|
188
194
|
def add_entry_to_batch(e)
|
189
195
|
line = e.entry['line']
|
190
196
|
# we don't want to send empty lines.
|
@@ -226,52 +232,68 @@ class LogStash::Outputs::EDGE_Loki < LogStash::Outputs::Base
|
|
226
232
|
end
|
227
233
|
|
228
234
|
def send(batch)
|
229
|
-
|
230
|
-
token = createClient()
|
235
|
+
#@logger.info("send: started method")
|
231
236
|
payload = batch.to_json
|
232
|
-
|
233
|
-
|
234
|
-
if res.is_a?(Net::HTTPSuccess)
|
235
|
-
@logger.debug("Successfully pushed data to loki")
|
237
|
+
if @proxy_url == ""
|
238
|
+
client = createClient()
|
236
239
|
else
|
237
|
-
|
240
|
+
client = createClientwithProxy()
|
238
241
|
end
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
res = loki_http_request(token, payload)
|
246
|
-
if res.is_a?(Net::HTTPSuccess)
|
247
|
-
@logger.debug("Successfully pushed data to loki")
|
242
|
+
access = getToken(client)
|
243
|
+
res = edge_http_request(access, payload)
|
244
|
+
if res == "200"
|
245
|
+
cntLines = JSON.parse(payload)["streams"][0]["values"].size
|
246
|
+
@logger.info("Successfully pushed data to edge_loki")
|
247
|
+
@logger.info("Sent #{cntLines} lines to edge-loki")
|
248
248
|
else
|
249
|
-
@logger.debug("failed payload", :payload => payload)
|
249
|
+
@logger.debug("failed payload", :payload => JSON.parse(payload)["streams"][0]["values"])
|
250
250
|
end
|
251
251
|
end
|
252
252
|
|
253
|
-
def
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
|
267
|
-
|
253
|
+
def createClient()
|
254
|
+
#@logger.info("createClient: started method")
|
255
|
+
client = OAuth2::Client.new(@client_id, @client_secret, auth_scheme: "basic_auth", site: @tokenurl_domain,:token_url => @tokenurl_endpoint)
|
256
|
+
return client
|
257
|
+
end
|
258
|
+
|
259
|
+
def createClientwithProxy()
|
260
|
+
#@logger.info("createClient: started method")
|
261
|
+
conn_opts = {}
|
262
|
+
conn_opts = conn_opts.merge('ssl' => {verify: false})
|
263
|
+
conn_opts = conn_opts.merge('proxy' => @proxy_url)
|
264
|
+
client = OAuth2::Client.new(@client_id, @client_secret, auth_scheme: "basic_auth", site: @tokenurl_domain,:token_url => @tokenurl_endpoint, :connection_opts => conn_opts)
|
265
|
+
return client
|
266
|
+
end
|
267
|
+
|
268
|
+
def getToken(client)
|
269
|
+
params = {}
|
270
|
+
opts = {}
|
271
|
+
params = params.merge('scope' => @scopes)
|
272
|
+
access = client.client_credentials.get_token(params, opts)
|
273
|
+
return access
|
274
|
+
end
|
275
|
+
|
276
|
+
def edge_http_request(token, payload)
|
277
|
+
#@logger.info("edge_http_request: started method")
|
278
|
+
headers = {}
|
279
|
+
cntLines = JSON.parse(payload)["streams"][0]["values"].size
|
280
|
+
@logger.info("processing #{cntLines} lines to edge-loki")
|
268
281
|
retry_count = 0
|
269
282
|
delay = @min_delay
|
270
283
|
begin
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
|
284
|
+
puts payload
|
285
|
+
headers["Content-Type"] = "application/json"
|
286
|
+
# Compress the payload and add appropriate header
|
287
|
+
if @http_compression == true
|
288
|
+
headers["Content-Encoding"] = "gzip"
|
289
|
+
payload1 = payload
|
290
|
+
payload = gzip(payload1)
|
291
|
+
end
|
292
|
+
puts payload
|
293
|
+
res = token.post(@uri, {:body => payload, :headers => headers})
|
294
|
+
status_code = "#{res.status}"
|
295
|
+
return status_code if !status_code.nil? && status_code.to_i != 429 && status_code.to_i.div(100) != 5
|
296
|
+
return res
|
275
297
|
raise StandardError.new res
|
276
298
|
rescue StandardError => e
|
277
299
|
retry_count += 1
|
@@ -291,21 +313,14 @@ class LogStash::Outputs::EDGE_Loki < LogStash::Outputs::Base
|
|
291
313
|
end
|
292
314
|
end
|
293
315
|
|
294
|
-
|
295
|
-
|
296
|
-
|
297
|
-
|
298
|
-
|
299
|
-
|
300
|
-
|
301
|
-
|
302
|
-
|
303
|
-
def pensive_http_request(token, payload)
|
304
|
-
@logger.info("pensive_http_request: started method")
|
305
|
-
response = token.get(@endpoint, :headers => { 'Accept' => 'application/json', 'X-Scope-OrgID', @tenant_id }, :params => { page: 1 })
|
306
|
-
hash = JSON.parse(response.body)
|
307
|
-
@logger.info("pensive_http_request: ", :token_output => hash)
|
308
|
-
return hash
|
316
|
+
# gzip data
|
317
|
+
def gzip(data)
|
318
|
+
gz = StringIO.new
|
319
|
+
gz.set_encoding("BINARY")
|
320
|
+
z = Zlib::GzipWriter.new(gz)
|
321
|
+
z.write(data)
|
322
|
+
z.close
|
323
|
+
gz.string
|
309
324
|
end
|
310
|
-
end
|
311
325
|
|
326
|
+
end
|
@@ -0,0 +1,303 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/outputs/base"
|
3
|
+
require "logstash/outputs/loki/entry"
|
4
|
+
require "logstash/outputs/loki/batch"
|
5
|
+
require "logstash/namespace"
|
6
|
+
require 'net/http'
|
7
|
+
require 'time'
|
8
|
+
require 'uri'
|
9
|
+
require 'json'
|
10
|
+
require 'oauth2'
|
11
|
+
|
12
|
+
class LogStash::Outputs::EDGE_Loki < LogStash::Outputs::Base
|
13
|
+
include Loki
|
14
|
+
config_name "edge_loki"
|
15
|
+
|
16
|
+
## 'A single instance of the Output will be shared among the pipeline worker threads'
|
17
|
+
concurrency :single
|
18
|
+
|
19
|
+
## 'Loki URL'
|
20
|
+
config :url, :validate => :string, :required => true
|
21
|
+
|
22
|
+
## 'BasicAuth credentials'
|
23
|
+
config :client_id, :validate => :string, :required => true
|
24
|
+
config :client_secret, :validate => :string, secret: true, :required => true
|
25
|
+
|
26
|
+
## 'Loki Token URL Domain'
|
27
|
+
config :tokenurl_domain, :validate => :string, :required => true
|
28
|
+
|
29
|
+
## 'Loki Token URL Endpoint'
|
30
|
+
config :tokenurl_endpoint, :validate => :string, :default => "oauth2/v2.0/token", :required => false
|
31
|
+
|
32
|
+
## 'Scopes'
|
33
|
+
config :scopes, :validate => :string, :required => true
|
34
|
+
|
35
|
+
## 'Proxy URL'
|
36
|
+
config :proxy_url, :validate => :string, :required => false
|
37
|
+
|
38
|
+
## 'Disable server certificate verification'
|
39
|
+
config :insecure_skip_verify, :validate => :boolean, :default => false, :required => false
|
40
|
+
|
41
|
+
## 'Client certificate'
|
42
|
+
config :cert, :validate => :path, :required => false
|
43
|
+
config :key, :validate => :path, :required => false
|
44
|
+
|
45
|
+
## 'TLS'
|
46
|
+
config :ca_cert, :validate => :path, :required => false
|
47
|
+
|
48
|
+
## 'Maximum batch size to accrue before pushing to loki. Defaults to 102400 bytes'
|
49
|
+
config :batch_size, :validate => :number, :default => 102400, :required => false
|
50
|
+
|
51
|
+
## 'Interval in seconds to wait before pushing a batch of records to loki. Defaults to 1 second'
|
52
|
+
config :batch_wait, :validate => :number, :default => 1, :required => false
|
53
|
+
|
54
|
+
## 'Log line field to pick from logstash. Defaults to "message"'
|
55
|
+
config :message_field, :validate => :string, :default => "message", :required => false
|
56
|
+
|
57
|
+
## 'Backoff configuration. Initial backoff time between retries. Default 1s'
|
58
|
+
config :min_delay, :validate => :number, :default => 1, :required => false
|
59
|
+
|
60
|
+
## 'An array of fields to map to labels, if defined only fields in this list will be mapped.'
|
61
|
+
config :include_fields, :validate => :array, :default => [], :required => false
|
62
|
+
|
63
|
+
## 'Backoff configuration. Maximum backoff time between retries. Default 300s'
|
64
|
+
config :max_delay, :validate => :number, :default => 300, :required => false
|
65
|
+
|
66
|
+
## 'Backoff configuration. Maximum number of retries to do'
|
67
|
+
config :retries, :validate => :number, :default => 10, :required => false
|
68
|
+
|
69
|
+
attr_reader :batch
|
70
|
+
public
|
71
|
+
def register
|
72
|
+
@uri = URI.parse(@url)
|
73
|
+
unless @uri.is_a?(URI::HTTP) || @uri.is_a?(URI::HTTPS)
|
74
|
+
raise LogStash::ConfigurationError, "url parameter must be valid HTTP, currently '#{@url}'"
|
75
|
+
end
|
76
|
+
|
77
|
+
if @min_delay > @max_delay
|
78
|
+
raise LogStash::ConfigurationError, "Min delay should be less than Max delay, currently 'Min delay is #{@min_delay} and Max delay is #{@max_delay}'"
|
79
|
+
end
|
80
|
+
|
81
|
+
@logger.info("Loki output plugin", :class => self.class.name)
|
82
|
+
|
83
|
+
# initialize Queue and Mutex
|
84
|
+
@entries = Queue.new
|
85
|
+
@mutex = Mutex.new
|
86
|
+
@stop = false
|
87
|
+
|
88
|
+
# create nil batch object.
|
89
|
+
@batch = nil
|
90
|
+
|
91
|
+
# validate certs
|
92
|
+
if ssl_cert?
|
93
|
+
load_ssl
|
94
|
+
validate_ssl_key
|
95
|
+
end
|
96
|
+
|
97
|
+
# start batch_max_wait and batch_max_size threads
|
98
|
+
@batch_wait_thread = Thread.new{max_batch_wait()}
|
99
|
+
@batch_size_thread = Thread.new{max_batch_size()}
|
100
|
+
end
|
101
|
+
|
102
|
+
def max_batch_size
|
103
|
+
#@logger.info("max_batch_size: started method")
|
104
|
+
loop do
|
105
|
+
@mutex.synchronize do
|
106
|
+
return if @stop
|
107
|
+
end
|
108
|
+
|
109
|
+
e = @entries.deq
|
110
|
+
return if e.nil?
|
111
|
+
|
112
|
+
@mutex.synchronize do
|
113
|
+
if !add_entry_to_batch(e)
|
114
|
+
@logger.debug("Max batch_size is reached. Sending batch to edge-loki")
|
115
|
+
send(@batch)
|
116
|
+
@batch = Batch.new(e)
|
117
|
+
end
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|
121
|
+
|
122
|
+
def max_batch_wait
|
123
|
+
# minimum wait frequency is 10 milliseconds
|
124
|
+
min_wait_checkfrequency = 1/100
|
125
|
+
max_wait_checkfrequency = @batch_wait
|
126
|
+
if max_wait_checkfrequency < min_wait_checkfrequency
|
127
|
+
max_wait_checkfrequency = min_wait_checkfrequency
|
128
|
+
end
|
129
|
+
|
130
|
+
loop do
|
131
|
+
@mutex.synchronize do
|
132
|
+
return if @stop
|
133
|
+
end
|
134
|
+
|
135
|
+
sleep(max_wait_checkfrequency)
|
136
|
+
if is_batch_expired
|
137
|
+
@mutex.synchronize do
|
138
|
+
@logger.debug("Max batch_wait time is reached. Sending batch to loki")
|
139
|
+
send(@batch)
|
140
|
+
@batch = nil
|
141
|
+
end
|
142
|
+
end
|
143
|
+
end
|
144
|
+
end
|
145
|
+
|
146
|
+
def ssl_cert?
|
147
|
+
!@key.nil? && !@cert.nil?
|
148
|
+
end
|
149
|
+
|
150
|
+
def load_ssl
|
151
|
+
@cert = OpenSSL::X509::Certificate.new(File.read(@cert)) if @cert
|
152
|
+
@key = OpenSSL::PKey.read(File.read(@key)) if @key
|
153
|
+
end
|
154
|
+
|
155
|
+
def validate_ssl_key
|
156
|
+
if !@key.is_a?(OpenSSL::PKey::RSA) && !@key.is_a?(OpenSSL::PKey::DSA)
|
157
|
+
raise LogStash::ConfigurationError, "Unsupported private key type '#{@key.class}''"
|
158
|
+
end
|
159
|
+
end
|
160
|
+
|
161
|
+
def ssl_opts(uri)
|
162
|
+
opts = {
|
163
|
+
use_ssl: uri.scheme == 'https'
|
164
|
+
}
|
165
|
+
|
166
|
+
# disable server certificate verification
|
167
|
+
if @insecure_skip_verify
|
168
|
+
opts = opts.merge(
|
169
|
+
verify_mode: OpenSSL::SSL::VERIFY_NONE
|
170
|
+
)
|
171
|
+
end
|
172
|
+
|
173
|
+
if !@cert.nil? && !@key.nil?
|
174
|
+
opts = opts.merge(
|
175
|
+
verify_mode: OpenSSL::SSL::VERIFY_PEER,
|
176
|
+
cert: @cert,
|
177
|
+
key: @key
|
178
|
+
)
|
179
|
+
end
|
180
|
+
|
181
|
+
unless @ca_cert.nil?
|
182
|
+
opts = opts.merge(
|
183
|
+
ca_file: @ca_cert
|
184
|
+
)
|
185
|
+
end
|
186
|
+
opts
|
187
|
+
end
|
188
|
+
|
189
|
+
# Add an entry to the current batch returns false if the batch is full and the entry can't be added.
|
190
|
+
def add_entry_to_batch(e)
|
191
|
+
line = e.entry['line']
|
192
|
+
# we don't want to send empty lines.
|
193
|
+
return true if line.to_s.strip.empty?
|
194
|
+
|
195
|
+
if @batch.nil?
|
196
|
+
@batch = Batch.new(e)
|
197
|
+
return true
|
198
|
+
end
|
199
|
+
|
200
|
+
if @batch.size_bytes_after(line) > @batch_size
|
201
|
+
return false
|
202
|
+
end
|
203
|
+
@batch.add(e)
|
204
|
+
return true
|
205
|
+
end
|
206
|
+
|
207
|
+
def is_batch_expired
|
208
|
+
return !@batch.nil? && @batch.age() >= @batch_wait
|
209
|
+
end
|
210
|
+
|
211
|
+
## Receives logstash events
|
212
|
+
public
|
213
|
+
def receive(event)
|
214
|
+
@entries << Entry.new(event, @message_field, @include_fields)
|
215
|
+
end
|
216
|
+
|
217
|
+
def close
|
218
|
+
@entries.close
|
219
|
+
@mutex.synchronize do
|
220
|
+
@stop = true
|
221
|
+
end
|
222
|
+
@batch_wait_thread.join
|
223
|
+
@batch_size_thread.join
|
224
|
+
|
225
|
+
# if by any chance we still have a forming batch, we need to send it.
|
226
|
+
send(@batch) if !@batch.nil?
|
227
|
+
@batch = nil
|
228
|
+
end
|
229
|
+
|
230
|
+
def send(batch)
|
231
|
+
#@logger.info("send: started method")
|
232
|
+
payload = batch.to_json
|
233
|
+
if @proxy_url == ""
|
234
|
+
client = createClient()
|
235
|
+
else
|
236
|
+
client = createClientwithProxy()
|
237
|
+
end
|
238
|
+
access = getToken(client)
|
239
|
+
res = edge_http_request(access, payload)
|
240
|
+
if res == "200"
|
241
|
+
cntLines = JSON.parse(payload)["streams"][0]["values"].size
|
242
|
+
@logger.info("Successfully pushed data to edge_loki")
|
243
|
+
@logger.info("Sent #{cntLines} lines to edge-loki")
|
244
|
+
else
|
245
|
+
@logger.debug("failed payload", :payload => JSON.parse(payload)["streams"][0]["values"])
|
246
|
+
end
|
247
|
+
end
|
248
|
+
|
249
|
+
def createClient()
|
250
|
+
#@logger.info("createClient: started method")
|
251
|
+
client = OAuth2::Client.new(@client_id, @client_secret, auth_scheme: "basic_auth", site: @tokenurl_domain,:token_url => @tokenurl_endpoint)
|
252
|
+
return client
|
253
|
+
end
|
254
|
+
|
255
|
+
def createClientwithProxy()
|
256
|
+
#@logger.info("createClient: started method")
|
257
|
+
conn_opts = {}
|
258
|
+
conn_opts = conn_opts.merge('ssl' => {verify: false})
|
259
|
+
conn_opts = conn_opts.merge('proxy' => @proxy_url)
|
260
|
+
client = OAuth2::Client.new(@client_id, @client_secret, auth_scheme: "basic_auth", site: @tokenurl_domain,:token_url => @tokenurl_endpoint, :connection_opts => conn_opts)
|
261
|
+
return client
|
262
|
+
end
|
263
|
+
|
264
|
+
def getToken(client)
|
265
|
+
params = {}
|
266
|
+
opts = {}
|
267
|
+
params = params.merge('scope' => @scopes)
|
268
|
+
access = client.client_credentials.get_token(params, opts)
|
269
|
+
return access
|
270
|
+
end
|
271
|
+
|
272
|
+
def edge_http_request(token, payload)
|
273
|
+
#@logger.info("edge_http_request: started method")
|
274
|
+
cntLines = JSON.parse(payload)["streams"][0]["values"].size
|
275
|
+
@logger.info("processing #{cntLines} lines to edge-loki")
|
276
|
+
retry_count = 0
|
277
|
+
delay = @min_delay
|
278
|
+
begin
|
279
|
+
puts payload
|
280
|
+
res = token.post(@uri, {:body => payload, :headers => {'Content-Type' => 'application/json'}})
|
281
|
+
status_code = "#{res.status}"
|
282
|
+
return status_code if !status_code.nil? && status_code.to_i != 429 && status_code.to_i.div(100) != 5
|
283
|
+
return res
|
284
|
+
raise StandardError.new res
|
285
|
+
rescue StandardError => e
|
286
|
+
retry_count += 1
|
287
|
+
@logger.warn("Failed to send batch, attempt: #{retry_count}/#{@retries}", :error_inspect => e.inspect, :error => e)
|
288
|
+
if retry_count < @retries
|
289
|
+
sleep delay
|
290
|
+
if delay * 2 <= @max_delay
|
291
|
+
delay = delay * 2
|
292
|
+
else
|
293
|
+
delay = @max_delay
|
294
|
+
end
|
295
|
+
retry
|
296
|
+
else
|
297
|
+
@logger.error("Failed to send batch", :error_inspect => e.inspect, :error => e)
|
298
|
+
return res
|
299
|
+
end
|
300
|
+
end
|
301
|
+
end
|
302
|
+
|
303
|
+
end
|
File without changes
|
File without changes
|
@@ -1,11 +1,11 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-output-edge_loki'
|
3
|
-
s.version = '1.0.
|
3
|
+
s.version = '1.0.11'
|
4
4
|
s.authors = ['Britto Prabhu']
|
5
5
|
s.email = ['britto.prabhu@apmterminals.com']
|
6
6
|
|
7
|
-
s.summary = 'Output plugin to ship logs to
|
8
|
-
s.description = 'Output plugin to ship logs to
|
7
|
+
s.summary = 'Output plugin to ship logs to Loki server with oAuth2'
|
8
|
+
s.description = 'Output plugin to ship logs to Loki server with oAuth2'
|
9
9
|
s.homepage = 'https://github.com/Maersk-Global/apmt-observability-deployment'
|
10
10
|
s.license = 'Apache-2.0'
|
11
11
|
s.require_paths = ["lib"]
|
File without changes
|
data/spec/outputs/loki_spec.rb
CHANGED
File without changes
|
metadata
CHANGED
@@ -1,30 +1,31 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-edge_loki
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.11
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Britto Prabhu
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2024-08-06 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
+
name: oauth2
|
14
15
|
requirement: !ruby/object:Gem::Requirement
|
15
16
|
requirements:
|
16
17
|
- - "~>"
|
17
18
|
- !ruby/object:Gem::Version
|
18
19
|
version: 2.0.9
|
19
|
-
name: oauth2
|
20
|
-
prerelease: false
|
21
20
|
type: :runtime
|
21
|
+
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: 2.0.9
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
|
+
name: logstash-core-plugin-api
|
28
29
|
requirement: !ruby/object:Gem::Requirement
|
29
30
|
requirements:
|
30
31
|
- - ">="
|
@@ -33,9 +34,8 @@ dependencies:
|
|
33
34
|
- - "<="
|
34
35
|
- !ruby/object:Gem::Version
|
35
36
|
version: '2.99'
|
36
|
-
name: logstash-core-plugin-api
|
37
|
-
prerelease: false
|
38
37
|
type: :runtime
|
38
|
+
prerelease: false
|
39
39
|
version_requirements: !ruby/object:Gem::Requirement
|
40
40
|
requirements:
|
41
41
|
- - ">="
|
@@ -45,34 +45,34 @@ dependencies:
|
|
45
45
|
- !ruby/object:Gem::Version
|
46
46
|
version: '2.99'
|
47
47
|
- !ruby/object:Gem::Dependency
|
48
|
+
name: logstash-codec-plain
|
48
49
|
requirement: !ruby/object:Gem::Requirement
|
49
50
|
requirements:
|
50
51
|
- - '='
|
51
52
|
- !ruby/object:Gem::Version
|
52
53
|
version: 3.1.0
|
53
|
-
name: logstash-codec-plain
|
54
|
-
prerelease: false
|
55
54
|
type: :runtime
|
55
|
+
prerelease: false
|
56
56
|
version_requirements: !ruby/object:Gem::Requirement
|
57
57
|
requirements:
|
58
58
|
- - '='
|
59
59
|
- !ruby/object:Gem::Version
|
60
60
|
version: 3.1.0
|
61
61
|
- !ruby/object:Gem::Dependency
|
62
|
+
name: logstash-devutils
|
62
63
|
requirement: !ruby/object:Gem::Requirement
|
63
64
|
requirements:
|
64
65
|
- - '='
|
65
66
|
- !ruby/object:Gem::Version
|
66
67
|
version: 2.0.2
|
67
|
-
name: logstash-devutils
|
68
|
-
prerelease: false
|
69
68
|
type: :development
|
69
|
+
prerelease: false
|
70
70
|
version_requirements: !ruby/object:Gem::Requirement
|
71
71
|
requirements:
|
72
72
|
- - '='
|
73
73
|
- !ruby/object:Gem::Version
|
74
74
|
version: 2.0.2
|
75
|
-
description: Output plugin to ship logs to
|
75
|
+
description: Output plugin to ship logs to Loki server with oAuth2
|
76
76
|
email:
|
77
77
|
- britto.prabhu@apmterminals.com
|
78
78
|
executables: []
|
@@ -82,6 +82,7 @@ files:
|
|
82
82
|
- Gemfile
|
83
83
|
- README.md
|
84
84
|
- lib/logstash/outputs/edge_loki.rb
|
85
|
+
- lib/logstash/outputs/edge_loki_20240722.rb
|
85
86
|
- lib/logstash/outputs/loki/batch.rb
|
86
87
|
- lib/logstash/outputs/loki/entry.rb
|
87
88
|
- logstash-output-edge_loki.gemspec
|
@@ -93,7 +94,7 @@ licenses:
|
|
93
94
|
metadata:
|
94
95
|
logstash_plugin: 'true'
|
95
96
|
logstash_group: output
|
96
|
-
post_install_message:
|
97
|
+
post_install_message:
|
97
98
|
rdoc_options: []
|
98
99
|
require_paths:
|
99
100
|
- lib
|
@@ -108,10 +109,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
108
109
|
- !ruby/object:Gem::Version
|
109
110
|
version: '0'
|
110
111
|
requirements: []
|
111
|
-
rubygems_version: 3.
|
112
|
-
signing_key:
|
112
|
+
rubygems_version: 3.0.3.1
|
113
|
+
signing_key:
|
113
114
|
specification_version: 4
|
114
|
-
summary: Output plugin to ship logs to
|
115
|
+
summary: Output plugin to ship logs to Loki server with oAuth2
|
115
116
|
test_files:
|
116
117
|
- spec/outputs/loki/entry_spec.rb
|
117
118
|
- spec/outputs/loki_spec.rb
|