logstash-output-edge_loki 1.0.8 → 1.0.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/logstash/outputs/edge_loki.rb +64 -73
- data/logstash-output-edge_loki.gemspec +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 459c97179f8e5871f7992ce20df5ba1ef2dc95ec0959eecbb1dbfda58c2dd81c
|
4
|
+
data.tar.gz: 672e5fc63bb488cd69fe6522cd210235fd6824d7677a713b45e16d8adc7864ce
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0727eb5e0abbf2cf0aeba08606b13a4081aa662f5b31afe1caebf7670d6ff62ed6059b1701d6662867feca4f1ab4a2b052eff678d02a7f2ee7849c3c7437b562
|
7
|
+
data.tar.gz: 2417817e5581a05067965c93a5e7e61a0f1403f4db0301c8f80d58eb03eca6e48eeacb9b583a046edb188e716793a15d2b3695ce15ad06eda6dfc4703bd1d2ac
|
@@ -16,31 +16,34 @@ class LogStash::Outputs::EDGE_Loki < LogStash::Outputs::Base
|
|
16
16
|
## 'A single instance of the Output will be shared among the pipeline worker threads'
|
17
17
|
concurrency :single
|
18
18
|
|
19
|
-
## 'Loki
|
20
|
-
config :
|
21
|
-
|
22
|
-
## 'Loki Endpoint'
|
23
|
-
config :endpoint, :validate => :string, :required => true
|
19
|
+
## 'Loki URL'
|
20
|
+
config :url, :validate => :string, :required => true
|
24
21
|
|
25
22
|
## 'BasicAuth credentials'
|
26
|
-
config :client_id, :validate => :string, :required =>
|
27
|
-
config :client_secret, :validate => :string, secret: true, :required =>
|
23
|
+
config :client_id, :validate => :string, :required => true
|
24
|
+
config :client_secret, :validate => :string, secret: true, :required => true
|
28
25
|
|
29
|
-
## '
|
30
|
-
config :
|
31
|
-
config :key, :validate => :path, :required => false
|
26
|
+
## 'Loki Token URL Domain'
|
27
|
+
config :tokenurl_domain, :validate => :string, :required => true
|
32
28
|
|
33
|
-
## '
|
34
|
-
config :
|
29
|
+
## 'Loki Token URL Endpoint'
|
30
|
+
config :tokenurl_endpoint, :validate => :string, :default => "oauth2/v2.0/token", :required => false
|
31
|
+
|
32
|
+
## 'Scopes'
|
33
|
+
config :scopes, :validate => :string, :required => true
|
34
|
+
|
35
|
+
## 'Proxy URL'
|
36
|
+
config :proxy_url, :validate => :string, :required => false
|
35
37
|
|
36
38
|
## 'Disable server certificate verification'
|
37
39
|
config :insecure_skip_verify, :validate => :boolean, :default => false, :required => false
|
38
40
|
|
39
|
-
## '
|
40
|
-
config :
|
41
|
+
## 'Client certificate'
|
42
|
+
config :cert, :validate => :path, :required => false
|
43
|
+
config :key, :validate => :path, :required => false
|
41
44
|
|
42
|
-
## '
|
43
|
-
config :
|
45
|
+
## 'TLS'
|
46
|
+
config :ca_cert, :validate => :path, :required => false
|
44
47
|
|
45
48
|
## 'Maximum batch size to accrue before pushing to loki. Defaults to 102400 bytes'
|
46
49
|
config :batch_size, :validate => :number, :default => 102400, :required => false
|
@@ -97,7 +100,7 @@ class LogStash::Outputs::EDGE_Loki < LogStash::Outputs::Base
|
|
97
100
|
end
|
98
101
|
|
99
102
|
def max_batch_size
|
100
|
-
|
103
|
+
#@logger.info("max_batch_size: started method")
|
101
104
|
loop do
|
102
105
|
@mutex.synchronize do
|
103
106
|
return if @stop
|
@@ -108,7 +111,7 @@ class LogStash::Outputs::EDGE_Loki < LogStash::Outputs::Base
|
|
108
111
|
|
109
112
|
@mutex.synchronize do
|
110
113
|
if !add_entry_to_batch(e)
|
111
|
-
@logger.debug("Max batch_size is reached. Sending batch to loki")
|
114
|
+
@logger.debug("Max batch_size is reached. Sending batch to edge-loki")
|
112
115
|
send(@batch)
|
113
116
|
@batch = Batch.new(e)
|
114
117
|
end
|
@@ -183,8 +186,7 @@ class LogStash::Outputs::EDGE_Loki < LogStash::Outputs::Base
|
|
183
186
|
opts
|
184
187
|
end
|
185
188
|
|
186
|
-
# Add an entry to the current batch returns false if the batch is full
|
187
|
-
# and the entry can't be added.
|
189
|
+
# Add an entry to the current batch returns false if the batch is full and the entry can't be added.
|
188
190
|
def add_entry_to_batch(e)
|
189
191
|
line = e.entry['line']
|
190
192
|
# we don't want to send empty lines.
|
@@ -226,52 +228,58 @@ class LogStash::Outputs::EDGE_Loki < LogStash::Outputs::Base
|
|
226
228
|
end
|
227
229
|
|
228
230
|
def send(batch)
|
229
|
-
|
230
|
-
token = createClient()
|
231
|
+
#@logger.info("send: started method")
|
231
232
|
payload = batch.to_json
|
232
|
-
|
233
|
-
|
234
|
-
if res.is_a?(Net::HTTPSuccess)
|
235
|
-
@logger.debug("Successfully pushed data to loki")
|
233
|
+
if @proxy_url == ""
|
234
|
+
client = createClient()
|
236
235
|
else
|
237
|
-
|
236
|
+
client = createClientwithProxy()
|
238
237
|
end
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
res = loki_http_request(token, payload)
|
246
|
-
if res.is_a?(Net::HTTPSuccess)
|
247
|
-
@logger.debug("Successfully pushed data to loki")
|
238
|
+
access = getToken(client)
|
239
|
+
res = edge_http_request(access, payload)
|
240
|
+
if res == "200"
|
241
|
+
cntLines = JSON.parse(payload)["streams"][0]["values"].size
|
242
|
+
@logger.info("Successfully pushed data to edge_loki")
|
243
|
+
@logger.info("Sent #{cntLines} lines to edge-loki")
|
248
244
|
else
|
249
|
-
@logger.debug("failed payload", :payload => payload)
|
245
|
+
@logger.debug("failed payload", :payload => JSON.parse(payload)["streams"][0]["values"])
|
250
246
|
end
|
251
247
|
end
|
252
248
|
|
253
|
-
def
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
|
267
|
-
|
249
|
+
def createClient()
|
250
|
+
#@logger.info("createClient: started method")
|
251
|
+
client = OAuth2::Client.new(@client_id, @client_secret, auth_scheme: "basic_auth", site: @tokenurl_domain,:token_url => @tokenurl_endpoint)
|
252
|
+
return client
|
253
|
+
end
|
254
|
+
|
255
|
+
def createClientwithProxy()
|
256
|
+
#@logger.info("createClient: started method")
|
257
|
+
conn_opts = {}
|
258
|
+
conn_opts = conn_opts.merge('ssl' => {verify: false})
|
259
|
+
conn_opts = conn_opts.merge('proxy' => @proxy_url)
|
260
|
+
client = OAuth2::Client.new(@client_id, @client_secret, auth_scheme: "basic_auth", site: @tokenurl_domain,:token_url => @tokenurl_endpoint, :connection_opts => conn_opts)
|
261
|
+
return client
|
262
|
+
end
|
263
|
+
|
264
|
+
def getToken(client)
|
265
|
+
params = {}
|
266
|
+
opts = {}
|
267
|
+
params = params.merge('scope' => @scopes)
|
268
|
+
access = client.client_credentials.get_token(params, opts)
|
269
|
+
return access
|
270
|
+
end
|
271
|
+
|
272
|
+
def edge_http_request(token, payload)
|
273
|
+
#@logger.info("edge_http_request: started method")
|
274
|
+
cntLines = JSON.parse(payload)["streams"][0]["values"].size
|
275
|
+
@logger.info("processing #{cntLines} lines to edge-loki")
|
268
276
|
retry_count = 0
|
269
277
|
delay = @min_delay
|
270
278
|
begin
|
271
|
-
res =
|
272
|
-
|
273
|
-
|
274
|
-
return res
|
279
|
+
res = token.post(@uri, {:body => payload, :headers => {'Content-Type' => 'application/json'}})
|
280
|
+
status_code = "#{res.status}"
|
281
|
+
return status_code if !status_code.nil? && status_code.to_i != 429 && status_code.to_i.div(100) != 5
|
282
|
+
return res
|
275
283
|
raise StandardError.new res
|
276
284
|
rescue StandardError => e
|
277
285
|
retry_count += 1
|
@@ -291,21 +299,4 @@ class LogStash::Outputs::EDGE_Loki < LogStash::Outputs::Base
|
|
291
299
|
end
|
292
300
|
end
|
293
301
|
|
294
|
-
def createClient
|
295
|
-
@logger.info("createClient: started method")
|
296
|
-
client = OAuth2::Client.new(@client_id, @client_secret, site: @domain + @endpoint)
|
297
|
-
client.auth_code.authorize_url(redirect_uri: @token_url)
|
298
|
-
token = client.auth_code.get_token(code, :redirect_uri => @token_url)
|
299
|
-
@logger.info("createClient: started method", :client => client)
|
300
|
-
return token
|
301
|
-
end
|
302
|
-
|
303
|
-
def pensive_http_request(token, payload)
|
304
|
-
@logger.info("pensive_http_request: started method")
|
305
|
-
response = token.get(@endpoint, :headers => { 'Accept' => 'application/json', 'X-Scope-OrgID', @tenant_id }, :params => { page: 1 })
|
306
|
-
hash = JSON.parse(response.body)
|
307
|
-
@logger.info("pensive_http_request: ", :token_output => hash)
|
308
|
-
return hash
|
309
|
-
end
|
310
302
|
end
|
311
|
-
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-edge_loki
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.10
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Britto Prabhu
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-09-
|
11
|
+
date: 2023-09-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|