logstash-output-edge_loki 1.0.8 → 1.0.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: a5ec89dff9f872d7371462dc0e39118ac9e0fa9d662fa5eecd4e99e9565f29d7
4
- data.tar.gz: b598d411be65be2e16c9ce26d6d954cc63496085b714e66bc6274af6c42ef765
3
+ metadata.gz: 459c97179f8e5871f7992ce20df5ba1ef2dc95ec0959eecbb1dbfda58c2dd81c
4
+ data.tar.gz: 672e5fc63bb488cd69fe6522cd210235fd6824d7677a713b45e16d8adc7864ce
5
5
  SHA512:
6
- metadata.gz: 8eef5ab0b6d08ab8968575e33cb60b88ecc6b45775df95c7abde04229bcc80577f394a976d4041bdd954772f598b67b94ce9da028c4f4ce32cd67a03bd5739da
7
- data.tar.gz: b7f7529cc7d32de3e6c47a579f5db49595f2671d156660add48091ff4be26a83621ecb1196b3769854d7eae22ade1e0dbf4ad9bbd3008327a9c6fe83b0b6ee9f
6
+ metadata.gz: 0727eb5e0abbf2cf0aeba08606b13a4081aa662f5b31afe1caebf7670d6ff62ed6059b1701d6662867feca4f1ab4a2b052eff678d02a7f2ee7849c3c7437b562
7
+ data.tar.gz: 2417817e5581a05067965c93a5e7e61a0f1403f4db0301c8f80d58eb03eca6e48eeacb9b583a046edb188e716793a15d2b3695ce15ad06eda6dfc4703bd1d2ac
@@ -16,31 +16,34 @@ class LogStash::Outputs::EDGE_Loki < LogStash::Outputs::Base
16
16
  ## 'A single instance of the Output will be shared among the pipeline worker threads'
17
17
  concurrency :single
18
18
 
19
- ## 'Loki Domain'
20
- config :domain, :validate => :string, :required => true
21
-
22
- ## 'Loki Endpoint'
23
- config :endpoint, :validate => :string, :required => true
19
+ ## 'Loki URL'
20
+ config :url, :validate => :string, :required => true
24
21
 
25
22
  ## 'BasicAuth credentials'
26
- config :client_id, :validate => :string, :required => false
27
- config :client_secret, :validate => :string, secret: true, :required => false
23
+ config :client_id, :validate => :string, :required => true
24
+ config :client_secret, :validate => :string, secret: true, :required => true
28
25
 
29
- ## 'Client certificate'
30
- config :cert, :validate => :path, :required => false
31
- config :key, :validate => :path, :required => false
26
+ ## 'Loki Token URL Domain'
27
+ config :tokenurl_domain, :validate => :string, :required => true
32
28
 
33
- ## 'TLS'
34
- config :ca_cert, :validate => :path, :required => false
29
+ ## 'Loki Token URL Endpoint'
30
+ config :tokenurl_endpoint, :validate => :string, :default => "oauth2/v2.0/token", :required => false
31
+
32
+ ## 'Scopes'
33
+ config :scopes, :validate => :string, :required => true
34
+
35
+ ## 'Proxy URL'
36
+ config :proxy_url, :validate => :string, :required => false
35
37
 
36
38
  ## 'Disable server certificate verification'
37
39
  config :insecure_skip_verify, :validate => :boolean, :default => false, :required => false
38
40
 
39
- ## 'Loki Tenant ID'
40
- config :tenant_id, :validate => :string, :required => false
41
+ ## 'Client certificate'
42
+ config :cert, :validate => :path, :required => false
43
+ config :key, :validate => :path, :required => false
41
44
 
42
- ## 'Loki Token URL'
43
- config :token_url, :validate => :string, :required => false
45
+ ## 'TLS'
46
+ config :ca_cert, :validate => :path, :required => false
44
47
 
45
48
  ## 'Maximum batch size to accrue before pushing to loki. Defaults to 102400 bytes'
46
49
  config :batch_size, :validate => :number, :default => 102400, :required => false
@@ -97,7 +100,7 @@ class LogStash::Outputs::EDGE_Loki < LogStash::Outputs::Base
97
100
  end
98
101
 
99
102
  def max_batch_size
100
- @logger.info("max_batch_size: started method")
103
+ #@logger.info("max_batch_size: started method")
101
104
  loop do
102
105
  @mutex.synchronize do
103
106
  return if @stop
@@ -108,7 +111,7 @@ class LogStash::Outputs::EDGE_Loki < LogStash::Outputs::Base
108
111
 
109
112
  @mutex.synchronize do
110
113
  if !add_entry_to_batch(e)
111
- @logger.debug("Max batch_size is reached. Sending batch to loki")
114
+ @logger.debug("Max batch_size is reached. Sending batch to edge-loki")
112
115
  send(@batch)
113
116
  @batch = Batch.new(e)
114
117
  end
@@ -183,8 +186,7 @@ class LogStash::Outputs::EDGE_Loki < LogStash::Outputs::Base
183
186
  opts
184
187
  end
185
188
 
186
- # Add an entry to the current batch returns false if the batch is full
187
- # and the entry can't be added.
189
+ # Add an entry to the current batch returns false if the batch is full and the entry can't be added.
188
190
  def add_entry_to_batch(e)
189
191
  line = e.entry['line']
190
192
  # we don't want to send empty lines.
@@ -226,52 +228,58 @@ class LogStash::Outputs::EDGE_Loki < LogStash::Outputs::Base
226
228
  end
227
229
 
228
230
  def send(batch)
229
- @logger.info("send: started method")
230
- token = createClient()
231
+ #@logger.info("send: started method")
231
232
  payload = batch.to_json
232
- res = pensive_http_request(token, payload)
233
- @logger.info("send result: ", res => res)
234
- if res.is_a?(Net::HTTPSuccess)
235
- @logger.debug("Successfully pushed data to loki")
233
+ if @proxy_url == ""
234
+ client = createClient()
236
235
  else
237
- @logger.debug("failed payload", :payload => payload)
236
+ client = createClientwithProxy()
238
237
  end
239
- end
240
-
241
- def send1(batch)
242
- @logger.info("send: started method")
243
- token = createClient()
244
- payload = batch.to_json
245
- res = loki_http_request(token, payload)
246
- if res.is_a?(Net::HTTPSuccess)
247
- @logger.debug("Successfully pushed data to loki")
238
+ access = getToken(client)
239
+ res = edge_http_request(access, payload)
240
+ if res == "200"
241
+ cntLines = JSON.parse(payload)["streams"][0]["values"].size
242
+ @logger.info("Successfully pushed data to edge_loki")
243
+ @logger.info("Sent #{cntLines} lines to edge-loki")
248
244
  else
249
- @logger.debug("failed payload", :payload => payload)
245
+ @logger.debug("failed payload", :payload => JSON.parse(payload)["streams"][0]["values"])
250
246
  end
251
247
  end
252
248
 
253
- def loki_http_request(token, payload)
254
- @logger.info("loki_http_request: started method")
255
- req = Net::HTTP::Post.new(
256
- @uri.request_uri
257
- )
258
-
259
- req.add_field('Content-Type', 'application/json')
260
- req.add_field('X-Scope-OrgID', @tenant_id) if @tenant_id
261
- req['User-Agent']= 'loki-logstash'
262
- req.basic_auth(@client_id, @client_secret) if @client_id
263
- req.body = payload
264
-
265
- opts = ssl_opts(@uri)
266
-
267
- @logger.debug("sending #{req.body.length} bytes to loki")
249
+ def createClient()
250
+ #@logger.info("createClient: started method")
251
+ client = OAuth2::Client.new(@client_id, @client_secret, auth_scheme: "basic_auth", site: @tokenurl_domain,:token_url => @tokenurl_endpoint)
252
+ return client
253
+ end
254
+
255
+ def createClientwithProxy()
256
+ #@logger.info("createClient: started method")
257
+ conn_opts = {}
258
+ conn_opts = conn_opts.merge('ssl' => {verify: false})
259
+ conn_opts = conn_opts.merge('proxy' => @proxy_url)
260
+ client = OAuth2::Client.new(@client_id, @client_secret, auth_scheme: "basic_auth", site: @tokenurl_domain,:token_url => @tokenurl_endpoint, :connection_opts => conn_opts)
261
+ return client
262
+ end
263
+
264
+ def getToken(client)
265
+ params = {}
266
+ opts = {}
267
+ params = params.merge('scope' => @scopes)
268
+ access = client.client_credentials.get_token(params, opts)
269
+ return access
270
+ end
271
+
272
+ def edge_http_request(token, payload)
273
+ #@logger.info("edge_http_request: started method")
274
+ cntLines = JSON.parse(payload)["streams"][0]["values"].size
275
+ @logger.info("processing #{cntLines} lines to edge-loki")
268
276
  retry_count = 0
269
277
  delay = @min_delay
270
278
  begin
271
- res = Net::HTTP.start(@uri.host, @uri.port, **opts) { |http|
272
- http.request(req)
273
- }
274
- return res if !res.nil? && res.code.to_i != 429 && res.code.to_i.div(100) != 5
279
+ res = token.post(@uri, {:body => payload, :headers => {'Content-Type' => 'application/json'}})
280
+ status_code = "#{res.status}"
281
+ return status_code if !status_code.nil? && status_code.to_i != 429 && status_code.to_i.div(100) != 5
282
+ return res
275
283
  raise StandardError.new res
276
284
  rescue StandardError => e
277
285
  retry_count += 1
@@ -291,21 +299,4 @@ class LogStash::Outputs::EDGE_Loki < LogStash::Outputs::Base
291
299
  end
292
300
  end
293
301
 
294
- def createClient
295
- @logger.info("createClient: started method")
296
- client = OAuth2::Client.new(@client_id, @client_secret, site: @domain + @endpoint)
297
- client.auth_code.authorize_url(redirect_uri: @token_url)
298
- token = client.auth_code.get_token(code, :redirect_uri => @token_url)
299
- @logger.info("createClient: started method", :client => client)
300
- return token
301
- end
302
-
303
- def pensive_http_request(token, payload)
304
- @logger.info("pensive_http_request: started method")
305
- response = token.get(@endpoint, :headers => { 'Accept' => 'application/json', 'X-Scope-OrgID', @tenant_id }, :params => { page: 1 })
306
- hash = JSON.parse(response.body)
307
- @logger.info("pensive_http_request: ", :token_output => hash)
308
- return hash
309
- end
310
302
  end
311
-
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-output-edge_loki'
3
- s.version = '1.0.8'
3
+ s.version = '1.0.10'
4
4
  s.authors = ['Britto Prabhu']
5
5
  s.email = ['britto.prabhu@apmterminals.com']
6
6
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-edge_loki
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.8
4
+ version: 1.0.10
5
5
  platform: ruby
6
6
  authors:
7
7
  - Britto Prabhu
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2023-09-04 00:00:00.000000000 Z
11
+ date: 2023-09-08 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement