logstash-output-percy 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: d3dde7fb57c4e513bea9952afc8eeb4b5510e75265e3f48640c856a1e2e3ef3f
4
+ data.tar.gz: 52e167b5885761a5b35219f704919aa0c8d276936328eb630f2b8801ec36179c
5
+ SHA512:
6
+ metadata.gz: 9fd85d5864293f59894175f484298d29648d77474e0a4b87a2e8b1603133c93319945207d027e7a6d8b69e81474b187f72f9fd653fda6990e9e453c8effad237
7
+ data.tar.gz: 3c628883ae080d7cb20f855a6a3920e49a04e063d010c729566b15707125da6fed4d0070c9a857df724411d1fb473f2fd660269b636e97974624515743d46b86
data/Gemfile ADDED
@@ -0,0 +1,20 @@
1
+ source 'https://rubygems.org'
2
+
3
+ gemspec
4
+
5
+ logstash_path = ENV["LOGSTASH_PATH"] || "./logstash"
6
+
7
+ if Dir.exist?(logstash_path)
8
+ gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
9
+ gem 'logstash-mixin-http_client', :path => "#{logstash_path}/vendor/bundle/jruby/2.6.0/gems/logstash-mixin-http_client"
10
+ else
11
+ raise 'missing logstash vendoring'
12
+ end
13
+
14
+ gem "oauth2", "~> 2.0.9"
15
+ gem "webmock", "~> 3.8"
16
+ gem 'logstash-devutils', "~> 2"
17
+ gem 'sinatra'
18
+ gem 'webrick'
19
+ gem "prometheus-client", "~> 1.0.0"
20
+ gem 'snappy'
data/README.md ADDED
@@ -0,0 +1,88 @@
1
+ # Contributing to Loki Logstash Output Plugin
2
+
3
+ For information about how to use this plugin see this [documentation](../../docs/sources/clients/logstash/_index.md).
4
+
5
+ ## Install dependencies
6
+
7
+ First, make sure you have JDK version `8` or `11` installed and you have set the `JAVA_HOME` environment variable.
8
+
9
+ You need to setup JRuby environment to build this plugin. Refer https://github.com/rbenv/rbenv for setting up your rbenv environment.
10
+
11
+ After setting up `rbenv`. Install JRuby
12
+
13
+ ```bash
14
+ rbenv install jruby-9.2.10.0
15
+ rbenv local jruby-9.2.10.0
16
+ ```
17
+
18
+ Check that the environment is configured
19
+
20
+ ```bash
21
+ ruby --version
22
+ jruby 9.2.10
23
+ ```
24
+
25
+ You should make sure you are running `jruby` and not `ruby`. If the command `ruby --version` still shows `ruby` and not `jruby`, check that PATH contains `$HOME/.rbenv/shims` and `$HOME/.rbenv/bin`. Also verify that you have this in your bash profile:
26
+
27
+ ```bash
28
+ export PATH="$HOME/.rbenv/bin:$PATH"
29
+ eval "$(rbenv init -)"
30
+ ```
31
+
32
+ Then install bundler:
33
+
34
+ ```bash
35
+ gem install bundler:2.1.4
36
+ ```
37
+
38
+ Follow those instructions to [install logstash](https://www.elastic.co/guide/en/logstash/current/installing-logstash.html) before moving to the next section.
39
+
40
+ ## Build and test the plugin
41
+
42
+ ### Install required packages
43
+
44
+ ```bash
45
+ git clone git@github.com:elastic/logstash.git
46
+ cd logstash
47
+ git checkout tags/v7.16.1
48
+ export LOGSTASH_PATH="$(pwd)"
49
+ export GEM_PATH="$LOGSTASH_PATH/vendor/bundle/jruby/2.5.0"
50
+ export GEM_HOME="$LOGSTASH_PATH/vendor/bundle/jruby/2.5.0"
51
+ ./gradlew assemble
52
+ cd ..
53
+ ruby -S bundle /config set --local path "$LOGSTASH_PATH/vendor/bundle"
54
+ ruby -S bundle install
55
+ ruby -S bundle exec rake vendor
56
+ ```
57
+
58
+ ### Build the plugin
59
+
60
+ ```bash
61
+ gem build logstash-output-loki.gemspec
62
+ ```
63
+
64
+ ### Test
65
+
66
+ ```bash
67
+ ruby -S bundle exec rspec
68
+ ```
69
+
70
+ Alternatively if you don't want to install JRuby. Enter inside logstash-loki container.
71
+
72
+ ```bash
73
+ docker build -t logstash-loki ./
74
+ docker run -v $(pwd)/spec:/home/logstash/spec -it --rm --entrypoint /bin/sh logstash-loki
75
+ bundle exec rspec
76
+ ```
77
+
78
+ ## Install plugin to local logstash
79
+
80
+ ```bash
81
+ bin/logstash-plugin install --no-verify --local logstash-output-loki-1.0.0.gem
82
+ ```
83
+
84
+ ## Send sample event and check plugin is working
85
+
86
+ ```bash
87
+ bin/logstash -f loki.conf
88
+ ```
@@ -0,0 +1,15 @@
1
+ require "logstash/outputs/edge/sample"
2
+ require "logstash/outputs/edge/label"
3
+
4
+ module Edge
5
+ class TimeSeries
6
+ attr_reader :labels, :samples
7
+
8
+ def initialize(labels_param, sample_param)
9
+ labels = labels_param
10
+ samples = sample_param
11
+ end
12
+
13
+ end
14
+
15
+ end
@@ -0,0 +1,63 @@
1
+ require 'time'
2
+
3
+ module Edge
4
+ class Batch
5
+ attr_reader :streams
6
+ def initialize(e)
7
+ @bytes = 0
8
+ @createdAt = Time.now
9
+ @streams = {}
10
+ add(e)
11
+ end
12
+
13
+ def size_bytes
14
+ return @bytes
15
+ end
16
+
17
+ def add(e)
18
+ @bytes = @bytes + e.entry['line'].length
19
+
20
+ # Append the entry to an already existing stream (if any)
21
+ labels = e.labels.sort.to_h
22
+ labelkey = labels.to_s
23
+ if @streams.has_key?(labelkey)
24
+ stream = @streams[labelkey]
25
+ stream['entries'].append(e.entry)
26
+ return
27
+ else
28
+ # Add the entry as a new stream
29
+ @streams[labelkey] = {
30
+ "labels" => labels,
31
+ "entries" => [e.entry],
32
+ }
33
+ end
34
+ end
35
+
36
+ def size_bytes_after(line)
37
+ return @bytes + line.length
38
+ end
39
+
40
+ def age()
41
+ return Time.now - @createdAt
42
+ end
43
+
44
+ def to_json
45
+ streams = []
46
+ @streams.each { |_ , stream|
47
+ streams.append(build_stream(stream))
48
+ }
49
+ return {"streams"=>streams}.to_json
50
+ end
51
+
52
+ def build_stream(stream)
53
+ values = []
54
+ stream['entries'].each { |entry|
55
+ values.append([entry['ts'].to_s, entry['line']])
56
+ }
57
+ return {
58
+ 'stream'=>stream['labels'],
59
+ 'values' => values
60
+ }
61
+ end
62
+ end
63
+ end
@@ -0,0 +1,26 @@
1
+ module Edge
2
+ def to_ns(s)
3
+ (s.to_f * (10**9)).to_i
4
+ end
5
+ class Entry
6
+ include Edge
7
+ attr_reader :labels, :entry
8
+ def initialize(event,message_field,include_fields)
9
+ @entry = {
10
+ "ts" => to_ns(event.get("@timestamp")),
11
+ "line" => event.get(message_field).to_s
12
+ }
13
+ event = event.clone()
14
+ event.remove(message_field)
15
+ event.remove("@timestamp")
16
+
17
+ @labels = {}
18
+ event.to_hash.each { |key,value|
19
+ next if key.start_with?('@')
20
+ next if value.is_a?(Hash)
21
+ next if include_fields.length() > 0 and not include_fields.include?(key)
22
+ @labels[key] = value.to_s
23
+ }
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,16 @@
1
+
2
+ module Edge
3
+ class Label
4
+ attr_reader :name, :value
5
+
6
+ def initialize(name_param, value_param)
7
+ puts "name"
8
+ puts name_param
9
+ puts "value"
10
+ puts value_param
11
+ name = name_param
12
+ value = value_param
13
+ end
14
+
15
+ end
16
+ end
@@ -0,0 +1,11 @@
1
+
2
+ module Edge
3
+ class Sample
4
+ attr_reader :value, :timestamp
5
+
6
+ def initialize(timestamp_param, value_param)
7
+ timestamp = timestamp_param
8
+ value = value_param
9
+ end
10
+ end
11
+ end
@@ -0,0 +1,369 @@
1
+ # encoding: utf-8
2
+ require "logstash/outputs/base"
3
+ require "logstash/outputs/edge/entry"
4
+ require "logstash/outputs/edge/batch"
5
+ require "logstash/outputs/edge/TimeSeries"
6
+ require "logstash/outputs/edge/sample"
7
+ require "logstash/outputs/edge/label"
8
+ require "logstash/namespace"
9
+ require 'net/http'
10
+ require 'time'
11
+ require 'uri'
12
+ require 'json'
13
+ require 'prometheus/client'
14
+ require 'prometheus/client/push'
15
+ require 'prometheus/client/formats/text'
16
+ require 'oauth2'
17
+ require 'snappy'
18
+
19
+ class LogStash::Outputs::Percy < LogStash::Outputs::Base
20
+ include Edge
21
+ config_name "lopro"
22
+
23
+ ## 'A single instance of the Output will be shared among the pipeline worker threads'
24
+ concurrency :single
25
+
26
+ ## 'Prom URL'
27
+ config :url, :validate => :string, :required => true
28
+
29
+ ## 'BasicAuth credentials'
30
+ config :client_id, :validate => :string, :required => true
31
+ config :client_secret, :validate => :string, secret: true, :required => true
32
+
33
+ ## 'Prom Token URL Domain'
34
+ config :tokenurl_domain, :validate => :string, :required => true
35
+
36
+ ## 'Prom Token URL Endpoint'
37
+ config :tokenurl_endpoint, :validate => :string, :default => "oauth2/v2.0/token", :required => false
38
+
39
+ ## 'Scopes'
40
+ config :scopes, :validate => :string, :required => true
41
+
42
+ ## 'Proxy URL'
43
+ config :proxy_url, :validate => :string, :required => false
44
+
45
+ ## 'Disable server certificate verification'
46
+ config :insecure_skip_verify, :validate => :boolean, :default => false, :required => false
47
+
48
+ ## 'Client certificate'
49
+ config :cert, :validate => :path, :required => false
50
+ config :key, :validate => :path, :required => false
51
+
52
+ ## 'TLS'
53
+ config :ca_cert, :validate => :path, :required => false
54
+
55
+ ## 'Maximum batch size to accrue before pushing to loki. Defaults to 102400 bytes'
56
+ config :batch_size, :validate => :number, :default => 102400, :required => false
57
+
58
+ ## 'Interval in seconds to wait before pushing a batch of records to loki. Defaults to 1 second'
59
+ config :batch_wait, :validate => :number, :default => 1, :required => false
60
+
61
+ ## 'Log line field to pick from logstash. Defaults to "message"'
62
+ config :message_field, :validate => :string, :default => "message", :required => false
63
+
64
+ ## 'Backoff configuration. Initial backoff time between retries. Default 1s'
65
+ config :min_delay, :validate => :number, :default => 1, :required => false
66
+
67
+ ## 'An array of fields to map to labels, if defined only fields in this list will be mapped.'
68
+ config :include_fields, :validate => :array, :default => [], :required => false
69
+
70
+ ## 'Backoff configuration. Maximum backoff time between retries. Default 300s'
71
+ config :max_delay, :validate => :number, :default => 300, :required => false
72
+
73
+ ## 'Backoff configuration. Maximum number of retries to do'
74
+ config :retries, :validate => :number, :default => 10, :required => false
75
+
76
+ attr_reader :batch
77
+ public
78
+ def register
79
+ puts "1"
80
+ @uri = URI.parse(@url)
81
+
82
+ @registry = Prometheus::Client.registry
83
+ @http_requests = Prometheus::Client::Gauge.new(:commandsAborted_summation, docstring: 'prometheus', labels: [:app, :env, :product_id, :provider, :region, :site_code])
84
+ @registry.register(@http_requests)
85
+
86
+ unless @uri.is_a?(URI::HTTP) || @uri.is_a?(URI::HTTPS)
87
+ raise LogStash::ConfigurationError, "url parameter must be valid HTTP, currently '#{@url}'"
88
+ end
89
+
90
+ if @min_delay > @max_delay
91
+ raise LogStash::ConfigurationError, "Min delay should be less than Max delay, currently 'Min delay is #{@min_delay} and Max delay is #{@max_delay}'"
92
+ end
93
+
94
+ @logger.info("Prom output plugin", :class => self.class.name)
95
+
96
+ # initialize Queue and Mutex
97
+ @entries = Queue.new
98
+ @mutex = Mutex.new
99
+ @stop = false
100
+
101
+ # create nil batch object.
102
+ @batch = nil
103
+
104
+ # validate certs
105
+ if ssl_cert?
106
+ load_ssl
107
+ validate_ssl_key
108
+ end
109
+
110
+ # start batch_max_wait and batch_max_size threads
111
+ @batch_wait_thread = Thread.new{max_batch_wait()}
112
+ @batch_size_thread = Thread.new{max_batch_size()}
113
+ end
114
+
115
+ def max_batch_size
116
+ @logger.info("max_batch_size: started method")
117
+ loop do
118
+ @mutex.synchronize do
119
+ return if @stop
120
+ end
121
+
122
+ e = @entries.deq
123
+ return if e.nil?
124
+
125
+ @mutex.synchronize do
126
+ if !add_entry_to_batch(e)
127
+ @logger.debug("Max batch_size is reached. Sending batch to edge-loki")
128
+ send(@batch)
129
+ @batch = Batch.new(e)
130
+ end
131
+ end
132
+ end
133
+ end
134
+
135
+ def max_batch_wait
136
+ # minimum wait frequency is 10 milliseconds
137
+ min_wait_checkfrequency = 1/100
138
+ max_wait_checkfrequency = @batch_wait
139
+ if max_wait_checkfrequency < min_wait_checkfrequency
140
+ max_wait_checkfrequency = min_wait_checkfrequency
141
+ end
142
+
143
+ loop do
144
+ @mutex.synchronize do
145
+ return if @stop
146
+ end
147
+
148
+ sleep(max_wait_checkfrequency)
149
+ if is_batch_expired
150
+ @mutex.synchronize do
151
+ @logger.debug("Max batch_wait time is reached. Sending batch to loki")
152
+ send(@batch)
153
+ @batch = nil
154
+ end
155
+ end
156
+ end
157
+ end
158
+
159
+ def ssl_cert?
160
+ !@key.nil? && !@cert.nil?
161
+ end
162
+
163
+ def load_ssl
164
+ @cert = OpenSSL::X509::Certificate.new(File.read(@cert)) if @cert
165
+ @key = OpenSSL::PKey.read(File.read(@key)) if @key
166
+ end
167
+
168
+ def validate_ssl_key
169
+ if !@key.is_a?(OpenSSL::PKey::RSA) && !@key.is_a?(OpenSSL::PKey::DSA)
170
+ raise LogStash::ConfigurationError, "Unsupported private key type '#{@key.class}''"
171
+ end
172
+ end
173
+
174
+ def ssl_opts(uri)
175
+ opts = {
176
+ use_ssl: uri.scheme == 'https'
177
+ }
178
+
179
+ # disable server certificate verification
180
+ if @insecure_skip_verify
181
+ opts = opts.merge(
182
+ verify_mode: OpenSSL::SSL::VERIFY_NONE
183
+ )
184
+ end
185
+
186
+ if !@cert.nil? && !@key.nil?
187
+ opts = opts.merge(
188
+ verify_mode: OpenSSL::SSL::VERIFY_PEER,
189
+ cert: @cert,
190
+ key: @key
191
+ )
192
+ end
193
+
194
+ unless @ca_cert.nil?
195
+ opts = opts.merge(
196
+ ca_file: @ca_cert
197
+ )
198
+ end
199
+ opts
200
+ end
201
+
202
+ # Add an entry to the current batch returns false if the batch is full and the entry can't be added.
203
+ def add_entry_to_batch(e)
204
+ line = e.entry['line']
205
+ # we don't want to send empty lines.
206
+ return true if line.to_s.strip.empty?
207
+
208
+ if @batch.nil?
209
+ @batch = Batch.new(e)
210
+ return true
211
+ end
212
+
213
+ if @batch.size_bytes_after(line) > @batch_size
214
+ return false
215
+ end
216
+ @batch.add(e)
217
+ return true
218
+ end
219
+
220
+ def is_batch_expired
221
+ return !@batch.nil? && @batch.age() >= @batch_wait
222
+ end
223
+
224
+ ## Receives logstash events
225
+ public
226
+ def receive(event)
227
+ @entries << Entry.new(event, @message_field, @include_fields)
228
+ end
229
+
230
+ def close
231
+ @entries.close
232
+ @mutex.synchronize do
233
+ @stop = true
234
+ end
235
+ @batch_wait_thread.join
236
+ @batch_size_thread.join
237
+
238
+ # if by any chance we still have a forming batch, we need to send it.
239
+ send(@batch) if !@batch.nil?
240
+ @batch = nil
241
+ end
242
+
243
+ def send(batch)
244
+ @logger.info("send: started method")
245
+ payload = batch.to_json
246
+ client = createClient()
247
+ access = getToken(client)
248
+ @http_requests.set(21.534, labels: label_set_for({ app: 'apmt_metric_test', env: 'test', product_id: 'apmt_test', provider: 'onprem', region: 'westeurope', site_code: 'USTIS01'}))
249
+ #puts "token"
250
+ #puts token
251
+ #JSON.parse(token.ac)
252
+ #puts access.token
253
+ #puts access.headers
254
+ push = Prometheus::Client::Push.new('prometheus').edge_add(access.token, @uri, @registry)
255
+ #Prometheus::Client::Push.new('my-batch-job').add(@registry)
256
+ #push = Prometheus::Client::Push.new('prom_batch', 'edge_prom', 'https://telemetry.pensieve.maersk-digital.net/api/v1/push').add(@registry)
257
+ #push.basic_auth("user", "password")
258
+ #push.add_field 'Authorization', 'Bearer ' + access.token
259
+ #WriteRequest(access)
260
+ end
261
+
262
+ def label_set_for(labels)
263
+ #@logger.info("label_set_for: started method")
264
+ return stringify_values(labels) if !labels.empty?
265
+ end
266
+
267
+ def stringify_values(labels)
268
+ #@logger.info("stringify_values: started method")
269
+ stringified = {}
270
+ labels.each { |k,v| stringified[k] = v.to_s }
271
+
272
+ stringified
273
+ end
274
+
275
+ def setLabels(labels_param)
276
+ stringified = []
277
+ labels_param.each { |k,v| stringified.append(Label.new(k, v)) }
278
+
279
+ stringified
280
+ end
281
+
282
+ def setSamples(sample_ts_param, sample_val_param)
283
+ return Sample.new(sample_ts_param, sample_val_param)
284
+ end
285
+
286
+ def WriteRequest(token)
287
+ #labels = setLabels(label_set_for({ '__name__':'vsphere_vm', app: 'apmt_metric_test', env: 'test', product_id: 'apmt_test', provider: 'onprem', region: 'westeurope', site_code: 'USTIS01'}))
288
+ #puts labels
289
+ #samples = setSamples(Time.now.to_f(), 10.785)
290
+ #puts samples
291
+ #ts = Edge::TimeSeries.new(labels, samples)
292
+ puts "TimeSeries"
293
+ payload1 = {"TimeSeries": {Labels: label_set_for({ '__name__':'vsphere_vm', app: 'apmt_logs_test', env: 'test', product_id: 'apmt_test', provider: 'onprem', region: 'westeurope', site_code: 'USTIS01'}), Samples: [Timestamp: Time.now.to_f(), Value: 10.5 ]}}.to_json
294
+ puts payload1
295
+ #payload1 = {Labels: labels, Samples: samples}
296
+ #payload = Marshal.dump(ts)
297
+ payload = Marshal.dump(payload1)
298
+ source = Snappy.deflate(payload)
299
+
300
+ puts "Payload"
301
+ puts payload
302
+ puts "Snappy"
303
+ puts source
304
+ response = token.post(@uri, {:body => source, :headers => {'Content-Type' => 'application/x-protobuf', 'Content-Encoding' => 'snappy', 'User-Agent' => 'mop-edge-1.0.0', 'X-Prometheus-Remote-Write-Version' => '0.1.0'}})
305
+ return response
306
+ end
307
+
308
+ def createClient()
309
+ @logger.info("createClient: started method")
310
+ client = OAuth2::Client.new(@client_id, @client_secret, auth_scheme: "basic_auth", site: @tokenurl_domain,:token_url => @tokenurl_endpoint)
311
+ return client
312
+ end
313
+
314
+ def createClientwithProxy()
315
+ #@logger.info("createClient: started method")
316
+ conn_opts = {}
317
+ conn_opts = conn_opts.merge('ssl' => {verify: false})
318
+ conn_opts = conn_opts.merge('proxy' => @proxy_url)
319
+ client = OAuth2::Client.new(@client_id, @client_secret, auth_scheme: "basic_auth", site: @tokenurl_domain,:token_url => @tokenurl_endpoint, :connection_opts => conn_opts)
320
+ return client
321
+ end
322
+
323
+ def getToken(client)
324
+ params = {}
325
+ opts = {}
326
+ params = params.merge('scope' => @scopes)
327
+ access = client.client_credentials.get_token(params, opts)
328
+ return access
329
+ end
330
+
331
+ def edge_http_request(token, payload)
332
+ @logger.info("edge_http_request: started method")
333
+ cntLines = JSON.parse(payload)["streams"][0]["values"].size
334
+ @logger.info("processing #{cntLines} lines to edge-prom")
335
+ retry_count = 0
336
+ delay = @min_delay
337
+ begin
338
+ res = token.post(@uri, {:body => payload1, :headers => params})
339
+ status_code = "#{res.status}"
340
+ @logger.info("send: status_code", :status_code => status_code)
341
+ begin
342
+ @logger.info("send: res_body", :res_body => res)
343
+ #JSON.parse(res.body)
344
+ #@logger.info("send res parsed", :res.parsed => res.parsed)
345
+ rescue StandardError => err
346
+ @logger.warn("Failed to send batch, attempt: ", :error_inspect => err.inspect, :error => err)
347
+ end
348
+ return status_code if !status_code.nil? && status_code.to_i != 429 && status_code.to_i.div(100) != 5
349
+ return res
350
+ raise StandardError.new res
351
+ rescue StandardError => e
352
+ retry_count += 1
353
+ @logger.warn("Failed to send batch, attempt: #{retry_count}/#{@retries}", :error_inspect => e.inspect, :error => e)
354
+ if retry_count < @retries
355
+ sleep delay
356
+ if delay * 2 <= @max_delay
357
+ delay = delay * 2
358
+ else
359
+ delay = @max_delay
360
+ end
361
+ retry
362
+ else
363
+ @logger.error("Failed to send batch", :error_inspect => e.inspect, :error => e)
364
+ return res
365
+ end
366
+ end
367
+ end
368
+
369
+ end
@@ -0,0 +1,77 @@
1
+ # encoding: utf-8
2
+ require 'net/http'
3
+ require 'time'
4
+ require 'uri'
5
+ require 'json'
6
+ require 'oauth2'
7
+ require 'snappy'
8
+
9
+ class Promtest
10
+
11
+ def register
12
+ @url = "https://telemetry.pensieve.maersk-digital.net/api/v1/push"
13
+ @client_id = "d6c2ea51-64db-4a46-b168-dc5751d30b38"
14
+ @client_secret = "Nfi8Q~cSuI3PQrCxCNTy5m363cHUSNJ.JaGaYar5"
15
+ @tokenurl_domain = "https://login.microsoftonline.com/05d75c05-fa1a-42e7-9cf1-eb416c396f2d/"
16
+ @tokenurl_endpoint = "oauth2/v2.0/token"
17
+ @scopes = "api://ingestion.pensieve/.default"
18
+
19
+ @uri = URI.parse(@url)
20
+
21
+ send()
22
+ end
23
+
24
+ def send()
25
+ puts("send: started method")
26
+ payload = batch.to_json
27
+ client = createClient()
28
+ access = getToken(client)
29
+ puts "token"
30
+ puts token
31
+
32
+ WriteRequest(access)
33
+ end
34
+
35
+ def label_set_for(labels)
36
+ #@logger.info("label_set_for: started method")
37
+ return stringify_values(labels) if !labels.empty?
38
+ end
39
+
40
+ def stringify_values(labels)
41
+ #@logger.info("stringify_values: started method")
42
+ stringified = {}
43
+ labels.each { |k,v| stringified[k] = v.to_s }
44
+
45
+ stringified
46
+ end
47
+
48
+ def WriteRequest(token)
49
+ puts "TimeSeries"
50
+ payload1 = {"TimeSeries": {Labels: label_set_for({ '__name__':'vsphere_vm', app: 'apmt_logs_test', env: 'test', product_id: 'apmt_test', provider: 'onprem', region: 'westeurope', site_code: 'USTIS01'}), Samples: [Timestamp: Time.now.to_f(), Value: 10.5 ]}}.to_json
51
+ puts payload1
52
+ payload = Marshal.dump(payload1)
53
+ source = Snappy.deflate(payload)
54
+
55
+ puts "Payload"
56
+ puts payload
57
+ puts "Snappy"
58
+ puts source
59
+ response = token.post(@uri, {:body => source, :headers => {'Content-Type' => 'application/x-protobuf', 'Content-Encoding' => 'snappy', 'User-Agent' => 'mop-edge-1.0.0', 'X-Prometheus-Remote-Write-Version' => '0.1.0'}})
60
+ return response
61
+ end
62
+
63
+ def createClient()
64
+ @logger.info("createClient: started method")
65
+ client = OAuth2::Client.new(@client_id, @client_secret, auth_scheme: "basic_auth", site: @tokenurl_domain,:token_url => @tokenurl_endpoint)
66
+ return client
67
+ end
68
+
69
+ def getToken(client)
70
+ params = {}
71
+ opts = {}
72
+ params = params.merge('scope' => @scopes)
73
+ access = client.client_credentials.get_token(params, opts)
74
+ return access
75
+ end
76
+
77
+ end
@@ -0,0 +1,33 @@
1
+ Gem::Specification.new do |s|
2
+ s.name = 'logstash-output-percy'
3
+ s.version = '1.0.0'
4
+ s.authors = ['Britto Prabhu']
5
+ s.email = ['britto.prabhu@apmterminals.com']
6
+
7
+ s.summary = 'Output plugin to ship logs to Loki server with oauth2 token'
8
+ s.description = 'This Output plugin to ship logs to Loki server with oauth2 token and proxy support'
9
+ s.summary = "Sends events to a generic HTTP or HTTPS endpoint with additional oauth2 token support"
10
+ s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program. This plugin supports oauth2 token."
11
+ s.homepage = 'https://github.com/Maersk-Global/apmt-observability-deployment'
12
+ s.license = 'Apache-2.0'
13
+ s.require_paths = ["lib"]
14
+
15
+ # Files
16
+ s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
17
+
18
+ # Tests
19
+ s.test_files = s.files.grep(%r{^(test|spec|features)/})
20
+
21
+ # Special flag to let us know this is actually a logstash plugin
22
+ s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
23
+
24
+ # Gem dependencies
25
+ s.add_runtime_dependency "oauth2", "~> 2.0.9"
26
+ s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
27
+ s.add_runtime_dependency "logstash-mixin-http_client", ">= 7.2.0", "< 8.0.0"
28
+ s.add_runtime_dependency "prometheus-client", "1.0.0"
29
+
30
+ s.add_development_dependency 'logstash-devutils'
31
+ s.add_development_dependency 'sinatra'
32
+ s.add_development_dependency 'webrick'
33
+ end
metadata ADDED
@@ -0,0 +1,168 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: logstash-output-percy
3
+ version: !ruby/object:Gem::Version
4
+ version: 1.0.0
5
+ platform: ruby
6
+ authors:
7
+ - Britto Prabhu
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2024-08-07 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: oauth2
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: 2.0.9
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: 2.0.9
27
+ - !ruby/object:Gem::Dependency
28
+ name: logstash-core-plugin-api
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: '1.60'
34
+ - - "<="
35
+ - !ruby/object:Gem::Version
36
+ version: '2.99'
37
+ type: :runtime
38
+ prerelease: false
39
+ version_requirements: !ruby/object:Gem::Requirement
40
+ requirements:
41
+ - - ">="
42
+ - !ruby/object:Gem::Version
43
+ version: '1.60'
44
+ - - "<="
45
+ - !ruby/object:Gem::Version
46
+ version: '2.99'
47
+ - !ruby/object:Gem::Dependency
48
+ name: logstash-mixin-http_client
49
+ requirement: !ruby/object:Gem::Requirement
50
+ requirements:
51
+ - - ">="
52
+ - !ruby/object:Gem::Version
53
+ version: 7.2.0
54
+ - - "<"
55
+ - !ruby/object:Gem::Version
56
+ version: 8.0.0
57
+ type: :runtime
58
+ prerelease: false
59
+ version_requirements: !ruby/object:Gem::Requirement
60
+ requirements:
61
+ - - ">="
62
+ - !ruby/object:Gem::Version
63
+ version: 7.2.0
64
+ - - "<"
65
+ - !ruby/object:Gem::Version
66
+ version: 8.0.0
67
+ - !ruby/object:Gem::Dependency
68
+ name: prometheus-client
69
+ requirement: !ruby/object:Gem::Requirement
70
+ requirements:
71
+ - - '='
72
+ - !ruby/object:Gem::Version
73
+ version: 1.0.0
74
+ type: :runtime
75
+ prerelease: false
76
+ version_requirements: !ruby/object:Gem::Requirement
77
+ requirements:
78
+ - - '='
79
+ - !ruby/object:Gem::Version
80
+ version: 1.0.0
81
+ - !ruby/object:Gem::Dependency
82
+ name: logstash-devutils
83
+ requirement: !ruby/object:Gem::Requirement
84
+ requirements:
85
+ - - ">="
86
+ - !ruby/object:Gem::Version
87
+ version: '0'
88
+ type: :development
89
+ prerelease: false
90
+ version_requirements: !ruby/object:Gem::Requirement
91
+ requirements:
92
+ - - ">="
93
+ - !ruby/object:Gem::Version
94
+ version: '0'
95
+ - !ruby/object:Gem::Dependency
96
+ name: sinatra
97
+ requirement: !ruby/object:Gem::Requirement
98
+ requirements:
99
+ - - ">="
100
+ - !ruby/object:Gem::Version
101
+ version: '0'
102
+ type: :development
103
+ prerelease: false
104
+ version_requirements: !ruby/object:Gem::Requirement
105
+ requirements:
106
+ - - ">="
107
+ - !ruby/object:Gem::Version
108
+ version: '0'
109
+ - !ruby/object:Gem::Dependency
110
+ name: webrick
111
+ requirement: !ruby/object:Gem::Requirement
112
+ requirements:
113
+ - - ">="
114
+ - !ruby/object:Gem::Version
115
+ version: '0'
116
+ type: :development
117
+ prerelease: false
118
+ version_requirements: !ruby/object:Gem::Requirement
119
+ requirements:
120
+ - - ">="
121
+ - !ruby/object:Gem::Version
122
+ version: '0'
123
+ description: This gem is a Logstash plugin required to be installed on top of the
124
+ Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
125
+ gem is not a stand-alone program. This plugin supports oauth2 token.
126
+ email:
127
+ - britto.prabhu@apmterminals.com
128
+ executables: []
129
+ extensions: []
130
+ extra_rdoc_files: []
131
+ files:
132
+ - Gemfile
133
+ - README.md
134
+ - lib/logstash/outputs/Edge/TimeSeries.rb
135
+ - lib/logstash/outputs/Edge/batch.rb
136
+ - lib/logstash/outputs/Edge/entry.rb
137
+ - lib/logstash/outputs/Edge/label.rb
138
+ - lib/logstash/outputs/Edge/sample.rb
139
+ - lib/logstash/outputs/percy.rb
140
+ - lib/logstash/outputs/prom_test.rb
141
+ - logstash-output-percy.gemspec
142
+ homepage: https://github.com/Maersk-Global/apmt-observability-deployment
143
+ licenses:
144
+ - Apache-2.0
145
+ metadata:
146
+ logstash_plugin: 'true'
147
+ logstash_group: output
148
+ post_install_message:
149
+ rdoc_options: []
150
+ require_paths:
151
+ - lib
152
+ required_ruby_version: !ruby/object:Gem::Requirement
153
+ requirements:
154
+ - - ">="
155
+ - !ruby/object:Gem::Version
156
+ version: '0'
157
+ required_rubygems_version: !ruby/object:Gem::Requirement
158
+ requirements:
159
+ - - ">="
160
+ - !ruby/object:Gem::Version
161
+ version: '0'
162
+ requirements: []
163
+ rubygems_version: 3.0.3.1
164
+ signing_key:
165
+ specification_version: 4
166
+ summary: Sends events to a generic HTTP or HTTPS endpoint with additional oauth2 token
167
+ support
168
+ test_files: []