fluent-plugin-grafana-loki 1.2.14 → 1.2.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 1fd9308b0afb758b4718b7ad46fc5a3c154400a93514ea79fe3453279abe5cdc
4
- data.tar.gz: b67621f7aa29c775f6e81f546623dec74d326097071f2ce5bd04523ffd865769
3
+ metadata.gz: b4a8acb3f25f565acd7f4071ea961ab1d7a76b7a2f80068fbcc77c3e380f3f5d
4
+ data.tar.gz: 9b0c3124a9a12d2c5f2f5a1a96d885fe257040b0c58db62bd323d8799a7c62bc
5
5
  SHA512:
6
- metadata.gz: 5e6fee1480cdd594f027cbbb22e503a7ecc0dd2384a6b5d46083eed2a6cdbb820d3c3bfe8f98f2cd197d89cbee2876d8a6659d57a070980b609e6b0cc99fec7a
7
- data.tar.gz: 5452556eed1158c3ccaf5bcb5509a773ff89ad4f64e7b3e778603aef0d221cbd2d1ea5ff0e7bed47b1d68b51abae9bf58fccab5996b9ca67bcbfaf81a977cafe
6
+ metadata.gz: 65384975b8afc4d2fe377d7551caf461b14ea92dd9b21d254bf3cd001582807918671f724e902d012694fd7907601eb738c8d9f7adb14b2474163154a79eaa8b
7
+ data.tar.gz: f2fb6366e91d22068797f43cf7b439b06d788a4dac33936bd62fa7895056f1f03e5b439ba2e31e0523152af8c0b2a6c380cc15eb1b35502cefcd5a8daeaeb601
data/README.md CHANGED
@@ -6,14 +6,17 @@ See [docs/client/fluentd/README.md](../../docs/sources/clients/fluentd/_index.md
6
6
 
7
7
  ## Development
8
8
 
9
- After checking out the repo, run `bin/setup` to install dependencies. Then, run `bundle exec rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
9
+ After checking out the repo, run `bin/setup` to install dependencies. Then, run `bin/test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
10
10
 
11
- To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `fluent-plugin-grafana-loki.gemspec`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
11
+ To install this gem onto your local machine, run `ruby -S bundle exec rake install`. To release a new version, update the version number in `fluent-plugin-grafana-loki.gemspec`, and then run `ruby -S bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
12
12
 
13
- To create the gem: `gem build fluent-plugin-grafana-loki.gemspec`
13
+ To create the gem: `ruby -S gem build fluent-plugin-grafana-loki.gemspec`
14
14
 
15
15
  Useful additions:
16
- `gem install rubocop`
16
+
17
+ ```bash
18
+ ruby -S gem install rubocop
19
+ ```
17
20
 
18
21
  ## Testing
19
22
 
data/bin/setup CHANGED
@@ -1,7 +1,9 @@
1
1
  #!/usr/bin/env bash
2
+
2
3
  set -euo pipefail
3
- IFS=$'\n\t'
4
- set -vx
5
4
 
6
- gem install bundler
7
- bundle install
5
+ ruby --version
6
+ echo ""
7
+ ruby -S gem install bundler --version 2.3.4
8
+ ruby -S bundle config set --local path $(pwd)/vendor/bundle
9
+ ruby -S bundle install
data/bin/test ADDED
@@ -0,0 +1,5 @@
1
+ #!/usr/bin/env bash
2
+
3
+ set -euo pipefail
4
+
5
+ ruby -S bundle exec rspec
@@ -35,21 +35,24 @@ module Fluent
35
35
 
36
36
  DEFAULT_BUFFER_TYPE = 'memory'
37
37
 
38
- desc 'url of loki server'
38
+ desc 'Loki API base URL'
39
39
  config_param :url, :string, default: 'https://logs-prod-us-central1.grafana.net'
40
40
 
41
- desc 'BasicAuth credentials'
41
+ desc 'Authentication: basic auth credentials'
42
42
  config_param :username, :string, default: nil
43
43
  config_param :password, :string, default: nil, secret: true
44
44
 
45
- desc 'Client certificate'
45
+ desc 'Authentication: Authorization header with Bearer token scheme'
46
+ config_param :bearer_token_file, :string, default: nil
47
+
48
+ desc 'TLS: parameters for presenting a client certificate'
46
49
  config_param :cert, :string, default: nil
47
50
  config_param :key, :string, default: nil
48
51
 
49
- desc 'TLS'
52
+ desc 'TLS: CA certificate file for server certificate verification'
50
53
  config_param :ca_cert, :string, default: nil
51
54
 
52
- desc 'Disable server certificate verification'
55
+ desc 'TLS: disable server certificate verification'
53
56
  config_param :insecure_tls, :bool, default: false
54
57
 
55
58
  desc 'Loki tenant id'
@@ -80,7 +83,7 @@ module Fluent
80
83
  super
81
84
  @uri = URI.parse(@url + '/loki/api/v1/push')
82
85
  unless @uri.is_a?(URI::HTTP) || @uri.is_a?(URI::HTTPS)
83
- raise Fluent::ConfigError, 'url parameter must be valid HTTP'
86
+ raise Fluent::ConfigError, 'URL parameter must have HTTP/HTTPS scheme'
84
87
  end
85
88
 
86
89
  @record_accessors = {}
@@ -96,24 +99,42 @@ module Fluent
96
99
  @remove_keys_accessors.push(record_accessor_create(key))
97
100
  end
98
101
 
99
- if ssl_cert?
100
- load_ssl
101
- validate_ssl_key
102
+ # If configured, load and validate client certificate (and corresponding key)
103
+ if client_cert_configured?
104
+ load_client_cert
105
+ validate_client_cert_key
106
+ end
107
+
108
+ raise "bearer_token_file #{@bearer_token_file} not found" if !@bearer_token_file.nil? && !File.exist?(@bearer_token_file)
109
+
110
+ @auth_token_bearer = nil
111
+ if !@bearer_token_file.nil?
112
+ if !File.exist?(@bearer_token_file)
113
+ raise "bearer_token_file #{@bearer_token_file} not found"
114
+ end
115
+
116
+ # Read the file once, assume long-lived authentication token.
117
+ @auth_token_bearer = File.read(@bearer_token_file)
118
+ if @auth_token_bearer.empty?
119
+ raise "bearer_token_file #{@bearer_token_file} is empty"
120
+ end
121
+ log.info "will use Bearer token from bearer_token_file #{@bearer_token_file} in Authorization header"
102
122
  end
103
123
 
124
+
104
125
  raise "CA certificate file #{@ca_cert} not found" if !@ca_cert.nil? && !File.exist?(@ca_cert)
105
126
  end
106
127
 
107
- def ssl_cert?
128
+ def client_cert_configured?
108
129
  !@key.nil? && !@cert.nil?
109
130
  end
110
131
 
111
- def load_ssl
132
+ def load_client_cert
112
133
  @cert = OpenSSL::X509::Certificate.new(File.read(@cert)) if @cert
113
134
  @key = OpenSSL::PKey.read(File.read(@key)) if @key
114
135
  end
115
136
 
116
- def validate_ssl_key
137
+ def validate_client_cert_key
117
138
  if !@key.is_a?(OpenSSL::PKey::RSA) && !@key.is_a?(OpenSSL::PKey::DSA)
118
139
  raise "Unsupported private key type #{key.class}"
119
140
  end
@@ -123,13 +144,6 @@ module Fluent
123
144
  true
124
145
  end
125
146
 
126
- def http_opts(uri)
127
- opts = {
128
- use_ssl: uri.scheme == 'https'
129
- }
130
- opts
131
- end
132
-
133
147
  # flush a chunk to loki
134
148
  def write(chunk)
135
149
  # streams by label
@@ -141,7 +155,10 @@ module Fluent
141
155
  # add ingest path to loki url
142
156
  res = loki_http_request(body, tenant)
143
157
 
144
- return if res.is_a?(Net::HTTPSuccess)
158
+ if res.is_a?(Net::HTTPSuccess)
159
+ log.debug "POST request was responded to with status code #{res.code}"
160
+ return
161
+ end
145
162
 
146
163
  res_summary = "#{res.code} #{res.message} #{res.body}"
147
164
  log.warn "failed to write post to #{@uri} (#{res_summary})"
@@ -151,19 +168,19 @@ module Fluent
151
168
  raise(LogPostError, res_summary) if res.is_a?(Net::HTTPTooManyRequests) || res.is_a?(Net::HTTPServerError)
152
169
  end
153
170
 
154
- def ssl_opts(uri)
171
+ def http_request_opts(uri)
155
172
  opts = {
156
173
  use_ssl: uri.scheme == 'https'
157
174
  }
158
175
 
159
- # Disable server TLS certificate verification
176
+ # Optionally disable server server certificate verification.
160
177
  if @insecure_tls
161
178
  opts = opts.merge(
162
179
  verify_mode: OpenSSL::SSL::VERIFY_NONE
163
180
  )
164
181
  end
165
182
 
166
- # Verify client TLS certificate
183
+ # Optionally present client certificate
167
184
  if !@cert.nil? && !@key.nil?
168
185
  opts = opts.merge(
169
186
  cert: @cert,
@@ -171,7 +188,8 @@ module Fluent
171
188
  )
172
189
  end
173
190
 
174
- # Specify custom certificate authority
191
+ # For server certificate verification: set custom CA bundle.
192
+ # Only takes effect when `insecure_tls` is not set.
175
193
  unless @ca_cert.nil?
176
194
  opts = opts.merge(
177
195
  ca_file: @ca_cert
@@ -194,11 +212,12 @@ module Fluent
194
212
  @uri.request_uri
195
213
  )
196
214
  req.add_field('Content-Type', 'application/json')
215
+ req.add_field('Authorization', "Bearer #{@auth_token_bearer}") if !@auth_token_bearer.nil?
197
216
  req.add_field('X-Scope-OrgID', tenant) if tenant
198
217
  req.body = Yajl.dump(body)
199
218
  req.basic_auth(@username, @password) if @username
200
219
 
201
- opts = ssl_opts(@uri)
220
+ opts = http_request_opts(@uri)
202
221
 
203
222
  msg = "sending #{req.body.length} bytes to loki"
204
223
  msg += " (tenant: \"#{tenant}\")" if tenant
@@ -259,7 +278,17 @@ module Fluent
259
278
  when :key_value
260
279
  formatted_labels = []
261
280
  record.each do |k, v|
262
- formatted_labels.push(%(#{k}="#{v}"))
281
+ # Remove non UTF-8 characters by force-encoding the string
282
+ if v.is_a?(String)
283
+ v = v.encode('utf-8', invalid: :replace)
284
+ end
285
+ # Escape double quotes and backslashes by prefixing them with a backslash
286
+ v = v.to_s.gsub(%r{(["\\])}, '\\\\\1')
287
+ if v.include?(' ') || v.include?('=')
288
+ formatted_labels.push(%(#{k}="#{v}"))
289
+ else
290
+ formatted_labels.push(%(#{k}=#{v}))
291
+ end
263
292
  end
264
293
  line = formatted_labels.join(' ')
265
294
  end
@@ -267,7 +296,6 @@ module Fluent
267
296
  line
268
297
  end
269
298
 
270
- #
271
299
  # convert a line to loki line with labels
272
300
  def line_to_loki(record)
273
301
  chunk_labels = {}
@@ -281,9 +309,11 @@ module Fluent
281
309
 
282
310
  if @extract_kubernetes_labels && record.key?('kubernetes')
283
311
  kubernetes_labels = record['kubernetes']['labels']
284
- kubernetes_labels.each_key do |l|
285
- new_key = l.gsub(%r{[.\-\/]}, '_')
286
- chunk_labels[new_key] = kubernetes_labels[l]
312
+ if !kubernetes_labels.nil?
313
+ kubernetes_labels.each_key do |l|
314
+ new_key = l.gsub(%r{[.\-\/]}, '_')
315
+ chunk_labels[new_key] = kubernetes_labels[l]
316
+ end
287
317
  end
288
318
  end
289
319
 
@@ -316,10 +346,8 @@ module Fluent
316
346
  # iterate through each chunk and create a loki stream entry
317
347
  def chunk_to_loki(chunk)
318
348
  streams = {}
319
- last_time = nil
320
349
  chunk.each do |time, record|
321
350
  # each chunk has a unique set of labels
322
- last_time = time if last_time.nil?
323
351
  result = line_to_loki(record)
324
352
  chunk_labels = result[:labels]
325
353
  # initialize a new stream with the chunk_labels if it does not exist
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-grafana-loki
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.2.14
4
+ version: 1.2.18
5
5
  platform: ruby
6
6
  authors:
7
7
  - woodsaj
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2020-07-23 00:00:00.000000000 Z
13
+ date: 2022-01-17 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: fluentd
@@ -129,6 +129,7 @@ files:
129
129
  - README.md
130
130
  - bin/console
131
131
  - bin/setup
132
+ - bin/test
132
133
  - lib/fluent/plugin/out_loki.rb
133
134
  homepage: https://github.com/grafana/loki/
134
135
  licenses:
@@ -149,7 +150,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
149
150
  - !ruby/object:Gem::Version
150
151
  version: '0'
151
152
  requirements: []
152
- rubygems_version: 3.0.3
153
+ rubygems_version: 3.0.3.1
153
154
  signing_key:
154
155
  specification_version: 4
155
156
  summary: Output plugin to ship logs to a Grafana Loki server