fluent-plugin-grafana-loki 1.2.14 → 1.2.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +7 -4
- data/bin/setup +6 -4
- data/bin/test +5 -0
- data/lib/fluent/plugin/out_loki.rb +60 -32
- metadata +4 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b4a8acb3f25f565acd7f4071ea961ab1d7a76b7a2f80068fbcc77c3e380f3f5d
|
4
|
+
data.tar.gz: 9b0c3124a9a12d2c5f2f5a1a96d885fe257040b0c58db62bd323d8799a7c62bc
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 65384975b8afc4d2fe377d7551caf461b14ea92dd9b21d254bf3cd001582807918671f724e902d012694fd7907601eb738c8d9f7adb14b2474163154a79eaa8b
|
7
|
+
data.tar.gz: f2fb6366e91d22068797f43cf7b439b06d788a4dac33936bd62fa7895056f1f03e5b439ba2e31e0523152af8c0b2a6c380cc15eb1b35502cefcd5a8daeaeb601
|
data/README.md
CHANGED
@@ -6,14 +6,17 @@ See [docs/client/fluentd/README.md](../../docs/sources/clients/fluentd/_index.md
|
|
6
6
|
|
7
7
|
## Development
|
8
8
|
|
9
|
-
After checking out the repo, run `bin/setup` to install dependencies. Then, run `
|
9
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `bin/test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
10
10
|
|
11
|
-
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `fluent-plugin-grafana-loki.gemspec`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
11
|
+
To install this gem onto your local machine, run `ruby -S bundle exec rake install`. To release a new version, update the version number in `fluent-plugin-grafana-loki.gemspec`, and then run `ruby -S bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
12
12
|
|
13
|
-
To create the gem: `gem build fluent-plugin-grafana-loki.gemspec`
|
13
|
+
To create the gem: `ruby -S gem build fluent-plugin-grafana-loki.gemspec`
|
14
14
|
|
15
15
|
Useful additions:
|
16
|
-
|
16
|
+
|
17
|
+
```bash
|
18
|
+
ruby -S gem install rubocop
|
19
|
+
```
|
17
20
|
|
18
21
|
## Testing
|
19
22
|
|
data/bin/setup
CHANGED
@@ -1,7 +1,9 @@
|
|
1
1
|
#!/usr/bin/env bash
|
2
|
+
|
2
3
|
set -euo pipefail
|
3
|
-
IFS=$'\n\t'
|
4
|
-
set -vx
|
5
4
|
|
6
|
-
|
7
|
-
|
5
|
+
ruby --version
|
6
|
+
echo ""
|
7
|
+
ruby -S gem install bundler --version 2.3.4
|
8
|
+
ruby -S bundle config set --local path $(pwd)/vendor/bundle
|
9
|
+
ruby -S bundle install
|
data/bin/test
ADDED
@@ -35,21 +35,24 @@ module Fluent
|
|
35
35
|
|
36
36
|
DEFAULT_BUFFER_TYPE = 'memory'
|
37
37
|
|
38
|
-
desc '
|
38
|
+
desc 'Loki API base URL'
|
39
39
|
config_param :url, :string, default: 'https://logs-prod-us-central1.grafana.net'
|
40
40
|
|
41
|
-
desc '
|
41
|
+
desc 'Authentication: basic auth credentials'
|
42
42
|
config_param :username, :string, default: nil
|
43
43
|
config_param :password, :string, default: nil, secret: true
|
44
44
|
|
45
|
-
desc '
|
45
|
+
desc 'Authentication: Authorization header with Bearer token scheme'
|
46
|
+
config_param :bearer_token_file, :string, default: nil
|
47
|
+
|
48
|
+
desc 'TLS: parameters for presenting a client certificate'
|
46
49
|
config_param :cert, :string, default: nil
|
47
50
|
config_param :key, :string, default: nil
|
48
51
|
|
49
|
-
desc 'TLS'
|
52
|
+
desc 'TLS: CA certificate file for server certificate verification'
|
50
53
|
config_param :ca_cert, :string, default: nil
|
51
54
|
|
52
|
-
desc '
|
55
|
+
desc 'TLS: disable server certificate verification'
|
53
56
|
config_param :insecure_tls, :bool, default: false
|
54
57
|
|
55
58
|
desc 'Loki tenant id'
|
@@ -80,7 +83,7 @@ module Fluent
|
|
80
83
|
super
|
81
84
|
@uri = URI.parse(@url + '/loki/api/v1/push')
|
82
85
|
unless @uri.is_a?(URI::HTTP) || @uri.is_a?(URI::HTTPS)
|
83
|
-
raise Fluent::ConfigError, '
|
86
|
+
raise Fluent::ConfigError, 'URL parameter must have HTTP/HTTPS scheme'
|
84
87
|
end
|
85
88
|
|
86
89
|
@record_accessors = {}
|
@@ -96,24 +99,42 @@ module Fluent
|
|
96
99
|
@remove_keys_accessors.push(record_accessor_create(key))
|
97
100
|
end
|
98
101
|
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
+
# If configured, load and validate client certificate (and corresponding key)
|
103
|
+
if client_cert_configured?
|
104
|
+
load_client_cert
|
105
|
+
validate_client_cert_key
|
106
|
+
end
|
107
|
+
|
108
|
+
raise "bearer_token_file #{@bearer_token_file} not found" if !@bearer_token_file.nil? && !File.exist?(@bearer_token_file)
|
109
|
+
|
110
|
+
@auth_token_bearer = nil
|
111
|
+
if !@bearer_token_file.nil?
|
112
|
+
if !File.exist?(@bearer_token_file)
|
113
|
+
raise "bearer_token_file #{@bearer_token_file} not found"
|
114
|
+
end
|
115
|
+
|
116
|
+
# Read the file once, assume long-lived authentication token.
|
117
|
+
@auth_token_bearer = File.read(@bearer_token_file)
|
118
|
+
if @auth_token_bearer.empty?
|
119
|
+
raise "bearer_token_file #{@bearer_token_file} is empty"
|
120
|
+
end
|
121
|
+
log.info "will use Bearer token from bearer_token_file #{@bearer_token_file} in Authorization header"
|
102
122
|
end
|
103
123
|
|
124
|
+
|
104
125
|
raise "CA certificate file #{@ca_cert} not found" if !@ca_cert.nil? && !File.exist?(@ca_cert)
|
105
126
|
end
|
106
127
|
|
107
|
-
def
|
128
|
+
def client_cert_configured?
|
108
129
|
!@key.nil? && !@cert.nil?
|
109
130
|
end
|
110
131
|
|
111
|
-
def
|
132
|
+
def load_client_cert
|
112
133
|
@cert = OpenSSL::X509::Certificate.new(File.read(@cert)) if @cert
|
113
134
|
@key = OpenSSL::PKey.read(File.read(@key)) if @key
|
114
135
|
end
|
115
136
|
|
116
|
-
def
|
137
|
+
def validate_client_cert_key
|
117
138
|
if !@key.is_a?(OpenSSL::PKey::RSA) && !@key.is_a?(OpenSSL::PKey::DSA)
|
118
139
|
raise "Unsupported private key type #{key.class}"
|
119
140
|
end
|
@@ -123,13 +144,6 @@ module Fluent
|
|
123
144
|
true
|
124
145
|
end
|
125
146
|
|
126
|
-
def http_opts(uri)
|
127
|
-
opts = {
|
128
|
-
use_ssl: uri.scheme == 'https'
|
129
|
-
}
|
130
|
-
opts
|
131
|
-
end
|
132
|
-
|
133
147
|
# flush a chunk to loki
|
134
148
|
def write(chunk)
|
135
149
|
# streams by label
|
@@ -141,7 +155,10 @@ module Fluent
|
|
141
155
|
# add ingest path to loki url
|
142
156
|
res = loki_http_request(body, tenant)
|
143
157
|
|
144
|
-
|
158
|
+
if res.is_a?(Net::HTTPSuccess)
|
159
|
+
log.debug "POST request was responded to with status code #{res.code}"
|
160
|
+
return
|
161
|
+
end
|
145
162
|
|
146
163
|
res_summary = "#{res.code} #{res.message} #{res.body}"
|
147
164
|
log.warn "failed to write post to #{@uri} (#{res_summary})"
|
@@ -151,19 +168,19 @@ module Fluent
|
|
151
168
|
raise(LogPostError, res_summary) if res.is_a?(Net::HTTPTooManyRequests) || res.is_a?(Net::HTTPServerError)
|
152
169
|
end
|
153
170
|
|
154
|
-
def
|
171
|
+
def http_request_opts(uri)
|
155
172
|
opts = {
|
156
173
|
use_ssl: uri.scheme == 'https'
|
157
174
|
}
|
158
175
|
|
159
|
-
#
|
176
|
+
# Optionally disable server server certificate verification.
|
160
177
|
if @insecure_tls
|
161
178
|
opts = opts.merge(
|
162
179
|
verify_mode: OpenSSL::SSL::VERIFY_NONE
|
163
180
|
)
|
164
181
|
end
|
165
182
|
|
166
|
-
#
|
183
|
+
# Optionally present client certificate
|
167
184
|
if !@cert.nil? && !@key.nil?
|
168
185
|
opts = opts.merge(
|
169
186
|
cert: @cert,
|
@@ -171,7 +188,8 @@ module Fluent
|
|
171
188
|
)
|
172
189
|
end
|
173
190
|
|
174
|
-
#
|
191
|
+
# For server certificate verification: set custom CA bundle.
|
192
|
+
# Only takes effect when `insecure_tls` is not set.
|
175
193
|
unless @ca_cert.nil?
|
176
194
|
opts = opts.merge(
|
177
195
|
ca_file: @ca_cert
|
@@ -194,11 +212,12 @@ module Fluent
|
|
194
212
|
@uri.request_uri
|
195
213
|
)
|
196
214
|
req.add_field('Content-Type', 'application/json')
|
215
|
+
req.add_field('Authorization', "Bearer #{@auth_token_bearer}") if !@auth_token_bearer.nil?
|
197
216
|
req.add_field('X-Scope-OrgID', tenant) if tenant
|
198
217
|
req.body = Yajl.dump(body)
|
199
218
|
req.basic_auth(@username, @password) if @username
|
200
219
|
|
201
|
-
opts =
|
220
|
+
opts = http_request_opts(@uri)
|
202
221
|
|
203
222
|
msg = "sending #{req.body.length} bytes to loki"
|
204
223
|
msg += " (tenant: \"#{tenant}\")" if tenant
|
@@ -259,7 +278,17 @@ module Fluent
|
|
259
278
|
when :key_value
|
260
279
|
formatted_labels = []
|
261
280
|
record.each do |k, v|
|
262
|
-
|
281
|
+
# Remove non UTF-8 characters by force-encoding the string
|
282
|
+
if v.is_a?(String)
|
283
|
+
v = v.encode('utf-8', invalid: :replace)
|
284
|
+
end
|
285
|
+
# Escape double quotes and backslashes by prefixing them with a backslash
|
286
|
+
v = v.to_s.gsub(%r{(["\\])}, '\\\\\1')
|
287
|
+
if v.include?(' ') || v.include?('=')
|
288
|
+
formatted_labels.push(%(#{k}="#{v}"))
|
289
|
+
else
|
290
|
+
formatted_labels.push(%(#{k}=#{v}))
|
291
|
+
end
|
263
292
|
end
|
264
293
|
line = formatted_labels.join(' ')
|
265
294
|
end
|
@@ -267,7 +296,6 @@ module Fluent
|
|
267
296
|
line
|
268
297
|
end
|
269
298
|
|
270
|
-
#
|
271
299
|
# convert a line to loki line with labels
|
272
300
|
def line_to_loki(record)
|
273
301
|
chunk_labels = {}
|
@@ -281,9 +309,11 @@ module Fluent
|
|
281
309
|
|
282
310
|
if @extract_kubernetes_labels && record.key?('kubernetes')
|
283
311
|
kubernetes_labels = record['kubernetes']['labels']
|
284
|
-
kubernetes_labels.
|
285
|
-
|
286
|
-
|
312
|
+
if !kubernetes_labels.nil?
|
313
|
+
kubernetes_labels.each_key do |l|
|
314
|
+
new_key = l.gsub(%r{[.\-\/]}, '_')
|
315
|
+
chunk_labels[new_key] = kubernetes_labels[l]
|
316
|
+
end
|
287
317
|
end
|
288
318
|
end
|
289
319
|
|
@@ -316,10 +346,8 @@ module Fluent
|
|
316
346
|
# iterate through each chunk and create a loki stream entry
|
317
347
|
def chunk_to_loki(chunk)
|
318
348
|
streams = {}
|
319
|
-
last_time = nil
|
320
349
|
chunk.each do |time, record|
|
321
350
|
# each chunk has a unique set of labels
|
322
|
-
last_time = time if last_time.nil?
|
323
351
|
result = line_to_loki(record)
|
324
352
|
chunk_labels = result[:labels]
|
325
353
|
# initialize a new stream with the chunk_labels if it does not exist
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-grafana-loki
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.2.
|
4
|
+
version: 1.2.18
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- woodsaj
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date:
|
13
|
+
date: 2022-01-17 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: fluentd
|
@@ -129,6 +129,7 @@ files:
|
|
129
129
|
- README.md
|
130
130
|
- bin/console
|
131
131
|
- bin/setup
|
132
|
+
- bin/test
|
132
133
|
- lib/fluent/plugin/out_loki.rb
|
133
134
|
homepage: https://github.com/grafana/loki/
|
134
135
|
licenses:
|
@@ -149,7 +150,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
149
150
|
- !ruby/object:Gem::Version
|
150
151
|
version: '0'
|
151
152
|
requirements: []
|
152
|
-
rubygems_version: 3.0.3
|
153
|
+
rubygems_version: 3.0.3.1
|
153
154
|
signing_key:
|
154
155
|
specification_version: 4
|
155
156
|
summary: Output plugin to ship logs to a Grafana Loki server
|