fluent-plugin-kubernetes_metadata_filter 2.0.0 → 2.1.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 3e76bf94f6dd2d7d375b01379c3a0533670291e7
4
- data.tar.gz: 0cf265d4423a322313a6233a9360c0438081f878
3
+ metadata.gz: 58bdc9ed97a93eb46eeec306ad5851b94d3b5656
4
+ data.tar.gz: 56806c61750f533d2190c0a0915fb1fed0b4d46e
5
5
  SHA512:
6
- metadata.gz: c52356d4f5edc444bfb26b9f812257cafc250328f7de81708bb7f3e64711c1c85895a46877537d7fd5d17e486f75328900a848d80625e6450922672005b7ce6a
7
- data.tar.gz: 4d0e4b4cdfd0734736419d32fe4237130e81fd04cc6612a5ea5f1eca22c69b8300b8ea4baeeca943b63dad93f256aeb7f35bd160f753a1a28ad650d395ba43b4
6
+ metadata.gz: e38271b6f13d95781df47a4c99474ec9d54bd8926a48a55d2613086079997728902d74b1207986325e36f9ea647b65786b0602f280a8ae5d4c37f3ae4737973a
7
+ data.tar.gz: 90a46813a80eec7928f35387adc4778d3c07e1e6f61eea0cf3ca1725ef8195bf73816f733bb7c5cb0464cd69269981a8fb7562bc2a3e17ffaf00384624821b74
data/README.md CHANGED
@@ -42,8 +42,6 @@ This must used named capture groups for `container_name`, `pod_name` & `namespac
42
42
  * `cache_size` - size of the cache of Kubernetes metadata to reduce requests to the API server (default: `1000`)
43
43
  * `cache_ttl` - TTL in seconds of each cached element. Set to negative value to disable TTL eviction (default: `3600` - 1 hour)
44
44
  * `watch` - set up a watch on pods on the API server for updates to metadata (default: `true`)
45
- * `merge_json_log` - merge logs in JSON format as top level keys (default: `true`)
46
- * `preserve_json_log` - preserve JSON logs in raw form in the `log` key, only used if the previous option is true (default: `true`)
47
45
  * `de_dot` - replace dots in labels and annotations with configured `de_dot_separator`, required for ElasticSearch 2.x compatibility (default: `true`)
48
46
  * `de_dot_separator` - separator to use if `de_dot` is enabled (default: `_`)
49
47
  * `use_journal` - If false (default), messages are expected to be formatted and tagged as if read by the fluentd in\_tail plugin with wildcard filename. If true, messages are expected to be formatted as if read from the systemd journal. The `MESSAGE` field has the full message. The `CONTAINER_NAME` field has the encoded k8s metadata (see below). The `CONTAINER_ID_FULL` field has the full container uuid. This requires docker to use the `--log-driver=journald` log driver.
@@ -55,6 +53,12 @@ when true (default: `true`)
55
53
  * `orphaned_namespace_name` - The namespace to associate with records where the namespace can not be determined (default: `.orphaned`)
56
54
  * `orphaned_namespace_id` - The namespace id to associate with records where the namespace can not be determined (default: `orphaned`)
57
55
 
56
+ **NOTE:** As of the release 1.1.x of this plugin, it no longer supports parsing the source message into JSON and attaching it to the
57
+ payload. The following configuration options are removed:
58
+
59
+ * `merge_json_log`
60
+ * `preserve_json_log`
61
+
58
62
  Reading from the JSON formatted log files with `in_tail` and wildcard filenames:
59
63
  ```
60
64
  <source>
@@ -104,6 +108,22 @@ Reading from the systemd journal (requires the fluentd `fluent-plugin-systemd` a
104
108
  </match>
105
109
  ```
106
110
 
111
+ ## Environment variables for Kubernetes
112
+
113
+ If the name of the Kubernetes node the plugin is running on is set as
114
+ an environment variable with the name `K8S_NODE_NAME`, it will reduce cache
115
+ misses and needless calls to the Kubernetes API.
116
+
117
+ In the Kubernetes container definition, this is easily accomplished by:
118
+
119
+ ```yaml
120
+ env:
121
+ - name: K8S_NODE_NAME
122
+ valueFrom:
123
+ fieldRef:
124
+ fieldPath: spec.nodeName
125
+ ```
126
+
107
127
  ## Example input/output
108
128
 
109
129
  Kubernetes creates symlinks to Docker log files in `/var/log/containers/*.log`. Docker logs in JSON format.
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
4
 
5
5
  Gem::Specification.new do |gem|
6
6
  gem.name = "fluent-plugin-kubernetes_metadata_filter"
7
- gem.version = "2.0.0"
7
+ gem.version = "2.1.0"
8
8
  gem.authors = ["Jimmi Dyson"]
9
9
  gem.email = ["jimmidyson@gmail.com"]
10
10
  gem.description = %q{Filter plugin to add Kubernetes metadata}
@@ -50,8 +50,6 @@ module Fluent::Plugin
50
50
  :string,
51
51
  :default => 'var\.log\.containers\.(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$'
52
52
  config_param :bearer_token_file, :string, default: nil
53
- config_param :merge_json_log, :bool, default: true
54
- config_param :preserve_json_log, :bool, default: true
55
53
  config_param :secret_dir, :string, default: '/var/run/secrets/kubernetes.io/serviceaccount'
56
54
  config_param :de_dot, :bool, default: true
57
55
  config_param :de_dot_separator, :string, default: '_'
@@ -247,11 +245,9 @@ module Fluent::Plugin
247
245
  end
248
246
  if @use_journal
249
247
  log.debug "Will stream from the journal"
250
- @merge_json_log_key = 'MESSAGE'
251
248
  self.class.class_eval { alias_method :filter_stream, :filter_stream_from_journal }
252
249
  else
253
250
  log.debug "Will stream from the files"
254
- @merge_json_log_key = 'log'
255
251
  self.class.class_eval { alias_method :filter_stream, :filter_stream_from_files }
256
252
  end
257
253
 
@@ -309,13 +305,10 @@ module Fluent::Plugin
309
305
  }
310
306
  end
311
307
 
312
- es.each { |time, record|
313
- record = merge_json_log(record) if @merge_json_log
314
-
308
+ es.each do |time, record|
315
309
  record = record.merge(Marshal.load(Marshal.dump(metadata))) if metadata
316
-
317
310
  new_es.add(time, record)
318
- }
311
+ end
319
312
  dump_stats
320
313
  new_es
321
314
  end
@@ -323,8 +316,7 @@ module Fluent::Plugin
323
316
  def filter_stream_from_journal(tag, es)
324
317
  new_es = Fluent::MultiEventStream.new
325
318
  batch_miss_cache = {}
326
- es.each { |time, record|
327
- record = merge_json_log(record) if @merge_json_log
319
+ es.each do |time, record|
328
320
  metadata = nil
329
321
  if record.has_key?('CONTAINER_NAME') && record.has_key?('CONTAINER_ID_FULL')
330
322
  metadata = record['CONTAINER_NAME'].match(@container_name_to_kubernetes_regexp_compiled) do |match_data|
@@ -347,35 +339,15 @@ module Fluent::Plugin
347
339
  @stats.bump(:container_name_id_missing)
348
340
  end
349
341
 
350
- if metadata
351
- record = record.merge(metadata)
352
- end
342
+ record = record.merge(metadata) if metadata
353
343
 
354
344
  new_es.add(time, record)
355
- }
345
+ end
356
346
 
357
347
  dump_stats
358
348
  new_es
359
349
  end
360
350
 
361
- def merge_json_log(record)
362
- if record.has_key?(@merge_json_log_key)
363
- value = record[@merge_json_log_key].strip
364
- if value[0].eql?('{') && value[-1].eql?('}')
365
- begin
366
- record = JSON.parse(value).merge(record)
367
- unless @preserve_json_log
368
- record.delete(@merge_json_log_key)
369
- end
370
- rescue JSON::ParserError=>e
371
- @stats.bump(:merge_json_parse_errors)
372
- log.debug(e)
373
- end
374
- end
375
- end
376
- record
377
- end
378
-
379
351
  def de_dot!(h)
380
352
  h.keys.each do |ref|
381
353
  if h[ref] && ref =~ /\./
@@ -42,6 +42,9 @@ module KubernetesMetadata
42
42
  if cached
43
43
  @cache[cache_key] = parse_pod_metadata(notice.object)
44
44
  @stats.bump(:pod_cache_watch_updates)
45
+ elsif ENV['K8S_NODE_NAME'] == notice.object['spec']['nodeName'] then
46
+ @cache[cache_key] = parse_pod_metadata(notice.object)
47
+ @stats.bump(:pod_cache_host_updates)
45
48
  else
46
49
  @stats.bump(:pod_cache_watch_misses)
47
50
  end
@@ -381,45 +381,6 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
381
381
  assert_false(filtered[0].has_key?(:kubernetes))
382
382
  end
383
383
 
384
- test 'merges json log data' do
385
- json_log = {
386
- 'hello' => 'world'
387
- }
388
- msg = {
389
- 'log' => "#{json_log.to_json}"
390
- }
391
- filtered = emit_with_tag('non-kubernetes', msg, '')
392
- assert_equal(msg.merge(json_log), filtered[0])
393
- end
394
-
395
- test 'merges json log data in MESSAGE' do
396
- json_log = {
397
- 'hello' => 'world'
398
- }
399
- msg = {
400
- 'MESSAGE' => "#{json_log.to_json}"
401
- }
402
- filtered = emit_with_tag('non-kubernetes', msg, 'use_journal true')
403
- assert_equal(msg.merge(json_log), filtered[0])
404
- end
405
-
406
- test 'merges json log data with message field' do
407
- json_log = {
408
- 'timeMillis' => 1459853347608,
409
- 'thread' => 'main',
410
- 'level' => 'INFO',
411
- 'loggerName' => 'org.apache.camel.spring.SpringCamelContext',
412
- 'message' => 'Total 1 routes, of which 1 is started.',
413
- 'endOfBatch' => false,
414
- 'loggerFqcn' => 'org.apache.logging.slf4j.Log4jLogger'
415
- }
416
- msg = {
417
- 'log' => "#{json_log.to_json}"
418
- }
419
- filtered = emit_with_tag('non-kubernetes', msg, '')
420
- assert_equal(msg.merge(json_log), filtered[0])
421
- end
422
-
423
384
  test 'ignores invalid json in log field' do
424
385
  json_log = "{'foo':123}"
425
386
  msg = {
@@ -429,50 +390,6 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
429
390
  assert_equal(msg, filtered[0])
430
391
  end
431
392
 
432
- test 'merges json log data with message field in MESSAGE' do
433
- json_log = {
434
- 'timeMillis' => 1459853347608,
435
- 'thread' => 'main',
436
- 'level' => 'INFO',
437
- 'loggerName' => 'org.apache.camel.spring.SpringCamelContext',
438
- 'message' => 'Total 1 routes, of which 1 is started.',
439
- 'endOfBatch' => false,
440
- 'loggerFqcn' => 'org.apache.logging.slf4j.Log4jLogger'
441
- }
442
- msg = {
443
- 'MESSAGE' => "#{json_log.to_json}"
444
- }
445
- filtered = emit_with_tag('non-kubernetes', msg, 'use_journal true')
446
- assert_equal(msg.merge(json_log), filtered[0])
447
- end
448
-
449
- test 'emit individual fields from json, throw out whole original string' do
450
- json_log = {
451
- 'hello' => 'world',
452
- 'more' => 'data'
453
- }
454
- msg = {
455
- 'log' => "#{json_log.to_json}"
456
- }
457
- filtered = emit_with_tag('non-kubernetes', msg, 'preserve_json_log false')
458
- assert_equal(json_log, filtered[0])
459
- end
460
-
461
- test 'emit individual fields from json, throw out whole original string in MESSAGE' do
462
- json_log = {
463
- 'hello' => 'world',
464
- 'more' => 'data'
465
- }
466
- msg = {
467
- 'MESSAGE' => "#{json_log.to_json}"
468
- }
469
- filtered = emit_with_tag('non-kubernetes', msg, '
470
- preserve_json_log false
471
- use_journal true
472
- ')
473
- assert_equal(json_log, filtered[0])
474
- end
475
-
476
393
  test 'with kubernetes dotted labels, de_dot enabled' do
477
394
  VCR.use_cassette('kubernetes_docker_metadata_dotted_labels') do
478
395
  filtered = emit({}, '
@@ -81,6 +81,17 @@ class DefaultPodWatchStrategyTest < WatchTest
81
81
  end
82
82
  end
83
83
 
84
+ test 'pod MODIFIED cached when hostname matches' do
85
+ orig_env_val = ENV['K8S_NODE_NAME']
86
+ ENV['K8S_NODE_NAME'] = 'aNodeName'
87
+ @client.stub :watch_pods, [@modified] do
88
+ start_pod_watch
89
+ assert_equal(true, @cache.key?('modified_uid'))
90
+ assert_equal(1, @stats[:pod_cache_host_updates])
91
+ end
92
+ ENV['K8S_NODE_NAME'] = orig_env_val
93
+ end
94
+
84
95
  test 'pod watch notice is updated when MODIFIED is received' do
85
96
  @cache['modified_uid'] = {}
86
97
  @client.stub :watch_pods, [@modified] do
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-kubernetes_metadata_filter
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.0.0
4
+ version: 2.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jimmi Dyson
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-04-19 00:00:00.000000000 Z
11
+ date: 2018-05-14 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd