fluent-plugin-kubernetes_metadata_filter 2.12.0 → 3.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +0 -1
- data/Gemfile.lock +7 -7
- data/README.md +5 -52
- data/doc/benchmark/5m-1-2500lps-256b-baseline-01/cpu.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-baseline-01/latency.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-baseline-01/loss.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-baseline-01/mem.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-baseline-01/readme.md +88 -0
- data/doc/benchmark/5m-1-2500lps-256b-baseline-01/results.html +127 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/cpu.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/latency.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/loss.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/mem.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/readme.md +97 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/results.html +136 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/cpu.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/latency.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/loss.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/mem.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/readme.md +97 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/results.html +136 -0
- data/fluent-plugin-kubernetes_metadata_filter.gemspec +2 -2
- data/lib/fluent/plugin/filter_kubernetes_metadata.rb +15 -77
- data/lib/fluent/plugin/kubernetes_metadata_cache_strategy.rb +2 -2
- data/lib/fluent/plugin/kubernetes_metadata_common.rb +1 -16
- data/lib/fluent/plugin/kubernetes_metadata_stats.rb +16 -0
- data/lib/fluent/plugin/kubernetes_metadata_util.rb +0 -20
- data/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb +1 -1
- data/test/plugin/test_filter_kubernetes_metadata.rb +23 -411
- metadata +21 -4
- data/test/plugin/test_utils.rb +0 -56
@@ -41,6 +41,24 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
|
|
41
41
|
assert_equal(1000, d.instance.cache_size)
|
42
42
|
end
|
43
43
|
|
44
|
+
sub_test_case 'stats_interval' do
|
45
|
+
|
46
|
+
test 'enables stats when greater than zero' do
|
47
|
+
d = create_driver('stats_interval 1')
|
48
|
+
assert_equal(1, d.instance.stats_interval)
|
49
|
+
d.instance.dump_stats
|
50
|
+
assert_false(d.instance.instance_variable_get("@curr_time").nil?)
|
51
|
+
end
|
52
|
+
|
53
|
+
test 'disables stats when <= zero' do
|
54
|
+
d = create_driver('stats_interval 0')
|
55
|
+
assert_equal(0, d.instance.stats_interval)
|
56
|
+
d.instance.dump_stats
|
57
|
+
assert_nil(d.instance.instance_variable_get("@curr_time"))
|
58
|
+
end
|
59
|
+
|
60
|
+
end
|
61
|
+
|
44
62
|
test 'check test_api_adapter' do
|
45
63
|
d = create_driver('test_api_adapter KubernetesMetadata::TestApiAdapter')
|
46
64
|
assert_equal('KubernetesMetadata::TestApiAdapter', d.instance.test_api_adapter)
|
@@ -494,318 +512,6 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
|
|
494
512
|
assert_equal(msg, filtered[0])
|
495
513
|
end
|
496
514
|
|
497
|
-
test 'with kubernetes dotted and slashed labels, de_dot and de_slash enabled' do
|
498
|
-
VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' }, { name: 'kubernetes_get_api_v1' },
|
499
|
-
{ name: 'kubernetes_docker_metadata_dotted_slashed_labels' }]) do
|
500
|
-
filtered = emit({}, '
|
501
|
-
kubernetes_url https://localhost:8443
|
502
|
-
watch false
|
503
|
-
cache_size 1
|
504
|
-
de_dot true
|
505
|
-
de_slash true
|
506
|
-
')
|
507
|
-
expected_kube_metadata = {
|
508
|
-
'docker' => {
|
509
|
-
'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
|
510
|
-
},
|
511
|
-
'kubernetes' => {
|
512
|
-
'host' => 'jimmi-redhat.localnet',
|
513
|
-
'pod_name' => 'fabric8-console-controller-98rqc',
|
514
|
-
'container_name' => 'fabric8-console-container',
|
515
|
-
'container_image' => 'fabric8/hawtio-kubernetes:latest',
|
516
|
-
'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
|
517
|
-
'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c',
|
518
|
-
'namespace_labels' => {
|
519
|
-
'kubernetes_io__namespacetest' => 'somevalue'
|
520
|
-
},
|
521
|
-
'namespace_name' => 'default',
|
522
|
-
'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d',
|
523
|
-
'pod_ip' => '172.17.0.8',
|
524
|
-
'master_url' => 'https://localhost:8443',
|
525
|
-
'labels' => {
|
526
|
-
'kubernetes_io__test' => 'somevalue'
|
527
|
-
}
|
528
|
-
}
|
529
|
-
}
|
530
|
-
assert_equal(expected_kube_metadata, filtered[0])
|
531
|
-
end
|
532
|
-
end
|
533
|
-
|
534
|
-
test 'with kubernetes dotted and slashed labels, de_dot and de_slash disabled' do
|
535
|
-
VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' }, { name: 'kubernetes_get_api_v1' },
|
536
|
-
{ name: 'kubernetes_docker_metadata_dotted_slashed_labels' }]) do
|
537
|
-
filtered = emit({}, '
|
538
|
-
kubernetes_url https://localhost:8443
|
539
|
-
watch false
|
540
|
-
cache_size 1
|
541
|
-
de_dot false
|
542
|
-
de_slash false
|
543
|
-
')
|
544
|
-
expected_kube_metadata = {
|
545
|
-
'docker' => {
|
546
|
-
'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
|
547
|
-
},
|
548
|
-
'kubernetes' => {
|
549
|
-
'host' => 'jimmi-redhat.localnet',
|
550
|
-
'pod_name' => 'fabric8-console-controller-98rqc',
|
551
|
-
'container_name' => 'fabric8-console-container',
|
552
|
-
'container_image' => 'fabric8/hawtio-kubernetes:latest',
|
553
|
-
'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
|
554
|
-
'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c',
|
555
|
-
'namespace_labels' => {
|
556
|
-
'kubernetes.io/namespacetest' => 'somevalue'
|
557
|
-
},
|
558
|
-
'namespace_name' => 'default',
|
559
|
-
'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d',
|
560
|
-
'pod_ip' => '172.17.0.8',
|
561
|
-
'master_url' => 'https://localhost:8443',
|
562
|
-
'labels' => {
|
563
|
-
'kubernetes.io/test' => 'somevalue'
|
564
|
-
}
|
565
|
-
}
|
566
|
-
}
|
567
|
-
assert_equal(expected_kube_metadata, filtered[0])
|
568
|
-
end
|
569
|
-
end
|
570
|
-
|
571
|
-
test 'invalid de_dot_separator config' do
|
572
|
-
assert_raise Fluent::ConfigError do
|
573
|
-
create_driver('
|
574
|
-
de_dot true
|
575
|
-
de_dot_separator contains.
|
576
|
-
')
|
577
|
-
end
|
578
|
-
end
|
579
|
-
|
580
|
-
test 'invalid de_slash_separator config' do
|
581
|
-
assert_raise Fluent::ConfigError do
|
582
|
-
create_driver('
|
583
|
-
de_slash true
|
584
|
-
de_slash_separator contains/
|
585
|
-
')
|
586
|
-
end
|
587
|
-
end
|
588
|
-
|
589
|
-
test 'with records from journald and docker & kubernetes metadata' do
|
590
|
-
# with use_journal true should ignore tags and use CONTAINER_NAME and CONTAINER_ID_FULL
|
591
|
-
tag = 'var.log.containers.junk1_junk2_junk3-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed450.log'
|
592
|
-
msg = {
|
593
|
-
'CONTAINER_NAME' => 'k8s_fabric8-console-container.db89db89_fabric8-console-controller-98rqc_default_c76927af-f563-11e4-b32d-54ee7527188d_89db89db',
|
594
|
-
'CONTAINER_ID_FULL' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459',
|
595
|
-
'randomfield' => 'randomvalue'
|
596
|
-
}
|
597
|
-
VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' }, { name: 'kubernetes_get_api_v1' }, { name: 'kubernetes_get_pod' },
|
598
|
-
{ name: 'kubernetes_get_namespace_default' }]) do
|
599
|
-
filtered = emit_with_tag(tag, msg, '
|
600
|
-
kubernetes_url https://localhost:8443
|
601
|
-
watch false
|
602
|
-
cache_size 1
|
603
|
-
use_journal true
|
604
|
-
')
|
605
|
-
expected_kube_metadata = {
|
606
|
-
'docker' => {
|
607
|
-
'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
|
608
|
-
},
|
609
|
-
'kubernetes' => {
|
610
|
-
'host' => 'jimmi-redhat.localnet',
|
611
|
-
'pod_name' => 'fabric8-console-controller-98rqc',
|
612
|
-
'container_name' => 'fabric8-console-container',
|
613
|
-
'container_image' => 'fabric8/hawtio-kubernetes:latest',
|
614
|
-
'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
|
615
|
-
'namespace_name' => 'default',
|
616
|
-
'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c',
|
617
|
-
'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d',
|
618
|
-
'pod_ip' => '172.17.0.8',
|
619
|
-
'master_url' => 'https://localhost:8443',
|
620
|
-
'labels' => {
|
621
|
-
'component' => 'fabric8Console'
|
622
|
-
}
|
623
|
-
}
|
624
|
-
}.merge(msg)
|
625
|
-
assert_equal(expected_kube_metadata, filtered[0])
|
626
|
-
end
|
627
|
-
end
|
628
|
-
|
629
|
-
test 'with records from journald and docker & kubernetes metadata & namespace_id enabled' do
|
630
|
-
# with use_journal true should ignore tags and use CONTAINER_NAME and CONTAINER_ID_FULL
|
631
|
-
tag = 'var.log.containers.junk1_junk2_junk3-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed450.log'
|
632
|
-
msg = {
|
633
|
-
'CONTAINER_NAME' => 'k8s_fabric8-console-container.db89db89_fabric8-console-controller-98rqc_default_c76927af-f563-11e4-b32d-54ee7527188d_89db89db',
|
634
|
-
'CONTAINER_ID_FULL' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459',
|
635
|
-
'randomfield' => 'randomvalue'
|
636
|
-
}
|
637
|
-
VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' }, { name: 'kubernetes_get_api_v1' }, { name: 'kubernetes_get_pod' },
|
638
|
-
{ name: 'kubernetes_get_namespace_default', options: { allow_playback_repeats: true } }]) do
|
639
|
-
filtered = emit_with_tag(tag, msg, '
|
640
|
-
kubernetes_url https://localhost:8443
|
641
|
-
watch false
|
642
|
-
cache_size 1
|
643
|
-
use_journal true
|
644
|
-
')
|
645
|
-
expected_kube_metadata = {
|
646
|
-
'docker' => {
|
647
|
-
'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
|
648
|
-
},
|
649
|
-
'kubernetes' => {
|
650
|
-
'host' => 'jimmi-redhat.localnet',
|
651
|
-
'pod_name' => 'fabric8-console-controller-98rqc',
|
652
|
-
'container_name' => 'fabric8-console-container',
|
653
|
-
'container_image' => 'fabric8/hawtio-kubernetes:latest',
|
654
|
-
'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
|
655
|
-
'namespace_name' => 'default',
|
656
|
-
'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c',
|
657
|
-
'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d',
|
658
|
-
'pod_ip' => '172.17.0.8',
|
659
|
-
'master_url' => 'https://localhost:8443',
|
660
|
-
'labels' => {
|
661
|
-
'component' => 'fabric8Console'
|
662
|
-
}
|
663
|
-
}
|
664
|
-
}.merge(msg)
|
665
|
-
assert_equal(expected_kube_metadata, filtered[0])
|
666
|
-
end
|
667
|
-
end
|
668
|
-
|
669
|
-
test 'with records from journald and docker & kubernetes metadata with use_journal unset' do
|
670
|
-
# with use_journal unset, should still use the journal fields instead of tag fields
|
671
|
-
tag = 'var.log.containers.fabric8-console-controller-98rqc_default_fabric8-console-container-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459.log'
|
672
|
-
msg = {
|
673
|
-
'CONTAINER_NAME' => 'k8s_journald-container-name.db89db89_journald-pod-name_journald-namespace-name_c76927af-f563-11e4-b32d-54ee7527188d_89db89db',
|
674
|
-
'CONTAINER_ID_FULL' => '838350c64bacba968d39a30a50789b2795291fceca6ccbff55298671d46b0e3b',
|
675
|
-
'kubernetes' => {
|
676
|
-
'namespace_name' => 'k8s-namespace-name',
|
677
|
-
'pod_name' => 'k8s-pod-name',
|
678
|
-
'container_name' => 'k8s-container-name'
|
679
|
-
},
|
680
|
-
'docker' => { 'container_id' => 'e463bc0d3ae38f5c89d92dca49b30e049e899799920b79d4d5f705acbe82ba95' },
|
681
|
-
'randomfield' => 'randomvalue'
|
682
|
-
}
|
683
|
-
VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' }, { name: 'kubernetes_get_api_v1' }, { name: 'kubernetes_get_pod' },
|
684
|
-
{ name: 'kubernetes_get_namespace_default' },
|
685
|
-
{ name: 'metadata_from_tag_journald_and_kubernetes_fields' }]) do
|
686
|
-
es = emit_with_tag(tag, msg, '
|
687
|
-
kubernetes_url https://localhost:8443
|
688
|
-
watch false
|
689
|
-
cache_size 1
|
690
|
-
')
|
691
|
-
expected_kube_metadata = {
|
692
|
-
'docker' => {
|
693
|
-
'container_id' => 'e463bc0d3ae38f5c89d92dca49b30e049e899799920b79d4d5f705acbe82ba95'
|
694
|
-
},
|
695
|
-
'kubernetes' => {
|
696
|
-
'host' => 'jimmi-redhat.localnet',
|
697
|
-
'pod_name' => 'k8s-pod-name',
|
698
|
-
'container_name' => 'k8s-container-name',
|
699
|
-
'container_image' => 'k8s-container-image:latest',
|
700
|
-
'container_image_id' => 'docker://d78c5217c41e9af08d37d9ae2cb070afa1fe3da6bc77bfb18879a8b4bfdf8a34',
|
701
|
-
'namespace_name' => 'k8s-namespace-name',
|
702
|
-
'namespace_id' => '8e0dc8fc-59f2-49f7-a3e2-ed0913e19d9f',
|
703
|
-
'pod_id' => 'ebabf749-5fcd-4750-a3f0-aedd89476da8',
|
704
|
-
'pod_ip' => '172.17.0.8',
|
705
|
-
'master_url' => 'https://localhost:8443',
|
706
|
-
'labels' => {
|
707
|
-
'component' => 'k8s-test'
|
708
|
-
}
|
709
|
-
}
|
710
|
-
}.merge(msg) { |key, oldval, newval| (key == 'kubernetes') || (key == 'docker') ? oldval : newval }
|
711
|
-
assert_equal(expected_kube_metadata, es[0])
|
712
|
-
end
|
713
|
-
end
|
714
|
-
|
715
|
-
test 'with records from journald and docker & kubernetes metadata with lookup_from_k8s_field false' do
|
716
|
-
# with use_journal unset, should still use the journal fields instead of tag fields
|
717
|
-
tag = 'var.log.containers.fabric8-console-controller-98rqc_default_fabric8-console-container-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459.log'
|
718
|
-
msg = {
|
719
|
-
'CONTAINER_NAME' => 'k8s_journald-container-name.db89db89_journald-pod-name_journald-namespace-name_c76927af-f563-11e4-b32d-54ee7527188d_89db89db',
|
720
|
-
'CONTAINER_ID_FULL' => '838350c64bacba968d39a30a50789b2795291fceca6ccbff55298671d46b0e3b',
|
721
|
-
'kubernetes' => {
|
722
|
-
'namespace_name' => 'k8s-namespace-name',
|
723
|
-
'pod_name' => 'k8s-pod-name',
|
724
|
-
'container_name' => 'k8s-container-name'
|
725
|
-
},
|
726
|
-
'docker' => { 'container_id' => 'e463bc0d3ae38f5c89d92dca49b30e049e899799920b79d4d5f705acbe82ba95' },
|
727
|
-
'randomfield' => 'randomvalue'
|
728
|
-
}
|
729
|
-
VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' }, { name: 'kubernetes_get_api_v1' }, { name: 'kubernetes_get_pod' },
|
730
|
-
{ name: 'kubernetes_get_namespace_default', options: { allow_playback_repeats: true } },
|
731
|
-
{ name: 'metadata_from_tag_and_journald_fields' }]) do
|
732
|
-
es = emit_with_tag(tag, msg, '
|
733
|
-
kubernetes_url https://localhost:8443
|
734
|
-
watch false
|
735
|
-
cache_size 1
|
736
|
-
lookup_from_k8s_field false
|
737
|
-
')
|
738
|
-
expected_kube_metadata = {
|
739
|
-
'docker' => {
|
740
|
-
'container_id' => '838350c64bacba968d39a30a50789b2795291fceca6ccbff55298671d46b0e3b'
|
741
|
-
},
|
742
|
-
'kubernetes' => {
|
743
|
-
'host' => 'jimmi-redhat.localnet',
|
744
|
-
'pod_name' => 'journald-pod-name',
|
745
|
-
'container_name' => 'journald-container-name',
|
746
|
-
'container_image' => 'journald-container-image:latest',
|
747
|
-
'container_image_id' => 'docker://dda4c95705fb7050b701b10a7fe928ca5bc971a1dcb521ae3c339194cbf36b47',
|
748
|
-
'namespace_name' => 'journald-namespace-name',
|
749
|
-
'namespace_id' => '8282888f-733f-4f23-a3d3-1fdfa3bdacf2',
|
750
|
-
'pod_id' => '5e1c1e27-b637-4e81-80b6-6d8a8c404d3b',
|
751
|
-
'pod_ip' => '172.17.0.8',
|
752
|
-
'master_url' => 'https://localhost:8443',
|
753
|
-
'labels' => {
|
754
|
-
'component' => 'journald-test'
|
755
|
-
}
|
756
|
-
}
|
757
|
-
}.merge(msg) { |key, oldval, newval| (key == 'kubernetes') || (key == 'docker') ? oldval : newval }
|
758
|
-
assert_equal(expected_kube_metadata, es[0])
|
759
|
-
end
|
760
|
-
end
|
761
|
-
|
762
|
-
test 'uses metadata from tag if use_journal false and lookup_from_k8s_field false' do
|
763
|
-
# with use_journal unset, should still use the journal fields instead of tag fields
|
764
|
-
msg = {
|
765
|
-
'CONTAINER_NAME' => 'k8s_journald-container-name.db89db89_journald-pod-name_journald-namespace-name_c76927af-f563-11e4-b32d-54ee7527188d_89db89db',
|
766
|
-
'CONTAINER_ID_FULL' => '838350c64bacba968d39a30a50789b2795291fceca6ccbff55298671d46b0e3b',
|
767
|
-
'kubernetes' => {
|
768
|
-
'namespace_name' => 'k8s-namespace-name',
|
769
|
-
'pod_name' => 'k8s-pod-name',
|
770
|
-
'container_name' => 'k8s-container-name'
|
771
|
-
},
|
772
|
-
'docker' => { 'container_id' => 'e463bc0d3ae38f5c89d92dca49b30e049e899799920b79d4d5f705acbe82ba95' },
|
773
|
-
'randomfield' => 'randomvalue'
|
774
|
-
}
|
775
|
-
VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' }, { name: 'kubernetes_get_api_v1' }, { name: 'kubernetes_get_pod' },
|
776
|
-
{ name: 'kubernetes_get_namespace_default', options: { allow_playback_repeats: true } },
|
777
|
-
{ name: 'metadata_from_tag_and_journald_fields' }]) do
|
778
|
-
es = emit_with_tag(VAR_LOG_CONTAINER_TAG, msg, "
|
779
|
-
kubernetes_url https://localhost:8443
|
780
|
-
watch false
|
781
|
-
cache_size 1
|
782
|
-
lookup_from_k8s_field false
|
783
|
-
use_journal false
|
784
|
-
")
|
785
|
-
expected_kube_metadata = {
|
786
|
-
'docker' => {
|
787
|
-
'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
|
788
|
-
},
|
789
|
-
'kubernetes' => {
|
790
|
-
'host' => 'jimmi-redhat.localnet',
|
791
|
-
'pod_name' => 'fabric8-console-controller-98rqc',
|
792
|
-
'container_name' => 'fabric8-console-container',
|
793
|
-
'container_image' => 'fabric8/hawtio-kubernetes:latest',
|
794
|
-
'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
|
795
|
-
'namespace_name' => 'default',
|
796
|
-
'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c',
|
797
|
-
'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d',
|
798
|
-
'pod_ip' => '172.17.0.8',
|
799
|
-
'master_url' => 'https://localhost:8443',
|
800
|
-
'labels' => {
|
801
|
-
'component' => 'fabric8Console'
|
802
|
-
}
|
803
|
-
}
|
804
|
-
}.merge(msg) { |key, oldval, newval| (key == 'kubernetes') || (key == 'docker') ? oldval : newval }
|
805
|
-
assert_equal(expected_kube_metadata, es[0])
|
806
|
-
end
|
807
|
-
end
|
808
|
-
|
809
515
|
test 'with kubernetes annotations' do
|
810
516
|
VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' }, { name: 'kubernetes_get_api_v1' },
|
811
517
|
{ name: 'kubernetes_docker_metadata_annotations' },
|
@@ -835,8 +541,8 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
|
|
835
541
|
'component' => 'fabric8Console'
|
836
542
|
},
|
837
543
|
'annotations' => {
|
838
|
-
'
|
839
|
-
'
|
544
|
+
'custom.field1' => 'hello_kitty',
|
545
|
+
'field.two' => 'value'
|
840
546
|
}
|
841
547
|
}
|
842
548
|
}
|
@@ -844,51 +550,6 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
|
|
844
550
|
end
|
845
551
|
end
|
846
552
|
|
847
|
-
test 'with records from journald and docker & kubernetes metadata, alternate form' do
|
848
|
-
# with use_journal true should ignore tags and use CONTAINER_NAME and CONTAINER_ID_FULL
|
849
|
-
tag = 'var.log.containers.junk1_junk2_junk3-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed450.log'
|
850
|
-
msg = {
|
851
|
-
'CONTAINER_NAME' => 'alt_fabric8-console-container_fabric8-console-controller-98rqc_default_c76927af-f563-11e4-b32d-54ee7527188d_0',
|
852
|
-
'CONTAINER_ID_FULL' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459',
|
853
|
-
'randomfield' => 'randomvalue'
|
854
|
-
}
|
855
|
-
VCR.use_cassettes([
|
856
|
-
{ name: 'valid_kubernetes_api_server' },
|
857
|
-
{ name: 'kubernetes_get_api_v1' },
|
858
|
-
{ name: 'kubernetes_get_pod' },
|
859
|
-
{ name: 'kubernetes_get_namespace_default' },
|
860
|
-
{ name: 'metadata_from_tag_and_journald_fields' }
|
861
|
-
]) do
|
862
|
-
filtered = emit_with_tag(tag, msg, '
|
863
|
-
kubernetes_url https://localhost:8443
|
864
|
-
watch false
|
865
|
-
cache_size 1
|
866
|
-
use_journal true
|
867
|
-
')
|
868
|
-
expected_kube_metadata = {
|
869
|
-
'docker' => {
|
870
|
-
'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
|
871
|
-
},
|
872
|
-
'kubernetes' => {
|
873
|
-
'host' => 'jimmi-redhat.localnet',
|
874
|
-
'pod_name' => 'fabric8-console-controller-98rqc',
|
875
|
-
'container_name' => 'fabric8-console-container',
|
876
|
-
'container_image' => 'fabric8/hawtio-kubernetes:latest',
|
877
|
-
'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
|
878
|
-
'namespace_name' => 'default',
|
879
|
-
'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c',
|
880
|
-
'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d',
|
881
|
-
'pod_ip' => '172.17.0.8',
|
882
|
-
'master_url' => 'https://localhost:8443',
|
883
|
-
'labels' => {
|
884
|
-
'component' => 'fabric8Console'
|
885
|
-
}
|
886
|
-
}
|
887
|
-
}.merge(msg)
|
888
|
-
assert_equal(expected_kube_metadata, filtered[0])
|
889
|
-
end
|
890
|
-
end
|
891
|
-
|
892
553
|
test 'with kubernetes namespace annotations' do
|
893
554
|
VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' }, { name: 'kubernetes_get_api_v1' },
|
894
555
|
{ name: 'kubernetes_docker_metadata_annotations' },
|
@@ -918,8 +579,8 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
|
|
918
579
|
'component' => 'fabric8Console'
|
919
580
|
},
|
920
581
|
'annotations' => {
|
921
|
-
'
|
922
|
-
'
|
582
|
+
'custom.field1' => 'hello_kitty',
|
583
|
+
'field.two' => 'value'
|
923
584
|
},
|
924
585
|
'namespace_annotations' => {
|
925
586
|
'workspaceId' => 'myWorkspaceName'
|
@@ -964,56 +625,6 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
|
|
964
625
|
end
|
965
626
|
end
|
966
627
|
|
967
|
-
test 'with CONTAINER_NAME that does not match' do
|
968
|
-
tag = 'var.log.containers.junk4_junk5_junk6-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed450.log'
|
969
|
-
msg = {
|
970
|
-
'CONTAINER_NAME' => 'does_not_match',
|
971
|
-
'CONTAINER_ID_FULL' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459',
|
972
|
-
'randomfield' => 'randomvalue'
|
973
|
-
}
|
974
|
-
VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' }, { name: 'kubernetes_get_api_v1' },
|
975
|
-
{ name: 'kubernetes_docker_metadata_annotations' },
|
976
|
-
{ name: 'kubernetes_get_namespace_default' }]) do
|
977
|
-
filtered = emit_with_tag(tag, msg, '
|
978
|
-
kubernetes_url https://localhost:8443
|
979
|
-
watch false
|
980
|
-
cache_size 1
|
981
|
-
use_journal true
|
982
|
-
')
|
983
|
-
expected_kube_metadata = {
|
984
|
-
'CONTAINER_NAME' => 'does_not_match',
|
985
|
-
'CONTAINER_ID_FULL' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459',
|
986
|
-
'randomfield' => 'randomvalue'
|
987
|
-
}
|
988
|
-
assert_equal(expected_kube_metadata, filtered[0])
|
989
|
-
end
|
990
|
-
end
|
991
|
-
|
992
|
-
test 'with CONTAINER_NAME starts with k8s_ that does not match' do
|
993
|
-
tag = 'var.log.containers.junk4_junk5_junk6-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed450.log'
|
994
|
-
msg = {
|
995
|
-
'CONTAINER_NAME' => 'k8s_doesnotmatch',
|
996
|
-
'CONTAINER_ID_FULL' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459',
|
997
|
-
'randomfield' => 'randomvalue'
|
998
|
-
}
|
999
|
-
VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' }, { name: 'kubernetes_get_api_v1' },
|
1000
|
-
{ name: 'kubernetes_docker_metadata_annotations' },
|
1001
|
-
{ name: 'kubernetes_get_namespace_default' }]) do
|
1002
|
-
filtered = emit_with_tag(tag, msg, '
|
1003
|
-
kubernetes_url https://localhost:8443
|
1004
|
-
watch false
|
1005
|
-
cache_size 1
|
1006
|
-
use_journal true
|
1007
|
-
')
|
1008
|
-
expected_kube_metadata = {
|
1009
|
-
'CONTAINER_NAME' => 'k8s_doesnotmatch',
|
1010
|
-
'CONTAINER_ID_FULL' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459',
|
1011
|
-
'randomfield' => 'randomvalue'
|
1012
|
-
}
|
1013
|
-
assert_equal(expected_kube_metadata, filtered[0])
|
1014
|
-
end
|
1015
|
-
end
|
1016
|
-
|
1017
628
|
test 'processes all events when reading from MessagePackEventStream' do
|
1018
629
|
VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' },
|
1019
630
|
{ name: 'kubernetes_get_api_v1' },
|
@@ -1027,6 +638,7 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
|
|
1027
638
|
kubernetes_url https://localhost:8443
|
1028
639
|
watch false
|
1029
640
|
cache_size 1
|
641
|
+
stats_interval 0
|
1030
642
|
')
|
1031
643
|
d.run do
|
1032
644
|
d.feed(VAR_LOG_CONTAINER_TAG, msgpack_stream)
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-kubernetes_metadata_filter
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 3.1.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- OpenShift Cluster Logging
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2022-
|
12
|
+
date: 2022-09-08 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|
@@ -221,6 +221,24 @@ files:
|
|
221
221
|
- LICENSE.txt
|
222
222
|
- README.md
|
223
223
|
- Rakefile
|
224
|
+
- doc/benchmark/5m-1-2500lps-256b-baseline-01/cpu.png
|
225
|
+
- doc/benchmark/5m-1-2500lps-256b-baseline-01/latency.png
|
226
|
+
- doc/benchmark/5m-1-2500lps-256b-baseline-01/loss.png
|
227
|
+
- doc/benchmark/5m-1-2500lps-256b-baseline-01/mem.png
|
228
|
+
- doc/benchmark/5m-1-2500lps-256b-baseline-01/readme.md
|
229
|
+
- doc/benchmark/5m-1-2500lps-256b-baseline-01/results.html
|
230
|
+
- doc/benchmark/5m-1-2500lps-256b-kube-01-01/cpu.png
|
231
|
+
- doc/benchmark/5m-1-2500lps-256b-kube-01-01/latency.png
|
232
|
+
- doc/benchmark/5m-1-2500lps-256b-kube-01-01/loss.png
|
233
|
+
- doc/benchmark/5m-1-2500lps-256b-kube-01-01/mem.png
|
234
|
+
- doc/benchmark/5m-1-2500lps-256b-kube-01-01/readme.md
|
235
|
+
- doc/benchmark/5m-1-2500lps-256b-kube-01-01/results.html
|
236
|
+
- doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/cpu.png
|
237
|
+
- doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/latency.png
|
238
|
+
- doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/loss.png
|
239
|
+
- doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/mem.png
|
240
|
+
- doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/readme.md
|
241
|
+
- doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/results.html
|
224
242
|
- fluent-plugin-kubernetes_metadata_filter.gemspec
|
225
243
|
- lib/fluent/plugin/filter_kubernetes_metadata.rb
|
226
244
|
- lib/fluent/plugin/kubernetes_metadata_cache_strategy.rb
|
@@ -250,7 +268,6 @@ files:
|
|
250
268
|
- test/plugin/test_cache_stats.rb
|
251
269
|
- test/plugin/test_cache_strategy.rb
|
252
270
|
- test/plugin/test_filter_kubernetes_metadata.rb
|
253
|
-
- test/plugin/test_utils.rb
|
254
271
|
- test/plugin/test_watch_namespaces.rb
|
255
272
|
- test/plugin/test_watch_pods.rb
|
256
273
|
- test/plugin/watch_test.rb
|
@@ -266,7 +283,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
266
283
|
requirements:
|
267
284
|
- - ">="
|
268
285
|
- !ruby/object:Gem::Version
|
269
|
-
version: 2.
|
286
|
+
version: 2.6.0
|
270
287
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
271
288
|
requirements:
|
272
289
|
- - ">="
|
data/test/plugin/test_utils.rb
DELETED
@@ -1,56 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
#
|
4
|
-
# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
|
5
|
-
# Kubernetes metadata
|
6
|
-
#
|
7
|
-
# Copyright 2015 Red Hat, Inc.
|
8
|
-
#
|
9
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
10
|
-
# you may not use this file except in compliance with the License.
|
11
|
-
# You may obtain a copy of the License at
|
12
|
-
#
|
13
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
14
|
-
#
|
15
|
-
# Unless required by applicable law or agreed to in writing, software
|
16
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
17
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
18
|
-
# See the License for the specific language governing permissions and
|
19
|
-
# limitations under the License.
|
20
|
-
#
|
21
|
-
class KubernetesMetadataCacheStatsTest < Test::Unit::TestCase
|
22
|
-
include KubernetesMetadata::Util
|
23
|
-
|
24
|
-
def setup
|
25
|
-
@time_fields = ['time']
|
26
|
-
@internal_time = Time.now
|
27
|
-
end
|
28
|
-
|
29
|
-
test '#create_time_from_record when time is empty' do
|
30
|
-
record = { 'time' => ' ' }
|
31
|
-
assert_equal(@internal_time.to_i, create_time_from_record(record, @internal_time).to_i)
|
32
|
-
end
|
33
|
-
test '#create_time_from_record when time is nil' do
|
34
|
-
record = {}
|
35
|
-
assert_equal(@internal_time.to_i, create_time_from_record(record, @internal_time).to_i)
|
36
|
-
end
|
37
|
-
|
38
|
-
test '#create_time_from_record when time is an integer' do
|
39
|
-
exp_time = Time.now
|
40
|
-
record = { 'time' => exp_time.to_i }
|
41
|
-
assert_equal(exp_time.to_i, create_time_from_record(record, @internal_time).to_i)
|
42
|
-
end
|
43
|
-
|
44
|
-
test '#create_time_from_record when time is a string' do
|
45
|
-
exp_time = Time.now
|
46
|
-
record = { 'time' => exp_time.to_s }
|
47
|
-
assert_equal(exp_time.to_i, create_time_from_record(record, @internal_time).to_i)
|
48
|
-
end
|
49
|
-
|
50
|
-
test '#create_time_from_record when timefields include journal time fields' do
|
51
|
-
@time_fields = ['_SOURCE_REALTIME_TIMESTAMP']
|
52
|
-
exp_time = Time.now
|
53
|
-
record = { '_SOURCE_REALTIME_TIMESTAMP' => exp_time.to_i.to_s }
|
54
|
-
assert_equal(Time.at(exp_time.to_i / 1_000_000, exp_time.to_i % 1_000_000).to_i, create_time_from_record(record, @internal_time).to_i)
|
55
|
-
end
|
56
|
-
end
|