fluent-plugin-k8s-metrics-agg 1.1.0 → 1.1.5

data/test/helper.rb

@@ -1,5 +1,4 @@
 require 'simplecov'
-require 'fluent/plugin/in_kubernetes_metrics_aggregator.rb'
 SimpleCov.start
 
 $LOAD_PATH.unshift(File.expand_path('..', __dir__))
@@ -37,33 +36,16 @@ module PluginTestHelper
     k8s_url + '/v1/nodes/generics-aws-node-three:10255/proxy/stats/summary'
   end
 
-  def stub_api_port_10250
-    WebMock.stub_request(:get, 'https://node.fakedestination.com:10250/api')
-           .with(
-             headers: {
-               'Accept' => '*/*',
-               'Accept-Encoding' => 'gzip, deflate',
-               'Host' => 'node.fakedestination.com:10250'
-             }
-           )
-           .to_return(status: 200,
-                      body: File.open(File.expand_path('../v1.json', __FILE__)),
-                      headers: {})
-  end
-
   def stub_api_port_10255
     WebMock.stub_request(:get,
                          'https://node.fakedestination.com:10255/api')
            .with(
              headers: {
-               'Accept' => '*/*',
-               'Accept-Encoding' => 'gzip, deflate',
                'Host' => 'node.fakedestination.com:10255'
              }
            )
            .to_return(status: 200,
-                      body: File.open(File.expand_path('../v1.json', __FILE__)),
-                      headers: {})
+                      body: File.open(File.expand_path('../v1.json', __FILE__)))
   end
 
   def stub_api_v1
@@ -71,29 +53,27 @@ module PluginTestHelper
                          'https://node.fakedestination.com:10255/api/v1')
            .with(
              headers: {
-               'Accept' => '*/*',
-               'Accept-Encoding' => 'gzip, deflate',
                'Host' => 'node.fakedestination.com:10255'
              }
            )
            .to_return(status: 200,
-                      body: File.open(File.expand_path('../v1.json', __FILE__)),
-                      headers: {})
-  end
+                      body: File.open(File.expand_path('../v1.json', __FILE__)))
+  end
+
+  def stub_api_pods(timeout=false)
+    get_pods = WebMock.stub_request(:get,
+                                    'https://node.fakedestination.com:10255/api/v1/pods')
+                   .with(
+                       headers: {
+                           'Host' => 'node.fakedestination.com:10255'
+                       }
+                   )
+    if timeout
+      get_pods = get_pods.to_timeout.then
+    end
 
-  def stub_api_pods
-    WebMock.stub_request(:get,
-                         'https://node.fakedestination.com:10255/api/v1/pods')
-           .with(
-             headers: {
-               'Accept' => '*/*',
-               'Accept-Encoding' => 'gzip, deflate',
-               'Host' => 'node.fakedestination.com:10255'
-             }
-           )
-           .to_return(status: 200,
-                      body: File.open(File.expand_path('../pods.json', __FILE__)),
-                      headers: {})
+    get_pods.to_return(status: 200,
+                       body: File.open(File.expand_path('../pods.json', __FILE__)))
   end
 
   def stub_api_node_1
@@ -101,29 +81,28 @@ module PluginTestHelper
                          'https://node.fakedestination.com:10255/api/v1/nodes/generics-aws-node-one:10255/proxy/stats/summary')
            .with(
              headers: {
-               'Accept' => '*/*',
-               'Accept-Encoding' => 'gzip, deflate',
                'Host' => 'node.fakedestination.com:10255'
              }
            )
            .to_return(status: 200,
-                      body: File.open(File.expand_path('../node1.json', __FILE__)),
-                      headers: {})
+                      body: File.open(File.expand_path('../node1.json', __FILE__)))
   end
 
-  def stub_api_node_2
-    WebMock.stub_request(:get,
-                         'https://node.fakedestination.com:10255/api/v1/nodes/generics-aws-node-two:10255/proxy/stats/summary')
+  def stub_api_node_2(timeout=false)
+    get_node_summary = WebMock.stub_request(:get,
+                                            'https://node.fakedestination.com:10255/api/v1/nodes/generics-aws-node-two:10255/proxy/stats/summary')
            .with(
              headers: {
-               'Accept' => '*/*',
-               'Accept-Encoding' => 'gzip, deflate',
                'Host' => 'node.fakedestination.com:10255'
              }
            )
-           .to_return(status: 200,
-                      body: File.open(File.expand_path('../node2.json', __FILE__)),
-                      headers: {})
+
+    if timeout
+      get_node_summary = get_node_summary.to_timeout
+    end
+
+    get_node_summary.to_return(status: 200,
+                      body: File.open(File.expand_path('../node2.json', __FILE__)))
   end
 
   def stub_api_node_3
@@ -131,40 +110,45 @@ module PluginTestHelper
                          'https://node.fakedestination.com:10255/api/v1/nodes/generics-aws-node-three:10255/proxy/stats/summary')
            .with(
              headers: {
-               'Accept' => '*/*',
-               'Accept-Encoding' => 'gzip, deflate',
                'Host' => 'node.fakedestination.com:10255'
              }
            )
            .to_return(status: 200,
-                      body: File.open(File.expand_path('../node3.json', __FILE__)),
-                      headers: {})
+                      body: File.open(File.expand_path('../node3.json', __FILE__)))
   end
 
-  def stub_api_nodes
-    WebMock.stub_request(:get,
-                         'https://node.fakedestination.com:10255/api/v1/nodes')
-           .with(
-             headers: {
-               'Accept' => '*/*',
-               'Accept-Encoding' => 'gzip, deflate',
-               'Host' => 'node.fakedestination.com:10255'
-             }
-           )
-           .to_return(status: 200,
-                      body: File.open(File.expand_path('../nodes.json', __FILE__)),
-                      headers: {})
+  def stub_api_nodes(timeout=false)
+    get_nodes = WebMock.stub_request(:get, 'https://node.fakedestination.com:10255/api/v1/nodes')
+                    .with(
+                        headers: {
+                            'Host' => 'node.fakedestination.com:10255'
+                        }
+                    )
+
+    if timeout
+      get_nodes = get_nodes.to_timeout.times(2) # Nodes endpoint is called from two timers so must fail in both cases
+    end
+
+    get_nodes.to_return(status: 200,
+               body: File.open(File.expand_path('../nodes.json', __FILE__)))
   end
 
-  def stub_k8s_requests
-    stub_api_port_10250
+  def stub_k8s_init_requests
+    WebMock.reset!
+
+    stub_api_port_10255
+  end
+
+  def stub_k8s_requests(nodes_timeout: false, node_summary_timeout: false, pods_timeout: false)
+    WebMock.reset!
+
     stub_api_port_10255
     stub_api_v1
-    stub_api_pods
-    stub_api_nodes
+    stub_api_pods(pods_timeout)
+    stub_api_nodes(nodes_timeout)
     stub_api_node_1
+    stub_api_node_2(node_summary_timeout)
     stub_api_node_3
-    stub_api_node_2
   end
 
   def get_parsed_file(file_name)

data/test/plugin/test_in_kubernetes_metrics_aggregator.rb

@@ -5,10 +5,6 @@ class KubernetesMetricsAggInputTest < Test::Unit::TestCase
   include Fluent::Test::Helpers
   include PluginTestHelper
 
-  @driver = nil
-  @driver_test = nil
-  @@hash_map_test = {}
-
   ZERO_CONFIG = %([
   ]).freeze
 
@@ -34,20 +30,22 @@ class KubernetesMetricsAggInputTest < Test::Unit::TestCase
   METRIC_TEST_CONFIG = %([
       kubernetes_url https://node.fakedestination.com
       kubelet_port 10255
+      interval 5s
+      tag kube.*
+  ]).freeze
+
+  TIMEOUT_TEST_CONFIG = %([
+      kubernetes_url https://node.fakedestination.com
+      kubelet_port 10255
+      interval 5s
       tag kube.*
   ]).freeze
 
   setup do
     Fluent::Test.setup
+
     ENV['KUBERNETES_SERVICE_HOST'] = "node.fakedestination.com"
     ENV['KUBERNETES_SERVICE_PORT'] = "10255"
-    stub_k8s_requests
-    @driver = create_driver(METRIC_TEST_CONFIG)
-    @driver.run timeout: 20, expect_emits: 200, shutdown: false
-
-    @driver.events.each do |tag, time, record|
-      @@hash_map_test[tag] = tag, time, record
-    end
   end
 
   def create_driver(conf = BASIC_CONFIG)
@@ -60,6 +58,8 @@ class KubernetesMetricsAggInputTest < Test::Unit::TestCase
 
   sub_test_case 'default parameter configuration' do
     test 'test default params' do
+      stub_k8s_init_requests
+
       d = create_driver(ZERO_CONFIG)
       assert_equal 10_250, d.instance.kubelet_port
       assert_equal 'kubernetes.metrics.*', d.instance.tag
@@ -78,141 +78,174 @@ class KubernetesMetricsAggInputTest < Test::Unit::TestCase
 
   sub_test_case 'modify parameter changes' do
     test 'test kubelet_port and supplied kubernetes URL parameters' do
+      stub_k8s_init_requests
+
       d = create_driver(ADVANCED_CONFIG_NO_CERTS)
       assert_equal 'https://node.fakedestination.com', d.instance.kubernetes_url
       assert_equal 10_255, d.instance.kubelet_port
     end
 
     test 'test tag and interval parameters' do
+      stub_k8s_init_requests
+
       d = create_driver(ADVANCED_CONFIG_NO_CERTS)
       assert_equal 'test.tag.check', d.instance.tag
       assert_equal 120, d.instance.interval
     end
 
     test 'test insecure_ssl and cluster_name parameters ' do
+      stub_k8s_init_requests
+
       d = create_driver(ADVANCED_CONFIG_NO_CERTS)
       assert_true d.instance.insecure_ssl
       assert_equal 'awesome_cluster', d.instance.cluster_name
     end
   end
 
-  sub_test_case 'Test metrics exist, limits_request_scraper - limits' do
-    test 'Testing kube.container.memory.limit' do
-      assert_true @@hash_map_test.key?('kube.container.memory.limit')
-    end
-
-    test 'Testing kube.namespace.cpu.limit' do
-      assert_true @@hash_map_test.key?('kube.namespace.cpu.limit')
-    end
-
-    test 'Testing kube.namespace.memory.limit	' do
-      assert_true @@hash_map_test.key?('kube.namespace.memory.limit')
-    end
-
-    test 'Testing kube.container.cpu.limit' do
-      assert_true @@hash_map_test.key?('kube.container.cpu.limit')
-    end
-
-    test 'Testing kube.pod.cpu.limit' do
-      assert_true @@hash_map_test.key?('kube.pod.cpu.limit')
-    end
-
-    test 'Testing kube.cluster.memory.limit	' do
-      assert_true @@hash_map_test.key?('kube.cluster.memory.limit')
-    end
-
-    test 'Testing kube.pod.memory.limit	' do
-      assert_true @@hash_map_test.key?('kube.pod.memory.limit')
-    end
-
-    test 'Testing kube.cluster.cpu.limit' do
-      assert_true @@hash_map_test.key?('kube.cluster.cpu.limit')
+  sub_test_case 'Test metrics exist' do
+    test 'Testing all expected metrics are emitted' do
+      stub_k8s_requests
+
+      hash_map_test = {}
+
+      d = create_driver(METRIC_TEST_CONFIG)
+      d.run timeout: 12, expect_emits: 200, shutdown: false
+
+      d.events.each do |tag, time, record|
+        hash_map_test[tag] = tag, time, record
+      end
+
+      # Test metrics exist, limits_request_scraper - limits
+      assert_true hash_map_test.key?('kube.namespace.cpu.limit')
+      assert_true hash_map_test.key?('kube.namespace.memory.limit')
+      assert_true hash_map_test.key?('kube.container.cpu.limit')
+      assert_true hash_map_test.key?('kube.pod.cpu.limit')
+      assert_true hash_map_test.key?('kube.cluster.memory.limit')
+      assert_true hash_map_test.key?('kube.pod.memory.limit')
+      assert_true hash_map_test.key?('kube.cluster.cpu.limit')
+
+      # Test metrics exist, limits_request_scraper - request
+      assert_true hash_map_test.key?('kube.cluster.memory.request')
+      assert_true hash_map_test.key?('kube.container.memory.request')
+      assert_true hash_map_test.key?('kube.pod.memory.request')
+      assert_true hash_map_test.key?('kube.namespace.memory.request')
+      assert_true hash_map_test.key?('kube.container.cpu.request')
+      assert_true hash_map_test.key?('kube.namespace.cpu.request')
+      assert_true hash_map_test.key?('kube.pod.cpu.request')
+      assert_true hash_map_test.key?('kube.cluster.cpu.request')
+
+      # Test metrics exist, node_scraper/resource_usage_scraper 1
+      assert_true hash_map_test.key?('kube.node.cpu.capacity')
+      assert_true hash_map_test.key?('kube.node.memory.capacity')
+      assert_true hash_map_test.key?('kube.node.memory.allocatable')
+      assert_true hash_map_test.key?('kube.node.cpu.utilization')
+      assert_true hash_map_test.key?('kube.node.memory.reservation')
+      assert_true hash_map_test.key?('kube.node.memory.utilization')
+
+      # Test metrics exist, node_scraper/resource_usage_scraper 2
+      assert_true hash_map_test.key?('kube.namespace.memory.usage')
+      assert_true hash_map_test.key?('kube.cluster.memory.usage')
+      assert_true hash_map_test.key?('kube.namespace.cpu.usage')
+      assert_true hash_map_test.key?('kube.node.cpu.allocatable')
+      assert_true hash_map_test.key?('kube.node.cpu.reservation')
+      assert_true hash_map_test.key?('kube.cluster.cpu.usage')
+
+      d.instance_shutdown
     end
   end
 
-  sub_test_case 'Test metrics exist, limits_request_scraper - request' do
-    test 'Testing kube.cluster.memory.request	' do
-      assert_true @@hash_map_test.key?('kube.cluster.memory.request')
-    end
+  sub_test_case 'Test handles request timeouts' do
 
-    test 'Testing kube.container.memory.request' do
-      assert_true @@hash_map_test.key?('kube.container.memory.request')
-    end
+    test 'Testing event count with nodes call timeout' do
+      stub_k8s_requests(nodes_timeout: true)
 
-    test 'Testing kube.pod.memory.request' do
-      assert_true @@hash_map_test.key?('kube.pod.memory.request')
-    end
+      namespace_event_count = 0
+      pod_event_count = 0
+      node_event_count = 0
 
-    test 'Testing kube.namespace.memory.request	' do
-      assert_true @@hash_map_test.key?('kube.namespace.memory.request')
-    end
+      d = create_driver(TIMEOUT_TEST_CONFIG)
+      # Should run for two intervals, the first call to node 1 which has the only 'default' namespace pod should timeout the first time
+      d.run timeout: 12, expect_emits: 500, shutdown: false
 
-    test 'Testing kube.container.cpu.request' do
-      assert_true @@hash_map_test.key?('kube.container.cpu.request')
-    end
+      d.events.each do |tag, _time, record|
+        # Limit to one events that should be emitted once per interval
+        if tag == 'kube.pod.cpu.limit' && record['name'] == 'new-metrics-test-final-splunk-kubernetes-metrics-fgszl'
+          pod_event_count += 1
+        end
+        if tag == 'kube.namespace.cpu.usage' && record['name'] == 'kube-system'
+          namespace_event_count += 1
+        end
+        if tag == 'kube.node.cpu.capacity' && record['node'] == 'generics-aws-node-one'
+          node_event_count += 1
+        end
+      end
 
-    test 'Testing kube.namespace.cpu.request' do
-      assert_true @@hash_map_test.key?('kube.namespace.cpu.request')
-    end
+      # 2 intervals - first call times out but timer continues emitting successfully next interval
+      assert_equal 1, node_event_count, 'Number of node events emitted was wrong'
+      # 2 intervals - first call times out but timer continues emitting successfully next interval
+      assert_equal 1, namespace_event_count, 'Number of namespace events emitted was wrong'
+      # 2 intervals - not timeouts
+      assert_equal 2, pod_event_count, 'Number of pod events emitted was wrong'
 
-    test 'Testing kube.pod.cpu.request' do
-      assert_true @@hash_map_test.key?('kube.pod.cpu.request')
+      d.instance_shutdown
     end
 
-    test 'Testing kube.cluster.cpu.request' do
-      assert_true @@hash_map_test.key?('kube.cluster.cpu.request')
-    end
-  end
+    test 'Testing event count with pods call timeout' do
+      stub_k8s_requests(pods_timeout: true)
 
-  sub_test_case 'Test metrics exist, node_scraper/resource_usage_scraper 1' do
-    test 'Testing kube.node.cpu.capacity' do
-      assert_true @@hash_map_test.key?('kube.node.cpu.capacity')
-    end
+      pod_event_count = 0
+      node_event_count = 0
 
-    test 'Testing kube.node.memory.capacity	' do
-      assert_true @@hash_map_test.key?('kube.node.memory.capacity')
-    end
+      d = create_driver(TIMEOUT_TEST_CONFIG)
+      # Should run for two intervals, the first call to node 1 which has the only 'default' namespace pod should timeout the first time
+      d.run timeout: 12, expect_emits: 500, shutdown: false
 
-    test 'Testing kube.node.memory.allocatable' do
-      assert_true @@hash_map_test.key?('kube.node.memory.allocatable')
-    end
+      d.events.each do |tag, _time, record|
+        # Limit to one events that should be emitted once per interval
+        if tag == 'kube.pod.cpu.limit' && record['name'] == 'new-metrics-test-final-splunk-kubernetes-metrics-fgszl'
+          pod_event_count += 1
+        end
 
-    test 'Testing kube.node.cpu.utilization	' do
-      assert_true @@hash_map_test.key?('kube.node.cpu.utilization')
-    end
+        if tag == 'kube.node.cpu.utilization' && record['node'] == 'generics-aws-node-one'
+          node_event_count += 1
+        end
+      end
 
-    test 'Testing kube.node.memory.reservation' do
-      assert_true @@hash_map_test.key?('kube.node.memory.reservation')
-    end
+      # 2 intervals - first call times out but timer continues emitting successfully next interval
+      assert_equal 1, pod_event_count, 'Number of pod events emitted was wrong'
+      # 2 intervals - not timeouts
+      assert_equal 2, node_event_count, 'Number of namespace events emitted was wrong'
 
-    test 'Testing kube.node.memory.utilization' do
-      assert_true @@hash_map_test.key?('kube.node.memory.utilization')
+      d.instance_shutdown
     end
-  end
 
-  sub_test_case 'Test metrics exist, node_scraper/resource_usage_scraper 2' do
-    test 'Testing kube.namespace.memory.usage	' do
-      assert_true @@hash_map_test.key?('kube.namespace.memory.usage')
-    end
+    test 'Testing event count with node summary call timeout' do
+      stub_k8s_requests(node_summary_timeout: true)
 
-    test 'Testing kube.cluster.memory.usage' do
-      assert_true @@hash_map_test.key?('kube.cluster.memory.usage')
-    end
+      namespace_event_count = 0
+      pod_event_count = 0
 
-    test 'Testing kube.namespace.cpu.usage' do
-      assert_true @@hash_map_test.key?('kube.namespace.cpu.usage')
-    end
+      d = create_driver(TIMEOUT_TEST_CONFIG)
+      # Should run for two intervals, the first call to node 1 which has the only 'default' namespace pod should timeout the first time
+      d.run timeout: 12, expect_emits: 500, shutdown: false
 
-    test 'Testing kube.node.cpu.allocatable	' do
-      assert_true @@hash_map_test.key?('kube.node.cpu.allocatable')
-    end
+      d.events.each do |tag, _time, record|
+        # Limit to one events that should be emitted once per interval
+        if tag == 'kube.namespace.cpu.usage' && record['name'] == 'kube-system'
+          namespace_event_count += 1
+        end
+        if tag == 'kube.pod.cpu.limit' && record['name'] == 'new-metrics-test-final-splunk-kubernetes-metrics-fgszl'
+          pod_event_count += 1
+        end
+      end
 
-    test 'Testing kube.node.cpu.reservation	' do
-      assert_true @@hash_map_test.key?('kube.node.cpu.reservation')
-    end
+      # 2 intervals - first call times out but timer continues emitting successfully next interval
+      assert_equal 1, namespace_event_count, 'Number of namespace events emitted was wrong'
+      # 2 intervals - not timeouts
+      assert_equal 2, pod_event_count, 'Number of pod events emitted was wrong'
 
-    test 'Testing kube.cluster.cpu.usage' do
-      assert_true @@hash_map_test.key?('kube.cluster.cpu.usage')
+      d.instance_shutdown
     end
+
   end
 end