fluent-plugin-datadog-log 0.1.0.rc1

data/test/plugin/base_test.rb

@@ -0,0 +1,173 @@
+# Copyright 2017 Yusuke KUOKA All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'helper'
+require 'mocha/test_unit'
+require 'webmock/test_unit'
+require 'prometheus/client'
+require 'fluent/test/driver/output'
+require 'fluent/test/helpers'
+
+require_relative 'constants'
+
+# Unit tests for Datadog plugin
+module BaseTest
+  include Constants
+  include Fluent::Test::Helpers
+
+  def setup
+    Fluent::Test.setup
+    require 'fluent/plugin/out_datadog_log'
+    @logs_sent = []
+  end
+
+  def create_driver(conf = APPLICATION_DEFAULT_CONFIG)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::DatadogOutput)
+      .configure(conf)
+  end
+
+  private
+
+  def setup_no_metadata_service_stubs
+    # Simulate a machine with no metadata service present
+    stub_request(:any, %r{http://169.254.169.254/.*})
+      .to_raise(Errno::EHOSTUNREACH)
+  end
+
+  def setup_ec2_metadata_stubs
+    # Stub the root, used for platform detection.
+    stub_request(:get, 'http://169.254.169.254')
+      .to_return(status: 200, headers: { 'Server' => 'EC2ws' })
+
+    # Stub the identity document lookup made by the agent.
+    stub_request(:get, 'http://169.254.169.254/latest/dynamic/' \
+                 'instance-identity/document')
+      .to_return(body: EC2_IDENTITY_DOCUMENT, status: 200,
+                 headers: { 'Content-Length' => EC2_IDENTITY_DOCUMENT.length })
+  end
+
+  def setup_logging_stubs
+    yield
+  end
+
+  def setup_prometheus
+    Prometheus::Client.registry.instance_variable_set('@metrics', {})
+  end
+
+  # Provide a stub context that initializes @logs_sent, executes the block and
+  # resets WebMock at the end.
+  def new_stub_context
+    @logs_sent = []
+    yield
+    WebMock.reset!
+  end
+
+  # Container.
+
+  def container_tag_with_container_name(container_name)
+    "kubernetes.#{CONTAINER_POD_NAME}_#{CONTAINER_NAMESPACE_NAME}_" \
+      "#{container_name}"
+  end
+
+  def container_log_entry_with_metadata(
+      log, container_name = CONTAINER_CONTAINER_NAME)
+    {
+      log: log,
+      stream: CONTAINER_STREAM,
+      time: CONTAINER_TIMESTAMP,
+      kubernetes: {
+        namespace_id: CONTAINER_NAMESPACE_ID,
+        namespace_name: CONTAINER_NAMESPACE_NAME,
+        pod_id: CONTAINER_POD_ID,
+        pod_name: CONTAINER_POD_NAME,
+        container_name: container_name,
+        labels: {
+          CONTAINER_LABEL_KEY => CONTAINER_LABEL_VALUE
+        }
+      }
+    }
+  end
+
+  def container_log_entry(log, stream = CONTAINER_STREAM)
+    {
+      log: log,
+      stream: stream,
+      time: CONTAINER_TIMESTAMP
+    }
+  end
+
+  # Docker Container.
+
+  def docker_container_stdout_stderr_log_entry(
+    log, stream = DOCKER_CONTAINER_STREAM_STDOUT)
+    severity = if stream == DOCKER_CONTAINER_STREAM_STDOUT
+                 'INFO'
+               else
+                 'ERROR'
+               end
+    {
+      log: log,
+      source: stream,
+      severity: severity
+    }
+  end
+
+  def docker_container_application_log_entry(log)
+    {
+      log: log,
+      time: DOCKER_CONTAINER_TIMESTAMP
+    }
+  end
+
+  def log_entry(i)
+    "test log entry #{i}"
+  end
+
+  # This module expects the methods below to be overridden.
+
+  def assert_prometheus_metric_value(metric_name, expected_value, labels = {})
+    metric = Prometheus::Client.registry.get(metric_name)
+    assert_not_nil(metric)
+    assert_equal(expected_value, metric.get(labels))
+  end
+
+  # Get the fields of the payload.
+  def get_fields(_payload)
+    _undefined
+  end
+
+  # Get the value of a struct field.
+  def get_struct(_field)
+    _undefined
+  end
+
+  # Get the value of a string field.
+  def get_string(_field)
+    _undefined
+  end
+
+  # Get the value of a number field.
+  def get_number(_field)
+    _undefined
+  end
+
+  # The null value.
+  def null_value(_field)
+    _undefined
+  end
+
+  def _undefined
+    fail "Method #{__callee__} is unimplemented and needs to be overridden."
+  end
+end

data/test/plugin/constants.rb

@@ -0,0 +1,188 @@
+# Copyright 2017 Yusuke KUOKA All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Constants used by unit tests for Datadog plugin.
+module Constants
+  # Generic attributes.
+  HOSTNAME = Socket.gethostname
+
+  # TODO(qingling128) Separate constants into different submodules.
+  # Attributes used for the GCE metadata service.
+  ZONE = 'us-central1-b'
+  VM_ID = '9876543210'
+
+  # Attributes used for the Metadata Agent resources.
+  METADATA_ZONE = 'us-central1-c'
+  METADATA_VM_ID = '0123456789'
+
+  # Attributes used for custom (overridden) configs.
+  CUSTOM_PROJECT_ID = 'test-custom-project-id'
+  CUSTOM_ZONE = 'us-custom-central1-b'
+  CUSTOM_VM_ID = 'C9876543210'
+  CUSTOM_HOSTNAME = 'custom.hostname.org'
+
+  # Attributes used for the EC2 metadata service.
+  EC2_PROJECT_ID = 'test-ec2-project-id'
+  EC2_ZONE = 'us-west-2b'
+  EC2_PREFIXED_ZONE = 'aws:' + EC2_ZONE
+  EC2_VM_ID = 'i-81c16767'
+  EC2_ACCOUNT_ID = '123456789012'
+
+  # The formatting here matches the format used on the VM.
+  EC2_IDENTITY_DOCUMENT = %({
+    "accountId" : "#{EC2_ACCOUNT_ID}",
+    "availabilityZone" : "#{EC2_ZONE}",
+    "instanceId" : "#{EC2_VM_ID}"
+  })
+
+  # Managed VMs specific labels.
+  MANAGED_VM_BACKEND_NAME = 'default'
+  MANAGED_VM_BACKEND_VERSION = 'guestbook2.0'
+
+  # Docker Container labels.
+  DOCKER_CONTAINER_ID = '0d0f03ff8d3c42688692536d1af77a28cd135c0a5c531f25a31'
+  DOCKER_CONTAINER_NAME = 'happy_hippo'
+  DOCKER_CONTAINER_STREAM_STDOUT = 'stdout'
+  DOCKER_CONTAINER_STREAM_STDERR = 'stderr'
+  # Timestamp for 1234567890 seconds and 987654321 nanoseconds since epoch.
+  DOCKER_CONTAINER_TIMESTAMP = '2009-02-13T23:31:30.987654321Z'
+  DOCKER_CONTAINER_SECONDS_EPOCH = 1_234_567_890
+  DOCKER_CONTAINER_NANOS = 987_654_321
+
+  # Container Engine / Kubernetes specific labels.
+  CONTAINER_CLUSTER_NAME = 'cluster-1'
+  CONTAINER_NAMESPACE_ID = '898268c8-4a36-11e5-9d81-42010af0194c'
+  CONTAINER_NAMESPACE_NAME = 'kube-system'
+  CONTAINER_POD_ID = 'cad3c3c4-4b9c-11e5-9d81-42010af0194c'
+  CONTAINER_POD_NAME = 'redis-master-c0l82.foo.bar'
+  CONTAINER_CONTAINER_NAME = 'redis'
+  CONTAINER_LABEL_KEY = 'component'
+  CONTAINER_LABEL_VALUE = 'redis-component'
+  CONTAINER_STREAM = 'stdout'
+  CONTAINER_SEVERITY = 'INFO'
+  # Timestamp for 1234567890 seconds and 987654321 nanoseconds since epoch.
+  CONTAINER_TIMESTAMP = '2009-02-13T23:31:30.987654321Z'
+  CONTAINER_SECONDS_EPOCH = 1_234_567_890
+  CONTAINER_NANOS = 987_654_321
+
+  # Cloud Functions specific labels.
+  CLOUDFUNCTIONS_FUNCTION_NAME = '$My_Function.Name-@1'
+  CLOUDFUNCTIONS_REGION = 'us-central1'
+  CLOUDFUNCTIONS_EXECUTION_ID = '123-0'
+  CLOUDFUNCTIONS_CLUSTER_NAME = 'cluster-1'
+  CLOUDFUNCTIONS_NAMESPACE_NAME = 'default'
+  CLOUDFUNCTIONS_POD_NAME = 'd.dc.myu.uc.functionp.pc.name-a.a1.987-c0l82'
+  CLOUDFUNCTIONS_CONTAINER_NAME = 'worker'
+
+  # Dataflow specific labels.
+  DATAFLOW_REGION = 'us-central1'
+  DATAFLOW_JOB_NAME = 'job_name_1'
+  DATAFLOW_JOB_ID = 'job_id_1'
+  DATAFLOW_STEP_ID = 'step_1'
+  DATAFLOW_TAG = 'dataflow-worker'
+
+  # Dataproc specific labels.
+  DATAPROC_CLUSTER_NAME = 'test-cluster'
+  DATAPROC_CLUSTER_UUID = '00000000-0000-0000-0000-000000000000'
+  DATAPROC_REGION = 'unittest'
+
+  # ML specific labels.
+  ML_REGION = 'us-central1'
+  ML_JOB_ID = 'job_name_1'
+  ML_TASK_NAME = 'task_name_1'
+  ML_TRIAL_ID = 'trial_id_1'
+  ML_LOG_AREA = 'log_area_1'
+  ML_TAG = 'master-replica-0'
+
+  # Parameters used for authentication.
+  AUTH_GRANT_TYPE = 'urn:ietf:params:oauth:grant-type:jwt-bearer'
+  FAKE_AUTH_TOKEN = 'abc123'
+
+  # Information about test credentials files.
+  # path: Path to the credentials file.
+  # project_id: ID of the project, which must correspond to the file contents.
+  IAM_CREDENTIALS = {
+    path: 'test/plugin/data/iam-credentials.json',
+    project_id: 'fluent-test-project'
+  }
+  LEGACY_CREDENTIALS = {
+    path: 'test/plugin/data/credentials.json',
+    project_id: '847859579879'
+  }
+  INVALID_CREDENTIALS = {
+    path: 'test/plugin/data/invalid_credentials.json',
+    project_id: ''
+  }
+
+  # Configuration files for various test scenarios.
+  APPLICATION_DEFAULT_CONFIG = %(
+
+  )
+
+  DETECT_JSON_CONFIG = %(
+    detect_json true
+  )
+
+  PARTIAL_SUCCESS_CONFIG = %(
+    partial_success true
+  )
+
+  REQUIRE_VALID_TAGS_CONFIG = %(
+    require_valid_tags true
+  )
+
+  NO_METADATA_SERVICE_CONFIG = %(
+    use_metadata_service false
+  )
+
+  PROMETHEUS_ENABLE_CONFIG = %(
+    enable_monitoring true
+    monitoring_type prometheus
+  )
+
+  CUSTOM_METADATA_CONFIG = %(
+    zone #{CUSTOM_ZONE}
+    vm_id #{CUSTOM_VM_ID}
+    vm_name #{CUSTOM_HOSTNAME}
+  )
+
+  # Service configurations for various services.
+
+  CUSTOM_LABELS_MESSAGE = {
+    'customKey' => 'value'
+  }
+  # Tags and their sanitized and encoded version.
+  VALID_TAGS = {
+    'test' => 'test',
+    'germanß' => 'german%C3%9F',
+    'chinese中' => 'chinese%E4%B8%AD',
+    'specialCharacter/_-.' => 'specialCharacter%2F_-.',
+    'abc@&^$*' => 'abc%40%26%5E%24%2A',
+    '@&^$*' => '%40%26%5E%24%2A'
+  }
+  INVALID_TAGS = {
+    # Non-string tags.
+    123 => '123',
+    1.23 => '1.23',
+    [1, 2, 3] => '%5B1%2C%202%2C%203%5D',
+    { key: 'value' } => '%7B%22key%22%3D%3E%22value%22%7D',
+    # Non-utf8 string tags.
+    "nonutf8#{[0x92].pack('C*')}" => 'nonutf8%20',
+    "abc#{[0x92].pack('C*')}" => 'abc%20',
+    "#{[0x92].pack('C*')}" => '%20',
+    # Empty string tag.
+    '' => '_'
+  }
+  ALL_TAGS = VALID_TAGS.merge(INVALID_TAGS)
+end

data/test/plugin/test_out_datadog_log.rb

@@ -0,0 +1,206 @@
+# Copyright 2017 Yusuke KUOKA All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require_relative 'base_test'
+
+# Unit tests for Datadog Log plugin
+class DatadogLogOutputTest < Test::Unit::TestCase
+  include BaseTest
+
+  def test_configure
+    new_stub_context do
+      setup_ec2_metadata_stubs
+
+      d = create_driver(<<-EOC)
+        type datadog_log
+        api_key myapikey
+        service myservice
+        source mysource
+      EOC
+
+      assert_equal 'myapikey', d.instance.api_key
+      assert_equal 'myservice', d.instance.service
+      assert_equal 'mysource', d.instance.source
+    end
+  end
+
+  def test_write
+    new_stub_context do
+      setup_ec2_metadata_stubs
+
+      timestamp_str = '2006-01-02T15:04:05.000000+00:00'
+      t = DateTime.rfc3339(timestamp_str).to_time
+      time = Fluent::EventTime.from_time(t)
+      d = create_driver(<<-EOC)
+        type datadog_log
+        api_key myapikey
+        service myservice
+        source mysource
+        source_category mysourcecategory
+        logset mylogset
+        log_level debug
+      EOC
+      conn = StubConn.new
+      fluentd_tag = 'mytag'
+      Net::TCPClient.stubs(:new)
+        .with(server: ':10516', ssl: true)
+        .returns(conn)
+      d.run(default_tag: fluentd_tag) do
+        record = {
+          'log' => 'mymsg',
+          'kubernetes' => {
+            'namespace' => 'myns',
+            'pod_name' => 'mypod',
+            'container_name' => 'mycontainer',
+            'labels' => {
+              'k8s-app' => 'myapp'
+            }
+          }
+        }
+        d.feed(time, record)
+      end
+
+      # fail d.logs.inspect
+      assert_equal(1, d.logs.count { |l| l =~ /Sent payload to Datadog/ })
+      assert_equal(1, conn.sent.size)
+      # rubocop:disable LineLength
+      payload = %(myapikey/mylogset <46>0 2006-01-02T15:04:05.000000+00:00 i-81c16767 myservice - - [dd ddsource="mysource"][dd ddsourcecategory="mysourcecategory"][dd ddtags="pod_name=mypod,container_name=mycontainer,kube_k8s-app=myapp,host=i-81c16767,zone=aws:us-west-2b,aws_account_id=123456789012"] mymsg\n)
+      # rubocop:enable LineLength
+      assert_equal(payload, conn.sent.first)
+    end
+  end
+
+  def test_prometheus_metrics
+    new_stub_context do
+      setup_ec2_metadata_stubs
+      timestamp_str = '2006-01-02T15:04:05.000000+00:00'
+      t = DateTime.rfc3339(timestamp_str).to_time
+      time = Fluent::EventTime.from_time(t)
+      [
+        # Single successful request.
+        [false, 0, 1, 1, [1, 0, 1, 0, 0]],
+        # Several successful requests.
+        [false, 0, 2, 1, [2, 0, 2, 0, 0]]
+      ].each do |_should_fail, _code, request_count, entry_count, metric_values|
+        setup_prometheus
+        (1..request_count).each do
+          d = create_driver(<<-EOC)
+            type datadog_log
+            api_key myapikey
+            service myservice
+            source mysource
+            source_category mysourcecategory
+            logset mylogset
+            log_level debug
+            enable_monitoring true
+          EOC
+          conn = StubConn.new
+          Net::TCPClient.stubs(:new)
+            .with(server: ':10516', ssl: true)
+            .returns(conn)
+          d.run(default_tag: 'mytag') do
+            (1..entry_count).each do |i|
+              d.feed time, 'message' => log_entry(i.to_s)
+            end
+          end
+        end
+        successful_requests_count, failed_requests_count,
+          ingested_entries_count, dropped_entries_count,
+          retried_entries_count = metric_values
+        assert_prometheus_metric_value(:datadog_successful_requests_count,
+                                       successful_requests_count)
+        assert_prometheus_metric_value(:datadog_failed_requests_count,
+                                       failed_requests_count)
+        assert_prometheus_metric_value(:datadog_ingested_entries_count,
+                                       ingested_entries_count)
+        assert_prometheus_metric_value(:datadog_dropped_entries_count,
+                                       dropped_entries_count)
+        assert_prometheus_metric_value(:datadog_retried_entries_count,
+                                       retried_entries_count)
+      end
+    end
+  end
+
+  def test_struct_payload_non_utf8_log
+    # d.emit('msg' => log_entry(0),
+    #        'normal_key' => "test#{non_utf8_character}non utf8",
+    #        "non_utf8#{non_utf8_character}key" => 5000,
+    #        'nested_struct' => { "non_utf8#{non_utf8_character}key" => \
+    #                             "test#{non_utf8_character}non utf8" },
+    #        'null_field' => nil)
+  end
+
+  class StubConn
+    attr_reader :sent
+
+    def initialize
+      @sent = []
+    end
+
+    def write(payload)
+      @sent << payload
+    end
+
+    def close
+    end
+  end
+
+  private
+
+  # Use the right single quotation mark as the sample non-utf8 character.
+  def non_utf8_character
+    [0x92].pack('C*')
+  end
+
+  # For an optional field with default values, Protobuf omits the field when it
+  # is deserialized to json. So we need to add an extra check for gRPC which
+  # uses Protobuf.
+  #
+  # An optional block can be passed in if we need to assert something other than
+  # a plain equal. e.g. assert_in_delta.
+  def assert_equal_with_default(field, expected_value, default_value, entry)
+    if expected_value == default_value
+      assert_nil field
+    elsif block_given?
+      yield
+    else
+      assert_equal expected_value, field, entry
+    end
+  end
+
+  # Get the fields of the payload.
+  def get_fields(payload)
+    payload['fields']
+  end
+
+  # Get the value of a struct field.
+  def get_struct(field)
+    field['structValue']
+  end
+
+  # Get the value of a string field.
+  def get_string(field)
+    field['stringValue']
+  end
+
+  # Get the value of a number field.
+  def get_number(field)
+    field['numberValue']
+  end
+
+  # The null value.
+  def null_value
+    { 'nullValue' => 'NULL_VALUE' }
+  end
+end