fluent-plugin-google-cloud 0.8.5 → 0.10.0

data/test/plugin/data/google-fluentd-baseline.conf

@@ -0,0 +1,24 @@
+<source>
+  @type syslog
+  port 514
+  protocol_type tcp
+  bind 127.0.0.1
+  tag syslog
+</source>
+
+<source>
+  @type tail
+  path /var/log/apache*/access.log,/var/log/apache*/access_log,/var/log/httpd/access.log,/var/log/httpd/access_log
+  pos_file /var/lib/google-fluentd/pos/apache-access.pos
+  tag apache-access
+</source>
+
+<filter **>
+  @type add_insert_ids
+</filter>
+
+<match **>
+  @type google_cloud
+  adjust_invalid_timestamps    true
+  autoformat_stackdriver_trace true
+</match>

data/test/plugin/data/google-fluentd-custom.conf

@@ -0,0 +1,40 @@
+<source>
+  @type syslog
+  port 514
+  protocol_type tcp
+  bind 127.0.0.1
+  tag syslog
+</source>
+
+<source>
+  @type tail
+  path /var/log/apache*/access.log,/var/log/apache*/access_log,/var/log/httpd/access.log,/var/log/httpd/access_log
+  pos_file /var/lib/google-fluentd/pos/apache-access.pos
+  tag apache-access
+</source>
+
+<filter **>
+  @type add_insert_ids
+</filter>
+
+<match **>
+  @type google_cloud
+  adjust_invalid_timestamps    true
+  autoformat_stackdriver_trace false
+  coerce_to_utf8               true
+</match>
+
+<filter **>
+  @type some_custom_filter
+</filter>
+
+<filter **>
+  @type record_transformer
+  <record>
+    host_param "#{Socket.gethostname}"
+  </record>
+</filter>
+
+<match "app.#{ENV['FLUENTD_TAG']}">
+  @type stdout
+</match>

data/test/plugin/test_driver.rb

@@ -16,19 +16,6 @@ require 'fluent/engine'
 require 'fluent/event'
 require 'fluent/test/input_test'
 
-module Fluent
-  module Test
-    # rubocop:disable Style/ClassVars
-    class BufferedOutputTestDriver < InputTestDriver
-      @@run_method = BufferedOutputTestDriver.instance_method(:run)
-      def run(num_waits = 0)
-        @@run_method.bind(self).call(num_waits)
-      end
-    end
-    # rubocop:enable Style/ClassVars
-  end
-end
-
 module Fluent
   module Test
     # Similar to the standard BufferedOutputTestDriver, but allows multiple tags
@@ -46,7 +33,7 @@ module Fluent
         self
       end
 
-      def run(num_waits = 0)
+      def run(num_waits = 10)
         result = nil
         super(num_waits) do
           chunk = @instance.buffer.generate_chunk(

data/test/plugin/test_filter_analyze_config.rb

@@ -0,0 +1,200 @@
+# Copyright 2020 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require_relative '../helper'
+require_relative 'asserts'
+require_relative 'constants'
+require_relative 'utils'
+
+require 'fluent/test/driver/filter'
+require 'fluent/plugin/filter_analyze_config'
+
+# Unit tests for filter_analyze_config plugin.
+class FilterAnalyzeConfigTest < Test::Unit::TestCase
+  include Asserts
+  include Constants
+  include Fluent::AnalyzeConfigFilter::Constants
+  include Utils
+
+  APPLICATION_DEFAULT_CONFIG = ''.freeze
+
+  def setup
+    Fluent::Test.setup
+    delete_env_vars
+  end
+
+  def test_config_file_does_not_exist
+    # By default, the FilterTestDriver.new does not set up a config file at:
+    # /etc/google-fluentd/google-fluentd.conf. The plugin should still proceed.
+    create_driver
+    # No exceptions were thrown.
+  end
+
+  def test_analyze_config
+    setup_auth_stubs('https://oauth2.googleapis.com/token')
+    setup_gce_metadata_stubs
+    [
+      [CONFIG_ANALYZE_CONFIG_PROMETHEUS,
+       method(:assert_prometheus_metric_value)],
+      [CONFIG_ANALYZE_CONFIG_OPENCENSUS,
+       method(:assert_opencensus_metric_value)]
+    ].each do |config, assert_metric_value|
+      clear_metrics
+      create_driver(config)
+
+      # Default plugins, with default config.
+      assert_metric_value.call(
+        :stackdriver_enabled_plugins,
+        1,
+        plugin_name: 'source/syslog/tcp',
+        is_default_plugin: true,
+        has_default_value: true,
+        has_ruby_snippet: false)
+      assert_metric_value.call(
+        :stackdriver_enabled_plugins,
+        1,
+        plugin_name: 'source/tail/apache-access',
+        is_default_plugin: true,
+        has_default_value: true,
+        has_ruby_snippet: false)
+      assert_metric_value.call(
+        :stackdriver_enabled_plugins,
+        1,
+        plugin_name: 'filter/add_insert_ids',
+        is_default_plugin: true,
+        has_default_value: true,
+        has_ruby_snippet: false)
+
+      # Default plugins, with custom config.
+      assert_metric_value.call(
+        :stackdriver_enabled_plugins,
+        1,
+        plugin_name: 'match/google_cloud',
+        is_default_plugin: true,
+        has_default_value: false,
+        has_ruby_snippet: false)
+
+      # Custom plugins, some with embedded Ruby.
+      assert_metric_value.call(
+        :stackdriver_enabled_plugins,
+        1,
+        plugin_name: 'filter',
+        is_default_plugin: false,
+        has_default_value: false,
+        has_ruby_snippet: false)
+      assert_metric_value.call(
+        :stackdriver_enabled_plugins,
+        1,
+        plugin_name: 'filter/record_transformer',
+        is_default_plugin: false,
+        has_default_value: false,
+        has_ruby_snippet: true)
+      assert_metric_value.call(
+        :stackdriver_enabled_plugins,
+        1,
+        plugin_name: 'match/stdout',
+        is_default_plugin: false,
+        has_default_value: false,
+        has_ruby_snippet: true)
+
+      # For out_google_cloud, 3 params are present.
+      assert_metric_value.call(
+        :stackdriver_config_usage,
+        1,
+        plugin_name: 'google_cloud',
+        param: 'adjust_invalid_timestamps',
+        is_present: true,
+        has_default_value: true)
+      assert_metric_value.call(
+        :stackdriver_config_usage,
+        1,
+        plugin_name: 'google_cloud',
+        param: 'autoformat_stackdriver_trace',
+        is_present: true,
+        has_default_value: false)
+      assert_metric_value.call(
+        :stackdriver_config_usage,
+        1,
+        plugin_name: 'google_cloud',
+        param: 'coerce_to_utf8',
+        is_present: true,
+        has_default_value: false)
+      # The remaining "google_cloud" params are not present.
+      # The are no params for "detect_exceptions".
+      %w(
+        auth_method
+        detect_json
+        enable_monitoring
+        gcm_service_address
+        grpc_compression_algorithm
+        http_request_key
+        insert_id_key
+        label_map
+        labels
+        labels_key
+        logging_api_url
+        monitoring_type
+        non_utf8_replacement_string
+        operation_key
+        private_key_email
+        private_key_passphrase
+        private_key_path
+        project_id
+        source_location_key
+        span_id_key
+        statusz_port
+        trace_key
+        trace_sampled_key
+        use_grpc
+        use_metadata_service
+        vm_id
+        vm_name
+        zone
+      ).each do |p|
+        assert_metric_value.call(
+          :stackdriver_config_usage,
+          1,
+          plugin_name: 'google_cloud',
+          param: p,
+          is_present: false,
+          has_default_value: false)
+      end
+
+      # We also export values for the bools.
+      assert_metric_value.call(
+        :stackdriver_config_bool_values,
+        1,
+        plugin_name: 'google_cloud',
+        param: 'adjust_invalid_timestamps',
+        value: true)
+      assert_metric_value.call(
+        :stackdriver_config_bool_values,
+        1,
+        plugin_name: 'google_cloud',
+        param: 'autoformat_stackdriver_trace',
+        value: false)
+      assert_metric_value.call(
+        :stackdriver_config_bool_values,
+        1,
+        plugin_name: 'google_cloud',
+        param: 'coerce_to_utf8',
+        value: true)
+    end
+  end
+
+  def create_driver(conf = APPLICATION_DEFAULT_CONFIG)
+    Fluent::Test::FilterTestDriver.new(
+      Fluent::AnalyzeConfigFilter).configure(conf, true)
+  end
+end

data/test/plugin/utils.rb

@@ -0,0 +1,147 @@
+# Copyright 2020 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require_relative 'constants'
+
+require 'prometheus/client'
+require 'webmock/test_unit'
+
+module Utils
+  include Constants
+
+  def delete_env_vars
+    # delete environment variables that googleauth uses to find credentials.
+    ENV.delete(CREDENTIALS_PATH_ENV_VAR)
+    # service account env.
+    ENV.delete(PRIVATE_KEY_VAR)
+    ENV.delete(CLIENT_EMAIL_VAR)
+    ENV.delete(PROJECT_ID_VAR)
+    # authorized_user env.
+    ENV.delete(CLIENT_ID_VAR)
+    ENV.delete(CLIENT_SECRET_VAR)
+    ENV.delete(REFRESH_TOKEN_VAR)
+    # home var, which is used to find $HOME/.gcloud/...
+    ENV.delete('HOME')
+  end
+
+  def stub_metadata_request(metadata_path, response_body)
+    stub_request(:get, 'http://169.254.169.254/computeMetadata/v1/' +
+                 metadata_path)
+      .to_return(body: response_body, status: 200,
+                 headers: { 'Content-Length' => response_body.length })
+  end
+
+  def setup_no_metadata_service_stubs
+    # Simulate a machine with no metadata service present
+    stub_request(:any, %r{http://169.254.169.254/.*})
+      .to_raise(Errno::EHOSTUNREACH)
+  end
+
+  def setup_gce_metadata_stubs
+    # Stub the root, used for platform detection by the plugin and 'googleauth'.
+    stub_request(:get, 'http://169.254.169.254')
+      .to_return(status: 200, headers: { 'Metadata-Flavor' => 'Google' })
+
+    # Create stubs for all the GCE metadata lookups the agent needs to make.
+    stub_metadata_request('project/project-id', PROJECT_ID)
+    stub_metadata_request('instance/zone', FULLY_QUALIFIED_ZONE)
+    stub_metadata_request('instance/id', VM_ID)
+    stub_metadata_request('instance/attributes/',
+                          "attribute1\nattribute2\nattribute3")
+
+    # Used by 'googleauth' to fetch the default service account credentials.
+    stub_request(:get, 'http://169.254.169.254/computeMetadata/v1/' \
+                 'instance/service-accounts/default/token')
+      .to_return(body: %({"access_token": "#{FAKE_AUTH_TOKEN}"}),
+                 status: 200,
+                 headers: { 'Content-Length' => FAKE_AUTH_TOKEN.length,
+                            'Content-Type' => 'application/json' })
+  end
+
+  def setup_ec2_metadata_stubs
+    # Stub the root, used for platform detection.
+    stub_request(:get, 'http://169.254.169.254')
+      .to_return(status: 200, headers: { 'Server' => 'EC2ws' })
+
+    # Stub the identity document lookup made by the agent.
+    stub_request(:get, 'http://169.254.169.254/latest/dynamic/' \
+                 'instance-identity/document')
+      .to_return(body: EC2_IDENTITY_DOCUMENT, status: 200,
+                 headers: { 'Content-Length' => EC2_IDENTITY_DOCUMENT.length })
+  end
+
+  def setup_auth_stubs(base_url)
+    # Used when loading credentials from a JSON file.
+    stub_request(:post, base_url)
+      .with(body: hash_including(grant_type: AUTH_GRANT_TYPE))
+      .to_return(body: %({"access_token": "#{FAKE_AUTH_TOKEN}"}),
+                 status: 200,
+                 headers: { 'Content-Length' => FAKE_AUTH_TOKEN.length,
+                            'Content-Type' => 'application/json' })
+
+    stub_request(:post, base_url)
+      .with(body: hash_including(grant_type: 'refresh_token'))
+      .to_return(body: %({"access_token": "#{FAKE_AUTH_TOKEN}"}),
+                 status: 200,
+                 headers: { 'Content-Length' => FAKE_AUTH_TOKEN.length,
+                            'Content-Type' => 'application/json' })
+  end
+
+  def setup_managed_vm_metadata_stubs
+    stub_metadata_request(
+      'instance/attributes/',
+      "attribute1\ngae_backend_name\ngae_backend_version\nlast_attribute")
+    stub_metadata_request('instance/attributes/gae_backend_name',
+                          MANAGED_VM_BACKEND_NAME)
+    stub_metadata_request('instance/attributes/gae_backend_version',
+                          MANAGED_VM_BACKEND_VERSION)
+  end
+
+  def setup_k8s_metadata_stubs(should_respond = true)
+    if should_respond
+      stub_metadata_request(
+        'instance/attributes/',
+        "attribute1\ncluster-location\ncluster-name\nlast_attribute")
+      stub_metadata_request('instance/attributes/cluster-location',
+                            K8S_LOCATION2)
+      stub_metadata_request('instance/attributes/cluster-name',
+                            K8S_CLUSTER_NAME)
+    else
+      ['cluster-location', 'cluster-name'].each do |metadata_name|
+        stub_request(:get, %r{.*instance/attributes/#{metadata_name}.*})
+          .to_return(status: 404,
+                     body: 'The requested URL /computeMetadata/v1/instance/' \
+                           "attributes/#{metadata_name} was not found on this" \
+                           ' server.')
+      end
+    end
+  end
+
+  def setup_dataproc_metadata_stubs
+    stub_metadata_request(
+      'instance/attributes/',
+      "attribute1\ndataproc-cluster-uuid\ndataproc-cluster-name")
+    stub_metadata_request('instance/attributes/dataproc-cluster-name',
+                          DATAPROC_CLUSTER_NAME)
+    stub_metadata_request('instance/attributes/dataproc-cluster-uuid',
+                          DATAPROC_CLUSTER_UUID)
+    stub_metadata_request('instance/attributes/dataproc-region',
+                          DATAPROC_REGION)
+  end
+
+  def clear_metrics
+    Prometheus::Client.registry.instance_variable_set('@metrics', {})
+    OpenCensus::Stats.ensure_recorder.clear_stats
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-google-cloud
 version: !ruby/object:Gem::Version
-  version: 0.8.5
+  version: 0.10.0
 platform: ruby
 authors:
 - Stackdriver Agents Team
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-13 00:00:00.000000000 Z
+date: 2020-08-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.7.4
+        version: 1.11.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.7.4
+        version: 1.11.2
 - !ruby/object:Gem::Dependency
   name: googleapis-common-protos
   requirement: !ruby/object:Gem::Requirement
@@ -86,28 +86,28 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 3.9.0
+        version: 3.12.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 3.9.0
+        version: 3.12.2
 - !ruby/object:Gem::Dependency
   name: grpc
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.22.0
+        version: 1.30.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.22.0
+        version: 1.30.2
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
@@ -266,23 +266,30 @@ files:
 - README.rdoc
 - Rakefile
 - fluent-plugin-google-cloud.gemspec
+- lib/fluent/plugin/common.rb
 - lib/fluent/plugin/filter_add_insert_ids.rb
+- lib/fluent/plugin/filter_analyze_config.rb
 - lib/fluent/plugin/in_object_space_dump.rb
 - lib/fluent/plugin/monitoring.rb
 - lib/fluent/plugin/out_google_cloud.rb
 - lib/fluent/plugin/statusz.rb
 - test/helper.rb
+- test/plugin/asserts.rb
 - test/plugin/base_test.rb
 - test/plugin/constants.rb
 - test/plugin/data/c31e573fd7f62ed495c9ca3821a5a85cb036dee1-privatekey.p12
 - test/plugin/data/credentials.json
+- test/plugin/data/google-fluentd-baseline.conf
+- test/plugin/data/google-fluentd-custom.conf
 - test/plugin/data/iam-credentials.json
 - test/plugin/data/invalid_credentials.json
 - test/plugin/data/new-style-credentials.json
 - test/plugin/test_driver.rb
 - test/plugin/test_filter_add_insert_ids.rb
+- test/plugin/test_filter_analyze_config.rb
 - test/plugin/test_out_google_cloud.rb
 - test/plugin/test_out_google_cloud_grpc.rb
+- test/plugin/utils.rb
 homepage: https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud
 licenses:
 - Apache-2.0
@@ -308,14 +315,19 @@ specification_version: 4
 summary: fluentd plugins for the Stackdriver Logging API
 test_files:
 - test/helper.rb
+- test/plugin/asserts.rb
 - test/plugin/base_test.rb
 - test/plugin/constants.rb
 - test/plugin/data/c31e573fd7f62ed495c9ca3821a5a85cb036dee1-privatekey.p12
 - test/plugin/data/credentials.json
+- test/plugin/data/google-fluentd-baseline.conf
+- test/plugin/data/google-fluentd-custom.conf
 - test/plugin/data/iam-credentials.json
 - test/plugin/data/invalid_credentials.json
 - test/plugin/data/new-style-credentials.json
 - test/plugin/test_driver.rb
 - test/plugin/test_filter_add_insert_ids.rb
+- test/plugin/test_filter_analyze_config.rb
 - test/plugin/test_out_google_cloud.rb
 - test/plugin/test_out_google_cloud_grpc.rb
+- test/plugin/utils.rb