fluent-plugin-google-cloud 0.8.7 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 851c496e93becdfb6a735f9c71aceecfdeccfff14519ce8311bb3dd42c7ed382
-  data.tar.gz: d893b865df5a2b10bac43c65381abe1b95e27333ff77ac45c6cb988c2f446036
+  metadata.gz: 2969a92cd254d0b7e3a8fa3fcf891df2ec8cbaa3b94c5d683db3d6c43ee2d11f
+  data.tar.gz: 78b2418688ff2ac6a939ebf1d19d81e61a40a1cea7793c35c2044ac6969f44ac
 SHA512:
-  metadata.gz: 6f840e9c06e995be5dd4abd0dcd9d04042c6d78507896fc842ab2b1fb8cb82eac4535fdb6259656cc703e99955e074e64c1ec0292c31d77647cbc95d54e7b919
-  data.tar.gz: c1573f562d3691a83b8b3dd2b8e8a0ed1585edac5f67df6179bc8c62af4a3728ce97b90be8827008718921069806eb6e0d2e81fd5c7f6de4c6603a1c2738774e
+  metadata.gz: bd4ccb079e3f1af0d0d9efb8d7165a04dcbfdcb2b21f8471a0b80f5c157b4d4e5a968f63e0c6a7c92f746fd34c2d46319037b883dec6abd301df78771ca201ed
+  data.tar.gz: bcf8016d3dbbd7fbfb6c2cc1bc2de490e89c61c1a51fe7c91324034fcca2a96d8b55059d197bbbe2ad3acfabfcb632ff3e917c7f54aa3061d36b6f0c2b9f1959

data/Gemfile.lock

@@ -1,14 +1,14 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-google-cloud (0.8.7)
+    fluent-plugin-google-cloud (0.9.0)
       fluentd (= 1.7.4)
       google-api-client (= 0.30.8)
       google-cloud-logging (= 1.6.6)
-      google-protobuf (= 3.9.0)
+      google-protobuf (= 3.12.2)
       googleapis-common-protos (= 1.3.9)
       googleauth (= 0.9.0)
-      grpc (= 1.22.0)
+      grpc (= 1.30.1)
       json (= 2.2.0)
       opencensus (= 0.5.0)
       opencensus-stackdriver (= 0.3.2)
@@ -29,7 +29,7 @@ GEM
       tins (~> 1.6)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
-    declarative (0.0.10)
+    declarative (0.0.20)
     declarative-option (0.1.0)
     dig_rb (1.0.1)
     docile (1.3.2)
@@ -57,7 +57,7 @@ GEM
     google-cloud-core (1.5.0)
       google-cloud-env (~> 1.0)
       google-cloud-errors (~> 1.0)
-    google-cloud-env (1.3.2)
+    google-cloud-env (1.3.3)
       faraday (>= 0.17.3, < 2.0)
     google-cloud-errors (1.0.1)
     google-cloud-logging (1.6.6)
@@ -66,27 +66,30 @@ GEM
       google-gax (~> 1.7)
       googleapis-common-protos-types (>= 1.0.2)
       stackdriver-core (~> 1.3)
-    google-cloud-monitoring (0.32.0)
-      google-gax (~> 1.7)
-      googleapis-common-protos-types (>= 1.0.2)
-    google-cloud-trace (0.35.0)
+    google-cloud-monitoring (0.38.0)
+      google-gax (~> 1.8)
+      googleapis-common-protos (>= 1.3.9, < 2.0)
+      googleapis-common-protos-types (>= 1.0.4, < 2.0)
+    google-cloud-trace (0.39.0)
       concurrent-ruby (~> 1.1)
       google-cloud-core (~> 1.2)
-      google-gax (~> 1.7)
+      google-gax (~> 1.8)
+      googleapis-common-protos (>= 1.3.9, < 2.0)
+      googleapis-common-protos-types (>= 1.0.4, < 2.0)
       stackdriver-core (~> 1.3)
-    google-gax (1.7.1)
-      google-protobuf (~> 3.2)
-      googleapis-common-protos (>= 1.3.5, < 2.0)
-      googleauth (>= 0.6.2, < 0.10.0)
-      grpc (>= 1.7.2, < 2.0)
+    google-gax (1.8.1)
+      google-protobuf (~> 3.9)
+      googleapis-common-protos (>= 1.3.9, < 2.0)
+      googleauth (~> 0.9)
+      grpc (~> 1.24)
       rly (~> 0.2.3)
-    google-protobuf (3.9.0)
+    google-protobuf (3.12.2)
     googleapis-common-protos (1.3.9)
       google-protobuf (~> 3.0)
       googleapis-common-protos-types (~> 1.0)
       grpc (~> 1.0)
-    googleapis-common-protos-types (1.0.4)
-      google-protobuf (~> 3.0)
+    googleapis-common-protos-types (1.0.5)
+      google-protobuf (~> 3.11)
     googleauth (0.9.0)
       faraday (~> 0.12)
       jwt (>= 1.4, < 3.0)
@@ -94,8 +97,8 @@ GEM
       multi_json (~> 1.11)
       os (>= 0.9, < 2.0)
       signet (~> 0.7)
-    grpc (1.22.0)
-      google-protobuf (~> 3.8)
+    grpc (1.30.1)
+      google-protobuf (~> 3.12)
       googleapis-common-protos-types (~> 1.0)
     hashdiff (1.0.1)
     http_parser.rb (0.6.0)
@@ -108,7 +111,7 @@ GEM
     mocha (1.9.0)
       metaclass (~> 0.0.1)
     msgpack (1.3.3)
-    multi_json (1.14.1)
+    multi_json (1.15.0)
     multipart-post (2.1.1)
     opencensus (0.5.0)
     opencensus-stackdriver (0.3.2)
@@ -117,8 +120,8 @@ GEM
       google-cloud-trace (~> 0.35)
       opencensus (~> 0.5)
     os (1.1.0)
-    parser (2.7.1.3)
-      ast (~> 2.4.0)
+    parser (2.7.1.4)
+      ast (~> 2.4.1)
     power_assert (1.2.0)
     powerpack (0.1.2)
     prometheus-client (0.9.0)

data/fluent-plugin-google-cloud.gemspec

@@ -10,7 +10,7 @@ eos
   gem.homepage      =
     'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
   gem.license       = 'Apache-2.0'
-  gem.version       = '0.8.7'
+  gem.version       = '0.9.0'
   gem.authors       = ['Stackdriver Agents Team']
   gem.email         = ['stackdriver-agents@google.com']
   gem.required_ruby_version = Gem::Requirement.new('>= 2.2')
@@ -24,8 +24,8 @@ eos
   gem.add_runtime_dependency 'googleauth', '0.9.0'
   gem.add_runtime_dependency 'google-api-client', '0.30.8'
   gem.add_runtime_dependency 'google-cloud-logging', '1.6.6'
-  gem.add_runtime_dependency 'google-protobuf', '3.9.0'
-  gem.add_runtime_dependency 'grpc', '1.22.0'
+  gem.add_runtime_dependency 'google-protobuf', '3.12.2'
+  gem.add_runtime_dependency 'grpc', '1.30.1'
   gem.add_runtime_dependency 'json', '2.2.0'
   gem.add_runtime_dependency 'opencensus', '0.5.0'
   gem.add_runtime_dependency 'opencensus-stackdriver', '0.3.2'

data/lib/fluent/plugin/common.rb

@@ -219,6 +219,12 @@ module Common
       @log.error 'Failed to obtain location: ', error: e
     end
 
+    # Create a monitored resource from type and labels.
+    def create_monitored_resource(type, labels)
+      Google::Apis::LoggingV2::MonitoredResource.new(
+        type: type, labels: labels.to_h)
+    end
+
     # Retrieve monitored resource via the legacy way.
     #
     # Note: This is just a failover plan if we fail to get metadata from
@@ -226,13 +232,12 @@ module Common
     # returns.
     def determine_agent_level_monitored_resource_via_legacy(
           platform, subservice_name, detect_subservice, vm_id, zone)
-      resource = Google::Apis::LoggingV2::MonitoredResource.new(
-        labels: {})
-      resource.type = determine_agent_level_monitored_resource_type(
+      resource_type = determine_agent_level_monitored_resource_type(
         platform, subservice_name, detect_subservice)
-      resource.labels = determine_agent_level_monitored_resource_labels(
-        platform, resource.type, vm_id, zone)
-      resource
+      create_monitored_resource(
+        resource_type,
+        determine_agent_level_monitored_resource_labels(
+          platform, resource_type, vm_id, zone))
     end
 
     # Determine agent level monitored resource type.

data/lib/fluent/plugin/filter_analyze_config.rb

@@ -17,6 +17,7 @@ require 'fluent/config'
 require 'fluent/config/v1_parser'
 require 'set'
 
+require_relative 'common'
 require_relative 'monitoring'
 
 module Fluent
@@ -122,16 +123,33 @@ module Fluent
 
     include self::Constants
 
+    # Disable this warning to conform to fluentd config_param conventions.
+    # rubocop:disable Style/HashSyntax
+
     # The root configuration file of google-fluentd package.
     # This only applies to Linux.
     config_param :google_fluentd_config_path,
                  :string,
-                 default: '/etc/google-fluentd/google-fluentd.conf'
+                 :default => '/etc/google-fluentd/google-fluentd.conf'
     # Baseline configuration for comparing with local
     # customizations.
     config_param :google_fluentd_baseline_config_path,
                  :string,
-                 default: '/etc/google-fluentd/baseline/google-fluentd.conf'
+                 :default => '/etc/google-fluentd/baseline/google-fluentd.conf'
+
+    # What system to use when collecting metrics. Possible values are:
+    #   - 'prometheus', in this case default registry in the Prometheus
+    #     client library is used, without actually exposing the endpoint
+    #     to serve metrics in the Prometheus format.
+    #    - any other value will result in the absence of metrics.
+    config_param :monitoring_type, :string,
+                 :default => Monitoring::PrometheusMonitoringRegistry.name
+
+    # Override for the Google Cloud Monitoring service hostname, or
+    # `nil` to leave as the default.
+    config_param :gcm_service_address, :string, :default => nil
+
+    # rubocop:enable Style/HashSyntax
 
     def start
       super
@@ -196,9 +214,29 @@ module Fluent
           " #{@google_fluentd_baseline_config_path}. " \
           'google-fluentd Analyzing configuration.')
 
-        # TODO: Add OpenCensus support.
+        utils = Common::Utils.new(@log)
+        platform = utils.detect_platform(true)
+        project_id = utils.get_project_id(platform, nil)
+        vm_id = utils.get_vm_id(platform, nil)
+        zone = utils.get_location(platform, nil, true)
+
+        # All metadata parameters must now be set.
+        utils.check_required_metadata_variables(
+          platform, project_id, zone, vm_id)
+
+        # Retrieve monitored resource.
+        # Fail over to retrieve monitored resource via the legacy path if we
+        # fail to get it from Metadata Agent.
+        resource = utils.determine_agent_level_monitored_resource_via_legacy(
+          platform, nil, false, vm_id, zone)
+
+        unless Monitoring::MonitoringRegistryFactory.supports_monitoring_type(
+          @monitoring_type)
+          @log.warn "monitoring_type '#{@monitoring_type}' is unknown; "\
+                    'there will be no metrics'
+        end
         registry = Monitoring::MonitoringRegistryFactory.create(
-          Monitoring::PrometheusMonitoringRegistry.name, nil, nil, nil)
+          @monitoring_type, project_id, resource, @gcm_service_address)
 
         plugin_usage = registry.counter(
           :stackdriver_enabled_plugins,

data/lib/fluent/plugin/out_google_cloud.rb

@@ -370,10 +370,20 @@ module Fluent
     #   - 'prometheus', in this case default registry in the Prometheus
     #     client library is used, without actually exposing the endpoint
     #     to serve metrics in the Prometheus format.
-    #    - any other value will result in the absence of metrics.
+    #   - 'opencensus', in this case the OpenCensus implementation is
+    #     used to send metrics directly to Google Cloud Monitoring.
+    #   - any other value will result in the absence of metrics.
     config_param :monitoring_type, :string,
                  :default => Monitoring::PrometheusMonitoringRegistry.name
 
+    # The monitored resource to use for OpenCensus metrics. Only valid
+    # when monitoring_type is set to 'opencensus'. This value is a hash in
+    # the form:
+    # {"type":"gce_instance","labels":{"instance_id":"aaa","zone":"bbb"} (JSON)
+    # or type:gce_instance,labels.instance_id:aaa,labels.zone:bbb (Hash)
+    config_param :metrics_resource, :hash,
+                 :symbolize_keys => true, :default => nil
+
     # Whether to call metadata agent to retrieve monitored resource. This flag
     # is kept for backwards compatibility, and is no longer used.
     # TODO: Breaking change. Remove this flag in Logging Agent 2.0.0 release.
@@ -423,6 +433,7 @@ module Fluent
     attr_reader :vm_id
     attr_reader :resource
     attr_reader :common_labels
+    attr_reader :monitoring_resource
 
     def initialize
       super
@@ -493,6 +504,43 @@ module Fluent
       @resource ||= @utils.determine_agent_level_monitored_resource_via_legacy(
         @platform, @subservice_name, @detect_subservice, @vm_id, @zone)
 
+      if @metrics_resource
+        unless @metrics_resource[:type].is_a?(String)
+          raise Fluent::ConfigError,
+                'metrics_resource.type must be a string:' \
+                " #{@metrics_resource}."
+        end
+        if @metrics_resource.key?(:labels)
+          unless @metrics_resource[:labels].is_a?(Hash)
+            raise Fluent::ConfigError,
+                  'metrics_resource.labels must be a hash:' \
+                  " #{@metrics_resource}."
+          end
+          extra_keys = @metrics_resource.reject do |k, _|
+            k == :type || k == :labels
+          end
+          unless extra_keys.empty?
+            raise Fluent::ConfigError,
+                  "metrics_resource has unrecognized keys: #{extra_keys.keys}."
+          end
+        else
+          extra_keys = @metrics_resource.reject do |k, _|
+            k == :type || k.to_s.start_with?('labels.')
+          end
+          unless extra_keys.empty?
+            raise Fluent::ConfigError,
+                  "metrics_resource has unrecognized keys: #{extra_keys.keys}."
+          end
+          # Transform the Hash form of the metrics_resource config if necessary.
+          resource_type = @metrics_resource[:type]
+          resource_labels = @metrics_resource.each_with_object({}) \
+            do |(k, v), h|
+              h[k.to_s.sub('labels.', '')] = v if k.to_s.start_with? 'labels.'
+            end
+          @metrics_resource = { type: resource_type, labels: resource_labels }
+        end
+      end
+
       # If monitoring is enabled, register metrics in the default registry
       # and store metric objects for future use.
       if @enable_monitoring
@@ -501,9 +549,15 @@ module Fluent
           @log.warn "monitoring_type '#{@monitoring_type}' is unknown; "\
                     'there will be no metrics'
         end
+        if @metrics_resource
+          @monitoring_resource = @utils.create_monitored_resource(
+            @metrics_resource[:type], @metrics_resource[:labels])
+        else
+          @monitoring_resource = @resource
+        end
         @registry = Monitoring::MonitoringRegistryFactory
-                    .create(@monitoring_type, @project_id, @resource,
-                            @gcm_service_address)
+                    .create(@monitoring_type, @project_id,
+                            @monitoring_resource, @gcm_service_address)
         # Export metrics every 60 seconds.
         timer_execute(:export_metrics, 60) { @registry.export }
         # Uptime should be a gauge, but the metric definition is a counter and

data/test/plugin/asserts.rb

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+require 'prometheus/client'
+
 # Additional assert functions.
 module Asserts
   # For an optional field with default values, Protobuf omits the field when it

data/test/plugin/base_test.rb

@@ -19,11 +19,12 @@ Coveralls.wear!
 require 'google/apis'
 require 'helper'
 require 'mocha/test_unit'
-require 'webmock/test_unit'
 require 'prometheus/client'
+require 'webmock/test_unit'
 
 require_relative 'asserts'
 require_relative 'constants'
+require_relative 'utils'
 
 module Monitoring
   # Prevent OpenCensus from writing to the network.
@@ -38,21 +39,11 @@ end
 module BaseTest
   include Asserts
   include Constants
+  include Utils
 
   def setup
     Fluent::Test.setup
-    # delete environment variables that googleauth uses to find credentials.
-    ENV.delete(CREDENTIALS_PATH_ENV_VAR)
-    # service account env.
-    ENV.delete(PRIVATE_KEY_VAR)
-    ENV.delete(CLIENT_EMAIL_VAR)
-    ENV.delete(PROJECT_ID_VAR)
-    # authorized_user env.
-    ENV.delete(CLIENT_ID_VAR)
-    ENV.delete(CLIENT_SECRET_VAR)
-    ENV.delete(REFRESH_TOKEN_VAR)
-    # home var, which is used to find $HOME/.gcloud/...
-    ENV.delete('HOME')
+    delete_env_vars
 
     # Unregister Prometheus metrics.
     registry = Prometheus::Client.registry
@@ -62,7 +53,7 @@ module BaseTest
     registry.unregister(:stackdriver_dropped_entries_count)
     registry.unregister(:stackdriver_retried_entries_count)
 
-    setup_auth_stubs
+    setup_auth_stubs('https://www.googleapis.com/oauth2/v4/token')
     @logs_sent = []
   end
 
@@ -131,8 +122,8 @@ module BaseTest
 
   def test_configure_ignores_unknown_monitoring_type
     # Verify that driver creation succeeds when monitoring type is not
-    # "prometheus" (in which case, we simply don't record metrics),
-    # and that the counters are set to nil.
+    # "prometheus" or "opencensus" (in which case, we simply don't record
+    # metrics), and that the counters are set to nil.
     setup_gce_metadata_stubs
     create_driver(CONFIG_UNKNOWN_MONITORING_TYPE)
     assert_nil(Prometheus::Client.registry.get(
@@ -145,6 +136,67 @@ module BaseTest
                  :stackdriver_dropped_entries_count))
     assert_nil(Prometheus::Client.registry.get(
                  :stackdriver_retried_entries_count))
+    assert_nil(OpenCensus::Stats::MeasureRegistry.get(
+                 Monitoring::MetricTranslator.new(
+                   :stackdriver_successful_requests_count, {})))
+    assert_nil(OpenCensus::Stats::MeasureRegistry.get(
+                 Monitoring::MetricTranslator.new(
+                   :stackdriver_failed_requests_count, {})))
+    assert_nil(OpenCensus::Stats::MeasureRegistry.get(
+                 Monitoring::MetricTranslator.new(
+                   :stackdriver_ingested_entries_count, {})))
+    assert_nil(OpenCensus::Stats::MeasureRegistry.get(
+                 Monitoring::MetricTranslator.new(
+                   :stackdriver_dropped_entries_count, {})))
+    assert_nil(OpenCensus::Stats::MeasureRegistry.get(
+                 Monitoring::MetricTranslator.new(
+                   :stackdriver_retried_entries_count, {})))
+  end
+
+  def test_configure_uses_metrics_resource
+    setup_gce_metadata_stubs
+    [CONFIG_METRICS_RESOURCE_JSON,
+     CONFIG_METRICS_RESOURCE_HASH,
+     CONFIG_METRICS_RESOURCE_JSON_HASH
+    ].each_with_index do |config, index|
+      d = create_driver(config)
+      assert_equal 'custom_resource', d.instance.monitoring_resource.type, \
+                   "Index #{index}"
+      assert_equal '123', d.instance.monitoring_resource.labels['label1'], \
+                   "Index #{index}"
+      assert_equal 'abc', d.instance.monitoring_resource.labels['label2'], \
+                   "Index #{index}"
+      assert_true d.instance.instance_variable_get(:@enable_monitoring)
+      registry = d.instance.instance_variable_get(:@registry)
+      assert_not_nil registry
+      exporter = registry.instance_variable_get(:@exporter)
+      assert_equal 'custom_resource', exporter.resource_type, "Index #{index}"
+      assert_equal({ 'label1' => '123', 'label2' => 'abc' },
+                   exporter.resource_labels, "Index #{index}")
+    end
+  end
+
+  def test_configure_metrics_resource_validation
+    setup_gce_metadata_stubs
+    {
+      CONFIG_METRICS_RESOURCE_JSON_NO_TYPE => /type must be a string/,
+      CONFIG_METRICS_RESOURCE_JSON_BAD_LABELS => /labels must be a hash/,
+      CONFIG_METRICS_RESOURCE_JSON_BAD_KEYS =>
+        /unrecognized keys: \[:random\]/,
+      CONFIG_METRICS_RESOURCE_JSON_BAD_KEYS_LABELS =>
+        /unrecognized keys: \[:"labels\.random"\]/,
+      CONFIG_METRICS_RESOURCE_JSON_BAD_KEYS_NO_LABELS =>
+        /unrecognized keys: \[:random\]/
+    }.each_with_index do |(config, pattern), index|
+      begin
+        create_driver(config)
+        assert false,
+               "Invalid config at index #{index} should have raised an error."
+      rescue Fluent::ConfigError => error
+        assert error.message.match?(pattern), \
+               "Index #{index} failed: got #{error.message}."
+      end
+    end
   end
 
   def test_metadata_loading
@@ -1925,116 +1977,6 @@ module BaseTest
 
   private
 
-  def stub_metadata_request(metadata_path, response_body)
-    stub_request(:get, 'http://169.254.169.254/computeMetadata/v1/' +
-                 metadata_path)
-      .to_return(body: response_body, status: 200,
-                 headers: { 'Content-Length' => response_body.length })
-  end
-
-  def setup_no_metadata_service_stubs
-    # Simulate a machine with no metadata service present
-    stub_request(:any, %r{http://169.254.169.254/.*})
-      .to_raise(Errno::EHOSTUNREACH)
-  end
-
-  def setup_gce_metadata_stubs
-    # Stub the root, used for platform detection by the plugin and 'googleauth'.
-    stub_request(:get, 'http://169.254.169.254')
-      .to_return(status: 200, headers: { 'Metadata-Flavor' => 'Google' })
-
-    # Create stubs for all the GCE metadata lookups the agent needs to make.
-    stub_metadata_request('project/project-id', PROJECT_ID)
-    stub_metadata_request('instance/zone', FULLY_QUALIFIED_ZONE)
-    stub_metadata_request('instance/id', VM_ID)
-    stub_metadata_request('instance/attributes/',
-                          "attribute1\nattribute2\nattribute3")
-
-    # Used by 'googleauth' to fetch the default service account credentials.
-    stub_request(:get, 'http://169.254.169.254/computeMetadata/v1/' \
-                 'instance/service-accounts/default/token')
-      .to_return(body: %({"access_token": "#{FAKE_AUTH_TOKEN}"}),
-                 status: 200,
-                 headers: { 'Content-Length' => FAKE_AUTH_TOKEN.length,
-                            'Content-Type' => 'application/json' })
-  end
-
-  def setup_ec2_metadata_stubs
-    # Stub the root, used for platform detection.
-    stub_request(:get, 'http://169.254.169.254')
-      .to_return(status: 200, headers: { 'Server' => 'EC2ws' })
-
-    # Stub the identity document lookup made by the agent.
-    stub_request(:get, 'http://169.254.169.254/latest/dynamic/' \
-                 'instance-identity/document')
-      .to_return(body: EC2_IDENTITY_DOCUMENT, status: 200,
-                 headers: { 'Content-Length' => EC2_IDENTITY_DOCUMENT.length })
-  end
-
-  def setup_auth_stubs
-    # Used when loading credentials from a JSON file.
-    stub_request(:post, 'https://www.googleapis.com/oauth2/v4/token')
-      .with(body: hash_including(grant_type: AUTH_GRANT_TYPE))
-      .to_return(body: %({"access_token": "#{FAKE_AUTH_TOKEN}"}),
-                 status: 200,
-                 headers: { 'Content-Length' => FAKE_AUTH_TOKEN.length,
-                            'Content-Type' => 'application/json' })
-
-    stub_request(:post, 'https://www.googleapis.com/oauth2/v4/token')
-      .with(body: hash_including(grant_type: 'refresh_token'))
-      .to_return(body: %({"access_token": "#{FAKE_AUTH_TOKEN}"}),
-                 status: 200,
-                 headers: { 'Content-Length' => FAKE_AUTH_TOKEN.length,
-                            'Content-Type' => 'application/json' })
-  end
-
-  def setup_managed_vm_metadata_stubs
-    stub_metadata_request(
-      'instance/attributes/',
-      "attribute1\ngae_backend_name\ngae_backend_version\nlast_attribute")
-    stub_metadata_request('instance/attributes/gae_backend_name',
-                          MANAGED_VM_BACKEND_NAME)
-    stub_metadata_request('instance/attributes/gae_backend_version',
-                          MANAGED_VM_BACKEND_VERSION)
-  end
-
-  def setup_k8s_metadata_stubs(should_respond = true)
-    if should_respond
-      stub_metadata_request(
-        'instance/attributes/',
-        "attribute1\ncluster-location\ncluster-name\nlast_attribute")
-      stub_metadata_request('instance/attributes/cluster-location',
-                            K8S_LOCATION2)
-      stub_metadata_request('instance/attributes/cluster-name',
-                            K8S_CLUSTER_NAME)
-    else
-      ['cluster-location', 'cluster-name'].each do |metadata_name|
-        stub_request(:get, %r{.*instance/attributes/#{metadata_name}.*})
-          .to_return(status: 404,
-                     body: 'The requested URL /computeMetadata/v1/instance/' \
-                           "attributes/#{metadata_name} was not found on this" \
-                           ' server.')
-      end
-    end
-  end
-
-  def setup_dataproc_metadata_stubs
-    stub_metadata_request(
-      'instance/attributes/',
-      "attribute1\ndataproc-cluster-uuid\ndataproc-cluster-name")
-    stub_metadata_request('instance/attributes/dataproc-cluster-name',
-                          DATAPROC_CLUSTER_NAME)
-    stub_metadata_request('instance/attributes/dataproc-cluster-uuid',
-                          DATAPROC_CLUSTER_UUID)
-    stub_metadata_request('instance/attributes/dataproc-region',
-                          DATAPROC_REGION)
-  end
-
-  def clear_metrics
-    Prometheus::Client.registry.instance_variable_set('@metrics', {})
-    OpenCensus::Stats.ensure_recorder.clear_stats
-  end
-
   # Provide a stub context that initializes @logs_sent, executes the block and
   # resets WebMock at the end.
   def new_stub_context

data/test/plugin/constants.rb

@@ -433,6 +433,56 @@ module Constants
     monitoring_type    not_prometheus
   ).freeze
 
+  # rubocop:disable Metrics/LineLength
+  CONFIG_METRICS_RESOURCE_JSON = %(
+    enable_monitoring  true
+    monitoring_type    opencensus
+    metrics_resource   {"type":"custom_resource","labels":{"label1":"123","label2":"abc"}}
+  ).freeze
+
+  CONFIG_METRICS_RESOURCE_HASH = %(
+    enable_monitoring  true
+    monitoring_type    opencensus
+    metrics_resource   type:custom_resource, labels.label1:123, labels.label2:abc
+  ).freeze
+
+  CONFIG_METRICS_RESOURCE_JSON_HASH = %(
+    enable_monitoring  true
+    monitoring_type    opencensus
+    metrics_resource   {"type":"custom_resource","labels.label1":"123","labels.label2":"abc"}
+  ).freeze
+
+  CONFIG_METRICS_RESOURCE_JSON_NO_TYPE = %(
+    enable_monitoring  true
+    monitoring_type    opencensus
+    metrics_resource   {"labels":{"label1":"123","label2":"abc"}}
+  ).freeze
+
+  CONFIG_METRICS_RESOURCE_JSON_BAD_LABELS = %(
+    enable_monitoring  true
+    monitoring_type    opencensus
+    metrics_resource   {"type":"custom_resource","labels":"123"}
+  ).freeze
+
+  CONFIG_METRICS_RESOURCE_JSON_BAD_KEYS = %(
+    enable_monitoring  true
+    monitoring_type    opencensus
+    metrics_resource   {"type":"custom_resource","labels":{"label1":"123"},"random":"x"}
+  ).freeze
+
+  CONFIG_METRICS_RESOURCE_JSON_BAD_KEYS_LABELS = %(
+    enable_monitoring  true
+    monitoring_type    opencensus
+    metrics_resource   {"type":"custom_resource","labels":{"label1":"123"},"labels.random":"x"}
+  ).freeze
+
+  CONFIG_METRICS_RESOURCE_JSON_BAD_KEYS_NO_LABELS = %(
+    enable_monitoring  true
+    monitoring_type    opencensus
+    metrics_resource   {"type":"custom_resource","labels.label1":"123","random":"x"}
+  ).freeze
+  # rubocop:enable Metrics/LineLength
+
   # For statusz.
   CONFIG_STATUSZ = %(
     statusz_port                  5678
@@ -472,11 +522,19 @@ module Constants
   ).freeze
 
   # For analyze_config.
-  CONFIG_ANALYZE_CONFIG = %(
+  CONFIG_ANALYZE_CONFIG_PROMETHEUS = %(
+    google_fluentd_config_path          \
+      test/plugin/data/google-fluentd-custom.conf
+    google_fluentd_baseline_config_path \
+      test/plugin/data/google-fluentd-baseline.conf
+    monitoring_type prometheus
+  ).freeze
+  CONFIG_ANALYZE_CONFIG_OPENCENSUS = %(
     google_fluentd_config_path          \
       test/plugin/data/google-fluentd-custom.conf
     google_fluentd_baseline_config_path \
       test/plugin/data/google-fluentd-baseline.conf
+    monitoring_type opencensus
   ).freeze
 
   # Service configurations for various services.

data/test/plugin/test_driver.rb

@@ -16,19 +16,6 @@ require 'fluent/engine'
 require 'fluent/event'
 require 'fluent/test/input_test'
 
-module Fluent
-  module Test
-    # rubocop:disable Style/ClassVars
-    class BufferedOutputTestDriver < InputTestDriver
-      @@run_method = BufferedOutputTestDriver.instance_method(:run)
-      def run(num_waits = 0)
-        @@run_method.bind(self).call(num_waits)
-      end
-    end
-    # rubocop:enable Style/ClassVars
-  end
-end
-
 module Fluent
   module Test
     # Similar to the standard BufferedOutputTestDriver, but allows multiple tags
@@ -46,7 +33,7 @@ module Fluent
         self
       end
 
-      def run(num_waits = 0)
+      def run(num_waits = 10)
         result = nil
         super(num_waits) do
           chunk = @instance.buffer.generate_chunk(

data/test/plugin/test_filter_analyze_config.rb

@@ -15,6 +15,7 @@
 require_relative '../helper'
 require_relative 'asserts'
 require_relative 'constants'
+require_relative 'utils'
 
 require 'fluent/test/driver/filter'
 require 'fluent/plugin/filter_analyze_config'
@@ -24,11 +25,13 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
   include Asserts
   include Constants
   include Fluent::AnalyzeConfigFilter::Constants
+  include Utils
 
   APPLICATION_DEFAULT_CONFIG = ''.freeze
 
   def setup
     Fluent::Test.setup
+    delete_env_vars
   end
 
   def test_config_file_does_not_exist
@@ -39,145 +42,155 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
   end
 
   def test_analyze_config
-    create_driver(CONFIG_ANALYZE_CONFIG)
+    setup_auth_stubs('https://oauth2.googleapis.com/token')
+    setup_gce_metadata_stubs
+    [
+      [CONFIG_ANALYZE_CONFIG_PROMETHEUS,
+       method(:assert_prometheus_metric_value)],
+      [CONFIG_ANALYZE_CONFIG_OPENCENSUS,
+       method(:assert_opencensus_metric_value)]
+    ].each do |config, assert_metric_value|
+      clear_metrics
+      create_driver(config)
 
-    # Default plugins, with default config.
-    assert_prometheus_metric_value(
-      :stackdriver_enabled_plugins,
-      1,
-      plugin_name: 'source/syslog/tcp',
-      is_default_plugin: true,
-      has_default_value: true,
-      has_ruby_snippet: false)
-    assert_prometheus_metric_value(
-      :stackdriver_enabled_plugins,
-      1,
-      plugin_name: 'source/tail/apache-access',
-      is_default_plugin: true,
-      has_default_value: true,
-      has_ruby_snippet: false)
-    assert_prometheus_metric_value(
-      :stackdriver_enabled_plugins,
-      1,
-      plugin_name: 'filter/add_insert_ids',
-      is_default_plugin: true,
-      has_default_value: true,
-      has_ruby_snippet: false)
+      # Default plugins, with default config.
+      assert_metric_value.call(
+        :stackdriver_enabled_plugins,
+        1,
+        plugin_name: 'source/syslog/tcp',
+        is_default_plugin: true,
+        has_default_value: true,
+        has_ruby_snippet: false)
+      assert_metric_value.call(
+        :stackdriver_enabled_plugins,
+        1,
+        plugin_name: 'source/tail/apache-access',
+        is_default_plugin: true,
+        has_default_value: true,
+        has_ruby_snippet: false)
+      assert_metric_value.call(
+        :stackdriver_enabled_plugins,
+        1,
+        plugin_name: 'filter/add_insert_ids',
+        is_default_plugin: true,
+        has_default_value: true,
+        has_ruby_snippet: false)
 
-    # Default plugins, with custom config.
-    assert_prometheus_metric_value(
-      :stackdriver_enabled_plugins,
-      1,
-      plugin_name: 'match/google_cloud',
-      is_default_plugin: true,
-      has_default_value: false,
-      has_ruby_snippet: false)
+      # Default plugins, with custom config.
+      assert_metric_value.call(
+        :stackdriver_enabled_plugins,
+        1,
+        plugin_name: 'match/google_cloud',
+        is_default_plugin: true,
+        has_default_value: false,
+        has_ruby_snippet: false)
 
-    # Custom plugins, some with embedded Ruby.
-    assert_prometheus_metric_value(
-      :stackdriver_enabled_plugins,
-      1,
-      plugin_name: 'filter',
-      is_default_plugin: false,
-      has_default_value: false,
-      has_ruby_snippet: false)
-    assert_prometheus_metric_value(
-      :stackdriver_enabled_plugins,
-      1,
-      plugin_name: 'filter/record_transformer',
-      is_default_plugin: false,
-      has_default_value: false,
-      has_ruby_snippet: true)
-    assert_prometheus_metric_value(
-      :stackdriver_enabled_plugins,
-      1,
-      plugin_name: 'match/stdout',
-      is_default_plugin: false,
-      has_default_value: false,
-      has_ruby_snippet: true)
+      # Custom plugins, some with embedded Ruby.
+      assert_metric_value.call(
+        :stackdriver_enabled_plugins,
+        1,
+        plugin_name: 'filter',
+        is_default_plugin: false,
+        has_default_value: false,
+        has_ruby_snippet: false)
+      assert_metric_value.call(
+        :stackdriver_enabled_plugins,
+        1,
+        plugin_name: 'filter/record_transformer',
+        is_default_plugin: false,
+        has_default_value: false,
+        has_ruby_snippet: true)
+      assert_metric_value.call(
+        :stackdriver_enabled_plugins,
+        1,
+        plugin_name: 'match/stdout',
+        is_default_plugin: false,
+        has_default_value: false,
+        has_ruby_snippet: true)
 
-    # For out_google_cloud, 3 params are present.
-    assert_prometheus_metric_value(
-      :stackdriver_config_usage,
-      1,
-      plugin_name: 'google_cloud',
-      param: 'adjust_invalid_timestamps',
-      is_present: true,
-      has_default_value: true)
-    assert_prometheus_metric_value(
-      :stackdriver_config_usage,
-      1,
-      plugin_name: 'google_cloud',
-      param: 'autoformat_stackdriver_trace',
-      is_present: true,
-      has_default_value: false)
-    assert_prometheus_metric_value(
-      :stackdriver_config_usage,
-      1,
-      plugin_name: 'google_cloud',
-      param: 'coerce_to_utf8',
-      is_present: true,
-      has_default_value: false)
-    # The remaining "google_cloud" params are not present.
-    # The are no params for "detect_exceptions".
-    %w(
-      auth_method
-      detect_json
-      enable_monitoring
-      gcm_service_address
-      grpc_compression_algorithm
-      http_request_key
-      insert_id_key
-      label_map
-      labels
-      labels_key
-      logging_api_url
-      monitoring_type
-      non_utf8_replacement_string
-      operation_key
-      private_key_email
-      private_key_passphrase
-      private_key_path
-      project_id
-      source_location_key
-      span_id_key
-      statusz_port
-      trace_key
-      trace_sampled_key
-      use_grpc
-      use_metadata_service
-      vm_id
-      vm_name
-      zone
-    ).each do |p|
-      assert_prometheus_metric_value(
+      # For out_google_cloud, 3 params are present.
+      assert_metric_value.call(
         :stackdriver_config_usage,
         1,
         plugin_name: 'google_cloud',
-        param: p,
-        is_present: false,
+        param: 'adjust_invalid_timestamps',
+        is_present: true,
+        has_default_value: true)
+      assert_metric_value.call(
+        :stackdriver_config_usage,
+        1,
+        plugin_name: 'google_cloud',
+        param: 'autoformat_stackdriver_trace',
+        is_present: true,
         has_default_value: false)
-    end
+      assert_metric_value.call(
+        :stackdriver_config_usage,
+        1,
+        plugin_name: 'google_cloud',
+        param: 'coerce_to_utf8',
+        is_present: true,
+        has_default_value: false)
+      # The remaining "google_cloud" params are not present.
+      # The are no params for "detect_exceptions".
+      %w(
+        auth_method
+        detect_json
+        enable_monitoring
+        gcm_service_address
+        grpc_compression_algorithm
+        http_request_key
+        insert_id_key
+        label_map
+        labels
+        labels_key
+        logging_api_url
+        monitoring_type
+        non_utf8_replacement_string
+        operation_key
+        private_key_email
+        private_key_passphrase
+        private_key_path
+        project_id
+        source_location_key
+        span_id_key
+        statusz_port
+        trace_key
+        trace_sampled_key
+        use_grpc
+        use_metadata_service
+        vm_id
+        vm_name
+        zone
+      ).each do |p|
+        assert_metric_value.call(
+          :stackdriver_config_usage,
+          1,
+          plugin_name: 'google_cloud',
+          param: p,
+          is_present: false,
+          has_default_value: false)
+      end
 
-    # We also export values for the bools.
-    assert_prometheus_metric_value(
-      :stackdriver_config_bool_values,
-      1,
-      plugin_name: 'google_cloud',
-      param: 'adjust_invalid_timestamps',
-      value: true)
-    assert_prometheus_metric_value(
-      :stackdriver_config_bool_values,
-      1,
-      plugin_name: 'google_cloud',
-      param: 'autoformat_stackdriver_trace',
-      value: false)
-    assert_prometheus_metric_value(
-      :stackdriver_config_bool_values,
-      1,
-      plugin_name: 'google_cloud',
-      param: 'coerce_to_utf8',
-      value: true)
+      # We also export values for the bools.
+      assert_metric_value.call(
+        :stackdriver_config_bool_values,
+        1,
+        plugin_name: 'google_cloud',
+        param: 'adjust_invalid_timestamps',
+        value: true)
+      assert_metric_value.call(
+        :stackdriver_config_bool_values,
+        1,
+        plugin_name: 'google_cloud',
+        param: 'autoformat_stackdriver_trace',
+        value: false)
+      assert_metric_value.call(
+        :stackdriver_config_bool_values,
+        1,
+        plugin_name: 'google_cloud',
+        param: 'coerce_to_utf8',
+        value: true)
+    end
   end
 
   def create_driver(conf = APPLICATION_DEFAULT_CONFIG)

data/test/plugin/utils.rb

@@ -0,0 +1,147 @@
+# Copyright 2020 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require_relative 'constants'
+
+require 'prometheus/client'
+require 'webmock/test_unit'
+
+module Utils
+  include Constants
+
+  def delete_env_vars
+    # delete environment variables that googleauth uses to find credentials.
+    ENV.delete(CREDENTIALS_PATH_ENV_VAR)
+    # service account env.
+    ENV.delete(PRIVATE_KEY_VAR)
+    ENV.delete(CLIENT_EMAIL_VAR)
+    ENV.delete(PROJECT_ID_VAR)
+    # authorized_user env.
+    ENV.delete(CLIENT_ID_VAR)
+    ENV.delete(CLIENT_SECRET_VAR)
+    ENV.delete(REFRESH_TOKEN_VAR)
+    # home var, which is used to find $HOME/.gcloud/...
+    ENV.delete('HOME')
+  end
+
+  def stub_metadata_request(metadata_path, response_body)
+    stub_request(:get, 'http://169.254.169.254/computeMetadata/v1/' +
+                 metadata_path)
+      .to_return(body: response_body, status: 200,
+                 headers: { 'Content-Length' => response_body.length })
+  end
+
+  def setup_no_metadata_service_stubs
+    # Simulate a machine with no metadata service present
+    stub_request(:any, %r{http://169.254.169.254/.*})
+      .to_raise(Errno::EHOSTUNREACH)
+  end
+
+  def setup_gce_metadata_stubs
+    # Stub the root, used for platform detection by the plugin and 'googleauth'.
+    stub_request(:get, 'http://169.254.169.254')
+      .to_return(status: 200, headers: { 'Metadata-Flavor' => 'Google' })
+
+    # Create stubs for all the GCE metadata lookups the agent needs to make.
+    stub_metadata_request('project/project-id', PROJECT_ID)
+    stub_metadata_request('instance/zone', FULLY_QUALIFIED_ZONE)
+    stub_metadata_request('instance/id', VM_ID)
+    stub_metadata_request('instance/attributes/',
+                          "attribute1\nattribute2\nattribute3")
+
+    # Used by 'googleauth' to fetch the default service account credentials.
+    stub_request(:get, 'http://169.254.169.254/computeMetadata/v1/' \
+                 'instance/service-accounts/default/token')
+      .to_return(body: %({"access_token": "#{FAKE_AUTH_TOKEN}"}),
+                 status: 200,
+                 headers: { 'Content-Length' => FAKE_AUTH_TOKEN.length,
+                            'Content-Type' => 'application/json' })
+  end
+
+  def setup_ec2_metadata_stubs
+    # Stub the root, used for platform detection.
+    stub_request(:get, 'http://169.254.169.254')
+      .to_return(status: 200, headers: { 'Server' => 'EC2ws' })
+
+    # Stub the identity document lookup made by the agent.
+    stub_request(:get, 'http://169.254.169.254/latest/dynamic/' \
+                 'instance-identity/document')
+      .to_return(body: EC2_IDENTITY_DOCUMENT, status: 200,
+                 headers: { 'Content-Length' => EC2_IDENTITY_DOCUMENT.length })
+  end
+
+  def setup_auth_stubs(base_url)
+    # Used when loading credentials from a JSON file.
+    stub_request(:post, base_url)
+      .with(body: hash_including(grant_type: AUTH_GRANT_TYPE))
+      .to_return(body: %({"access_token": "#{FAKE_AUTH_TOKEN}"}),
+                 status: 200,
+                 headers: { 'Content-Length' => FAKE_AUTH_TOKEN.length,
+                            'Content-Type' => 'application/json' })
+
+    stub_request(:post, base_url)
+      .with(body: hash_including(grant_type: 'refresh_token'))
+      .to_return(body: %({"access_token": "#{FAKE_AUTH_TOKEN}"}),
+                 status: 200,
+                 headers: { 'Content-Length' => FAKE_AUTH_TOKEN.length,
+                            'Content-Type' => 'application/json' })
+  end
+
+  def setup_managed_vm_metadata_stubs
+    stub_metadata_request(
+      'instance/attributes/',
+      "attribute1\ngae_backend_name\ngae_backend_version\nlast_attribute")
+    stub_metadata_request('instance/attributes/gae_backend_name',
+                          MANAGED_VM_BACKEND_NAME)
+    stub_metadata_request('instance/attributes/gae_backend_version',
+                          MANAGED_VM_BACKEND_VERSION)
+  end
+
+  def setup_k8s_metadata_stubs(should_respond = true)
+    if should_respond
+      stub_metadata_request(
+        'instance/attributes/',
+        "attribute1\ncluster-location\ncluster-name\nlast_attribute")
+      stub_metadata_request('instance/attributes/cluster-location',
+                            K8S_LOCATION2)
+      stub_metadata_request('instance/attributes/cluster-name',
+                            K8S_CLUSTER_NAME)
+    else
+      ['cluster-location', 'cluster-name'].each do |metadata_name|
+        stub_request(:get, %r{.*instance/attributes/#{metadata_name}.*})
+          .to_return(status: 404,
+                     body: 'The requested URL /computeMetadata/v1/instance/' \
+                           "attributes/#{metadata_name} was not found on this" \
+                           ' server.')
+      end
+    end
+  end
+
+  def setup_dataproc_metadata_stubs
+    stub_metadata_request(
+      'instance/attributes/',
+      "attribute1\ndataproc-cluster-uuid\ndataproc-cluster-name")
+    stub_metadata_request('instance/attributes/dataproc-cluster-name',
+                          DATAPROC_CLUSTER_NAME)
+    stub_metadata_request('instance/attributes/dataproc-cluster-uuid',
+                          DATAPROC_CLUSTER_UUID)
+    stub_metadata_request('instance/attributes/dataproc-region',
+                          DATAPROC_REGION)
+  end
+
+  def clear_metrics
+    Prometheus::Client.registry.instance_variable_set('@metrics', {})
+    OpenCensus::Stats.ensure_recorder.clear_stats
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-google-cloud
 version: !ruby/object:Gem::Version
-  version: 0.8.7
+  version: 0.9.0
 platform: ruby
 authors:
 - Stackdriver Agents Team
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-06-11 00:00:00.000000000 Z
+date: 2020-07-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -86,28 +86,28 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 3.9.0
+        version: 3.12.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 3.9.0
+        version: 3.12.2
 - !ruby/object:Gem::Dependency
   name: grpc
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.22.0
+        version: 1.30.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.22.0
+        version: 1.30.1
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
@@ -289,6 +289,7 @@ files:
 - test/plugin/test_filter_analyze_config.rb
 - test/plugin/test_out_google_cloud.rb
 - test/plugin/test_out_google_cloud_grpc.rb
+- test/plugin/utils.rb
 homepage: https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud
 licenses:
 - Apache-2.0
@@ -329,3 +330,4 @@ test_files:
 - test/plugin/test_filter_analyze_config.rb
 - test/plugin/test_out_google_cloud.rb
 - test/plugin/test_out_google_cloud_grpc.rb
+- test/plugin/utils.rb