fluent-plugin-google-cloud 0.7.13 → 0.7.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 121ed882fad27b0eb910d9fb8e95e7840fce8430d20ee7b85e53b39280b8f68c
-  data.tar.gz: aa3d64df2c47d080323032bbbf7aa7faee7b7fd58cb5796007bbdb4fac84d02c
+  metadata.gz: 66acba1fe70f426b7ee4aa89bb1451404971f76a0af963c639b239f67db280f3
+  data.tar.gz: 391bc3714af4c0a474bb630458e9a073f4c3b4298bd7818e90c83d32c7c82cf9
 SHA512:
-  metadata.gz: b94c6ac4c29f6d7f2312ab9976dd1dabab6f532ccb52e717191a6b225dd65defef117623067131648d72e2494ae5f380a6475d633db87c568dd9153932895c94
-  data.tar.gz: 2bda121660d6d9eebac56db54283ef49cbfb4db1d482e933fb33549fd1eac4fa33038b0f1024555c05ee0f4d3333953b9ead825e9afc468d27a31fc7beee7fc0
+  metadata.gz: 3adaecad447e1cf9c0665c11468578e7f0808ab76044b491b2314972bcf60fe56afe94e7f442d4d92db93c206e77d80bd9ef05841491da1d76bbd49c951d2830
+  data.tar.gz: a34643f3e4693041280192de7586f363d14f204dc88964fb25b3022e96a4e25e064072672286720f02e5685db001088ca9549c534c9049ec5be689735a53ab0e

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-google-cloud (0.7.13)
+    fluent-plugin-google-cloud (0.7.14)
       fluentd (= 1.4.2)
       google-api-client (= 0.28.4)
       google-cloud-logging (= 1.6.0)
@@ -47,7 +47,7 @@ GEM
       signet (~> 0.10)
     google-cloud-core (1.3.0)
       google-cloud-env (~> 1.0)
-    google-cloud-env (1.0.5)
+    google-cloud-env (1.2.0)
       faraday (~> 0.11)
     google-cloud-logging (1.6.0)
       concurrent-ruby (~> 1.0)
@@ -55,7 +55,7 @@ GEM
       google-gax (~> 1.3)
       googleapis-common-protos-types (>= 1.0.2)
       stackdriver-core (~> 1.3)
-    google-gax (1.5.0)
+    google-gax (1.6.3)
       google-protobuf (~> 3.2)
       googleapis-common-protos (>= 1.3.5, < 2.0)
       googleauth (>= 0.6.2, < 0.10.0)
@@ -78,7 +78,7 @@ GEM
     grpc (1.14.2)
       google-protobuf (~> 3.1)
       googleapis-common-protos-types (~> 1.0.0)
-    hashdiff (0.3.9)
+    hashdiff (0.4.0)
     http_parser.rb (0.6.0)
     httpclient (2.8.3)
     json (2.1.0)
@@ -88,9 +88,9 @@ GEM
     mime-types (3.2.2)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2019.0331)
-    mocha (1.8.0)
+    mocha (1.9.0)
       metaclass (~> 0.0.1)
-    msgpack (1.2.10)
+    msgpack (1.3.0)
     multi_json (1.13.1)
     multipart-post (2.1.1)
     os (1.0.1)
@@ -100,7 +100,7 @@ GEM
     powerpack (0.1.2)
     prometheus-client (0.7.1)
       quantile (~> 0.2.0)
-    public_suffix (3.0.3)
+    public_suffix (3.1.0)
     quantile (0.2.1)
     rainbow (2.2.2)
       rake
@@ -117,7 +117,7 @@ GEM
       rainbow (>= 1.99.1, < 3.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (~> 1.0, >= 1.0.1)
-    ruby-progressbar (1.10.0)
+    ruby-progressbar (1.10.1)
     safe_yaml (1.0.5)
     serverengine (2.1.1)
       sigdump (~> 0.2.2)

data/fluent-plugin-google-cloud.gemspec

@@ -10,7 +10,7 @@ eos
   gem.homepage      =
     'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
   gem.license       = 'Apache-2.0'
-  gem.version       = '0.7.13'
+  gem.version       = '0.7.14'
   gem.authors       = ['Stackdriver Agents Team']
   gem.email         = ['stackdriver-agents@google.com']
   gem.required_ruby_version = Gem::Requirement.new('>= 2.2')

data/lib/fluent/plugin/out_google_cloud.rb

@@ -60,14 +60,6 @@ module Fluent
         resource_type: 'gae_app',
         metadata_attributes: %w(gae_backend_name gae_backend_version)
       }.freeze
-      CLOUDFUNCTIONS_CONSTANTS = {
-        service: 'cloudfunctions.googleapis.com',
-        resource_type: 'cloud_function',
-        stream_severity_map: {
-          'stdout' => 'INFO',
-          'stderr' => 'ERROR'
-        }
-      }.freeze
       COMPUTE_CONSTANTS = {
         service: 'compute.googleapis.com',
         resource_type: 'gce_instance'
@@ -541,21 +533,9 @@ module Fluent
       @resource ||= determine_agent_level_monitored_resource_via_legacy
 
       # Set regexp that we should match tags against later on. Using a list
-      # instead of a map to ensure order. For example, tags will be matched
-      # against Cloud Functions first, then GKE.
+      # instead of a map to ensure order.
       @tag_regexp_list = []
       if @resource.type == GKE_CONSTANTS[:resource_type]
-        # We only support Cloud Functions logs for GKE right now.
-        if fetch_gce_metadata('instance/attributes/'
-                             ).split.include?('gcf_region')
-          # Fetch this info and store it to avoid recurring
-          # metadata server calls.
-          @gcf_region = fetch_gce_metadata('instance/attributes/gcf_region')
-          @tag_regexp_list << [
-            CLOUDFUNCTIONS_CONSTANTS[:resource_type],
-            @compiled_cloudfunctions_tag_regexp
-          ]
-        end
         @tag_regexp_list << [
           GKE_CONSTANTS[:resource_type], @compiled_kubernetes_tag_regexp
         ]
@@ -667,8 +647,7 @@ module Fluent
             end
           end
 
-          ts_secs, ts_nanos = compute_timestamp(
-            entry_level_resource.type, record, time)
+          ts_secs, ts_nanos = compute_timestamp(record, time)
           severity = compute_severity(
             entry_level_resource.type, record, entry_level_common_labels)
 
@@ -1087,14 +1066,6 @@ module Fluent
       @compiled_kubernetes_tag_regexp = Regexp.new(@kubernetes_tag_regexp) if
         @kubernetes_tag_regexp
 
-      @compiled_cloudfunctions_tag_regexp =
-        /\.(?<encoded_function_name>.+)\.\d+-[^-]+_default_worker$/
-      @compiled_cloudfunctions_log_regexp = /^
-        (?:\[(?<severity>.)\])?
-        \[(?<timestamp>.{24})\]
-        (?:\[(?<execution_id>[^\]]+)\])?
-        [ ](?<text>.*)$/x
-
       @compiled_http_latency_regexp =
         /^\s*(?<seconds>\d+)(?<decimal>\.\d+)?\s*s\s*$/
     end
@@ -1308,7 +1279,7 @@ module Fluent
       chunk.msgpack_each do |tag, time, record|
         unless record.is_a?(Hash)
           @log.warn 'Dropping log entries with malformed record: ' \
-                    "'#{record.inspect}'. " \
+                    "'#{record.inspect}' from tag '#{tag}' at '#{time}'. " \
                     'A log record should be in JSON format.'
           next
         end
@@ -1364,23 +1335,6 @@ module Fluent
 
       # Once the resource type is settled down, determine the labels.
       case resource.type
-      # Cloud Functions.
-      when CLOUDFUNCTIONS_CONSTANTS[:resource_type]
-        resource.labels.merge!(
-          'region' => @gcf_region,
-          'function_name' => decode_cloudfunctions_function_name(
-            matched_regexp_group['encoded_function_name'])
-        )
-        instance_id = resource.labels.delete('instance_id')
-        common_labels.merge!(
-          "#{GKE_CONSTANTS[:service]}/instance_id" => instance_id,
-          "#{COMPUTE_CONSTANTS[:service]}/resource_id" => instance_id,
-          "#{GKE_CONSTANTS[:service]}/cluster_name" =>
-            resource.labels.delete('cluster_name'),
-          "#{COMPUTE_CONSTANTS[:service]}/zone" =>
-            resource.labels.delete('zone')
-        )
-
       # GKE container.
       when GKE_CONSTANTS[:resource_type]
         if matched_regexp_group
@@ -1475,17 +1429,6 @@ module Fluent
       common_labels = group_level_common_labels.dup
 
       case resource.type
-      # Cloud Functions.
-      when CLOUDFUNCTIONS_CONSTANTS[:resource_type]
-        if record.key?('log')
-          @cloudfunctions_log_match =
-            @compiled_cloudfunctions_log_regexp.match(record['log'])
-          common_labels['execution_id'] =
-            @cloudfunctions_log_match['execution_id'] if
-              @cloudfunctions_log_match &&
-              @cloudfunctions_log_match['execution_id']
-        end
-
       # GKE container.
       when GKE_CONSTANTS[:resource_type]
         # Move the stdout/stderr annotation from the record into a label.
@@ -1627,7 +1570,7 @@ module Fluent
       nil
     end
 
-    def compute_timestamp(resource_type, record, time)
+    def compute_timestamp(record, time)
       current_time = Time.now
       if record.key?('timestamp') &&
          record['timestamp'].is_a?(Hash) &&
@@ -1655,12 +1598,6 @@ module Fluent
             'timestampSeconds and timestampNanos instead.'
         end
         timestamp = time_or_nil(ts_secs, ts_nanos)
-      elsif resource_type == CLOUDFUNCTIONS_CONSTANTS[:resource_type] &&
-            @cloudfunctions_log_match
-        timestamp = DateTime.parse(
-          @cloudfunctions_log_match['timestamp']).to_time
-        ts_secs = timestamp.tv_sec
-        ts_nanos = timestamp.tv_nsec
       elsif record.key?('time')
         # k8s ISO8601 timestamp
         begin
@@ -1718,14 +1655,7 @@ module Fluent
     end
 
     def compute_severity(resource_type, record, entry_level_common_labels)
-      if resource_type == CLOUDFUNCTIONS_CONSTANTS[:resource_type]
-        if @cloudfunctions_log_match && @cloudfunctions_log_match['severity']
-          return parse_severity(@cloudfunctions_log_match['severity'])
-        elsif record.key?('stream')
-          return CLOUDFUNCTIONS_CONSTANTS[:stream_severity_map].fetch(
-            record.delete('stream'), 'DEFAULT')
-        end
-      elsif record.key?('severity')
+      if record.key?('severity')
         return parse_severity(record.delete('severity'))
       elsif resource_type == GKE_CONSTANTS[:resource_type]
         stream = entry_level_common_labels["#{GKE_CONSTANTS[:service]}/stream"]
@@ -1946,14 +1876,6 @@ module Fluent
       end
     end
 
-    def decode_cloudfunctions_function_name(function_name)
-      function_name.gsub(/c\.[a-z]/) { |s| s.upcase[-1] }
-                   .gsub('u.u', '_')
-                   .gsub('d.d', '$')
-                   .gsub('a.a', '@')
-                   .gsub('p.p', '.')
-    end
-
     def format(tag, time, record)
       Fluent::Engine.msgpack_factory.packer.write([tag, time, record]).to_s
     end
@@ -2035,19 +1957,11 @@ module Fluent
       # Only one of {text_payload, json_payload} will be set.
       text_payload = nil
       json_payload = nil
-      # If this is a Cloud Functions log that matched the expected regexp,
-      # use text payload. Otherwise, use JSON if we found valid JSON, or text
-      # payload in the following cases:
-      # 1. This is a Cloud Functions log and the 'log' key is available
-      # 2. This is an unstructured Container log and the 'log' key is available
-      # 3. The only remaining key is 'message'
-      if resource_type == CLOUDFUNCTIONS_CONSTANTS[:resource_type] &&
-         @cloudfunctions_log_match
-        text_payload = @cloudfunctions_log_match['text']
-      elsif resource_type == CLOUDFUNCTIONS_CONSTANTS[:resource_type] &&
-            record.key?('log')
-        text_payload = record['log']
-      elsif is_json
+      # Use JSON if we found valid JSON, or text payload in the following
+      # cases:
+      # 1. This is an unstructured Container log and the 'log' key is available
+      # 2. The only remaining key is 'message'
+      if is_json
         json_payload = record
       elsif [GKE_CONSTANTS[:resource_type],
              DOCKER_CONSTANTS[:resource_type]].include?(resource_type) &&
@@ -2076,9 +1990,7 @@ module Fluent
     end
 
     def log_name(tag, resource)
-      if resource.type == CLOUDFUNCTIONS_CONSTANTS[:resource_type]
-        tag = 'cloud-functions'
-      elsif resource.type == APPENGINE_CONSTANTS[:resource_type]
+      if resource.type == APPENGINE_CONSTANTS[:resource_type]
         # Add a prefix to Managed VM logs to prevent namespace collisions.
         tag = "#{APPENGINE_CONSTANTS[:service]}/#{tag}"
       elsif resource.type == GKE_CONSTANTS[:resource_type]

data/test/plugin/base_test.rb

@@ -1223,70 +1223,6 @@ module BaseTest
     end
   end
 
-  def test_cloudfunctions_log
-    setup_gce_metadata_stubs
-    setup_cloudfunctions_metadata_stubs
-    [1, 2, 3, 5, 11, 50].each do |n|
-      setup_logging_stubs do
-        d = create_driver(APPLICATION_DEFAULT_CONFIG, CLOUDFUNCTIONS_TAG)
-        # The test driver doesn't clear its buffer of entries after running, so
-        # do it manually here.
-        d.instance_variable_get('@entries').clear
-        @logs_sent = []
-        n.times { |i| d.emit(cloudfunctions_log_entry(i)) }
-        d.run
-      end
-      verify_log_entries(n, CLOUDFUNCTIONS_PARAMS) do |entry, i|
-        verify_default_log_entry_text(entry['textPayload'], i, entry)
-        assert_equal 'DEBUG', entry['severity'],
-                     "Test with #{n} logs failed. \n#{entry}"
-      end
-    end
-  end
-
-  def test_cloudfunctions_logs_text_not_matched
-    setup_gce_metadata_stubs
-    setup_cloudfunctions_metadata_stubs
-    [1, 2, 3, 5, 11, 50].each do |n|
-      @logs_sent = []
-      setup_logging_stubs do
-        d = create_driver(APPLICATION_DEFAULT_CONFIG, CLOUDFUNCTIONS_TAG)
-        # The test driver doesn't clear its buffer of entries after running, so
-        # do it manually here.
-        d.instance_variable_get('@entries').clear
-        n.times { |i| d.emit(cloudfunctions_log_entry_text_not_matched(i)) }
-        d.run
-      end
-      verify_log_entries(
-        n, CLOUDFUNCTIONS_TEXT_NOT_MATCHED_PARAMS) do |entry|
-        assert_equal 'INFO', entry['severity'],
-                     "Test with #{n} logs failed. \n#{entry}"
-      end
-    end
-  end
-
-  def test_multiple_cloudfunctions_logs_tag_not_matched
-    setup_gce_metadata_stubs
-    setup_cloudfunctions_metadata_stubs
-    [1, 2, 3, 5, 11, 50].each do |n|
-      @logs_sent = []
-      setup_logging_stubs do
-        d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_TAG)
-        # The test driver doesn't clear its buffer of entries after running, so
-        # do it manually here.
-        d.instance_variable_get('@entries').clear
-        n.times { |i| d.emit(cloudfunctions_log_entry(i)) }
-        d.run
-      end
-      verify_log_entries(n, CONTAINER_FROM_TAG_PARAMS, 'textPayload') \
-        do |entry, i|
-          assert_equal '[D][2015-09-25T12:34:56.789Z][123-0] test log entry ' \
-                       "#{i}", entry['textPayload'],
-                       "Test with #{n} logs failed. \n#{entry}"
-        end
-    end
-  end
-
   def test_dataproc_log
     setup_gce_metadata_stubs
     setup_dataproc_metadata_stubs
@@ -2173,18 +2109,6 @@ module BaseTest
     end
   end
 
-  def setup_cloudfunctions_metadata_stubs
-    stub_metadata_request(
-      'instance/attributes/',
-      "attribute1\ncluster-location\ncluster-name\ngcf_region\nlast_attribute")
-    stub_metadata_request('instance/attributes/cluster-location',
-                          K8S_LOCATION2)
-    stub_metadata_request('instance/attributes/cluster-name',
-                          K8S_CLUSTER_NAME)
-    stub_metadata_request('instance/attributes/gcf_region',
-                          CLOUDFUNCTIONS_REGION)
-  end
-
   def setup_dataproc_metadata_stubs
     stub_metadata_request(
       'instance/attributes/',
@@ -2339,20 +2263,6 @@ module BaseTest
     }
   end
 
-  def cloudfunctions_log_entry(i)
-    {
-      stream: 'stdout',
-      log: '[D][2015-09-25T12:34:56.789Z][123-0] ' + log_entry(i)
-    }
-  end
-
-  def cloudfunctions_log_entry_text_not_matched(i)
-    {
-      stream: 'stdout',
-      log: log_entry(i)
-    }
-  end
-
   def dataflow_log_entry(i)
     {
       step: DATAFLOW_STEP_ID,

data/test/plugin/constants.rb

@@ -178,16 +178,6 @@ module Constants
   CONTAINER_SEVERITY = 'INFO'.freeze
   CONTAINER_LOCAL_RESOURCE_ID_PREFIX = 'gke_container'.freeze
 
-  # Cloud Functions specific labels.
-  CLOUDFUNCTIONS_FUNCTION_NAME = '$My_Function.Name-@1'.freeze
-  CLOUDFUNCTIONS_REGION = 'us-central1'.freeze
-  CLOUDFUNCTIONS_EXECUTION_ID = '123-0'.freeze
-  CLOUDFUNCTIONS_CLUSTER_NAME = 'cluster-1'.freeze
-  CLOUDFUNCTIONS_NAMESPACE_NAME = 'default'.freeze
-  CLOUDFUNCTIONS_POD_NAME =
-    'd.dc.myu.uc.functionp.pc.name-a.a1.987-c0l82'.freeze
-  CLOUDFUNCTIONS_CONTAINER_NAME = 'worker'.freeze
-
   # Dataflow specific labels.
   DATAFLOW_REGION = 'us-central1'.freeze
   DATAFLOW_JOB_NAME = 'job_name_1'.freeze
@@ -709,52 +699,6 @@ module Constants
   DOCKER_CONTAINER_PARAMS_NO_STREAM =
     DOCKER_CONTAINER_PARAMS.merge(labels: {}).freeze
 
-  # Cloud Functions.
-  CLOUDFUNCTIONS_TAG = "kubernetes.#{CLOUDFUNCTIONS_POD_NAME}_" \
-                        "#{CLOUDFUNCTIONS_NAMESPACE_NAME}_" \
-                        "#{CLOUDFUNCTIONS_CONTAINER_NAME}".freeze
-
-  CLOUDFUNCTIONS_PARAMS = {
-    resource: {
-      type: CLOUDFUNCTIONS_CONSTANTS[:resource_type],
-      labels: {
-        'function_name' => CLOUDFUNCTIONS_FUNCTION_NAME,
-        'region' => CLOUDFUNCTIONS_REGION
-      }
-    },
-    log_name: 'cloud-functions',
-    project_id: PROJECT_ID,
-    labels: {
-      'execution_id' => CLOUDFUNCTIONS_EXECUTION_ID,
-      "#{GKE_CONSTANTS[:service]}/instance_id" => VM_ID,
-      "#{GKE_CONSTANTS[:service]}/cluster_name" =>
-        CLOUDFUNCTIONS_CLUSTER_NAME,
-      "#{COMPUTE_CONSTANTS[:service]}/resource_id" => VM_ID,
-      "#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME,
-      "#{COMPUTE_CONSTANTS[:service]}/zone" => ZONE
-    }
-  }.freeze
-
-  CLOUDFUNCTIONS_TEXT_NOT_MATCHED_PARAMS = {
-    resource: {
-      type: CLOUDFUNCTIONS_CONSTANTS[:resource_type],
-      labels: {
-        'function_name' => CLOUDFUNCTIONS_FUNCTION_NAME,
-        'region' => CLOUDFUNCTIONS_REGION
-      }
-    },
-    log_name: 'cloud-functions',
-    project_id: PROJECT_ID,
-    labels: {
-      "#{GKE_CONSTANTS[:service]}/instance_id" => VM_ID,
-      "#{GKE_CONSTANTS[:service]}/cluster_name" =>
-        CLOUDFUNCTIONS_CLUSTER_NAME,
-      "#{COMPUTE_CONSTANTS[:service]}/resource_id" => VM_ID,
-      "#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME,
-      "#{COMPUTE_CONSTANTS[:service]}/zone" => ZONE
-    }
-  }.freeze
-
   # Cloud Dataflow.
   DATAFLOW_PARAMS = {
     resource: {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-google-cloud
 version: !ruby/object:Gem::Version
-  version: 0.7.13
+  version: 0.7.14
 platform: ruby
 authors:
 - Stackdriver Agents Team
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-05-25 00:00:00.000000000 Z
+date: 2019-06-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -260,7 +260,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.9
+rubygems_version: 2.7.10
 signing_key: 
 specification_version: 4
 summary: fluentd plugins for the Stackdriver Logging API