fluent-plugin-google-cloud 0.6.25.1 → 0.7.0.pre.1

data/test/plugin/constants.rb

@@ -98,14 +98,6 @@ module Constants
   MANAGED_VM_BACKEND_NAME = 'default'.freeze
   MANAGED_VM_BACKEND_VERSION = 'guestbook2.0'.freeze
 
-  # LogEntry fields for extraction.
-  TRACE = 'projects/proj1/traces/1234567890abcdef1234567890abcdef'.freeze
-  TRACE2 = 'projects/proj1/traces/1234567890abcdef1234567890fedcba'.freeze
-  SPAN_ID = '000000000000004a'.freeze
-  SPAN_ID2 = '000000000000007e'.freeze
-  INSERT_ID = 'fah7yr7iw64tg857y'.freeze
-  INSERT_ID2 = 'fah7yr7iw64tgaeuf'.freeze
-
   # Docker Container labels.
   DOCKER_CONTAINER_ID =
     '0d0f03ff8d3c42688692536d1af77a28cd135c0a5c531f25a31'.freeze
@@ -143,11 +135,20 @@ module Constants
     ".#{K8S_CONTAINER_NAME}".freeze
 
   # Container Engine / Kubernetes specific labels.
+  CONTAINER_CLUSTER_NAME = 'cluster-1'.freeze
   CONTAINER_NAMESPACE_ID = '898268c8-4a36-11e5-9d81-42010af0194c'.freeze
+  CONTAINER_NAMESPACE_NAME = 'kube-system'.freeze
   CONTAINER_POD_ID = 'cad3c3c4-4b9c-11e5-9d81-42010af0194c'.freeze
+  CONTAINER_POD_NAME = 'redis-master-c0l82.foo.bar'.freeze
+  CONTAINER_CONTAINER_NAME = 'redis'.freeze
   CONTAINER_LABEL_KEY = 'component'.freeze
   CONTAINER_LABEL_VALUE = 'redis-component'.freeze
+  CONTAINER_STREAM = 'stdout'.freeze
   CONTAINER_SEVERITY = 'INFO'.freeze
+  # Timestamp for 1234567890 seconds and 987654321 nanoseconds since epoch.
+  CONTAINER_TIMESTAMP = '2009-02-13T23:31:30.987654321Z'.freeze
+  CONTAINER_SECONDS_EPOCH = 1_234_567_890
+  CONTAINER_NANOS = 987_654_321
   CONTAINER_LOCAL_RESOURCE_ID_PREFIX = 'gke_container'.freeze
 
   # Cloud Functions specific labels.
@@ -399,27 +400,28 @@ module Constants
 
   # GKE Container.
   CONTAINER_TAG =
-    "kubernetes.#{K8S_POD_NAME}_#{K8S_NAMESPACE_NAME}_" \
-    "#{K8S_CONTAINER_NAME}".freeze
+    "kubernetes.#{CONTAINER_POD_NAME}_#{CONTAINER_NAMESPACE_NAME}_" \
+    "#{CONTAINER_CONTAINER_NAME}".freeze
 
   CONTAINER_FROM_METADATA_PARAMS = {
     resource: {
       type: GKE_CONSTANTS[:resource_type],
       labels: {
-        'cluster_name' => K8S_CLUSTER_NAME,
+        'cluster_name' => CONTAINER_CLUSTER_NAME,
         'namespace_id' => CONTAINER_NAMESPACE_ID,
         'instance_id' => VM_ID,
         'pod_id' => CONTAINER_POD_ID,
-        'container_name' => K8S_CONTAINER_NAME,
+        'container_name' => CONTAINER_CONTAINER_NAME,
         'zone' => ZONE
       }
     },
-    log_name: K8S_CONTAINER_NAME,
+    log_name: CONTAINER_CONTAINER_NAME,
     project_id: PROJECT_ID,
     labels: {
-      "#{GKE_CONSTANTS[:service]}/namespace_name" => K8S_NAMESPACE_NAME,
-      "#{GKE_CONSTANTS[:service]}/pod_name" => K8S_POD_NAME,
-      "#{GKE_CONSTANTS[:service]}/stream" => K8S_STREAM,
+      "#{GKE_CONSTANTS[:service]}/namespace_name" =>
+        CONTAINER_NAMESPACE_NAME,
+      "#{GKE_CONSTANTS[:service]}/pod_name" => CONTAINER_POD_NAME,
+      "#{GKE_CONSTANTS[:service]}/stream" => CONTAINER_STREAM,
       "label/#{CONTAINER_LABEL_KEY}" => CONTAINER_LABEL_VALUE,
       "#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME
     }
@@ -431,20 +433,21 @@ module Constants
     resource: {
       type: GKE_CONSTANTS[:resource_type],
       labels: {
-        'cluster_name' => K8S_CLUSTER_NAME,
-        'namespace_id' => K8S_NAMESPACE_NAME,
+        'cluster_name' => CONTAINER_CLUSTER_NAME,
+        'namespace_id' => CONTAINER_NAMESPACE_NAME,
         'instance_id' => VM_ID,
-        'pod_id' => K8S_POD_NAME,
-        'container_name' => K8S_CONTAINER_NAME,
+        'pod_id' => CONTAINER_POD_NAME,
+        'container_name' => CONTAINER_CONTAINER_NAME,
         'zone' => ZONE
       }
     },
-    log_name: K8S_CONTAINER_NAME,
+    log_name: CONTAINER_CONTAINER_NAME,
     project_id: PROJECT_ID,
     labels: {
-      "#{GKE_CONSTANTS[:service]}/namespace_name" => K8S_NAMESPACE_NAME,
-      "#{GKE_CONSTANTS[:service]}/pod_name" => K8S_POD_NAME,
-      "#{GKE_CONSTANTS[:service]}/stream" => K8S_STREAM,
+      "#{GKE_CONSTANTS[:service]}/namespace_name" =>
+        CONTAINER_NAMESPACE_NAME,
+      "#{GKE_CONSTANTS[:service]}/pod_name" => CONTAINER_POD_NAME,
+      "#{GKE_CONSTANTS[:service]}/stream" => CONTAINER_STREAM,
       "#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME
     }
   }.freeze
@@ -453,11 +456,11 @@ module Constants
     resource: {
       type: GKE_CONSTANTS[:resource_type],
       labels: {
-        'cluster_name' => K8S_CLUSTER_NAME,
+        'cluster_name' => CONTAINER_CLUSTER_NAME,
         'namespace_id' => CONTAINER_NAMESPACE_ID,
         'instance_id' => VM_ID,
         'pod_id' => CONTAINER_POD_ID,
-        'container_name' => K8S_CONTAINER_NAME,
+        'container_name' => CONTAINER_CONTAINER_NAME,
         'zone' => ZONE
       }
     },
@@ -480,6 +483,7 @@ module Constants
         'location' => K8S_LOCATION
       }
     },
+    log_name: 'test',
     project_id: PROJECT_ID,
     labels: {}
   }.freeze
@@ -498,10 +502,6 @@ module Constants
       )
     )
   ).freeze
-  # Used in k8s fallback tests.
-  K8S_CONTAINER_PARAMS_FROM_FALLBACK = COMPUTE_PARAMS_NO_LOG_NAME.merge(
-    log_name: CONTAINER_TAG
-  ).freeze
 
   # K8s Node.
   K8S_NODE_PARAMS = {
@@ -777,12 +777,12 @@ module Constants
       }.to_json,
     # GKE container logs.
     "#{CONTAINER_LOCAL_RESOURCE_ID_PREFIX}.#{CONTAINER_NAMESPACE_ID}" \
-    ".#{K8S_POD_NAME}.#{K8S_CONTAINER_NAME}" =>
+    ".#{CONTAINER_POD_NAME}.#{CONTAINER_CONTAINER_NAME}" =>
       {
         'type' => GKE_CONSTANTS[:resource_type],
         'labels' => {
-          'cluster_name' => K8S_CLUSTER_NAME,
-          'container_name' => K8S_CONTAINER_NAME,
+          'cluster_name' => CONTAINER_CLUSTER_NAME,
+          'container_name' => CONTAINER_CONTAINER_NAME,
           'instance_id' => VM_ID,
           'namespace_id' => CONTAINER_NAMESPACE_ID,
           'pod_id' => CONTAINER_POD_ID,

metadata

@@ -1,14 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-google-cloud
 version: !ruby/object:Gem::Version
-  version: 0.6.25.1
+  version: 0.7.0.pre.1
 platform: ruby
 authors:
-- Stackdriver Agents Team
+- Ling Huang
+- Igor Peshansky
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-09-13 00:00:00.000000000 Z
+date: 2018-08-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -16,14 +17,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.10'
+        version: 1.2.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.10'
+        version: 1.2.5
 - !ruby/object:Gem::Dependency
   name: googleapis-common-protos
   requirement: !ruby/object:Gem::Requirement
@@ -106,14 +107,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '1.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '1.8'
 - !ruby/object:Gem::Dependency
   name: google-protobuf
   requirement: !ruby/object:Gem::Requirement
@@ -219,9 +220,9 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.7.1
 description: |2
-     Fluentd plugins for the Stackdriver Logging API, which will make logs
-     viewable in the Stackdriver Logs Viewer and can optionally store them
-     in Google Cloud Storage and/or BigQuery.
+     Fluentd output plugin for the Stackdriver Logging API, which will make
+     logs viewable in the Developer Console's log viewer and can optionally
+     store them in Google Cloud Storage and/or BigQuery.
      This is an official Google Ruby gem.
 email:
 - stackdriver-agents@google.com
@@ -235,7 +236,6 @@ files:
 - README.rdoc
 - Rakefile
 - fluent-plugin-google-cloud.gemspec
-- lib/fluent/plugin/filter_add_insert_ids.rb
 - lib/fluent/plugin/monitoring.rb
 - lib/fluent/plugin/out_google_cloud.rb
 - test/helper.rb
@@ -247,7 +247,6 @@ files:
 - test/plugin/data/invalid_credentials.json
 - test/plugin/data/new-style-credentials.json
 - test/plugin/test_driver.rb
-- test/plugin/test_filter_add_insert_ids.rb
 - test/plugin/test_out_google_cloud.rb
 - test/plugin/test_out_google_cloud_grpc.rb
 homepage: https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud
@@ -265,15 +264,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.2'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.6.14
 signing_key: 
 specification_version: 4
-summary: fluentd plugins for the Stackdriver Logging API
+summary: fluentd output plugin for the Stackdriver Logging API
 test_files:
 - test/helper.rb
 - test/plugin/base_test.rb
@@ -284,6 +283,5 @@ test_files:
 - test/plugin/data/invalid_credentials.json
 - test/plugin/data/new-style-credentials.json
 - test/plugin/test_driver.rb
-- test/plugin/test_filter_add_insert_ids.rb
 - test/plugin/test_out_google_cloud.rb
 - test/plugin/test_out_google_cloud_grpc.rb

data/lib/fluent/plugin/filter_add_insert_ids.rb

@@ -1,99 +0,0 @@
-# Copyright 2018 Google Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-module Fluent
-  # Fluentd filter plugin for adding insertIds to guarantee log entry order
-  # and uniqueness.
-  # Sample log entries enriched by this plugin:
-  # {
-  #   "timestamp": "2017-08-22 13:35:28",
-  #   "message": "1",
-  #   "logging.googleapis.com/insertId": "aye7eakuf23h41aef0"
-  # }
-  # {
-  #   "timestamp": "2017-08-22 13:35:28",
-  #   "message": "2",
-  #   "logging.googleapis.com/insertId": "aye7eakuf23h41aef1"
-  # }
-  # {
-  #   "timestamp": "2017-08-22 13:35:28",
-  #   "message": "3",
-  #   "logging.googleapis.com/insertId": "aye7eakuf23h41aef2"
-  # }
-  class AddInsertIdsFilter < Filter
-    Fluent::Plugin.register_filter('add_insert_ids', self)
-
-    # Constants for configuration.
-    module ConfigConstants
-      # The default field name of insertIds in the log entry.
-      DEFAULT_INSERT_ID_KEY = 'logging.googleapis.com/insertId'.freeze
-      # The character size of the insertIds. This matches the setup in the
-      # Stackdriver Logging backend.
-      INSERT_ID_SIZE = 17
-      # The characters that are allowed in the insertIds. This matches the
-      # allowed collection by the Stackdriver Logging Backend.
-      ALLOWED_CHARS = (Array(0..9) + Array('a'..'z')).freeze
-    end
-
-    include self::ConfigConstants
-
-    desc 'The field name for insertIds in the log record.'
-    config_param :insert_id_key, :string, default: DEFAULT_INSERT_ID_KEY
-
-    # Expose attr_readers for testing.
-    attr_reader :insert_id_key
-
-    def start
-      super
-      @log = $log # rubocop:disable Style/GlobalVars
-
-      # Initialize the insertID.
-      @log.info "Started the add_insert_ids plugin with #{@insert_id_key}" \
-                ' as the insert ID key.'
-      @insert_id = generate_initial_insert_id
-      @log.info "Initialized the insert ID key to #{@insert_id}."
-    end
-
-    def configure(conf)
-      super
-    end
-
-    def shutdown
-      super
-    end
-
-    # rubocop:disable Style/UnusedMethodArgument
-    def filter(tag, time, record)
-      # Only generate and add an insertId field if the record is a hash and
-      # the insert ID field is not already set (or set to an empty string).
-      if record.is_a?(Hash) && record[@insert_id_key].to_s.empty?
-        record[@insert_id_key] = increment_insert_id
-      end
-      record
-    end
-    # rubocop:enable Style/UnusedMethodArgument
-
-    private
-
-    # Generate a random string as the initial insertId.
-    def generate_initial_insert_id
-      Array.new(INSERT_ID_SIZE) { ALLOWED_CHARS.sample }.join
-    end
-
-    # Increment the insertId and return the new value.
-    def increment_insert_id
-      @insert_id = @insert_id.next
-    end
-  end
-end

data/test/plugin/test_filter_add_insert_ids.rb

@@ -1,135 +0,0 @@
-# Copyright 2018 Google Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-require_relative '../helper'
-
-require 'fluent/test/driver/filter'
-require 'fluent/plugin/filter_add_insert_ids'
-
-# Unit tests for filter_add_insert_ids plugin.
-class FilterAddInsertIdsTest < Test::Unit::TestCase
-  include Fluent::AddInsertIdsFilter::ConfigConstants
-
-  CUSTOM_INSERT_ID_KEY = 'custom_insert_id_key'.freeze
-  INSERT_ID = 'aeyr82r92h249gh9h'.freeze
-  TEST_MESSAGE = 'test message for add_insert_ids plugin.'.freeze
-  APPLICATION_DEFAULT_CONFIG = ''.freeze
-  INSERT_ID_KEY_CONFIG = %(
-    insert_id_key #{CUSTOM_INSERT_ID_KEY}
-  ).freeze
-
-  def setup
-    Fluent::Test.setup
-  end
-
-  def test_configure_insert_id_key
-    {
-      APPLICATION_DEFAULT_CONFIG => DEFAULT_INSERT_ID_KEY,
-      INSERT_ID_KEY_CONFIG => CUSTOM_INSERT_ID_KEY
-    }.each do |config, insert_id_key|
-      d = create_driver(config)
-      assert_equal insert_id_key, d.instance.insert_id_key
-    end
-  end
-
-  def test_add_insert_ids
-    total_entry_count = 1000
-    d = create_driver
-    d.run do
-      total_entry_count.times do |index|
-        d.emit(log_entry(index))
-      end
-    end
-    filtered_events = d.filtered_as_array
-
-    assert_equal total_entry_count, filtered_events.size,
-                 "#{total_entry_count} log entries after filtering is" \
-                 " expected. Only #{filtered_events.size} are detected."
-    # The expected insertId will be assigned as we scan the first log entry.
-    expected_insert_id = nil
-    unique_insert_ids = Set.new
-    filtered_events.each_with_index do |event, index|
-      assert_equal 3, event.size, "Index #{index} failed. Log event should" \
-                   ' include 3 elements: tag, time and record.'
-      record = event[2]
-      assert_true record.is_a?(Hash), "Index #{index} failed. Log record" \
-                  " #{record} should be a hash."
-      assert_equal index, record['id'], "Index #{index} failed. Log entries" \
-                   ' should come in order.'
-      assert_equal TEST_MESSAGE, record['message'], "Index #{index} failed."
-
-      # Get the first insertID.
-      expected_insert_id = record[DEFAULT_INSERT_ID_KEY] if index == 0
-      insert_id = record[DEFAULT_INSERT_ID_KEY]
-      assert_equal expected_insert_id, insert_id, "Index #{index} failed."
-      expected_insert_id = expected_insert_id.next
-      assert_true insert_id < expected_insert_id,
-                  "Index #{index} failed. #{insert_id}" \
-                  " < #{expected_insert_id} is false."
-      unique_insert_ids << insert_id
-    end
-    assert_equal total_entry_count, unique_insert_ids.size,
-                 "Expected #{total_entry_count} unique insertIds." \
-                 " Only #{unique_insert_ids.size} found."
-  end
-
-  def test_insert_ids_not_added_if_present
-    log_entry_with_empty_insert_id = log_entry(0).merge(
-      DEFAULT_INSERT_ID_KEY => '')
-    {
-      log_entry(0).merge(DEFAULT_INSERT_ID_KEY => INSERT_ID) => true,
-      # Still generate insertId if it's an empty string
-      log_entry_with_empty_insert_id => false
-    }.each do |test_data|
-      input_log_entry, retain_original_insert_id = test_data
-      # Make a copy because the log entry gets modified by the filter plugin.
-      log_entry = input_log_entry.dup
-      d = create_driver
-      d.run do
-        d.emit(log_entry)
-      end
-      filtered_events = d.filtered_as_array
-
-      assert_equal 1, filtered_events.size, 'Exact 1 log entry after' \
-                   " filtering is expected. Test data: #{test_data}."
-      event = filtered_events[0]
-      assert_equal 3, event.size, 'Log event should include 3 elements: tag,' \
-                   " time and record. Test data: #{test_data}."
-      record = event[2]
-      assert_true record.is_a?(Hash), "Log record #{record} should be a hash." \
-                  " Test data: #{test_data}."
-      assert_equal 0, record['id'], "Test data: #{test_data}."
-      assert_equal TEST_MESSAGE, record['message'], "Test data: #{test_data}."
-      insert_id = record[DEFAULT_INSERT_ID_KEY]
-      assert_false insert_id.to_s.empty?, 'Insert ID should not be empty.' \
-                   " Test data: #{test_data}."
-      assert_equal retain_original_insert_id,
-                   input_log_entry[DEFAULT_INSERT_ID_KEY] == insert_id,
-                   "Input value is #{input_log_entry[DEFAULT_INSERT_ID_KEY]}." \
-                   " Output value is #{insert_id}. Test data: #{test_data}."
-    end
-  end
-
-  def create_driver(conf = APPLICATION_DEFAULT_CONFIG)
-    Fluent::Test::FilterTestDriver.new(
-      Fluent::AddInsertIdsFilter).configure(conf, true)
-  end
-
-  def log_entry(index)
-    {
-      'id' => index,
-      'message' => TEST_MESSAGE
-    }
-  end
-end