fluent-plugin-google-cloud 0.6.4.pre.3 → 0.6.4

data/test/plugin/constants.rb

@@ -23,7 +23,6 @@ module Constants
   # Attributes used for the GCE metadata service.
   PROJECT_ID = 'test-project-id'
   ZONE = 'us-central1-b'
-  ZONE2 = 'us-central1-c'
   FULLY_QUALIFIED_ZONE = 'projects/' + PROJECT_ID + '/zones/' + ZONE
   VM_ID = '9876543210'
 
@@ -52,19 +51,6 @@ module Constants
   MANAGED_VM_BACKEND_NAME = 'default'
   MANAGED_VM_BACKEND_VERSION = 'guestbook2.0'
 
-  # Docker Container labels.
-  DOCKER_CONTAINER_ID = '0d0f03ff8d3c42688692536d1af77a28cd135c0a5c531f25a31'
-  DOCKER_CONTAINER_ID2 = '42688692536d1af77a28cd135c0a5c531f25a0317d37da427e3'
-  DOCKER_CONTAINER_NAME = 'happy_hippo'
-  DOCKER_CONTAINER_NAME2 = 'sad_panda'
-  DOCKER_CONTAINER_STREAM_STDOUT = 'stdout'
-  DOCKER_CONTAINER_STREAM_STDERR = 'stderr'
-  # Timestamp for 1234567890 seconds and 987654321 nanoseconds since epoch.
-  DOCKER_CONTAINER_TIMESTAMP = '2009-02-13T23:31:30.987654321Z'
-  DOCKER_CONTAINER_SECONDS_EPOCH = 1_234_567_890
-  DOCKER_CONTAINER_NANOS = 987_654_321
-  DOCKER_CONTAINER_APPLICATION_LOG_TAG = 'nginx-access-log'
-
   # Container Engine / Kubernetes specific labels.
   CONTAINER_CLUSTER_NAME = 'cluster-1'
   CONTAINER_NAMESPACE_ID = '898268c8-4a36-11e5-9d81-42010af0194c'
@@ -154,12 +140,9 @@ module Constants
     detect_subservice false
   )
 
-  ENABLE_METADATA_AGENT_CONFIG = %(
-    enable_metadata_agent true
-  )
-
-  DISABLE_METADATA_AGENT_CONFIG = %(
-    enable_metadata_agent false
+  PROMETHEUS_ENABLE_CONFIG = %(
+    enable_monitoring true
+    monitoring_type prometheus
   )
 
   CUSTOM_METADATA_CONFIG = %(
@@ -214,8 +197,6 @@ module Constants
   )
 
   # Service configurations for various services.
-
-  # GCE.
   COMPUTE_PARAMS = {
     resource: {
       type: COMPUTE_CONSTANTS[:resource_type],
@@ -231,7 +212,6 @@ module Constants
     }
   }
 
-  # GAE.
   VMENGINE_PARAMS = {
     resource: {
       type: APPENGINE_CONSTANTS[:resource_type],
@@ -249,7 +229,6 @@ module Constants
     }
   }
 
-  # GKE Container.
   CONTAINER_TAG = "kubernetes.#{CONTAINER_POD_NAME}_" \
                   "#{CONTAINER_NAMESPACE_NAME}_#{CONTAINER_CONTAINER_NAME}"
 
@@ -302,60 +281,6 @@ module Constants
     }
   }
 
-  # Docker Container.
-  DOCKER_CONTAINER_TAG = "container.#{DOCKER_CONTAINER_ID}." \
-                         "#{DOCKER_CONTAINER_NAME}"
-  DOCKER_CONTAINER_TAG_WITH_APPLICATION =
-    "application-container.#{DOCKER_CONTAINER_NAME}." \
-    "#{DOCKER_CONTAINER_APPLICATION_LOG_TAG}"
-
-  DOCKER_CONTAINER_PARAMS = {
-    resource: {
-      type: DOCKER_CONSTANTS[:resource_type],
-      labels: {
-        'container_id' => DOCKER_CONTAINER_ID,
-        'location' => ZONE
-      }
-    },
-    log_name: "container.#{DOCKER_CONTAINER_ID}.#{DOCKER_CONTAINER_NAME}",
-    project_id: PROJECT_ID,
-    labels: {
-      "#{COMPUTE_CONSTANTS[:service]}/resource_id" => VM_ID,
-      "#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME,
-      "#{DOCKER_CONSTANTS[:service]}/container_name" => DOCKER_CONTAINER_NAME
-    }
-  }
-  DOCKER_CONTAINER_PARAMS_WITH_STREAM_STDOUT = DOCKER_CONTAINER_PARAMS.merge(
-    labels: DOCKER_CONTAINER_PARAMS[:labels].merge(
-      "#{DOCKER_CONSTANTS[:service]}/stream" => DOCKER_CONTAINER_STREAM_STDOUT
-    )
-  )
-  DOCKER_CONTAINER_PARAMS_WITH_METADATA_OVERWRITTEN = \
-    DOCKER_CONTAINER_PARAMS.merge(
-      resource: DOCKER_CONTAINER_PARAMS[:resource].merge(
-        labels: DOCKER_CONTAINER_PARAMS[:resource][:labels].merge(
-          'container_id' => DOCKER_CONTAINER_ID2
-        )
-      ),
-      labels: DOCKER_CONTAINER_PARAMS[:labels].merge(
-        "#{DOCKER_CONSTANTS[:service]}/container_name" =>
-          DOCKER_CONTAINER_NAME2,
-        "#{DOCKER_CONSTANTS[:service]}/stream" =>
-          DOCKER_CONTAINER_STREAM_STDERR
-      )
-    )
-  DOCKER_CONTAINER_PARAMS_WITH_ZONE2 = DOCKER_CONTAINER_PARAMS.merge(
-    resource: DOCKER_CONTAINER_PARAMS[:resource].merge(
-      labels: DOCKER_CONTAINER_PARAMS[:resource][:labels].merge(
-        'location' => ZONE2
-      )
-    )
-  )
-  DOCKER_CONTAINER_WITH_APPLICATION_PARAMS = DOCKER_CONTAINER_PARAMS.merge(
-    log_name: DOCKER_CONTAINER_TAG_WITH_APPLICATION
-  )
-
-  # Cloud Functions.
   CLOUDFUNCTIONS_TAG = "kubernetes.#{CLOUDFUNCTIONS_POD_NAME}_" \
                         "#{CLOUDFUNCTIONS_NAMESPACE_NAME}_" \
                         "#{CLOUDFUNCTIONS_CONTAINER_NAME}"
@@ -401,7 +326,6 @@ module Constants
     }
   }
 
-  # Cloud Dataflow.
   DATAFLOW_PARAMS = {
     resource: {
       type: DATAFLOW_CONSTANTS[:resource_type],
@@ -421,7 +345,6 @@ module Constants
     }
   }
 
-  # Cloud Dataproc.
   DATAPROC_PARAMS = {
     resource: {
       type: DATAPROC_CONSTANTS[:resource_type],
@@ -440,7 +363,6 @@ module Constants
     }
   }
 
-  # Cloud ML.
   ML_PARAMS = {
     resource: {
       type: ML_CONSTANTS[:resource_type],
@@ -527,41 +449,4 @@ module Constants
     '' => '_'
   }
   ALL_TAGS = VALID_TAGS.merge(INVALID_TAGS)
-
-  # Stub value for Monitored resources from Metadata Agent.
-  MONITORED_RESOURCE_STUBS = {
-    # Implicit GCE instance.
-    IMPLICIT_MONITORED_RESOURCE_UNIQUE_KEY => {
-      'unique_id' => IMPLICIT_MONITORED_RESOURCE_UNIQUE_KEY,
-      'monitored_resource' => '{' \
-        "\"type\": \"#{COMPUTE_CONSTANTS[:resource_type]}\"," \
-        '"labels": {' \
-          "\"zone\": \"#{ZONE}\"," \
-          "\"instance_id\": \"#{VM_ID}\"" \
-        '}' \
-      '}'
-    },
-    # Docker container stderr / stdout logs.
-    DOCKER_CONSTANTS[:resource_type] => {
-      'unique_id' => "container.#{DOCKER_CONTAINER_ID}",
-      'monitored_resource' => '{' \
-        "\"type\": \"#{DOCKER_CONSTANTS[:resource_type]}\"," \
-        '"labels": {' \
-          "\"location\": \"#{ZONE2}\"," \
-          "\"container_id\": \"#{DOCKER_CONTAINER_ID}\"" \
-        '}' \
-      '}'
-    },
-    # Docker container application logs.
-    "#{DOCKER_CONSTANTS[:resource_type]}_application" => {
-      'unique_id' => "containerName.#{DOCKER_CONTAINER_NAME}",
-      'monitored_resource' => '{' \
-        "\"type\": \"#{DOCKER_CONSTANTS[:resource_type]}\"," \
-        '"labels": {' \
-          "\"location\": \"#{ZONE}\"," \
-          "\"container_id\": \"#{DOCKER_CONTAINER_ID}\"" \
-        '}' \
-      '}'
-    }
-  }
 end

data/test/plugin/test_out_google_cloud.rb

@@ -70,6 +70,54 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     assert_equal 1, exception_count
   end
 
+  # TODO: The code in the non-gRPC and gRPC tests is nearly identical.
+  # Refactor and remove duplication.
+  # TODO: Use status codes instead of int literals.
+  def test_prometheus_metrics
+    setup_gce_metadata_stubs
+    [
+      # Single successful request.
+      [200, 1, 1, [1, 0, 1, 0]],
+      # Several successful requests.
+      [200, 2, 1, [2, 0, 2, 0]],
+      # Single successful request with several entries.
+      [200, 1, 2, [1, 0, 2, 0]],
+      # Single failed request that causes logs to be dropped.
+      [401, 1, 1, [0, 1, 0, 1]],
+      # Single failed request that escalates without logs being dropped.
+      [500, 1, 1, [0, 1, 0, 0]]
+    ].each do |code, request_count, entry_count, metric_values|
+      setup_prometheus
+      # TODO: Do this as part of setup_logging_stubs.
+      stub_request(:post, WRITE_LOG_ENTRIES_URI)
+        .to_return(status: code, body: 'Some Message')
+      (1..request_count).each do
+        d = create_driver(PROMETHEUS_ENABLE_CONFIG)
+        (1..entry_count).each do |i|
+          d.emit('message' => log_entry(i.to_s))
+        end
+        # rubocop:disable Lint/HandleExceptions
+        begin
+          d.run
+        rescue Google::Apis::AuthorizationError
+        rescue Google::Apis::ServerError
+        end
+        # rubocop:enable Lint/HandleExceptions
+      end
+      successful_requests_count, failed_requests_count,
+        ingested_entries_count, dropped_entries_count = metric_values
+      assert_prometheus_metric_value(:stackdriver_successful_requests_count,
+                                     successful_requests_count, grpc: false)
+      assert_prometheus_metric_value(:stackdriver_failed_requests_count,
+                                     failed_requests_count,
+                                     grpc: false, code: code)
+      assert_prometheus_metric_value(:stackdriver_ingested_entries_count,
+                                     ingested_entries_count)
+      assert_prometheus_metric_value(:stackdriver_dropped_entries_count,
+                                     dropped_entries_count)
+    end
+  end
+
   # This test looks similar between the grpc and non-grpc paths except that when
   # parsing "105", the grpc path responds with "DEBUG", while the non-grpc path
   # responds with "100".

data/test/plugin/test_out_google_cloud_grpc.rb

@@ -72,6 +72,53 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
     end
   end
 
+  # TODO: The code in the non-gRPC and gRPC tests is nearly identical.
+  # Refactor and remove duplication.
+  # TODO: Use status codes instead of int literals.
+  def test_prometheus_metrics
+    setup_gce_metadata_stubs
+    [
+      # Single successful request.
+      [false, 0, 1, 1, [1, 0, 1, 0]],
+      # Several successful requests.
+      [false, 0, 2, 1, [2, 0, 2, 0]],
+      # Single successful request with several entries.
+      [false, 0, 1, 2, [1, 0, 2, 0]],
+      # Single failed request that causes logs to be dropped.
+      [true, 16, 1, 1, [0, 1, 0, 1]],
+      # Single failed request that escalates without logs being dropped.
+      [true, 13, 1, 1, [0, 1, 0, 0]]
+    ].each do |should_fail, code, request_count, entry_count, metric_values|
+      setup_prometheus
+      setup_logging_stubs(should_fail, code, 'SomeMessage') do
+        (1..request_count).each do
+          d = create_driver(USE_GRPC_CONFIG + PROMETHEUS_ENABLE_CONFIG, 'test',
+                            GRPCLoggingMockFailingService.rpc_stub_class)
+          (1..entry_count).each do |i|
+            d.emit('message' => log_entry(i.to_s))
+          end
+          # rubocop:disable Lint/HandleExceptions
+          begin
+            d.run
+          rescue GRPC::BadStatus
+          end
+          # rubocop:enable Lint/HandleExceptions
+        end
+      end
+      successful_requests_count, failed_requests_count,
+        ingested_entries_count, dropped_entries_count = metric_values
+      assert_prometheus_metric_value(:stackdriver_successful_requests_count,
+                                     successful_requests_count, grpc: true)
+      assert_prometheus_metric_value(:stackdriver_failed_requests_count,
+                                     failed_requests_count,
+                                     grpc: true, code: code)
+      assert_prometheus_metric_value(:stackdriver_ingested_entries_count,
+                                     ingested_entries_count)
+      assert_prometheus_metric_value(:stackdriver_dropped_entries_count,
+                                     dropped_entries_count)
+    end
+  end
+
   # This test looks similar between the grpc and non-grpc paths except that when
   # parsing "105", the grpc path responds with "DEBUG", while the non-grpc path
   # responds with "100".

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-google-cloud
 version: !ruby/object:Gem::Version
-  version: 0.6.4.pre.3
+  version: 0.6.4
 platform: ruby
 authors:
 - Todd Derr
@@ -9,22 +9,8 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-07-25 00:00:00.000000000 Z
+date: 2017-07-11 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: excon
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.57.1
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.57.1
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
@@ -199,6 +185,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: prometheus-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.1
 description: |2
      Fluentd output plugin for the Stackdriver Logging API, which will make
      logs viewable in the Developer Console's log viewer and can optionally
@@ -212,10 +212,12 @@ extra_rdoc_files: []
 files:
 - CONTRIBUTING
 - Gemfile
+- Gemfile.lock
 - LICENSE
 - README.rdoc
 - Rakefile
 - fluent-plugin-google-cloud.gemspec
+- lib/fluent/plugin/monitoring.rb
 - lib/fluent/plugin/out_google_cloud.rb
 - test/helper.rb
 - test/plugin/base_test.rb
@@ -241,12 +243,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.8
+rubygems_version: 2.6.12
 signing_key: 
 specification_version: 4
 summary: fluentd output plugin for the Stackdriver Logging API