fluent-plugin-stackdriver-monitoring 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0152a3a88115ba576c4fcc2deff4acfc308e96e8
-  data.tar.gz: 3caa5074ce36b5063bfd3645ca74ab4dbd230719
+  metadata.gz: 0840d73e9fb3fb65918eecaf78b02e6e969e2ad9
+  data.tar.gz: 48731da8b94611bb1fde363e3a9b8aaf82f04be6
 SHA512:
-  metadata.gz: c26c808d1bfc6bab1310e91f6605d495ad02ce1e9343f4287447db306ceab75f16c3daa4d9e5fb1345e9a395e6f1208803f01091f49c8a0847775d02ccd30d3b
-  data.tar.gz: de8ad272f8fc05772a52e01cade86e08f7dc232158a96cc458636e235bbf18235f642fb0461e1df97eeca263e0e8c34f59497812b6a6f8c3a4d4030fb2771668
+  metadata.gz: 35bd642a6ccd1883e438de46febffe778300c033efe3ee577cc411cef6438156990f92c347aa077fb298214cad97ee4145593cf7f917ae9c4ea4c7a95765adc5
+  data.tar.gz: 00ea14ff7803677ed925da418c3fc785f5381a3012c0dd0c4f614d269ee2b630692d307f0d2f514f1360fe76642acf8ecc5f096f4e70b4689562aabb1057864b

data/fluent-plugin-stackdriver-monitoring.gemspec

@@ -6,7 +6,7 @@ Gem::Specification.new do |gem|
   gem.license     = 'MIT'
   gem.homepage    = 'https://github.com/mia-0032/fluent-plugin-stackdriver-monitoring'
   gem.summary     = gem.description
-  gem.version     = '0.0.3'
+  gem.version     = '0.0.4'
   gem.authors     = ['Yoshihiro MIYAI']
   gem.email       = 'msparrow17@gmail.com'
   gem.has_rdoc    = false

data/lib/fluent/plugin/out_stackdriver_monitoring.rb

@@ -1,8 +1,5 @@
 require 'fluent/output'
-require 'google/api/metric_pb'
-require 'google/cloud/monitoring/v3/metric_service_client'
-require 'google/protobuf/repeated_field'
-require 'google/protobuf/timestamp_pb'
+require_relative 'stackdriver/client'
 
 module Fluent
   class StackdriverMonitoringOutput < BufferedOutput
@@ -18,6 +15,7 @@ module Fluent
     end
 
     TYPE_PREFIX = 'custom.googleapis.com/'.freeze
+    PAST_DATA_TIME_LIMIT = 60 * 60 * 24  # 24h
 
     def configure(conf)
       super
@@ -35,15 +33,12 @@ module Fluent
         end
       end
 
-      @project_name = Google::Cloud::Monitoring::V3::MetricServiceClient.project_path @project
-      @metric_name = Google::Cloud::Monitoring::V3::MetricServiceClient.metric_descriptor_path @project, @custom_metrics.type
+      @client = Fluent::StackdriverMonitoring::Writer.new @project, @custom_metrics, log
     end
 
     def start
       super
-
-      @metric_service_client = Google::Cloud::Monitoring::V3::MetricServiceClient.new
-      @metric_descriptor = create_metric_descriptor
+      @client.start
     end
 
     def format(tag, time, record)
@@ -51,18 +46,17 @@ module Fluent
     end
 
     def write(chunk)
-      chunk.msgpack_each do |tag, time, record|
-        time_series = create_time_series
-        value = record[@custom_metrics.key]
+      current_time = Time.now.to_i
 
-        point = Google::Monitoring::V3::Point.new
-        point.interval = create_time_interval time, @custom_metrics.time_interval
-        point.value = create_typed_value value
-        time_series.points.push point
+      chunk.msgpack_each do |tag, time, record|
+        if (current_time - time) >= PAST_DATA_TIME_LIMIT
+          log.warn 'Drop data point because it cannot be written more than 24h in the past', time: Time.at(time).to_s, metric_name: @metric_name
+          next
+        end
 
-        log.debug "Create time series", time: Time.at(time).to_s, value: value, metric_name: @metric_name
-        # Only one point can be written per TimeSeries per request.
-        @metric_service_client.create_time_series @project_name, [time_series]
+        value = record[@custom_metrics.key]
+        start_time = time - @custom_metrics.time_interval
+        @client.write start_time, time, value
       end
     end
 
@@ -70,61 +64,5 @@ module Fluent
     def is_custom_metric?(metric_type)
       metric_type.start_with? TYPE_PREFIX
     end
-
-    def create_metric_descriptor
-      begin
-        metric_descriptor = @metric_service_client.get_metric_descriptor(@metric_name)
-        log.info "Succeed to get metric descripter", metric_name: @metric_name
-        return metric_descriptor
-      rescue Google::Gax::RetryError
-        log.info "Failed to get metric descripter", metric_name: @metric_name
-      end
-
-      metric_descriptor = Google::Api::MetricDescriptor.new
-      metric_descriptor.type = @custom_metrics.type
-      metric_descriptor.metric_kind = @custom_metrics.metric_kind
-      metric_descriptor.value_type = @custom_metrics.value_type
-      metric_descriptor = @metric_service_client.create_metric_descriptor(@project_name, metric_descriptor)
-      log.info "Succeed to create metric descripter", metric_name: @metric_name
-
-      metric_descriptor
-    end
-
-    def create_time_series
-      time_series = Google::Monitoring::V3::TimeSeries.new
-
-      metric = Google::Api::Metric.new
-      metric.type = @metric_descriptor.type
-      time_series.metric = metric
-
-      time_series.metric_kind = @metric_descriptor.metric_kind
-      time_series.value_type = @metric_descriptor.value_type
-
-      time_series
-    end
-
-    def create_time_interval(time, interval)
-      time_interval = Google::Monitoring::V3::TimeInterval.new
-      time_interval.start_time = Google::Protobuf::Timestamp.new seconds: (time - interval)
-      time_interval.end_time = Google::Protobuf::Timestamp.new seconds: time
-
-      time_interval
-    end
-
-    def create_typed_value(value)
-      typed_value = Google::Monitoring::V3::TypedValue.new
-      case @metric_descriptor.value_type
-      when :BOOL
-        typed_value.bool_value = !!value
-      when :INT64
-        typed_value.int64_value = value.to_i
-      when :DOUBLE
-        typed_value.double_value = value.to_f
-      else
-        raise 'Unknown value_type!'
-      end
-
-      typed_value
-    end
   end
 end

data/lib/fluent/plugin/stackdriver/client.rb

@@ -0,0 +1,97 @@
+require 'google/api/metric_pb'
+require 'google/cloud/monitoring/v3/metric_service_client'
+require 'google/protobuf/repeated_field'
+require 'google/protobuf/timestamp_pb'
+
+module Fluent
+  module StackdriverMonitoring
+    class Error < StandardError
+    end
+
+    class Writer
+      def initialize(project, custom_metrics, log)
+        @custom_metrics = custom_metrics
+        @project_name = Google::Cloud::Monitoring::V3::MetricServiceClient.project_path project
+        @metric_name = Google::Cloud::Monitoring::V3::MetricServiceClient.metric_descriptor_path project, custom_metrics.type
+        @log = log
+      end
+
+      def start
+        @metric_service_client = Google::Cloud::Monitoring::V3::MetricServiceClient.new
+        @metric_descriptor = create_metric_descriptor
+      end
+
+      def write(start_time, end_time, value)
+        time_series = create_time_series
+        point = Google::Monitoring::V3::Point.new
+        point.interval = create_time_interval start_time, end_time
+        point.value = create_typed_value value
+        time_series.points.push point
+
+        log.debug "Create time series", start_time: Time.at(start_time).to_s, end_time: Time.at(end_time).to_s, value: value, metric_name: @metric_name
+        # Only one point can be written per TimeSeries per request.
+        @metric_service_client.create_time_series @project_name, [time_series]
+      end
+
+      private
+      def log
+        @log
+      end
+
+      def create_metric_descriptor
+        begin
+          metric_descriptor = @metric_service_client.get_metric_descriptor(@metric_name)
+          log.info "Succeed to get metric descripter", metric_name: @metric_name
+          return metric_descriptor
+        rescue Google::Gax::RetryError
+          log.info "Failed to get metric descripter", metric_name: @metric_name
+        end
+
+        metric_descriptor = Google::Api::MetricDescriptor.new
+        metric_descriptor.type = @custom_metrics.type
+        metric_descriptor.metric_kind = @custom_metrics.metric_kind
+        metric_descriptor.value_type = @custom_metrics.value_type
+        metric_descriptor = @metric_service_client.create_metric_descriptor(@project_name, metric_descriptor)
+        log.info "Succeed to create metric descripter", metric_name: @metric_name
+
+        metric_descriptor
+      end
+
+      def create_time_series
+        time_series = Google::Monitoring::V3::TimeSeries.new
+
+        metric = Google::Api::Metric.new
+        metric.type = @metric_descriptor.type
+
+        time_series.metric = metric
+        time_series.metric_kind = @metric_descriptor.metric_kind
+        time_series.value_type = @metric_descriptor.value_type
+
+        time_series
+      end
+
+      def create_time_interval(start_time, end_time)
+        time_interval = Google::Monitoring::V3::TimeInterval.new
+        time_interval.start_time = Google::Protobuf::Timestamp.new seconds: start_time
+        time_interval.end_time = Google::Protobuf::Timestamp.new seconds: end_time
+        time_interval
+      end
+
+      def create_typed_value(value)
+        typed_value = Google::Monitoring::V3::TypedValue.new
+        case @metric_descriptor.value_type
+        when :BOOL
+          typed_value.bool_value = !!value
+        when :INT64
+          typed_value.int64_value = value.to_i
+        when :DOUBLE
+          typed_value.double_value = value.to_f
+        else
+          raise Error.new 'Unknown value_type!'
+        end
+
+        typed_value
+      end
+    end
+  end
+end

data/test/plugin/test_out_stackdriver_monitoring.rb

@@ -4,7 +4,7 @@ class StackdriverMonitoringOutputTest < Test::Unit::TestCase
   CONFIG = %[
     project project-test
     <custom_metrics>
-      key hoge_key
+      key test_key
       type custom.googleapis.com/hoge_type
       metric_kind GAUGE
       value_type INT64
@@ -24,7 +24,7 @@ class StackdriverMonitoringOutputTest < Test::Unit::TestCase
       d = create_driver
 
       assert_equal('project-test', d.instance.project)
-      assert_equal('hoge_key', d.instance.custom_metrics.key)
+      assert_equal('test_key', d.instance.custom_metrics.key)
       assert_equal('custom.googleapis.com/hoge_type', d.instance.custom_metrics.type)
       assert_equal(:GAUGE, d.instance.custom_metrics.metric_kind)
       assert_equal(:INT64, d.instance.custom_metrics.value_type)
@@ -36,7 +36,7 @@ class StackdriverMonitoringOutputTest < Test::Unit::TestCase
         create_driver(%[
           project project-test
           <custom_metrics>
-            key hoge_key
+            key test_key
             type invalid_type
             metric_kind GAUGE
             value_type INT64
@@ -50,7 +50,7 @@ class StackdriverMonitoringOutputTest < Test::Unit::TestCase
         create_driver(%[
           project project-test
           <custom_metrics>
-            key hoge_key
+            key test_key
             type invalid_type
             metric_kind CUMULATIVE
             value_type INT64
@@ -65,7 +65,7 @@ class StackdriverMonitoringOutputTest < Test::Unit::TestCase
         create_driver(%[
           project project-test
           <custom_metrics>
-            key hoge_key
+            key test_key
             type invalid_type
             metric_kind CUMULATIVE
             value_type BOOL
@@ -74,4 +74,48 @@ class StackdriverMonitoringOutputTest < Test::Unit::TestCase
       end
     end
   end
+
+  sub_test_case 'write' do
+    setup do
+      @client = mock!
+      @client.start.once
+      stub(Fluent::StackdriverMonitoring::Writer).new(anything, anything, anything).once { @client }
+    end
+
+    test 'basic data' do
+      now = Time.now.to_i
+      @client.write(now, now, 1).once
+
+      d = create_driver
+      d.emit({'test_key' => 1}, now)
+      d.run
+    end
+
+    test 'data with time_interval' do
+      now = Time.now.to_i
+      @client.write(now - 10, now, 1).once
+
+      d = create_driver(%[
+        project project-test
+        <custom_metrics>
+          key test_key
+          type custom.googleapis.com/hoge_type
+          metric_kind GAUGE
+          value_type INT64
+          time_interval 10s
+        </custom_metrics>
+      ])
+      d.emit({'test_key' => 1}, now)
+      d.run
+    end
+
+    test 'past data are dropped' do
+      past_time = Time.now.to_i - (60 * 60 * 24 + 1)
+      @client.write.times(0)
+
+      d = create_driver
+      d.emit({'test_key' => 1}, past_time)
+      d.run
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-stackdriver-monitoring
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - Yoshihiro MIYAI
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-04-14 00:00:00.000000000 Z
+date: 2017-04-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -115,6 +115,7 @@ files:
 - Rakefile
 - fluent-plugin-stackdriver-monitoring.gemspec
 - lib/fluent/plugin/out_stackdriver_monitoring.rb
+- lib/fluent/plugin/stackdriver/client.rb
 - test/plugin/test_out_stackdriver_monitoring.rb
 - test/test_helper.rb
 homepage: https://github.com/mia-0032/fluent-plugin-stackdriver-monitoring