fluent-plugin-stackdriver-monitoring 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e30388f7688cbb1f15ba85437d4ea5dd4402527f
+  data.tar.gz: f957f5d472d0fa713cdeaff4852aadc79784103a
+SHA512:
+  metadata.gz: a0634a724daddae317b894aa131691ce6f29713062e91821a7bd2543a3b98fcb3206a8d88899519e8e62d0f68ba78f7275b2cbf6a4baa95e626c17cd56348302
+  data.tar.gz: ee4aceb22ae5c4327291b834771d65821be38fd8251dbd224c36a437205f99e717aa4161a9b4e41de8fe419d8aac6c1613f2d402654c7ed7cc61e9d53749883d

data/.gitignore

@@ -0,0 +1,50 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/spec/examples.txt
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+# Used by dotenv library to load environment variables.
+# .env
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+*.bridgesupport
+build-iPhoneOS/
+build-iPhoneSimulator/
+
+## Specific to RubyMotion (use of CocoaPods):
+#
+# We recommend against adding the Pods directory to your .gitignore. However
+# you should judge for yourself, the pros and cons are mentioned at:
+# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
+#
+# vendor/Pods/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalization:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+Gemfile.lock
+.ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc

data/.travis.yml

@@ -0,0 +1,30 @@
+language: ruby
+
+rvm:
+  - 2.0
+  - 2.1
+  - 2.2.5
+  - 2.3.1
+  - ruby-head
+  - rbx
+
+gemfile:
+ - Gemfile
+ - Gemfile.v0.12
+
+branches:
+  only:
+    - master
+
+before_install: gem update bundler
+script: bundle exec rake test
+
+sudo: false
+
+matrix:
+  allow_failures:
+    - rvm: ruby-head
+    - rvm: rbx
+  exclude:
+    - rvm: 2.0
+      gemfile: Gemfile

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/Gemfile.v0.12

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+
+gem 'fluentd', '~> 0.12.0'
+
+gemspec

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2016 Yoshihiro MIYAI
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,52 @@
+# fluent-plugin-stackdriver-monitoring
+
+[Stackdriver Monitoring](https://cloud.google.com/monitoring/) custom metrics output plugin for [Fluentd](http://www.fluentd.org/)
+
+## Installation
+
+```shell
+$ gem install fluent-plugin-stackdriver-monitoring
+```
+
+## Authentication
+
+See [google-cloud-monitoring gem document](https://github.com/GoogleCloudPlatform/google-cloud-ruby/tree/master/google-cloud-monitoring#setup-authentication).
+
+## Configuration
+
+Sample configuration is below.
+
+```
+<match your.tag>
+  @type stackdriver_monitoring
+  project {{PROJECT_NAME}}
+
+  <custom_metrics>
+    key {{KEY_NAME}}
+    type custom.googleapis.com/{{METRICS_NAME}}
+    metric_kind GAUGE
+    value_type INT64
+  </custom_metrics>
+
+  flush_interval 1s  # must be 1(sec) or above
+</match>
+```
+
+- project (string, required)
+  - Set your Stackdriver project id.
+- <custom_metrics>
+  - key (string, required)
+    - Specify field name in your log to send to Stackdriver.
+  - type (string, required)
+    - Set name of descriptor. It must start with `custom.googleapis.com/`.
+  - metric_kind(enum, required)
+    - See [metric kind](https://cloud.google.com/monitoring/api/ref_v3/rest/v3/projects.metricDescriptors#MetricKind).
+    - You can specify `GAUGE`, `DELTA` or `CUMULATIVE`.
+  - value_type(enum, required)
+    - See [value type](https://cloud.google.com/monitoring/api/ref_v3/rest/v3/projects.metricDescriptors#valuetype).
+    - You can specify `BOOL`, `INT64`, `DOUBLE` or `STRING`.
+
+## TODO
+
+- Add test!
+- Support Unit and Monitored Resource in custom_metrics.

data/Rakefile

@@ -0,0 +1,12 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/plugin/test_*.rb']
+  test.verbose = true
+end
+
+task :default => [:build]

data/fluent-plugin-stackdriver-monitoring.gemspec

@@ -0,0 +1,25 @@
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = 'fluent-plugin-stackdriver-monitoring'
+  gem.description = 'Stackdriver Monitoring custom metrics output plugin for Fluentd'
+  gem.license     = 'MIT'
+  gem.homepage    = 'https://github.com/mia-0032/fluent-plugin-stackdriver-monitoring'
+  gem.summary     = gem.description
+  gem.version     = '0.0.1'
+  gem.authors     = ['Yoshihiro MIYAI']
+  gem.email       = 'msparrow17@gmail.com'
+  gem.has_rdoc    = false
+  gem.files       = `git ls-files`.split("\n")
+  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_runtime_dependency 'fluentd', ['>= 0.12.0', '< 2']
+  gem.add_runtime_dependency 'google-cloud-monitoring', '~> 0.21.0'
+
+  gem.add_development_dependency 'bundler'
+  gem.add_development_dependency 'rake'
+  gem.add_development_dependency 'test-unit'
+  gem.add_development_dependency 'test-unit-rr'
+end

data/lib/fluent/plugin/out_stackdriver_monitoring.rb

@@ -0,0 +1,118 @@
+require 'fluent/output'
+require 'google/api/metric_pb'
+require 'google/cloud/monitoring/v3/metric_service_api'
+require 'google/protobuf/repeated_field'
+require 'google/protobuf/timestamp_pb'
+
+module Fluent
+  class StackdriverMonitoringOutput < BufferedOutput
+    Fluent::Plugin.register_output('stackdriver_monitoring', self)
+
+    config_param :project, :string
+    config_section :custom_metrics, required: true, multi: false do
+      config_param :key, :string
+      config_param :type, :string
+      config_param :metric_kind, :enum, list: [:GAUGE, :DELTA, :CUMULATIVE]
+      config_param :value_type, :enum, list: [:BOOL, :INT64, :DOUBLE, :STRING] # todo: implement :DISTRIBUTION, :MONEY
+    end
+
+    TYPE_PREFIX = 'custom.googleapis.com/'.freeze
+
+    def configure(conf)
+      super
+
+      unless @custom_metrics.type.start_with? TYPE_PREFIX
+        raise "custom_metrics.type must start with \"#{TYPE_PREFIX}\""
+      end
+
+      @project_name = Google::Cloud::Monitoring::V3::MetricServiceApi.project_path @project
+      @metric_name = Google::Cloud::Monitoring::V3::MetricServiceApi.metric_descriptor_path @project, @custom_metrics.type
+    end
+
+    def start
+      super
+
+      @metric_service_api = Google::Cloud::Monitoring::V3::MetricServiceApi.new
+      @metric_descriptor = create_metric_descriptor
+    end
+
+    def format(tag, time, record)
+      [tag, time, record].to_msgpack
+    end
+
+    def write(chunk)
+      chunk.msgpack_each do |tag, time, record|
+        time_series = create_time_series
+        value = record[@custom_metrics.key]
+
+        point = Google::Monitoring::V3::Point.new
+        point.interval = create_time_interval time
+        point.value = create_typed_value value
+        time_series.points.push point
+
+        log.debug "Create time series", time: Time.at(time).to_s, value: value, metric_name: @metric_name
+        # Only one point can be written per TimeSeries per request.
+        @metric_service_api.create_time_series @project_name, [time_series]
+      end
+    end
+
+    private
+    def create_metric_descriptor
+      begin
+        metric_descriptor = @metric_service_api.get_metric_descriptor(@metric_name)
+        log.info "Succeed to get metric descripter", metric_name: @metric_name
+        return metric_descriptor
+      rescue Google::Gax::RetryError
+        log.info "Failed to get metric descripter", metric_name: @metric_name
+      end
+
+      metric_descriptor = Google::Api::MetricDescriptor.new
+      metric_descriptor.type = @custom_metrics.type
+      metric_descriptor.metric_kind = @custom_metrics.metric_kind
+      metric_descriptor.value_type = @custom_metrics.value_type
+      metric_descriptor = @metric_service_api.create_metric_descriptor(@project_name, metric_descriptor)
+      log.info "Succeed to create metric descripter", metric_name: @metric_name
+
+      metric_descriptor
+    end
+
+    def create_time_series
+      time_series = Google::Monitoring::V3::TimeSeries.new
+
+      metric = Google::Api::Metric.new
+      metric.type = @metric_descriptor.type
+      time_series.metric = metric
+
+      time_series.metric_kind = @metric_descriptor.metric_kind
+      time_series.value_type = @metric_descriptor.value_type
+
+      time_series
+    end
+
+    def create_time_interval(time)
+      time_interval = Google::Monitoring::V3::TimeInterval.new
+      time_interval.start_time = Google::Protobuf::Timestamp.new seconds: time
+      time_interval.end_time = Google::Protobuf::Timestamp.new seconds: time
+
+      time_interval
+    end
+
+    def create_typed_value(value)
+      typed_value = Google::Monitoring::V3::TypedValue.new
+      case @metric_descriptor.value_type
+      when :BOOL
+        typed_value.bool_value = value.to_bool
+      when :INT64
+        typed_value.int64_value = value.to_i
+      when :DOUBLE
+        typed_value.double_value = value.to_f
+      when :STRING
+        typed_value.string_value = value.to_s
+      else
+        raise 'Unknown value_type!'
+      end
+
+      typed_value
+    end
+  end
+end

metadata

@@ -0,0 +1,142 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-stackdriver-monitoring
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Yoshihiro MIYAI
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-11-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.12.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.12.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: google-cloud-monitoring
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.21.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.21.0
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit-rr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Stackdriver Monitoring custom metrics output plugin for Fluentd
+email: msparrow17@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- Gemfile.v0.12
+- LICENSE
+- README.md
+- Rakefile
+- fluent-plugin-stackdriver-monitoring.gemspec
+- lib/fluent/plugin/out_stackdriver_monitoring.rb
+homepage: https://github.com/mia-0032/fluent-plugin-stackdriver-monitoring
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Stackdriver Monitoring custom metrics output plugin for Fluentd
+test_files: []