observed 0.1.1

data/plugins/observed-fluentd/features/plugin.feature

@@ -0,0 +1,61 @@
+Feature: Receives Observed's input and send it to Fluentd
+
+  In order to pass data from Observed to Fluentd
+
+  I want to configure Observed to use observed-fluentd plugin
+
+  Scenario: Write configuration files for Observed and Fluentd, then run Fluentd, and then run Observed
+    Given a file named "test.rb" with:
+    """
+    require 'observed'
+    require 'observed/http'
+    require 'observed/fluentd'
+
+    include Observed
+
+    observe 'myservice', via: 'http', with: {
+      method: 'get',
+      url: 'http://google.com/',
+      timeout_in_milliseconds: 3000,
+    }
+
+    report /myservice.*/, via: 'fluentd', with: {
+      host: 'localhost',
+      port: 24224,
+      tag: 'debug.myservice'
+    }
+
+    run 'myservice'
+    """
+    Given a file named "fluent.conf" with:
+    """
+    <source>
+      type forward
+    </source>
+
+    <match debug.**>
+      type file
+      path fluent.out
+      time_slice_format foo
+      utc
+      flush_interval 1s
+    </match>
+    """
+    When I run `pwd`
+    When I run `fluentd -d pid_file -c fluent.conf`
+#    When I start my daemon with "fluentd -c tmp/aruba/fluent.conf"
+    When I run `sleep 3`
+    When I run `ruby test.rb`
+    #Then a daemon called "fluentd" should be running
+    When I run `sleep 3`
+    When I run the command "kill $(cat tmp/aruba/pid_file)"
+    When I run `cat fluent.out.foo_0.log`
+    Then the output should contain:
+    """
+    elapsed_time
+    """
+    Then the output should contain:
+    """
+    debug.myservice
+    """
+

data/plugins/observed-fluentd/features/support/env.rb

@@ -0,0 +1,32 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__) + '/../../lib')
+require 'aruba/cucumber'
+
+World(Aruba::Api)
+
+Before do
+  @aruba_timeout_seconds = 120
+end
+
+When /^I run the command "([^"]+)"$/ do |cmd|
+  system cmd
+end
+
+When /^I start my daemon with "([^"]*)"$/ do |cmd|
+  @root = Pathname.new(File.dirname(__FILE__)).parent.parent.expand_path
+  command = "#{@root.join('bin')}/#{cmd}"
+
+  puts "In the working directory: #{Dir.pwd}"
+  puts "Running #{command}"
+
+  @pipe = IO.popen(command, "r")
+  sleep 2 # so the daemon has a chance to boot
+
+  # clean up the daemon when the tests finish
+  at_exit do
+    Process.kill("KILL", @pipe.pid)
+  end
+end
+
+Then /^a daemon called "([^"]*)" should be running$/ do |daemon|
+  `ps -eo command |grep #{daemon}`.size.should > 0
+end

data/plugins/observed-fluentd/fluent.d/fluent.conf

@@ -0,0 +1,93 @@
+
+## built-in TCP input
+## $ echo <json> | fluent-cat <tag>
+<source>
+  type forward
+</source>
+
+## built-in UNIX socket input
+#<source>
+#  type unix
+#</source>
+
+# HTTP input
+# http://localhost:8888/<tag>?json=<json>
+<source>
+  type http
+  port 8888
+</source>
+
+## File input
+## read apache logs with tag=apache.access
+#<source>
+#  type tail
+#  format apache
+#  path /var/log/httpd-access.log
+#  tag apache.access
+#</source>
+
+# Listen HTTP for monitoring
+# http://localhost:24220/api/plugins
+# http://localhost:24220/api/plugins?type=TYPE
+# http://localhost:24220/api/plugins?tag=MYTAG
+<source>
+  type monitor_agent
+  port 24220
+</source>
+
+# Listen DRb for debug
+<source>
+  type debug_agent
+  port 24230
+</source>
+
+
+## match tag=apache.access and write to file
+#<match apache.access>
+#  type file
+#  path /var/log/fluent/access
+#</match>
+
+## match tag=debug.** and dump to console
+<match debug.**>
+  type stdout
+</match>
+
+# match tag=system.** and forward to another fluent server
+<match system.**>
+  type forward
+  host 192.168.0.11
+  <secondary>
+    host 192.168.0.12
+  </secondary>
+</match>
+
+## match tag=myapp.** and forward and write to file
+#<match myapp.**>
+#  type copy
+#  <store>
+#    type forward
+#    host 192.168.0.13
+#    buffer_type file
+#    buffer_path /var/log/fluent/myapp-forward
+#    retry_limit 50
+#    flush_interval 10s
+#  </store>
+#  <store>
+#    type file
+#    path /var/log/fluent/myapp
+#  </store>
+#</match>
+
+## match fluent's internal events
+#<match fluent.**>
+#  type null
+#</match>
+
+## match not matched logs and write to file
+#<match **>
+#  type file
+#  path /var/log/fluent/else
+#  compress gz
+#</match>
+

data/plugins/observed-fluentd/fluentd.pid

@@ -0,0 +1 @@
+3921

data/plugins/observed-fluentd/lib/observed/fluentd/version.rb

@@ -0,0 +1,5 @@
+module Observed
+  class Fluentd
+    VERSION = "0.1.0"
+  end
+end

data/plugins/observed-fluentd/lib/observed/fluentd.rb

@@ -0,0 +1,30 @@
+require 'observed/fluentd/version'
+require 'observed/reporter'
+require 'observed/reporter/regexp_matching'
+require 'fluent-logger'
+
+module Observed
+  module Plugins
+    class Fluentd < Observed::Reporter
+
+      include Observed::Reporter::RegexpMatching
+
+      plugin_name 'fluentd'
+
+      attribute :tag
+      attribute :host
+      attribute :port, default: 24224
+      attribute :transform, default: ->(data){ data }
+
+      def report(tag, time, data)
+        fluent_logger.post(self.tag, transform.call(data))
+      end
+
+      private
+
+      def fluent_logger
+        @fluent_logger ||= Fluent::Logger::FluentLogger.new(nil, host: host, port: port)
+      end
+    end
+  end
+end

data/plugins/observed-fluentd/observe.d/clockwork.rb

@@ -0,0 +1,9 @@
+require 'clockwork'
+require 'observed/clockwork'
+
+include Clockwork
+include Observed::Clockwork
+
+observed :config_file => 'observed.conf'
+
+every(10.seconds, 'myservice')

data/plugins/observed-fluentd/observe.d/observed.conf

@@ -0,0 +1,17 @@
+require 'observed/builtin_plugins'
+require 'observed/http'
+require 'observed/fluentd'
+
+observe 'myservice', {
+  plugin: 'http',
+  method: 'get',
+  url: 'http://localhost/',
+  timeout_in_milliseconds: 3000,
+}
+
+match /myservice.*/, {
+  plugin: 'fluentd',
+  host: 'localhost',
+  port: 24224,
+  tag: 'debug.myservice'
+}

data/plugins/observed-fluentd/observed-fluentd.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'observed/fluentd/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'observed-fluentd'
+  spec.version       = Observed::Fluentd::VERSION
+  spec.authors       = ['KUOKA Yusuke']
+  spec.email         = %w(yusuke.kuoka@gree.net)
+  spec.description   = %q{Observed Fluentd Output Plugin}
+  spec.summary       = %q{observed-fluentd is an Observed output plugin for sending observed data to Fluentd}
+  spec.homepage      = ''
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = %w(lib)
+
+  spec.add_dependency 'fluent-logger', '~> 0.4.6'
+
+  spec.add_development_dependency 'bundler', '~> 1.3'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'rspec'
+end

data/plugins/observed-gauge/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*~

data/plugins/observed-gauge/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in observed-gauge.gemspec
+gemspec

data/plugins/observed-gauge/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 GREE, Inc.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/plugins/observed-gauge/README.md

@@ -0,0 +1,29 @@
+# Observed::Gauge
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'observed-gauge'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install observed-gauge
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/plugins/observed-gauge/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/plugins/observed-gauge/lib/observed/gauge/version.rb

@@ -0,0 +1,5 @@
+module Observed
+  module Gauge
+    VERSION = "0.1.0"
+  end
+end

data/plugins/observed-gauge/lib/observed/gauge.rb

@@ -0,0 +1,117 @@
+require 'observed/reporter'
+require 'observed/reporter/regexp_matching'
+require 'observed/gauge/version'
+require 'logger'
+require 'rrd'
+
+module Observed
+  module Plugins
+    class Gauge < Observed::Reporter
+
+      plugin_name 'gauge'
+
+      include Observed::Reporter::RegexpMatching
+
+      attribute :tag
+      attribute :key_path
+      attribute :coerce, default: ->(data){ data }
+      attribute :rrd
+      attribute :step
+      attribute :period
+
+      def report(tag, time, data)
+        rewrote = update_value_for_key_path(data, key_path) do |v|
+          sample = coerce.call(v)
+          average = get_cdp_updated_with(time, sample)
+          average
+        end
+        unless fetch_value_for_key_path(rewrote, key_path).nan?
+          system.report(self.tag, rewrote)
+        end
+      end
+
+      def prepare_rrd(args)
+        start = args[:start]
+        logger.debug "Creating a rrd file named '#{args[:rrd]}' with options {:start => #{start}}"
+        result = RRD::Builder.new(args[:rrd], start: start, step: step.seconds).tap do |builder|
+          builder.datasource data_source, :type => :gauge, :heartbeat => period.seconds, :min => 0, :max => :unlimited
+          builder.archive :average, :every => period.seconds, :during => period.seconds
+          builder.save
+        end
+        logger.debug "Builder#save returned: #{result.inspect}"
+      end
+
+      private
+
+      def update_value_for_key_path(data, key_path, &block)
+        first, *rest = split_key_path(key_path)
+        data = data.dup
+        dug_data = data[first] || data[first.intern]
+
+        if rest.empty?
+          hash_update(data, first, block.call(dug_data))
+        else
+          hash_update(data, first, update_value_for_key_path(dug_data, rest, &block))
+        end
+
+        data
+      end
+
+      def hash_update(hash, key, value)
+        if hash[key]
+          hash[key] = value
+        else
+          hash[key.intern] = value
+        end
+      end
+
+      def fetch_value_for_key_path(data, key_path)
+        first, *rest = split_key_path(key_path)
+        dug_data = data[first] || data[first.intern]
+        if rest.empty?
+          dug_data
+        else
+          fetch_value_for_key_path(dug_data, rest)
+        end
+      end
+
+      def split_key_path(key_path)
+        case key_path
+        when Array
+          key_path
+        when String
+          key_path.split('.')
+        else
+          fail "Unexpected type of key_path met. Expected an Array or a String, but it was a(n) #{key_path.class}"
+        end
+      end
+
+      def data_source
+        self.key_path.gsub('.', '_')
+      end
+
+      # @param [Time] time
+      def get_cdp_updated_with(time, value)
+        rrd_path = self.rrd
+        t = time.to_i
+
+        rrd = RRD::Base.new(rrd_path)
+
+        unless File.exist? rrd_path
+          prepare_rrd(rrd: rrd_path, start: t)
+        end
+
+        logger.debug "Updating the data source '#{data_source}' with the value #{value} with timestamp #{t}"
+        rrd.update t, value
+
+        logger.debug rrd.fetch!(:average)[-2..-1]
+
+        rrd.fetch(:average)[-2..-1].first.last
+      end
+
+      def logger
+        @logger ||= Logger.new(STDOUT)
+      end
+    end
+  end
+end

data/plugins/observed-gauge/observed-gauge.gemspec

@@ -0,0 +1,30 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'observed/gauge/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'observed-gauge'
+  spec.version       = Observed::Gauge::VERSION
+  spec.authors       = ['KUOKA Yusuke']
+  spec.email         = ['yusuke.kuoka@gree.net']
+  spec.description   = %q{Gauge plugin for Observed}
+  spec.summary       = %q{A plugin to consolidate outputs from other Observed output plugins by averaging values by averaging them over configured periods}
+  spec.homepage      = ''
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'observed', '~> 0.1.0'
+  spec.add_dependency 'rrd-ffi', '~> 0.2.14'
+
+  spec.add_development_dependency 'bundler', '~> 1.3'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'mocha'
+  spec.add_development_dependency 'fakefs'
+  spec.add_development_dependency 'simplecov'
+end