fluent-plugin-statsite 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 2e12dccb3b7db8188b8fc47b913d1ee260773d1e
+  data.tar.gz: f09fa661f3b430f1bcc2f1b9b03f3cc4876989ce
+SHA512:
+  metadata.gz: 0259eae4b813fa352773f84dbc465008463d4be9f9aa5e2c7f6dc4fa8c31a2e291fe2974b6a0de81cb856e5fbd0f6eae3601b3d0ba07cf7127457fbd42955219
+  data.tar.gz: 55b89401b72406cbcbfe3942b5ad5318e897a2e05c21c68d7836d9b7085ec22bc244c7fe6585fa3ee24c0a6e864ff141bd6409d87bf8d6a6047527a57b62c1de

data/.gitignore

@@ -0,0 +1,22 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/.gitmodules

@@ -0,0 +1,3 @@
+[submodule "vendor/statsite"]
+	path = vendor/statsite
+	url = https://github.com/armon/statsite

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-statsite.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,13 @@
+Copyright 2014 OKUNO Akihiro
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,66 @@
+# Statsite Fluentd Plugin
+
+This plugin calculates various useful metrics using [Statsite by armon](http://armon.github.io/statsite/).
+
+ [Statsite](http://armon.github.io/statsite/) is very cool software. Statsite works as daemon service, receiving events from tcp/udp, aggregating these events with specified methods, and sending the results via pluggable sinks. Statsite is written in C, cpu and memory efficient, and employ some approximate algorithms for unique sets and percentiles.
+
+ You may think this as standard output plugin which just sends events to a daemon process, such as [mongodb plugin](https://github.com/fluent/fluent-plugin-mongo). It is true that this plugin is registered as output plugin, but this works as the so-called **Filter Plugin**, which means that this plugin sends matched events to Statsite process, recieves results aggregated by the Statsite, then re-emitting these results as events.
+
+ Statsite process is launched as a child process from this plugin internally. All you have to do place statsite the binary under $PATH, or set the path of statsite binary as parameter. Neither config files or daemon process is not required. Besides, the communication between the plugin and the Statsite process takes place through STDIN/STDOUT, so no network port will be used.
+
+## Installation
+
+`$ fluent-gem install fluent-plugin-statsite`
+
+### Statsite Installation
+
+ Statsite can work as sinble binary with few dependency. You probably could get it working just by downloading source files and executing make command.
+
+Please refer to [Statsite official page](http://armon.github.io/statsite/).
+
+## Configuration
+
+It is strongly recommended to use '[V1 config format](http://docs.fluentd.org/articles/config-file#v1-format)' because this plugin requires to set deeply nested parameters. 
+
+### Example
+
+```
+<match **>
+  type statsite
+  tag statsite
+  metrics [
+    "${status}:1|c",
+    {"key": "request_time", "value_field": "request_time", "type": "ms"}
+  ]
+  histograms [
+    {"prefix": "request_time" "min": 0, "max": 1, "width": 0.1}
+  ]
+  statsite_path "statsite"
+  statsite_flush_interval 1s
+  timer_eps 0.01
+  set_eps 0.01
+  child_respawn 5
+</match>
+```
+
+### Parameter
+
+TODO
+
+### Metrics Format
+
+You can specify metrics in two format, string style, and hash style.
+
+#### String style
+
+TODO
+
+#### Hash style
+
+TODO
+
+## Copyright
+
+* Copyright (c) 2014- OKUNO Akihiro
+* License
+    * Apache License, version 2.0

data/Rakefile

@@ -0,0 +1,25 @@
+require "bundler/gem_tasks"
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib' << 'test'
+  t.test_files = FileList['test/test*.rb']
+  t.verbose = true
+end
+task :test => 'statsite:build'
+
+namespace :statsite do
+  desc 'Build statsite binary'
+  task :'build' => 'vendor/statsite/statsite'
+
+  desc 'Clean statsite artifacts'
+  task :'clean' do
+    sh 'make -C vendor/statsite clean'
+  end
+
+  file 'vendor/statsite/statsite' do
+    sh 'make -C vendor/statsite'
+  end
+end
+
+task :default => :test

data/example.conf

@@ -0,0 +1,27 @@
+<source>
+  type forward
+</source>
+
+<match event.*>
+  type statsite
+  tag statsite
+  metrics [
+    "k1:1|kv",
+    "k2:1|g",
+    "k3:1|ms",
+    "k4:1|c",
+    "k5:1|s"
+  ]
+  histograms [
+    {"prefix": "bar", "min": 0, "max": 10, "width": 1.0}
+  ]
+  statsite_path "vendor/statsite/statsite"
+  statsite_flush_interval 1s
+  timer_eps 0.01
+  set_eps 0.02
+  child_respawn 5
+</match>
+
+<match *>
+    type stdout
+</match>

data/fluent-plugin-statsite.gemspec

@@ -0,0 +1,25 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-statsite"
+  spec.version       = "0.0.2"
+  spec.authors       = ["OKUNO Akihiro"]
+  spec.email         = ["choplin.choplin@gmail.com"]
+  spec.summary       = %q{Fluentd statsite plugin}
+  spec.description   = %q{Fluentd plugin which caluculate statistics using statsite}
+  spec.homepage      = ""
+  spec.homepage         = "https://github.com/choplin/fluent-plugin-statsite"
+  spec.license       = "Apache-2.0"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "fluentd"
+
+  spec.add_development_dependency "bundler", "~> 1.6"
+  spec.add_development_dependency "rake"
+end

data/lib/fluent/plugin/out_statsite.rb

@@ -0,0 +1,156 @@
+require_relative 'statsite/child_process'
+require_relative 'statsite/format'
+require_relative 'statsite/metric'
+require_relative 'statsite/histogram'
+
+require 'tempfile'
+require 'timeout'
+
+module Fluent
+  class StatsiteOutput < Fluent::BufferedOutput
+    include Fluent::StatsitePlugin
+
+    Fluent::Plugin.register_output('statsite', self)
+
+    # TODO: should be configurable?
+    CONFIG_VALIDATION_WAIT = 0.1
+
+    config_param :tag,                     :string
+    config_param :metrics,                 :array
+    config_param :histograms,              :array,    :default => []
+    config_param :statsite_path,           :string,   :default => 'statsite'
+    config_param :statsite_flush_interval, :time,     :default => 10
+    config_param :timer_eps,               :float,    :default => 0.01
+    config_param :set_eps,                 :float,    :default => 0.02
+    config_param :child_respawn,           :string,   :default => nil
+    # TODO: should support input_counter?
+
+    def initialize
+      super
+    end
+
+    def configure(conf)
+      super
+      @metrics = validate_metrics
+      @metrics.each{|m| @log.info "out_statsite: #{m}"}
+
+      @histograms = validate_histograms
+
+      @respawns = configure_respawns
+
+      @parser = StatsiteParser.new(method(:on_message))
+      @formatter = StatsiteFormatter.new(@metrics)
+
+      $log.info "out_statsite: statsite config\n\n#{config}"
+      @conf = Tempfile.new('fluent-plugin-statsite-')
+      @conf.puts config
+      @conf.flush
+
+      @child = ChildProcess.new(@parser, @respawns, log)
+
+      @cmd = "#{@statsite_path} -f #{@conf.path}"
+      validate_statsite_confg
+    end
+
+    def start
+      super
+
+      begin
+        $log.info "out_statsite: launching statsite process", cmd: @cmd
+        @child.start(@cmd)
+      rescue
+        shutdown
+        raise
+      end
+    end
+
+    def before_shutdown
+      super
+      $log.debug "out_statsite#before_shutdown called"
+      @child.finished = true
+      sleep 0.5  # TODO wait time before killing child process
+    end
+
+    def shutdown
+      super
+      @conf.close
+      @child.shutdown
+    end
+
+    def format(tag, time, record)
+      @formatter.call(record)
+    end
+
+    def write(chunk)
+      @child.write chunk
+    end
+
+    private
+
+    def run
+      @loop.run
+    rescue => e
+      log.error "out_statsite: unexpected error", :error => e, :error_class => e.class
+      log.error_backtrace
+    end
+
+    def configure_respawns
+      if @child_respawn.nil? or @child_respawn == 'none' or @child_respawn == '0'
+        0
+      elsif @child_respawn == 'inf' or @child_respawn == '-1'
+        -1
+      elsif @child_respawn =~ /^\d+$/
+        @child_respawn.to_i
+      else
+        raise ConfigError, "child_respawn option argument invalid: none(or 0), inf(or -1) or positive number"
+      end
+    end
+
+    def validate_metrics
+      @metrics.map {|m| Metric.validate(m)}
+    end
+
+    def validate_histograms
+      @histograms.map {|h| Histogram.validate(h)}
+    end
+
+    def validate_statsite_confg
+      $log.debug "lanuch statsite process to validate statsite config"
+      pid = spawn(@cmd, out: '/dev/null')
+      if pid.nil?
+        raise ConfigError, 'failed to launch statsite process', cmd: @cmd
+      else
+        begin
+          Timeout::timeout(CONFIG_VALIDATION_WAIT) do
+            Process.waitpid(pid)
+          end
+          raise ConfigError, 'Statsite process cannot be launched correctly. A config is probably invalid.'
+        rescue Timeout::Error
+          # launched correctly
+          Process.kill(:KILL, pid)
+          Process.waitpid(pid)
+        end
+      end
+    end
+
+    def config
+      <<-CONFIG
+[statsite]
+port = 0
+udp_port = 0
+parse_stdin = 1
+log_level = INFO
+flush_interval = #{@statsite_flush_interval}
+timer_eps = #{@timer_eps}
+set_eps = #{@set_eps}
+stream_cmd = cat
+
+#{@histograms.map(&:to_ini).join("\n\n")}
+      CONFIG
+    end
+
+    def on_message(time, record)
+      Engine.emit(@tag, time, record)
+    end
+  end
+end

data/lib/fluent/plugin/statsite/child_process.rb

@@ -0,0 +1,101 @@
+module Fluent
+  module StatsitePlugin
+    class ChildProcess
+      attr_accessor :finished
+
+      def initialize(parser, respawns=0, log = $log)
+        @pid = nil
+        @thread = nil
+        @parser = parser
+        @respawns = respawns
+        @mutex = Mutex.new
+        @finished = nil
+        @log = log
+      end
+
+      def start(command)
+        @command = command
+        @mutex.synchronize do
+          @io = IO.popen(command, "r+")
+          @pid = @io.pid
+          @io.sync = true
+          @thread = Thread.new(&method(:run))
+        end
+        @finished = false
+      end
+
+      def kill_child(join_wait)
+        begin
+          Process.kill(:TERM, @pid)
+        rescue #Errno::ECHILD, Errno::ESRCH, Errno::EPERM
+          # Errno::ESRCH 'No such process', ignore
+          # child process killed by signal chained from fluentd process
+        end
+        if @thread.join(join_wait)
+          # @thread successfully shutdown
+          return
+        end
+        begin
+          Process.kill(:KILL, @pid)
+        rescue #Errno::ECHILD, Errno::ESRCH, Errno::EPERM
+          # ignore if successfully killed by :TERM
+        end
+        @thread.join
+      end
+
+      def shutdown
+        @finished = true
+        @mutex.synchronize do
+          kill_child(60) # TODO wait time
+        end
+      end
+
+      def write(chunk)
+        begin
+          chunk.write_to(@io)
+        rescue Errno::EPIPE => e
+          # Broken pipe (child process unexpectedly exited)
+          @log.warn "statsite Broken pipe, child process maybe exited.", :command => @command
+          if try_respawn
+            retry # retry chunk#write_to with child respawned
+          else
+            raise e # to retry #write with other ChildProcess instance (when num_children > 1)
+          end
+        end
+      end
+
+      def try_respawn
+        return false if @respawns == 0
+        @mutex.synchronize do
+          return false if @respawns == 0
+
+          kill_child(5) # TODO wait time
+
+          @io = IO.popen(@command, "r+")
+          @pid = @io.pid
+          @io.sync = true
+          @thread = Thread.new(&method(:run))
+
+          @respawns -= 1 if @respawns > 0
+        end
+        @log.warn "statsite child process successfully respawned.", :command => @command, :respawns => @respawns
+        true
+      end
+
+      def run
+        @parser.call(@io)
+      rescue
+        @log.error "statsite thread unexpectedly failed with an error.", :command=>@command, :error=>$!.to_s
+        @log.warn_backtrace $!.backtrace
+      ensure
+        pid, stat = Process.waitpid2(@pid)
+        unless @finished
+          @log.error "statsite process unexpectedly exited.", :command=>@command, :ecode=>stat.to_i
+          unless @respawns == 0
+            @log.warn "statsite child process will respawn for next input data (respawns #{@respawns})."
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/statsite/format.rb

@@ -0,0 +1,47 @@
+module Fluent
+  module StatsitePlugin
+    class StatsiteParser
+      def initialize(on_message)
+        @on_message = on_message
+      end
+
+      def call(io)
+        io.each_line(&method(:each_line))
+      end
+
+      def each_line(line)
+        k,v,t = line.chomp.split('|')
+        type, key, statistic, range = k.split(".", 4)
+
+        record = case type
+                 when 'timers' then 1
+                   if statistic == 'histogram'
+                     {type: type, key: key, value: v.to_i, statistic: statistic, range: range[4..-1]}
+                   elsif statistic == 'count'
+                     {type: type, key: key, value: v.to_i, statistic: statistic}
+                   else
+                     {type: type, key: key, value: v.to_f, statistic: statistic}
+                   end
+                 when 'kv', 'gauges', 'counts'
+                   {type: type, key: key, value: v.to_f}
+                 when 'sets'
+                   {type: type, key: key, value: v.to_i}
+                 end
+
+        raise "out_statsite: failed to parse a line. '#{line}'" if record.nil?
+
+        @on_message.call(t.to_i, record)
+      end
+    end
+
+    class StatsiteFormatter
+      def initialize(metrics)
+        @metrics = metrics
+      end
+
+      def call(record)
+        @metrics.map{|m| m.convert(record)}.select{|m| not m.nil?}.join('')
+      end
+    end
+  end
+end

data/lib/fluent/plugin/statsite/histogram.rb

@@ -0,0 +1,65 @@
+module Fluent
+  module StatsitePlugin
+    class Histogram
+      FIELD = %w(
+        min
+        max
+        width
+      )
+
+      OPTIONAL_FIELD = %w(prefix)
+
+      FLOATING_FIELD = %w(
+        min
+        max
+        width
+      )
+
+      def initialize(prefix, min, max, width)
+        @prefix = prefix
+        @min = min
+        @max = max
+        @width = width
+
+        @section = prefix.nil? ? "default" : prefix
+      end
+
+      def to_ini
+        <<-INI
+[histogram_#{@section}]
+prefix=#{@prefix}
+min=#{@min}
+max=#{@max}
+width=#{@width}
+        INI
+      end
+
+      def self.validate(h)
+        if h.class != Hash
+          raise ConfigError, "a type of histogram element must be Hash, but specified as #{h.class}"
+        end
+
+        FIELD.each do |f|
+          if not h.has_key?(f)
+            raise ConfigError, "histogram element must contain '#{f}'"
+          end
+        end
+
+        h.keys.each do |k|
+          if not FIELD.member?(k) and not OPTIONAL_FIELD.member?(k)
+            raise ConfigError, "invalid histogram hash key: #{k}"
+          end
+        end
+
+        FLOATING_FIELD.each do |f|
+          cls = h[f].class
+          if cls != Fixnum and cls != Float
+            raise ConfigError, "#{f} value of histogram must be Fixnum or Float"
+          end
+        end
+
+        new(h['prefix'], h['min'], h['max'], h['width'])
+      end
+    end
+  end
+end

data/lib/fluent/plugin/statsite/metric.rb

@@ -0,0 +1,81 @@
+module Fluent
+  module StatsitePlugin
+    class Metric
+      TYPE = %w(kv g ms h c s)
+
+      HASH_FIELD = %w(
+        type
+        key
+        key_field
+        value
+        value_field
+      )
+
+      FIELD = '\w+|\$\{\w+\}'
+
+      STRING_PATTERN = /^(#{FIELD}):(#{FIELD})\|(#{TYPE.join('|')})$/
+      STRING_EXAMPLE = "key_field:value_field|type"
+
+      def initialize(key, key_field, value, value_field, type)
+        @key = key
+        @key_field = key_field
+        @value = value
+        @value_field = value_field
+        @type = type
+      end
+
+      def convert(record)
+        k = @key.nil? ? record[@key_field] : @key
+        v = @value.nil? ? record[@value_field] : @value
+        (k.nil? or v.nil?) ? nil : "#{k}:#{v}|#{@type}\n"
+      end
+
+      def to_s
+        k = @key.nil? ? "key_field=#{@key_field}" :"key=#{@key}"
+        v = @value.nil? ? "value_field=#{@value_field}" :"value=#{@value}"
+        "Metric(#{k}, #{v}, type=#{@type})"
+      end
+
+      def self.validate(m)
+        if not (m.class == Hash or m.class == String)
+          raise ConfigError, "a type of metrics element must be Hash or String, but specified as #{m.class}"
+        end
+
+        case m
+        when Hash
+          m.keys.each do |k|
+            if not HASH_FIELD.member?(k)
+              raise ConfigError, "invalid metrics element hash key: #{k}"
+            end
+          end
+
+          if not m.has_key?('key') ^ m.has_key?('key_field')
+              raise ConfigError, "metrics element must contain either one of 'key' or 'key_field'"
+          end
+
+          if not m.has_key?('value') ^ m.has_key?('value_field')
+              raise ConfigError, "metrics element must contain either one of 'value' or 'value_field'"
+          end
+
+          if not m.has_key?('type')
+              raise ConfigError, "metrics element must contain 'type'"
+          end
+
+          if not TYPE.member?(m['type'])
+            raise ConfigError, "metrics type must be one of the following: #{TYPE.join(' ')}, but specified as #{m['type']}"
+          end
+
+          new(m['key'], m['value_field'], m['value'], m['value_field'], m['type'])
+        when String
+          if (STRING_PATTERN =~ m).nil?
+            raise ConfigError, "metrics string must be #{STRING_PATTERN}, but specified as #{m}"
+          end
+
+          key, key_field = $1.start_with?('$') ? [nil, $1[2..-2]] : [$1, nil]
+          value, value_field = $2.start_with?('$') ? [nil, $2[2..-2]] : [$2, nil]
+          new(key, key_field, value, value_field, $3)
+        end
+      end
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,62 @@
+require 'test/unit'
+require 'fluent/log'
+require 'fluent/test'
+
+$log = Fluent::Log.new(Fluent::Test::DummyLogDevice.new, Fluent::Log::LEVEL_WARN)
+
+def records
+  [
+    {
+      "remote_addr" => "114.170.6.118",
+      "remote_user" => "-",
+      "time_local" => "20/Jul/2014:18:25:50 +0000",
+      "request" => "GET /foo HTTP/1.1",
+      "status" => "200",
+      "body_bytes_sent" => "911",
+      "http_referer" => "-",
+      "http_user_agent" => "Mozilla/5.0 (Linux; U; Android 4.2.2; ja-jp; SO-04E Build/10.3.1.B.0.256) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30",
+      "request_time" => "0.058",
+      "upstream_addr" => "192.168.222.180:80",
+      "upstream_response_time" => "0.058"
+    },
+    {
+      "remote_addr" => "180.214.48.86",
+      "remote_user" => "-",
+      "time_local" => "20/Jul/2014:18:25:50 +0000",
+      "request" => "POST /bar HTTP/1.1",
+      "status" => "200",
+      "body_bytes_sent" => "57",
+      "http_referer" => "-",
+      "http_user_agent" => "Mozilla/5.0 (Linux; U; Android 4.2.2; ja-jp; SO-04E Build/10.3.1.B.0.256) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30",
+      "request_time" => "0.041",
+      "upstream_addr" => "10.0.0.143:80",
+      "upstream_response_time" => "0.041"
+    },
+    {
+      "remote_addr" => "153.160.159.80",
+      "remote_user" => "-",
+      "time_local" => "20/Jul/2014:18:25:50 +0000",
+      "request" => "GET /foo HTTP/1.1",
+      "status" => "200",
+      "body_bytes_sent" => "34139",
+      "http_referer" => "/bar",
+      "http_user_agent" => "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:30.0) Gecko/20100101 Firefox/30.0",
+      "request_time" => "0.000",
+      "upstream_addr" => "-",
+      "upstream_response_time" => "-"
+    },
+    {
+      "remote_addr" => "172.56.33.226",
+      "remote_user" => "-",
+      "time_local" => "20/Jul/2014:18:25:50 +0000",
+      "request" => "GET /foo HTTP/1.1",
+      "status" => "200",
+      "body_bytes_sent" => "791",
+      "http_referer" => "-",
+      "http_user_agent" => "en;Mozilla/5.0 (Linux; U; Android 4.3; en-us; SGH-T999 Build/JSS15J) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30",
+      "request_time" => "0.073",
+      "upstream_addr" => "192.168.222.209:80",
+      "upstream_response_time" => "0.073"
+    }
+  ]
+end

data/test/test_formatter.rb

@@ -0,0 +1,25 @@
+require 'helper'
+require 'fluent/plugin/statsite/format'
+
+include Fluent::StatsitePlugin
+
+class StatsiteFormatterTest < Test::Unit::TestCase
+  def setup
+    metrics = [
+      Metric.validate('${k}:${v}|g'),
+      Metric.validate('${k}:v|g')
+    ]
+    @formatter = StatsiteFormatter.new(metrics)
+  end
+
+  def test_call
+    record = {'v' => 'value'}
+    assert_equal "", @formatter.call(record)
+
+    record = {'k' => 'key'}
+    assert_equal "key:v|g\n", @formatter.call(record)
+
+    record = {'k' => 'key', 'v' => 'value'}
+    assert_equal "key:value|g\nkey:v|g\n", @formatter.call(record)
+  end
+end

data/test/test_histogram.rb

@@ -0,0 +1,79 @@
+require 'helper'
+require 'fluent/plugin/statsite/histogram'
+
+include Fluent::StatsitePlugin
+
+class Histogram
+  attr_reader :section, :prefix, :min, :max, :width
+end
+
+class HistogramTest < Test::Unit::TestCase
+  def valid_config
+    {
+      'prefix' => 'pre',
+      'min' => 0.0,
+      'max' => 10.0,
+      'width' => 1.0,
+    }
+  end
+
+  def test_validate_object_type
+    config = []
+    assert_raises(Fluent::ConfigError) { Histogram.validate(config) }
+  end
+
+  def test_validate_mandatory_field
+    Histogram::FIELD.each do |f|
+      config = valid_config
+      config.delete(f)
+      assert_raises(Fluent::ConfigError) { Histogram.validate(config) }
+    end
+  end
+
+  def test_validate_extra_field
+    config = valid_config
+    config['foo'] = 'bar'
+    assert_raises(Fluent::ConfigError) { Histogram.validate(config) }
+  end
+
+  def test_validate_floating_field
+    Histogram::FLOATING_FIELD.each do |f|
+      config = valid_config
+      config[f] = 'foo'
+      assert_raises(Fluent::ConfigError) { Histogram.validate(config) }
+    end
+  end
+
+  def test_validate_result
+    c = valid_config
+    h = Histogram.validate(c)
+    assert_equal c['prefix'], h.section
+    assert_equal c['prefix'], h.prefix
+    assert_equal c['min'], h.min
+    assert_equal c['max'], h.max
+    assert_equal c['width'], h.width
+  end
+
+  def test_to_init
+    c = valid_config
+
+    ini = Histogram.validate(c).to_ini
+    assert_equal ini, <<-INI
+[histogram_#{c['prefix']}]
+prefix=#{c['prefix']}
+min=#{c['min']}
+max=#{c['max']}
+width=#{c['width']}
+    INI
+
+    c.delete('prefix')
+    ini = Histogram.validate(c).to_ini
+    assert_equal ini, <<-INI
+[histogram_default]
+prefix=
+min=#{c['min']}
+max=#{c['max']}
+width=#{c['width']}
+    INI
+  end
+end

data/test/test_metric.rb

@@ -0,0 +1,100 @@
+require 'helper'
+require 'fluent/plugin/statsite/metric'
+
+include Fluent::StatsitePlugin
+
+class Metric
+  attr_reader :key, :key_field, :value, :value_field, :type
+end
+
+class MetricTest < Test::Unit::TestCase
+
+  def valid_config
+    {'key' => 'k', 'value' => 'v', 'type' => 'kv'}
+  end
+
+  def test_validate_object_type
+    config = []
+    assert_raises(Fluent::ConfigError) { Metric.validate(config) }
+  end
+
+  def test_validate_extra_field
+    config = valid_config
+    config['foo'] = 'bar'
+    assert_raises(Fluent::ConfigError) { Metric.validate(config) }
+  end
+
+  def test_validate_key
+    config = (valid_config)['key_field'] = 'k'
+    assert_raises(Fluent::ConfigError) { Metric.validate(config) }
+
+    config = (valid_config)
+    config.delete('key')
+    assert_raises(Fluent::ConfigError) { Metric.validate(config) }
+  end
+
+  def test_validate_value
+    config = (valid_config)['value_field'] = 'k'
+    assert_raises(Fluent::ConfigError) { Metric.validate(config) }
+
+    config = (valid_config)
+    config.delete('value')
+    assert_raises(Fluent::ConfigError) { Metric.validate(config) }
+  end
+
+  def test_validate_type
+    config = (valid_config)
+    config.delete('type')
+    assert_raises(Fluent::ConfigError) { Metric.validate(config) }
+
+    config = (valid_config)
+    config['type'] = 'foo'
+    assert_raises(Fluent::ConfigError) { Metric.validate(config) }
+  end
+
+  def test_validate_string
+    config = "foo"
+    assert_raises(Fluent::ConfigError) { Metric.validate(config) }
+
+    config = "k:v|foo"
+    assert_raises(Fluent::ConfigError) { Metric.validate(config) }
+  end
+
+  def test_validate_result
+    m = Metric.validate(valid_config)
+    assert_equal 'k', m.key
+    assert_nil m.key_field
+    assert_equal 'v', m.value
+    assert_nil m.value_field
+    assert_equal 'kv', m.type
+  end
+
+  def test_validate_result_string
+    m = Metric.validate('k:v|kv')
+    assert_equal 'k', m.key
+    assert_nil m.key_field
+    assert_equal 'v', m.value
+    assert_nil m.value_field
+    assert_equal 'kv', m.type
+
+    m = Metric.validate('${k}:${v}|kv')
+    assert_nil m.key
+    assert_equal 'k', m.key_field
+    assert_nil m.value
+    assert_equal 'v', m.value_field
+    assert_equal 'kv', m.type
+  end
+
+  def test_convert
+    m = Metric.validate('${k}:${v}|kv')
+
+    record = {'k' => 'key'}
+    assert_nil m.convert(record)
+
+    record = {'v' => 'value'}
+    assert_nil m.convert(record)
+
+    record = {'k' => 'key', 'v' => 'value'}
+    assert_equal "key:value|kv\n", m.convert(record)
+  end
+end

data/test/test_out_statsite.rb

@@ -0,0 +1,65 @@
+require 'helper'
+require 'fluent/plugin/out_statsite'
+
+class Fluent::StatsiteOutput
+  attr_reader :respawns
+end
+
+class StatsiteOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  ROOT = File.expand_path('..', File.dirname(__FILE__))
+  PATH = ROOT + '/vendor/statsite/statsite'
+
+  CONFIG = %[
+    type statsite
+    tag statsite
+    metrics [
+      "${status}:1|c"
+    ]
+    histograms [
+      {"prefix": "k", "min": 0, "max": 10, "width": 1.0}
+    ]
+    statsite_path "#{PATH}"
+    statsite_flush_interval 1s
+    timer_eps 0.01
+    set_eps 0.02
+    child_respawn 5
+  ]
+
+  RECORDS = [
+  ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::OutputTestDriver.new(Fluent::StatsiteOutput).configure(conf, true)
+  end
+
+  def test_configure
+    d = create_driver
+
+    assert_equal 'statsite', d.instance.tag
+    assert d.instance.metrics.all?{|m| m.class == Fluent::StatsitePlugin::Metric}
+    assert d.instance.histograms.all?{|m| m.class == Fluent::StatsitePlugin::Histogram}
+    assert_equal PATH, d.instance.statsite_path
+    assert_equal 1, d.instance.statsite_flush_interval
+    assert_equal 0.01, d.instance.timer_eps
+    assert_equal 0.02, d.instance.set_eps
+    assert_equal 5, d.instance.respawns
+  end
+
+  def test_emit
+    d = create_driver
+
+    d.run do
+      records.each { |r| d.emit(r, Time.now) }
+    end
+
+    emits = d.emits
+
+    count_result = emits.pop
+    assert_equal 'statsite', count_result[0]
+    assert_equal({type: 'counts', key: '200', value: 4.0}, count_result[2])
+  end
+end

data/test/test_parser.rb

@@ -0,0 +1,87 @@
+require 'helper'
+require 'fluent/plugin/statsite/format'
+
+include Fluent::StatsitePlugin
+
+class StatsiteParserTest < Test::Unit::TestCase
+  def setup
+    @res = []
+    proc = Proc.new {|time, record| @res << {time: time, record:record } }
+    @parser = StatsiteParser.new(proc)
+  end
+
+  def test_eachline_kv
+    line = 'kv.k|1.000000|1405869579'
+    @parser.each_line(line)
+    expected = {
+      time: 1405869579,
+      record: { type: 'kv', key: 'k', value: 1.000000 }
+    }
+    assert_equal expected, @res.pop
+  end
+
+  def test_eachline_gauge
+    line = 'gauges.k|1.000000|1405869579'
+    @parser.each_line(line)
+    expected = {
+      time: 1405869579,
+      record: { type: 'gauges', key: 'k', value: 1.000000 }
+    }
+    assert_equal expected, @res.pop
+  end
+
+  def test_eachline_counts
+    line = 'counts.k|1.000000|1405869579'
+    @parser.each_line(line)
+    expected = {
+      time: 1405869579,
+      record: { type: 'counts', key: 'k', value: 1.000000 }
+    }
+    assert_equal expected, @res.pop
+  end
+
+  def test_eachline_sets
+    line = 'sets.k|1|1405869579'
+    @parser.each_line(line)
+    expected = {
+      time: 1405869579,
+      record: { type: 'sets', key: 'k', value: 1 }
+    }
+    assert_equal expected, @res.pop
+  end
+
+  def test_eachline_timers
+    line = 'timers.k.sum|1.000000|1405869579'
+    @parser.each_line(line)
+    expected = {
+      time: 1405869579,
+      record: { type: 'timers', key: 'k', value: 1.000000, statistic: 'sum' }
+    }
+    assert_equal expected, @res.pop
+  end
+
+  def test_eachline_timers_count
+    line = 'timers.k.count|1|1405869579'
+    @parser.each_line(line)
+    expected = {
+      time: 1405869579,
+      record: { type: 'timers', key: 'k', value: 1, statistic: 'count' }
+    }
+    assert_equal expected, @res.pop
+  end
+
+  def test_eachline_timers_histogram
+    line = 'timers.k.histogram.bin_<0.00|0|1405869579'
+    @parser.each_line(line)
+    expected = {
+      time: 1405869579,
+      record: { type: 'timers', key: 'k', value: 0, statistic: 'histogram', range: '<0.00' }
+    }
+    assert_equal expected, @res.pop
+  end
+
+  def test_eachline_invalid_line
+    line = 'test'
+    assert_raise(RuntimeError) { @parser.each_line(line) }
+  end
+end

metadata

@@ -0,0 +1,111 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-statsite
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: ruby
+authors:
+- OKUNO Akihiro
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-07-20 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Fluentd plugin which caluculate statistics using statsite
+email:
+- choplin.choplin@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".gitmodules"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- example.conf
+- fluent-plugin-statsite.gemspec
+- lib/fluent/plugin/out_statsite.rb
+- lib/fluent/plugin/statsite/child_process.rb
+- lib/fluent/plugin/statsite/format.rb
+- lib/fluent/plugin/statsite/histogram.rb
+- lib/fluent/plugin/statsite/metric.rb
+- test/helper.rb
+- test/test_formatter.rb
+- test/test_histogram.rb
+- test/test_metric.rb
+- test/test_out_statsite.rb
+- test/test_parser.rb
+homepage: https://github.com/choplin/fluent-plugin-statsite
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Fluentd statsite plugin
+test_files:
+- test/helper.rb
+- test/test_formatter.rb
+- test/test_histogram.rb
+- test/test_metric.rb
+- test/test_out_statsite.rb
+- test/test_parser.rb