fluent-plugin-docker-metrics 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: fe3c77f3d0e10766bb18060d471de817455fd9cb
+  data.tar.gz: f56a4465bb6a566240a4c27276bca97d07081222
+SHA512:
+  metadata.gz: bcbcfd347ac2a1a7cb1e06bc0cdce38fce40968a3405c75577b50a0782cb5ea341c41cdad4a56b42139a4a450278347dba0777001799d8c85e4fb941454e05d5
+  data.tar.gz: 6d4ad921e6da4e8be9c4da6581a79253fe17cbbe692ab6e88d18e24778173da75e4c503b1fc0ee2d5bf953270dd376f367f9c6d52434a2a4063a15675048d1aa

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/README.md

@@ -0,0 +1,37 @@
+# Fluentd Docker Metrics Input Plugin
+
+This is a [Fluentd](http://www.fluentd.org) plugin to collect Docker metrics periodically.
+
+## How it works
+
+It's assumed to run on the host server. It periodically runs `docker ps --no-trunc -q`
+to get a list of running Docker container IDs, and it looks at `/sys/fs/cgroups/<metric_type>/docker/<container_id>/`
+for relevant stats. You can say this is an implementation of the metric collection strategy outlined in [this blog post](http://blog.docker.com/2013/10/gathering-lxc-docker-containers-metrics/).
+
+## Installing
+
+to be uploaded on Rubygems
+
+## Example config
+
+```
+<source>
+  type docker_metrics
+  stats_interval 1m
+</source>
+```
+
+## Parameters
+
+* **stats_interval**: how often to poll Docker containers for stats. The default is every minute.
+* **cgroup_path**: The path to cgroups pseudofiles. The default is `sys/fs/cgroup`.
+* **tag_prefix**: The tag prefix. The default value is "docker"
+
+## Example output
+
+```
+2014-06-26 18:16:43 +0000 docker.memory.stat: {"key":"memory_stat_total_active_anon","value":26025984,"source":"docker:precise64:b7f17c393775476bc0999cb6dcb4c6416e94b0473317375b9a245985dc6e91c5"}
+2014-06-26 18:16:43 +0000 docker.memory.stat: {"key":"memory_stat_total_inactive_file","value":131072,"source":"docker:precise64:b7f17c393775476bc0999cb6dcb4c6416e94b0473317375b9a245985dc6e91c5"}
+```
+
+In particular, each event is a key-value pair of individual metrics. Also, it has a field whose value is "\<tag_prefix\>:\<hostname\>:\<container_id\>"

data/fluent-plugin-docker-metrics.gemspec

@@ -0,0 +1,22 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-docker-metrics"
+  spec.version       = "0.0.1"
+  spec.authors       = ["kiyoto"]
+  spec.email         = ["kiyoto@treasure-data.com"]
+  spec.summary       = %q{Fluentd plugin to collect Docker container metrics}
+  spec.homepage      = "https://github.com/kiyoto/fluent-plugin-docker-metrics"
+  spec.license       = "Apache License, Version 2.0"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler"
+  spec.add_development_dependency "rake"
+  spec.add_runtime_dependency "fluentd"
+end

data/lib/fluent/plugin/in_docker_metrics.rb

@@ -0,0 +1,136 @@
+module Fluent
+  class DockerMetricsInput < Input
+    Plugin.register_input('docker_metrics', self)
+
+    config_param :cgroup_path, :string, :default => '/sys/fs/cgroup'
+    config_param :stats_interval, :time, :default => 60 # every minute
+    config_param :tag_prefix, :string, :default => "docker"
+
+    def initialize
+      super
+      require 'socket'
+      @hostname = Socket.gethostname
+    end
+
+    def configure(conf)
+      super
+
+    end
+
+    def start
+      @loop = Coolio::Loop.new
+      tw = TimerWatcher.new(@stats_interval, true, @log, &method(:get_metrics))
+      tw.attach(@loop)
+      @thread = Thread.new(&method(:run))
+    end
+    def run
+      @loop.run
+    rescue
+      log.error "unexpected error", :error=>$!.to_s
+      log.error_backtrace
+    end
+
+    # Metrics collection methods
+    def get_metrics
+      list_container_ids.each do |id|
+        emit_container_metric(id, 'memory', 'memory.stat') 
+        emit_container_metric(id, 'cpuacct', 'cpuacct.stat') 
+        emit_container_metric(id, 'blkio', 'blkio.io_serviced') 
+        emit_container_metric(id, 'blkio', 'blkio.io_service_bytes') 
+        emit_container_metric(id, 'blkio', 'blkio.io_service_queued') 
+        emit_container_metric(id, 'blkio', 'blkio.sectors') 
+      end
+    end
+
+    def list_container_ids
+      `docker ps --no-trunc -q`.split /\s+/
+    end
+   
+    def emit_container_metric(id, metric_type, metric_filename, opts = {})
+      path = "#{@cgroup_path}/#{metric_type}/docker/#{id}/#{metric_filename}"
+      if File.exists?(path)
+        parser = if metric_type != 'blkio'
+                   KeyValueStatsParser.new(path, metric_filename.gsub('.', '_'))
+                 else 
+                   BlkioStatsParser.new(path, metric_filename.gsub('.', '_'))
+                 end
+        time = Engine.now
+        tag = "#{@tag_prefix}.#{metric_filename}"
+        mes = MultiEventStream.new
+        parser.parse_each_line do |data|
+          next if not data
+          # TODO: address this more elegantly
+          if data[:key] =~ /^(?:cpuacct|blkio|memory_stat_pg)/
+            data[:type] = 'counter'
+          end
+          data["source"] = "#{@tag_prefix}:#{@hostname}:#{id}"
+          mes.add(time, data)
+        end
+        Engine.emit_stream(tag, mes)
+      else
+        nil
+      end
+    end
+
+    def shutdown
+      @loop.stop
+      @thread.join
+    end
+
+    class TimerWatcher < Coolio::TimerWatcher
+
+      def initialize(interval, repeat, log, &callback)
+        @callback = callback
+        @log = log
+        super(interval, repeat)
+      end
+      def on_timer
+        @callback.call
+      rescue
+        @log.error $!.to_s
+        @log.error_backtrace
+      end
+    end
+
+    class CGroupStatsParser
+      def initialize(path, metric_type)
+        raise ConfigError if not File.exists?(path)
+        @path = path
+        @metric_type = metric_type
+      end
+
+      def parse_line(line)
+      end
+
+      def parse_each_line(&block)
+        File.new(@path).each_line do |line|
+          block.call(parse_line(line))
+        end
+      end
+    end
+
+    class KeyValueStatsParser < CGroupStatsParser
+      def parse_line(line)
+        k, v = line.split(/\s+/, 2)
+        if k and v
+          { key: @metric_type + "_" + k, value: v.to_i }
+        else
+          nil
+        end
+      end
+    end
+
+    class BlkioStatsParser < CGroupStatsParser
+      BlkioLineRegexp = /^(?<major>\d+):(?<minor>\d+) (?<key>[^ ]+) (?<value>\d+)/
+      
+      def parse_line(line)
+        m = BlkioLineRegexp.match(line)
+        if m
+          { key: @metric_type + "_" + m["key"].downcase, value: m["value"] }
+        else
+          nil
+        end
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,90 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-docker-metrics
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- kiyoto
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-07-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- kiyoto@treasure-data.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- README.md
+- fluent-plugin-docker-metrics.gemspec
+- lib/fluent/plugin/in_docker_metrics.rb
+homepage: https://github.com/kiyoto/fluent-plugin-docker-metrics
+licenses:
+- Apache License, Version 2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Fluentd plugin to collect Docker container metrics
+test_files: []