fluent-plugin-docker-metrics 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fe3c77f3d0e10766bb18060d471de817455fd9cb
-  data.tar.gz: f56a4465bb6a566240a4c27276bca97d07081222
+  metadata.gz: 587119d524daa8765eebadec104dc588369eaf7a
+  data.tar.gz: 3b7bbb2601a73bafb2e5bb77fa35506eabc7b79c
 SHA512:
-  metadata.gz: bcbcfd347ac2a1a7cb1e06bc0cdce38fce40968a3405c75577b50a0782cb5ea341c41cdad4a56b42139a4a450278347dba0777001799d8c85e4fb941454e05d5
-  data.tar.gz: 6d4ad921e6da4e8be9c4da6581a79253fe17cbbe692ab6e88d18e24778173da75e4c503b1fc0ee2d5bf953270dd376f367f9c6d52434a2a4063a15675048d1aa
+  metadata.gz: e56678bbbfbc5ff862e4ac841e7381e0e4220a54b274f55aaed8927c63634cc934155c243bdab00313fb97c956efba662ba727f83c420279aa0e920c6e59191a
+  data.tar.gz: 2dd6d2b41b5e78d8165f4720a9a637e1219ff7d5d6ec4cdd23d47e790d2c675c6882558c0545395409858f1febe238d27fd1a97062a476be9ac8be284be01c57

data/CHANGELOG

@@ -0,0 +1,7 @@
+## v0.0.2 2014/11/22
+
+- Change the output format. The "source" field is deprecated. Instead, it has "hostname", "id", and "container".
+
+## v0.0.1 2014/06/27
+
+- Initial release

data/README.md

@@ -30,8 +30,13 @@ to be uploaded on Rubygems
 ## Example output
 
 ```
-2014-06-26 18:16:43 +0000 docker.memory.stat: {"key":"memory_stat_total_active_anon","value":26025984,"source":"docker:precise64:b7f17c393775476bc0999cb6dcb4c6416e94b0473317375b9a245985dc6e91c5"}
-2014-06-26 18:16:43 +0000 docker.memory.stat: {"key":"memory_stat_total_inactive_file","value":131072,"source":"docker:precise64:b7f17c393775476bc0999cb6dcb4c6416e94b0473317375b9a245985dc6e91c5"}
+2014-11-22 17:48:26 +0000 docker.blkio.io_queued: {"key":"blkio_io_queued_total","value":0,"type":"counter","hostname":"precise64","id":"24f5fb3bfc429e88aa3dbacd704667899dc496067cedcfa58dd84da42e7cb3cf","name":"/world"}
+2014-11-22 17:48:26 +0000 docker.blkio.sectors: {"key":"blkio_sectors","value":136,"type":"counter","hostname":"precise64","id":"24f5fb3bfc429e88aa3dbacd704667899dc496067cedcfa58dd84da42e7cb3cf","name":"/world"}
 ```
 
-In particular, each event is a key-value pair of individual metrics. Also, it has a field whose value is "\<tag_prefix\>:\<hostname\>:\<container_id\>"
+In particular, each event is a key-value pair of individual metrics. Also, it has
+
+- `hostname` is the hostname of the Docker host
+- `id` is the ID of the container
+- `name` is the descriptive name of the container (a la `docker inspect --format '{{ .Names }}'`)
+

data/Rakefile

@@ -0,0 +1,10 @@
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << "test" << "lib"
+  test.pattern = 'test/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/fluent-plugin-docker-metrics.gemspec

@@ -4,7 +4,8 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-docker-metrics"
-  spec.version       = "0.0.1"
+  spec.version       = "0.0.2"
+  spec.description   = 'Fluentd input plugin to collect container metrics periodically'
   spec.authors       = ["kiyoto"]
   spec.email         = ["kiyoto@treasure-data.com"]
   spec.summary       = %q{Fluentd plugin to collect Docker container metrics}
@@ -16,7 +17,9 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler"
-  spec.add_development_dependency "rake"
-  spec.add_runtime_dependency "fluentd"
+  spec.add_development_dependency "bundler", '~> 1.7'
+  spec.add_development_dependency "rake", '~> 10.1'
+  spec.add_development_dependency "fakefs", '~> 0.6'
+  spec.add_runtime_dependency "fluentd", '~> 0.10'
+  spec.add_runtime_dependency "docker-api", '~> 1.15'
 end

data/lib/fluent/plugin/in_docker_metrics.rb

@@ -5,10 +5,12 @@ module Fluent
     config_param :cgroup_path, :string, :default => '/sys/fs/cgroup'
     config_param :stats_interval, :time, :default => 60 # every minute
     config_param :tag_prefix, :string, :default => "docker"
+    config_param :container_ids, :array, :default => nil # mainly for testing
 
     def initialize
       super
       require 'socket'
+      require 'docker'
       @hostname = Socket.gethostname
     end
 
@@ -32,38 +34,49 @@ module Fluent
 
     # Metrics collection methods
     def get_metrics
-      list_container_ids.each do |id|
-        emit_container_metric(id, 'memory', 'memory.stat') 
-        emit_container_metric(id, 'cpuacct', 'cpuacct.stat') 
-        emit_container_metric(id, 'blkio', 'blkio.io_serviced') 
-        emit_container_metric(id, 'blkio', 'blkio.io_service_bytes') 
-        emit_container_metric(id, 'blkio', 'blkio.io_service_queued') 
-        emit_container_metric(id, 'blkio', 'blkio.sectors') 
+      ids = @container_ids || list_container_ids
+      ids.each do |id, name|
+        emit_container_metric(id, name, 'memory', 'memory.stat') 
+        emit_container_metric(id, name, 'cpuacct', 'cpuacct.stat') 
+        emit_container_metric(id, name, 'blkio', 'blkio.io_serviced') 
+        emit_container_metric(id, name, 'blkio', 'blkio.io_service_bytes') 
+        emit_container_metric(id, name, 'blkio', 'blkio.io_queued') 
+        emit_container_metric(id, name, 'blkio', 'blkio.sectors') 
       end
     end
 
     def list_container_ids
-      `docker ps --no-trunc -q`.split /\s+/
+      Docker::Container.all.map do |container|
+        [container.id, container.info["Names"].first]
+      end
     end
-   
-    def emit_container_metric(id, metric_type, metric_filename, opts = {})
+
+    def emit_container_metric(id, name, metric_type, metric_filename, opts = {})
       path = "#{@cgroup_path}/#{metric_type}/docker/#{id}/#{metric_filename}"
+
       if File.exists?(path)
-        parser = if metric_type != 'blkio'
-                   KeyValueStatsParser.new(path, metric_filename.gsub('.', '_'))
-                 else 
-                   BlkioStatsParser.new(path, metric_filename.gsub('.', '_'))
-                 end
+        # the order of these two if's matters
+        if metric_filename == 'blkio.sectors'
+          parser = BlkioSectorsParser.new(path, metric_filename.gsub('.', '_'))
+        elsif metric_type == 'blkio'
+          parser = BlkioStatsParser.new(path, metric_filename.gsub('.', '_'))
+        else
+          parser = KeyValueStatsParser.new(path, metric_filename.gsub('.', '_'))
+        end
         time = Engine.now
         tag = "#{@tag_prefix}.#{metric_filename}"
         mes = MultiEventStream.new
         parser.parse_each_line do |data|
           next if not data
           # TODO: address this more elegantly
-          if data[:key] =~ /^(?:cpuacct|blkio|memory_stat_pg)/
-            data[:type] = 'counter'
+          if data['key'] =~ /^(?:cpuacct|blkio|memory_stat_pg)/
+            data['type'] = 'counter'
+          else
+            data['type'] = 'gauge'
           end
-          data["source"] = "#{@tag_prefix}:#{@hostname}:#{id}"
+          data["hostname"] = @hostname
+          data["id"] = id
+          data["name"] = name
           mes.add(time, data)
         end
         Engine.emit_stream(tag, mes)
@@ -113,7 +126,7 @@ module Fluent
       def parse_line(line)
         k, v = line.split(/\s+/, 2)
         if k and v
-          { key: @metric_type + "_" + k, value: v.to_i }
+          { 'key' => @metric_type + "_" + k, 'value' => v.to_i }
         else
           nil
         end
@@ -126,7 +139,20 @@ module Fluent
       def parse_line(line)
         m = BlkioLineRegexp.match(line)
         if m
-          { key: @metric_type + "_" + m["key"].downcase, value: m["value"] }
+          { 'key' => @metric_type + "_" + m["key"].downcase, 'value' => m["value"].to_i }
+        else
+          nil
+        end
+      end
+    end
+
+    class BlkioSectorsParser < CGroupStatsParser
+      BlkioSectorsLineRegexp = /^(?<major>\d+):(?<minor>\d+) (?<value>\d+)/
+
+      def parse_line(line)
+        m = BlkioSectorsLineRegexp.match(line)
+        if m
+          { 'key' => @metric_type, 'value' => m["value"].to_i }
         else
           nil
         end
@@ -134,3 +160,4 @@ module Fluent
     end
   end
 end
+  

data/test/data/blkio.io_queued

@@ -0,0 +1,6 @@
+8:0 Read 0
+8:0 Write 0
+8:0 Sync 0
+8:0 Async 0
+8:0 Total 0
+Total 0

data/test/data/blkio.io_service_bytes

@@ -0,0 +1,6 @@
+8:0 Read 14970880
+8:0 Write 4096
+8:0 Sync 14974976
+8:0 Async 0
+8:0 Total 14974976
+Total 14974976

data/test/data/blkio.io_serviced

@@ -0,0 +1,6 @@
+8:0 Read 822
+8:0 Write 1
+8:0 Sync 823
+8:0 Async 0
+8:0 Total 823
+Total 823

data/test/data/blkio.sectors

@@ -0,0 +1 @@
+8:0 816

data/test/data/cpuacct.stat

@@ -0,0 +1,2 @@
+user 0
+system 0

data/test/data/memory.stat

@@ -0,0 +1,28 @@
+cache 32768
+rss 471040
+mapped_file 0
+pgpgin 293
+pgpgout 170
+swap 0
+pgfault 1254
+pgmajfault 0
+inactive_anon 20480
+active_anon 483328
+inactive_file 0
+active_file 0
+unevictable 0
+hierarchical_memory_limit 9223372036854775807
+hierarchical_memsw_limit 9223372036854775807
+total_cache 32768
+total_rss 471040
+total_mapped_file 0
+total_pgpgin 293
+total_pgpgout 170
+total_swap 0
+total_pgfault 1254
+total_pgmajfault 0
+total_inactive_anon 20480
+total_active_anon 483328
+total_inactive_file 0
+total_active_file 0
+total_unevictable 0

data/test/test_in_docker_metrics.rb

@@ -0,0 +1,134 @@
+require 'fluent/test'
+require 'fluent/plugin/in_docker_metrics'
+require 'fakefs/safe'
+
+class TestDockerMetricsInput < MiniTest::Unit::TestCase
+  METRICS = [
+      ['memory', 'memory.stat'], 
+      ['cpuacct', 'cpuacct.stat'], 
+      ['blkio', 'blkio.io_serviced'], 
+      ['blkio', 'blkio.io_service_bytes'], 
+      ['blkio', 'blkio.io_queued'],
+      ['blkio', 'blkio.sectors']
+    ]
+
+  def setup
+    @container_id = 'sadais1337hacker'
+    @container_name = 'sample_container'
+    @mock_metrics = read_mock_metrics
+    FakeFS.activate!
+    setup_proc_files
+  end
+
+  def read_mock_metrics
+    metrics = {}
+    METRICS.each do |_, file|
+      p = "#{File.dirname(File.expand_path(__FILE__))}/data/#{file}"
+      if not File.exists?(p)
+        raise IOError, p
+      end
+      metrics[file] = File.new(p).read 
+    end
+    metrics
+  end
+
+  def setup_proc_files
+    METRICS.each do |type, file|
+      path = "/sys/fs/cgroup/#{type}/docker/#{@container_id}"
+      FileUtils.mkdir_p(path)
+      fh = File.new("#{path}/#{file}", "w")
+      fh.write(@mock_metrics[file])
+      fh.close
+    end
+  end
+
+  def create_driver
+    Fluent::Test::InputTestDriver.new(Fluent::DockerMetricsInput).configure(%[
+      container_ids [["#{@container_id}", "#{@container_name}"]]
+      stats_interval 5s
+    ])
+  end
+
+  def test_outputs
+    d = create_driver
+    d.run do
+      sleep 2
+    end
+
+    emits = d.emits
+    check_metric_type(emits, 'memory.stat', [
+        {"key"=>"memory_stat_cache", "value"=>32768},
+        {"key"=>"memory_stat_rss", "value"=>471040},
+        {"key"=>"memory_stat_mapped_file", "value"=>0},
+        {"key"=>"memory_stat_pgpgin", "value"=>293},
+        {"key"=>"memory_stat_pgpgout", "value"=>170},
+        {"key"=>"memory_stat_swap", "value"=>0},
+        {"key"=>"memory_stat_pgfault", "value"=>1254},
+        {"key"=>"memory_stat_pgmajfault", "value"=>0},
+        {"key"=>"memory_stat_inactive_anon", "value"=>20480},
+        {"key"=>"memory_stat_active_anon", "value"=>483328},
+        {"key"=>"memory_stat_inactive_file", "value"=>0},
+        {"key"=>"memory_stat_active_file", "value"=>0},
+        {"key"=>"memory_stat_unevictable", "value"=>0},
+        {"key"=>"memory_stat_hierarchical_memory_limit", "value"=>9223372036854775807},
+        {"key"=>"memory_stat_hierarchical_memsw_limit", "value"=>9223372036854775807},
+        {"key"=>"memory_stat_total_cache", "value"=>32768},
+        {"key"=>"memory_stat_total_rss", "value"=>471040},
+        {"key"=>"memory_stat_total_mapped_file", "value"=>0},
+        {"key"=>"memory_stat_total_pgpgin", "value"=>293},
+        {"key"=>"memory_stat_total_pgpgout", "value"=>170},
+        {"key"=>"memory_stat_total_swap", "value"=>0},
+        {"key"=>"memory_stat_total_pgfault", "value"=>1254},
+        {"key"=>"memory_stat_total_pgmajfault", "value"=>0},
+        {"key"=>"memory_stat_total_inactive_anon", "value"=>20480},
+        {"key"=>"memory_stat_total_active_anon", "value"=>483328},
+        {"key"=>"memory_stat_total_inactive_file", "value"=>0},
+        {"key"=>"memory_stat_total_active_file", "value"=>0},
+        {"key"=>"memory_stat_total_unevictable", "value"=>0}
+      ])
+    check_metric_type(emits, 'cpuacct.stat', [
+        {"key"=>"cpuacct_stat_user", "value"=>0},
+        {"key"=>"cpuacct_stat_system", "value"=>0}
+      ])
+    check_metric_type(emits, 'blkio.io_queued', [
+        {"key"=>"blkio_io_queued_read", "value"=>0},
+        {"key"=>"blkio_io_queued_write", "value"=>0},
+        {"key"=>"blkio_io_queued_sync", "value"=>0},
+        {"key"=>"blkio_io_queued_async", "value"=>0},
+        {"key"=>"blkio_io_queued_total", "value"=>0}
+      ])
+    check_metric_type(emits, 'blkio.io_serviced', [
+        {"key"=>"blkio_io_serviced_read", "value"=>822},
+        {"key"=>"blkio_io_serviced_write", "value"=>1},
+        {"key"=>"blkio_io_serviced_sync", "value"=>823},
+        {"key"=>"blkio_io_serviced_async", "value"=>0},
+        {"key"=>"blkio_io_serviced_total", "value"=>823}
+      ])
+    check_metric_type(emits, 'blkio.sectors', [
+        {"key"=>"blkio_sectors", "value"=>816}
+      ])
+  end
+
+  def check_metric_type(emits, type, records)
+    stats = emits.select do |tag, time, record| tag == "docker.#{type}" end
+    assert_equal records.length, stats.length, "Mismatch for #{type}"
+    assert_equal @container_id, emits.first[2]["id"]
+    assert_equal @container_name, emits.first[2]["name"]
+    records.each do |record|
+      find_metric(stats, record)
+    end
+  end
+
+  def find_metric(emits, expected_record)
+    match = emits.select do |_, _, record|
+      record["key"] == expected_record["key"] &&
+      record["value"] == expected_record["value"]
+    end
+
+    assert_equal 1, match.length, "Didn't find #{expected_record.to_json} among #{emits.to_json}"
+  end
+
+  def teardown
+    FakeFS.deactivate!
+  end
+end 

metadata

@@ -1,68 +1,105 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-docker-metrics
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - kiyoto
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-23 00:00:00.000000000 Z
+date: 2014-11-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.7'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.7'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '10.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '10.1'
+- !ruby/object:Gem::Dependency
+  name: fakefs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+- !ruby/object:Gem::Dependency
+  name: docker-api
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.15'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-description: 
+        version: '1.15'
+description: Fluentd input plugin to collect container metrics periodically
 email:
 - kiyoto@treasure-data.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- CHANGELOG
 - Gemfile
 - README.md
+- Rakefile
 - fluent-plugin-docker-metrics.gemspec
 - lib/fluent/plugin/in_docker_metrics.rb
+- test/data/blkio.io_queued
+- test/data/blkio.io_service_bytes
+- test/data/blkio.io_serviced
+- test/data/blkio.sectors
+- test/data/cpuacct.stat
+- test/data/memory.stat
+- test/test_in_docker_metrics.rb
 homepage: https://github.com/kiyoto/fluent-plugin-docker-metrics
 licenses:
 - Apache License, Version 2.0
@@ -87,4 +124,11 @@ rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Fluentd plugin to collect Docker container metrics
-test_files: []
+test_files:
+- test/data/blkio.io_queued
+- test/data/blkio.io_service_bytes
+- test/data/blkio.io_serviced
+- test/data/blkio.sectors
+- test/data/cpuacct.stat
+- test/data/memory.stat
+- test/test_in_docker_metrics.rb