fluent-plugin-docker-metrics 0.0.1 → 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: fe3c77f3d0e10766bb18060d471de817455fd9cb
4
- data.tar.gz: f56a4465bb6a566240a4c27276bca97d07081222
3
+ metadata.gz: 587119d524daa8765eebadec104dc588369eaf7a
4
+ data.tar.gz: 3b7bbb2601a73bafb2e5bb77fa35506eabc7b79c
5
5
  SHA512:
6
- metadata.gz: bcbcfd347ac2a1a7cb1e06bc0cdce38fce40968a3405c75577b50a0782cb5ea341c41cdad4a56b42139a4a450278347dba0777001799d8c85e4fb941454e05d5
7
- data.tar.gz: 6d4ad921e6da4e8be9c4da6581a79253fe17cbbe692ab6e88d18e24778173da75e4c503b1fc0ee2d5bf953270dd376f367f9c6d52434a2a4063a15675048d1aa
6
+ metadata.gz: e56678bbbfbc5ff862e4ac841e7381e0e4220a54b274f55aaed8927c63634cc934155c243bdab00313fb97c956efba662ba727f83c420279aa0e920c6e59191a
7
+ data.tar.gz: 2dd6d2b41b5e78d8165f4720a9a637e1219ff7d5d6ec4cdd23d47e790d2c675c6882558c0545395409858f1febe238d27fd1a97062a476be9ac8be284be01c57
@@ -0,0 +1,7 @@
1
+ ## v0.0.2 2014/11/22
2
+
3
+ - Change the output format. The "source" field is deprecated. Instead, it has "hostname", "id", and "container".
4
+
5
+ ## v0.0.1 2014/06/27
6
+
7
+ - Initial release
data/README.md CHANGED
@@ -30,8 +30,13 @@ to be uploaded on Rubygems
30
30
  ## Example output
31
31
 
32
32
  ```
33
- 2014-06-26 18:16:43 +0000 docker.memory.stat: {"key":"memory_stat_total_active_anon","value":26025984,"source":"docker:precise64:b7f17c393775476bc0999cb6dcb4c6416e94b0473317375b9a245985dc6e91c5"}
34
- 2014-06-26 18:16:43 +0000 docker.memory.stat: {"key":"memory_stat_total_inactive_file","value":131072,"source":"docker:precise64:b7f17c393775476bc0999cb6dcb4c6416e94b0473317375b9a245985dc6e91c5"}
33
+ 2014-11-22 17:48:26 +0000 docker.blkio.io_queued: {"key":"blkio_io_queued_total","value":0,"type":"counter","hostname":"precise64","id":"24f5fb3bfc429e88aa3dbacd704667899dc496067cedcfa58dd84da42e7cb3cf","name":"/world"}
34
+ 2014-11-22 17:48:26 +0000 docker.blkio.sectors: {"key":"blkio_sectors","value":136,"type":"counter","hostname":"precise64","id":"24f5fb3bfc429e88aa3dbacd704667899dc496067cedcfa58dd84da42e7cb3cf","name":"/world"}
35
35
  ```
36
36
 
37
- In particular, each event is a key-value pair of individual metrics. Also, it has a field whose value is "\<tag_prefix\>:\<hostname\>:\<container_id\>"
37
+ In particular, each event is a key-value pair of individual metrics. Also, it has
38
+
39
+ - `hostname` is the hostname of the Docker host
40
+ - `id` is the ID of the container
41
+ - `name` is the descriptive name of the container (a la `docker inspect --format '{{ .Names }}'`)
42
+
@@ -0,0 +1,10 @@
1
+ require 'bundler/gem_tasks'
2
+ require 'rake/testtask'
3
+
4
+ Rake::TestTask.new(:test) do |test|
5
+ test.libs << "test" << "lib"
6
+ test.pattern = 'test/test_*.rb'
7
+ test.verbose = true
8
+ end
9
+
10
+ task :default => :test
@@ -4,7 +4,8 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
4
 
5
5
  Gem::Specification.new do |spec|
6
6
  spec.name = "fluent-plugin-docker-metrics"
7
- spec.version = "0.0.1"
7
+ spec.version = "0.0.2"
8
+ spec.description = 'Fluentd input plugin to collect container metrics periodically'
8
9
  spec.authors = ["kiyoto"]
9
10
  spec.email = ["kiyoto@treasure-data.com"]
10
11
  spec.summary = %q{Fluentd plugin to collect Docker container metrics}
@@ -16,7 +17,9 @@ Gem::Specification.new do |spec|
16
17
  spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
17
18
  spec.require_paths = ["lib"]
18
19
 
19
- spec.add_development_dependency "bundler"
20
- spec.add_development_dependency "rake"
21
- spec.add_runtime_dependency "fluentd"
20
+ spec.add_development_dependency "bundler", '~> 1.7'
21
+ spec.add_development_dependency "rake", '~> 10.1'
22
+ spec.add_development_dependency "fakefs", '~> 0.6'
23
+ spec.add_runtime_dependency "fluentd", '~> 0.10'
24
+ spec.add_runtime_dependency "docker-api", '~> 1.15'
22
25
  end
@@ -5,10 +5,12 @@ module Fluent
5
5
  config_param :cgroup_path, :string, :default => '/sys/fs/cgroup'
6
6
  config_param :stats_interval, :time, :default => 60 # every minute
7
7
  config_param :tag_prefix, :string, :default => "docker"
8
+ config_param :container_ids, :array, :default => nil # mainly for testing
8
9
 
9
10
  def initialize
10
11
  super
11
12
  require 'socket'
13
+ require 'docker'
12
14
  @hostname = Socket.gethostname
13
15
  end
14
16
 
@@ -32,38 +34,49 @@ module Fluent
32
34
 
33
35
  # Metrics collection methods
34
36
  def get_metrics
35
- list_container_ids.each do |id|
36
- emit_container_metric(id, 'memory', 'memory.stat')
37
- emit_container_metric(id, 'cpuacct', 'cpuacct.stat')
38
- emit_container_metric(id, 'blkio', 'blkio.io_serviced')
39
- emit_container_metric(id, 'blkio', 'blkio.io_service_bytes')
40
- emit_container_metric(id, 'blkio', 'blkio.io_service_queued')
41
- emit_container_metric(id, 'blkio', 'blkio.sectors')
37
+ ids = @container_ids || list_container_ids
38
+ ids.each do |id, name|
39
+ emit_container_metric(id, name, 'memory', 'memory.stat')
40
+ emit_container_metric(id, name, 'cpuacct', 'cpuacct.stat')
41
+ emit_container_metric(id, name, 'blkio', 'blkio.io_serviced')
42
+ emit_container_metric(id, name, 'blkio', 'blkio.io_service_bytes')
43
+ emit_container_metric(id, name, 'blkio', 'blkio.io_queued')
44
+ emit_container_metric(id, name, 'blkio', 'blkio.sectors')
42
45
  end
43
46
  end
44
47
 
45
48
  def list_container_ids
46
- `docker ps --no-trunc -q`.split /\s+/
49
+ Docker::Container.all.map do |container|
50
+ [container.id, container.info["Names"].first]
51
+ end
47
52
  end
48
-
49
- def emit_container_metric(id, metric_type, metric_filename, opts = {})
53
+
54
+ def emit_container_metric(id, name, metric_type, metric_filename, opts = {})
50
55
  path = "#{@cgroup_path}/#{metric_type}/docker/#{id}/#{metric_filename}"
56
+
51
57
  if File.exists?(path)
52
- parser = if metric_type != 'blkio'
53
- KeyValueStatsParser.new(path, metric_filename.gsub('.', '_'))
54
- else
55
- BlkioStatsParser.new(path, metric_filename.gsub('.', '_'))
56
- end
58
+ # the order of these two if's matters
59
+ if metric_filename == 'blkio.sectors'
60
+ parser = BlkioSectorsParser.new(path, metric_filename.gsub('.', '_'))
61
+ elsif metric_type == 'blkio'
62
+ parser = BlkioStatsParser.new(path, metric_filename.gsub('.', '_'))
63
+ else
64
+ parser = KeyValueStatsParser.new(path, metric_filename.gsub('.', '_'))
65
+ end
57
66
  time = Engine.now
58
67
  tag = "#{@tag_prefix}.#{metric_filename}"
59
68
  mes = MultiEventStream.new
60
69
  parser.parse_each_line do |data|
61
70
  next if not data
62
71
  # TODO: address this more elegantly
63
- if data[:key] =~ /^(?:cpuacct|blkio|memory_stat_pg)/
64
- data[:type] = 'counter'
72
+ if data['key'] =~ /^(?:cpuacct|blkio|memory_stat_pg)/
73
+ data['type'] = 'counter'
74
+ else
75
+ data['type'] = 'gauge'
65
76
  end
66
- data["source"] = "#{@tag_prefix}:#{@hostname}:#{id}"
77
+ data["hostname"] = @hostname
78
+ data["id"] = id
79
+ data["name"] = name
67
80
  mes.add(time, data)
68
81
  end
69
82
  Engine.emit_stream(tag, mes)
@@ -113,7 +126,7 @@ module Fluent
113
126
  def parse_line(line)
114
127
  k, v = line.split(/\s+/, 2)
115
128
  if k and v
116
- { key: @metric_type + "_" + k, value: v.to_i }
129
+ { 'key' => @metric_type + "_" + k, 'value' => v.to_i }
117
130
  else
118
131
  nil
119
132
  end
@@ -126,7 +139,20 @@ module Fluent
126
139
  def parse_line(line)
127
140
  m = BlkioLineRegexp.match(line)
128
141
  if m
129
- { key: @metric_type + "_" + m["key"].downcase, value: m["value"] }
142
+ { 'key' => @metric_type + "_" + m["key"].downcase, 'value' => m["value"].to_i }
143
+ else
144
+ nil
145
+ end
146
+ end
147
+ end
148
+
149
+ class BlkioSectorsParser < CGroupStatsParser
150
+ BlkioSectorsLineRegexp = /^(?<major>\d+):(?<minor>\d+) (?<value>\d+)/
151
+
152
+ def parse_line(line)
153
+ m = BlkioSectorsLineRegexp.match(line)
154
+ if m
155
+ { 'key' => @metric_type, 'value' => m["value"].to_i }
130
156
  else
131
157
  nil
132
158
  end
@@ -134,3 +160,4 @@ module Fluent
134
160
  end
135
161
  end
136
162
  end
163
+
@@ -0,0 +1,6 @@
1
+ 8:0 Read 0
2
+ 8:0 Write 0
3
+ 8:0 Sync 0
4
+ 8:0 Async 0
5
+ 8:0 Total 0
6
+ Total 0
@@ -0,0 +1,6 @@
1
+ 8:0 Read 14970880
2
+ 8:0 Write 4096
3
+ 8:0 Sync 14974976
4
+ 8:0 Async 0
5
+ 8:0 Total 14974976
6
+ Total 14974976
@@ -0,0 +1,6 @@
1
+ 8:0 Read 822
2
+ 8:0 Write 1
3
+ 8:0 Sync 823
4
+ 8:0 Async 0
5
+ 8:0 Total 823
6
+ Total 823
@@ -0,0 +1 @@
1
+ 8:0 816
@@ -0,0 +1,2 @@
1
+ user 0
2
+ system 0
@@ -0,0 +1,28 @@
1
+ cache 32768
2
+ rss 471040
3
+ mapped_file 0
4
+ pgpgin 293
5
+ pgpgout 170
6
+ swap 0
7
+ pgfault 1254
8
+ pgmajfault 0
9
+ inactive_anon 20480
10
+ active_anon 483328
11
+ inactive_file 0
12
+ active_file 0
13
+ unevictable 0
14
+ hierarchical_memory_limit 9223372036854775807
15
+ hierarchical_memsw_limit 9223372036854775807
16
+ total_cache 32768
17
+ total_rss 471040
18
+ total_mapped_file 0
19
+ total_pgpgin 293
20
+ total_pgpgout 170
21
+ total_swap 0
22
+ total_pgfault 1254
23
+ total_pgmajfault 0
24
+ total_inactive_anon 20480
25
+ total_active_anon 483328
26
+ total_inactive_file 0
27
+ total_active_file 0
28
+ total_unevictable 0
@@ -0,0 +1,134 @@
1
+ require 'fluent/test'
2
+ require 'fluent/plugin/in_docker_metrics'
3
+ require 'fakefs/safe'
4
+
5
+ class TestDockerMetricsInput < MiniTest::Unit::TestCase
6
+ METRICS = [
7
+ ['memory', 'memory.stat'],
8
+ ['cpuacct', 'cpuacct.stat'],
9
+ ['blkio', 'blkio.io_serviced'],
10
+ ['blkio', 'blkio.io_service_bytes'],
11
+ ['blkio', 'blkio.io_queued'],
12
+ ['blkio', 'blkio.sectors']
13
+ ]
14
+
15
+ def setup
16
+ @container_id = 'sadais1337hacker'
17
+ @container_name = 'sample_container'
18
+ @mock_metrics = read_mock_metrics
19
+ FakeFS.activate!
20
+ setup_proc_files
21
+ end
22
+
23
+ def read_mock_metrics
24
+ metrics = {}
25
+ METRICS.each do |_, file|
26
+ p = "#{File.dirname(File.expand_path(__FILE__))}/data/#{file}"
27
+ if not File.exists?(p)
28
+ raise IOError, p
29
+ end
30
+ metrics[file] = File.new(p).read
31
+ end
32
+ metrics
33
+ end
34
+
35
+ def setup_proc_files
36
+ METRICS.each do |type, file|
37
+ path = "/sys/fs/cgroup/#{type}/docker/#{@container_id}"
38
+ FileUtils.mkdir_p(path)
39
+ fh = File.new("#{path}/#{file}", "w")
40
+ fh.write(@mock_metrics[file])
41
+ fh.close
42
+ end
43
+ end
44
+
45
+ def create_driver
46
+ Fluent::Test::InputTestDriver.new(Fluent::DockerMetricsInput).configure(%[
47
+ container_ids [["#{@container_id}", "#{@container_name}"]]
48
+ stats_interval 5s
49
+ ])
50
+ end
51
+
52
+ def test_outputs
53
+ d = create_driver
54
+ d.run do
55
+ sleep 2
56
+ end
57
+
58
+ emits = d.emits
59
+ check_metric_type(emits, 'memory.stat', [
60
+ {"key"=>"memory_stat_cache", "value"=>32768},
61
+ {"key"=>"memory_stat_rss", "value"=>471040},
62
+ {"key"=>"memory_stat_mapped_file", "value"=>0},
63
+ {"key"=>"memory_stat_pgpgin", "value"=>293},
64
+ {"key"=>"memory_stat_pgpgout", "value"=>170},
65
+ {"key"=>"memory_stat_swap", "value"=>0},
66
+ {"key"=>"memory_stat_pgfault", "value"=>1254},
67
+ {"key"=>"memory_stat_pgmajfault", "value"=>0},
68
+ {"key"=>"memory_stat_inactive_anon", "value"=>20480},
69
+ {"key"=>"memory_stat_active_anon", "value"=>483328},
70
+ {"key"=>"memory_stat_inactive_file", "value"=>0},
71
+ {"key"=>"memory_stat_active_file", "value"=>0},
72
+ {"key"=>"memory_stat_unevictable", "value"=>0},
73
+ {"key"=>"memory_stat_hierarchical_memory_limit", "value"=>9223372036854775807},
74
+ {"key"=>"memory_stat_hierarchical_memsw_limit", "value"=>9223372036854775807},
75
+ {"key"=>"memory_stat_total_cache", "value"=>32768},
76
+ {"key"=>"memory_stat_total_rss", "value"=>471040},
77
+ {"key"=>"memory_stat_total_mapped_file", "value"=>0},
78
+ {"key"=>"memory_stat_total_pgpgin", "value"=>293},
79
+ {"key"=>"memory_stat_total_pgpgout", "value"=>170},
80
+ {"key"=>"memory_stat_total_swap", "value"=>0},
81
+ {"key"=>"memory_stat_total_pgfault", "value"=>1254},
82
+ {"key"=>"memory_stat_total_pgmajfault", "value"=>0},
83
+ {"key"=>"memory_stat_total_inactive_anon", "value"=>20480},
84
+ {"key"=>"memory_stat_total_active_anon", "value"=>483328},
85
+ {"key"=>"memory_stat_total_inactive_file", "value"=>0},
86
+ {"key"=>"memory_stat_total_active_file", "value"=>0},
87
+ {"key"=>"memory_stat_total_unevictable", "value"=>0}
88
+ ])
89
+ check_metric_type(emits, 'cpuacct.stat', [
90
+ {"key"=>"cpuacct_stat_user", "value"=>0},
91
+ {"key"=>"cpuacct_stat_system", "value"=>0}
92
+ ])
93
+ check_metric_type(emits, 'blkio.io_queued', [
94
+ {"key"=>"blkio_io_queued_read", "value"=>0},
95
+ {"key"=>"blkio_io_queued_write", "value"=>0},
96
+ {"key"=>"blkio_io_queued_sync", "value"=>0},
97
+ {"key"=>"blkio_io_queued_async", "value"=>0},
98
+ {"key"=>"blkio_io_queued_total", "value"=>0}
99
+ ])
100
+ check_metric_type(emits, 'blkio.io_serviced', [
101
+ {"key"=>"blkio_io_serviced_read", "value"=>822},
102
+ {"key"=>"blkio_io_serviced_write", "value"=>1},
103
+ {"key"=>"blkio_io_serviced_sync", "value"=>823},
104
+ {"key"=>"blkio_io_serviced_async", "value"=>0},
105
+ {"key"=>"blkio_io_serviced_total", "value"=>823}
106
+ ])
107
+ check_metric_type(emits, 'blkio.sectors', [
108
+ {"key"=>"blkio_sectors", "value"=>816}
109
+ ])
110
+ end
111
+
112
+ def check_metric_type(emits, type, records)
113
+ stats = emits.select do |tag, time, record| tag == "docker.#{type}" end
114
+ assert_equal records.length, stats.length, "Mismatch for #{type}"
115
+ assert_equal @container_id, emits.first[2]["id"]
116
+ assert_equal @container_name, emits.first[2]["name"]
117
+ records.each do |record|
118
+ find_metric(stats, record)
119
+ end
120
+ end
121
+
122
+ def find_metric(emits, expected_record)
123
+ match = emits.select do |_, _, record|
124
+ record["key"] == expected_record["key"] &&
125
+ record["value"] == expected_record["value"]
126
+ end
127
+
128
+ assert_equal 1, match.length, "Didn't find #{expected_record.to_json} among #{emits.to_json}"
129
+ end
130
+
131
+ def teardown
132
+ FakeFS.deactivate!
133
+ end
134
+ end
metadata CHANGED
@@ -1,68 +1,105 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-docker-metrics
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.1
4
+ version: 0.0.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - kiyoto
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-07-23 00:00:00.000000000 Z
11
+ date: 2014-11-22 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
- - - ">="
17
+ - - "~>"
18
18
  - !ruby/object:Gem::Version
19
- version: '0'
19
+ version: '1.7'
20
20
  type: :development
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
- - - ">="
24
+ - - "~>"
25
25
  - !ruby/object:Gem::Version
26
- version: '0'
26
+ version: '1.7'
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: rake
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
- - - ">="
31
+ - - "~>"
32
32
  - !ruby/object:Gem::Version
33
- version: '0'
33
+ version: '10.1'
34
34
  type: :development
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
- - - ">="
38
+ - - "~>"
39
39
  - !ruby/object:Gem::Version
40
- version: '0'
40
+ version: '10.1'
41
+ - !ruby/object:Gem::Dependency
42
+ name: fakefs
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '0.6'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '0.6'
41
55
  - !ruby/object:Gem::Dependency
42
56
  name: fluentd
43
57
  requirement: !ruby/object:Gem::Requirement
44
58
  requirements:
45
- - - ">="
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '0.10'
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: '0.10'
69
+ - !ruby/object:Gem::Dependency
70
+ name: docker-api
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - "~>"
46
74
  - !ruby/object:Gem::Version
47
- version: '0'
75
+ version: '1.15'
48
76
  type: :runtime
49
77
  prerelease: false
50
78
  version_requirements: !ruby/object:Gem::Requirement
51
79
  requirements:
52
- - - ">="
80
+ - - "~>"
53
81
  - !ruby/object:Gem::Version
54
- version: '0'
55
- description:
82
+ version: '1.15'
83
+ description: Fluentd input plugin to collect container metrics periodically
56
84
  email:
57
85
  - kiyoto@treasure-data.com
58
86
  executables: []
59
87
  extensions: []
60
88
  extra_rdoc_files: []
61
89
  files:
90
+ - CHANGELOG
62
91
  - Gemfile
63
92
  - README.md
93
+ - Rakefile
64
94
  - fluent-plugin-docker-metrics.gemspec
65
95
  - lib/fluent/plugin/in_docker_metrics.rb
96
+ - test/data/blkio.io_queued
97
+ - test/data/blkio.io_service_bytes
98
+ - test/data/blkio.io_serviced
99
+ - test/data/blkio.sectors
100
+ - test/data/cpuacct.stat
101
+ - test/data/memory.stat
102
+ - test/test_in_docker_metrics.rb
66
103
  homepage: https://github.com/kiyoto/fluent-plugin-docker-metrics
67
104
  licenses:
68
105
  - Apache License, Version 2.0
@@ -87,4 +124,11 @@ rubygems_version: 2.2.2
87
124
  signing_key:
88
125
  specification_version: 4
89
126
  summary: Fluentd plugin to collect Docker container metrics
90
- test_files: []
127
+ test_files:
128
+ - test/data/blkio.io_queued
129
+ - test/data/blkio.io_service_bytes
130
+ - test/data/blkio.io_serviced
131
+ - test/data/blkio.sectors
132
+ - test/data/cpuacct.stat
133
+ - test/data/memory.stat
134
+ - test/test_in_docker_metrics.rb