fluent-plugin-cloudwatch 1.0.0 → 1.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (6) hide show
  1. data/README.md +143 -0
  2. data/fluent-plugin-cloudwatch.gemspec +9 -10
  3. data/lib/fluent/plugin/in_cloudwatch.rb +45 -44
  4. data/test/plugin/test_in_cloudwatch.rb +47 -7
  5. metadata +16 -14
  6. data/README.rdoc +0 -49
data/README.md ADDED
@@ -0,0 +1,143 @@
1
+ # fluent-plugin-cloudwatch
2
+
3
+
4
+ ## Overview
5
+ ***AWS CloudWatch*** input plugin.
6
+
7
+ this plugin is simple.
8
+ get metrics from cloudwatch.
9
+
10
+ 1. get every 60 seconds from AWS CloudWatch API(sleep 1sec)
11
+ 2. 60 seconds of data
12
+
13
+ ## Configuration
14
+
15
+ ```config
16
+ <source>
17
+ type cloudwatch
18
+ tag cloudwatch
19
+ aws_key_id YOUR_AWS_KEY_ID
20
+ aws_sec_key YOUR_AWS_SECRET_KEY
21
+ cw_endpoint ENDPOINT
22
+
23
+ namespace [namespace]
24
+ statistics [statistics] (default: Average)
25
+ metric_name [metric name]
26
+ dimensions_name [dimensions_name]
27
+ dimensions_value [dimensions value]
28
+ period [period] (default: 60)
29
+ </source>
30
+
31
+ ```
32
+
33
+ ### GET RDS Metric
34
+
35
+ ```config
36
+ <source>
37
+ type cloudwatch
38
+ tag cloudwatch
39
+ aws_key_id YOUR_AWS_KEY_ID
40
+ aws_sec_key YOUR_AWS_SECRET_KEY
41
+ cw_endpoint monitoring.ap-northeast-1.amazonaws.com
42
+
43
+ namespace AWS/RDS
44
+ metric_name CPUUtilization,FreeStorageSpace,DiskQueueDepth,FreeableMemory,SwapUsage,ReadIOPS,ReadLatency,ReadThroughput,WriteIOPS,WriteLatency,WriteThroughput
45
+ dimensions_name DBInstanceIdentifier
46
+ dimensions_value rds01
47
+ </source>
48
+
49
+ <match cloudwatch>
50
+ type copy
51
+ <store>
52
+ type file
53
+ path /var/log/td-agent/test
54
+ </store>
55
+ </match>
56
+
57
+ ```
58
+
59
+ #### output data format
60
+
61
+ ```
62
+ 2013-02-24T13:40:00+09:00 cloudwatch {"CPUUtilization":2.0}
63
+ 2013-02-24T13:40:00+09:00 cloudwatch {"FreeStorageSpace":104080723968.0}
64
+ 2013-02-24T13:39:00+09:00 cloudwatch {"DiskQueueDepth":0.002000233360558732}
65
+ 2013-02-24T13:40:00+09:00 cloudwatch {"FreeableMemory":6047948800.0}
66
+ 2013-02-24T13:40:00+09:00 cloudwatch {"SwapUsage":0.0}
67
+ 2013-02-24T13:40:00+09:00 cloudwatch {"ReadIOPS":0.4832769510223807}
68
+ 2013-02-24T13:40:00+09:00 cloudwatch {"ReadLatency":0.0}
69
+ 2013-02-24T13:39:00+09:00 cloudwatch {"ReadThroughput":0.0}
70
+ 2013-02-24T13:40:00+09:00 cloudwatch {"WriteIOPS":5.116069791857616}
71
+ 2013-02-24T13:40:00+09:00 cloudwatch {"WriteLatency":0.004106280193236715}
72
+ 2013-02-24T13:39:00+09:00 cloudwatch {"WriteThroughput":54074.40992132284}
73
+ ```
74
+
75
+
76
+ ### GET ELB Metirc
77
+
78
+ ```config
79
+ <source>
80
+ type cloudwatch
81
+ tag cloudwatch
82
+ aws_key_id YOUR_AWS_KEY_ID
83
+ aws_sec_key YOUR_AWS_SECRET/KE
84
+ cw_endpoint monitoring.ap-northeast-1.amazonaws.com
85
+
86
+ namespace AWS/ELB
87
+ metric_name HealthyHostCount,HTTPCode_Backend_2XX,HTTPCode_Backend_3XX,HTTPCode_Backend_4XX,HTTPCode_Backend_5XX,HTTPCode_ELB_4XX,Latency,RequestCount,UnHealthyHostCount
88
+ dimensions_name LoadBalancerName
89
+ dimensions_value YOUR_ELB_NAME
90
+ </source>
91
+ ```
92
+
93
+ #### output data format
94
+
95
+ ```
96
+
97
+ 2013-03-21T14:08:00+09:00 cloudwatch {"HealthyHostCount":2.0}
98
+ 2013-03-21T14:08:00+09:00 cloudwatch {"HTTPCode_Backend_2XX":1.0}
99
+ 2013-03-21T14:08:00+09:00 cloudwatch {"Latency":0.004025}
100
+ 2013-03-21T14:08:00+09:00 cloudwatch {"RequestCount":1.0}
101
+ 2013-03-21T14:09:00+09:00 cloudwatch {"UnHealthyHostCount":0.0}
102
+
103
+ ```
104
+
105
+
106
+ ### GET EC2 Metirc
107
+
108
+ ```config
109
+ <source>
110
+ type cloudwatch
111
+ tag cloudwatch
112
+ aws_key_id YOUR_AWS_KEY_ID
113
+ aws_sec_key YOUR_AWS_SECRET/KE
114
+ cw_endpoint monitoring.ap-northeast-1.amazonaws.com
115
+
116
+ namespace AWS/EC2
117
+ metric_name CPUUtilization,FreeStorageSpace,DiskQueueDepth,FreeableMemory,SwapUsage,ReadIOPS,ReadLatency,ReadThroughput,WriteIOPS,WriteLatency,WriteThroughput
118
+ dimensions_name InstanceId
119
+ dimensions_value YOUR_INSTANCE_ID
120
+ </source>
121
+ ```
122
+
123
+ #### output data format
124
+
125
+ ```
126
+
127
+ 2013-02-25T00:44:00+09:00 cloudwatch {"CPUUtilization":1.58}
128
+ 2013-02-25T00:44:00+09:00 cloudwatch {"DiskReadBytes":0.0}
129
+ 2013-02-25T00:44:00+09:00 cloudwatch {"DiskReadBytes":0.0}
130
+ 2013-02-25T00:44:00+09:00 cloudwatch {"DiskWriteBytes":0.0}
131
+ 2013-02-25T00:44:00+09:00 cloudwatch {"DiskWriteOps":0.0}
132
+ 2013-02-25T00:44:00+09:00 cloudwatch {"NetworkIn":95183.0}
133
+ 2013-02-25T00:44:00+09:00 cloudwatch {"NetworkOut":95645.0}
134
+
135
+ ```
136
+
137
+ ## Contributing
138
+
139
+ 1. Fork it
140
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
141
+ 3. Commit your changes (`git commit -am 'Added some feature'`)
142
+ 4. Push to the branch (`git push origin my-new-feature`)
143
+ 5. Create new Pull Request
data/fluent-plugin-cloudwatch.gemspec CHANGED
@@ -1,20 +1,19 @@
1
1
  # -*- encoding: utf-8 -*-
2
- $:.push File.expand_path("../lib", __FILE__)
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
3
4
 
4
5
  Gem::Specification.new do |gem|
5
- gem.authors = ["Yusuke Nomura"]
6
- gem.email = ["yunomu@gmail.com"]
6
+ gem.name = "fluent-plugin-cloudwatch"
7
+ gem.version = "1.1.0"
8
+ gem.authors = ["Yusuke Nomura", "kenjiskywalker"]
9
+ gem.email = ["yunomu@gmail.com", "git@kenjiskywalker.org"]
7
10
  gem.description = %q{Input plugin for AWS CloudWatch.}
8
- gem.summary = %q{Input plugin for AWS CloudWatch}
9
11
  gem.homepage = "https://github.com/yunomu/fluent-plugin-cloudwatch"
10
-
12
+ gem.summary = gem.description
11
13
  gem.files = `git ls-files`.split($\)
12
14
  gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
13
15
  gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
14
- gem.name = "fluent-plugin-cloudwatch"
15
16
  gem.require_paths = ["lib"]
16
- gem.version = "1.0.0"
17
-
18
- gem.add_development_dependency "fluentd"
19
- gem.add_runtime_dependency "fluentd"
17
+ gem.add_dependency "fluentd", "~> 0.10.30"
18
+ gem.add_dependency "aws-sdk", "= 1.8.3"
20
19
  end
data/lib/fluent/plugin/in_cloudwatch.rb CHANGED
@@ -1,33 +1,31 @@
1
1
  class Fluent::CloudwatchInput < Fluent::Input
2
2
  Fluent::Plugin.register_input("cloudwatch", self)
3
3
 
4
- config_param :tag, :string
5
- config_param :access_key, :string
6
- config_param :secret_key, :string
7
- config_param :endpoint, :string, :default => "monitoring.amazonaws.com"
4
+ config_param :tag, :string
5
+ config_param :aws_key_id, :string, :default => nil
6
+ config_param :aws_sec_key, :string, :default => nil
7
+ config_param :cw_endpoint, :string, :default => nil
8
8
 
9
- def initialize
9
+ config_param :namespace, :string, :default => nil
10
+ config_param :metric_name, :string, :default => nil
11
+ config_param :statistics, :string, :default => "Average"
12
+ config_param :dimensions_name, :string, :default => nil
13
+ config_param :dimensions_value, :string, :default => nil
14
+ config_param :period, :integer, :default => 60
15
+
16
+
17
+ def initialize
10
18
  super
11
- require 'rubygems'
12
- require 'AWS'
13
- require 'time'
19
+ require 'aws-sdk'
14
20
  end
15
21
 
16
22
  def configure(conf)
17
23
  super
18
- @mon = AWS::Cloudwatch::Base.new(
19
- :access_key_id => access_key,
20
- :secret_access_key => secret_key,
21
- :server => endpoint)
22
-
23
- @metrics = conf.elements.select {|e|
24
- e.name == 'metric'
25
- }.map {|e|
26
- {:namespace => e['namespace'],
27
- :statistics => e['statistics'],
28
- :dimensions => e['dimensions'],
29
- :measure_name => e['metric_name']}
30
- }
24
+ @cw = AWS::CloudWatch.new(
25
+ :access_key_id => @aws_key_id,
26
+ :secret_access_key => @aws_sec_key,
27
+ :cloud_watch_endpoint => @cw_endpoint
28
+ ).client
31
29
  end
32
30
 
33
31
  def start
@@ -44,34 +42,37 @@ class Fluent::CloudwatchInput < Fluent::Input
44
42
  private
45
43
  def watch
46
44
  while true
47
- Fluent::Engine.emit(tag, Fluent::Engine.now, output)
48
- sleep 60 * 5
45
+ sleep 60
46
+ output
49
47
  end
50
48
  end
51
49
 
52
50
  def output
53
- end_time = Time.now
54
- start_time = end_time - 60 * 60
55
- @metrics.map {|m|
56
- m[:start_time] = start_time
57
- m[:end_time] = end_time
58
- format m, @mon.get_metric_statistics(m)
59
- }
60
- end
61
-
62
- def format(m, s)
63
- d = s["GetMetricStatisticsResult"]["Datapoints"]["member"].sort {|a,b|
64
- time(a) <=> time(b)
65
- }.last
51
+ @metric_name.split(",").each {|m|
52
+ statistics = @cw.get_metric_statistics({
53
+ :namespace => @namespace,
54
+ :metric_name => m,
55
+ :statistics => [@statistics],
56
+ :dimensions => [{
57
+ :name => @dimensions_name,
58
+ :value => @dimensions_value
59
+ }],
60
+ :start_time => (Time.now - 120).iso8601,
61
+ :end_time => Time.now.iso8601,
62
+ :period => @period
63
+ })
64
+ unless statistics[:datapoints].empty?
65
+ stat = @statistics.downcase.to_sym
66
+ data = statistics[:datapoints][0][stat]
66
67
 
67
- d["MetricName"] = m[:measure_name]
68
- d["Statistics"] = m[:statistics]
69
- d["Value"] = d.delete(m[:statistics])
70
- d
71
- end
68
+ # unix time
69
+ catch_time = statistics[:datapoints][0][:timestamp].to_i
72
70
 
73
- def time(d)
74
- Time.parse d["Timestamp"]
71
+ # no output_data.to_json
72
+ output_data = {m => data}
73
+ Fluent::Engine.emit(tag, catch_time, output_data)
74
+ end
75
+ }
76
+ sleep 1
75
77
  end
76
78
  end
77
-
data/test/plugin/test_in_cloudwatch.rb CHANGED
@@ -5,21 +5,61 @@ class CloudwatchInputTest < Test::Unit::TestCase
5
5
  Fluent::Test.setup
6
6
  end
7
7
 
8
- CONFIG = %[
8
+ ### for RDS
9
+ CONFIG_RDS = %[
10
+ tag cloudwatch
11
+ aws_key_id test_key_id
12
+ aws_sec_key test_sec_key
13
+ cw_endpoint test_cloud_watch_endpoint
14
+ namespace AWS/RDS
15
+ metric_name CPUUtilization,FreeStorageSpace,DiskQueueDepth,FreeableMemory,SwapUsage,ReadIOPS,ReadLatency,ReadThroughput,WriteIOPS,WriteLatency,WriteThroughput
16
+ dimensions_name DBInstanceIdentifier
17
+ dimensions_value rds01
9
18
  ]
10
19
 
11
- def create_driver(conf = CONFIG, tag = 'test')
20
+ def create_driver_rds(conf = CONFIG_RDS)
12
21
  Fluent::Test::InputTestDriver.new(Fluent::CloudwatchInput).configure(conf)
13
22
  end
14
23
 
15
- def test_configure
24
+ def test_configure_rds
25
+ d = create_driver_rds
26
+ assert_equal 'cloudwatch', d.instance.tag
27
+ assert_equal 'test_key_id', d.instance.aws_key_id
28
+ assert_equal 'test_sec_key', d.instance.aws_sec_key
29
+ assert_equal 'test_cloud_watch_endpoint', d.instance.cw_endpoint
30
+ assert_equal 'AWS/RDS', d.instance.namespace
31
+ assert_equal 'CPUUtilization,FreeStorageSpace,DiskQueueDepth,FreeableMemory,SwapUsage,ReadIOPS,ReadLatency,ReadThroughput,WriteIOPS,WriteLatency,WriteThroughput', d.instance.metric_name
32
+ assert_equal 'DBInstanceIdentifier', d.instance.dimensions_name
33
+ assert_equal 'rds01', d.instance.dimensions_value
16
34
  end
17
35
 
18
- def test_format
19
- d = create_driver
36
+
37
+ ### for EC2
38
+ CONFIG_ECTWO = %[
39
+ tag cloudwatch
40
+ aws_key_id test_key_id
41
+ aws_sec_key test_sec_key
42
+ cw_endpoint test_cloud_watch_endpoint
43
+ namespace AWS/EC2
44
+ metric_name CPUUtilization,FreeStorageSpace,DiskQueueDepth,FreeableMemory,SwapUsage,ReadIOPS,ReadLatency,ReadThroughput,WriteIOPS,WriteLatency,WriteThroughput
45
+ dimensions_name InstanceId
46
+ dimensions_value ec2-01
47
+ ]
48
+
49
+ def create_driver_ectwo(conf = CONFIG_ECTWO)
50
+ Fluent::Test::InputTestDriver.new(Fluent::CloudwatchInput).configure(conf)
20
51
  end
21
52
 
22
- def test_write
23
- d = create_driver
53
+ def test_configure_ectwo
54
+ d = create_driver_ectwo
55
+ assert_equal 'cloudwatch', d.instance.tag
56
+ assert_equal 'test_key_id', d.instance.aws_key_id
57
+ assert_equal 'test_sec_key', d.instance.aws_sec_key
58
+ assert_equal 'test_cloud_watch_endpoint', d.instance.cw_endpoint
59
+ assert_equal 'AWS/EC2', d.instance.namespace
60
+ assert_equal 'CPUUtilization,FreeStorageSpace,DiskQueueDepth,FreeableMemory,SwapUsage,ReadIOPS,ReadLatency,ReadThroughput,WriteIOPS,WriteLatency,WriteThroughput', d.instance.metric_name
61
+ assert_equal 'InstanceId', d.instance.dimensions_name
62
+ assert_equal 'ec2-01', d.instance.dimensions_value
24
63
  end
64
+
25
65
  end
metadata CHANGED
@@ -1,51 +1,53 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-cloudwatch
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.0
4
+ version: 1.1.0
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
8
8
  - Yusuke Nomura
9
+ - kenjiskywalker
9
10
  autorequire:
10
11
  bindir: bin
11
12
  cert_chain: []
12
- date: 2013-01-14 00:00:00.000000000 Z
13
+ date: 2013-04-07 00:00:00.000000000 Z
13
14
  dependencies:
14
15
  - !ruby/object:Gem::Dependency
15
16
  name: fluentd
16
17
  requirement: !ruby/object:Gem::Requirement
17
18
  none: false
18
19
  requirements:
19
- - - ! '>='
20
+ - - ~>
20
21
  - !ruby/object:Gem::Version
21
- version: '0'
22
- type: :development
22
+ version: 0.10.30
23
+ type: :runtime
23
24
  prerelease: false
24
25
  version_requirements: !ruby/object:Gem::Requirement
25
26
  none: false
26
27
  requirements:
27
- - - ! '>='
28
+ - - ~>
28
29
  - !ruby/object:Gem::Version
29
- version: '0'
30
+ version: 0.10.30
30
31
  - !ruby/object:Gem::Dependency
31
- name: fluentd
32
+ name: aws-sdk
32
33
  requirement: !ruby/object:Gem::Requirement
33
34
  none: false
34
35
  requirements:
35
- - - ! '>='
36
+ - - '='
36
37
  - !ruby/object:Gem::Version
37
- version: '0'
38
+ version: 1.8.3
38
39
  type: :runtime
39
40
  prerelease: false
40
41
  version_requirements: !ruby/object:Gem::Requirement
41
42
  none: false
42
43
  requirements:
43
- - - ! '>='
44
+ - - '='
44
45
  - !ruby/object:Gem::Version
45
- version: '0'
46
+ version: 1.8.3
46
47
  description: Input plugin for AWS CloudWatch.
47
48
  email:
48
49
  - yunomu@gmail.com
50
+ - git@kenjiskywalker.org
49
51
  executables: []
50
52
  extensions: []
51
53
  extra_rdoc_files: []
@@ -53,7 +55,7 @@ files:
53
55
  - .gitignore
54
56
  - Gemfile
55
57
  - LICENSE
56
- - README.rdoc
58
+ - README.md
57
59
  - Rakefile
58
60
  - fluent-plugin-cloudwatch.gemspec
59
61
  - lib/fluent/plugin/in_cloudwatch.rb
@@ -82,7 +84,7 @@ rubyforge_project:
82
84
  rubygems_version: 1.8.23
83
85
  signing_key:
84
86
  specification_version: 3
85
- summary: Input plugin for AWS CloudWatch
87
+ summary: Input plugin for AWS CloudWatch.
86
88
  test_files:
87
89
  - test/helper.rb
88
90
  - test/plugin/test_in_cloudwatch.rb
data/README.rdoc DELETED
@@ -1,49 +0,0 @@
1
- = Fluent::Plugin::Cloudwatch
2
-
3
- Plugin for AWS CloudWatch.
4
-
5
- == Installation
6
-
7
- Add this line to your application's Gemfile:
8
-
9
- gem 'fluent-plugin-cloudwatch'
10
-
11
- And then execute:
12
-
13
- $ bundle
14
-
15
- Or install it yourself as:
16
-
17
- $ gem install fluent-plugin-cloudwatch
18
-
19
- == Configuration
20
-
21
- <source>
22
- type cloudwatch
23
- tag cloudwatch
24
- access_key XXXXXXXXXX
25
- secret_key XXXXXXXXXX
26
- endpoint monitoring.ap-northeast-1.amazonaws.com
27
-
28
- <metric>
29
- namespace AWS/EC2
30
- statistics Average
31
- dimensions InstanceId=i-xxxxxxxx
32
- metric_name CPUUtilization
33
- </metric>
34
- <metric>
35
- namespace AWS/EC2
36
- statistics Sum
37
- dimensions InstanceId=i-xxxxxxxx
38
- metric_name NetworkIn
39
- </metric>
40
- </source>
41
-
42
-
43
- == Contributing
44
-
45
- 1. Fork it
46
- 2. Create your feature branch (`git checkout -b my-new-feature`)
47
- 3. Commit your changes (`git commit -am 'Added some feature'`)
48
- 4. Push to the branch (`git push origin my-new-feature`)
49
- 5. Create new Pull Request