fluent-plugin-cloudwatch 1.0.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (6) hide show
  1. data/README.md +143 -0
  2. data/fluent-plugin-cloudwatch.gemspec +9 -10
  3. data/lib/fluent/plugin/in_cloudwatch.rb +45 -44
  4. data/test/plugin/test_in_cloudwatch.rb +47 -7
  5. metadata +16 -14
  6. data/README.rdoc +0 -49
data/README.md ADDED
@@ -0,0 +1,143 @@
1
+ # fluent-plugin-cloudwatch
2
+
3
+
4
+ ## Overview
5
+ ***AWS CloudWatch*** input plugin.
6
+
7
+ this plugin is simple.
8
+ get metrics from cloudwatch.
9
+
10
+ 1. get every 60 seconds from AWS CloudWatch API(sleep 1sec)
11
+ 2. 60 seconds of data
12
+
13
+ ## Configuration
14
+
15
+ ```config
16
+ <source>
17
+ type cloudwatch
18
+ tag cloudwatch
19
+ aws_key_id YOUR_AWS_KEY_ID
20
+ aws_sec_key YOUR_AWS_SECRET_KEY
21
+ cw_endpoint ENDPOINT
22
+
23
+ namespace [namespace]
24
+ statistics [statistics] (default: Average)
25
+ metric_name [metric name]
26
+ dimensions_name [dimensions_name]
27
+ dimensions_value [dimensions value]
28
+ period [period] (default: 60)
29
+ </source>
30
+
31
+ ```
32
+
33
+ ### GET RDS Metric
34
+
35
+ ```config
36
+ <source>
37
+ type cloudwatch
38
+ tag cloudwatch
39
+ aws_key_id YOUR_AWS_KEY_ID
40
+ aws_sec_key YOUR_AWS_SECRET_KEY
41
+ cw_endpoint monitoring.ap-northeast-1.amazonaws.com
42
+
43
+ namespace AWS/RDS
44
+ metric_name CPUUtilization,FreeStorageSpace,DiskQueueDepth,FreeableMemory,SwapUsage,ReadIOPS,ReadLatency,ReadThroughput,WriteIOPS,WriteLatency,WriteThroughput
45
+ dimensions_name DBInstanceIdentifier
46
+ dimensions_value rds01
47
+ </source>
48
+
49
+ <match cloudwatch>
50
+ type copy
51
+ <store>
52
+ type file
53
+ path /var/log/td-agent/test
54
+ </store>
55
+ </match>
56
+
57
+ ```
58
+
59
+ #### output data format
60
+
61
+ ```
62
+ 2013-02-24T13:40:00+09:00 cloudwatch {"CPUUtilization":2.0}
63
+ 2013-02-24T13:40:00+09:00 cloudwatch {"FreeStorageSpace":104080723968.0}
64
+ 2013-02-24T13:39:00+09:00 cloudwatch {"DiskQueueDepth":0.002000233360558732}
65
+ 2013-02-24T13:40:00+09:00 cloudwatch {"FreeableMemory":6047948800.0}
66
+ 2013-02-24T13:40:00+09:00 cloudwatch {"SwapUsage":0.0}
67
+ 2013-02-24T13:40:00+09:00 cloudwatch {"ReadIOPS":0.4832769510223807}
68
+ 2013-02-24T13:40:00+09:00 cloudwatch {"ReadLatency":0.0}
69
+ 2013-02-24T13:39:00+09:00 cloudwatch {"ReadThroughput":0.0}
70
+ 2013-02-24T13:40:00+09:00 cloudwatch {"WriteIOPS":5.116069791857616}
71
+ 2013-02-24T13:40:00+09:00 cloudwatch {"WriteLatency":0.004106280193236715}
72
+ 2013-02-24T13:39:00+09:00 cloudwatch {"WriteThroughput":54074.40992132284}
73
+ ```
74
+
75
+
76
+ ### GET ELB Metirc
77
+
78
+ ```config
79
+ <source>
80
+ type cloudwatch
81
+ tag cloudwatch
82
+ aws_key_id YOUR_AWS_KEY_ID
83
+ aws_sec_key YOUR_AWS_SECRET/KE
84
+ cw_endpoint monitoring.ap-northeast-1.amazonaws.com
85
+
86
+ namespace AWS/ELB
87
+ metric_name HealthyHostCount,HTTPCode_Backend_2XX,HTTPCode_Backend_3XX,HTTPCode_Backend_4XX,HTTPCode_Backend_5XX,HTTPCode_ELB_4XX,Latency,RequestCount,UnHealthyHostCount
88
+ dimensions_name LoadBalancerName
89
+ dimensions_value YOUR_ELB_NAME
90
+ </source>
91
+ ```
92
+
93
+ #### output data format
94
+
95
+ ```
96
+
97
+ 2013-03-21T14:08:00+09:00 cloudwatch {"HealthyHostCount":2.0}
98
+ 2013-03-21T14:08:00+09:00 cloudwatch {"HTTPCode_Backend_2XX":1.0}
99
+ 2013-03-21T14:08:00+09:00 cloudwatch {"Latency":0.004025}
100
+ 2013-03-21T14:08:00+09:00 cloudwatch {"RequestCount":1.0}
101
+ 2013-03-21T14:09:00+09:00 cloudwatch {"UnHealthyHostCount":0.0}
102
+
103
+ ```
104
+
105
+
106
+ ### GET EC2 Metirc
107
+
108
+ ```config
109
+ <source>
110
+ type cloudwatch
111
+ tag cloudwatch
112
+ aws_key_id YOUR_AWS_KEY_ID
113
+ aws_sec_key YOUR_AWS_SECRET/KE
114
+ cw_endpoint monitoring.ap-northeast-1.amazonaws.com
115
+
116
+ namespace AWS/EC2
117
+ metric_name CPUUtilization,FreeStorageSpace,DiskQueueDepth,FreeableMemory,SwapUsage,ReadIOPS,ReadLatency,ReadThroughput,WriteIOPS,WriteLatency,WriteThroughput
118
+ dimensions_name InstanceId
119
+ dimensions_value YOUR_INSTANCE_ID
120
+ </source>
121
+ ```
122
+
123
+ #### output data format
124
+
125
+ ```
126
+
127
+ 2013-02-25T00:44:00+09:00 cloudwatch {"CPUUtilization":1.58}
128
+ 2013-02-25T00:44:00+09:00 cloudwatch {"DiskReadBytes":0.0}
129
+ 2013-02-25T00:44:00+09:00 cloudwatch {"DiskReadBytes":0.0}
130
+ 2013-02-25T00:44:00+09:00 cloudwatch {"DiskWriteBytes":0.0}
131
+ 2013-02-25T00:44:00+09:00 cloudwatch {"DiskWriteOps":0.0}
132
+ 2013-02-25T00:44:00+09:00 cloudwatch {"NetworkIn":95183.0}
133
+ 2013-02-25T00:44:00+09:00 cloudwatch {"NetworkOut":95645.0}
134
+
135
+ ```
136
+
137
+ ## Contributing
138
+
139
+ 1. Fork it
140
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
141
+ 3. Commit your changes (`git commit -am 'Added some feature'`)
142
+ 4. Push to the branch (`git push origin my-new-feature`)
143
+ 5. Create new Pull Request
data/fluent-plugin-cloudwatch.gemspec CHANGED
@@ -1,20 +1,19 @@
1
1
  # -*- encoding: utf-8 -*-
2
- $:.push File.expand_path("../lib", __FILE__)
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
3
4
 
4
5
  Gem::Specification.new do |gem|
5
- gem.authors = ["Yusuke Nomura"]
6
- gem.email = ["yunomu@gmail.com"]
6
+ gem.name = "fluent-plugin-cloudwatch"
7
+ gem.version = "1.1.0"
8
+ gem.authors = ["Yusuke Nomura", "kenjiskywalker"]
9
+ gem.email = ["yunomu@gmail.com", "git@kenjiskywalker.org"]
7
10
  gem.description = %q{Input plugin for AWS CloudWatch.}
8
- gem.summary = %q{Input plugin for AWS CloudWatch}
9
11
  gem.homepage = "https://github.com/yunomu/fluent-plugin-cloudwatch"
10
-
12
+ gem.summary = gem.description
11
13
  gem.files = `git ls-files`.split($\)
12
14
  gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
13
15
  gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
14
- gem.name = "fluent-plugin-cloudwatch"
15
16
  gem.require_paths = ["lib"]
16
- gem.version = "1.0.0"
17
-
18
- gem.add_development_dependency "fluentd"
19
- gem.add_runtime_dependency "fluentd"
17
+ gem.add_dependency "fluentd", "~> 0.10.30"
18
+ gem.add_dependency "aws-sdk", "= 1.8.3"
20
19
  end
data/lib/fluent/plugin/in_cloudwatch.rb CHANGED
@@ -1,33 +1,31 @@
1
1
  class Fluent::CloudwatchInput < Fluent::Input
2
2
  Fluent::Plugin.register_input("cloudwatch", self)
3
3
 
4
- config_param :tag, :string
5
- config_param :access_key, :string
6
- config_param :secret_key, :string
7
- config_param :endpoint, :string, :default => "monitoring.amazonaws.com"
4
+ config_param :tag, :string
5
+ config_param :aws_key_id, :string, :default => nil
6
+ config_param :aws_sec_key, :string, :default => nil
7
+ config_param :cw_endpoint, :string, :default => nil
8
8
 
9
- def initialize
9
+ config_param :namespace, :string, :default => nil
10
+ config_param :metric_name, :string, :default => nil
11
+ config_param :statistics, :string, :default => "Average"
12
+ config_param :dimensions_name, :string, :default => nil
13
+ config_param :dimensions_value, :string, :default => nil
14
+ config_param :period, :integer, :default => 60
15
+
16
+
17
+ def initialize
10
18
  super
11
- require 'rubygems'
12
- require 'AWS'
13
- require 'time'
19
+ require 'aws-sdk'
14
20
  end
15
21
 
16
22
  def configure(conf)
17
23
  super
18
- @mon = AWS::Cloudwatch::Base.new(
19
- :access_key_id => access_key,
20
- :secret_access_key => secret_key,
21
- :server => endpoint)
22
-
23
- @metrics = conf.elements.select {|e|
24
- e.name == 'metric'
25
- }.map {|e|
26
- {:namespace => e['namespace'],
27
- :statistics => e['statistics'],
28
- :dimensions => e['dimensions'],
29
- :measure_name => e['metric_name']}
30
- }
24
+ @cw = AWS::CloudWatch.new(
25
+ :access_key_id => @aws_key_id,
26
+ :secret_access_key => @aws_sec_key,
27
+ :cloud_watch_endpoint => @cw_endpoint
28
+ ).client
31
29
  end
32
30
 
33
31
  def start
@@ -44,34 +42,37 @@ class Fluent::CloudwatchInput < Fluent::Input
44
42
  private
45
43
  def watch
46
44
  while true
47
- Fluent::Engine.emit(tag, Fluent::Engine.now, output)
48
- sleep 60 * 5
45
+ sleep 60
46
+ output
49
47
  end
50
48
  end
51
49
 
52
50
  def output
53
- end_time = Time.now
54
- start_time = end_time - 60 * 60
55
- @metrics.map {|m|
56
- m[:start_time] = start_time
57
- m[:end_time] = end_time
58
- format m, @mon.get_metric_statistics(m)
59
- }
60
- end
61
-
62
- def format(m, s)
63
- d = s["GetMetricStatisticsResult"]["Datapoints"]["member"].sort {|a,b|
64
- time(a) <=> time(b)
65
- }.last
51
+ @metric_name.split(",").each {|m|
52
+ statistics = @cw.get_metric_statistics({
53
+ :namespace => @namespace,
54
+ :metric_name => m,
55
+ :statistics => [@statistics],
56
+ :dimensions => [{
57
+ :name => @dimensions_name,
58
+ :value => @dimensions_value
59
+ }],
60
+ :start_time => (Time.now - 120).iso8601,
61
+ :end_time => Time.now.iso8601,
62
+ :period => @period
63
+ })
64
+ unless statistics[:datapoints].empty?
65
+ stat = @statistics.downcase.to_sym
66
+ data = statistics[:datapoints][0][stat]
66
67
 
67
- d["MetricName"] = m[:measure_name]
68
- d["Statistics"] = m[:statistics]
69
- d["Value"] = d.delete(m[:statistics])
70
- d
71
- end
68
+ # unix time
69
+ catch_time = statistics[:datapoints][0][:timestamp].to_i
72
70
 
73
- def time(d)
74
- Time.parse d["Timestamp"]
71
+ # no output_data.to_json
72
+ output_data = {m => data}
73
+ Fluent::Engine.emit(tag, catch_time, output_data)
74
+ end
75
+ }
76
+ sleep 1
75
77
  end
76
78
  end
77
-
data/test/plugin/test_in_cloudwatch.rb CHANGED
@@ -5,21 +5,61 @@ class CloudwatchInputTest < Test::Unit::TestCase
5
5
  Fluent::Test.setup
6
6
  end
7
7
 
8
- CONFIG = %[
8
+ ### for RDS
9
+ CONFIG_RDS = %[
10
+ tag cloudwatch
11
+ aws_key_id test_key_id
12
+ aws_sec_key test_sec_key
13
+ cw_endpoint test_cloud_watch_endpoint
14
+ namespace AWS/RDS
15
+ metric_name CPUUtilization,FreeStorageSpace,DiskQueueDepth,FreeableMemory,SwapUsage,ReadIOPS,ReadLatency,ReadThroughput,WriteIOPS,WriteLatency,WriteThroughput
16
+ dimensions_name DBInstanceIdentifier
17
+ dimensions_value rds01
9
18
  ]
10
19
 
11
- def create_driver(conf = CONFIG, tag = 'test')
20
+ def create_driver_rds(conf = CONFIG_RDS)
12
21
  Fluent::Test::InputTestDriver.new(Fluent::CloudwatchInput).configure(conf)
13
22
  end
14
23
 
15
- def test_configure
24
+ def test_configure_rds
25
+ d = create_driver_rds
26
+ assert_equal 'cloudwatch', d.instance.tag
27
+ assert_equal 'test_key_id', d.instance.aws_key_id
28
+ assert_equal 'test_sec_key', d.instance.aws_sec_key
29
+ assert_equal 'test_cloud_watch_endpoint', d.instance.cw_endpoint
30
+ assert_equal 'AWS/RDS', d.instance.namespace
31
+ assert_equal 'CPUUtilization,FreeStorageSpace,DiskQueueDepth,FreeableMemory,SwapUsage,ReadIOPS,ReadLatency,ReadThroughput,WriteIOPS,WriteLatency,WriteThroughput', d.instance.metric_name
32
+ assert_equal 'DBInstanceIdentifier', d.instance.dimensions_name
33
+ assert_equal 'rds01', d.instance.dimensions_value
16
34
  end
17
35
 
18
- def test_format
19
- d = create_driver
36
+
37
+ ### for EC2
38
+ CONFIG_ECTWO = %[
39
+ tag cloudwatch
40
+ aws_key_id test_key_id
41
+ aws_sec_key test_sec_key
42
+ cw_endpoint test_cloud_watch_endpoint
43
+ namespace AWS/EC2
44
+ metric_name CPUUtilization,FreeStorageSpace,DiskQueueDepth,FreeableMemory,SwapUsage,ReadIOPS,ReadLatency,ReadThroughput,WriteIOPS,WriteLatency,WriteThroughput
45
+ dimensions_name InstanceId
46
+ dimensions_value ec2-01
47
+ ]
48
+
49
+ def create_driver_ectwo(conf = CONFIG_ECTWO)
50
+ Fluent::Test::InputTestDriver.new(Fluent::CloudwatchInput).configure(conf)
20
51
  end
21
52
 
22
- def test_write
23
- d = create_driver
53
+ def test_configure_ectwo
54
+ d = create_driver_ectwo
55
+ assert_equal 'cloudwatch', d.instance.tag
56
+ assert_equal 'test_key_id', d.instance.aws_key_id
57
+ assert_equal 'test_sec_key', d.instance.aws_sec_key
58
+ assert_equal 'test_cloud_watch_endpoint', d.instance.cw_endpoint
59
+ assert_equal 'AWS/EC2', d.instance.namespace
60
+ assert_equal 'CPUUtilization,FreeStorageSpace,DiskQueueDepth,FreeableMemory,SwapUsage,ReadIOPS,ReadLatency,ReadThroughput,WriteIOPS,WriteLatency,WriteThroughput', d.instance.metric_name
61
+ assert_equal 'InstanceId', d.instance.dimensions_name
62
+ assert_equal 'ec2-01', d.instance.dimensions_value
24
63
  end
64
+
25
65
  end
metadata CHANGED
@@ -1,51 +1,53 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-cloudwatch
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.0
4
+ version: 1.1.0
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
8
8
  - Yusuke Nomura
9
+ - kenjiskywalker
9
10
  autorequire:
10
11
  bindir: bin
11
12
  cert_chain: []
12
- date: 2013-01-14 00:00:00.000000000 Z
13
+ date: 2013-04-07 00:00:00.000000000 Z
13
14
  dependencies:
14
15
  - !ruby/object:Gem::Dependency
15
16
  name: fluentd
16
17
  requirement: !ruby/object:Gem::Requirement
17
18
  none: false
18
19
  requirements:
19
- - - ! '>='
20
+ - - ~>
20
21
  - !ruby/object:Gem::Version
21
- version: '0'
22
- type: :development
22
+ version: 0.10.30
23
+ type: :runtime
23
24
  prerelease: false
24
25
  version_requirements: !ruby/object:Gem::Requirement
25
26
  none: false
26
27
  requirements:
27
- - - ! '>='
28
+ - - ~>
28
29
  - !ruby/object:Gem::Version
29
- version: '0'
30
+ version: 0.10.30
30
31
  - !ruby/object:Gem::Dependency
31
- name: fluentd
32
+ name: aws-sdk
32
33
  requirement: !ruby/object:Gem::Requirement
33
34
  none: false
34
35
  requirements:
35
- - - ! '>='
36
+ - - '='
36
37
  - !ruby/object:Gem::Version
37
- version: '0'
38
+ version: 1.8.3
38
39
  type: :runtime
39
40
  prerelease: false
40
41
  version_requirements: !ruby/object:Gem::Requirement
41
42
  none: false
42
43
  requirements:
43
- - - ! '>='
44
+ - - '='
44
45
  - !ruby/object:Gem::Version
45
- version: '0'
46
+ version: 1.8.3
46
47
  description: Input plugin for AWS CloudWatch.
47
48
  email:
48
49
  - yunomu@gmail.com
50
+ - git@kenjiskywalker.org
49
51
  executables: []
50
52
  extensions: []
51
53
  extra_rdoc_files: []
@@ -53,7 +55,7 @@ files:
53
55
  - .gitignore
54
56
  - Gemfile
55
57
  - LICENSE
56
- - README.rdoc
58
+ - README.md
57
59
  - Rakefile
58
60
  - fluent-plugin-cloudwatch.gemspec
59
61
  - lib/fluent/plugin/in_cloudwatch.rb
@@ -82,7 +84,7 @@ rubyforge_project:
82
84
  rubygems_version: 1.8.23
83
85
  signing_key:
84
86
  specification_version: 3
85
- summary: Input plugin for AWS CloudWatch
87
+ summary: Input plugin for AWS CloudWatch.
86
88
  test_files:
87
89
  - test/helper.rb
88
90
  - test/plugin/test_in_cloudwatch.rb
data/README.rdoc DELETED
@@ -1,49 +0,0 @@
1
- = Fluent::Plugin::Cloudwatch
2
-
3
- Plugin for AWS CloudWatch.
4
-
5
- == Installation
6
-
7
- Add this line to your application's Gemfile:
8
-
9
- gem 'fluent-plugin-cloudwatch'
10
-
11
- And then execute:
12
-
13
- $ bundle
14
-
15
- Or install it yourself as:
16
-
17
- $ gem install fluent-plugin-cloudwatch
18
-
19
- == Configuration
20
-
21
- <source>
22
- type cloudwatch
23
- tag cloudwatch
24
- access_key XXXXXXXXXX
25
- secret_key XXXXXXXXXX
26
- endpoint monitoring.ap-northeast-1.amazonaws.com
27
-
28
- <metric>
29
- namespace AWS/EC2
30
- statistics Average
31
- dimensions InstanceId=i-xxxxxxxx
32
- metric_name CPUUtilization
33
- </metric>
34
- <metric>
35
- namespace AWS/EC2
36
- statistics Sum
37
- dimensions InstanceId=i-xxxxxxxx
38
- metric_name NetworkIn
39
- </metric>
40
- </source>
41
-
42
-
43
- == Contributing
44
-
45
- 1. Fork it
46
- 2. Create your feature branch (`git checkout -b my-new-feature`)
47
- 3. Commit your changes (`git commit -am 'Added some feature'`)
48
- 4. Push to the branch (`git push origin my-new-feature`)
49
- 5. Create new Pull Request