fluent-plugin-cloudwatch-logs 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 82f4aa50c7c50dd4fb39644824405fe597e44dbc
-  data.tar.gz: 7af64b0786713b5afaa6aa9e15c4a5c2b14cf41e
+  metadata.gz: 9ba82ca82f0ebbaf1df3fb81e42606433d99b520
+  data.tar.gz: e29e78f866b41a7623d27c3d1af5428fd23b00ce
 SHA512:
-  metadata.gz: 0e96bd6e04ce768e9b5ace8492ff5e77d8de77f8203416711a4acb7efdfe5ddea805503c875434884d6e8b6cbebab991b19b7f492c7c9fce2fdb53a3a7ac8370
-  data.tar.gz: c204752a267e1083c8ff1e5ff2237611fc44a2293d59247761242fc8c5d1ffaddc9da64773ecd2f00cbe21506604b5ec072d14b2d17aca733b5af69fba9126d9
+  metadata.gz: 12e1b6c8596c9da1fa8945ca8cee1e553b0b5019812c55d2c082c11df74332683f61f7d07d9d99e8a0a51448ae8c92070343d37740f80cb0f1c1279dd66b3dfc
+  data.tar.gz: db5edcaa122522537ad34ec72dbeb49e9f420dc6f29ef02b7f47a24180abde671f0643904710d930e1c415d3c991d2fbf3e12b867589343e186fb9e178700c92

data/README.md

@@ -1,11 +1,42 @@
 # fluent-plugin-cloudwatch-logs
 
-CloudWatch Logs Plugin for Fluentd
+[CloudWatch Logs](http://aws.amazon.com/blogs/aws/cloudwatch-log-service/) Plugin for Fluentd
 
 ## Installation
 
     $ gem install fluent-plugin-cloudwatch-logs
 
+## Preparation
+
+Create IAM user with a policy like the following:
+
+```json
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "logs:*",
+        "s3:GetObject"
+      ],
+      "Resource": [
+        "arn:aws:logs:us-east-1:*:*",
+        "arn:aws:s3:::*"
+      ]
+    }
+  ]
+}
+```
+
+Set region and credentials:
+
+```
+$ export AWS_REGION=us-east-1
+$ export AWS_ACCESS_KEY_ID="YOUR_ACCESS_KEY"
+$ export AWS_SECRET_ACCESS_KEY="YOUR_SECRET_ACCESS_KEY"
+```
+
 ## Example
 
 Start fluentd:
@@ -27,7 +58,8 @@ Fetch sample log from CloudWatch Logs:
 2014-07-17 00:28:02 +0900 test.cloudwatch_logs.in: {"hello":"world"}
 ```
 
-## out_cloudwatch_logs
+## Configuration
+### out_cloudwatch_logs
 
 ```
 <match tag>
@@ -39,7 +71,12 @@ Fetch sample log from CloudWatch Logs:
 </match>
 ```
 
-## in_cloudwatch_logs
+* `log_group_name`: name of log group to store logs
+* `log_stream_name`: name of log stream to store logs
+* `sequence_token_file`: file to store next sequence token
+* `auto_create_stream`: to create log group and stream automatically
+
+### in_cloudwatch_logs
 
 ```
 <source>
@@ -51,6 +88,11 @@ Fetch sample log from CloudWatch Logs:
 </source>
 ```
 
+* `tag`: fluentd tag
+* `log_group_name`: name of log group to fetch logs
+* `log_stream_name`: name of log stream to fetch logs
+* `state_file`: file to store current state (e.g. next\_forward\_token)
+
 ## Test
 
 Set credentials:
@@ -67,6 +109,12 @@ Run tests:
 $ rake test
 ```
 
+Or, If you do not want to use IAM roll or ENV(this is just like writing to configuration file) :
+
+```
+$ rake aws_key_id=YOUR_ACCESS_KEY aws_sec_key=YOUR_SECRET_KEY region=us-east-1 test
+```
+
 ## TODO
 
 * out_cloudwatch_logs

data/lib/fluent/plugin/cloudwatch/logs/version.rb

@@ -2,7 +2,7 @@ module Fluent
   module Plugin
     module Cloudwatch
       module Logs
-        VERSION = "0.0.1"
+        VERSION = "0.0.2"
       end
     end
   end

data/lib/fluent/plugin/in_cloudwatch_logs.rb

@@ -2,6 +2,9 @@ module Fluent
   class CloudwatchLogsInput < Input
     Plugin.register_input('cloudwatch_logs', self)
 
+    config_param :aws_key_id, :string, :default => nil
+    config_param :aws_sec_key, :string, :default => nil
+    config_param :region, :string, :default => nil
     config_param :tag, :string
     config_param :log_group_name, :string
     config_param :log_stream_name, :string
@@ -12,10 +15,14 @@ module Fluent
       super
 
       require 'aws-sdk-core'
-      @logs = Aws::CloudWatchLogs.new
     end
 
     def start
+      options = {}
+      options[:credentials] = Aws::Credentials.new(@aws_key_id, @aws_sec_key) if @aws_key_id && @aws_sec_key
+      options[:region] = @region if @region
+      @logs = Aws::CloudWatchLogs.new(options)
+
       @finished = false
       @thread = Thread.new(&method(:run))
     end

data/lib/fluent/plugin/out_cloudwatch_logs.rb

@@ -2,6 +2,9 @@ module Fluent
   class CloudwatchLogsOutput < BufferedOutput
     Plugin.register_output('cloudwatch_logs', self)
 
+    config_param :aws_key_id, :string, :default => nil
+    config_param :aws_sec_key, :string, :default => nil
+    config_param :region, :string, :default => nil
     config_param :log_group_name, :string
     config_param :log_stream_name, :string
     config_param :sequence_token_file, :string
@@ -15,12 +18,16 @@ module Fluent
       super
 
       require 'aws-sdk-core'
-      @logs = Aws::CloudWatchLogs.new
     end
 
-    def configure(conf)
+    def start
       super
 
+      options = {}
+      options[:credentials] = Aws::Credentials.new(@aws_key_id, @aws_sec_key) if @aws_key_id && @aws_sec_key
+      options[:region] = @region if @region
+      @logs = Aws::CloudWatchLogs.new(options)
+
       create_stream if @auto_create_stream
     end
 
@@ -78,4 +85,3 @@ module Fluent
     end
   end
 end
-

data/test/plugin/test_in_cloudwatch_logs.rb

@@ -16,12 +16,18 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
   def test_configure
     d = create_driver(<<-EOC)
       type cloudwatch_logs
+      aws_key_id test_id
+      aws_sec_key test_key
+      region us-east-1
       tag test
       log_group_name group
       log_stream_name stream
       state_file /tmp/state
     EOC
 
+    assert_equal('test_id', d.instance.aws_key_id)
+    assert_equal('test_key', d.instance.aws_sec_key)
+    assert_equal('us-east-1', d.instance.region)
     assert_equal('test', d.instance.tag)
     assert_equal('group', d.instance.log_group_name)
     assert_equal('stream', d.instance.log_stream_name)
@@ -43,7 +49,7 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
     d.run do
       sleep 5
     end
-    
+
     emits = d.emits
     assert_equal(2, emits.size)
     assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs1'}], emits[0])
@@ -58,6 +64,9 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
       log_group_name #{log_group_name}
       log_stream_name #{log_stream_name}
       state_file /tmp/state
+      #{aws_key_id}
+      #{aws_sec_key}
+      #{region}
     EOC
   end
 
@@ -65,4 +74,3 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
     Fluent::Test::InputTestDriver.new(Fluent::CloudwatchLogsInput).configure(conf)
   end
 end
-

data/test/plugin/test_out_cloudwatch_logs.rb

@@ -8,24 +8,29 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
     Fluent::Test.setup
     require 'fluent/plugin/out_cloudwatch_logs'
 
-    @logs = Aws::CloudWatchLogs.new
   end
 
   def teardown
     clear_log_group
     FileUtils.rm_f(sequence_token_file)
   end
-  
+
 
   def test_configure
     d = create_driver(<<-EOC)
       type cloudwatch_logs
+      aws_key_id test_id
+      aws_sec_key test_key
+      region us-east-1
       log_group_name test_group
       log_stream_name test_stream
       sequence_token_file /tmp/sq
       auto_create_stream false
     EOC
 
+    assert_equal('test_id', d.instance.aws_key_id)
+    assert_equal('test_key', d.instance.aws_sec_key)
+    assert_equal('us-east-1', d.instance.region)
     assert_equal('test_group', d.instance.log_group_name)
     assert_equal('test_stream', d.instance.log_stream_name)
     assert_equal('/tmp/sq', d.instance.sequence_token_file)
@@ -41,7 +46,7 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
     d.emit({'cloudwatch' => 'logs2'}, time.to_i + 1)
     d.run
 
-    sleep 5
+    sleep 20
 
     events = get_log_events
     assert_equal(2, events.size)
@@ -59,6 +64,9 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
     log_stream_name #{log_stream_name}
     sequence_token_file #{sequence_token_file}
     auto_create_stream true
+    #{aws_key_id}
+    #{aws_sec_key}
+    #{region}
     EOC
   end
 

data/test/test_helper.rb

@@ -6,13 +6,28 @@ require 'aws-sdk-core'
 module CloudwatchLogsTestHelper
   private
   def logs
-    @logs ||= Aws::CloudWatchLogs.new
+    options = {}
+    options[:credentials] = Aws::Credentials.new(ENV['aws_key_id'], ENV['aws_sec_key']) if ENV['aws_key_id'] && ENV['aws_sec_key']
+    options[:region] = ENV['region'] if ENV['region']
+    @logs ||= Aws::CloudWatchLogs.new(options)
   end
 
   def log_group_name
     @log_group_name ||= "fluent-plugin-cloudwatch-test-#{Time.now.to_f}"
   end
 
+  def aws_key_id
+    "aws_key_id #{ENV['aws_key_id']}" if ENV['aws_key_id']
+  end
+
+  def aws_sec_key
+    "aws_sec_key #{ENV['aws_sec_key']}" if ENV['aws_sec_key']
+  end
+
+  def region
+    "region #{ENV['region']}" if ENV['region']
+  end
+
   def log_stream_name
     if !@log_stream_name
       new_log_stream
@@ -57,4 +72,3 @@ module CloudwatchLogsTestHelper
     logs.put_log_events(args)
   end
 end
-

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cloudwatch-logs
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Ryota Arai
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-16 00:00:00.000000000 Z
+date: 2014-07-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd