fluent-plugin-cloudwatch-logs 0.0.9 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8baebdf33279c41e8ff155a945366d3f4b47c801
-  data.tar.gz: bf2f6284c6550db5599de2b27e03ad7031037d7a
+  metadata.gz: d63e33f244adf82bff43836bd21d935eebb5fee8
+  data.tar.gz: 6e14fdd4432c7b21c0b0cf6b9c1f6574565a34b9
 SHA512:
-  metadata.gz: 6528ea074580ac7f2964a4416c11ac17f7b8bdac8324830728478bc049074bed07b613615a68c027b48948265fcca96b4bc495c80b1aac58a5acdb91eed9a4f1
-  data.tar.gz: cc21ab19ad8ead46f28eecb9355bf26afa560e83a4899e1f154ee638e3f18a0bc2b431e12e30ca1cacb900d5dc7481bad41536ae94923841ee4b684fec5ae5c1
+  metadata.gz: 87332e167639ac1d7a43404849f05bb1820faac2190e2e30ce73672159b2e52f91aa68c7bb3433f815d3969c87fcd0d46c9408a84612bed7243c66afc0e53d60
+  data.tar.gz: 52698378e0c3d7a935aec5117eceba6ae3799dedc09e125bc99dd9f2f0608448cc640c08138a3a4d6703884371e868e1252af97773bfbb30e3c117baa02dc429

data/README.md

@@ -1,5 +1,7 @@
 # fluent-plugin-cloudwatch-logs
 
+[![Gem Version](https://badge.fury.io/rb/fluent-plugin-cloudwatch-logs.svg)](http://badge.fury.io/rb/fluent-plugin-cloudwatch-logs)
+
 [CloudWatch Logs](http://aws.amazon.com/blogs/aws/cloudwatch-log-service/) Plugin for Fluentd
 
 ## Installation

data/fluent-plugin-cloudwatch-logs.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency 'fluentd'
-  spec.add_dependency 'aws-sdk-core', '~> 2.0.7'
+  spec.add_dependency 'aws-sdk-core', '>= 2.0.7'
 
   spec.add_development_dependency "bundler", "~> 1.6"
   spec.add_development_dependency "rake"

data/lib/fluent/plugin/cloudwatch/logs/version.rb

@@ -2,7 +2,7 @@ module Fluent
   module Plugin
     module Cloudwatch
       module Logs
-        VERSION = "0.0.9"
+        VERSION = "0.1.0"
       end
     end
   end

data/lib/fluent/plugin/out_cloudwatch_logs.rb

@@ -6,11 +6,12 @@ module Fluent
     config_param :aws_sec_key, :string, :default => nil, :secret => true
     config_param :region, :string, :default => nil
     config_param :log_group_name, :string, :default => nil
-    config_param :log_stream_name, :string
+    config_param :log_stream_name, :string, :default => nil
     config_param :auto_create_stream, :bool, default: false
     config_param :message_keys, :string, :default => nil
     config_param :max_message_length, :integer, :default => nil
     config_param :use_tag_as_group, :bool, :default => false
+    config_param :use_tag_as_stream, :bool, :default => false
     config_param :http_proxy, :string, default: nil
 
     MAX_EVENTS_SIZE = 1_048_576
@@ -46,6 +47,7 @@ module Fluent
         tag
       }.each {|tag, rs|
         group_name = @use_tag_as_group ? tag : @log_group_name
+        stream_name = @use_tag_as_stream ? tag : @log_stream_name
 
         unless log_group_exists?(group_name)
           if @auto_create_stream
@@ -56,11 +58,11 @@ module Fluent
           end
         end
 
-        unless log_stream_exists?(group_name, @log_stream_name)
+        unless log_stream_exists?(group_name, stream_name)
           if @auto_create_stream
-            create_log_stream(group_name, @log_stream_name)
+            create_log_stream(group_name, stream_name)
           else
-            log.warn "Log stream '#{@log_stream_name}' dose not exists"
+            log.warn "Log stream '#{stream_name}' dose not exists"
             next
           end
         end
@@ -85,7 +87,7 @@ module Fluent
         # The log events in the batch must be in chronological ordered by their timestamp.
         # http://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_PutLogEvents.html
         events = events.sort_by {|e| e[:timestamp] }
-        put_events_by_chunk(group_name, events)
+        put_events_by_chunk(group_name, stream_name, events)
       }
     end
 
@@ -98,14 +100,14 @@ module Fluent
       @sequence_tokens[group_name][stream_name] = token
     end
 
-    def put_events_by_chunk(group_name, events)
+    def put_events_by_chunk(group_name, stream_name, events)
       chunk = []
 
       # The maximum batch size is 1,048,576 bytes, and this size is calculated as the sum of all event messages in UTF-8, plus 26 bytes for each log event.
       # http://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_PutLogEvents.html
       while event = events.shift
         if (chunk + [event]).inject(0) {|sum, e| sum + e[:message].length + 26 } > MAX_EVENTS_SIZE
-          put_events(group_name, chunk)
+          put_events(group_name, stream_name, chunk)
           chunk = [event]
         else
           chunk << event
@@ -113,21 +115,21 @@ module Fluent
       end
 
       unless chunk.empty?
-        put_events(group_name, chunk)
+        put_events(group_name, stream_name, chunk)
       end
     end
 
-    def put_events(group_name, events)
+    def put_events(group_name, stream_name, events)
       args = {
         log_events: events,
         log_group_name: group_name,
-        log_stream_name: @log_stream_name,
+        log_stream_name: stream_name,
       }
-      token = next_sequence_token(group_name, @log_stream_name)
+      token = next_sequence_token(group_name, stream_name)
       args[:sequence_token] = token if token
 
       response = @logs.put_log_events(args)
-      store_next_sequence_token(group_name, @log_stream_name, response.next_sequence_token)
+      store_next_sequence_token(group_name, stream_name, response.next_sequence_token)
     end
 
     def create_log_group(group_name)

data/test/plugin/test_out_cloudwatch_logs.rb

@@ -125,6 +125,30 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
     assert_equal('message2 logs2', events[1].message)
   end
 
+  def test_write_use_tag_as_stream
+    new_log_stream
+
+    d = create_driver(<<-EOC)
+    #{default_config}
+    message_keys message,cloudwatch
+    use_tag_as_stream true
+    EOC
+
+    time = Time.now
+    d.emit({'cloudwatch' => 'logs1', 'message' => 'message1'}, time.to_i)
+    d.emit({'cloudwatch' => 'logs2', 'message' => 'message2'}, time.to_i + 1)
+    d.run
+
+    sleep 20
+
+    events = get_log_events(log_group_name, fluentd_tag)
+    assert_equal(2, events.size)
+    assert_equal(time.to_i * 1000, events[0].timestamp)
+    assert_equal('message1 logs1', events[0].message)
+    assert_equal((time.to_i + 1) * 1000, events[1].timestamp)
+    assert_equal('message2 logs2', events[1].message)
+  end
+
   private
   def default_config
     <<-EOC

data/test/test_helper.rb

@@ -10,7 +10,7 @@ module CloudwatchLogsTestHelper
     options[:credentials] = Aws::Credentials.new(ENV['aws_key_id'], ENV['aws_sec_key']) if ENV['aws_key_id'] && ENV['aws_sec_key']
     options[:region] = ENV['region'] if ENV['region']
     options[:http_proxy] = ENV['http_proxy'] if ENV['http_proxy']
-    @logs ||= Aws::CloudWatchLogs.new(options)
+    @logs ||= Aws::CloudWatchLogs::Client.new(options)
   end
 
   def log_group_name
@@ -68,8 +68,8 @@ module CloudwatchLogsTestHelper
     end
   end
 
-  def get_log_events(tag = nil)
-    logs.get_log_events(log_group_name: tag || log_group_name, log_stream_name: log_stream_name).events
+  def get_log_events(group = log_group_name, stream = log_stream_name)
+    logs.get_log_events(log_group_name: group, log_stream_name: stream).events
   end
 
   def put_log_events(events)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cloudwatch-logs
 version: !ruby/object:Gem::Version
-  version: 0.0.9
+  version: 0.1.0
 platform: ruby
 authors:
 - Ryota Arai
@@ -28,14 +28,14 @@ dependencies:
   name: aws-sdk-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 2.0.7
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 2.0.7
 - !ruby/object:Gem::Dependency