fluent-plugin-cloudwatch-logs 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4515d3ff1d51227199dd771c142b25f543c04ef0
-  data.tar.gz: 784e81df02faae8bad44ddafca8397464c95b967
+  metadata.gz: c919aefdaa503c87f9e0e031c4bb26154da3f5ce
+  data.tar.gz: f6be0442947f82c255e17a2d7bb7a4e63e76ebc7
 SHA512:
-  metadata.gz: 189047028dc9a27bba67e2aa6603064a0fd0be5f25aece8e347e8838a905460ca0ae4428f1c640c382cae42801d70defa7f54112e06f8757744c60a28bc8e0cd
-  data.tar.gz: ea197c9342d43a29c218baeb772dd9da217726cf90f35eed7d8c03e37878be5c50905369cb24c4c5a59904d62614c0fd2629e0c06dd239645d70f5de09377671
+  metadata.gz: c77b226dcd3ed30b749edda0df7579cf1bacd68c51d53db5c6811cbd4193b459ffed3e716b992fd9e3808f5a154f99eddf7809927dee708ec81019013d928c18
+  data.tar.gz: 1ffe8e5f35cfac5e4ed7484ad07270933d259910a4714f5c52a9b436eb18b5f340edcab0b875ef2e33eb79d14cbb9d54ec75a732cd90ad3a212a27b306402e52

data/README.md

@@ -66,10 +66,10 @@ Fetch sample log from CloudWatch Logs:
   type cloudwatch_logs
   log_group_name log-group-name
   log_stream_name log-stream-name
-  sequence_token_file /var/lib/fluent/group_stream.out.seq
   auto_create_stream true
   #message_keys key1,key2,key3,...
   #max_message_length 32768
+  #use_tag_as_group false
 </match>
 ```
 
@@ -79,6 +79,7 @@ Fetch sample log from CloudWatch Logs:
 * `auto_create_stream`: to create log group and stream automatically
 * `message_keys`: keys to send messages as events
 * `max_message_length`: maximum length of the message
+* `use_tag_as_group`: to use tag as a group name
 
 ### in_cloudwatch_logs
 

data/lib/fluent/plugin/cloudwatch/logs/version.rb

@@ -2,7 +2,7 @@ module Fluent
   module Plugin
     module Cloudwatch
       module Logs
-        VERSION = "0.0.4"
+        VERSION = "0.0.5"
       end
     end
   end

data/lib/fluent/plugin/out_cloudwatch_logs.rb

@@ -5,12 +5,12 @@ module Fluent
     config_param :aws_key_id, :string, :default => nil
     config_param :aws_sec_key, :string, :default => nil
     config_param :region, :string, :default => nil
-    config_param :log_group_name, :string
+    config_param :log_group_name, :string, :default => nil
     config_param :log_stream_name, :string
-    config_param :sequence_token_file, :string
     config_param :auto_create_stream, :bool, default: false
     config_param :message_keys, :string, :default => nil
     config_param :max_message_length, :integer, :default => nil
+    config_param :use_tag_as_group, :bool, :default => false
 
     unless method_defined?(:log)
       define_method(:log) { $log }
@@ -29,8 +29,7 @@ module Fluent
       options[:credentials] = Aws::Credentials.new(@aws_key_id, @aws_sec_key) if @aws_key_id && @aws_sec_key
       options[:region] = @region if @region
       @logs = Aws::CloudWatchLogs::Client.new(options)
-
-      create_stream if @auto_create_stream
+      @sequence_tokens = {}
     end
 
     def format(tag, time, record)
@@ -39,62 +38,110 @@ module Fluent
 
     def write(chunk)
       events = []
-      chunk.msgpack_each do |tag, time, record|
-        time_ms = time * 1000
-
-        if @message_keys
-          message = @message_keys.split(',').map {|k| record[k].to_s }.join(' ')
-        else
-          message = record.to_json
+      chunk.enum_for(:msgpack_each).chunk {|tag, time, record|
+        tag
+      }.each {|tag, rs|
+        group_name = @use_tag_as_group ? tag : @log_group_name
+
+        unless log_group_exists?(group_name)
+          if @auto_create_stream
+            create_log_group(group_name)
+          else
+            log.warn "Log group '#{group_name}' dose not exists"
+            next
+          end
         end
 
-        if @max_message_length
-          message.force_encoding('ASCII-8BIT')
-          message = message.slice(0, @max_message_length)
+        unless log_stream_exists?(group_name, @log_stream_name)
+          if @auto_create_stream
+            create_log_stream(group_name, @log_stream_name)
+          else
+            log.warn "Log stream '#{@log_stream_name}' dose not exists"
+            next
+          end
         end
 
-        events << {timestamp: time_ms, message: message}
-      end
-      put_events(events)
+        rs.each do |t, time, record|
+          time_ms = time * 1000
+
+          if @message_keys
+            message = @message_keys.split(',').map {|k| record[k].to_s }.join(' ')
+          else
+            message = record.to_json
+          end
+
+          if @max_message_length
+            message.force_encoding('ASCII-8BIT')
+            message = message.slice(0, @max_message_length)
+          end
+
+          events << {timestamp: time_ms, message: message}
+        end
+        put_events(group_name, events)
+      }
     end
 
     private
-    def next_sequence_token
-      return nil unless File.exist?(@sequence_token_file)
-      open(@sequence_token_file) {|f| f.read }.chomp
+    def next_sequence_token(group_name, stream_name)
+      @sequence_tokens[group_name][stream_name]
     end
 
-    def store_next_sequence_token(token)
-      open(@sequence_token_file, 'w') do |f|
-        f.write token
-      end
+    def store_next_sequence_token(group_name, stream_name, token)
+      @sequence_tokens[group_name][stream_name] = token
     end
 
-    def put_events(events)
+    def put_events(group_name, events)
       args = {
         log_events: events,
-        log_group_name: @log_group_name,
+        log_group_name: group_name,
         log_stream_name: @log_stream_name,
       }
-      args[:sequence_token] = next_sequence_token if next_sequence_token
+      token = next_sequence_token(group_name, @log_stream_name)
+      args[:sequence_token] = token if token
 
       response = @logs.put_log_events(args)
-      store_next_sequence_token(response.next_sequence_token)
+      store_next_sequence_token(group_name, @log_stream_name, response.next_sequence_token)
     end
 
-    def create_stream
-      log.debug "Creating log stream '#{@log_stream_name}' in log group '#{@log_group_name}'"
-
+    def create_log_group(group_name)
       begin
-        @logs.create_log_group(log_group_name: @log_group_name)
+        @logs.create_log_group(log_group_name: group_name)
+        @sequence_tokens[group_name] = {}
       rescue Aws::CloudWatchLogs::Errors::ResourceAlreadyExistsException
-        log.debug "Log group '#{@log_group_name}' already exists"
+        log.debug "Log group '#{group_name}' already exists"
       end
+    end
 
+    def create_log_stream(group_name, stream_name)
       begin
-        @logs.create_log_stream(log_group_name: @log_group_name, log_stream_name: @log_stream_name)
+        @logs.create_log_stream(log_group_name: group_name, log_stream_name: stream_name)
+        @sequence_tokens[group_name][stream_name] = nil
       rescue Aws::CloudWatchLogs::Errors::ResourceAlreadyExistsException
-        log.debug "Log stream '#{@log_stream_name}' already exists"
+        log.debug "Log stream '#{stream_name}' already exists"
+      end
+    end
+
+    def log_group_exists?(group_name)
+      if @sequence_tokens[group_name]
+        true
+      elsif @logs.describe_log_groups.log_groups.any? {|i| i.log_group_name == group_name }
+        @sequence_tokens[group_name] = {}
+        true
+      else
+        false
+      end
+    end
+
+    def log_stream_exists?(group_name, stream_name)
+      if not @sequence_tokens[group_name]
+        false
+      elsif @sequence_tokens[group_name].has_key?(stream_name)
+        true
+      elsif (log_stream = @logs.describe_log_streams(log_group_name: group_name).log_streams.find {|i| i.log_stream_name  == stream_name })
+        @sequence_tokens[group_name][stream_name] = log_stream.upload_sequence_token
+        true
+      else
+        false
       end
     end
   end

data/test/plugin/test_out_cloudwatch_logs.rb

@@ -24,7 +24,6 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
       region us-east-1
       log_group_name test_group
       log_stream_name test_stream
-      sequence_token_file /tmp/sq
       auto_create_stream false
     EOC
 
@@ -33,7 +32,6 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
     assert_equal('us-east-1', d.instance.region)
     assert_equal('test_group', d.instance.log_group_name)
     assert_equal('test_stream', d.instance.log_stream_name)
-    assert_equal('/tmp/sq', d.instance.sequence_token_file)
     assert_equal(false, d.instance.auto_create_stream)
   end
 
@@ -103,6 +101,30 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
     assert_equal('message2 l', events[1].message)
   end
 
+  def test_write_use_tag_as_group
+    new_log_stream
+
+    d = create_driver(<<-EOC)
+    #{default_config}
+    message_keys message,cloudwatch
+    use_tag_as_group true
+    EOC
+
+    time = Time.now
+    d.emit({'cloudwatch' => 'logs1', 'message' => 'message1'}, time.to_i)
+    d.emit({'cloudwatch' => 'logs2', 'message' => 'message2'}, time.to_i + 1)
+    d.run
+
+    sleep 20
+
+    events = get_log_events(fluentd_tag)
+    assert_equal(2, events.size)
+    assert_equal(time.to_i * 1000, events[0].timestamp)
+    assert_equal('message1 logs1', events[0].message)
+    assert_equal((time.to_i + 1) * 1000, events[1].timestamp)
+    assert_equal('message2 logs2', events[1].message)
+  end
+
   private
   def default_config
     <<-EOC
@@ -123,6 +145,6 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
 
 
   def create_driver(conf = default_config)
-    Fluent::Test::BufferedOutputTestDriver.new(Fluent::CloudwatchLogsOutput).configure(conf)
+    Fluent::Test::BufferedOutputTestDriver.new(Fluent::CloudwatchLogsOutput, fluentd_tag).configure(conf)
   end
 end

data/test/test_helper.rb

@@ -40,9 +40,17 @@ module CloudwatchLogsTestHelper
   end
 
   def clear_log_group
-    logs.delete_log_group(log_group_name: log_group_name)
-  rescue Aws::CloudWatchLogs::Errors::ResourceNotFoundException
-    # pass
+    [log_group_name, fluentd_tag].each do |name|
+      begin
+        logs.delete_log_group(log_group_name: name)
+      rescue Aws::CloudWatchLogs::Errors::ResourceNotFoundException
+        # pass
+      end
+    end
+  end
+
+  def fluentd_tag
+    @fluentd_tag ||= "fluent.plugin.cloudwatch.test.#{Time.now.to_f}"
   end
 
   def create_log_stream
@@ -59,8 +67,8 @@ module CloudwatchLogsTestHelper
     end
   end
 
-  def get_log_events
-    logs.get_log_events(log_group_name: log_group_name, log_stream_name: log_stream_name).events
+  def get_log_events(tag = nil)
+    logs.get_log_events(log_group_name: tag || log_group_name, log_stream_name: log_stream_name).events
   end
 
   def put_log_events(events)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cloudwatch-logs
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - Ryota Arai
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-19 00:00:00.000000000 Z
+date: 2014-08-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd