fluent-plugin-cloudwatch-logs 0.0.7 → 0.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 26d61c290a6fd56f27524732758ea6e4f8cacd96
-  data.tar.gz: 678b4500dc46217e141328d3cd915470a691a2ad
+  metadata.gz: 92dbd2c06330d3511fdc20c7ed1668c8b3984522
+  data.tar.gz: 7fb872f59507e8513201aba54f90089a22023fed
 SHA512:
-  metadata.gz: 7b6d98c07e99f57022f041d1d4c60738bf847aea9512f8bed75c47de7f25fd5dca7ca3b867289784eb08901e6312f98fe3b929edb7022ed9b2032ca7b52796a9
-  data.tar.gz: 8e86a20a0f129546c5300af5811d8f0d0d7d50907bc5622fe44fa03e8115669d8eb5411339533be5f39f0eb4b9733df4a8ced4235e6ca7ffe29c22f6f544bc90
+  metadata.gz: 0ff3b9b908e63926e095391b91d60df659d37dbeead2725f3e726bc9a4482839cbcda17f41d0377aea14fefd8a6af9fc66770f5a3df4d1aa48ce7debaaed8b25
+  data.tar.gz: 49e564f50e9fc4d3d19f11761d1c9f77ab1d8b780b5de006786da03d2c16a796c25e871960f47e9e0965d91eb935ec424a21620c40a3b4ad212aa26203d2af75

data/lib/fluent/plugin/cloudwatch/logs/version.rb

@@ -2,7 +2,7 @@ module Fluent
   module Plugin
     module Cloudwatch
       module Logs
-        VERSION = "0.0.7"
+        VERSION = "0.0.8"
       end
     end
   end

data/lib/fluent/plugin/in_cloudwatch_logs.rb

@@ -2,14 +2,15 @@ module Fluent
   class CloudwatchLogsInput < Input
     Plugin.register_input('cloudwatch_logs', self)
 
-    config_param :aws_key_id, :string, :default => nil
-    config_param :aws_sec_key, :string, :default => nil
+    config_param :aws_key_id, :string, :default => nil, :secret => true
+    config_param :aws_sec_key, :string, :default => nil, :secret => true
     config_param :region, :string, :default => nil
     config_param :tag, :string
     config_param :log_group_name, :string
     config_param :log_stream_name, :string
     config_param :state_file, :string
     config_param :fetch_interval, :time, default: 60
+    config_param :http_proxy, :string, default: nil
 
     def initialize
       super
@@ -17,10 +18,16 @@ module Fluent
       require 'aws-sdk-core'
     end
 
+    def configure(conf)
+      super
+      configure_parser(conf)
+    end
+
     def start
       options = {}
       options[:credentials] = Aws::Credentials.new(@aws_key_id, @aws_sec_key) if @aws_key_id && @aws_sec_key
       options[:region] = @region if @region
+      options[:http_proxy] = @http_proxy if @http_proxy
       @logs = Aws::CloudWatchLogs::Client.new(options)
 
       @finished = false
@@ -33,6 +40,13 @@ module Fluent
     end
 
     private
+    def configure_parser(conf)
+      if conf['format']
+        @parser = TextParser.new
+        @parser.configure(conf)
+      end
+    end
+
     def next_token
       return nil unless File.exist?(@state_file)
       File.read(@state_file).chomp
@@ -53,9 +67,14 @@ module Fluent
 
           events = get_events
           events.each do |event|
-            time = (event.timestamp / 1000).floor
-            record = JSON.parse(event.message)
-            Engine.emit(@tag, time, record)
+            if @parser
+              record = @parser.parse(event.message)
+              Engine.emit(@tag, record[0], record[1])
+            else
+              time = (event.timestamp / 1000).floor
+              record = JSON.parse(event.message)
+              Engine.emit(@tag, time, record)
+            end
           end
         end
         sleep 1

data/lib/fluent/plugin/out_cloudwatch_logs.rb

@@ -2,8 +2,8 @@ module Fluent
   class CloudwatchLogsOutput < BufferedOutput
     Plugin.register_output('cloudwatch_logs', self)
 
-    config_param :aws_key_id, :string, :default => nil
-    config_param :aws_sec_key, :string, :default => nil
+    config_param :aws_key_id, :string, :default => nil, :secret => true
+    config_param :aws_sec_key, :string, :default => nil, :secret => true
     config_param :region, :string, :default => nil
     config_param :log_group_name, :string, :default => nil
     config_param :log_stream_name, :string
@@ -11,8 +11,9 @@ module Fluent
     config_param :message_keys, :string, :default => nil
     config_param :max_message_length, :integer, :default => nil
     config_param :use_tag_as_group, :bool, :default => false
+    config_param :http_proxy, :string, default: nil
 
-    MAX_EVENTS_SIZE = 30720
+    MAX_EVENTS_SIZE = 1_048_576
 
     unless method_defined?(:log)
       define_method(:log) { $log }
@@ -30,6 +31,7 @@ module Fluent
       options = {}
       options[:credentials] = Aws::Credentials.new(@aws_key_id, @aws_sec_key) if @aws_key_id && @aws_sec_key
       options[:region] = @region if @region
+      options[:http_proxy] = @http_proxy if @http_proxy
       @logs = Aws::CloudWatchLogs::Client.new(options)
       @sequence_tokens = {}
     end
@@ -99,10 +101,10 @@ module Fluent
     def put_events_by_chunk(group_name, events)
       chunk = []
 
-      # The maximum batch size is 32,768 bytes, and this size is calculated as the sum of all event messages in UTF-8, plus 26 bytes for each log event.
+      # The maximum batch size is 1,048,576 bytes, and this size is calculated as the sum of all event messages in UTF-8, plus 26 bytes for each log event.
       # http://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_PutLogEvents.html
       while event = events.shift
-        if (chunk + [event]).inject(0) {|sum, e| sum + e[:message].length } > MAX_EVENTS_SIZE
+        if (chunk + [event]).inject(0) {|sum, e| sum + e[:message].length + 26 } > MAX_EVENTS_SIZE
           put_events(group_name, chunk)
           chunk = [event]
         else
@@ -162,12 +164,27 @@ module Fluent
         false
       elsif @sequence_tokens[group_name].has_key?(stream_name)
         true
-      elsif (log_stream = @logs.describe_log_streams(log_group_name: group_name).log_streams.find {|i| i.log_stream_name  == stream_name })
+      elsif (log_stream = find_log_stream(group_name, stream_name))
         @sequence_tokens[group_name][stream_name] = log_stream.upload_sequence_token
         true
       else
         false
       end
     end
+
+    def find_log_stream(group_name, stream_name)
+      next_token = nil
+      loop do
+        response = @logs.describe_log_streams(log_group_name: group_name, next_token: next_token)
+        if (log_stream = response.log_streams.find {|i| i.log_stream_name == stream_name })
+          return log_stream
+        end
+        if response.next_token.nil?
+          break
+        end
+        next_token = response.next_token
+      end
+      nil
+    end
   end
 end

data/test/plugin/test_in_cloudwatch_logs.rb

@@ -56,6 +56,43 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
     assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs2'}], emits[1])
   end
 
+  def test_emit_width_format
+    create_log_stream
+
+    time_ms = (Time.now.to_f * 1000).floor
+    put_log_events([
+      {timestamp: time_ms, message: 'logs1'},
+      {timestamp: time_ms, message: 'logs2'},
+    ])
+
+    sleep 5
+
+    d = create_driver(<<-EOC)
+      tag test
+      type cloudwatch_logs
+      log_group_name #{log_group_name}
+      log_stream_name #{log_stream_name}
+      state_file /tmp/state
+      format /^(?<cloudwatch>[^ ]*)?/
+      #{aws_key_id}
+      #{aws_sec_key}
+      #{region}
+    EOC
+
+    d.run do
+      sleep 5
+    end
+
+    emits = d.emits
+    assert_equal(2, emits.size)
+    assert_equal('test', emits[0][0])
+    assert_in_delta((time_ms / 1000).floor, emits[0][1], 10)
+    assert_equal({'cloudwatch' => 'logs1'}, emits[0][2])
+    assert_equal('test', emits[1][0])
+    assert_in_delta((time_ms / 1000).floor, emits[1][1], 10)
+    assert_equal({'cloudwatch' => 'logs2'}, emits[1][2])
+  end
+
   private
   def default_config
     <<-EOC

data/test/test_helper.rb

@@ -9,6 +9,7 @@ module CloudwatchLogsTestHelper
     options = {}
     options[:credentials] = Aws::Credentials.new(ENV['aws_key_id'], ENV['aws_sec_key']) if ENV['aws_key_id'] && ENV['aws_sec_key']
     options[:region] = ENV['region'] if ENV['region']
+    options[:http_proxy] = ENV['http_proxy'] if ENV['http_proxy']
     @logs ||= Aws::CloudWatchLogs.new(options)
   end
 

metadata

@@ -1,69 +1,69 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cloudwatch-logs
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.0.8
 platform: ruby
 authors:
 - Ryota Arai
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-11-10 00:00:00.000000000 Z
+date: 2015-09-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: aws-sdk-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 2.0.7
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 2.0.7
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.6'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.6'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: 
@@ -73,7 +73,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
+- ".gitignore"
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -97,17 +97,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.14
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: CloudWatch Logs Plugin for Fluentd