fluent-plugin-cloudwatch-logs 0.1.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: d63e33f244adf82bff43836bd21d935eebb5fee8
4
- data.tar.gz: 6e14fdd4432c7b21c0b0cf6b9c1f6574565a34b9
3
+ metadata.gz: 88a161bb1764e810bbf9a56b73fc7e47b05ebeb3
4
+ data.tar.gz: 04838391a2d452903c11bf0f5878f06c554b92f2
5
5
  SHA512:
6
- metadata.gz: 87332e167639ac1d7a43404849f05bb1820faac2190e2e30ce73672159b2e52f91aa68c7bb3433f815d3969c87fcd0d46c9408a84612bed7243c66afc0e53d60
7
- data.tar.gz: 52698378e0c3d7a935aec5117eceba6ae3799dedc09e125bc99dd9f2f0608448cc640c08138a3a4d6703884371e868e1252af97773bfbb30e3c117baa02dc429
6
+ metadata.gz: 00153e08af56da4d830290799b162b3935c9f88bc415467f558e96c191dd6bdeebcd4497f1c51975deb6a597841d35c04dba1ec1f2b19cdf851772357c8d49a7
7
+ data.tar.gz: 2ae3cf9e79a22595d80f99c043b3808a640f39fe7c11ae299b4c92641fd9fa2bb2dfd6cc45b4dc7230136ae56af1e845b420f95d0c0c2fd93c86b28e3fe9abf6
data/README.md CHANGED
@@ -72,6 +72,7 @@ Fetch sample log from CloudWatch Logs:
72
72
  #message_keys key1,key2,key3,...
73
73
  #max_message_length 32768
74
74
  #use_tag_as_group false
75
+ #use_tag_as_stream false
75
76
  </match>
76
77
  ```
77
78
 
@@ -81,7 +82,9 @@ Fetch sample log from CloudWatch Logs:
81
82
  * `auto_create_stream`: to create log group and stream automatically
82
83
  * `message_keys`: keys to send messages as events
83
84
  * `max_message_length`: maximum length of the message
85
+ * `max_events_per_batch`: maximum number of events to send at once (default 10000)
84
86
  * `use_tag_as_group`: to use tag as a group name
87
+ * `use_tag_as_stream`: to use tag as a stream name
85
88
 
86
89
  ### in_cloudwatch_logs
87
90
 
@@ -2,7 +2,7 @@ module Fluent
2
2
  module Plugin
3
3
  module Cloudwatch
4
4
  module Logs
5
- VERSION = "0.1.0"
5
+ VERSION = "0.1.1"
6
6
  end
7
7
  end
8
8
  end
@@ -10,6 +10,7 @@ module Fluent
10
10
  config_param :auto_create_stream, :bool, default: false
11
11
  config_param :message_keys, :string, :default => nil
12
12
  config_param :max_message_length, :integer, :default => nil
13
+ config_param :max_events_per_batch, :integer, :default => 10000
13
14
  config_param :use_tag_as_group, :bool, :default => false
14
15
  config_param :use_tag_as_stream, :bool, :default => false
15
16
  config_param :http_proxy, :string, default: nil
@@ -106,7 +107,11 @@ module Fluent
106
107
  # The maximum batch size is 1,048,576 bytes, and this size is calculated as the sum of all event messages in UTF-8, plus 26 bytes for each log event.
107
108
  # http://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_PutLogEvents.html
108
109
  while event = events.shift
109
- if (chunk + [event]).inject(0) {|sum, e| sum + e[:message].length + 26 } > MAX_EVENTS_SIZE
110
+ new_chunk = chunk + [event]
111
+ chunk_span_too_big = new_chunk.size > 1 && new_chunk[-1][:timestamp] - new_chunk[0][:timestamp] >= 1000 * 60 * 60 * 24
112
+ chunk_too_big = new_chunk.inject(0) {|sum, e| sum + e[:message].length + 26 } > MAX_EVENTS_SIZE
113
+ chunk_too_long = @max_events_per_batch && chunk.size >= @max_events_per_batch
114
+ if chunk_too_big or chunk_span_too_big or chunk_too_long
110
115
  put_events(group_name, stream_name, chunk)
111
116
  chunk = [event]
112
117
  else
@@ -54,6 +54,28 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
54
54
  assert_equal('{"cloudwatch":"logs2"}', events[1].message)
55
55
  end
56
56
 
57
+ def test_write_24h_apart
58
+ new_log_stream
59
+
60
+ d = create_driver
61
+ time = Time.now
62
+ d.emit({'cloudwatch' => 'logs0'}, time.to_i - 60 * 60 * 25)
63
+ d.emit({'cloudwatch' => 'logs1'}, time.to_i)
64
+ d.emit({'cloudwatch' => 'logs2'}, time.to_i + 1)
65
+ d.run
66
+
67
+ sleep 20
68
+
69
+ events = get_log_events
70
+ assert_equal(3, events.size)
71
+ assert_equal((time.to_i - 60 * 60 * 25) * 1000, events[0].timestamp)
72
+ assert_equal('{"cloudwatch":"logs0"}', events[0].message)
73
+ assert_equal((time.to_i ) * 1000, events[1].timestamp)
74
+ assert_equal('{"cloudwatch":"logs1"}', events[1].message)
75
+ assert_equal((time.to_i + 1) * 1000, events[2].timestamp)
76
+ assert_equal('{"cloudwatch":"logs2"}', events[2].message)
77
+ end
78
+
57
79
  def test_write_with_message_keys
58
80
  new_log_stream
59
81
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-cloudwatch-logs
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0
4
+ version: 0.1.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ryota Arai
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-09-29 00:00:00.000000000 Z
11
+ date: 2015-10-06 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd