fluent-plugin-cloudwatch-logs 0.6.0 → 0.6.1
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f3809f992ae8d9967a7f1edae7ac40614496d98e38baf2d6aa18bcfbae5c72ff
|
4
|
+
data.tar.gz: 4eb7d7741fb3ef6e1c3d698f305c36ac49ee94df4d9350f5794a2e887e29984c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8da9d33a1217cae533af682e950314d5a0bd5dbaac8019f0309ef541f26434027ef4b0133bbf2e3030a87c685ba00839c766a99336b0dd0c9f1304a65698a647
|
7
|
+
data.tar.gz: a2df61af2a98d7dab6d2fb4b5aecbb904fd582aa80e9ae3dd4b15c00f8d320dface8b3fbfb8ba3b1cdf47641a8d4edce8e2e03eaf2eafc16ba976cf68edf9514
|
data/README.md
CHANGED
@@ -132,6 +132,7 @@ Fetch sample log from CloudWatch Logs:
|
|
132
132
|
* `aws_sts_role_arn`: the role ARN to assume when using cross-account sts authentication
|
133
133
|
* `aws_sts_session_name`: the session name to use with sts authentication (default: `fluentd`)
|
134
134
|
* `json_handler`: name of the library to be used to handle JSON data. For now, supported libraries are `json` (default) and `yajl`.
|
135
|
+
* `use_todays_log_stream`: use todays and yesterdays date as log stream name prefix (formatted YYYY/MM/DD). (default: `false`)
|
135
136
|
|
136
137
|
This plugin uses [fluent-mixin-config-placeholders](https://github.com/tagomoris/fluent-mixin-config-placeholders) and you can use addtional variables such as %{hostname}, %{uuid}, etc. These variables are useful to put hostname in `log_stream_name`.
|
137
138
|
|
@@ -1,3 +1,4 @@
|
|
1
|
+
require 'date'
|
1
2
|
require 'fluent/plugin/input'
|
2
3
|
require 'fluent/plugin/parser'
|
3
4
|
require 'yajl'
|
@@ -17,12 +18,13 @@ module Fluent::Plugin
|
|
17
18
|
config_param :endpoint, :string, :default => nil
|
18
19
|
config_param :tag, :string
|
19
20
|
config_param :log_group_name, :string
|
20
|
-
config_param :log_stream_name, :string
|
21
|
+
config_param :log_stream_name, :string, :default => nil
|
21
22
|
config_param :use_log_stream_name_prefix, :bool, default: false
|
22
23
|
config_param :state_file, :string
|
23
24
|
config_param :fetch_interval, :time, default: 60
|
24
25
|
config_param :http_proxy, :string, default: nil
|
25
26
|
config_param :json_handler, :enum, list: [:yajl, :json], :default => :yajl
|
27
|
+
config_param :use_todays_log_stream, :bool, default: false
|
26
28
|
|
27
29
|
config_section :parse do
|
28
30
|
config_set_default :@type, 'none'
|
@@ -105,8 +107,10 @@ module Fluent::Plugin
|
|
105
107
|
if Time.now > @next_fetch_time
|
106
108
|
@next_fetch_time += @fetch_interval
|
107
109
|
|
108
|
-
if @use_log_stream_name_prefix
|
109
|
-
|
110
|
+
if @use_log_stream_name_prefix || @use_todays_log_stream
|
111
|
+
log_stream_name_prefix = @use_todays_log_stream ? get_todays_date : @log_stream_name
|
112
|
+
log_streams = describe_log_streams(log_stream_name_prefix)
|
113
|
+
log_streams.concat(describe_log_streams(get_yesterdays_date)) if @use_todays_log_stream
|
110
114
|
log_streams.each do |log_stream|
|
111
115
|
log_stream_name = log_stream.log_stream_name
|
112
116
|
events = get_events(log_stream_name)
|
@@ -142,19 +146,22 @@ module Fluent::Plugin
|
|
142
146
|
log_group_name: @log_group_name,
|
143
147
|
log_stream_name: log_stream_name
|
144
148
|
}
|
145
|
-
|
149
|
+
log_next_token = next_token(log_stream_name)
|
150
|
+
request[:next_token] = log_next_token if !log_next_token.nil? && !log_next_token.empty?
|
146
151
|
response = @logs.get_log_events(request)
|
147
|
-
|
152
|
+
if valid_next_token(log_next_token, response.next_forward_token)
|
153
|
+
store_next_token(response.next_forward_token, log_stream_name)
|
154
|
+
end
|
148
155
|
|
149
156
|
response.events
|
150
157
|
end
|
151
158
|
|
152
|
-
def describe_log_streams(log_streams = nil, next_token = nil)
|
159
|
+
def describe_log_streams(log_stream_name_prefix, log_streams = nil, next_token = nil)
|
153
160
|
request = {
|
154
161
|
log_group_name: @log_group_name
|
155
162
|
}
|
156
163
|
request[:next_token] = next_token if next_token
|
157
|
-
request[:log_stream_name_prefix] =
|
164
|
+
request[:log_stream_name_prefix] = log_stream_name_prefix
|
158
165
|
response = @logs.describe_log_streams(request)
|
159
166
|
if log_streams
|
160
167
|
log_streams.concat(response.log_streams)
|
@@ -162,9 +169,21 @@ module Fluent::Plugin
|
|
162
169
|
log_streams = response.log_streams
|
163
170
|
end
|
164
171
|
if response.next_token
|
165
|
-
log_streams = describe_log_streams(log_streams, response.next_token)
|
172
|
+
log_streams = describe_log_streams(log_stream_name_prefix, log_streams, response.next_token)
|
166
173
|
end
|
167
174
|
log_streams
|
168
175
|
end
|
176
|
+
|
177
|
+
def valid_next_token(prev_token, next_token)
|
178
|
+
return prev_token != next_token.chomp && !next_token.nil?
|
179
|
+
end
|
180
|
+
|
181
|
+
def get_todays_date
|
182
|
+
return Date.today.strftime("%Y/%m/%d")
|
183
|
+
end
|
184
|
+
|
185
|
+
def get_yesterdays_date
|
186
|
+
return (Date.today - 1).strftime("%Y/%m/%d")
|
187
|
+
end
|
169
188
|
end
|
170
189
|
end
|
@@ -343,6 +343,7 @@ module Fluent::Plugin
|
|
343
343
|
"log_stream" => stream_name,
|
344
344
|
"new_sequence_token" => token,
|
345
345
|
}
|
346
|
+
retry_count += 1
|
346
347
|
rescue Aws::CloudWatchLogs::Errors::ResourceNotFoundException => err
|
347
348
|
if @auto_create_stream && err.message == 'The specified log stream does not exist.'
|
348
349
|
log.warn 'Creating log stream because "The specified log stream does not exist." error is got', {
|
@@ -352,6 +353,7 @@ module Fluent::Plugin
|
|
352
353
|
}
|
353
354
|
create_log_stream(group_name, stream_name)
|
354
355
|
delete_sequence_token(group_name, stream_name)
|
356
|
+
retry_count += 1
|
355
357
|
else
|
356
358
|
raise err
|
357
359
|
end
|
@@ -1,6 +1,7 @@
|
|
1
1
|
require 'test_helper'
|
2
2
|
require 'fluent/test/driver/input'
|
3
3
|
require 'fluent/test/helpers'
|
4
|
+
require 'date'
|
4
5
|
|
5
6
|
class CloudwatchLogsInputTest < Test::Unit::TestCase
|
6
7
|
include CloudwatchLogsTestHelper
|
@@ -137,6 +138,87 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
|
|
137
138
|
assert_true(emits.include? ['test', ((time_ms + 4000) / 1000).floor, {'cloudwatch' => 'logs4'}])
|
138
139
|
end
|
139
140
|
|
141
|
+
def test_emit_with_todays_log_stream
|
142
|
+
new_log_stream("testprefix")
|
143
|
+
create_log_stream
|
144
|
+
|
145
|
+
today = DateTime.now.strftime("%Y/%m/%d")
|
146
|
+
yesterday = (Date.today - 1).strftime("%Y/%m/%d")
|
147
|
+
tomorrow = (Date.today + 1).strftime("%Y/%m/%d")
|
148
|
+
|
149
|
+
|
150
|
+
time_ms = (Time.now.to_f * 1000).floor
|
151
|
+
put_log_events([
|
152
|
+
{timestamp: time_ms + 1000, message: '{"cloudwatch":"logs1"}'},
|
153
|
+
{timestamp: time_ms + 2000, message: '{"cloudwatch":"logs2"}'},
|
154
|
+
])
|
155
|
+
|
156
|
+
new_log_stream(today)
|
157
|
+
create_log_stream
|
158
|
+
put_log_events([
|
159
|
+
{timestamp: time_ms + 3000, message: '{"cloudwatch":"logs3"}'},
|
160
|
+
{timestamp: time_ms + 4000, message: '{"cloudwatch":"logs4"}'},
|
161
|
+
])
|
162
|
+
|
163
|
+
new_log_stream(yesterday)
|
164
|
+
create_log_stream
|
165
|
+
put_log_events([
|
166
|
+
{timestamp: time_ms + 5000, message: '{"cloudwatch":"logs5"}'},
|
167
|
+
{timestamp: time_ms + 6000, message: '{"cloudwatch":"logs6"}'},
|
168
|
+
])
|
169
|
+
|
170
|
+
new_log_stream(tomorrow)
|
171
|
+
create_log_stream
|
172
|
+
put_log_events([
|
173
|
+
{timestamp: time_ms + 7000, message: '{"cloudwatch":"logs7"}'},
|
174
|
+
{timestamp: time_ms + 8000, message: '{"cloudwatch":"logs8"}'},
|
175
|
+
])
|
176
|
+
|
177
|
+
new_log_stream(today)
|
178
|
+
create_log_stream
|
179
|
+
put_log_events([
|
180
|
+
{timestamp: time_ms + 9000, message: '{"cloudwatch":"logs9"}'},
|
181
|
+
{timestamp: time_ms + 10000, message: '{"cloudwatch":"logs10"}'},
|
182
|
+
])
|
183
|
+
|
184
|
+
new_log_stream(yesterday)
|
185
|
+
create_log_stream
|
186
|
+
put_log_events([
|
187
|
+
{timestamp: time_ms + 11000, message: '{"cloudwatch":"logs11"}'},
|
188
|
+
{timestamp: time_ms + 12000, message: '{"cloudwatch":"logs12"}'},
|
189
|
+
])
|
190
|
+
|
191
|
+
sleep 15
|
192
|
+
|
193
|
+
d = create_driver(<<-EOC)
|
194
|
+
tag test
|
195
|
+
@type cloudwatch_logs
|
196
|
+
log_group_name #{log_group_name}
|
197
|
+
use_todays_log_stream true
|
198
|
+
state_file /tmp/state
|
199
|
+
#{aws_key_id}
|
200
|
+
#{aws_sec_key}
|
201
|
+
#{region}
|
202
|
+
#{endpoint}
|
203
|
+
EOC
|
204
|
+
d.run(expect_emits: 8, timeout: 15)
|
205
|
+
|
206
|
+
emits = d.events
|
207
|
+
assert_equal(8, emits.size)
|
208
|
+
assert_false(emits.include? ['test', ((time_ms + 1000) / 1000).floor, {'cloudwatch' => 'logs1'}])
|
209
|
+
assert_false(emits.include? ['test', ((time_ms + 2000) / 1000).floor, {'cloudwatch' => 'logs2'}])
|
210
|
+
assert_true(emits.include? ['test', ((time_ms + 3000) / 1000).floor, {'cloudwatch' => 'logs3'}])
|
211
|
+
assert_true(emits.include? ['test', ((time_ms + 4000) / 1000).floor, {'cloudwatch' => 'logs4'}])
|
212
|
+
assert_true(emits.include? ['test', ((time_ms + 5000) / 1000).floor, {'cloudwatch' => 'logs5'}])
|
213
|
+
assert_true(emits.include? ['test', ((time_ms + 6000) / 1000).floor, {'cloudwatch' => 'logs6'}])
|
214
|
+
assert_false(emits.include? ['test', ((time_ms + 7000) / 1000).floor, {'cloudwatch' => 'logs7'}])
|
215
|
+
assert_false(emits.include? ['test', ((time_ms + 8000) / 1000).floor, {'cloudwatch' => 'logs8'}])
|
216
|
+
assert_true(emits.include? ['test', ((time_ms + 9000) / 1000).floor, {'cloudwatch' => 'logs9'}])
|
217
|
+
assert_true(emits.include? ['test', ((time_ms + 10000) / 1000).floor, {'cloudwatch' => 'logs10'}])
|
218
|
+
assert_true(emits.include? ['test', ((time_ms + 11000) / 1000).floor, {'cloudwatch' => 'logs11'}])
|
219
|
+
assert_true(emits.include? ['test', ((time_ms + 12000) / 1000).floor, {'cloudwatch' => 'logs12'}])
|
220
|
+
end
|
221
|
+
|
140
222
|
private
|
141
223
|
def default_config
|
142
224
|
<<-EOC
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-cloudwatch-logs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.6.
|
4
|
+
version: 0.6.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ryota Arai
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-
|
11
|
+
date: 2018-08-22 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|