fluent-plugin-cloudwatch-logs 0.9.0 → 0.9.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +6 -0
- data/lib/fluent/plugin/cloudwatch/logs/version.rb +1 -1
- data/lib/fluent/plugin/in_cloudwatch_logs.rb +12 -0
- data/test/plugin/test_in_cloudwatch_logs.rb +53 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 849137579f60a74de04a845cdad60ec5c31a7903b8af5248c1122a06a4d66b3a
|
4
|
+
data.tar.gz: 4911ef8c0a16b88501f08cfe2dd217e458025d68496b1f279662bef606a53d2d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 72233f6fe060a41813673688fe5ca00ec36de7e6e21bc52fcdb7751091a2e7b0198110e398ad5233021527496f74eee22a3c18490d2f35c847cea3587dcf5c40
|
7
|
+
data.tar.gz: 13357b5705f53a84a5598132238c00fc9caf1c0b317e943e38a1ff8b3ac96d1e427a01499ce89f911291b0571ac43876528c3faaa0bd821a600d20805fe76ec7
|
data/README.md
CHANGED
@@ -170,6 +170,9 @@ Please refer to [the PutRetentionPolicy column in documentation](https://docs.aw
|
|
170
170
|
state_file /var/lib/fluent/group_stream.in.state
|
171
171
|
#endpoint http://localhost:5000/
|
172
172
|
#json_handler json
|
173
|
+
# start_time "2020-03-01 00:00:00Z"
|
174
|
+
# end_time "2020-04-30 15:00:00Z"
|
175
|
+
# time_range_format "%Y-%m-%d %H:%M:%S%z"
|
173
176
|
# Users can use `format` or `<parse>` directive to parse non-JSON CloudwatchLogs' log
|
174
177
|
# format none # or csv, tsv, regexp etc.
|
175
178
|
#<parse>
|
@@ -195,6 +198,9 @@ Please refer to [the PutRetentionPolicy column in documentation](https://docs.aw
|
|
195
198
|
* `use_log_stream_name_prefix`: to use `log_stream_name` as log stream name prefix (default false)
|
196
199
|
* `use_todays_log_stream`: use todays and yesterdays date as log stream name prefix (formatted YYYY/MM/DD). (default: `false`)
|
197
200
|
* `use_aws_timestamp`: get timestamp from Cloudwatch event for non json logs, otherwise fluentd will parse the log to get the timestamp (default `false`)
|
201
|
+
* `start_time`: specify starting time range for obtaining logs. (default: `nil`)
|
202
|
+
* `end_time`: specify ending time range for obtaining logs. (default: `nil`)
|
203
|
+
* `time_range_format`: specify time format for time range. (default: `%Y-%m-%d %H:%M:%S`)
|
198
204
|
* `format`: specify CloudWatchLogs' log format. (default `nil`)
|
199
205
|
* `<parse>`: specify parser plugin configuration. see also: https://docs.fluentd.org/v/1.0/parser#how-to-use
|
200
206
|
|
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'date'
|
2
|
+
require 'time'
|
2
3
|
require 'fluent/plugin/input'
|
3
4
|
require 'fluent/plugin/parser'
|
4
5
|
require 'yajl'
|
@@ -26,6 +27,9 @@ module Fluent::Plugin
|
|
26
27
|
config_param :json_handler, :enum, list: [:yajl, :json], default: :yajl
|
27
28
|
config_param :use_todays_log_stream, :bool, default: false
|
28
29
|
config_param :use_aws_timestamp, :bool, default: false
|
30
|
+
config_param :start_time, :string, default: nil
|
31
|
+
config_param :end_time, :string, default: nil
|
32
|
+
config_param :time_range_format, :string, default: "%Y-%m-%d %H:%M:%S"
|
29
33
|
|
30
34
|
config_section :parse do
|
31
35
|
config_set_default :@type, 'none'
|
@@ -42,6 +46,12 @@ module Fluent::Plugin
|
|
42
46
|
compat_parameters_convert(conf, :parser)
|
43
47
|
super
|
44
48
|
configure_parser(conf)
|
49
|
+
|
50
|
+
@start_time = (Time.strptime(@start_time, @time_range_format).to_f * 1000).floor if @start_time
|
51
|
+
@end_time = (Time.strptime(@end_time, @time_range_format).to_f * 1000).floor if @end_time
|
52
|
+
if @start_time && @end_time && (@end_time < @start_time)
|
53
|
+
raise Fluent::ConfigError, "end_time(#{@end_time}) should be greater than start_time(#{@start_time})."
|
54
|
+
end
|
45
55
|
end
|
46
56
|
|
47
57
|
def start
|
@@ -163,6 +173,8 @@ module Fluent::Plugin
|
|
163
173
|
log_group_name: @log_group_name,
|
164
174
|
log_stream_name: log_stream_name
|
165
175
|
}
|
176
|
+
request.merge!(start_time: @start_time) if @start_time
|
177
|
+
request.merge!(end_time: @end_time) if @end_time
|
166
178
|
log_next_token = next_token(log_stream_name)
|
167
179
|
request[:next_token] = log_next_token if !log_next_token.nil? && !log_next_token.empty?
|
168
180
|
response = @logs.get_log_events(request)
|
@@ -25,6 +25,9 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
|
|
25
25
|
use_log_stream_name_prefix true
|
26
26
|
state_file /tmp/state
|
27
27
|
use_aws_timestamp true
|
28
|
+
start_time "2019-06-18 00:00:00Z"
|
29
|
+
end_time "2020-01-18 00:00:00Z"
|
30
|
+
time_range_format "%Y-%m-%d %H:%M:%S%z"
|
28
31
|
EOC
|
29
32
|
|
30
33
|
assert_equal('test_id', d.instance.aws_key_id)
|
@@ -37,6 +40,29 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
|
|
37
40
|
assert_equal('/tmp/state', d.instance.state_file)
|
38
41
|
assert_equal(:yajl, d.instance.json_handler)
|
39
42
|
assert_equal(true, d.instance.use_aws_timestamp)
|
43
|
+
assert_equal(1560816000000, d.instance.start_time)
|
44
|
+
assert_equal(1579305600000, d.instance.end_time)
|
45
|
+
assert_equal("%Y-%m-%d %H:%M:%S%z", d.instance.time_range_format)
|
46
|
+
end
|
47
|
+
|
48
|
+
test 'invalid time range' do
|
49
|
+
assert_raise(Fluent::ConfigError) do
|
50
|
+
create_driver(<<-EOC)
|
51
|
+
@type cloudwatch_logs
|
52
|
+
aws_key_id test_id
|
53
|
+
aws_sec_key test_key
|
54
|
+
region us-east-1
|
55
|
+
tag test
|
56
|
+
log_group_name group
|
57
|
+
log_stream_name stream
|
58
|
+
use_log_stream_name_prefix true
|
59
|
+
state_file /tmp/state
|
60
|
+
use_aws_timestamp true
|
61
|
+
start_time "2019-06-18 00:00:00Z"
|
62
|
+
end_time "2019-01-18 00:00:00Z"
|
63
|
+
time_range_format "%Y-%m-%d %H:%M:%S%z"
|
64
|
+
EOC
|
65
|
+
end
|
40
66
|
end
|
41
67
|
end
|
42
68
|
|
@@ -92,6 +118,33 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
|
|
92
118
|
assert_equal(['test', (time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs2"}], emits[1])
|
93
119
|
end
|
94
120
|
|
121
|
+
def test_emit_with_aws_timestamp_and_time_range
|
122
|
+
create_log_stream
|
123
|
+
|
124
|
+
time_ms = (Time.now.to_f * 1000).floor
|
125
|
+
before_6h_time_ms = ((Time.now.to_f - 60*60*6) * 1000).floor
|
126
|
+
log_time_ms = time_ms - 10000
|
127
|
+
put_log_events([
|
128
|
+
{timestamp: before_6h_time_ms, message: Time.at((before_6h_time_ms - 10000)/1000.floor).to_s + ",Cloudwatch non json logs1"},
|
129
|
+
{timestamp: before_6h_time_ms, message: Time.at((before_6h_time_ms - 10000)/1000.floor).to_s + ",Cloudwatch non json logs2"},
|
130
|
+
{timestamp: time_ms, message: Time.at(log_time_ms/1000.floor).to_s + ",Cloudwatch non json logs3"},
|
131
|
+
])
|
132
|
+
|
133
|
+
sleep 5
|
134
|
+
|
135
|
+
d = create_driver(csv_format_config_aws_timestamp + %[
|
136
|
+
start_time #{Time.at(Time.now.to_f - 60*60*8).to_s}
|
137
|
+
end_time #{Time.at(Time.now.to_f - 60*60*4).to_s}
|
138
|
+
time_range_format "%Y-%m-%d %H:%M:%S %z"
|
139
|
+
])
|
140
|
+
d.run(expect_emits: 2, timeout: 5)
|
141
|
+
|
142
|
+
emits = d.events
|
143
|
+
assert_equal(2, emits.size)
|
144
|
+
assert_equal(['test', (before_6h_time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs1"}], emits[0])
|
145
|
+
assert_equal(['test', (before_6h_time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs2"}], emits[1])
|
146
|
+
end
|
147
|
+
|
95
148
|
def test_emit_with_log_timestamp
|
96
149
|
create_log_stream
|
97
150
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-cloudwatch-logs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.9.
|
4
|
+
version: 0.9.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ryota Arai
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-03-
|
11
|
+
date: 2020-03-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|