fluent-plugin-cloudwatch-logs 0.8.0 → 0.8.1
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: df810968d48ed4f1c2f11cfd66972864d9d4c0594ad03453c7c1a534cd5c367d
|
4
|
+
data.tar.gz: a0e21faa91fa6a5e530a7b7ed145de5a016f6e1b092282682e54f02025e729b0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d20bd70811d9e52969d0e0aa90d99ed4a94fc53b2de060571a00ccad58ebe9ab9fd8532d82466f0659ac3ac8547d8662ed88ab493997346c951e6568e609e40c
|
7
|
+
data.tar.gz: 19385acfe587971bcfa1b8ae911c021e00d406132633f190dec8f14946aa5b048c5ba68a6233942672d9f4b4b38671b7c0c3df57819544ad605a7fd895f07be4
|
data/README.md
CHANGED
@@ -48,7 +48,7 @@ Create IAM user with a policy like the following:
|
|
48
48
|
There are several methods to provide authentication credentials. Be aware that there are various tradeoffs for these methods,
|
49
49
|
although most of these tradeoffs are highly dependent on the specific environment.
|
50
50
|
|
51
|
-
### Environment
|
51
|
+
### Environment
|
52
52
|
|
53
53
|
Set region and credentials via the environment:
|
54
54
|
|
@@ -62,7 +62,7 @@ Note: For this to work persistently the enviornment will need to be set in the s
|
|
62
62
|
|
63
63
|
### AWS Configuration
|
64
64
|
|
65
|
-
The plugin will look for the `$HOME/.aws/config` and `$HOME/.aws/credentials` for configuration information. To setup, as the
|
65
|
+
The plugin will look for the `$HOME/.aws/config` and `$HOME/.aws/credentials` for configuration information. To setup, as the
|
66
66
|
fluentd user, run:
|
67
67
|
|
68
68
|
```sh
|
@@ -71,7 +71,7 @@ aws configure
|
|
71
71
|
|
72
72
|
### Configuration Parameters
|
73
73
|
|
74
|
-
The authentication information can also be set
|
74
|
+
The authentication information can also be set
|
75
75
|
|
76
76
|
## Example
|
77
77
|
|
@@ -170,6 +170,11 @@ Please refer to [the PutRetentionPolicy column in documentation](https://docs.aw
|
|
170
170
|
state_file /var/lib/fluent/group_stream.in.state
|
171
171
|
#endpoint http://localhost:5000/
|
172
172
|
#json_handler json
|
173
|
+
# Users can use `format` or `<parse>` directive to parse non-JSON CloudwatchLogs' log
|
174
|
+
# format none # or csv, tsv, regexp etc.
|
175
|
+
#<parse>
|
176
|
+
# @type none # or csv, tsv, regexp etc.
|
177
|
+
#</parse>
|
173
178
|
</source>
|
174
179
|
```
|
175
180
|
|
@@ -190,6 +195,8 @@ Please refer to [the PutRetentionPolicy column in documentation](https://docs.aw
|
|
190
195
|
* `use_log_stream_name_prefix`: to use `log_stream_name` as log stream name prefix (default false)
|
191
196
|
* `use_todays_log_stream`: use todays and yesterdays date as log stream name prefix (formatted YYYY/MM/DD). (default: `false`)
|
192
197
|
* `use_aws_timestamp`: get timestamp from Cloudwatch event for non json logs, otherwise fluentd will parse the log to get the timestamp (default `false`)
|
198
|
+
* `format`: specify CloudWatchLogs' log format. (default `nil`)
|
199
|
+
* `<parse>`: specify parser plugin configuration. see also: https://docs.fluentd.org/v/1.0/parser#how-to-use
|
193
200
|
|
194
201
|
## Test
|
195
202
|
|
@@ -34,6 +34,7 @@ module Fluent::Plugin
|
|
34
34
|
def initialize
|
35
35
|
super
|
36
36
|
|
37
|
+
@parser = nil
|
37
38
|
require 'aws-sdk-cloudwatchlogs'
|
38
39
|
end
|
39
40
|
|
@@ -82,6 +83,8 @@ module Fluent::Plugin
|
|
82
83
|
def configure_parser(conf)
|
83
84
|
if conf['format']
|
84
85
|
@parser = parser_create
|
86
|
+
elsif parser_config = conf.elements('parse').first
|
87
|
+
@parser = parser_create(conf: parser_config)
|
85
88
|
end
|
86
89
|
end
|
87
90
|
|
@@ -113,6 +113,43 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
|
|
113
113
|
assert_equal(['test', (log_time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs2"}], emits[1])
|
114
114
|
end
|
115
115
|
|
116
|
+
test "emit with <parse> csv" do
|
117
|
+
cloudwatch_config = {'tag' => "test",
|
118
|
+
'@type' => 'cloudwatch_logs',
|
119
|
+
'log_group_name' => "#{log_group_name}",
|
120
|
+
'log_stream_name' => "#{log_stream_name}",
|
121
|
+
'state_file' => '/tmp/state',
|
122
|
+
}
|
123
|
+
cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_key_id)) if ENV['aws_key_id']
|
124
|
+
cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_sec_key)) if ENV['aws_sec_key']
|
125
|
+
cloudwatch_config = cloudwatch_config.merge!(config_elementify(region)) if ENV['region']
|
126
|
+
cloudwatch_config = cloudwatch_config.merge!(config_elementify(endpoint)) if ENV['endpoint']
|
127
|
+
|
128
|
+
csv_format_config = config_element('ROOT', '', cloudwatch_config, [
|
129
|
+
config_element('parse', '', {'@type' => 'csv',
|
130
|
+
'keys' => 'time,message',
|
131
|
+
'time_key' => 'time'})
|
132
|
+
])
|
133
|
+
create_log_stream
|
134
|
+
|
135
|
+
time_ms = (Time.now.to_f * 1000).floor
|
136
|
+
log_time_ms = time_ms - 10000
|
137
|
+
put_log_events([
|
138
|
+
{timestamp: time_ms, message: Time.at(log_time_ms/1000.floor).to_s + ",Cloudwatch non json logs1"},
|
139
|
+
{timestamp: time_ms, message: Time.at(log_time_ms/1000.floor).to_s + ",Cloudwatch non json logs2"},
|
140
|
+
])
|
141
|
+
|
142
|
+
sleep 5
|
143
|
+
|
144
|
+
d = create_driver(csv_format_config)
|
145
|
+
d.run(expect_emits: 2, timeout: 5)
|
146
|
+
|
147
|
+
emits = d.events
|
148
|
+
assert_equal(2, emits.size)
|
149
|
+
assert_equal(['test', (log_time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs1"}], emits[0])
|
150
|
+
assert_equal(['test', (log_time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs2"}], emits[1])
|
151
|
+
end
|
152
|
+
|
116
153
|
def test_emit_width_format
|
117
154
|
create_log_stream
|
118
155
|
|
@@ -149,6 +186,47 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
|
|
149
186
|
assert_equal({'cloudwatch' => 'logs2'}, emits[1][2])
|
150
187
|
end
|
151
188
|
|
189
|
+
test "emit with <parse> regexp" do
|
190
|
+
cloudwatch_config = {'tag' => "test",
|
191
|
+
'@type' => 'cloudwatch_logs',
|
192
|
+
'log_group_name' => "#{log_group_name}",
|
193
|
+
'log_stream_name' => "#{log_stream_name}",
|
194
|
+
'state_file' => '/tmp/state',
|
195
|
+
}
|
196
|
+
cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_key_id)) if ENV['aws_key_id']
|
197
|
+
cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_sec_key)) if ENV['aws_sec_key']
|
198
|
+
cloudwatch_config = cloudwatch_config.merge!(config_elementify(region)) if ENV['region']
|
199
|
+
cloudwatch_config = cloudwatch_config.merge!(config_elementify(endpoint)) if ENV['endpoint']
|
200
|
+
|
201
|
+
regex_format_config = config_element('ROOT', '', cloudwatch_config, [
|
202
|
+
config_element('parse', '', {'@type' => 'regexp',
|
203
|
+
'expression' => "/^(?<cloudwatch>[^ ]*)?/",
|
204
|
+
})
|
205
|
+
])
|
206
|
+
create_log_stream
|
207
|
+
|
208
|
+
time_ms = (Time.now.to_f * 1000).floor
|
209
|
+
put_log_events([
|
210
|
+
{timestamp: time_ms, message: 'logs1'},
|
211
|
+
{timestamp: time_ms, message: 'logs2'},
|
212
|
+
])
|
213
|
+
|
214
|
+
sleep 5
|
215
|
+
|
216
|
+
d = create_driver(regex_format_config)
|
217
|
+
|
218
|
+
d.run(expect_emits: 2, timeout: 5)
|
219
|
+
|
220
|
+
emits = d.events
|
221
|
+
assert_equal(2, emits.size)
|
222
|
+
assert_equal('test', emits[0][0])
|
223
|
+
assert_in_delta((time_ms / 1000).floor, emits[0][1], 10)
|
224
|
+
assert_equal({'cloudwatch' => 'logs1'}, emits[0][2])
|
225
|
+
assert_equal('test', emits[1][0])
|
226
|
+
assert_in_delta((time_ms / 1000).floor, emits[1][1], 10)
|
227
|
+
assert_equal({'cloudwatch' => 'logs2'}, emits[1][2])
|
228
|
+
end
|
229
|
+
|
152
230
|
def test_emit_with_prefix
|
153
231
|
new_log_stream("testprefix")
|
154
232
|
create_log_stream
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-cloudwatch-logs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.8.
|
4
|
+
version: 0.8.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ryota Arai
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-03-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|