fluent-plugin-cloudwatch-ingest 0.5.4 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 97564b12944b487966ac98b5f0175b1fed34b2d9
|
4
|
+
data.tar.gz: 7bde517b6c51e6b78902da36d090d3ddb7594e4e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1d5ee8f52a6c7669ab9386b9874f3f227d20f5983550d65bcfd389925b6af3b84bcfa76f59f788547902930fad9ddcfabb60c1b0ff92e93d86497ec01050c064
|
7
|
+
data.tar.gz: 86f6914db8742f1c4eb852a7faea833c11714563ffee10c7f0c3a37acf83d5f713e5baaa2717ac4811f72ae1cba99cea2515f28782c3ad6ab8b189eec579a2b0
|
data/README.md
CHANGED
@@ -39,10 +39,12 @@ Or install it yourself as:
|
|
39
39
|
limit_events 10000
|
40
40
|
state_file_name /mnt/nfs/cloudwatch.state
|
41
41
|
interval 60
|
42
|
-
api_interval 5
|
43
|
-
limit_events 10000
|
44
|
-
event_start_time 0
|
45
|
-
oldest_logs_first false
|
42
|
+
api_interval 5 # Time to wait between API call failures before retry
|
43
|
+
limit_events 10000 # Number of events to fetch in any given iteration
|
44
|
+
event_start_time 0 # Do not fetch events before this time (UNIX epoch, miliseconds)
|
45
|
+
oldest_logs_first false # When true fetch the oldest logs first
|
46
|
+
telemetry false # Produce statsd telemetry
|
47
|
+
statsd_endpoint localhost # Endpoint to which telemetry should be sent
|
46
48
|
<parse>
|
47
49
|
@type cloudwatch_ingest
|
48
50
|
expression /^(?<message>.+)$/
|
@@ -24,7 +24,7 @@ module Fluent::Plugin
|
|
24
24
|
desc 'Log stream name or prefix. Not setting means "all"'
|
25
25
|
config_param :log_stream_name_prefix, :string, default: ''
|
26
26
|
desc 'State file name'
|
27
|
-
config_param :state_file_name, :string, default: '/var/spool/td-agent/cloudwatch.state' # rubocop:disable
|
27
|
+
config_param :state_file_name, :string, default: '/var/spool/td-agent/cloudwatch.state' # rubocop:disable LineLength
|
28
28
|
desc 'Fetch logs every interval'
|
29
29
|
config_param :interval, :time, default: 60
|
30
30
|
desc 'Time to pause between API call failures and limits'
|
@@ -39,6 +39,10 @@ module Fluent::Plugin
|
|
39
39
|
config_param :event_start_time, :integer, default: 0
|
40
40
|
desc 'Fetch the oldest logs first'
|
41
41
|
config_param :oldest_logs_first, :bool, default: false
|
42
|
+
desc 'Turn on telemetry'
|
43
|
+
config_param :telemetry, :bool, default: false
|
44
|
+
desc 'Statsd endpoint to which telemetry should be written'
|
45
|
+
config_param :statsd_endpoint, :string, default: 'localhost'
|
42
46
|
config_section :parse do
|
43
47
|
config_set_default :@type, 'cloudwatch_ingest'
|
44
48
|
desc 'Regular expression with which to parse the event message'
|
@@ -58,6 +62,15 @@ module Fluent::Plugin
|
|
58
62
|
log.info('Starting fluentd-plugin-cloudwatch-ingest')
|
59
63
|
end
|
60
64
|
|
65
|
+
def metric(method, name, value = 0)
|
66
|
+
case method
|
67
|
+
when :increment
|
68
|
+
@statsd.send(method, name) if @telemetry
|
69
|
+
else
|
70
|
+
@statsd.send(method, name, value) if @telemetry
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
61
74
|
def configure(conf)
|
62
75
|
super
|
63
76
|
compat_parameters_convert(conf, :parser)
|
@@ -72,6 +85,9 @@ module Fluent::Plugin
|
|
72
85
|
raise Fluent::ConfigError, 'parse/event_time is required.'
|
73
86
|
end
|
74
87
|
|
88
|
+
# Configure telemetry, if enabled
|
89
|
+
@statsd = Statsd.new @statsd_endpoint, 8125 if @telemetry
|
90
|
+
|
75
91
|
@parser = parser_create(conf: parser_config)
|
76
92
|
log.info('Configured fluentd-plugin-cloudwatch-ingest')
|
77
93
|
end
|
@@ -92,7 +108,7 @@ module Fluent::Plugin
|
|
92
108
|
role_session_name: @sts_session_name
|
93
109
|
)
|
94
110
|
|
95
|
-
log.info("Using STS for authentication with source account ARN: #{@sts_arn}, session name: #{@sts_session_name}") # rubocop:disable
|
111
|
+
log.info("Using STS for authentication with source account ARN: #{@sts_arn}, session name: #{@sts_session_name}") # rubocop:disable LineLength
|
96
112
|
else
|
97
113
|
log.info('Using local instance IAM role for authentication')
|
98
114
|
end
|
@@ -111,6 +127,7 @@ module Fluent::Plugin
|
|
111
127
|
def emit(event, log_group_name, log_stream_name)
|
112
128
|
@parser.parse(event, log_group_name, log_stream_name) do |time, record|
|
113
129
|
router.emit(@tag, time, record)
|
130
|
+
metric(:increment, 'events.emitted.success')
|
114
131
|
end
|
115
132
|
end
|
116
133
|
|
@@ -121,6 +138,7 @@ module Fluent::Plugin
|
|
121
138
|
next_token = nil
|
122
139
|
loop do
|
123
140
|
begin
|
141
|
+
metric(:increment, 'api.calls.describeloggroups.attempted')
|
124
142
|
response = if !log_group_prefix.empty?
|
125
143
|
@aws.describe_log_groups(
|
126
144
|
log_group_name_prefix: log_group_prefix,
|
@@ -137,6 +155,7 @@ module Fluent::Plugin
|
|
137
155
|
next_token = response.next_token
|
138
156
|
rescue => boom
|
139
157
|
log.error("Unable to retrieve log groups: #{boom}")
|
158
|
+
metric(:increment, 'api.calls.describeloggroups.failed')
|
140
159
|
next_token = nil
|
141
160
|
sleep @api_interval
|
142
161
|
retry
|
@@ -152,6 +171,7 @@ module Fluent::Plugin
|
|
152
171
|
next_token = nil
|
153
172
|
loop do
|
154
173
|
begin
|
174
|
+
metric(:increment, 'api.calls.describelogstreams.attempted')
|
155
175
|
response = if !log_stream_name_prefix.empty?
|
156
176
|
@aws.describe_log_streams(
|
157
177
|
log_group_name: log_group_name,
|
@@ -169,7 +189,8 @@ module Fluent::Plugin
|
|
169
189
|
break unless response.next_token
|
170
190
|
next_token = response.next_token
|
171
191
|
rescue => boom
|
172
|
-
log.error("Unable to retrieve log streams for group #{log_group_name} with stream prefix #{log_stream_name_prefix}: #{boom}") # rubocop:disable
|
192
|
+
log.error("Unable to retrieve log streams for group #{log_group_name} with stream prefix #{log_stream_name_prefix}: #{boom}") # rubocop:disable LineLength
|
193
|
+
metric(:increment, 'api.calls.describelogstreams.failed')
|
173
194
|
log_streams = []
|
174
195
|
next_token = nil
|
175
196
|
sleep @api_interval
|
@@ -206,6 +227,7 @@ module Fluent::Plugin
|
|
206
227
|
end
|
207
228
|
|
208
229
|
begin
|
230
|
+
metric(:increment, 'api.calls.getlogevents.attempted')
|
209
231
|
response = @aws.get_log_events(
|
210
232
|
log_group_name: group,
|
211
233
|
log_stream_name: stream,
|
@@ -220,6 +242,7 @@ module Fluent::Plugin
|
|
220
242
|
emit(e, group, stream)
|
221
243
|
rescue => boom
|
222
244
|
log.error("Failed to emit event #{e}: #{boom}")
|
245
|
+
metric(:increment, 'events.emitted.failed')
|
223
246
|
end
|
224
247
|
end
|
225
248
|
|
@@ -227,7 +250,8 @@ module Fluent::Plugin
|
|
227
250
|
# in this iteration, store the forward token
|
228
251
|
state.store[group][stream] = response.next_forward_token
|
229
252
|
rescue => boom
|
230
|
-
log.error("Unable to retrieve events for stream #{stream} in group #{group}: #{boom}") # rubocop:disable
|
253
|
+
log.error("Unable to retrieve events for stream #{stream} in group #{group}: #{boom}") # rubocop:disable LineLength
|
254
|
+
metric(:increment, 'api.calls.getlogevents.failed')
|
231
255
|
sleep @api_interval
|
232
256
|
retry
|
233
257
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-cloudwatch-ingest
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.6.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sam Pointer
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-06-
|
11
|
+
date: 2017-06-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -108,6 +108,20 @@ dependencies:
|
|
108
108
|
- - "~>"
|
109
109
|
- !ruby/object:Gem::Version
|
110
110
|
version: '1.12'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: statsd-ruby
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - "~>"
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: 1.4.0
|
118
|
+
type: :runtime
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - "~>"
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: 1.4.0
|
111
125
|
description: Fluentd plugin to ingest AWS Cloudwatch logs
|
112
126
|
email:
|
113
127
|
- san@outsidethe.net
|