fluent-plugin-cloudwatch-ingest 0.5.4 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 2ae5771e3b9826c304a17a701823a7f04d756cfe
4
- data.tar.gz: 73fbfc8239d034a6b48de309347f7104792587ac
3
+ metadata.gz: 97564b12944b487966ac98b5f0175b1fed34b2d9
4
+ data.tar.gz: 7bde517b6c51e6b78902da36d090d3ddb7594e4e
5
5
  SHA512:
6
- metadata.gz: 028695acdce8b76c387b6acbace10522b508a025289a4ed18b8cbf85ecb5e3dd5249541d8888ee083ce0f9c907bebaeaa9808274fe504b9d96d113955a9bdc25
7
- data.tar.gz: c62d7a7ceae54c17183fc77e85963ef851769094fb91f33ebf5af6631133b604091c186b3aca0970ca1c8b909c21ce395c51cf5d21992d61c6516856f97e178c
6
+ metadata.gz: 1d5ee8f52a6c7669ab9386b9874f3f227d20f5983550d65bcfd389925b6af3b84bcfa76f59f788547902930fad9ddcfabb60c1b0ff92e93d86497ec01050c064
7
+ data.tar.gz: 86f6914db8742f1c4eb852a7faea833c11714563ffee10c7f0c3a37acf83d5f713e5baaa2717ac4811f72ae1cba99cea2515f28782c3ad6ab8b189eec579a2b0
data/README.md CHANGED
@@ -39,10 +39,12 @@ Or install it yourself as:
39
39
  limit_events 10000
40
40
  state_file_name /mnt/nfs/cloudwatch.state
41
41
  interval 60
42
- api_interval 5 # Time to wait between API call failures before retry
43
- limit_events 10000 # Number of events to fetch in any given iteration
44
- event_start_time 0 # Do not fetch events before this time (UNIX epoch, miliseconds)
45
- oldest_logs_first false # When true fetch the oldest logs first
42
+ api_interval 5 # Time to wait between API call failures before retry
43
+ limit_events 10000 # Number of events to fetch in any given iteration
44
+ event_start_time 0 # Do not fetch events before this time (UNIX epoch, miliseconds)
45
+ oldest_logs_first false # When true fetch the oldest logs first
46
+ telemetry false # Produce statsd telemetry
47
+ statsd_endpoint localhost # Endpoint to which telemetry should be sent
46
48
  <parse>
47
49
  @type cloudwatch_ingest
48
50
  expression /^(?<message>.+)$/
@@ -35,4 +35,5 @@ Gem::Specification.new do |spec|
35
35
  spec.add_dependency 'fluentd', '~>0.14.13'
36
36
  spec.add_dependency 'aws-sdk', '~>2.8.4'
37
37
  spec.add_dependency 'multi_json', '~>1.12'
38
+ spec.add_dependency 'statsd-ruby', '~>1.4.0'
38
39
  end
@@ -2,7 +2,7 @@ module Fluent
2
2
  module Plugin
3
3
  module Cloudwatch
4
4
  module Ingest
5
- VERSION = '0.5.4'.freeze
5
+ VERSION = '0.6.0'.freeze
6
6
  end
7
7
  end
8
8
  end
@@ -24,7 +24,7 @@ module Fluent::Plugin
24
24
  desc 'Log stream name or prefix. Not setting means "all"'
25
25
  config_param :log_stream_name_prefix, :string, default: ''
26
26
  desc 'State file name'
27
- config_param :state_file_name, :string, default: '/var/spool/td-agent/cloudwatch.state' # rubocop:disable all
27
+ config_param :state_file_name, :string, default: '/var/spool/td-agent/cloudwatch.state' # rubocop:disable LineLength
28
28
  desc 'Fetch logs every interval'
29
29
  config_param :interval, :time, default: 60
30
30
  desc 'Time to pause between API call failures and limits'
@@ -39,6 +39,10 @@ module Fluent::Plugin
39
39
  config_param :event_start_time, :integer, default: 0
40
40
  desc 'Fetch the oldest logs first'
41
41
  config_param :oldest_logs_first, :bool, default: false
42
+ desc 'Turn on telemetry'
43
+ config_param :telemetry, :bool, default: false
44
+ desc 'Statsd endpoint to which telemetry should be written'
45
+ config_param :statsd_endpoint, :string, default: 'localhost'
42
46
  config_section :parse do
43
47
  config_set_default :@type, 'cloudwatch_ingest'
44
48
  desc 'Regular expression with which to parse the event message'
@@ -58,6 +62,15 @@ module Fluent::Plugin
58
62
  log.info('Starting fluentd-plugin-cloudwatch-ingest')
59
63
  end
60
64
 
65
+ def metric(method, name, value = 0)
66
+ case method
67
+ when :increment
68
+ @statsd.send(method, name) if @telemetry
69
+ else
70
+ @statsd.send(method, name, value) if @telemetry
71
+ end
72
+ end
73
+
61
74
  def configure(conf)
62
75
  super
63
76
  compat_parameters_convert(conf, :parser)
@@ -72,6 +85,9 @@ module Fluent::Plugin
72
85
  raise Fluent::ConfigError, 'parse/event_time is required.'
73
86
  end
74
87
 
88
+ # Configure telemetry, if enabled
89
+ @statsd = Statsd.new @statsd_endpoint, 8125 if @telemetry
90
+
75
91
  @parser = parser_create(conf: parser_config)
76
92
  log.info('Configured fluentd-plugin-cloudwatch-ingest')
77
93
  end
@@ -92,7 +108,7 @@ module Fluent::Plugin
92
108
  role_session_name: @sts_session_name
93
109
  )
94
110
 
95
- log.info("Using STS for authentication with source account ARN: #{@sts_arn}, session name: #{@sts_session_name}") # rubocop:disable all
111
+ log.info("Using STS for authentication with source account ARN: #{@sts_arn}, session name: #{@sts_session_name}") # rubocop:disable LineLength
96
112
  else
97
113
  log.info('Using local instance IAM role for authentication')
98
114
  end
@@ -111,6 +127,7 @@ module Fluent::Plugin
111
127
  def emit(event, log_group_name, log_stream_name)
112
128
  @parser.parse(event, log_group_name, log_stream_name) do |time, record|
113
129
  router.emit(@tag, time, record)
130
+ metric(:increment, 'events.emitted.success')
114
131
  end
115
132
  end
116
133
 
@@ -121,6 +138,7 @@ module Fluent::Plugin
121
138
  next_token = nil
122
139
  loop do
123
140
  begin
141
+ metric(:increment, 'api.calls.describeloggroups.attempted')
124
142
  response = if !log_group_prefix.empty?
125
143
  @aws.describe_log_groups(
126
144
  log_group_name_prefix: log_group_prefix,
@@ -137,6 +155,7 @@ module Fluent::Plugin
137
155
  next_token = response.next_token
138
156
  rescue => boom
139
157
  log.error("Unable to retrieve log groups: #{boom}")
158
+ metric(:increment, 'api.calls.describeloggroups.failed')
140
159
  next_token = nil
141
160
  sleep @api_interval
142
161
  retry
@@ -152,6 +171,7 @@ module Fluent::Plugin
152
171
  next_token = nil
153
172
  loop do
154
173
  begin
174
+ metric(:increment, 'api.calls.describelogstreams.attempted')
155
175
  response = if !log_stream_name_prefix.empty?
156
176
  @aws.describe_log_streams(
157
177
  log_group_name: log_group_name,
@@ -169,7 +189,8 @@ module Fluent::Plugin
169
189
  break unless response.next_token
170
190
  next_token = response.next_token
171
191
  rescue => boom
172
- log.error("Unable to retrieve log streams for group #{log_group_name} with stream prefix #{log_stream_name_prefix}: #{boom}") # rubocop:disable all
192
+ log.error("Unable to retrieve log streams for group #{log_group_name} with stream prefix #{log_stream_name_prefix}: #{boom}") # rubocop:disable LineLength
193
+ metric(:increment, 'api.calls.describelogstreams.failed')
173
194
  log_streams = []
174
195
  next_token = nil
175
196
  sleep @api_interval
@@ -206,6 +227,7 @@ module Fluent::Plugin
206
227
  end
207
228
 
208
229
  begin
230
+ metric(:increment, 'api.calls.getlogevents.attempted')
209
231
  response = @aws.get_log_events(
210
232
  log_group_name: group,
211
233
  log_stream_name: stream,
@@ -220,6 +242,7 @@ module Fluent::Plugin
220
242
  emit(e, group, stream)
221
243
  rescue => boom
222
244
  log.error("Failed to emit event #{e}: #{boom}")
245
+ metric(:increment, 'events.emitted.failed')
223
246
  end
224
247
  end
225
248
 
@@ -227,7 +250,8 @@ module Fluent::Plugin
227
250
  # in this iteration, store the forward token
228
251
  state.store[group][stream] = response.next_forward_token
229
252
  rescue => boom
230
- log.error("Unable to retrieve events for stream #{stream} in group #{group}: #{boom}") # rubocop:disable all
253
+ log.error("Unable to retrieve events for stream #{stream} in group #{group}: #{boom}") # rubocop:disable LineLength
254
+ metric(:increment, 'api.calls.getlogevents.failed')
231
255
  sleep @api_interval
232
256
  retry
233
257
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-cloudwatch-ingest
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.5.4
4
+ version: 0.6.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sam Pointer
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2017-06-16 00:00:00.000000000 Z
11
+ date: 2017-06-23 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -108,6 +108,20 @@ dependencies:
108
108
  - - "~>"
109
109
  - !ruby/object:Gem::Version
110
110
  version: '1.12'
111
+ - !ruby/object:Gem::Dependency
112
+ name: statsd-ruby
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - "~>"
116
+ - !ruby/object:Gem::Version
117
+ version: 1.4.0
118
+ type: :runtime
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - "~>"
123
+ - !ruby/object:Gem::Version
124
+ version: 1.4.0
111
125
  description: Fluentd plugin to ingest AWS Cloudwatch logs
112
126
  email:
113
127
  - san@outsidethe.net