fluent-plugin-google-cloud 0.10.1 → 0.10.6

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 343edd2c59a22d1f35757f12629e19b7e01314dd55bf78bbbfbe6914530dcf27
4
- data.tar.gz: d811dfea9e90007dd3dee571df75428a50e2d7b9ce9ab61d6581c21c4d9d5423
3
+ metadata.gz: d962a86da4d815fd57a1c6bb8f6462995d2b5bce96de8cf69869d59e08d44f92
4
+ data.tar.gz: f092dca63d15d65ff0983cd719b544093a2865bfd67f7d88eb1f80fb9f1c15c4
5
5
  SHA512:
6
- metadata.gz: 8a8b30d92c16eeaea8a6608dccb8e97d72ca1b7e13126e5523582f8be56bd150a2b26bda4e56c247cbf6c3680f307bb71eb248588f6a7c2d87cf30379096f75d
7
- data.tar.gz: ced9235516d07cba8af6a045481c969bf8a2abe9b67b891b14904cc5e150ebb82f888d0c7f2d2c3ea55686c1cc3bff53ff29bace8d455f6abf84a6264366e87d
6
+ metadata.gz: cdfdc357fea1964db223fd2ee5df419c6c2bb8e17171915368b94a5518a35b8904c986f80175c328bf57080e019a370fd882f575bca5731538eeede2f2d25f1a
7
+ data.tar.gz: ae6ef51d1fc54c8e8c65b351ce15409ccab8d6285123640d54cfea7840e1bfb84e0209b9cd2dd6b41dde752203b4ff29b9541fcf87036bf2bf715aa371d0e75f
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- fluent-plugin-google-cloud (0.10.1)
4
+ fluent-plugin-google-cloud (0.10.6)
5
5
  fluentd (= 1.11.2)
6
6
  google-api-client (= 0.30.8)
7
7
  google-cloud-logging (= 1.6.6)
@@ -18,20 +18,20 @@ GEM
18
18
  specs:
19
19
  addressable (2.7.0)
20
20
  public_suffix (>= 2.0.2, < 5.0)
21
- ast (2.4.1)
22
- concurrent-ruby (1.1.7)
23
- cool.io (1.6.0)
21
+ ast (2.4.2)
22
+ concurrent-ruby (1.1.8)
23
+ cool.io (1.7.0)
24
24
  coveralls (0.8.23)
25
25
  json (>= 1.8, < 3)
26
26
  simplecov (~> 0.16.1)
27
27
  term-ansicolor (~> 1.3)
28
28
  thor (>= 0.19.4, < 2.0)
29
29
  tins (~> 1.6)
30
- crack (0.4.3)
31
- safe_yaml (~> 1.0.0)
30
+ crack (0.4.5)
31
+ rexml
32
32
  declarative (0.0.20)
33
33
  declarative-option (0.1.0)
34
- docile (1.3.2)
34
+ docile (1.3.5)
35
35
  faraday (0.17.3)
36
36
  multipart-post (>= 1.2, < 3)
37
37
  fluentd (1.11.2)
@@ -61,7 +61,7 @@ GEM
61
61
  google-cloud-core (1.5.0)
62
62
  google-cloud-env (~> 1.0)
63
63
  google-cloud-errors (~> 1.0)
64
- google-cloud-env (1.3.3)
64
+ google-cloud-env (1.4.0)
65
65
  faraday (>= 0.17.3, < 2.0)
66
66
  google-cloud-errors (1.0.1)
67
67
  google-cloud-logging (1.6.6)
@@ -80,10 +80,10 @@ GEM
80
80
  google-cloud-trace-v1 (~> 0.0)
81
81
  google-cloud-trace-v2 (~> 0.0)
82
82
  stackdriver-core (~> 1.3)
83
- google-cloud-trace-v1 (0.1.2)
83
+ google-cloud-trace-v1 (0.2.0)
84
84
  gapic-common (~> 0.3)
85
85
  google-cloud-errors (~> 1.0)
86
- google-cloud-trace-v2 (0.1.2)
86
+ google-cloud-trace-v2 (0.2.0)
87
87
  gapic-common (~> 0.3)
88
88
  google-cloud-errors (~> 1.0)
89
89
  google-gax (1.8.1)
@@ -119,7 +119,7 @@ GEM
119
119
  mini_mime (1.0.2)
120
120
  mocha (1.9.0)
121
121
  metaclass (~> 0.0.1)
122
- msgpack (1.3.3)
122
+ msgpack (1.4.2)
123
123
  multi_json (1.15.0)
124
124
  multipart-post (2.1.1)
125
125
  opencensus (0.5.0)
@@ -129,13 +129,13 @@ GEM
129
129
  google-cloud-trace (~> 0.35)
130
130
  opencensus (~> 0.5)
131
131
  os (1.1.1)
132
- parser (2.7.1.4)
132
+ parser (2.7.2.0)
133
133
  ast (~> 2.4.1)
134
- power_assert (1.2.0)
135
- powerpack (0.1.2)
134
+ power_assert (2.0.0)
135
+ powerpack (0.1.3)
136
136
  prometheus-client (0.9.0)
137
137
  quantile (~> 0.2.1)
138
- public_suffix (4.0.5)
138
+ public_suffix (4.0.6)
139
139
  quantile (0.2.1)
140
140
  rainbow (2.2.2)
141
141
  rake
@@ -145,6 +145,7 @@ GEM
145
145
  declarative-option (< 0.2.0)
146
146
  uber (< 0.2.0)
147
147
  retriable (3.1.2)
148
+ rexml (3.2.4)
148
149
  rly (0.2.3)
149
150
  rubocop (0.39.0)
150
151
  parser (>= 2.3.0.7, < 3.0)
@@ -152,12 +153,11 @@ GEM
152
153
  rainbow (>= 1.99.1, < 3.0)
153
154
  ruby-progressbar (~> 1.7)
154
155
  unicode-display_width (~> 1.0, >= 1.0.1)
155
- ruby-progressbar (1.10.1)
156
- safe_yaml (1.0.5)
157
- serverengine (2.2.1)
156
+ ruby-progressbar (1.11.0)
157
+ serverengine (2.2.2)
158
158
  sigdump (~> 0.2.2)
159
159
  sigdump (0.2.4)
160
- signet (0.14.0)
160
+ signet (0.14.1)
161
161
  addressable (~> 2.3)
162
162
  faraday (>= 0.17.3, < 2.0)
163
163
  jwt (>= 1.5, < 3.0)
@@ -169,18 +169,18 @@ GEM
169
169
  simplecov-html (0.10.2)
170
170
  stackdriver-core (1.4.0)
171
171
  google-cloud-core (~> 1.2)
172
- strptime (0.2.4)
172
+ strptime (0.2.5)
173
173
  sync (0.5.0)
174
174
  term-ansicolor (1.7.1)
175
175
  tins (~> 1.0)
176
176
  test-unit (3.3.3)
177
177
  power_assert
178
- thor (1.0.1)
179
- tins (1.25.0)
178
+ thor (1.1.0)
179
+ tins (1.28.0)
180
180
  sync
181
- tzinfo (2.0.2)
181
+ tzinfo (2.0.4)
182
182
  concurrent-ruby (~> 1.0)
183
- tzinfo-data (1.2020.1)
183
+ tzinfo-data (1.2021.1)
184
184
  tzinfo (>= 1.0.0)
185
185
  uber (0.1.0)
186
186
  unicode-display_width (1.7.0)
@@ -10,7 +10,7 @@ eos
10
10
  gem.homepage =
11
11
  'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
12
12
  gem.license = 'Apache-2.0'
13
- gem.version = '0.10.1'
13
+ gem.version = '0.10.6'
14
14
  gem.authors = ['Stackdriver Agents Team']
15
15
  gem.email = ['stackdriver-agents@google.com']
16
16
  gem.required_ruby_version = Gem::Requirement.new('>= 2.2')
@@ -15,6 +15,10 @@
15
15
  require 'fileutils'
16
16
  require 'fluent/config'
17
17
  require 'fluent/config/v1_parser'
18
+ require 'fluent/plugin_helper'
19
+ require 'googleauth'
20
+ require 'google/apis/logging_v2'
21
+ require 'open-uri'
18
22
  require 'set'
19
23
 
20
24
  require_relative 'common'
@@ -29,6 +33,10 @@ module Fluent
29
33
  include Fluent::Config
30
34
  Fluent::Plugin.register_filter('analyze_config', self)
31
35
 
36
+ # Required for the timer_execute method below
37
+ include PluginHelper::Mixin
38
+ helpers :timer
39
+
32
40
  module Constants
33
41
  # Built-in plugins that are ok to reference in metrics.
34
42
  KNOWN_PLUGINS = {
@@ -155,8 +163,8 @@ module Fluent
155
163
  super
156
164
  @log = $log # rubocop:disable Style/GlobalVars
157
165
 
158
- # Initialize the insertID.
159
- @log.info 'Started the analyze_config plugin to analyze configuration.'
166
+ @log.info(
167
+ 'analyze_config plugin: Started the plugin to analyze configuration.')
160
168
  end
161
169
 
162
170
  def parse_config(path)
@@ -205,10 +213,11 @@ module Fluent
205
213
 
206
214
  def configure(conf)
207
215
  super
216
+ @log.info('analyze_config plugin: Starting to configure the plugin.')
208
217
  if File.file?(@google_fluentd_config_path) &&
209
218
  File.file?(@google_fluentd_baseline_config_path)
210
219
  @log.info(
211
- 'google-fluentd configuration file found at' \
220
+ 'analyze_config plugin: google-fluentd configuration file found at' \
212
221
  " #{@google_fluentd_config_path}. " \
213
222
  'google-fluentd baseline configuration file found at' \
214
223
  " #{@google_fluentd_baseline_config_path}. " \
@@ -232,27 +241,53 @@ module Fluent
232
241
 
233
242
  unless Monitoring::MonitoringRegistryFactory.supports_monitoring_type(
234
243
  @monitoring_type)
235
- @log.warn "monitoring_type '#{@monitoring_type}' is unknown; "\
236
- 'there will be no metrics'
244
+ @log.warn(
245
+ "analyze_config plugin: monitoring_type #{@monitoring_type} is " \
246
+ 'unknown; there will be no metrics.')
237
247
  end
238
- registry = Monitoring::MonitoringRegistryFactory.create(
248
+
249
+ @registry = Monitoring::MonitoringRegistryFactory.create(
239
250
  @monitoring_type, project_id, resource, @gcm_service_address)
251
+ # Export metrics every 60 seconds.
252
+ timer_execute(:export_config_analysis_metrics, 60) { @registry.export }
240
253
 
241
- plugin_usage = registry.counter(
242
- :stackdriver_enabled_plugins,
243
- [:plugin_name, :is_default_plugin, :has_default_value],
244
- 'Enabled plugins')
245
- config_usage = registry.counter(
246
- :stackdriver_config_usage,
247
- [:plugin_name, :param, :is_present, :has_default_value],
248
- 'Parameter usage for Google Cloud plugins')
249
- config_bool_values = registry.counter(
250
- :stackdriver_config_bool_values,
254
+ @log.info('analyze_config plugin: Registering counters.')
255
+ enabled_plugins_counter = @registry.counter(
256
+ :enabled_plugins,
257
+ [:plugin_name, :is_default_plugin,
258
+ :has_default_config, :has_ruby_snippet],
259
+ 'Enabled plugins',
260
+ 'agent.googleapis.com/agent/internal/logging/config',
261
+ 'GAUGE')
262
+ @log.info(
263
+ 'analyze_config plugin: registered enable_plugins counter. ' \
264
+ "#{enabled_plugins_counter}")
265
+ plugin_config_counter = @registry.counter(
266
+ :plugin_config,
267
+ [:plugin_name, :param, :is_present, :has_default_config],
268
+ 'Configuration parameter usage for plugins relevant to Google Cloud.',
269
+ 'agent.googleapis.com/agent/internal/logging/config',
270
+ 'GAUGE')
271
+ @log.info('analyze_config plugin: registered plugin_config counter. ' \
272
+ "#{plugin_config_counter}")
273
+ config_bool_values_counter = @registry.counter(
274
+ :config_bool_values,
251
275
  [:plugin_name, :param, :value],
252
- 'Values for bool parameters in Google Cloud plugins')
276
+ 'Values for bool parameters in Google Cloud plugins',
277
+ 'agent.googleapis.com/agent/internal/logging/config',
278
+ 'GAUGE')
279
+ @log.info('analyze_config plugin: registered config_bool_values ' \
280
+ "counter. #{config_bool_values_counter}")
253
281
 
254
282
  config = parse_config(@google_fluentd_config_path)
283
+ @log.debug(
284
+ 'analyze_config plugin: successfully parsed google-fluentd' \
285
+ " configuration file at #{@google_fluentd_config_path}. #{config}")
255
286
  baseline_config = parse_config(@google_fluentd_baseline_config_path)
287
+ @log.debug(
288
+ 'analyze_config plugin: successfully parsed google-fluentd' \
289
+ ' baseline configuration file at' \
290
+ " #{@google_fluentd_baseline_config_path}: #{baseline_config}")
256
291
 
257
292
  # Create hash of all baseline elements by their plugin names.
258
293
  baseline_elements = Hash[baseline_config.elements.collect do |e|
@@ -276,17 +311,17 @@ module Fluent
276
311
  plugin_name = default_plugin_name(e)
277
312
  if baseline_elements.key?(plugin_name)
278
313
  is_default_plugin = true
279
- has_default_value = (baseline_elements[plugin_name] == e)
314
+ has_default_config = (baseline_elements[plugin_name] == e)
280
315
  else
281
316
  plugin_name = custom_plugin_name(e)
282
317
  is_default_plugin = false
283
- has_default_value = false
318
+ has_default_config = false
284
319
  end
285
- plugin_usage.increment(
320
+ enabled_plugins_counter.increment(
286
321
  labels: {
287
322
  plugin_name: plugin_name,
288
323
  is_default_plugin: is_default_plugin,
289
- has_default_value: has_default_value,
324
+ has_default_config: has_default_config,
290
325
  has_ruby_snippet: embedded_ruby?(e)
291
326
  },
292
327
  by: 1)
@@ -295,18 +330,18 @@ module Fluent
295
330
  # detect_exceptions).
296
331
  next unless GOOGLE_PLUGIN_PARAMS.key?(e['@type'])
297
332
  GOOGLE_PLUGIN_PARAMS[e['@type']].each do |p|
298
- config_usage.increment(
333
+ plugin_config_counter.increment(
299
334
  labels: {
300
335
  plugin_name: e['@type'],
301
336
  param: p,
302
337
  is_present: e.key?(p),
303
- has_default_value: (e.key?(p) &&
338
+ has_default_config: (e.key?(p) &&
304
339
  baseline_google_element.key?(p) &&
305
340
  e[p] == baseline_google_element[p])
306
341
  },
307
342
  by: 1)
308
343
  next unless e.key?(p) && %w(true false).include?(e[p])
309
- config_bool_values.increment(
344
+ config_bool_values_counter.increment(
310
345
  labels: {
311
346
  plugin_name: e['@type'],
312
347
  param: p,
@@ -315,23 +350,29 @@ module Fluent
315
350
  by: 1)
316
351
  end
317
352
  end
353
+ @log.info(
354
+ 'analyze_config plugin: Successfully finished analyzing config.')
318
355
  else
319
356
  @log.info(
320
- 'google-fluentd configuration file does not exist at' \
321
- " #{@google_fluentd_config_path} or " \
322
- 'google-fluentd baseline configuration file does not exist at' \
323
- " #{@google_fluentd_baseline_config_path} or " \
324
- '. Skipping configuration analysis.')
357
+ 'analyze_config plugin: google-fluentd configuration file does not ' \
358
+ "exist at #{@google_fluentd_config_path} or google-fluentd " \
359
+ 'baseline configuration file does not exist at' \
360
+ " #{@google_fluentd_baseline_config_path}. Skipping configuration " \
361
+ 'analysis.')
325
362
  end
326
363
  rescue => e
327
364
  # Do not crash the agent due to configuration analysis failures.
328
365
  @log.warn(
329
- 'Failed to optionally analyze the google-fluentd configuration' \
330
- " file. Proceeding anyway. Error: #{e}")
366
+ 'analyze_config plugin: Failed to optionally analyze the ' \
367
+ "google-fluentd configuration file. Proceeding anyway. Error: #{e}. " \
368
+ "Trace: #{e.backtrace}")
331
369
  end
332
370
 
333
371
  def shutdown
334
372
  super
373
+ # Export metrics on shutdown. This is a best-effort attempt, and it might
374
+ # fail, for instance if there was a recent write to the same time series.
375
+ @registry.export unless @registry.nil?
335
376
  end
336
377
 
337
378
  # rubocop:disable Lint/UnusedMethodArgument
@@ -52,7 +52,7 @@ module Monitoring
52
52
  def initialize(_project_id, _monitored_resource, _gcm_service_address)
53
53
  end
54
54
 
55
- def counter(_name, _labels, _docstring)
55
+ def counter(_name, _labels, _docstring, _prefix, _aggregation)
56
56
  BaseCounter.new
57
57
  end
58
58
 
@@ -75,7 +75,7 @@ module Monitoring
75
75
  end
76
76
 
77
77
  # Exception-driven behavior to avoid synchronization errors.
78
- def counter(name, _labels, docstring)
78
+ def counter(name, _labels, docstring, _prefix, _aggregation)
79
79
  # When we upgrade to Prometheus client 0.10.0 or higher, pass the
80
80
  # labels in the metric constructor. The 'labels' field in
81
81
  # Prometheus client 0.9.0 has a different function and will not
@@ -97,42 +97,85 @@ module Monitoring
97
97
  require 'opencensus'
98
98
  require 'opencensus-stackdriver'
99
99
  @log = $log # rubocop:disable Style/GlobalVars
100
- @recorder = OpenCensus::Stats.ensure_recorder
101
- @exporter = OpenCensus::Stats::Exporters::Stackdriver.new(
102
- project_id: project_id,
103
- metric_prefix: 'agent.googleapis.com/agent',
104
- resource_type: monitored_resource.type,
105
- resource_labels: monitored_resource.labels,
106
- gcm_service_address: gcm_service_address
107
- )
108
- OpenCensus.configure do |c|
109
- c.stats.exporter = @exporter
110
- end
111
- @log.debug "OpenCensus config=#{OpenCensus.config}"
100
+ @project_id = project_id
101
+ @metrics_monitored_resource = monitored_resource
102
+ @gcm_service_address = gcm_service_address
103
+ @recorders = {}
104
+ @exporters = {}
105
+ @log.info(
106
+ 'monitoring module: Successfully initialized Open Census monitoring ' \
107
+ 'registry.')
112
108
  end
113
109
 
114
- def counter(name, labels, docstring)
110
+ def counter(name, labels, docstring, prefix, aggregation)
115
111
  translator = MetricTranslator.new(name, labels)
116
112
  measure = OpenCensus::Stats::MeasureRegistry.get(translator.name)
117
113
  if measure.nil?
114
+ @log.info(
115
+ 'monitoring module: Registering a new measure registry for ' \
116
+ "#{translator.name}")
118
117
  measure = OpenCensus::Stats.create_measure_int(
119
118
  name: translator.name,
120
119
  unit: OpenCensus::Stats::Measure::UNIT_NONE,
121
120
  description: docstring
122
121
  )
123
122
  end
124
- OpenCensus::Stats.create_and_register_view(
125
- name: translator.name,
126
- measure: measure,
127
- aggregation: OpenCensus::Stats.create_sum_aggregation,
128
- description: docstring,
129
- columns: translator.view_labels.map(&:to_s)
123
+ unless @exporters.keys.include?(prefix)
124
+ @log.info(
125
+ 'monitoring module: Registering a new exporter for ' \
126
+ "#{prefix}")
127
+ @recorders[prefix] = OpenCensus::Stats::Recorder.new
128
+ @exporters[prefix] = \
129
+ OpenCensus::Stats::Exporters::Stackdriver.new(
130
+ project_id: @project_id,
131
+ metric_prefix: prefix,
132
+ resource_type: @metrics_monitored_resource.type,
133
+ resource_labels: @metrics_monitored_resource.labels,
134
+ gcm_service_address: @gcm_service_address
135
+ )
136
+ @log.info(
137
+ 'monitoring module: Registered recorders and exporters for ' \
138
+ "#{prefix}.\n#{@exporters[prefix]}")
139
+ end
140
+ if aggregation == 'GAUGE'
141
+ stats_aggregation = OpenCensus::Stats.create_last_value_aggregation
142
+ else
143
+ stats_aggregation = OpenCensus::Stats.create_sum_aggregation
144
+ end
145
+ @recorders[prefix].register_view(
146
+ OpenCensus::Stats::View.new(
147
+ name: translator.name,
148
+ measure: measure,
149
+ aggregation: stats_aggregation,
150
+ description: docstring,
151
+ columns: translator.view_labels.map(&:to_s)
152
+ )
130
153
  )
131
- OpenCensusCounter.new(@recorder, measure, translator)
154
+ counter = OpenCensusCounter.new(@recorders[prefix], measure, translator)
155
+ @log.info(
156
+ 'monitoring module: Successfully initialized Open Census counter for ' \
157
+ "#{prefix}/#{name}.")
158
+ counter
159
+ rescue StandardError => e
160
+ @log.warn "Failed to count metrics for #{name}.", error: e
161
+ raise e
132
162
  end
133
163
 
134
164
  def export
135
- @exporter.export @recorder.views_data
165
+ @log.debug(
166
+ "monitoring module: Exporting metrics for #{@exporters.keys}.")
167
+ @exporters.keys.each do |prefix|
168
+ @log.debug(
169
+ "monitoring module: Exporting metrics for #{prefix}. " \
170
+ "#{@recorders[prefix].views_data}")
171
+ @exporters[prefix].export @recorders[prefix].views_data
172
+ end
173
+ rescue StandardError => e
174
+ # TODO(lingshi): Fix the error handling here. Seems like the export is
175
+ # done asynchronously. So any failure happens silently. More details at
176
+ # https://github.com/census-ecosystem/opencensus-ruby-exporter-stackdriver/blob/f8de506204972548ca535eff6010d15f328df6c3/lib/opencensus/stats/exporters/stackdriver.rb#L156
177
+ @log.warn 'Failed to export some metrics.', error: e
178
+ raise e
136
179
  end
137
180
  end
138
181