fluent-plugin-google-cloud 0.10.2 → 0.10.7

Sign up to get free protection for your applications and to get access to all the features.
@@ -563,32 +563,38 @@ module Fluent
563
563
  # Uptime should be a gauge, but the metric definition is a counter and
564
564
  # we can't change it.
565
565
  @uptime_metric = @registry.counter(
566
- :uptime, [:version], 'Uptime of Logging agent')
566
+ :uptime, [:version], 'Uptime of Logging agent',
567
+ 'agent.googleapis.com/agent', 'CUMULATIVE')
567
568
  update_uptime
568
569
  timer_execute(:update_uptime, 1) { update_uptime }
569
570
  @successful_requests_count = @registry.counter(
570
571
  :stackdriver_successful_requests_count,
571
572
  [:grpc, :code],
572
- 'A number of successful requests to the Stackdriver Logging API')
573
+ 'A number of successful requests to the Stackdriver Logging API',
574
+ 'agent.googleapis.com/agent', 'CUMULATIVE')
573
575
  @failed_requests_count = @registry.counter(
574
576
  :stackdriver_failed_requests_count,
575
577
  [:grpc, :code],
576
578
  'A number of failed requests to the Stackdriver Logging '\
577
- 'API, broken down by the error code')
579
+ 'API, broken down by the error code',
580
+ 'agent.googleapis.com/agent', 'CUMULATIVE')
578
581
  @ingested_entries_count = @registry.counter(
579
582
  :stackdriver_ingested_entries_count,
580
583
  [:grpc, :code],
581
- 'A number of log entries ingested by Stackdriver Logging')
584
+ 'A number of log entries ingested by Stackdriver Logging',
585
+ 'agent.googleapis.com/agent', 'CUMULATIVE')
582
586
  @dropped_entries_count = @registry.counter(
583
587
  :stackdriver_dropped_entries_count,
584
588
  [:grpc, :code],
585
- 'A number of log entries dropped by the Stackdriver output plugin')
589
+ 'A number of log entries dropped by the Stackdriver output plugin',
590
+ 'agent.googleapis.com/agent', 'CUMULATIVE')
586
591
  @retried_entries_count = @registry.counter(
587
592
  :stackdriver_retried_entries_count,
588
593
  [:grpc, :code],
589
594
  'The number of log entries that failed to be ingested by '\
590
595
  'the Stackdriver output plugin due to a transient error '\
591
- 'and were retried')
596
+ 'and were retried',
597
+ 'agent.googleapis.com/agent', 'CUMULATIVE')
592
598
  @ok_code = @use_grpc ? GRPC::Core::StatusCodes::OK : 200
593
599
  end
594
600
 
@@ -37,7 +37,10 @@ module Asserts
37
37
  assert_in_delta expected_ts.tv_nsec, ts_nanos, 600, entry
38
38
  end
39
39
 
40
- def assert_prometheus_metric_value(metric_name, expected_value, labels = {})
40
+ # rubocop:disable Metrics/ParameterLists
41
+ def assert_prometheus_metric_value(metric_name, expected_value, _prefix,
42
+ _aggregation, _test_driver, labels = {})
43
+ # rubocop:enable Metrics/ParameterLists
41
44
  metric = Prometheus::Client.registry.get(metric_name)
42
45
  assert_not_nil(metric)
43
46
  metric_value = if labels == :aggregate
@@ -49,7 +52,10 @@ module Asserts
49
52
  assert_equal(expected_value, metric_value)
50
53
  end
51
54
 
52
- def assert_opencensus_metric_value(metric_name, expected_value, labels = {})
55
+ # rubocop:disable Metrics/ParameterLists
56
+ def assert_opencensus_metric_value(metric_name, expected_value, prefix,
57
+ aggregation, test_driver, labels = {})
58
+ # rubocop:enable Metrics/ParameterLists
53
59
  translator = Monitoring::MetricTranslator.new(metric_name, labels)
54
60
  metric_name = translator.name
55
61
  labels = translator.translate_labels(labels)
@@ -60,18 +66,22 @@ module Asserts
60
66
  # label that never changes during runtime.
61
67
  labels.select! { |k, _| translator.view_labels.include? k }
62
68
  labels = labels.map { |k, v| [k.to_s, v.to_s] }.to_h
63
- stats_recorder = OpenCensus::Stats.ensure_recorder
64
- view_data = stats_recorder.view_data metric_name
65
- assert_not_nil(view_data)
69
+
70
+ registry = test_driver.instance.instance_variable_get(:@registry)
71
+ recorder = registry.instance_variable_get(:@recorders)[prefix]
72
+ view_data = recorder.measure_views_data[metric_name][0].data
73
+ view = recorder.instance_variable_get(:@views)[metric_name]
74
+
75
+ # Assert values in the view.
76
+ assert_kind_of(aggregation, view.aggregation)
77
+ assert_equal(labels.keys, view.columns)
78
+ assert_equal(metric_name, view.measure.name)
79
+ assert_equal('INT64', view.measure.type)
80
+
66
81
  # For now assume all metrics are counters.
67
- assert_kind_of(OpenCensus::Stats::Aggregation::Sum,
68
- view_data.view.aggregation)
69
- assert_true(view_data.view.measure.int64?)
70
- tag_values = view_data.view.columns.map { |column| labels[column] }
82
+ tag_values = view.columns.map { |column| labels[column] }
71
83
  metric_value = 0
72
- if view_data.data.key? tag_values
73
- metric_value = view_data.data[tag_values].value
74
- end
84
+ metric_value = view_data[tag_values].value if view_data.key? tag_values
75
85
  assert_equal(expected_value, metric_value)
76
86
  end
77
87
  end
@@ -169,10 +169,11 @@ module BaseTest
169
169
  assert_true d.instance.instance_variable_get(:@enable_monitoring)
170
170
  registry = d.instance.instance_variable_get(:@registry)
171
171
  assert_not_nil registry
172
- exporter = registry.instance_variable_get(:@exporter)
173
- assert_equal 'custom_resource', exporter.resource_type, "Index #{index}"
172
+ monitored_resource = registry.instance_variable_get(
173
+ :@metrics_monitored_resource)
174
+ assert_equal('custom_resource', monitored_resource.type, "Index #{index}")
174
175
  assert_equal({ 'label1' => '123', 'label2' => 'abc' },
175
- exporter.resource_labels, "Index #{index}")
176
+ monitored_resource.labels, "Index #{index}")
176
177
  end
177
178
  end
178
179
 
@@ -942,22 +943,30 @@ module BaseTest
942
943
  d.emit("tag#{i}", 'message' => log_entry(i))
943
944
  end
944
945
  d.run
946
+ @logs_sent.zip(request_log_names).each do |request, log_name|
947
+ assert_equal log_name, request['logName']
948
+ end
949
+ verify_log_entries(log_entry_count, COMPUTE_PARAMS_NO_LOG_NAME,
950
+ 'textPayload') do |entry, entry_index|
951
+ verify_default_log_entry_text(entry['textPayload'], entry_index,
952
+ entry)
953
+ assert_equal entry_log_names[entry_index], entry['logName']
954
+ end
955
+ # Verify the number of requests is different based on whether the
956
+ # 'split_logs_by_tag' flag is enabled.
957
+ assert_prometheus_metric_value(
958
+ :stackdriver_successful_requests_count,
959
+ request_count,
960
+ 'agent.googleapis.com/agent',
961
+ OpenCensus::Stats::Aggregation::Sum, d,
962
+ :aggregate)
963
+ assert_prometheus_metric_value(
964
+ :stackdriver_ingested_entries_count,
965
+ log_entry_count,
966
+ 'agent.googleapis.com/agent',
967
+ OpenCensus::Stats::Aggregation::Sum, d,
968
+ :aggregate)
945
969
  end
946
- @logs_sent.zip(request_log_names).each do |request, log_name|
947
- assert_equal log_name, request['logName']
948
- end
949
- verify_log_entries(log_entry_count, COMPUTE_PARAMS_NO_LOG_NAME,
950
- 'textPayload') do |entry, entry_index|
951
- verify_default_log_entry_text(entry['textPayload'], entry_index,
952
- entry)
953
- assert_equal entry_log_names[entry_index], entry['logName']
954
- end
955
- # Verify the number of requests is different based on whether the
956
- # 'split_logs_by_tag' flag is enabled.
957
- assert_prometheus_metric_value(:stackdriver_successful_requests_count,
958
- request_count, :aggregate)
959
- assert_prometheus_metric_value(:stackdriver_ingested_entries_count,
960
- log_entry_count, :aggregate)
961
970
  end
962
971
  end
963
972
 
@@ -1967,7 +1976,9 @@ module BaseTest
1967
1976
  expected = Time.now.to_i - start_time
1968
1977
  d.instance.update_uptime
1969
1978
  assert_metric_value.call(
1970
- :uptime, expected, version: Fluent::GoogleCloudOutput.version_string)
1979
+ :uptime, expected, 'agent.googleapis.com/agent',
1980
+ OpenCensus::Stats::Aggregation::Sum, d,
1981
+ version: Fluent::GoogleCloudOutput.version_string)
1971
1982
  rescue Test::Unit::AssertionFailedError
1972
1983
  retry if (retries += 1) < 3
1973
1984
  end
@@ -1975,6 +1986,102 @@ module BaseTest
1975
1986
  end
1976
1987
  end
1977
1988
 
1989
+ def test_metrics
1990
+ setup_gce_metadata_stubs
1991
+ [
1992
+ [ENABLE_PROMETHEUS_CONFIG, method(:assert_prometheus_metric_value)],
1993
+ [ENABLE_OPENCENSUS_CONFIG, method(:assert_opencensus_metric_value)]
1994
+ ].each do |config, assert_metric_value|
1995
+ [
1996
+ # Single successful request.
1997
+ [ok_status_code, 1, 1, [0, 0, 0]],
1998
+ # Several successful requests.
1999
+ [ok_status_code, 2, 1, [0, 0, 0]],
2000
+ # Single successful request with several entries.
2001
+ [ok_status_code, 1, 2, [0, 0, 0]],
2002
+ # Single failed request that causes logs to be dropped.
2003
+ [client_error_status_code, 1, 1, [1, 1, 0]],
2004
+ # Single failed request that escalates without logs being dropped with
2005
+ # several entries.
2006
+ [server_error_status_code, 1, 2, [0, 0, 2]]
2007
+ ].each do |code, request_count, entry_count, metric_values|
2008
+ clear_metrics
2009
+ setup_logging_stubs(nil, code, 'SomeMessage') do
2010
+ (1..request_count).each do |request_index|
2011
+ d = create_driver(config)
2012
+ (1..entry_count).each do |entry_index|
2013
+ d.emit('message' => log_entry(entry_index.to_s))
2014
+ end
2015
+ # rubocop:disable Lint/HandleExceptions
2016
+ begin
2017
+ d.run
2018
+ rescue mock_error_type
2019
+ end
2020
+ # rubocop:enable Lint/HandleExceptions
2021
+ failed_requests_count, dropped_entries_count,
2022
+ retried_entries_count = metric_values
2023
+
2024
+ successful_requests_count = \
2025
+ if code != ok_status_code
2026
+ 0
2027
+ elsif config == ENABLE_OPENCENSUS_CONFIG
2028
+ # TODO(b/173215689) Improve the Open Census side of testing.
2029
+ # The test driver instance variables can not survive between
2030
+ # test driver runs. So the open cencensus side counter gets
2031
+ # reset as expected.
2032
+ 1
2033
+ else
2034
+ request_index
2035
+ end
2036
+
2037
+ ingested_entries_count = \
2038
+ if code != ok_status_code
2039
+ 0
2040
+ elsif config == ENABLE_OPENCENSUS_CONFIG
2041
+ # TODO(b/173215689) Improve the Open Census side of testing.
2042
+ # The test driver instance variables can not survive between
2043
+ # test driver runs. So the open cencensus side counter gets
2044
+ # reset as expected.
2045
+ entry_count
2046
+ else
2047
+ request_index * entry_count
2048
+ end
2049
+
2050
+ assert_metric_value.call(:stackdriver_successful_requests_count,
2051
+ successful_requests_count,
2052
+ 'agent.googleapis.com/agent',
2053
+ OpenCensus::Stats::Aggregation::Sum, d,
2054
+ grpc: use_grpc, code: ok_status_code)
2055
+ assert_metric_value.call(:stackdriver_ingested_entries_count,
2056
+ ingested_entries_count,
2057
+ 'agent.googleapis.com/agent',
2058
+ OpenCensus::Stats::Aggregation::Sum, d,
2059
+ grpc: use_grpc, code: ok_status_code)
2060
+ assert_metric_value.call(:stackdriver_retried_entries_count,
2061
+ retried_entries_count,
2062
+ 'agent.googleapis.com/agent',
2063
+ OpenCensus::Stats::Aggregation::Sum, d,
2064
+ grpc: use_grpc, code: code)
2065
+ # Skip failure assertions when code indicates success, because the
2066
+ # assertion will fail in the case when a single metric contains time
2067
+ # series with success and failure events.
2068
+ next if code == ok_status_code
2069
+ assert_metric_value.call(:stackdriver_failed_requests_count,
2070
+ failed_requests_count,
2071
+ 'agent.googleapis.com/agent',
2072
+ OpenCensus::Stats::Aggregation::Sum, d,
2073
+ grpc: use_grpc, code: code)
2074
+ assert_metric_value.call(:stackdriver_dropped_entries_count,
2075
+ dropped_entries_count,
2076
+ 'agent.googleapis.com/agent',
2077
+ OpenCensus::Stats::Aggregation::Sum, d,
2078
+ grpc: use_grpc, code: code)
2079
+ end
2080
+ end
2081
+ end
2082
+ end
2083
+ end
2084
+
1978
2085
  private
1979
2086
 
1980
2087
  # Provide a stub context that initializes @logs_sent, executes the block and
@@ -2455,7 +2562,32 @@ module BaseTest
2455
2562
  end
2456
2563
 
2457
2564
  # Set up http or grpc stubs to mock the external calls.
2458
- def setup_logging_stubs
2565
+ def setup_logging_stubs(_error = nil, _code = nil, _message = nil)
2566
+ _undefined
2567
+ end
2568
+
2569
+ # Whether this is the grpc path
2570
+ def use_grpc
2571
+ _undefined
2572
+ end
2573
+
2574
+ # The OK status code.
2575
+ def ok_status_code
2576
+ _undefined
2577
+ end
2578
+
2579
+ # A client side error status code.
2580
+ def client_error_status_code
2581
+ _undefined
2582
+ end
2583
+
2584
+ # A server side error status code.
2585
+ def server_error_status_code
2586
+ _undefined
2587
+ end
2588
+
2589
+ # The parent error type to expect in the mock
2590
+ def mock_error_type
2459
2591
  _undefined
2460
2592
  end
2461
2593
 
@@ -51,85 +51,115 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
51
51
  method(:assert_opencensus_metric_value)]
52
52
  ].each do |config, assert_metric_value|
53
53
  clear_metrics
54
- create_driver(config)
54
+ d = create_driver(config)
55
55
 
56
56
  # Default plugins, with default config.
57
57
  assert_metric_value.call(
58
- :stackdriver_enabled_plugins,
58
+ :enabled_plugins,
59
59
  1,
60
+ 'agent.googleapis.com/agent/internal/logging/config',
61
+ OpenCensus::Stats::Aggregation::LastValue,
62
+ d,
60
63
  plugin_name: 'source/syslog/tcp',
61
64
  is_default_plugin: true,
62
- has_default_value: true,
65
+ has_default_config: true,
63
66
  has_ruby_snippet: false)
64
67
  assert_metric_value.call(
65
- :stackdriver_enabled_plugins,
68
+ :enabled_plugins,
66
69
  1,
70
+ 'agent.googleapis.com/agent/internal/logging/config',
71
+ OpenCensus::Stats::Aggregation::LastValue,
72
+ d,
67
73
  plugin_name: 'source/tail/apache-access',
68
74
  is_default_plugin: true,
69
- has_default_value: true,
75
+ has_default_config: true,
70
76
  has_ruby_snippet: false)
71
77
  assert_metric_value.call(
72
- :stackdriver_enabled_plugins,
78
+ :enabled_plugins,
73
79
  1,
80
+ 'agent.googleapis.com/agent/internal/logging/config',
81
+ OpenCensus::Stats::Aggregation::LastValue,
82
+ d,
74
83
  plugin_name: 'filter/add_insert_ids',
75
84
  is_default_plugin: true,
76
- has_default_value: true,
85
+ has_default_config: true,
77
86
  has_ruby_snippet: false)
78
87
 
79
88
  # Default plugins, with custom config.
80
89
  assert_metric_value.call(
81
- :stackdriver_enabled_plugins,
90
+ :enabled_plugins,
82
91
  1,
92
+ 'agent.googleapis.com/agent/internal/logging/config',
93
+ OpenCensus::Stats::Aggregation::LastValue,
94
+ d,
83
95
  plugin_name: 'match/google_cloud',
84
96
  is_default_plugin: true,
85
- has_default_value: false,
97
+ has_default_config: false,
86
98
  has_ruby_snippet: false)
87
99
 
88
100
  # Custom plugins, some with embedded Ruby.
89
101
  assert_metric_value.call(
90
- :stackdriver_enabled_plugins,
102
+ :enabled_plugins,
91
103
  1,
104
+ 'agent.googleapis.com/agent/internal/logging/config',
105
+ OpenCensus::Stats::Aggregation::LastValue,
106
+ d,
92
107
  plugin_name: 'filter',
93
108
  is_default_plugin: false,
94
- has_default_value: false,
109
+ has_default_config: false,
95
110
  has_ruby_snippet: false)
96
111
  assert_metric_value.call(
97
- :stackdriver_enabled_plugins,
112
+ :enabled_plugins,
98
113
  1,
114
+ 'agent.googleapis.com/agent/internal/logging/config',
115
+ OpenCensus::Stats::Aggregation::LastValue,
116
+ d,
99
117
  plugin_name: 'filter/record_transformer',
100
118
  is_default_plugin: false,
101
- has_default_value: false,
119
+ has_default_config: false,
102
120
  has_ruby_snippet: true)
103
121
  assert_metric_value.call(
104
- :stackdriver_enabled_plugins,
122
+ :enabled_plugins,
105
123
  1,
124
+ 'agent.googleapis.com/agent/internal/logging/config',
125
+ OpenCensus::Stats::Aggregation::LastValue,
126
+ d,
106
127
  plugin_name: 'match/stdout',
107
128
  is_default_plugin: false,
108
- has_default_value: false,
129
+ has_default_config: false,
109
130
  has_ruby_snippet: true)
110
131
 
111
132
  # For out_google_cloud, 3 params are present.
112
133
  assert_metric_value.call(
113
- :stackdriver_config_usage,
134
+ :plugin_config,
114
135
  1,
136
+ 'agent.googleapis.com/agent/internal/logging/config',
137
+ OpenCensus::Stats::Aggregation::LastValue,
138
+ d,
115
139
  plugin_name: 'google_cloud',
116
140
  param: 'adjust_invalid_timestamps',
117
141
  is_present: true,
118
- has_default_value: true)
142
+ has_default_config: true)
119
143
  assert_metric_value.call(
120
- :stackdriver_config_usage,
144
+ :plugin_config,
121
145
  1,
146
+ 'agent.googleapis.com/agent/internal/logging/config',
147
+ OpenCensus::Stats::Aggregation::LastValue,
148
+ d,
122
149
  plugin_name: 'google_cloud',
123
150
  param: 'autoformat_stackdriver_trace',
124
151
  is_present: true,
125
- has_default_value: false)
152
+ has_default_config: false)
126
153
  assert_metric_value.call(
127
- :stackdriver_config_usage,
154
+ :plugin_config,
128
155
  1,
156
+ 'agent.googleapis.com/agent/internal/logging/config',
157
+ OpenCensus::Stats::Aggregation::LastValue,
158
+ d,
129
159
  plugin_name: 'google_cloud',
130
160
  param: 'coerce_to_utf8',
131
161
  is_present: true,
132
- has_default_value: false)
162
+ has_default_config: false)
133
163
  # The remaining "google_cloud" params are not present.
134
164
  # The are no params for "detect_exceptions".
135
165
  %w(
@@ -163,30 +193,42 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
163
193
  zone
164
194
  ).each do |p|
165
195
  assert_metric_value.call(
166
- :stackdriver_config_usage,
196
+ :plugin_config,
167
197
  1,
198
+ 'agent.googleapis.com/agent/internal/logging/config',
199
+ OpenCensus::Stats::Aggregation::LastValue,
200
+ d,
168
201
  plugin_name: 'google_cloud',
169
202
  param: p,
170
203
  is_present: false,
171
- has_default_value: false)
204
+ has_default_config: false)
172
205
  end
173
206
 
174
207
  # We also export values for the bools.
175
208
  assert_metric_value.call(
176
- :stackdriver_config_bool_values,
209
+ :config_bool_values,
177
210
  1,
211
+ 'agent.googleapis.com/agent/internal/logging/config',
212
+ OpenCensus::Stats::Aggregation::LastValue,
213
+ d,
178
214
  plugin_name: 'google_cloud',
179
215
  param: 'adjust_invalid_timestamps',
180
216
  value: true)
181
217
  assert_metric_value.call(
182
- :stackdriver_config_bool_values,
218
+ :config_bool_values,
183
219
  1,
220
+ 'agent.googleapis.com/agent/internal/logging/config',
221
+ OpenCensus::Stats::Aggregation::LastValue,
222
+ d,
184
223
  plugin_name: 'google_cloud',
185
224
  param: 'autoformat_stackdriver_trace',
186
225
  value: false)
187
226
  assert_metric_value.call(
188
- :stackdriver_config_bool_values,
227
+ :config_bool_values,
189
228
  1,
229
+ 'agent.googleapis.com/agent/internal/logging/config',
230
+ OpenCensus::Stats::Aggregation::LastValue,
231
+ d,
190
232
  plugin_name: 'google_cloud',
191
233
  param: 'coerce_to_utf8',
192
234
  value: true)