fluent-plugin-vadimberezniker-gcp 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,265 @@
1
+ # Copyright 2017 Google Inc. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ module Monitoring
16
+ # Base class for the counter.
17
+ class BaseCounter
18
+ def increment(by: 1, labels: {})
19
+ # No default behavior
20
+ end
21
+ end
22
+
23
+ # Prometheus implementation of counters.
24
+ class PrometheusCounter < BaseCounter
25
+ def initialize(prometheus_counter)
26
+ super()
27
+ @counter = prometheus_counter
28
+ end
29
+
30
+ def increment(by: 1, labels: {})
31
+ @counter.increment(labels, by)
32
+ end
33
+ end
34
+
35
+ # OpenCensus implementation of counters.
36
+ class OpenCensusCounter < BaseCounter
37
+ def initialize(recorder, measure, translator)
38
+ super()
39
+ raise ArgumentError, 'measure must not be nil' if measure.nil?
40
+
41
+ @recorder = recorder
42
+ @measure = measure
43
+ @translator = translator
44
+ end
45
+
46
+ def increment(by: 1, labels: {})
47
+ labels = @translator.translate_labels(labels)
48
+ tag_map = OpenCensus::Tags::TagMap.new(
49
+ labels.map { |k, v| [k.to_s, v.to_s] }.to_h
50
+ )
51
+ @recorder.record(@measure.create_measurement(value: by, tags: tag_map))
52
+ end
53
+ end
54
+
55
+ # Base class for the monitoring registry.
56
+ class BaseMonitoringRegistry
57
+ def initialize(_project_id, _monitored_resource, _gcm_service_address)
58
+ # no default behavior
59
+ end
60
+
61
+ def counter(_name, _labels, _docstring, _prefix, _aggregation)
62
+ BaseCounter.new
63
+ end
64
+
65
+ def export
66
+ nil
67
+ end
68
+ end
69
+
70
+ # Prometheus implementation of the monitoring registry, that uses the default
71
+ # registry in the official Prometheus client library.
72
+ class PrometheusMonitoringRegistry < BaseMonitoringRegistry
73
+ def self.name
74
+ 'prometheus'
75
+ end
76
+
77
+ def initialize(_project_id, _monitored_resource, _gcm_service_address)
78
+ super
79
+ require 'prometheus/client'
80
+ @registry = Prometheus::Client.registry
81
+ end
82
+
83
+ # Exception-driven behavior to avoid synchronization errors.
84
+ def counter(name, _labels, docstring, _prefix, _aggregation)
85
+ # When we upgrade to Prometheus client 0.10.0 or higher, pass the
86
+ # labels in the metric constructor. The 'labels' field in
87
+ # Prometheus client 0.9.0 has a different function and will not
88
+ # work as intended.
89
+ PrometheusCounter.new(@registry.counter(name, docstring))
90
+ rescue Prometheus::Client::Registry::AlreadyRegisteredError
91
+ PrometheusCounter.new(@registry.get(name))
92
+ end
93
+ end
94
+
95
+ # OpenCensus implementation of the monitoring registry.
96
+ class OpenCensusMonitoringRegistry < BaseMonitoringRegistry
97
+ def self.name
98
+ 'opencensus'
99
+ end
100
+
101
+ def initialize(project_id, monitored_resource, gcm_service_address)
102
+ super
103
+ require 'opencensus'
104
+ require 'opencensus-stackdriver'
105
+ @log = $log # rubocop:disable Style/GlobalVars
106
+ @project_id = project_id
107
+ @metrics_monitored_resource = monitored_resource
108
+ @gcm_service_address = gcm_service_address
109
+ @recorders = {}
110
+ @exporters = {}
111
+ @log.info(
112
+ 'monitoring module: Successfully initialized Open Census monitoring ' \
113
+ 'registry.'
114
+ )
115
+ end
116
+
117
+ def counter(name, labels, docstring, prefix, aggregation)
118
+ translator = MetricTranslator.new(name, labels)
119
+ measure = OpenCensus::Stats::MeasureRegistry.get(translator.name)
120
+ if measure.nil?
121
+ @log.info(
122
+ 'monitoring module: Registering a new measure registry for ' \
123
+ "#{translator.name}"
124
+ )
125
+ measure = OpenCensus::Stats.create_measure_int(
126
+ name: translator.name,
127
+ unit: OpenCensus::Stats::Measure::UNIT_NONE,
128
+ description: docstring
129
+ )
130
+ end
131
+ unless @exporters.keys.include?(prefix)
132
+ @log.info(
133
+ 'monitoring module: Registering a new exporter for ' \
134
+ "#{prefix}"
135
+ )
136
+ @recorders[prefix] = OpenCensus::Stats::Recorder.new
137
+ @exporters[prefix] = \
138
+ OpenCensus::Stats::Exporters::Stackdriver.new(
139
+ project_id: @project_id,
140
+ metric_prefix: prefix,
141
+ resource_type: @metrics_monitored_resource.type,
142
+ resource_labels: @metrics_monitored_resource.labels,
143
+ gcm_service_address: @gcm_service_address
144
+ )
145
+ @log.info(
146
+ 'monitoring module: Registered recorders and exporters for ' \
147
+ "#{prefix}.\n#{@exporters[prefix]}"
148
+ )
149
+ end
150
+ stats_aggregation = if aggregation == 'GAUGE'
151
+ OpenCensus::Stats.create_last_value_aggregation
152
+ else
153
+ OpenCensus::Stats.create_sum_aggregation
154
+ end
155
+ @recorders[prefix].register_view(
156
+ OpenCensus::Stats::View.new(
157
+ name: translator.name,
158
+ measure: measure,
159
+ aggregation: stats_aggregation,
160
+ description: docstring,
161
+ columns: translator.view_labels.map(&:to_s)
162
+ )
163
+ )
164
+ counter = OpenCensusCounter.new(@recorders[prefix], measure, translator)
165
+ @log.info(
166
+ 'monitoring module: Successfully initialized Open Census counter for ' \
167
+ "#{prefix}/#{name}."
168
+ )
169
+ counter
170
+ rescue StandardError => e
171
+ @log.warn "Failed to count metrics for #{name}.", error: e
172
+ raise e
173
+ end
174
+
175
+ # Update timestamps for each existing AggregationData without altering tags
176
+ # or values.
177
+ # This is currently only used for config analysis metrics, because we want
178
+ # to repeatedly send the exact same metrics as created at start-up.
179
+ def update_timestamps(prefix)
180
+ new_time = Time.now.utc
181
+ recorder = @recorders[prefix]
182
+ recorder.views_data.each do |view_data|
183
+ view_data.data.each_value do |aggr_data|
184
+ # Apply this only to GAUGE metrics. This could fail if the metric uses
185
+ # Distribution or other fancier aggregators.
186
+ aggr_data.add aggr_data.value, new_time if aggr_data.is_a? OpenCensus::Stats::AggregationData::LastValue
187
+ end
188
+ end
189
+ end
190
+
191
+ def export
192
+ @log.debug(
193
+ "monitoring module: Exporting metrics for #{@exporters.keys}."
194
+ )
195
+ @exporters.each_key do |prefix|
196
+ @log.debug(
197
+ "monitoring module: Exporting metrics for #{prefix}. " \
198
+ "#{@recorders[prefix].views_data}"
199
+ )
200
+ @exporters[prefix].export @recorders[prefix].views_data
201
+ end
202
+ rescue StandardError => e
203
+ # TODO(lingshi): Fix the error handling here. Seems like the export is
204
+ # done asynchronously. So any failure happens silently. More details at
205
+ # https://github.com/census-ecosystem/opencensus-ruby-exporter-stackdriver/blob/f8de506204972548ca535eff6010d15f328df6c3/lib/opencensus/stats/exporters/stackdriver.rb#L156
206
+ @log.warn 'Failed to export some metrics.', error: e
207
+ raise e
208
+ end
209
+ end
210
+
211
+ # Factory that is used to create a monitoring registry based on
212
+ # the monitoring solution name.
213
+ class MonitoringRegistryFactory
214
+ @known_registry_types = {
215
+ PrometheusMonitoringRegistry.name =>
216
+ PrometheusMonitoringRegistry,
217
+ OpenCensusMonitoringRegistry.name =>
218
+ OpenCensusMonitoringRegistry
219
+ }
220
+
221
+ def self.supports_monitoring_type(name)
222
+ @known_registry_types.key?(name)
223
+ end
224
+
225
+ def self.create(name, project_id, monitored_resource, gcm_service_address)
226
+ registry = @known_registry_types[name] || BaseMonitoringRegistry
227
+ registry.new(project_id, monitored_resource, gcm_service_address)
228
+ end
229
+ end
230
+
231
+ # Translate the internal metrics to the curated metrics in Stackdriver. The
232
+ # Prometheus metrics are collected by Google Kubernetes Engine's monitoring,
233
+ # so we can't redefine them.
234
+ # Avoid this mechanism for new metrics by defining them in their final form,
235
+ # so they don't need translation.
236
+ class MetricTranslator
237
+ attr_reader :name, :view_labels
238
+
239
+ def initialize(name, metric_labels)
240
+ @legacy = true
241
+ case name
242
+ when :stackdriver_successful_requests_count,
243
+ :stackdriver_failed_requests_count
244
+ @name = :request_count
245
+ when :stackdriver_ingested_entries_count,
246
+ :stackdriver_dropped_entries_count
247
+ @name = :log_entry_count
248
+ when :stackdriver_retried_entries_count
249
+ @name = :log_entry_retry_count
250
+ else
251
+ @name = name
252
+ @legacy = false
253
+ end
254
+ # Collapsed from [:response_code, :grpc]
255
+ @view_labels = @legacy ? [:response_code] : metric_labels
256
+ end
257
+
258
+ def translate_labels(labels)
259
+ return labels unless @legacy
260
+
261
+ translation = { code: :response_code, grpc: :grpc }
262
+ labels.transform_keys { |k| translation[k] }
263
+ end
264
+ end
265
+ end