fluent-plugin-google-cloud 0.6.4.pre.3 → 0.6.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +159 -0
- data/fluent-plugin-google-cloud.gemspec +2 -2
- data/lib/fluent/plugin/monitoring.rb +55 -0
- data/lib/fluent/plugin/out_google_cloud.rb +377 -769
- data/test/plugin/base_test.rb +86 -282
- data/test/plugin/constants.rb +3 -118
- data/test/plugin/test_out_google_cloud.rb +48 -0
- data/test/plugin/test_out_google_cloud_grpc.rb +47 -0
- metadata +21 -19
data/test/plugin/constants.rb
CHANGED
@@ -23,7 +23,6 @@ module Constants
|
|
23
23
|
# Attributes used for the GCE metadata service.
|
24
24
|
PROJECT_ID = 'test-project-id'
|
25
25
|
ZONE = 'us-central1-b'
|
26
|
-
ZONE2 = 'us-central1-c'
|
27
26
|
FULLY_QUALIFIED_ZONE = 'projects/' + PROJECT_ID + '/zones/' + ZONE
|
28
27
|
VM_ID = '9876543210'
|
29
28
|
|
@@ -52,19 +51,6 @@ module Constants
|
|
52
51
|
MANAGED_VM_BACKEND_NAME = 'default'
|
53
52
|
MANAGED_VM_BACKEND_VERSION = 'guestbook2.0'
|
54
53
|
|
55
|
-
# Docker Container labels.
|
56
|
-
DOCKER_CONTAINER_ID = '0d0f03ff8d3c42688692536d1af77a28cd135c0a5c531f25a31'
|
57
|
-
DOCKER_CONTAINER_ID2 = '42688692536d1af77a28cd135c0a5c531f25a0317d37da427e3'
|
58
|
-
DOCKER_CONTAINER_NAME = 'happy_hippo'
|
59
|
-
DOCKER_CONTAINER_NAME2 = 'sad_panda'
|
60
|
-
DOCKER_CONTAINER_STREAM_STDOUT = 'stdout'
|
61
|
-
DOCKER_CONTAINER_STREAM_STDERR = 'stderr'
|
62
|
-
# Timestamp for 1234567890 seconds and 987654321 nanoseconds since epoch.
|
63
|
-
DOCKER_CONTAINER_TIMESTAMP = '2009-02-13T23:31:30.987654321Z'
|
64
|
-
DOCKER_CONTAINER_SECONDS_EPOCH = 1_234_567_890
|
65
|
-
DOCKER_CONTAINER_NANOS = 987_654_321
|
66
|
-
DOCKER_CONTAINER_APPLICATION_LOG_TAG = 'nginx-access-log'
|
67
|
-
|
68
54
|
# Container Engine / Kubernetes specific labels.
|
69
55
|
CONTAINER_CLUSTER_NAME = 'cluster-1'
|
70
56
|
CONTAINER_NAMESPACE_ID = '898268c8-4a36-11e5-9d81-42010af0194c'
|
@@ -154,12 +140,9 @@ module Constants
|
|
154
140
|
detect_subservice false
|
155
141
|
)
|
156
142
|
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
DISABLE_METADATA_AGENT_CONFIG = %(
|
162
|
-
enable_metadata_agent false
|
143
|
+
PROMETHEUS_ENABLE_CONFIG = %(
|
144
|
+
enable_monitoring true
|
145
|
+
monitoring_type prometheus
|
163
146
|
)
|
164
147
|
|
165
148
|
CUSTOM_METADATA_CONFIG = %(
|
@@ -214,8 +197,6 @@ module Constants
|
|
214
197
|
)
|
215
198
|
|
216
199
|
# Service configurations for various services.
|
217
|
-
|
218
|
-
# GCE.
|
219
200
|
COMPUTE_PARAMS = {
|
220
201
|
resource: {
|
221
202
|
type: COMPUTE_CONSTANTS[:resource_type],
|
@@ -231,7 +212,6 @@ module Constants
|
|
231
212
|
}
|
232
213
|
}
|
233
214
|
|
234
|
-
# GAE.
|
235
215
|
VMENGINE_PARAMS = {
|
236
216
|
resource: {
|
237
217
|
type: APPENGINE_CONSTANTS[:resource_type],
|
@@ -249,7 +229,6 @@ module Constants
|
|
249
229
|
}
|
250
230
|
}
|
251
231
|
|
252
|
-
# GKE Container.
|
253
232
|
CONTAINER_TAG = "kubernetes.#{CONTAINER_POD_NAME}_" \
|
254
233
|
"#{CONTAINER_NAMESPACE_NAME}_#{CONTAINER_CONTAINER_NAME}"
|
255
234
|
|
@@ -302,60 +281,6 @@ module Constants
|
|
302
281
|
}
|
303
282
|
}
|
304
283
|
|
305
|
-
# Docker Container.
|
306
|
-
DOCKER_CONTAINER_TAG = "container.#{DOCKER_CONTAINER_ID}." \
|
307
|
-
"#{DOCKER_CONTAINER_NAME}"
|
308
|
-
DOCKER_CONTAINER_TAG_WITH_APPLICATION =
|
309
|
-
"application-container.#{DOCKER_CONTAINER_NAME}." \
|
310
|
-
"#{DOCKER_CONTAINER_APPLICATION_LOG_TAG}"
|
311
|
-
|
312
|
-
DOCKER_CONTAINER_PARAMS = {
|
313
|
-
resource: {
|
314
|
-
type: DOCKER_CONSTANTS[:resource_type],
|
315
|
-
labels: {
|
316
|
-
'container_id' => DOCKER_CONTAINER_ID,
|
317
|
-
'location' => ZONE
|
318
|
-
}
|
319
|
-
},
|
320
|
-
log_name: "container.#{DOCKER_CONTAINER_ID}.#{DOCKER_CONTAINER_NAME}",
|
321
|
-
project_id: PROJECT_ID,
|
322
|
-
labels: {
|
323
|
-
"#{COMPUTE_CONSTANTS[:service]}/resource_id" => VM_ID,
|
324
|
-
"#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME,
|
325
|
-
"#{DOCKER_CONSTANTS[:service]}/container_name" => DOCKER_CONTAINER_NAME
|
326
|
-
}
|
327
|
-
}
|
328
|
-
DOCKER_CONTAINER_PARAMS_WITH_STREAM_STDOUT = DOCKER_CONTAINER_PARAMS.merge(
|
329
|
-
labels: DOCKER_CONTAINER_PARAMS[:labels].merge(
|
330
|
-
"#{DOCKER_CONSTANTS[:service]}/stream" => DOCKER_CONTAINER_STREAM_STDOUT
|
331
|
-
)
|
332
|
-
)
|
333
|
-
DOCKER_CONTAINER_PARAMS_WITH_METADATA_OVERWRITTEN = \
|
334
|
-
DOCKER_CONTAINER_PARAMS.merge(
|
335
|
-
resource: DOCKER_CONTAINER_PARAMS[:resource].merge(
|
336
|
-
labels: DOCKER_CONTAINER_PARAMS[:resource][:labels].merge(
|
337
|
-
'container_id' => DOCKER_CONTAINER_ID2
|
338
|
-
)
|
339
|
-
),
|
340
|
-
labels: DOCKER_CONTAINER_PARAMS[:labels].merge(
|
341
|
-
"#{DOCKER_CONSTANTS[:service]}/container_name" =>
|
342
|
-
DOCKER_CONTAINER_NAME2,
|
343
|
-
"#{DOCKER_CONSTANTS[:service]}/stream" =>
|
344
|
-
DOCKER_CONTAINER_STREAM_STDERR
|
345
|
-
)
|
346
|
-
)
|
347
|
-
DOCKER_CONTAINER_PARAMS_WITH_ZONE2 = DOCKER_CONTAINER_PARAMS.merge(
|
348
|
-
resource: DOCKER_CONTAINER_PARAMS[:resource].merge(
|
349
|
-
labels: DOCKER_CONTAINER_PARAMS[:resource][:labels].merge(
|
350
|
-
'location' => ZONE2
|
351
|
-
)
|
352
|
-
)
|
353
|
-
)
|
354
|
-
DOCKER_CONTAINER_WITH_APPLICATION_PARAMS = DOCKER_CONTAINER_PARAMS.merge(
|
355
|
-
log_name: DOCKER_CONTAINER_TAG_WITH_APPLICATION
|
356
|
-
)
|
357
|
-
|
358
|
-
# Cloud Functions.
|
359
284
|
CLOUDFUNCTIONS_TAG = "kubernetes.#{CLOUDFUNCTIONS_POD_NAME}_" \
|
360
285
|
"#{CLOUDFUNCTIONS_NAMESPACE_NAME}_" \
|
361
286
|
"#{CLOUDFUNCTIONS_CONTAINER_NAME}"
|
@@ -401,7 +326,6 @@ module Constants
|
|
401
326
|
}
|
402
327
|
}
|
403
328
|
|
404
|
-
# Cloud Dataflow.
|
405
329
|
DATAFLOW_PARAMS = {
|
406
330
|
resource: {
|
407
331
|
type: DATAFLOW_CONSTANTS[:resource_type],
|
@@ -421,7 +345,6 @@ module Constants
|
|
421
345
|
}
|
422
346
|
}
|
423
347
|
|
424
|
-
# Cloud Dataproc.
|
425
348
|
DATAPROC_PARAMS = {
|
426
349
|
resource: {
|
427
350
|
type: DATAPROC_CONSTANTS[:resource_type],
|
@@ -440,7 +363,6 @@ module Constants
|
|
440
363
|
}
|
441
364
|
}
|
442
365
|
|
443
|
-
# Cloud ML.
|
444
366
|
ML_PARAMS = {
|
445
367
|
resource: {
|
446
368
|
type: ML_CONSTANTS[:resource_type],
|
@@ -527,41 +449,4 @@ module Constants
|
|
527
449
|
'' => '_'
|
528
450
|
}
|
529
451
|
ALL_TAGS = VALID_TAGS.merge(INVALID_TAGS)
|
530
|
-
|
531
|
-
# Stub value for Monitored resources from Metadata Agent.
|
532
|
-
MONITORED_RESOURCE_STUBS = {
|
533
|
-
# Implicit GCE instance.
|
534
|
-
IMPLICIT_MONITORED_RESOURCE_UNIQUE_KEY => {
|
535
|
-
'unique_id' => IMPLICIT_MONITORED_RESOURCE_UNIQUE_KEY,
|
536
|
-
'monitored_resource' => '{' \
|
537
|
-
"\"type\": \"#{COMPUTE_CONSTANTS[:resource_type]}\"," \
|
538
|
-
'"labels": {' \
|
539
|
-
"\"zone\": \"#{ZONE}\"," \
|
540
|
-
"\"instance_id\": \"#{VM_ID}\"" \
|
541
|
-
'}' \
|
542
|
-
'}'
|
543
|
-
},
|
544
|
-
# Docker container stderr / stdout logs.
|
545
|
-
DOCKER_CONSTANTS[:resource_type] => {
|
546
|
-
'unique_id' => "container.#{DOCKER_CONTAINER_ID}",
|
547
|
-
'monitored_resource' => '{' \
|
548
|
-
"\"type\": \"#{DOCKER_CONSTANTS[:resource_type]}\"," \
|
549
|
-
'"labels": {' \
|
550
|
-
"\"location\": \"#{ZONE2}\"," \
|
551
|
-
"\"container_id\": \"#{DOCKER_CONTAINER_ID}\"" \
|
552
|
-
'}' \
|
553
|
-
'}'
|
554
|
-
},
|
555
|
-
# Docker container application logs.
|
556
|
-
"#{DOCKER_CONSTANTS[:resource_type]}_application" => {
|
557
|
-
'unique_id' => "containerName.#{DOCKER_CONTAINER_NAME}",
|
558
|
-
'monitored_resource' => '{' \
|
559
|
-
"\"type\": \"#{DOCKER_CONSTANTS[:resource_type]}\"," \
|
560
|
-
'"labels": {' \
|
561
|
-
"\"location\": \"#{ZONE}\"," \
|
562
|
-
"\"container_id\": \"#{DOCKER_CONTAINER_ID}\"" \
|
563
|
-
'}' \
|
564
|
-
'}'
|
565
|
-
}
|
566
|
-
}
|
567
452
|
end
|
@@ -70,6 +70,54 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
70
70
|
assert_equal 1, exception_count
|
71
71
|
end
|
72
72
|
|
73
|
+
# TODO: The code in the non-gRPC and gRPC tests is nearly identical.
|
74
|
+
# Refactor and remove duplication.
|
75
|
+
# TODO: Use status codes instead of int literals.
|
76
|
+
def test_prometheus_metrics
|
77
|
+
setup_gce_metadata_stubs
|
78
|
+
[
|
79
|
+
# Single successful request.
|
80
|
+
[200, 1, 1, [1, 0, 1, 0]],
|
81
|
+
# Several successful requests.
|
82
|
+
[200, 2, 1, [2, 0, 2, 0]],
|
83
|
+
# Single successful request with several entries.
|
84
|
+
[200, 1, 2, [1, 0, 2, 0]],
|
85
|
+
# Single failed request that causes logs to be dropped.
|
86
|
+
[401, 1, 1, [0, 1, 0, 1]],
|
87
|
+
# Single failed request that escalates without logs being dropped.
|
88
|
+
[500, 1, 1, [0, 1, 0, 0]]
|
89
|
+
].each do |code, request_count, entry_count, metric_values|
|
90
|
+
setup_prometheus
|
91
|
+
# TODO: Do this as part of setup_logging_stubs.
|
92
|
+
stub_request(:post, WRITE_LOG_ENTRIES_URI)
|
93
|
+
.to_return(status: code, body: 'Some Message')
|
94
|
+
(1..request_count).each do
|
95
|
+
d = create_driver(PROMETHEUS_ENABLE_CONFIG)
|
96
|
+
(1..entry_count).each do |i|
|
97
|
+
d.emit('message' => log_entry(i.to_s))
|
98
|
+
end
|
99
|
+
# rubocop:disable Lint/HandleExceptions
|
100
|
+
begin
|
101
|
+
d.run
|
102
|
+
rescue Google::Apis::AuthorizationError
|
103
|
+
rescue Google::Apis::ServerError
|
104
|
+
end
|
105
|
+
# rubocop:enable Lint/HandleExceptions
|
106
|
+
end
|
107
|
+
successful_requests_count, failed_requests_count,
|
108
|
+
ingested_entries_count, dropped_entries_count = metric_values
|
109
|
+
assert_prometheus_metric_value(:stackdriver_successful_requests_count,
|
110
|
+
successful_requests_count, grpc: false)
|
111
|
+
assert_prometheus_metric_value(:stackdriver_failed_requests_count,
|
112
|
+
failed_requests_count,
|
113
|
+
grpc: false, code: code)
|
114
|
+
assert_prometheus_metric_value(:stackdriver_ingested_entries_count,
|
115
|
+
ingested_entries_count)
|
116
|
+
assert_prometheus_metric_value(:stackdriver_dropped_entries_count,
|
117
|
+
dropped_entries_count)
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
73
121
|
# This test looks similar between the grpc and non-grpc paths except that when
|
74
122
|
# parsing "105", the grpc path responds with "DEBUG", while the non-grpc path
|
75
123
|
# responds with "100".
|
@@ -72,6 +72,53 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
|
|
72
72
|
end
|
73
73
|
end
|
74
74
|
|
75
|
+
# TODO: The code in the non-gRPC and gRPC tests is nearly identical.
|
76
|
+
# Refactor and remove duplication.
|
77
|
+
# TODO: Use status codes instead of int literals.
|
78
|
+
def test_prometheus_metrics
|
79
|
+
setup_gce_metadata_stubs
|
80
|
+
[
|
81
|
+
# Single successful request.
|
82
|
+
[false, 0, 1, 1, [1, 0, 1, 0]],
|
83
|
+
# Several successful requests.
|
84
|
+
[false, 0, 2, 1, [2, 0, 2, 0]],
|
85
|
+
# Single successful request with several entries.
|
86
|
+
[false, 0, 1, 2, [1, 0, 2, 0]],
|
87
|
+
# Single failed request that causes logs to be dropped.
|
88
|
+
[true, 16, 1, 1, [0, 1, 0, 1]],
|
89
|
+
# Single failed request that escalates without logs being dropped.
|
90
|
+
[true, 13, 1, 1, [0, 1, 0, 0]]
|
91
|
+
].each do |should_fail, code, request_count, entry_count, metric_values|
|
92
|
+
setup_prometheus
|
93
|
+
setup_logging_stubs(should_fail, code, 'SomeMessage') do
|
94
|
+
(1..request_count).each do
|
95
|
+
d = create_driver(USE_GRPC_CONFIG + PROMETHEUS_ENABLE_CONFIG, 'test',
|
96
|
+
GRPCLoggingMockFailingService.rpc_stub_class)
|
97
|
+
(1..entry_count).each do |i|
|
98
|
+
d.emit('message' => log_entry(i.to_s))
|
99
|
+
end
|
100
|
+
# rubocop:disable Lint/HandleExceptions
|
101
|
+
begin
|
102
|
+
d.run
|
103
|
+
rescue GRPC::BadStatus
|
104
|
+
end
|
105
|
+
# rubocop:enable Lint/HandleExceptions
|
106
|
+
end
|
107
|
+
end
|
108
|
+
successful_requests_count, failed_requests_count,
|
109
|
+
ingested_entries_count, dropped_entries_count = metric_values
|
110
|
+
assert_prometheus_metric_value(:stackdriver_successful_requests_count,
|
111
|
+
successful_requests_count, grpc: true)
|
112
|
+
assert_prometheus_metric_value(:stackdriver_failed_requests_count,
|
113
|
+
failed_requests_count,
|
114
|
+
grpc: true, code: code)
|
115
|
+
assert_prometheus_metric_value(:stackdriver_ingested_entries_count,
|
116
|
+
ingested_entries_count)
|
117
|
+
assert_prometheus_metric_value(:stackdriver_dropped_entries_count,
|
118
|
+
dropped_entries_count)
|
119
|
+
end
|
120
|
+
end
|
121
|
+
|
75
122
|
# This test looks similar between the grpc and non-grpc paths except that when
|
76
123
|
# parsing "105", the grpc path responds with "DEBUG", while the non-grpc path
|
77
124
|
# responds with "100".
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-google-cloud
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.6.4
|
4
|
+
version: 0.6.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Todd Derr
|
@@ -9,22 +9,8 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2017-07-
|
12
|
+
date: 2017-07-11 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
|
-
- !ruby/object:Gem::Dependency
|
15
|
-
name: excon
|
16
|
-
requirement: !ruby/object:Gem::Requirement
|
17
|
-
requirements:
|
18
|
-
- - "~>"
|
19
|
-
- !ruby/object:Gem::Version
|
20
|
-
version: 0.57.1
|
21
|
-
type: :runtime
|
22
|
-
prerelease: false
|
23
|
-
version_requirements: !ruby/object:Gem::Requirement
|
24
|
-
requirements:
|
25
|
-
- - "~>"
|
26
|
-
- !ruby/object:Gem::Version
|
27
|
-
version: 0.57.1
|
28
14
|
- !ruby/object:Gem::Dependency
|
29
15
|
name: fluentd
|
30
16
|
requirement: !ruby/object:Gem::Requirement
|
@@ -199,6 +185,20 @@ dependencies:
|
|
199
185
|
- - "~>"
|
200
186
|
- !ruby/object:Gem::Version
|
201
187
|
version: '3.0'
|
188
|
+
- !ruby/object:Gem::Dependency
|
189
|
+
name: prometheus-client
|
190
|
+
requirement: !ruby/object:Gem::Requirement
|
191
|
+
requirements:
|
192
|
+
- - "~>"
|
193
|
+
- !ruby/object:Gem::Version
|
194
|
+
version: 0.7.1
|
195
|
+
type: :development
|
196
|
+
prerelease: false
|
197
|
+
version_requirements: !ruby/object:Gem::Requirement
|
198
|
+
requirements:
|
199
|
+
- - "~>"
|
200
|
+
- !ruby/object:Gem::Version
|
201
|
+
version: 0.7.1
|
202
202
|
description: |2
|
203
203
|
Fluentd output plugin for the Stackdriver Logging API, which will make
|
204
204
|
logs viewable in the Developer Console's log viewer and can optionally
|
@@ -212,10 +212,12 @@ extra_rdoc_files: []
|
|
212
212
|
files:
|
213
213
|
- CONTRIBUTING
|
214
214
|
- Gemfile
|
215
|
+
- Gemfile.lock
|
215
216
|
- LICENSE
|
216
217
|
- README.rdoc
|
217
218
|
- Rakefile
|
218
219
|
- fluent-plugin-google-cloud.gemspec
|
220
|
+
- lib/fluent/plugin/monitoring.rb
|
219
221
|
- lib/fluent/plugin/out_google_cloud.rb
|
220
222
|
- test/helper.rb
|
221
223
|
- test/plugin/base_test.rb
|
@@ -241,12 +243,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
241
243
|
version: '2.0'
|
242
244
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
243
245
|
requirements:
|
244
|
-
- - "
|
246
|
+
- - ">="
|
245
247
|
- !ruby/object:Gem::Version
|
246
|
-
version:
|
248
|
+
version: '0'
|
247
249
|
requirements: []
|
248
250
|
rubyforge_project:
|
249
|
-
rubygems_version: 2.
|
251
|
+
rubygems_version: 2.6.12
|
250
252
|
signing_key:
|
251
253
|
specification_version: 4
|
252
254
|
summary: fluentd output plugin for the Stackdriver Logging API
|