fluent-plugin-google-cloud 0.6.4.pre.3 → 0.6.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -23,7 +23,6 @@ module Constants
23
23
  # Attributes used for the GCE metadata service.
24
24
  PROJECT_ID = 'test-project-id'
25
25
  ZONE = 'us-central1-b'
26
- ZONE2 = 'us-central1-c'
27
26
  FULLY_QUALIFIED_ZONE = 'projects/' + PROJECT_ID + '/zones/' + ZONE
28
27
  VM_ID = '9876543210'
29
28
 
@@ -52,19 +51,6 @@ module Constants
52
51
  MANAGED_VM_BACKEND_NAME = 'default'
53
52
  MANAGED_VM_BACKEND_VERSION = 'guestbook2.0'
54
53
 
55
- # Docker Container labels.
56
- DOCKER_CONTAINER_ID = '0d0f03ff8d3c42688692536d1af77a28cd135c0a5c531f25a31'
57
- DOCKER_CONTAINER_ID2 = '42688692536d1af77a28cd135c0a5c531f25a0317d37da427e3'
58
- DOCKER_CONTAINER_NAME = 'happy_hippo'
59
- DOCKER_CONTAINER_NAME2 = 'sad_panda'
60
- DOCKER_CONTAINER_STREAM_STDOUT = 'stdout'
61
- DOCKER_CONTAINER_STREAM_STDERR = 'stderr'
62
- # Timestamp for 1234567890 seconds and 987654321 nanoseconds since epoch.
63
- DOCKER_CONTAINER_TIMESTAMP = '2009-02-13T23:31:30.987654321Z'
64
- DOCKER_CONTAINER_SECONDS_EPOCH = 1_234_567_890
65
- DOCKER_CONTAINER_NANOS = 987_654_321
66
- DOCKER_CONTAINER_APPLICATION_LOG_TAG = 'nginx-access-log'
67
-
68
54
  # Container Engine / Kubernetes specific labels.
69
55
  CONTAINER_CLUSTER_NAME = 'cluster-1'
70
56
  CONTAINER_NAMESPACE_ID = '898268c8-4a36-11e5-9d81-42010af0194c'
@@ -154,12 +140,9 @@ module Constants
154
140
  detect_subservice false
155
141
  )
156
142
 
157
- ENABLE_METADATA_AGENT_CONFIG = %(
158
- enable_metadata_agent true
159
- )
160
-
161
- DISABLE_METADATA_AGENT_CONFIG = %(
162
- enable_metadata_agent false
143
+ PROMETHEUS_ENABLE_CONFIG = %(
144
+ enable_monitoring true
145
+ monitoring_type prometheus
163
146
  )
164
147
 
165
148
  CUSTOM_METADATA_CONFIG = %(
@@ -214,8 +197,6 @@ module Constants
214
197
  )
215
198
 
216
199
  # Service configurations for various services.
217
-
218
- # GCE.
219
200
  COMPUTE_PARAMS = {
220
201
  resource: {
221
202
  type: COMPUTE_CONSTANTS[:resource_type],
@@ -231,7 +212,6 @@ module Constants
231
212
  }
232
213
  }
233
214
 
234
- # GAE.
235
215
  VMENGINE_PARAMS = {
236
216
  resource: {
237
217
  type: APPENGINE_CONSTANTS[:resource_type],
@@ -249,7 +229,6 @@ module Constants
249
229
  }
250
230
  }
251
231
 
252
- # GKE Container.
253
232
  CONTAINER_TAG = "kubernetes.#{CONTAINER_POD_NAME}_" \
254
233
  "#{CONTAINER_NAMESPACE_NAME}_#{CONTAINER_CONTAINER_NAME}"
255
234
 
@@ -302,60 +281,6 @@ module Constants
302
281
  }
303
282
  }
304
283
 
305
- # Docker Container.
306
- DOCKER_CONTAINER_TAG = "container.#{DOCKER_CONTAINER_ID}." \
307
- "#{DOCKER_CONTAINER_NAME}"
308
- DOCKER_CONTAINER_TAG_WITH_APPLICATION =
309
- "application-container.#{DOCKER_CONTAINER_NAME}." \
310
- "#{DOCKER_CONTAINER_APPLICATION_LOG_TAG}"
311
-
312
- DOCKER_CONTAINER_PARAMS = {
313
- resource: {
314
- type: DOCKER_CONSTANTS[:resource_type],
315
- labels: {
316
- 'container_id' => DOCKER_CONTAINER_ID,
317
- 'location' => ZONE
318
- }
319
- },
320
- log_name: "container.#{DOCKER_CONTAINER_ID}.#{DOCKER_CONTAINER_NAME}",
321
- project_id: PROJECT_ID,
322
- labels: {
323
- "#{COMPUTE_CONSTANTS[:service]}/resource_id" => VM_ID,
324
- "#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME,
325
- "#{DOCKER_CONSTANTS[:service]}/container_name" => DOCKER_CONTAINER_NAME
326
- }
327
- }
328
- DOCKER_CONTAINER_PARAMS_WITH_STREAM_STDOUT = DOCKER_CONTAINER_PARAMS.merge(
329
- labels: DOCKER_CONTAINER_PARAMS[:labels].merge(
330
- "#{DOCKER_CONSTANTS[:service]}/stream" => DOCKER_CONTAINER_STREAM_STDOUT
331
- )
332
- )
333
- DOCKER_CONTAINER_PARAMS_WITH_METADATA_OVERWRITTEN = \
334
- DOCKER_CONTAINER_PARAMS.merge(
335
- resource: DOCKER_CONTAINER_PARAMS[:resource].merge(
336
- labels: DOCKER_CONTAINER_PARAMS[:resource][:labels].merge(
337
- 'container_id' => DOCKER_CONTAINER_ID2
338
- )
339
- ),
340
- labels: DOCKER_CONTAINER_PARAMS[:labels].merge(
341
- "#{DOCKER_CONSTANTS[:service]}/container_name" =>
342
- DOCKER_CONTAINER_NAME2,
343
- "#{DOCKER_CONSTANTS[:service]}/stream" =>
344
- DOCKER_CONTAINER_STREAM_STDERR
345
- )
346
- )
347
- DOCKER_CONTAINER_PARAMS_WITH_ZONE2 = DOCKER_CONTAINER_PARAMS.merge(
348
- resource: DOCKER_CONTAINER_PARAMS[:resource].merge(
349
- labels: DOCKER_CONTAINER_PARAMS[:resource][:labels].merge(
350
- 'location' => ZONE2
351
- )
352
- )
353
- )
354
- DOCKER_CONTAINER_WITH_APPLICATION_PARAMS = DOCKER_CONTAINER_PARAMS.merge(
355
- log_name: DOCKER_CONTAINER_TAG_WITH_APPLICATION
356
- )
357
-
358
- # Cloud Functions.
359
284
  CLOUDFUNCTIONS_TAG = "kubernetes.#{CLOUDFUNCTIONS_POD_NAME}_" \
360
285
  "#{CLOUDFUNCTIONS_NAMESPACE_NAME}_" \
361
286
  "#{CLOUDFUNCTIONS_CONTAINER_NAME}"
@@ -401,7 +326,6 @@ module Constants
401
326
  }
402
327
  }
403
328
 
404
- # Cloud Dataflow.
405
329
  DATAFLOW_PARAMS = {
406
330
  resource: {
407
331
  type: DATAFLOW_CONSTANTS[:resource_type],
@@ -421,7 +345,6 @@ module Constants
421
345
  }
422
346
  }
423
347
 
424
- # Cloud Dataproc.
425
348
  DATAPROC_PARAMS = {
426
349
  resource: {
427
350
  type: DATAPROC_CONSTANTS[:resource_type],
@@ -440,7 +363,6 @@ module Constants
440
363
  }
441
364
  }
442
365
 
443
- # Cloud ML.
444
366
  ML_PARAMS = {
445
367
  resource: {
446
368
  type: ML_CONSTANTS[:resource_type],
@@ -527,41 +449,4 @@ module Constants
527
449
  '' => '_'
528
450
  }
529
451
  ALL_TAGS = VALID_TAGS.merge(INVALID_TAGS)
530
-
531
- # Stub value for Monitored resources from Metadata Agent.
532
- MONITORED_RESOURCE_STUBS = {
533
- # Implicit GCE instance.
534
- IMPLICIT_MONITORED_RESOURCE_UNIQUE_KEY => {
535
- 'unique_id' => IMPLICIT_MONITORED_RESOURCE_UNIQUE_KEY,
536
- 'monitored_resource' => '{' \
537
- "\"type\": \"#{COMPUTE_CONSTANTS[:resource_type]}\"," \
538
- '"labels": {' \
539
- "\"zone\": \"#{ZONE}\"," \
540
- "\"instance_id\": \"#{VM_ID}\"" \
541
- '}' \
542
- '}'
543
- },
544
- # Docker container stderr / stdout logs.
545
- DOCKER_CONSTANTS[:resource_type] => {
546
- 'unique_id' => "container.#{DOCKER_CONTAINER_ID}",
547
- 'monitored_resource' => '{' \
548
- "\"type\": \"#{DOCKER_CONSTANTS[:resource_type]}\"," \
549
- '"labels": {' \
550
- "\"location\": \"#{ZONE2}\"," \
551
- "\"container_id\": \"#{DOCKER_CONTAINER_ID}\"" \
552
- '}' \
553
- '}'
554
- },
555
- # Docker container application logs.
556
- "#{DOCKER_CONSTANTS[:resource_type]}_application" => {
557
- 'unique_id' => "containerName.#{DOCKER_CONTAINER_NAME}",
558
- 'monitored_resource' => '{' \
559
- "\"type\": \"#{DOCKER_CONSTANTS[:resource_type]}\"," \
560
- '"labels": {' \
561
- "\"location\": \"#{ZONE}\"," \
562
- "\"container_id\": \"#{DOCKER_CONTAINER_ID}\"" \
563
- '}' \
564
- '}'
565
- }
566
- }
567
452
  end
@@ -70,6 +70,54 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
70
70
  assert_equal 1, exception_count
71
71
  end
72
72
 
73
+ # TODO: The code in the non-gRPC and gRPC tests is nearly identical.
74
+ # Refactor and remove duplication.
75
+ # TODO: Use status codes instead of int literals.
76
+ def test_prometheus_metrics
77
+ setup_gce_metadata_stubs
78
+ [
79
+ # Single successful request.
80
+ [200, 1, 1, [1, 0, 1, 0]],
81
+ # Several successful requests.
82
+ [200, 2, 1, [2, 0, 2, 0]],
83
+ # Single successful request with several entries.
84
+ [200, 1, 2, [1, 0, 2, 0]],
85
+ # Single failed request that causes logs to be dropped.
86
+ [401, 1, 1, [0, 1, 0, 1]],
87
+ # Single failed request that escalates without logs being dropped.
88
+ [500, 1, 1, [0, 1, 0, 0]]
89
+ ].each do |code, request_count, entry_count, metric_values|
90
+ setup_prometheus
91
+ # TODO: Do this as part of setup_logging_stubs.
92
+ stub_request(:post, WRITE_LOG_ENTRIES_URI)
93
+ .to_return(status: code, body: 'Some Message')
94
+ (1..request_count).each do
95
+ d = create_driver(PROMETHEUS_ENABLE_CONFIG)
96
+ (1..entry_count).each do |i|
97
+ d.emit('message' => log_entry(i.to_s))
98
+ end
99
+ # rubocop:disable Lint/HandleExceptions
100
+ begin
101
+ d.run
102
+ rescue Google::Apis::AuthorizationError
103
+ rescue Google::Apis::ServerError
104
+ end
105
+ # rubocop:enable Lint/HandleExceptions
106
+ end
107
+ successful_requests_count, failed_requests_count,
108
+ ingested_entries_count, dropped_entries_count = metric_values
109
+ assert_prometheus_metric_value(:stackdriver_successful_requests_count,
110
+ successful_requests_count, grpc: false)
111
+ assert_prometheus_metric_value(:stackdriver_failed_requests_count,
112
+ failed_requests_count,
113
+ grpc: false, code: code)
114
+ assert_prometheus_metric_value(:stackdriver_ingested_entries_count,
115
+ ingested_entries_count)
116
+ assert_prometheus_metric_value(:stackdriver_dropped_entries_count,
117
+ dropped_entries_count)
118
+ end
119
+ end
120
+
73
121
  # This test looks similar between the grpc and non-grpc paths except that when
74
122
  # parsing "105", the grpc path responds with "DEBUG", while the non-grpc path
75
123
  # responds with "100".
@@ -72,6 +72,53 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
72
72
  end
73
73
  end
74
74
 
75
+ # TODO: The code in the non-gRPC and gRPC tests is nearly identical.
76
+ # Refactor and remove duplication.
77
+ # TODO: Use status codes instead of int literals.
78
+ def test_prometheus_metrics
79
+ setup_gce_metadata_stubs
80
+ [
81
+ # Single successful request.
82
+ [false, 0, 1, 1, [1, 0, 1, 0]],
83
+ # Several successful requests.
84
+ [false, 0, 2, 1, [2, 0, 2, 0]],
85
+ # Single successful request with several entries.
86
+ [false, 0, 1, 2, [1, 0, 2, 0]],
87
+ # Single failed request that causes logs to be dropped.
88
+ [true, 16, 1, 1, [0, 1, 0, 1]],
89
+ # Single failed request that escalates without logs being dropped.
90
+ [true, 13, 1, 1, [0, 1, 0, 0]]
91
+ ].each do |should_fail, code, request_count, entry_count, metric_values|
92
+ setup_prometheus
93
+ setup_logging_stubs(should_fail, code, 'SomeMessage') do
94
+ (1..request_count).each do
95
+ d = create_driver(USE_GRPC_CONFIG + PROMETHEUS_ENABLE_CONFIG, 'test',
96
+ GRPCLoggingMockFailingService.rpc_stub_class)
97
+ (1..entry_count).each do |i|
98
+ d.emit('message' => log_entry(i.to_s))
99
+ end
100
+ # rubocop:disable Lint/HandleExceptions
101
+ begin
102
+ d.run
103
+ rescue GRPC::BadStatus
104
+ end
105
+ # rubocop:enable Lint/HandleExceptions
106
+ end
107
+ end
108
+ successful_requests_count, failed_requests_count,
109
+ ingested_entries_count, dropped_entries_count = metric_values
110
+ assert_prometheus_metric_value(:stackdriver_successful_requests_count,
111
+ successful_requests_count, grpc: true)
112
+ assert_prometheus_metric_value(:stackdriver_failed_requests_count,
113
+ failed_requests_count,
114
+ grpc: true, code: code)
115
+ assert_prometheus_metric_value(:stackdriver_ingested_entries_count,
116
+ ingested_entries_count)
117
+ assert_prometheus_metric_value(:stackdriver_dropped_entries_count,
118
+ dropped_entries_count)
119
+ end
120
+ end
121
+
75
122
  # This test looks similar between the grpc and non-grpc paths except that when
76
123
  # parsing "105", the grpc path responds with "DEBUG", while the non-grpc path
77
124
  # responds with "100".
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-google-cloud
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.6.4.pre.3
4
+ version: 0.6.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Todd Derr
@@ -9,22 +9,8 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2017-07-25 00:00:00.000000000 Z
12
+ date: 2017-07-11 00:00:00.000000000 Z
13
13
  dependencies:
14
- - !ruby/object:Gem::Dependency
15
- name: excon
16
- requirement: !ruby/object:Gem::Requirement
17
- requirements:
18
- - - "~>"
19
- - !ruby/object:Gem::Version
20
- version: 0.57.1
21
- type: :runtime
22
- prerelease: false
23
- version_requirements: !ruby/object:Gem::Requirement
24
- requirements:
25
- - - "~>"
26
- - !ruby/object:Gem::Version
27
- version: 0.57.1
28
14
  - !ruby/object:Gem::Dependency
29
15
  name: fluentd
30
16
  requirement: !ruby/object:Gem::Requirement
@@ -199,6 +185,20 @@ dependencies:
199
185
  - - "~>"
200
186
  - !ruby/object:Gem::Version
201
187
  version: '3.0'
188
+ - !ruby/object:Gem::Dependency
189
+ name: prometheus-client
190
+ requirement: !ruby/object:Gem::Requirement
191
+ requirements:
192
+ - - "~>"
193
+ - !ruby/object:Gem::Version
194
+ version: 0.7.1
195
+ type: :development
196
+ prerelease: false
197
+ version_requirements: !ruby/object:Gem::Requirement
198
+ requirements:
199
+ - - "~>"
200
+ - !ruby/object:Gem::Version
201
+ version: 0.7.1
202
202
  description: |2
203
203
  Fluentd output plugin for the Stackdriver Logging API, which will make
204
204
  logs viewable in the Developer Console's log viewer and can optionally
@@ -212,10 +212,12 @@ extra_rdoc_files: []
212
212
  files:
213
213
  - CONTRIBUTING
214
214
  - Gemfile
215
+ - Gemfile.lock
215
216
  - LICENSE
216
217
  - README.rdoc
217
218
  - Rakefile
218
219
  - fluent-plugin-google-cloud.gemspec
220
+ - lib/fluent/plugin/monitoring.rb
219
221
  - lib/fluent/plugin/out_google_cloud.rb
220
222
  - test/helper.rb
221
223
  - test/plugin/base_test.rb
@@ -241,12 +243,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
241
243
  version: '2.0'
242
244
  required_rubygems_version: !ruby/object:Gem::Requirement
243
245
  requirements:
244
- - - ">"
246
+ - - ">="
245
247
  - !ruby/object:Gem::Version
246
- version: 1.3.1
248
+ version: '0'
247
249
  requirements: []
248
250
  rubyforge_project:
249
- rubygems_version: 2.4.8
251
+ rubygems_version: 2.6.12
250
252
  signing_key:
251
253
  specification_version: 4
252
254
  summary: fluentd output plugin for the Stackdriver Logging API