vmik-fluent-plugin-google-cloud 0.5.5 → 0.6.4.pre.alpha

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,452 @@
1
+ # Copyright 2017 Google Inc. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ # Constants used by unit tests for Google Cloud Logging plugin.
16
+ module Constants
17
+ include Fluent::GoogleCloudOutput::Constants
18
+
19
+ # Generic attributes.
20
+ HOSTNAME = Socket.gethostname
21
+
22
+ # TODO(qingling128) Separate constants into different submodules.
23
+ # Attributes used for the GCE metadata service.
24
+ PROJECT_ID = 'test-project-id'
25
+ ZONE = 'us-central1-b'
26
+ FULLY_QUALIFIED_ZONE = 'projects/' + PROJECT_ID + '/zones/' + ZONE
27
+ VM_ID = '9876543210'
28
+
29
+ # Attributes used for custom (overridden) configs.
30
+ CUSTOM_PROJECT_ID = 'test-custom-project-id'
31
+ CUSTOM_ZONE = 'us-custom-central1-b'
32
+ CUSTOM_FULLY_QUALIFIED_ZONE = 'projects/' + PROJECT_ID + '/zones/' + ZONE
33
+ CUSTOM_VM_ID = 'C9876543210'
34
+ CUSTOM_HOSTNAME = 'custom.hostname.org'
35
+
36
+ # Attributes used for the EC2 metadata service.
37
+ EC2_PROJECT_ID = 'test-ec2-project-id'
38
+ EC2_ZONE = 'us-west-2b'
39
+ EC2_PREFIXED_ZONE = 'aws:' + EC2_ZONE
40
+ EC2_VM_ID = 'i-81c16767'
41
+ EC2_ACCOUNT_ID = '123456789012'
42
+
43
+ # The formatting here matches the format used on the VM.
44
+ EC2_IDENTITY_DOCUMENT = %({
45
+ "accountId" : "#{EC2_ACCOUNT_ID}",
46
+ "availabilityZone" : "#{EC2_ZONE}",
47
+ "instanceId" : "#{EC2_VM_ID}"
48
+ })
49
+
50
+ # Managed VMs specific labels.
51
+ MANAGED_VM_BACKEND_NAME = 'default'
52
+ MANAGED_VM_BACKEND_VERSION = 'guestbook2.0'
53
+
54
+ # Container Engine / Kubernetes specific labels.
55
+ CONTAINER_CLUSTER_NAME = 'cluster-1'
56
+ CONTAINER_NAMESPACE_ID = '898268c8-4a36-11e5-9d81-42010af0194c'
57
+ CONTAINER_NAMESPACE_NAME = 'kube-system'
58
+ CONTAINER_POD_ID = 'cad3c3c4-4b9c-11e5-9d81-42010af0194c'
59
+ CONTAINER_POD_NAME = 'redis-master-c0l82.foo.bar'
60
+ CONTAINER_CONTAINER_NAME = 'redis'
61
+ CONTAINER_LABEL_KEY = 'component'
62
+ CONTAINER_LABEL_VALUE = 'redis-component'
63
+ CONTAINER_STREAM = 'stdout'
64
+ CONTAINER_SEVERITY = 'INFO'
65
+ # Timestamp for 1234567890 seconds and 987654321 nanoseconds since epoch.
66
+ CONTAINER_TIMESTAMP = '2009-02-13T23:31:30.987654321Z'
67
+ CONTAINER_SECONDS_EPOCH = 1_234_567_890
68
+ CONTAINER_NANOS = 987_654_321
69
+
70
+ # Cloud Functions specific labels.
71
+ CLOUDFUNCTIONS_FUNCTION_NAME = '$My_Function.Name-@1'
72
+ CLOUDFUNCTIONS_REGION = 'us-central1'
73
+ CLOUDFUNCTIONS_EXECUTION_ID = '123-0'
74
+ CLOUDFUNCTIONS_CLUSTER_NAME = 'cluster-1'
75
+ CLOUDFUNCTIONS_NAMESPACE_NAME = 'default'
76
+ CLOUDFUNCTIONS_POD_NAME = 'd.dc.myu.uc.functionp.pc.name-a.a1.987-c0l82'
77
+ CLOUDFUNCTIONS_CONTAINER_NAME = 'worker'
78
+
79
+ # Dataflow specific labels.
80
+ DATAFLOW_REGION = 'us-central1'
81
+ DATAFLOW_JOB_NAME = 'job_name_1'
82
+ DATAFLOW_JOB_ID = 'job_id_1'
83
+ DATAFLOW_STEP_ID = 'step_1'
84
+ DATAFLOW_TAG = 'dataflow.googleapis.com/worker'
85
+
86
+ # Dataproc specific labels.
87
+ DATAPROC_CLUSTER_NAME = 'test-cluster'
88
+ DATAPROC_CLUSTER_UUID = '00000000-0000-0000-0000-000000000000'
89
+ DATAPROC_REGION = 'unittest'
90
+
91
+ # ML specific labels.
92
+ ML_REGION = 'us-central1'
93
+ ML_JOB_ID = 'job_name_1'
94
+ ML_TASK_NAME = 'task_name_1'
95
+ ML_TRIAL_ID = 'trial_id_1'
96
+ ML_LOG_AREA = 'log_area_1'
97
+ ML_TAG = 'master-replica-0'
98
+
99
+ # Parameters used for authentication.
100
+ AUTH_GRANT_TYPE = 'urn:ietf:params:oauth:grant-type:jwt-bearer'
101
+ FAKE_AUTH_TOKEN = 'abc123'
102
+
103
+ # Information about test credentials files.
104
+ # path: Path to the credentials file.
105
+ # project_id: ID of the project, which must correspond to the file contents.
106
+ IAM_CREDENTIALS = {
107
+ path: 'test/plugin/data/iam-credentials.json',
108
+ project_id: 'fluent-test-project'
109
+ }
110
+ LEGACY_CREDENTIALS = {
111
+ path: 'test/plugin/data/credentials.json',
112
+ project_id: '847859579879'
113
+ }
114
+ INVALID_CREDENTIALS = {
115
+ path: 'test/plugin/data/invalid_credentials.json',
116
+ project_id: ''
117
+ }
118
+
119
+ # Configuration files for various test scenarios.
120
+ APPLICATION_DEFAULT_CONFIG = %(
121
+ )
122
+
123
+ # rubocop:disable Metrics/LineLength
124
+ PRIVATE_KEY_CONFIG = %(
125
+ auth_method private_key
126
+ private_key_email 271661262351-ft99kc9kjro9rrihq3k2n3s2inbplu0q@developer.gserviceaccount.com
127
+ private_key_path test/plugin/data/c31e573fd7f62ed495c9ca3821a5a85cb036dee1-privatekey.p12
128
+ )
129
+ # rubocop:enable Metrics/LineLength
130
+
131
+ REQUIRE_VALID_TAGS_CONFIG = %(
132
+ require_valid_tags true
133
+ )
134
+
135
+ NO_METADATA_SERVICE_CONFIG = %(
136
+ use_metadata_service false
137
+ )
138
+
139
+ NO_DETECT_SUBSERVICE_CONFIG = %(
140
+ detect_subservice false
141
+ )
142
+
143
+ PROMETHEUS_ENABLE_CONFIG = %(
144
+ monitoring_enabled true
145
+ monitoring_type prometheus
146
+ )
147
+
148
+ CUSTOM_METADATA_CONFIG = %(
149
+ project_id #{CUSTOM_PROJECT_ID}
150
+ zone #{CUSTOM_ZONE}
151
+ vm_id #{CUSTOM_VM_ID}
152
+ vm_name #{CUSTOM_HOSTNAME}
153
+ )
154
+
155
+ CONFIG_MISSING_METADATA_PROJECT_ID = %(
156
+ zone #{CUSTOM_ZONE}
157
+ vm_id #{CUSTOM_VM_ID}
158
+ )
159
+ CONFIG_MISSING_METADATA_ZONE = %(
160
+ project_id #{CUSTOM_PROJECT_ID}
161
+ vm_id #{CUSTOM_VM_ID}
162
+ )
163
+ CONFIG_MISSING_METADATA_VM_ID = %(
164
+ project_id #{CUSTOM_PROJECT_ID}
165
+ zone #{CUSTOM_ZONE}
166
+ )
167
+ CONFIG_MISSING_METADATA_ALL = %(
168
+ )
169
+
170
+ CONFIG_EC2_PROJECT_ID = %(
171
+ project_id #{EC2_PROJECT_ID}
172
+ )
173
+
174
+ CONFIG_EC2_PROJECT_ID_AND_CUSTOM_VM_ID = %(
175
+ project_id #{EC2_PROJECT_ID}
176
+ vm_id #{CUSTOM_VM_ID}
177
+ )
178
+
179
+ CONFIG_DATAFLOW = %(
180
+ subservice_name "#{DATAFLOW_CONSTANTS[:service]}"
181
+ labels {
182
+ "#{DATAFLOW_CONSTANTS[:service]}/region" : "#{DATAFLOW_REGION}",
183
+ "#{DATAFLOW_CONSTANTS[:service]}/job_name" : "#{DATAFLOW_JOB_NAME}",
184
+ "#{DATAFLOW_CONSTANTS[:service]}/job_id" : "#{DATAFLOW_JOB_ID}"
185
+ }
186
+ label_map { "step": "#{DATAFLOW_CONSTANTS[:service]}/step_id" }
187
+ )
188
+
189
+ CONFIG_ML = %(
190
+ subservice_name "#{ML_CONSTANTS[:service]}"
191
+ labels {
192
+ "#{ML_CONSTANTS[:service]}/job_id" : "#{ML_JOB_ID}",
193
+ "#{ML_CONSTANTS[:service]}/task_name" : "#{ML_TASK_NAME}",
194
+ "#{ML_CONSTANTS[:service]}/trial_id" : "#{ML_TRIAL_ID}"
195
+ }
196
+ label_map { "name": "#{ML_CONSTANTS[:service]}/job_id/log_area" }
197
+ )
198
+
199
+ # Service configurations for various services.
200
+ COMPUTE_PARAMS = {
201
+ resource: {
202
+ type: COMPUTE_CONSTANTS[:resource_type],
203
+ labels: {
204
+ 'instance_id' => VM_ID,
205
+ 'zone' => ZONE
206
+ }
207
+ },
208
+ log_name: 'test',
209
+ project_id: PROJECT_ID,
210
+ labels: {
211
+ "#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME
212
+ }
213
+ }
214
+
215
+ VMENGINE_PARAMS = {
216
+ resource: {
217
+ type: APPENGINE_CONSTANTS[:resource_type],
218
+ labels: {
219
+ 'module_id' => MANAGED_VM_BACKEND_NAME,
220
+ 'version_id' => MANAGED_VM_BACKEND_VERSION
221
+ }
222
+ },
223
+ log_name: "#{APPENGINE_CONSTANTS[:service]}%2Ftest",
224
+ project_id: PROJECT_ID,
225
+ labels: {
226
+ "#{COMPUTE_CONSTANTS[:service]}/resource_id" => VM_ID,
227
+ "#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME,
228
+ "#{COMPUTE_CONSTANTS[:service]}/zone" => ZONE
229
+ }
230
+ }
231
+
232
+ CONTAINER_TAG = "kubernetes.#{CONTAINER_POD_NAME}_" \
233
+ "#{CONTAINER_NAMESPACE_NAME}_#{CONTAINER_CONTAINER_NAME}"
234
+
235
+ CONTAINER_FROM_METADATA_PARAMS = {
236
+ resource: {
237
+ type: CONTAINER_CONSTANTS[:resource_type],
238
+ labels: {
239
+ 'cluster_name' => CONTAINER_CLUSTER_NAME,
240
+ 'namespace_id' => CONTAINER_NAMESPACE_ID,
241
+ 'instance_id' => VM_ID,
242
+ 'pod_id' => CONTAINER_POD_ID,
243
+ 'container_name' => CONTAINER_CONTAINER_NAME,
244
+ 'zone' => ZONE
245
+ }
246
+ },
247
+ log_name: CONTAINER_CONTAINER_NAME,
248
+ project_id: PROJECT_ID,
249
+ labels: {
250
+ "#{CONTAINER_CONSTANTS[:service]}/namespace_name" =>
251
+ CONTAINER_NAMESPACE_NAME,
252
+ "#{CONTAINER_CONSTANTS[:service]}/pod_name" => CONTAINER_POD_NAME,
253
+ "#{CONTAINER_CONSTANTS[:service]}/stream" => CONTAINER_STREAM,
254
+ "label/#{CONTAINER_LABEL_KEY}" => CONTAINER_LABEL_VALUE,
255
+ "#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME
256
+ }
257
+ }
258
+
259
+ # Almost the same as from metadata, but namespace_id and pod_id come from
260
+ # namespace and pod names.
261
+ CONTAINER_FROM_TAG_PARAMS = {
262
+ resource: {
263
+ type: CONTAINER_CONSTANTS[:resource_type],
264
+ labels: {
265
+ 'cluster_name' => CONTAINER_CLUSTER_NAME,
266
+ 'namespace_id' => CONTAINER_NAMESPACE_NAME,
267
+ 'instance_id' => VM_ID,
268
+ 'pod_id' => CONTAINER_POD_NAME,
269
+ 'container_name' => CONTAINER_CONTAINER_NAME,
270
+ 'zone' => ZONE
271
+ }
272
+ },
273
+ log_name: CONTAINER_CONTAINER_NAME,
274
+ project_id: PROJECT_ID,
275
+ labels: {
276
+ "#{CONTAINER_CONSTANTS[:service]}/namespace_name" =>
277
+ CONTAINER_NAMESPACE_NAME,
278
+ "#{CONTAINER_CONSTANTS[:service]}/pod_name" => CONTAINER_POD_NAME,
279
+ "#{CONTAINER_CONSTANTS[:service]}/stream" => CONTAINER_STREAM,
280
+ "#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME
281
+ }
282
+ }
283
+
284
+ CLOUDFUNCTIONS_TAG = "kubernetes.#{CLOUDFUNCTIONS_POD_NAME}_" \
285
+ "#{CLOUDFUNCTIONS_NAMESPACE_NAME}_" \
286
+ "#{CLOUDFUNCTIONS_CONTAINER_NAME}"
287
+
288
+ CLOUDFUNCTIONS_PARAMS = {
289
+ resource: {
290
+ type: CLOUDFUNCTIONS_CONSTANTS[:resource_type],
291
+ labels: {
292
+ 'function_name' => CLOUDFUNCTIONS_FUNCTION_NAME,
293
+ 'region' => CLOUDFUNCTIONS_REGION
294
+ }
295
+ },
296
+ log_name: 'cloud-functions',
297
+ project_id: PROJECT_ID,
298
+ labels: {
299
+ 'execution_id' => CLOUDFUNCTIONS_EXECUTION_ID,
300
+ "#{CONTAINER_CONSTANTS[:service]}/instance_id" => VM_ID,
301
+ "#{CONTAINER_CONSTANTS[:service]}/cluster_name" =>
302
+ CLOUDFUNCTIONS_CLUSTER_NAME,
303
+ "#{COMPUTE_CONSTANTS[:service]}/resource_id" => VM_ID,
304
+ "#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME,
305
+ "#{COMPUTE_CONSTANTS[:service]}/zone" => ZONE
306
+ }
307
+ }
308
+
309
+ CLOUDFUNCTIONS_TEXT_NOT_MATCHED_PARAMS = {
310
+ resource: {
311
+ type: CLOUDFUNCTIONS_CONSTANTS[:resource_type],
312
+ labels: {
313
+ 'function_name' => CLOUDFUNCTIONS_FUNCTION_NAME,
314
+ 'region' => CLOUDFUNCTIONS_REGION
315
+ }
316
+ },
317
+ log_name: 'cloud-functions',
318
+ project_id: PROJECT_ID,
319
+ labels: {
320
+ "#{CONTAINER_CONSTANTS[:service]}/instance_id" => VM_ID,
321
+ "#{CONTAINER_CONSTANTS[:service]}/cluster_name" =>
322
+ CLOUDFUNCTIONS_CLUSTER_NAME,
323
+ "#{COMPUTE_CONSTANTS[:service]}/resource_id" => VM_ID,
324
+ "#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME,
325
+ "#{COMPUTE_CONSTANTS[:service]}/zone" => ZONE
326
+ }
327
+ }
328
+
329
+ DATAFLOW_PARAMS = {
330
+ resource: {
331
+ type: DATAFLOW_CONSTANTS[:resource_type],
332
+ labels: {
333
+ 'job_name' => DATAFLOW_JOB_NAME,
334
+ 'job_id' => DATAFLOW_JOB_ID,
335
+ 'step_id' => DATAFLOW_STEP_ID,
336
+ 'region' => DATAFLOW_REGION
337
+ }
338
+ },
339
+ log_name: DATAFLOW_TAG,
340
+ project_id: PROJECT_ID,
341
+ labels: {
342
+ "#{COMPUTE_CONSTANTS[:service]}/resource_id" => VM_ID,
343
+ "#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME,
344
+ "#{COMPUTE_CONSTANTS[:service]}/zone" => ZONE
345
+ }
346
+ }
347
+
348
+ DATAPROC_PARAMS = {
349
+ resource: {
350
+ type: DATAPROC_CONSTANTS[:resource_type],
351
+ labels: {
352
+ 'cluster_name' => DATAPROC_CLUSTER_NAME,
353
+ 'cluster_uuid' => DATAPROC_CLUSTER_UUID,
354
+ 'region' => DATAPROC_REGION
355
+ }
356
+ },
357
+ log_name: 'test',
358
+ project_id: PROJECT_ID,
359
+ labels: {
360
+ "#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME,
361
+ "#{COMPUTE_CONSTANTS[:service]}/resource_id" => VM_ID,
362
+ "#{COMPUTE_CONSTANTS[:service]}/zone" => ZONE
363
+ }
364
+ }
365
+
366
+ ML_PARAMS = {
367
+ resource: {
368
+ type: ML_CONSTANTS[:resource_type],
369
+ labels: {
370
+ 'job_id' => ML_JOB_ID,
371
+ 'task_name' => ML_TASK_NAME
372
+ }
373
+ },
374
+ log_name: ML_TAG,
375
+ project_id: PROJECT_ID,
376
+ labels: {
377
+ "#{ML_CONSTANTS[:service]}/trial_id" => ML_TRIAL_ID,
378
+ "#{ML_CONSTANTS[:service]}/job_id/log_area" => ML_LOG_AREA,
379
+ "#{COMPUTE_CONSTANTS[:service]}/resource_id" => VM_ID,
380
+ "#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME,
381
+ "#{COMPUTE_CONSTANTS[:service]}/zone" => ZONE
382
+ }
383
+ }
384
+
385
+ CUSTOM_PARAMS = {
386
+ resource: {
387
+ type: COMPUTE_CONSTANTS[:resource_type],
388
+ labels: {
389
+ 'instance_id' => CUSTOM_VM_ID,
390
+ 'zone' => CUSTOM_ZONE
391
+ }
392
+ },
393
+ log_name: 'test',
394
+ project_id: CUSTOM_PROJECT_ID,
395
+ labels: {
396
+ "#{COMPUTE_CONSTANTS[:service]}/resource_name" => CUSTOM_HOSTNAME
397
+ }
398
+ }
399
+
400
+ EC2_PARAMS = {
401
+ resource: {
402
+ type: EC2_CONSTANTS[:resource_type],
403
+ labels: {
404
+ 'instance_id' => EC2_VM_ID,
405
+ 'region' => EC2_PREFIXED_ZONE,
406
+ 'aws_account' => EC2_ACCOUNT_ID
407
+ }
408
+ },
409
+ log_name: 'test',
410
+ project_id: EC2_PROJECT_ID,
411
+ labels: {
412
+ "#{EC2_CONSTANTS[:service]}/resource_name" => HOSTNAME
413
+ }
414
+ }
415
+
416
+ HTTP_REQUEST_MESSAGE = {
417
+ 'requestMethod' => 'POST',
418
+ 'requestUrl' => 'http://example/',
419
+ 'requestSize' => 210,
420
+ 'status' => 200,
421
+ 'responseSize' => 65,
422
+ 'userAgent' => 'USER AGENT 1.0',
423
+ 'remoteIp' => '55.55.55.55',
424
+ 'referer' => 'http://referer/',
425
+ 'cacheHit' => true,
426
+ 'cacheValidatedWithOriginServer' => true
427
+ }
428
+
429
+ # Tags and their sanitized and encoded version.
430
+ VALID_TAGS = {
431
+ 'test' => 'test',
432
+ 'germanß' => 'german%C3%9F',
433
+ 'chinese中' => 'chinese%E4%B8%AD',
434
+ 'specialCharacter/_-.' => 'specialCharacter%2F_-.',
435
+ 'abc@&^$*' => 'abc%40%26%5E%24%2A',
436
+ '@&^$*' => '%40%26%5E%24%2A'
437
+ }
438
+ INVALID_TAGS = {
439
+ # Non-string tags.
440
+ 123 => '123',
441
+ 1.23 => '1.23',
442
+ [1, 2, 3] => '%5B1%2C%202%2C%203%5D',
443
+ { key: 'value' } => '%7B%22key%22%3D%3E%22value%22%7D',
444
+ # Non-utf8 string tags.
445
+ "nonutf8#{[0x92].pack('C*')}" => 'nonutf8%20',
446
+ "abc#{[0x92].pack('C*')}" => 'abc%20',
447
+ "#{[0x92].pack('C*')}" => '%20',
448
+ # Empty string tag.
449
+ '' => '_'
450
+ }
451
+ ALL_TAGS = VALID_TAGS.merge(INVALID_TAGS)
452
+ end
@@ -28,18 +28,18 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
28
28
  setup_gce_metadata_stubs
29
29
  # The API Client should not retry this and the plugin should consume
30
30
  # the exception.
31
- stub_request(:post, uri_for_log(COMPUTE_PARAMS))
31
+ stub_request(:post, WRITE_LOG_ENTRIES_URI)
32
32
  .to_return(status: 400, body: 'Bad Request')
33
33
  d = create_driver
34
34
  d.emit('message' => log_entry(0))
35
35
  d.run
36
- assert_requested(:post, uri_for_log(COMPUTE_PARAMS), times: 1)
36
+ assert_requested(:post, WRITE_LOG_ENTRIES_URI, times: 1)
37
37
  end
38
38
 
39
39
  # All credentials errors resolve to a 401.
40
40
  def test_client_401
41
41
  setup_gce_metadata_stubs
42
- stub_request(:post, uri_for_log(COMPUTE_PARAMS))
42
+ stub_request(:post, WRITE_LOG_ENTRIES_URI)
43
43
  .to_return(status: 401, body: 'Unauthorized')
44
44
  d = create_driver
45
45
  d.emit('message' => log_entry(0))
@@ -48,14 +48,14 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
48
48
  rescue Google::Apis::AuthorizationError => error
49
49
  assert_equal 'Unauthorized', error.message
50
50
  end
51
- assert_requested(:post, uri_for_log(COMPUTE_PARAMS), times: 2)
51
+ assert_requested(:post, WRITE_LOG_ENTRIES_URI, times: 2)
52
52
  end
53
53
 
54
54
  def test_server_error
55
55
  setup_gce_metadata_stubs
56
56
  # The API client should retry this once, then throw an exception which
57
57
  # gets propagated through the plugin.
58
- stub_request(:post, uri_for_log(COMPUTE_PARAMS))
58
+ stub_request(:post, WRITE_LOG_ENTRIES_URI)
59
59
  .to_return(status: 500, body: 'Server Error')
60
60
  d = create_driver
61
61
  d.emit('message' => log_entry(0))
@@ -66,25 +66,55 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
66
66
  assert_equal 'Server error', error.message
67
67
  exception_count += 1
68
68
  end
69
- assert_requested(:post, uri_for_log(COMPUTE_PARAMS), times: 1)
69
+ assert_requested(:post, WRITE_LOG_ENTRIES_URI, times: 1)
70
70
  assert_equal 1, exception_count
71
71
  end
72
72
 
73
- def test_http_request_from_record_with_referer_nil
73
+ # TODO: The code in the non-gRPC and gRPC tests is nearly identical.
74
+ # Refactor and remove duplication.
75
+ # TODO: Use status codes instead of int literals.
76
+ def test_prometheus_metrics
74
77
  setup_gce_metadata_stubs
75
- setup_logging_stubs do
76
- d = create_driver
77
- d.emit('httpRequest' => http_request_message_with_nil_referer)
78
- d.run
79
- end
80
- verify_log_entries(1, COMPUTE_PARAMS, 'httpRequest') do |entry|
81
- # The request we send to Logging API has json like:
82
- # "httpRequest": { "referer": null }, but eventually the stored LogEntry
83
- # would be "httpRequest": {}, since 'referer' is defined as a string in
84
- # the proto.
85
- assert_equal http_request_message_with_nil_referer,
86
- entry['httpRequest'], entry
87
- assert_nil get_fields(entry['structPayload'])['httpRequest'], entry
78
+ [
79
+ # Single successful request.
80
+ [200, 1, 1, [1, 0, 1, 0]],
81
+ # Several successful requests.
82
+ [200, 2, 1, [2, 0, 2, 0]],
83
+ # Single successful request with several entries.
84
+ [200, 1, 2, [1, 0, 2, 0]],
85
+ # Single failed request that causes logs to be dropped.
86
+ [401, 1, 1, [0, 1, 0, 1]],
87
+ # Single failed request that escalates without logs being dropped.
88
+ [500, 1, 1, [0, 1, 0, 0]]
89
+ ].each do |code, request_count, entry_count, metric_values|
90
+ setup_prometheus
91
+ # TODO: Do this as part of setup_logging_stubs.
92
+ stub_request(:post, WRITE_LOG_ENTRIES_URI)
93
+ .to_return(status: code, body: 'Some Message')
94
+ (1..request_count).each do
95
+ d = create_driver(PROMETHEUS_ENABLE_CONFIG)
96
+ (1..entry_count).each do |i|
97
+ d.emit('message' => log_entry(i.to_s))
98
+ end
99
+ # rubocop:disable Lint/HandleExceptions
100
+ begin
101
+ d.run
102
+ rescue Google::Apis::AuthorizationError
103
+ rescue Google::Apis::ServerError
104
+ end
105
+ # rubocop:enable Lint/HandleExceptions
106
+ end
107
+ successful_requests_count, failed_requests_count,
108
+ ingested_entries_count, dropped_entries_count = metric_values
109
+ assert_prometheus_metric_value(:stackdriver_successful_requests_count,
110
+ successful_requests_count, grpc: false)
111
+ assert_prometheus_metric_value(:stackdriver_failed_requests_count,
112
+ failed_requests_count,
113
+ grpc: false, code: code)
114
+ assert_prometheus_metric_value(:stackdriver_ingested_entries_count,
115
+ ingested_entries_count)
116
+ assert_prometheus_metric_value(:stackdriver_dropped_entries_count,
117
+ dropped_entries_count)
88
118
  end
89
119
  end
90
120
 
@@ -113,7 +143,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
113
143
  verify_index = 0
114
144
  verify_log_entries(emit_index, COMPUTE_PARAMS) do |entry|
115
145
  assert_equal expected_severity[verify_index],
116
- entry['metadata']['severity'], entry
146
+ entry['severity'], entry
117
147
  verify_index += 1
118
148
  end
119
149
  end
@@ -181,7 +211,6 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
181
211
  assert_equal('ERROR', test_obj.parse_severity('ERROR '))
182
212
  assert_equal('ERROR', test_obj.parse_severity(' ERROR '))
183
213
  assert_equal('ERROR', test_obj.parse_severity("\t ERROR "))
184
-
185
214
  # space in the middle should not be stripped.
186
215
  assert_equal('DEFAULT', test_obj.parse_severity('ER ROR'))
187
216
 
@@ -208,38 +237,25 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
208
237
  d.run
209
238
  end
210
239
  verify_log_entries(1, COMPUTE_PARAMS) do |entry|
211
- assert_equal timestamp, entry['metadata']['timestamp'],
212
- "Test with timestamp '#{timestamp}' failed for " \
213
- "entry: '#{entry}'."
240
+ assert_equal timestamp, entry['timestamp'], 'Test with timestamp ' \
241
+ "'#{timestamp}' failed for entry: '#{entry}'."
214
242
  end
215
243
  end
216
244
  end
217
245
 
218
246
  private
219
247
 
248
+ WRITE_LOG_ENTRIES_URI = 'https://logging.googleapis.com/v2beta1/entries:write'
249
+
220
250
  def rename_key(hash, old_key, new_key)
221
251
  hash.merge(new_key => hash[old_key]).reject { |k, _| k == old_key }
222
252
  end
223
253
 
224
- # The REST path uses old bindings that were generated prior to the field
225
- # rename, and has to use the old name, which is 'validatedWithOriginServer'.
226
- def http_request_message
227
- rename_key(super, 'cacheValidatedWithOriginServer',
228
- 'validatedWithOriginServer')
229
- end
230
-
231
254
  # Set up http stubs to mock the external calls.
232
- def setup_logging_stubs(override_stub_params = nil)
233
- stub_params = override_stub_params || \
234
- [COMPUTE_PARAMS, VMENGINE_PARAMS, CONTAINER_FROM_TAG_PARAMS,
235
- CONTAINER_FROM_METADATA_PARAMS, CLOUDFUNCTIONS_PARAMS,
236
- CUSTOM_PARAMS, EC2_PARAMS]
237
- stub_params.each do |params|
238
- stub_request(:post, uri_for_log(params)).to_return do |request|
239
- log_name = "projects/#{params[:project_id]}/logs/#{params[:log_name]}"
240
- @logs_sent << JSON.parse(request.body).merge('logName' => log_name)
241
- { body: '' }
242
- end
255
+ def setup_logging_stubs
256
+ stub_request(:post, WRITE_LOG_ENTRIES_URI).to_return do |request|
257
+ @logs_sent << JSON.parse(request.body)
258
+ { body: '' }
243
259
  end
244
260
  yield
245
261
  end
@@ -270,9 +286,9 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
270
286
  end
271
287
  end
272
288
 
273
- # Get the fields of the struct payload.
274
- def get_fields(struct_payload)
275
- struct_payload
289
+ # Get the fields of the payload.
290
+ def get_fields(payload)
291
+ payload
276
292
  end
277
293
 
278
294
  # Get the value of a struct field.