vmik-fluent-plugin-google-cloud 0.5.5 → 0.6.4.pre.alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/Gemfile.lock +159 -0
- data/fluent-plugin-google-cloud.gemspec +9 -7
- data/lib/fluent/plugin/monitoring.rb +67 -0
- data/lib/fluent/plugin/out_google_cloud.rb +481 -224
- data/test/plugin/base_test.rb +263 -447
- data/test/plugin/constants.rb +452 -0
- data/test/plugin/test_out_google_cloud.rb +62 -46
- data/test/plugin/test_out_google_cloud_grpc.rb +66 -61
- metadata +83 -109
- data/lib/google/logging/type/http_request_pb.rb +0 -30
- data/lib/google/logging/type/log_severity_pb.rb +0 -26
- data/lib/google/logging/v1/log_entry_pb.rb +0 -52
- data/lib/google/logging/v1/logging_pb.rb +0 -84
- data/lib/google/logging/v1/logging_services_pb.rb +0 -150
@@ -0,0 +1,452 @@
|
|
1
|
+
# Copyright 2017 Google Inc. All rights reserved.
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
# Constants used by unit tests for Google Cloud Logging plugin.
|
16
|
+
module Constants
|
17
|
+
include Fluent::GoogleCloudOutput::Constants
|
18
|
+
|
19
|
+
# Generic attributes.
|
20
|
+
HOSTNAME = Socket.gethostname
|
21
|
+
|
22
|
+
# TODO(qingling128) Separate constants into different submodules.
|
23
|
+
# Attributes used for the GCE metadata service.
|
24
|
+
PROJECT_ID = 'test-project-id'
|
25
|
+
ZONE = 'us-central1-b'
|
26
|
+
FULLY_QUALIFIED_ZONE = 'projects/' + PROJECT_ID + '/zones/' + ZONE
|
27
|
+
VM_ID = '9876543210'
|
28
|
+
|
29
|
+
# Attributes used for custom (overridden) configs.
|
30
|
+
CUSTOM_PROJECT_ID = 'test-custom-project-id'
|
31
|
+
CUSTOM_ZONE = 'us-custom-central1-b'
|
32
|
+
CUSTOM_FULLY_QUALIFIED_ZONE = 'projects/' + PROJECT_ID + '/zones/' + ZONE
|
33
|
+
CUSTOM_VM_ID = 'C9876543210'
|
34
|
+
CUSTOM_HOSTNAME = 'custom.hostname.org'
|
35
|
+
|
36
|
+
# Attributes used for the EC2 metadata service.
|
37
|
+
EC2_PROJECT_ID = 'test-ec2-project-id'
|
38
|
+
EC2_ZONE = 'us-west-2b'
|
39
|
+
EC2_PREFIXED_ZONE = 'aws:' + EC2_ZONE
|
40
|
+
EC2_VM_ID = 'i-81c16767'
|
41
|
+
EC2_ACCOUNT_ID = '123456789012'
|
42
|
+
|
43
|
+
# The formatting here matches the format used on the VM.
|
44
|
+
EC2_IDENTITY_DOCUMENT = %({
|
45
|
+
"accountId" : "#{EC2_ACCOUNT_ID}",
|
46
|
+
"availabilityZone" : "#{EC2_ZONE}",
|
47
|
+
"instanceId" : "#{EC2_VM_ID}"
|
48
|
+
})
|
49
|
+
|
50
|
+
# Managed VMs specific labels.
|
51
|
+
MANAGED_VM_BACKEND_NAME = 'default'
|
52
|
+
MANAGED_VM_BACKEND_VERSION = 'guestbook2.0'
|
53
|
+
|
54
|
+
# Container Engine / Kubernetes specific labels.
|
55
|
+
CONTAINER_CLUSTER_NAME = 'cluster-1'
|
56
|
+
CONTAINER_NAMESPACE_ID = '898268c8-4a36-11e5-9d81-42010af0194c'
|
57
|
+
CONTAINER_NAMESPACE_NAME = 'kube-system'
|
58
|
+
CONTAINER_POD_ID = 'cad3c3c4-4b9c-11e5-9d81-42010af0194c'
|
59
|
+
CONTAINER_POD_NAME = 'redis-master-c0l82.foo.bar'
|
60
|
+
CONTAINER_CONTAINER_NAME = 'redis'
|
61
|
+
CONTAINER_LABEL_KEY = 'component'
|
62
|
+
CONTAINER_LABEL_VALUE = 'redis-component'
|
63
|
+
CONTAINER_STREAM = 'stdout'
|
64
|
+
CONTAINER_SEVERITY = 'INFO'
|
65
|
+
# Timestamp for 1234567890 seconds and 987654321 nanoseconds since epoch.
|
66
|
+
CONTAINER_TIMESTAMP = '2009-02-13T23:31:30.987654321Z'
|
67
|
+
CONTAINER_SECONDS_EPOCH = 1_234_567_890
|
68
|
+
CONTAINER_NANOS = 987_654_321
|
69
|
+
|
70
|
+
# Cloud Functions specific labels.
|
71
|
+
CLOUDFUNCTIONS_FUNCTION_NAME = '$My_Function.Name-@1'
|
72
|
+
CLOUDFUNCTIONS_REGION = 'us-central1'
|
73
|
+
CLOUDFUNCTIONS_EXECUTION_ID = '123-0'
|
74
|
+
CLOUDFUNCTIONS_CLUSTER_NAME = 'cluster-1'
|
75
|
+
CLOUDFUNCTIONS_NAMESPACE_NAME = 'default'
|
76
|
+
CLOUDFUNCTIONS_POD_NAME = 'd.dc.myu.uc.functionp.pc.name-a.a1.987-c0l82'
|
77
|
+
CLOUDFUNCTIONS_CONTAINER_NAME = 'worker'
|
78
|
+
|
79
|
+
# Dataflow specific labels.
|
80
|
+
DATAFLOW_REGION = 'us-central1'
|
81
|
+
DATAFLOW_JOB_NAME = 'job_name_1'
|
82
|
+
DATAFLOW_JOB_ID = 'job_id_1'
|
83
|
+
DATAFLOW_STEP_ID = 'step_1'
|
84
|
+
DATAFLOW_TAG = 'dataflow.googleapis.com/worker'
|
85
|
+
|
86
|
+
# Dataproc specific labels.
|
87
|
+
DATAPROC_CLUSTER_NAME = 'test-cluster'
|
88
|
+
DATAPROC_CLUSTER_UUID = '00000000-0000-0000-0000-000000000000'
|
89
|
+
DATAPROC_REGION = 'unittest'
|
90
|
+
|
91
|
+
# ML specific labels.
|
92
|
+
ML_REGION = 'us-central1'
|
93
|
+
ML_JOB_ID = 'job_name_1'
|
94
|
+
ML_TASK_NAME = 'task_name_1'
|
95
|
+
ML_TRIAL_ID = 'trial_id_1'
|
96
|
+
ML_LOG_AREA = 'log_area_1'
|
97
|
+
ML_TAG = 'master-replica-0'
|
98
|
+
|
99
|
+
# Parameters used for authentication.
|
100
|
+
AUTH_GRANT_TYPE = 'urn:ietf:params:oauth:grant-type:jwt-bearer'
|
101
|
+
FAKE_AUTH_TOKEN = 'abc123'
|
102
|
+
|
103
|
+
# Information about test credentials files.
|
104
|
+
# path: Path to the credentials file.
|
105
|
+
# project_id: ID of the project, which must correspond to the file contents.
|
106
|
+
IAM_CREDENTIALS = {
|
107
|
+
path: 'test/plugin/data/iam-credentials.json',
|
108
|
+
project_id: 'fluent-test-project'
|
109
|
+
}
|
110
|
+
LEGACY_CREDENTIALS = {
|
111
|
+
path: 'test/plugin/data/credentials.json',
|
112
|
+
project_id: '847859579879'
|
113
|
+
}
|
114
|
+
INVALID_CREDENTIALS = {
|
115
|
+
path: 'test/plugin/data/invalid_credentials.json',
|
116
|
+
project_id: ''
|
117
|
+
}
|
118
|
+
|
119
|
+
# Configuration files for various test scenarios.
|
120
|
+
APPLICATION_DEFAULT_CONFIG = %(
|
121
|
+
)
|
122
|
+
|
123
|
+
# rubocop:disable Metrics/LineLength
|
124
|
+
PRIVATE_KEY_CONFIG = %(
|
125
|
+
auth_method private_key
|
126
|
+
private_key_email 271661262351-ft99kc9kjro9rrihq3k2n3s2inbplu0q@developer.gserviceaccount.com
|
127
|
+
private_key_path test/plugin/data/c31e573fd7f62ed495c9ca3821a5a85cb036dee1-privatekey.p12
|
128
|
+
)
|
129
|
+
# rubocop:enable Metrics/LineLength
|
130
|
+
|
131
|
+
REQUIRE_VALID_TAGS_CONFIG = %(
|
132
|
+
require_valid_tags true
|
133
|
+
)
|
134
|
+
|
135
|
+
NO_METADATA_SERVICE_CONFIG = %(
|
136
|
+
use_metadata_service false
|
137
|
+
)
|
138
|
+
|
139
|
+
NO_DETECT_SUBSERVICE_CONFIG = %(
|
140
|
+
detect_subservice false
|
141
|
+
)
|
142
|
+
|
143
|
+
PROMETHEUS_ENABLE_CONFIG = %(
|
144
|
+
monitoring_enabled true
|
145
|
+
monitoring_type prometheus
|
146
|
+
)
|
147
|
+
|
148
|
+
CUSTOM_METADATA_CONFIG = %(
|
149
|
+
project_id #{CUSTOM_PROJECT_ID}
|
150
|
+
zone #{CUSTOM_ZONE}
|
151
|
+
vm_id #{CUSTOM_VM_ID}
|
152
|
+
vm_name #{CUSTOM_HOSTNAME}
|
153
|
+
)
|
154
|
+
|
155
|
+
CONFIG_MISSING_METADATA_PROJECT_ID = %(
|
156
|
+
zone #{CUSTOM_ZONE}
|
157
|
+
vm_id #{CUSTOM_VM_ID}
|
158
|
+
)
|
159
|
+
CONFIG_MISSING_METADATA_ZONE = %(
|
160
|
+
project_id #{CUSTOM_PROJECT_ID}
|
161
|
+
vm_id #{CUSTOM_VM_ID}
|
162
|
+
)
|
163
|
+
CONFIG_MISSING_METADATA_VM_ID = %(
|
164
|
+
project_id #{CUSTOM_PROJECT_ID}
|
165
|
+
zone #{CUSTOM_ZONE}
|
166
|
+
)
|
167
|
+
CONFIG_MISSING_METADATA_ALL = %(
|
168
|
+
)
|
169
|
+
|
170
|
+
CONFIG_EC2_PROJECT_ID = %(
|
171
|
+
project_id #{EC2_PROJECT_ID}
|
172
|
+
)
|
173
|
+
|
174
|
+
CONFIG_EC2_PROJECT_ID_AND_CUSTOM_VM_ID = %(
|
175
|
+
project_id #{EC2_PROJECT_ID}
|
176
|
+
vm_id #{CUSTOM_VM_ID}
|
177
|
+
)
|
178
|
+
|
179
|
+
CONFIG_DATAFLOW = %(
|
180
|
+
subservice_name "#{DATAFLOW_CONSTANTS[:service]}"
|
181
|
+
labels {
|
182
|
+
"#{DATAFLOW_CONSTANTS[:service]}/region" : "#{DATAFLOW_REGION}",
|
183
|
+
"#{DATAFLOW_CONSTANTS[:service]}/job_name" : "#{DATAFLOW_JOB_NAME}",
|
184
|
+
"#{DATAFLOW_CONSTANTS[:service]}/job_id" : "#{DATAFLOW_JOB_ID}"
|
185
|
+
}
|
186
|
+
label_map { "step": "#{DATAFLOW_CONSTANTS[:service]}/step_id" }
|
187
|
+
)
|
188
|
+
|
189
|
+
CONFIG_ML = %(
|
190
|
+
subservice_name "#{ML_CONSTANTS[:service]}"
|
191
|
+
labels {
|
192
|
+
"#{ML_CONSTANTS[:service]}/job_id" : "#{ML_JOB_ID}",
|
193
|
+
"#{ML_CONSTANTS[:service]}/task_name" : "#{ML_TASK_NAME}",
|
194
|
+
"#{ML_CONSTANTS[:service]}/trial_id" : "#{ML_TRIAL_ID}"
|
195
|
+
}
|
196
|
+
label_map { "name": "#{ML_CONSTANTS[:service]}/job_id/log_area" }
|
197
|
+
)
|
198
|
+
|
199
|
+
# Service configurations for various services.
|
200
|
+
COMPUTE_PARAMS = {
|
201
|
+
resource: {
|
202
|
+
type: COMPUTE_CONSTANTS[:resource_type],
|
203
|
+
labels: {
|
204
|
+
'instance_id' => VM_ID,
|
205
|
+
'zone' => ZONE
|
206
|
+
}
|
207
|
+
},
|
208
|
+
log_name: 'test',
|
209
|
+
project_id: PROJECT_ID,
|
210
|
+
labels: {
|
211
|
+
"#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME
|
212
|
+
}
|
213
|
+
}
|
214
|
+
|
215
|
+
VMENGINE_PARAMS = {
|
216
|
+
resource: {
|
217
|
+
type: APPENGINE_CONSTANTS[:resource_type],
|
218
|
+
labels: {
|
219
|
+
'module_id' => MANAGED_VM_BACKEND_NAME,
|
220
|
+
'version_id' => MANAGED_VM_BACKEND_VERSION
|
221
|
+
}
|
222
|
+
},
|
223
|
+
log_name: "#{APPENGINE_CONSTANTS[:service]}%2Ftest",
|
224
|
+
project_id: PROJECT_ID,
|
225
|
+
labels: {
|
226
|
+
"#{COMPUTE_CONSTANTS[:service]}/resource_id" => VM_ID,
|
227
|
+
"#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME,
|
228
|
+
"#{COMPUTE_CONSTANTS[:service]}/zone" => ZONE
|
229
|
+
}
|
230
|
+
}
|
231
|
+
|
232
|
+
CONTAINER_TAG = "kubernetes.#{CONTAINER_POD_NAME}_" \
|
233
|
+
"#{CONTAINER_NAMESPACE_NAME}_#{CONTAINER_CONTAINER_NAME}"
|
234
|
+
|
235
|
+
CONTAINER_FROM_METADATA_PARAMS = {
|
236
|
+
resource: {
|
237
|
+
type: CONTAINER_CONSTANTS[:resource_type],
|
238
|
+
labels: {
|
239
|
+
'cluster_name' => CONTAINER_CLUSTER_NAME,
|
240
|
+
'namespace_id' => CONTAINER_NAMESPACE_ID,
|
241
|
+
'instance_id' => VM_ID,
|
242
|
+
'pod_id' => CONTAINER_POD_ID,
|
243
|
+
'container_name' => CONTAINER_CONTAINER_NAME,
|
244
|
+
'zone' => ZONE
|
245
|
+
}
|
246
|
+
},
|
247
|
+
log_name: CONTAINER_CONTAINER_NAME,
|
248
|
+
project_id: PROJECT_ID,
|
249
|
+
labels: {
|
250
|
+
"#{CONTAINER_CONSTANTS[:service]}/namespace_name" =>
|
251
|
+
CONTAINER_NAMESPACE_NAME,
|
252
|
+
"#{CONTAINER_CONSTANTS[:service]}/pod_name" => CONTAINER_POD_NAME,
|
253
|
+
"#{CONTAINER_CONSTANTS[:service]}/stream" => CONTAINER_STREAM,
|
254
|
+
"label/#{CONTAINER_LABEL_KEY}" => CONTAINER_LABEL_VALUE,
|
255
|
+
"#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME
|
256
|
+
}
|
257
|
+
}
|
258
|
+
|
259
|
+
# Almost the same as from metadata, but namespace_id and pod_id come from
|
260
|
+
# namespace and pod names.
|
261
|
+
CONTAINER_FROM_TAG_PARAMS = {
|
262
|
+
resource: {
|
263
|
+
type: CONTAINER_CONSTANTS[:resource_type],
|
264
|
+
labels: {
|
265
|
+
'cluster_name' => CONTAINER_CLUSTER_NAME,
|
266
|
+
'namespace_id' => CONTAINER_NAMESPACE_NAME,
|
267
|
+
'instance_id' => VM_ID,
|
268
|
+
'pod_id' => CONTAINER_POD_NAME,
|
269
|
+
'container_name' => CONTAINER_CONTAINER_NAME,
|
270
|
+
'zone' => ZONE
|
271
|
+
}
|
272
|
+
},
|
273
|
+
log_name: CONTAINER_CONTAINER_NAME,
|
274
|
+
project_id: PROJECT_ID,
|
275
|
+
labels: {
|
276
|
+
"#{CONTAINER_CONSTANTS[:service]}/namespace_name" =>
|
277
|
+
CONTAINER_NAMESPACE_NAME,
|
278
|
+
"#{CONTAINER_CONSTANTS[:service]}/pod_name" => CONTAINER_POD_NAME,
|
279
|
+
"#{CONTAINER_CONSTANTS[:service]}/stream" => CONTAINER_STREAM,
|
280
|
+
"#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME
|
281
|
+
}
|
282
|
+
}
|
283
|
+
|
284
|
+
CLOUDFUNCTIONS_TAG = "kubernetes.#{CLOUDFUNCTIONS_POD_NAME}_" \
|
285
|
+
"#{CLOUDFUNCTIONS_NAMESPACE_NAME}_" \
|
286
|
+
"#{CLOUDFUNCTIONS_CONTAINER_NAME}"
|
287
|
+
|
288
|
+
CLOUDFUNCTIONS_PARAMS = {
|
289
|
+
resource: {
|
290
|
+
type: CLOUDFUNCTIONS_CONSTANTS[:resource_type],
|
291
|
+
labels: {
|
292
|
+
'function_name' => CLOUDFUNCTIONS_FUNCTION_NAME,
|
293
|
+
'region' => CLOUDFUNCTIONS_REGION
|
294
|
+
}
|
295
|
+
},
|
296
|
+
log_name: 'cloud-functions',
|
297
|
+
project_id: PROJECT_ID,
|
298
|
+
labels: {
|
299
|
+
'execution_id' => CLOUDFUNCTIONS_EXECUTION_ID,
|
300
|
+
"#{CONTAINER_CONSTANTS[:service]}/instance_id" => VM_ID,
|
301
|
+
"#{CONTAINER_CONSTANTS[:service]}/cluster_name" =>
|
302
|
+
CLOUDFUNCTIONS_CLUSTER_NAME,
|
303
|
+
"#{COMPUTE_CONSTANTS[:service]}/resource_id" => VM_ID,
|
304
|
+
"#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME,
|
305
|
+
"#{COMPUTE_CONSTANTS[:service]}/zone" => ZONE
|
306
|
+
}
|
307
|
+
}
|
308
|
+
|
309
|
+
CLOUDFUNCTIONS_TEXT_NOT_MATCHED_PARAMS = {
|
310
|
+
resource: {
|
311
|
+
type: CLOUDFUNCTIONS_CONSTANTS[:resource_type],
|
312
|
+
labels: {
|
313
|
+
'function_name' => CLOUDFUNCTIONS_FUNCTION_NAME,
|
314
|
+
'region' => CLOUDFUNCTIONS_REGION
|
315
|
+
}
|
316
|
+
},
|
317
|
+
log_name: 'cloud-functions',
|
318
|
+
project_id: PROJECT_ID,
|
319
|
+
labels: {
|
320
|
+
"#{CONTAINER_CONSTANTS[:service]}/instance_id" => VM_ID,
|
321
|
+
"#{CONTAINER_CONSTANTS[:service]}/cluster_name" =>
|
322
|
+
CLOUDFUNCTIONS_CLUSTER_NAME,
|
323
|
+
"#{COMPUTE_CONSTANTS[:service]}/resource_id" => VM_ID,
|
324
|
+
"#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME,
|
325
|
+
"#{COMPUTE_CONSTANTS[:service]}/zone" => ZONE
|
326
|
+
}
|
327
|
+
}
|
328
|
+
|
329
|
+
DATAFLOW_PARAMS = {
|
330
|
+
resource: {
|
331
|
+
type: DATAFLOW_CONSTANTS[:resource_type],
|
332
|
+
labels: {
|
333
|
+
'job_name' => DATAFLOW_JOB_NAME,
|
334
|
+
'job_id' => DATAFLOW_JOB_ID,
|
335
|
+
'step_id' => DATAFLOW_STEP_ID,
|
336
|
+
'region' => DATAFLOW_REGION
|
337
|
+
}
|
338
|
+
},
|
339
|
+
log_name: DATAFLOW_TAG,
|
340
|
+
project_id: PROJECT_ID,
|
341
|
+
labels: {
|
342
|
+
"#{COMPUTE_CONSTANTS[:service]}/resource_id" => VM_ID,
|
343
|
+
"#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME,
|
344
|
+
"#{COMPUTE_CONSTANTS[:service]}/zone" => ZONE
|
345
|
+
}
|
346
|
+
}
|
347
|
+
|
348
|
+
DATAPROC_PARAMS = {
|
349
|
+
resource: {
|
350
|
+
type: DATAPROC_CONSTANTS[:resource_type],
|
351
|
+
labels: {
|
352
|
+
'cluster_name' => DATAPROC_CLUSTER_NAME,
|
353
|
+
'cluster_uuid' => DATAPROC_CLUSTER_UUID,
|
354
|
+
'region' => DATAPROC_REGION
|
355
|
+
}
|
356
|
+
},
|
357
|
+
log_name: 'test',
|
358
|
+
project_id: PROJECT_ID,
|
359
|
+
labels: {
|
360
|
+
"#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME,
|
361
|
+
"#{COMPUTE_CONSTANTS[:service]}/resource_id" => VM_ID,
|
362
|
+
"#{COMPUTE_CONSTANTS[:service]}/zone" => ZONE
|
363
|
+
}
|
364
|
+
}
|
365
|
+
|
366
|
+
ML_PARAMS = {
|
367
|
+
resource: {
|
368
|
+
type: ML_CONSTANTS[:resource_type],
|
369
|
+
labels: {
|
370
|
+
'job_id' => ML_JOB_ID,
|
371
|
+
'task_name' => ML_TASK_NAME
|
372
|
+
}
|
373
|
+
},
|
374
|
+
log_name: ML_TAG,
|
375
|
+
project_id: PROJECT_ID,
|
376
|
+
labels: {
|
377
|
+
"#{ML_CONSTANTS[:service]}/trial_id" => ML_TRIAL_ID,
|
378
|
+
"#{ML_CONSTANTS[:service]}/job_id/log_area" => ML_LOG_AREA,
|
379
|
+
"#{COMPUTE_CONSTANTS[:service]}/resource_id" => VM_ID,
|
380
|
+
"#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME,
|
381
|
+
"#{COMPUTE_CONSTANTS[:service]}/zone" => ZONE
|
382
|
+
}
|
383
|
+
}
|
384
|
+
|
385
|
+
CUSTOM_PARAMS = {
|
386
|
+
resource: {
|
387
|
+
type: COMPUTE_CONSTANTS[:resource_type],
|
388
|
+
labels: {
|
389
|
+
'instance_id' => CUSTOM_VM_ID,
|
390
|
+
'zone' => CUSTOM_ZONE
|
391
|
+
}
|
392
|
+
},
|
393
|
+
log_name: 'test',
|
394
|
+
project_id: CUSTOM_PROJECT_ID,
|
395
|
+
labels: {
|
396
|
+
"#{COMPUTE_CONSTANTS[:service]}/resource_name" => CUSTOM_HOSTNAME
|
397
|
+
}
|
398
|
+
}
|
399
|
+
|
400
|
+
EC2_PARAMS = {
|
401
|
+
resource: {
|
402
|
+
type: EC2_CONSTANTS[:resource_type],
|
403
|
+
labels: {
|
404
|
+
'instance_id' => EC2_VM_ID,
|
405
|
+
'region' => EC2_PREFIXED_ZONE,
|
406
|
+
'aws_account' => EC2_ACCOUNT_ID
|
407
|
+
}
|
408
|
+
},
|
409
|
+
log_name: 'test',
|
410
|
+
project_id: EC2_PROJECT_ID,
|
411
|
+
labels: {
|
412
|
+
"#{EC2_CONSTANTS[:service]}/resource_name" => HOSTNAME
|
413
|
+
}
|
414
|
+
}
|
415
|
+
|
416
|
+
HTTP_REQUEST_MESSAGE = {
|
417
|
+
'requestMethod' => 'POST',
|
418
|
+
'requestUrl' => 'http://example/',
|
419
|
+
'requestSize' => 210,
|
420
|
+
'status' => 200,
|
421
|
+
'responseSize' => 65,
|
422
|
+
'userAgent' => 'USER AGENT 1.0',
|
423
|
+
'remoteIp' => '55.55.55.55',
|
424
|
+
'referer' => 'http://referer/',
|
425
|
+
'cacheHit' => true,
|
426
|
+
'cacheValidatedWithOriginServer' => true
|
427
|
+
}
|
428
|
+
|
429
|
+
# Tags and their sanitized and encoded version.
|
430
|
+
VALID_TAGS = {
|
431
|
+
'test' => 'test',
|
432
|
+
'germanß' => 'german%C3%9F',
|
433
|
+
'chinese中' => 'chinese%E4%B8%AD',
|
434
|
+
'specialCharacter/_-.' => 'specialCharacter%2F_-.',
|
435
|
+
'abc@&^$*' => 'abc%40%26%5E%24%2A',
|
436
|
+
'@&^$*' => '%40%26%5E%24%2A'
|
437
|
+
}
|
438
|
+
INVALID_TAGS = {
|
439
|
+
# Non-string tags.
|
440
|
+
123 => '123',
|
441
|
+
1.23 => '1.23',
|
442
|
+
[1, 2, 3] => '%5B1%2C%202%2C%203%5D',
|
443
|
+
{ key: 'value' } => '%7B%22key%22%3D%3E%22value%22%7D',
|
444
|
+
# Non-utf8 string tags.
|
445
|
+
"nonutf8#{[0x92].pack('C*')}" => 'nonutf8%20',
|
446
|
+
"abc#{[0x92].pack('C*')}" => 'abc%20',
|
447
|
+
"#{[0x92].pack('C*')}" => '%20',
|
448
|
+
# Empty string tag.
|
449
|
+
'' => '_'
|
450
|
+
}
|
451
|
+
ALL_TAGS = VALID_TAGS.merge(INVALID_TAGS)
|
452
|
+
end
|
@@ -28,18 +28,18 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
28
28
|
setup_gce_metadata_stubs
|
29
29
|
# The API Client should not retry this and the plugin should consume
|
30
30
|
# the exception.
|
31
|
-
stub_request(:post,
|
31
|
+
stub_request(:post, WRITE_LOG_ENTRIES_URI)
|
32
32
|
.to_return(status: 400, body: 'Bad Request')
|
33
33
|
d = create_driver
|
34
34
|
d.emit('message' => log_entry(0))
|
35
35
|
d.run
|
36
|
-
assert_requested(:post,
|
36
|
+
assert_requested(:post, WRITE_LOG_ENTRIES_URI, times: 1)
|
37
37
|
end
|
38
38
|
|
39
39
|
# All credentials errors resolve to a 401.
|
40
40
|
def test_client_401
|
41
41
|
setup_gce_metadata_stubs
|
42
|
-
stub_request(:post,
|
42
|
+
stub_request(:post, WRITE_LOG_ENTRIES_URI)
|
43
43
|
.to_return(status: 401, body: 'Unauthorized')
|
44
44
|
d = create_driver
|
45
45
|
d.emit('message' => log_entry(0))
|
@@ -48,14 +48,14 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
48
48
|
rescue Google::Apis::AuthorizationError => error
|
49
49
|
assert_equal 'Unauthorized', error.message
|
50
50
|
end
|
51
|
-
assert_requested(:post,
|
51
|
+
assert_requested(:post, WRITE_LOG_ENTRIES_URI, times: 2)
|
52
52
|
end
|
53
53
|
|
54
54
|
def test_server_error
|
55
55
|
setup_gce_metadata_stubs
|
56
56
|
# The API client should retry this once, then throw an exception which
|
57
57
|
# gets propagated through the plugin.
|
58
|
-
stub_request(:post,
|
58
|
+
stub_request(:post, WRITE_LOG_ENTRIES_URI)
|
59
59
|
.to_return(status: 500, body: 'Server Error')
|
60
60
|
d = create_driver
|
61
61
|
d.emit('message' => log_entry(0))
|
@@ -66,25 +66,55 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
66
66
|
assert_equal 'Server error', error.message
|
67
67
|
exception_count += 1
|
68
68
|
end
|
69
|
-
assert_requested(:post,
|
69
|
+
assert_requested(:post, WRITE_LOG_ENTRIES_URI, times: 1)
|
70
70
|
assert_equal 1, exception_count
|
71
71
|
end
|
72
72
|
|
73
|
-
|
73
|
+
# TODO: The code in the non-gRPC and gRPC tests is nearly identical.
|
74
|
+
# Refactor and remove duplication.
|
75
|
+
# TODO: Use status codes instead of int literals.
|
76
|
+
def test_prometheus_metrics
|
74
77
|
setup_gce_metadata_stubs
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
#
|
83
|
-
|
84
|
-
#
|
85
|
-
|
86
|
-
|
87
|
-
|
78
|
+
[
|
79
|
+
# Single successful request.
|
80
|
+
[200, 1, 1, [1, 0, 1, 0]],
|
81
|
+
# Several successful requests.
|
82
|
+
[200, 2, 1, [2, 0, 2, 0]],
|
83
|
+
# Single successful request with several entries.
|
84
|
+
[200, 1, 2, [1, 0, 2, 0]],
|
85
|
+
# Single failed request that causes logs to be dropped.
|
86
|
+
[401, 1, 1, [0, 1, 0, 1]],
|
87
|
+
# Single failed request that escalates without logs being dropped.
|
88
|
+
[500, 1, 1, [0, 1, 0, 0]]
|
89
|
+
].each do |code, request_count, entry_count, metric_values|
|
90
|
+
setup_prometheus
|
91
|
+
# TODO: Do this as part of setup_logging_stubs.
|
92
|
+
stub_request(:post, WRITE_LOG_ENTRIES_URI)
|
93
|
+
.to_return(status: code, body: 'Some Message')
|
94
|
+
(1..request_count).each do
|
95
|
+
d = create_driver(PROMETHEUS_ENABLE_CONFIG)
|
96
|
+
(1..entry_count).each do |i|
|
97
|
+
d.emit('message' => log_entry(i.to_s))
|
98
|
+
end
|
99
|
+
# rubocop:disable Lint/HandleExceptions
|
100
|
+
begin
|
101
|
+
d.run
|
102
|
+
rescue Google::Apis::AuthorizationError
|
103
|
+
rescue Google::Apis::ServerError
|
104
|
+
end
|
105
|
+
# rubocop:enable Lint/HandleExceptions
|
106
|
+
end
|
107
|
+
successful_requests_count, failed_requests_count,
|
108
|
+
ingested_entries_count, dropped_entries_count = metric_values
|
109
|
+
assert_prometheus_metric_value(:stackdriver_successful_requests_count,
|
110
|
+
successful_requests_count, grpc: false)
|
111
|
+
assert_prometheus_metric_value(:stackdriver_failed_requests_count,
|
112
|
+
failed_requests_count,
|
113
|
+
grpc: false, code: code)
|
114
|
+
assert_prometheus_metric_value(:stackdriver_ingested_entries_count,
|
115
|
+
ingested_entries_count)
|
116
|
+
assert_prometheus_metric_value(:stackdriver_dropped_entries_count,
|
117
|
+
dropped_entries_count)
|
88
118
|
end
|
89
119
|
end
|
90
120
|
|
@@ -113,7 +143,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
113
143
|
verify_index = 0
|
114
144
|
verify_log_entries(emit_index, COMPUTE_PARAMS) do |entry|
|
115
145
|
assert_equal expected_severity[verify_index],
|
116
|
-
entry['
|
146
|
+
entry['severity'], entry
|
117
147
|
verify_index += 1
|
118
148
|
end
|
119
149
|
end
|
@@ -181,7 +211,6 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
181
211
|
assert_equal('ERROR', test_obj.parse_severity('ERROR '))
|
182
212
|
assert_equal('ERROR', test_obj.parse_severity(' ERROR '))
|
183
213
|
assert_equal('ERROR', test_obj.parse_severity("\t ERROR "))
|
184
|
-
|
185
214
|
# space in the middle should not be stripped.
|
186
215
|
assert_equal('DEFAULT', test_obj.parse_severity('ER ROR'))
|
187
216
|
|
@@ -208,38 +237,25 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
208
237
|
d.run
|
209
238
|
end
|
210
239
|
verify_log_entries(1, COMPUTE_PARAMS) do |entry|
|
211
|
-
assert_equal timestamp, entry['
|
212
|
-
"
|
213
|
-
"entry: '#{entry}'."
|
240
|
+
assert_equal timestamp, entry['timestamp'], 'Test with timestamp ' \
|
241
|
+
"'#{timestamp}' failed for entry: '#{entry}'."
|
214
242
|
end
|
215
243
|
end
|
216
244
|
end
|
217
245
|
|
218
246
|
private
|
219
247
|
|
248
|
+
WRITE_LOG_ENTRIES_URI = 'https://logging.googleapis.com/v2beta1/entries:write'
|
249
|
+
|
220
250
|
def rename_key(hash, old_key, new_key)
|
221
251
|
hash.merge(new_key => hash[old_key]).reject { |k, _| k == old_key }
|
222
252
|
end
|
223
253
|
|
224
|
-
# The REST path uses old bindings that were generated prior to the field
|
225
|
-
# rename, and has to use the old name, which is 'validatedWithOriginServer'.
|
226
|
-
def http_request_message
|
227
|
-
rename_key(super, 'cacheValidatedWithOriginServer',
|
228
|
-
'validatedWithOriginServer')
|
229
|
-
end
|
230
|
-
|
231
254
|
# Set up http stubs to mock the external calls.
|
232
|
-
def setup_logging_stubs
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
CUSTOM_PARAMS, EC2_PARAMS]
|
237
|
-
stub_params.each do |params|
|
238
|
-
stub_request(:post, uri_for_log(params)).to_return do |request|
|
239
|
-
log_name = "projects/#{params[:project_id]}/logs/#{params[:log_name]}"
|
240
|
-
@logs_sent << JSON.parse(request.body).merge('logName' => log_name)
|
241
|
-
{ body: '' }
|
242
|
-
end
|
255
|
+
def setup_logging_stubs
|
256
|
+
stub_request(:post, WRITE_LOG_ENTRIES_URI).to_return do |request|
|
257
|
+
@logs_sent << JSON.parse(request.body)
|
258
|
+
{ body: '' }
|
243
259
|
end
|
244
260
|
yield
|
245
261
|
end
|
@@ -270,9 +286,9 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
270
286
|
end
|
271
287
|
end
|
272
288
|
|
273
|
-
# Get the fields of the
|
274
|
-
def get_fields(
|
275
|
-
|
289
|
+
# Get the fields of the payload.
|
290
|
+
def get_fields(payload)
|
291
|
+
payload
|
276
292
|
end
|
277
293
|
|
278
294
|
# Get the value of a struct field.
|