fluent-plugin-vadimberezniker-gcp 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CONTRIBUTING +24 -0
- data/Gemfile +3 -0
- data/LICENSE +201 -0
- data/README.rdoc +53 -0
- data/Rakefile +43 -0
- data/fluent-plugin-google-cloud.gemspec +43 -0
- data/fluent-plugin-vadimberezniker-gcp-0.13.2.gem +0 -0
- data/lib/fluent/plugin/common.rb +399 -0
- data/lib/fluent/plugin/filter_add_insert_ids.rb +86 -0
- data/lib/fluent/plugin/filter_analyze_config.rb +410 -0
- data/lib/fluent/plugin/in_object_space_dump.rb +62 -0
- data/lib/fluent/plugin/monitoring.rb +265 -0
- data/lib/fluent/plugin/out_google_cloud.rb +2209 -0
- data/lib/fluent/plugin/statusz.rb +124 -0
- data/test/helper.rb +46 -0
- data/test/plugin/asserts.rb +87 -0
- data/test/plugin/base_test.rb +2680 -0
- data/test/plugin/constants.rb +1114 -0
- data/test/plugin/data/c31e573fd7f62ed495c9ca3821a5a85cb036dee1-privatekey.p12 +0 -0
- data/test/plugin/data/credentials.json +7 -0
- data/test/plugin/data/google-fluentd-baseline.conf +24 -0
- data/test/plugin/data/google-fluentd-custom.conf +40 -0
- data/test/plugin/data/iam-credentials.json +11 -0
- data/test/plugin/data/invalid_credentials.json +8 -0
- data/test/plugin/data/new-style-credentials.json +12 -0
- data/test/plugin/test_driver.rb +56 -0
- data/test/plugin/test_filter_add_insert_ids.rb +137 -0
- data/test/plugin/test_filter_analyze_config.rb +257 -0
- data/test/plugin/test_out_google_cloud.rb +465 -0
- data/test/plugin/test_out_google_cloud_grpc.rb +478 -0
- data/test/plugin/utils.rb +148 -0
- metadata +347 -0
@@ -0,0 +1,1114 @@
|
|
1
|
+
# Copyright 2017 Google Inc. All rights reserved.
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
require 'google/rpc/error_details_pb'
|
16
|
+
|
17
|
+
# Add some helper methods to standard classes.
|
18
|
+
String.class_eval do
|
19
|
+
def inspect_octal
|
20
|
+
specials = {
|
21
|
+
'a' => '\\007',
|
22
|
+
'b' => '\\010',
|
23
|
+
'v' => '\\013',
|
24
|
+
'f' => '\\014',
|
25
|
+
'r' => '\\015'
|
26
|
+
}.freeze
|
27
|
+
inspect.gsub(/\\([abvfr])/) { specials[Regexp.last_match(1)] } \
|
28
|
+
.gsub(/\\x([0-9A-F][0-9A-F])/) do
|
29
|
+
format('\\%03o', Regexp.last_match(1).to_i(16))
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
# Constants used by unit tests for Google Cloud Logging plugin.
|
35
|
+
module Constants
|
36
|
+
include Common::ServiceConstants
|
37
|
+
include Fluent::GoogleCloudOutput::ConfigConstants
|
38
|
+
include Fluent::GoogleCloudOutput::InternalConstants
|
39
|
+
|
40
|
+
# Generic attributes.
|
41
|
+
HOSTNAME = Socket.gethostname
|
42
|
+
CUSTOM_LOGGING_API_URL = 'http://localhost:52000'.freeze
|
43
|
+
|
44
|
+
# TODO(qingling128) Separate constants into different submodules.
|
45
|
+
# Attributes used for the GCE metadata service.
|
46
|
+
PROJECT_ID = 'test-project-id'.freeze
|
47
|
+
ZONE = 'us-central1-b'.freeze
|
48
|
+
FULLY_QUALIFIED_ZONE = "projects/#{PROJECT_ID}/zones/#{ZONE}".freeze
|
49
|
+
VM_ID = '9876543210'.freeze
|
50
|
+
|
51
|
+
RANDOM_LOCAL_RESOURCE_ID = 'ehb.jjk.poq.ll'.freeze
|
52
|
+
|
53
|
+
# Attributes used for the Metadata Agent resources.
|
54
|
+
METADATA_ZONE = 'us-central1-c'.freeze
|
55
|
+
METADATA_VM_ID = '0123456789'.freeze
|
56
|
+
|
57
|
+
# Attributes used for custom (overridden) configs.
|
58
|
+
CUSTOM_PROJECT_ID = 'test-custom-project-id'.freeze
|
59
|
+
CUSTOM_ZONE = 'us-custom-central1-b'.freeze
|
60
|
+
CUSTOM_FULLY_QUALIFIED_ZONE = "projects/#{PROJECT_ID}/zones/#{ZONE}".freeze
|
61
|
+
CUSTOM_VM_ID = 'C9876543210'.freeze
|
62
|
+
CUSTOM_HOSTNAME = 'custom.hostname.org'.freeze
|
63
|
+
# Kubernetes-specific attributes.
|
64
|
+
CUSTOM_K8S_CLUSTER_NAME = 'kubernetes-cluster'.freeze
|
65
|
+
CUSTOM_K8S_LOCATION = 'kubernetes-location'.freeze
|
66
|
+
|
67
|
+
# Attributes used for the EC2 metadata service.
|
68
|
+
EC2_PROJECT_ID = 'test-ec2-project-id'.freeze
|
69
|
+
EC2_ZONE = 'us-west-2b'.freeze
|
70
|
+
EC2_PREFIXED_ZONE = "aws:#{EC2_ZONE}".freeze
|
71
|
+
EC2_REGION = 'us-west-2'.freeze
|
72
|
+
EC2_PREFIXED_REGION = "aws:#{EC2_REGION}".freeze
|
73
|
+
EC2_VM_ID = 'i-81c16767'.freeze
|
74
|
+
EC2_ACCOUNT_ID = '123456789012'.freeze
|
75
|
+
|
76
|
+
# The formatting here matches the format used on the VM.
|
77
|
+
EC2_IDENTITY_DOCUMENT = %({
|
78
|
+
"accountId" : "#{EC2_ACCOUNT_ID}",
|
79
|
+
"availabilityZone" : "#{EC2_ZONE}",
|
80
|
+
"region" : "#{EC2_REGION}",
|
81
|
+
"instanceId" : "#{EC2_VM_ID}"
|
82
|
+
}).freeze
|
83
|
+
|
84
|
+
# Managed VMs specific labels.
|
85
|
+
MANAGED_VM_BACKEND_NAME = 'default'.freeze
|
86
|
+
MANAGED_VM_BACKEND_VERSION = 'guestbook2.0'.freeze
|
87
|
+
|
88
|
+
# LogEntry fields for extraction.
|
89
|
+
INSERT_ID = 'fah7yr7iw64tg857y'.freeze
|
90
|
+
INSERT_ID2 = 'fah7yr7iw64tgaeuf'.freeze
|
91
|
+
SPAN_ID = '000000000000004a'.freeze
|
92
|
+
SPAN_ID2 = '000000000000007e'.freeze
|
93
|
+
TRACE = 'projects/proj1/traces/1234567890abcdef1234567890abcdef'.freeze
|
94
|
+
TRACE2 = 'projects/proj1/traces/1234567890abcdef1234567890fedcba'.freeze
|
95
|
+
TRACE_SAMPLED = true
|
96
|
+
TRACE_SAMPLED2 = false
|
97
|
+
|
98
|
+
STACKDRIVER_TRACE_ID = '1234567890abcdef1234567890abcdef'.freeze
|
99
|
+
FULL_STACKDRIVER_TRACE = \
|
100
|
+
"projects/#{PROJECT_ID}/traces/#{STACKDRIVER_TRACE_ID}".freeze
|
101
|
+
|
102
|
+
# Invalid trace id for stackdriver.
|
103
|
+
EMPTY_STRING = ''.freeze
|
104
|
+
INVALID_SHORT_STACKDRIVER_TRACE_ID = '1234567890abcdef'.freeze
|
105
|
+
INVALID_LONG_STACKDRIVER_TRACE_ID = \
|
106
|
+
'1234567890abcdef1234567890abcdef123'.freeze
|
107
|
+
INVALID_NON_HEX_STACKDRIVER_TRACE_ID = \
|
108
|
+
'1234567890abcdef1234567890abcdeZ'.freeze
|
109
|
+
|
110
|
+
# Invalid full format of stackdriver trace.
|
111
|
+
INVALID_TRACE_NO_TRACE_ID = "projects/#{PROJECT_ID}/traces/".freeze
|
112
|
+
INVALID_TRACE_NO_PROJECT_ID = \
|
113
|
+
"projects//traces/#{STACKDRIVER_TRACE_ID}".freeze
|
114
|
+
INVALID_TRACE_WITH_SHORT_TRACE_ID = \
|
115
|
+
"projects/#{PROJECT_ID}/traces/#{INVALID_SHORT_STACKDRIVER_TRACE_ID}".freeze
|
116
|
+
INVALID_TRACE_WITH_LONG_TRACE_ID = \
|
117
|
+
"projects/#{PROJECT_ID}/traces/#{INVALID_LONG_STACKDRIVER_TRACE_ID}".freeze
|
118
|
+
INVALID_TRACE_WITH_NON_HEX_TRACE_ID = \
|
119
|
+
"projects/#{PROJECT_ID}/" \
|
120
|
+
"traces/#{INVALID_NON_HEX_STACKDRIVER_TRACE_ID}".freeze
|
121
|
+
|
122
|
+
# New K8s resource constants.
|
123
|
+
K8S_LOCATION = 'us-central1-b'.freeze
|
124
|
+
K8S_LOCATION2 = 'us-central1-c'.freeze
|
125
|
+
K8S_CLUSTER_NAME = 'cluster-1'.freeze
|
126
|
+
K8S_NAMESPACE_NAME = 'kube-system'.freeze
|
127
|
+
K8S_NODE_NAME = 'performance--default-pool-cabf1342-08jc'.freeze
|
128
|
+
K8S_POD_NAME = 'redis-master-c0l82.foo.bar'.freeze
|
129
|
+
K8S_CONTAINER_NAME = 'redis'.freeze
|
130
|
+
K8S_STREAM = 'stdout'.freeze
|
131
|
+
# Timestamp for 1234567890 seconds and 987654321 nanoseconds since epoch.
|
132
|
+
K8S_TIMESTAMP = '2009-02-13T23:31:30.987654321Z'.freeze
|
133
|
+
K8S_SECONDS_EPOCH = 1_234_567_890
|
134
|
+
K8S_NANOS = 987_654_321
|
135
|
+
K8S_CONTAINER_LOCAL_RESOURCE_ID_PREFIX = 'k8s_container'.freeze
|
136
|
+
K8S_POD_LOCAL_RESOURCE_ID_PREFIX = 'k8s_pod'.freeze
|
137
|
+
K8S_NODE_LOCAL_RESOURCE_ID_PREFIX = 'k8s_node'.freeze
|
138
|
+
K8S_TAG =
|
139
|
+
"var.log.containers.#{K8S_NAMESPACE_NAME}_#{K8S_POD_NAME}_" \
|
140
|
+
"#{K8S_CONTAINER_NAME}.log".freeze
|
141
|
+
K8S_LOCAL_RESOURCE_ID =
|
142
|
+
"#{K8S_CONTAINER_LOCAL_RESOURCE_ID_PREFIX}" \
|
143
|
+
".#{K8S_NAMESPACE_NAME}" \
|
144
|
+
".#{K8S_POD_NAME}" \
|
145
|
+
".#{K8S_CONTAINER_NAME}".freeze
|
146
|
+
|
147
|
+
# Container Engine / Kubernetes specific labels.
|
148
|
+
CONTAINER_NAMESPACE_ID = '898268c8-4a36-11e5-9d81-42010af0194c'.freeze
|
149
|
+
CONTAINER_POD_ID = 'cad3c3c4-4b9c-11e5-9d81-42010af0194c'.freeze
|
150
|
+
CONTAINER_LABEL_KEY = 'component'.freeze
|
151
|
+
CONTAINER_LABEL_VALUE = 'redis-component'.freeze
|
152
|
+
CONTAINER_SEVERITY = 'INFO'.freeze
|
153
|
+
CONTAINER_LOCAL_RESOURCE_ID_PREFIX = 'gke_container'.freeze
|
154
|
+
|
155
|
+
# Dataflow specific labels.
|
156
|
+
DATAFLOW_REGION = 'us-central1'.freeze
|
157
|
+
DATAFLOW_JOB_NAME = 'job_name_1'.freeze
|
158
|
+
DATAFLOW_JOB_ID = 'job_id_1'.freeze
|
159
|
+
DATAFLOW_STEP_ID = 'step_1'.freeze
|
160
|
+
DATAFLOW_TAG = 'dataflow-worker'.freeze
|
161
|
+
|
162
|
+
# Dataproc specific labels.
|
163
|
+
DATAPROC_CLUSTER_NAME = 'test-cluster'.freeze
|
164
|
+
DATAPROC_CLUSTER_UUID = '00000000-0000-0000-0000-000000000000'.freeze
|
165
|
+
DATAPROC_REGION = 'unittest'.freeze
|
166
|
+
|
167
|
+
# ML specific labels.
|
168
|
+
ML_REGION = 'us-central1'.freeze
|
169
|
+
ML_JOB_ID = 'job_name_1'.freeze
|
170
|
+
ML_TASK_NAME = 'task_name_1'.freeze
|
171
|
+
ML_TRIAL_ID = 'trial_id_1'.freeze
|
172
|
+
ML_LOG_AREA = 'log_area_1'.freeze
|
173
|
+
ML_TAG = 'master-replica-0'.freeze
|
174
|
+
|
175
|
+
# Parameters used for authentication.
|
176
|
+
AUTH_GRANT_TYPE = 'urn:ietf:params:oauth:grant-type:jwt-bearer'.freeze
|
177
|
+
FAKE_AUTH_TOKEN = 'abc123'.freeze
|
178
|
+
|
179
|
+
# Information about test credentials files.
|
180
|
+
# path: Path to the credentials file.
|
181
|
+
# project_id: ID of the project, which must correspond to the file contents.
|
182
|
+
IAM_CREDENTIALS = {
|
183
|
+
path: 'test/plugin/data/iam-credentials.json',
|
184
|
+
project_id: 'fluent-test-project'
|
185
|
+
}.freeze
|
186
|
+
NEW_STYLE_CREDENTIALS = {
|
187
|
+
path: 'test/plugin/data/new-style-credentials.json',
|
188
|
+
project_id: 'fluent-test-project'
|
189
|
+
}.freeze
|
190
|
+
LEGACY_CREDENTIALS = {
|
191
|
+
path: 'test/plugin/data/credentials.json',
|
192
|
+
project_id: '847859579879'
|
193
|
+
}.freeze
|
194
|
+
INVALID_CREDENTIALS = {
|
195
|
+
path: 'test/plugin/data/invalid_credentials.json',
|
196
|
+
project_id: ''
|
197
|
+
}.freeze
|
198
|
+
|
199
|
+
# Special googleauth environment variables.
|
200
|
+
PROJECT_ID_VAR = 'GOOGLE_PROJECT_ID'.freeze
|
201
|
+
PRIVATE_KEY_VAR = 'GOOGLE_PRIVATE_KEY'.freeze
|
202
|
+
CLIENT_EMAIL_VAR = 'GOOGLE_CLIENT_EMAIL'.freeze
|
203
|
+
CLIENT_ID_VAR = 'GOOGLE_CLIENT_ID'.freeze
|
204
|
+
CLIENT_SECRET_VAR = 'GOOGLE_CLIENT_SECRET'.freeze
|
205
|
+
REFRESH_TOKEN_VAR = 'GOOGLE_REFRESH_TOKEN'.freeze
|
206
|
+
|
207
|
+
# Configuration files for various test scenarios.
|
208
|
+
APPLICATION_DEFAULT_CONFIG = %(
|
209
|
+
).freeze
|
210
|
+
|
211
|
+
CUSTOM_LOGGING_API_URL_CONFIG = %(
|
212
|
+
logging_api_url #{CUSTOM_LOGGING_API_URL}
|
213
|
+
).freeze
|
214
|
+
|
215
|
+
DETECT_JSON_CONFIG = %(
|
216
|
+
detect_json true
|
217
|
+
).freeze
|
218
|
+
|
219
|
+
PRIVATE_KEY_CONFIG = %(
|
220
|
+
auth_method private_key
|
221
|
+
private_key_email 271661262351-ft99kc9kjro9rrihq3k2n3s2inbplu0q@developer.gserviceaccount.com
|
222
|
+
private_key_path test/plugin/data/c31e573fd7f62ed495c9ca3821a5a85cb036dee1-privatekey.p12
|
223
|
+
).freeze
|
224
|
+
|
225
|
+
REQUIRE_VALID_TAGS_CONFIG = %(
|
226
|
+
require_valid_tags true
|
227
|
+
).freeze
|
228
|
+
|
229
|
+
NO_METADATA_SERVICE_CONFIG = %(
|
230
|
+
use_metadata_service false
|
231
|
+
).freeze
|
232
|
+
|
233
|
+
NO_DETECT_SUBSERVICE_CONFIG = %(
|
234
|
+
detect_subservice false
|
235
|
+
).freeze
|
236
|
+
|
237
|
+
ENABLE_SPLIT_LOGS_BY_TAG_CONFIG = %(
|
238
|
+
split_logs_by_tag true
|
239
|
+
).freeze
|
240
|
+
|
241
|
+
ENABLE_PROMETHEUS_CONFIG = %(
|
242
|
+
enable_monitoring true
|
243
|
+
monitoring_type prometheus
|
244
|
+
).freeze
|
245
|
+
|
246
|
+
ENABLE_OPENCENSUS_CONFIG = %(
|
247
|
+
enable_monitoring true
|
248
|
+
monitoring_type opencensus
|
249
|
+
).freeze
|
250
|
+
|
251
|
+
ENABLE_AUTOFORMAT_STACKDRIVER_TRACE_CONFIG = %(
|
252
|
+
autoformat_stackdriver_trace true
|
253
|
+
).freeze
|
254
|
+
|
255
|
+
DISABLE_AUTOFORMAT_STACKDRIVER_TRACE_CONFIG = %(
|
256
|
+
autoformat_stackdriver_trace false
|
257
|
+
).freeze
|
258
|
+
|
259
|
+
CUSTOM_METADATA_CONFIG = %(
|
260
|
+
project_id #{CUSTOM_PROJECT_ID}
|
261
|
+
zone #{CUSTOM_ZONE}
|
262
|
+
vm_id #{CUSTOM_VM_ID}
|
263
|
+
vm_name #{CUSTOM_HOSTNAME}
|
264
|
+
).freeze
|
265
|
+
|
266
|
+
CONFIG_MISSING_METADATA_PROJECT_ID = %(
|
267
|
+
zone #{CUSTOM_ZONE}
|
268
|
+
vm_id #{CUSTOM_VM_ID}
|
269
|
+
).freeze
|
270
|
+
CONFIG_MISSING_METADATA_ZONE = %(
|
271
|
+
project_id #{CUSTOM_PROJECT_ID}
|
272
|
+
vm_id #{CUSTOM_VM_ID}
|
273
|
+
).freeze
|
274
|
+
CONFIG_MISSING_METADATA_VM_ID = %(
|
275
|
+
project_id #{CUSTOM_PROJECT_ID}
|
276
|
+
zone #{CUSTOM_ZONE}
|
277
|
+
).freeze
|
278
|
+
CONFIG_MISSING_METADATA_ALL = %(
|
279
|
+
).freeze
|
280
|
+
|
281
|
+
CUSTOM_K8S_CONFIG = %(
|
282
|
+
k8s_cluster_name #{CUSTOM_K8S_CLUSTER_NAME}
|
283
|
+
k8s_cluster_location #{CUSTOM_K8S_LOCATION}
|
284
|
+
).freeze
|
285
|
+
|
286
|
+
EMPTY_K8S_CONFIG = %(
|
287
|
+
k8s_cluster_name ""
|
288
|
+
k8s_cluster_location ""
|
289
|
+
).freeze
|
290
|
+
|
291
|
+
CONFIG_EC2_PROJECT_ID = %(
|
292
|
+
project_id #{EC2_PROJECT_ID}
|
293
|
+
).freeze
|
294
|
+
|
295
|
+
CONFIG_EC2_PROJECT_ID_AND_CUSTOM_VM_ID = %(
|
296
|
+
project_id #{EC2_PROJECT_ID}
|
297
|
+
vm_id #{CUSTOM_VM_ID}
|
298
|
+
).freeze
|
299
|
+
|
300
|
+
CONFIG_EC2_PROJECT_ID_USE_REGION = %(
|
301
|
+
project_id #{EC2_PROJECT_ID}
|
302
|
+
use_aws_availability_zone false
|
303
|
+
).freeze
|
304
|
+
|
305
|
+
CONFIG_DATAFLOW = %(
|
306
|
+
subservice_name "#{DATAFLOW_CONSTANTS[:service]}"
|
307
|
+
labels {
|
308
|
+
"#{DATAFLOW_CONSTANTS[:service]}/region" : "#{DATAFLOW_REGION}",
|
309
|
+
"#{DATAFLOW_CONSTANTS[:service]}/job_name" : "#{DATAFLOW_JOB_NAME}",
|
310
|
+
"#{DATAFLOW_CONSTANTS[:service]}/job_id" : "#{DATAFLOW_JOB_ID}"
|
311
|
+
}
|
312
|
+
label_map { "step": "#{DATAFLOW_CONSTANTS[:service]}/step_id" }
|
313
|
+
).freeze
|
314
|
+
|
315
|
+
CONFIG_ML = %(
|
316
|
+
subservice_name "#{ML_CONSTANTS[:service]}"
|
317
|
+
labels {
|
318
|
+
"#{ML_CONSTANTS[:service]}/job_id" : "#{ML_JOB_ID}",
|
319
|
+
"#{ML_CONSTANTS[:service]}/task_name" : "#{ML_TASK_NAME}",
|
320
|
+
"#{ML_CONSTANTS[:service]}/trial_id" : "#{ML_TRIAL_ID}"
|
321
|
+
}
|
322
|
+
label_map { "name": "#{ML_CONSTANTS[:service]}/job_id/log_area" }
|
323
|
+
).freeze
|
324
|
+
|
325
|
+
CONFIG_CUSTOM_INSERT_ID_KEY_SPECIFIED = %(
|
326
|
+
insert_id_key custom_insert_id_key
|
327
|
+
).freeze
|
328
|
+
|
329
|
+
CONFIG_CUSTOM_LABELS_KEY_SPECIFIED = %(
|
330
|
+
labels_key custom_labels_key
|
331
|
+
).freeze
|
332
|
+
|
333
|
+
CONFIG_CUSTOM_OPERATION_KEY_SPECIFIED = %(
|
334
|
+
operation_key custom_operation_key
|
335
|
+
).freeze
|
336
|
+
|
337
|
+
CONFIG_CUSTOM_SOURCE_LOCATION_KEY_SPECIFIED = %(
|
338
|
+
source_location_key custom_source_location_key
|
339
|
+
).freeze
|
340
|
+
|
341
|
+
CONFIG_CUSTOM_SPAN_ID_KEY_SPECIFIED = %(
|
342
|
+
span_id_key custom_span_id_key
|
343
|
+
).freeze
|
344
|
+
|
345
|
+
CONFIG_CUSTOM_TRACE_KEY_SPECIFIED = %(
|
346
|
+
trace_key custom_trace_key
|
347
|
+
).freeze
|
348
|
+
|
349
|
+
CONFIG_CUSTOM_TRACE_SAMPLED_KEY_SPECIFIED = %(
|
350
|
+
trace_sampled_key custom_trace_sampled_key
|
351
|
+
).freeze
|
352
|
+
|
353
|
+
# For 'labels' config.
|
354
|
+
LABELS_FROM_LABELS_CONFIG = {
|
355
|
+
'a_label_from_labels_config' => 'some_value',
|
356
|
+
'another_label_from_labels_config' => 'some_value'
|
357
|
+
}.freeze
|
358
|
+
CONFIG_LABELS = %(
|
359
|
+
labels #{LABELS_FROM_LABELS_CONFIG.to_json}
|
360
|
+
).freeze
|
361
|
+
|
362
|
+
# For 'label_map' config.
|
363
|
+
LABEL_MAP_HASH = {
|
364
|
+
'target_field_from_payload' => 'a_label_from_label_map_config',
|
365
|
+
'another_target_field_from_payload' => 'another_label_from_label_map_config'
|
366
|
+
}.freeze
|
367
|
+
PAYLOAD_FOR_LABEL_MAP = {
|
368
|
+
'target_field_from_payload' => 'a_value',
|
369
|
+
'another_target_field_from_payload' => 'b_value'
|
370
|
+
}.freeze
|
371
|
+
LABELS_FROM_LABEL_MAP_CONFIG = {
|
372
|
+
'a_label_from_label_map_config' => 'a_value',
|
373
|
+
'another_label_from_label_map_config' => 'b_value'
|
374
|
+
}.freeze
|
375
|
+
CONFIG_LABEL_MAP = %(
|
376
|
+
label_map #{LABEL_MAP_HASH.to_json}
|
377
|
+
).freeze
|
378
|
+
|
379
|
+
CONFIG_LABLES_AND_LABLE_MAP = %(
|
380
|
+
#{CONFIG_LABELS}
|
381
|
+
#{CONFIG_LABEL_MAP}
|
382
|
+
).freeze
|
383
|
+
|
384
|
+
# For conflicting labels.
|
385
|
+
CONFLICTING_LABEL_NAME = 'conflicting_label_key'.freeze
|
386
|
+
CONFLICTING_LABEL_VALUE1 = 'conflicting_value_1'.freeze
|
387
|
+
CONFLICTING_LABEL_VALUE2 = 'conflicting_value_2'.freeze
|
388
|
+
CONFLICTING_LABEL_VALUE3 = 'conflicting_value_3'.freeze
|
389
|
+
LABELS_FROM_PAYLOAD_CONFLICTING = {
|
390
|
+
CONFLICTING_LABEL_NAME => CONFLICTING_LABEL_VALUE1
|
391
|
+
}.freeze
|
392
|
+
LABELS_FROM_LABEL_MAP_CONFIG_CONFLICTING = {
|
393
|
+
CONFLICTING_LABEL_NAME => CONFLICTING_LABEL_VALUE2
|
394
|
+
}.freeze
|
395
|
+
LABELS_FROM_LABELS_CONFIG_CONFLICTING = {
|
396
|
+
CONFLICTING_LABEL_NAME => CONFLICTING_LABEL_VALUE3
|
397
|
+
}.freeze
|
398
|
+
|
399
|
+
LABEL_MAP_HASH_CONFLICTING = {
|
400
|
+
'target_field_from_payload' => CONFLICTING_LABEL_NAME
|
401
|
+
}.freeze
|
402
|
+
PAYLOAD_FOR_LABEL_MAP_CONFLICTING = {
|
403
|
+
'target_field_from_payload' => CONFLICTING_LABEL_VALUE2
|
404
|
+
}.freeze
|
405
|
+
CONFIG_LABEL_MAP_CONFLICTING = %(
|
406
|
+
label_map #{LABEL_MAP_HASH_CONFLICTING.to_json}
|
407
|
+
).freeze
|
408
|
+
CONFIG_LABELS_CONFLICTING = %(
|
409
|
+
labels #{LABELS_FROM_LABELS_CONFIG_CONFLICTING.to_json}
|
410
|
+
).freeze
|
411
|
+
CONFIG_LABLES_AND_LABLE_MAP_CONFLICTING = %(
|
412
|
+
#{CONFIG_LABELS_CONFLICTING}
|
413
|
+
#{CONFIG_LABEL_MAP_CONFLICTING}
|
414
|
+
).freeze
|
415
|
+
|
416
|
+
# For monitoring config.
|
417
|
+
CONFIG_UNKNOWN_MONITORING_TYPE = %(
|
418
|
+
enable_monitoring true
|
419
|
+
monitoring_type not_prometheus
|
420
|
+
).freeze
|
421
|
+
|
422
|
+
CONFIG_METRICS_RESOURCE_JSON = %(
|
423
|
+
enable_monitoring true
|
424
|
+
monitoring_type opencensus
|
425
|
+
metrics_resource {"type":"custom_resource","labels":{"label1":"123","label2":"abc"}}
|
426
|
+
).freeze
|
427
|
+
|
428
|
+
CONFIG_METRICS_RESOURCE_HASH = %(
|
429
|
+
enable_monitoring true
|
430
|
+
monitoring_type opencensus
|
431
|
+
metrics_resource type:custom_resource, labels.label1:123, labels.label2:abc
|
432
|
+
).freeze
|
433
|
+
|
434
|
+
CONFIG_METRICS_RESOURCE_JSON_HASH = %(
|
435
|
+
enable_monitoring true
|
436
|
+
monitoring_type opencensus
|
437
|
+
metrics_resource {"type":"custom_resource","labels.label1":"123","labels.label2":"abc"}
|
438
|
+
).freeze
|
439
|
+
|
440
|
+
CONFIG_METRICS_RESOURCE_JSON_NO_TYPE = %(
|
441
|
+
enable_monitoring true
|
442
|
+
monitoring_type opencensus
|
443
|
+
metrics_resource {"labels":{"label1":"123","label2":"abc"}}
|
444
|
+
).freeze
|
445
|
+
|
446
|
+
CONFIG_METRICS_RESOURCE_JSON_BAD_LABELS = %(
|
447
|
+
enable_monitoring true
|
448
|
+
monitoring_type opencensus
|
449
|
+
metrics_resource {"type":"custom_resource","labels":"123"}
|
450
|
+
).freeze
|
451
|
+
|
452
|
+
CONFIG_METRICS_RESOURCE_JSON_BAD_KEYS = %(
|
453
|
+
enable_monitoring true
|
454
|
+
monitoring_type opencensus
|
455
|
+
metrics_resource {"type":"custom_resource","labels":{"label1":"123"},"random":"x"}
|
456
|
+
).freeze
|
457
|
+
|
458
|
+
CONFIG_METRICS_RESOURCE_JSON_BAD_KEYS_LABELS = %(
|
459
|
+
enable_monitoring true
|
460
|
+
monitoring_type opencensus
|
461
|
+
metrics_resource {"type":"custom_resource","labels":{"label1":"123"},"labels.random":"x"}
|
462
|
+
).freeze
|
463
|
+
|
464
|
+
CONFIG_METRICS_RESOURCE_JSON_BAD_KEYS_NO_LABELS = %(
|
465
|
+
enable_monitoring true
|
466
|
+
monitoring_type opencensus
|
467
|
+
metrics_resource {"type":"custom_resource","labels.label1":"123","random":"x"}
|
468
|
+
).freeze
|
469
|
+
|
470
|
+
# For statusz.
|
471
|
+
CONFIG_STATUSZ = %(
|
472
|
+
statusz_port 5678
|
473
|
+
|
474
|
+
adjust_invalid_timestamps false
|
475
|
+
autoformat_stackdriver_trace false
|
476
|
+
coerce_to_utf8 false
|
477
|
+
detect_json true
|
478
|
+
detect_subservice false
|
479
|
+
enable_monitoring true
|
480
|
+
http_request_key test_http_request_key
|
481
|
+
insert_id_key test_insert_id_key
|
482
|
+
k8s_cluster_location test-k8s-cluster-location
|
483
|
+
k8s_cluster_name test-k8s-cluster-name
|
484
|
+
kubernetes_tag_regexp .*test-regexp.*
|
485
|
+
label_map { "label_map_key": "label_map_value" }
|
486
|
+
labels_key test_labels_key
|
487
|
+
labels { "labels_key": "labels_value" }
|
488
|
+
logging_api_url http://localhost:52000
|
489
|
+
monitoring_type not_prometheus
|
490
|
+
non_utf8_replacement_string zzz
|
491
|
+
operation_key test_operation_key
|
492
|
+
project_id test-project-id-123
|
493
|
+
require_valid_tags true
|
494
|
+
source_location_key test_source_location_key
|
495
|
+
span_id_key test_span_id_key
|
496
|
+
split_logs_by_tag true
|
497
|
+
subservice_name test_subservice_name
|
498
|
+
trace_key test_trace_key
|
499
|
+
trace_sampled_key test_trace_sampled_key
|
500
|
+
use_aws_availability_zone false
|
501
|
+
use_grpc true
|
502
|
+
use_metadata_service false
|
503
|
+
vm_id 12345
|
504
|
+
vm_name test.hostname.org
|
505
|
+
zone asia-east2
|
506
|
+
).freeze
|
507
|
+
|
508
|
+
# For analyze_config.
|
509
|
+
CONFIG_ANALYZE_CONFIG_PROMETHEUS = %(
|
510
|
+
google_fluentd_config_path \
|
511
|
+
test/plugin/data/google-fluentd-custom.conf
|
512
|
+
google_fluentd_baseline_config_path \
|
513
|
+
test/plugin/data/google-fluentd-baseline.conf
|
514
|
+
monitoring_type prometheus
|
515
|
+
).freeze
|
516
|
+
CONFIG_ANALYZE_CONFIG_OPENCENSUS = %(
|
517
|
+
google_fluentd_config_path \
|
518
|
+
test/plugin/data/google-fluentd-custom.conf
|
519
|
+
google_fluentd_baseline_config_path \
|
520
|
+
test/plugin/data/google-fluentd-baseline.conf
|
521
|
+
monitoring_type opencensus
|
522
|
+
).freeze
|
523
|
+
|
524
|
+
# Service configurations for various services.
|
525
|
+
|
526
|
+
# GCE.
|
527
|
+
COMPUTE_PARAMS_NO_LOG_NAME = {
|
528
|
+
resource: {
|
529
|
+
type: COMPUTE_CONSTANTS[:resource_type],
|
530
|
+
labels: {
|
531
|
+
'instance_id' => VM_ID,
|
532
|
+
'zone' => ZONE
|
533
|
+
}
|
534
|
+
},
|
535
|
+
project_id: PROJECT_ID,
|
536
|
+
labels: {
|
537
|
+
"#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME
|
538
|
+
}
|
539
|
+
}.freeze
|
540
|
+
COMPUTE_PARAMS = COMPUTE_PARAMS_NO_LOG_NAME.merge(
|
541
|
+
log_name: 'test'
|
542
|
+
).freeze
|
543
|
+
COMPUTE_PARAMS_WITH_METADATA_VM_ID_AND_ZONE = COMPUTE_PARAMS.merge(
|
544
|
+
resource: COMPUTE_PARAMS[:resource].merge(
|
545
|
+
labels: {
|
546
|
+
'instance_id' => METADATA_VM_ID,
|
547
|
+
'zone' => METADATA_ZONE
|
548
|
+
}
|
549
|
+
)
|
550
|
+
).freeze
|
551
|
+
|
552
|
+
# GAE.
|
553
|
+
VMENGINE_PARAMS = {
|
554
|
+
resource: {
|
555
|
+
type: APPENGINE_CONSTANTS[:resource_type],
|
556
|
+
labels: {
|
557
|
+
'module_id' => MANAGED_VM_BACKEND_NAME,
|
558
|
+
'version_id' => MANAGED_VM_BACKEND_VERSION
|
559
|
+
}
|
560
|
+
},
|
561
|
+
log_name: "#{APPENGINE_CONSTANTS[:service]}%2Ftest",
|
562
|
+
project_id: PROJECT_ID,
|
563
|
+
labels: {
|
564
|
+
"#{COMPUTE_CONSTANTS[:service]}/resource_id" => VM_ID,
|
565
|
+
"#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME,
|
566
|
+
"#{COMPUTE_CONSTANTS[:service]}/zone" => ZONE
|
567
|
+
}
|
568
|
+
}.freeze
|
569
|
+
|
570
|
+
# GKE Container.
|
571
|
+
CONTAINER_TAG =
|
572
|
+
"kubernetes.#{K8S_POD_NAME}_#{K8S_NAMESPACE_NAME}_" \
|
573
|
+
"#{K8S_CONTAINER_NAME}".freeze
|
574
|
+
|
575
|
+
CONTAINER_FROM_METADATA_PARAMS = {
|
576
|
+
resource: {
|
577
|
+
type: GKE_CONSTANTS[:resource_type],
|
578
|
+
labels: {
|
579
|
+
'cluster_name' => K8S_CLUSTER_NAME,
|
580
|
+
'namespace_id' => CONTAINER_NAMESPACE_ID,
|
581
|
+
'instance_id' => VM_ID,
|
582
|
+
'pod_id' => CONTAINER_POD_ID,
|
583
|
+
'container_name' => K8S_CONTAINER_NAME,
|
584
|
+
'zone' => ZONE
|
585
|
+
}
|
586
|
+
},
|
587
|
+
log_name: K8S_CONTAINER_NAME,
|
588
|
+
project_id: PROJECT_ID,
|
589
|
+
labels: {
|
590
|
+
"#{GKE_CONSTANTS[:service]}/namespace_name" => K8S_NAMESPACE_NAME,
|
591
|
+
"#{GKE_CONSTANTS[:service]}/pod_name" => K8S_POD_NAME,
|
592
|
+
"#{GKE_CONSTANTS[:service]}/stream" => K8S_STREAM,
|
593
|
+
"label/#{CONTAINER_LABEL_KEY}" => CONTAINER_LABEL_VALUE,
|
594
|
+
"#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME
|
595
|
+
}
|
596
|
+
}.freeze
|
597
|
+
|
598
|
+
# Almost the same as from metadata, but namespace_id and pod_id come from
|
599
|
+
# namespace and pod names.
|
600
|
+
CONTAINER_FROM_TAG_PARAMS = {
|
601
|
+
resource: {
|
602
|
+
type: GKE_CONSTANTS[:resource_type],
|
603
|
+
labels: {
|
604
|
+
'cluster_name' => K8S_CLUSTER_NAME,
|
605
|
+
'namespace_id' => K8S_NAMESPACE_NAME,
|
606
|
+
'instance_id' => VM_ID,
|
607
|
+
'pod_id' => K8S_POD_NAME,
|
608
|
+
'container_name' => K8S_CONTAINER_NAME,
|
609
|
+
'zone' => ZONE
|
610
|
+
}
|
611
|
+
},
|
612
|
+
log_name: K8S_CONTAINER_NAME,
|
613
|
+
project_id: PROJECT_ID,
|
614
|
+
labels: {
|
615
|
+
"#{GKE_CONSTANTS[:service]}/namespace_name" => K8S_NAMESPACE_NAME,
|
616
|
+
"#{GKE_CONSTANTS[:service]}/pod_name" => K8S_POD_NAME,
|
617
|
+
"#{GKE_CONSTANTS[:service]}/stream" => K8S_STREAM,
|
618
|
+
"#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME
|
619
|
+
}
|
620
|
+
}.freeze
|
621
|
+
|
622
|
+
CONTAINER_FROM_APPLICATION_PARAMS = {
|
623
|
+
resource: {
|
624
|
+
type: GKE_CONSTANTS[:resource_type],
|
625
|
+
labels: {
|
626
|
+
'cluster_name' => K8S_CLUSTER_NAME,
|
627
|
+
'namespace_id' => CONTAINER_NAMESPACE_ID,
|
628
|
+
'instance_id' => VM_ID,
|
629
|
+
'pod_id' => CONTAINER_POD_ID,
|
630
|
+
'container_name' => K8S_CONTAINER_NAME,
|
631
|
+
'zone' => ZONE
|
632
|
+
}
|
633
|
+
},
|
634
|
+
log_name: 'redis',
|
635
|
+
project_id: PROJECT_ID,
|
636
|
+
labels: {
|
637
|
+
"#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME
|
638
|
+
}
|
639
|
+
}.freeze
|
640
|
+
|
641
|
+
# K8s Container.
|
642
|
+
K8S_CONTAINER_PARAMS = {
|
643
|
+
resource: {
|
644
|
+
type: K8S_CONTAINER_CONSTANTS[:resource_type],
|
645
|
+
labels: {
|
646
|
+
'namespace_name' => K8S_NAMESPACE_NAME,
|
647
|
+
'pod_name' => K8S_POD_NAME,
|
648
|
+
'container_name' => K8S_CONTAINER_NAME,
|
649
|
+
'cluster_name' => K8S_CLUSTER_NAME,
|
650
|
+
'location' => K8S_LOCATION
|
651
|
+
}
|
652
|
+
},
|
653
|
+
project_id: PROJECT_ID,
|
654
|
+
labels: {}
|
655
|
+
}.freeze
|
656
|
+
K8S_CONTAINER_PARAMS_FROM_LOCAL = K8S_CONTAINER_PARAMS.merge(
|
657
|
+
resource: K8S_CONTAINER_PARAMS[:resource].merge(
|
658
|
+
labels: K8S_CONTAINER_PARAMS[:resource][:labels].merge(
|
659
|
+
'location' => K8S_LOCATION2
|
660
|
+
)
|
661
|
+
)
|
662
|
+
).freeze
|
663
|
+
K8S_CONTAINER_PARAMS_CUSTOM = K8S_CONTAINER_PARAMS.merge(
|
664
|
+
resource: K8S_CONTAINER_PARAMS[:resource].merge(
|
665
|
+
labels: K8S_CONTAINER_PARAMS[:resource][:labels].merge(
|
666
|
+
'cluster_name' => CUSTOM_K8S_CLUSTER_NAME,
|
667
|
+
'location' => CUSTOM_K8S_LOCATION
|
668
|
+
)
|
669
|
+
)
|
670
|
+
).freeze
|
671
|
+
# Used in k8s fallback tests.
|
672
|
+
K8S_CONTAINER_PARAMS_FROM_FALLBACK = COMPUTE_PARAMS_NO_LOG_NAME.merge(
|
673
|
+
log_name: CONTAINER_TAG
|
674
|
+
).freeze
|
675
|
+
|
676
|
+
# K8s Pod.
|
677
|
+
K8S_POD_PARAMS = {
|
678
|
+
resource: {
|
679
|
+
type: K8S_POD_CONSTANTS[:resource_type],
|
680
|
+
labels: {
|
681
|
+
'namespace_name' => K8S_NAMESPACE_NAME,
|
682
|
+
'pod_name' => K8S_POD_NAME,
|
683
|
+
'cluster_name' => K8S_CLUSTER_NAME,
|
684
|
+
'location' => K8S_LOCATION
|
685
|
+
}
|
686
|
+
},
|
687
|
+
project_id: PROJECT_ID,
|
688
|
+
labels: {}
|
689
|
+
}.freeze
|
690
|
+
K8S_POD_PARAMS_FROM_LOCAL = K8S_POD_PARAMS.merge(
|
691
|
+
resource: K8S_POD_PARAMS[:resource].merge(
|
692
|
+
labels: K8S_POD_PARAMS[:resource][:labels].merge(
|
693
|
+
'location' => K8S_LOCATION2
|
694
|
+
)
|
695
|
+
)
|
696
|
+
).freeze
|
697
|
+
K8S_POD_PARAMS_CUSTOM = K8S_POD_PARAMS.merge(
|
698
|
+
resource: K8S_POD_PARAMS[:resource].merge(
|
699
|
+
labels: K8S_POD_PARAMS[:resource][:labels].merge(
|
700
|
+
'cluster_name' => CUSTOM_K8S_CLUSTER_NAME,
|
701
|
+
'location' => CUSTOM_K8S_LOCATION
|
702
|
+
)
|
703
|
+
)
|
704
|
+
).freeze
|
705
|
+
|
706
|
+
# K8s Node.
|
707
|
+
K8S_NODE_PARAMS = {
|
708
|
+
resource: {
|
709
|
+
type: K8S_NODE_CONSTANTS[:resource_type],
|
710
|
+
labels: {
|
711
|
+
'node_name' => K8S_NODE_NAME,
|
712
|
+
'cluster_name' => K8S_CLUSTER_NAME,
|
713
|
+
'location' => K8S_LOCATION
|
714
|
+
}
|
715
|
+
},
|
716
|
+
log_name: 'test',
|
717
|
+
project_id: PROJECT_ID,
|
718
|
+
labels: {}
|
719
|
+
}.freeze
|
720
|
+
K8S_NODE_PARAMS_FROM_LOCAL = K8S_NODE_PARAMS.merge(
|
721
|
+
resource: K8S_NODE_PARAMS[:resource].merge(
|
722
|
+
labels: K8S_NODE_PARAMS[:resource][:labels].merge(
|
723
|
+
'location' => K8S_LOCATION2
|
724
|
+
)
|
725
|
+
)
|
726
|
+
).freeze
|
727
|
+
K8S_NODE_PARAMS_CUSTOM = K8S_NODE_PARAMS.merge(
|
728
|
+
resource: K8S_NODE_PARAMS[:resource].merge(
|
729
|
+
labels: K8S_NODE_PARAMS[:resource][:labels].merge(
|
730
|
+
'cluster_name' => CUSTOM_K8S_CLUSTER_NAME,
|
731
|
+
'location' => CUSTOM_K8S_LOCATION
|
732
|
+
)
|
733
|
+
)
|
734
|
+
).freeze
|
735
|
+
|
736
|
+
# Cloud Dataflow.
|
737
|
+
DATAFLOW_PARAMS = {
|
738
|
+
resource: {
|
739
|
+
type: DATAFLOW_CONSTANTS[:resource_type],
|
740
|
+
labels: {
|
741
|
+
'job_name' => DATAFLOW_JOB_NAME,
|
742
|
+
'job_id' => DATAFLOW_JOB_ID,
|
743
|
+
'step_id' => DATAFLOW_STEP_ID,
|
744
|
+
'region' => DATAFLOW_REGION
|
745
|
+
}
|
746
|
+
},
|
747
|
+
log_name: DATAFLOW_TAG,
|
748
|
+
project_id: PROJECT_ID,
|
749
|
+
labels: {
|
750
|
+
"#{COMPUTE_CONSTANTS[:service]}/resource_id" => VM_ID,
|
751
|
+
"#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME,
|
752
|
+
"#{COMPUTE_CONSTANTS[:service]}/zone" => ZONE
|
753
|
+
}
|
754
|
+
}.freeze
|
755
|
+
|
756
|
+
# Cloud Dataproc.
|
757
|
+
DATAPROC_PARAMS = {
|
758
|
+
resource: {
|
759
|
+
type: DATAPROC_CONSTANTS[:resource_type],
|
760
|
+
labels: {
|
761
|
+
'cluster_name' => DATAPROC_CLUSTER_NAME,
|
762
|
+
'cluster_uuid' => DATAPROC_CLUSTER_UUID,
|
763
|
+
'region' => DATAPROC_REGION
|
764
|
+
}
|
765
|
+
},
|
766
|
+
log_name: 'test',
|
767
|
+
project_id: PROJECT_ID,
|
768
|
+
labels: {
|
769
|
+
"#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME,
|
770
|
+
"#{COMPUTE_CONSTANTS[:service]}/resource_id" => VM_ID,
|
771
|
+
"#{COMPUTE_CONSTANTS[:service]}/zone" => ZONE
|
772
|
+
}
|
773
|
+
}.freeze
|
774
|
+
|
775
|
+
# Cloud ML.
|
776
|
+
ML_PARAMS = {
|
777
|
+
resource: {
|
778
|
+
type: ML_CONSTANTS[:resource_type],
|
779
|
+
labels: {
|
780
|
+
'job_id' => ML_JOB_ID,
|
781
|
+
'task_name' => ML_TASK_NAME
|
782
|
+
}
|
783
|
+
},
|
784
|
+
log_name: ML_TAG,
|
785
|
+
project_id: PROJECT_ID,
|
786
|
+
labels: {
|
787
|
+
"#{ML_CONSTANTS[:service]}/trial_id" => ML_TRIAL_ID,
|
788
|
+
"#{ML_CONSTANTS[:service]}/job_id/log_area" => ML_LOG_AREA,
|
789
|
+
"#{COMPUTE_CONSTANTS[:service]}/resource_id" => VM_ID,
|
790
|
+
"#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME,
|
791
|
+
"#{COMPUTE_CONSTANTS[:service]}/zone" => ZONE
|
792
|
+
}
|
793
|
+
}.freeze
|
794
|
+
|
795
|
+
CUSTOM_PARAMS = {
|
796
|
+
resource: {
|
797
|
+
type: COMPUTE_CONSTANTS[:resource_type],
|
798
|
+
labels: {
|
799
|
+
'instance_id' => CUSTOM_VM_ID,
|
800
|
+
'zone' => CUSTOM_ZONE
|
801
|
+
}
|
802
|
+
},
|
803
|
+
log_name: 'test',
|
804
|
+
project_id: CUSTOM_PROJECT_ID,
|
805
|
+
labels: {
|
806
|
+
"#{COMPUTE_CONSTANTS[:service]}/resource_name" => CUSTOM_HOSTNAME
|
807
|
+
}
|
808
|
+
}.freeze
|
809
|
+
|
810
|
+
EC2_REGION_PARAMS = {
|
811
|
+
resource: {
|
812
|
+
type: EC2_CONSTANTS[:resource_type],
|
813
|
+
labels: {
|
814
|
+
'instance_id' => EC2_VM_ID,
|
815
|
+
'region' => EC2_PREFIXED_REGION,
|
816
|
+
'aws_account' => EC2_ACCOUNT_ID
|
817
|
+
}
|
818
|
+
},
|
819
|
+
log_name: 'test',
|
820
|
+
project_id: EC2_PROJECT_ID,
|
821
|
+
labels: {
|
822
|
+
"#{EC2_CONSTANTS[:service]}/resource_name" => HOSTNAME
|
823
|
+
}
|
824
|
+
}.freeze
|
825
|
+
|
826
|
+
EC2_ZONE_PARAMS = EC2_REGION_PARAMS.merge(
|
827
|
+
resource: EC2_REGION_PARAMS[:resource].merge(
|
828
|
+
labels: EC2_REGION_PARAMS[:resource][:labels].merge(
|
829
|
+
'region' => EC2_PREFIXED_ZONE
|
830
|
+
)
|
831
|
+
)
|
832
|
+
).freeze
|
833
|
+
|
834
|
+
HTTP_REQUEST_MESSAGE = {
|
835
|
+
'cacheFillBytes' => '6653',
|
836
|
+
'cacheHit' => true,
|
837
|
+
'cacheLookup' => true,
|
838
|
+
'cacheValidatedWithOriginServer' => true,
|
839
|
+
'protocol' => 'HTTP/1.1',
|
840
|
+
'referer' => 'http://referer/',
|
841
|
+
'remoteIp' => '55.55.55.55',
|
842
|
+
'responseSize' => '65',
|
843
|
+
'requestMethod' => 'POST',
|
844
|
+
'requestSize' => '210',
|
845
|
+
'requestUrl' => 'http://example/',
|
846
|
+
'serverIp' => '66.66.66.66',
|
847
|
+
'status' => 200,
|
848
|
+
'userAgent' => 'USER AGENT 1.0'
|
849
|
+
}.freeze
|
850
|
+
|
851
|
+
SOURCE_LOCATION_MESSAGE = {
|
852
|
+
'file' => 'source/file',
|
853
|
+
'function' => 'my_function',
|
854
|
+
'line' => '18'
|
855
|
+
}.freeze
|
856
|
+
|
857
|
+
SOURCE_LOCATION_MESSAGE2 = {
|
858
|
+
'file' => 'src/file',
|
859
|
+
'function' => 'my_func',
|
860
|
+
'line' => '8'
|
861
|
+
}.freeze
|
862
|
+
|
863
|
+
OPERATION_MESSAGE = {
|
864
|
+
'id' => 'op_id',
|
865
|
+
'producer' => 'my/app',
|
866
|
+
'last' => true
|
867
|
+
}.freeze
|
868
|
+
|
869
|
+
OPERATION_MESSAGE2 = {
|
870
|
+
'id' => 'op_id2',
|
871
|
+
'producer' => 'my/app2',
|
872
|
+
'last' => false
|
873
|
+
}.freeze
|
874
|
+
|
875
|
+
LABELS_MESSAGE = {
|
876
|
+
'component' => 'front-end',
|
877
|
+
'source' => 'user',
|
878
|
+
'app' => 'request-router'
|
879
|
+
}.freeze
|
880
|
+
|
881
|
+
LABELS_MESSAGE2 = {
|
882
|
+
'component' => 'front-end',
|
883
|
+
'source' => 'system',
|
884
|
+
'app' => 'request-router'
|
885
|
+
}.freeze
|
886
|
+
|
887
|
+
CUSTOM_LABELS_MESSAGE = {
|
888
|
+
'customKey' => 'value'
|
889
|
+
}.freeze
|
890
|
+
CONFLICTING_LABEL_KEY = "#{COMPUTE_CONSTANTS[:service]}/resource_name".freeze
|
891
|
+
|
892
|
+
# Tags and their sanitized and encoded version.
|
893
|
+
VALID_TAGS = {
|
894
|
+
'test' => 'test',
|
895
|
+
'germanß' => 'german%C3%9F',
|
896
|
+
'chinese中' => 'chinese%E4%B8%AD',
|
897
|
+
'specialCharacter/_-.' => 'specialCharacter%2F_-.',
|
898
|
+
'abc@&^$*' => 'abc%40%26%5E%24%2A',
|
899
|
+
'@&^$*' => '%40%26%5E%24%2A'
|
900
|
+
}.freeze
|
901
|
+
INVALID_TAGS = {
|
902
|
+
# Non-string tags.
|
903
|
+
123 => '123',
|
904
|
+
1.23 => '1.23',
|
905
|
+
[1, 2, 3] => '%5B1%2C%202%2C%203%5D',
|
906
|
+
{ key: 'value' } => '%7B%22key%22%3D%3E%22value%22%7D',
|
907
|
+
# Non-utf8 string tags.
|
908
|
+
"nonutf8#{[0x92].pack('C*')}" => 'nonutf8%20',
|
909
|
+
"abc#{[0x92].pack('C*')}" => 'abc%20',
|
910
|
+
[0x92].pack('C*') => '%20',
|
911
|
+
# Empty string tag.
|
912
|
+
'' => '_'
|
913
|
+
}.freeze
|
914
|
+
ALL_TAGS = VALID_TAGS.merge(INVALID_TAGS)
|
915
|
+
|
916
|
+
# Stub value for Monitored resources from Metadata Agent.
|
917
|
+
# Map from the local_resource_id to the retrieved monitored resource.
|
918
|
+
MONITORED_RESOURCE_STUBS = {
|
919
|
+
# GKE container logs.
|
920
|
+
"#{CONTAINER_LOCAL_RESOURCE_ID_PREFIX}.#{CONTAINER_NAMESPACE_ID}" \
|
921
|
+
".#{K8S_POD_NAME}.#{K8S_CONTAINER_NAME}" =>
|
922
|
+
{
|
923
|
+
'type' => GKE_CONSTANTS[:resource_type],
|
924
|
+
'labels' => {
|
925
|
+
'cluster_name' => K8S_CLUSTER_NAME,
|
926
|
+
'container_name' => K8S_CONTAINER_NAME,
|
927
|
+
'instance_id' => VM_ID,
|
928
|
+
'namespace_id' => CONTAINER_NAMESPACE_ID,
|
929
|
+
'pod_id' => CONTAINER_POD_ID,
|
930
|
+
'zone' => ZONE
|
931
|
+
}
|
932
|
+
}.to_json,
|
933
|
+
# K8s container logs.
|
934
|
+
"#{K8S_CONTAINER_LOCAL_RESOURCE_ID_PREFIX}.#{K8S_NAMESPACE_NAME}" \
|
935
|
+
".#{K8S_POD_NAME}.#{K8S_CONTAINER_NAME}" =>
|
936
|
+
{
|
937
|
+
'type' => K8S_CONTAINER_CONSTANTS[:resource_type],
|
938
|
+
'labels' => {
|
939
|
+
'namespace_name' => K8S_NAMESPACE_NAME,
|
940
|
+
'pod_name' => K8S_POD_NAME,
|
941
|
+
'container_name' => K8S_CONTAINER_NAME,
|
942
|
+
'cluster_name' => K8S_CLUSTER_NAME,
|
943
|
+
'location' => K8S_LOCATION
|
944
|
+
}
|
945
|
+
}.to_json,
|
946
|
+
# K8s pod logs.
|
947
|
+
"#{K8S_POD_LOCAL_RESOURCE_ID_PREFIX}.#{K8S_NAMESPACE_NAME}" \
|
948
|
+
".#{K8S_POD_NAME}" =>
|
949
|
+
{
|
950
|
+
'type' => K8S_POD_CONSTANTS[:resource_type],
|
951
|
+
'labels' => {
|
952
|
+
'namespace_name' => K8S_NAMESPACE_NAME,
|
953
|
+
'pod_name' => K8S_POD_NAME,
|
954
|
+
'cluster_name' => K8S_CLUSTER_NAME,
|
955
|
+
'location' => K8S_LOCATION
|
956
|
+
}
|
957
|
+
}.to_json,
|
958
|
+
# K8s node logs.
|
959
|
+
"#{K8S_NODE_LOCAL_RESOURCE_ID_PREFIX}.#{K8S_NODE_NAME}" =>
|
960
|
+
{
|
961
|
+
'type' => K8S_NODE_CONSTANTS[:resource_type],
|
962
|
+
'labels' => {
|
963
|
+
'node_name' => K8S_NODE_NAME,
|
964
|
+
'cluster_name' => K8S_CLUSTER_NAME,
|
965
|
+
'location' => K8S_LOCATION
|
966
|
+
}
|
967
|
+
}.to_json
|
968
|
+
}.freeze
|
969
|
+
|
970
|
+
PARTIAL_SUCCESS_RESPONSE_BODY = {
|
971
|
+
'error' => {
|
972
|
+
'code' => 403,
|
973
|
+
'message' => 'User not authorized.',
|
974
|
+
'status' => 'PERMISSION_DENIED',
|
975
|
+
'details' => [
|
976
|
+
{
|
977
|
+
'@type' => 'type.googleapis.com/google.logging.v2.WriteLogEntriesPa' \
|
978
|
+
'rtialErrors',
|
979
|
+
'logEntryErrors' => {
|
980
|
+
'0' => {
|
981
|
+
'code' => 7,
|
982
|
+
'message' => 'User not authorized.'
|
983
|
+
},
|
984
|
+
'1' => {
|
985
|
+
'code' => 3,
|
986
|
+
'message' => 'Log name contains illegal character :'
|
987
|
+
},
|
988
|
+
'2' => {
|
989
|
+
'code' => 3,
|
990
|
+
'message' => 'Log name contains illegal character :'
|
991
|
+
}
|
992
|
+
}
|
993
|
+
},
|
994
|
+
{
|
995
|
+
'@type' => 'type.googleapis.com/google.rpc.DebugInfo',
|
996
|
+
'detail' => '[ORIGINAL ERROR] generic::permission_denied: User not ' \
|
997
|
+
'authorized. [google.rpc.error_details_ext] { message: \"User not' \
|
998
|
+
' authorized.\" details { type_url: \"type.googleapis.com/google.' \
|
999
|
+
'logging.v2.WriteLogEntriesPartialErrors\" value: \"\\n\\034\\010' \
|
1000
|
+
'\\000\\022\\030\\010\\007\\022\\024User not authorized.\\n-\\010' \
|
1001
|
+
'\\001\\022)\\010\\003\\022%Log name contains illegal character :' \
|
1002
|
+
'\\n-\\010\\002\\022)\\010\\003\\022%Log name contains illegal ch' \
|
1003
|
+
'aracter :\" } }'
|
1004
|
+
}
|
1005
|
+
]
|
1006
|
+
}
|
1007
|
+
}.freeze
|
1008
|
+
|
1009
|
+
PARTIAL_SUCCESS_GRPC_METADATA = begin
|
1010
|
+
partial_errors = Google::Cloud::Logging::V2::WriteLogEntriesPartialErrors.new(
|
1011
|
+
log_entry_errors: {
|
1012
|
+
0 => Google::Rpc::Status.new(
|
1013
|
+
code: GRPC::Core::StatusCodes::PERMISSION_DENIED,
|
1014
|
+
message: 'User not authorized.',
|
1015
|
+
details: []
|
1016
|
+
),
|
1017
|
+
1 => Google::Rpc::Status.new(
|
1018
|
+
code: GRPC::Core::StatusCodes::INVALID_ARGUMENT,
|
1019
|
+
message: 'Log name contains illegal character :',
|
1020
|
+
details: []
|
1021
|
+
),
|
1022
|
+
3 => Google::Rpc::Status.new(
|
1023
|
+
code: GRPC::Core::StatusCodes::INVALID_ARGUMENT,
|
1024
|
+
message: 'Log name contains illegal character :',
|
1025
|
+
details: []
|
1026
|
+
)
|
1027
|
+
}
|
1028
|
+
)
|
1029
|
+
status = Google::Rpc::Status.new(
|
1030
|
+
message: 'User not authorized.',
|
1031
|
+
details: [Google::Protobuf::Any.pack(partial_errors)]
|
1032
|
+
)
|
1033
|
+
debug_info = Google::Rpc::DebugInfo.new(
|
1034
|
+
detail: '[ORIGINAL ERROR] generic::permission_denied: User not' \
|
1035
|
+
' authorized. [google.rpc.error_details_ext] { message:' \
|
1036
|
+
" #{status.message.inspect} details { type_url:" \
|
1037
|
+
" #{status.details[0].type_url.inspect} value:" \
|
1038
|
+
" #{status.details[0].value.inspect_octal} } }"
|
1039
|
+
)
|
1040
|
+
status_details = Google::Rpc::Status.new(
|
1041
|
+
code: 7, message: 'User not authorized.',
|
1042
|
+
details: [Google::Protobuf::Any.pack(partial_errors),
|
1043
|
+
Google::Protobuf::Any.pack(debug_info)]
|
1044
|
+
)
|
1045
|
+
{
|
1046
|
+
'google.logging.v2.writelogentriespartialerrors-bin' =>
|
1047
|
+
partial_errors.to_proto,
|
1048
|
+
'google.rpc.debuginfo-bin' => debug_info.to_proto,
|
1049
|
+
'grpc-status-details-bin' => status_details.to_proto
|
1050
|
+
}.freeze
|
1051
|
+
end
|
1052
|
+
|
1053
|
+
PARSE_ERROR_RESPONSE_BODY = {
|
1054
|
+
'error' => {
|
1055
|
+
'code' => 400,
|
1056
|
+
'message' => 'Request contains an invalid argument.',
|
1057
|
+
'status' => 'INVALID_ARGUMENT',
|
1058
|
+
'details' => [
|
1059
|
+
{
|
1060
|
+
'@type' => 'type.googleapis.com/google.rpc.DebugInfo',
|
1061
|
+
'detail' =>
|
1062
|
+
'[ORIGINAL ERROR] RPC::CLIENT_ERROR: server could not parse' \
|
1063
|
+
" request sent by client; initialization error is: ''"
|
1064
|
+
}
|
1065
|
+
]
|
1066
|
+
}
|
1067
|
+
}.freeze
|
1068
|
+
|
1069
|
+
PARSE_ERROR_GRPC_METADATA = begin
|
1070
|
+
debug_info = Google::Rpc::DebugInfo.new(
|
1071
|
+
detail: '[ORIGINAL ERROR] RPC::CLIENT_ERROR: server could not parse' \
|
1072
|
+
" request sent by client; initialization error is: ''"
|
1073
|
+
)
|
1074
|
+
status_details = Google::Rpc::Status.new(
|
1075
|
+
code: 3, message: 'internal client error',
|
1076
|
+
details: [Google::Protobuf::Any.pack(debug_info)]
|
1077
|
+
)
|
1078
|
+
{
|
1079
|
+
'google.rpc.debuginfo-bin' => debug_info.to_proto,
|
1080
|
+
'grpc-status-details-bin' => status_details.to_proto
|
1081
|
+
}.freeze
|
1082
|
+
end
|
1083
|
+
|
1084
|
+
PRESERVED_KEYS_TIMESTAMP_FIELDS = [
|
1085
|
+
{
|
1086
|
+
'time' => K8S_TIMESTAMP
|
1087
|
+
},
|
1088
|
+
{
|
1089
|
+
'timeNanos' => K8S_NANOS
|
1090
|
+
},
|
1091
|
+
{
|
1092
|
+
'timestamp' => {
|
1093
|
+
'nanos' => K8S_NANOS,
|
1094
|
+
'seconds' => K8S_SECONDS_EPOCH
|
1095
|
+
}
|
1096
|
+
},
|
1097
|
+
{
|
1098
|
+
'timestampNanos' => K8S_NANOS,
|
1099
|
+
'timestampSeconds' => K8S_SECONDS_EPOCH
|
1100
|
+
}
|
1101
|
+
].freeze
|
1102
|
+
|
1103
|
+
PRESERVED_KEYS_MAP = {
|
1104
|
+
'severity' => CONTAINER_SEVERITY,
|
1105
|
+
DEFAULT_HTTP_REQUEST_KEY => HTTP_REQUEST_MESSAGE,
|
1106
|
+
DEFAULT_INSERT_ID_KEY => INSERT_ID,
|
1107
|
+
DEFAULT_LABELS_KEY => LABELS_MESSAGE,
|
1108
|
+
DEFAULT_OPERATION_KEY => OPERATION_MESSAGE,
|
1109
|
+
DEFAULT_SOURCE_LOCATION_KEY => SOURCE_LOCATION_MESSAGE,
|
1110
|
+
DEFAULT_SPAN_ID_KEY => SPAN_ID,
|
1111
|
+
DEFAULT_TRACE_KEY => TRACE,
|
1112
|
+
DEFAULT_TRACE_SAMPLED_KEY => TRACE_SAMPLED
|
1113
|
+
}.freeze
|
1114
|
+
end
|