fluent-plugin-google-cloud 0.6.5.pre.1 → 0.6.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +8 -10
- data/fluent-plugin-google-cloud.gemspec +1 -2
- data/lib/fluent/plugin/out_google_cloud.rb +405 -614
- data/test/plugin/base_test.rb +134 -348
- data/test/plugin/constants.rb +24 -159
- metadata +4 -18
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d7a29c9e8d247a3ca399605a375529a371856aa2
|
4
|
+
data.tar.gz: 49254ff8d84d7ab097ae2087f830882c7c990f8d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 582f2b49c2a5fbbefdeef47e124b6de5ac197028630c8ee2fe784cfa00ae914158c358ac04e88e06bab539c3969b29ba8b700d7048b0d5436a54733cba229808
|
7
|
+
data.tar.gz: 00decd87d0b8b4b4f1d1be8a77ec332d2b276f0c29edb7b0d6e34e0f950eb72e48b32948db3be40edfd8e3483cfe5ff166f5b51fef96128c296e0a867483f418
|
data/Gemfile.lock
CHANGED
@@ -1,8 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
fluent-plugin-google-cloud (0.6.5
|
5
|
-
excon (~> 0.57.1)
|
4
|
+
fluent-plugin-google-cloud (0.6.5)
|
6
5
|
fluentd (~> 0.10)
|
7
6
|
google-api-client (~> 0.9.0)
|
8
7
|
google-cloud-logging (= 0.24.1)
|
@@ -22,8 +21,7 @@ GEM
|
|
22
21
|
cool.io (1.5.1)
|
23
22
|
crack (0.4.3)
|
24
23
|
safe_yaml (~> 1.0.0)
|
25
|
-
|
26
|
-
faraday (0.12.2)
|
24
|
+
faraday (0.13.1)
|
27
25
|
multipart-post (>= 1.2, < 3)
|
28
26
|
fluentd (0.14.20)
|
29
27
|
cool.io (>= 1.4.5, < 2.0.0)
|
@@ -51,13 +49,13 @@ GEM
|
|
51
49
|
google-cloud-core (~> 0.21.1)
|
52
50
|
google-gax (~> 0.8.0)
|
53
51
|
stackdriver-core (~> 0.21.0)
|
54
|
-
google-gax (0.8.
|
52
|
+
google-gax (0.8.6)
|
55
53
|
google-protobuf (~> 3.2)
|
56
54
|
googleapis-common-protos (~> 1.3.5)
|
57
55
|
googleauth (~> 0.5.1)
|
58
56
|
grpc (~> 1.0)
|
59
57
|
rly (~> 0.2.3)
|
60
|
-
google-protobuf (3.
|
58
|
+
google-protobuf (3.4.0.2)
|
61
59
|
googleapis-common-protos (1.3.5)
|
62
60
|
google-protobuf (~> 3.2)
|
63
61
|
grpc (~> 1.0)
|
@@ -69,10 +67,10 @@ GEM
|
|
69
67
|
multi_json (~> 1.11)
|
70
68
|
os (~> 0.9)
|
71
69
|
signet (~> 0.7)
|
72
|
-
grpc (1.2.5
|
70
|
+
grpc (1.2.5)
|
73
71
|
google-protobuf (~> 3.1)
|
74
72
|
googleauth (~> 0.5.1)
|
75
|
-
hashdiff (0.3.
|
73
|
+
hashdiff (0.3.6)
|
76
74
|
http_parser.rb (0.6.0)
|
77
75
|
httpclient (2.8.3)
|
78
76
|
hurley (0.2)
|
@@ -95,7 +93,7 @@ GEM
|
|
95
93
|
os (0.9.6)
|
96
94
|
parser (2.4.0.0)
|
97
95
|
ast (~> 2.2)
|
98
|
-
power_assert (1.0
|
96
|
+
power_assert (1.1.0)
|
99
97
|
powerpack (0.1.1)
|
100
98
|
prometheus-client (0.7.1)
|
101
99
|
quantile (~> 0.2.0)
|
@@ -156,4 +154,4 @@ DEPENDENCIES
|
|
156
154
|
webmock (~> 2.3.1)
|
157
155
|
|
158
156
|
BUNDLED WITH
|
159
|
-
1.15.
|
157
|
+
1.15.4
|
@@ -10,7 +10,7 @@ eos
|
|
10
10
|
gem.homepage = \
|
11
11
|
'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
|
12
12
|
gem.license = 'Apache-2.0'
|
13
|
-
gem.version = '0.6.5
|
13
|
+
gem.version = '0.6.5'
|
14
14
|
gem.authors = ['Todd Derr', 'Alex Robinson']
|
15
15
|
gem.email = ['salty@google.com']
|
16
16
|
gem.required_ruby_version = Gem::Requirement.new('>= 2.0')
|
@@ -19,7 +19,6 @@ eos
|
|
19
19
|
gem.test_files = gem.files.grep(/^(test)/)
|
20
20
|
gem.require_paths = ['lib']
|
21
21
|
|
22
|
-
gem.add_runtime_dependency 'excon', '~> 0.57.1'
|
23
22
|
gem.add_runtime_dependency 'fluentd', '~> 0.10'
|
24
23
|
gem.add_runtime_dependency 'googleapis-common-protos', '~> 1.3'
|
25
24
|
gem.add_runtime_dependency 'google-api-client', '~> 0.9.0'
|
@@ -11,11 +11,9 @@
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
12
|
# See the License for the specific language governing permissions and
|
13
13
|
# limitations under the License.
|
14
|
-
require 'excon'
|
15
14
|
require 'grpc'
|
16
15
|
require 'json'
|
17
16
|
require 'open-uri'
|
18
|
-
require 'rubygems'
|
19
17
|
require 'socket'
|
20
18
|
require 'time'
|
21
19
|
require 'yaml'
|
@@ -40,13 +38,12 @@ end
|
|
40
38
|
module Fluent
|
41
39
|
# fluentd output plugin for the Stackdriver Logging API
|
42
40
|
class GoogleCloudOutput < BufferedOutput
|
43
|
-
# Constants.
|
41
|
+
# Constants for service names and resource types.
|
44
42
|
module Constants
|
45
|
-
# Service names and resource types.
|
46
|
-
|
47
43
|
APPENGINE_CONSTANTS = {
|
48
44
|
service: 'appengine.googleapis.com',
|
49
|
-
resource_type: 'gae_app'
|
45
|
+
resource_type: 'gae_app',
|
46
|
+
metadata_attributes: %w(gae_backend_name gae_backend_version).to_set
|
50
47
|
}
|
51
48
|
CLOUDFUNCTIONS_CONSTANTS = {
|
52
49
|
service: 'cloudfunctions.googleapis.com',
|
@@ -58,19 +55,21 @@ module Fluent
|
|
58
55
|
}
|
59
56
|
CONTAINER_CONSTANTS = {
|
60
57
|
service: 'container.googleapis.com',
|
61
|
-
resource_type: 'container'
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
resource_type: 'docker_container'
|
58
|
+
resource_type: 'container',
|
59
|
+
extra_resource_labels: %w(namespace_id pod_id container_name),
|
60
|
+
extra_common_labels: %w(namespace_name pod_name),
|
61
|
+
metadata_attributes: %w(kube-env).to_set
|
66
62
|
}
|
67
63
|
DATAFLOW_CONSTANTS = {
|
68
64
|
service: 'dataflow.googleapis.com',
|
69
|
-
resource_type: 'dataflow_step'
|
65
|
+
resource_type: 'dataflow_step',
|
66
|
+
extra_common_labels: %w(region job_name job_id step_id)
|
70
67
|
}
|
71
68
|
DATAPROC_CONSTANTS = {
|
72
69
|
service: 'cluster.dataproc.googleapis.com',
|
73
|
-
resource_type: 'cloud_dataproc_cluster'
|
70
|
+
resource_type: 'cloud_dataproc_cluster',
|
71
|
+
metadata_attributes:
|
72
|
+
%w(dataproc-cluster-uuid dataproc-cluster-name).to_set
|
74
73
|
}
|
75
74
|
EC2_CONSTANTS = {
|
76
75
|
service: 'ec2.amazonaws.com',
|
@@ -78,25 +77,83 @@ module Fluent
|
|
78
77
|
}
|
79
78
|
ML_CONSTANTS = {
|
80
79
|
service: 'ml.googleapis.com',
|
81
|
-
resource_type: 'ml_job'
|
80
|
+
resource_type: 'ml_job',
|
81
|
+
extra_common_labels: %w(job_id task_name)
|
82
82
|
}
|
83
83
|
|
84
|
-
#
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
#
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
#
|
99
|
-
|
84
|
+
# The map between a subservice name and a resource type.
|
85
|
+
SUBSERVICE_MAP = \
|
86
|
+
[APPENGINE_CONSTANTS, CONTAINER_CONSTANTS, DATAFLOW_CONSTANTS,
|
87
|
+
DATAPROC_CONSTANTS, ML_CONSTANTS]
|
88
|
+
.map { |consts| [consts[:service], consts[:resource_type]] }.to_h
|
89
|
+
# Default back to GCE if invalid value is detected.
|
90
|
+
SUBSERVICE_MAP.default = COMPUTE_CONSTANTS[:resource_type]
|
91
|
+
|
92
|
+
# The map between a resource type and expected subservice attributes.
|
93
|
+
SUBSERVICE_METADATA_ATTRIBUTES = \
|
94
|
+
[APPENGINE_CONSTANTS, CONTAINER_CONSTANTS, DATAPROC_CONSTANTS]
|
95
|
+
.map { |consts| [consts[:resource_type], consts[:metadata_attributes]] }
|
96
|
+
.to_h
|
97
|
+
|
98
|
+
# Default values for JSON payload keys to set the "trace",
|
99
|
+
# "sourceLocation", "operation" and "labels" fields in the LogEntry.
|
100
|
+
DEFAULT_PAYLOAD_KEY_PREFIX = 'logging.googleapis.com'
|
101
|
+
DEFAULT_LABELS_KEY = "#{DEFAULT_PAYLOAD_KEY_PREFIX}/labels"
|
102
|
+
DEFAULT_HTTP_REQUEST_KEY = 'httpRequest'
|
103
|
+
DEFAULT_OPERATION_KEY = "#{DEFAULT_PAYLOAD_KEY_PREFIX}/operation"
|
104
|
+
DEFAULT_SOURCE_LOCATION_KEY =
|
105
|
+
"#{DEFAULT_PAYLOAD_KEY_PREFIX}/sourceLocation"
|
106
|
+
DEFAULT_TRACE_KEY = "#{DEFAULT_PAYLOAD_KEY_PREFIX}/trace"
|
107
|
+
|
108
|
+
# Map from each field name under LogEntry to corresponding variables
|
109
|
+
# required to perform field value extraction from the log record.
|
110
|
+
LOG_ENTRY_FIELDS_MAP = {
|
111
|
+
'http_request' => [
|
112
|
+
# The config to specify label name for field extraction from record.
|
113
|
+
'@http_request_key',
|
114
|
+
# Map from subfields' names to their types.
|
115
|
+
[
|
116
|
+
# subfield key in the payload, destination key, cast lambda (opt)
|
117
|
+
%w(requestMethod request_method parse_string),
|
118
|
+
%w(requestUrl request_url parse_string),
|
119
|
+
%w(requestSize request_size parse_int),
|
120
|
+
%w(status status parse_int),
|
121
|
+
%w(responseSize response_size parse_int),
|
122
|
+
%w(userAgent user_agent parse_string),
|
123
|
+
%w(remoteIp remote_ip parse_string),
|
124
|
+
%w(referer referer parse_string),
|
125
|
+
%w(cacheHit cache_hit parse_bool),
|
126
|
+
%w(cacheValidatedWithOriginServer
|
127
|
+
cache_validated_with_origin_server parse_bool),
|
128
|
+
%w(latency latency parse_latency)
|
129
|
+
],
|
130
|
+
# The grpc version class name.
|
131
|
+
'Google::Logging::Type::HttpRequest',
|
132
|
+
# The non-grpc version class name.
|
133
|
+
'Google::Apis::LoggingV2beta1::HttpRequest'
|
134
|
+
],
|
135
|
+
'source_location' => [
|
136
|
+
'@source_location_key',
|
137
|
+
[
|
138
|
+
%w(file file parse_string),
|
139
|
+
%w(function function parse_string),
|
140
|
+
%w(line line parse_int)
|
141
|
+
],
|
142
|
+
'Google::Logging::V2::LogEntrySourceLocation',
|
143
|
+
'Google::Apis::LoggingV2beta1::LogEntrySourceLocation'
|
144
|
+
],
|
145
|
+
'operation' => [
|
146
|
+
'@operation_key',
|
147
|
+
[
|
148
|
+
%w(id id parse_string),
|
149
|
+
%w(producer producer parse_string),
|
150
|
+
%w(first first parse_bool),
|
151
|
+
%w(last last parse_bool)
|
152
|
+
],
|
153
|
+
'Google::Logging::V2::LogEntryOperation',
|
154
|
+
'Google::Apis::LoggingV2beta1::LogEntryOperation'
|
155
|
+
]
|
156
|
+
}
|
100
157
|
end
|
101
158
|
|
102
159
|
include self::Constants
|
@@ -104,7 +161,7 @@ module Fluent
|
|
104
161
|
Fluent::Plugin.register_output('google_cloud', self)
|
105
162
|
|
106
163
|
PLUGIN_NAME = 'Fluentd Google Cloud Logging plugin'
|
107
|
-
PLUGIN_VERSION = '0.6.5
|
164
|
+
PLUGIN_VERSION = '0.6.5'
|
108
165
|
|
109
166
|
# Name of the the Google cloud logging write scope.
|
110
167
|
LOGGING_SCOPE = 'https://www.googleapis.com/auth/logging.write'
|
@@ -134,10 +191,14 @@ module Fluent
|
|
134
191
|
config_param :vm_id, :string, :default => nil
|
135
192
|
config_param :vm_name, :string, :default => nil
|
136
193
|
|
137
|
-
#
|
194
|
+
# Map keys from a JSON payload to corresponding LogEntry fields.
|
195
|
+
config_param :labels_key, :string, :default => DEFAULT_LABELS_KEY
|
196
|
+
config_param :http_request_key, :string, :default =>
|
197
|
+
DEFAULT_HTTP_REQUEST_KEY
|
198
|
+
config_param :operation_key, :string, :default => DEFAULT_OPERATION_KEY
|
199
|
+
config_param :source_location_key, :string, :default =>
|
200
|
+
DEFAULT_SOURCE_LOCATION_KEY
|
138
201
|
config_param :trace_key, :string, :default => DEFAULT_TRACE_KEY
|
139
|
-
# Whether to also keep the trace key/value in the payload.
|
140
|
-
config_param :keep_trace_key, :bool, :default => false
|
141
202
|
|
142
203
|
# Whether to try to detect if the VM is owned by a "subservice" such as App
|
143
204
|
# Engine of Kubernetes, rather than just associating the logs with the
|
@@ -231,18 +292,6 @@ module Fluent
|
|
231
292
|
config_param :monitoring_type, :string,
|
232
293
|
:default => Monitoring::PrometheusMonitoringRegistry.name
|
233
294
|
|
234
|
-
# Whether to call metadata agent to retrieve monitored resource.
|
235
|
-
config_param :enable_metadata_agent, :bool, :default => false
|
236
|
-
config_param :metadata_agent_url, :string,
|
237
|
-
:default => 'http://local-metadata-agent.stackdriver.com:8000'
|
238
|
-
|
239
|
-
# Whether to call Docker Remote API locally when Metadata Agent is not
|
240
|
-
# enabled or if the request fails.
|
241
|
-
config_param :call_docker_api_locally, :bool, :default => true
|
242
|
-
# Docker Remote API unix socket path.
|
243
|
-
config_param :docker_remote_api_socket_path, :string,
|
244
|
-
:default => DEFAULT_DOCKER_API_SOCKET_PATH
|
245
|
-
|
246
295
|
# rubocop:enable Style/HashSyntax
|
247
296
|
|
248
297
|
# TODO: Add a log_name config option rather than just using the tag?
|
@@ -261,35 +310,6 @@ module Fluent
|
|
261
310
|
@log = $log # rubocop:disable Style/GlobalVars
|
262
311
|
end
|
263
312
|
|
264
|
-
# Set up regex patterns used to parse tags and logs.
|
265
|
-
def setup_regex_patterns
|
266
|
-
@compiled_kubernetes_tag_regexp = nil
|
267
|
-
if @kubernetes_tag_regexp
|
268
|
-
@compiled_kubernetes_tag_regexp = Regexp.new(@kubernetes_tag_regexp)
|
269
|
-
end
|
270
|
-
|
271
|
-
@cloudfunctions_tag_regexp =
|
272
|
-
/\.(?<encoded_function_name>.+)\.\d+-[^-]+_default_worker$/
|
273
|
-
@cloudfunctions_log_regexp = /^
|
274
|
-
(?:\[(?<severity>.)\])?
|
275
|
-
\[(?<timestamp>.{24})\]
|
276
|
-
(?:\[(?<execution_id>[^\]]+)\])?
|
277
|
-
[ ](?<text>.*)$/x
|
278
|
-
|
279
|
-
# Docker container tag format:
|
280
|
-
# "container.<container_id>.<container_name>".
|
281
|
-
@dockercontainer_tag_regexp =
|
282
|
-
/^container\.(?<container_id>[a-zA-Z0-9]+)\.
|
283
|
-
(?<container_name>[a-zA-Z0-9_.-]+)$/x
|
284
|
-
# Docker container with application tag format:
|
285
|
-
# "application-container.<container_name>.<additional_tag>".
|
286
|
-
@dockercontainer_tag_with_application_regexp =
|
287
|
-
/^application-container\.(?<container_name>[a-zA-Z0-9_.-]+)\.
|
288
|
-
(?<additional_tag>.+)$/x
|
289
|
-
|
290
|
-
@http_latency_regexp = /^\s*(?<seconds>\d+)(?<decimal>\.\d+)?\s*s\s*$/
|
291
|
-
end
|
292
|
-
|
293
313
|
def configure(conf)
|
294
314
|
super
|
295
315
|
|
@@ -327,49 +347,46 @@ module Fluent
|
|
327
347
|
extra.join(' ')
|
328
348
|
end
|
329
349
|
|
330
|
-
|
350
|
+
set_regexp_patterns
|
331
351
|
|
332
352
|
@platform = detect_platform
|
333
353
|
|
334
|
-
# Set
|
335
|
-
#
|
336
|
-
#
|
337
|
-
|
338
|
-
|
339
|
-
|
340
|
-
# explicitly send the key.
|
341
|
-
@resource = call_metadata_agent_for_monitored_resource(
|
342
|
-
IMPLICIT_MONITORED_RESOURCE_UNIQUE_KEY)
|
343
|
-
end
|
344
|
-
|
345
|
-
# Set required variables: @project_id, @vm_id, @vm_name and @zone.
|
346
|
-
# If any info above is included in the response from Metadata Agent, make
|
347
|
-
# use of that. Otherwise make some additional requests to metadata server.
|
354
|
+
# Set required variables: @project_id, @vm_id, @vm_name and @zone by
|
355
|
+
# making some requests to metadata server.
|
356
|
+
#
|
357
|
+
# Note: Once we support metadata injection at Logging API side, we might
|
358
|
+
# no longer need to require all these metadata in logging agent. But for
|
359
|
+
# now, they are still required.
|
348
360
|
#
|
349
|
-
#
|
350
|
-
#
|
351
|
-
# for now, they are still required.
|
361
|
+
# TODO(qingling128): After Metadata Agent support is added, try extracting
|
362
|
+
# these info from responses from Metadata Agent first.
|
352
363
|
set_required_metadata_variables
|
353
364
|
|
354
|
-
#
|
355
|
-
#
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
|
360
|
-
#
|
361
|
-
|
362
|
-
#
|
363
|
-
|
364
|
-
|
365
|
-
# We
|
366
|
-
|
367
|
-
|
368
|
-
|
369
|
-
|
370
|
-
|
371
|
-
|
372
|
-
|
365
|
+
# Retrieve monitored resource.
|
366
|
+
#
|
367
|
+
# TODO(qingling128): After Metadata Agent support is added, try retrieving
|
368
|
+
# the monitored resource from Metadata Agent first.
|
369
|
+
@resource = determine_agent_level_monitored_resource_via_legacy
|
370
|
+
|
371
|
+
# Set regexp that we should match tags against later on. Using a list
|
372
|
+
# instead of a map to ensure order. For example, tags will be matched
|
373
|
+
# against Cloud Functions first, then GKE.
|
374
|
+
@tag_regexp_list = []
|
375
|
+
if @resource.type == CONTAINER_CONSTANTS[:resource_type]
|
376
|
+
# We only support Cloud Functions logs for GKE right now.
|
377
|
+
if fetch_gce_metadata('instance/attributes/'
|
378
|
+
).split.include?('gcf_region')
|
379
|
+
# Fetch this info and store it to avoid recurring
|
380
|
+
# metadata server calls.
|
381
|
+
@gcf_region = fetch_gce_metadata('instance/attributes/gcf_region')
|
382
|
+
@tag_regexp_list << [
|
383
|
+
CLOUDFUNCTIONS_CONSTANTS[:resource_type],
|
384
|
+
@compiled_cloudfunctions_tag_regexp
|
385
|
+
]
|
386
|
+
end
|
387
|
+
@tag_regexp_list << [
|
388
|
+
CONTAINER_CONSTANTS[:resource_type], @compiled_kubernetes_tag_regexp
|
389
|
+
]
|
373
390
|
end
|
374
391
|
|
375
392
|
# Determine the common labels that should be added to all log entries
|
@@ -384,7 +401,7 @@ module Fluent
|
|
384
401
|
|
385
402
|
# Log an informational message containing the Logs viewer URL
|
386
403
|
@log.info 'Logs viewer address: https://console.cloud.google.com/logs/',
|
387
|
-
"viewer?project=#{@project_id}&resource=#{@
|
404
|
+
"viewer?project=#{@project_id}&resource=#{@resource_type}/",
|
388
405
|
"instance_id/#{@vm_id}"
|
389
406
|
end
|
390
407
|
|
@@ -421,16 +438,14 @@ module Fluent
|
|
421
438
|
arr.each do |time, record|
|
422
439
|
next unless record.is_a?(Hash)
|
423
440
|
|
424
|
-
|
441
|
+
extracted_resource_labels, extracted_common_labels = \
|
425
442
|
determine_entry_level_labels(group_resource, record)
|
426
443
|
entry_resource = group_resource.dup
|
427
|
-
entry_resource.type = resource_type
|
428
444
|
entry_resource.labels.merge!(extracted_resource_labels)
|
429
445
|
entry_common_labels = \
|
430
446
|
group_common_labels.merge(extracted_common_labels)
|
431
447
|
|
432
|
-
if
|
433
|
-
DOCKER_CONSTANTS[:resource_type]].include?(entry_resource.type)
|
448
|
+
if entry_resource.type == CONTAINER_CONSTANTS[:resource_type]
|
434
449
|
# Save the timestamp if available, then clear it out to allow for
|
435
450
|
# determining whether we should parse the log or message field.
|
436
451
|
timestamp = record.key?('time') ? record['time'] : nil
|
@@ -459,13 +474,16 @@ module Fluent
|
|
459
474
|
severity = compute_severity(
|
460
475
|
entry_resource.type, record, entry_common_labels)
|
461
476
|
|
462
|
-
|
463
|
-
|
464
|
-
|
465
|
-
|
466
|
-
|
467
|
-
|
468
|
-
|
477
|
+
ts_secs = begin
|
478
|
+
Integer ts_secs
|
479
|
+
rescue ArgumentError, TypeError
|
480
|
+
ts_secs
|
481
|
+
end
|
482
|
+
ts_nanos = begin
|
483
|
+
Integer ts_nanos
|
484
|
+
rescue ArgumentError, TypeError
|
485
|
+
ts_nanos
|
486
|
+
end
|
469
487
|
if @use_grpc
|
470
488
|
entry = Google::Logging::V2::LogEntry.new(
|
471
489
|
labels: entry_common_labels,
|
@@ -475,7 +493,6 @@ module Fluent
|
|
475
493
|
),
|
476
494
|
severity: grpc_severity(severity)
|
477
495
|
)
|
478
|
-
entry.trace = fq_trace_id if fq_trace_id
|
479
496
|
# If "seconds" is null or not an integer, we will omit the timestamp
|
480
497
|
# field and defer the decision on how to handle it to the downstream
|
481
498
|
# Logging API. If "nanos" is null or not an integer, it will be set
|
@@ -487,8 +504,6 @@ module Fluent
|
|
487
504
|
nanos: ts_nanos
|
488
505
|
)
|
489
506
|
end
|
490
|
-
set_http_request(record, entry)
|
491
|
-
set_payload_grpc(entry_resource.type, record, entry, is_json)
|
492
507
|
else
|
493
508
|
# Remove the labels if we didn't populate them with anything.
|
494
509
|
entry_resource.labels = nil if entry_resource.labels.empty?
|
@@ -501,8 +516,18 @@ module Fluent
|
|
501
516
|
nanos: ts_nanos
|
502
517
|
}
|
503
518
|
)
|
504
|
-
|
505
|
-
|
519
|
+
end
|
520
|
+
|
521
|
+
# Get fully-qualified trace id for LogEntry "trace" field per config.
|
522
|
+
fq_trace_id = record.delete(@trace_key)
|
523
|
+
entry.trace = fq_trace_id if fq_trace_id
|
524
|
+
|
525
|
+
set_log_entry_fields(record, entry)
|
526
|
+
set_labels(record, entry)
|
527
|
+
|
528
|
+
if @use_grpc
|
529
|
+
set_payload_grpc(entry_resource.type, record, entry, is_json)
|
530
|
+
else
|
506
531
|
set_payload(entry_resource.type, record, entry, is_json)
|
507
532
|
end
|
508
533
|
|
@@ -722,20 +747,19 @@ module Fluent
|
|
722
747
|
|
723
748
|
# Set regexp patterns to parse tags and logs.
|
724
749
|
def set_regexp_patterns
|
725
|
-
@compiled_kubernetes_tag_regexp =
|
726
|
-
|
727
|
-
@compiled_kubernetes_tag_regexp = Regexp.new(@kubernetes_tag_regexp)
|
728
|
-
end
|
750
|
+
@compiled_kubernetes_tag_regexp = Regexp.new(@kubernetes_tag_regexp) if
|
751
|
+
@kubernetes_tag_regexp
|
729
752
|
|
730
|
-
@
|
753
|
+
@compiled_cloudfunctions_tag_regexp =
|
731
754
|
/\.(?<encoded_function_name>.+)\.\d+-[^-]+_default_worker$/
|
732
|
-
@
|
755
|
+
@compiled_cloudfunctions_log_regexp = /^
|
733
756
|
(?:\[(?<severity>.)\])?
|
734
757
|
\[(?<timestamp>.{24})\]
|
735
758
|
(?:\[(?<execution_id>[^\]]+)\])?
|
736
759
|
[ ](?<text>.*)$/x
|
737
760
|
|
738
|
-
@
|
761
|
+
@compiled_http_latency_regexp =
|
762
|
+
/^\s*(?<seconds>\d+)(?<decimal>\.\d+)?\s*s\s*$/
|
739
763
|
end
|
740
764
|
|
741
765
|
# Set required variables like @project_id, @vm_id, @vm_name and @zone.
|
@@ -746,11 +770,11 @@ module Fluent
|
|
746
770
|
set_location
|
747
771
|
|
748
772
|
# All metadata parameters must now be set.
|
749
|
-
return if @project_id && @zone && @vm_id
|
750
773
|
missing = []
|
751
774
|
missing << 'project_id' unless @project_id
|
752
775
|
missing << 'zone' unless @zone
|
753
776
|
missing << 'vm_id' unless @vm_id
|
777
|
+
return if missing.empty?
|
754
778
|
fail Fluent::ConfigError, 'Unable to obtain metadata parameters: ' +
|
755
779
|
missing.join(' ')
|
756
780
|
end
|
@@ -767,11 +791,8 @@ module Fluent
|
|
767
791
|
end
|
768
792
|
|
769
793
|
# 1. Return the value if it is explicitly set in the config already.
|
770
|
-
# 2. If not,
|
771
|
-
# 3. If not, try to retrieve it by calling metadata servers directly.
|
794
|
+
# 2. If not, try to retrieve it by calling metadata servers directly.
|
772
795
|
def set_vm_id
|
773
|
-
@vm_id ||= @resource.labels['instance_id'] if
|
774
|
-
!@resource.nil? && @resource.labels.key?('instance_id')
|
775
796
|
@vm_id ||= fetch_gce_metadata('instance/id') if @platform == Platform::GCE
|
776
797
|
@vm_id ||= ec2_metadata['instanceId'] if @platform == Platform::EC2
|
777
798
|
rescue StandardError => e
|
@@ -779,29 +800,16 @@ module Fluent
|
|
779
800
|
end
|
780
801
|
|
781
802
|
# 1. Return the value if it is explicitly set in the config already.
|
782
|
-
# 2. If not,
|
783
|
-
# 3. If not, try to retrieve it locally.
|
803
|
+
# 2. If not, try to retrieve it locally.
|
784
804
|
def set_vm_name
|
785
|
-
@vm_name ||= @resource.labels['instance_name'] if
|
786
|
-
!@resource.nil? && @resource.labels.key?('instance_name')
|
787
805
|
@vm_name ||= Socket.gethostname
|
788
806
|
rescue StandardError => e
|
789
807
|
@log.error 'Failed to obtain vm name: ', error: e
|
790
808
|
end
|
791
809
|
|
792
810
|
# 1. Return the value if it is explicitly set in the config already.
|
793
|
-
# 2. If not,
|
794
|
-
# 3. If not, try to retrieve it locally.
|
811
|
+
# 2. If not, try to retrieve it locally.
|
795
812
|
def set_location
|
796
|
-
unless @resource.nil?
|
797
|
-
@zone ||= @resource.labels['location'] if
|
798
|
-
@resource.type == DOCKER_CONSTANTS[:resource_type] &&
|
799
|
-
@resource.labels.key?('location')
|
800
|
-
@zone ||= @resource.labels['zone'] if
|
801
|
-
@platform == Platform::GCE && @resource.labels.key?('zone')
|
802
|
-
@zone ||= @resource.labels['region'] if
|
803
|
-
@platform == Platform::EC2 && @resource.labels.key?('region')
|
804
|
-
end
|
805
813
|
# Response format: "projects/<number>/zones/<zone>"
|
806
814
|
@zone ||= fetch_gce_metadata('instance/zone').rpartition('/')[2] if
|
807
815
|
@platform == Platform::GCE
|
@@ -813,9 +821,8 @@ module Fluent
|
|
813
821
|
|
814
822
|
# Retrieve monitored resource via the legacy way.
|
815
823
|
#
|
816
|
-
#
|
817
|
-
#
|
818
|
-
# returns.
|
824
|
+
# TODO(qingling128): Use this as only a fallback plan after Metadata Agent
|
825
|
+
# support is added.
|
819
826
|
def determine_agent_level_monitored_resource_via_legacy
|
820
827
|
resource = Google::Apis::LoggingV2beta1::MonitoredResource.new(
|
821
828
|
labels: {})
|
@@ -827,108 +834,92 @@ module Fluent
|
|
827
834
|
|
828
835
|
# Determine agent level monitored resource type.
|
829
836
|
def determine_agent_level_monitored_resource_type
|
830
|
-
|
831
|
-
|
832
|
-
|
837
|
+
case @platform
|
838
|
+
when Platform::OTHER
|
839
|
+
# Unknown platform will be defaulted to GCE instance.
|
840
|
+
return COMPUTE_CONSTANTS[:resource_type]
|
833
841
|
|
834
|
-
|
835
|
-
|
836
|
-
@platform == Platform::OTHER
|
837
|
-
|
838
|
-
# Resource types determined by @subservice_name config.
|
839
|
-
# Cloud Dataflow.
|
840
|
-
return DATAFLOW_CONSTANTS[:resource_type] if
|
841
|
-
@subservice_name == DATAFLOW_CONSTANTS[:service]
|
842
|
-
# Cloud ML.
|
843
|
-
return ML_CONSTANTS[:resource_type] if
|
844
|
-
@subservice_name == ML_CONSTANTS[:service]
|
845
|
-
# Default back to GCE if invalid value is detected.
|
846
|
-
return COMPUTE_CONSTANTS[:resource_type] if
|
847
|
-
@subservice_name
|
842
|
+
when Platform::EC2
|
843
|
+
return EC2_CONSTANTS[:resource_type]
|
848
844
|
|
849
|
-
|
850
|
-
|
851
|
-
|
852
|
-
|
853
|
-
|
854
|
-
|
845
|
+
when Platform::GCE
|
846
|
+
# Resource types determined by @subservice_name config.
|
847
|
+
return SUBSERVICE_MAP[@subservice_name] if @subservice_name
|
848
|
+
|
849
|
+
# Resource types determined by @detect_subservice config.
|
850
|
+
if @detect_subservice
|
851
|
+
begin
|
852
|
+
attributes = fetch_gce_metadata('instance/attributes/').split.to_set
|
853
|
+
SUBSERVICE_METADATA_ATTRIBUTES.each do |resource_type, expected|
|
854
|
+
return resource_type if attributes.superset?(expected)
|
855
|
+
end
|
856
|
+
rescue StandardError => e
|
857
|
+
@log.error 'Failed to detect subservice: ', error: e
|
858
|
+
end
|
855
859
|
end
|
856
|
-
|
857
|
-
|
858
|
-
|
859
|
-
attributes.include?('gae_backend_version')
|
860
|
-
# GKE container.
|
861
|
-
return CONTAINER_CONSTANTS[:resource_type] if
|
862
|
-
attributes.include?('kube-env')
|
863
|
-
# Cloud Dataproc.
|
864
|
-
return DATAPROC_CONSTANTS[:resource_type] if
|
865
|
-
attributes.include?('dataproc-cluster-uuid') &&
|
866
|
-
attributes.include?('dataproc-cluster-name')
|
860
|
+
|
861
|
+
# GCE instance.
|
862
|
+
return COMPUTE_CONSTANTS[:resource_type]
|
867
863
|
end
|
868
|
-
# GCE instance.
|
869
|
-
COMPUTE_CONSTANTS[:resource_type]
|
870
864
|
end
|
871
865
|
|
872
866
|
# Determine agent level monitored resource labels based on the resource
|
873
867
|
# type. Each resource type has its own labels that need to be filled in.
|
874
868
|
def determine_agent_level_monitored_resource_labels(type)
|
875
|
-
labels = {}
|
876
|
-
|
877
869
|
case type
|
878
|
-
|
879
870
|
# GAE app.
|
880
871
|
when APPENGINE_CONSTANTS[:resource_type]
|
881
|
-
|
882
|
-
|
883
|
-
'instance/attributes/gae_backend_name')
|
884
|
-
|
885
|
-
'instance/attributes/gae_backend_version')
|
886
|
-
|
887
|
-
@log.error 'Failed to set monitored resource labels for GAE: ',
|
888
|
-
error: e
|
889
|
-
end
|
872
|
+
return {
|
873
|
+
'module_id' =>
|
874
|
+
fetch_gce_metadata('instance/attributes/gae_backend_name'),
|
875
|
+
'version_id' =>
|
876
|
+
fetch_gce_metadata('instance/attributes/gae_backend_version')
|
877
|
+
}
|
890
878
|
|
891
879
|
# GCE.
|
892
880
|
when COMPUTE_CONSTANTS[:resource_type]
|
893
|
-
|
894
|
-
|
881
|
+
return {
|
882
|
+
'instance_id' => @vm_id,
|
883
|
+
'zone' => @zone
|
884
|
+
}
|
895
885
|
|
896
886
|
# GKE container.
|
897
887
|
when CONTAINER_CONSTANTS[:resource_type]
|
898
|
-
|
899
|
-
|
900
|
-
|
901
|
-
|
902
|
-
|
903
|
-
|
904
|
-
|
905
|
-
rescue StandardError => e
|
906
|
-
@log.error 'Failed to set monitored resource labels for GKE: ',
|
907
|
-
error: e
|
908
|
-
end
|
888
|
+
raw_kube_env = fetch_gce_metadata('instance/attributes/kube-env')
|
889
|
+
kube_env = YAML.load(raw_kube_env)
|
890
|
+
return {
|
891
|
+
'instance_id' => @vm_id,
|
892
|
+
'zone' => @zone,
|
893
|
+
'cluster_name' => cluster_name_from_kube_env(kube_env)
|
894
|
+
}
|
909
895
|
|
910
896
|
# Cloud Dataproc.
|
911
897
|
when DATAPROC_CONSTANTS[:resource_type]
|
912
|
-
|
913
|
-
|
914
|
-
fetch_gce_metadata('instance/attributes/dataproc-cluster-uuid')
|
915
|
-
|
916
|
-
fetch_gce_metadata('instance/attributes/dataproc-cluster-name')
|
917
|
-
|
898
|
+
return {
|
899
|
+
'cluster_uuid' =>
|
900
|
+
fetch_gce_metadata('instance/attributes/dataproc-cluster-uuid'),
|
901
|
+
'cluster_name' =>
|
902
|
+
fetch_gce_metadata('instance/attributes/dataproc-cluster-name'),
|
903
|
+
'region' =>
|
918
904
|
fetch_gce_metadata('instance/attributes/dataproc-region')
|
919
|
-
|
920
|
-
@log.error 'Failed to set monitored resource labels for Cloud ' \
|
921
|
-
'Dataproc: ', error: e
|
922
|
-
end
|
905
|
+
}
|
923
906
|
|
924
907
|
# EC2.
|
925
908
|
when EC2_CONSTANTS[:resource_type]
|
926
|
-
labels
|
927
|
-
|
909
|
+
labels = {
|
910
|
+
'instance_id' => @vm_id,
|
911
|
+
'region' => @zone
|
912
|
+
}
|
928
913
|
labels['aws_account'] = ec2_metadata['accountId'] if
|
929
914
|
ec2_metadata.key?('accountId')
|
915
|
+
return labels
|
930
916
|
end
|
931
|
-
|
917
|
+
|
918
|
+
{}
|
919
|
+
rescue StandardError => e
|
920
|
+
@log.error "Failed to set monitored resource labels for #{type}: ",
|
921
|
+
error: e
|
922
|
+
return {}
|
932
923
|
end
|
933
924
|
|
934
925
|
# Determine the common labels that should be added to all log entries
|
@@ -936,38 +927,30 @@ module Fluent
|
|
936
927
|
def determine_agent_level_common_labels
|
937
928
|
labels = {}
|
938
929
|
# User can specify labels via config. We want to capture those as well.
|
939
|
-
# TODO: Send instance tags as labels as well?
|
940
930
|
labels.merge!(@labels) if @labels
|
941
931
|
|
942
932
|
case @resource.type
|
943
|
-
|
944
|
-
|
945
|
-
|
946
|
-
|
947
|
-
|
948
|
-
labels
|
949
|
-
|
950
|
-
|
933
|
+
# GAE, Cloud Dataflow, Cloud Dataproc and Cloud ML.
|
934
|
+
when APPENGINE_CONSTANTS[:resource_type],
|
935
|
+
DATAFLOW_CONSTANTS[:resource_type],
|
936
|
+
DATAPROC_CONSTANTS[:resource_type],
|
937
|
+
ML_CONSTANTS[:resource_type]
|
938
|
+
labels.merge!(
|
939
|
+
"#{COMPUTE_CONSTANTS[:service]}/resource_id" => @vm_id,
|
940
|
+
"#{COMPUTE_CONSTANTS[:service]}/resource_name" => @vm_name,
|
941
|
+
"#{COMPUTE_CONSTANTS[:service]}/zone" => @zone
|
942
|
+
)
|
943
|
+
|
944
|
+
# GCE instance and GKE container.
|
951
945
|
when COMPUTE_CONSTANTS[:resource_type],
|
952
946
|
CONTAINER_CONSTANTS[:resource_type]
|
953
|
-
labels
|
954
|
-
|
955
|
-
# Cloud Dataflow and Cloud Dataproc.
|
956
|
-
when DATAFLOW_CONSTANTS[:resource_type],
|
957
|
-
DATAPROC_CONSTANTS[:resource_type]
|
958
|
-
labels["#{COMPUTE_CONSTANTS[:service]}/resource_id"] = @vm_id
|
959
|
-
labels["#{COMPUTE_CONSTANTS[:service]}/resource_name"] = @vm_name
|
960
|
-
labels["#{COMPUTE_CONSTANTS[:service]}/zone"] = @zone
|
947
|
+
labels.merge!(
|
948
|
+
"#{COMPUTE_CONSTANTS[:service]}/resource_name" => @vm_name)
|
961
949
|
|
962
950
|
# EC2.
|
963
951
|
when EC2_CONSTANTS[:resource_type]
|
964
|
-
labels
|
965
|
-
|
966
|
-
# Cloud ML.
|
967
|
-
when ML_CONSTANTS[:resource_type]
|
968
|
-
labels["#{COMPUTE_CONSTANTS[:service]}/resource_id"] = @vm_id
|
969
|
-
labels["#{COMPUTE_CONSTANTS[:service]}/resource_name"] = @vm_name
|
970
|
-
labels["#{COMPUTE_CONSTANTS[:service]}/zone"] = @zone
|
952
|
+
labels.merge!(
|
953
|
+
"#{EC2_CONSTANTS[:service]}/resource_name" => @vm_name)
|
971
954
|
end
|
972
955
|
labels
|
973
956
|
end
|
@@ -977,14 +960,13 @@ module Fluent
|
|
977
960
|
def determine_group_level_monitored_resource_and_labels(tag)
|
978
961
|
# Determine group level monitored resource type. For certain types,
|
979
962
|
# extract useful info from the tag and store those in
|
980
|
-
#
|
981
|
-
group_resource_type,
|
963
|
+
# matched_regex_group.
|
964
|
+
group_resource_type, matched_regex_group =
|
982
965
|
determine_group_level_monitored_resource_type(tag)
|
983
966
|
|
984
967
|
# Determine group level monitored resource labels and common labels.
|
985
|
-
|
986
|
-
|
987
|
-
group_resource_type, matched_regexp_group)
|
968
|
+
group_resource_labels, group_common_labels =
|
969
|
+
determine_group_level_labels(group_resource_type, matched_regex_group)
|
988
970
|
|
989
971
|
group_resource = Google::Apis::LoggingV2beta1::MonitoredResource.new(
|
990
972
|
type: group_resource_type,
|
@@ -1002,202 +984,84 @@ module Fluent
|
|
1002
984
|
|
1003
985
|
# Determine group level monitored resource type shared by a collection of
|
1004
986
|
# entries.
|
1005
|
-
#
|
1006
|
-
#
|
1007
|
-
#
|
987
|
+
# Return the resource type and tag regexp matched groups. The matched groups
|
988
|
+
# only apply to some resource types. Return nil if not applicable or if
|
989
|
+
# there is no match.
|
1008
990
|
def determine_group_level_monitored_resource_type(tag)
|
1009
|
-
|
1010
|
-
|
1011
|
-
|
1012
|
-
|
1013
|
-
matched_regexp_group] if matched_regexp_group
|
991
|
+
@tag_regexp_list.each do |derived_type, tag_regexp|
|
992
|
+
matched_regex_group = tag_regexp.match(tag)
|
993
|
+
return [derived_type, matched_regex_group] if
|
994
|
+
matched_regex_group
|
1014
995
|
end
|
1015
|
-
|
1016
|
-
# Match tag against Docker container stderr / stdout log format and
|
1017
|
-
# Docker container application log format.
|
1018
|
-
matched_regexp_group =
|
1019
|
-
# Format: "container.<container_id>.<container_name>"
|
1020
|
-
@dockercontainer_tag_regexp.match(tag) ||
|
1021
|
-
# Format: "application-container.<container_name>.<additional_tag>"
|
1022
|
-
@dockercontainer_tag_with_application_regexp.match(tag)
|
1023
|
-
return [DOCKER_CONSTANTS[:resource_type], matched_regexp_group] if
|
1024
|
-
matched_regexp_group
|
1025
|
-
|
1026
|
-
# Match tag against GKE Container format.
|
1027
|
-
if @resource.type == CONTAINER_CONSTANTS[:resource_type] &&
|
1028
|
-
@compiled_kubernetes_tag_regexp
|
1029
|
-
# Container logs in Kubernetes are tagged based on where they came from,
|
1030
|
-
# so we can extract useful metadata from the tag. Do this here to avoid
|
1031
|
-
# having to repeat it for each record.
|
1032
|
-
matched_regexp_group = @compiled_kubernetes_tag_regexp.match(tag)
|
1033
|
-
return [@resource.type, matched_regexp_group] if matched_regexp_group
|
1034
|
-
end
|
1035
|
-
|
1036
|
-
# Otherwise, return the original type.
|
1037
996
|
[@resource.type, nil]
|
1038
997
|
end
|
1039
998
|
|
1040
999
|
# Determine group level monitored resource labels and common labels. These
|
1041
|
-
# labels will be shared by a collection of entries.
|
1042
|
-
|
1043
|
-
def determine_group_level_labels_and_adjust_type(group_resource_type,
|
1044
|
-
matched_regexp_group)
|
1000
|
+
# labels will be shared by a collection of entries.
|
1001
|
+
def determine_group_level_labels(group_resource_type, matched_regex_group)
|
1045
1002
|
group_resource_labels = @resource.labels.dup
|
1046
1003
|
group_common_labels = @common_labels.dup
|
1047
1004
|
|
1048
1005
|
case group_resource_type
|
1049
|
-
|
1050
1006
|
# Cloud Functions.
|
1051
1007
|
when CLOUDFUNCTIONS_CONSTANTS[:resource_type]
|
1052
|
-
group_resource_labels
|
1053
|
-
|
1054
|
-
decode_cloudfunctions_function_name(
|
1055
|
-
|
1008
|
+
group_resource_labels.merge!(
|
1009
|
+
'region' => @gcf_region,
|
1010
|
+
'function_name' => decode_cloudfunctions_function_name(
|
1011
|
+
matched_regex_group['encoded_function_name'])
|
1012
|
+
)
|
1013
|
+
|
1056
1014
|
instance_id = group_resource_labels.delete('instance_id')
|
1057
|
-
group_common_labels
|
1058
|
-
|
1059
|
-
|
1060
|
-
|
1061
|
-
|
1062
|
-
|
1063
|
-
|
1064
|
-
|
1015
|
+
group_common_labels.merge!(
|
1016
|
+
"#{CONTAINER_CONSTANTS[:service]}/instance_id" => instance_id,
|
1017
|
+
"#{COMPUTE_CONSTANTS[:service]}/resource_id" => instance_id,
|
1018
|
+
"#{CONTAINER_CONSTANTS[:service]}/cluster_name" =>
|
1019
|
+
group_resource_labels.delete('cluster_name'),
|
1020
|
+
"#{COMPUTE_CONSTANTS[:service]}/zone" =>
|
1021
|
+
group_resource_labels.delete('zone')
|
1022
|
+
)
|
1065
1023
|
|
1066
1024
|
# GKE container.
|
1067
1025
|
when CONTAINER_CONSTANTS[:resource_type]
|
1068
|
-
if
|
1069
|
-
|
1070
|
-
|
1071
|
-
|
1072
|
-
|
1073
|
-
|
1074
|
-
|
1075
|
-
|
1076
|
-
|
1077
|
-
|
1078
|
-
|
1079
|
-
|
1080
|
-
|
1081
|
-
|
1082
|
-
|
1083
|
-
|
1084
|
-
|
1085
|
-
|
1086
|
-
|
1087
|
-
|
1088
|
-
|
1089
|
-
|
1090
|
-
# <additional_tag>", thus 'container_id' info is unknown yet.
|
1091
|
-
# 'container_name' info on the other hand is always available.
|
1092
|
-
container_id = matched_regexp_group['container_id'] if
|
1093
|
-
matched_regexp_group.names.include? 'container_id'
|
1094
|
-
container_name = matched_regexp_group['container_name']
|
1095
|
-
|
1096
|
-
if @enable_metadata_agent
|
1097
|
-
# Call Metadata Agent with "container.<container_id>" or
|
1098
|
-
# "application-container.<container_name>" as the locally-unique key
|
1099
|
-
# to retrieve monitored resource. This should be different from the
|
1100
|
-
# original @resource value that got initiated when the agent starts up
|
1101
|
-
# because that one is always at the VM level.
|
1102
|
-
if container_id
|
1103
|
-
locally_unique_id = "container.#{container_id}"
|
1104
|
-
else
|
1105
|
-
locally_unique_id = "containerName.#{container_name}"
|
1106
|
-
end
|
1107
|
-
retrieved_resource = call_metadata_agent_for_monitored_resource(
|
1108
|
-
locally_unique_id)
|
1109
|
-
end
|
1110
|
-
|
1111
|
-
if !retrieved_resource.nil?
|
1112
|
-
# If we successfully get a monitored resource from Metadata Agent,
|
1113
|
-
# use this one instead of the original instance monitored resource.
|
1114
|
-
group_resource_labels = retrieved_resource.labels.dup
|
1115
|
-
@log.debug 'Retrieved monitored resource from Metadata Agent: ' \
|
1116
|
-
"#{retrieved_resource.inspect}."
|
1117
|
-
else
|
1118
|
-
# If Metadata Agent is not enabled, or we failed to get a monitored
|
1119
|
-
# resource, we need to have some backup plan.
|
1120
|
-
@log.debug 'Metadata Agent not enabled or failed to retrieve ' \
|
1121
|
-
'docker container monitored resource from Metadata ' \
|
1122
|
-
'Agent.'
|
1123
|
-
|
1124
|
-
# 1. Check if 'container_id' is set already. It should be available
|
1125
|
-
# for stdout / stderr). If so, use that.
|
1126
|
-
# 2. If not, call Docker Remote API to retrieve the container ID from
|
1127
|
-
# container name, but only if @call_docker_api_locally is true.
|
1128
|
-
container_id ||= retrieve_container_id_by_name_locally(
|
1129
|
-
container_name) if @call_docker_api_locally
|
1130
|
-
unless container_id
|
1131
|
-
@log.debug 'No docker container id retrieved. Falling back to
|
1132
|
-
instance monitored resource.'
|
1133
|
-
# If a container id is not available, fall back to the instance
|
1134
|
-
# monitored resource.
|
1135
|
-
return [COMPUTE_CONSTANTS[:resource_type], group_resource_labels,
|
1136
|
-
group_common_labels]
|
1137
|
-
end
|
1138
|
-
group_resource_labels['container_id'] = container_id
|
1139
|
-
# 'zone' for GCP and 'region' for EC2 must have been set at this
|
1140
|
-
# point. Rename them to 'location'.
|
1141
|
-
group_resource_labels['location'] = @zone
|
1142
|
-
if @platform == Platform::EC2
|
1143
|
-
group_resource_labels.delete('region')
|
1144
|
-
else
|
1145
|
-
group_resource_labels.delete('zone')
|
1146
|
-
end
|
1147
|
-
# vm id info should be reported as a metadata label instead.
|
1148
|
-
group_resource_labels.delete('instance_id')
|
1149
|
-
|
1026
|
+
if matched_regex_group
|
1027
|
+
# We only expect one occurrence of each key in the match group.
|
1028
|
+
resource_labels_candidates =
|
1029
|
+
matched_regex_group.names.zip(matched_regex_group.captures).to_h
|
1030
|
+
common_labels_candidates =
|
1031
|
+
resource_labels_candidates.dup
|
1032
|
+
group_resource_labels.merge!(
|
1033
|
+
delete_and_extract_labels(
|
1034
|
+
resource_labels_candidates,
|
1035
|
+
# The kubernetes_tag_regexp is poorly named. 'namespace_name' is
|
1036
|
+
# in fact 'namespace_id'. 'pod_name' is in fact 'pod_id'.
|
1037
|
+
# TODO(qingling128): Figure out how to put this map into
|
1038
|
+
# constants like CONTAINER_CONSTANTS[:extra_resource_labels].
|
1039
|
+
'container_name' => 'container_name',
|
1040
|
+
'namespace_name' => 'namespace_id',
|
1041
|
+
'pod_name' => 'pod_id'))
|
1042
|
+
|
1043
|
+
group_common_labels.merge!(
|
1044
|
+
delete_and_extract_labels(
|
1045
|
+
common_labels_candidates,
|
1046
|
+
CONTAINER_CONSTANTS[:extra_common_labels]
|
1047
|
+
.map { |l| [l, "#{CONTAINER_CONSTANTS[:service]}/#{l}"] }.to_h))
|
1150
1048
|
end
|
1151
|
-
# Set metadata labels.
|
1152
|
-
group_common_labels["#{DOCKER_CONSTANTS[:service]}/container_name"] =
|
1153
|
-
matched_regexp_group['container_name']
|
1154
|
-
group_common_labels["#{COMPUTE_CONSTANTS[:service]}/resource_id"] =
|
1155
|
-
@vm_id
|
1156
1049
|
end
|
1157
1050
|
|
1158
|
-
[
|
1051
|
+
[group_resource_labels, group_common_labels]
|
1159
1052
|
end
|
1160
1053
|
|
1161
1054
|
# Extract entry resource and common labels that should be applied to
|
1162
1055
|
# individual entries from the group resource.
|
1163
1056
|
def determine_entry_level_labels(group_resource, record)
|
1164
|
-
resource_type = group_resource.type
|
1165
1057
|
resource_labels = {}
|
1166
1058
|
common_labels = {}
|
1167
1059
|
|
1168
|
-
# The format of the locally unique key varies by monitored resource.
|
1169
|
-
#
|
1170
|
-
# Docker container:
|
1171
|
-
# "container.<container_id>"
|
1172
|
-
# "containerName.<container_name>"
|
1173
|
-
# GKE container:
|
1174
|
-
# "gke_containerName.<namespace_id>.<pod_name>.<container_name>"
|
1175
|
-
if @enable_metadata_agent && record.key?(LOCALLY_UNIQUE_ID_LABEL_NAME)
|
1176
|
-
locally_unique_id = record.delete(LOCALLY_UNIQUE_ID_LABEL_NAME)
|
1177
|
-
@log.debug 'Calling metadata agent with locally unique id: ' \
|
1178
|
-
"#{locally_unique_id}."
|
1179
|
-
retrieved_resource = call_metadata_agent_for_monitored_resource(
|
1180
|
-
locally_unique_id)
|
1181
|
-
@log.debug 'Retrieved monitored resource from metadata agent: ' \
|
1182
|
-
"#{retrieved_resource.inspect}."
|
1183
|
-
unless retrieved_resource.nil?
|
1184
|
-
resource_type = retrieved_resource.type
|
1185
|
-
# Temporarily renaming 'gke_container' to 'container'.
|
1186
|
-
resource_type = 'container' if resource_type == 'gke_container'
|
1187
|
-
# If we successfully get a monitored resource from Metadata Agent,
|
1188
|
-
# use this one instead of the original VM-level monitored resource.
|
1189
|
-
resource_labels = retrieved_resource.labels.dup
|
1190
|
-
@log.debug 'Retrieved gke_container monitored resource from' \
|
1191
|
-
'Stackdriver Metadata agent: ' \
|
1192
|
-
"#{retrieved_resource.inspect}."
|
1193
|
-
end
|
1194
|
-
end
|
1195
|
-
|
1196
1060
|
# Cloud Functions.
|
1197
|
-
if
|
1061
|
+
if group_resource.type == CLOUDFUNCTIONS_CONSTANTS[:resource_type] &&
|
1198
1062
|
record.key?('log')
|
1199
1063
|
@cloudfunctions_log_match =
|
1200
|
-
@
|
1064
|
+
@compiled_cloudfunctions_log_regexp.match(record['log'])
|
1201
1065
|
common_labels['execution_id'] =
|
1202
1066
|
@cloudfunctions_log_match['execution_id'] if \
|
1203
1067
|
@cloudfunctions_log_match &&
|
@@ -1205,7 +1069,7 @@ module Fluent
|
|
1205
1069
|
end
|
1206
1070
|
|
1207
1071
|
# GKE containers.
|
1208
|
-
if
|
1072
|
+
if group_resource.type == CONTAINER_CONSTANTS[:resource_type]
|
1209
1073
|
# Move the stdout/stderr annotation from the record into a label.
|
1210
1074
|
common_labels.merge!(
|
1211
1075
|
delete_and_extract_labels(
|
@@ -1217,17 +1081,18 @@ module Fluent
|
|
1217
1081
|
if record.key?('kubernetes')
|
1218
1082
|
resource_labels.merge!(
|
1219
1083
|
delete_and_extract_labels(
|
1220
|
-
record['kubernetes'],
|
1084
|
+
record['kubernetes'], CONTAINER_CONSTANTS[:extra_resource_labels]
|
1221
1085
|
.map { |l| [l, l] }.to_h))
|
1222
1086
|
common_labels.merge!(
|
1223
1087
|
delete_and_extract_labels(
|
1224
|
-
record['kubernetes'],
|
1088
|
+
record['kubernetes'], CONTAINER_CONSTANTS[:extra_common_labels]
|
1225
1089
|
.map { |l| [l, "#{CONTAINER_CONSTANTS[:service]}/#{l}"] }.to_h))
|
1226
1090
|
# Prepend label/ to all user-defined labels' keys.
|
1227
1091
|
if record['kubernetes'].key?('labels')
|
1228
|
-
|
1229
|
-
|
1230
|
-
|
1092
|
+
common_labels.merge!(
|
1093
|
+
delete_and_extract_labels(
|
1094
|
+
record['kubernetes']['labels'], record['kubernetes']['labels']
|
1095
|
+
.map { |key, _| [key, "label/#{key}"] }.to_h))
|
1231
1096
|
end
|
1232
1097
|
# We've explicitly consumed all the fields we care about -- don't
|
1233
1098
|
# litter the log entries with the remaining fields that the kubernetes
|
@@ -1237,96 +1102,24 @@ module Fluent
|
|
1237
1102
|
end
|
1238
1103
|
end
|
1239
1104
|
|
1240
|
-
# Docker containers.
|
1241
|
-
if resource_type == DOCKER_CONSTANTS[:resource_type]
|
1242
|
-
# For logs coming from Docker Fluentd Logging Driver, the log record
|
1243
|
-
# has 4 fields: 'container_id', 'container_name', 'source' and 'log'.
|
1244
|
-
# Extract 'container_id', 'container_name' and 'source' from json
|
1245
|
-
# record, set corresponding labels, and remove these fields from record.
|
1246
|
-
{
|
1247
|
-
'container_name' => 'container_name',
|
1248
|
-
'source' => 'stream'
|
1249
|
-
}.each do |field_name, label_name|
|
1250
|
-
common_labels.merge!(
|
1251
|
-
delete_and_extract_labels(
|
1252
|
-
record,
|
1253
|
-
field_name => "#{DOCKER_CONSTANTS[:service]}/#{label_name}"
|
1254
|
-
)
|
1255
|
-
)
|
1256
|
-
end
|
1257
|
-
resource_labels.merge!(
|
1258
|
-
delete_and_extract_labels(record, 'container_id' => 'container_id'))
|
1259
|
-
end
|
1260
|
-
|
1261
1105
|
# If the name of a field in the record is present in the @label_map
|
1262
1106
|
# configured by users, report its value as a label and do not send that
|
1263
1107
|
# field as part of the payload.
|
1264
1108
|
common_labels.merge!(delete_and_extract_labels(record, @label_map))
|
1265
1109
|
|
1266
|
-
# Cloud Dataflow.
|
1267
|
-
# These labels can be set via configuring 'labels' or 'label_map'.
|
1268
|
-
# Report them as monitored resource labels instead of common labels.
|
1269
|
-
if group_resource.type == DATAFLOW_CONSTANTS[:resource_type]
|
1270
|
-
resource_labels.merge!(
|
1271
|
-
delete_and_extract_labels(
|
1272
|
-
common_labels, %w(region job_name job_id step_id)
|
1273
|
-
.map { |l| ["#{DATAFLOW_CONSTANTS[:service]}/#{l}", l] }.to_h))
|
1274
|
-
end
|
1275
|
-
|
1276
|
-
# Cloud ML.
|
1110
|
+
# Cloud Dataflow and Cloud ML.
|
1277
1111
|
# These labels can be set via configuring 'labels' or 'label_map'.
|
1278
1112
|
# Report them as monitored resource labels instead of common labels.
|
1279
|
-
|
1113
|
+
# e.g. "dataflow.googleapis.com/job_id" => "job_id"
|
1114
|
+
[DATAFLOW_CONSTANTS, ML_CONSTANTS].each do |service_constants|
|
1115
|
+
next unless group_resource.type == service_constants[:resource_type]
|
1280
1116
|
resource_labels.merge!(
|
1281
1117
|
delete_and_extract_labels(
|
1282
|
-
common_labels,
|
1283
|
-
.map { |l| ["#{
|
1118
|
+
common_labels, service_constants[:extra_common_labels]
|
1119
|
+
.map { |l| ["#{service_constants[:service]}/#{l}", l] }.to_h))
|
1284
1120
|
end
|
1285
1121
|
|
1286
|
-
[
|
1287
|
-
end
|
1288
|
-
|
1289
|
-
# Call Metadata Agent to get monitored resource information and parse
|
1290
|
-
# response to Google::Api::MonitoredResource.
|
1291
|
-
def call_metadata_agent_for_monitored_resource(unique_key)
|
1292
|
-
response = call_metadata_agent("monitoredResource/#{unique_key}")
|
1293
|
-
return nil if response.nil?
|
1294
|
-
begin
|
1295
|
-
resource = Google::Api::MonitoredResource.decode_json(response.to_json)
|
1296
|
-
rescue Google::Protobuf::ParseError, ArgumentError => e
|
1297
|
-
@log.error 'Error paring monitored resource from Metadata Agent. ' \
|
1298
|
-
"response: #{response.inspect}", error: e
|
1299
|
-
return nil
|
1300
|
-
end
|
1301
|
-
|
1302
|
-
# TODO(qingling128): Use Google::Api::MonitoredResource directly after we
|
1303
|
-
# upgrade gRPC version to include the fix for the protobuf map
|
1304
|
-
# corruption issue.
|
1305
|
-
Google::Apis::LoggingV2beta1::MonitoredResource.new(
|
1306
|
-
type: resource.type,
|
1307
|
-
labels: resource.labels.to_h
|
1308
|
-
)
|
1309
|
-
end
|
1310
|
-
|
1311
|
-
# Call Metadata Agent and parse response to json. Return nil in case of any
|
1312
|
-
# error / failure.
|
1313
|
-
def call_metadata_agent(path)
|
1314
|
-
url = "#{@metadata_agent_url}/#{path}"
|
1315
|
-
@log.debug("Calling Metadata Agent: #{url}")
|
1316
|
-
open(url) do |f|
|
1317
|
-
response = f.read
|
1318
|
-
parsed_hash = parse_json_or_nil(response)
|
1319
|
-
if parsed_hash.nil?
|
1320
|
-
@log.error 'Response from Metadata Agent is not in valid json ' \
|
1321
|
-
"format: '#{response.inspect}'."
|
1322
|
-
return nil
|
1323
|
-
end
|
1324
|
-
@log.debug "Response from Metadata Agent: #{parsed_hash}"
|
1325
|
-
return parsed_hash
|
1326
|
-
end
|
1327
|
-
rescue StandardError => e
|
1328
|
-
@log.error 'Error calling Metadata Agent.', error: e
|
1329
|
-
return nil
|
1122
|
+
[resource_labels, common_labels]
|
1330
1123
|
end
|
1331
1124
|
|
1332
1125
|
# TODO: This functionality should eventually be available in another
|
@@ -1364,30 +1157,6 @@ module Fluent
|
|
1364
1157
|
end
|
1365
1158
|
end
|
1366
1159
|
|
1367
|
-
# Calling Docker Remote API to get container id by name.
|
1368
|
-
def retrieve_container_id_by_name_locally(container_name)
|
1369
|
-
response = Excon.get(
|
1370
|
-
"unix:///containers/#{container_name}/json",
|
1371
|
-
socket: @docker_remote_api_socket_path)
|
1372
|
-
@log.debug "Response from Docker API with name '#{container_name}': " \
|
1373
|
-
"#{response.inspect}."
|
1374
|
-
return parse_container_id_from_docker_api_response(response)
|
1375
|
-
rescue StandardError => e
|
1376
|
-
@log.error 'Error calling Docker API to get container id.', error: e
|
1377
|
-
return nil
|
1378
|
-
end
|
1379
|
-
|
1380
|
-
# Parse the container id from Docker Remote API response.
|
1381
|
-
# TODO(qingling128) Add a config for Docker API version to support parsing
|
1382
|
-
# different versions of Docker Remote API when the format varies.
|
1383
|
-
def parse_container_id_from_docker_api_response(response)
|
1384
|
-
JSON.parse(response.data[:body])['Id']
|
1385
|
-
rescue StandardError => e
|
1386
|
-
@log.error 'Error parsing Docker API response to get container id.',
|
1387
|
-
error: e
|
1388
|
-
return nil
|
1389
|
-
end
|
1390
|
-
|
1391
1160
|
def cluster_name_from_kube_env(kube_env)
|
1392
1161
|
return kube_env['CLUSTER_NAME'] if kube_env.key?('CLUSTER_NAME')
|
1393
1162
|
instance_prefix = kube_env['INSTANCE_PREFIX']
|
@@ -1458,14 +1227,9 @@ module Fluent
|
|
1458
1227
|
end
|
1459
1228
|
elsif record.key?('severity')
|
1460
1229
|
return parse_severity(record.delete('severity'))
|
1461
|
-
elsif
|
1462
|
-
|
1463
|
-
stream = entry_common_labels[
|
1464
|
-
"#{CONTAINER_CONSTANTS[:service]}/stream"] if
|
1465
|
-
resource_type == CONTAINER_CONSTANTS[:resource_type]
|
1466
|
-
stream = entry_common_labels[
|
1467
|
-
"#{DOCKER_CONSTANTS[:service]}/stream"] if
|
1468
|
-
resource_type == DOCKER_CONSTANTS[:resource_type]
|
1230
|
+
elsif resource_type == CONTAINER_CONSTANTS[:resource_type] &&
|
1231
|
+
entry_common_labels.key?("#{CONTAINER_CONSTANTS[:service]}/stream")
|
1232
|
+
stream = entry_common_labels["#{CONTAINER_CONSTANTS[:service]}/stream"]
|
1469
1233
|
if stream == 'stdout'
|
1470
1234
|
return 'INFO'
|
1471
1235
|
elsif stream == 'stderr'
|
@@ -1478,71 +1242,61 @@ module Fluent
|
|
1478
1242
|
end
|
1479
1243
|
end
|
1480
1244
|
|
1481
|
-
|
1245
|
+
def set_log_entry_fields(record, entry)
|
1246
|
+
LOG_ENTRY_FIELDS_MAP.each do |field_name, config|
|
1247
|
+
payload_key, subfields, grpc_class, non_grpc_class = config
|
1248
|
+
begin
|
1249
|
+
payload_key = instance_variable_get(payload_key)
|
1250
|
+
fields = record[payload_key]
|
1251
|
+
next unless fields.is_a?(Hash)
|
1252
|
+
|
1253
|
+
extracted_subfields = subfields.each_with_object({}) \
|
1254
|
+
do |(original_key, destination_key, cast_fn), extracted_fields|
|
1255
|
+
value = fields.delete(original_key)
|
1256
|
+
next if value.nil?
|
1257
|
+
begin
|
1258
|
+
casted_value = send(cast_fn, value)
|
1259
|
+
rescue TypeError
|
1260
|
+
@log.error "Failed to #{cast_fn} for #{field_name}." \
|
1261
|
+
"#{original_key} with value #{value.inspect}.", err
|
1262
|
+
next
|
1263
|
+
end
|
1264
|
+
next if casted_value.nil?
|
1265
|
+
extracted_fields[destination_key] = casted_value
|
1266
|
+
end
|
1267
|
+
|
1268
|
+
next unless extracted_subfields
|
1482
1269
|
|
1483
|
-
def set_http_request(record, entry)
|
1484
|
-
return nil unless record['httpRequest'].is_a?(Hash)
|
1485
|
-
input = record['httpRequest']
|
1486
|
-
if @use_grpc
|
1487
|
-
output = Google::Logging::Type::HttpRequest.new
|
1488
|
-
else
|
1489
|
-
output = Google::Apis::LoggingV2beta1::HttpRequest.new
|
1490
|
-
end
|
1491
|
-
# We need to delete each field from 'httpRequest' even if its value is
|
1492
|
-
# nil. However we do not want to assign this nil value to the constructed
|
1493
|
-
# json or proto.
|
1494
|
-
request_method = input.delete('requestMethod')
|
1495
|
-
output.request_method = request_method unless request_method.nil?
|
1496
|
-
request_url = input.delete('requestUrl')
|
1497
|
-
output.request_url = request_url unless request_url.nil?
|
1498
|
-
request_size = input.delete('requestSize')
|
1499
|
-
output.request_size = request_size.to_i unless request_size.nil?
|
1500
|
-
status = input.delete('status')
|
1501
|
-
output.status = status.to_i unless status.nil?
|
1502
|
-
response_size = input.delete('responseSize')
|
1503
|
-
output.response_size = response_size.to_i unless response_size.nil?
|
1504
|
-
user_agent = input.delete('userAgent')
|
1505
|
-
output.user_agent = user_agent unless user_agent.nil?
|
1506
|
-
remote_ip = input.delete('remoteIp')
|
1507
|
-
output.remote_ip = remote_ip unless remote_ip.nil?
|
1508
|
-
referer = input.delete('referer')
|
1509
|
-
output.referer = referer unless referer.nil?
|
1510
|
-
cache_hit = input.delete('cacheHit')
|
1511
|
-
output.cache_hit = cache_hit unless cache_hit.nil?
|
1512
|
-
cache_validated_with_origin_server = \
|
1513
|
-
input.delete('cacheValidatedWithOriginServer')
|
1514
|
-
output.cache_validated_with_origin_server = \
|
1515
|
-
cache_validated_with_origin_server \
|
1516
|
-
unless cache_validated_with_origin_server.nil?
|
1517
|
-
|
1518
|
-
latency = input.delete('latency')
|
1519
|
-
unless latency.nil?
|
1520
|
-
# Parse latency. If no valid format is detected, skip setting latency.
|
1521
|
-
# Format: whitespace (optional) + integer + point & decimal (optional)
|
1522
|
-
# + whitespace (optional) + "s" + whitespace (optional)
|
1523
|
-
# e.g.: "1.42 s"
|
1524
|
-
match = @http_latency_regexp.match(latency)
|
1525
|
-
if match
|
1526
|
-
# Split the integer and decimal parts in order to calculate seconds
|
1527
|
-
# and nanos.
|
1528
|
-
latency_seconds = match['seconds'].to_i
|
1529
|
-
latency_nanos = (match['decimal'].to_f * NANOS_IN_A_SECOND).round
|
1530
1270
|
if @use_grpc
|
1531
|
-
output
|
1532
|
-
seconds: latency_seconds,
|
1533
|
-
nanos: latency_nanos
|
1534
|
-
)
|
1271
|
+
output = Object.const_get(grpc_class).new
|
1535
1272
|
else
|
1536
|
-
output
|
1537
|
-
seconds: latency_seconds,
|
1538
|
-
nanos: latency_nanos
|
1539
|
-
}.delete_if { |_, v| v == 0 }
|
1273
|
+
output = Object.const_get(non_grpc_class).new
|
1540
1274
|
end
|
1275
|
+
extracted_subfields.each do |key, value|
|
1276
|
+
output.send("#{key}=", value)
|
1277
|
+
end
|
1278
|
+
|
1279
|
+
record.delete(payload_key) if fields.empty?
|
1280
|
+
|
1281
|
+
entry.send("#{field_name}=", output)
|
1282
|
+
rescue StandardError => err
|
1283
|
+
@log.error "Failed to set log entry field for #{field_name}.", err
|
1284
|
+
end
|
1285
|
+
end
|
1286
|
+
end
|
1287
|
+
|
1288
|
+
def set_labels(record, entry)
|
1289
|
+
record_labels = record[@labels_key]
|
1290
|
+
return nil unless record_labels.is_a?(Hash)
|
1291
|
+
|
1292
|
+
record_labels.each do |key, value|
|
1293
|
+
unless entry.labels.key?(key)
|
1294
|
+
record_labels.delete(key)
|
1295
|
+
entry.labels[key] = value
|
1541
1296
|
end
|
1542
1297
|
end
|
1543
1298
|
|
1544
|
-
record.delete(
|
1545
|
-
entry.http_request = output
|
1299
|
+
record.delete(@labels_key) if record_labels.empty?
|
1546
1300
|
end
|
1547
1301
|
|
1548
1302
|
# Values permitted by the API for 'severity' (which is an enum).
|
@@ -1640,6 +1394,45 @@ module Fluent
|
|
1640
1394
|
severity
|
1641
1395
|
end
|
1642
1396
|
|
1397
|
+
def parse_string(value)
|
1398
|
+
value.to_s
|
1399
|
+
end
|
1400
|
+
|
1401
|
+
def parse_int(value)
|
1402
|
+
value.to_i
|
1403
|
+
end
|
1404
|
+
|
1405
|
+
def parse_bool(value)
|
1406
|
+
[true, 'true', 1].include?(value)
|
1407
|
+
end
|
1408
|
+
|
1409
|
+
def parse_latency(latency)
|
1410
|
+
# Parse latency.
|
1411
|
+
# If no valid format is detected, return nil so we can later skip
|
1412
|
+
# setting latency.
|
1413
|
+
# Format: whitespace (opt.) + integer + point & decimal (opt.)
|
1414
|
+
# + whitespace (opt.) + "s" + whitespace (opt.)
|
1415
|
+
# e.g.: "1.42 s"
|
1416
|
+
match = @compiled_http_latency_regexp.match(latency)
|
1417
|
+
return nil unless match
|
1418
|
+
|
1419
|
+
# Split the integer and decimal parts in order to calculate
|
1420
|
+
# seconds and nanos.
|
1421
|
+
seconds = match['seconds'].to_i
|
1422
|
+
nanos = (match['decimal'].to_f * 1000 * 1000 * 1000).round
|
1423
|
+
if @use_grpc
|
1424
|
+
return Google::Protobuf::Duration.new(
|
1425
|
+
seconds: seconds,
|
1426
|
+
nanos: nanos
|
1427
|
+
)
|
1428
|
+
else
|
1429
|
+
return {
|
1430
|
+
seconds: seconds,
|
1431
|
+
nanos: nanos
|
1432
|
+
}.delete_if { |_, v| v == 0 }
|
1433
|
+
end
|
1434
|
+
end
|
1435
|
+
|
1643
1436
|
def decode_cloudfunctions_function_name(function_name)
|
1644
1437
|
function_name.gsub(/c\.[a-z]/) { |s| s.upcase[-1] }
|
1645
1438
|
.gsub('u.u', '_').gsub('d.d', '$').gsub('a.a', '@').gsub('p.p', '.')
|
@@ -1664,16 +1457,16 @@ module Fluent
|
|
1664
1457
|
end
|
1665
1458
|
|
1666
1459
|
# For every original_label => new_label pair in the label_map, delete the
|
1667
|
-
# original_label from the
|
1668
|
-
#
|
1669
|
-
def delete_and_extract_labels(
|
1460
|
+
# original_label from the hash map if it exists, and extract the value to
|
1461
|
+
# form a map with the new_label as the key.
|
1462
|
+
def delete_and_extract_labels(hash, label_map)
|
1670
1463
|
return {} if label_map.nil? || !label_map.is_a?(Hash) ||
|
1671
|
-
|
1464
|
+
hash.nil? || !hash.is_a?(Hash)
|
1672
1465
|
label_map.each_with_object({}) \
|
1673
1466
|
do |(original_label, new_label), extracted_labels|
|
1674
|
-
extracted_labels[new_label] =
|
1675
|
-
|
1676
|
-
|
1467
|
+
extracted_labels[new_label] =
|
1468
|
+
convert_to_utf8(hash.delete(original_label).to_s) if
|
1469
|
+
hash.key?(original_label)
|
1677
1470
|
end
|
1678
1471
|
end
|
1679
1472
|
|
@@ -1692,8 +1485,7 @@ module Fluent
|
|
1692
1485
|
entry.text_payload = record['log']
|
1693
1486
|
elsif is_json
|
1694
1487
|
entry.json_payload = record
|
1695
|
-
elsif
|
1696
|
-
DOCKER_CONSTANTS[:resource_type]].include?(resource_type) &&
|
1488
|
+
elsif resource_type == CONTAINER_CONSTANTS[:resource_type] &&
|
1697
1489
|
record.key?('log')
|
1698
1490
|
entry.text_payload = record['log']
|
1699
1491
|
elsif record.size == 1 && record.key?('message')
|
@@ -1763,8 +1555,7 @@ module Fluent
|
|
1763
1555
|
entry.text_payload = convert_to_utf8(record['log'])
|
1764
1556
|
elsif is_json
|
1765
1557
|
entry.json_payload = struct_from_ruby(record)
|
1766
|
-
elsif
|
1767
|
-
DOCKER_CONSTANTS[:resource_type]].include?(resource_type) &&
|
1558
|
+
elsif resource_type == CONTAINER_CONSTANTS[:resource_type] &&
|
1768
1559
|
record.key?('log')
|
1769
1560
|
entry.text_payload = convert_to_utf8(record['log'])
|
1770
1561
|
elsif record.size == 1 && record.key?('message')
|