fluent-plugin-google-cloud 0.7.30 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +13 -13
- data/fluent-plugin-google-cloud.gemspec +2 -2
- data/lib/fluent/plugin/out_google_cloud.rb +36 -123
- data/lib/fluent/plugin/statusz.rb +0 -3
- data/test/plugin/base_test.rb +17 -351
- data/test/plugin/constants.rb +2 -84
- data/test/plugin/test_out_google_cloud.rb +0 -3
- metadata +4 -4
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: db48fa801fa732a1558906459e4bebfd2c7eb66b35aa471e01629f8d2a95989c
|
|
4
|
+
data.tar.gz: 8dd3e21883be568e0a8eae1ae5d79cda7693e2bca747df493b4a1d77ac09a5e9
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 7f1d13d9c74cd06b3be961724b5cd59e4bd28870ae31e7b94166928598b44b2ec951da5850a92accfe2ea244553f6b4fd3093bd0e66c8bbb8ff261eac17a2b14
|
|
7
|
+
data.tar.gz: a99993cd8944ebb0d338d8c285550c78338b25fff8e6be0dba7e194eb7a1607f2924869c0df6b6a894d28dbb4a582a07a344ffa45018440b010f564f74991685
|
data/Gemfile.lock
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
fluent-plugin-google-cloud (0.
|
|
4
|
+
fluent-plugin-google-cloud (0.8.0)
|
|
5
5
|
fluentd (= 1.6.3)
|
|
6
6
|
google-api-client (= 0.30.8)
|
|
7
7
|
google-cloud-logging (= 1.6.6)
|
|
@@ -11,7 +11,7 @@ PATH
|
|
|
11
11
|
grpc (= 1.22.0)
|
|
12
12
|
json (= 2.2.0)
|
|
13
13
|
opencensus (= 0.5.0)
|
|
14
|
-
opencensus-stackdriver (= 0.3.
|
|
14
|
+
opencensus-stackdriver (= 0.3.2)
|
|
15
15
|
|
|
16
16
|
GEM
|
|
17
17
|
remote: https://rubygems.org/
|
|
@@ -19,7 +19,7 @@ GEM
|
|
|
19
19
|
addressable (2.7.0)
|
|
20
20
|
public_suffix (>= 2.0.2, < 5.0)
|
|
21
21
|
ast (2.4.0)
|
|
22
|
-
concurrent-ruby (1.1.
|
|
22
|
+
concurrent-ruby (1.1.6)
|
|
23
23
|
cool.io (1.6.0)
|
|
24
24
|
coveralls (0.8.23)
|
|
25
25
|
json (>= 1.8, < 3)
|
|
@@ -66,8 +66,8 @@ GEM
|
|
|
66
66
|
google-gax (~> 1.7)
|
|
67
67
|
googleapis-common-protos-types (>= 1.0.2)
|
|
68
68
|
stackdriver-core (~> 1.3)
|
|
69
|
-
google-cloud-monitoring (0.
|
|
70
|
-
google-gax (~> 1.
|
|
69
|
+
google-cloud-monitoring (0.32.0)
|
|
70
|
+
google-gax (~> 1.7)
|
|
71
71
|
googleapis-common-protos-types (>= 1.0.2)
|
|
72
72
|
google-cloud-trace (0.35.0)
|
|
73
73
|
concurrent-ruby (~> 1.1)
|
|
@@ -111,15 +111,15 @@ GEM
|
|
|
111
111
|
multi_json (1.14.1)
|
|
112
112
|
multipart-post (2.1.1)
|
|
113
113
|
opencensus (0.5.0)
|
|
114
|
-
opencensus-stackdriver (0.3.
|
|
114
|
+
opencensus-stackdriver (0.3.2)
|
|
115
115
|
concurrent-ruby (~> 1.0)
|
|
116
|
-
google-cloud-monitoring (~> 0.
|
|
117
|
-
google-cloud-trace (~> 0.
|
|
116
|
+
google-cloud-monitoring (~> 0.32)
|
|
117
|
+
google-cloud-trace (~> 0.35)
|
|
118
118
|
opencensus (~> 0.5)
|
|
119
119
|
os (1.0.1)
|
|
120
|
-
parser (2.7.0.
|
|
120
|
+
parser (2.7.0.3)
|
|
121
121
|
ast (~> 2.4.0)
|
|
122
|
-
power_assert (1.1.
|
|
122
|
+
power_assert (1.1.6)
|
|
123
123
|
powerpack (0.1.2)
|
|
124
124
|
prometheus-client (0.9.0)
|
|
125
125
|
quantile (~> 0.2.1)
|
|
@@ -145,9 +145,9 @@ GEM
|
|
|
145
145
|
serverengine (2.2.1)
|
|
146
146
|
sigdump (~> 0.2.2)
|
|
147
147
|
sigdump (0.2.4)
|
|
148
|
-
signet (0.
|
|
148
|
+
signet (0.13.0)
|
|
149
149
|
addressable (~> 2.3)
|
|
150
|
-
faraday (
|
|
150
|
+
faraday (>= 0.17.3, < 2.0)
|
|
151
151
|
jwt (>= 1.5, < 3.0)
|
|
152
152
|
multi_json (~> 1.10)
|
|
153
153
|
simplecov (0.16.1)
|
|
@@ -165,7 +165,7 @@ GEM
|
|
|
165
165
|
power_assert
|
|
166
166
|
thor (1.0.1)
|
|
167
167
|
thread_safe (0.3.6)
|
|
168
|
-
tins (1.24.
|
|
168
|
+
tins (1.24.1)
|
|
169
169
|
sync
|
|
170
170
|
tzinfo (1.2.6)
|
|
171
171
|
thread_safe (~> 0.1)
|
|
@@ -10,7 +10,7 @@ eos
|
|
|
10
10
|
gem.homepage =
|
|
11
11
|
'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
|
|
12
12
|
gem.license = 'Apache-2.0'
|
|
13
|
-
gem.version = '0.
|
|
13
|
+
gem.version = '0.8.0'
|
|
14
14
|
gem.authors = ['Stackdriver Agents Team']
|
|
15
15
|
gem.email = ['stackdriver-agents@google.com']
|
|
16
16
|
gem.required_ruby_version = Gem::Requirement.new('>= 2.2')
|
|
@@ -28,7 +28,7 @@ eos
|
|
|
28
28
|
gem.add_runtime_dependency 'grpc', '1.22.0'
|
|
29
29
|
gem.add_runtime_dependency 'json', '2.2.0'
|
|
30
30
|
gem.add_runtime_dependency 'opencensus', '0.5.0'
|
|
31
|
-
gem.add_runtime_dependency 'opencensus-stackdriver', '0.3.
|
|
31
|
+
gem.add_runtime_dependency 'opencensus-stackdriver', '0.3.2'
|
|
32
32
|
|
|
33
33
|
gem.add_development_dependency 'mocha', '1.9.0'
|
|
34
34
|
# Keep this the same as in
|
|
@@ -121,10 +121,6 @@ module Fluent
|
|
|
121
121
|
K8S_NODE_CONSTANTS = {
|
|
122
122
|
resource_type: 'k8s_node'
|
|
123
123
|
}.freeze
|
|
124
|
-
DOCKER_CONSTANTS = {
|
|
125
|
-
service: 'docker.googleapis.com',
|
|
126
|
-
resource_type: 'docker_container'
|
|
127
|
-
}.freeze
|
|
128
124
|
DATAFLOW_CONSTANTS = {
|
|
129
125
|
service: 'dataflow.googleapis.com',
|
|
130
126
|
resource_type: 'dataflow_step',
|
|
@@ -174,10 +170,6 @@ module Fluent
|
|
|
174
170
|
DEFAULT_SPAN_ID_KEY = 'logging.googleapis.com/spanId'.freeze
|
|
175
171
|
DEFAULT_TRACE_KEY = 'logging.googleapis.com/trace'.freeze
|
|
176
172
|
DEFAULT_TRACE_SAMPLED_KEY = 'logging.googleapis.com/trace_sampled'.freeze
|
|
177
|
-
|
|
178
|
-
DEFAULT_METADATA_AGENT_URL =
|
|
179
|
-
'http://local-metadata-agent.stackdriver.com:8000'.freeze
|
|
180
|
-
METADATA_AGENT_URL_ENV_VAR = 'STACKDRIVER_METADATA_AGENT_URL'.freeze
|
|
181
173
|
end
|
|
182
174
|
|
|
183
175
|
# Internal constants.
|
|
@@ -346,7 +338,7 @@ module Fluent
|
|
|
346
338
|
# running on GCE.
|
|
347
339
|
#
|
|
348
340
|
# The initial motivation for this is to separate out Kubernetes node
|
|
349
|
-
# component (
|
|
341
|
+
# component (Kubelet, etc.) logs from container logs.
|
|
350
342
|
config_param :detect_subservice, :bool, :default => true
|
|
351
343
|
# The subservice_name overrides the subservice detection, if provided.
|
|
352
344
|
config_param :subservice_name, :string, :default => nil
|
|
@@ -408,8 +400,13 @@ module Fluent
|
|
|
408
400
|
|
|
409
401
|
# Whether valid entries should be written even if some other entries fail
|
|
410
402
|
# due to INVALID_ARGUMENT or PERMISSION_DENIED errors when communicating to
|
|
411
|
-
# the Stackdriver Logging API. This is
|
|
412
|
-
|
|
403
|
+
# the Stackdriver Logging API. This flag is no longer used, and is kept for
|
|
404
|
+
# backwards compatibility, partial_success is enabled for all requests.
|
|
405
|
+
# TODO: Breaking change. Remove this flag in Logging Agent 2.0.0 release.
|
|
406
|
+
config_param :partial_success, :bool,
|
|
407
|
+
:default => true,
|
|
408
|
+
:skip_accessor => true,
|
|
409
|
+
:deprecated => 'This feature is permanently enabled'
|
|
413
410
|
|
|
414
411
|
# Whether to allow non-UTF-8 characters in user logs. If set to true, any
|
|
415
412
|
# non-UTF-8 character would be replaced by the string specified by
|
|
@@ -449,15 +446,21 @@ module Fluent
|
|
|
449
446
|
config_param :monitoring_type, :string,
|
|
450
447
|
:default => Monitoring::PrometheusMonitoringRegistry.name
|
|
451
448
|
|
|
452
|
-
# Whether to call metadata agent to retrieve monitored resource.
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
449
|
+
# Whether to call metadata agent to retrieve monitored resource. This flag
|
|
450
|
+
# is kept for backwards compatibility, and is no longer used.
|
|
451
|
+
# TODO: Breaking change. Remove this flag in Logging Agent 2.0.0 release.
|
|
452
|
+
config_param :enable_metadata_agent, :bool,
|
|
453
|
+
:default => false,
|
|
454
|
+
:skip_accessor => true,
|
|
455
|
+
:deprecated => 'This feature is permanently disabled'
|
|
456
|
+
|
|
457
|
+
# The URL of the Metadata Agent. This flag is kept for backwards
|
|
458
|
+
# compatibility, and is no longer used.
|
|
459
|
+
# TODO: Breaking change. Remove this flag in Logging Agent 2.0.0 release.
|
|
460
|
+
config_param :metadata_agent_url, :string,
|
|
461
|
+
:default => nil,
|
|
462
|
+
:skip_accessor => true,
|
|
463
|
+
:deprecated => 'This feature is permanently disabled'
|
|
461
464
|
|
|
462
465
|
# Whether to split log entries with different log tags into different
|
|
463
466
|
# requests when talking to Stackdriver Logging API.
|
|
@@ -520,20 +523,6 @@ module Fluent
|
|
|
520
523
|
' ignored.'
|
|
521
524
|
end
|
|
522
525
|
|
|
523
|
-
# 1. If @metadata_agent_url is customized (aka not nil), use that.
|
|
524
|
-
# 2. Otherwise check the presence of the environment variable
|
|
525
|
-
# STACKDRIVER_METADATA_AGENT_URL and use that if set.
|
|
526
|
-
# 3. Fall back to the default if neither is set.
|
|
527
|
-
if @enable_metadata_agent
|
|
528
|
-
# Convert to string to capture empty string.
|
|
529
|
-
@metadata_agent_url ||=
|
|
530
|
-
if ENV[METADATA_AGENT_URL_ENV_VAR].to_s.empty?
|
|
531
|
-
DEFAULT_METADATA_AGENT_URL
|
|
532
|
-
else
|
|
533
|
-
ENV[METADATA_AGENT_URL_ENV_VAR]
|
|
534
|
-
end
|
|
535
|
-
end
|
|
536
|
-
|
|
537
526
|
# Alert on old authentication configuration.
|
|
538
527
|
unless @auth_method.nil? && @private_key_email.nil? &&
|
|
539
528
|
@private_key_path.nil? && @private_key_passphrase.nil?
|
|
@@ -903,7 +892,7 @@ module Fluent
|
|
|
903
892
|
labels: labels.map do |k, v|
|
|
904
893
|
[k.encode('utf-8'), convert_to_utf8(v)]
|
|
905
894
|
end.to_h,
|
|
906
|
-
partial_success:
|
|
895
|
+
partial_success: true
|
|
907
896
|
)
|
|
908
897
|
increment_successful_requests_count
|
|
909
898
|
increment_ingested_entries_count(entries_count)
|
|
@@ -964,7 +953,7 @@ module Fluent
|
|
|
964
953
|
@log.warn "Dropping #{entries_count} log message(s)",
|
|
965
954
|
error: error.to_s, error_code: error.code.to_s
|
|
966
955
|
|
|
967
|
-
#
|
|
956
|
+
# As partial_success is enabled, valid entries should have been
|
|
968
957
|
# written even if some other entries fail due to InvalidArgument or
|
|
969
958
|
# PermissionDenied errors. Only invalid entries will be dropped.
|
|
970
959
|
when \
|
|
@@ -1029,7 +1018,7 @@ module Fluent
|
|
|
1029
1018
|
log_name: log_name,
|
|
1030
1019
|
resource: resource,
|
|
1031
1020
|
labels: labels,
|
|
1032
|
-
partial_success:
|
|
1021
|
+
partial_success: true
|
|
1033
1022
|
),
|
|
1034
1023
|
options: { api_format_version: '2' }
|
|
1035
1024
|
)
|
|
@@ -1431,8 +1420,10 @@ module Fluent
|
|
|
1431
1420
|
# Metadata Agent with this key.
|
|
1432
1421
|
#
|
|
1433
1422
|
# Examples:
|
|
1434
|
-
#
|
|
1435
|
-
# "k8s_pod.<namespace_name>.<pod_name>"
|
|
1423
|
+
# // GKE Pod.
|
|
1424
|
+
# "k8s_pod.<namespace_name>.<pod_name>"
|
|
1425
|
+
# // GKE container.
|
|
1426
|
+
# "k8s_container.<namespace_name>.<pod_name>.<container_name>"
|
|
1436
1427
|
if local_resource_id
|
|
1437
1428
|
converted_resource = monitored_resource_from_local_resource_id(
|
|
1438
1429
|
local_resource_id)
|
|
@@ -1466,12 +1457,6 @@ module Fluent
|
|
|
1466
1457
|
.map { |l| [l, "#{GKE_CONSTANTS[:service]}/#{l}"] }.to_h))
|
|
1467
1458
|
end
|
|
1468
1459
|
|
|
1469
|
-
# Docker container.
|
|
1470
|
-
# TODO(qingling128): Remove this logic once the resource is retrieved at a
|
|
1471
|
-
# proper time (b/65175256).
|
|
1472
|
-
when DOCKER_CONSTANTS[:resource_type]
|
|
1473
|
-
common_labels.delete("#{COMPUTE_CONSTANTS[:service]}/resource_name")
|
|
1474
|
-
|
|
1475
1460
|
# TODO(qingling128): Temporary fallback for metadata agent restarts.
|
|
1476
1461
|
# K8s resources.
|
|
1477
1462
|
when K8S_CONTAINER_CONSTANTS[:resource_type],
|
|
@@ -1500,32 +1485,6 @@ module Fluent
|
|
|
1500
1485
|
[resource, common_labels]
|
|
1501
1486
|
end
|
|
1502
1487
|
|
|
1503
|
-
# Take a locally unique resource id and convert it to the globally unique
|
|
1504
|
-
# monitored resource.
|
|
1505
|
-
def monitored_resource_from_local_resource_id(local_resource_id)
|
|
1506
|
-
return unless local_resource_id
|
|
1507
|
-
if @enable_metadata_agent
|
|
1508
|
-
@log.debug 'Calling metadata agent with local_resource_id: ' \
|
|
1509
|
-
"#{local_resource_id}."
|
|
1510
|
-
resource = query_metadata_agent_for_monitored_resource(
|
|
1511
|
-
local_resource_id)
|
|
1512
|
-
@log.debug 'Retrieved monitored resource from metadata agent: ' \
|
|
1513
|
-
"#{resource.inspect}."
|
|
1514
|
-
if resource
|
|
1515
|
-
# TODO(qingling128): Fix this temporary renaming from 'gke_container'
|
|
1516
|
-
# to 'container'.
|
|
1517
|
-
resource.type = 'container' if resource.type == 'gke_container'
|
|
1518
|
-
return resource
|
|
1519
|
-
end
|
|
1520
|
-
@log.debug('Failed to retrieve monitored resource from Metadata' \
|
|
1521
|
-
" Agent with local_resource_id #{local_resource_id}.")
|
|
1522
|
-
end
|
|
1523
|
-
# Fall back to constructing monitored resource locally.
|
|
1524
|
-
# TODO(qingling128): This entire else clause is temporary until we
|
|
1525
|
-
# implement buffering and caching.
|
|
1526
|
-
construct_k8s_resource_locally(local_resource_id)
|
|
1527
|
-
end
|
|
1528
|
-
|
|
1529
1488
|
# Extract entry level monitored resource and common labels that should be
|
|
1530
1489
|
# applied to individual entries.
|
|
1531
1490
|
def determine_entry_level_monitored_resource_and_labels(
|
|
@@ -1589,49 +1548,6 @@ module Fluent
|
|
|
1589
1548
|
[resource, common_labels]
|
|
1590
1549
|
end
|
|
1591
1550
|
|
|
1592
|
-
# Call Metadata Agent to get monitored resource information and parse
|
|
1593
|
-
# response to Google::Api::MonitoredResource.
|
|
1594
|
-
def query_metadata_agent_for_monitored_resource(local_resource_id)
|
|
1595
|
-
response = query_metadata_agent("monitoredResource/#{local_resource_id}")
|
|
1596
|
-
return nil if response.nil?
|
|
1597
|
-
begin
|
|
1598
|
-
resource = Google::Api::MonitoredResource.decode_json(response.to_json)
|
|
1599
|
-
rescue Google::Protobuf::ParseError, ArgumentError => e
|
|
1600
|
-
@log.error 'Error parsing monitored resource from Metadata Agent. ' \
|
|
1601
|
-
"response: #{response.inspect}", error: e
|
|
1602
|
-
return nil
|
|
1603
|
-
end
|
|
1604
|
-
|
|
1605
|
-
# TODO(qingling128): Use Google::Api::MonitoredResource directly after we
|
|
1606
|
-
# upgrade gRPC version to include the fix for the protobuf map
|
|
1607
|
-
# corruption issue.
|
|
1608
|
-
Google::Apis::LoggingV2::MonitoredResource.new(
|
|
1609
|
-
type: resource.type,
|
|
1610
|
-
labels: resource.labels.to_h
|
|
1611
|
-
)
|
|
1612
|
-
end
|
|
1613
|
-
|
|
1614
|
-
# Issue a request to the Metadata Agent's local API and parse the response
|
|
1615
|
-
# to JSON. Return nil in case of failure.
|
|
1616
|
-
def query_metadata_agent(path)
|
|
1617
|
-
url = "#{@metadata_agent_url}/#{path}"
|
|
1618
|
-
@log.debug("Calling Metadata Agent: #{url}")
|
|
1619
|
-
open(url) do |f|
|
|
1620
|
-
response = f.read
|
|
1621
|
-
parsed_hash = parse_json_or_nil(response)
|
|
1622
|
-
if parsed_hash.nil?
|
|
1623
|
-
@log.error 'Response from Metadata Agent is not in valid json ' \
|
|
1624
|
-
"format: '#{response.inspect}'."
|
|
1625
|
-
return nil
|
|
1626
|
-
end
|
|
1627
|
-
@log.debug "Response from Metadata Agent: #{parsed_hash}"
|
|
1628
|
-
return parsed_hash
|
|
1629
|
-
end
|
|
1630
|
-
rescue StandardError => e
|
|
1631
|
-
@log.error "Error calling Metadata Agent at #{url}.", error: e
|
|
1632
|
-
nil
|
|
1633
|
-
end
|
|
1634
|
-
|
|
1635
1551
|
# TODO: This functionality should eventually be available in another
|
|
1636
1552
|
# library, but implement it ourselves for now.
|
|
1637
1553
|
module CredentialsInfo
|
|
@@ -2079,9 +1995,7 @@ module Fluent
|
|
|
2079
1995
|
# 2. The only remaining key is 'message'
|
|
2080
1996
|
if is_json
|
|
2081
1997
|
json_payload = record
|
|
2082
|
-
elsif
|
|
2083
|
-
DOCKER_CONSTANTS[:resource_type]].include?(resource_type) &&
|
|
2084
|
-
record.key?('log')
|
|
1998
|
+
elsif GKE_CONSTANTS[:resource_type] == resource_type && record.key?('log')
|
|
2085
1999
|
text_payload = record['log']
|
|
2086
2000
|
elsif record.size == 1 && record.key?('message')
|
|
2087
2001
|
text_payload = record['message']
|
|
@@ -2208,7 +2122,7 @@ module Fluent
|
|
|
2208
2122
|
end
|
|
2209
2123
|
|
|
2210
2124
|
# Extract a map of error details from a potentially partially successful
|
|
2211
|
-
# REST request.
|
|
2125
|
+
# REST request.
|
|
2212
2126
|
#
|
|
2213
2127
|
# The keys in this map are [error_code, error_message] pairs, and the values
|
|
2214
2128
|
# are a list of stringified indexes of log entries that failed due to this
|
|
@@ -2283,7 +2197,6 @@ module Fluent
|
|
|
2283
2197
|
# [3, 'Log name contains illegal character :']: ['1', '3']
|
|
2284
2198
|
# }
|
|
2285
2199
|
def construct_error_details_map(error)
|
|
2286
|
-
return {} unless @partial_success
|
|
2287
2200
|
error_details_map = Hash.new { |h, k| h[k] = [] }
|
|
2288
2201
|
|
|
2289
2202
|
error_details = ensure_array(
|
|
@@ -2312,7 +2225,7 @@ module Fluent
|
|
|
2312
2225
|
end
|
|
2313
2226
|
|
|
2314
2227
|
# Extract a map of error details from a potentially partially successful
|
|
2315
|
-
# gRPC request.
|
|
2228
|
+
# gRPC request.
|
|
2316
2229
|
#
|
|
2317
2230
|
# The keys in this map are [error_code, error_message] pairs, and the values
|
|
2318
2231
|
# are a list of indexes of log entries that failed due to this error.
|
|
@@ -2352,7 +2265,6 @@ module Fluent
|
|
|
2352
2265
|
# [3, 'Log name contains illegal character :']: [1, 3]
|
|
2353
2266
|
# }
|
|
2354
2267
|
def construct_error_details_map_grpc(gax_error)
|
|
2355
|
-
return {} unless @partial_success
|
|
2356
2268
|
error_details_map = Hash.new { |h, k| h[k] = [] }
|
|
2357
2269
|
error_details = ensure_array(gax_error.status_details)
|
|
2358
2270
|
raise JSON::ParserError, 'The error details are empty.' if
|
|
@@ -2372,8 +2284,9 @@ module Fluent
|
|
|
2372
2284
|
{}
|
|
2373
2285
|
end
|
|
2374
2286
|
|
|
2375
|
-
#
|
|
2376
|
-
|
|
2287
|
+
# Take a locally unique resource id and convert it to the globally unique
|
|
2288
|
+
# monitored resource.
|
|
2289
|
+
def monitored_resource_from_local_resource_id(local_resource_id)
|
|
2377
2290
|
return unless
|
|
2378
2291
|
/^
|
|
2379
2292
|
(?<resource_type>k8s_container)
|
|
@@ -40,7 +40,6 @@ CONFIG_KEYS = %w(
|
|
|
40
40
|
coerce_to_utf8
|
|
41
41
|
detect_json
|
|
42
42
|
detect_subservice
|
|
43
|
-
enable_metadata_agent
|
|
44
43
|
enable_monitoring
|
|
45
44
|
http_request_key
|
|
46
45
|
insert_id_key
|
|
@@ -51,11 +50,9 @@ CONFIG_KEYS = %w(
|
|
|
51
50
|
labels_key
|
|
52
51
|
labels
|
|
53
52
|
logging_api_url
|
|
54
|
-
metadata_agent_url
|
|
55
53
|
monitoring_type
|
|
56
54
|
non_utf8_replacement_string
|
|
57
55
|
operation_key
|
|
58
|
-
partial_success
|
|
59
56
|
project_id
|
|
60
57
|
require_valid_tags
|
|
61
58
|
source_location_key
|
data/test/plugin/base_test.rb
CHANGED
|
@@ -127,42 +127,6 @@ module BaseTest
|
|
|
127
127
|
end
|
|
128
128
|
end
|
|
129
129
|
|
|
130
|
-
def test_configure_partial_success
|
|
131
|
-
setup_gce_metadata_stubs
|
|
132
|
-
{
|
|
133
|
-
APPLICATION_DEFAULT_CONFIG => true,
|
|
134
|
-
PARTIAL_SUCCESS_DISABLED_CONFIG => false
|
|
135
|
-
}.each do |(config, partial_success)|
|
|
136
|
-
d = create_driver(config)
|
|
137
|
-
assert_equal partial_success,
|
|
138
|
-
d.instance.instance_variable_get(:@partial_success)
|
|
139
|
-
end
|
|
140
|
-
end
|
|
141
|
-
|
|
142
|
-
def test_metadata_agent_url_customization
|
|
143
|
-
[
|
|
144
|
-
# If @metadata_agent_url is set, use that even if the environment
|
|
145
|
-
# variable is set.
|
|
146
|
-
[CUSTOM_METADATA_AGENT_URL_CONFIG, true, CUSTOM_METADATA_AGENT_URL],
|
|
147
|
-
# If @metadata_agent_url is set and the environment variable is
|
|
148
|
-
# not set, use @metadata_agent_url.
|
|
149
|
-
[CUSTOM_METADATA_AGENT_URL_CONFIG, false, CUSTOM_METADATA_AGENT_URL],
|
|
150
|
-
# If @metadata_agent_url is not set and the environment variable is set,
|
|
151
|
-
# use the env.
|
|
152
|
-
[APPLICATION_DEFAULT_CONFIG, true, METADATA_AGENT_URL_FROM_ENV],
|
|
153
|
-
# If @metadata_agent_url is not set and the environment variable is
|
|
154
|
-
# not set, fall back to the default.
|
|
155
|
-
[APPLICATION_DEFAULT_CONFIG, false, DEFAULT_METADATA_AGENT_URL]
|
|
156
|
-
].each do |(config, url_from_env, expected_url)|
|
|
157
|
-
ENV[METADATA_AGENT_URL_ENV_VAR] = METADATA_AGENT_URL_FROM_ENV if
|
|
158
|
-
url_from_env
|
|
159
|
-
setup_gce_metadata_stubs
|
|
160
|
-
d = create_driver(ENABLE_METADATA_AGENT_CONFIG + config)
|
|
161
|
-
assert_equal expected_url, d.instance.metadata_agent_url
|
|
162
|
-
ENV.delete(METADATA_AGENT_URL_ENV_VAR)
|
|
163
|
-
end
|
|
164
|
-
end
|
|
165
|
-
|
|
166
130
|
def test_configure_ignores_unknown_monitoring_type
|
|
167
131
|
# Verify that driver creation succeeds when monitoring type is not
|
|
168
132
|
# "prometheus" (in which case, we simply don't record metrics),
|
|
@@ -1758,176 +1722,48 @@ module BaseTest
|
|
|
1758
1722
|
end
|
|
1759
1723
|
end
|
|
1760
1724
|
|
|
1761
|
-
#
|
|
1762
|
-
|
|
1763
|
-
# Test enable_metadata_agent not set or set to false.
|
|
1764
|
-
def test_configure_enable_metadata_agent_default_and_false
|
|
1765
|
-
setup_gce_metadata_stubs
|
|
1766
|
-
[create_driver, create_driver(DISABLE_METADATA_AGENT_CONFIG)].each do |d|
|
|
1767
|
-
assert_false d.instance.instance_variable_get(:@enable_metadata_agent)
|
|
1768
|
-
end
|
|
1769
|
-
end
|
|
1770
|
-
|
|
1771
|
-
# Test enable_metadata_agent set to true.
|
|
1772
|
-
def test_configure_enable_metadata_agent_true
|
|
1773
|
-
new_stub_context do
|
|
1774
|
-
setup_gce_metadata_stubs
|
|
1775
|
-
setup_metadata_agent_stubs
|
|
1776
|
-
d = create_driver(ENABLE_METADATA_AGENT_CONFIG)
|
|
1777
|
-
assert_true d.instance.instance_variable_get(:@enable_metadata_agent)
|
|
1778
|
-
end
|
|
1779
|
-
end
|
|
1780
|
-
|
|
1781
|
-
# Docker Container.
|
|
1782
|
-
|
|
1783
|
-
# Test textPayload logs from Docker container stdout / stderr.
|
|
1784
|
-
def test_docker_container_stdout_stderr_logs_text_payload
|
|
1785
|
-
[1, 2, 3, 5, 11, 50].each do |n|
|
|
1786
|
-
new_stub_context do
|
|
1787
|
-
setup_gce_metadata_stubs
|
|
1788
|
-
setup_metadata_agent_stubs
|
|
1789
|
-
setup_logging_stubs do
|
|
1790
|
-
d = create_driver(DOCKER_CONTAINER_CONFIG)
|
|
1791
|
-
n.times do |i|
|
|
1792
|
-
d.emit(docker_container_stdout_stderr_log_entry(log_entry(i)))
|
|
1793
|
-
end
|
|
1794
|
-
d.run
|
|
1795
|
-
end
|
|
1796
|
-
verify_log_entries(n, DOCKER_CONTAINER_PARAMS)
|
|
1797
|
-
assert_requested_metadata_agent_stub("container.#{DOCKER_CONTAINER_ID}")
|
|
1798
|
-
end
|
|
1799
|
-
end
|
|
1800
|
-
end
|
|
1801
|
-
|
|
1802
|
-
# Test jsonPayload logs from Docker container stdout / stderr.
|
|
1803
|
-
def test_docker_container_stdout_stderr_logs_json_payload
|
|
1804
|
-
[1, 2, 3, 5, 11, 50].each do |n|
|
|
1805
|
-
new_stub_context do
|
|
1806
|
-
setup_gce_metadata_stubs
|
|
1807
|
-
setup_metadata_agent_stubs
|
|
1808
|
-
setup_logging_stubs do
|
|
1809
|
-
d = create_driver(DOCKER_CONTAINER_CONFIG)
|
|
1810
|
-
n.times do
|
|
1811
|
-
d.emit(docker_container_stdout_stderr_log_entry(
|
|
1812
|
-
'{"msg": "test log entry ' \
|
|
1813
|
-
"#{n}" \
|
|
1814
|
-
'", "tag2": "test", "data": ' \
|
|
1815
|
-
'5000, "severity": "WARNING"}'))
|
|
1816
|
-
end
|
|
1817
|
-
d.run
|
|
1818
|
-
end
|
|
1819
|
-
verify_log_entries(n, DOCKER_CONTAINER_PARAMS, 'jsonPayload') do |entry|
|
|
1820
|
-
fields = entry['jsonPayload']
|
|
1821
|
-
assert_equal 3, fields.size, entry
|
|
1822
|
-
assert_equal "test log entry #{n}", fields['msg'], entry
|
|
1823
|
-
assert_equal 'test', fields['tag2'], entry
|
|
1824
|
-
assert_equal 5000, fields['data'], entry
|
|
1825
|
-
end
|
|
1826
|
-
assert_requested_metadata_agent_stub("container.#{DOCKER_CONTAINER_ID}")
|
|
1827
|
-
end
|
|
1828
|
-
end
|
|
1829
|
-
end
|
|
1830
|
-
|
|
1831
|
-
# Test logs from applications running in Docker containers. These logs have
|
|
1832
|
-
# the label "logging.googleapis.com/local_resource_id" set in the format of
|
|
1833
|
-
# "container.<container_name>".
|
|
1834
|
-
def test_docker_container_application_logs
|
|
1835
|
-
new_stub_context do
|
|
1836
|
-
setup_gce_metadata_stubs
|
|
1837
|
-
setup_metadata_agent_stubs
|
|
1838
|
-
setup_logging_stubs do
|
|
1839
|
-
# Metadata Agent is not enabled. Will call Docker Remote API for
|
|
1840
|
-
# container info.
|
|
1841
|
-
d = create_driver(ENABLE_METADATA_AGENT_CONFIG)
|
|
1842
|
-
d.emit(docker_container_application_log_entry(log_entry(0)))
|
|
1843
|
-
d.run
|
|
1844
|
-
end
|
|
1845
|
-
verify_log_entries(1, DOCKER_CONTAINER_PARAMS_NO_STREAM)
|
|
1846
|
-
assert_requested_metadata_agent_stub(
|
|
1847
|
-
"#{DOCKER_CONTAINER_LOCAL_RESOURCE_ID_PREFIX}.#{DOCKER_CONTAINER_NAME}")
|
|
1848
|
-
end
|
|
1849
|
-
end
|
|
1850
|
-
|
|
1851
|
-
# Test k8s_container monitored resource including the fallback when Metadata
|
|
1852
|
-
# Agent restarts.
|
|
1853
|
-
def test_k8s_container_monitored_resource_fallback
|
|
1725
|
+
# Test k8s_container monitored resource.
|
|
1726
|
+
def test_k8s_container_monitored_resource
|
|
1854
1727
|
[
|
|
1855
|
-
# When enable_metadata_agent is false.
|
|
1856
1728
|
{
|
|
1857
1729
|
config: APPLICATION_DEFAULT_CONFIG,
|
|
1858
|
-
setup_metadata_agent_stub: false,
|
|
1859
1730
|
setup_k8s_stub: false,
|
|
1860
1731
|
log_entry: k8s_container_log_entry(log_entry(0)),
|
|
1861
1732
|
expected_params: K8S_CONTAINER_PARAMS_FROM_FALLBACK
|
|
1862
1733
|
},
|
|
1863
1734
|
{
|
|
1864
1735
|
config: APPLICATION_DEFAULT_CONFIG,
|
|
1865
|
-
setup_metadata_agent_stub: true,
|
|
1866
1736
|
setup_k8s_stub: false,
|
|
1867
1737
|
log_entry: k8s_container_log_entry(log_entry(0)),
|
|
1868
1738
|
expected_params: K8S_CONTAINER_PARAMS_FROM_FALLBACK
|
|
1869
1739
|
},
|
|
1870
1740
|
{
|
|
1871
1741
|
config: APPLICATION_DEFAULT_CONFIG,
|
|
1872
|
-
setup_metadata_agent_stub: true,
|
|
1873
1742
|
setup_k8s_stub: true,
|
|
1874
1743
|
log_entry: k8s_container_log_entry(log_entry(0)),
|
|
1875
1744
|
expected_params: K8S_CONTAINER_PARAMS_FROM_LOCAL
|
|
1876
1745
|
},
|
|
1877
1746
|
{
|
|
1878
1747
|
config: APPLICATION_DEFAULT_CONFIG,
|
|
1879
|
-
setup_metadata_agent_stub: false,
|
|
1880
1748
|
setup_k8s_stub: true,
|
|
1881
1749
|
log_entry: k8s_container_log_entry(log_entry(0)),
|
|
1882
1750
|
expected_params: K8S_CONTAINER_PARAMS_FROM_LOCAL
|
|
1883
1751
|
},
|
|
1884
|
-
# When enable_metadata_agent is true.
|
|
1885
1752
|
{
|
|
1886
|
-
config:
|
|
1887
|
-
setup_metadata_agent_stub: false,
|
|
1888
|
-
setup_k8s_stub: false,
|
|
1889
|
-
log_entry: k8s_container_log_entry(log_entry(0)),
|
|
1890
|
-
expected_params: K8S_CONTAINER_PARAMS_FROM_FALLBACK
|
|
1891
|
-
},
|
|
1892
|
-
{
|
|
1893
|
-
config: ENABLE_METADATA_AGENT_CONFIG,
|
|
1894
|
-
setup_metadata_agent_stub: false,
|
|
1895
|
-
setup_k8s_stub: true,
|
|
1896
|
-
log_entry: k8s_container_log_entry(log_entry(0)),
|
|
1897
|
-
expected_params: K8S_CONTAINER_PARAMS_FROM_LOCAL
|
|
1898
|
-
},
|
|
1899
|
-
{
|
|
1900
|
-
config: CUSTOM_K8S_ENABLE_METADATA_AGENT_CONFIG,
|
|
1901
|
-
setup_metadata_agent_stub: false,
|
|
1753
|
+
config: CUSTOM_K8S_CONFIG,
|
|
1902
1754
|
setup_k8s_stub: false,
|
|
1903
1755
|
log_entry: k8s_container_log_entry(log_entry(0)),
|
|
1904
1756
|
expected_params: K8S_CONTAINER_PARAMS_CUSTOM
|
|
1905
1757
|
},
|
|
1906
1758
|
{
|
|
1907
|
-
config:
|
|
1908
|
-
setup_metadata_agent_stub: true,
|
|
1909
|
-
setup_k8s_stub: true,
|
|
1910
|
-
log_entry: k8s_container_log_entry(log_entry(0)),
|
|
1911
|
-
expected_params: K8S_CONTAINER_PARAMS
|
|
1912
|
-
},
|
|
1913
|
-
{
|
|
1914
|
-
config: ENABLE_METADATA_AGENT_CONFIG,
|
|
1915
|
-
setup_metadata_agent_stub: true,
|
|
1916
|
-
setup_k8s_stub: false,
|
|
1917
|
-
log_entry: k8s_container_log_entry(log_entry(0)),
|
|
1918
|
-
expected_params: K8S_CONTAINER_PARAMS
|
|
1919
|
-
},
|
|
1920
|
-
{
|
|
1921
|
-
config: ENABLE_METADATA_AGENT_CONFIG,
|
|
1922
|
-
setup_metadata_agent_stub: true,
|
|
1759
|
+
config: EMPTY_K8S_CONFIG,
|
|
1923
1760
|
setup_k8s_stub: true,
|
|
1924
1761
|
log_entry: k8s_container_log_entry(log_entry(0)),
|
|
1925
|
-
expected_params:
|
|
1762
|
+
expected_params: K8S_CONTAINER_PARAMS_FROM_LOCAL
|
|
1926
1763
|
}
|
|
1927
1764
|
].each do |test_params|
|
|
1928
1765
|
new_stub_context do
|
|
1929
1766
|
setup_gce_metadata_stubs
|
|
1930
|
-
setup_metadata_agent_stubs(test_params[:setup_metadata_agent_stub])
|
|
1931
1767
|
setup_k8s_metadata_stubs(test_params[:setup_k8s_stub])
|
|
1932
1768
|
setup_logging_stubs do
|
|
1933
1769
|
d = create_driver(test_params[:config], CONTAINER_TAG)
|
|
@@ -1949,16 +1785,14 @@ module BaseTest
|
|
|
1949
1785
|
[
|
|
1950
1786
|
# When local_resource_id is not present or does not match k8s regexes.
|
|
1951
1787
|
{
|
|
1952
|
-
config:
|
|
1953
|
-
setup_metadata_agent_stub: true,
|
|
1788
|
+
config: APPLICATION_DEFAULT_CONFIG,
|
|
1954
1789
|
setup_k8s_stub: true,
|
|
1955
1790
|
log_entry: k8s_container_log_entry(
|
|
1956
1791
|
log_entry(0)).reject { |k, _| k == LOCAL_RESOURCE_ID_KEY },
|
|
1957
1792
|
expected_params: CONTAINER_FROM_TAG_PARAMS
|
|
1958
1793
|
},
|
|
1959
1794
|
{
|
|
1960
|
-
config:
|
|
1961
|
-
setup_metadata_agent_stub: true,
|
|
1795
|
+
config: APPLICATION_DEFAULT_CONFIG,
|
|
1962
1796
|
setup_k8s_stub: true,
|
|
1963
1797
|
log_entry: k8s_container_log_entry(
|
|
1964
1798
|
log_entry(0),
|
|
@@ -1968,7 +1802,6 @@ module BaseTest
|
|
|
1968
1802
|
].each do |test_params|
|
|
1969
1803
|
new_stub_context do
|
|
1970
1804
|
setup_gce_metadata_stubs
|
|
1971
|
-
setup_metadata_agent_stubs(test_params[:setup_metadata_agent_stub])
|
|
1972
1805
|
setup_k8s_metadata_stubs(test_params[:setup_k8s_stub])
|
|
1973
1806
|
setup_logging_stubs do
|
|
1974
1807
|
d = create_driver(test_params[:config], CONTAINER_TAG)
|
|
@@ -1982,49 +1815,30 @@ module BaseTest
|
|
|
1982
1815
|
end
|
|
1983
1816
|
end
|
|
1984
1817
|
|
|
1985
|
-
# Test k8s_pod monitored resource
|
|
1986
|
-
|
|
1987
|
-
def test_k8s_pod_monitored_resource_fallback
|
|
1818
|
+
# Test k8s_pod monitored resource.
|
|
1819
|
+
def test_k8s_pod_monitored_resource
|
|
1988
1820
|
[
|
|
1989
1821
|
{
|
|
1990
1822
|
config: APPLICATION_DEFAULT_CONFIG,
|
|
1991
|
-
setup_metadata_agent_stub: true,
|
|
1992
1823
|
setup_k8s_stub: true,
|
|
1993
1824
|
log_entry: k8s_pod_log_entry(log_entry(0)),
|
|
1994
1825
|
expected_params: K8S_POD_PARAMS_FROM_LOCAL
|
|
1995
1826
|
},
|
|
1996
1827
|
{
|
|
1997
|
-
config:
|
|
1998
|
-
setup_metadata_agent_stub: false,
|
|
1999
|
-
setup_k8s_stub: true,
|
|
2000
|
-
log_entry: k8s_pod_log_entry(log_entry(0)),
|
|
2001
|
-
expected_params: K8S_POD_PARAMS_FROM_LOCAL
|
|
2002
|
-
},
|
|
2003
|
-
{
|
|
2004
|
-
config: CUSTOM_K8S_ENABLE_METADATA_AGENT_CONFIG,
|
|
2005
|
-
setup_metadata_agent_stub: false,
|
|
1828
|
+
config: CUSTOM_K8S_CONFIG,
|
|
2006
1829
|
setup_k8s_stub: false,
|
|
2007
1830
|
log_entry: k8s_pod_log_entry(log_entry(0)),
|
|
2008
1831
|
expected_params: K8S_POD_PARAMS_CUSTOM
|
|
2009
1832
|
},
|
|
2010
1833
|
{
|
|
2011
|
-
config:
|
|
2012
|
-
setup_metadata_agent_stub: true,
|
|
2013
|
-
setup_k8s_stub: true,
|
|
2014
|
-
log_entry: k8s_pod_log_entry(log_entry(0)),
|
|
2015
|
-
expected_params: K8S_POD_PARAMS
|
|
2016
|
-
},
|
|
2017
|
-
{
|
|
2018
|
-
config: ENABLE_METADATA_AGENT_CONFIG,
|
|
2019
|
-
setup_metadata_agent_stub: true,
|
|
1834
|
+
config: EMPTY_K8S_CONFIG,
|
|
2020
1835
|
setup_k8s_stub: true,
|
|
2021
1836
|
log_entry: k8s_pod_log_entry(log_entry(0)),
|
|
2022
|
-
expected_params:
|
|
1837
|
+
expected_params: K8S_POD_PARAMS_FROM_LOCAL
|
|
2023
1838
|
}
|
|
2024
1839
|
].each do |test_params|
|
|
2025
1840
|
new_stub_context do
|
|
2026
1841
|
setup_gce_metadata_stubs
|
|
2027
|
-
setup_metadata_agent_stubs(test_params[:setup_metadata_agent_stub])
|
|
2028
1842
|
setup_k8s_metadata_stubs(test_params[:setup_k8s_stub])
|
|
2029
1843
|
setup_logging_stubs do
|
|
2030
1844
|
d = create_driver(test_params[:config])
|
|
@@ -2042,49 +1856,30 @@ module BaseTest
|
|
|
2042
1856
|
end
|
|
2043
1857
|
end
|
|
2044
1858
|
|
|
2045
|
-
# Test k8s_node monitored resource
|
|
2046
|
-
|
|
2047
|
-
def test_k8s_node_monitored_resource_fallback
|
|
1859
|
+
# Test k8s_node monitored resource.
|
|
1860
|
+
def test_k8s_node_monitored_resource
|
|
2048
1861
|
[
|
|
2049
1862
|
{
|
|
2050
1863
|
config: APPLICATION_DEFAULT_CONFIG,
|
|
2051
|
-
setup_metadata_agent_stub: true,
|
|
2052
1864
|
setup_k8s_stub: true,
|
|
2053
1865
|
log_entry: k8s_node_log_entry(log_entry(0)),
|
|
2054
1866
|
expected_params: K8S_NODE_PARAMS_FROM_LOCAL
|
|
2055
1867
|
},
|
|
2056
1868
|
{
|
|
2057
|
-
config:
|
|
2058
|
-
setup_metadata_agent_stub: false,
|
|
2059
|
-
setup_k8s_stub: true,
|
|
2060
|
-
log_entry: k8s_node_log_entry(log_entry(0)),
|
|
2061
|
-
expected_params: K8S_NODE_PARAMS_FROM_LOCAL
|
|
2062
|
-
},
|
|
2063
|
-
{
|
|
2064
|
-
config: CUSTOM_K8S_ENABLE_METADATA_AGENT_CONFIG,
|
|
2065
|
-
setup_metadata_agent_stub: false,
|
|
1869
|
+
config: CUSTOM_K8S_CONFIG,
|
|
2066
1870
|
setup_k8s_stub: false,
|
|
2067
1871
|
log_entry: k8s_node_log_entry(log_entry(0)),
|
|
2068
1872
|
expected_params: K8S_NODE_PARAMS_CUSTOM
|
|
2069
1873
|
},
|
|
2070
1874
|
{
|
|
2071
|
-
config:
|
|
2072
|
-
setup_metadata_agent_stub: true,
|
|
1875
|
+
config: EMPTY_K8S_CONFIG,
|
|
2073
1876
|
setup_k8s_stub: true,
|
|
2074
1877
|
log_entry: k8s_node_log_entry(log_entry(0)),
|
|
2075
|
-
expected_params:
|
|
2076
|
-
},
|
|
2077
|
-
{
|
|
2078
|
-
config: ENABLE_METADATA_AGENT_CONFIG,
|
|
2079
|
-
setup_metadata_agent_stub: true,
|
|
2080
|
-
setup_k8s_stub: true,
|
|
2081
|
-
log_entry: k8s_node_log_entry(log_entry(0)),
|
|
2082
|
-
expected_params: K8S_NODE_PARAMS
|
|
1878
|
+
expected_params: K8S_NODE_PARAMS_FROM_LOCAL
|
|
2083
1879
|
}
|
|
2084
1880
|
].each do |test_params|
|
|
2085
1881
|
new_stub_context do
|
|
2086
1882
|
setup_gce_metadata_stubs
|
|
2087
|
-
setup_metadata_agent_stubs(test_params[:setup_metadata_agent_stub])
|
|
2088
1883
|
setup_k8s_metadata_stubs(test_params[:setup_k8s_stub])
|
|
2089
1884
|
setup_logging_stubs do
|
|
2090
1885
|
d = create_driver(test_params[:config])
|
|
@@ -2102,83 +1897,6 @@ module BaseTest
|
|
|
2102
1897
|
end
|
|
2103
1898
|
end
|
|
2104
1899
|
|
|
2105
|
-
# Test that the 'time' field from the json record is extracted and set to
|
|
2106
|
-
# entry.timestamp for Docker container logs.
|
|
2107
|
-
def test_time_field_extraction_for_docker_container_logs
|
|
2108
|
-
new_stub_context do
|
|
2109
|
-
setup_gce_metadata_stubs
|
|
2110
|
-
setup_metadata_agent_stubs
|
|
2111
|
-
setup_logging_stubs do
|
|
2112
|
-
d = create_driver(ENABLE_METADATA_AGENT_CONFIG)
|
|
2113
|
-
d.emit(docker_container_application_log_entry(log_entry(0)))
|
|
2114
|
-
d.run
|
|
2115
|
-
end
|
|
2116
|
-
verify_log_entries(1, DOCKER_CONTAINER_PARAMS_NO_STREAM) do |entry, i|
|
|
2117
|
-
verify_default_log_entry_text(entry['textPayload'], i, entry)
|
|
2118
|
-
# Timestamp in 'time' field from log entry should be set properly.
|
|
2119
|
-
actual_timestamp = timestamp_parse(entry['timestamp'])
|
|
2120
|
-
assert_equal DOCKER_CONTAINER_SECONDS_EPOCH,
|
|
2121
|
-
actual_timestamp['seconds'], entry
|
|
2122
|
-
assert_equal DOCKER_CONTAINER_NANOS, actual_timestamp['nanos'], entry
|
|
2123
|
-
end
|
|
2124
|
-
assert_requested_metadata_agent_stub(
|
|
2125
|
-
"#{DOCKER_CONTAINER_LOCAL_RESOURCE_ID_PREFIX}.#{DOCKER_CONTAINER_NAME}")
|
|
2126
|
-
end
|
|
2127
|
-
end
|
|
2128
|
-
|
|
2129
|
-
# Test that the 'source' field is properly extracted from the record json and
|
|
2130
|
-
# set as a common label 'stream'.
|
|
2131
|
-
def test_source_for_docker_container_logs
|
|
2132
|
-
{
|
|
2133
|
-
docker_container_stdout_stderr_log_entry(
|
|
2134
|
-
log_entry(0), DOCKER_CONTAINER_STREAM_STDOUT) =>
|
|
2135
|
-
DOCKER_CONTAINER_PARAMS,
|
|
2136
|
-
docker_container_stdout_stderr_log_entry(
|
|
2137
|
-
log_entry(0), DOCKER_CONTAINER_STREAM_STDERR) =>
|
|
2138
|
-
DOCKER_CONTAINER_PARAMS_STREAM_STDERR,
|
|
2139
|
-
docker_container_application_log_entry(log_entry(0)) =>
|
|
2140
|
-
DOCKER_CONTAINER_PARAMS_NO_STREAM,
|
|
2141
|
-
docker_container_application_log_entry(log_entry(0)) \
|
|
2142
|
-
.merge('severity' => 'warning') =>
|
|
2143
|
-
DOCKER_CONTAINER_PARAMS_NO_STREAM
|
|
2144
|
-
}.each do |log_entry, expected_params|
|
|
2145
|
-
new_stub_context do
|
|
2146
|
-
setup_gce_metadata_stubs
|
|
2147
|
-
setup_metadata_agent_stubs
|
|
2148
|
-
setup_logging_stubs do
|
|
2149
|
-
d = create_driver(DOCKER_CONTAINER_CONFIG)
|
|
2150
|
-
d.emit(log_entry)
|
|
2151
|
-
d.run
|
|
2152
|
-
end
|
|
2153
|
-
verify_log_entries(1, expected_params)
|
|
2154
|
-
end
|
|
2155
|
-
end
|
|
2156
|
-
end
|
|
2157
|
-
|
|
2158
|
-
# Test GKE container logs. These logs have the label
|
|
2159
|
-
# "logging.googleapis.com/local_resource_id" set in the format of
|
|
2160
|
-
# "gke_container.<namespace_id>.<pod_name>.<container_name>".
|
|
2161
|
-
def test_gke_container_logs
|
|
2162
|
-
[1, 2, 3, 5, 11, 50].each do |n|
|
|
2163
|
-
new_stub_context do
|
|
2164
|
-
setup_gce_metadata_stubs
|
|
2165
|
-
setup_k8s_metadata_stubs
|
|
2166
|
-
setup_metadata_agent_stubs
|
|
2167
|
-
setup_logging_stubs do
|
|
2168
|
-
d = create_driver(ENABLE_METADATA_AGENT_CONFIG)
|
|
2169
|
-
n.times do |i|
|
|
2170
|
-
d.emit(gke_container_log_entry(log_entry(i)))
|
|
2171
|
-
end
|
|
2172
|
-
d.run
|
|
2173
|
-
end
|
|
2174
|
-
verify_log_entries(n, CONTAINER_FROM_APPLICATION_PARAMS)
|
|
2175
|
-
assert_requested_metadata_agent_stub(
|
|
2176
|
-
"#{CONTAINER_LOCAL_RESOURCE_ID_PREFIX}.#{CONTAINER_NAMESPACE_ID}" \
|
|
2177
|
-
".#{K8S_POD_NAME}.#{K8S_CONTAINER_NAME}")
|
|
2178
|
-
end
|
|
2179
|
-
end
|
|
2180
|
-
end
|
|
2181
|
-
|
|
2182
1900
|
def test_uptime_metric
|
|
2183
1901
|
setup_gce_metadata_stubs
|
|
2184
1902
|
[
|
|
@@ -2315,12 +2033,6 @@ module BaseTest
|
|
|
2315
2033
|
OpenCensus::Stats.ensure_recorder.clear_stats
|
|
2316
2034
|
end
|
|
2317
2035
|
|
|
2318
|
-
# Metadata Agent.
|
|
2319
|
-
|
|
2320
|
-
def metadata_request_url(local_resource_id)
|
|
2321
|
-
"#{DEFAULT_METADATA_AGENT_URL}/monitoredResource/#{local_resource_id}"
|
|
2322
|
-
end
|
|
2323
|
-
|
|
2324
2036
|
# Provide a stub context that initializes @logs_sent, executes the block and
|
|
2325
2037
|
# resets WebMock at the end.
|
|
2326
2038
|
def new_stub_context
|
|
@@ -2329,26 +2041,6 @@ module BaseTest
|
|
|
2329
2041
|
WebMock.reset!
|
|
2330
2042
|
end
|
|
2331
2043
|
|
|
2332
|
-
def setup_metadata_agent_stubs(should_respond = true)
|
|
2333
|
-
if should_respond
|
|
2334
|
-
MONITORED_RESOURCE_STUBS.each do |local_resource_id, resource|
|
|
2335
|
-
stub_request(:get, metadata_request_url(local_resource_id))
|
|
2336
|
-
.to_return(status: 200, body: resource)
|
|
2337
|
-
end
|
|
2338
|
-
stub_request(:get, metadata_request_url(RANDOM_LOCAL_RESOURCE_ID))
|
|
2339
|
-
.to_return(status: 404, body: '')
|
|
2340
|
-
else
|
|
2341
|
-
# Simulate an environment with no metadata agent endpoint present.
|
|
2342
|
-
stub_request(:get,
|
|
2343
|
-
%r{#{DEFAULT_METADATA_AGENT_URL}\/monitoredResource/.*})
|
|
2344
|
-
.to_raise(Errno::EHOSTUNREACH)
|
|
2345
|
-
end
|
|
2346
|
-
end
|
|
2347
|
-
|
|
2348
|
-
def assert_requested_metadata_agent_stub(local_resource_id)
|
|
2349
|
-
assert_requested :get, metadata_request_url(local_resource_id)
|
|
2350
|
-
end
|
|
2351
|
-
|
|
2352
2044
|
# GKE Container.
|
|
2353
2045
|
|
|
2354
2046
|
def container_tag_with_container_name(container_name)
|
|
@@ -2391,32 +2083,6 @@ module BaseTest
|
|
|
2391
2083
|
}
|
|
2392
2084
|
end
|
|
2393
2085
|
|
|
2394
|
-
# Docker Container.
|
|
2395
|
-
|
|
2396
|
-
def docker_container_stdout_stderr_log_entry(
|
|
2397
|
-
log, stream = DOCKER_CONTAINER_STREAM_STDOUT)
|
|
2398
|
-
severity = if stream == DOCKER_CONTAINER_STREAM_STDOUT
|
|
2399
|
-
'INFO'
|
|
2400
|
-
else
|
|
2401
|
-
'ERROR'
|
|
2402
|
-
end
|
|
2403
|
-
{
|
|
2404
|
-
log: log,
|
|
2405
|
-
source: stream,
|
|
2406
|
-
severity: severity,
|
|
2407
|
-
LOCAL_RESOURCE_ID_KEY => "container.#{DOCKER_CONTAINER_ID}"
|
|
2408
|
-
}
|
|
2409
|
-
end
|
|
2410
|
-
|
|
2411
|
-
def docker_container_application_log_entry(log)
|
|
2412
|
-
{
|
|
2413
|
-
log: log,
|
|
2414
|
-
time: DOCKER_CONTAINER_TIMESTAMP,
|
|
2415
|
-
LOCAL_RESOURCE_ID_KEY => "#{DOCKER_CONTAINER_LOCAL_RESOURCE_ID_PREFIX}." \
|
|
2416
|
-
"#{DOCKER_CONTAINER_NAME}"
|
|
2417
|
-
}
|
|
2418
|
-
end
|
|
2419
|
-
|
|
2420
2086
|
# TODO(qingling128): Temporary fallback for metadata agent restarts.
|
|
2421
2087
|
# k8s resources.
|
|
2422
2088
|
|
data/test/plugin/constants.rb
CHANGED
|
@@ -52,8 +52,6 @@ module Constants
|
|
|
52
52
|
# Generic attributes.
|
|
53
53
|
HOSTNAME = Socket.gethostname
|
|
54
54
|
CUSTOM_LOGGING_API_URL = 'http://localhost:52000'.freeze
|
|
55
|
-
CUSTOM_METADATA_AGENT_URL = 'http://localhost:12345'.freeze
|
|
56
|
-
METADATA_AGENT_URL_FROM_ENV = 'http://localhost:54321'.freeze
|
|
57
55
|
|
|
58
56
|
# TODO(qingling128) Separate constants into different submodules.
|
|
59
57
|
# Attributes used for the GCE metadata service.
|
|
@@ -133,18 +131,6 @@ module Constants
|
|
|
133
131
|
"projects/#{PROJECT_ID}/" \
|
|
134
132
|
"traces/#{INVALID_NON_HEX_STACKDRIVER_TRACE_ID}".freeze
|
|
135
133
|
|
|
136
|
-
# Docker Container labels.
|
|
137
|
-
DOCKER_CONTAINER_ID =
|
|
138
|
-
'0d0f03ff8d3c42688692536d1af77a28cd135c0a5c531f25a31'.freeze
|
|
139
|
-
DOCKER_CONTAINER_NAME = 'happy_hippo'.freeze
|
|
140
|
-
DOCKER_CONTAINER_STREAM_STDOUT = 'stdout'.freeze
|
|
141
|
-
DOCKER_CONTAINER_STREAM_STDERR = 'stderr'.freeze
|
|
142
|
-
# Timestamp for 1234567890 seconds and 987654321 nanoseconds since epoch.
|
|
143
|
-
DOCKER_CONTAINER_TIMESTAMP = '2009-02-13T23:31:30.987654321Z'.freeze
|
|
144
|
-
DOCKER_CONTAINER_SECONDS_EPOCH = 1_234_567_890
|
|
145
|
-
DOCKER_CONTAINER_NANOS = 987_654_321
|
|
146
|
-
DOCKER_CONTAINER_LOCAL_RESOURCE_ID_PREFIX = 'container'.freeze
|
|
147
|
-
|
|
148
134
|
# New K8s resource constants.
|
|
149
135
|
K8S_LOCATION = 'us-central1-b'.freeze
|
|
150
136
|
K8S_LOCATION2 = 'us-central1-c'.freeze
|
|
@@ -238,18 +224,10 @@ module Constants
|
|
|
238
224
|
logging_api_url #{CUSTOM_LOGGING_API_URL}
|
|
239
225
|
).freeze
|
|
240
226
|
|
|
241
|
-
CUSTOM_METADATA_AGENT_URL_CONFIG = %(
|
|
242
|
-
metadata_agent_url #{CUSTOM_METADATA_AGENT_URL}
|
|
243
|
-
).freeze
|
|
244
|
-
|
|
245
227
|
DETECT_JSON_CONFIG = %(
|
|
246
228
|
detect_json true
|
|
247
229
|
).freeze
|
|
248
230
|
|
|
249
|
-
PARTIAL_SUCCESS_DISABLED_CONFIG = %(
|
|
250
|
-
partial_success false
|
|
251
|
-
).freeze
|
|
252
|
-
|
|
253
231
|
# rubocop:disable Metrics/LineLength
|
|
254
232
|
PRIVATE_KEY_CONFIG = %(
|
|
255
233
|
auth_method private_key
|
|
@@ -284,14 +262,6 @@ module Constants
|
|
|
284
262
|
monitoring_type opencensus
|
|
285
263
|
).freeze
|
|
286
264
|
|
|
287
|
-
ENABLE_METADATA_AGENT_CONFIG = %(
|
|
288
|
-
enable_metadata_agent true
|
|
289
|
-
).freeze
|
|
290
|
-
|
|
291
|
-
DISABLE_METADATA_AGENT_CONFIG = %(
|
|
292
|
-
enable_metadata_agent false
|
|
293
|
-
).freeze
|
|
294
|
-
|
|
295
265
|
ENABLE_AUTOFORMAT_STACKDRIVER_TRACE_CONFIG = %(
|
|
296
266
|
autoformat_stackdriver_trace true
|
|
297
267
|
).freeze
|
|
@@ -300,12 +270,6 @@ module Constants
|
|
|
300
270
|
autoformat_stackdriver_trace false
|
|
301
271
|
).freeze
|
|
302
272
|
|
|
303
|
-
DOCKER_CONTAINER_CONFIG = %(
|
|
304
|
-
enable_metadata_agent true
|
|
305
|
-
label_map { "source": "#{DOCKER_CONSTANTS[:service]}/stream" }
|
|
306
|
-
detect_json true
|
|
307
|
-
).freeze
|
|
308
|
-
|
|
309
273
|
CUSTOM_METADATA_CONFIG = %(
|
|
310
274
|
project_id #{CUSTOM_PROJECT_ID}
|
|
311
275
|
zone #{CUSTOM_ZONE}
|
|
@@ -328,14 +292,12 @@ module Constants
|
|
|
328
292
|
CONFIG_MISSING_METADATA_ALL = %(
|
|
329
293
|
).freeze
|
|
330
294
|
|
|
331
|
-
|
|
332
|
-
enable_metadata_agent true
|
|
295
|
+
CUSTOM_K8S_CONFIG = %(
|
|
333
296
|
k8s_cluster_name #{CUSTOM_K8S_CLUSTER_NAME}
|
|
334
297
|
k8s_cluster_location #{CUSTOM_K8S_LOCATION}
|
|
335
298
|
).freeze
|
|
336
299
|
|
|
337
|
-
|
|
338
|
-
enable_metadata_agent true
|
|
300
|
+
EMPTY_K8S_CONFIG = %(
|
|
339
301
|
k8s_cluster_name ""
|
|
340
302
|
k8s_cluster_location ""
|
|
341
303
|
).freeze
|
|
@@ -480,7 +442,6 @@ module Constants
|
|
|
480
442
|
coerce_to_utf8 false
|
|
481
443
|
detect_json true
|
|
482
444
|
detect_subservice false
|
|
483
|
-
enable_metadata_agent true
|
|
484
445
|
enable_monitoring true
|
|
485
446
|
http_request_key test_http_request_key
|
|
486
447
|
insert_id_key test_insert_id_key
|
|
@@ -491,11 +452,9 @@ module Constants
|
|
|
491
452
|
labels_key test_labels_key
|
|
492
453
|
labels { "labels_key": "labels_value" }
|
|
493
454
|
logging_api_url http://localhost:52000
|
|
494
|
-
metadata_agent_url http://localhost:12345
|
|
495
455
|
monitoring_type not_prometheus
|
|
496
456
|
non_utf8_replacement_string zzz
|
|
497
457
|
operation_key test_operation_key
|
|
498
|
-
partial_success false
|
|
499
458
|
project_id test-project-id-123
|
|
500
459
|
require_valid_tags true
|
|
501
460
|
source_location_key test_source_location_key
|
|
@@ -724,29 +683,6 @@ module Constants
|
|
|
724
683
|
)
|
|
725
684
|
).freeze
|
|
726
685
|
|
|
727
|
-
# Docker Container.
|
|
728
|
-
DOCKER_CONTAINER_PARAMS = {
|
|
729
|
-
resource: {
|
|
730
|
-
type: DOCKER_CONSTANTS[:resource_type],
|
|
731
|
-
labels: {
|
|
732
|
-
'container_id' => DOCKER_CONTAINER_ID,
|
|
733
|
-
'location' => ZONE
|
|
734
|
-
}
|
|
735
|
-
},
|
|
736
|
-
log_name: 'test',
|
|
737
|
-
project_id: PROJECT_ID,
|
|
738
|
-
labels: {
|
|
739
|
-
"#{DOCKER_CONSTANTS[:service]}/stream" => DOCKER_CONTAINER_STREAM_STDOUT
|
|
740
|
-
}
|
|
741
|
-
}.freeze
|
|
742
|
-
DOCKER_CONTAINER_PARAMS_STREAM_STDERR = DOCKER_CONTAINER_PARAMS.merge(
|
|
743
|
-
labels: DOCKER_CONTAINER_PARAMS[:labels].merge(
|
|
744
|
-
"#{DOCKER_CONSTANTS[:service]}/stream" => DOCKER_CONTAINER_STREAM_STDERR
|
|
745
|
-
)
|
|
746
|
-
).freeze
|
|
747
|
-
DOCKER_CONTAINER_PARAMS_NO_STREAM =
|
|
748
|
-
DOCKER_CONTAINER_PARAMS.merge(labels: {}).freeze
|
|
749
|
-
|
|
750
686
|
# Cloud Dataflow.
|
|
751
687
|
DATAFLOW_PARAMS = {
|
|
752
688
|
resource: {
|
|
@@ -930,24 +866,6 @@ module Constants
|
|
|
930
866
|
# Stub value for Monitored resources from Metadata Agent.
|
|
931
867
|
# Map from the local_resource_id to the retrieved monitored resource.
|
|
932
868
|
MONITORED_RESOURCE_STUBS = {
|
|
933
|
-
# Docker container stderr / stdout logs.
|
|
934
|
-
"#{DOCKER_CONTAINER_LOCAL_RESOURCE_ID_PREFIX}.#{DOCKER_CONTAINER_ID}" =>
|
|
935
|
-
{
|
|
936
|
-
'type' => DOCKER_CONSTANTS[:resource_type],
|
|
937
|
-
'labels' => {
|
|
938
|
-
'location' => ZONE,
|
|
939
|
-
'container_id' => DOCKER_CONTAINER_ID
|
|
940
|
-
}
|
|
941
|
-
}.to_json,
|
|
942
|
-
# Docker container application logs.
|
|
943
|
-
"#{DOCKER_CONTAINER_LOCAL_RESOURCE_ID_PREFIX}.#{DOCKER_CONTAINER_NAME}" =>
|
|
944
|
-
{
|
|
945
|
-
'type' => DOCKER_CONSTANTS[:resource_type],
|
|
946
|
-
'labels' => {
|
|
947
|
-
'location' => ZONE,
|
|
948
|
-
'container_id' => DOCKER_CONTAINER_ID
|
|
949
|
-
}
|
|
950
|
-
}.to_json,
|
|
951
869
|
# GKE container logs.
|
|
952
870
|
"#{CONTAINER_LOCAL_RESOURCE_ID_PREFIX}.#{CONTAINER_NAMESPACE_ID}" \
|
|
953
871
|
".#{K8S_POD_NAME}.#{K8S_CONTAINER_NAME}" =>
|
|
@@ -353,7 +353,6 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
|
353
353
|
'\bcoerce_to_utf8\b.*\bfalse\b',
|
|
354
354
|
'\bdetect_json\b.*\btrue\b',
|
|
355
355
|
'\bdetect_subservice\b.*\bfalse\b',
|
|
356
|
-
'\benable_metadata_agent\b.*\btrue\b',
|
|
357
356
|
'\benable_monitoring\b.*\btrue\b',
|
|
358
357
|
'\bhttp_request_key\b.*\btest_http_request_key\b',
|
|
359
358
|
'\binsert_id_key\b.*\btest_insert_id_key\b',
|
|
@@ -364,11 +363,9 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
|
364
363
|
'\blabels_key\b.*\btest_labels_key\b',
|
|
365
364
|
'\blabels\b.*{"labels_key"=>"labels_value"}',
|
|
366
365
|
'\blogging_api_url\b.*\bhttp://localhost:52000\b',
|
|
367
|
-
'\bmetadata_agent_url\b.*\bhttp://localhost:12345\b',
|
|
368
366
|
'\bmonitoring_type\b.*\bnot_prometheus\b',
|
|
369
367
|
'\bnon_utf8_replacement_string\b.*\bzzz\b',
|
|
370
368
|
'\boperation_key\b.*\btest_operation_key\b',
|
|
371
|
-
'\bpartial_success\b.*\bfalse\b',
|
|
372
369
|
'\bproject_id\b.*\btest-project-id-123\b',
|
|
373
370
|
'\brequire_valid_tags\b.*\btrue\b',
|
|
374
371
|
'\bsource_location_key\b.*\btest_source_location_key\b',
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluent-plugin-google-cloud
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.8.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Stackdriver Agents Team
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2020-02-
|
|
11
|
+
date: 2020-02-27 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: fluentd
|
|
@@ -142,14 +142,14 @@ dependencies:
|
|
|
142
142
|
requirements:
|
|
143
143
|
- - '='
|
|
144
144
|
- !ruby/object:Gem::Version
|
|
145
|
-
version: 0.3.
|
|
145
|
+
version: 0.3.2
|
|
146
146
|
type: :runtime
|
|
147
147
|
prerelease: false
|
|
148
148
|
version_requirements: !ruby/object:Gem::Requirement
|
|
149
149
|
requirements:
|
|
150
150
|
- - '='
|
|
151
151
|
- !ruby/object:Gem::Version
|
|
152
|
-
version: 0.3.
|
|
152
|
+
version: 0.3.2
|
|
153
153
|
- !ruby/object:Gem::Dependency
|
|
154
154
|
name: mocha
|
|
155
155
|
requirement: !ruby/object:Gem::Requirement
|