fluent-plugin-google-cloud 0.7.30 → 0.8.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +13 -13
- data/fluent-plugin-google-cloud.gemspec +2 -2
- data/lib/fluent/plugin/out_google_cloud.rb +36 -123
- data/lib/fluent/plugin/statusz.rb +0 -3
- data/test/plugin/base_test.rb +17 -351
- data/test/plugin/constants.rb +2 -84
- data/test/plugin/test_out_google_cloud.rb +0 -3
- metadata +4 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: db48fa801fa732a1558906459e4bebfd2c7eb66b35aa471e01629f8d2a95989c
|
4
|
+
data.tar.gz: 8dd3e21883be568e0a8eae1ae5d79cda7693e2bca747df493b4a1d77ac09a5e9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7f1d13d9c74cd06b3be961724b5cd59e4bd28870ae31e7b94166928598b44b2ec951da5850a92accfe2ea244553f6b4fd3093bd0e66c8bbb8ff261eac17a2b14
|
7
|
+
data.tar.gz: a99993cd8944ebb0d338d8c285550c78338b25fff8e6be0dba7e194eb7a1607f2924869c0df6b6a894d28dbb4a582a07a344ffa45018440b010f564f74991685
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
fluent-plugin-google-cloud (0.
|
4
|
+
fluent-plugin-google-cloud (0.8.0)
|
5
5
|
fluentd (= 1.6.3)
|
6
6
|
google-api-client (= 0.30.8)
|
7
7
|
google-cloud-logging (= 1.6.6)
|
@@ -11,7 +11,7 @@ PATH
|
|
11
11
|
grpc (= 1.22.0)
|
12
12
|
json (= 2.2.0)
|
13
13
|
opencensus (= 0.5.0)
|
14
|
-
opencensus-stackdriver (= 0.3.
|
14
|
+
opencensus-stackdriver (= 0.3.2)
|
15
15
|
|
16
16
|
GEM
|
17
17
|
remote: https://rubygems.org/
|
@@ -19,7 +19,7 @@ GEM
|
|
19
19
|
addressable (2.7.0)
|
20
20
|
public_suffix (>= 2.0.2, < 5.0)
|
21
21
|
ast (2.4.0)
|
22
|
-
concurrent-ruby (1.1.
|
22
|
+
concurrent-ruby (1.1.6)
|
23
23
|
cool.io (1.6.0)
|
24
24
|
coveralls (0.8.23)
|
25
25
|
json (>= 1.8, < 3)
|
@@ -66,8 +66,8 @@ GEM
|
|
66
66
|
google-gax (~> 1.7)
|
67
67
|
googleapis-common-protos-types (>= 1.0.2)
|
68
68
|
stackdriver-core (~> 1.3)
|
69
|
-
google-cloud-monitoring (0.
|
70
|
-
google-gax (~> 1.
|
69
|
+
google-cloud-monitoring (0.32.0)
|
70
|
+
google-gax (~> 1.7)
|
71
71
|
googleapis-common-protos-types (>= 1.0.2)
|
72
72
|
google-cloud-trace (0.35.0)
|
73
73
|
concurrent-ruby (~> 1.1)
|
@@ -111,15 +111,15 @@ GEM
|
|
111
111
|
multi_json (1.14.1)
|
112
112
|
multipart-post (2.1.1)
|
113
113
|
opencensus (0.5.0)
|
114
|
-
opencensus-stackdriver (0.3.
|
114
|
+
opencensus-stackdriver (0.3.2)
|
115
115
|
concurrent-ruby (~> 1.0)
|
116
|
-
google-cloud-monitoring (~> 0.
|
117
|
-
google-cloud-trace (~> 0.
|
116
|
+
google-cloud-monitoring (~> 0.32)
|
117
|
+
google-cloud-trace (~> 0.35)
|
118
118
|
opencensus (~> 0.5)
|
119
119
|
os (1.0.1)
|
120
|
-
parser (2.7.0.
|
120
|
+
parser (2.7.0.3)
|
121
121
|
ast (~> 2.4.0)
|
122
|
-
power_assert (1.1.
|
122
|
+
power_assert (1.1.6)
|
123
123
|
powerpack (0.1.2)
|
124
124
|
prometheus-client (0.9.0)
|
125
125
|
quantile (~> 0.2.1)
|
@@ -145,9 +145,9 @@ GEM
|
|
145
145
|
serverengine (2.2.1)
|
146
146
|
sigdump (~> 0.2.2)
|
147
147
|
sigdump (0.2.4)
|
148
|
-
signet (0.
|
148
|
+
signet (0.13.0)
|
149
149
|
addressable (~> 2.3)
|
150
|
-
faraday (
|
150
|
+
faraday (>= 0.17.3, < 2.0)
|
151
151
|
jwt (>= 1.5, < 3.0)
|
152
152
|
multi_json (~> 1.10)
|
153
153
|
simplecov (0.16.1)
|
@@ -165,7 +165,7 @@ GEM
|
|
165
165
|
power_assert
|
166
166
|
thor (1.0.1)
|
167
167
|
thread_safe (0.3.6)
|
168
|
-
tins (1.24.
|
168
|
+
tins (1.24.1)
|
169
169
|
sync
|
170
170
|
tzinfo (1.2.6)
|
171
171
|
thread_safe (~> 0.1)
|
@@ -10,7 +10,7 @@ eos
|
|
10
10
|
gem.homepage =
|
11
11
|
'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
|
12
12
|
gem.license = 'Apache-2.0'
|
13
|
-
gem.version = '0.
|
13
|
+
gem.version = '0.8.0'
|
14
14
|
gem.authors = ['Stackdriver Agents Team']
|
15
15
|
gem.email = ['stackdriver-agents@google.com']
|
16
16
|
gem.required_ruby_version = Gem::Requirement.new('>= 2.2')
|
@@ -28,7 +28,7 @@ eos
|
|
28
28
|
gem.add_runtime_dependency 'grpc', '1.22.0'
|
29
29
|
gem.add_runtime_dependency 'json', '2.2.0'
|
30
30
|
gem.add_runtime_dependency 'opencensus', '0.5.0'
|
31
|
-
gem.add_runtime_dependency 'opencensus-stackdriver', '0.3.
|
31
|
+
gem.add_runtime_dependency 'opencensus-stackdriver', '0.3.2'
|
32
32
|
|
33
33
|
gem.add_development_dependency 'mocha', '1.9.0'
|
34
34
|
# Keep this the same as in
|
@@ -121,10 +121,6 @@ module Fluent
|
|
121
121
|
K8S_NODE_CONSTANTS = {
|
122
122
|
resource_type: 'k8s_node'
|
123
123
|
}.freeze
|
124
|
-
DOCKER_CONSTANTS = {
|
125
|
-
service: 'docker.googleapis.com',
|
126
|
-
resource_type: 'docker_container'
|
127
|
-
}.freeze
|
128
124
|
DATAFLOW_CONSTANTS = {
|
129
125
|
service: 'dataflow.googleapis.com',
|
130
126
|
resource_type: 'dataflow_step',
|
@@ -174,10 +170,6 @@ module Fluent
|
|
174
170
|
DEFAULT_SPAN_ID_KEY = 'logging.googleapis.com/spanId'.freeze
|
175
171
|
DEFAULT_TRACE_KEY = 'logging.googleapis.com/trace'.freeze
|
176
172
|
DEFAULT_TRACE_SAMPLED_KEY = 'logging.googleapis.com/trace_sampled'.freeze
|
177
|
-
|
178
|
-
DEFAULT_METADATA_AGENT_URL =
|
179
|
-
'http://local-metadata-agent.stackdriver.com:8000'.freeze
|
180
|
-
METADATA_AGENT_URL_ENV_VAR = 'STACKDRIVER_METADATA_AGENT_URL'.freeze
|
181
173
|
end
|
182
174
|
|
183
175
|
# Internal constants.
|
@@ -346,7 +338,7 @@ module Fluent
|
|
346
338
|
# running on GCE.
|
347
339
|
#
|
348
340
|
# The initial motivation for this is to separate out Kubernetes node
|
349
|
-
# component (
|
341
|
+
# component (Kubelet, etc.) logs from container logs.
|
350
342
|
config_param :detect_subservice, :bool, :default => true
|
351
343
|
# The subservice_name overrides the subservice detection, if provided.
|
352
344
|
config_param :subservice_name, :string, :default => nil
|
@@ -408,8 +400,13 @@ module Fluent
|
|
408
400
|
|
409
401
|
# Whether valid entries should be written even if some other entries fail
|
410
402
|
# due to INVALID_ARGUMENT or PERMISSION_DENIED errors when communicating to
|
411
|
-
# the Stackdriver Logging API. This is
|
412
|
-
|
403
|
+
# the Stackdriver Logging API. This flag is no longer used, and is kept for
|
404
|
+
# backwards compatibility, partial_success is enabled for all requests.
|
405
|
+
# TODO: Breaking change. Remove this flag in Logging Agent 2.0.0 release.
|
406
|
+
config_param :partial_success, :bool,
|
407
|
+
:default => true,
|
408
|
+
:skip_accessor => true,
|
409
|
+
:deprecated => 'This feature is permanently enabled'
|
413
410
|
|
414
411
|
# Whether to allow non-UTF-8 characters in user logs. If set to true, any
|
415
412
|
# non-UTF-8 character would be replaced by the string specified by
|
@@ -449,15 +446,21 @@ module Fluent
|
|
449
446
|
config_param :monitoring_type, :string,
|
450
447
|
:default => Monitoring::PrometheusMonitoringRegistry.name
|
451
448
|
|
452
|
-
# Whether to call metadata agent to retrieve monitored resource.
|
453
|
-
|
454
|
-
|
455
|
-
|
456
|
-
|
457
|
-
|
458
|
-
|
459
|
-
|
460
|
-
|
449
|
+
# Whether to call metadata agent to retrieve monitored resource. This flag
|
450
|
+
# is kept for backwards compatibility, and is no longer used.
|
451
|
+
# TODO: Breaking change. Remove this flag in Logging Agent 2.0.0 release.
|
452
|
+
config_param :enable_metadata_agent, :bool,
|
453
|
+
:default => false,
|
454
|
+
:skip_accessor => true,
|
455
|
+
:deprecated => 'This feature is permanently disabled'
|
456
|
+
|
457
|
+
# The URL of the Metadata Agent. This flag is kept for backwards
|
458
|
+
# compatibility, and is no longer used.
|
459
|
+
# TODO: Breaking change. Remove this flag in Logging Agent 2.0.0 release.
|
460
|
+
config_param :metadata_agent_url, :string,
|
461
|
+
:default => nil,
|
462
|
+
:skip_accessor => true,
|
463
|
+
:deprecated => 'This feature is permanently disabled'
|
461
464
|
|
462
465
|
# Whether to split log entries with different log tags into different
|
463
466
|
# requests when talking to Stackdriver Logging API.
|
@@ -520,20 +523,6 @@ module Fluent
|
|
520
523
|
' ignored.'
|
521
524
|
end
|
522
525
|
|
523
|
-
# 1. If @metadata_agent_url is customized (aka not nil), use that.
|
524
|
-
# 2. Otherwise check the presence of the environment variable
|
525
|
-
# STACKDRIVER_METADATA_AGENT_URL and use that if set.
|
526
|
-
# 3. Fall back to the default if neither is set.
|
527
|
-
if @enable_metadata_agent
|
528
|
-
# Convert to string to capture empty string.
|
529
|
-
@metadata_agent_url ||=
|
530
|
-
if ENV[METADATA_AGENT_URL_ENV_VAR].to_s.empty?
|
531
|
-
DEFAULT_METADATA_AGENT_URL
|
532
|
-
else
|
533
|
-
ENV[METADATA_AGENT_URL_ENV_VAR]
|
534
|
-
end
|
535
|
-
end
|
536
|
-
|
537
526
|
# Alert on old authentication configuration.
|
538
527
|
unless @auth_method.nil? && @private_key_email.nil? &&
|
539
528
|
@private_key_path.nil? && @private_key_passphrase.nil?
|
@@ -903,7 +892,7 @@ module Fluent
|
|
903
892
|
labels: labels.map do |k, v|
|
904
893
|
[k.encode('utf-8'), convert_to_utf8(v)]
|
905
894
|
end.to_h,
|
906
|
-
partial_success:
|
895
|
+
partial_success: true
|
907
896
|
)
|
908
897
|
increment_successful_requests_count
|
909
898
|
increment_ingested_entries_count(entries_count)
|
@@ -964,7 +953,7 @@ module Fluent
|
|
964
953
|
@log.warn "Dropping #{entries_count} log message(s)",
|
965
954
|
error: error.to_s, error_code: error.code.to_s
|
966
955
|
|
967
|
-
#
|
956
|
+
# As partial_success is enabled, valid entries should have been
|
968
957
|
# written even if some other entries fail due to InvalidArgument or
|
969
958
|
# PermissionDenied errors. Only invalid entries will be dropped.
|
970
959
|
when \
|
@@ -1029,7 +1018,7 @@ module Fluent
|
|
1029
1018
|
log_name: log_name,
|
1030
1019
|
resource: resource,
|
1031
1020
|
labels: labels,
|
1032
|
-
partial_success:
|
1021
|
+
partial_success: true
|
1033
1022
|
),
|
1034
1023
|
options: { api_format_version: '2' }
|
1035
1024
|
)
|
@@ -1431,8 +1420,10 @@ module Fluent
|
|
1431
1420
|
# Metadata Agent with this key.
|
1432
1421
|
#
|
1433
1422
|
# Examples:
|
1434
|
-
#
|
1435
|
-
# "k8s_pod.<namespace_name>.<pod_name>"
|
1423
|
+
# // GKE Pod.
|
1424
|
+
# "k8s_pod.<namespace_name>.<pod_name>"
|
1425
|
+
# // GKE container.
|
1426
|
+
# "k8s_container.<namespace_name>.<pod_name>.<container_name>"
|
1436
1427
|
if local_resource_id
|
1437
1428
|
converted_resource = monitored_resource_from_local_resource_id(
|
1438
1429
|
local_resource_id)
|
@@ -1466,12 +1457,6 @@ module Fluent
|
|
1466
1457
|
.map { |l| [l, "#{GKE_CONSTANTS[:service]}/#{l}"] }.to_h))
|
1467
1458
|
end
|
1468
1459
|
|
1469
|
-
# Docker container.
|
1470
|
-
# TODO(qingling128): Remove this logic once the resource is retrieved at a
|
1471
|
-
# proper time (b/65175256).
|
1472
|
-
when DOCKER_CONSTANTS[:resource_type]
|
1473
|
-
common_labels.delete("#{COMPUTE_CONSTANTS[:service]}/resource_name")
|
1474
|
-
|
1475
1460
|
# TODO(qingling128): Temporary fallback for metadata agent restarts.
|
1476
1461
|
# K8s resources.
|
1477
1462
|
when K8S_CONTAINER_CONSTANTS[:resource_type],
|
@@ -1500,32 +1485,6 @@ module Fluent
|
|
1500
1485
|
[resource, common_labels]
|
1501
1486
|
end
|
1502
1487
|
|
1503
|
-
# Take a locally unique resource id and convert it to the globally unique
|
1504
|
-
# monitored resource.
|
1505
|
-
def monitored_resource_from_local_resource_id(local_resource_id)
|
1506
|
-
return unless local_resource_id
|
1507
|
-
if @enable_metadata_agent
|
1508
|
-
@log.debug 'Calling metadata agent with local_resource_id: ' \
|
1509
|
-
"#{local_resource_id}."
|
1510
|
-
resource = query_metadata_agent_for_monitored_resource(
|
1511
|
-
local_resource_id)
|
1512
|
-
@log.debug 'Retrieved monitored resource from metadata agent: ' \
|
1513
|
-
"#{resource.inspect}."
|
1514
|
-
if resource
|
1515
|
-
# TODO(qingling128): Fix this temporary renaming from 'gke_container'
|
1516
|
-
# to 'container'.
|
1517
|
-
resource.type = 'container' if resource.type == 'gke_container'
|
1518
|
-
return resource
|
1519
|
-
end
|
1520
|
-
@log.debug('Failed to retrieve monitored resource from Metadata' \
|
1521
|
-
" Agent with local_resource_id #{local_resource_id}.")
|
1522
|
-
end
|
1523
|
-
# Fall back to constructing monitored resource locally.
|
1524
|
-
# TODO(qingling128): This entire else clause is temporary until we
|
1525
|
-
# implement buffering and caching.
|
1526
|
-
construct_k8s_resource_locally(local_resource_id)
|
1527
|
-
end
|
1528
|
-
|
1529
1488
|
# Extract entry level monitored resource and common labels that should be
|
1530
1489
|
# applied to individual entries.
|
1531
1490
|
def determine_entry_level_monitored_resource_and_labels(
|
@@ -1589,49 +1548,6 @@ module Fluent
|
|
1589
1548
|
[resource, common_labels]
|
1590
1549
|
end
|
1591
1550
|
|
1592
|
-
# Call Metadata Agent to get monitored resource information and parse
|
1593
|
-
# response to Google::Api::MonitoredResource.
|
1594
|
-
def query_metadata_agent_for_monitored_resource(local_resource_id)
|
1595
|
-
response = query_metadata_agent("monitoredResource/#{local_resource_id}")
|
1596
|
-
return nil if response.nil?
|
1597
|
-
begin
|
1598
|
-
resource = Google::Api::MonitoredResource.decode_json(response.to_json)
|
1599
|
-
rescue Google::Protobuf::ParseError, ArgumentError => e
|
1600
|
-
@log.error 'Error parsing monitored resource from Metadata Agent. ' \
|
1601
|
-
"response: #{response.inspect}", error: e
|
1602
|
-
return nil
|
1603
|
-
end
|
1604
|
-
|
1605
|
-
# TODO(qingling128): Use Google::Api::MonitoredResource directly after we
|
1606
|
-
# upgrade gRPC version to include the fix for the protobuf map
|
1607
|
-
# corruption issue.
|
1608
|
-
Google::Apis::LoggingV2::MonitoredResource.new(
|
1609
|
-
type: resource.type,
|
1610
|
-
labels: resource.labels.to_h
|
1611
|
-
)
|
1612
|
-
end
|
1613
|
-
|
1614
|
-
# Issue a request to the Metadata Agent's local API and parse the response
|
1615
|
-
# to JSON. Return nil in case of failure.
|
1616
|
-
def query_metadata_agent(path)
|
1617
|
-
url = "#{@metadata_agent_url}/#{path}"
|
1618
|
-
@log.debug("Calling Metadata Agent: #{url}")
|
1619
|
-
open(url) do |f|
|
1620
|
-
response = f.read
|
1621
|
-
parsed_hash = parse_json_or_nil(response)
|
1622
|
-
if parsed_hash.nil?
|
1623
|
-
@log.error 'Response from Metadata Agent is not in valid json ' \
|
1624
|
-
"format: '#{response.inspect}'."
|
1625
|
-
return nil
|
1626
|
-
end
|
1627
|
-
@log.debug "Response from Metadata Agent: #{parsed_hash}"
|
1628
|
-
return parsed_hash
|
1629
|
-
end
|
1630
|
-
rescue StandardError => e
|
1631
|
-
@log.error "Error calling Metadata Agent at #{url}.", error: e
|
1632
|
-
nil
|
1633
|
-
end
|
1634
|
-
|
1635
1551
|
# TODO: This functionality should eventually be available in another
|
1636
1552
|
# library, but implement it ourselves for now.
|
1637
1553
|
module CredentialsInfo
|
@@ -2079,9 +1995,7 @@ module Fluent
|
|
2079
1995
|
# 2. The only remaining key is 'message'
|
2080
1996
|
if is_json
|
2081
1997
|
json_payload = record
|
2082
|
-
elsif
|
2083
|
-
DOCKER_CONSTANTS[:resource_type]].include?(resource_type) &&
|
2084
|
-
record.key?('log')
|
1998
|
+
elsif GKE_CONSTANTS[:resource_type] == resource_type && record.key?('log')
|
2085
1999
|
text_payload = record['log']
|
2086
2000
|
elsif record.size == 1 && record.key?('message')
|
2087
2001
|
text_payload = record['message']
|
@@ -2208,7 +2122,7 @@ module Fluent
|
|
2208
2122
|
end
|
2209
2123
|
|
2210
2124
|
# Extract a map of error details from a potentially partially successful
|
2211
|
-
# REST request.
|
2125
|
+
# REST request.
|
2212
2126
|
#
|
2213
2127
|
# The keys in this map are [error_code, error_message] pairs, and the values
|
2214
2128
|
# are a list of stringified indexes of log entries that failed due to this
|
@@ -2283,7 +2197,6 @@ module Fluent
|
|
2283
2197
|
# [3, 'Log name contains illegal character :']: ['1', '3']
|
2284
2198
|
# }
|
2285
2199
|
def construct_error_details_map(error)
|
2286
|
-
return {} unless @partial_success
|
2287
2200
|
error_details_map = Hash.new { |h, k| h[k] = [] }
|
2288
2201
|
|
2289
2202
|
error_details = ensure_array(
|
@@ -2312,7 +2225,7 @@ module Fluent
|
|
2312
2225
|
end
|
2313
2226
|
|
2314
2227
|
# Extract a map of error details from a potentially partially successful
|
2315
|
-
# gRPC request.
|
2228
|
+
# gRPC request.
|
2316
2229
|
#
|
2317
2230
|
# The keys in this map are [error_code, error_message] pairs, and the values
|
2318
2231
|
# are a list of indexes of log entries that failed due to this error.
|
@@ -2352,7 +2265,6 @@ module Fluent
|
|
2352
2265
|
# [3, 'Log name contains illegal character :']: [1, 3]
|
2353
2266
|
# }
|
2354
2267
|
def construct_error_details_map_grpc(gax_error)
|
2355
|
-
return {} unless @partial_success
|
2356
2268
|
error_details_map = Hash.new { |h, k| h[k] = [] }
|
2357
2269
|
error_details = ensure_array(gax_error.status_details)
|
2358
2270
|
raise JSON::ParserError, 'The error details are empty.' if
|
@@ -2372,8 +2284,9 @@ module Fluent
|
|
2372
2284
|
{}
|
2373
2285
|
end
|
2374
2286
|
|
2375
|
-
#
|
2376
|
-
|
2287
|
+
# Take a locally unique resource id and convert it to the globally unique
|
2288
|
+
# monitored resource.
|
2289
|
+
def monitored_resource_from_local_resource_id(local_resource_id)
|
2377
2290
|
return unless
|
2378
2291
|
/^
|
2379
2292
|
(?<resource_type>k8s_container)
|
@@ -40,7 +40,6 @@ CONFIG_KEYS = %w(
|
|
40
40
|
coerce_to_utf8
|
41
41
|
detect_json
|
42
42
|
detect_subservice
|
43
|
-
enable_metadata_agent
|
44
43
|
enable_monitoring
|
45
44
|
http_request_key
|
46
45
|
insert_id_key
|
@@ -51,11 +50,9 @@ CONFIG_KEYS = %w(
|
|
51
50
|
labels_key
|
52
51
|
labels
|
53
52
|
logging_api_url
|
54
|
-
metadata_agent_url
|
55
53
|
monitoring_type
|
56
54
|
non_utf8_replacement_string
|
57
55
|
operation_key
|
58
|
-
partial_success
|
59
56
|
project_id
|
60
57
|
require_valid_tags
|
61
58
|
source_location_key
|
data/test/plugin/base_test.rb
CHANGED
@@ -127,42 +127,6 @@ module BaseTest
|
|
127
127
|
end
|
128
128
|
end
|
129
129
|
|
130
|
-
def test_configure_partial_success
|
131
|
-
setup_gce_metadata_stubs
|
132
|
-
{
|
133
|
-
APPLICATION_DEFAULT_CONFIG => true,
|
134
|
-
PARTIAL_SUCCESS_DISABLED_CONFIG => false
|
135
|
-
}.each do |(config, partial_success)|
|
136
|
-
d = create_driver(config)
|
137
|
-
assert_equal partial_success,
|
138
|
-
d.instance.instance_variable_get(:@partial_success)
|
139
|
-
end
|
140
|
-
end
|
141
|
-
|
142
|
-
def test_metadata_agent_url_customization
|
143
|
-
[
|
144
|
-
# If @metadata_agent_url is set, use that even if the environment
|
145
|
-
# variable is set.
|
146
|
-
[CUSTOM_METADATA_AGENT_URL_CONFIG, true, CUSTOM_METADATA_AGENT_URL],
|
147
|
-
# If @metadata_agent_url is set and the environment variable is
|
148
|
-
# not set, use @metadata_agent_url.
|
149
|
-
[CUSTOM_METADATA_AGENT_URL_CONFIG, false, CUSTOM_METADATA_AGENT_URL],
|
150
|
-
# If @metadata_agent_url is not set and the environment variable is set,
|
151
|
-
# use the env.
|
152
|
-
[APPLICATION_DEFAULT_CONFIG, true, METADATA_AGENT_URL_FROM_ENV],
|
153
|
-
# If @metadata_agent_url is not set and the environment variable is
|
154
|
-
# not set, fall back to the default.
|
155
|
-
[APPLICATION_DEFAULT_CONFIG, false, DEFAULT_METADATA_AGENT_URL]
|
156
|
-
].each do |(config, url_from_env, expected_url)|
|
157
|
-
ENV[METADATA_AGENT_URL_ENV_VAR] = METADATA_AGENT_URL_FROM_ENV if
|
158
|
-
url_from_env
|
159
|
-
setup_gce_metadata_stubs
|
160
|
-
d = create_driver(ENABLE_METADATA_AGENT_CONFIG + config)
|
161
|
-
assert_equal expected_url, d.instance.metadata_agent_url
|
162
|
-
ENV.delete(METADATA_AGENT_URL_ENV_VAR)
|
163
|
-
end
|
164
|
-
end
|
165
|
-
|
166
130
|
def test_configure_ignores_unknown_monitoring_type
|
167
131
|
# Verify that driver creation succeeds when monitoring type is not
|
168
132
|
# "prometheus" (in which case, we simply don't record metrics),
|
@@ -1758,176 +1722,48 @@ module BaseTest
|
|
1758
1722
|
end
|
1759
1723
|
end
|
1760
1724
|
|
1761
|
-
#
|
1762
|
-
|
1763
|
-
# Test enable_metadata_agent not set or set to false.
|
1764
|
-
def test_configure_enable_metadata_agent_default_and_false
|
1765
|
-
setup_gce_metadata_stubs
|
1766
|
-
[create_driver, create_driver(DISABLE_METADATA_AGENT_CONFIG)].each do |d|
|
1767
|
-
assert_false d.instance.instance_variable_get(:@enable_metadata_agent)
|
1768
|
-
end
|
1769
|
-
end
|
1770
|
-
|
1771
|
-
# Test enable_metadata_agent set to true.
|
1772
|
-
def test_configure_enable_metadata_agent_true
|
1773
|
-
new_stub_context do
|
1774
|
-
setup_gce_metadata_stubs
|
1775
|
-
setup_metadata_agent_stubs
|
1776
|
-
d = create_driver(ENABLE_METADATA_AGENT_CONFIG)
|
1777
|
-
assert_true d.instance.instance_variable_get(:@enable_metadata_agent)
|
1778
|
-
end
|
1779
|
-
end
|
1780
|
-
|
1781
|
-
# Docker Container.
|
1782
|
-
|
1783
|
-
# Test textPayload logs from Docker container stdout / stderr.
|
1784
|
-
def test_docker_container_stdout_stderr_logs_text_payload
|
1785
|
-
[1, 2, 3, 5, 11, 50].each do |n|
|
1786
|
-
new_stub_context do
|
1787
|
-
setup_gce_metadata_stubs
|
1788
|
-
setup_metadata_agent_stubs
|
1789
|
-
setup_logging_stubs do
|
1790
|
-
d = create_driver(DOCKER_CONTAINER_CONFIG)
|
1791
|
-
n.times do |i|
|
1792
|
-
d.emit(docker_container_stdout_stderr_log_entry(log_entry(i)))
|
1793
|
-
end
|
1794
|
-
d.run
|
1795
|
-
end
|
1796
|
-
verify_log_entries(n, DOCKER_CONTAINER_PARAMS)
|
1797
|
-
assert_requested_metadata_agent_stub("container.#{DOCKER_CONTAINER_ID}")
|
1798
|
-
end
|
1799
|
-
end
|
1800
|
-
end
|
1801
|
-
|
1802
|
-
# Test jsonPayload logs from Docker container stdout / stderr.
|
1803
|
-
def test_docker_container_stdout_stderr_logs_json_payload
|
1804
|
-
[1, 2, 3, 5, 11, 50].each do |n|
|
1805
|
-
new_stub_context do
|
1806
|
-
setup_gce_metadata_stubs
|
1807
|
-
setup_metadata_agent_stubs
|
1808
|
-
setup_logging_stubs do
|
1809
|
-
d = create_driver(DOCKER_CONTAINER_CONFIG)
|
1810
|
-
n.times do
|
1811
|
-
d.emit(docker_container_stdout_stderr_log_entry(
|
1812
|
-
'{"msg": "test log entry ' \
|
1813
|
-
"#{n}" \
|
1814
|
-
'", "tag2": "test", "data": ' \
|
1815
|
-
'5000, "severity": "WARNING"}'))
|
1816
|
-
end
|
1817
|
-
d.run
|
1818
|
-
end
|
1819
|
-
verify_log_entries(n, DOCKER_CONTAINER_PARAMS, 'jsonPayload') do |entry|
|
1820
|
-
fields = entry['jsonPayload']
|
1821
|
-
assert_equal 3, fields.size, entry
|
1822
|
-
assert_equal "test log entry #{n}", fields['msg'], entry
|
1823
|
-
assert_equal 'test', fields['tag2'], entry
|
1824
|
-
assert_equal 5000, fields['data'], entry
|
1825
|
-
end
|
1826
|
-
assert_requested_metadata_agent_stub("container.#{DOCKER_CONTAINER_ID}")
|
1827
|
-
end
|
1828
|
-
end
|
1829
|
-
end
|
1830
|
-
|
1831
|
-
# Test logs from applications running in Docker containers. These logs have
|
1832
|
-
# the label "logging.googleapis.com/local_resource_id" set in the format of
|
1833
|
-
# "container.<container_name>".
|
1834
|
-
def test_docker_container_application_logs
|
1835
|
-
new_stub_context do
|
1836
|
-
setup_gce_metadata_stubs
|
1837
|
-
setup_metadata_agent_stubs
|
1838
|
-
setup_logging_stubs do
|
1839
|
-
# Metadata Agent is not enabled. Will call Docker Remote API for
|
1840
|
-
# container info.
|
1841
|
-
d = create_driver(ENABLE_METADATA_AGENT_CONFIG)
|
1842
|
-
d.emit(docker_container_application_log_entry(log_entry(0)))
|
1843
|
-
d.run
|
1844
|
-
end
|
1845
|
-
verify_log_entries(1, DOCKER_CONTAINER_PARAMS_NO_STREAM)
|
1846
|
-
assert_requested_metadata_agent_stub(
|
1847
|
-
"#{DOCKER_CONTAINER_LOCAL_RESOURCE_ID_PREFIX}.#{DOCKER_CONTAINER_NAME}")
|
1848
|
-
end
|
1849
|
-
end
|
1850
|
-
|
1851
|
-
# Test k8s_container monitored resource including the fallback when Metadata
|
1852
|
-
# Agent restarts.
|
1853
|
-
def test_k8s_container_monitored_resource_fallback
|
1725
|
+
# Test k8s_container monitored resource.
|
1726
|
+
def test_k8s_container_monitored_resource
|
1854
1727
|
[
|
1855
|
-
# When enable_metadata_agent is false.
|
1856
1728
|
{
|
1857
1729
|
config: APPLICATION_DEFAULT_CONFIG,
|
1858
|
-
setup_metadata_agent_stub: false,
|
1859
1730
|
setup_k8s_stub: false,
|
1860
1731
|
log_entry: k8s_container_log_entry(log_entry(0)),
|
1861
1732
|
expected_params: K8S_CONTAINER_PARAMS_FROM_FALLBACK
|
1862
1733
|
},
|
1863
1734
|
{
|
1864
1735
|
config: APPLICATION_DEFAULT_CONFIG,
|
1865
|
-
setup_metadata_agent_stub: true,
|
1866
1736
|
setup_k8s_stub: false,
|
1867
1737
|
log_entry: k8s_container_log_entry(log_entry(0)),
|
1868
1738
|
expected_params: K8S_CONTAINER_PARAMS_FROM_FALLBACK
|
1869
1739
|
},
|
1870
1740
|
{
|
1871
1741
|
config: APPLICATION_DEFAULT_CONFIG,
|
1872
|
-
setup_metadata_agent_stub: true,
|
1873
1742
|
setup_k8s_stub: true,
|
1874
1743
|
log_entry: k8s_container_log_entry(log_entry(0)),
|
1875
1744
|
expected_params: K8S_CONTAINER_PARAMS_FROM_LOCAL
|
1876
1745
|
},
|
1877
1746
|
{
|
1878
1747
|
config: APPLICATION_DEFAULT_CONFIG,
|
1879
|
-
setup_metadata_agent_stub: false,
|
1880
1748
|
setup_k8s_stub: true,
|
1881
1749
|
log_entry: k8s_container_log_entry(log_entry(0)),
|
1882
1750
|
expected_params: K8S_CONTAINER_PARAMS_FROM_LOCAL
|
1883
1751
|
},
|
1884
|
-
# When enable_metadata_agent is true.
|
1885
1752
|
{
|
1886
|
-
config:
|
1887
|
-
setup_metadata_agent_stub: false,
|
1888
|
-
setup_k8s_stub: false,
|
1889
|
-
log_entry: k8s_container_log_entry(log_entry(0)),
|
1890
|
-
expected_params: K8S_CONTAINER_PARAMS_FROM_FALLBACK
|
1891
|
-
},
|
1892
|
-
{
|
1893
|
-
config: ENABLE_METADATA_AGENT_CONFIG,
|
1894
|
-
setup_metadata_agent_stub: false,
|
1895
|
-
setup_k8s_stub: true,
|
1896
|
-
log_entry: k8s_container_log_entry(log_entry(0)),
|
1897
|
-
expected_params: K8S_CONTAINER_PARAMS_FROM_LOCAL
|
1898
|
-
},
|
1899
|
-
{
|
1900
|
-
config: CUSTOM_K8S_ENABLE_METADATA_AGENT_CONFIG,
|
1901
|
-
setup_metadata_agent_stub: false,
|
1753
|
+
config: CUSTOM_K8S_CONFIG,
|
1902
1754
|
setup_k8s_stub: false,
|
1903
1755
|
log_entry: k8s_container_log_entry(log_entry(0)),
|
1904
1756
|
expected_params: K8S_CONTAINER_PARAMS_CUSTOM
|
1905
1757
|
},
|
1906
1758
|
{
|
1907
|
-
config:
|
1908
|
-
setup_metadata_agent_stub: true,
|
1909
|
-
setup_k8s_stub: true,
|
1910
|
-
log_entry: k8s_container_log_entry(log_entry(0)),
|
1911
|
-
expected_params: K8S_CONTAINER_PARAMS
|
1912
|
-
},
|
1913
|
-
{
|
1914
|
-
config: ENABLE_METADATA_AGENT_CONFIG,
|
1915
|
-
setup_metadata_agent_stub: true,
|
1916
|
-
setup_k8s_stub: false,
|
1917
|
-
log_entry: k8s_container_log_entry(log_entry(0)),
|
1918
|
-
expected_params: K8S_CONTAINER_PARAMS
|
1919
|
-
},
|
1920
|
-
{
|
1921
|
-
config: ENABLE_METADATA_AGENT_CONFIG,
|
1922
|
-
setup_metadata_agent_stub: true,
|
1759
|
+
config: EMPTY_K8S_CONFIG,
|
1923
1760
|
setup_k8s_stub: true,
|
1924
1761
|
log_entry: k8s_container_log_entry(log_entry(0)),
|
1925
|
-
expected_params:
|
1762
|
+
expected_params: K8S_CONTAINER_PARAMS_FROM_LOCAL
|
1926
1763
|
}
|
1927
1764
|
].each do |test_params|
|
1928
1765
|
new_stub_context do
|
1929
1766
|
setup_gce_metadata_stubs
|
1930
|
-
setup_metadata_agent_stubs(test_params[:setup_metadata_agent_stub])
|
1931
1767
|
setup_k8s_metadata_stubs(test_params[:setup_k8s_stub])
|
1932
1768
|
setup_logging_stubs do
|
1933
1769
|
d = create_driver(test_params[:config], CONTAINER_TAG)
|
@@ -1949,16 +1785,14 @@ module BaseTest
|
|
1949
1785
|
[
|
1950
1786
|
# When local_resource_id is not present or does not match k8s regexes.
|
1951
1787
|
{
|
1952
|
-
config:
|
1953
|
-
setup_metadata_agent_stub: true,
|
1788
|
+
config: APPLICATION_DEFAULT_CONFIG,
|
1954
1789
|
setup_k8s_stub: true,
|
1955
1790
|
log_entry: k8s_container_log_entry(
|
1956
1791
|
log_entry(0)).reject { |k, _| k == LOCAL_RESOURCE_ID_KEY },
|
1957
1792
|
expected_params: CONTAINER_FROM_TAG_PARAMS
|
1958
1793
|
},
|
1959
1794
|
{
|
1960
|
-
config:
|
1961
|
-
setup_metadata_agent_stub: true,
|
1795
|
+
config: APPLICATION_DEFAULT_CONFIG,
|
1962
1796
|
setup_k8s_stub: true,
|
1963
1797
|
log_entry: k8s_container_log_entry(
|
1964
1798
|
log_entry(0),
|
@@ -1968,7 +1802,6 @@ module BaseTest
|
|
1968
1802
|
].each do |test_params|
|
1969
1803
|
new_stub_context do
|
1970
1804
|
setup_gce_metadata_stubs
|
1971
|
-
setup_metadata_agent_stubs(test_params[:setup_metadata_agent_stub])
|
1972
1805
|
setup_k8s_metadata_stubs(test_params[:setup_k8s_stub])
|
1973
1806
|
setup_logging_stubs do
|
1974
1807
|
d = create_driver(test_params[:config], CONTAINER_TAG)
|
@@ -1982,49 +1815,30 @@ module BaseTest
|
|
1982
1815
|
end
|
1983
1816
|
end
|
1984
1817
|
|
1985
|
-
# Test k8s_pod monitored resource
|
1986
|
-
|
1987
|
-
def test_k8s_pod_monitored_resource_fallback
|
1818
|
+
# Test k8s_pod monitored resource.
|
1819
|
+
def test_k8s_pod_monitored_resource
|
1988
1820
|
[
|
1989
1821
|
{
|
1990
1822
|
config: APPLICATION_DEFAULT_CONFIG,
|
1991
|
-
setup_metadata_agent_stub: true,
|
1992
1823
|
setup_k8s_stub: true,
|
1993
1824
|
log_entry: k8s_pod_log_entry(log_entry(0)),
|
1994
1825
|
expected_params: K8S_POD_PARAMS_FROM_LOCAL
|
1995
1826
|
},
|
1996
1827
|
{
|
1997
|
-
config:
|
1998
|
-
setup_metadata_agent_stub: false,
|
1999
|
-
setup_k8s_stub: true,
|
2000
|
-
log_entry: k8s_pod_log_entry(log_entry(0)),
|
2001
|
-
expected_params: K8S_POD_PARAMS_FROM_LOCAL
|
2002
|
-
},
|
2003
|
-
{
|
2004
|
-
config: CUSTOM_K8S_ENABLE_METADATA_AGENT_CONFIG,
|
2005
|
-
setup_metadata_agent_stub: false,
|
1828
|
+
config: CUSTOM_K8S_CONFIG,
|
2006
1829
|
setup_k8s_stub: false,
|
2007
1830
|
log_entry: k8s_pod_log_entry(log_entry(0)),
|
2008
1831
|
expected_params: K8S_POD_PARAMS_CUSTOM
|
2009
1832
|
},
|
2010
1833
|
{
|
2011
|
-
config:
|
2012
|
-
setup_metadata_agent_stub: true,
|
2013
|
-
setup_k8s_stub: true,
|
2014
|
-
log_entry: k8s_pod_log_entry(log_entry(0)),
|
2015
|
-
expected_params: K8S_POD_PARAMS
|
2016
|
-
},
|
2017
|
-
{
|
2018
|
-
config: ENABLE_METADATA_AGENT_CONFIG,
|
2019
|
-
setup_metadata_agent_stub: true,
|
1834
|
+
config: EMPTY_K8S_CONFIG,
|
2020
1835
|
setup_k8s_stub: true,
|
2021
1836
|
log_entry: k8s_pod_log_entry(log_entry(0)),
|
2022
|
-
expected_params:
|
1837
|
+
expected_params: K8S_POD_PARAMS_FROM_LOCAL
|
2023
1838
|
}
|
2024
1839
|
].each do |test_params|
|
2025
1840
|
new_stub_context do
|
2026
1841
|
setup_gce_metadata_stubs
|
2027
|
-
setup_metadata_agent_stubs(test_params[:setup_metadata_agent_stub])
|
2028
1842
|
setup_k8s_metadata_stubs(test_params[:setup_k8s_stub])
|
2029
1843
|
setup_logging_stubs do
|
2030
1844
|
d = create_driver(test_params[:config])
|
@@ -2042,49 +1856,30 @@ module BaseTest
|
|
2042
1856
|
end
|
2043
1857
|
end
|
2044
1858
|
|
2045
|
-
# Test k8s_node monitored resource
|
2046
|
-
|
2047
|
-
def test_k8s_node_monitored_resource_fallback
|
1859
|
+
# Test k8s_node monitored resource.
|
1860
|
+
def test_k8s_node_monitored_resource
|
2048
1861
|
[
|
2049
1862
|
{
|
2050
1863
|
config: APPLICATION_DEFAULT_CONFIG,
|
2051
|
-
setup_metadata_agent_stub: true,
|
2052
1864
|
setup_k8s_stub: true,
|
2053
1865
|
log_entry: k8s_node_log_entry(log_entry(0)),
|
2054
1866
|
expected_params: K8S_NODE_PARAMS_FROM_LOCAL
|
2055
1867
|
},
|
2056
1868
|
{
|
2057
|
-
config:
|
2058
|
-
setup_metadata_agent_stub: false,
|
2059
|
-
setup_k8s_stub: true,
|
2060
|
-
log_entry: k8s_node_log_entry(log_entry(0)),
|
2061
|
-
expected_params: K8S_NODE_PARAMS_FROM_LOCAL
|
2062
|
-
},
|
2063
|
-
{
|
2064
|
-
config: CUSTOM_K8S_ENABLE_METADATA_AGENT_CONFIG,
|
2065
|
-
setup_metadata_agent_stub: false,
|
1869
|
+
config: CUSTOM_K8S_CONFIG,
|
2066
1870
|
setup_k8s_stub: false,
|
2067
1871
|
log_entry: k8s_node_log_entry(log_entry(0)),
|
2068
1872
|
expected_params: K8S_NODE_PARAMS_CUSTOM
|
2069
1873
|
},
|
2070
1874
|
{
|
2071
|
-
config:
|
2072
|
-
setup_metadata_agent_stub: true,
|
1875
|
+
config: EMPTY_K8S_CONFIG,
|
2073
1876
|
setup_k8s_stub: true,
|
2074
1877
|
log_entry: k8s_node_log_entry(log_entry(0)),
|
2075
|
-
expected_params:
|
2076
|
-
},
|
2077
|
-
{
|
2078
|
-
config: ENABLE_METADATA_AGENT_CONFIG,
|
2079
|
-
setup_metadata_agent_stub: true,
|
2080
|
-
setup_k8s_stub: true,
|
2081
|
-
log_entry: k8s_node_log_entry(log_entry(0)),
|
2082
|
-
expected_params: K8S_NODE_PARAMS
|
1878
|
+
expected_params: K8S_NODE_PARAMS_FROM_LOCAL
|
2083
1879
|
}
|
2084
1880
|
].each do |test_params|
|
2085
1881
|
new_stub_context do
|
2086
1882
|
setup_gce_metadata_stubs
|
2087
|
-
setup_metadata_agent_stubs(test_params[:setup_metadata_agent_stub])
|
2088
1883
|
setup_k8s_metadata_stubs(test_params[:setup_k8s_stub])
|
2089
1884
|
setup_logging_stubs do
|
2090
1885
|
d = create_driver(test_params[:config])
|
@@ -2102,83 +1897,6 @@ module BaseTest
|
|
2102
1897
|
end
|
2103
1898
|
end
|
2104
1899
|
|
2105
|
-
# Test that the 'time' field from the json record is extracted and set to
|
2106
|
-
# entry.timestamp for Docker container logs.
|
2107
|
-
def test_time_field_extraction_for_docker_container_logs
|
2108
|
-
new_stub_context do
|
2109
|
-
setup_gce_metadata_stubs
|
2110
|
-
setup_metadata_agent_stubs
|
2111
|
-
setup_logging_stubs do
|
2112
|
-
d = create_driver(ENABLE_METADATA_AGENT_CONFIG)
|
2113
|
-
d.emit(docker_container_application_log_entry(log_entry(0)))
|
2114
|
-
d.run
|
2115
|
-
end
|
2116
|
-
verify_log_entries(1, DOCKER_CONTAINER_PARAMS_NO_STREAM) do |entry, i|
|
2117
|
-
verify_default_log_entry_text(entry['textPayload'], i, entry)
|
2118
|
-
# Timestamp in 'time' field from log entry should be set properly.
|
2119
|
-
actual_timestamp = timestamp_parse(entry['timestamp'])
|
2120
|
-
assert_equal DOCKER_CONTAINER_SECONDS_EPOCH,
|
2121
|
-
actual_timestamp['seconds'], entry
|
2122
|
-
assert_equal DOCKER_CONTAINER_NANOS, actual_timestamp['nanos'], entry
|
2123
|
-
end
|
2124
|
-
assert_requested_metadata_agent_stub(
|
2125
|
-
"#{DOCKER_CONTAINER_LOCAL_RESOURCE_ID_PREFIX}.#{DOCKER_CONTAINER_NAME}")
|
2126
|
-
end
|
2127
|
-
end
|
2128
|
-
|
2129
|
-
# Test that the 'source' field is properly extracted from the record json and
|
2130
|
-
# set as a common label 'stream'.
|
2131
|
-
def test_source_for_docker_container_logs
|
2132
|
-
{
|
2133
|
-
docker_container_stdout_stderr_log_entry(
|
2134
|
-
log_entry(0), DOCKER_CONTAINER_STREAM_STDOUT) =>
|
2135
|
-
DOCKER_CONTAINER_PARAMS,
|
2136
|
-
docker_container_stdout_stderr_log_entry(
|
2137
|
-
log_entry(0), DOCKER_CONTAINER_STREAM_STDERR) =>
|
2138
|
-
DOCKER_CONTAINER_PARAMS_STREAM_STDERR,
|
2139
|
-
docker_container_application_log_entry(log_entry(0)) =>
|
2140
|
-
DOCKER_CONTAINER_PARAMS_NO_STREAM,
|
2141
|
-
docker_container_application_log_entry(log_entry(0)) \
|
2142
|
-
.merge('severity' => 'warning') =>
|
2143
|
-
DOCKER_CONTAINER_PARAMS_NO_STREAM
|
2144
|
-
}.each do |log_entry, expected_params|
|
2145
|
-
new_stub_context do
|
2146
|
-
setup_gce_metadata_stubs
|
2147
|
-
setup_metadata_agent_stubs
|
2148
|
-
setup_logging_stubs do
|
2149
|
-
d = create_driver(DOCKER_CONTAINER_CONFIG)
|
2150
|
-
d.emit(log_entry)
|
2151
|
-
d.run
|
2152
|
-
end
|
2153
|
-
verify_log_entries(1, expected_params)
|
2154
|
-
end
|
2155
|
-
end
|
2156
|
-
end
|
2157
|
-
|
2158
|
-
# Test GKE container logs. These logs have the label
|
2159
|
-
# "logging.googleapis.com/local_resource_id" set in the format of
|
2160
|
-
# "gke_container.<namespace_id>.<pod_name>.<container_name>".
|
2161
|
-
def test_gke_container_logs
|
2162
|
-
[1, 2, 3, 5, 11, 50].each do |n|
|
2163
|
-
new_stub_context do
|
2164
|
-
setup_gce_metadata_stubs
|
2165
|
-
setup_k8s_metadata_stubs
|
2166
|
-
setup_metadata_agent_stubs
|
2167
|
-
setup_logging_stubs do
|
2168
|
-
d = create_driver(ENABLE_METADATA_AGENT_CONFIG)
|
2169
|
-
n.times do |i|
|
2170
|
-
d.emit(gke_container_log_entry(log_entry(i)))
|
2171
|
-
end
|
2172
|
-
d.run
|
2173
|
-
end
|
2174
|
-
verify_log_entries(n, CONTAINER_FROM_APPLICATION_PARAMS)
|
2175
|
-
assert_requested_metadata_agent_stub(
|
2176
|
-
"#{CONTAINER_LOCAL_RESOURCE_ID_PREFIX}.#{CONTAINER_NAMESPACE_ID}" \
|
2177
|
-
".#{K8S_POD_NAME}.#{K8S_CONTAINER_NAME}")
|
2178
|
-
end
|
2179
|
-
end
|
2180
|
-
end
|
2181
|
-
|
2182
1900
|
def test_uptime_metric
|
2183
1901
|
setup_gce_metadata_stubs
|
2184
1902
|
[
|
@@ -2315,12 +2033,6 @@ module BaseTest
|
|
2315
2033
|
OpenCensus::Stats.ensure_recorder.clear_stats
|
2316
2034
|
end
|
2317
2035
|
|
2318
|
-
# Metadata Agent.
|
2319
|
-
|
2320
|
-
def metadata_request_url(local_resource_id)
|
2321
|
-
"#{DEFAULT_METADATA_AGENT_URL}/monitoredResource/#{local_resource_id}"
|
2322
|
-
end
|
2323
|
-
|
2324
2036
|
# Provide a stub context that initializes @logs_sent, executes the block and
|
2325
2037
|
# resets WebMock at the end.
|
2326
2038
|
def new_stub_context
|
@@ -2329,26 +2041,6 @@ module BaseTest
|
|
2329
2041
|
WebMock.reset!
|
2330
2042
|
end
|
2331
2043
|
|
2332
|
-
def setup_metadata_agent_stubs(should_respond = true)
|
2333
|
-
if should_respond
|
2334
|
-
MONITORED_RESOURCE_STUBS.each do |local_resource_id, resource|
|
2335
|
-
stub_request(:get, metadata_request_url(local_resource_id))
|
2336
|
-
.to_return(status: 200, body: resource)
|
2337
|
-
end
|
2338
|
-
stub_request(:get, metadata_request_url(RANDOM_LOCAL_RESOURCE_ID))
|
2339
|
-
.to_return(status: 404, body: '')
|
2340
|
-
else
|
2341
|
-
# Simulate an environment with no metadata agent endpoint present.
|
2342
|
-
stub_request(:get,
|
2343
|
-
%r{#{DEFAULT_METADATA_AGENT_URL}\/monitoredResource/.*})
|
2344
|
-
.to_raise(Errno::EHOSTUNREACH)
|
2345
|
-
end
|
2346
|
-
end
|
2347
|
-
|
2348
|
-
def assert_requested_metadata_agent_stub(local_resource_id)
|
2349
|
-
assert_requested :get, metadata_request_url(local_resource_id)
|
2350
|
-
end
|
2351
|
-
|
2352
2044
|
# GKE Container.
|
2353
2045
|
|
2354
2046
|
def container_tag_with_container_name(container_name)
|
@@ -2391,32 +2083,6 @@ module BaseTest
|
|
2391
2083
|
}
|
2392
2084
|
end
|
2393
2085
|
|
2394
|
-
# Docker Container.
|
2395
|
-
|
2396
|
-
def docker_container_stdout_stderr_log_entry(
|
2397
|
-
log, stream = DOCKER_CONTAINER_STREAM_STDOUT)
|
2398
|
-
severity = if stream == DOCKER_CONTAINER_STREAM_STDOUT
|
2399
|
-
'INFO'
|
2400
|
-
else
|
2401
|
-
'ERROR'
|
2402
|
-
end
|
2403
|
-
{
|
2404
|
-
log: log,
|
2405
|
-
source: stream,
|
2406
|
-
severity: severity,
|
2407
|
-
LOCAL_RESOURCE_ID_KEY => "container.#{DOCKER_CONTAINER_ID}"
|
2408
|
-
}
|
2409
|
-
end
|
2410
|
-
|
2411
|
-
def docker_container_application_log_entry(log)
|
2412
|
-
{
|
2413
|
-
log: log,
|
2414
|
-
time: DOCKER_CONTAINER_TIMESTAMP,
|
2415
|
-
LOCAL_RESOURCE_ID_KEY => "#{DOCKER_CONTAINER_LOCAL_RESOURCE_ID_PREFIX}." \
|
2416
|
-
"#{DOCKER_CONTAINER_NAME}"
|
2417
|
-
}
|
2418
|
-
end
|
2419
|
-
|
2420
2086
|
# TODO(qingling128): Temporary fallback for metadata agent restarts.
|
2421
2087
|
# k8s resources.
|
2422
2088
|
|
data/test/plugin/constants.rb
CHANGED
@@ -52,8 +52,6 @@ module Constants
|
|
52
52
|
# Generic attributes.
|
53
53
|
HOSTNAME = Socket.gethostname
|
54
54
|
CUSTOM_LOGGING_API_URL = 'http://localhost:52000'.freeze
|
55
|
-
CUSTOM_METADATA_AGENT_URL = 'http://localhost:12345'.freeze
|
56
|
-
METADATA_AGENT_URL_FROM_ENV = 'http://localhost:54321'.freeze
|
57
55
|
|
58
56
|
# TODO(qingling128) Separate constants into different submodules.
|
59
57
|
# Attributes used for the GCE metadata service.
|
@@ -133,18 +131,6 @@ module Constants
|
|
133
131
|
"projects/#{PROJECT_ID}/" \
|
134
132
|
"traces/#{INVALID_NON_HEX_STACKDRIVER_TRACE_ID}".freeze
|
135
133
|
|
136
|
-
# Docker Container labels.
|
137
|
-
DOCKER_CONTAINER_ID =
|
138
|
-
'0d0f03ff8d3c42688692536d1af77a28cd135c0a5c531f25a31'.freeze
|
139
|
-
DOCKER_CONTAINER_NAME = 'happy_hippo'.freeze
|
140
|
-
DOCKER_CONTAINER_STREAM_STDOUT = 'stdout'.freeze
|
141
|
-
DOCKER_CONTAINER_STREAM_STDERR = 'stderr'.freeze
|
142
|
-
# Timestamp for 1234567890 seconds and 987654321 nanoseconds since epoch.
|
143
|
-
DOCKER_CONTAINER_TIMESTAMP = '2009-02-13T23:31:30.987654321Z'.freeze
|
144
|
-
DOCKER_CONTAINER_SECONDS_EPOCH = 1_234_567_890
|
145
|
-
DOCKER_CONTAINER_NANOS = 987_654_321
|
146
|
-
DOCKER_CONTAINER_LOCAL_RESOURCE_ID_PREFIX = 'container'.freeze
|
147
|
-
|
148
134
|
# New K8s resource constants.
|
149
135
|
K8S_LOCATION = 'us-central1-b'.freeze
|
150
136
|
K8S_LOCATION2 = 'us-central1-c'.freeze
|
@@ -238,18 +224,10 @@ module Constants
|
|
238
224
|
logging_api_url #{CUSTOM_LOGGING_API_URL}
|
239
225
|
).freeze
|
240
226
|
|
241
|
-
CUSTOM_METADATA_AGENT_URL_CONFIG = %(
|
242
|
-
metadata_agent_url #{CUSTOM_METADATA_AGENT_URL}
|
243
|
-
).freeze
|
244
|
-
|
245
227
|
DETECT_JSON_CONFIG = %(
|
246
228
|
detect_json true
|
247
229
|
).freeze
|
248
230
|
|
249
|
-
PARTIAL_SUCCESS_DISABLED_CONFIG = %(
|
250
|
-
partial_success false
|
251
|
-
).freeze
|
252
|
-
|
253
231
|
# rubocop:disable Metrics/LineLength
|
254
232
|
PRIVATE_KEY_CONFIG = %(
|
255
233
|
auth_method private_key
|
@@ -284,14 +262,6 @@ module Constants
|
|
284
262
|
monitoring_type opencensus
|
285
263
|
).freeze
|
286
264
|
|
287
|
-
ENABLE_METADATA_AGENT_CONFIG = %(
|
288
|
-
enable_metadata_agent true
|
289
|
-
).freeze
|
290
|
-
|
291
|
-
DISABLE_METADATA_AGENT_CONFIG = %(
|
292
|
-
enable_metadata_agent false
|
293
|
-
).freeze
|
294
|
-
|
295
265
|
ENABLE_AUTOFORMAT_STACKDRIVER_TRACE_CONFIG = %(
|
296
266
|
autoformat_stackdriver_trace true
|
297
267
|
).freeze
|
@@ -300,12 +270,6 @@ module Constants
|
|
300
270
|
autoformat_stackdriver_trace false
|
301
271
|
).freeze
|
302
272
|
|
303
|
-
DOCKER_CONTAINER_CONFIG = %(
|
304
|
-
enable_metadata_agent true
|
305
|
-
label_map { "source": "#{DOCKER_CONSTANTS[:service]}/stream" }
|
306
|
-
detect_json true
|
307
|
-
).freeze
|
308
|
-
|
309
273
|
CUSTOM_METADATA_CONFIG = %(
|
310
274
|
project_id #{CUSTOM_PROJECT_ID}
|
311
275
|
zone #{CUSTOM_ZONE}
|
@@ -328,14 +292,12 @@ module Constants
|
|
328
292
|
CONFIG_MISSING_METADATA_ALL = %(
|
329
293
|
).freeze
|
330
294
|
|
331
|
-
|
332
|
-
enable_metadata_agent true
|
295
|
+
CUSTOM_K8S_CONFIG = %(
|
333
296
|
k8s_cluster_name #{CUSTOM_K8S_CLUSTER_NAME}
|
334
297
|
k8s_cluster_location #{CUSTOM_K8S_LOCATION}
|
335
298
|
).freeze
|
336
299
|
|
337
|
-
|
338
|
-
enable_metadata_agent true
|
300
|
+
EMPTY_K8S_CONFIG = %(
|
339
301
|
k8s_cluster_name ""
|
340
302
|
k8s_cluster_location ""
|
341
303
|
).freeze
|
@@ -480,7 +442,6 @@ module Constants
|
|
480
442
|
coerce_to_utf8 false
|
481
443
|
detect_json true
|
482
444
|
detect_subservice false
|
483
|
-
enable_metadata_agent true
|
484
445
|
enable_monitoring true
|
485
446
|
http_request_key test_http_request_key
|
486
447
|
insert_id_key test_insert_id_key
|
@@ -491,11 +452,9 @@ module Constants
|
|
491
452
|
labels_key test_labels_key
|
492
453
|
labels { "labels_key": "labels_value" }
|
493
454
|
logging_api_url http://localhost:52000
|
494
|
-
metadata_agent_url http://localhost:12345
|
495
455
|
monitoring_type not_prometheus
|
496
456
|
non_utf8_replacement_string zzz
|
497
457
|
operation_key test_operation_key
|
498
|
-
partial_success false
|
499
458
|
project_id test-project-id-123
|
500
459
|
require_valid_tags true
|
501
460
|
source_location_key test_source_location_key
|
@@ -724,29 +683,6 @@ module Constants
|
|
724
683
|
)
|
725
684
|
).freeze
|
726
685
|
|
727
|
-
# Docker Container.
|
728
|
-
DOCKER_CONTAINER_PARAMS = {
|
729
|
-
resource: {
|
730
|
-
type: DOCKER_CONSTANTS[:resource_type],
|
731
|
-
labels: {
|
732
|
-
'container_id' => DOCKER_CONTAINER_ID,
|
733
|
-
'location' => ZONE
|
734
|
-
}
|
735
|
-
},
|
736
|
-
log_name: 'test',
|
737
|
-
project_id: PROJECT_ID,
|
738
|
-
labels: {
|
739
|
-
"#{DOCKER_CONSTANTS[:service]}/stream" => DOCKER_CONTAINER_STREAM_STDOUT
|
740
|
-
}
|
741
|
-
}.freeze
|
742
|
-
DOCKER_CONTAINER_PARAMS_STREAM_STDERR = DOCKER_CONTAINER_PARAMS.merge(
|
743
|
-
labels: DOCKER_CONTAINER_PARAMS[:labels].merge(
|
744
|
-
"#{DOCKER_CONSTANTS[:service]}/stream" => DOCKER_CONTAINER_STREAM_STDERR
|
745
|
-
)
|
746
|
-
).freeze
|
747
|
-
DOCKER_CONTAINER_PARAMS_NO_STREAM =
|
748
|
-
DOCKER_CONTAINER_PARAMS.merge(labels: {}).freeze
|
749
|
-
|
750
686
|
# Cloud Dataflow.
|
751
687
|
DATAFLOW_PARAMS = {
|
752
688
|
resource: {
|
@@ -930,24 +866,6 @@ module Constants
|
|
930
866
|
# Stub value for Monitored resources from Metadata Agent.
|
931
867
|
# Map from the local_resource_id to the retrieved monitored resource.
|
932
868
|
MONITORED_RESOURCE_STUBS = {
|
933
|
-
# Docker container stderr / stdout logs.
|
934
|
-
"#{DOCKER_CONTAINER_LOCAL_RESOURCE_ID_PREFIX}.#{DOCKER_CONTAINER_ID}" =>
|
935
|
-
{
|
936
|
-
'type' => DOCKER_CONSTANTS[:resource_type],
|
937
|
-
'labels' => {
|
938
|
-
'location' => ZONE,
|
939
|
-
'container_id' => DOCKER_CONTAINER_ID
|
940
|
-
}
|
941
|
-
}.to_json,
|
942
|
-
# Docker container application logs.
|
943
|
-
"#{DOCKER_CONTAINER_LOCAL_RESOURCE_ID_PREFIX}.#{DOCKER_CONTAINER_NAME}" =>
|
944
|
-
{
|
945
|
-
'type' => DOCKER_CONSTANTS[:resource_type],
|
946
|
-
'labels' => {
|
947
|
-
'location' => ZONE,
|
948
|
-
'container_id' => DOCKER_CONTAINER_ID
|
949
|
-
}
|
950
|
-
}.to_json,
|
951
869
|
# GKE container logs.
|
952
870
|
"#{CONTAINER_LOCAL_RESOURCE_ID_PREFIX}.#{CONTAINER_NAMESPACE_ID}" \
|
953
871
|
".#{K8S_POD_NAME}.#{K8S_CONTAINER_NAME}" =>
|
@@ -353,7 +353,6 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
353
353
|
'\bcoerce_to_utf8\b.*\bfalse\b',
|
354
354
|
'\bdetect_json\b.*\btrue\b',
|
355
355
|
'\bdetect_subservice\b.*\bfalse\b',
|
356
|
-
'\benable_metadata_agent\b.*\btrue\b',
|
357
356
|
'\benable_monitoring\b.*\btrue\b',
|
358
357
|
'\bhttp_request_key\b.*\btest_http_request_key\b',
|
359
358
|
'\binsert_id_key\b.*\btest_insert_id_key\b',
|
@@ -364,11 +363,9 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
364
363
|
'\blabels_key\b.*\btest_labels_key\b',
|
365
364
|
'\blabels\b.*{"labels_key"=>"labels_value"}',
|
366
365
|
'\blogging_api_url\b.*\bhttp://localhost:52000\b',
|
367
|
-
'\bmetadata_agent_url\b.*\bhttp://localhost:12345\b',
|
368
366
|
'\bmonitoring_type\b.*\bnot_prometheus\b',
|
369
367
|
'\bnon_utf8_replacement_string\b.*\bzzz\b',
|
370
368
|
'\boperation_key\b.*\btest_operation_key\b',
|
371
|
-
'\bpartial_success\b.*\bfalse\b',
|
372
369
|
'\bproject_id\b.*\btest-project-id-123\b',
|
373
370
|
'\brequire_valid_tags\b.*\btrue\b',
|
374
371
|
'\bsource_location_key\b.*\btest_source_location_key\b',
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-google-cloud
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.8.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Stackdriver Agents Team
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-02-
|
11
|
+
date: 2020-02-27 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -142,14 +142,14 @@ dependencies:
|
|
142
142
|
requirements:
|
143
143
|
- - '='
|
144
144
|
- !ruby/object:Gem::Version
|
145
|
-
version: 0.3.
|
145
|
+
version: 0.3.2
|
146
146
|
type: :runtime
|
147
147
|
prerelease: false
|
148
148
|
version_requirements: !ruby/object:Gem::Requirement
|
149
149
|
requirements:
|
150
150
|
- - '='
|
151
151
|
- !ruby/object:Gem::Version
|
152
|
-
version: 0.3.
|
152
|
+
version: 0.3.2
|
153
153
|
- !ruby/object:Gem::Dependency
|
154
154
|
name: mocha
|
155
155
|
requirement: !ruby/object:Gem::Requirement
|