fluent-plugin-google-cloud 0.6.9.pre.1 → 0.6.10.pre.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +23 -25
- data/fluent-plugin-google-cloud.gemspec +4 -4
- data/lib/fluent/plugin/out_google_cloud.rb +93 -77
- data/test/plugin/base_test.rb +37 -24
- data/test/plugin/constants.rb +119 -110
- data/test/plugin/test_out_google_cloud.rb +2 -1
- data/test/plugin/test_out_google_cloud_grpc.rb +41 -24
- metadata +4 -10
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 39661a11e58b52f626ece8c74056504cc8357f85
|
4
|
+
data.tar.gz: db78d67f4c56435d85011a66332f01c4fd6d69df
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2d856be37dcd8bf41c8c35dec9a1e712bf05e684cf2d824dc76a75c279c5f65ea8ebc9b8f3d01922f6e15479cfd0aa2fa87359f407c56227364a40922d0c9037
|
7
|
+
data.tar.gz: 58c0e3b7eba803069baa7bc411269e0f829a43cf76d8555427f03d8cc350e3151e567216504daa4ea10c91162af1e0a375e03ac58e263383a76b85a0667c25e8
|
data/Gemfile.lock
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
fluent-plugin-google-cloud (0.6.
|
4
|
+
fluent-plugin-google-cloud (0.6.10.pre.1)
|
5
5
|
fluentd (~> 0.10)
|
6
6
|
google-api-client (~> 0.14)
|
7
7
|
google-cloud-logging (~> 1.2, >= 1.2.3)
|
8
8
|
googleapis-common-protos (~> 1.3)
|
9
9
|
googleauth (~> 0.4, < 0.5.2)
|
10
|
-
grpc (~> 1.0
|
10
|
+
grpc (~> 1.0)
|
11
11
|
json (~> 1.8)
|
12
12
|
|
13
13
|
GEM
|
@@ -16,8 +16,6 @@ GEM
|
|
16
16
|
addressable (2.5.2)
|
17
17
|
public_suffix (>= 2.0.2, < 4.0)
|
18
18
|
ast (2.3.0)
|
19
|
-
astrolabe (1.3.1)
|
20
|
-
parser (~> 2.2)
|
21
19
|
cool.io (1.5.1)
|
22
20
|
crack (0.4.3)
|
23
21
|
safe_yaml (~> 1.0.0)
|
@@ -25,7 +23,7 @@ GEM
|
|
25
23
|
declarative-option (0.1.0)
|
26
24
|
faraday (0.13.1)
|
27
25
|
multipart-post (>= 1.2, < 3)
|
28
|
-
fluentd (0.14.
|
26
|
+
fluentd (0.14.22)
|
29
27
|
cool.io (>= 1.4.5, < 2.0.0)
|
30
28
|
http_parser.rb (>= 0.5.1, < 0.7.0)
|
31
29
|
msgpack (>= 0.7.0, < 2.0.0)
|
@@ -36,9 +34,9 @@ GEM
|
|
36
34
|
tzinfo (~> 1.0)
|
37
35
|
tzinfo-data (~> 1.0)
|
38
36
|
yajl-ruby (~> 1.0)
|
39
|
-
google-api-client (0.
|
37
|
+
google-api-client (0.17.1)
|
40
38
|
addressable (~> 2.5, >= 2.5.1)
|
41
|
-
googleauth (
|
39
|
+
googleauth (>= 0.5, < 0.7.0)
|
42
40
|
httpclient (>= 2.8.1, < 3.0)
|
43
41
|
mime-types (~> 3.0)
|
44
42
|
representable (~> 3.0)
|
@@ -59,11 +57,11 @@ GEM
|
|
59
57
|
grpc (~> 1.0)
|
60
58
|
rly (~> 0.2.3)
|
61
59
|
google-protobuf (3.4.1.1)
|
62
|
-
googleapis-common-protos (1.3.
|
60
|
+
googleapis-common-protos (1.3.7)
|
63
61
|
google-protobuf (~> 3.0)
|
64
62
|
googleapis-common-protos-types (~> 1.0)
|
65
63
|
grpc (~> 1.0)
|
66
|
-
googleapis-common-protos-types (1.0.
|
64
|
+
googleapis-common-protos-types (1.0.1)
|
67
65
|
google-protobuf (~> 3.0)
|
68
66
|
googleauth (0.5.1)
|
69
67
|
faraday (~> 0.9)
|
@@ -73,9 +71,10 @@ GEM
|
|
73
71
|
multi_json (~> 1.11)
|
74
72
|
os (~> 0.9)
|
75
73
|
signet (~> 0.7)
|
76
|
-
grpc (1.2
|
74
|
+
grpc (1.7.2)
|
77
75
|
google-protobuf (~> 3.1)
|
78
|
-
|
76
|
+
googleapis-common-protos-types (~> 1.0.0)
|
77
|
+
googleauth (>= 0.5.1, < 0.7)
|
79
78
|
hashdiff (0.3.7)
|
80
79
|
http_parser.rb (0.6.0)
|
81
80
|
httpclient (2.8.3)
|
@@ -96,13 +95,13 @@ GEM
|
|
96
95
|
multi_json (1.12.2)
|
97
96
|
multipart-post (2.0.0)
|
98
97
|
os (0.9.6)
|
99
|
-
parser (2.4.0.
|
100
|
-
ast (~> 2.
|
101
|
-
power_assert (1.1.
|
98
|
+
parser (2.4.0.2)
|
99
|
+
ast (~> 2.3)
|
100
|
+
power_assert (1.1.1)
|
102
101
|
powerpack (0.1.1)
|
103
102
|
prometheus-client (0.7.1)
|
104
103
|
quantile (~> 0.2.0)
|
105
|
-
public_suffix (3.0.
|
104
|
+
public_suffix (3.0.1)
|
106
105
|
quantile (0.2.0)
|
107
106
|
rainbow (2.2.2)
|
108
107
|
rake
|
@@ -113,13 +112,12 @@ GEM
|
|
113
112
|
uber (< 0.2.0)
|
114
113
|
retriable (3.1.1)
|
115
114
|
rly (0.2.3)
|
116
|
-
rubocop (0.
|
117
|
-
|
118
|
-
parser (>= 2.2.3.0, < 3.0)
|
115
|
+
rubocop (0.39.0)
|
116
|
+
parser (>= 2.3.0.7, < 3.0)
|
119
117
|
powerpack (~> 0.1)
|
120
118
|
rainbow (>= 1.99.1, < 3.0)
|
121
119
|
ruby-progressbar (~> 1.7)
|
122
|
-
|
120
|
+
unicode-display_width (~> 1.0, >= 1.0.1)
|
123
121
|
ruby-progressbar (1.9.0)
|
124
122
|
ruby_dig (0.0.2)
|
125
123
|
safe_yaml (1.0.4)
|
@@ -136,17 +134,17 @@ GEM
|
|
136
134
|
test-unit (3.2.6)
|
137
135
|
power_assert
|
138
136
|
thread_safe (0.3.6)
|
139
|
-
|
140
|
-
tzinfo (1.2.3)
|
137
|
+
tzinfo (1.2.4)
|
141
138
|
thread_safe (~> 0.1)
|
142
|
-
tzinfo-data (1.2017.
|
139
|
+
tzinfo-data (1.2017.3)
|
143
140
|
tzinfo (>= 1.0.0)
|
144
141
|
uber (0.1.0)
|
142
|
+
unicode-display_width (1.3.0)
|
145
143
|
webmock (2.3.2)
|
146
144
|
addressable (>= 2.3.6)
|
147
145
|
crack (>= 0.3.2)
|
148
146
|
hashdiff
|
149
|
-
yajl-ruby (1.3.
|
147
|
+
yajl-ruby (1.3.1)
|
150
148
|
|
151
149
|
PLATFORMS
|
152
150
|
ruby
|
@@ -156,9 +154,9 @@ DEPENDENCIES
|
|
156
154
|
mocha (~> 1.1)
|
157
155
|
prometheus-client (~> 0.7.1)
|
158
156
|
rake (~> 10.3)
|
159
|
-
rubocop (~> 0.
|
157
|
+
rubocop (~> 0.39.0)
|
160
158
|
test-unit (~> 3.0)
|
161
159
|
webmock (~> 2.3.1)
|
162
160
|
|
163
161
|
BUNDLED WITH
|
164
|
-
1.
|
162
|
+
1.16.0
|
@@ -7,10 +7,10 @@ Gem::Specification.new do |gem|
|
|
7
7
|
This is an official Google Ruby gem.
|
8
8
|
eos
|
9
9
|
gem.summary = 'fluentd output plugin for the Stackdriver Logging API'
|
10
|
-
gem.homepage =
|
10
|
+
gem.homepage =
|
11
11
|
'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
|
12
12
|
gem.license = 'Apache-2.0'
|
13
|
-
gem.version = '0.6.
|
13
|
+
gem.version = '0.6.10.pre.1'
|
14
14
|
gem.authors = ['Todd Derr', 'Alex Robinson']
|
15
15
|
gem.email = ['salty@google.com']
|
16
16
|
gem.required_ruby_version = Gem::Requirement.new('>= 2.0')
|
@@ -24,12 +24,12 @@ eos
|
|
24
24
|
gem.add_runtime_dependency 'google-api-client', '~> 0.14'
|
25
25
|
gem.add_runtime_dependency 'google-cloud-logging', '~> 1.2', '>= 1.2.3'
|
26
26
|
gem.add_runtime_dependency 'googleauth', '~> 0.4', '< 0.5.2'
|
27
|
-
gem.add_runtime_dependency 'grpc', '~> 1.0'
|
27
|
+
gem.add_runtime_dependency 'grpc', '~> 1.0'
|
28
28
|
gem.add_runtime_dependency 'json', '~> 1.8'
|
29
29
|
|
30
30
|
gem.add_development_dependency 'mocha', '~> 1.1'
|
31
31
|
gem.add_development_dependency 'rake', '~> 10.3'
|
32
|
-
gem.add_development_dependency 'rubocop', '~> 0.
|
32
|
+
gem.add_development_dependency 'rubocop', '~> 0.39.0'
|
33
33
|
gem.add_development_dependency 'webmock', '~> 2.3.1'
|
34
34
|
gem.add_development_dependency 'test-unit', '~> 3.0'
|
35
35
|
gem.add_development_dependency 'prometheus-client', '~> 0.7.1'
|
@@ -31,7 +31,7 @@ module Google
|
|
31
31
|
module Protobuf
|
32
32
|
# Alias the has_key? method to have the same interface as a regular map.
|
33
33
|
class Map
|
34
|
-
|
34
|
+
alias key? has_key?
|
35
35
|
end
|
36
36
|
end
|
37
37
|
end
|
@@ -45,80 +45,92 @@ module Fluent
|
|
45
45
|
service: 'appengine.googleapis.com',
|
46
46
|
resource_type: 'gae_app',
|
47
47
|
metadata_attributes: %w(gae_backend_name gae_backend_version)
|
48
|
-
}
|
48
|
+
}.freeze
|
49
49
|
CLOUDFUNCTIONS_CONSTANTS = {
|
50
50
|
service: 'cloudfunctions.googleapis.com',
|
51
|
-
resource_type: 'cloud_function'
|
52
|
-
|
51
|
+
resource_type: 'cloud_function',
|
52
|
+
stream_severity_map: {
|
53
|
+
'stdout' => 'INFO',
|
54
|
+
'stderr' => 'ERROR'
|
55
|
+
}
|
56
|
+
}.freeze
|
53
57
|
COMPUTE_CONSTANTS = {
|
54
58
|
service: 'compute.googleapis.com',
|
55
59
|
resource_type: 'gce_instance'
|
56
|
-
}
|
60
|
+
}.freeze
|
57
61
|
GKE_CONSTANTS = {
|
58
62
|
service: 'container.googleapis.com',
|
59
63
|
resource_type: 'container',
|
60
64
|
extra_resource_labels: %w(namespace_id pod_id container_name),
|
61
65
|
extra_common_labels: %w(namespace_name pod_name),
|
62
|
-
metadata_attributes: %w(kube-env)
|
63
|
-
|
66
|
+
metadata_attributes: %w(kube-env),
|
67
|
+
stream_severity_map: {
|
68
|
+
'stdout' => 'INFO',
|
69
|
+
'stderr' => 'ERROR'
|
70
|
+
}
|
71
|
+
}.freeze
|
64
72
|
DOCKER_CONSTANTS = {
|
65
73
|
service: 'docker.googleapis.com',
|
66
74
|
resource_type: 'docker_container'
|
67
|
-
}
|
75
|
+
}.freeze
|
68
76
|
DATAFLOW_CONSTANTS = {
|
69
77
|
service: 'dataflow.googleapis.com',
|
70
78
|
resource_type: 'dataflow_step',
|
71
79
|
extra_resource_labels: %w(region job_name job_id step_id)
|
72
|
-
}
|
80
|
+
}.freeze
|
73
81
|
DATAPROC_CONSTANTS = {
|
74
82
|
service: 'cluster.dataproc.googleapis.com',
|
75
83
|
resource_type: 'cloud_dataproc_cluster',
|
76
84
|
metadata_attributes: %w(dataproc-cluster-uuid dataproc-cluster-name)
|
77
|
-
}
|
85
|
+
}.freeze
|
78
86
|
EC2_CONSTANTS = {
|
79
87
|
service: 'ec2.amazonaws.com',
|
80
88
|
resource_type: 'aws_ec2_instance'
|
81
|
-
}
|
89
|
+
}.freeze
|
82
90
|
ML_CONSTANTS = {
|
83
91
|
service: 'ml.googleapis.com',
|
84
92
|
resource_type: 'ml_job',
|
85
93
|
extra_resource_labels: %w(job_id task_name)
|
86
|
-
}
|
94
|
+
}.freeze
|
87
95
|
|
88
96
|
# The map between a subservice name and a resource type.
|
89
|
-
SUBSERVICE_MAP =
|
97
|
+
SUBSERVICE_MAP =
|
90
98
|
[APPENGINE_CONSTANTS, GKE_CONSTANTS, DATAFLOW_CONSTANTS,
|
91
99
|
DATAPROC_CONSTANTS, ML_CONSTANTS]
|
92
100
|
.map { |consts| [consts[:service], consts[:resource_type]] }.to_h
|
93
101
|
# Default back to GCE if invalid value is detected.
|
94
102
|
SUBSERVICE_MAP.default = COMPUTE_CONSTANTS[:resource_type]
|
103
|
+
SUBSERVICE_MAP.freeze
|
95
104
|
|
96
105
|
# The map between a resource type and expected subservice attributes.
|
97
|
-
SUBSERVICE_METADATA_ATTRIBUTES =
|
106
|
+
SUBSERVICE_METADATA_ATTRIBUTES =
|
98
107
|
[APPENGINE_CONSTANTS, GKE_CONSTANTS, DATAPROC_CONSTANTS].map do |consts|
|
99
108
|
[consts[:resource_type], consts[:metadata_attributes].to_set]
|
100
|
-
end.to_h
|
109
|
+
end.to_h.freeze
|
101
110
|
end
|
102
111
|
|
103
112
|
# Constants for configuration.
|
104
113
|
module ConfigConstants
|
105
114
|
# Default values for JSON payload keys to set the "httpRequest",
|
106
115
|
# "operation", "sourceLocation", "trace" fields in the LogEntry.
|
107
|
-
DEFAULT_HTTP_REQUEST_KEY = 'httpRequest'
|
108
|
-
DEFAULT_OPERATION_KEY = 'logging.googleapis.com/operation'
|
109
|
-
DEFAULT_SOURCE_LOCATION_KEY =
|
110
|
-
|
116
|
+
DEFAULT_HTTP_REQUEST_KEY = 'httpRequest'.freeze
|
117
|
+
DEFAULT_OPERATION_KEY = 'logging.googleapis.com/operation'.freeze
|
118
|
+
DEFAULT_SOURCE_LOCATION_KEY =
|
119
|
+
'logging.googleapis.com/sourceLocation'.freeze
|
120
|
+
DEFAULT_TRACE_KEY = 'logging.googleapis.com/trace'.freeze
|
111
121
|
|
112
122
|
DEFAULT_METADATA_AGENT_URL =
|
113
|
-
'http://local-metadata-agent.stackdriver.com:8000'
|
123
|
+
'http://local-metadata-agent.stackdriver.com:8000'.freeze
|
114
124
|
end
|
115
125
|
|
116
126
|
# Internal constants.
|
117
127
|
module InternalConstants
|
128
|
+
DEFAULT_LOGGING_API_URL = 'logging.googleapis.com'.freeze
|
129
|
+
|
118
130
|
# The label name of local_resource_id in the json payload. When a record
|
119
131
|
# has this field in the payload, we will use the value to retrieve
|
120
132
|
# monitored resource from Stackdriver Metadata agent.
|
121
|
-
LOCAL_RESOURCE_ID_KEY = 'logging.googleapis.com/local_resource_id'
|
133
|
+
LOCAL_RESOURCE_ID_KEY = 'logging.googleapis.com/local_resource_id'.freeze
|
122
134
|
|
123
135
|
# Map from each field name under LogEntry to corresponding variables
|
124
136
|
# required to perform field value extraction from the log record.
|
@@ -168,11 +180,12 @@ module Fluent
|
|
168
180
|
'Google::Logging::V2::LogEntryOperation',
|
169
181
|
'Google::Apis::LoggingV2::LogEntryOperation'
|
170
182
|
]
|
171
|
-
}
|
183
|
+
}.freeze
|
172
184
|
|
173
185
|
# The name of the WriteLogEntriesPartialErrors field in the error details.
|
174
186
|
PARTIAL_ERROR_FIELD =
|
175
|
-
'type.googleapis.com/google.logging.v2.WriteLogEntriesPartialErrors'
|
187
|
+
'type.googleapis.com/google.logging.v2.WriteLogEntriesPartialErrors' \
|
188
|
+
.freeze
|
176
189
|
end
|
177
190
|
|
178
191
|
include self::ServiceConstants
|
@@ -181,14 +194,14 @@ module Fluent
|
|
181
194
|
|
182
195
|
Fluent::Plugin.register_output('google_cloud', self)
|
183
196
|
|
184
|
-
PLUGIN_NAME = 'Fluentd Google Cloud Logging plugin'
|
185
|
-
PLUGIN_VERSION = '0.6.
|
197
|
+
PLUGIN_NAME = 'Fluentd Google Cloud Logging plugin'.freeze
|
198
|
+
PLUGIN_VERSION = '0.6.10.pre.1'.freeze
|
186
199
|
|
187
200
|
# Name of the the Google cloud logging write scope.
|
188
|
-
LOGGING_SCOPE = 'https://www.googleapis.com/auth/logging.write'
|
201
|
+
LOGGING_SCOPE = 'https://www.googleapis.com/auth/logging.write'.freeze
|
189
202
|
|
190
203
|
# Address of the metadata service.
|
191
|
-
METADATA_SERVICE_ADDR = '169.254.169.254'
|
204
|
+
METADATA_SERVICE_ADDR = '169.254.169.254'.freeze
|
192
205
|
|
193
206
|
# Disable this warning to conform to fluentd config_param conventions.
|
194
207
|
# rubocop:disable Style/HashSyntax
|
@@ -310,6 +323,10 @@ module Fluent
|
|
310
323
|
:default => nil,
|
311
324
|
:secret => true
|
312
325
|
|
326
|
+
# The URL of Stackdriver Logging API. Right now this only works with the
|
327
|
+
# gRPC path (use_grpc = true).
|
328
|
+
config_param :logging_api_url, :string, :default => DEFAULT_LOGGING_API_URL
|
329
|
+
|
313
330
|
# Whether to collect metrics about the plugin usage. The mechanism for
|
314
331
|
# collecting and exposing metrics is controlled by the monitoring_type
|
315
332
|
# parameter.
|
@@ -358,6 +375,15 @@ module Fluent
|
|
358
375
|
' partial_success flag will be ignored.'
|
359
376
|
end
|
360
377
|
|
378
|
+
# TODO(qingling128): Remove this warning after the support is added. Also
|
379
|
+
# remove the comment in the description of this configuration.
|
380
|
+
unless @logging_api_url == DEFAULT_LOGGING_API_URL || @use_grpc
|
381
|
+
@log.warn 'Detected customized logging_api_url while use_grpc is not' \
|
382
|
+
' enabled. Customized logging_api_url for the non-gRPC path' \
|
383
|
+
' is not supported. The logging_api_url option will be' \
|
384
|
+
' ignored.'
|
385
|
+
end
|
386
|
+
|
361
387
|
# If monitoring is enabled, register metrics in the default registry
|
362
388
|
# and store metric objects for future use.
|
363
389
|
if @enable_monitoring
|
@@ -391,10 +417,9 @@ module Fluent
|
|
391
417
|
extra << 'private_key_path' unless @private_key_path.nil?
|
392
418
|
extra << 'private_key_passphrase' unless @private_key_passphrase.nil?
|
393
419
|
|
394
|
-
|
395
|
-
|
396
|
-
|
397
|
-
extra.join(' ')
|
420
|
+
raise Fluent::ConfigError,
|
421
|
+
"#{PLUGIN_NAME} no longer supports auth_method.\n" \
|
422
|
+
"Please remove configuration parameters: #{extra.join(' ')}"
|
398
423
|
end
|
399
424
|
|
400
425
|
set_regexp_patterns
|
@@ -641,7 +666,7 @@ module Fluent
|
|
641
666
|
end
|
642
667
|
else
|
643
668
|
begin
|
644
|
-
write_request =
|
669
|
+
write_request =
|
645
670
|
Google::Apis::LoggingV2::WriteLogEntriesRequest.new(
|
646
671
|
log_name: log_name,
|
647
672
|
resource: group_level_resource,
|
@@ -767,7 +792,7 @@ module Fluent
|
|
767
792
|
end
|
768
793
|
|
769
794
|
def fetch_gce_metadata(metadata_path)
|
770
|
-
|
795
|
+
raise "Called fetch_gce_metadata with platform=#{@platform}" unless
|
771
796
|
@platform == Platform::GCE
|
772
797
|
# See https://cloud.google.com/compute/docs/metadata
|
773
798
|
open('http://' + METADATA_SERVICE_ADDR + '/computeMetadata/v1/' +
|
@@ -777,7 +802,7 @@ module Fluent
|
|
777
802
|
# EC2 Metadata server returns everything in one call. Store it after the
|
778
803
|
# first fetch to avoid making multiple calls.
|
779
804
|
def ec2_metadata
|
780
|
-
|
805
|
+
raise "Called ec2_metadata with platform=#{@platform}" unless
|
781
806
|
@platform == Platform::EC2
|
782
807
|
unless @ec2_metadata
|
783
808
|
# See http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html
|
@@ -821,8 +846,8 @@ module Fluent
|
|
821
846
|
missing << 'zone' unless @zone
|
822
847
|
missing << 'vm_id' unless @vm_id
|
823
848
|
return if missing.empty?
|
824
|
-
|
825
|
-
|
849
|
+
raise Fluent::ConfigError,
|
850
|
+
"Unable to obtain metadata parameters: #{missing.join(' ')}"
|
826
851
|
end
|
827
852
|
|
828
853
|
# 1. Return the value if it is explicitly set in the config already.
|
@@ -991,13 +1016,11 @@ module Fluent
|
|
991
1016
|
# GCE instance and GKE container.
|
992
1017
|
when COMPUTE_CONSTANTS[:resource_type],
|
993
1018
|
GKE_CONSTANTS[:resource_type]
|
994
|
-
labels
|
995
|
-
"#{COMPUTE_CONSTANTS[:service]}/resource_name" => @vm_name)
|
1019
|
+
labels["#{COMPUTE_CONSTANTS[:service]}/resource_name"] = @vm_name
|
996
1020
|
|
997
1021
|
# EC2.
|
998
1022
|
when EC2_CONSTANTS[:resource_type]
|
999
|
-
labels
|
1000
|
-
"#{EC2_CONSTANTS[:service]}/resource_name" => @vm_name)
|
1023
|
+
labels["#{EC2_CONSTANTS[:service]}/resource_name"] = @vm_name
|
1001
1024
|
end
|
1002
1025
|
labels
|
1003
1026
|
end
|
@@ -1354,29 +1377,17 @@ module Fluent
|
|
1354
1377
|
if resource_type == CLOUDFUNCTIONS_CONSTANTS[:resource_type]
|
1355
1378
|
if @cloudfunctions_log_match && @cloudfunctions_log_match['severity']
|
1356
1379
|
return parse_severity(@cloudfunctions_log_match['severity'])
|
1357
|
-
elsif record.key?('stream')
|
1358
|
-
|
1359
|
-
|
1360
|
-
elsif record.key?('stream') && record['stream'] == 'stderr'
|
1361
|
-
record.delete('stream')
|
1362
|
-
return 'ERROR'
|
1363
|
-
else
|
1364
|
-
return 'DEFAULT'
|
1380
|
+
elsif record.key?('stream')
|
1381
|
+
return CLOUDFUNCTIONS_CONSTANTS[:stream_severity_map].fetch(
|
1382
|
+
record.delete('stream'), 'DEFAULT')
|
1365
1383
|
end
|
1366
1384
|
elsif record.key?('severity')
|
1367
1385
|
return parse_severity(record.delete('severity'))
|
1368
1386
|
elsif resource_type == GKE_CONSTANTS[:resource_type]
|
1369
1387
|
stream = entry_level_common_labels["#{GKE_CONSTANTS[:service]}/stream"]
|
1370
|
-
|
1371
|
-
return 'INFO'
|
1372
|
-
elsif stream == 'stderr'
|
1373
|
-
return 'ERROR'
|
1374
|
-
else
|
1375
|
-
return 'DEFAULT'
|
1376
|
-
end
|
1377
|
-
else
|
1378
|
-
return 'DEFAULT'
|
1388
|
+
return GKE_CONSTANTS[:stream_severity_map].fetch(stream, 'DEFAULT')
|
1379
1389
|
end
|
1390
|
+
'DEFAULT'
|
1380
1391
|
end
|
1381
1392
|
|
1382
1393
|
def set_log_entry_fields(record, entry)
|
@@ -1404,11 +1415,11 @@ module Fluent
|
|
1404
1415
|
|
1405
1416
|
next unless extracted_subfields
|
1406
1417
|
|
1407
|
-
if @use_grpc
|
1408
|
-
|
1409
|
-
|
1410
|
-
|
1411
|
-
|
1418
|
+
output = if @use_grpc
|
1419
|
+
Object.const_get(grpc_class).new
|
1420
|
+
else
|
1421
|
+
Object.const_get(non_grpc_class).new
|
1422
|
+
end
|
1412
1423
|
extracted_subfields.each do |key, value|
|
1413
1424
|
output.send("#{key}=", value)
|
1414
1425
|
end
|
@@ -1424,7 +1435,8 @@ module Fluent
|
|
1424
1435
|
|
1425
1436
|
# Values permitted by the API for 'severity' (which is an enum).
|
1426
1437
|
VALID_SEVERITIES = Set.new(
|
1427
|
-
%w(DEFAULT DEBUG INFO NOTICE WARNING ERROR CRITICAL ALERT EMERGENCY)
|
1438
|
+
%w(DEFAULT DEBUG INFO NOTICE WARNING ERROR CRITICAL ALERT EMERGENCY)
|
1439
|
+
).freeze
|
1428
1440
|
|
1429
1441
|
# Translates other severity strings to one of the valid values above.
|
1430
1442
|
SEVERITY_TRANSLATIONS = {
|
@@ -1453,7 +1465,7 @@ module Fluent
|
|
1453
1465
|
# other misc. translations.
|
1454
1466
|
'ERR' => 'ERROR',
|
1455
1467
|
'F' => 'CRITICAL'
|
1456
|
-
}
|
1468
|
+
}.freeze
|
1457
1469
|
|
1458
1470
|
def parse_severity(severity_str)
|
1459
1471
|
# The API is case insensitive, but uppercase to make things simpler.
|
@@ -1464,12 +1476,13 @@ module Fluent
|
|
1464
1476
|
|
1465
1477
|
# If the severity is an integer (string) return it as an integer,
|
1466
1478
|
# truncated to the closest valid value (multiples of 100 between 0-800).
|
1467
|
-
if /\A\d+\z
|
1479
|
+
if /\A\d+\z/ =~ severity
|
1468
1480
|
begin
|
1469
1481
|
numeric_severity = (severity.to_i / 100) * 100
|
1470
|
-
|
1482
|
+
case
|
1483
|
+
when numeric_severity < 0
|
1471
1484
|
return 0
|
1472
|
-
|
1485
|
+
when numeric_severity > 800
|
1473
1486
|
return 800
|
1474
1487
|
else
|
1475
1488
|
return numeric_severity
|
@@ -1507,7 +1520,7 @@ module Fluent
|
|
1507
1520
|
600 => Google::Logging::Type::LogSeverity::CRITICAL,
|
1508
1521
|
700 => Google::Logging::Type::LogSeverity::ALERT,
|
1509
1522
|
800 => Google::Logging::Type::LogSeverity::EMERGENCY
|
1510
|
-
}
|
1523
|
+
}.freeze
|
1511
1524
|
|
1512
1525
|
def grpc_severity(severity)
|
1513
1526
|
# TODO: find out why this doesn't work.
|
@@ -1561,11 +1574,14 @@ module Fluent
|
|
1561
1574
|
|
1562
1575
|
def decode_cloudfunctions_function_name(function_name)
|
1563
1576
|
function_name.gsub(/c\.[a-z]/) { |s| s.upcase[-1] }
|
1564
|
-
|
1577
|
+
.gsub('u.u', '_')
|
1578
|
+
.gsub('d.d', '$')
|
1579
|
+
.gsub('a.a', '@')
|
1580
|
+
.gsub('p.p', '.')
|
1565
1581
|
end
|
1566
1582
|
|
1567
1583
|
def format(tag, time, record)
|
1568
|
-
[tag, time, record].
|
1584
|
+
Fluent::Engine.msgpack_factory.packer.write([tag, time, record]).to_s
|
1569
1585
|
end
|
1570
1586
|
|
1571
1587
|
# Given a tag, returns the corresponding valid tag if possible, or nil if
|
@@ -1618,7 +1634,7 @@ module Fluent
|
|
1618
1634
|
ret.list_value = list_from_ruby(value)
|
1619
1635
|
else
|
1620
1636
|
@log.error "Unknown type: #{value.class}"
|
1621
|
-
|
1637
|
+
raise Google::Protobuf::Error, "Unknown type: #{value.class}"
|
1622
1638
|
end
|
1623
1639
|
ret
|
1624
1640
|
end
|
@@ -1717,7 +1733,7 @@ module Fluent
|
|
1717
1733
|
creds = GRPC::Core::CallCredentials.new(authentication.updater_proc)
|
1718
1734
|
creds = ssl_creds.compose(creds)
|
1719
1735
|
@client = Google::Logging::V2::LoggingServiceV2::Stub.new(
|
1720
|
-
|
1736
|
+
@logging_api_url, creds)
|
1721
1737
|
else
|
1722
1738
|
unless @client.authorization.expired?
|
1723
1739
|
begin
|
@@ -1839,7 +1855,7 @@ module Fluent
|
|
1839
1855
|
error_details = ensure_array(
|
1840
1856
|
ensure_hash(ensure_hash(JSON.parse(error.body))['error'])['details'])
|
1841
1857
|
partial_errors = error_details.detect(
|
1842
|
-
-> {
|
1858
|
+
-> { raise JSON::ParserError, "No type #{PARTIAL_ERROR_FIELD}." }
|
1843
1859
|
) do |error_detail|
|
1844
1860
|
ensure_hash(error_detail)['@type'] == PARTIAL_ERROR_FIELD
|
1845
1861
|
end
|
@@ -1847,8 +1863,8 @@ module Fluent
|
|
1847
1863
|
ensure_hash(partial_errors)['logEntryErrors'])
|
1848
1864
|
log_entry_errors.each do |index, log_entry_error|
|
1849
1865
|
error_hash = ensure_hash(log_entry_error)
|
1850
|
-
|
1851
|
-
|
1866
|
+
raise JSON::ParserError,
|
1867
|
+
"Entry #{index} is missing 'code' or 'message'." unless
|
1852
1868
|
error_hash['code'] && error_hash['message']
|
1853
1869
|
error_key = [error_hash['code'], error_hash['message']].freeze
|
1854
1870
|
# TODO(qingling128): Convert indexes to integers.
|
@@ -1862,11 +1878,11 @@ module Fluent
|
|
1862
1878
|
end
|
1863
1879
|
|
1864
1880
|
def ensure_array(value)
|
1865
|
-
Array.try_convert(value) || (
|
1881
|
+
Array.try_convert(value) || (raise JSON::ParserError, value.class.to_s)
|
1866
1882
|
end
|
1867
1883
|
|
1868
1884
|
def ensure_hash(value)
|
1869
|
-
Hash.try_convert(value) || (
|
1885
|
+
Hash.try_convert(value) || (raise JSON::ParserError, value.class.to_s)
|
1870
1886
|
end
|
1871
1887
|
|
1872
1888
|
# Increment the metric for the number of successful requests.
|