fluent-plugin-google-cloud 0.6.9.pre.1 → 0.6.10.pre.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +23 -25
- data/fluent-plugin-google-cloud.gemspec +4 -4
- data/lib/fluent/plugin/out_google_cloud.rb +93 -77
- data/test/plugin/base_test.rb +37 -24
- data/test/plugin/constants.rb +119 -110
- data/test/plugin/test_out_google_cloud.rb +2 -1
- data/test/plugin/test_out_google_cloud_grpc.rb +41 -24
- metadata +4 -10
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 39661a11e58b52f626ece8c74056504cc8357f85
|
4
|
+
data.tar.gz: db78d67f4c56435d85011a66332f01c4fd6d69df
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2d856be37dcd8bf41c8c35dec9a1e712bf05e684cf2d824dc76a75c279c5f65ea8ebc9b8f3d01922f6e15479cfd0aa2fa87359f407c56227364a40922d0c9037
|
7
|
+
data.tar.gz: 58c0e3b7eba803069baa7bc411269e0f829a43cf76d8555427f03d8cc350e3151e567216504daa4ea10c91162af1e0a375e03ac58e263383a76b85a0667c25e8
|
data/Gemfile.lock
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
fluent-plugin-google-cloud (0.6.
|
4
|
+
fluent-plugin-google-cloud (0.6.10.pre.1)
|
5
5
|
fluentd (~> 0.10)
|
6
6
|
google-api-client (~> 0.14)
|
7
7
|
google-cloud-logging (~> 1.2, >= 1.2.3)
|
8
8
|
googleapis-common-protos (~> 1.3)
|
9
9
|
googleauth (~> 0.4, < 0.5.2)
|
10
|
-
grpc (~> 1.0
|
10
|
+
grpc (~> 1.0)
|
11
11
|
json (~> 1.8)
|
12
12
|
|
13
13
|
GEM
|
@@ -16,8 +16,6 @@ GEM
|
|
16
16
|
addressable (2.5.2)
|
17
17
|
public_suffix (>= 2.0.2, < 4.0)
|
18
18
|
ast (2.3.0)
|
19
|
-
astrolabe (1.3.1)
|
20
|
-
parser (~> 2.2)
|
21
19
|
cool.io (1.5.1)
|
22
20
|
crack (0.4.3)
|
23
21
|
safe_yaml (~> 1.0.0)
|
@@ -25,7 +23,7 @@ GEM
|
|
25
23
|
declarative-option (0.1.0)
|
26
24
|
faraday (0.13.1)
|
27
25
|
multipart-post (>= 1.2, < 3)
|
28
|
-
fluentd (0.14.
|
26
|
+
fluentd (0.14.22)
|
29
27
|
cool.io (>= 1.4.5, < 2.0.0)
|
30
28
|
http_parser.rb (>= 0.5.1, < 0.7.0)
|
31
29
|
msgpack (>= 0.7.0, < 2.0.0)
|
@@ -36,9 +34,9 @@ GEM
|
|
36
34
|
tzinfo (~> 1.0)
|
37
35
|
tzinfo-data (~> 1.0)
|
38
36
|
yajl-ruby (~> 1.0)
|
39
|
-
google-api-client (0.
|
37
|
+
google-api-client (0.17.1)
|
40
38
|
addressable (~> 2.5, >= 2.5.1)
|
41
|
-
googleauth (
|
39
|
+
googleauth (>= 0.5, < 0.7.0)
|
42
40
|
httpclient (>= 2.8.1, < 3.0)
|
43
41
|
mime-types (~> 3.0)
|
44
42
|
representable (~> 3.0)
|
@@ -59,11 +57,11 @@ GEM
|
|
59
57
|
grpc (~> 1.0)
|
60
58
|
rly (~> 0.2.3)
|
61
59
|
google-protobuf (3.4.1.1)
|
62
|
-
googleapis-common-protos (1.3.
|
60
|
+
googleapis-common-protos (1.3.7)
|
63
61
|
google-protobuf (~> 3.0)
|
64
62
|
googleapis-common-protos-types (~> 1.0)
|
65
63
|
grpc (~> 1.0)
|
66
|
-
googleapis-common-protos-types (1.0.
|
64
|
+
googleapis-common-protos-types (1.0.1)
|
67
65
|
google-protobuf (~> 3.0)
|
68
66
|
googleauth (0.5.1)
|
69
67
|
faraday (~> 0.9)
|
@@ -73,9 +71,10 @@ GEM
|
|
73
71
|
multi_json (~> 1.11)
|
74
72
|
os (~> 0.9)
|
75
73
|
signet (~> 0.7)
|
76
|
-
grpc (1.2
|
74
|
+
grpc (1.7.2)
|
77
75
|
google-protobuf (~> 3.1)
|
78
|
-
|
76
|
+
googleapis-common-protos-types (~> 1.0.0)
|
77
|
+
googleauth (>= 0.5.1, < 0.7)
|
79
78
|
hashdiff (0.3.7)
|
80
79
|
http_parser.rb (0.6.0)
|
81
80
|
httpclient (2.8.3)
|
@@ -96,13 +95,13 @@ GEM
|
|
96
95
|
multi_json (1.12.2)
|
97
96
|
multipart-post (2.0.0)
|
98
97
|
os (0.9.6)
|
99
|
-
parser (2.4.0.
|
100
|
-
ast (~> 2.
|
101
|
-
power_assert (1.1.
|
98
|
+
parser (2.4.0.2)
|
99
|
+
ast (~> 2.3)
|
100
|
+
power_assert (1.1.1)
|
102
101
|
powerpack (0.1.1)
|
103
102
|
prometheus-client (0.7.1)
|
104
103
|
quantile (~> 0.2.0)
|
105
|
-
public_suffix (3.0.
|
104
|
+
public_suffix (3.0.1)
|
106
105
|
quantile (0.2.0)
|
107
106
|
rainbow (2.2.2)
|
108
107
|
rake
|
@@ -113,13 +112,12 @@ GEM
|
|
113
112
|
uber (< 0.2.0)
|
114
113
|
retriable (3.1.1)
|
115
114
|
rly (0.2.3)
|
116
|
-
rubocop (0.
|
117
|
-
|
118
|
-
parser (>= 2.2.3.0, < 3.0)
|
115
|
+
rubocop (0.39.0)
|
116
|
+
parser (>= 2.3.0.7, < 3.0)
|
119
117
|
powerpack (~> 0.1)
|
120
118
|
rainbow (>= 1.99.1, < 3.0)
|
121
119
|
ruby-progressbar (~> 1.7)
|
122
|
-
|
120
|
+
unicode-display_width (~> 1.0, >= 1.0.1)
|
123
121
|
ruby-progressbar (1.9.0)
|
124
122
|
ruby_dig (0.0.2)
|
125
123
|
safe_yaml (1.0.4)
|
@@ -136,17 +134,17 @@ GEM
|
|
136
134
|
test-unit (3.2.6)
|
137
135
|
power_assert
|
138
136
|
thread_safe (0.3.6)
|
139
|
-
|
140
|
-
tzinfo (1.2.3)
|
137
|
+
tzinfo (1.2.4)
|
141
138
|
thread_safe (~> 0.1)
|
142
|
-
tzinfo-data (1.2017.
|
139
|
+
tzinfo-data (1.2017.3)
|
143
140
|
tzinfo (>= 1.0.0)
|
144
141
|
uber (0.1.0)
|
142
|
+
unicode-display_width (1.3.0)
|
145
143
|
webmock (2.3.2)
|
146
144
|
addressable (>= 2.3.6)
|
147
145
|
crack (>= 0.3.2)
|
148
146
|
hashdiff
|
149
|
-
yajl-ruby (1.3.
|
147
|
+
yajl-ruby (1.3.1)
|
150
148
|
|
151
149
|
PLATFORMS
|
152
150
|
ruby
|
@@ -156,9 +154,9 @@ DEPENDENCIES
|
|
156
154
|
mocha (~> 1.1)
|
157
155
|
prometheus-client (~> 0.7.1)
|
158
156
|
rake (~> 10.3)
|
159
|
-
rubocop (~> 0.
|
157
|
+
rubocop (~> 0.39.0)
|
160
158
|
test-unit (~> 3.0)
|
161
159
|
webmock (~> 2.3.1)
|
162
160
|
|
163
161
|
BUNDLED WITH
|
164
|
-
1.
|
162
|
+
1.16.0
|
@@ -7,10 +7,10 @@ Gem::Specification.new do |gem|
|
|
7
7
|
This is an official Google Ruby gem.
|
8
8
|
eos
|
9
9
|
gem.summary = 'fluentd output plugin for the Stackdriver Logging API'
|
10
|
-
gem.homepage =
|
10
|
+
gem.homepage =
|
11
11
|
'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
|
12
12
|
gem.license = 'Apache-2.0'
|
13
|
-
gem.version = '0.6.
|
13
|
+
gem.version = '0.6.10.pre.1'
|
14
14
|
gem.authors = ['Todd Derr', 'Alex Robinson']
|
15
15
|
gem.email = ['salty@google.com']
|
16
16
|
gem.required_ruby_version = Gem::Requirement.new('>= 2.0')
|
@@ -24,12 +24,12 @@ eos
|
|
24
24
|
gem.add_runtime_dependency 'google-api-client', '~> 0.14'
|
25
25
|
gem.add_runtime_dependency 'google-cloud-logging', '~> 1.2', '>= 1.2.3'
|
26
26
|
gem.add_runtime_dependency 'googleauth', '~> 0.4', '< 0.5.2'
|
27
|
-
gem.add_runtime_dependency 'grpc', '~> 1.0'
|
27
|
+
gem.add_runtime_dependency 'grpc', '~> 1.0'
|
28
28
|
gem.add_runtime_dependency 'json', '~> 1.8'
|
29
29
|
|
30
30
|
gem.add_development_dependency 'mocha', '~> 1.1'
|
31
31
|
gem.add_development_dependency 'rake', '~> 10.3'
|
32
|
-
gem.add_development_dependency 'rubocop', '~> 0.
|
32
|
+
gem.add_development_dependency 'rubocop', '~> 0.39.0'
|
33
33
|
gem.add_development_dependency 'webmock', '~> 2.3.1'
|
34
34
|
gem.add_development_dependency 'test-unit', '~> 3.0'
|
35
35
|
gem.add_development_dependency 'prometheus-client', '~> 0.7.1'
|
@@ -31,7 +31,7 @@ module Google
|
|
31
31
|
module Protobuf
|
32
32
|
# Alias the has_key? method to have the same interface as a regular map.
|
33
33
|
class Map
|
34
|
-
|
34
|
+
alias key? has_key?
|
35
35
|
end
|
36
36
|
end
|
37
37
|
end
|
@@ -45,80 +45,92 @@ module Fluent
|
|
45
45
|
service: 'appengine.googleapis.com',
|
46
46
|
resource_type: 'gae_app',
|
47
47
|
metadata_attributes: %w(gae_backend_name gae_backend_version)
|
48
|
-
}
|
48
|
+
}.freeze
|
49
49
|
CLOUDFUNCTIONS_CONSTANTS = {
|
50
50
|
service: 'cloudfunctions.googleapis.com',
|
51
|
-
resource_type: 'cloud_function'
|
52
|
-
|
51
|
+
resource_type: 'cloud_function',
|
52
|
+
stream_severity_map: {
|
53
|
+
'stdout' => 'INFO',
|
54
|
+
'stderr' => 'ERROR'
|
55
|
+
}
|
56
|
+
}.freeze
|
53
57
|
COMPUTE_CONSTANTS = {
|
54
58
|
service: 'compute.googleapis.com',
|
55
59
|
resource_type: 'gce_instance'
|
56
|
-
}
|
60
|
+
}.freeze
|
57
61
|
GKE_CONSTANTS = {
|
58
62
|
service: 'container.googleapis.com',
|
59
63
|
resource_type: 'container',
|
60
64
|
extra_resource_labels: %w(namespace_id pod_id container_name),
|
61
65
|
extra_common_labels: %w(namespace_name pod_name),
|
62
|
-
metadata_attributes: %w(kube-env)
|
63
|
-
|
66
|
+
metadata_attributes: %w(kube-env),
|
67
|
+
stream_severity_map: {
|
68
|
+
'stdout' => 'INFO',
|
69
|
+
'stderr' => 'ERROR'
|
70
|
+
}
|
71
|
+
}.freeze
|
64
72
|
DOCKER_CONSTANTS = {
|
65
73
|
service: 'docker.googleapis.com',
|
66
74
|
resource_type: 'docker_container'
|
67
|
-
}
|
75
|
+
}.freeze
|
68
76
|
DATAFLOW_CONSTANTS = {
|
69
77
|
service: 'dataflow.googleapis.com',
|
70
78
|
resource_type: 'dataflow_step',
|
71
79
|
extra_resource_labels: %w(region job_name job_id step_id)
|
72
|
-
}
|
80
|
+
}.freeze
|
73
81
|
DATAPROC_CONSTANTS = {
|
74
82
|
service: 'cluster.dataproc.googleapis.com',
|
75
83
|
resource_type: 'cloud_dataproc_cluster',
|
76
84
|
metadata_attributes: %w(dataproc-cluster-uuid dataproc-cluster-name)
|
77
|
-
}
|
85
|
+
}.freeze
|
78
86
|
EC2_CONSTANTS = {
|
79
87
|
service: 'ec2.amazonaws.com',
|
80
88
|
resource_type: 'aws_ec2_instance'
|
81
|
-
}
|
89
|
+
}.freeze
|
82
90
|
ML_CONSTANTS = {
|
83
91
|
service: 'ml.googleapis.com',
|
84
92
|
resource_type: 'ml_job',
|
85
93
|
extra_resource_labels: %w(job_id task_name)
|
86
|
-
}
|
94
|
+
}.freeze
|
87
95
|
|
88
96
|
# The map between a subservice name and a resource type.
|
89
|
-
SUBSERVICE_MAP =
|
97
|
+
SUBSERVICE_MAP =
|
90
98
|
[APPENGINE_CONSTANTS, GKE_CONSTANTS, DATAFLOW_CONSTANTS,
|
91
99
|
DATAPROC_CONSTANTS, ML_CONSTANTS]
|
92
100
|
.map { |consts| [consts[:service], consts[:resource_type]] }.to_h
|
93
101
|
# Default back to GCE if invalid value is detected.
|
94
102
|
SUBSERVICE_MAP.default = COMPUTE_CONSTANTS[:resource_type]
|
103
|
+
SUBSERVICE_MAP.freeze
|
95
104
|
|
96
105
|
# The map between a resource type and expected subservice attributes.
|
97
|
-
SUBSERVICE_METADATA_ATTRIBUTES =
|
106
|
+
SUBSERVICE_METADATA_ATTRIBUTES =
|
98
107
|
[APPENGINE_CONSTANTS, GKE_CONSTANTS, DATAPROC_CONSTANTS].map do |consts|
|
99
108
|
[consts[:resource_type], consts[:metadata_attributes].to_set]
|
100
|
-
end.to_h
|
109
|
+
end.to_h.freeze
|
101
110
|
end
|
102
111
|
|
103
112
|
# Constants for configuration.
|
104
113
|
module ConfigConstants
|
105
114
|
# Default values for JSON payload keys to set the "httpRequest",
|
106
115
|
# "operation", "sourceLocation", "trace" fields in the LogEntry.
|
107
|
-
DEFAULT_HTTP_REQUEST_KEY = 'httpRequest'
|
108
|
-
DEFAULT_OPERATION_KEY = 'logging.googleapis.com/operation'
|
109
|
-
DEFAULT_SOURCE_LOCATION_KEY =
|
110
|
-
|
116
|
+
DEFAULT_HTTP_REQUEST_KEY = 'httpRequest'.freeze
|
117
|
+
DEFAULT_OPERATION_KEY = 'logging.googleapis.com/operation'.freeze
|
118
|
+
DEFAULT_SOURCE_LOCATION_KEY =
|
119
|
+
'logging.googleapis.com/sourceLocation'.freeze
|
120
|
+
DEFAULT_TRACE_KEY = 'logging.googleapis.com/trace'.freeze
|
111
121
|
|
112
122
|
DEFAULT_METADATA_AGENT_URL =
|
113
|
-
'http://local-metadata-agent.stackdriver.com:8000'
|
123
|
+
'http://local-metadata-agent.stackdriver.com:8000'.freeze
|
114
124
|
end
|
115
125
|
|
116
126
|
# Internal constants.
|
117
127
|
module InternalConstants
|
128
|
+
DEFAULT_LOGGING_API_URL = 'logging.googleapis.com'.freeze
|
129
|
+
|
118
130
|
# The label name of local_resource_id in the json payload. When a record
|
119
131
|
# has this field in the payload, we will use the value to retrieve
|
120
132
|
# monitored resource from Stackdriver Metadata agent.
|
121
|
-
LOCAL_RESOURCE_ID_KEY = 'logging.googleapis.com/local_resource_id'
|
133
|
+
LOCAL_RESOURCE_ID_KEY = 'logging.googleapis.com/local_resource_id'.freeze
|
122
134
|
|
123
135
|
# Map from each field name under LogEntry to corresponding variables
|
124
136
|
# required to perform field value extraction from the log record.
|
@@ -168,11 +180,12 @@ module Fluent
|
|
168
180
|
'Google::Logging::V2::LogEntryOperation',
|
169
181
|
'Google::Apis::LoggingV2::LogEntryOperation'
|
170
182
|
]
|
171
|
-
}
|
183
|
+
}.freeze
|
172
184
|
|
173
185
|
# The name of the WriteLogEntriesPartialErrors field in the error details.
|
174
186
|
PARTIAL_ERROR_FIELD =
|
175
|
-
'type.googleapis.com/google.logging.v2.WriteLogEntriesPartialErrors'
|
187
|
+
'type.googleapis.com/google.logging.v2.WriteLogEntriesPartialErrors' \
|
188
|
+
.freeze
|
176
189
|
end
|
177
190
|
|
178
191
|
include self::ServiceConstants
|
@@ -181,14 +194,14 @@ module Fluent
|
|
181
194
|
|
182
195
|
Fluent::Plugin.register_output('google_cloud', self)
|
183
196
|
|
184
|
-
PLUGIN_NAME = 'Fluentd Google Cloud Logging plugin'
|
185
|
-
PLUGIN_VERSION = '0.6.
|
197
|
+
PLUGIN_NAME = 'Fluentd Google Cloud Logging plugin'.freeze
|
198
|
+
PLUGIN_VERSION = '0.6.10.pre.1'.freeze
|
186
199
|
|
187
200
|
# Name of the the Google cloud logging write scope.
|
188
|
-
LOGGING_SCOPE = 'https://www.googleapis.com/auth/logging.write'
|
201
|
+
LOGGING_SCOPE = 'https://www.googleapis.com/auth/logging.write'.freeze
|
189
202
|
|
190
203
|
# Address of the metadata service.
|
191
|
-
METADATA_SERVICE_ADDR = '169.254.169.254'
|
204
|
+
METADATA_SERVICE_ADDR = '169.254.169.254'.freeze
|
192
205
|
|
193
206
|
# Disable this warning to conform to fluentd config_param conventions.
|
194
207
|
# rubocop:disable Style/HashSyntax
|
@@ -310,6 +323,10 @@ module Fluent
|
|
310
323
|
:default => nil,
|
311
324
|
:secret => true
|
312
325
|
|
326
|
+
# The URL of Stackdriver Logging API. Right now this only works with the
|
327
|
+
# gRPC path (use_grpc = true).
|
328
|
+
config_param :logging_api_url, :string, :default => DEFAULT_LOGGING_API_URL
|
329
|
+
|
313
330
|
# Whether to collect metrics about the plugin usage. The mechanism for
|
314
331
|
# collecting and exposing metrics is controlled by the monitoring_type
|
315
332
|
# parameter.
|
@@ -358,6 +375,15 @@ module Fluent
|
|
358
375
|
' partial_success flag will be ignored.'
|
359
376
|
end
|
360
377
|
|
378
|
+
# TODO(qingling128): Remove this warning after the support is added. Also
|
379
|
+
# remove the comment in the description of this configuration.
|
380
|
+
unless @logging_api_url == DEFAULT_LOGGING_API_URL || @use_grpc
|
381
|
+
@log.warn 'Detected customized logging_api_url while use_grpc is not' \
|
382
|
+
' enabled. Customized logging_api_url for the non-gRPC path' \
|
383
|
+
' is not supported. The logging_api_url option will be' \
|
384
|
+
' ignored.'
|
385
|
+
end
|
386
|
+
|
361
387
|
# If monitoring is enabled, register metrics in the default registry
|
362
388
|
# and store metric objects for future use.
|
363
389
|
if @enable_monitoring
|
@@ -391,10 +417,9 @@ module Fluent
|
|
391
417
|
extra << 'private_key_path' unless @private_key_path.nil?
|
392
418
|
extra << 'private_key_passphrase' unless @private_key_passphrase.nil?
|
393
419
|
|
394
|
-
|
395
|
-
|
396
|
-
|
397
|
-
extra.join(' ')
|
420
|
+
raise Fluent::ConfigError,
|
421
|
+
"#{PLUGIN_NAME} no longer supports auth_method.\n" \
|
422
|
+
"Please remove configuration parameters: #{extra.join(' ')}"
|
398
423
|
end
|
399
424
|
|
400
425
|
set_regexp_patterns
|
@@ -641,7 +666,7 @@ module Fluent
|
|
641
666
|
end
|
642
667
|
else
|
643
668
|
begin
|
644
|
-
write_request =
|
669
|
+
write_request =
|
645
670
|
Google::Apis::LoggingV2::WriteLogEntriesRequest.new(
|
646
671
|
log_name: log_name,
|
647
672
|
resource: group_level_resource,
|
@@ -767,7 +792,7 @@ module Fluent
|
|
767
792
|
end
|
768
793
|
|
769
794
|
def fetch_gce_metadata(metadata_path)
|
770
|
-
|
795
|
+
raise "Called fetch_gce_metadata with platform=#{@platform}" unless
|
771
796
|
@platform == Platform::GCE
|
772
797
|
# See https://cloud.google.com/compute/docs/metadata
|
773
798
|
open('http://' + METADATA_SERVICE_ADDR + '/computeMetadata/v1/' +
|
@@ -777,7 +802,7 @@ module Fluent
|
|
777
802
|
# EC2 Metadata server returns everything in one call. Store it after the
|
778
803
|
# first fetch to avoid making multiple calls.
|
779
804
|
def ec2_metadata
|
780
|
-
|
805
|
+
raise "Called ec2_metadata with platform=#{@platform}" unless
|
781
806
|
@platform == Platform::EC2
|
782
807
|
unless @ec2_metadata
|
783
808
|
# See http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html
|
@@ -821,8 +846,8 @@ module Fluent
|
|
821
846
|
missing << 'zone' unless @zone
|
822
847
|
missing << 'vm_id' unless @vm_id
|
823
848
|
return if missing.empty?
|
824
|
-
|
825
|
-
|
849
|
+
raise Fluent::ConfigError,
|
850
|
+
"Unable to obtain metadata parameters: #{missing.join(' ')}"
|
826
851
|
end
|
827
852
|
|
828
853
|
# 1. Return the value if it is explicitly set in the config already.
|
@@ -991,13 +1016,11 @@ module Fluent
|
|
991
1016
|
# GCE instance and GKE container.
|
992
1017
|
when COMPUTE_CONSTANTS[:resource_type],
|
993
1018
|
GKE_CONSTANTS[:resource_type]
|
994
|
-
labels
|
995
|
-
"#{COMPUTE_CONSTANTS[:service]}/resource_name" => @vm_name)
|
1019
|
+
labels["#{COMPUTE_CONSTANTS[:service]}/resource_name"] = @vm_name
|
996
1020
|
|
997
1021
|
# EC2.
|
998
1022
|
when EC2_CONSTANTS[:resource_type]
|
999
|
-
labels
|
1000
|
-
"#{EC2_CONSTANTS[:service]}/resource_name" => @vm_name)
|
1023
|
+
labels["#{EC2_CONSTANTS[:service]}/resource_name"] = @vm_name
|
1001
1024
|
end
|
1002
1025
|
labels
|
1003
1026
|
end
|
@@ -1354,29 +1377,17 @@ module Fluent
|
|
1354
1377
|
if resource_type == CLOUDFUNCTIONS_CONSTANTS[:resource_type]
|
1355
1378
|
if @cloudfunctions_log_match && @cloudfunctions_log_match['severity']
|
1356
1379
|
return parse_severity(@cloudfunctions_log_match['severity'])
|
1357
|
-
elsif record.key?('stream')
|
1358
|
-
|
1359
|
-
|
1360
|
-
elsif record.key?('stream') && record['stream'] == 'stderr'
|
1361
|
-
record.delete('stream')
|
1362
|
-
return 'ERROR'
|
1363
|
-
else
|
1364
|
-
return 'DEFAULT'
|
1380
|
+
elsif record.key?('stream')
|
1381
|
+
return CLOUDFUNCTIONS_CONSTANTS[:stream_severity_map].fetch(
|
1382
|
+
record.delete('stream'), 'DEFAULT')
|
1365
1383
|
end
|
1366
1384
|
elsif record.key?('severity')
|
1367
1385
|
return parse_severity(record.delete('severity'))
|
1368
1386
|
elsif resource_type == GKE_CONSTANTS[:resource_type]
|
1369
1387
|
stream = entry_level_common_labels["#{GKE_CONSTANTS[:service]}/stream"]
|
1370
|
-
|
1371
|
-
return 'INFO'
|
1372
|
-
elsif stream == 'stderr'
|
1373
|
-
return 'ERROR'
|
1374
|
-
else
|
1375
|
-
return 'DEFAULT'
|
1376
|
-
end
|
1377
|
-
else
|
1378
|
-
return 'DEFAULT'
|
1388
|
+
return GKE_CONSTANTS[:stream_severity_map].fetch(stream, 'DEFAULT')
|
1379
1389
|
end
|
1390
|
+
'DEFAULT'
|
1380
1391
|
end
|
1381
1392
|
|
1382
1393
|
def set_log_entry_fields(record, entry)
|
@@ -1404,11 +1415,11 @@ module Fluent
|
|
1404
1415
|
|
1405
1416
|
next unless extracted_subfields
|
1406
1417
|
|
1407
|
-
if @use_grpc
|
1408
|
-
|
1409
|
-
|
1410
|
-
|
1411
|
-
|
1418
|
+
output = if @use_grpc
|
1419
|
+
Object.const_get(grpc_class).new
|
1420
|
+
else
|
1421
|
+
Object.const_get(non_grpc_class).new
|
1422
|
+
end
|
1412
1423
|
extracted_subfields.each do |key, value|
|
1413
1424
|
output.send("#{key}=", value)
|
1414
1425
|
end
|
@@ -1424,7 +1435,8 @@ module Fluent
|
|
1424
1435
|
|
1425
1436
|
# Values permitted by the API for 'severity' (which is an enum).
|
1426
1437
|
VALID_SEVERITIES = Set.new(
|
1427
|
-
%w(DEFAULT DEBUG INFO NOTICE WARNING ERROR CRITICAL ALERT EMERGENCY)
|
1438
|
+
%w(DEFAULT DEBUG INFO NOTICE WARNING ERROR CRITICAL ALERT EMERGENCY)
|
1439
|
+
).freeze
|
1428
1440
|
|
1429
1441
|
# Translates other severity strings to one of the valid values above.
|
1430
1442
|
SEVERITY_TRANSLATIONS = {
|
@@ -1453,7 +1465,7 @@ module Fluent
|
|
1453
1465
|
# other misc. translations.
|
1454
1466
|
'ERR' => 'ERROR',
|
1455
1467
|
'F' => 'CRITICAL'
|
1456
|
-
}
|
1468
|
+
}.freeze
|
1457
1469
|
|
1458
1470
|
def parse_severity(severity_str)
|
1459
1471
|
# The API is case insensitive, but uppercase to make things simpler.
|
@@ -1464,12 +1476,13 @@ module Fluent
|
|
1464
1476
|
|
1465
1477
|
# If the severity is an integer (string) return it as an integer,
|
1466
1478
|
# truncated to the closest valid value (multiples of 100 between 0-800).
|
1467
|
-
if /\A\d+\z
|
1479
|
+
if /\A\d+\z/ =~ severity
|
1468
1480
|
begin
|
1469
1481
|
numeric_severity = (severity.to_i / 100) * 100
|
1470
|
-
|
1482
|
+
case
|
1483
|
+
when numeric_severity < 0
|
1471
1484
|
return 0
|
1472
|
-
|
1485
|
+
when numeric_severity > 800
|
1473
1486
|
return 800
|
1474
1487
|
else
|
1475
1488
|
return numeric_severity
|
@@ -1507,7 +1520,7 @@ module Fluent
|
|
1507
1520
|
600 => Google::Logging::Type::LogSeverity::CRITICAL,
|
1508
1521
|
700 => Google::Logging::Type::LogSeverity::ALERT,
|
1509
1522
|
800 => Google::Logging::Type::LogSeverity::EMERGENCY
|
1510
|
-
}
|
1523
|
+
}.freeze
|
1511
1524
|
|
1512
1525
|
def grpc_severity(severity)
|
1513
1526
|
# TODO: find out why this doesn't work.
|
@@ -1561,11 +1574,14 @@ module Fluent
|
|
1561
1574
|
|
1562
1575
|
def decode_cloudfunctions_function_name(function_name)
|
1563
1576
|
function_name.gsub(/c\.[a-z]/) { |s| s.upcase[-1] }
|
1564
|
-
|
1577
|
+
.gsub('u.u', '_')
|
1578
|
+
.gsub('d.d', '$')
|
1579
|
+
.gsub('a.a', '@')
|
1580
|
+
.gsub('p.p', '.')
|
1565
1581
|
end
|
1566
1582
|
|
1567
1583
|
def format(tag, time, record)
|
1568
|
-
[tag, time, record].
|
1584
|
+
Fluent::Engine.msgpack_factory.packer.write([tag, time, record]).to_s
|
1569
1585
|
end
|
1570
1586
|
|
1571
1587
|
# Given a tag, returns the corresponding valid tag if possible, or nil if
|
@@ -1618,7 +1634,7 @@ module Fluent
|
|
1618
1634
|
ret.list_value = list_from_ruby(value)
|
1619
1635
|
else
|
1620
1636
|
@log.error "Unknown type: #{value.class}"
|
1621
|
-
|
1637
|
+
raise Google::Protobuf::Error, "Unknown type: #{value.class}"
|
1622
1638
|
end
|
1623
1639
|
ret
|
1624
1640
|
end
|
@@ -1717,7 +1733,7 @@ module Fluent
|
|
1717
1733
|
creds = GRPC::Core::CallCredentials.new(authentication.updater_proc)
|
1718
1734
|
creds = ssl_creds.compose(creds)
|
1719
1735
|
@client = Google::Logging::V2::LoggingServiceV2::Stub.new(
|
1720
|
-
|
1736
|
+
@logging_api_url, creds)
|
1721
1737
|
else
|
1722
1738
|
unless @client.authorization.expired?
|
1723
1739
|
begin
|
@@ -1839,7 +1855,7 @@ module Fluent
|
|
1839
1855
|
error_details = ensure_array(
|
1840
1856
|
ensure_hash(ensure_hash(JSON.parse(error.body))['error'])['details'])
|
1841
1857
|
partial_errors = error_details.detect(
|
1842
|
-
-> {
|
1858
|
+
-> { raise JSON::ParserError, "No type #{PARTIAL_ERROR_FIELD}." }
|
1843
1859
|
) do |error_detail|
|
1844
1860
|
ensure_hash(error_detail)['@type'] == PARTIAL_ERROR_FIELD
|
1845
1861
|
end
|
@@ -1847,8 +1863,8 @@ module Fluent
|
|
1847
1863
|
ensure_hash(partial_errors)['logEntryErrors'])
|
1848
1864
|
log_entry_errors.each do |index, log_entry_error|
|
1849
1865
|
error_hash = ensure_hash(log_entry_error)
|
1850
|
-
|
1851
|
-
|
1866
|
+
raise JSON::ParserError,
|
1867
|
+
"Entry #{index} is missing 'code' or 'message'." unless
|
1852
1868
|
error_hash['code'] && error_hash['message']
|
1853
1869
|
error_key = [error_hash['code'], error_hash['message']].freeze
|
1854
1870
|
# TODO(qingling128): Convert indexes to integers.
|
@@ -1862,11 +1878,11 @@ module Fluent
|
|
1862
1878
|
end
|
1863
1879
|
|
1864
1880
|
def ensure_array(value)
|
1865
|
-
Array.try_convert(value) || (
|
1881
|
+
Array.try_convert(value) || (raise JSON::ParserError, value.class.to_s)
|
1866
1882
|
end
|
1867
1883
|
|
1868
1884
|
def ensure_hash(value)
|
1869
|
-
Hash.try_convert(value) || (
|
1885
|
+
Hash.try_convert(value) || (raise JSON::ParserError, value.class.to_s)
|
1870
1886
|
end
|
1871
1887
|
|
1872
1888
|
# Increment the metric for the number of successful requests.
|