fluent-plugin-google-cloud 0.4.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile.lock +19 -4
- data/README.rdoc +27 -16
- data/Rakefile +10 -0
- data/fluent-plugin-google-cloud.gemspec +18 -10
- data/lib/fluent/plugin/out_google_cloud.rb +102 -103
- data/test/helper.rb +7 -9
- data/test/plugin/test_out_google_cloud.rb +190 -169
- metadata +40 -24
data/Gemfile.lock
CHANGED
@@ -1,9 +1,9 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
fluent-plugin-google-cloud (0.4.
|
4
|
+
fluent-plugin-google-cloud (0.4.3)
|
5
5
|
fluentd (>= 0.10)
|
6
|
-
google-api-client (
|
6
|
+
google-api-client (~> 0.8.6)
|
7
7
|
googleauth (~> 0.4)
|
8
8
|
json (~> 1.8.2)
|
9
9
|
|
@@ -17,6 +17,9 @@ GEM
|
|
17
17
|
thread_safe (~> 0.3, >= 0.3.4)
|
18
18
|
tzinfo (~> 1.1)
|
19
19
|
addressable (2.3.8)
|
20
|
+
ast (2.0.0)
|
21
|
+
astrolabe (1.3.1)
|
22
|
+
parser (~> 2.2)
|
20
23
|
autoparse (0.3.3)
|
21
24
|
addressable (>= 2.3.1)
|
22
25
|
extlib (>= 0.9.15)
|
@@ -27,7 +30,7 @@ GEM
|
|
27
30
|
extlib (0.9.16)
|
28
31
|
faraday (0.9.1)
|
29
32
|
multipart-post (>= 1.2, < 3)
|
30
|
-
fluentd (0.12.
|
33
|
+
fluentd (0.12.14)
|
31
34
|
cool.io (>= 1.2.2, < 2.0.0)
|
32
35
|
http_parser.rb (>= 0.5.1, < 0.7.0)
|
33
36
|
json (>= 1.4.3)
|
@@ -73,9 +76,20 @@ GEM
|
|
73
76
|
msgpack (0.5.12)
|
74
77
|
multi_json (1.11.0)
|
75
78
|
multipart-post (2.0.0)
|
76
|
-
|
79
|
+
parser (2.2.2.6)
|
80
|
+
ast (>= 1.1, < 3.0)
|
81
|
+
power_assert (0.2.4)
|
82
|
+
powerpack (0.1.1)
|
83
|
+
rainbow (2.0.0)
|
77
84
|
rake (10.4.2)
|
78
85
|
retriable (1.4.1)
|
86
|
+
rubocop (0.32.1)
|
87
|
+
astrolabe (~> 1.3)
|
88
|
+
parser (>= 2.2.2.5, < 3.0)
|
89
|
+
powerpack (~> 0.1)
|
90
|
+
rainbow (>= 1.99.1, < 3.0)
|
91
|
+
ruby-progressbar (~> 1.4)
|
92
|
+
ruby-progressbar (1.7.5)
|
79
93
|
safe_yaml (1.0.4)
|
80
94
|
sigdump (0.2.3)
|
81
95
|
signet (0.6.1)
|
@@ -104,6 +118,7 @@ DEPENDENCIES
|
|
104
118
|
fluent-plugin-google-cloud!
|
105
119
|
mocha (~> 1.1)
|
106
120
|
rake (>= 10.3.2)
|
121
|
+
rubocop (~> 0.30)
|
107
122
|
test-unit (~> 3.0.2)
|
108
123
|
webmock (>= 1.17.0)
|
109
124
|
|
data/README.rdoc
CHANGED
@@ -1,33 +1,44 @@
|
|
1
|
-
= Google Cloud
|
1
|
+
= Google Cloud Logging plugin for {fluentd}[http://github.com/fluent/fluentd]
|
2
2
|
|
3
|
-
fluent-plugin-google-cloud
|
3
|
+
fluent-plugin-google-cloud is an
|
4
|
+
{output plugin for fluentd}[http://docs.fluentd.org/articles/output-plugin-overview]
|
5
|
+
which sends logs to the
|
6
|
+
{Google Cloud Logging API}[https://cloud.google.com/logging/docs/api/].
|
7
|
+
|
8
|
+
This is an official Google Ruby gem.
|
9
|
+
|
10
|
+
{<img src="https://badge.fury.io/rb/fluent-plugin-google-cloud.svg" alt="Gem Version" />}[http://badge.fury.io/rb/fluent-plugin-google-cloud]
|
11
|
+
{<img src="https://secure.travis-ci.org/google/google-auth-library-ruby.png" alt="Build Status" />}[https://travis-ci.org/GoogleCloudPlatform/fluent-plugin-google-cloud]
|
4
12
|
|
5
13
|
== Installation
|
6
14
|
|
7
|
-
|
15
|
+
This gem is hosted at
|
16
|
+
{RubyGems.org}[https://rubygems.org/gems/fluent-plugin-google-cloud]
|
17
|
+
and can be installed using:
|
8
18
|
|
9
19
|
$ gem install fluent-plugin-google-cloud
|
10
20
|
|
21
|
+
Installing {google-fluentd}[https://cloud.google.com/logging/docs/agent/]
|
22
|
+
will also install and configure the gem.
|
23
|
+
|
11
24
|
== Configuration
|
12
25
|
|
26
|
+
To send logs to Google Cloud Logging, specify <code>type google_cloud</code>
|
27
|
+
in a
|
28
|
+
{match clause}[http://docs.fluentd.org/articles/config-file#2-ldquomatchrdquo-tell-fluentd-what-to-do]
|
29
|
+
of your fluentd configuration file, for example:
|
30
|
+
|
13
31
|
<match **>
|
14
32
|
type google_cloud
|
15
|
-
auth_method <method>
|
16
|
-
private_key_email <address>
|
17
|
-
private_key_path <path>
|
18
33
|
</match>
|
19
34
|
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
private_key_email and private_key_path are required if auth_method is
|
26
|
-
'private_key', otherwise they are ignored.
|
27
|
-
|
28
|
-
== Caveats
|
35
|
+
No further configuration is required. The plugin uses
|
36
|
+
{Google Application Default Credentials}[https://developers.google.com/identity/protocols/application-default-credentials]
|
37
|
+
for authorization - for additional information see
|
38
|
+
{here}[https://cloud.google.com/logging/docs/agent/authorization].
|
29
39
|
|
30
|
-
|
40
|
+
<em>The previously documented parameters auth_method, private_key_email,
|
41
|
+
and private_key_path are deprecated and should no longer be used.</em>
|
31
42
|
|
32
43
|
== Copyright
|
33
44
|
|
data/Rakefile
CHANGED
@@ -4,9 +4,19 @@ require 'bundler'
|
|
4
4
|
Bundler::GemHelper.install_tasks
|
5
5
|
|
6
6
|
require 'rake/testtask'
|
7
|
+
require 'rubocop/rake_task'
|
7
8
|
|
9
|
+
desc 'Run Rubocop to check for style violations'
|
10
|
+
RuboCop::RakeTask.new
|
11
|
+
|
12
|
+
desc 'Run unit tests'
|
8
13
|
Rake::TestTask.new(:test) do |test|
|
9
14
|
test.libs << 'lib' << 'test'
|
10
15
|
test.test_files = FileList['test/plugin/*.rb']
|
11
16
|
test.verbose = true
|
12
17
|
end
|
18
|
+
|
19
|
+
desc 'Does rubocop lint and runs tests'
|
20
|
+
task all: [:rubocop, :test]
|
21
|
+
|
22
|
+
task default: :all
|
@@ -1,23 +1,31 @@
|
|
1
1
|
Gem::Specification.new do |gem|
|
2
2
|
gem.name = 'fluent-plugin-google-cloud'
|
3
|
-
gem.description =
|
4
|
-
|
5
|
-
|
3
|
+
gem.description = <<-eos
|
4
|
+
Fluentd output plugin for the Google Cloud Logging API, which will make
|
5
|
+
loge viewable in the Developer Console's log viewer and can optionally
|
6
|
+
store them in Google Cloud Storage and/or BigQuery.
|
7
|
+
This is an official Google Ruby gem.'
|
8
|
+
eos
|
9
|
+
gem.summary = 'fluentd output plugin for the Google Cloud Logging API'
|
10
|
+
gem.homepage = \
|
11
|
+
'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
|
6
12
|
gem.license = 'Apache 2.0'
|
7
|
-
gem.version = '0.4.
|
13
|
+
gem.version = '0.4.3'
|
8
14
|
gem.authors = ['Todd Derr', 'Alex Robinson']
|
9
15
|
gem.email = ['salty@google.com']
|
10
16
|
|
11
17
|
gem.files = Dir['**/*'].keep_if { |file| File.file?(file) }
|
12
|
-
gem.test_files = gem.files.grep(
|
18
|
+
gem.test_files = gem.files.grep(/^(test)/)
|
13
19
|
gem.require_paths = ['lib']
|
14
20
|
|
15
21
|
gem.add_runtime_dependency 'fluentd', '>= 0.10'
|
16
|
-
gem.add_runtime_dependency 'google-api-client', '
|
22
|
+
gem.add_runtime_dependency 'google-api-client', '~> 0.8.6'
|
17
23
|
gem.add_runtime_dependency 'googleauth', '~> 0.4'
|
18
24
|
gem.add_runtime_dependency 'json', '~> 1.8.2'
|
19
|
-
|
20
|
-
gem.add_development_dependency
|
21
|
-
gem.add_development_dependency
|
22
|
-
gem.add_development_dependency
|
25
|
+
|
26
|
+
gem.add_development_dependency 'mocha', '~> 1.1'
|
27
|
+
gem.add_development_dependency 'rake', '>= 10.3.2'
|
28
|
+
gem.add_development_dependency 'rubocop', '~> 0.30'
|
29
|
+
gem.add_development_dependency 'webmock', '>= 1.17.0'
|
30
|
+
gem.add_development_dependency 'test-unit', '~> 3.0.2'
|
23
31
|
end
|
@@ -13,6 +13,7 @@
|
|
13
13
|
# limitations under the License.
|
14
14
|
|
15
15
|
module Fluent
|
16
|
+
# fluentd output plugin for the Google Cloud Logging API
|
16
17
|
class GoogleCloudOutput < BufferedOutput
|
17
18
|
Fluent::Plugin.register_output('google_cloud', self)
|
18
19
|
|
@@ -28,6 +29,9 @@ module Fluent
|
|
28
29
|
# Address of the metadata service.
|
29
30
|
METADATA_SERVICE_ADDR = '169.254.169.254'
|
30
31
|
|
32
|
+
# Disable this warning to conform to fluentd config_param conventions.
|
33
|
+
# rubocop:disable Style/HashSyntax
|
34
|
+
|
31
35
|
# DEPRECATED: auth_method (and support for 'private_key') is deprecated in
|
32
36
|
# favor of Google Application Default Credentials as documented at:
|
33
37
|
# https://developers.google.com/identity/protocols/application-default-credentials
|
@@ -74,6 +78,8 @@ module Fluent
|
|
74
78
|
# }
|
75
79
|
config_param :label_map, :hash, :default => nil
|
76
80
|
|
81
|
+
# rubocop:enable Style/HashSyntax
|
82
|
+
|
77
83
|
# TODO: Add a log_name config option rather than just using the tag?
|
78
84
|
|
79
85
|
# Expose attr_readers to make testing of metadata more direct than only
|
@@ -95,24 +101,27 @@ module Fluent
|
|
95
101
|
require 'googleauth'
|
96
102
|
require 'json'
|
97
103
|
require 'open-uri'
|
104
|
+
|
105
|
+
# use the global logger
|
106
|
+
@log = $log # rubocop:disable Style/GlobalVars
|
98
107
|
end
|
99
108
|
|
100
109
|
def configure(conf)
|
101
110
|
super
|
102
111
|
|
103
|
-
|
104
|
-
|
105
|
-
|
112
|
+
unless @auth_method.nil?
|
113
|
+
@log.warn 'auth_method is deprecated; please migrate to using ' \
|
114
|
+
'Application Default Credentials.'
|
106
115
|
if @auth_method == 'private_key'
|
107
116
|
if !@private_key_email
|
108
|
-
|
109
|
-
|
117
|
+
fail Fluent::ConfigError, '"private_key_email" must be ' \
|
118
|
+
'specified if auth_method is "private_key"'
|
110
119
|
elsif !@private_key_path
|
111
|
-
|
112
|
-
|
120
|
+
fail Fluent::ConfigError, '"private_key_path" must be ' \
|
121
|
+
'specified if auth_method is "private_key"'
|
113
122
|
elsif !@private_key_passphrase
|
114
|
-
|
115
|
-
|
123
|
+
fail Fluent::ConfigError, '"private_key_passphrase" must be ' \
|
124
|
+
'specified if auth_method is "private_key"'
|
116
125
|
end
|
117
126
|
end
|
118
127
|
end
|
@@ -133,34 +142,32 @@ module Fluent
|
|
133
142
|
fully_qualified_zone = fetch_gce_metadata('instance/zone')
|
134
143
|
@zone = fully_qualified_zone.rpartition('/')[2]
|
135
144
|
end
|
136
|
-
if @vm_id.nil?
|
137
|
-
@vm_id = fetch_gce_metadata('instance/id')
|
138
|
-
end
|
145
|
+
@vm_id = fetch_gce_metadata('instance/id') if @vm_id.nil?
|
139
146
|
when Platform::EC2
|
140
147
|
metadata = fetch_ec2_metadata
|
141
|
-
if @zone.nil? && metadata.
|
148
|
+
if @zone.nil? && metadata.key?('availabilityZone')
|
142
149
|
@zone = 'aws:' + metadata['availabilityZone']
|
143
150
|
end
|
144
|
-
if @vm_id.nil? && metadata.
|
151
|
+
if @vm_id.nil? && metadata.key?('instanceId')
|
145
152
|
@vm_id = metadata['instanceId']
|
146
153
|
end
|
147
|
-
if metadata.
|
154
|
+
if metadata.key?('accountId')
|
148
155
|
common_labels["#{EC2_SERVICE}/account_id"] = metadata['accountId']
|
149
156
|
end
|
150
157
|
when Platform::OTHER
|
151
158
|
# do nothing
|
152
159
|
else
|
153
|
-
|
160
|
+
fail Fluent::ConfigError, 'Unknown platform ' + @platform
|
154
161
|
end
|
155
162
|
|
156
163
|
# all metadata parameters must now be set
|
157
164
|
unless @project_id && @zone && @vm_id
|
158
165
|
missing = []
|
159
|
-
missing <<
|
160
|
-
missing <<
|
161
|
-
missing <<
|
162
|
-
|
163
|
-
|
166
|
+
missing << 'project_id' unless @project_id
|
167
|
+
missing << 'zone' unless @zone
|
168
|
+
missing << 'vm_id' unless @vm_id
|
169
|
+
fail Fluent::ConfigError, 'Unable to obtain metadata parameters: ' +
|
170
|
+
missing.join(' ')
|
164
171
|
end
|
165
172
|
|
166
173
|
# Default this to false; it is only overwritten if we detect Managed VM.
|
@@ -173,8 +180,8 @@ module Fluent
|
|
173
180
|
# Check for specialized GCE environments (Managed VM or Dataflow).
|
174
181
|
# TODO: Add config options for these to allow for running outside GCE?
|
175
182
|
attributes = fetch_gce_metadata('instance/attributes/').split
|
176
|
-
if
|
177
|
-
|
183
|
+
if attributes.include?('gae_backend_name') &&
|
184
|
+
attributes.include?('gae_backend_version')
|
178
185
|
# Managed VM
|
179
186
|
@running_on_managed_vm = true
|
180
187
|
@gae_backend_name =
|
@@ -185,7 +192,7 @@ module Fluent
|
|
185
192
|
common_labels["#{APPENGINE_SERVICE}/module_id"] = @gae_backend_name
|
186
193
|
common_labels["#{APPENGINE_SERVICE}/version_id"] =
|
187
194
|
@gae_backend_version
|
188
|
-
elsif
|
195
|
+
elsif attributes.include?('job_id')
|
189
196
|
# Dataflow
|
190
197
|
@service_name = DATAFLOW_SERVICE
|
191
198
|
@dataflow_job_id = fetch_gce_metadata('instance/attributes/job_id')
|
@@ -212,7 +219,7 @@ module Fluent
|
|
212
219
|
def start
|
213
220
|
super
|
214
221
|
|
215
|
-
init_api_client
|
222
|
+
init_api_client
|
216
223
|
|
217
224
|
@successful_call = false
|
218
225
|
@timenanos_warning = false
|
@@ -230,43 +237,41 @@ module Fluent
|
|
230
237
|
# Group the entries since we have to make one call per tag.
|
231
238
|
grouped_entries = {}
|
232
239
|
chunk.msgpack_each do |tag, *arr|
|
233
|
-
|
234
|
-
grouped_entries[tag] = []
|
235
|
-
end
|
240
|
+
grouped_entries[tag] = [] unless grouped_entries.key?(tag)
|
236
241
|
grouped_entries[tag].push(arr)
|
237
242
|
end
|
238
243
|
|
239
244
|
grouped_entries.each do |tag, arr|
|
240
245
|
write_log_entries_request = {
|
241
246
|
'commonLabels' => @common_labels,
|
242
|
-
'entries' => []
|
247
|
+
'entries' => []
|
243
248
|
}
|
244
249
|
arr.each do |time, record|
|
245
250
|
next unless record.is_a?(Hash)
|
246
|
-
if
|
247
|
-
|
248
|
-
|
249
|
-
|
251
|
+
if record.key?('timestamp') &&
|
252
|
+
record['timestamp'].is_a?(Hash) &&
|
253
|
+
record['timestamp'].key?('seconds') &&
|
254
|
+
record['timestamp'].key?('nanos')
|
250
255
|
ts_secs = record['timestamp']['seconds']
|
251
256
|
ts_nanos = record['timestamp']['nanos']
|
252
257
|
record.delete('timestamp')
|
253
|
-
elsif
|
254
|
-
|
258
|
+
elsif record.key?('timestampSeconds') &&
|
259
|
+
record.key?('timestampNanos')
|
255
260
|
ts_secs = record['timestampSeconds']
|
256
261
|
ts_nanos = record['timestampNanos']
|
257
262
|
record.delete('timestampSeconds')
|
258
263
|
record.delete('timestampNanos')
|
259
|
-
elsif
|
264
|
+
elsif record.key?('timeNanos')
|
260
265
|
# This is deprecated since the precision is insufficient.
|
261
266
|
# Use timestampSeconds/timestampNanos instead
|
262
|
-
ts_secs = (record['timeNanos'] /
|
263
|
-
ts_nanos = record['timeNanos'] %
|
267
|
+
ts_secs = (record['timeNanos'] / 1_000_000_000).to_i
|
268
|
+
ts_nanos = record['timeNanos'] % 1_000_000_000
|
264
269
|
record.delete('timeNanos')
|
265
|
-
|
270
|
+
unless @timenanos_warning
|
266
271
|
# Warn the user this is deprecated, but only once to avoid spam.
|
267
272
|
@timenanos_warning = true
|
268
|
-
|
269
|
-
|
273
|
+
@log.warn 'timeNanos is deprecated - please use ' \
|
274
|
+
'timestampSeconds and timestampNanos instead.'
|
270
275
|
end
|
271
276
|
else
|
272
277
|
timestamp = Time.at(time)
|
@@ -281,10 +286,10 @@ module Fluent
|
|
281
286
|
'timestamp' => {
|
282
287
|
'seconds' => ts_secs,
|
283
288
|
'nanos' => ts_nanos
|
284
|
-
}
|
285
|
-
}
|
289
|
+
}
|
290
|
+
}
|
286
291
|
}
|
287
|
-
if record.
|
292
|
+
if record.key?('severity')
|
288
293
|
entry['metadata']['severity'] = parse_severity(record['severity'])
|
289
294
|
record.delete('severity')
|
290
295
|
else
|
@@ -294,22 +299,20 @@ module Fluent
|
|
294
299
|
# If a field is present in the label_map, send its value as a label
|
295
300
|
# (mapping the field name to label name as specified in the config)
|
296
301
|
# and do not send that field as part of the payload.
|
297
|
-
|
302
|
+
unless label_map.nil?
|
298
303
|
labels = {}
|
299
304
|
@label_map.each do |field, label|
|
300
|
-
if record.
|
305
|
+
if record.key?(field)
|
301
306
|
labels[label] = record[field]
|
302
307
|
record.delete(field)
|
303
308
|
end
|
304
309
|
end
|
305
|
-
|
306
|
-
entry['metadata']['labels'] = labels
|
307
|
-
end
|
310
|
+
entry['metadata']['labels'] = labels unless labels.empty?
|
308
311
|
end
|
309
312
|
|
310
313
|
# use textPayload if the only remainaing key is 'message',
|
311
314
|
# otherwise use a struct.
|
312
|
-
if
|
315
|
+
if record.size == 1 && record.key?('message')
|
313
316
|
entry['textPayload'] = record['message']
|
314
317
|
else
|
315
318
|
entry['structPayload'] = record
|
@@ -321,24 +324,24 @@ module Fluent
|
|
321
324
|
|
322
325
|
# Add a prefix to VMEngines logs to prevent namespace collisions,
|
323
326
|
# and also escape the log name.
|
324
|
-
log_name = CGI
|
325
|
-
|
326
|
-
url =
|
327
|
-
|
327
|
+
log_name = CGI.escape(
|
328
|
+
@running_on_managed_vm ? "#{APPENGINE_SERVICE}/#{tag}" : tag)
|
329
|
+
url = 'https://logging.googleapis.com/v1beta3/projects/' \
|
330
|
+
"#{@project_id}/logs/#{log_name}/entries:write"
|
328
331
|
begin
|
329
|
-
client = api_client
|
330
|
-
request = client.generate_request(
|
331
|
-
:
|
332
|
-
:
|
333
|
-
:
|
334
|
-
:
|
335
|
-
|
332
|
+
client = api_client
|
333
|
+
request = client.generate_request(
|
334
|
+
uri: url,
|
335
|
+
body_object: write_log_entries_request,
|
336
|
+
http_method: 'POST',
|
337
|
+
authenticated: true
|
338
|
+
)
|
336
339
|
client.execute!(request)
|
337
340
|
# Let the user explicitly know when the first call succeeded,
|
338
341
|
# to aid with verification and troubleshooting.
|
339
|
-
|
342
|
+
unless @successful_call
|
340
343
|
@successful_call = true
|
341
|
-
|
344
|
+
@log.info 'Successfully sent to Google Cloud Logging API.'
|
342
345
|
end
|
343
346
|
# Allow most exceptions to propagate, which will cause fluentd to
|
344
347
|
# retry (with backoff), but in some cases we catch the error and
|
@@ -347,9 +350,7 @@ module Fluent
|
|
347
350
|
# Most ClientErrors indicate a problem with the request itself and
|
348
351
|
# should not be retried, unless it is an authentication issue, in
|
349
352
|
# which case we will retry the request via re-raising the exception.
|
350
|
-
if (
|
351
|
-
raise error
|
352
|
-
end
|
353
|
+
raise error if retriable_client_error?(error)
|
353
354
|
log_write_failure(write_log_entries_request, error)
|
354
355
|
rescue JSON::GeneratorError => error
|
355
356
|
# This happens if the request contains illegal characters;
|
@@ -365,17 +366,18 @@ module Fluent
|
|
365
366
|
'Invalid Credentials',
|
366
367
|
'Request had invalid credentials.',
|
367
368
|
'The caller does not have permission',
|
368
|
-
'Project has not enabled the API. Please use Google Developers
|
369
|
+
'Project has not enabled the API. Please use Google Developers ' \
|
370
|
+
'Console to activate the API for your project.',
|
369
371
|
'Unable to fetch access token (no scopes configured?)']
|
370
372
|
|
371
|
-
def
|
372
|
-
|
373
|
+
def retriable_client_error?(error)
|
374
|
+
RETRIABLE_CLIENT_ERRORS.include?(error.message)
|
373
375
|
end
|
374
376
|
|
375
377
|
def log_write_failure(request, error)
|
376
378
|
dropped = request['entries'].length
|
377
|
-
|
378
|
-
|
379
|
+
@log.warn "Dropping #{dropped} log message(s)",
|
380
|
+
error_class: error.class.to_s, error: error.to_s
|
379
381
|
end
|
380
382
|
|
381
383
|
# "enum" of Platform values
|
@@ -388,55 +390,54 @@ module Fluent
|
|
388
390
|
# Determine what platform we are running on by consulting the metadata
|
389
391
|
# service (unless the user has explicitly disabled using that).
|
390
392
|
def detect_platform
|
391
|
-
|
392
|
-
|
393
|
+
unless @use_metadata_service
|
394
|
+
@log.info 'use_metadata_service is false; not detecting platform'
|
393
395
|
return Platform::OTHER
|
394
396
|
end
|
395
397
|
|
396
398
|
begin
|
397
399
|
open('http://' + METADATA_SERVICE_ADDR) do |f|
|
398
400
|
if (f.meta['metadata-flavor'] == 'Google')
|
399
|
-
|
401
|
+
@log.info 'Detected GCE platform'
|
400
402
|
return Platform::GCE
|
401
403
|
end
|
402
404
|
if (f.meta['server'] == 'EC2ws')
|
403
|
-
|
405
|
+
@log.info 'Detected EC2 platform'
|
404
406
|
return Platform::EC2
|
405
407
|
end
|
406
408
|
end
|
407
|
-
rescue
|
408
|
-
|
409
|
+
rescue StandardError => e
|
410
|
+
@log.debug 'Failed to access metadata service: ', error: e
|
409
411
|
end
|
410
412
|
|
411
|
-
|
412
|
-
|
413
|
+
@log.info 'Unable to determine platform'
|
414
|
+
Platform::OTHER
|
413
415
|
end
|
414
416
|
|
415
417
|
def fetch_gce_metadata(metadata_path)
|
416
|
-
|
418
|
+
fail "Called fetch_gce_metadata with platform=#{@platform}" unless
|
417
419
|
@platform == Platform::GCE
|
418
420
|
# See https://cloud.google.com/compute/docs/metadata
|
419
421
|
open('http://' + METADATA_SERVICE_ADDR + '/computeMetadata/v1/' +
|
420
|
-
metadata_path,
|
422
|
+
metadata_path, 'Metadata-Flavor' => 'Google') do |f|
|
421
423
|
f.read
|
422
424
|
end
|
423
425
|
end
|
424
426
|
|
425
427
|
def fetch_ec2_metadata
|
426
|
-
|
428
|
+
fail "Called fetch_ec2_metadata with platform=#{@platform}" unless
|
427
429
|
@platform == Platform::EC2
|
428
430
|
# See http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html
|
429
431
|
open('http://' + METADATA_SERVICE_ADDR +
|
430
432
|
'/latest/dynamic/instance-identity/document') do |f|
|
431
|
-
contents = f.read
|
433
|
+
contents = f.read
|
432
434
|
return JSON.parse(contents)
|
433
435
|
end
|
434
436
|
end
|
435
437
|
|
436
438
|
# Values permitted by the API for 'severity' (which is an enum).
|
437
|
-
VALID_SEVERITIES = Set.new
|
438
|
-
|
439
|
-
'ALERT', 'EMERGENCY']
|
439
|
+
VALID_SEVERITIES = Set.new(
|
440
|
+
%w(DEFAULT DEBUG INFO NOTICE WARNING ERROR CRITICAL ALERT EMERGENCY))
|
440
441
|
|
441
442
|
# Translates other severity strings to one of the valid values above.
|
442
443
|
SEVERITY_TRANSLATIONS = {
|
@@ -457,7 +458,7 @@ module Fluent
|
|
457
458
|
'C' => 'CRITICAL',
|
458
459
|
'A' => 'ALERT',
|
459
460
|
# other misc. translations.
|
460
|
-
'ERR' => 'ERROR'
|
461
|
+
'ERR' => 'ERROR'
|
461
462
|
}
|
462
463
|
|
463
464
|
def parse_severity(severity_str)
|
@@ -465,18 +466,16 @@ module Fluent
|
|
465
466
|
severity = severity_str.upcase.strip
|
466
467
|
|
467
468
|
# If the severity is already valid, just return it.
|
468
|
-
if
|
469
|
-
return severity
|
470
|
-
end
|
469
|
+
return severity if VALID_SEVERITIES.include?(severity)
|
471
470
|
|
472
471
|
# If the severity is an integer (string) return it as an integer,
|
473
472
|
# truncated to the closest valid value (multiples of 100 between 0-800).
|
474
|
-
if
|
473
|
+
if /\A\d+\z/.match(severity)
|
475
474
|
begin
|
476
475
|
numeric_severity = (severity.to_i / 100) * 100
|
477
|
-
if
|
476
|
+
if numeric_severity < 0
|
478
477
|
return 0
|
479
|
-
elsif
|
478
|
+
elsif numeric_severity > 800
|
480
479
|
return 800
|
481
480
|
else
|
482
481
|
return numeric_severity
|
@@ -487,19 +486,19 @@ module Fluent
|
|
487
486
|
end
|
488
487
|
|
489
488
|
# Try to translate the severity.
|
490
|
-
if
|
489
|
+
if SEVERITY_TRANSLATIONS.key?(severity)
|
491
490
|
return SEVERITY_TRANSLATIONS[severity]
|
492
491
|
end
|
493
492
|
|
494
493
|
# If all else fails, use 'DEFAULT'.
|
495
|
-
|
494
|
+
'DEFAULT'
|
496
495
|
end
|
497
496
|
|
498
497
|
def init_api_client
|
499
498
|
@client = Google::APIClient.new(
|
500
|
-
:
|
501
|
-
:
|
502
|
-
:
|
499
|
+
application_name: 'Fluentd Google Cloud Logging plugin',
|
500
|
+
application_version: '0.4.3',
|
501
|
+
retries: 1)
|
503
502
|
|
504
503
|
if @auth_method == 'private_key'
|
505
504
|
key = Google::APIClient::PKCS12.load_key(@private_key_path,
|
@@ -510,22 +509,22 @@ module Fluent
|
|
510
509
|
@client.authorization.expiry = 3600 # 3600s is the max allowed value
|
511
510
|
else
|
512
511
|
@client.authorization = Google::Auth.get_application_default(
|
513
|
-
|
512
|
+
LOGGING_SCOPE)
|
514
513
|
end
|
515
514
|
end
|
516
515
|
|
517
516
|
def api_client
|
518
|
-
|
517
|
+
unless @client.authorization.expired?
|
519
518
|
begin
|
520
519
|
@client.authorization.fetch_access_token!
|
521
520
|
rescue MultiJson::ParseError
|
522
521
|
# Workaround an issue in the API client; just re-raise a more
|
523
522
|
# descriptive error for the user (which will still cause a retry).
|
524
|
-
raise Google::APIClient::ClientError,
|
525
|
-
'
|
523
|
+
raise Google::APIClient::ClientError, 'Unable to fetch access ' \
|
524
|
+
'token (no scopes configured?)'
|
526
525
|
end
|
527
526
|
end
|
528
|
-
|
527
|
+
@client
|
529
528
|
end
|
530
529
|
end
|
531
530
|
end
|