fluent-plugin-google-cloud 0.4.2 → 0.4.3
Sign up to get free protection for your applications and to get access to all the features.
- data/Gemfile.lock +19 -4
- data/README.rdoc +27 -16
- data/Rakefile +10 -0
- data/fluent-plugin-google-cloud.gemspec +18 -10
- data/lib/fluent/plugin/out_google_cloud.rb +102 -103
- data/test/helper.rb +7 -9
- data/test/plugin/test_out_google_cloud.rb +190 -169
- metadata +40 -24
data/Gemfile.lock
CHANGED
@@ -1,9 +1,9 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
fluent-plugin-google-cloud (0.4.
|
4
|
+
fluent-plugin-google-cloud (0.4.3)
|
5
5
|
fluentd (>= 0.10)
|
6
|
-
google-api-client (
|
6
|
+
google-api-client (~> 0.8.6)
|
7
7
|
googleauth (~> 0.4)
|
8
8
|
json (~> 1.8.2)
|
9
9
|
|
@@ -17,6 +17,9 @@ GEM
|
|
17
17
|
thread_safe (~> 0.3, >= 0.3.4)
|
18
18
|
tzinfo (~> 1.1)
|
19
19
|
addressable (2.3.8)
|
20
|
+
ast (2.0.0)
|
21
|
+
astrolabe (1.3.1)
|
22
|
+
parser (~> 2.2)
|
20
23
|
autoparse (0.3.3)
|
21
24
|
addressable (>= 2.3.1)
|
22
25
|
extlib (>= 0.9.15)
|
@@ -27,7 +30,7 @@ GEM
|
|
27
30
|
extlib (0.9.16)
|
28
31
|
faraday (0.9.1)
|
29
32
|
multipart-post (>= 1.2, < 3)
|
30
|
-
fluentd (0.12.
|
33
|
+
fluentd (0.12.14)
|
31
34
|
cool.io (>= 1.2.2, < 2.0.0)
|
32
35
|
http_parser.rb (>= 0.5.1, < 0.7.0)
|
33
36
|
json (>= 1.4.3)
|
@@ -73,9 +76,20 @@ GEM
|
|
73
76
|
msgpack (0.5.12)
|
74
77
|
multi_json (1.11.0)
|
75
78
|
multipart-post (2.0.0)
|
76
|
-
|
79
|
+
parser (2.2.2.6)
|
80
|
+
ast (>= 1.1, < 3.0)
|
81
|
+
power_assert (0.2.4)
|
82
|
+
powerpack (0.1.1)
|
83
|
+
rainbow (2.0.0)
|
77
84
|
rake (10.4.2)
|
78
85
|
retriable (1.4.1)
|
86
|
+
rubocop (0.32.1)
|
87
|
+
astrolabe (~> 1.3)
|
88
|
+
parser (>= 2.2.2.5, < 3.0)
|
89
|
+
powerpack (~> 0.1)
|
90
|
+
rainbow (>= 1.99.1, < 3.0)
|
91
|
+
ruby-progressbar (~> 1.4)
|
92
|
+
ruby-progressbar (1.7.5)
|
79
93
|
safe_yaml (1.0.4)
|
80
94
|
sigdump (0.2.3)
|
81
95
|
signet (0.6.1)
|
@@ -104,6 +118,7 @@ DEPENDENCIES
|
|
104
118
|
fluent-plugin-google-cloud!
|
105
119
|
mocha (~> 1.1)
|
106
120
|
rake (>= 10.3.2)
|
121
|
+
rubocop (~> 0.30)
|
107
122
|
test-unit (~> 3.0.2)
|
108
123
|
webmock (>= 1.17.0)
|
109
124
|
|
data/README.rdoc
CHANGED
@@ -1,33 +1,44 @@
|
|
1
|
-
= Google Cloud
|
1
|
+
= Google Cloud Logging plugin for {fluentd}[http://github.com/fluent/fluentd]
|
2
2
|
|
3
|
-
fluent-plugin-google-cloud
|
3
|
+
fluent-plugin-google-cloud is an
|
4
|
+
{output plugin for fluentd}[http://docs.fluentd.org/articles/output-plugin-overview]
|
5
|
+
which sends logs to the
|
6
|
+
{Google Cloud Logging API}[https://cloud.google.com/logging/docs/api/].
|
7
|
+
|
8
|
+
This is an official Google Ruby gem.
|
9
|
+
|
10
|
+
{<img src="https://badge.fury.io/rb/fluent-plugin-google-cloud.svg" alt="Gem Version" />}[http://badge.fury.io/rb/fluent-plugin-google-cloud]
|
11
|
+
{<img src="https://secure.travis-ci.org/google/google-auth-library-ruby.png" alt="Build Status" />}[https://travis-ci.org/GoogleCloudPlatform/fluent-plugin-google-cloud]
|
4
12
|
|
5
13
|
== Installation
|
6
14
|
|
7
|
-
|
15
|
+
This gem is hosted at
|
16
|
+
{RubyGems.org}[https://rubygems.org/gems/fluent-plugin-google-cloud]
|
17
|
+
and can be installed using:
|
8
18
|
|
9
19
|
$ gem install fluent-plugin-google-cloud
|
10
20
|
|
21
|
+
Installing {google-fluentd}[https://cloud.google.com/logging/docs/agent/]
|
22
|
+
will also install and configure the gem.
|
23
|
+
|
11
24
|
== Configuration
|
12
25
|
|
26
|
+
To send logs to Google Cloud Logging, specify <code>type google_cloud</code>
|
27
|
+
in a
|
28
|
+
{match clause}[http://docs.fluentd.org/articles/config-file#2-ldquomatchrdquo-tell-fluentd-what-to-do]
|
29
|
+
of your fluentd configuration file, for example:
|
30
|
+
|
13
31
|
<match **>
|
14
32
|
type google_cloud
|
15
|
-
auth_method <method>
|
16
|
-
private_key_email <address>
|
17
|
-
private_key_path <path>
|
18
33
|
</match>
|
19
34
|
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
private_key_email and private_key_path are required if auth_method is
|
26
|
-
'private_key', otherwise they are ignored.
|
27
|
-
|
28
|
-
== Caveats
|
35
|
+
No further configuration is required. The plugin uses
|
36
|
+
{Google Application Default Credentials}[https://developers.google.com/identity/protocols/application-default-credentials]
|
37
|
+
for authorization - for additional information see
|
38
|
+
{here}[https://cloud.google.com/logging/docs/agent/authorization].
|
29
39
|
|
30
|
-
|
40
|
+
<em>The previously documented parameters auth_method, private_key_email,
|
41
|
+
and private_key_path are deprecated and should no longer be used.</em>
|
31
42
|
|
32
43
|
== Copyright
|
33
44
|
|
data/Rakefile
CHANGED
@@ -4,9 +4,19 @@ require 'bundler'
|
|
4
4
|
Bundler::GemHelper.install_tasks
|
5
5
|
|
6
6
|
require 'rake/testtask'
|
7
|
+
require 'rubocop/rake_task'
|
7
8
|
|
9
|
+
desc 'Run Rubocop to check for style violations'
|
10
|
+
RuboCop::RakeTask.new
|
11
|
+
|
12
|
+
desc 'Run unit tests'
|
8
13
|
Rake::TestTask.new(:test) do |test|
|
9
14
|
test.libs << 'lib' << 'test'
|
10
15
|
test.test_files = FileList['test/plugin/*.rb']
|
11
16
|
test.verbose = true
|
12
17
|
end
|
18
|
+
|
19
|
+
desc 'Does rubocop lint and runs tests'
|
20
|
+
task all: [:rubocop, :test]
|
21
|
+
|
22
|
+
task default: :all
|
@@ -1,23 +1,31 @@
|
|
1
1
|
Gem::Specification.new do |gem|
|
2
2
|
gem.name = 'fluent-plugin-google-cloud'
|
3
|
-
gem.description =
|
4
|
-
|
5
|
-
|
3
|
+
gem.description = <<-eos
|
4
|
+
Fluentd output plugin for the Google Cloud Logging API, which will make
|
5
|
+
loge viewable in the Developer Console's log viewer and can optionally
|
6
|
+
store them in Google Cloud Storage and/or BigQuery.
|
7
|
+
This is an official Google Ruby gem.'
|
8
|
+
eos
|
9
|
+
gem.summary = 'fluentd output plugin for the Google Cloud Logging API'
|
10
|
+
gem.homepage = \
|
11
|
+
'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
|
6
12
|
gem.license = 'Apache 2.0'
|
7
|
-
gem.version = '0.4.
|
13
|
+
gem.version = '0.4.3'
|
8
14
|
gem.authors = ['Todd Derr', 'Alex Robinson']
|
9
15
|
gem.email = ['salty@google.com']
|
10
16
|
|
11
17
|
gem.files = Dir['**/*'].keep_if { |file| File.file?(file) }
|
12
|
-
gem.test_files = gem.files.grep(
|
18
|
+
gem.test_files = gem.files.grep(/^(test)/)
|
13
19
|
gem.require_paths = ['lib']
|
14
20
|
|
15
21
|
gem.add_runtime_dependency 'fluentd', '>= 0.10'
|
16
|
-
gem.add_runtime_dependency 'google-api-client', '
|
22
|
+
gem.add_runtime_dependency 'google-api-client', '~> 0.8.6'
|
17
23
|
gem.add_runtime_dependency 'googleauth', '~> 0.4'
|
18
24
|
gem.add_runtime_dependency 'json', '~> 1.8.2'
|
19
|
-
|
20
|
-
gem.add_development_dependency
|
21
|
-
gem.add_development_dependency
|
22
|
-
gem.add_development_dependency
|
25
|
+
|
26
|
+
gem.add_development_dependency 'mocha', '~> 1.1'
|
27
|
+
gem.add_development_dependency 'rake', '>= 10.3.2'
|
28
|
+
gem.add_development_dependency 'rubocop', '~> 0.30'
|
29
|
+
gem.add_development_dependency 'webmock', '>= 1.17.0'
|
30
|
+
gem.add_development_dependency 'test-unit', '~> 3.0.2'
|
23
31
|
end
|
@@ -13,6 +13,7 @@
|
|
13
13
|
# limitations under the License.
|
14
14
|
|
15
15
|
module Fluent
|
16
|
+
# fluentd output plugin for the Google Cloud Logging API
|
16
17
|
class GoogleCloudOutput < BufferedOutput
|
17
18
|
Fluent::Plugin.register_output('google_cloud', self)
|
18
19
|
|
@@ -28,6 +29,9 @@ module Fluent
|
|
28
29
|
# Address of the metadata service.
|
29
30
|
METADATA_SERVICE_ADDR = '169.254.169.254'
|
30
31
|
|
32
|
+
# Disable this warning to conform to fluentd config_param conventions.
|
33
|
+
# rubocop:disable Style/HashSyntax
|
34
|
+
|
31
35
|
# DEPRECATED: auth_method (and support for 'private_key') is deprecated in
|
32
36
|
# favor of Google Application Default Credentials as documented at:
|
33
37
|
# https://developers.google.com/identity/protocols/application-default-credentials
|
@@ -74,6 +78,8 @@ module Fluent
|
|
74
78
|
# }
|
75
79
|
config_param :label_map, :hash, :default => nil
|
76
80
|
|
81
|
+
# rubocop:enable Style/HashSyntax
|
82
|
+
|
77
83
|
# TODO: Add a log_name config option rather than just using the tag?
|
78
84
|
|
79
85
|
# Expose attr_readers to make testing of metadata more direct than only
|
@@ -95,24 +101,27 @@ module Fluent
|
|
95
101
|
require 'googleauth'
|
96
102
|
require 'json'
|
97
103
|
require 'open-uri'
|
104
|
+
|
105
|
+
# use the global logger
|
106
|
+
@log = $log # rubocop:disable Style/GlobalVars
|
98
107
|
end
|
99
108
|
|
100
109
|
def configure(conf)
|
101
110
|
super
|
102
111
|
|
103
|
-
|
104
|
-
|
105
|
-
|
112
|
+
unless @auth_method.nil?
|
113
|
+
@log.warn 'auth_method is deprecated; please migrate to using ' \
|
114
|
+
'Application Default Credentials.'
|
106
115
|
if @auth_method == 'private_key'
|
107
116
|
if !@private_key_email
|
108
|
-
|
109
|
-
|
117
|
+
fail Fluent::ConfigError, '"private_key_email" must be ' \
|
118
|
+
'specified if auth_method is "private_key"'
|
110
119
|
elsif !@private_key_path
|
111
|
-
|
112
|
-
|
120
|
+
fail Fluent::ConfigError, '"private_key_path" must be ' \
|
121
|
+
'specified if auth_method is "private_key"'
|
113
122
|
elsif !@private_key_passphrase
|
114
|
-
|
115
|
-
|
123
|
+
fail Fluent::ConfigError, '"private_key_passphrase" must be ' \
|
124
|
+
'specified if auth_method is "private_key"'
|
116
125
|
end
|
117
126
|
end
|
118
127
|
end
|
@@ -133,34 +142,32 @@ module Fluent
|
|
133
142
|
fully_qualified_zone = fetch_gce_metadata('instance/zone')
|
134
143
|
@zone = fully_qualified_zone.rpartition('/')[2]
|
135
144
|
end
|
136
|
-
if @vm_id.nil?
|
137
|
-
@vm_id = fetch_gce_metadata('instance/id')
|
138
|
-
end
|
145
|
+
@vm_id = fetch_gce_metadata('instance/id') if @vm_id.nil?
|
139
146
|
when Platform::EC2
|
140
147
|
metadata = fetch_ec2_metadata
|
141
|
-
if @zone.nil? && metadata.
|
148
|
+
if @zone.nil? && metadata.key?('availabilityZone')
|
142
149
|
@zone = 'aws:' + metadata['availabilityZone']
|
143
150
|
end
|
144
|
-
if @vm_id.nil? && metadata.
|
151
|
+
if @vm_id.nil? && metadata.key?('instanceId')
|
145
152
|
@vm_id = metadata['instanceId']
|
146
153
|
end
|
147
|
-
if metadata.
|
154
|
+
if metadata.key?('accountId')
|
148
155
|
common_labels["#{EC2_SERVICE}/account_id"] = metadata['accountId']
|
149
156
|
end
|
150
157
|
when Platform::OTHER
|
151
158
|
# do nothing
|
152
159
|
else
|
153
|
-
|
160
|
+
fail Fluent::ConfigError, 'Unknown platform ' + @platform
|
154
161
|
end
|
155
162
|
|
156
163
|
# all metadata parameters must now be set
|
157
164
|
unless @project_id && @zone && @vm_id
|
158
165
|
missing = []
|
159
|
-
missing <<
|
160
|
-
missing <<
|
161
|
-
missing <<
|
162
|
-
|
163
|
-
|
166
|
+
missing << 'project_id' unless @project_id
|
167
|
+
missing << 'zone' unless @zone
|
168
|
+
missing << 'vm_id' unless @vm_id
|
169
|
+
fail Fluent::ConfigError, 'Unable to obtain metadata parameters: ' +
|
170
|
+
missing.join(' ')
|
164
171
|
end
|
165
172
|
|
166
173
|
# Default this to false; it is only overwritten if we detect Managed VM.
|
@@ -173,8 +180,8 @@ module Fluent
|
|
173
180
|
# Check for specialized GCE environments (Managed VM or Dataflow).
|
174
181
|
# TODO: Add config options for these to allow for running outside GCE?
|
175
182
|
attributes = fetch_gce_metadata('instance/attributes/').split
|
176
|
-
if
|
177
|
-
|
183
|
+
if attributes.include?('gae_backend_name') &&
|
184
|
+
attributes.include?('gae_backend_version')
|
178
185
|
# Managed VM
|
179
186
|
@running_on_managed_vm = true
|
180
187
|
@gae_backend_name =
|
@@ -185,7 +192,7 @@ module Fluent
|
|
185
192
|
common_labels["#{APPENGINE_SERVICE}/module_id"] = @gae_backend_name
|
186
193
|
common_labels["#{APPENGINE_SERVICE}/version_id"] =
|
187
194
|
@gae_backend_version
|
188
|
-
elsif
|
195
|
+
elsif attributes.include?('job_id')
|
189
196
|
# Dataflow
|
190
197
|
@service_name = DATAFLOW_SERVICE
|
191
198
|
@dataflow_job_id = fetch_gce_metadata('instance/attributes/job_id')
|
@@ -212,7 +219,7 @@ module Fluent
|
|
212
219
|
def start
|
213
220
|
super
|
214
221
|
|
215
|
-
init_api_client
|
222
|
+
init_api_client
|
216
223
|
|
217
224
|
@successful_call = false
|
218
225
|
@timenanos_warning = false
|
@@ -230,43 +237,41 @@ module Fluent
|
|
230
237
|
# Group the entries since we have to make one call per tag.
|
231
238
|
grouped_entries = {}
|
232
239
|
chunk.msgpack_each do |tag, *arr|
|
233
|
-
|
234
|
-
grouped_entries[tag] = []
|
235
|
-
end
|
240
|
+
grouped_entries[tag] = [] unless grouped_entries.key?(tag)
|
236
241
|
grouped_entries[tag].push(arr)
|
237
242
|
end
|
238
243
|
|
239
244
|
grouped_entries.each do |tag, arr|
|
240
245
|
write_log_entries_request = {
|
241
246
|
'commonLabels' => @common_labels,
|
242
|
-
'entries' => []
|
247
|
+
'entries' => []
|
243
248
|
}
|
244
249
|
arr.each do |time, record|
|
245
250
|
next unless record.is_a?(Hash)
|
246
|
-
if
|
247
|
-
|
248
|
-
|
249
|
-
|
251
|
+
if record.key?('timestamp') &&
|
252
|
+
record['timestamp'].is_a?(Hash) &&
|
253
|
+
record['timestamp'].key?('seconds') &&
|
254
|
+
record['timestamp'].key?('nanos')
|
250
255
|
ts_secs = record['timestamp']['seconds']
|
251
256
|
ts_nanos = record['timestamp']['nanos']
|
252
257
|
record.delete('timestamp')
|
253
|
-
elsif
|
254
|
-
|
258
|
+
elsif record.key?('timestampSeconds') &&
|
259
|
+
record.key?('timestampNanos')
|
255
260
|
ts_secs = record['timestampSeconds']
|
256
261
|
ts_nanos = record['timestampNanos']
|
257
262
|
record.delete('timestampSeconds')
|
258
263
|
record.delete('timestampNanos')
|
259
|
-
elsif
|
264
|
+
elsif record.key?('timeNanos')
|
260
265
|
# This is deprecated since the precision is insufficient.
|
261
266
|
# Use timestampSeconds/timestampNanos instead
|
262
|
-
ts_secs = (record['timeNanos'] /
|
263
|
-
ts_nanos = record['timeNanos'] %
|
267
|
+
ts_secs = (record['timeNanos'] / 1_000_000_000).to_i
|
268
|
+
ts_nanos = record['timeNanos'] % 1_000_000_000
|
264
269
|
record.delete('timeNanos')
|
265
|
-
|
270
|
+
unless @timenanos_warning
|
266
271
|
# Warn the user this is deprecated, but only once to avoid spam.
|
267
272
|
@timenanos_warning = true
|
268
|
-
|
269
|
-
|
273
|
+
@log.warn 'timeNanos is deprecated - please use ' \
|
274
|
+
'timestampSeconds and timestampNanos instead.'
|
270
275
|
end
|
271
276
|
else
|
272
277
|
timestamp = Time.at(time)
|
@@ -281,10 +286,10 @@ module Fluent
|
|
281
286
|
'timestamp' => {
|
282
287
|
'seconds' => ts_secs,
|
283
288
|
'nanos' => ts_nanos
|
284
|
-
}
|
285
|
-
}
|
289
|
+
}
|
290
|
+
}
|
286
291
|
}
|
287
|
-
if record.
|
292
|
+
if record.key?('severity')
|
288
293
|
entry['metadata']['severity'] = parse_severity(record['severity'])
|
289
294
|
record.delete('severity')
|
290
295
|
else
|
@@ -294,22 +299,20 @@ module Fluent
|
|
294
299
|
# If a field is present in the label_map, send its value as a label
|
295
300
|
# (mapping the field name to label name as specified in the config)
|
296
301
|
# and do not send that field as part of the payload.
|
297
|
-
|
302
|
+
unless label_map.nil?
|
298
303
|
labels = {}
|
299
304
|
@label_map.each do |field, label|
|
300
|
-
if record.
|
305
|
+
if record.key?(field)
|
301
306
|
labels[label] = record[field]
|
302
307
|
record.delete(field)
|
303
308
|
end
|
304
309
|
end
|
305
|
-
|
306
|
-
entry['metadata']['labels'] = labels
|
307
|
-
end
|
310
|
+
entry['metadata']['labels'] = labels unless labels.empty?
|
308
311
|
end
|
309
312
|
|
310
313
|
# use textPayload if the only remainaing key is 'message',
|
311
314
|
# otherwise use a struct.
|
312
|
-
if
|
315
|
+
if record.size == 1 && record.key?('message')
|
313
316
|
entry['textPayload'] = record['message']
|
314
317
|
else
|
315
318
|
entry['structPayload'] = record
|
@@ -321,24 +324,24 @@ module Fluent
|
|
321
324
|
|
322
325
|
# Add a prefix to VMEngines logs to prevent namespace collisions,
|
323
326
|
# and also escape the log name.
|
324
|
-
log_name = CGI
|
325
|
-
|
326
|
-
url =
|
327
|
-
|
327
|
+
log_name = CGI.escape(
|
328
|
+
@running_on_managed_vm ? "#{APPENGINE_SERVICE}/#{tag}" : tag)
|
329
|
+
url = 'https://logging.googleapis.com/v1beta3/projects/' \
|
330
|
+
"#{@project_id}/logs/#{log_name}/entries:write"
|
328
331
|
begin
|
329
|
-
client = api_client
|
330
|
-
request = client.generate_request(
|
331
|
-
:
|
332
|
-
:
|
333
|
-
:
|
334
|
-
:
|
335
|
-
|
332
|
+
client = api_client
|
333
|
+
request = client.generate_request(
|
334
|
+
uri: url,
|
335
|
+
body_object: write_log_entries_request,
|
336
|
+
http_method: 'POST',
|
337
|
+
authenticated: true
|
338
|
+
)
|
336
339
|
client.execute!(request)
|
337
340
|
# Let the user explicitly know when the first call succeeded,
|
338
341
|
# to aid with verification and troubleshooting.
|
339
|
-
|
342
|
+
unless @successful_call
|
340
343
|
@successful_call = true
|
341
|
-
|
344
|
+
@log.info 'Successfully sent to Google Cloud Logging API.'
|
342
345
|
end
|
343
346
|
# Allow most exceptions to propagate, which will cause fluentd to
|
344
347
|
# retry (with backoff), but in some cases we catch the error and
|
@@ -347,9 +350,7 @@ module Fluent
|
|
347
350
|
# Most ClientErrors indicate a problem with the request itself and
|
348
351
|
# should not be retried, unless it is an authentication issue, in
|
349
352
|
# which case we will retry the request via re-raising the exception.
|
350
|
-
if (
|
351
|
-
raise error
|
352
|
-
end
|
353
|
+
raise error if retriable_client_error?(error)
|
353
354
|
log_write_failure(write_log_entries_request, error)
|
354
355
|
rescue JSON::GeneratorError => error
|
355
356
|
# This happens if the request contains illegal characters;
|
@@ -365,17 +366,18 @@ module Fluent
|
|
365
366
|
'Invalid Credentials',
|
366
367
|
'Request had invalid credentials.',
|
367
368
|
'The caller does not have permission',
|
368
|
-
'Project has not enabled the API. Please use Google Developers
|
369
|
+
'Project has not enabled the API. Please use Google Developers ' \
|
370
|
+
'Console to activate the API for your project.',
|
369
371
|
'Unable to fetch access token (no scopes configured?)']
|
370
372
|
|
371
|
-
def
|
372
|
-
|
373
|
+
def retriable_client_error?(error)
|
374
|
+
RETRIABLE_CLIENT_ERRORS.include?(error.message)
|
373
375
|
end
|
374
376
|
|
375
377
|
def log_write_failure(request, error)
|
376
378
|
dropped = request['entries'].length
|
377
|
-
|
378
|
-
|
379
|
+
@log.warn "Dropping #{dropped} log message(s)",
|
380
|
+
error_class: error.class.to_s, error: error.to_s
|
379
381
|
end
|
380
382
|
|
381
383
|
# "enum" of Platform values
|
@@ -388,55 +390,54 @@ module Fluent
|
|
388
390
|
# Determine what platform we are running on by consulting the metadata
|
389
391
|
# service (unless the user has explicitly disabled using that).
|
390
392
|
def detect_platform
|
391
|
-
|
392
|
-
|
393
|
+
unless @use_metadata_service
|
394
|
+
@log.info 'use_metadata_service is false; not detecting platform'
|
393
395
|
return Platform::OTHER
|
394
396
|
end
|
395
397
|
|
396
398
|
begin
|
397
399
|
open('http://' + METADATA_SERVICE_ADDR) do |f|
|
398
400
|
if (f.meta['metadata-flavor'] == 'Google')
|
399
|
-
|
401
|
+
@log.info 'Detected GCE platform'
|
400
402
|
return Platform::GCE
|
401
403
|
end
|
402
404
|
if (f.meta['server'] == 'EC2ws')
|
403
|
-
|
405
|
+
@log.info 'Detected EC2 platform'
|
404
406
|
return Platform::EC2
|
405
407
|
end
|
406
408
|
end
|
407
|
-
rescue
|
408
|
-
|
409
|
+
rescue StandardError => e
|
410
|
+
@log.debug 'Failed to access metadata service: ', error: e
|
409
411
|
end
|
410
412
|
|
411
|
-
|
412
|
-
|
413
|
+
@log.info 'Unable to determine platform'
|
414
|
+
Platform::OTHER
|
413
415
|
end
|
414
416
|
|
415
417
|
def fetch_gce_metadata(metadata_path)
|
416
|
-
|
418
|
+
fail "Called fetch_gce_metadata with platform=#{@platform}" unless
|
417
419
|
@platform == Platform::GCE
|
418
420
|
# See https://cloud.google.com/compute/docs/metadata
|
419
421
|
open('http://' + METADATA_SERVICE_ADDR + '/computeMetadata/v1/' +
|
420
|
-
metadata_path,
|
422
|
+
metadata_path, 'Metadata-Flavor' => 'Google') do |f|
|
421
423
|
f.read
|
422
424
|
end
|
423
425
|
end
|
424
426
|
|
425
427
|
def fetch_ec2_metadata
|
426
|
-
|
428
|
+
fail "Called fetch_ec2_metadata with platform=#{@platform}" unless
|
427
429
|
@platform == Platform::EC2
|
428
430
|
# See http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html
|
429
431
|
open('http://' + METADATA_SERVICE_ADDR +
|
430
432
|
'/latest/dynamic/instance-identity/document') do |f|
|
431
|
-
contents = f.read
|
433
|
+
contents = f.read
|
432
434
|
return JSON.parse(contents)
|
433
435
|
end
|
434
436
|
end
|
435
437
|
|
436
438
|
# Values permitted by the API for 'severity' (which is an enum).
|
437
|
-
VALID_SEVERITIES = Set.new
|
438
|
-
|
439
|
-
'ALERT', 'EMERGENCY']
|
439
|
+
VALID_SEVERITIES = Set.new(
|
440
|
+
%w(DEFAULT DEBUG INFO NOTICE WARNING ERROR CRITICAL ALERT EMERGENCY))
|
440
441
|
|
441
442
|
# Translates other severity strings to one of the valid values above.
|
442
443
|
SEVERITY_TRANSLATIONS = {
|
@@ -457,7 +458,7 @@ module Fluent
|
|
457
458
|
'C' => 'CRITICAL',
|
458
459
|
'A' => 'ALERT',
|
459
460
|
# other misc. translations.
|
460
|
-
'ERR' => 'ERROR'
|
461
|
+
'ERR' => 'ERROR'
|
461
462
|
}
|
462
463
|
|
463
464
|
def parse_severity(severity_str)
|
@@ -465,18 +466,16 @@ module Fluent
|
|
465
466
|
severity = severity_str.upcase.strip
|
466
467
|
|
467
468
|
# If the severity is already valid, just return it.
|
468
|
-
if
|
469
|
-
return severity
|
470
|
-
end
|
469
|
+
return severity if VALID_SEVERITIES.include?(severity)
|
471
470
|
|
472
471
|
# If the severity is an integer (string) return it as an integer,
|
473
472
|
# truncated to the closest valid value (multiples of 100 between 0-800).
|
474
|
-
if
|
473
|
+
if /\A\d+\z/.match(severity)
|
475
474
|
begin
|
476
475
|
numeric_severity = (severity.to_i / 100) * 100
|
477
|
-
if
|
476
|
+
if numeric_severity < 0
|
478
477
|
return 0
|
479
|
-
elsif
|
478
|
+
elsif numeric_severity > 800
|
480
479
|
return 800
|
481
480
|
else
|
482
481
|
return numeric_severity
|
@@ -487,19 +486,19 @@ module Fluent
|
|
487
486
|
end
|
488
487
|
|
489
488
|
# Try to translate the severity.
|
490
|
-
if
|
489
|
+
if SEVERITY_TRANSLATIONS.key?(severity)
|
491
490
|
return SEVERITY_TRANSLATIONS[severity]
|
492
491
|
end
|
493
492
|
|
494
493
|
# If all else fails, use 'DEFAULT'.
|
495
|
-
|
494
|
+
'DEFAULT'
|
496
495
|
end
|
497
496
|
|
498
497
|
def init_api_client
|
499
498
|
@client = Google::APIClient.new(
|
500
|
-
:
|
501
|
-
:
|
502
|
-
:
|
499
|
+
application_name: 'Fluentd Google Cloud Logging plugin',
|
500
|
+
application_version: '0.4.3',
|
501
|
+
retries: 1)
|
503
502
|
|
504
503
|
if @auth_method == 'private_key'
|
505
504
|
key = Google::APIClient::PKCS12.load_key(@private_key_path,
|
@@ -510,22 +509,22 @@ module Fluent
|
|
510
509
|
@client.authorization.expiry = 3600 # 3600s is the max allowed value
|
511
510
|
else
|
512
511
|
@client.authorization = Google::Auth.get_application_default(
|
513
|
-
|
512
|
+
LOGGING_SCOPE)
|
514
513
|
end
|
515
514
|
end
|
516
515
|
|
517
516
|
def api_client
|
518
|
-
|
517
|
+
unless @client.authorization.expired?
|
519
518
|
begin
|
520
519
|
@client.authorization.fetch_access_token!
|
521
520
|
rescue MultiJson::ParseError
|
522
521
|
# Workaround an issue in the API client; just re-raise a more
|
523
522
|
# descriptive error for the user (which will still cause a retry).
|
524
|
-
raise Google::APIClient::ClientError,
|
525
|
-
'
|
523
|
+
raise Google::APIClient::ClientError, 'Unable to fetch access ' \
|
524
|
+
'token (no scopes configured?)'
|
526
525
|
end
|
527
526
|
end
|
528
|
-
|
527
|
+
@client
|
529
528
|
end
|
530
529
|
end
|
531
530
|
end
|