fluent-plugin-google-cloud 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile.lock +11 -10
- data/fluent-plugin-google-cloud.gemspec +2 -1
- data/lib/fluent/plugin/out_google_cloud.rb +156 -55
- data/test/plugin/test_out_google_cloud.rb +193 -17
- metadata +18 -2
data/Gemfile.lock
CHANGED
@@ -1,10 +1,11 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
fluent-plugin-google-cloud (0.3.
|
4
|
+
fluent-plugin-google-cloud (0.3.1)
|
5
5
|
fluentd (>= 0.10)
|
6
6
|
google-api-client (>= 0.8)
|
7
7
|
googleauth (~> 0.4)
|
8
|
+
json (~> 1.8.3)
|
8
9
|
|
9
10
|
GEM
|
10
11
|
remote: https://rubygems.org/
|
@@ -20,13 +21,13 @@ GEM
|
|
20
21
|
addressable (>= 2.3.1)
|
21
22
|
extlib (>= 0.9.15)
|
22
23
|
multi_json (>= 1.0.0)
|
23
|
-
cool.io (1.3.
|
24
|
+
cool.io (1.3.1)
|
24
25
|
crack (0.4.2)
|
25
26
|
safe_yaml (~> 1.0.0)
|
26
27
|
extlib (0.9.16)
|
27
28
|
faraday (0.9.1)
|
28
29
|
multipart-post (>= 1.2, < 3)
|
29
|
-
fluentd (0.12.
|
30
|
+
fluentd (0.12.11)
|
30
31
|
cool.io (>= 1.2.2, < 2.0.0)
|
31
32
|
http_parser.rb (>= 0.5.1, < 0.7.0)
|
32
33
|
json (>= 1.4.3)
|
@@ -56,8 +57,8 @@ GEM
|
|
56
57
|
signet (~> 0.6)
|
57
58
|
http_parser.rb (0.6.0)
|
58
59
|
i18n (0.7.0)
|
59
|
-
json (1.8.
|
60
|
-
jwt (1.
|
60
|
+
json (1.8.3)
|
61
|
+
jwt (1.5.0)
|
61
62
|
launchy (2.4.3)
|
62
63
|
addressable (~> 2.3)
|
63
64
|
little-plugger (1.1.3)
|
@@ -66,22 +67,22 @@ GEM
|
|
66
67
|
multi_json (~> 1.10)
|
67
68
|
memoist (0.12.0)
|
68
69
|
metaclass (0.0.4)
|
69
|
-
minitest (5.
|
70
|
+
minitest (5.7.0)
|
70
71
|
mocha (1.1.0)
|
71
72
|
metaclass (~> 0.0.1)
|
72
|
-
msgpack (0.5.
|
73
|
+
msgpack (0.5.12)
|
73
74
|
multi_json (1.11.0)
|
74
75
|
multipart-post (2.0.0)
|
75
76
|
power_assert (0.2.3)
|
76
77
|
rake (10.4.2)
|
77
78
|
retriable (1.4.1)
|
78
79
|
safe_yaml (1.0.4)
|
79
|
-
sigdump (0.2.
|
80
|
-
signet (0.6.
|
80
|
+
sigdump (0.2.3)
|
81
|
+
signet (0.6.1)
|
81
82
|
addressable (~> 2.3)
|
82
83
|
extlib (~> 0.9)
|
83
84
|
faraday (~> 0.9)
|
84
|
-
jwt (~> 1.
|
85
|
+
jwt (~> 1.5)
|
85
86
|
multi_json (~> 1.10)
|
86
87
|
string-scrub (0.0.5)
|
87
88
|
test-unit (3.0.9)
|
@@ -4,7 +4,7 @@ Gem::Specification.new do |gem|
|
|
4
4
|
gem.summary = %q{Fluentd plugin to stream logs to the Google Cloud Platform's logging API}
|
5
5
|
gem.homepage = 'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
|
6
6
|
gem.license = 'Apache 2.0'
|
7
|
-
gem.version = '0.
|
7
|
+
gem.version = '0.4.0'
|
8
8
|
gem.authors = ['Todd Derr', 'Alex Robinson']
|
9
9
|
gem.email = ['salty@google.com']
|
10
10
|
|
@@ -15,6 +15,7 @@ Gem::Specification.new do |gem|
|
|
15
15
|
gem.add_runtime_dependency 'fluentd', '>= 0.10'
|
16
16
|
gem.add_runtime_dependency 'google-api-client', '>= 0.8'
|
17
17
|
gem.add_runtime_dependency 'googleauth', '~> 0.4'
|
18
|
+
gem.add_runtime_dependency 'json', '~> 1.8.3'
|
18
19
|
gem.add_development_dependency "rake", '>= 10.3.2'
|
19
20
|
gem.add_development_dependency "webmock", '>= 1.17.0'
|
20
21
|
gem.add_development_dependency "test-unit", "~> 3.0.2"
|
@@ -16,14 +16,18 @@ module Fluent
|
|
16
16
|
class GoogleCloudOutput < BufferedOutput
|
17
17
|
Fluent::Plugin.register_output('google_cloud', self)
|
18
18
|
|
19
|
-
# Constants for
|
19
|
+
# Constants for service names.
|
20
20
|
APPENGINE_SERVICE = 'appengine.googleapis.com'
|
21
21
|
COMPUTE_SERVICE = 'compute.googleapis.com'
|
22
22
|
DATAFLOW_SERVICE = 'dataflow.googleapis.com'
|
23
|
+
EC2_SERVICE = 'ec2.amazonaws.com'
|
23
24
|
|
24
25
|
# Name of the the Google cloud logging write scope.
|
25
26
|
LOGGING_SCOPE = 'https://www.googleapis.com/auth/logging.write'
|
26
27
|
|
28
|
+
# Address of the metadata service.
|
29
|
+
METADATA_SERVICE_ADDR = '169.254.169.254'
|
30
|
+
|
27
31
|
# DEPRECATED: auth_method (and support for 'private_key') is deprecated in
|
28
32
|
# favor of Google Application Default Credentials as documented at:
|
29
33
|
# https://developers.google.com/identity/protocols/application-default-credentials
|
@@ -36,10 +40,16 @@ module Fluent
|
|
36
40
|
config_param :private_key_path, :string, :default => nil
|
37
41
|
config_param :private_key_passphrase, :string, :default => 'notasecret'
|
38
42
|
|
39
|
-
#
|
40
|
-
#
|
41
|
-
#
|
43
|
+
# Specify project/instance metadata.
|
44
|
+
#
|
45
|
+
# project_id, zone, and vm_id are required to have valid values, which
|
46
|
+
# can be obtained from the metadata service or set explicitly.
|
47
|
+
# Otherwise, the plugin will fail to initialize.
|
48
|
+
#
|
49
|
+
# Whether to attempt to obtain metadata from the local metadata service.
|
50
|
+
# It is safe to specify 'true' even on platforms with no metadata service.
|
42
51
|
config_param :use_metadata_service, :bool, :default => true
|
52
|
+
# These parameters override any values obtained from the metadata service.
|
43
53
|
config_param :project_id, :string, :default => nil
|
44
54
|
config_param :zone, :string, :default => nil
|
45
55
|
config_param :vm_id, :string, :default => nil
|
@@ -63,6 +73,7 @@ module Fluent
|
|
63
73
|
require 'google/api_client'
|
64
74
|
require 'google/api_client/auth/compute_service_account'
|
65
75
|
require 'googleauth'
|
76
|
+
require 'json'
|
66
77
|
require 'open-uri'
|
67
78
|
end
|
68
79
|
|
@@ -86,11 +97,95 @@ module Fluent
|
|
86
97
|
end
|
87
98
|
end
|
88
99
|
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
100
|
+
# TODO: Send instance tags and/or hostname as labels as well?
|
101
|
+
@common_labels = {}
|
102
|
+
|
103
|
+
# set attributes from metadata (unless overriden by static config)
|
104
|
+
@platform = detect_platform
|
105
|
+
case @platform
|
106
|
+
when Platform::GCE
|
107
|
+
if @project_id.nil?
|
108
|
+
@project_id = fetch_gce_metadata('project/project-id')
|
109
|
+
end
|
110
|
+
if @zone.nil?
|
111
|
+
# this returns "projects/<number>/zones/<zone>"; we only want
|
112
|
+
# the part after the final slash.
|
113
|
+
fully_qualified_zone = fetch_gce_metadata('instance/zone')
|
114
|
+
@zone = fully_qualified_zone.rpartition('/')[2]
|
115
|
+
end
|
116
|
+
if @vm_id.nil?
|
117
|
+
@vm_id = fetch_gce_metadata('instance/id')
|
118
|
+
end
|
119
|
+
when Platform::EC2
|
120
|
+
metadata = fetch_ec2_metadata
|
121
|
+
if @zone.nil? && metadata.has_key?('availabilityZone')
|
122
|
+
@zone = 'aws:' + metadata['availabilityZone']
|
93
123
|
end
|
124
|
+
if @vm_id.nil? && metadata.has_key?('instanceId')
|
125
|
+
@vm_id = metadata['instanceId']
|
126
|
+
end
|
127
|
+
if metadata.has_key?('accountId')
|
128
|
+
common_labels["#{EC2_SERVICE}/account_id"] = metadata['accountId']
|
129
|
+
end
|
130
|
+
when Platform::OTHER
|
131
|
+
# do nothing
|
132
|
+
else
|
133
|
+
raise Fluent::ConfigError, 'Unknown platform ' + @platform
|
134
|
+
end
|
135
|
+
|
136
|
+
# all metadata parameters must now be set
|
137
|
+
unless @project_id && @zone && @vm_id
|
138
|
+
missing = []
|
139
|
+
missing << "project_id" unless @project_id
|
140
|
+
missing << "zone" unless @zone
|
141
|
+
missing << "vm_id" unless @vm_id
|
142
|
+
raise Fluent::ConfigError,
|
143
|
+
('Unable to obtain metadata parameters: ' + missing.join(' '))
|
144
|
+
end
|
145
|
+
|
146
|
+
# Default this to false; it is only overwritten if we detect Managed VM.
|
147
|
+
@running_on_managed_vm = false
|
148
|
+
|
149
|
+
# Set labels, etc. based on the config
|
150
|
+
case @platform
|
151
|
+
when Platform::GCE
|
152
|
+
@service_name = COMPUTE_SERVICE
|
153
|
+
# Check for specialized GCE environments (Managed VM or Dataflow).
|
154
|
+
# TODO: Add config options for these to allow for running outside GCE?
|
155
|
+
attributes = fetch_gce_metadata('instance/attributes/').split
|
156
|
+
if (attributes.include?('gae_backend_name') &&
|
157
|
+
attributes.include?('gae_backend_version'))
|
158
|
+
# Managed VM
|
159
|
+
@running_on_managed_vm = true
|
160
|
+
@gae_backend_name =
|
161
|
+
fetch_gce_metadata('instance/attributes/gae_backend_name')
|
162
|
+
@gae_backend_version =
|
163
|
+
fetch_gce_metadata('instance/attributes/gae_backend_version')
|
164
|
+
@service_name = APPENGINE_SERVICE
|
165
|
+
common_labels["#{APPENGINE_SERVICE}/module_id"] = @gae_backend_name
|
166
|
+
common_labels["#{APPENGINE_SERVICE}/version_id"] =
|
167
|
+
@gae_backend_version
|
168
|
+
elsif (attributes.include?('job_id'))
|
169
|
+
# Dataflow
|
170
|
+
@service_name = DATAFLOW_SERVICE
|
171
|
+
@dataflow_job_id = fetch_gce_metadata('instance/attributes/job_id')
|
172
|
+
common_labels["#{DATAFLOW_SERVICE}/job_id"] = @dataflow_job_id
|
173
|
+
end
|
174
|
+
# include GCE labels unless we're running on dataflow, which
|
175
|
+
# uses their own labels exclusively.
|
176
|
+
if (@service_name != DATAFLOW_SERVICE)
|
177
|
+
common_labels["#{COMPUTE_SERVICE}/resource_type"] = 'instance'
|
178
|
+
common_labels["#{COMPUTE_SERVICE}/resource_id"] = @vm_id
|
179
|
+
end
|
180
|
+
when Platform::EC2
|
181
|
+
@service_name = EC2_SERVICE
|
182
|
+
common_labels["#{EC2_SERVICE}/resource_type"] = 'instance'
|
183
|
+
common_labels["#{EC2_SERVICE}/resource_id"] = @vm_id
|
184
|
+
when Platform::OTHER
|
185
|
+
# Use COMPUTE_SERVICE as the default environment.
|
186
|
+
@service_name = COMPUTE_SERVICE
|
187
|
+
common_labels["#{COMPUTE_SERVICE}/resource_type"] = 'instance'
|
188
|
+
common_labels["#{COMPUTE_SERVICE}/resource_id"] = @vm_id
|
94
189
|
end
|
95
190
|
end
|
96
191
|
|
@@ -101,47 +196,6 @@ module Fluent
|
|
101
196
|
|
102
197
|
@successful_call = false
|
103
198
|
@timenanos_warning = false
|
104
|
-
|
105
|
-
if @use_metadata_service
|
106
|
-
# Grab metadata about the Google Compute Engine instance that we're on.
|
107
|
-
@project_id = fetch_metadata('project/project-id')
|
108
|
-
fully_qualified_zone = fetch_metadata('instance/zone')
|
109
|
-
@zone = fully_qualified_zone.rpartition('/')[2]
|
110
|
-
@vm_id = fetch_metadata('instance/id')
|
111
|
-
end
|
112
|
-
# TODO: Send instance tags and/or hostname with the logs as well?
|
113
|
-
@common_labels = {}
|
114
|
-
|
115
|
-
# If this is running on a Managed VM, grab the relevant App Engine
|
116
|
-
# metadata as well.
|
117
|
-
# TODO: Add config options for these to allow for running outside GCE?
|
118
|
-
attributes_string = @use_metadata_service ?
|
119
|
-
fetch_metadata('instance/attributes/') : ""
|
120
|
-
attributes = attributes_string.split
|
121
|
-
if (attributes.include?('gae_backend_name') &&
|
122
|
-
attributes.include?('gae_backend_version'))
|
123
|
-
@running_on_managed_vm = true
|
124
|
-
@gae_backend_name =
|
125
|
-
fetch_metadata('instance/attributes/gae_backend_name')
|
126
|
-
@gae_backend_version =
|
127
|
-
fetch_metadata('instance/attributes/gae_backend_version')
|
128
|
-
@service_name = APPENGINE_SERVICE
|
129
|
-
common_labels["#{APPENGINE_SERVICE}/module_id"] = @gae_backend_name
|
130
|
-
common_labels["#{APPENGINE_SERVICE}/version_id"] = @gae_backend_version
|
131
|
-
elsif (attributes.include?('job_id'))
|
132
|
-
@running_on_managed_vm = false
|
133
|
-
@service_name = DATAFLOW_SERVICE
|
134
|
-
@dataflow_job_id = fetch_metadata('instance/attributes/job_id')
|
135
|
-
common_labels["#{DATAFLOW_SERVICE}/job_id"] = @dataflow_job_id
|
136
|
-
else
|
137
|
-
@running_on_managed_vm = false
|
138
|
-
@service_name = COMPUTE_SERVICE
|
139
|
-
end
|
140
|
-
|
141
|
-
if (@service_name != DATAFLOW_SERVICE)
|
142
|
-
common_labels["#{COMPUTE_SERVICE}/resource_type"] = 'instance'
|
143
|
-
common_labels["#{COMPUTE_SERVICE}/resource_id"] = @vm_id
|
144
|
-
end
|
145
199
|
end
|
146
200
|
|
147
201
|
def shutdown
|
@@ -284,17 +338,64 @@ module Fluent
|
|
284
338
|
def log_write_failure(request, error)
|
285
339
|
dropped = request['entries'].length
|
286
340
|
$log.warn "Dropping #{dropped} log message(s)",
|
287
|
-
:error_class=>error.class.to_s, :error=>error.to_s
|
341
|
+
:error_class => error.class.to_s, :error => error.to_s
|
342
|
+
end
|
343
|
+
|
344
|
+
# "enum" of Platform values
|
345
|
+
module Platform
|
346
|
+
OTHER = 0 # Other/unkown platform
|
347
|
+
GCE = 1 # Google Compute Engine
|
348
|
+
EC2 = 2 # Amazon EC2
|
349
|
+
end
|
350
|
+
|
351
|
+
# Determine what platform we are running on by consulting the metadata
|
352
|
+
# service (unless the user has explicitly disabled using that).
|
353
|
+
def detect_platform
|
354
|
+
if !@use_metadata_service
|
355
|
+
$log.info "use_metadata_service is false; not detecting platform"
|
356
|
+
return Platform::OTHER
|
357
|
+
end
|
358
|
+
|
359
|
+
begin
|
360
|
+
open('http://' + METADATA_SERVICE_ADDR) do |f|
|
361
|
+
if (f.meta['metadata-flavor'] == 'Google')
|
362
|
+
$log.info 'Detected GCE platform'
|
363
|
+
return Platform::GCE
|
364
|
+
end
|
365
|
+
if (f.meta['server'] == 'EC2ws')
|
366
|
+
$log.info 'Detected EC2 platform'
|
367
|
+
return Platform::EC2
|
368
|
+
end
|
369
|
+
end
|
370
|
+
rescue Exception => e
|
371
|
+
$log.debug "Failed to access metadata service: ", :error => e
|
372
|
+
end
|
373
|
+
|
374
|
+
$log.info 'Unable to determine platform'
|
375
|
+
return Platform::OTHER
|
288
376
|
end
|
289
377
|
|
290
|
-
def
|
291
|
-
|
292
|
-
|
293
|
-
|
378
|
+
def fetch_gce_metadata(metadata_path)
|
379
|
+
raise "Called fetch_gce_metadata with platform=#{@platform}" unless
|
380
|
+
@platform == Platform::GCE
|
381
|
+
# See https://cloud.google.com/compute/docs/metadata
|
382
|
+
open('http://' + METADATA_SERVICE_ADDR + '/computeMetadata/v1/' +
|
383
|
+
metadata_path, {'Metadata-Flavor' => 'Google'}) do |f|
|
294
384
|
f.read
|
295
385
|
end
|
296
386
|
end
|
297
387
|
|
388
|
+
def fetch_ec2_metadata
|
389
|
+
raise "Called fetch_ec2_metadata with platform=#{@platform}" unless
|
390
|
+
@platform == Platform::EC2
|
391
|
+
# See http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html
|
392
|
+
open('http://' + METADATA_SERVICE_ADDR +
|
393
|
+
'/latest/dynamic/instance-identity/document') do |f|
|
394
|
+
contents = f.read()
|
395
|
+
return JSON.parse(contents)
|
396
|
+
end
|
397
|
+
end
|
398
|
+
|
298
399
|
# Values permitted by the API for 'severity' (which is an enum).
|
299
400
|
VALID_SEVERITIES = Set.new [
|
300
401
|
'DEFAULT', 'DEBUG', 'INFO', 'NOTICE', 'WARNING', 'ERROR', 'CRITICAL',
|
@@ -360,7 +461,7 @@ module Fluent
|
|
360
461
|
def init_api_client
|
361
462
|
@client = Google::APIClient.new(
|
362
463
|
:application_name => 'Fluentd Google Cloud Logging plugin',
|
363
|
-
:application_version => '0.
|
464
|
+
:application_version => '0.4.0',
|
364
465
|
:retries => 1)
|
365
466
|
|
366
467
|
if @auth_method == 'private_key'
|
@@ -25,22 +25,41 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
25
25
|
@logs_sent = []
|
26
26
|
end
|
27
27
|
|
28
|
+
# attributes used for the GCE metadata service
|
28
29
|
PROJECT_ID = 'test-project-id'
|
29
30
|
ZONE = 'us-central1-b'
|
30
31
|
FULLY_QUALIFIED_ZONE = 'projects/' + PROJECT_ID + '/zones/' + ZONE
|
31
32
|
VM_ID = '9876543210'
|
32
33
|
|
34
|
+
# attributes used for custom (overridden) configs
|
33
35
|
CUSTOM_PROJECT_ID = 'test-custom-project-id'
|
34
36
|
CUSTOM_ZONE = 'us-custom-central1-b'
|
35
37
|
CUSTOM_FULLY_QUALIFIED_ZONE = 'projects/' + PROJECT_ID + '/zones/' + ZONE
|
36
38
|
CUSTOM_VM_ID = 'C9876543210'
|
37
39
|
|
40
|
+
# attributes used for the EC2 metadata service
|
41
|
+
EC2_PROJECT_ID = 'test-ec2-project-id'
|
42
|
+
EC2_ZONE = 'us-west-2b'
|
43
|
+
EC2_PREFIXED_ZONE = 'aws:' + EC2_ZONE
|
44
|
+
EC2_VM_ID = 'i-81c16767'
|
45
|
+
EC2_ACCOUNT_ID = '123456789012'
|
46
|
+
|
47
|
+
# The formatting here matches the format used on the VM.
|
48
|
+
EC2_IDENTITY_DOCUMENT = %[{
|
49
|
+
"accountId" : "#{EC2_ACCOUNT_ID}",
|
50
|
+
"availabilityZone" : "#{EC2_ZONE}",
|
51
|
+
"instanceId" : "#{EC2_VM_ID}"
|
52
|
+
}]
|
53
|
+
|
54
|
+
# Managed VMs specific labels
|
38
55
|
MANAGED_VM_BACKEND_NAME = 'default'
|
39
56
|
MANAGED_VM_BACKEND_VERSION = 'guestbook2.0'
|
40
57
|
|
58
|
+
# Parameters used for authentication
|
41
59
|
AUTH_GRANT_TYPE = 'urn:ietf:params:oauth:grant-type:jwt-bearer'
|
42
60
|
FAKE_AUTH_TOKEN = 'abc123'
|
43
61
|
|
62
|
+
# Configuration files for various test scenarios
|
44
63
|
APPLICATION_DEFAULT_CONFIG = %[
|
45
64
|
]
|
46
65
|
|
@@ -50,29 +69,52 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
50
69
|
private_key_path test/plugin/data/c31e573fd7f62ed495c9ca3821a5a85cb036dee1-privatekey.p12
|
51
70
|
]
|
52
71
|
|
53
|
-
|
72
|
+
NO_METADATA_SERVICE_CONFIG = %[
|
54
73
|
use_metadata_service false
|
74
|
+
]
|
75
|
+
|
76
|
+
CUSTOM_METADATA_CONFIG = %[
|
55
77
|
project_id #{CUSTOM_PROJECT_ID}
|
56
78
|
zone #{CUSTOM_ZONE}
|
57
79
|
vm_id #{CUSTOM_VM_ID}
|
58
80
|
]
|
59
81
|
|
60
|
-
|
82
|
+
CONFIG_MISSING_PRIVATE_KEY_PATH = %[
|
61
83
|
auth_method private_key
|
62
84
|
private_key_email nobody@example.com
|
63
85
|
]
|
64
|
-
|
86
|
+
CONFIG_MISSING_PRIVATE_KEY_EMAIL = %[
|
65
87
|
auth_method private_key
|
66
88
|
private_key_path /fake/path/to/key
|
67
89
|
]
|
68
|
-
|
69
|
-
|
90
|
+
CONFIG_MISSING_METADATA_PROJECT_ID = %[
|
91
|
+
zone #{CUSTOM_ZONE}
|
92
|
+
vm_id #{CUSTOM_VM_ID}
|
93
|
+
]
|
94
|
+
CONFIG_MISSING_METADATA_ZONE = %[
|
95
|
+
project_id #{CUSTOM_PROJECT_ID}
|
96
|
+
vm_id #{CUSTOM_VM_ID}
|
97
|
+
]
|
98
|
+
CONFIG_MISSING_METADATA_VM_ID = %[
|
70
99
|
project_id #{CUSTOM_PROJECT_ID}
|
71
100
|
zone #{CUSTOM_ZONE}
|
72
101
|
]
|
102
|
+
CONFIG_MISSING_METADATA_ALL = %[
|
103
|
+
]
|
73
104
|
|
105
|
+
CONFIG_EC2_PROJECT_ID = %[
|
106
|
+
project_id #{EC2_PROJECT_ID}
|
107
|
+
]
|
108
|
+
|
109
|
+
CONFIG_EC2_PROJECT_ID_AND_CUSTOM_VM_ID = %[
|
110
|
+
project_id #{EC2_PROJECT_ID}
|
111
|
+
vm_id #{CUSTOM_VM_ID}
|
112
|
+
]
|
113
|
+
|
114
|
+
# Service configurations for various services
|
74
115
|
COMPUTE_SERVICE_NAME = 'compute.googleapis.com'
|
75
116
|
APPENGINE_SERVICE_NAME = 'appengine.googleapis.com'
|
117
|
+
EC2_SERVICE_NAME = 'ec2.amazonaws.com'
|
76
118
|
|
77
119
|
COMPUTE_PARAMS = {
|
78
120
|
'service_name' => COMPUTE_SERVICE_NAME,
|
@@ -109,6 +151,18 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
109
151
|
}
|
110
152
|
}
|
111
153
|
|
154
|
+
EC2_PARAMS = {
|
155
|
+
'service_name' => EC2_SERVICE_NAME,
|
156
|
+
'log_name' => 'test',
|
157
|
+
'project_id' => EC2_PROJECT_ID,
|
158
|
+
'zone' => EC2_PREFIXED_ZONE,
|
159
|
+
'labels' => {
|
160
|
+
"#{EC2_SERVICE_NAME}/resource_type" => 'instance',
|
161
|
+
"#{EC2_SERVICE_NAME}/resource_id" => EC2_VM_ID,
|
162
|
+
"#{EC2_SERVICE_NAME}/account_id" => EC2_ACCOUNT_ID
|
163
|
+
}
|
164
|
+
}
|
165
|
+
|
112
166
|
def create_driver(conf=APPLICATION_DEFAULT_CONFIG)
|
113
167
|
Fluent::Test::BufferedOutputTestDriver.new(
|
114
168
|
Fluent::GoogleCloudOutput).configure(conf)
|
@@ -127,16 +181,17 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
127
181
|
end
|
128
182
|
|
129
183
|
def test_configure_custom_metadata
|
184
|
+
setup_no_metadata_service_stubs
|
130
185
|
d = create_driver(CUSTOM_METADATA_CONFIG)
|
131
186
|
assert_equal CUSTOM_PROJECT_ID, d.instance.project_id
|
132
187
|
assert_equal CUSTOM_ZONE, d.instance.zone
|
133
188
|
assert_equal CUSTOM_VM_ID, d.instance.vm_id
|
134
189
|
end
|
135
190
|
|
136
|
-
def
|
191
|
+
def test_configure_invalid_private_key_configs
|
137
192
|
exception_count = 0
|
138
193
|
begin
|
139
|
-
d = create_driver(
|
194
|
+
d = create_driver(CONFIG_MISSING_PRIVATE_KEY_PATH)
|
140
195
|
rescue Fluent::ConfigError => error
|
141
196
|
assert error.message.include? 'private_key_path'
|
142
197
|
exception_count += 1
|
@@ -145,18 +200,54 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
145
200
|
|
146
201
|
exception_count = 0
|
147
202
|
begin
|
148
|
-
d = create_driver(
|
203
|
+
d = create_driver(CONFIG_MISSING_PRIVATE_KEY_EMAIL)
|
149
204
|
rescue Fluent::ConfigError => error
|
150
205
|
assert error.message.include? 'private_key_email'
|
151
206
|
exception_count += 1
|
152
207
|
end
|
153
208
|
assert_equal 1, exception_count
|
209
|
+
end
|
210
|
+
|
211
|
+
def test_configure_invalid_metadata_configs_no_metadata_service
|
212
|
+
setup_no_metadata_service_stubs
|
213
|
+
exception_count = 0
|
214
|
+
begin
|
215
|
+
d = create_driver(CONFIG_MISSING_METADATA_PROJECT_ID)
|
216
|
+
rescue Fluent::ConfigError => error
|
217
|
+
assert error.message.include? 'Unable to obtain metadata parameters:'
|
218
|
+
assert error.message.include? 'project_id'
|
219
|
+
exception_count += 1
|
220
|
+
end
|
221
|
+
assert_equal 1, exception_count
|
222
|
+
|
223
|
+
exception_count = 0
|
224
|
+
begin
|
225
|
+
d = create_driver(CONFIG_MISSING_METADATA_ZONE)
|
226
|
+
rescue Fluent::ConfigError => error
|
227
|
+
assert error.message.include? 'Unable to obtain metadata parameters:'
|
228
|
+
assert error.message.include? 'zone'
|
229
|
+
exception_count += 1
|
230
|
+
end
|
231
|
+
assert_equal 1, exception_count
|
232
|
+
|
233
|
+
exception_count = 0
|
234
|
+
begin
|
235
|
+
d = create_driver(CONFIG_MISSING_METADATA_VM_ID)
|
236
|
+
rescue Fluent::ConfigError => error
|
237
|
+
assert error.message.include? 'Unable to obtain metadata parameters:'
|
238
|
+
assert error.message.include? 'vm_id'
|
239
|
+
exception_count += 1
|
240
|
+
end
|
241
|
+
assert_equal 1, exception_count
|
154
242
|
|
155
243
|
exception_count = 0
|
156
244
|
begin
|
157
|
-
d = create_driver(
|
245
|
+
d = create_driver(CONFIG_MISSING_METADATA_ALL)
|
158
246
|
rescue Fluent::ConfigError => error
|
159
|
-
assert error.message.include? '
|
247
|
+
assert error.message.include? 'Unable to obtain metadata parameters:'
|
248
|
+
assert error.message.include? 'project_id'
|
249
|
+
assert error.message.include? 'zone'
|
250
|
+
assert error.message.include? 'vm_id'
|
160
251
|
exception_count += 1
|
161
252
|
end
|
162
253
|
assert_equal 1, exception_count
|
@@ -187,6 +278,18 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
187
278
|
|
188
279
|
def test_gce_metadata_does_not_load_when_use_metadata_service_is_false
|
189
280
|
Fluent::GoogleCloudOutput.any_instance.expects(:fetch_metadata).never
|
281
|
+
d = create_driver(NO_METADATA_SERVICE_CONFIG + CUSTOM_METADATA_CONFIG)
|
282
|
+
d.run
|
283
|
+
assert_equal CUSTOM_PROJECT_ID, d.instance.project_id
|
284
|
+
assert_equal CUSTOM_ZONE, d.instance.zone
|
285
|
+
assert_equal CUSTOM_VM_ID, d.instance.vm_id
|
286
|
+
assert_equal false, d.instance.running_on_managed_vm
|
287
|
+
end
|
288
|
+
|
289
|
+
def test_metadata_overrides_on_gce
|
290
|
+
# In this case we are overriding all configured parameters so we should
|
291
|
+
# see all "custom" values rather than the ones from the metadata server.
|
292
|
+
setup_gce_metadata_stubs
|
190
293
|
d = create_driver(CUSTOM_METADATA_CONFIG)
|
191
294
|
d.run
|
192
295
|
assert_equal CUSTOM_PROJECT_ID, d.instance.project_id
|
@@ -195,6 +298,51 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
195
298
|
assert_equal false, d.instance.running_on_managed_vm
|
196
299
|
end
|
197
300
|
|
301
|
+
def test_metadata_partial_overrides_on_gce
|
302
|
+
# Similar to above, but we are not overriding project_id in this config
|
303
|
+
# so we should see the metadata value for project_id and "custom" otherwise.
|
304
|
+
setup_gce_metadata_stubs
|
305
|
+
d = create_driver(CONFIG_MISSING_METADATA_PROJECT_ID)
|
306
|
+
d.run
|
307
|
+
assert_equal PROJECT_ID, d.instance.project_id
|
308
|
+
assert_equal CUSTOM_ZONE, d.instance.zone
|
309
|
+
assert_equal CUSTOM_VM_ID, d.instance.vm_id
|
310
|
+
assert_equal false, d.instance.running_on_managed_vm
|
311
|
+
end
|
312
|
+
|
313
|
+
def test_ec2_metadata_loading
|
314
|
+
setup_ec2_metadata_stubs
|
315
|
+
d = create_driver(CONFIG_EC2_PROJECT_ID)
|
316
|
+
d.run
|
317
|
+
assert_equal EC2_PROJECT_ID, d.instance.project_id
|
318
|
+
assert_equal EC2_PREFIXED_ZONE, d.instance.zone
|
319
|
+
assert_equal EC2_VM_ID, d.instance.vm_id
|
320
|
+
assert_equal false, d.instance.running_on_managed_vm
|
321
|
+
end
|
322
|
+
|
323
|
+
def test_ec2_metadata_partial_override
|
324
|
+
setup_ec2_metadata_stubs
|
325
|
+
d = create_driver(CONFIG_EC2_PROJECT_ID_AND_CUSTOM_VM_ID)
|
326
|
+
d.run
|
327
|
+
assert_equal EC2_PROJECT_ID, d.instance.project_id
|
328
|
+
assert_equal EC2_PREFIXED_ZONE, d.instance.zone
|
329
|
+
assert_equal CUSTOM_VM_ID, d.instance.vm_id
|
330
|
+
assert_equal false, d.instance.running_on_managed_vm
|
331
|
+
end
|
332
|
+
|
333
|
+
def test_ec2_metadata_requires_project_id
|
334
|
+
setup_ec2_metadata_stubs
|
335
|
+
exception_count = 0
|
336
|
+
begin
|
337
|
+
d = create_driver()
|
338
|
+
rescue Fluent::ConfigError => error
|
339
|
+
assert error.message.include? 'Unable to obtain metadata parameters:'
|
340
|
+
assert error.message.include? 'project_id'
|
341
|
+
exception_count += 1
|
342
|
+
end
|
343
|
+
assert_equal 1, exception_count
|
344
|
+
end
|
345
|
+
|
198
346
|
def test_one_log
|
199
347
|
setup_gce_metadata_stubs
|
200
348
|
setup_logging_stubs
|
@@ -240,15 +388,26 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
240
388
|
end
|
241
389
|
|
242
390
|
def test_one_log_custom_metadata
|
391
|
+
# don't set up any metadata stubs, so the test will fail if we try to
|
392
|
+
# fetch metadata (and explicitly check this as well).
|
243
393
|
Fluent::GoogleCloudOutput.any_instance.expects(:fetch_metadata).never
|
244
394
|
ENV['GOOGLE_APPLICATION_CREDENTIALS'] = 'test/plugin/data/credentials.json'
|
245
395
|
setup_logging_stubs
|
246
|
-
d = create_driver(CUSTOM_METADATA_CONFIG)
|
396
|
+
d = create_driver(NO_METADATA_SERVICE_CONFIG + CUSTOM_METADATA_CONFIG)
|
247
397
|
d.emit({'message' => log_entry(0)})
|
248
398
|
d.run
|
249
399
|
verify_log_entries(1, CUSTOM_PARAMS)
|
250
400
|
end
|
251
401
|
|
402
|
+
def test_one_log_ec2
|
403
|
+
setup_ec2_metadata_stubs
|
404
|
+
setup_logging_stubs
|
405
|
+
d = create_driver(CONFIG_EC2_PROJECT_ID)
|
406
|
+
d.emit({'message' => log_entry(0)})
|
407
|
+
d.run
|
408
|
+
verify_log_entries(1, EC2_PARAMS)
|
409
|
+
end
|
410
|
+
|
252
411
|
def test_struct_payload_log
|
253
412
|
setup_gce_metadata_stubs
|
254
413
|
setup_logging_stubs
|
@@ -534,7 +693,17 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
534
693
|
:headers => {'Content-Length' => response_body.length})
|
535
694
|
end
|
536
695
|
|
696
|
+
def setup_no_metadata_service_stubs
|
697
|
+
# Simulate a machine with no metadata service present
|
698
|
+
stub_request(:any, /http:\/\/169.254.169.254\/.*/).
|
699
|
+
to_raise Errno::EHOSTUNREACH;
|
700
|
+
end
|
701
|
+
|
537
702
|
def setup_gce_metadata_stubs
|
703
|
+
# Stub the root, used for platform detection by the plugin and 'googleauth'.
|
704
|
+
stub_request(:get, 'http://169.254.169.254').
|
705
|
+
to_return(:status => 200, :headers => {'Metadata-Flavor' => 'Google'})
|
706
|
+
|
538
707
|
# Create stubs for all the GCE metadata lookups the agent needs to make.
|
539
708
|
stub_metadata_request('project/project-id', PROJECT_ID)
|
540
709
|
stub_metadata_request('instance/zone', FULLY_QUALIFIED_ZONE)
|
@@ -542,11 +711,6 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
542
711
|
stub_metadata_request('instance/attributes/',
|
543
712
|
"attribute1\nattribute2\nattribute3")
|
544
713
|
|
545
|
-
# Used by 'googleauth' to test whether we're running on GCE.
|
546
|
-
# It only cares about the request succeeding with Metdata-Flavor: Google.
|
547
|
-
stub_request(:get, 'http://169.254.169.254').
|
548
|
-
to_return(:status => 200, :headers => {'Metadata-Flavor' => 'Google'})
|
549
|
-
|
550
714
|
# Used by 'googleauth' to fetch the default service account credentials.
|
551
715
|
stub_request(:get, 'http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/token').
|
552
716
|
to_return(:body => "{\"access_token\": \"#{FAKE_AUTH_TOKEN}\"}",
|
@@ -555,8 +719,20 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
555
719
|
'Content-Type' => 'application/json' })
|
556
720
|
end
|
557
721
|
|
722
|
+
def setup_ec2_metadata_stubs
|
723
|
+
# Stub the root, used for platform detection
|
724
|
+
stub_request(:get, 'http://169.254.169.254').
|
725
|
+
to_return(:status => 200, :headers => {'Server' => 'EC2ws'})
|
726
|
+
|
727
|
+
# Stub the identity document lookup made by the agent.
|
728
|
+
stub_request(:get, 'http://169.254.169.254/latest/dynamic/instance-identity/document').
|
729
|
+
to_return(:body => EC2_IDENTITY_DOCUMENT, :status => 200,
|
730
|
+
:headers => {'Content-Length' => EC2_IDENTITY_DOCUMENT.length})
|
731
|
+
end
|
732
|
+
|
558
733
|
def setup_logging_stubs
|
559
|
-
[COMPUTE_PARAMS, VMENGINE_PARAMS, CUSTOM_PARAMS].
|
734
|
+
[COMPUTE_PARAMS, VMENGINE_PARAMS, CUSTOM_PARAMS, EC2_PARAMS].
|
735
|
+
each do |params|
|
560
736
|
stub_request(:post, uri_for_log(params)).to_return do |request|
|
561
737
|
@logs_sent << JSON.parse(request.body)
|
562
738
|
{:body => ''}
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-google-cloud
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2015-
|
13
|
+
date: 2015-06-11 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: fluentd
|
@@ -60,6 +60,22 @@ dependencies:
|
|
60
60
|
- - ~>
|
61
61
|
- !ruby/object:Gem::Version
|
62
62
|
version: '0.4'
|
63
|
+
- !ruby/object:Gem::Dependency
|
64
|
+
name: json
|
65
|
+
requirement: !ruby/object:Gem::Requirement
|
66
|
+
none: false
|
67
|
+
requirements:
|
68
|
+
- - ~>
|
69
|
+
- !ruby/object:Gem::Version
|
70
|
+
version: 1.8.3
|
71
|
+
type: :runtime
|
72
|
+
prerelease: false
|
73
|
+
version_requirements: !ruby/object:Gem::Requirement
|
74
|
+
none: false
|
75
|
+
requirements:
|
76
|
+
- - ~>
|
77
|
+
- !ruby/object:Gem::Version
|
78
|
+
version: 1.8.3
|
63
79
|
- !ruby/object:Gem::Dependency
|
64
80
|
name: rake
|
65
81
|
requirement: !ruby/object:Gem::Requirement
|