fluent-plugin-google-cloud 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/Gemfile.lock CHANGED
@@ -1,10 +1,11 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- fluent-plugin-google-cloud (0.3.0)
4
+ fluent-plugin-google-cloud (0.3.1)
5
5
  fluentd (>= 0.10)
6
6
  google-api-client (>= 0.8)
7
7
  googleauth (~> 0.4)
8
+ json (~> 1.8.3)
8
9
 
9
10
  GEM
10
11
  remote: https://rubygems.org/
@@ -20,13 +21,13 @@ GEM
20
21
  addressable (>= 2.3.1)
21
22
  extlib (>= 0.9.15)
22
23
  multi_json (>= 1.0.0)
23
- cool.io (1.3.0)
24
+ cool.io (1.3.1)
24
25
  crack (0.4.2)
25
26
  safe_yaml (~> 1.0.0)
26
27
  extlib (0.9.16)
27
28
  faraday (0.9.1)
28
29
  multipart-post (>= 1.2, < 3)
29
- fluentd (0.12.8)
30
+ fluentd (0.12.11)
30
31
  cool.io (>= 1.2.2, < 2.0.0)
31
32
  http_parser.rb (>= 0.5.1, < 0.7.0)
32
33
  json (>= 1.4.3)
@@ -56,8 +57,8 @@ GEM
56
57
  signet (~> 0.6)
57
58
  http_parser.rb (0.6.0)
58
59
  i18n (0.7.0)
59
- json (1.8.2)
60
- jwt (1.4.1)
60
+ json (1.8.3)
61
+ jwt (1.5.0)
61
62
  launchy (2.4.3)
62
63
  addressable (~> 2.3)
63
64
  little-plugger (1.1.3)
@@ -66,22 +67,22 @@ GEM
66
67
  multi_json (~> 1.10)
67
68
  memoist (0.12.0)
68
69
  metaclass (0.0.4)
69
- minitest (5.6.1)
70
+ minitest (5.7.0)
70
71
  mocha (1.1.0)
71
72
  metaclass (~> 0.0.1)
72
- msgpack (0.5.11)
73
+ msgpack (0.5.12)
73
74
  multi_json (1.11.0)
74
75
  multipart-post (2.0.0)
75
76
  power_assert (0.2.3)
76
77
  rake (10.4.2)
77
78
  retriable (1.4.1)
78
79
  safe_yaml (1.0.4)
79
- sigdump (0.2.2)
80
- signet (0.6.0)
80
+ sigdump (0.2.3)
81
+ signet (0.6.1)
81
82
  addressable (~> 2.3)
82
83
  extlib (~> 0.9)
83
84
  faraday (~> 0.9)
84
- jwt (~> 1.0)
85
+ jwt (~> 1.5)
85
86
  multi_json (~> 1.10)
86
87
  string-scrub (0.0.5)
87
88
  test-unit (3.0.9)
@@ -4,7 +4,7 @@ Gem::Specification.new do |gem|
4
4
  gem.summary = %q{Fluentd plugin to stream logs to the Google Cloud Platform's logging API}
5
5
  gem.homepage = 'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
6
6
  gem.license = 'Apache 2.0'
7
- gem.version = '0.3.1'
7
+ gem.version = '0.4.0'
8
8
  gem.authors = ['Todd Derr', 'Alex Robinson']
9
9
  gem.email = ['salty@google.com']
10
10
 
@@ -15,6 +15,7 @@ Gem::Specification.new do |gem|
15
15
  gem.add_runtime_dependency 'fluentd', '>= 0.10'
16
16
  gem.add_runtime_dependency 'google-api-client', '>= 0.8'
17
17
  gem.add_runtime_dependency 'googleauth', '~> 0.4'
18
+ gem.add_runtime_dependency 'json', '~> 1.8.3'
18
19
  gem.add_development_dependency "rake", '>= 10.3.2'
19
20
  gem.add_development_dependency "webmock", '>= 1.17.0'
20
21
  gem.add_development_dependency "test-unit", "~> 3.0.2"
@@ -16,14 +16,18 @@ module Fluent
16
16
  class GoogleCloudOutput < BufferedOutput
17
17
  Fluent::Plugin.register_output('google_cloud', self)
18
18
 
19
- # Constants for Google service names.
19
+ # Constants for service names.
20
20
  APPENGINE_SERVICE = 'appengine.googleapis.com'
21
21
  COMPUTE_SERVICE = 'compute.googleapis.com'
22
22
  DATAFLOW_SERVICE = 'dataflow.googleapis.com'
23
+ EC2_SERVICE = 'ec2.amazonaws.com'
23
24
 
24
25
  # Name of the the Google cloud logging write scope.
25
26
  LOGGING_SCOPE = 'https://www.googleapis.com/auth/logging.write'
26
27
 
28
+ # Address of the metadata service.
29
+ METADATA_SERVICE_ADDR = '169.254.169.254'
30
+
27
31
  # DEPRECATED: auth_method (and support for 'private_key') is deprecated in
28
32
  # favor of Google Application Default Credentials as documented at:
29
33
  # https://developers.google.com/identity/protocols/application-default-credentials
@@ -36,10 +40,16 @@ module Fluent
36
40
  config_param :private_key_path, :string, :default => nil
37
41
  config_param :private_key_passphrase, :string, :default => 'notasecret'
38
42
 
39
- # If use_metadata_service is set to true, we obtain the project_id, zone,
40
- # and vm_id from the GCE metadata service. Otherwise, those parameters
41
- # must be specified in the config file explicitly.
43
+ # Specify project/instance metadata.
44
+ #
45
+ # project_id, zone, and vm_id are required to have valid values, which
46
+ # can be obtained from the metadata service or set explicitly.
47
+ # Otherwise, the plugin will fail to initialize.
48
+ #
49
+ # Whether to attempt to obtain metadata from the local metadata service.
50
+ # It is safe to specify 'true' even on platforms with no metadata service.
42
51
  config_param :use_metadata_service, :bool, :default => true
52
+ # These parameters override any values obtained from the metadata service.
43
53
  config_param :project_id, :string, :default => nil
44
54
  config_param :zone, :string, :default => nil
45
55
  config_param :vm_id, :string, :default => nil
@@ -63,6 +73,7 @@ module Fluent
63
73
  require 'google/api_client'
64
74
  require 'google/api_client/auth/compute_service_account'
65
75
  require 'googleauth'
76
+ require 'json'
66
77
  require 'open-uri'
67
78
  end
68
79
 
@@ -86,11 +97,95 @@ module Fluent
86
97
  end
87
98
  end
88
99
 
89
- unless @use_metadata_service
90
- unless @project_id && @zone && @vm_id
91
- raise Fluent::ConfigError,
92
- ('Please specify "project_id", "zone" and "vm_id" if you set "use_metadata_service" to false')
100
+ # TODO: Send instance tags and/or hostname as labels as well?
101
+ @common_labels = {}
102
+
103
+ # set attributes from metadata (unless overriden by static config)
104
+ @platform = detect_platform
105
+ case @platform
106
+ when Platform::GCE
107
+ if @project_id.nil?
108
+ @project_id = fetch_gce_metadata('project/project-id')
109
+ end
110
+ if @zone.nil?
111
+ # this returns "projects/<number>/zones/<zone>"; we only want
112
+ # the part after the final slash.
113
+ fully_qualified_zone = fetch_gce_metadata('instance/zone')
114
+ @zone = fully_qualified_zone.rpartition('/')[2]
115
+ end
116
+ if @vm_id.nil?
117
+ @vm_id = fetch_gce_metadata('instance/id')
118
+ end
119
+ when Platform::EC2
120
+ metadata = fetch_ec2_metadata
121
+ if @zone.nil? && metadata.has_key?('availabilityZone')
122
+ @zone = 'aws:' + metadata['availabilityZone']
93
123
  end
124
+ if @vm_id.nil? && metadata.has_key?('instanceId')
125
+ @vm_id = metadata['instanceId']
126
+ end
127
+ if metadata.has_key?('accountId')
128
+ common_labels["#{EC2_SERVICE}/account_id"] = metadata['accountId']
129
+ end
130
+ when Platform::OTHER
131
+ # do nothing
132
+ else
133
+ raise Fluent::ConfigError, 'Unknown platform ' + @platform
134
+ end
135
+
136
+ # all metadata parameters must now be set
137
+ unless @project_id && @zone && @vm_id
138
+ missing = []
139
+ missing << "project_id" unless @project_id
140
+ missing << "zone" unless @zone
141
+ missing << "vm_id" unless @vm_id
142
+ raise Fluent::ConfigError,
143
+ ('Unable to obtain metadata parameters: ' + missing.join(' '))
144
+ end
145
+
146
+ # Default this to false; it is only overwritten if we detect Managed VM.
147
+ @running_on_managed_vm = false
148
+
149
+ # Set labels, etc. based on the config
150
+ case @platform
151
+ when Platform::GCE
152
+ @service_name = COMPUTE_SERVICE
153
+ # Check for specialized GCE environments (Managed VM or Dataflow).
154
+ # TODO: Add config options for these to allow for running outside GCE?
155
+ attributes = fetch_gce_metadata('instance/attributes/').split
156
+ if (attributes.include?('gae_backend_name') &&
157
+ attributes.include?('gae_backend_version'))
158
+ # Managed VM
159
+ @running_on_managed_vm = true
160
+ @gae_backend_name =
161
+ fetch_gce_metadata('instance/attributes/gae_backend_name')
162
+ @gae_backend_version =
163
+ fetch_gce_metadata('instance/attributes/gae_backend_version')
164
+ @service_name = APPENGINE_SERVICE
165
+ common_labels["#{APPENGINE_SERVICE}/module_id"] = @gae_backend_name
166
+ common_labels["#{APPENGINE_SERVICE}/version_id"] =
167
+ @gae_backend_version
168
+ elsif (attributes.include?('job_id'))
169
+ # Dataflow
170
+ @service_name = DATAFLOW_SERVICE
171
+ @dataflow_job_id = fetch_gce_metadata('instance/attributes/job_id')
172
+ common_labels["#{DATAFLOW_SERVICE}/job_id"] = @dataflow_job_id
173
+ end
174
+ # include GCE labels unless we're running on dataflow, which
175
+ # uses their own labels exclusively.
176
+ if (@service_name != DATAFLOW_SERVICE)
177
+ common_labels["#{COMPUTE_SERVICE}/resource_type"] = 'instance'
178
+ common_labels["#{COMPUTE_SERVICE}/resource_id"] = @vm_id
179
+ end
180
+ when Platform::EC2
181
+ @service_name = EC2_SERVICE
182
+ common_labels["#{EC2_SERVICE}/resource_type"] = 'instance'
183
+ common_labels["#{EC2_SERVICE}/resource_id"] = @vm_id
184
+ when Platform::OTHER
185
+ # Use COMPUTE_SERVICE as the default environment.
186
+ @service_name = COMPUTE_SERVICE
187
+ common_labels["#{COMPUTE_SERVICE}/resource_type"] = 'instance'
188
+ common_labels["#{COMPUTE_SERVICE}/resource_id"] = @vm_id
94
189
  end
95
190
  end
96
191
 
@@ -101,47 +196,6 @@ module Fluent
101
196
 
102
197
  @successful_call = false
103
198
  @timenanos_warning = false
104
-
105
- if @use_metadata_service
106
- # Grab metadata about the Google Compute Engine instance that we're on.
107
- @project_id = fetch_metadata('project/project-id')
108
- fully_qualified_zone = fetch_metadata('instance/zone')
109
- @zone = fully_qualified_zone.rpartition('/')[2]
110
- @vm_id = fetch_metadata('instance/id')
111
- end
112
- # TODO: Send instance tags and/or hostname with the logs as well?
113
- @common_labels = {}
114
-
115
- # If this is running on a Managed VM, grab the relevant App Engine
116
- # metadata as well.
117
- # TODO: Add config options for these to allow for running outside GCE?
118
- attributes_string = @use_metadata_service ?
119
- fetch_metadata('instance/attributes/') : ""
120
- attributes = attributes_string.split
121
- if (attributes.include?('gae_backend_name') &&
122
- attributes.include?('gae_backend_version'))
123
- @running_on_managed_vm = true
124
- @gae_backend_name =
125
- fetch_metadata('instance/attributes/gae_backend_name')
126
- @gae_backend_version =
127
- fetch_metadata('instance/attributes/gae_backend_version')
128
- @service_name = APPENGINE_SERVICE
129
- common_labels["#{APPENGINE_SERVICE}/module_id"] = @gae_backend_name
130
- common_labels["#{APPENGINE_SERVICE}/version_id"] = @gae_backend_version
131
- elsif (attributes.include?('job_id'))
132
- @running_on_managed_vm = false
133
- @service_name = DATAFLOW_SERVICE
134
- @dataflow_job_id = fetch_metadata('instance/attributes/job_id')
135
- common_labels["#{DATAFLOW_SERVICE}/job_id"] = @dataflow_job_id
136
- else
137
- @running_on_managed_vm = false
138
- @service_name = COMPUTE_SERVICE
139
- end
140
-
141
- if (@service_name != DATAFLOW_SERVICE)
142
- common_labels["#{COMPUTE_SERVICE}/resource_type"] = 'instance'
143
- common_labels["#{COMPUTE_SERVICE}/resource_id"] = @vm_id
144
- end
145
199
  end
146
200
 
147
201
  def shutdown
@@ -284,17 +338,64 @@ module Fluent
284
338
  def log_write_failure(request, error)
285
339
  dropped = request['entries'].length
286
340
  $log.warn "Dropping #{dropped} log message(s)",
287
- :error_class=>error.class.to_s, :error=>error.to_s
341
+ :error_class => error.class.to_s, :error => error.to_s
342
+ end
343
+
344
+ # "enum" of Platform values
345
+ module Platform
346
+ OTHER = 0 # Other/unkown platform
347
+ GCE = 1 # Google Compute Engine
348
+ EC2 = 2 # Amazon EC2
349
+ end
350
+
351
+ # Determine what platform we are running on by consulting the metadata
352
+ # service (unless the user has explicitly disabled using that).
353
+ def detect_platform
354
+ if !@use_metadata_service
355
+ $log.info "use_metadata_service is false; not detecting platform"
356
+ return Platform::OTHER
357
+ end
358
+
359
+ begin
360
+ open('http://' + METADATA_SERVICE_ADDR) do |f|
361
+ if (f.meta['metadata-flavor'] == 'Google')
362
+ $log.info 'Detected GCE platform'
363
+ return Platform::GCE
364
+ end
365
+ if (f.meta['server'] == 'EC2ws')
366
+ $log.info 'Detected EC2 platform'
367
+ return Platform::EC2
368
+ end
369
+ end
370
+ rescue Exception => e
371
+ $log.debug "Failed to access metadata service: ", :error => e
372
+ end
373
+
374
+ $log.info 'Unable to determine platform'
375
+ return Platform::OTHER
288
376
  end
289
377
 
290
- def fetch_metadata(metadata_path)
291
- # Fetch GCE metadata - see https://cloud.google.com/compute/docs/metadata
292
- open('http://169.254.169.254/computeMetadata/v1/' + metadata_path,
293
- {'Metadata-Flavor' => 'Google'}) do |f|
378
+ def fetch_gce_metadata(metadata_path)
379
+ raise "Called fetch_gce_metadata with platform=#{@platform}" unless
380
+ @platform == Platform::GCE
381
+ # See https://cloud.google.com/compute/docs/metadata
382
+ open('http://' + METADATA_SERVICE_ADDR + '/computeMetadata/v1/' +
383
+ metadata_path, {'Metadata-Flavor' => 'Google'}) do |f|
294
384
  f.read
295
385
  end
296
386
  end
297
387
 
388
+ def fetch_ec2_metadata
389
+ raise "Called fetch_ec2_metadata with platform=#{@platform}" unless
390
+ @platform == Platform::EC2
391
+ # See http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html
392
+ open('http://' + METADATA_SERVICE_ADDR +
393
+ '/latest/dynamic/instance-identity/document') do |f|
394
+ contents = f.read()
395
+ return JSON.parse(contents)
396
+ end
397
+ end
398
+
298
399
  # Values permitted by the API for 'severity' (which is an enum).
299
400
  VALID_SEVERITIES = Set.new [
300
401
  'DEFAULT', 'DEBUG', 'INFO', 'NOTICE', 'WARNING', 'ERROR', 'CRITICAL',
@@ -360,7 +461,7 @@ module Fluent
360
461
  def init_api_client
361
462
  @client = Google::APIClient.new(
362
463
  :application_name => 'Fluentd Google Cloud Logging plugin',
363
- :application_version => '0.3.1',
464
+ :application_version => '0.4.0',
364
465
  :retries => 1)
365
466
 
366
467
  if @auth_method == 'private_key'
@@ -25,22 +25,41 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
25
25
  @logs_sent = []
26
26
  end
27
27
 
28
+ # attributes used for the GCE metadata service
28
29
  PROJECT_ID = 'test-project-id'
29
30
  ZONE = 'us-central1-b'
30
31
  FULLY_QUALIFIED_ZONE = 'projects/' + PROJECT_ID + '/zones/' + ZONE
31
32
  VM_ID = '9876543210'
32
33
 
34
+ # attributes used for custom (overridden) configs
33
35
  CUSTOM_PROJECT_ID = 'test-custom-project-id'
34
36
  CUSTOM_ZONE = 'us-custom-central1-b'
35
37
  CUSTOM_FULLY_QUALIFIED_ZONE = 'projects/' + PROJECT_ID + '/zones/' + ZONE
36
38
  CUSTOM_VM_ID = 'C9876543210'
37
39
 
40
+ # attributes used for the EC2 metadata service
41
+ EC2_PROJECT_ID = 'test-ec2-project-id'
42
+ EC2_ZONE = 'us-west-2b'
43
+ EC2_PREFIXED_ZONE = 'aws:' + EC2_ZONE
44
+ EC2_VM_ID = 'i-81c16767'
45
+ EC2_ACCOUNT_ID = '123456789012'
46
+
47
+ # The formatting here matches the format used on the VM.
48
+ EC2_IDENTITY_DOCUMENT = %[{
49
+ "accountId" : "#{EC2_ACCOUNT_ID}",
50
+ "availabilityZone" : "#{EC2_ZONE}",
51
+ "instanceId" : "#{EC2_VM_ID}"
52
+ }]
53
+
54
+ # Managed VMs specific labels
38
55
  MANAGED_VM_BACKEND_NAME = 'default'
39
56
  MANAGED_VM_BACKEND_VERSION = 'guestbook2.0'
40
57
 
58
+ # Parameters used for authentication
41
59
  AUTH_GRANT_TYPE = 'urn:ietf:params:oauth:grant-type:jwt-bearer'
42
60
  FAKE_AUTH_TOKEN = 'abc123'
43
61
 
62
+ # Configuration files for various test scenarios
44
63
  APPLICATION_DEFAULT_CONFIG = %[
45
64
  ]
46
65
 
@@ -50,29 +69,52 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
50
69
  private_key_path test/plugin/data/c31e573fd7f62ed495c9ca3821a5a85cb036dee1-privatekey.p12
51
70
  ]
52
71
 
53
- CUSTOM_METADATA_CONFIG = %[
72
+ NO_METADATA_SERVICE_CONFIG = %[
54
73
  use_metadata_service false
74
+ ]
75
+
76
+ CUSTOM_METADATA_CONFIG = %[
55
77
  project_id #{CUSTOM_PROJECT_ID}
56
78
  zone #{CUSTOM_ZONE}
57
79
  vm_id #{CUSTOM_VM_ID}
58
80
  ]
59
81
 
60
- INVALID_CONFIG_MISSING_PRIVATE_KEY_PATH = %[
82
+ CONFIG_MISSING_PRIVATE_KEY_PATH = %[
61
83
  auth_method private_key
62
84
  private_key_email nobody@example.com
63
85
  ]
64
- INVALID_CONFIG_MISSING_PRIVATE_KEY_EMAIL = %[
86
+ CONFIG_MISSING_PRIVATE_KEY_EMAIL = %[
65
87
  auth_method private_key
66
88
  private_key_path /fake/path/to/key
67
89
  ]
68
- INVALID_CONFIG_MISSING_METADATA_VM_ID = %[
69
- use_metadata_service false
90
+ CONFIG_MISSING_METADATA_PROJECT_ID = %[
91
+ zone #{CUSTOM_ZONE}
92
+ vm_id #{CUSTOM_VM_ID}
93
+ ]
94
+ CONFIG_MISSING_METADATA_ZONE = %[
95
+ project_id #{CUSTOM_PROJECT_ID}
96
+ vm_id #{CUSTOM_VM_ID}
97
+ ]
98
+ CONFIG_MISSING_METADATA_VM_ID = %[
70
99
  project_id #{CUSTOM_PROJECT_ID}
71
100
  zone #{CUSTOM_ZONE}
72
101
  ]
102
+ CONFIG_MISSING_METADATA_ALL = %[
103
+ ]
73
104
 
105
+ CONFIG_EC2_PROJECT_ID = %[
106
+ project_id #{EC2_PROJECT_ID}
107
+ ]
108
+
109
+ CONFIG_EC2_PROJECT_ID_AND_CUSTOM_VM_ID = %[
110
+ project_id #{EC2_PROJECT_ID}
111
+ vm_id #{CUSTOM_VM_ID}
112
+ ]
113
+
114
+ # Service configurations for various services
74
115
  COMPUTE_SERVICE_NAME = 'compute.googleapis.com'
75
116
  APPENGINE_SERVICE_NAME = 'appengine.googleapis.com'
117
+ EC2_SERVICE_NAME = 'ec2.amazonaws.com'
76
118
 
77
119
  COMPUTE_PARAMS = {
78
120
  'service_name' => COMPUTE_SERVICE_NAME,
@@ -109,6 +151,18 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
109
151
  }
110
152
  }
111
153
 
154
+ EC2_PARAMS = {
155
+ 'service_name' => EC2_SERVICE_NAME,
156
+ 'log_name' => 'test',
157
+ 'project_id' => EC2_PROJECT_ID,
158
+ 'zone' => EC2_PREFIXED_ZONE,
159
+ 'labels' => {
160
+ "#{EC2_SERVICE_NAME}/resource_type" => 'instance',
161
+ "#{EC2_SERVICE_NAME}/resource_id" => EC2_VM_ID,
162
+ "#{EC2_SERVICE_NAME}/account_id" => EC2_ACCOUNT_ID
163
+ }
164
+ }
165
+
112
166
  def create_driver(conf=APPLICATION_DEFAULT_CONFIG)
113
167
  Fluent::Test::BufferedOutputTestDriver.new(
114
168
  Fluent::GoogleCloudOutput).configure(conf)
@@ -127,16 +181,17 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
127
181
  end
128
182
 
129
183
  def test_configure_custom_metadata
184
+ setup_no_metadata_service_stubs
130
185
  d = create_driver(CUSTOM_METADATA_CONFIG)
131
186
  assert_equal CUSTOM_PROJECT_ID, d.instance.project_id
132
187
  assert_equal CUSTOM_ZONE, d.instance.zone
133
188
  assert_equal CUSTOM_VM_ID, d.instance.vm_id
134
189
  end
135
190
 
136
- def test_configure_invalid_configs
191
+ def test_configure_invalid_private_key_configs
137
192
  exception_count = 0
138
193
  begin
139
- d = create_driver(INVALID_CONFIG_MISSING_PRIVATE_KEY_PATH)
194
+ d = create_driver(CONFIG_MISSING_PRIVATE_KEY_PATH)
140
195
  rescue Fluent::ConfigError => error
141
196
  assert error.message.include? 'private_key_path'
142
197
  exception_count += 1
@@ -145,18 +200,54 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
145
200
 
146
201
  exception_count = 0
147
202
  begin
148
- d = create_driver(INVALID_CONFIG_MISSING_PRIVATE_KEY_EMAIL)
203
+ d = create_driver(CONFIG_MISSING_PRIVATE_KEY_EMAIL)
149
204
  rescue Fluent::ConfigError => error
150
205
  assert error.message.include? 'private_key_email'
151
206
  exception_count += 1
152
207
  end
153
208
  assert_equal 1, exception_count
209
+ end
210
+
211
+ def test_configure_invalid_metadata_configs_no_metadata_service
212
+ setup_no_metadata_service_stubs
213
+ exception_count = 0
214
+ begin
215
+ d = create_driver(CONFIG_MISSING_METADATA_PROJECT_ID)
216
+ rescue Fluent::ConfigError => error
217
+ assert error.message.include? 'Unable to obtain metadata parameters:'
218
+ assert error.message.include? 'project_id'
219
+ exception_count += 1
220
+ end
221
+ assert_equal 1, exception_count
222
+
223
+ exception_count = 0
224
+ begin
225
+ d = create_driver(CONFIG_MISSING_METADATA_ZONE)
226
+ rescue Fluent::ConfigError => error
227
+ assert error.message.include? 'Unable to obtain metadata parameters:'
228
+ assert error.message.include? 'zone'
229
+ exception_count += 1
230
+ end
231
+ assert_equal 1, exception_count
232
+
233
+ exception_count = 0
234
+ begin
235
+ d = create_driver(CONFIG_MISSING_METADATA_VM_ID)
236
+ rescue Fluent::ConfigError => error
237
+ assert error.message.include? 'Unable to obtain metadata parameters:'
238
+ assert error.message.include? 'vm_id'
239
+ exception_count += 1
240
+ end
241
+ assert_equal 1, exception_count
154
242
 
155
243
  exception_count = 0
156
244
  begin
157
- d = create_driver(INVALID_CONFIG_MISSING_METADATA_VM_ID)
245
+ d = create_driver(CONFIG_MISSING_METADATA_ALL)
158
246
  rescue Fluent::ConfigError => error
159
- assert error.message.include? 'use_metadata_service'
247
+ assert error.message.include? 'Unable to obtain metadata parameters:'
248
+ assert error.message.include? 'project_id'
249
+ assert error.message.include? 'zone'
250
+ assert error.message.include? 'vm_id'
160
251
  exception_count += 1
161
252
  end
162
253
  assert_equal 1, exception_count
@@ -187,6 +278,18 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
187
278
 
188
279
  def test_gce_metadata_does_not_load_when_use_metadata_service_is_false
189
280
  Fluent::GoogleCloudOutput.any_instance.expects(:fetch_metadata).never
281
+ d = create_driver(NO_METADATA_SERVICE_CONFIG + CUSTOM_METADATA_CONFIG)
282
+ d.run
283
+ assert_equal CUSTOM_PROJECT_ID, d.instance.project_id
284
+ assert_equal CUSTOM_ZONE, d.instance.zone
285
+ assert_equal CUSTOM_VM_ID, d.instance.vm_id
286
+ assert_equal false, d.instance.running_on_managed_vm
287
+ end
288
+
289
+ def test_metadata_overrides_on_gce
290
+ # In this case we are overriding all configured parameters so we should
291
+ # see all "custom" values rather than the ones from the metadata server.
292
+ setup_gce_metadata_stubs
190
293
  d = create_driver(CUSTOM_METADATA_CONFIG)
191
294
  d.run
192
295
  assert_equal CUSTOM_PROJECT_ID, d.instance.project_id
@@ -195,6 +298,51 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
195
298
  assert_equal false, d.instance.running_on_managed_vm
196
299
  end
197
300
 
301
+ def test_metadata_partial_overrides_on_gce
302
+ # Similar to above, but we are not overriding project_id in this config
303
+ # so we should see the metadata value for project_id and "custom" otherwise.
304
+ setup_gce_metadata_stubs
305
+ d = create_driver(CONFIG_MISSING_METADATA_PROJECT_ID)
306
+ d.run
307
+ assert_equal PROJECT_ID, d.instance.project_id
308
+ assert_equal CUSTOM_ZONE, d.instance.zone
309
+ assert_equal CUSTOM_VM_ID, d.instance.vm_id
310
+ assert_equal false, d.instance.running_on_managed_vm
311
+ end
312
+
313
+ def test_ec2_metadata_loading
314
+ setup_ec2_metadata_stubs
315
+ d = create_driver(CONFIG_EC2_PROJECT_ID)
316
+ d.run
317
+ assert_equal EC2_PROJECT_ID, d.instance.project_id
318
+ assert_equal EC2_PREFIXED_ZONE, d.instance.zone
319
+ assert_equal EC2_VM_ID, d.instance.vm_id
320
+ assert_equal false, d.instance.running_on_managed_vm
321
+ end
322
+
323
+ def test_ec2_metadata_partial_override
324
+ setup_ec2_metadata_stubs
325
+ d = create_driver(CONFIG_EC2_PROJECT_ID_AND_CUSTOM_VM_ID)
326
+ d.run
327
+ assert_equal EC2_PROJECT_ID, d.instance.project_id
328
+ assert_equal EC2_PREFIXED_ZONE, d.instance.zone
329
+ assert_equal CUSTOM_VM_ID, d.instance.vm_id
330
+ assert_equal false, d.instance.running_on_managed_vm
331
+ end
332
+
333
+ def test_ec2_metadata_requires_project_id
334
+ setup_ec2_metadata_stubs
335
+ exception_count = 0
336
+ begin
337
+ d = create_driver()
338
+ rescue Fluent::ConfigError => error
339
+ assert error.message.include? 'Unable to obtain metadata parameters:'
340
+ assert error.message.include? 'project_id'
341
+ exception_count += 1
342
+ end
343
+ assert_equal 1, exception_count
344
+ end
345
+
198
346
  def test_one_log
199
347
  setup_gce_metadata_stubs
200
348
  setup_logging_stubs
@@ -240,15 +388,26 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
240
388
  end
241
389
 
242
390
  def test_one_log_custom_metadata
391
+ # don't set up any metadata stubs, so the test will fail if we try to
392
+ # fetch metadata (and explicitly check this as well).
243
393
  Fluent::GoogleCloudOutput.any_instance.expects(:fetch_metadata).never
244
394
  ENV['GOOGLE_APPLICATION_CREDENTIALS'] = 'test/plugin/data/credentials.json'
245
395
  setup_logging_stubs
246
- d = create_driver(CUSTOM_METADATA_CONFIG)
396
+ d = create_driver(NO_METADATA_SERVICE_CONFIG + CUSTOM_METADATA_CONFIG)
247
397
  d.emit({'message' => log_entry(0)})
248
398
  d.run
249
399
  verify_log_entries(1, CUSTOM_PARAMS)
250
400
  end
251
401
 
402
+ def test_one_log_ec2
403
+ setup_ec2_metadata_stubs
404
+ setup_logging_stubs
405
+ d = create_driver(CONFIG_EC2_PROJECT_ID)
406
+ d.emit({'message' => log_entry(0)})
407
+ d.run
408
+ verify_log_entries(1, EC2_PARAMS)
409
+ end
410
+
252
411
  def test_struct_payload_log
253
412
  setup_gce_metadata_stubs
254
413
  setup_logging_stubs
@@ -534,7 +693,17 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
534
693
  :headers => {'Content-Length' => response_body.length})
535
694
  end
536
695
 
696
+ def setup_no_metadata_service_stubs
697
+ # Simulate a machine with no metadata service present
698
+ stub_request(:any, /http:\/\/169.254.169.254\/.*/).
699
+ to_raise Errno::EHOSTUNREACH;
700
+ end
701
+
537
702
  def setup_gce_metadata_stubs
703
+ # Stub the root, used for platform detection by the plugin and 'googleauth'.
704
+ stub_request(:get, 'http://169.254.169.254').
705
+ to_return(:status => 200, :headers => {'Metadata-Flavor' => 'Google'})
706
+
538
707
  # Create stubs for all the GCE metadata lookups the agent needs to make.
539
708
  stub_metadata_request('project/project-id', PROJECT_ID)
540
709
  stub_metadata_request('instance/zone', FULLY_QUALIFIED_ZONE)
@@ -542,11 +711,6 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
542
711
  stub_metadata_request('instance/attributes/',
543
712
  "attribute1\nattribute2\nattribute3")
544
713
 
545
- # Used by 'googleauth' to test whether we're running on GCE.
546
- # It only cares about the request succeeding with Metdata-Flavor: Google.
547
- stub_request(:get, 'http://169.254.169.254').
548
- to_return(:status => 200, :headers => {'Metadata-Flavor' => 'Google'})
549
-
550
714
  # Used by 'googleauth' to fetch the default service account credentials.
551
715
  stub_request(:get, 'http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/token').
552
716
  to_return(:body => "{\"access_token\": \"#{FAKE_AUTH_TOKEN}\"}",
@@ -555,8 +719,20 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
555
719
  'Content-Type' => 'application/json' })
556
720
  end
557
721
 
722
+ def setup_ec2_metadata_stubs
723
+ # Stub the root, used for platform detection
724
+ stub_request(:get, 'http://169.254.169.254').
725
+ to_return(:status => 200, :headers => {'Server' => 'EC2ws'})
726
+
727
+ # Stub the identity document lookup made by the agent.
728
+ stub_request(:get, 'http://169.254.169.254/latest/dynamic/instance-identity/document').
729
+ to_return(:body => EC2_IDENTITY_DOCUMENT, :status => 200,
730
+ :headers => {'Content-Length' => EC2_IDENTITY_DOCUMENT.length})
731
+ end
732
+
558
733
  def setup_logging_stubs
559
- [COMPUTE_PARAMS, VMENGINE_PARAMS, CUSTOM_PARAMS].each do |params|
734
+ [COMPUTE_PARAMS, VMENGINE_PARAMS, CUSTOM_PARAMS, EC2_PARAMS].
735
+ each do |params|
560
736
  stub_request(:post, uri_for_log(params)).to_return do |request|
561
737
  @logs_sent << JSON.parse(request.body)
562
738
  {:body => ''}
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-google-cloud
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.1
4
+ version: 0.4.0
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2015-05-19 00:00:00.000000000 Z
13
+ date: 2015-06-11 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: fluentd
@@ -60,6 +60,22 @@ dependencies:
60
60
  - - ~>
61
61
  - !ruby/object:Gem::Version
62
62
  version: '0.4'
63
+ - !ruby/object:Gem::Dependency
64
+ name: json
65
+ requirement: !ruby/object:Gem::Requirement
66
+ none: false
67
+ requirements:
68
+ - - ~>
69
+ - !ruby/object:Gem::Version
70
+ version: 1.8.3
71
+ type: :runtime
72
+ prerelease: false
73
+ version_requirements: !ruby/object:Gem::Requirement
74
+ none: false
75
+ requirements:
76
+ - - ~>
77
+ - !ruby/object:Gem::Version
78
+ version: 1.8.3
63
79
  - !ruby/object:Gem::Dependency
64
80
  name: rake
65
81
  requirement: !ruby/object:Gem::Requirement