chef-provisioning-fog 0.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,381 @@
1
+ require 'chef/log'
2
+ require 'fog/aws'
3
+ require 'uri'
4
+
5
+ # fog:AWS:<account_id>:<region>
6
+ # fog:AWS:<profile_name>
7
+ # fog:AWS:<profile_name>:<region>
8
+ class Chef
9
+ module Provisioning
10
+ module FogDriver
11
+ module Providers
12
+ class AWS < FogDriver::Driver
13
+
14
+ require_relative 'aws/credentials'
15
+
16
+ Driver.register_provider_class('AWS', FogDriver::Providers::AWS)
17
+
18
+ def creator
19
+ driver_options[:aws_account_info][:aws_username]
20
+ end
21
+
22
+ def default_ssh_username
23
+ 'ubuntu'
24
+ end
25
+
26
+ def allocate_image(action_handler, image_spec, image_options, machine_spec)
27
+ if image_spec.location
28
+ image = compute.images.get(image_spec.location['image_id'])
29
+ if image
30
+ raise "The image already exists, why are you asking me to create it? I can't do that, Dave."
31
+ end
32
+ end
33
+ action_handler.perform_action "Create image #{image_spec.name} from machine #{machine_spec.name} with options #{image_options.inspect}" do
34
+ opt = image_options.dup
35
+ response = compute.create_image(machine_spec.location['server_id'],
36
+ image_spec.name,
37
+ opt.delete(:description) || "The image formerly and currently named '#{image_spec.name}'",
38
+ opt.delete(:no_reboot) || false,
39
+ opt)
40
+ image_spec.location = {
41
+ 'driver_url' => driver_url,
42
+ 'driver_version' => FogDriver::VERSION,
43
+ 'image_id' => response.body['imageId'],
44
+ 'creator' => creator,
45
+ 'allocated_at' => Time.now.to_i
46
+ }
47
+
48
+ image_spec.machine_options ||= {}
49
+ image_spec.machine_options.merge!({
50
+ :bootstrap_options => {
51
+ :image_id => image_spec.location['image_id']
52
+ }
53
+ })
54
+
55
+ end
56
+ end
57
+
58
+ def ready_image(action_handler, image_spec, image_options)
59
+ if !image_spec.location
60
+ raise "Cannot ready an image that does not exist"
61
+ end
62
+ image = compute.images.get(image_spec.location['image_id'])
63
+ if !image.ready?
64
+ action_handler.report_progress "Waiting for image to be ready ..."
65
+ # TODO timeout
66
+ image.wait_for { ready? }
67
+ action_handler.report_progress "Image is ready!"
68
+ end
69
+ end
70
+
71
+ def destroy_image(action_handler, image_spec, image_options)
72
+ if !image_spec.location
73
+ return
74
+ end
75
+ image = compute.images.get(image_spec.location['image_id'])
76
+ if !image
77
+ return
78
+ end
79
+ delete_snapshots = image_options[:delete_snapshots]
80
+ delete_snapshots = true if delete_snapshots.nil?
81
+ image.deregister(delete_snapshots)
82
+ end
83
+
84
+ def bootstrap_options_for(action_handler, machine_spec, machine_options)
85
+ bootstrap_options = symbolize_keys(machine_options[:bootstrap_options] || {})
86
+
87
+ if !bootstrap_options[:key_name]
88
+ bootstrap_options[:key_name] = overwrite_default_key_willy_nilly(action_handler, machine_spec)
89
+ end
90
+ bootstrap_options.delete(:tags) # we handle these separately for performance reasons
91
+ bootstrap_options
92
+ end
93
+
94
+ def create_servers(action_handler, specs_and_options, parallelizer)
95
+ super(action_handler, specs_and_options, parallelizer) do |machine_spec, server|
96
+ yield machine_spec, server if block_given?
97
+
98
+ machine_options = specs_and_options[machine_spec]
99
+ bootstrap_options = symbolize_keys(machine_options[:bootstrap_options] || {})
100
+ tags = default_tags(machine_spec, bootstrap_options[:tags] || {})
101
+
102
+ # Right now, not doing that in case manual tagging is going on
103
+ server_tags = server.tags || {}
104
+ extra_tags = tags.keys.select { |tag_name| !server_tags.has_key?(tag_name) }.to_a
105
+ different_tags = server_tags.select { |tag_name, tag_value| tags.has_key?(tag_name) && tags[tag_name] != tag_value }.to_a
106
+ if extra_tags.size > 0 || different_tags.size > 0
107
+ tags_description = [ "Update tags for #{machine_spec.name} on #{driver_url}" ]
108
+ tags_description += extra_tags.map { |tag| " Add #{tag} = #{tags[tag].inspect}" }
109
+ tags_description += different_tags.map { |tag_name, tag_value| " Update #{tag_name} from #{tag_value.inspect} to #{tags[tag_name].inspect}"}
110
+ action_handler.perform_action tags_description do
111
+ # TODO should we narrow this down to just extra/different tags or
112
+ # is it OK to just pass 'em all? Certainly easier to do the
113
+ # latter, and I can't think of a consequence for doing so offhand.
114
+ compute.create_tags(server.identity, tags)
115
+ end
116
+ end
117
+ end
118
+ end
119
+
120
+ def convergence_strategy_for(machine_spec, machine_options)
121
+ machine_options[:convergence_options] ||= {}
122
+ machine_options[:convergence_options][:ohai_hints] = { 'ec2' => ''}
123
+ super(machine_spec, machine_options)
124
+ end
125
+
126
+ def self.get_aws_profile(driver_options, aws_account_id)
127
+ aws_credentials = get_aws_credentials(driver_options)
128
+ compute_options = driver_options[:compute_options] || {}
129
+
130
+ # Order of operations:
131
+ # compute_options[:aws_access_key_id] / compute_options[:aws_secret_access_key] / compute_options[:aws_security_token] / compute_options[:region]
132
+ # compute_options[:aws_profile]
133
+ # ENV['AWS_ACCESS_KEY_ID'] / ENV['AWS_SECRET_ACCESS_KEY'] / ENV['AWS_SECURITY_TOKEN'] / ENV['AWS_DEFAULT_REGION']
134
+ # ENV['AWS_PROFILE']
135
+ # ENV['DEFAULT_PROFILE']
136
+ # 'default'
137
+ if compute_options[:aws_access_key_id]
138
+ Chef::Log.debug("Using AWS driver access key options")
139
+ aws_profile = {
140
+ :aws_access_key_id => compute_options[:aws_access_key_id],
141
+ :aws_secret_access_key => compute_options[:aws_secret_access_key],
142
+ :aws_security_token => compute_options[:aws_session_token],
143
+ :region => compute_options[:region]
144
+ }
145
+ elsif driver_options[:aws_profile]
146
+ Chef::Log.debug("Using AWS profile #{driver_options[:aws_profile]}")
147
+ aws_profile = aws_credentials[driver_options[:aws_profile]]
148
+ elsif ENV['AWS_ACCESS_KEY_ID'] || ENV['AWS_ACCESS_KEY']
149
+ Chef::Log.debug("Using AWS environment variable access keys")
150
+ aws_profile = {
151
+ :aws_access_key_id => ENV['AWS_ACCESS_KEY_ID'] || ENV['AWS_ACCESS_KEY'],
152
+ :aws_secret_access_key => ENV['AWS_SECRET_ACCESS_KEY'] || ENV['AWS_SECRET_KEY'],
153
+ :aws_security_token => ENV['AWS_SECURITY_TOKEN'],
154
+ :region => ENV['AWS_DEFAULT_REGION'] || ENV['AWS_REGION'] || ENV['EC2_REGION']
155
+ }
156
+ elsif ENV['AWS_PROFILE']
157
+ Chef::Log.debug("Using AWS profile #{ENV['AWS_PROFILE']} from AWS_PROFILE environment variable")
158
+ aws_profile = aws_credentials[ENV['AWS_PROFILE']]
159
+ if !aws_profile
160
+ raise "Environment variable AWS_PROFILE is set to #{ENV['AWS_PROFILE'].inspect} but your AWS config file does not contain that profile!"
161
+ end
162
+ else
163
+ Chef::Log.debug("Using AWS default profile")
164
+ aws_profile = aws_credentials.default
165
+ end
166
+
167
+ default_ec2_endpoint = compute_options[:ec2_endpoint] || ENV['EC2_URL']
168
+ default_iam_endpoint = compute_options[:iam_endpoint] || ENV['AWS_IAM_URL']
169
+
170
+ # Merge in account info for profile
171
+ if aws_profile
172
+ aws_profile = aws_profile.merge(aws_account_info_for(aws_profile, default_iam_endpoint))
173
+ end
174
+
175
+ # If no profile is found (or the profile is not the right account), search
176
+ # for a profile that matches the given account ID
177
+ if aws_account_id && (!aws_profile || aws_profile[:aws_account_id] != aws_account_id)
178
+ aws_profile = find_aws_profile_for_account_id(aws_credentials, aws_account_id, default_iam_endpoint)
179
+ end
180
+
181
+ fail 'No AWS profile specified! Are you missing something in the Chef config or ~/.aws/config?' unless aws_profile
182
+
183
+ aws_profile[:ec2_endpoint] ||= default_ec2_endpoint
184
+ aws_profile[:iam_endpoint] ||= default_iam_endpoint
185
+
186
+ aws_profile.delete_if { |key, value| value.nil? }
187
+ aws_profile
188
+ end
189
+
190
+ def self.find_aws_profile_for_account_id(aws_credentials, aws_account_id, default_iam_endpoint=nil)
191
+ aws_profile = nil
192
+ aws_credentials.each do |profile_name, profile|
193
+ begin
194
+ aws_account_info = aws_account_info_for(profile, default_iam_endpoint)
195
+ rescue
196
+ Chef::Log.warn("Could not connect to AWS profile #{aws_credentials[:name]}: #{$!}")
197
+ Chef::Log.debug($!.backtrace.join("\n"))
198
+ next
199
+ end
200
+ if aws_account_info[:aws_account_id] == aws_account_id
201
+ aws_profile = profile
202
+ aws_profile[:name] = profile_name
203
+ aws_profile = aws_profile.merge(aws_account_info)
204
+ break
205
+ end
206
+ end
207
+ if aws_profile
208
+ Chef::Log.info("Discovered AWS profile #{aws_profile[:name]} pointing at account #{aws_account_id}. Using ...")
209
+ else
210
+ raise "No AWS profile leads to account ##{aws_account_id}. Do you need to add profiles to ~/.aws/config?"
211
+ end
212
+ aws_profile
213
+ end
214
+
215
+ def self.aws_account_info_for(aws_profile, default_iam_endpoint = nil)
216
+ iam_endpoint = aws_profile[:iam_endpoint] || default_iam_endpoint
217
+
218
+ @@aws_account_info ||= {}
219
+ @@aws_account_info[aws_profile[:aws_access_key_id]] ||= begin
220
+ options = {
221
+ # Endpoint configuration
222
+ :aws_access_key_id => aws_profile[:aws_access_key_id],
223
+ :aws_secret_access_key => aws_profile[:aws_secret_access_key],
224
+ :aws_session_token => aws_profile[:aws_security_token]
225
+ }
226
+ if iam_endpoint
227
+ options[:host] = URI(iam_endpoint).host
228
+ options[:scheme] = URI(iam_endpoint).scheme
229
+ options[:port] = URI(iam_endpoint).port
230
+ options[:path] = URI(iam_endpoint).path
231
+ end
232
+ options.delete_if { |key, value| value.nil? }
233
+
234
+ iam = Fog::AWS::IAM.new(options)
235
+ arn = begin
236
+ # TODO it would be nice if Fog let you do this normally ...
237
+ iam.send(:request, {
238
+ 'Action' => 'GetUser',
239
+ :parser => Fog::Parsers::AWS::IAM::GetUser.new
240
+ }).body['User']['Arn']
241
+ rescue Fog::AWS::IAM::Error
242
+ # TODO Someone tell me there is a better way to find out your current
243
+ # user ID than this! This is what happens when you use an IAM user
244
+ # with default privileges.
245
+ if $!.message =~ /AccessDenied.+(arn:aws:iam::\d+:\S+)/
246
+ arn = $1
247
+ else
248
+ raise
249
+ end
250
+ end
251
+ arn_split = arn.split(':', 6)
252
+ {
253
+ :aws_account_id => arn_split[4],
254
+ :aws_username => arn_split[5],
255
+ :aws_user_arn => arn
256
+ }
257
+ end
258
+ end
259
+
260
+ def self.get_aws_credentials(driver_options)
261
+ # Grab the list of possible credentials
262
+ if driver_options[:aws_credentials]
263
+ aws_credentials = driver_options[:aws_credentials]
264
+ else
265
+ aws_credentials = Credentials.new
266
+ if driver_options[:aws_config_file]
267
+ aws_credentials.load_ini(driver_options.delete(:aws_config_file))
268
+ elsif driver_options[:aws_csv_file]
269
+ aws_credentials.load_csv(driver_options.delete(:aws_csv_file))
270
+ else
271
+ aws_credentials.load_default
272
+ end
273
+ end
274
+ aws_credentials
275
+ end
276
+
277
+ def self.compute_options_for(provider, id, config)
278
+ new_compute_options = {}
279
+ new_compute_options[:provider] = provider
280
+ new_config = { :driver_options => { :compute_options => new_compute_options }}
281
+ new_defaults = {
282
+ :driver_options => { :compute_options => {} },
283
+ :machine_options => { :bootstrap_options => {} }
284
+ }
285
+ result = Cheffish::MergedConfig.new(new_config, config, new_defaults)
286
+
287
+ if id && id != ''
288
+ # AWS canonical URLs are of the form fog:AWS:
289
+ if id =~ /^(\d{12})(:(.+))?$/
290
+ if $2
291
+ id = $1
292
+ new_compute_options[:region] = $3
293
+ else
294
+ Chef::Log.warn("Old-style AWS URL #{id} from an early beta of chef-metal (before 0.11-final) found. If you have servers in multiple regions on this account, you may see odd behavior like servers being recreated. To fix, edit any nodes with attribute chef_provisioning.location.driver_url to include the region like so: fog:AWS:#{id}:<region> (e.g. us-east-1)")
295
+ end
296
+ else
297
+ # Assume it is a profile name, and set that.
298
+ aws_profile, region = id.split(':', 2)
299
+ new_config[:driver_options][:aws_profile] = aws_profile
300
+ new_compute_options[:region] = region
301
+ id = nil
302
+ end
303
+ end
304
+
305
+ aws_profile = get_aws_profile(result[:driver_options], id)
306
+ new_compute_options[:aws_access_key_id] = aws_profile[:aws_access_key_id]
307
+ new_compute_options[:aws_secret_access_key] = aws_profile[:aws_secret_access_key]
308
+ new_compute_options[:aws_session_token] = aws_profile[:aws_security_token]
309
+ new_defaults[:driver_options][:compute_options][:region] = aws_profile[:region]
310
+ new_defaults[:driver_options][:compute_options][:endpoint] = aws_profile[:ec2_endpoint]
311
+
312
+ account_info = aws_account_info_for(result[:driver_options][:compute_options])
313
+ new_config[:driver_options][:aws_account_info] = account_info
314
+ id = "#{account_info[:aws_account_id]}:#{result[:driver_options][:compute_options][:region]}"
315
+
316
+ # Make sure we're using a reasonable default AMI, for now this is Ubuntu 14.04 LTS
317
+ result[:machine_options][:bootstrap_options][:image_id] ||=
318
+ default_ami_for_region(result[:driver_options][:compute_options][:region])
319
+
320
+ [result, id]
321
+ end
322
+
323
+ def create_many_servers(num_servers, bootstrap_options, parallelizer)
324
+ # Create all the servers in one request if we have a version of fog that can do that
325
+ if compute.servers.respond_to?(:create_many)
326
+ servers = compute.servers.create_many(num_servers, num_servers, bootstrap_options)
327
+ if block_given?
328
+ parallelizer.parallelize(servers) do |server|
329
+ yield server
330
+ end.to_a
331
+ end
332
+ servers
333
+ else
334
+ super
335
+ end
336
+ end
337
+
338
+ def servers_for(machine_specs)
339
+ # Grab all the servers in one request
340
+ instance_ids = machine_specs.map { |machine_spec| (machine_spec.location || {})['server_id'] }.select { |id| !id.nil? }
341
+ servers = compute.servers.all('instance-id' => instance_ids)
342
+ result = {}
343
+ machine_specs.each do |machine_spec|
344
+ if machine_spec.location
345
+ result[machine_spec] = servers.select { |s| s.id == machine_spec.location['server_id'] }.first
346
+ else
347
+ result[machine_spec] = nil
348
+ end
349
+ end
350
+ result
351
+ end
352
+
353
+ private
354
+ def self.default_ami_for_region(region)
355
+ Chef::Log.debug("Choosing default AMI for region '#{region}'")
356
+
357
+ case region
358
+ when 'ap-northeast-1'
359
+ 'ami-c786dcc6'
360
+ when 'ap-southeast-1'
361
+ 'ami-eefca7bc'
362
+ when 'ap-southeast-2'
363
+ 'ami-996706a3'
364
+ when 'eu-west-1'
365
+ 'ami-4ab46b3d'
366
+ when 'sa-east-1'
367
+ 'ami-6770d87a'
368
+ when 'us-east-1'
369
+ 'ami-d2ff23ba'
370
+ when 'us-west-1'
371
+ 'ami-73717d36'
372
+ when 'us-west-2'
373
+ 'ami-f1ce8bc1'
374
+ end
375
+ end
376
+
377
+ end
378
+ end
379
+ end
380
+ end
381
+ end
@@ -0,0 +1,87 @@
1
+ require 'inifile'
2
+ require 'csv'
3
+
4
+ class Chef
5
+ module Provisioning
6
+ module FogDriver
7
+ module Providers
8
+ class AWS
9
+ # Reads in a credentials file in Amazon's download format and presents the credentials to you
10
+ class Credentials
11
+ def initialize
12
+ @credentials = {}
13
+ end
14
+
15
+ include Enumerable
16
+
17
+ def default
18
+ if @credentials.size == 0
19
+ raise "No credentials loaded! Do you have a ~/.aws/config?"
20
+ end
21
+ @credentials[ENV['AWS_DEFAULT_PROFILE'] || 'default'] || @credentials.first[1]
22
+ end
23
+
24
+ def keys
25
+ @credentials.keys
26
+ end
27
+
28
+ def [](name)
29
+ @credentials[name]
30
+ end
31
+
32
+ def each(&block)
33
+ @credentials.each(&block)
34
+ end
35
+
36
+ def load_ini(credentials_ini_file)
37
+ inifile = IniFile.load(File.expand_path(credentials_ini_file))
38
+ if inifile
39
+ inifile.each_section do |section|
40
+ if section =~ /^\s*profile\s+(.+)$/ || section =~ /^\s*(default)\s*/
41
+ profile_name = $1.strip
42
+ profile = inifile[section].inject({}) do |result, pair|
43
+ result[pair[0].to_sym] = pair[1]
44
+ result
45
+ end
46
+ profile[:name] = profile_name
47
+ @credentials[profile_name] = profile
48
+ end
49
+ end
50
+ else
51
+ # Get it to throw an error
52
+ File.open(File.expand_path(credentials_ini_file)) do
53
+ end
54
+ end
55
+ end
56
+
57
+ def load_csv(credentials_csv_file)
58
+ CSV.new(File.open(credentials_csv_file), :headers => :first_row).each do |row|
59
+ @credentials[row['User Name']] = {
60
+ :name => row['User Name'],
61
+ :user_name => row['User Name'],
62
+ :aws_access_key_id => row['Access Key Id'],
63
+ :aws_secret_access_key => row['Secret Access Key']
64
+ }
65
+ end
66
+ end
67
+
68
+ def load_default
69
+ config_file = ENV['AWS_CONFIG_FILE'] || File.expand_path('~/.aws/config')
70
+ if File.file?(config_file)
71
+ load_ini(config_file)
72
+ end
73
+ end
74
+
75
+ def self.method_missing(name, *args, &block)
76
+ singleton.send(name, *args, &block)
77
+ end
78
+
79
+ def self.singleton
80
+ @aws_credentials ||= Credentials.new
81
+ end
82
+ end
83
+ end
84
+ end
85
+ end
86
+ end
87
+ end