ironfan 3.1.0.rc1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. data/.gitignore +51 -0
  2. data/.rspec +3 -0
  3. data/CHANGELOG.md +130 -0
  4. data/Gemfile +26 -0
  5. data/LICENSE.md +201 -0
  6. data/README.md +328 -0
  7. data/Rakefile +104 -0
  8. data/TODO.md +16 -0
  9. data/VERSION +1 -0
  10. data/chefignore +41 -0
  11. data/cluster_chef-knife.gemspec +123 -0
  12. data/cluster_chef.gemspec +111 -0
  13. data/config/client.rb +59 -0
  14. data/config/proxy.pac +12 -0
  15. data/config/ubuntu10.04-ironfan.erb +157 -0
  16. data/config/ubuntu11.10-ironfan.erb +145 -0
  17. data/ironfan.gemspec +121 -0
  18. data/lib/chef/knife/bootstrap/ubuntu10.04-ironfan.erb +157 -0
  19. data/lib/chef/knife/bootstrap/ubuntu11.10-ironfan.erb +145 -0
  20. data/lib/chef/knife/cluster_bootstrap.rb +74 -0
  21. data/lib/chef/knife/cluster_kick.rb +94 -0
  22. data/lib/chef/knife/cluster_kill.rb +73 -0
  23. data/lib/chef/knife/cluster_launch.rb +164 -0
  24. data/lib/chef/knife/cluster_list.rb +50 -0
  25. data/lib/chef/knife/cluster_proxy.rb +126 -0
  26. data/lib/chef/knife/cluster_show.rb +61 -0
  27. data/lib/chef/knife/cluster_ssh.rb +141 -0
  28. data/lib/chef/knife/cluster_start.rb +40 -0
  29. data/lib/chef/knife/cluster_stop.rb +43 -0
  30. data/lib/chef/knife/cluster_sync.rb +77 -0
  31. data/lib/chef/knife/generic_command.rb +66 -0
  32. data/lib/chef/knife/knife_common.rb +195 -0
  33. data/lib/ironfan.rb +143 -0
  34. data/lib/ironfan/chef_layer.rb +299 -0
  35. data/lib/ironfan/cloud.rb +412 -0
  36. data/lib/ironfan/cluster.rb +118 -0
  37. data/lib/ironfan/compute.rb +153 -0
  38. data/lib/ironfan/deprecated.rb +33 -0
  39. data/lib/ironfan/discovery.rb +177 -0
  40. data/lib/ironfan/dsl_object.rb +124 -0
  41. data/lib/ironfan/facet.rb +144 -0
  42. data/lib/ironfan/fog_layer.rb +150 -0
  43. data/lib/ironfan/private_key.rb +130 -0
  44. data/lib/ironfan/role_implications.rb +58 -0
  45. data/lib/ironfan/security_group.rb +119 -0
  46. data/lib/ironfan/server.rb +281 -0
  47. data/lib/ironfan/server_slice.rb +260 -0
  48. data/lib/ironfan/volume.rb +157 -0
  49. data/spec/ironfan/cluster_spec.rb +13 -0
  50. data/spec/ironfan/facet_spec.rb +69 -0
  51. data/spec/ironfan/server_slice_spec.rb +19 -0
  52. data/spec/ironfan/server_spec.rb +112 -0
  53. data/spec/ironfan_spec.rb +193 -0
  54. data/spec/spec_helper.rb +50 -0
  55. data/spec/spec_helper/dummy_chef.rb +25 -0
  56. data/spec/test_config.rb +20 -0
  57. data/tasks/chef_config.rake +38 -0
  58. data/tasks/jeweler_use_alt_branch.rake +53 -0
  59. metadata +217 -0
@@ -0,0 +1,124 @@
1
+ Mash.class_eval do
2
+ def reverse_merge!(other_hash)
3
+ # stupid mash doesn't take a block arg, which breaks the implementation of
4
+ # reverse_merge!
5
+ other_hash.each_pair do |key, value|
6
+ key = convert_key(key)
7
+ regular_writer(key, convert_value(value)) unless has_key?(key)
8
+ end
9
+ self
10
+ end
11
+ def to_mash
12
+ self.dup
13
+ end unless method_defined?(:to_mash)
14
+ end
15
+
16
+ Hash.class_eval do
17
+ def to_mash
18
+ Mash.new(self)
19
+ end unless method_defined?(:to_mash)
20
+ end
21
+
22
+ module Ironfan
23
+ #
24
+ # Provides magic methods, defined with has_keys
25
+ #
26
+ # @example
27
+ # class Mom < Ironfan::DslObject
28
+ # has_keys(:college, :combat_boots, :fat, :so_fat)
29
+ # end
30
+ #
31
+ # class Person
32
+ # def momma &block
33
+ # @momma ||= Mom.new
34
+ # @momma.configure(&block) if block
35
+ # end
36
+ # end
37
+ #
38
+ # yo = Person.new
39
+ # yo.mamma.combat_boots :wears
40
+ # yo.momma do
41
+ # fat true
42
+ # so_fat 'When she sits around the house, she sits *AROUND* the house'
43
+ # end
44
+ #
45
+ class DslObject
46
+ class_attribute :keys
47
+ self.keys = []
48
+
49
+ def initialize(attrs={}, &block)
50
+ @settings = Mash.new
51
+ configure(attrs, &block)
52
+ end
53
+
54
+ #
55
+ # Defines DSL attributes
56
+ #
57
+ # @params [Array(String)] key_names DSL attribute names
58
+ #
59
+ # @example
60
+ # class Mom < Ironfan::DslObject
61
+ # has_keys(:fat, :so_fat)
62
+ # end
63
+ # yer_mom = Mom.new
64
+ # yer_mom.fat :quite
65
+ #
66
+ def self.has_keys(*key_names)
67
+ key_names.map!(&:to_sym)
68
+ self.keys += key_names
69
+ self.keys.uniq!
70
+ key_names.each do |key|
71
+ next if method_defined?(key)
72
+ define_method(key){|*args| set(key, *args) }
73
+ end
74
+ end
75
+
76
+ #
77
+ # Sets the DSL attribute, unless the given value is nil.
78
+ #
79
+ def set(key, val=nil)
80
+ @settings[key.to_s] = val unless val.nil?
81
+ @settings[key.to_s]
82
+ end
83
+
84
+ def to_hash
85
+ @settings.to_hash
86
+ end
87
+
88
+ def to_mash
89
+ @settings.dup
90
+ end
91
+
92
+ def to_s
93
+ "<#{self.class} #{to_hash.inspect}>"
94
+ end
95
+
96
+ def reverse_merge!(hsh)
97
+ @settings.reverse_merge!(hsh.to_hash)
98
+ end
99
+
100
+ def configure(hsh={}, &block)
101
+ @settings.merge!(hsh.to_hash)
102
+ instance_eval(&block) if block
103
+ self
104
+ end
105
+
106
+ # delegate to the knife ui presenter
107
+ def ui() Ironfan.ui ; end
108
+ # delegate to the knife ui presenter
109
+ def self.ui() Ironfan.ui ; end
110
+
111
+ def step(desc, *style)
112
+ ui.info(" #{"%-15s" % (name.to_s+":")}\t#{ui.color(desc.to_s, *style)}")
113
+ end
114
+
115
+ # helper method for bombing out of a script
116
+ def die(*args) Ironfan.die(*args) ; end
117
+
118
+ # helper method for turning exceptions into warnings
119
+ def safely(*args, &block) Ironfan.safely(*args, &block) ; end
120
+
121
+ # helper method for debugging only
122
+ def dump(*args) args.each{|arg| Chef::Log.debug( arg.inspect ) } end
123
+ end
124
+ end
@@ -0,0 +1,144 @@
1
+ module Ironfan
2
+ class Facet < Ironfan::ComputeBuilder
3
+ attr_reader :cluster
4
+ has_keys :instances
5
+
6
+ def initialize cluster, facet_name, attrs={}
7
+ super(facet_name.to_sym, attrs)
8
+ @cluster = cluster
9
+ @servers = Mash.new
10
+ @chef_roles = []
11
+ @settings[:instances] ||= 1
12
+ create_facet_role
13
+ create_facet_security_group unless attrs[:no_security_group]
14
+ end
15
+
16
+ def cluster_name
17
+ cluster.name
18
+ end
19
+
20
+ def facet_name
21
+ name
22
+ end
23
+
24
+ # The auto-generated role for this facet.
25
+ # Instance-evals the given block in the context of that role,
26
+ #
27
+ # @example
28
+ # facet_role do
29
+ # override_attributes({
30
+ # :time_machine => { :transition_speed => 88 },
31
+ # })
32
+ # end
33
+ #
34
+ # @return [Chef::Role] The auto-generated role for this facet.
35
+ def facet_role(&block)
36
+ @facet_role.instance_eval( &block ) if block_given?
37
+ @facet_role
38
+ end
39
+
40
+ def assign_volume_ids(volume_name, *volume_ids)
41
+ volume_ids.flatten.zip(servers).each do |volume_id, server|
42
+ server.volume(volume_name){ volume_id(volume_id) } if server
43
+ end
44
+ end
45
+
46
+ #
47
+ # Retrieve or define the given server
48
+ #
49
+ # @param [Integer] idx -- the index of the desired server
50
+ # @param [Hash] attrs -- attributes to configure on the object
51
+ # @yield a block to execute in the context of the object
52
+ #
53
+ # @return [Ironfan::Facet]
54
+ #
55
+ def server(idx, attrs={}, &block)
56
+ idx = idx.to_i
57
+ @servers[idx] ||= Ironfan::Server.new(self, idx)
58
+ @servers[idx].configure(attrs, &block)
59
+ @servers[idx]
60
+ end
61
+
62
+ # if the server has been added to this facet or is in range
63
+ def has_server? idx
64
+ (idx.to_i < instances) || @servers.include?(idx.to_i)
65
+ end
66
+
67
+ #
68
+ # Slicing
69
+ #
70
+
71
+ # All servers in this facet
72
+ #
73
+ # @return [Ironfan::ServerSlice] slice containing all servers
74
+ def servers
75
+ slice(indexes)
76
+ end
77
+
78
+ #
79
+ # A slice of servers from this facet, in index order
80
+ #
81
+ # If +slice_indexes+ is nil, returns all servers.
82
+ # Otherwise, takes slice (given by +*args+) from the requested facet.
83
+ #
84
+ # @param [Array, String] slice_indexes -- servers in that facet (or nil for all in facet).
85
+ #
86
+ # @return [Ironfan::ServerSlice] the requested slice
87
+ def slice(slice_indexes=nil)
88
+ slice_indexes = self.indexes if slice_indexes.blank?
89
+ slice_indexes = indexes_from_intervals(slice_indexes) if slice_indexes.is_a?(String)
90
+ svrs = Array(slice_indexes).map(&:to_i).sort!.select{|idx| has_server?(idx) }.map{|idx| server(idx) }
91
+ Ironfan::ServerSlice.new(self.cluster, svrs)
92
+ end
93
+
94
+ # all valid server indexes
95
+ def valid_indexes
96
+ (0 ... instances).to_a # note the '...'
97
+ end
98
+
99
+ # indexes in the 0...instances range plus bogus ones that showed up
100
+ # (probably from chef or fog)
101
+ def indexes
102
+ [@servers.keys, valid_indexes].flatten.compact.uniq.sort
103
+ end
104
+
105
+ #
106
+ # Resolve:
107
+ #
108
+ def resolve!
109
+ servers.each(&:resolve!)
110
+ end
111
+
112
+ protected
113
+
114
+ def create_facet_security_group
115
+ cloud.security_group("#{cluster_name}-#{facet_name}")
116
+ end
117
+
118
+ # Creates a chef role named for the facet
119
+ def create_facet_role
120
+ @facet_role_name = "#{cluster_name}_#{facet_name}"
121
+ @facet_role = new_chef_role(@facet_role_name, cluster, self)
122
+ role(@facet_role_name, :own)
123
+ end
124
+
125
+ #
126
+ # Given a string enumerating indexes to select returns a flat array of
127
+ # indexes. The indexes will be unique but in an arbitrary order.
128
+ #
129
+ # @example
130
+ # facet = Ironfan::Facet.new('foo', 'bar')
131
+ # facet.indexes_from_intervals('1,2-3,8-9,7') # [1, 2, 3, 8, 9, 7]
132
+ # facet.indexes_from_intervals('1,3-5,4,7') # [1, 3, 4, 5, 7]
133
+ #
134
+ def indexes_from_intervals intervals
135
+ intervals.split(",").map do |term|
136
+ if term =~ /^(\d+)-(\d+)$/ then ($1.to_i .. $2.to_i).to_a
137
+ elsif term =~ /^(\d+)$/ then $1.to_i
138
+ else ui.warn("Bad interval: #{term}") ; nil
139
+ end
140
+ end.flatten.compact.uniq
141
+ end
142
+
143
+ end
144
+ end
@@ -0,0 +1,150 @@
1
+ module Ironfan
2
+ #
3
+ # Ironfan::Server methods that handle Fog action
4
+ #
5
+ Server.class_eval do
6
+
7
+ def fog_create_server
8
+ step(" creating cloud server", :green)
9
+ lint_fog
10
+ launch_desc = fog_launch_description
11
+ Chef::Log.debug(JSON.pretty_generate(launch_desc))
12
+ safely do
13
+ @fog_server = Ironfan.fog_connection.servers.create(launch_desc)
14
+ end
15
+ end
16
+
17
+ def lint_fog
18
+ unless cloud.image_id then raise "No image ID found: nothing in Chef::Config[:ec2_image_info] for AZ #{self.default_availability_zone} flavor #{cloud.flavor} backing #{cloud.backing} image name #{cloud.image_name}, and cloud.image_id was not set directly. See https://github.com/infochimps-labs/ironfan/wiki/machine-image-(AMI)-lookup-by-name - #{cloud.list_images}" end
19
+ unless cloud.image_id then cloud.list_flavors ; raise "No machine flavor found" ; end
20
+ end
21
+
22
+ def fog_launch_description
23
+ {
24
+ :image_id => cloud.image_id,
25
+ :flavor_id => cloud.flavor,
26
+ #
27
+ :groups => cloud.security_groups.keys,
28
+ :key_name => cloud.keypair.to_s,
29
+ # Fog does not actually create tags when it creates a server.
30
+ :tags => {
31
+ :cluster => cluster_name,
32
+ :facet => facet_name,
33
+ :index => facet_index, },
34
+ :user_data => JSON.pretty_generate(cloud.user_data),
35
+ :block_device_mapping => block_device_mapping,
36
+ # :disable_api_termination => cloud.permanent,
37
+ # :instance_initiated_shutdown_behavior => instance_initiated_shutdown_behavior,
38
+ :availability_zone => self.default_availability_zone,
39
+ :monitoring => cloud.monitoring,
40
+ }
41
+ end
42
+
43
+ #
44
+ # Takes key-value pairs and idempotently sets those tags on the cloud machine
45
+ #
46
+ def fog_create_tags(fog_obj, desc, tags)
47
+ tags_to_create = tags.reject{|key, val| fog_obj.tags[key] == val.to_s }
48
+ return if tags_to_create.empty?
49
+ step(" tagging #{desc} with #{tags_to_create.inspect}", :green)
50
+ tags_to_create.each do |key, value|
51
+ Chef::Log.debug( "tagging #{desc} with #{key} = #{value}" )
52
+ safely do
53
+ Ironfan.fog_connection.tags.create({
54
+ :key => key, :value => value.to_s, :resource_id => fog_obj.id })
55
+ end
56
+ end
57
+ end
58
+
59
+ def fog_address
60
+ address_str = self.cloud.public_ip or return
61
+ Ironfan.fog_addresses[address_str]
62
+ end
63
+
64
+ def discover_volumes!
65
+ composite_volumes.each do |vol_name, vol|
66
+ my_vol = volumes[vol_name]
67
+ next if my_vol.fog_volume
68
+ next if Ironfan.chef_config[:cloud] == false
69
+ my_vol.fog_volume = Ironfan.fog_volumes.find do |fv|
70
+ ( # matches the explicit volume id
71
+ (vol.volume_id && (fv.id == vol.volume_id) ) ||
72
+ # OR this server's machine exists, and this volume is attached to
73
+ # it, and in the right place
74
+ ( fog_server && fv.server_id && vol.device &&
75
+ (fv.server_id == fog_server.id) &&
76
+ (fv.device.to_s == vol.device.to_s) ) ||
77
+ # OR this volume is tagged as belonging to this machine
78
+ ( fv.tags.present? &&
79
+ (fv.tags['server'] == self.fullname) &&
80
+ (fv.tags['device'] == vol.device.to_s) )
81
+ )
82
+ end
83
+ next unless my_vol.fog_volume
84
+ my_vol.volume_id(my_vol.fog_volume.id) unless my_vol.volume_id.present?
85
+ my_vol.availability_zone(my_vol.fog_volume.availability_zone) unless my_vol.availability_zone.present?
86
+ check_server_id_pairing(my_vol.fog_volume, my_vol.desc)
87
+ end
88
+ end
89
+
90
+ def attach_volumes
91
+ return unless in_cloud?
92
+ discover_volumes!
93
+ return if composite_volumes.empty?
94
+ step(" attaching volumes")
95
+ composite_volumes.each do |vol_name, vol|
96
+ next if vol.volume_id.blank? || (vol.attachable != :ebs)
97
+ if (not vol.in_cloud?) then Chef::Log.debug("Volume not found: #{vol.desc}") ; next ; end
98
+ if (vol.has_server?) then check_server_id_pairing(vol.fog_volume, vol.desc) ; next ; end
99
+ step(" - attaching #{vol.desc} -- #{vol.inspect}", :blue)
100
+ safely do
101
+ vol.fog_volume.device = vol.device
102
+ vol.fog_volume.server = fog_server
103
+ end
104
+ end
105
+ end
106
+
107
+ def associate_public_ip
108
+ address = self.cloud.public_ip
109
+ return unless self.in_cloud? && address
110
+ desc = "elastic ip #{address} for #{self.fullname}"
111
+ if (fog_address && fog_address.server_id) then check_server_id_pairing(fog_address, desc) ; return ; end
112
+ safely do
113
+ step(" assigning #{desc}", :blue)
114
+ Ironfan.fog_connection.associate_address(self.fog_server.id, address)
115
+ end
116
+ end
117
+
118
+ def check_server_id_pairing thing, desc
119
+ return unless thing && thing.server_id && self.in_cloud?
120
+ type_of_thing = thing.class.to_s.gsub(/.*::/,"")
121
+ if thing.server_id != self.fog_server.id
122
+ ui.warn "#{type_of_thing} mismatch: #{desc} is on #{thing.server_id} not #{self.fog_server.id}: #{thing.inspect.gsub(/\s+/m,' ')}"
123
+ false
124
+ else
125
+ Chef::Log.debug("#{type_of_thing} paired: #{desc}")
126
+ true
127
+ end
128
+ end
129
+
130
+ end
131
+
132
+ class ServerSlice
133
+ def sync_keypairs
134
+ step("ensuring keypairs exist")
135
+ keypairs = servers.map{|svr| [svr.cluster.cloud.keypair, svr.cloud.keypair] }.flatten.map(&:to_s).reject(&:blank?).uniq
136
+ keypairs = keypairs - Ironfan.fog_keypairs.keys
137
+ keypairs.each do |keypair_name|
138
+ keypair_obj = Ironfan::Ec2Keypair.create!(keypair_name)
139
+ Ironfan.fog_keypairs[keypair_name] = keypair_obj
140
+ end
141
+ end
142
+
143
+ # Create security groups, their dependencies, and synchronize their permissions
144
+ def sync_security_groups
145
+ step("ensuring security groups exist and are correct")
146
+ security_groups.each{|name,group| group.run }
147
+ end
148
+
149
+ end
150
+ end
@@ -0,0 +1,130 @@
1
+ require 'fileutils'
2
+
3
+ module Ironfan
4
+ #
5
+ # A private key -- chef client key, ssh key, etc.
6
+ #
7
+ # The key is a pro
8
+ class PrivateKey < Ironfan::DslObject
9
+ attr_reader :name
10
+ attr_reader :proxy
11
+ attr_reader :on_update
12
+
13
+ #
14
+ # PrivateKey.new('bob')
15
+ #
16
+ # @yield a block, executed in caller's context, when the body is updated
17
+ # @yieldparam the updated body
18
+ def initialize(name, proxy=nil, &on_update)
19
+ @name = name
20
+ @proxy = proxy
21
+ @on_update = on_update
22
+ end
23
+
24
+ def filename
25
+ File.join(key_dir, "#{name}.pem")
26
+ end
27
+
28
+ def save
29
+ return unless @body
30
+ if Ironfan.chef_config[:dry_run]
31
+ Chef::Log.debug(" key #{name} - dry run, not writing out key")
32
+ return
33
+ end
34
+ ui.info( " key #{name} - writing to #{filename}" )
35
+ FileUtils.mkdir_p(File.dirname(filename))
36
+ File.open(filename, "w", 0600){|f| f.print( @body ) }
37
+ end
38
+
39
+ def load
40
+ return unless File.exists?(filename)
41
+ self.body = File.read(filename).chomp
42
+ end
43
+
44
+ def body=(content)
45
+ @body = content
46
+ on_update.call(content) if on_update
47
+ content
48
+ end
49
+
50
+ def self.create!(name, *args, &block)
51
+ obj = self.new(name, *args, &block)
52
+ obj.create_proxy!
53
+ obj
54
+ end
55
+
56
+ def to_s
57
+ [super[0..-2], @name, @proxy, @body.to_s[32..64], '...', @body.to_s[-60..-30]].join(" ").gsub(/[\r\n\t]+/,'') + '>'
58
+ end
59
+ end
60
+
61
+ class ChefClientKey < PrivateKey
62
+ def body
63
+ return @body if @body
64
+ if proxy && proxy.private_key && (not proxy.private_key.empty?)
65
+ @body = proxy.private_key
66
+ else
67
+ load
68
+ end
69
+ @body
70
+ end
71
+
72
+ def key_dir
73
+ Chef::Config.client_key_dir || '/tmp/client_keys'
74
+ end
75
+ end
76
+
77
+ class DataBagKey < PrivateKey
78
+ def body
79
+ return @body if @body
80
+ @body
81
+ end
82
+
83
+ def random_token
84
+ require "digest/sha2"
85
+ digest = Digest::SHA512.hexdigest( Time.now.to_s + (1..10).collect{ rand.to_s }.join )
86
+ 5.times{ digest = Digest::SHA512.hexdigest(digest) }
87
+ digest
88
+ end
89
+
90
+ def key_dir
91
+ return Chef::Config.data_bag_key_dir if Chef::Config.data_bag_key_dir
92
+ dir = "#{ENV['HOME']}/.chef/data_bag_keys"
93
+ warn "Please set 'data_bag_key_dir' in your knife.rb. Will use #{dir} as a default"
94
+ dir
95
+ end
96
+ end
97
+
98
+ class Ec2Keypair < PrivateKey
99
+ def body
100
+ return @body if @body
101
+ if proxy && proxy.private_key && (not proxy.private_key.empty?)
102
+ @body = proxy.private_key
103
+ else
104
+ load
105
+ end
106
+ @body
107
+ end
108
+
109
+ def create_proxy!
110
+ safely do
111
+ step(" key #{name} - creating", :green)
112
+ @proxy = Ironfan.fog_connection.key_pairs.create(:name => name.to_s)
113
+ end
114
+ Ironfan.fog_keypairs[name] = proxy
115
+ self.body = proxy.private_key
116
+ save
117
+ end
118
+
119
+ def key_dir
120
+ if Chef::Config.ec2_key_dir
121
+ return Chef::Config.ec2_key_dir
122
+ else
123
+ dir = "#{ENV['HOME']}/.chef/ec2_keys"
124
+ warn "Please set 'ec2_key_dir' in your knife.rb. Will use #{dir} as a default"
125
+ dir
126
+ end
127
+ end
128
+ end
129
+
130
+ end