bigbang 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/README +19 -0
- data/bin/bigbang +46 -0
- data/lib/bigbang/cluster.rb +10 -0
- data/lib/bigbang/config.rb +6 -0
- data/lib/bigbang/dsl.rb +25 -0
- data/lib/bigbang/ec2-git-bootstrap.rb +27 -0
- data/lib/bigbang/instance.rb +13 -0
- data/lib/bigbang/provider.rb +43 -0
- data/lib/bigbang/universe.rb +246 -0
- data/lib/bigbang.rb +2 -0
- data/src/bootstrap.sh +31 -0
- data/src/git-ssh-wrap.sh +3 -0
- data/src/post-extract.sh +8 -0
- metadata +106 -0
data/README
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
|
2
|
+
|
3
|
+
# creates the base images
|
4
|
+
bigbang base
|
5
|
+
|
6
|
+
# run the instances
|
7
|
+
bigbang explode <universe-info.rb>
|
8
|
+
|
9
|
+
# list universes
|
10
|
+
bigbang list
|
11
|
+
|
12
|
+
# kill a universe
|
13
|
+
bigbang kill <universe-name>
|
14
|
+
|
15
|
+
# resize clusters
|
16
|
+
bigbang resize
|
17
|
+
|
18
|
+
# test configuration
|
19
|
+
bigbang test
|
data/bin/bigbang
ADDED
@@ -0,0 +1,46 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + "/../lib/bigbang"
|
4
|
+
|
5
|
+
def help
|
6
|
+
puts "test"
|
7
|
+
end
|
8
|
+
|
9
|
+
$universe = nil
|
10
|
+
|
11
|
+
def self.universe(&block)
|
12
|
+
$universe = u = BigBang::Universe.new
|
13
|
+
u.instance_eval(&block)
|
14
|
+
end
|
15
|
+
|
16
|
+
if ARGV.empty?
|
17
|
+
help
|
18
|
+
exit 1
|
19
|
+
end
|
20
|
+
|
21
|
+
case ARGV[0]
|
22
|
+
when 'test':
|
23
|
+
load './universe.rb'
|
24
|
+
$universe.test
|
25
|
+
when 'explode':
|
26
|
+
if ARGV.size < 2 then
|
27
|
+
puts "#{$0} explode <universe name>"
|
28
|
+
exit 1
|
29
|
+
end
|
30
|
+
require 'universe.rb'
|
31
|
+
$universe.explode(ARGV[1])
|
32
|
+
when 'list':
|
33
|
+
require 'universe.rb'
|
34
|
+
$universe.list
|
35
|
+
when 'kill':
|
36
|
+
if ARGV.size < 2 then
|
37
|
+
puts "#{$0} kill <universe name>"
|
38
|
+
exit 1
|
39
|
+
end
|
40
|
+
require 'universe.rb'
|
41
|
+
$universe.kill(ARGV[1])
|
42
|
+
else
|
43
|
+
puts "unknown command #{ARGV[0]}"
|
44
|
+
exit 2
|
45
|
+
end
|
46
|
+
|
data/lib/bigbang/dsl.rb
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
require 'bigbang/config'
|
2
|
+
require 'bigbang/instance'
|
3
|
+
require 'bigbang/cluster'
|
4
|
+
|
5
|
+
module BigBang
|
6
|
+
module DSL
|
7
|
+
def cluster(name, &block)
|
8
|
+
@instances << Cluster.new(name).tap do |cluster|
|
9
|
+
cluster.instance_eval(&block)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def instance(name, &block)
|
14
|
+
@instances << Instance.new(name).tap do |i|
|
15
|
+
i.instance_eval(&block)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def config(&block)
|
20
|
+
@config = Config.new.tap do |c|
|
21
|
+
c.instance_eval(&block)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
class Ec2BootstrapGenerator
|
2
|
+
def generate_from_dir(path)
|
3
|
+
basedir = File.dirname(__FILE__) + "/../../"
|
4
|
+
tmp=%x(mktemp -d /tmp/gen-ec2-userdata.XXXXX).strip
|
5
|
+
%x(mkdir -p #{tmp}/data)
|
6
|
+
%x(cp -r #{path}/* #{tmp}/data/)
|
7
|
+
%x(cp #{basedir}/src/* #{tmp})
|
8
|
+
tmptar=%x(mktemp /tmp/gen-ec2-userdata.XXXXX.tar.gz).strip
|
9
|
+
%x(tar -czf #{tmptar} -C #{tmp} .)
|
10
|
+
|
11
|
+
File.new("#{basedir}/src/bootstrap.sh").read +
|
12
|
+
File.new(tmptar).read
|
13
|
+
end
|
14
|
+
|
15
|
+
def generate_from_hash(h)
|
16
|
+
tmp=%x(mktemp -d /tmp/gen-ec2-userdata.XXXXX).strip
|
17
|
+
h.each_pair do |k,v|
|
18
|
+
f = File.new("#{tmp}/#{k}", "w")
|
19
|
+
f.write(v)
|
20
|
+
f.close
|
21
|
+
end
|
22
|
+
|
23
|
+
generate_from_dir(tmp).tap do |r|
|
24
|
+
%x(rm -fr #{tmp})
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
module BigBang
|
2
|
+
class Provider
|
3
|
+
def initialize(config)
|
4
|
+
@config = config
|
5
|
+
end
|
6
|
+
|
7
|
+
def ec2
|
8
|
+
return @ec2 unless @ec2.nil?
|
9
|
+
@ec2 = AWS::EC2::Base.new(
|
10
|
+
:access_key_id => @config.access_key_id,
|
11
|
+
:secret_access_key => @config.secret_key)
|
12
|
+
end
|
13
|
+
|
14
|
+
def dns
|
15
|
+
return @dns unless @dns.nil?
|
16
|
+
@dns = Fog::DNS.new(@config.dns_opts)
|
17
|
+
end
|
18
|
+
|
19
|
+
def configured_zone
|
20
|
+
dns.zones.find { |z| z.domain == @config.domain }
|
21
|
+
end
|
22
|
+
|
23
|
+
def create_dns(domain, addr)
|
24
|
+
configured_zone.records.create(
|
25
|
+
:value => addr,
|
26
|
+
:name => domain,
|
27
|
+
:type => "A")
|
28
|
+
end
|
29
|
+
|
30
|
+
def eips
|
31
|
+
aset = ec2.describe_addresses.addressesSet
|
32
|
+
if aset.nil?
|
33
|
+
return []
|
34
|
+
else
|
35
|
+
return aset.item
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def free_eips
|
40
|
+
eips.find_all { |eip| eip.instanceId.nil? }
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,246 @@
|
|
1
|
+
require 'base64'
|
2
|
+
require 'AWS'
|
3
|
+
require 'ap'
|
4
|
+
require 'set'
|
5
|
+
require 'fog'
|
6
|
+
require 'bigbang/provider'
|
7
|
+
require 'bigbang/instance'
|
8
|
+
require 'bigbang/dsl'
|
9
|
+
require 'bigbang/ec2-git-bootstrap'
|
10
|
+
|
11
|
+
module BigBang
|
12
|
+
class Universe
|
13
|
+
def initialize
|
14
|
+
@clusters = []
|
15
|
+
@instances = []
|
16
|
+
end
|
17
|
+
|
18
|
+
include DSL
|
19
|
+
|
20
|
+
def provider
|
21
|
+
return @provider unless @provider.nil?
|
22
|
+
@provider = Provider.new(@config)
|
23
|
+
end
|
24
|
+
|
25
|
+
def get_instances
|
26
|
+
rset = provider.ec2.describe_instances.reservationSet
|
27
|
+
if rset.nil?
|
28
|
+
return []
|
29
|
+
else
|
30
|
+
instances = []
|
31
|
+
rset.item.each do |reservation|
|
32
|
+
reservation.instancesSet.item.each do |i|
|
33
|
+
instances << i
|
34
|
+
end
|
35
|
+
end
|
36
|
+
return instances
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def test
|
41
|
+
ap provider.eips
|
42
|
+
get_instances
|
43
|
+
puts "ec2 access OK"
|
44
|
+
@instances.collect { |i| i.ami }.to_set.each do |ami|
|
45
|
+
begin
|
46
|
+
provider.ec2.describe_images(:image_id => [ami])
|
47
|
+
rescue AWS::InvalidAMIIDNotFound => e
|
48
|
+
puts "ami #{ami} not found"
|
49
|
+
end
|
50
|
+
end
|
51
|
+
puts "AMI's OK"
|
52
|
+
if provider.configured_zone.nil?
|
53
|
+
puts "Configured DNS domain zone not found"
|
54
|
+
else
|
55
|
+
puts "DNS domain OK"
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
def wait_for_eips(expected)
|
60
|
+
print "waiting for eips to be allocated"
|
61
|
+
STDOUT.flush
|
62
|
+
addrs = provider.free_eips
|
63
|
+
while(addrs.size < expected)
|
64
|
+
sleep(1)
|
65
|
+
print "."
|
66
|
+
STDOUT.flush
|
67
|
+
addrs = provider.free_eips
|
68
|
+
end
|
69
|
+
puts
|
70
|
+
addrs
|
71
|
+
end
|
72
|
+
|
73
|
+
def allocate_addresses
|
74
|
+
instances = @instances.find_all { |i| i.elastic_ip == true }
|
75
|
+
if instances.empty?
|
76
|
+
puts "no need to allocate elastic ips"
|
77
|
+
return
|
78
|
+
end
|
79
|
+
free_ips = provider.free_eips
|
80
|
+
toalloc = instances.size
|
81
|
+
blacklist = free_ips
|
82
|
+
if free_ips.size > 0
|
83
|
+
n = nil
|
84
|
+
if free_ips.size >= instances.size
|
85
|
+
n = instances.size
|
86
|
+
else
|
87
|
+
n = free_ips.size
|
88
|
+
end
|
89
|
+
confirm("Use #{n} of your #{free_ips.size} free eips?") do
|
90
|
+
blacklist = []
|
91
|
+
toalloc -= n
|
92
|
+
end
|
93
|
+
end
|
94
|
+
puts "need to alloc: #{toalloc}"
|
95
|
+
1.upto(toalloc) do |i|
|
96
|
+
puts "allocating eip address #{i}"
|
97
|
+
provider.ec2.allocate_address
|
98
|
+
end
|
99
|
+
|
100
|
+
addrs = wait_for_eips(free_ips.size + toalloc)
|
101
|
+
avail_ips = addrs.collect { |a| a.publicIp }.to_set
|
102
|
+
black_ips = blacklist.collect { |a| a.publicIp }.to_set
|
103
|
+
(avail_ips - black_ips).to_a[0,instances.size]
|
104
|
+
end
|
105
|
+
|
106
|
+
def running_instances
|
107
|
+
get_instances.find_all { |i| i.instanceState.name == "running" }
|
108
|
+
end
|
109
|
+
|
110
|
+
def running_instances_count(ids)
|
111
|
+
running_instances.count { |i| ids.include?(i.instanceId) }
|
112
|
+
end
|
113
|
+
|
114
|
+
def wait_for_running(instances)
|
115
|
+
ids = instances.collect { |i| i.instanceId }.to_set
|
116
|
+
print "Waiting for all instances to be in running state. "
|
117
|
+
STDOUT.flush
|
118
|
+
while running_instances_count(ids) < instances.size
|
119
|
+
print "."
|
120
|
+
sleep(5)
|
121
|
+
STDOUT.flush
|
122
|
+
end
|
123
|
+
puts
|
124
|
+
end
|
125
|
+
|
126
|
+
def run_instances(name)
|
127
|
+
instances = []
|
128
|
+
gen = Ec2BootstrapGenerator.new
|
129
|
+
@instances.each do |instance|
|
130
|
+
userdata = Base64.encode64(gen.generate_from_hash(
|
131
|
+
"bootstrap-repo" => instance.bootstrap_repo
|
132
|
+
))
|
133
|
+
res = provider.ec2.run_instances(
|
134
|
+
:image_id => instance.ami,
|
135
|
+
:key_name => instance.key_name,
|
136
|
+
:instance_type => instance.type,
|
137
|
+
:user_data => userdata)
|
138
|
+
ap res
|
139
|
+
res.instancesSet.item.each do |i|
|
140
|
+
instances << i
|
141
|
+
provider.ec2.create_tags(:resource_id => i.instanceId,
|
142
|
+
:tag => [
|
143
|
+
{ 'bb_name' => instance.name },
|
144
|
+
{ 'bb_universe' => name},
|
145
|
+
])
|
146
|
+
end
|
147
|
+
end
|
148
|
+
instances.tap do |i|
|
149
|
+
wait_for_running(i)
|
150
|
+
end
|
151
|
+
end
|
152
|
+
|
153
|
+
def assign_addresses(instances, addrs)
|
154
|
+
if instances.size != addrs.size
|
155
|
+
raise "error: instances number and addresses don't match"
|
156
|
+
end
|
157
|
+
|
158
|
+
instances.each_index do |i|
|
159
|
+
puts "associating address #{addrs[i]} " +
|
160
|
+
"to instance #{instances[i].instanceId}"
|
161
|
+
provider.ec2.associate_address(
|
162
|
+
:instance_id => instances[i].instanceId,
|
163
|
+
:public_ip => addrs[i])
|
164
|
+
end
|
165
|
+
end
|
166
|
+
|
167
|
+
def create_dns_entries(name, instances, addrs)
|
168
|
+
@instances.each_index do |i|
|
169
|
+
domains = [@instances[i].domain]
|
170
|
+
if @instances[i].domain.is_a?(Array)
|
171
|
+
domains = @instances[i].domain
|
172
|
+
end
|
173
|
+
domains.each do |domain|
|
174
|
+
domain = "#{name}.#{domain}"
|
175
|
+
puts "creating domain #{domain}.#{@config.domain} to #{addrs[i]}"
|
176
|
+
provider.create_dns(domain, addrs[i])
|
177
|
+
if @instances[i].wildcard_domain == true
|
178
|
+
provider.create_dns("*.#{domain}", addrs[i])
|
179
|
+
end
|
180
|
+
end
|
181
|
+
end
|
182
|
+
end
|
183
|
+
|
184
|
+
def explode(name)
|
185
|
+
free_eips = allocate_addresses
|
186
|
+
confirm("Will assign the following ips:\n" +
|
187
|
+
"#{free_eips.join("\n")}\nConfirm") do
|
188
|
+
instances = run_instances(name)
|
189
|
+
assign_addresses(instances, free_eips)
|
190
|
+
create_dns_entries(name, instances, free_eips)
|
191
|
+
end
|
192
|
+
end
|
193
|
+
|
194
|
+
def tags
|
195
|
+
tag_set = provider.ec2.describe_tags(:filter =>
|
196
|
+
[{'key' => 'bb_universe'}]).tagSet
|
197
|
+
if tag_set.nil?
|
198
|
+
return []
|
199
|
+
end
|
200
|
+
|
201
|
+
tag_set.item
|
202
|
+
end
|
203
|
+
|
204
|
+
def list
|
205
|
+
universes = Set.new
|
206
|
+
tags.each do |tag|
|
207
|
+
universes << tag.value
|
208
|
+
end
|
209
|
+
universes.each do |u|
|
210
|
+
puts u
|
211
|
+
end
|
212
|
+
end
|
213
|
+
|
214
|
+
def confirm(msg)
|
215
|
+
print "#{msg}? [y/N] "
|
216
|
+
if STDIN.gets.strip == "y"
|
217
|
+
yield
|
218
|
+
return true
|
219
|
+
else
|
220
|
+
puts "skipping"
|
221
|
+
return false
|
222
|
+
end
|
223
|
+
end
|
224
|
+
|
225
|
+
def kill_instance(tag, running)
|
226
|
+
instance = running.find { |i| i.instanceId == tag.resourceId }
|
227
|
+
if instance.nil?
|
228
|
+
puts "instance #{tag.resourceId} is not running. skipping"
|
229
|
+
return
|
230
|
+
end
|
231
|
+
confirm("kill instance #{tag.resourceId}") do
|
232
|
+
provider.ec2.terminate_instances(:instance_id => [tag.resourceId])
|
233
|
+
puts "sent termination signal to #{tag.resourceId}"
|
234
|
+
end
|
235
|
+
end
|
236
|
+
|
237
|
+
def kill(name)
|
238
|
+
running = running_instances
|
239
|
+
tags.find_all do |tag|
|
240
|
+
tag.value == name
|
241
|
+
end.each do |tag|
|
242
|
+
send("kill_#{tag.resourceType}", tag, running)
|
243
|
+
end
|
244
|
+
end
|
245
|
+
end
|
246
|
+
end
|
data/lib/bigbang.rb
ADDED
data/src/bootstrap.sh
ADDED
@@ -0,0 +1,31 @@
|
|
1
|
+
#!/bin/bash
|
2
|
+
function extract()
|
3
|
+
{
|
4
|
+
SKIP=`awk '/^__TARFILE_FOLLOWS__/ { print NR + 1; exit 0; }' $0`
|
5
|
+
|
6
|
+
#remember our file name
|
7
|
+
THIS=`pwd`/$0
|
8
|
+
tmp=$(mktemp -d /tmp/bootstrap.XXXXX)
|
9
|
+
|
10
|
+
# take the tarfile and pipe it into tar
|
11
|
+
echo "decompressing to $tmp"
|
12
|
+
cd $tmp
|
13
|
+
tail -n +$SKIP $THIS | tar -xz
|
14
|
+
export EC2_GIT_BOOTSTRAP_PATH=$tmp/data
|
15
|
+
chmod -R a+Xr $tmp
|
16
|
+
./post-extract.sh
|
17
|
+
}
|
18
|
+
|
19
|
+
set -x
|
20
|
+
log=/var/log/ec2-git-bootstrap.log
|
21
|
+
extract > $log
|
22
|
+
echo "EC2-GIT-BOOTSTRAP: Completed!" >> $log
|
23
|
+
|
24
|
+
#
|
25
|
+
# place any bash script here you need.
|
26
|
+
# Any script here will happen after the tar file extract.
|
27
|
+
echo "Finished"
|
28
|
+
exit 0
|
29
|
+
|
30
|
+
# NOTE: Don't place any newline characters after the last line below.
|
31
|
+
__TARFILE_FOLLOWS__
|
data/src/git-ssh-wrap.sh
ADDED
data/src/post-extract.sh
ADDED
metadata
ADDED
@@ -0,0 +1,106 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: bigbang
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
hash: 29
|
5
|
+
prerelease:
|
6
|
+
segments:
|
7
|
+
- 0
|
8
|
+
- 0
|
9
|
+
- 1
|
10
|
+
version: 0.0.1
|
11
|
+
platform: ruby
|
12
|
+
authors:
|
13
|
+
- Giorgenes Gelatti
|
14
|
+
autorequire:
|
15
|
+
bindir: bin
|
16
|
+
cert_chain: []
|
17
|
+
|
18
|
+
date: 2011-07-15 00:00:00 +01:00
|
19
|
+
default_executable:
|
20
|
+
dependencies:
|
21
|
+
- !ruby/object:Gem::Dependency
|
22
|
+
name: fog
|
23
|
+
prerelease: false
|
24
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ">="
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
hash: 3
|
30
|
+
segments:
|
31
|
+
- 0
|
32
|
+
version: "0"
|
33
|
+
type: :runtime
|
34
|
+
version_requirements: *id001
|
35
|
+
- !ruby/object:Gem::Dependency
|
36
|
+
name: amazon-ec2
|
37
|
+
prerelease: false
|
38
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
39
|
+
none: false
|
40
|
+
requirements:
|
41
|
+
- - ">="
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
hash: 3
|
44
|
+
segments:
|
45
|
+
- 0
|
46
|
+
version: "0"
|
47
|
+
type: :runtime
|
48
|
+
version_requirements: *id002
|
49
|
+
description: ""
|
50
|
+
email: giorgenes@gmail.com
|
51
|
+
executables:
|
52
|
+
- bigbang
|
53
|
+
extensions: []
|
54
|
+
|
55
|
+
extra_rdoc_files: []
|
56
|
+
|
57
|
+
files:
|
58
|
+
- src/git-ssh-wrap.sh
|
59
|
+
- src/post-extract.sh
|
60
|
+
- src/bootstrap.sh
|
61
|
+
- bin/bigbang
|
62
|
+
- README
|
63
|
+
- lib/bigbang.rb
|
64
|
+
- lib/bigbang/instance.rb
|
65
|
+
- lib/bigbang/provider.rb
|
66
|
+
- lib/bigbang/dsl.rb
|
67
|
+
- lib/bigbang/universe.rb
|
68
|
+
- lib/bigbang/ec2-git-bootstrap.rb
|
69
|
+
- lib/bigbang/config.rb
|
70
|
+
- lib/bigbang/cluster.rb
|
71
|
+
has_rdoc: true
|
72
|
+
homepage: http://github.com/giorgenes/bigbang/
|
73
|
+
licenses: []
|
74
|
+
|
75
|
+
post_install_message:
|
76
|
+
rdoc_options: []
|
77
|
+
|
78
|
+
require_paths:
|
79
|
+
- lib
|
80
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
81
|
+
none: false
|
82
|
+
requirements:
|
83
|
+
- - ">="
|
84
|
+
- !ruby/object:Gem::Version
|
85
|
+
hash: 3
|
86
|
+
segments:
|
87
|
+
- 0
|
88
|
+
version: "0"
|
89
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
90
|
+
none: false
|
91
|
+
requirements:
|
92
|
+
- - ">="
|
93
|
+
- !ruby/object:Gem::Version
|
94
|
+
hash: 3
|
95
|
+
segments:
|
96
|
+
- 0
|
97
|
+
version: "0"
|
98
|
+
requirements: []
|
99
|
+
|
100
|
+
rubyforge_project:
|
101
|
+
rubygems_version: 1.6.2
|
102
|
+
signing_key:
|
103
|
+
specification_version: 3
|
104
|
+
summary: ""
|
105
|
+
test_files: []
|
106
|
+
|