nirvdrum-rubber 1.1.7

Sign up to get free protection for your applications and to get access to all the features.
Files changed (186) hide show
  1. data/CHANGELOG +146 -0
  2. data/COPYING +339 -0
  3. data/README +6 -0
  4. data/TODO +11 -0
  5. data/VERSION +1 -0
  6. data/bin/vulcanize +41 -0
  7. data/generators/vulcanize/USAGE +6 -0
  8. data/generators/vulcanize/templates/apache/config/rubber/deploy-apache.rb +51 -0
  9. data/generators/vulcanize/templates/apache/config/rubber/role/apache/deflate.conf +10 -0
  10. data/generators/vulcanize/templates/apache/config/rubber/role/apache/expires.conf +9 -0
  11. data/generators/vulcanize/templates/apache/config/rubber/role/apache/headers.conf +6 -0
  12. data/generators/vulcanize/templates/apache/config/rubber/role/apache/monit-apache.conf +8 -0
  13. data/generators/vulcanize/templates/apache/config/rubber/role/apache/ports.conf +5 -0
  14. data/generators/vulcanize/templates/apache/config/rubber/role/apache/setenvif.conf +52 -0
  15. data/generators/vulcanize/templates/apache/config/rubber/role/web_tools/tools-apache-vhost.conf +66 -0
  16. data/generators/vulcanize/templates/apache/config/rubber/role/web_tools/tools-apache.auth +7 -0
  17. data/generators/vulcanize/templates/apache/config/rubber/role/web_tools/tools-index.html +34 -0
  18. data/generators/vulcanize/templates/apache/config/rubber/rubber-apache.yml +6 -0
  19. data/generators/vulcanize/templates/apache/templates.yml +1 -0
  20. data/generators/vulcanize/templates/base/Capfile +14 -0
  21. data/generators/vulcanize/templates/base/config/deploy.rb +55 -0
  22. data/generators/vulcanize/templates/base/config/rubber/common/crontab +17 -0
  23. data/generators/vulcanize/templates/base/config/rubber/common/monit-postfix.conf +8 -0
  24. data/generators/vulcanize/templates/base/config/rubber/common/rubber.profile +14 -0
  25. data/generators/vulcanize/templates/base/config/rubber/deploy-setup.rb +84 -0
  26. data/generators/vulcanize/templates/base/config/rubber/rubber-dns.yml +79 -0
  27. data/generators/vulcanize/templates/base/config/rubber/rubber.yml +227 -0
  28. data/generators/vulcanize/templates/base/lib/tasks/rubber.rake +15 -0
  29. data/generators/vulcanize/templates/base/script/cron-rake +18 -0
  30. data/generators/vulcanize/templates/base/script/cron-runner +18 -0
  31. data/generators/vulcanize/templates/base/script/cron-sh +67 -0
  32. data/generators/vulcanize/templates/base/templates.yml +1 -0
  33. data/generators/vulcanize/templates/complete_mongrel_mysql/config/rubber/role/haproxy/haproxy-mongrel.conf +23 -0
  34. data/generators/vulcanize/templates/complete_mongrel_mysql/config/rubber/role/nginx/nginx-mongrel.conf +113 -0
  35. data/generators/vulcanize/templates/complete_mongrel_mysql/config/rubber/rubber-complete.yml +41 -0
  36. data/generators/vulcanize/templates/complete_mongrel_mysql/templates.yml +6 -0
  37. data/generators/vulcanize/templates/complete_passenger_mysql/config/rubber/role/haproxy/haproxy-passenger.conf +21 -0
  38. data/generators/vulcanize/templates/complete_passenger_mysql/config/rubber/rubber-complete.yml +40 -0
  39. data/generators/vulcanize/templates/complete_passenger_mysql/templates.yml +10 -0
  40. data/generators/vulcanize/templates/cruise/config/rubber/deploy-cruise.rb +72 -0
  41. data/generators/vulcanize/templates/cruise/config/rubber/role/cruise/cruise +40 -0
  42. data/generators/vulcanize/templates/cruise/config/rubber/role/cruise/my.cnf +165 -0
  43. data/generators/vulcanize/templates/cruise/config/rubber/role/cruise/production.rb +8 -0
  44. data/generators/vulcanize/templates/cruise/config/rubber/role/cruise/site_config.rb +76 -0
  45. data/generators/vulcanize/templates/cruise/config/rubber/role/web_tools/cruise-nginx.conf +11 -0
  46. data/generators/vulcanize/templates/cruise/config/rubber/rubber-cruise.yml +18 -0
  47. data/generators/vulcanize/templates/cruise/templates.yml +1 -0
  48. data/generators/vulcanize/templates/haproxy/config/rubber/deploy-haproxy.rb +45 -0
  49. data/generators/vulcanize/templates/haproxy/config/rubber/role/haproxy/haproxy-base.conf +26 -0
  50. data/generators/vulcanize/templates/haproxy/config/rubber/role/haproxy/haproxy-default.conf +8 -0
  51. data/generators/vulcanize/templates/haproxy/config/rubber/role/haproxy/monit-haproxy.conf +9 -0
  52. data/generators/vulcanize/templates/haproxy/config/rubber/role/haproxy/syslog-haproxy.conf +6 -0
  53. data/generators/vulcanize/templates/haproxy/config/rubber/role/haproxy/syslogd-default.conf +17 -0
  54. data/generators/vulcanize/templates/haproxy/config/rubber/role/web_tools/haproxy-nginx.conf +10 -0
  55. data/generators/vulcanize/templates/haproxy/config/rubber/rubber-haproxy.yml +7 -0
  56. data/generators/vulcanize/templates/haproxy/templates.yml +1 -0
  57. data/generators/vulcanize/templates/jetty/config/rubber/deploy-jetty.rb +59 -0
  58. data/generators/vulcanize/templates/jetty/config/rubber/role/jetty/jetty.sh +589 -0
  59. data/generators/vulcanize/templates/jetty/config/rubber/role/jetty/jetty.xml +199 -0
  60. data/generators/vulcanize/templates/jetty/config/rubber/role/jetty/monit-jetty.conf +9 -0
  61. data/generators/vulcanize/templates/jetty/config/rubber/rubber-jetty.yml +10 -0
  62. data/generators/vulcanize/templates/jetty/templates.yml +1 -0
  63. data/generators/vulcanize/templates/memcached/config/memcached.yml +28 -0
  64. data/generators/vulcanize/templates/memcached/config/rubber/common/memcached.yml +14 -0
  65. data/generators/vulcanize/templates/memcached/config/rubber/role/memcached/memcached.conf +52 -0
  66. data/generators/vulcanize/templates/memcached/config/rubber/role/memcached/memcached_munin_plugin +249 -0
  67. data/generators/vulcanize/templates/memcached/config/rubber/rubber-memcached.yml +7 -0
  68. data/generators/vulcanize/templates/memcached/templates.yml +1 -0
  69. data/generators/vulcanize/templates/minimal_mysql/templates.yml +7 -0
  70. data/generators/vulcanize/templates/minimal_nodb/templates.yml +6 -0
  71. data/generators/vulcanize/templates/mongrel/config/rubber/deploy-mongrel.rb +75 -0
  72. data/generators/vulcanize/templates/mongrel/config/rubber/role/mongrel/mongrel_cluster.yml +12 -0
  73. data/generators/vulcanize/templates/mongrel/config/rubber/role/mongrel/monit-mongrel.conf +20 -0
  74. data/generators/vulcanize/templates/mongrel/config/rubber/rubber-mongrel.yml +9 -0
  75. data/generators/vulcanize/templates/mongrel/templates.yml +1 -0
  76. data/generators/vulcanize/templates/monit/config/rubber/common/monit-default.conf +15 -0
  77. data/generators/vulcanize/templates/monit/config/rubber/common/monit.conf +251 -0
  78. data/generators/vulcanize/templates/monit/config/rubber/deploy-monit.rb +32 -0
  79. data/generators/vulcanize/templates/monit/config/rubber/role/web_tools/monit-admin-nginx.conf +10 -0
  80. data/generators/vulcanize/templates/monit/config/rubber/rubber-monit.yml +6 -0
  81. data/generators/vulcanize/templates/monit/templates.yml +1 -0
  82. data/generators/vulcanize/templates/munin/config/rubber/common/monit-munin.conf +8 -0
  83. data/generators/vulcanize/templates/munin/config/rubber/common/munin-node.conf +48 -0
  84. data/generators/vulcanize/templates/munin/config/rubber/common/munin-plugins.conf +9 -0
  85. data/generators/vulcanize/templates/munin/config/rubber/deploy-munin.rb +46 -0
  86. data/generators/vulcanize/templates/munin/config/rubber/role/web_tools/munin-nginx.conf +8 -0
  87. data/generators/vulcanize/templates/munin/config/rubber/role/web_tools/munin-plugins.conf +31 -0
  88. data/generators/vulcanize/templates/munin/config/rubber/role/web_tools/munin.conf +80 -0
  89. data/generators/vulcanize/templates/munin/config/rubber/rubber-munin.yml +8 -0
  90. data/generators/vulcanize/templates/munin/script/munin/example_mysql_query.rb +57 -0
  91. data/generators/vulcanize/templates/munin/script/munin/example_simple.rb +24 -0
  92. data/generators/vulcanize/templates/munin/templates.yml +1 -0
  93. data/generators/vulcanize/templates/mysql/config/rubber/common/database.yml +11 -0
  94. data/generators/vulcanize/templates/mysql/config/rubber/deploy-mysql.rb +156 -0
  95. data/generators/vulcanize/templates/mysql/config/rubber/role/db/crontab +14 -0
  96. data/generators/vulcanize/templates/mysql/config/rubber/role/db/monit-mysql.cnf +10 -0
  97. data/generators/vulcanize/templates/mysql/config/rubber/role/db/my.cnf +167 -0
  98. data/generators/vulcanize/templates/mysql/config/rubber/role/mysql_slave/mysql_slave_munin_plugin +51 -0
  99. data/generators/vulcanize/templates/mysql/config/rubber/rubber-mysql.yml +46 -0
  100. data/generators/vulcanize/templates/mysql/templates.yml +1 -0
  101. data/generators/vulcanize/templates/mysql_cluster/config/rubber/common/mysql_cluster_migrations.rb +13 -0
  102. data/generators/vulcanize/templates/mysql_cluster/config/rubber/deploy-mysql_cluster.rb +173 -0
  103. data/generators/vulcanize/templates/mysql_cluster/config/rubber/role/mysql_data/my.cnf +15 -0
  104. data/generators/vulcanize/templates/mysql_cluster/config/rubber/role/mysql_mgm/ndb_mgmd.cnf +39 -0
  105. data/generators/vulcanize/templates/mysql_cluster/config/rubber/role/mysql_sql/monit-mysql_cluster_sql.cnf +10 -0
  106. data/generators/vulcanize/templates/mysql_cluster/config/rubber/role/mysql_sql/my.cnf +23 -0
  107. data/generators/vulcanize/templates/mysql_cluster/config/rubber/rubber-mysql_cluster.yml +32 -0
  108. data/generators/vulcanize/templates/mysql_cluster/templates.yml +1 -0
  109. data/generators/vulcanize/templates/mysql_proxy/config/rubber/common/database.yml +16 -0
  110. data/generators/vulcanize/templates/mysql_proxy/config/rubber/common/monit-mysql_proxy.cnf +10 -0
  111. data/generators/vulcanize/templates/mysql_proxy/config/rubber/common/mysql-proxy +153 -0
  112. data/generators/vulcanize/templates/mysql_proxy/config/rubber/common/mysql-proxy.conf +10 -0
  113. data/generators/vulcanize/templates/mysql_proxy/config/rubber/common/mysql-proxy.lua +5 -0
  114. data/generators/vulcanize/templates/mysql_proxy/config/rubber/deploy-mysql_proxy.rb +52 -0
  115. data/generators/vulcanize/templates/mysql_proxy/config/rubber/rubber-mysql_proxy.yml +11 -0
  116. data/generators/vulcanize/templates/mysql_proxy/templates.yml +1 -0
  117. data/generators/vulcanize/templates/nginx/config/rubber/deploy-nginx.rb +45 -0
  118. data/generators/vulcanize/templates/nginx/config/rubber/role/nginx/crontab +9 -0
  119. data/generators/vulcanize/templates/nginx/config/rubber/role/nginx/monit-nginx.conf +8 -0
  120. data/generators/vulcanize/templates/nginx/config/rubber/role/nginx/nginx.conf +42 -0
  121. data/generators/vulcanize/templates/nginx/config/rubber/role/web_tools/nginx-tools.conf +55 -0
  122. data/generators/vulcanize/templates/nginx/config/rubber/role/web_tools/tools-index.html +30 -0
  123. data/generators/vulcanize/templates/nginx/config/rubber/role/web_tools/tools-nginx.auth +7 -0
  124. data/generators/vulcanize/templates/nginx/config/rubber/rubber-nginx.yml +10 -0
  125. data/generators/vulcanize/templates/nginx/templates.yml +1 -0
  126. data/generators/vulcanize/templates/passenger/config/rubber/deploy-passenger.rb +37 -0
  127. data/generators/vulcanize/templates/passenger/config/rubber/role/passenger/munin-passenger-memory.conf +34 -0
  128. data/generators/vulcanize/templates/passenger/config/rubber/role/passenger/munin-passenger-sudoers.conf +7 -0
  129. data/generators/vulcanize/templates/passenger/config/rubber/role/passenger/munin-passenger.conf +47 -0
  130. data/generators/vulcanize/templates/passenger/config/rubber/role/passenger/passenger-apache-vhost.conf +46 -0
  131. data/generators/vulcanize/templates/passenger/config/rubber/role/passenger/passenger.conf +10 -0
  132. data/generators/vulcanize/templates/passenger/config/rubber/rubber-passenger.yml +15 -0
  133. data/generators/vulcanize/templates/passenger/templates.yml +3 -0
  134. data/generators/vulcanize/templates/redis/config/rubber/deploy-redis.rb +36 -0
  135. data/generators/vulcanize/templates/redis/config/rubber/role/redis/crontab +8 -0
  136. data/generators/vulcanize/templates/redis/config/rubber/role/redis/monit-redis.conf +9 -0
  137. data/generators/vulcanize/templates/redis/config/rubber/role/redis/redis.conf +141 -0
  138. data/generators/vulcanize/templates/redis/config/rubber/rubber-redis.yml +4 -0
  139. data/generators/vulcanize/templates/redis/templates.yml +1 -0
  140. data/generators/vulcanize/templates/resque/config/rubber/deploy-resque-worker-default.rb +38 -0
  141. data/generators/vulcanize/templates/resque/config/rubber/deploy-resque.rb +39 -0
  142. data/generators/vulcanize/templates/resque/config/rubber/role/resque_worker_default/monit-resque_worker_default.conf +19 -0
  143. data/generators/vulcanize/templates/resque/config/rubber/rubber-resque.yml +10 -0
  144. data/generators/vulcanize/templates/resque/templates.yml +3 -0
  145. data/generators/vulcanize/templates/sphinx/config/rubber/common/sphinx.yml +46 -0
  146. data/generators/vulcanize/templates/sphinx/config/rubber/deploy-sphinx.rb +112 -0
  147. data/generators/vulcanize/templates/sphinx/config/rubber/role/sphinx/crontab +11 -0
  148. data/generators/vulcanize/templates/sphinx/config/rubber/role/sphinx/monit-sphinx.conf +10 -0
  149. data/generators/vulcanize/templates/sphinx/config/rubber/rubber-sphinx.yml +6 -0
  150. data/generators/vulcanize/templates/sphinx/templates.yml +1 -0
  151. data/generators/vulcanize/vulcanize_generator.rb +67 -0
  152. data/lib/capistrano/hostcmd.rb +12 -0
  153. data/lib/rubber.rb +38 -0
  154. data/lib/rubber/capistrano.rb +1 -0
  155. data/lib/rubber/cloud.rb +13 -0
  156. data/lib/rubber/cloud/aws.rb +334 -0
  157. data/lib/rubber/cloud/base.rb +16 -0
  158. data/lib/rubber/configuration.rb +47 -0
  159. data/lib/rubber/dns.rb +13 -0
  160. data/lib/rubber/dns/base.rb +84 -0
  161. data/lib/rubber/dns/dyndns.rb +78 -0
  162. data/lib/rubber/dns/nettica.rb +117 -0
  163. data/lib/rubber/dns/zerigo.rb +174 -0
  164. data/lib/rubber/environment.rb +169 -0
  165. data/lib/rubber/generator.rb +197 -0
  166. data/lib/rubber/instance.rb +166 -0
  167. data/lib/rubber/recipes/rubber.rb +89 -0
  168. data/lib/rubber/recipes/rubber/bundles.rb +28 -0
  169. data/lib/rubber/recipes/rubber/deploy.rb +90 -0
  170. data/lib/rubber/recipes/rubber/instances.rb +393 -0
  171. data/lib/rubber/recipes/rubber/load_balancers.rb +44 -0
  172. data/lib/rubber/recipes/rubber/security_groups.rb +189 -0
  173. data/lib/rubber/recipes/rubber/setup.rb +457 -0
  174. data/lib/rubber/recipes/rubber/spot_requests.rb +17 -0
  175. data/lib/rubber/recipes/rubber/static_ips.rb +107 -0
  176. data/lib/rubber/recipes/rubber/utils.rb +203 -0
  177. data/lib/rubber/recipes/rubber/volumes.rb +264 -0
  178. data/lib/rubber/tasks/rubber.rb +279 -0
  179. data/lib/rubber/util.rb +37 -0
  180. data/rails/init.rb +9 -0
  181. data/test/environment_test.rb +133 -0
  182. data/test/generator_test.rb +323 -0
  183. data/test/instance_test.rb +93 -0
  184. data/test/test_helper.rb +8 -0
  185. data/test/util_test.rb +16 -0
  186. metadata +298 -0
@@ -0,0 +1,44 @@
1
+ namespace :rubber do
2
+
3
+ #desc <<-DESC
4
+ # Sets up the network load balancers
5
+ #DESC
6
+ #required_task :setup_load_balancers do
7
+ # setup_load_balancers()
8
+ #end
9
+ #
10
+ #desc <<-DESC
11
+ # Describes the network load balancers
12
+ #DESC
13
+ #required_task :describe_load_balancers do
14
+ # lbs = cloud.describe_load_balancers()
15
+ # pp lbs
16
+ #end
17
+
18
+ def setup_load_balancers
19
+ # OPTIONAL: Automatically provision and assign instances to a Cloud provided
20
+ # load balancer.
21
+ #load_balancers:
22
+ # my_lb_name:
23
+ # listeners:
24
+ # - protocol: http
25
+ # port: 80
26
+ # instance_port: 8080
27
+ # - protocol: tcp
28
+ # port: 443
29
+ # instance_port: 8080
30
+ # target_roles: [app]
31
+ #
32
+ #isolate_load_balancers: true
33
+
34
+
35
+
36
+ # get remote lbs
37
+ # for each local not in remote, add it
38
+ # get all zones for all instances for roles, and make sure in lb
39
+ # warn if lb not balanced (count of instances per zone is equal)
40
+ # for each local that is in remote, sync listeners and zones
41
+ # for each remote not in local, remove it
42
+ end
43
+
44
+ end
@@ -0,0 +1,189 @@
1
+ namespace :rubber do
2
+
3
+ desc <<-DESC
4
+ Sets up the network security groups
5
+ All defined groups will be created, and any not defined will be removed.
6
+ Likewise, rules within a group will get created, and those not will be removed
7
+ DESC
8
+ required_task :setup_security_groups do
9
+ setup_security_groups()
10
+ end
11
+
12
+ desc <<-DESC
13
+ Describes the network security groups
14
+ DESC
15
+ required_task :describe_security_groups do
16
+ groups = cloud.describe_security_groups()
17
+ groups.each do |group|
18
+ puts "#{group[:name]}, #{group[:description]}"
19
+ group[:permissions].each do |perm|
20
+ puts " protocol: #{perm[:protocol]}"
21
+ puts " from_port: #{perm[:from_port]}"
22
+ puts " to_port: #{perm[:to_port]}"
23
+ puts " source_groups: #{perm[:source_groups].collect {|g| g[:name]}.join(", ") }" if perm[:source_groups]
24
+ puts " source_ips: #{perm[:source_ips].join(", ") }" if perm[:source_ips]
25
+ puts "\n"
26
+ end if group[:permissions]
27
+ puts "\n"
28
+ end
29
+ end
30
+
31
+
32
+ def get_assigned_security_groups(host=nil, roles=[])
33
+ env = rubber_cfg.environment.bind(roles, host)
34
+ security_groups = env.assigned_security_groups
35
+ if env.auto_security_groups
36
+ security_groups << host
37
+ security_groups += roles
38
+ end
39
+ security_groups = security_groups.uniq.compact.reject {|x| x.empty? }
40
+ security_groups = security_groups.collect {|x| isolate_group_name(x) } if env.isolate_security_groups
41
+ return security_groups
42
+ end
43
+
44
+ def setup_security_groups(host=nil, roles=[])
45
+ env = rubber_cfg.environment.bind(roles, host)
46
+ security_group_defns = env.security_groups
47
+ if env.auto_security_groups
48
+ sghosts = (rubber_instances.collect{|ic| ic.name } + [host]).uniq.compact
49
+ sgroles = (rubber_instances.all_roles + roles).uniq.compact
50
+ security_group_defns = inject_auto_security_groups(security_group_defns, sghosts, sgroles)
51
+ sync_security_groups(security_group_defns)
52
+ else
53
+ sync_security_groups(security_group_defns)
54
+ end
55
+ end
56
+
57
+ def inject_auto_security_groups(groups, hosts, roles)
58
+ hosts.each do |name|
59
+ group_name = name
60
+ groups[group_name] ||= {'description' => "Rubber automatic security group for host: #{name}", 'rules' => []}
61
+ end
62
+ roles.each do |name|
63
+ group_name = name
64
+ groups[group_name] ||= {'description' => "Rubber automatic security group for role: #{name}", 'rules' => []}
65
+ end
66
+ return groups
67
+ end
68
+
69
+ def isolate_prefix
70
+ return "#{rubber_env.app_name}_#{RUBBER_ENV}_"
71
+ end
72
+
73
+ def isolate_group_name(group_name)
74
+ new_name = "#{isolate_prefix}#{group_name}"
75
+ return new_name
76
+ end
77
+
78
+ def isolate_groups(groups)
79
+ renamed = {}
80
+ groups.each do |name, group|
81
+ new_name = name =~ /^#{isolate_prefix}/ ? name : isolate_group_name(name)
82
+ new_group = Marshal.load(Marshal.dump(group))
83
+ new_group['rules'].each do |rule|
84
+ old_ref_name = rule['source_group_name']
85
+ if old_ref_name && old_ref_name !~ /^#{isolate_prefix}/
86
+ rule['source_group_name'] = isolate_group_name(old_ref_name)
87
+ end
88
+ end
89
+ renamed[new_name] = new_group
90
+ end
91
+ return renamed
92
+ end
93
+
94
+ def sync_security_groups(groups)
95
+ return unless groups
96
+
97
+ groups = Rubber::Util::stringify(groups)
98
+ groups = isolate_groups(groups) if rubber_env.isolate_security_groups
99
+ group_keys = groups.keys.clone()
100
+
101
+ # For each group that does already exist in cloud
102
+ cloud_groups = cloud.describe_security_groups()
103
+ cloud_groups.each do |cloud_group|
104
+ group_name = cloud_group[:name]
105
+
106
+ # skip those groups that don't belong to this project/env
107
+ next if rubber_env.isolate_security_groups && group_name !~ /^#{isolate_prefix}/
108
+
109
+ if group_keys.delete(group_name)
110
+ # sync rules
111
+ logger.debug "Security Group already in cloud, syncing rules: #{group_name}"
112
+ group = groups[group_name]
113
+ rules = group['rules'].clone
114
+ rule_maps = []
115
+
116
+ # first collect the rule maps from the request (group/user pairs are duplicated for tcp/udp/icmp,
117
+ # so we need to do this up frnot and remove duplicates before checking against the local rubber rules)
118
+ cloud_group[:permissions].each do |rule|
119
+ if rule[:source_groups]
120
+ rule.source_groups.each do |source_group|
121
+ rule_map = {:source_group_name => source_group[:name], :source_group_account => source_group[:account]}
122
+ rule_map = Rubber::Util::stringify(rule_map)
123
+ rule_maps << rule_map unless rule_maps.include?(rule_map)
124
+ end
125
+ else
126
+ rule_map = Rubber::Util::stringify(rule)
127
+ rule_maps << rule_map unless rule_maps.include?(rule_map)
128
+ end
129
+ end if cloud_group[:permissions]
130
+ # For each rule, if it exists, do nothing, otherwise remove it as its no longer defined locally
131
+ rule_maps.each do |rule_map|
132
+ if rules.delete(rule_map)
133
+ # rules match, don't need to do anything
134
+ # logger.debug "Rule in sync: #{rule_map.inspect}"
135
+ else
136
+ # rules don't match, remove them from cloud and re-add below
137
+ answer = Capistrano::CLI.ui.ask("Rule '#{rule_map.inspect}' exists in cloud, but not locally, remove from cloud? [y/N]?: ")
138
+ rule_map = Rubber::Util::symbolize_keys(rule_map)
139
+ if rule_map[:source_group_name]
140
+ cloud.remove_security_group_rule(group_name, nil, nil, nil, {:name => rule_map[:source_group_name], :account => rule_map[:source_group_account]})
141
+ else
142
+ rule_map[:source_ips].each do |source_ip|
143
+ cloud.remove_security_group_rule(group_name, rule_map[:protocol], rule_map[:from_port], rule_map[:to_port], source_ip)
144
+ end if rule_map[:source_ips] && answer =~ /^y/
145
+ end
146
+ end
147
+ end
148
+
149
+ rules.each do |rule_map|
150
+ # create non-existing rules
151
+ logger.debug "Missing rule, creating: #{rule_map.inspect}"
152
+ rule_map = Rubber::Util::symbolize_keys(rule_map)
153
+ if rule_map[:source_group_name]
154
+ cloud.add_security_group_rule(group_name, nil, nil, nil, {:name => rule_map[:source_group_name], :account => rule_map[:source_group_account]})
155
+ else
156
+ rule_map[:source_ips].each do |source_ip|
157
+ cloud.add_security_group_rule(group_name, rule_map[:protocol], rule_map[:from_port], rule_map[:to_port], source_ip)
158
+ end if rule_map[:source_ips]
159
+ end
160
+ end
161
+ else
162
+ # delete group
163
+ answer = Capistrano::CLI.ui.ask("Security group '#{group_name}' exists in cloud but not locally, remove from cloud? [y/N]: ")
164
+ cloud.destroy_security_group(group_name) if answer =~ /^y/
165
+ end
166
+ end
167
+
168
+ # For each group that didnt already exist in cloud
169
+ group_keys.each do |group_name|
170
+ group = groups[group_name]
171
+ logger.debug "Creating new security group: #{group_name}"
172
+ # create each group
173
+ cloud.create_security_group(group_name, group['description'])
174
+ # create rules for group
175
+ group['rules'].each do |rule_map|
176
+ logger.debug "Creating new rule: #{rule_map.inspect}"
177
+ rule_map = Rubber::Util::symbolize_keys(rule_map)
178
+ if rule_map[:source_group_name]
179
+ cloud.add_security_group_rule(group_name, nil, nil, nil, {:name => rule_map[:source_group_name], :account => rule_map[:source_group_account]})
180
+ else
181
+ rule_map[:source_ips].each do |source_ip|
182
+ cloud.add_security_group_rule(group_name, rule_map[:protocol], rule_map[:from_port], rule_map[:to_port], source_ip)
183
+ end if rule_map[:source_ips]
184
+ end
185
+ end
186
+ end
187
+ end
188
+
189
+ end
@@ -0,0 +1,457 @@
1
+ namespace :rubber do
2
+
3
+ desc <<-DESC
4
+ Bootstraps instances by setting timezone, installing packages and gems
5
+ DESC
6
+ task :bootstrap do
7
+ set_timezone
8
+ link_bash
9
+ upgrade_packages
10
+ install_packages
11
+ setup_volumes
12
+ setup_gem_sources
13
+ install_gems
14
+ deploy.setup
15
+ end
16
+
17
+ desc <<-DESC
18
+ Sets up aliases for instance hostnames based on contents of instance.yml.
19
+ Generates /etc/hosts for local/remote machines and sets hostname on
20
+ remote instances, and sets values in dynamic dns entries
21
+ DESC
22
+ required_task :setup_aliases do
23
+ setup_local_aliases
24
+ setup_remote_aliases
25
+ setup_dns_aliases
26
+ end
27
+
28
+ desc <<-DESC
29
+ Sets up local aliases for instance hostnames based on contents of instance.yml.
30
+ Generates/etc/hosts for local machine
31
+ DESC
32
+ required_task :setup_local_aliases do
33
+ hosts_file = '/etc/hosts'
34
+
35
+ # Generate /etc/hosts contents for the local machine from instance config
36
+ delim = "## rubber config #{rubber_env.domain} #{RUBBER_ENV}"
37
+ local_hosts = delim + "\n"
38
+ rubber_instances.each do |ic|
39
+ # don't add unqualified hostname in local hosts file since user may be
40
+ # managing multiple domains with same aliases
41
+ hosts_data = [ic.full_name, ic.external_host, ic.internal_host].join(' ')
42
+ local_hosts << ic.external_ip << ' ' << hosts_data << "\n"
43
+ end
44
+ local_hosts << delim << "\n"
45
+
46
+ # Write out the hosts file for this machine, use sudo
47
+ filtered = File.read(hosts_file).gsub(/^#{delim}.*^#{delim}\n?/m, '')
48
+ logger.info "Writing out aliases into local machines #{hosts_file}, sudo access needed"
49
+ Rubber::Util::sudo_open(hosts_file, 'w') do |f|
50
+ f.write(filtered)
51
+ f.write(local_hosts)
52
+ end
53
+ end
54
+
55
+ desc <<-DESC
56
+ Sets up aliases in dynamic dns provider for instance hostnames based on contents of instance.yml.
57
+ DESC
58
+ required_task :setup_dns_aliases do
59
+ rubber_instances.each do |ic|
60
+ update_dyndns(ic)
61
+ end
62
+ end
63
+
64
+ desc <<-DESC
65
+ Sets up the additional dns records supplied in the dns_records config in rubber.yml
66
+ DESC
67
+ required_task :setup_dns_records do
68
+ records = rubber_env.dns_records
69
+ if records && rubber_env.dns_provider
70
+ provider = Rubber::Dns::get_provider(rubber_env.dns_provider, rubber_env)
71
+
72
+ # collect the round robin records (those with the same host/domain/type)
73
+ rr_records = []
74
+ records.each_with_index do |record, i|
75
+ m = records.find_all {|r| record['host'] == r['host'] && record['domain'] == r['domain'] && record['type'] == r['type']}
76
+ m = m.sort {|a,b| a.object_id <=> b.object_id}
77
+ rr_records << m if m.size > 1 && ! rr_records.include?(m)
78
+ end
79
+
80
+ # simple records are those that aren't round robin ones
81
+ simple_records = records - rr_records.flatten
82
+
83
+ # for each simple record, create or update as necessary
84
+ simple_records.each do |record|
85
+ matching = provider.find_host_records(:host => record['host'], :domain =>record['domain'], :type => record['type'])
86
+ if matching.size > 1
87
+ msg = "Multiple records in dns provider, but not in rubber.yml\n"
88
+ msg << "Round robin records need to be in both, or neither.\n"
89
+ msg << "Please fix manually:\n"
90
+ msg << matching.pretty_inspect
91
+ fatal(msg)
92
+ end
93
+
94
+ record = provider.setup_opts(record)
95
+ if matching.size == 1
96
+ match = matching.first
97
+ if provider.host_records_equal?(record, match)
98
+ logger.info "Simple dns record already up to date: #{record[:host]}.#{record[:domain]}:#{record[:type]} => #{record[:data]}"
99
+ else
100
+ logger.info "Updating simple dns record: #{record[:host]}.#{record[:domain]}:#{record[:type]} => #{record[:data]}"
101
+ provider.update_host_record(match, record)
102
+ end
103
+ else
104
+ logger.info "Creating simple dns record: #{record[:host]}.#{record[:domain]}:#{record[:type]} => #{record[:data]}"
105
+ provider.create_host_record(record)
106
+ end
107
+ end
108
+
109
+ # group round robin records
110
+ rr_records.each do |rr_group|
111
+ host = rr_group.first['host']
112
+ domain = rr_group.first['domain']
113
+ type = rr_group.first['type']
114
+ matching = provider.find_host_records(:host => host, :domain => domain, :type => type)
115
+
116
+ # remove from consideration the local records that are the same as remote ones
117
+ matching.clone.each do |r|
118
+ rr_group.delete_if {|rg| provider.host_records_equal?(r, rg) }
119
+ matching.delete_if {|rg| provider.host_records_equal?(r, rg) }
120
+ end
121
+ if rr_group.size == 0 && matching.size == 0
122
+ logger.info "Round robin dns records already up to date: #{host}.#{domain}:#{type}"
123
+ end
124
+
125
+ # create the local records that don't exist remotely
126
+ rr_group.each do |r|
127
+ r = provider.setup_opts(r)
128
+ logger.info "Creating round robin dns record: #{r[:host]}.#{r[:domain]}:#{r[:type]} => #{r[:data]}"
129
+ provider.create_host_record(r)
130
+ end
131
+
132
+ # remove the remote records that don't exist locally
133
+ matching.each do |r|
134
+ logger.info "Removing round robin dns record: #{r[:host]}.#{r[:domain]}:#{r[:type]} => #{r[:data]}"
135
+ provider.destroy_host_record(r)
136
+ end
137
+ end
138
+ end
139
+ end
140
+
141
+ desc <<-DESC
142
+ Sets up aliases for instance hostnames based on contents of instance.yml.
143
+ Generates /etc/hosts for remote machines and sets hostname on remote instances
144
+ DESC
145
+ task :setup_remote_aliases do
146
+ hosts_file = '/etc/hosts'
147
+
148
+ # Generate /etc/hosts contents for the remote instance from instance config
149
+ delim = "## rubber config"
150
+ delim = "#{delim} #{RUBBER_ENV}"
151
+ remote_hosts = delim + "\n"
152
+ rubber_instances.each do |ic|
153
+ hosts_data = [ic.name, ic.full_name, ic.external_host, ic.internal_host].join(' ')
154
+ remote_hosts << ic.internal_ip << ' ' << hosts_data << "\n"
155
+ end
156
+ remote_hosts << delim << "\n"
157
+ if rubber_instances.size > 0
158
+ # write out the hosts file for the remote instances
159
+ # NOTE that we use "capture" to get the existing hosts
160
+ # file, which only grabs the hosts file from the first host
161
+ filtered = (capture "cat #{hosts_file}").gsub(/^#{delim}.*^#{delim}\n?/m, '')
162
+ filtered = filtered + remote_hosts
163
+ # Put the generated hosts back on remote instance
164
+ put filtered, hosts_file
165
+
166
+ # Setup hostname on instance so shell, etcs have nice display
167
+ sudo "echo $CAPISTRANO:HOST$ > /etc/hostname && hostname $CAPISTRANO:HOST$"
168
+ end
169
+
170
+ # TODO
171
+ # /etc/resolv.conf to add search domain
172
+ # ~/.ssh/options to setup user/host/key aliases
173
+ end
174
+
175
+ desc <<-DESC
176
+ Update to the newest versions of all packages/gems.
177
+ DESC
178
+ task :update do
179
+ upgrade_packages
180
+ update_gems
181
+ end
182
+
183
+ desc <<-DESC
184
+ Upgrade to the newest versions of all Ubuntu packages.
185
+ DESC
186
+ task :upgrade_packages do
187
+ package_helper(true)
188
+ end
189
+
190
+ desc <<-DESC
191
+ Upgrade to the newest versions of all rubygems.
192
+ DESC
193
+ task :update_gems do
194
+ gem_helper(true)
195
+ end
196
+
197
+ desc <<-DESC
198
+ Install extra packages and gems.
199
+ DESC
200
+ task :install do
201
+ install_packages
202
+ install_gems
203
+ end
204
+
205
+ desc <<-DESC
206
+ Install Ubuntu packages. Set 'packages' in rubber.yml to \
207
+ be an array of strings.
208
+ DESC
209
+ task :install_packages do
210
+ package_helper(false)
211
+ end
212
+
213
+ desc <<-DESC
214
+ Install ruby gems. Set 'gems' in rubber.yml to \
215
+ be an array of strings.
216
+ DESC
217
+ task :install_gems do
218
+ gem_helper(false)
219
+ end
220
+
221
+ desc <<-DESC
222
+ Install ruby gems defined in the rails environment.rb
223
+ DESC
224
+ after "rubber:config", "rubber:install_rails_gems" if Rubber::Util.is_rails?
225
+ task :install_rails_gems do
226
+ sudo "sh -c 'cd #{current_path} && RAILS_ENV=#{RUBBER_ENV} rake gems:install'"
227
+ end
228
+
229
+ desc <<-DESC
230
+ Convenience task for installing your defined set of ruby gems locally.
231
+ DESC
232
+ required_task :install_local_gems do
233
+ fatal("install_local_gems can only be run in development") if RUBBER_ENV != 'development'
234
+ env = rubber_cfg.environment.bind(rubber_cfg.environment.known_roles)
235
+ gems = env['gems']
236
+ expanded_gem_list = []
237
+ gems.each do |gem_spec|
238
+ if gem_spec.is_a?(Array)
239
+ expanded_gem_list << "#{gem_spec[0]}:#{gem_spec[1]}"
240
+ else
241
+ expanded_gem_list << gem_spec
242
+ end
243
+ end
244
+ expanded_gem_list = expanded_gem_list.join(' ')
245
+
246
+ logger.info "Installing gems:#{expanded_gem_list}"
247
+ open("/tmp/gem_helper", "w") {|f| f.write(gem_helper_script)}
248
+ system "sh /tmp/gem_helper install #{expanded_gem_list}"
249
+ end
250
+
251
+ desc <<-DESC
252
+ Setup ruby gems sources. Set 'gemsources' in rubber.yml to \
253
+ be an array of URI strings.
254
+ DESC
255
+ task :setup_gem_sources do
256
+ if rubber_env.gemsources
257
+ script = prepare_script 'gem_sources_helper', <<-'ENDSCRIPT'
258
+ ruby - $@ <<-'EOF'
259
+
260
+ sources = ARGV
261
+
262
+ installed = []
263
+ `gem sources -l`.grep(/^[^*]/) do |line|
264
+ line = line.strip
265
+ installed << line if line.size > 0
266
+ end
267
+
268
+ to_install = sources - installed
269
+ to_remove = installed - sources
270
+
271
+ if to_install.size > 0
272
+ to_install.each do |source|
273
+ system "gem sources -a #{source}"
274
+ fail "Unable to add gem sources" if $?.exitstatus > 0
275
+ end
276
+ end
277
+ if to_remove.size > 0
278
+ to_remove.each do |source|
279
+ system "gem sources -r #{source}"
280
+ fail "Unable to remove gem sources" if $?.exitstatus > 0
281
+ end
282
+ end
283
+
284
+ 'EOF'
285
+ ENDSCRIPT
286
+
287
+ sudo "sh #{script} #{rubber_env.gemsources.join(' ')}"
288
+ end
289
+ end
290
+
291
+ desc <<-DESC
292
+ The ubuntu has /bin/sh linking to dash instead of bash, fix this
293
+ You can override this task if you don't want this to happen
294
+ DESC
295
+ task :link_bash do
296
+ sudo("ln -sf /bin/bash /bin/sh")
297
+ end
298
+
299
+ desc <<-DESC
300
+ Set the timezone using the value of the variable named timezone. \
301
+ Valid options for timezone can be determined by the contents of \
302
+ /usr/share/zoneinfo, which can be seen here: \
303
+ http://packages.ubuntu.com/cgi-bin/search_contents.pl?searchmode=filelist&word=tzdata&version=gutsy&arch=all&page=1&number=all \
304
+ Remove 'usr/share/zoneinfo/' from the filename, and use the last \
305
+ directory and file as the value. For example 'Africa/Abidjan' or \
306
+ 'posix/GMT' or 'Canada/Eastern'.
307
+ DESC
308
+ task :set_timezone do
309
+ opts = get_host_options('timezone')
310
+ sudo "bash -c 'echo $CAPISTRANO:VAR$ > /etc/timezone'", opts
311
+ sudo "cp /usr/share/zoneinfo/$CAPISTRANO:VAR$ /etc/localtime", opts
312
+ # restart syslog so that times match timezone
313
+ sudo "/etc/init.d/sysklogd restart"
314
+ end
315
+
316
+ def update_dyndns(instance_item)
317
+ env = rubber_cfg.environment.bind(instance_item.role_names, instance_item.name)
318
+ if env.dns_provider
319
+ provider = Rubber::Dns::get_provider(env.dns_provider, env)
320
+ provider.update(instance_item.name, instance_item.external_ip)
321
+ end
322
+ end
323
+
324
+ def destroy_dyndns(instance_item)
325
+ env = rubber_cfg.environment.bind(instance_item.role_names, instance_item.name)
326
+ if env.dns_provider
327
+ provider = Rubber::Dns::get_provider(env.dns_provider, env)
328
+ provider.destroy(instance_item.name)
329
+ end
330
+ end
331
+
332
+ def package_helper(upgrade=false)
333
+ opts = get_host_options('packages') do |pkg_list|
334
+ expanded_pkg_list = []
335
+ pkg_list.each do |pkg_spec|
336
+ if pkg_spec.is_a?(Array)
337
+ expanded_pkg_list << "#{pkg_spec[0]}=#{pkg_spec[1]}"
338
+ else
339
+ expanded_pkg_list << pkg_spec
340
+ end
341
+ end
342
+ expanded_pkg_list.join(' ')
343
+ end
344
+
345
+ sudo "apt-get -q update"
346
+ if upgrade
347
+ sudo "/bin/sh -c 'export DEBIAN_FRONTEND=noninteractive; apt-get -q -y --force-yes dist-upgrade'"
348
+ else
349
+ sudo "/bin/sh -c 'export DEBIAN_FRONTEND=noninteractive; apt-get -q -y --force-yes install $CAPISTRANO:VAR$'", opts
350
+ end
351
+ end
352
+
353
+ def custom_package(url_base, name, ver, install_test)
354
+ rubber.run_script "install_#{name}", <<-ENDSCRIPT
355
+ if [[ #{install_test} ]]; then
356
+ arch=`uname -m`
357
+ if [ "$arch" = "x86_64" ]; then
358
+ src="#{url_base}/#{name}_#{ver}_amd64.deb"
359
+ else
360
+ src="#{url_base}/#{name}_#{ver}_i386.deb"
361
+ fi
362
+ src_file="${src##*/}"
363
+ wget -qP /tmp ${src}
364
+ dpkg -i /tmp/${src_file}
365
+ fi
366
+ ENDSCRIPT
367
+ end
368
+
369
+ def handle_gem_prompt(ch, data, str)
370
+ ch[:data] ||= ""
371
+ ch[:data] << data
372
+ if data =~ />\s*$/
373
+ logger.info data
374
+ logger.info "The gem command is asking for a number:"
375
+ choice = STDIN.gets
376
+ ch.send_data(choice)
377
+ else
378
+ logger.info data
379
+ end
380
+ end
381
+
382
+ # Rubygems always installs even if the gem is already installed
383
+ # When providing versions, rubygems fails unless versions are provided for all gems
384
+ # This helper script works around these issues by installing gems only if they
385
+ # aren't already installed, and separates versioned/unversioned into two separate
386
+ # calls to rubygems
387
+ #
388
+ set :gem_helper_script, <<-'ENDSCRIPT'
389
+ ruby - $@ <<-'EOF'
390
+
391
+ gem_cmd = ARGV[0]
392
+ gems = ARGV[1..-1]
393
+ cmd = "gem #{gem_cmd} --no-rdoc --no-ri"
394
+
395
+ to_install = {}
396
+ to_install_ver = {}
397
+ # gem list passed in, possibly with versions, as "gem1 gem2:1.2 gem3"
398
+ gems.each do |gem_spec|
399
+ parts = gem_spec.split(':')
400
+ if parts[1]
401
+ to_install_ver[parts[0]] = parts[1]
402
+ else
403
+ to_install[parts[0]] = true
404
+ end
405
+ end
406
+
407
+ installed = {}
408
+ `gem list --local`.each do |line|
409
+ parts = line.scan(/(.*) \((.*)\)/).first
410
+ next unless parts && parts.size == 2
411
+ installed[parts[0]] = parts[1].split(",")
412
+ end
413
+
414
+ to_install.delete_if {|g, v| installed.has_key?(g) } if gem_cmd == 'install'
415
+ to_install_ver.delete_if {|g, v| installed.has_key?(g) && installed[g].include?(v) }
416
+
417
+ # rubygems can only do asingle versioned gem at a time so we need
418
+ # to do the two groups separately
419
+ # install versioned ones first so unversioned don't pull in a newer version
420
+ to_install_ver.each do |g, v|
421
+ system "#{cmd} #{g} -v #{v}"
422
+ fail "Unable to install versioned gem #{g}:#{v}" if $?.exitstatus > 0
423
+ end
424
+ if to_install.size > 0
425
+ gem_list = to_install.keys.join(' ')
426
+ system "#{cmd} #{gem_list}"
427
+ fail "Unable to install gems" if $?.exitstatus > 0
428
+ end
429
+
430
+ 'EOF'
431
+ ENDSCRIPT
432
+
433
+ # Helper for installing gems,allows one to respond to prompts
434
+ def gem_helper(update=false)
435
+ cmd = update ? "update" : "install"
436
+
437
+ opts = get_host_options('gems') do |gem_list|
438
+ expanded_gem_list = []
439
+ gem_list.each do |gem_spec|
440
+ if gem_spec.is_a?(Array)
441
+ expanded_gem_list << "#{gem_spec[0]}:#{gem_spec[1]}"
442
+ else
443
+ expanded_gem_list << gem_spec
444
+ end
445
+ end
446
+ expanded_gem_list.join(' ')
447
+ end
448
+
449
+ if opts.size > 0
450
+ script = prepare_script('gem_helper', gem_helper_script)
451
+ sudo "sh #{script} #{cmd} $CAPISTRANO:VAR$", opts do |ch, str, data|
452
+ handle_gem_prompt(ch, data, str)
453
+ end
454
+ end
455
+ end
456
+
457
+ end