nirvdrum-rubber 1.1.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (186) hide show
  1. data/CHANGELOG +146 -0
  2. data/COPYING +339 -0
  3. data/README +6 -0
  4. data/TODO +11 -0
  5. data/VERSION +1 -0
  6. data/bin/vulcanize +41 -0
  7. data/generators/vulcanize/USAGE +6 -0
  8. data/generators/vulcanize/templates/apache/config/rubber/deploy-apache.rb +51 -0
  9. data/generators/vulcanize/templates/apache/config/rubber/role/apache/deflate.conf +10 -0
  10. data/generators/vulcanize/templates/apache/config/rubber/role/apache/expires.conf +9 -0
  11. data/generators/vulcanize/templates/apache/config/rubber/role/apache/headers.conf +6 -0
  12. data/generators/vulcanize/templates/apache/config/rubber/role/apache/monit-apache.conf +8 -0
  13. data/generators/vulcanize/templates/apache/config/rubber/role/apache/ports.conf +5 -0
  14. data/generators/vulcanize/templates/apache/config/rubber/role/apache/setenvif.conf +52 -0
  15. data/generators/vulcanize/templates/apache/config/rubber/role/web_tools/tools-apache-vhost.conf +66 -0
  16. data/generators/vulcanize/templates/apache/config/rubber/role/web_tools/tools-apache.auth +7 -0
  17. data/generators/vulcanize/templates/apache/config/rubber/role/web_tools/tools-index.html +34 -0
  18. data/generators/vulcanize/templates/apache/config/rubber/rubber-apache.yml +6 -0
  19. data/generators/vulcanize/templates/apache/templates.yml +1 -0
  20. data/generators/vulcanize/templates/base/Capfile +14 -0
  21. data/generators/vulcanize/templates/base/config/deploy.rb +55 -0
  22. data/generators/vulcanize/templates/base/config/rubber/common/crontab +17 -0
  23. data/generators/vulcanize/templates/base/config/rubber/common/monit-postfix.conf +8 -0
  24. data/generators/vulcanize/templates/base/config/rubber/common/rubber.profile +14 -0
  25. data/generators/vulcanize/templates/base/config/rubber/deploy-setup.rb +84 -0
  26. data/generators/vulcanize/templates/base/config/rubber/rubber-dns.yml +79 -0
  27. data/generators/vulcanize/templates/base/config/rubber/rubber.yml +227 -0
  28. data/generators/vulcanize/templates/base/lib/tasks/rubber.rake +15 -0
  29. data/generators/vulcanize/templates/base/script/cron-rake +18 -0
  30. data/generators/vulcanize/templates/base/script/cron-runner +18 -0
  31. data/generators/vulcanize/templates/base/script/cron-sh +67 -0
  32. data/generators/vulcanize/templates/base/templates.yml +1 -0
  33. data/generators/vulcanize/templates/complete_mongrel_mysql/config/rubber/role/haproxy/haproxy-mongrel.conf +23 -0
  34. data/generators/vulcanize/templates/complete_mongrel_mysql/config/rubber/role/nginx/nginx-mongrel.conf +113 -0
  35. data/generators/vulcanize/templates/complete_mongrel_mysql/config/rubber/rubber-complete.yml +41 -0
  36. data/generators/vulcanize/templates/complete_mongrel_mysql/templates.yml +6 -0
  37. data/generators/vulcanize/templates/complete_passenger_mysql/config/rubber/role/haproxy/haproxy-passenger.conf +21 -0
  38. data/generators/vulcanize/templates/complete_passenger_mysql/config/rubber/rubber-complete.yml +40 -0
  39. data/generators/vulcanize/templates/complete_passenger_mysql/templates.yml +10 -0
  40. data/generators/vulcanize/templates/cruise/config/rubber/deploy-cruise.rb +72 -0
  41. data/generators/vulcanize/templates/cruise/config/rubber/role/cruise/cruise +40 -0
  42. data/generators/vulcanize/templates/cruise/config/rubber/role/cruise/my.cnf +165 -0
  43. data/generators/vulcanize/templates/cruise/config/rubber/role/cruise/production.rb +8 -0
  44. data/generators/vulcanize/templates/cruise/config/rubber/role/cruise/site_config.rb +76 -0
  45. data/generators/vulcanize/templates/cruise/config/rubber/role/web_tools/cruise-nginx.conf +11 -0
  46. data/generators/vulcanize/templates/cruise/config/rubber/rubber-cruise.yml +18 -0
  47. data/generators/vulcanize/templates/cruise/templates.yml +1 -0
  48. data/generators/vulcanize/templates/haproxy/config/rubber/deploy-haproxy.rb +45 -0
  49. data/generators/vulcanize/templates/haproxy/config/rubber/role/haproxy/haproxy-base.conf +26 -0
  50. data/generators/vulcanize/templates/haproxy/config/rubber/role/haproxy/haproxy-default.conf +8 -0
  51. data/generators/vulcanize/templates/haproxy/config/rubber/role/haproxy/monit-haproxy.conf +9 -0
  52. data/generators/vulcanize/templates/haproxy/config/rubber/role/haproxy/syslog-haproxy.conf +6 -0
  53. data/generators/vulcanize/templates/haproxy/config/rubber/role/haproxy/syslogd-default.conf +17 -0
  54. data/generators/vulcanize/templates/haproxy/config/rubber/role/web_tools/haproxy-nginx.conf +10 -0
  55. data/generators/vulcanize/templates/haproxy/config/rubber/rubber-haproxy.yml +7 -0
  56. data/generators/vulcanize/templates/haproxy/templates.yml +1 -0
  57. data/generators/vulcanize/templates/jetty/config/rubber/deploy-jetty.rb +59 -0
  58. data/generators/vulcanize/templates/jetty/config/rubber/role/jetty/jetty.sh +589 -0
  59. data/generators/vulcanize/templates/jetty/config/rubber/role/jetty/jetty.xml +199 -0
  60. data/generators/vulcanize/templates/jetty/config/rubber/role/jetty/monit-jetty.conf +9 -0
  61. data/generators/vulcanize/templates/jetty/config/rubber/rubber-jetty.yml +10 -0
  62. data/generators/vulcanize/templates/jetty/templates.yml +1 -0
  63. data/generators/vulcanize/templates/memcached/config/memcached.yml +28 -0
  64. data/generators/vulcanize/templates/memcached/config/rubber/common/memcached.yml +14 -0
  65. data/generators/vulcanize/templates/memcached/config/rubber/role/memcached/memcached.conf +52 -0
  66. data/generators/vulcanize/templates/memcached/config/rubber/role/memcached/memcached_munin_plugin +249 -0
  67. data/generators/vulcanize/templates/memcached/config/rubber/rubber-memcached.yml +7 -0
  68. data/generators/vulcanize/templates/memcached/templates.yml +1 -0
  69. data/generators/vulcanize/templates/minimal_mysql/templates.yml +7 -0
  70. data/generators/vulcanize/templates/minimal_nodb/templates.yml +6 -0
  71. data/generators/vulcanize/templates/mongrel/config/rubber/deploy-mongrel.rb +75 -0
  72. data/generators/vulcanize/templates/mongrel/config/rubber/role/mongrel/mongrel_cluster.yml +12 -0
  73. data/generators/vulcanize/templates/mongrel/config/rubber/role/mongrel/monit-mongrel.conf +20 -0
  74. data/generators/vulcanize/templates/mongrel/config/rubber/rubber-mongrel.yml +9 -0
  75. data/generators/vulcanize/templates/mongrel/templates.yml +1 -0
  76. data/generators/vulcanize/templates/monit/config/rubber/common/monit-default.conf +15 -0
  77. data/generators/vulcanize/templates/monit/config/rubber/common/monit.conf +251 -0
  78. data/generators/vulcanize/templates/monit/config/rubber/deploy-monit.rb +32 -0
  79. data/generators/vulcanize/templates/monit/config/rubber/role/web_tools/monit-admin-nginx.conf +10 -0
  80. data/generators/vulcanize/templates/monit/config/rubber/rubber-monit.yml +6 -0
  81. data/generators/vulcanize/templates/monit/templates.yml +1 -0
  82. data/generators/vulcanize/templates/munin/config/rubber/common/monit-munin.conf +8 -0
  83. data/generators/vulcanize/templates/munin/config/rubber/common/munin-node.conf +48 -0
  84. data/generators/vulcanize/templates/munin/config/rubber/common/munin-plugins.conf +9 -0
  85. data/generators/vulcanize/templates/munin/config/rubber/deploy-munin.rb +46 -0
  86. data/generators/vulcanize/templates/munin/config/rubber/role/web_tools/munin-nginx.conf +8 -0
  87. data/generators/vulcanize/templates/munin/config/rubber/role/web_tools/munin-plugins.conf +31 -0
  88. data/generators/vulcanize/templates/munin/config/rubber/role/web_tools/munin.conf +80 -0
  89. data/generators/vulcanize/templates/munin/config/rubber/rubber-munin.yml +8 -0
  90. data/generators/vulcanize/templates/munin/script/munin/example_mysql_query.rb +57 -0
  91. data/generators/vulcanize/templates/munin/script/munin/example_simple.rb +24 -0
  92. data/generators/vulcanize/templates/munin/templates.yml +1 -0
  93. data/generators/vulcanize/templates/mysql/config/rubber/common/database.yml +11 -0
  94. data/generators/vulcanize/templates/mysql/config/rubber/deploy-mysql.rb +156 -0
  95. data/generators/vulcanize/templates/mysql/config/rubber/role/db/crontab +14 -0
  96. data/generators/vulcanize/templates/mysql/config/rubber/role/db/monit-mysql.cnf +10 -0
  97. data/generators/vulcanize/templates/mysql/config/rubber/role/db/my.cnf +167 -0
  98. data/generators/vulcanize/templates/mysql/config/rubber/role/mysql_slave/mysql_slave_munin_plugin +51 -0
  99. data/generators/vulcanize/templates/mysql/config/rubber/rubber-mysql.yml +46 -0
  100. data/generators/vulcanize/templates/mysql/templates.yml +1 -0
  101. data/generators/vulcanize/templates/mysql_cluster/config/rubber/common/mysql_cluster_migrations.rb +13 -0
  102. data/generators/vulcanize/templates/mysql_cluster/config/rubber/deploy-mysql_cluster.rb +173 -0
  103. data/generators/vulcanize/templates/mysql_cluster/config/rubber/role/mysql_data/my.cnf +15 -0
  104. data/generators/vulcanize/templates/mysql_cluster/config/rubber/role/mysql_mgm/ndb_mgmd.cnf +39 -0
  105. data/generators/vulcanize/templates/mysql_cluster/config/rubber/role/mysql_sql/monit-mysql_cluster_sql.cnf +10 -0
  106. data/generators/vulcanize/templates/mysql_cluster/config/rubber/role/mysql_sql/my.cnf +23 -0
  107. data/generators/vulcanize/templates/mysql_cluster/config/rubber/rubber-mysql_cluster.yml +32 -0
  108. data/generators/vulcanize/templates/mysql_cluster/templates.yml +1 -0
  109. data/generators/vulcanize/templates/mysql_proxy/config/rubber/common/database.yml +16 -0
  110. data/generators/vulcanize/templates/mysql_proxy/config/rubber/common/monit-mysql_proxy.cnf +10 -0
  111. data/generators/vulcanize/templates/mysql_proxy/config/rubber/common/mysql-proxy +153 -0
  112. data/generators/vulcanize/templates/mysql_proxy/config/rubber/common/mysql-proxy.conf +10 -0
  113. data/generators/vulcanize/templates/mysql_proxy/config/rubber/common/mysql-proxy.lua +5 -0
  114. data/generators/vulcanize/templates/mysql_proxy/config/rubber/deploy-mysql_proxy.rb +52 -0
  115. data/generators/vulcanize/templates/mysql_proxy/config/rubber/rubber-mysql_proxy.yml +11 -0
  116. data/generators/vulcanize/templates/mysql_proxy/templates.yml +1 -0
  117. data/generators/vulcanize/templates/nginx/config/rubber/deploy-nginx.rb +45 -0
  118. data/generators/vulcanize/templates/nginx/config/rubber/role/nginx/crontab +9 -0
  119. data/generators/vulcanize/templates/nginx/config/rubber/role/nginx/monit-nginx.conf +8 -0
  120. data/generators/vulcanize/templates/nginx/config/rubber/role/nginx/nginx.conf +42 -0
  121. data/generators/vulcanize/templates/nginx/config/rubber/role/web_tools/nginx-tools.conf +55 -0
  122. data/generators/vulcanize/templates/nginx/config/rubber/role/web_tools/tools-index.html +30 -0
  123. data/generators/vulcanize/templates/nginx/config/rubber/role/web_tools/tools-nginx.auth +7 -0
  124. data/generators/vulcanize/templates/nginx/config/rubber/rubber-nginx.yml +10 -0
  125. data/generators/vulcanize/templates/nginx/templates.yml +1 -0
  126. data/generators/vulcanize/templates/passenger/config/rubber/deploy-passenger.rb +37 -0
  127. data/generators/vulcanize/templates/passenger/config/rubber/role/passenger/munin-passenger-memory.conf +34 -0
  128. data/generators/vulcanize/templates/passenger/config/rubber/role/passenger/munin-passenger-sudoers.conf +7 -0
  129. data/generators/vulcanize/templates/passenger/config/rubber/role/passenger/munin-passenger.conf +47 -0
  130. data/generators/vulcanize/templates/passenger/config/rubber/role/passenger/passenger-apache-vhost.conf +46 -0
  131. data/generators/vulcanize/templates/passenger/config/rubber/role/passenger/passenger.conf +10 -0
  132. data/generators/vulcanize/templates/passenger/config/rubber/rubber-passenger.yml +15 -0
  133. data/generators/vulcanize/templates/passenger/templates.yml +3 -0
  134. data/generators/vulcanize/templates/redis/config/rubber/deploy-redis.rb +36 -0
  135. data/generators/vulcanize/templates/redis/config/rubber/role/redis/crontab +8 -0
  136. data/generators/vulcanize/templates/redis/config/rubber/role/redis/monit-redis.conf +9 -0
  137. data/generators/vulcanize/templates/redis/config/rubber/role/redis/redis.conf +141 -0
  138. data/generators/vulcanize/templates/redis/config/rubber/rubber-redis.yml +4 -0
  139. data/generators/vulcanize/templates/redis/templates.yml +1 -0
  140. data/generators/vulcanize/templates/resque/config/rubber/deploy-resque-worker-default.rb +38 -0
  141. data/generators/vulcanize/templates/resque/config/rubber/deploy-resque.rb +39 -0
  142. data/generators/vulcanize/templates/resque/config/rubber/role/resque_worker_default/monit-resque_worker_default.conf +19 -0
  143. data/generators/vulcanize/templates/resque/config/rubber/rubber-resque.yml +10 -0
  144. data/generators/vulcanize/templates/resque/templates.yml +3 -0
  145. data/generators/vulcanize/templates/sphinx/config/rubber/common/sphinx.yml +46 -0
  146. data/generators/vulcanize/templates/sphinx/config/rubber/deploy-sphinx.rb +112 -0
  147. data/generators/vulcanize/templates/sphinx/config/rubber/role/sphinx/crontab +11 -0
  148. data/generators/vulcanize/templates/sphinx/config/rubber/role/sphinx/monit-sphinx.conf +10 -0
  149. data/generators/vulcanize/templates/sphinx/config/rubber/rubber-sphinx.yml +6 -0
  150. data/generators/vulcanize/templates/sphinx/templates.yml +1 -0
  151. data/generators/vulcanize/vulcanize_generator.rb +67 -0
  152. data/lib/capistrano/hostcmd.rb +12 -0
  153. data/lib/rubber.rb +38 -0
  154. data/lib/rubber/capistrano.rb +1 -0
  155. data/lib/rubber/cloud.rb +13 -0
  156. data/lib/rubber/cloud/aws.rb +334 -0
  157. data/lib/rubber/cloud/base.rb +16 -0
  158. data/lib/rubber/configuration.rb +47 -0
  159. data/lib/rubber/dns.rb +13 -0
  160. data/lib/rubber/dns/base.rb +84 -0
  161. data/lib/rubber/dns/dyndns.rb +78 -0
  162. data/lib/rubber/dns/nettica.rb +117 -0
  163. data/lib/rubber/dns/zerigo.rb +174 -0
  164. data/lib/rubber/environment.rb +169 -0
  165. data/lib/rubber/generator.rb +197 -0
  166. data/lib/rubber/instance.rb +166 -0
  167. data/lib/rubber/recipes/rubber.rb +89 -0
  168. data/lib/rubber/recipes/rubber/bundles.rb +28 -0
  169. data/lib/rubber/recipes/rubber/deploy.rb +90 -0
  170. data/lib/rubber/recipes/rubber/instances.rb +393 -0
  171. data/lib/rubber/recipes/rubber/load_balancers.rb +44 -0
  172. data/lib/rubber/recipes/rubber/security_groups.rb +189 -0
  173. data/lib/rubber/recipes/rubber/setup.rb +457 -0
  174. data/lib/rubber/recipes/rubber/spot_requests.rb +17 -0
  175. data/lib/rubber/recipes/rubber/static_ips.rb +107 -0
  176. data/lib/rubber/recipes/rubber/utils.rb +203 -0
  177. data/lib/rubber/recipes/rubber/volumes.rb +264 -0
  178. data/lib/rubber/tasks/rubber.rb +279 -0
  179. data/lib/rubber/util.rb +37 -0
  180. data/rails/init.rb +9 -0
  181. data/test/environment_test.rb +133 -0
  182. data/test/generator_test.rb +323 -0
  183. data/test/instance_test.rb +93 -0
  184. data/test/test_helper.rb +8 -0
  185. data/test/util_test.rb +16 -0
  186. metadata +298 -0
@@ -0,0 +1,44 @@
1
+ namespace :rubber do
2
+
3
+ #desc <<-DESC
4
+ # Sets up the network load balancers
5
+ #DESC
6
+ #required_task :setup_load_balancers do
7
+ # setup_load_balancers()
8
+ #end
9
+ #
10
+ #desc <<-DESC
11
+ # Describes the network load balancers
12
+ #DESC
13
+ #required_task :describe_load_balancers do
14
+ # lbs = cloud.describe_load_balancers()
15
+ # pp lbs
16
+ #end
17
+
18
+ def setup_load_balancers
19
+ # OPTIONAL: Automatically provision and assign instances to a Cloud provided
20
+ # load balancer.
21
+ #load_balancers:
22
+ # my_lb_name:
23
+ # listeners:
24
+ # - protocol: http
25
+ # port: 80
26
+ # instance_port: 8080
27
+ # - protocol: tcp
28
+ # port: 443
29
+ # instance_port: 8080
30
+ # target_roles: [app]
31
+ #
32
+ #isolate_load_balancers: true
33
+
34
+
35
+
36
+ # get remote lbs
37
+ # for each local not in remote, add it
38
+ # get all zones for all instances for roles, and make sure in lb
39
+ # warn if lb not balanced (count of instances per zone is equal)
40
+ # for each local that is in remote, sync listeners and zones
41
+ # for each remote not in local, remove it
42
+ end
43
+
44
+ end
@@ -0,0 +1,189 @@
1
+ namespace :rubber do
2
+
3
+ desc <<-DESC
4
+ Sets up the network security groups
5
+ All defined groups will be created, and any not defined will be removed.
6
+ Likewise, rules within a group will get created, and those not will be removed
7
+ DESC
8
+ required_task :setup_security_groups do
9
+ setup_security_groups()
10
+ end
11
+
12
+ desc <<-DESC
13
+ Describes the network security groups
14
+ DESC
15
+ required_task :describe_security_groups do
16
+ groups = cloud.describe_security_groups()
17
+ groups.each do |group|
18
+ puts "#{group[:name]}, #{group[:description]}"
19
+ group[:permissions].each do |perm|
20
+ puts " protocol: #{perm[:protocol]}"
21
+ puts " from_port: #{perm[:from_port]}"
22
+ puts " to_port: #{perm[:to_port]}"
23
+ puts " source_groups: #{perm[:source_groups].collect {|g| g[:name]}.join(", ") }" if perm[:source_groups]
24
+ puts " source_ips: #{perm[:source_ips].join(", ") }" if perm[:source_ips]
25
+ puts "\n"
26
+ end if group[:permissions]
27
+ puts "\n"
28
+ end
29
+ end
30
+
31
+
32
+ def get_assigned_security_groups(host=nil, roles=[])
33
+ env = rubber_cfg.environment.bind(roles, host)
34
+ security_groups = env.assigned_security_groups
35
+ if env.auto_security_groups
36
+ security_groups << host
37
+ security_groups += roles
38
+ end
39
+ security_groups = security_groups.uniq.compact.reject {|x| x.empty? }
40
+ security_groups = security_groups.collect {|x| isolate_group_name(x) } if env.isolate_security_groups
41
+ return security_groups
42
+ end
43
+
44
+ def setup_security_groups(host=nil, roles=[])
45
+ env = rubber_cfg.environment.bind(roles, host)
46
+ security_group_defns = env.security_groups
47
+ if env.auto_security_groups
48
+ sghosts = (rubber_instances.collect{|ic| ic.name } + [host]).uniq.compact
49
+ sgroles = (rubber_instances.all_roles + roles).uniq.compact
50
+ security_group_defns = inject_auto_security_groups(security_group_defns, sghosts, sgroles)
51
+ sync_security_groups(security_group_defns)
52
+ else
53
+ sync_security_groups(security_group_defns)
54
+ end
55
+ end
56
+
57
+ def inject_auto_security_groups(groups, hosts, roles)
58
+ hosts.each do |name|
59
+ group_name = name
60
+ groups[group_name] ||= {'description' => "Rubber automatic security group for host: #{name}", 'rules' => []}
61
+ end
62
+ roles.each do |name|
63
+ group_name = name
64
+ groups[group_name] ||= {'description' => "Rubber automatic security group for role: #{name}", 'rules' => []}
65
+ end
66
+ return groups
67
+ end
68
+
69
+ def isolate_prefix
70
+ return "#{rubber_env.app_name}_#{RUBBER_ENV}_"
71
+ end
72
+
73
+ def isolate_group_name(group_name)
74
+ new_name = "#{isolate_prefix}#{group_name}"
75
+ return new_name
76
+ end
77
+
78
+ def isolate_groups(groups)
79
+ renamed = {}
80
+ groups.each do |name, group|
81
+ new_name = name =~ /^#{isolate_prefix}/ ? name : isolate_group_name(name)
82
+ new_group = Marshal.load(Marshal.dump(group))
83
+ new_group['rules'].each do |rule|
84
+ old_ref_name = rule['source_group_name']
85
+ if old_ref_name && old_ref_name !~ /^#{isolate_prefix}/
86
+ rule['source_group_name'] = isolate_group_name(old_ref_name)
87
+ end
88
+ end
89
+ renamed[new_name] = new_group
90
+ end
91
+ return renamed
92
+ end
93
+
94
+ def sync_security_groups(groups)
95
+ return unless groups
96
+
97
+ groups = Rubber::Util::stringify(groups)
98
+ groups = isolate_groups(groups) if rubber_env.isolate_security_groups
99
+ group_keys = groups.keys.clone()
100
+
101
+ # For each group that does already exist in cloud
102
+ cloud_groups = cloud.describe_security_groups()
103
+ cloud_groups.each do |cloud_group|
104
+ group_name = cloud_group[:name]
105
+
106
+ # skip those groups that don't belong to this project/env
107
+ next if rubber_env.isolate_security_groups && group_name !~ /^#{isolate_prefix}/
108
+
109
+ if group_keys.delete(group_name)
110
+ # sync rules
111
+ logger.debug "Security Group already in cloud, syncing rules: #{group_name}"
112
+ group = groups[group_name]
113
+ rules = group['rules'].clone
114
+ rule_maps = []
115
+
116
+ # first collect the rule maps from the request (group/user pairs are duplicated for tcp/udp/icmp,
117
+ # so we need to do this up frnot and remove duplicates before checking against the local rubber rules)
118
+ cloud_group[:permissions].each do |rule|
119
+ if rule[:source_groups]
120
+ rule.source_groups.each do |source_group|
121
+ rule_map = {:source_group_name => source_group[:name], :source_group_account => source_group[:account]}
122
+ rule_map = Rubber::Util::stringify(rule_map)
123
+ rule_maps << rule_map unless rule_maps.include?(rule_map)
124
+ end
125
+ else
126
+ rule_map = Rubber::Util::stringify(rule)
127
+ rule_maps << rule_map unless rule_maps.include?(rule_map)
128
+ end
129
+ end if cloud_group[:permissions]
130
+ # For each rule, if it exists, do nothing, otherwise remove it as its no longer defined locally
131
+ rule_maps.each do |rule_map|
132
+ if rules.delete(rule_map)
133
+ # rules match, don't need to do anything
134
+ # logger.debug "Rule in sync: #{rule_map.inspect}"
135
+ else
136
+ # rules don't match, remove them from cloud and re-add below
137
+ answer = Capistrano::CLI.ui.ask("Rule '#{rule_map.inspect}' exists in cloud, but not locally, remove from cloud? [y/N]?: ")
138
+ rule_map = Rubber::Util::symbolize_keys(rule_map)
139
+ if rule_map[:source_group_name]
140
+ cloud.remove_security_group_rule(group_name, nil, nil, nil, {:name => rule_map[:source_group_name], :account => rule_map[:source_group_account]})
141
+ else
142
+ rule_map[:source_ips].each do |source_ip|
143
+ cloud.remove_security_group_rule(group_name, rule_map[:protocol], rule_map[:from_port], rule_map[:to_port], source_ip)
144
+ end if rule_map[:source_ips] && answer =~ /^y/
145
+ end
146
+ end
147
+ end
148
+
149
+ rules.each do |rule_map|
150
+ # create non-existing rules
151
+ logger.debug "Missing rule, creating: #{rule_map.inspect}"
152
+ rule_map = Rubber::Util::symbolize_keys(rule_map)
153
+ if rule_map[:source_group_name]
154
+ cloud.add_security_group_rule(group_name, nil, nil, nil, {:name => rule_map[:source_group_name], :account => rule_map[:source_group_account]})
155
+ else
156
+ rule_map[:source_ips].each do |source_ip|
157
+ cloud.add_security_group_rule(group_name, rule_map[:protocol], rule_map[:from_port], rule_map[:to_port], source_ip)
158
+ end if rule_map[:source_ips]
159
+ end
160
+ end
161
+ else
162
+ # delete group
163
+ answer = Capistrano::CLI.ui.ask("Security group '#{group_name}' exists in cloud but not locally, remove from cloud? [y/N]: ")
164
+ cloud.destroy_security_group(group_name) if answer =~ /^y/
165
+ end
166
+ end
167
+
168
+ # For each group that didnt already exist in cloud
169
+ group_keys.each do |group_name|
170
+ group = groups[group_name]
171
+ logger.debug "Creating new security group: #{group_name}"
172
+ # create each group
173
+ cloud.create_security_group(group_name, group['description'])
174
+ # create rules for group
175
+ group['rules'].each do |rule_map|
176
+ logger.debug "Creating new rule: #{rule_map.inspect}"
177
+ rule_map = Rubber::Util::symbolize_keys(rule_map)
178
+ if rule_map[:source_group_name]
179
+ cloud.add_security_group_rule(group_name, nil, nil, nil, {:name => rule_map[:source_group_name], :account => rule_map[:source_group_account]})
180
+ else
181
+ rule_map[:source_ips].each do |source_ip|
182
+ cloud.add_security_group_rule(group_name, rule_map[:protocol], rule_map[:from_port], rule_map[:to_port], source_ip)
183
+ end if rule_map[:source_ips]
184
+ end
185
+ end
186
+ end
187
+ end
188
+
189
+ end
@@ -0,0 +1,457 @@
1
+ namespace :rubber do
2
+
3
+ desc <<-DESC
4
+ Bootstraps instances by setting timezone, installing packages and gems
5
+ DESC
6
+ task :bootstrap do
7
+ set_timezone
8
+ link_bash
9
+ upgrade_packages
10
+ install_packages
11
+ setup_volumes
12
+ setup_gem_sources
13
+ install_gems
14
+ deploy.setup
15
+ end
16
+
17
+ desc <<-DESC
18
+ Sets up aliases for instance hostnames based on contents of instance.yml.
19
+ Generates /etc/hosts for local/remote machines and sets hostname on
20
+ remote instances, and sets values in dynamic dns entries
21
+ DESC
22
+ required_task :setup_aliases do
23
+ setup_local_aliases
24
+ setup_remote_aliases
25
+ setup_dns_aliases
26
+ end
27
+
28
+ desc <<-DESC
29
+ Sets up local aliases for instance hostnames based on contents of instance.yml.
30
+ Generates/etc/hosts for local machine
31
+ DESC
32
+ required_task :setup_local_aliases do
33
+ hosts_file = '/etc/hosts'
34
+
35
+ # Generate /etc/hosts contents for the local machine from instance config
36
+ delim = "## rubber config #{rubber_env.domain} #{RUBBER_ENV}"
37
+ local_hosts = delim + "\n"
38
+ rubber_instances.each do |ic|
39
+ # don't add unqualified hostname in local hosts file since user may be
40
+ # managing multiple domains with same aliases
41
+ hosts_data = [ic.full_name, ic.external_host, ic.internal_host].join(' ')
42
+ local_hosts << ic.external_ip << ' ' << hosts_data << "\n"
43
+ end
44
+ local_hosts << delim << "\n"
45
+
46
+ # Write out the hosts file for this machine, use sudo
47
+ filtered = File.read(hosts_file).gsub(/^#{delim}.*^#{delim}\n?/m, '')
48
+ logger.info "Writing out aliases into local machines #{hosts_file}, sudo access needed"
49
+ Rubber::Util::sudo_open(hosts_file, 'w') do |f|
50
+ f.write(filtered)
51
+ f.write(local_hosts)
52
+ end
53
+ end
54
+
55
+ desc <<-DESC
56
+ Sets up aliases in dynamic dns provider for instance hostnames based on contents of instance.yml.
57
+ DESC
58
+ required_task :setup_dns_aliases do
59
+ rubber_instances.each do |ic|
60
+ update_dyndns(ic)
61
+ end
62
+ end
63
+
64
+ desc <<-DESC
65
+ Sets up the additional dns records supplied in the dns_records config in rubber.yml
66
+ DESC
67
+ required_task :setup_dns_records do
68
+ records = rubber_env.dns_records
69
+ if records && rubber_env.dns_provider
70
+ provider = Rubber::Dns::get_provider(rubber_env.dns_provider, rubber_env)
71
+
72
+ # collect the round robin records (those with the same host/domain/type)
73
+ rr_records = []
74
+ records.each_with_index do |record, i|
75
+ m = records.find_all {|r| record['host'] == r['host'] && record['domain'] == r['domain'] && record['type'] == r['type']}
76
+ m = m.sort {|a,b| a.object_id <=> b.object_id}
77
+ rr_records << m if m.size > 1 && ! rr_records.include?(m)
78
+ end
79
+
80
+ # simple records are those that aren't round robin ones
81
+ simple_records = records - rr_records.flatten
82
+
83
+ # for each simple record, create or update as necessary
84
+ simple_records.each do |record|
85
+ matching = provider.find_host_records(:host => record['host'], :domain =>record['domain'], :type => record['type'])
86
+ if matching.size > 1
87
+ msg = "Multiple records in dns provider, but not in rubber.yml\n"
88
+ msg << "Round robin records need to be in both, or neither.\n"
89
+ msg << "Please fix manually:\n"
90
+ msg << matching.pretty_inspect
91
+ fatal(msg)
92
+ end
93
+
94
+ record = provider.setup_opts(record)
95
+ if matching.size == 1
96
+ match = matching.first
97
+ if provider.host_records_equal?(record, match)
98
+ logger.info "Simple dns record already up to date: #{record[:host]}.#{record[:domain]}:#{record[:type]} => #{record[:data]}"
99
+ else
100
+ logger.info "Updating simple dns record: #{record[:host]}.#{record[:domain]}:#{record[:type]} => #{record[:data]}"
101
+ provider.update_host_record(match, record)
102
+ end
103
+ else
104
+ logger.info "Creating simple dns record: #{record[:host]}.#{record[:domain]}:#{record[:type]} => #{record[:data]}"
105
+ provider.create_host_record(record)
106
+ end
107
+ end
108
+
109
+ # group round robin records
110
+ rr_records.each do |rr_group|
111
+ host = rr_group.first['host']
112
+ domain = rr_group.first['domain']
113
+ type = rr_group.first['type']
114
+ matching = provider.find_host_records(:host => host, :domain => domain, :type => type)
115
+
116
+ # remove from consideration the local records that are the same as remote ones
117
+ matching.clone.each do |r|
118
+ rr_group.delete_if {|rg| provider.host_records_equal?(r, rg) }
119
+ matching.delete_if {|rg| provider.host_records_equal?(r, rg) }
120
+ end
121
+ if rr_group.size == 0 && matching.size == 0
122
+ logger.info "Round robin dns records already up to date: #{host}.#{domain}:#{type}"
123
+ end
124
+
125
+ # create the local records that don't exist remotely
126
+ rr_group.each do |r|
127
+ r = provider.setup_opts(r)
128
+ logger.info "Creating round robin dns record: #{r[:host]}.#{r[:domain]}:#{r[:type]} => #{r[:data]}"
129
+ provider.create_host_record(r)
130
+ end
131
+
132
+ # remove the remote records that don't exist locally
133
+ matching.each do |r|
134
+ logger.info "Removing round robin dns record: #{r[:host]}.#{r[:domain]}:#{r[:type]} => #{r[:data]}"
135
+ provider.destroy_host_record(r)
136
+ end
137
+ end
138
+ end
139
+ end
140
+
141
+ desc <<-DESC
142
+ Sets up aliases for instance hostnames based on contents of instance.yml.
143
+ Generates /etc/hosts for remote machines and sets hostname on remote instances
144
+ DESC
145
+ task :setup_remote_aliases do
146
+ hosts_file = '/etc/hosts'
147
+
148
+ # Generate /etc/hosts contents for the remote instance from instance config
149
+ delim = "## rubber config"
150
+ delim = "#{delim} #{RUBBER_ENV}"
151
+ remote_hosts = delim + "\n"
152
+ rubber_instances.each do |ic|
153
+ hosts_data = [ic.name, ic.full_name, ic.external_host, ic.internal_host].join(' ')
154
+ remote_hosts << ic.internal_ip << ' ' << hosts_data << "\n"
155
+ end
156
+ remote_hosts << delim << "\n"
157
+ if rubber_instances.size > 0
158
+ # write out the hosts file for the remote instances
159
+ # NOTE that we use "capture" to get the existing hosts
160
+ # file, which only grabs the hosts file from the first host
161
+ filtered = (capture "cat #{hosts_file}").gsub(/^#{delim}.*^#{delim}\n?/m, '')
162
+ filtered = filtered + remote_hosts
163
+ # Put the generated hosts back on remote instance
164
+ put filtered, hosts_file
165
+
166
+ # Setup hostname on instance so shell, etcs have nice display
167
+ sudo "echo $CAPISTRANO:HOST$ > /etc/hostname && hostname $CAPISTRANO:HOST$"
168
+ end
169
+
170
+ # TODO
171
+ # /etc/resolv.conf to add search domain
172
+ # ~/.ssh/options to setup user/host/key aliases
173
+ end
174
+
175
+ desc <<-DESC
176
+ Update to the newest versions of all packages/gems.
177
+ DESC
178
+ task :update do
179
+ upgrade_packages
180
+ update_gems
181
+ end
182
+
183
+ desc <<-DESC
184
+ Upgrade to the newest versions of all Ubuntu packages.
185
+ DESC
186
+ task :upgrade_packages do
187
+ package_helper(true)
188
+ end
189
+
190
+ desc <<-DESC
191
+ Upgrade to the newest versions of all rubygems.
192
+ DESC
193
+ task :update_gems do
194
+ gem_helper(true)
195
+ end
196
+
197
+ desc <<-DESC
198
+ Install extra packages and gems.
199
+ DESC
200
+ task :install do
201
+ install_packages
202
+ install_gems
203
+ end
204
+
205
+ desc <<-DESC
206
+ Install Ubuntu packages. Set 'packages' in rubber.yml to \
207
+ be an array of strings.
208
+ DESC
209
+ task :install_packages do
210
+ package_helper(false)
211
+ end
212
+
213
+ desc <<-DESC
214
+ Install ruby gems. Set 'gems' in rubber.yml to \
215
+ be an array of strings.
216
+ DESC
217
+ task :install_gems do
218
+ gem_helper(false)
219
+ end
220
+
221
+ desc <<-DESC
222
+ Install ruby gems defined in the rails environment.rb
223
+ DESC
224
+ after "rubber:config", "rubber:install_rails_gems" if Rubber::Util.is_rails?
225
+ task :install_rails_gems do
226
+ sudo "sh -c 'cd #{current_path} && RAILS_ENV=#{RUBBER_ENV} rake gems:install'"
227
+ end
228
+
229
+ desc <<-DESC
230
+ Convenience task for installing your defined set of ruby gems locally.
231
+ DESC
232
+ required_task :install_local_gems do
233
+ fatal("install_local_gems can only be run in development") if RUBBER_ENV != 'development'
234
+ env = rubber_cfg.environment.bind(rubber_cfg.environment.known_roles)
235
+ gems = env['gems']
236
+ expanded_gem_list = []
237
+ gems.each do |gem_spec|
238
+ if gem_spec.is_a?(Array)
239
+ expanded_gem_list << "#{gem_spec[0]}:#{gem_spec[1]}"
240
+ else
241
+ expanded_gem_list << gem_spec
242
+ end
243
+ end
244
+ expanded_gem_list = expanded_gem_list.join(' ')
245
+
246
+ logger.info "Installing gems:#{expanded_gem_list}"
247
+ open("/tmp/gem_helper", "w") {|f| f.write(gem_helper_script)}
248
+ system "sh /tmp/gem_helper install #{expanded_gem_list}"
249
+ end
250
+
251
+ desc <<-DESC
252
+ Setup ruby gems sources. Set 'gemsources' in rubber.yml to \
253
+ be an array of URI strings.
254
+ DESC
255
+ task :setup_gem_sources do
256
+ if rubber_env.gemsources
257
+ script = prepare_script 'gem_sources_helper', <<-'ENDSCRIPT'
258
+ ruby - $@ <<-'EOF'
259
+
260
+ sources = ARGV
261
+
262
+ installed = []
263
+ `gem sources -l`.grep(/^[^*]/) do |line|
264
+ line = line.strip
265
+ installed << line if line.size > 0
266
+ end
267
+
268
+ to_install = sources - installed
269
+ to_remove = installed - sources
270
+
271
+ if to_install.size > 0
272
+ to_install.each do |source|
273
+ system "gem sources -a #{source}"
274
+ fail "Unable to add gem sources" if $?.exitstatus > 0
275
+ end
276
+ end
277
+ if to_remove.size > 0
278
+ to_remove.each do |source|
279
+ system "gem sources -r #{source}"
280
+ fail "Unable to remove gem sources" if $?.exitstatus > 0
281
+ end
282
+ end
283
+
284
+ 'EOF'
285
+ ENDSCRIPT
286
+
287
+ sudo "sh #{script} #{rubber_env.gemsources.join(' ')}"
288
+ end
289
+ end
290
+
291
+ desc <<-DESC
292
+ The ubuntu has /bin/sh linking to dash instead of bash, fix this
293
+ You can override this task if you don't want this to happen
294
+ DESC
295
+ task :link_bash do
296
+ sudo("ln -sf /bin/bash /bin/sh")
297
+ end
298
+
299
+ desc <<-DESC
300
+ Set the timezone using the value of the variable named timezone. \
301
+ Valid options for timezone can be determined by the contents of \
302
+ /usr/share/zoneinfo, which can be seen here: \
303
+ http://packages.ubuntu.com/cgi-bin/search_contents.pl?searchmode=filelist&word=tzdata&version=gutsy&arch=all&page=1&number=all \
304
+ Remove 'usr/share/zoneinfo/' from the filename, and use the last \
305
+ directory and file as the value. For example 'Africa/Abidjan' or \
306
+ 'posix/GMT' or 'Canada/Eastern'.
307
+ DESC
308
+ task :set_timezone do
309
+ opts = get_host_options('timezone')
310
+ sudo "bash -c 'echo $CAPISTRANO:VAR$ > /etc/timezone'", opts
311
+ sudo "cp /usr/share/zoneinfo/$CAPISTRANO:VAR$ /etc/localtime", opts
312
+ # restart syslog so that times match timezone
313
+ sudo "/etc/init.d/sysklogd restart"
314
+ end
315
+
316
+ def update_dyndns(instance_item)
317
+ env = rubber_cfg.environment.bind(instance_item.role_names, instance_item.name)
318
+ if env.dns_provider
319
+ provider = Rubber::Dns::get_provider(env.dns_provider, env)
320
+ provider.update(instance_item.name, instance_item.external_ip)
321
+ end
322
+ end
323
+
324
+ def destroy_dyndns(instance_item)
325
+ env = rubber_cfg.environment.bind(instance_item.role_names, instance_item.name)
326
+ if env.dns_provider
327
+ provider = Rubber::Dns::get_provider(env.dns_provider, env)
328
+ provider.destroy(instance_item.name)
329
+ end
330
+ end
331
+
332
+ def package_helper(upgrade=false)
333
+ opts = get_host_options('packages') do |pkg_list|
334
+ expanded_pkg_list = []
335
+ pkg_list.each do |pkg_spec|
336
+ if pkg_spec.is_a?(Array)
337
+ expanded_pkg_list << "#{pkg_spec[0]}=#{pkg_spec[1]}"
338
+ else
339
+ expanded_pkg_list << pkg_spec
340
+ end
341
+ end
342
+ expanded_pkg_list.join(' ')
343
+ end
344
+
345
+ sudo "apt-get -q update"
346
+ if upgrade
347
+ sudo "/bin/sh -c 'export DEBIAN_FRONTEND=noninteractive; apt-get -q -y --force-yes dist-upgrade'"
348
+ else
349
+ sudo "/bin/sh -c 'export DEBIAN_FRONTEND=noninteractive; apt-get -q -y --force-yes install $CAPISTRANO:VAR$'", opts
350
+ end
351
+ end
352
+
353
+ def custom_package(url_base, name, ver, install_test)
354
+ rubber.run_script "install_#{name}", <<-ENDSCRIPT
355
+ if [[ #{install_test} ]]; then
356
+ arch=`uname -m`
357
+ if [ "$arch" = "x86_64" ]; then
358
+ src="#{url_base}/#{name}_#{ver}_amd64.deb"
359
+ else
360
+ src="#{url_base}/#{name}_#{ver}_i386.deb"
361
+ fi
362
+ src_file="${src##*/}"
363
+ wget -qP /tmp ${src}
364
+ dpkg -i /tmp/${src_file}
365
+ fi
366
+ ENDSCRIPT
367
+ end
368
+
369
+ def handle_gem_prompt(ch, data, str)
370
+ ch[:data] ||= ""
371
+ ch[:data] << data
372
+ if data =~ />\s*$/
373
+ logger.info data
374
+ logger.info "The gem command is asking for a number:"
375
+ choice = STDIN.gets
376
+ ch.send_data(choice)
377
+ else
378
+ logger.info data
379
+ end
380
+ end
381
+
382
+ # Rubygems always installs even if the gem is already installed
383
+ # When providing versions, rubygems fails unless versions are provided for all gems
384
+ # This helper script works around these issues by installing gems only if they
385
+ # aren't already installed, and separates versioned/unversioned into two separate
386
+ # calls to rubygems
387
+ #
388
+ set :gem_helper_script, <<-'ENDSCRIPT'
389
+ ruby - $@ <<-'EOF'
390
+
391
+ gem_cmd = ARGV[0]
392
+ gems = ARGV[1..-1]
393
+ cmd = "gem #{gem_cmd} --no-rdoc --no-ri"
394
+
395
+ to_install = {}
396
+ to_install_ver = {}
397
+ # gem list passed in, possibly with versions, as "gem1 gem2:1.2 gem3"
398
+ gems.each do |gem_spec|
399
+ parts = gem_spec.split(':')
400
+ if parts[1]
401
+ to_install_ver[parts[0]] = parts[1]
402
+ else
403
+ to_install[parts[0]] = true
404
+ end
405
+ end
406
+
407
+ installed = {}
408
+ `gem list --local`.each do |line|
409
+ parts = line.scan(/(.*) \((.*)\)/).first
410
+ next unless parts && parts.size == 2
411
+ installed[parts[0]] = parts[1].split(",")
412
+ end
413
+
414
+ to_install.delete_if {|g, v| installed.has_key?(g) } if gem_cmd == 'install'
415
+ to_install_ver.delete_if {|g, v| installed.has_key?(g) && installed[g].include?(v) }
416
+
417
+ # rubygems can only do asingle versioned gem at a time so we need
418
+ # to do the two groups separately
419
+ # install versioned ones first so unversioned don't pull in a newer version
420
+ to_install_ver.each do |g, v|
421
+ system "#{cmd} #{g} -v #{v}"
422
+ fail "Unable to install versioned gem #{g}:#{v}" if $?.exitstatus > 0
423
+ end
424
+ if to_install.size > 0
425
+ gem_list = to_install.keys.join(' ')
426
+ system "#{cmd} #{gem_list}"
427
+ fail "Unable to install gems" if $?.exitstatus > 0
428
+ end
429
+
430
+ 'EOF'
431
+ ENDSCRIPT
432
+
433
+ # Helper for installing gems,allows one to respond to prompts
434
+ def gem_helper(update=false)
435
+ cmd = update ? "update" : "install"
436
+
437
+ opts = get_host_options('gems') do |gem_list|
438
+ expanded_gem_list = []
439
+ gem_list.each do |gem_spec|
440
+ if gem_spec.is_a?(Array)
441
+ expanded_gem_list << "#{gem_spec[0]}:#{gem_spec[1]}"
442
+ else
443
+ expanded_gem_list << gem_spec
444
+ end
445
+ end
446
+ expanded_gem_list.join(' ')
447
+ end
448
+
449
+ if opts.size > 0
450
+ script = prepare_script('gem_helper', gem_helper_script)
451
+ sudo "sh #{script} #{cmd} $CAPISTRANO:VAR$", opts do |ch, str, data|
452
+ handle_gem_prompt(ch, data, str)
453
+ end
454
+ end
455
+ end
456
+
457
+ end