rubber 1.0.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (163) hide show
  1. data/CHANGELOG +39 -0
  2. data/COPYING +339 -0
  3. data/README +6 -0
  4. data/TODO +11 -0
  5. data/VERSION +1 -0
  6. data/bin/vulcanize +41 -0
  7. data/generators/vulcanize/USAGE +6 -0
  8. data/generators/vulcanize/templates/apache/config/rubber/deploy-apache.rb +51 -0
  9. data/generators/vulcanize/templates/apache/config/rubber/role/apache/deflate.conf +10 -0
  10. data/generators/vulcanize/templates/apache/config/rubber/role/apache/expires.conf +9 -0
  11. data/generators/vulcanize/templates/apache/config/rubber/role/apache/headers.conf +6 -0
  12. data/generators/vulcanize/templates/apache/config/rubber/role/apache/monit-apache.conf +8 -0
  13. data/generators/vulcanize/templates/apache/config/rubber/role/apache/ports.conf +5 -0
  14. data/generators/vulcanize/templates/apache/config/rubber/role/apache/setenvif.conf +52 -0
  15. data/generators/vulcanize/templates/apache/config/rubber/role/web_tools/tools-apache-vhost.conf +62 -0
  16. data/generators/vulcanize/templates/apache/config/rubber/role/web_tools/tools-apache.auth +7 -0
  17. data/generators/vulcanize/templates/apache/config/rubber/role/web_tools/tools-index.html +30 -0
  18. data/generators/vulcanize/templates/apache/config/rubber/rubber-apache.yml +7 -0
  19. data/generators/vulcanize/templates/apache/templates.yml +1 -0
  20. data/generators/vulcanize/templates/base/Capfile +14 -0
  21. data/generators/vulcanize/templates/base/config/deploy.rb +55 -0
  22. data/generators/vulcanize/templates/base/config/rubber/common/crontab +16 -0
  23. data/generators/vulcanize/templates/base/config/rubber/common/profile.rc +9 -0
  24. data/generators/vulcanize/templates/base/config/rubber/deploy-setup.rb +104 -0
  25. data/generators/vulcanize/templates/base/config/rubber/rubber.yml +241 -0
  26. data/generators/vulcanize/templates/base/lib/tasks/rubber.rake +15 -0
  27. data/generators/vulcanize/templates/base/script/cron-rake +18 -0
  28. data/generators/vulcanize/templates/base/script/cron-runner +18 -0
  29. data/generators/vulcanize/templates/base/script/cron-sh +67 -0
  30. data/generators/vulcanize/templates/base/templates.yml +1 -0
  31. data/generators/vulcanize/templates/complete_mongrel_mysql/config/rubber/role/haproxy/haproxy-mongrel.conf +23 -0
  32. data/generators/vulcanize/templates/complete_mongrel_mysql/config/rubber/role/nginx/nginx-mongrel.conf +113 -0
  33. data/generators/vulcanize/templates/complete_mongrel_mysql/config/rubber/rubber-complete.yml +41 -0
  34. data/generators/vulcanize/templates/complete_mongrel_mysql/templates.yml +6 -0
  35. data/generators/vulcanize/templates/complete_passenger_mysql/config/rubber/role/haproxy/haproxy-passenger.conf +19 -0
  36. data/generators/vulcanize/templates/complete_passenger_mysql/config/rubber/rubber-complete.yml +40 -0
  37. data/generators/vulcanize/templates/complete_passenger_mysql/templates.yml +10 -0
  38. data/generators/vulcanize/templates/cruise/config/rubber/deploy-cruise.rb +72 -0
  39. data/generators/vulcanize/templates/cruise/config/rubber/role/cruise/cruise +40 -0
  40. data/generators/vulcanize/templates/cruise/config/rubber/role/cruise/my.cnf +165 -0
  41. data/generators/vulcanize/templates/cruise/config/rubber/role/cruise/production.rb +8 -0
  42. data/generators/vulcanize/templates/cruise/config/rubber/role/cruise/site_config.rb +76 -0
  43. data/generators/vulcanize/templates/cruise/config/rubber/role/web_tools/cruise-nginx.conf +11 -0
  44. data/generators/vulcanize/templates/cruise/config/rubber/rubber-cruise.yml +18 -0
  45. data/generators/vulcanize/templates/cruise/templates.yml +1 -0
  46. data/generators/vulcanize/templates/haproxy/config/rubber/deploy-haproxy.rb +45 -0
  47. data/generators/vulcanize/templates/haproxy/config/rubber/role/haproxy/haproxy-base.conf +26 -0
  48. data/generators/vulcanize/templates/haproxy/config/rubber/role/haproxy/haproxy-default.conf +8 -0
  49. data/generators/vulcanize/templates/haproxy/config/rubber/role/haproxy/monit-haproxy.conf +9 -0
  50. data/generators/vulcanize/templates/haproxy/config/rubber/role/haproxy/syslog-haproxy.conf +6 -0
  51. data/generators/vulcanize/templates/haproxy/config/rubber/role/haproxy/syslogd-default.conf +17 -0
  52. data/generators/vulcanize/templates/haproxy/config/rubber/role/web_tools/haproxy-nginx.conf +10 -0
  53. data/generators/vulcanize/templates/haproxy/config/rubber/rubber-haproxy.yml +7 -0
  54. data/generators/vulcanize/templates/haproxy/templates.yml +1 -0
  55. data/generators/vulcanize/templates/memcached/config/memcached.yml +28 -0
  56. data/generators/vulcanize/templates/memcached/config/rubber/common/memcached.yml +14 -0
  57. data/generators/vulcanize/templates/memcached/config/rubber/role/memcached/memcached.conf +52 -0
  58. data/generators/vulcanize/templates/memcached/config/rubber/role/memcached/memcached_munin_plugin +249 -0
  59. data/generators/vulcanize/templates/memcached/config/rubber/rubber-memcached.yml +7 -0
  60. data/generators/vulcanize/templates/memcached/templates.yml +1 -0
  61. data/generators/vulcanize/templates/minimal_mysql/templates.yml +7 -0
  62. data/generators/vulcanize/templates/minimal_nodb/templates.yml +6 -0
  63. data/generators/vulcanize/templates/mongrel/config/rubber/deploy-mongrel.rb +75 -0
  64. data/generators/vulcanize/templates/mongrel/config/rubber/role/mongrel/mongrel_cluster.yml +12 -0
  65. data/generators/vulcanize/templates/mongrel/config/rubber/role/mongrel/monit-mongrel.conf +20 -0
  66. data/generators/vulcanize/templates/mongrel/config/rubber/rubber-mongrel.yml +9 -0
  67. data/generators/vulcanize/templates/mongrel/templates.yml +1 -0
  68. data/generators/vulcanize/templates/monit/config/rubber/common/monit-default.conf +15 -0
  69. data/generators/vulcanize/templates/monit/config/rubber/common/monit.conf +251 -0
  70. data/generators/vulcanize/templates/monit/config/rubber/deploy-monit.rb +32 -0
  71. data/generators/vulcanize/templates/monit/config/rubber/role/web_tools/monit-admin-nginx.conf +10 -0
  72. data/generators/vulcanize/templates/monit/config/rubber/rubber-monit.yml +6 -0
  73. data/generators/vulcanize/templates/monit/templates.yml +1 -0
  74. data/generators/vulcanize/templates/munin/config/rubber/common/monit-munin.conf +8 -0
  75. data/generators/vulcanize/templates/munin/config/rubber/common/munin-node.conf +48 -0
  76. data/generators/vulcanize/templates/munin/config/rubber/deploy-munin.rb +46 -0
  77. data/generators/vulcanize/templates/munin/config/rubber/role/web_tools/munin-nginx.conf +8 -0
  78. data/generators/vulcanize/templates/munin/config/rubber/role/web_tools/munin-plugins.conf +31 -0
  79. data/generators/vulcanize/templates/munin/config/rubber/role/web_tools/munin.conf +80 -0
  80. data/generators/vulcanize/templates/munin/config/rubber/rubber-munin.yml +8 -0
  81. data/generators/vulcanize/templates/munin/script/munin/example_mysql_query.rb +57 -0
  82. data/generators/vulcanize/templates/munin/script/munin/example_simple.rb +24 -0
  83. data/generators/vulcanize/templates/munin/templates.yml +1 -0
  84. data/generators/vulcanize/templates/mysql/config/rubber/common/database.yml +11 -0
  85. data/generators/vulcanize/templates/mysql/config/rubber/deploy-mysql.rb +156 -0
  86. data/generators/vulcanize/templates/mysql/config/rubber/role/db/crontab +14 -0
  87. data/generators/vulcanize/templates/mysql/config/rubber/role/db/monit-mysql.cnf +10 -0
  88. data/generators/vulcanize/templates/mysql/config/rubber/role/db/my.cnf +167 -0
  89. data/generators/vulcanize/templates/mysql/config/rubber/role/mysql_slave/mysql_slave_munin_plugin +51 -0
  90. data/generators/vulcanize/templates/mysql/config/rubber/rubber-mysql.yml +46 -0
  91. data/generators/vulcanize/templates/mysql/templates.yml +1 -0
  92. data/generators/vulcanize/templates/mysql_cluster/config/rubber/common/mysql_cluster_migrations.rb +13 -0
  93. data/generators/vulcanize/templates/mysql_cluster/config/rubber/deploy-mysql_cluster.rb +173 -0
  94. data/generators/vulcanize/templates/mysql_cluster/config/rubber/role/mysql_data/my.cnf +15 -0
  95. data/generators/vulcanize/templates/mysql_cluster/config/rubber/role/mysql_mgm/ndb_mgmd.cnf +39 -0
  96. data/generators/vulcanize/templates/mysql_cluster/config/rubber/role/mysql_sql/monit-mysql_cluster_sql.cnf +10 -0
  97. data/generators/vulcanize/templates/mysql_cluster/config/rubber/role/mysql_sql/my.cnf +23 -0
  98. data/generators/vulcanize/templates/mysql_cluster/config/rubber/rubber-mysql_cluster.yml +32 -0
  99. data/generators/vulcanize/templates/mysql_cluster/templates.yml +1 -0
  100. data/generators/vulcanize/templates/mysql_proxy/config/rubber/common/database.yml +16 -0
  101. data/generators/vulcanize/templates/mysql_proxy/config/rubber/common/monit-mysql_proxy.cnf +10 -0
  102. data/generators/vulcanize/templates/mysql_proxy/config/rubber/common/mysql-proxy +153 -0
  103. data/generators/vulcanize/templates/mysql_proxy/config/rubber/common/mysql-proxy.conf +10 -0
  104. data/generators/vulcanize/templates/mysql_proxy/config/rubber/common/mysql-proxy.lua +5 -0
  105. data/generators/vulcanize/templates/mysql_proxy/config/rubber/deploy-mysql_proxy.rb +52 -0
  106. data/generators/vulcanize/templates/mysql_proxy/config/rubber/rubber-mysql_proxy.yml +11 -0
  107. data/generators/vulcanize/templates/mysql_proxy/templates.yml +1 -0
  108. data/generators/vulcanize/templates/nginx/config/rubber/deploy-nginx.rb +45 -0
  109. data/generators/vulcanize/templates/nginx/config/rubber/role/nginx/crontab +9 -0
  110. data/generators/vulcanize/templates/nginx/config/rubber/role/nginx/monit-nginx.conf +8 -0
  111. data/generators/vulcanize/templates/nginx/config/rubber/role/nginx/nginx.conf +42 -0
  112. data/generators/vulcanize/templates/nginx/config/rubber/role/web_tools/nginx-tools.conf +55 -0
  113. data/generators/vulcanize/templates/nginx/config/rubber/role/web_tools/tools-index.html +30 -0
  114. data/generators/vulcanize/templates/nginx/config/rubber/role/web_tools/tools-nginx.auth +7 -0
  115. data/generators/vulcanize/templates/nginx/config/rubber/rubber-nginx.yml +10 -0
  116. data/generators/vulcanize/templates/nginx/templates.yml +1 -0
  117. data/generators/vulcanize/templates/passenger/config/rubber/deploy-passenger.rb +37 -0
  118. data/generators/vulcanize/templates/passenger/config/rubber/role/passenger/munin-passenger-sudoers.conf +6 -0
  119. data/generators/vulcanize/templates/passenger/config/rubber/role/passenger/munin-passenger.conf +47 -0
  120. data/generators/vulcanize/templates/passenger/config/rubber/role/passenger/passenger-apache-vhost.conf +46 -0
  121. data/generators/vulcanize/templates/passenger/config/rubber/role/passenger/passenger.conf +10 -0
  122. data/generators/vulcanize/templates/passenger/config/rubber/rubber-passenger.yml +12 -0
  123. data/generators/vulcanize/templates/passenger/templates.yml +1 -0
  124. data/generators/vulcanize/templates/sphinx/config/rubber/common/sphinx.yml +46 -0
  125. data/generators/vulcanize/templates/sphinx/config/rubber/deploy-sphinx.rb +112 -0
  126. data/generators/vulcanize/templates/sphinx/config/rubber/role/sphinx/crontab +11 -0
  127. data/generators/vulcanize/templates/sphinx/config/rubber/role/sphinx/monit-sphinx.conf +10 -0
  128. data/generators/vulcanize/templates/sphinx/config/rubber/rubber-sphinx.yml +6 -0
  129. data/generators/vulcanize/templates/sphinx/templates.yml +1 -0
  130. data/generators/vulcanize/vulcanize_generator.rb +67 -0
  131. data/lib/capistrano/hostcmd.rb +12 -0
  132. data/lib/rubber.rb +38 -0
  133. data/lib/rubber/capistrano.rb +1 -0
  134. data/lib/rubber/cloud.rb +13 -0
  135. data/lib/rubber/cloud/aws.rb +305 -0
  136. data/lib/rubber/cloud/base.rb +16 -0
  137. data/lib/rubber/configuration.rb +47 -0
  138. data/lib/rubber/dns.rb +13 -0
  139. data/lib/rubber/dns/base.rb +69 -0
  140. data/lib/rubber/dns/dyndns.rb +63 -0
  141. data/lib/rubber/dns/nettica.rb +73 -0
  142. data/lib/rubber/dns/zerigo.rb +131 -0
  143. data/lib/rubber/environment.rb +161 -0
  144. data/lib/rubber/generator.rb +197 -0
  145. data/lib/rubber/instance.rb +165 -0
  146. data/lib/rubber/recipes/rubber.rb +89 -0
  147. data/lib/rubber/recipes/rubber/bundles.rb +28 -0
  148. data/lib/rubber/recipes/rubber/deploy.rb +90 -0
  149. data/lib/rubber/recipes/rubber/instances.rb +348 -0
  150. data/lib/rubber/recipes/rubber/load_balancers.rb +44 -0
  151. data/lib/rubber/recipes/rubber/security_groups.rb +189 -0
  152. data/lib/rubber/recipes/rubber/setup.rb +357 -0
  153. data/lib/rubber/recipes/rubber/static_ips.rb +107 -0
  154. data/lib/rubber/recipes/rubber/utils.rb +203 -0
  155. data/lib/rubber/recipes/rubber/volumes.rb +264 -0
  156. data/lib/rubber/tasks/rubber.rb +221 -0
  157. data/lib/rubber/util.rb +37 -0
  158. data/test/environment_test.rb +118 -0
  159. data/test/generator_test.rb +323 -0
  160. data/test/instance_test.rb +93 -0
  161. data/test/test_helper.rb +4 -0
  162. data/test/util_test.rb +16 -0
  163. metadata +274 -0
@@ -0,0 +1,221 @@
1
+ require 'fileutils'
2
+ require 'date'
3
+ require 'time'
4
+ require 'aws/s3'
5
+ require 'rubber'
6
+
7
+ namespace :rubber do
8
+
9
+ def rubber_env()
10
+ Rubber::Configuration.rubber_env
11
+ end
12
+
13
+ def rubber_instances()
14
+ Rubber::Configuration.rubber_instances
15
+ end
16
+
17
+ def cloud_provider
18
+ rubber_env.cloud_providers[rubber_env.cloud_provider]
19
+ end
20
+
21
+ def init_s3()
22
+ AWS::S3::Base.establish_connection!(:access_key_id => cloud_provider.access_key, :secret_access_key => cloud_provider.secret_access_key)
23
+ end
24
+
25
+ desc "Generate system config files by transforming the files in the config tree"
26
+ task :config do
27
+ cfg = Rubber::Configuration.get_configuration(RUBBER_ENV)
28
+ instance_alias = cfg.environment.current_host
29
+ instance = cfg.instance[instance_alias]
30
+ if instance
31
+ roles = instance.role_names
32
+ env = cfg.environment.bind(roles, instance_alias)
33
+ gen = Rubber::Configuration::Generator.new("#{RUBBER_ROOT}/config/rubber", roles, instance_alias)
34
+ elsif RUBBER_ENV == 'development'
35
+ roles = cfg.environment.known_roles
36
+ role_items = roles.collect do |r|
37
+ Rubber::Configuration::RoleItem.new(r, r == "db" ? {'primary' => true} : {})
38
+ end
39
+ env = cfg.environment.bind(roles, instance_alias)
40
+ domain = env.domain
41
+ instance = Rubber::Configuration::InstanceItem.new(instance_alias, domain, role_items, 'dummyid', ['dummygroup'])
42
+ instance.external_host = instance.full_name
43
+ instance.external_ip = "127.0.0.1"
44
+ instance.internal_host = instance.full_name
45
+ instance.internal_ip = "127.0.0.1"
46
+ cfg.instance.add(instance)
47
+ gen = Rubber::Configuration::Generator.new("#{RUBBER_ROOT}/config/rubber", roles, instance_alias)
48
+ gen.fake_root ="#{RUBBER_ROOT}/tmp/rubber"
49
+ else
50
+ puts "Instance not found for host: #{instance_alias}"
51
+ exit 1
52
+ end
53
+
54
+ if ENV['NO_POST']
55
+ gen.no_post = true
56
+ end
57
+ if ENV['FILE']
58
+ gen.file_pattern = ENV['FILE']
59
+ end
60
+ if ENV['FORCE']
61
+ gen.force = true
62
+ end
63
+ gen.stop_on_error_cmd = env.stop_on_error_cmd
64
+ gen.run
65
+
66
+ end
67
+
68
+ desc <<-DESC
69
+ Rotate rails app server logfiles. Should be run right after midnight.
70
+ The following arguments affect behavior:
71
+ LOG_DIR (required): Directory where log files are located
72
+ LOG_FILE (*.log): File pattern to match to find logs to rotate
73
+ LOG_AGE (7): Delete rotated logs older than this many days in the past
74
+ DESC
75
+ task :rotate_logs do
76
+ log_src_dir = get_env('LOG_DIR', true)
77
+ log_file_glob = get_env('LOG_FILES') || "*.log"
78
+ log_file_age = (get_env('LOG_AGE') || 7).to_i
79
+
80
+ rotated_date = (Date.today - 1).strftime('%Y%m%d')
81
+ puts "Rotating logfiles located at: #{log_src_dir}/#{log_file_glob}"
82
+ Dir["#{log_src_dir}/#{log_file_glob}"].each do |logfile|
83
+ rotated_file = "#{logfile}.#{rotated_date}"
84
+ if File.exist?(rotated_file)
85
+ rotated_file += "_#{Time.now.to_i}"
86
+ end
87
+ FileUtils.cp(logfile, rotated_file)
88
+ File.truncate logfile, 0
89
+ end
90
+
91
+ tdate = Date.today - log_file_age
92
+ threshold = Time.local(tdate.year, tdate.month, tdate.day)
93
+ puts "Cleaning rotated log files older than #{log_file_age} days"
94
+ Dir["#{log_src_dir}/#{log_file_glob}.[0-9]*"].each do |logfile|
95
+ if File.mtime(logfile) < threshold
96
+ FileUtils.rm_f(logfile)
97
+ end
98
+ end
99
+ end
100
+
101
+ desc <<-DESC
102
+ Backup database to given backup directory
103
+ The following arguments affect behavior:
104
+ BACKUP_DIR (required): Directory where db backups will be stored
105
+ BACKUP_AGE (7): Delete rotated logs older than this many days in the past
106
+ DBUSER (required) User to connect to the db as
107
+ DBPASS (optional): Pass to connect to the db with
108
+ DBHOST (required): Host where the db is
109
+ DBNAME (required): Database name to backup
110
+ DESC
111
+ task :backup_db do
112
+ dir = get_env('BACKUP_DIR', true)
113
+ age = (get_env('BACKUP_AGE') || 3).to_i
114
+ time_stamp = Time.now.strftime("%Y-%m-%d_%H-%M")
115
+ backup_file = "#{dir}/#{RUBBER_ENV}_dump_#{time_stamp}.sql.gz"
116
+ FileUtils.mkdir_p(File.dirname(backup_file))
117
+
118
+ user = get_env('DBUSER', true)
119
+ pass = get_env('DBPASS')
120
+ pass = nil if pass.strip.size == 0
121
+ host = get_env('DBHOST', true)
122
+ name = get_env('DBNAME', true)
123
+
124
+ raise "No db_backup_cmd defined in rubber.yml, cannot backup!" unless rubber_env.db_backup_cmd
125
+ db_backup_cmd = rubber_env.db_backup_cmd.gsub(/%([^%]+)%/, '#{\1}')
126
+ db_backup_cmd = eval('%Q{' + db_backup_cmd + '}')
127
+
128
+ puts "Backing up database with command:"
129
+ sh db_backup_cmd
130
+ puts "Created backup: #{backup_file}"
131
+
132
+ s3_prefix = "db/"
133
+ backup_bucket = cloud_provider.backup_bucket
134
+ if backup_bucket
135
+ init_s3
136
+ unless AWS::S3::Bucket.list.find { |b| b.name == backup_bucket }
137
+ AWS::S3::Bucket.create(backup_bucket)
138
+ end
139
+ dest = "#{s3_prefix}#{File.basename(backup_file)}"
140
+ puts "Saving db backup to S3: #{backup_bucket}:#{dest}"
141
+ AWS::S3::S3Object.store(dest, open(backup_file), backup_bucket)
142
+ end
143
+
144
+ tdate = Date.today - age
145
+ threshold = Time.local(tdate.year, tdate.month, tdate.day)
146
+ puts "Cleaning backups older than #{age} days"
147
+ Dir["#{dir}/*"].each do |file|
148
+ if File.mtime(file) < threshold
149
+ puts "Deleting #{file}"
150
+ FileUtils.rm_f(file)
151
+ end
152
+ end
153
+
154
+ if backup_bucket
155
+ puts "Cleaning S3 backups older than #{age} days from: #{backup_bucket}:#{s3_prefix}"
156
+ AWS::S3::Bucket.objects(backup_bucket, :prefix => s3_prefix).clone.each do |obj|
157
+ if Time.parse(obj.about["last-modified"]) < threshold
158
+ puts "Deleting #{obj.key}"
159
+ obj.delete
160
+ end
161
+ end
162
+ end
163
+ end
164
+
165
+ desc <<-DESC
166
+ Restores a database backup from s3.
167
+ This tries to find the last backup made or the s3 object identified by the
168
+ key FILENAME
169
+ The following arguments affect behavior:
170
+ FILENAME (optional): key of S3 object to use
171
+ DBUSER (required) User to connect to the db as
172
+ DBPASS (optional): Pass to connect to the db with
173
+ DBHOST (required): Host where the db is
174
+ DBNAME (required): Database name to backup
175
+ DESC
176
+ task :restore_db_s3 do
177
+ file = get_env('FILENAME')
178
+ user = get_env('DBUSER', true)
179
+ pass = get_env('DBPASS')
180
+ pass = nil if pass && pass.strip.size == 0
181
+ host = get_env('DBHOST', true)
182
+ name = get_env('DBNAME', true)
183
+
184
+ raise "No db_restore_cmd defined in rubber.yml" unless rubber_env.db_restore_cmd
185
+ db_restore_cmd = rubber_env.db_restore_cmd.gsub(/%([^%]+)%/, '#{\1}')
186
+ db_restore_cmd = eval('%Q{' + db_restore_cmd + '}')
187
+
188
+ # try to fetch a matching file from s3 (if backup_bucket given)
189
+ backup_bucket = cloud_provider.backup_bucket
190
+ raise "No backup_bucket defined in rubber.yml" unless backup_bucket
191
+ if (init_s3 &&
192
+ AWS::S3::Bucket.list.find { |b| b.name == backup_bucket })
193
+ s3objects = AWS::S3::Bucket.find(backup_bucket,
194
+ :prefix => 'db/')
195
+ if file
196
+ puts "trying to fetch #{file} from s3"
197
+ data = s3objects.detect { |o| file == o.key }
198
+ else
199
+ puts "trying to fetch last modified s3 backup"
200
+ data = s3objects.max {|a,b| a.about["last-modified"] <=> b.about["last-modified"] }
201
+ end
202
+ end
203
+ raise "could not access backup file via s3" unless data
204
+
205
+ puts "piping restore data to command [#{db_restore_cmd}]"
206
+ IO.popen(db_restore_cmd, mode='w') do |p|
207
+ data.value do |segment|
208
+ p.write segment
209
+ end
210
+ end
211
+
212
+ end
213
+
214
+
215
+ def get_env(name, required=false)
216
+ value = ENV[name]
217
+ raise("#{name} is required, pass using environment") if required && ! value
218
+ return value
219
+ end
220
+
221
+ end
@@ -0,0 +1,37 @@
1
+
2
+
3
+ module Rubber
4
+ module Util
5
+
6
+ def self.symbolize_keys(map)
7
+ map.inject({}) do |options, (key, value)|
8
+ options[key.to_sym || key] = value
9
+ options
10
+ end
11
+ end
12
+
13
+ def self.stringify(val)
14
+ case val
15
+ when String
16
+ val
17
+ when Hash
18
+ val.inject({}) {|h, a| h[stringify(a[0])] = stringify(a[1]); h}
19
+ when Enumerable
20
+ val.collect {|v| stringify(v)}
21
+ else
22
+ val.to_s
23
+ end
24
+
25
+ end
26
+
27
+ # Opens the file for writing by root
28
+ def self.sudo_open(path, perms, &block)
29
+ open("|sudo tee #{path} > /dev/null", perms, &block)
30
+ end
31
+
32
+ def self.is_rails?
33
+ File.exist?(File.join(RUBBER_ROOT, 'config', 'environment.rb'))
34
+ end
35
+
36
+ end
37
+ end
@@ -0,0 +1,118 @@
1
+ require 'test/unit'
2
+ require 'tempfile'
3
+ require 'test_helper'
4
+
5
+ class EnvironmentTest < Test::Unit::TestCase
6
+ include Rubber::Configuration
7
+
8
+ def test_known_roles
9
+ env = Rubber::Configuration::Environment.new("#{File.dirname(__FILE__)}/fixtures/basic")
10
+ assert_equal ['role1', 'role2'], env.known_roles, "list of know roles not correct"
11
+ end
12
+
13
+ def test_env
14
+ env = Rubber::Configuration::Environment.new("#{File.dirname(__FILE__)}/fixtures/basic")
15
+ e = env.bind()
16
+ assert_equal 'val1', e['var1'], 'env not retrieving right val'
17
+ assert_equal 'val2', e['var2'], 'env not retrieving right val'
18
+ assert_equal 'val1', e.var1, 'env not retrieving right val for method missing'
19
+ assert_equal 'val2', e.var2, 'env not retrieving right val for method missing'
20
+
21
+ assert_equal 'val3', e.var3, 'env not retrieving right val for config in supplemental file'
22
+
23
+ e = env.bind('role1', 'nohost')
24
+ assert_equal 'val1', e['var1'], 'env not retrieving right val'
25
+ assert_equal 'role1val2', e['var2'], 'env not retrieving right val'
26
+ assert_equal 'val1', e.var1, 'env not retrieving right val for method missing'
27
+ assert_equal 'role1val2', e.var2, 'env not retrieving right val for method missing'
28
+
29
+ e = env.bind('role1', 'host1')
30
+ assert_equal 'val1', e['var1'], 'env not retrieving right val'
31
+ assert_equal 'host1val2', e['var2'], 'env not retrieving right val'
32
+ assert_equal 'val1', e.var1, 'env not retrieving right val for method missing'
33
+ assert_equal 'host1val2', e.var2, 'env not retrieving right val for method missing'
34
+
35
+ e = env.bind('norole', 'host1')
36
+ assert_equal 'val1', e['var1'], 'env not retrieving right val'
37
+ assert_equal 'host1val2', e['var2'], 'env not retrieving right val'
38
+ assert_equal 'val1', e.var1, 'env not retrieving right val for method missing'
39
+ assert_equal 'host1val2', e.var2, 'env not retrieving right val for method missing'
40
+ end
41
+
42
+ def test_combine
43
+ env = Rubber::Configuration::Environment
44
+ assert_equal "new", env.combine("old", "new"), "Last should win for scalar combine"
45
+ assert_equal 5, env.combine(1, 5), "Last should win for scalar combine"
46
+ assert_equal [1, 2, 3, 4], env.combine([1, 2, 3], [3, 4]), "Arrays should be unioned when combined"
47
+ assert_equal({1 => "1", 2 => "2", 3 => "3", 4 => "4"}, env.combine({1 => "1", 2 => "2"}, {3 => "3", 4 => "4"}), "Maps should be unioned when combined")
48
+ assert_equal({1 => "2"}, env.combine({1 => "1"}, {1 => "2"}), "Last should win for scalars in maps when combined")
49
+ assert_equal({1 => {1 => "1", 2 => "2"}}, env.combine({1 => {1 => "1"}}, {1 => {2 => "2"}}), "Maps should be unioned recursively when combined")
50
+ end
51
+
52
+ def test_expansion
53
+ env = Rubber::Configuration::Environment.new("#{File.dirname(__FILE__)}/fixtures/expansion")
54
+ e = env.bind()
55
+ assert_equal 'val1', e['var1']
56
+ assert_equal 'val2', e['var2']
57
+ assert_equal 'val1', e['var3']
58
+ assert_equal '4 is val2', e['var4']
59
+ assert_equal 'val1', e['var5']
60
+ assert_equal %w[lv1 lv2 val1], e['list1']
61
+ expected = {'mk1' => 'mv1', 'mk2' => 'mv2', 'mk3' => 'val2'}
62
+ e.map1.each do |k, v|
63
+ assert_equal expected[k], v
64
+ end
65
+
66
+ e = env.bind('role1', 'nohost')
67
+ assert_equal 'role1val1', e['var1']
68
+ assert_equal 'role1val1', e['var3']
69
+ assert_equal %w[lv1 lv2 role1val1 role1lv1 role1val2], e['list1']
70
+
71
+ e = env.bind('role1', 'host1')
72
+ assert_equal 'host1val1', e['var1']
73
+ assert_equal 'host1val1', e['var3']
74
+ assert_equal %w[lv1 lv2 host1val1 role1lv1 host1val2 host1lv1], e['list1'] # lists are additive
75
+
76
+ e = env.bind('norole', 'host1')
77
+ assert_equal 'host1val1', e['var1']
78
+ assert_equal 'host1val1', e['var3']
79
+ assert_equal %w[lv1 lv2 host1val1 host1lv1 host1val2], e['list1']
80
+ end
81
+
82
+ def test_bool_expansion
83
+ env = Rubber::Configuration::Environment.new("#{File.dirname(__FILE__)}/fixtures/expansion")
84
+ e = env.bind()
85
+ assert_equal true, e['truevar']
86
+ assert_equal false, e['falsevar']
87
+ assert_equal true, e['truevar_exp']
88
+ assert 'true' != e['truevar_exp']
89
+ assert_equal false, e['falsevar_exp']
90
+ assert 'false' != e['falsevar_exp']
91
+ assert_equal 'true thing', e['faketruevar_exp']
92
+ end
93
+
94
+ def test_secret_env
95
+ env = Rubber::Configuration::Environment.new("#{File.dirname(__FILE__)}/fixtures/basic")
96
+ e = env.bind()
97
+ assert_nil e['rubber_secret'], 'env should not have secret set'
98
+
99
+ fixture_dir = File.expand_path("#{File.dirname(__FILE__)}/fixtures/secret")
100
+ env = Rubber::Configuration::Environment.new(fixture_dir)
101
+ e = env.bind()
102
+ assert_equal "#{fixture_dir}/secret.yml", e['rubber_secret'], 'env should have secret set'
103
+ assert_equal "secret_val", e['secret_key'], 'env should have gotten setting from secret file'
104
+ end
105
+
106
+ def test_nested_ref
107
+ env = Rubber::Configuration::Environment.new("#{File.dirname(__FILE__)}/fixtures/nested")
108
+ e = env.bind()
109
+ assert_equal 'val1', e.var1, 'env not retrieving right val'
110
+ assert_equal 'val3', e.var2.var3, 'env not retrieving right val'
111
+ assert_equal({'var5' => 'val5'}, e.var2.var4, 'env not retrieving right val')
112
+ assert_equal ['val6a', 'val6b'], e.var2.var6, 'env not retrieving right val'
113
+ assert_equal 'val1', e.var2.var7, 'env not retrieving right val'
114
+ assert_equal 'val3', e.var2.var8, 'env not retrieving right val'
115
+ assert_equal 'val5', e.var2.var9, 'env not retrieving right val'
116
+ end
117
+
118
+ end
@@ -0,0 +1,323 @@
1
+ require 'test/unit'
2
+ require 'tempfile'
3
+ require 'test_helper'
4
+
5
+ class GeneratorTest < Test::Unit::TestCase
6
+ include Rubber::Configuration
7
+
8
+ def test_validate
9
+ assert_raise RuntimeError do
10
+ src = <<-SRC
11
+ hello
12
+ SRC
13
+ Generator.new(nil, nil, nil).transform(src)
14
+ end
15
+ assert_raise RuntimeError do
16
+ src = <<-SRC
17
+ <%
18
+ @read_cmd = 'ls'
19
+ %>
20
+ hello
21
+ SRC
22
+ Generator.new(nil, nil, nil).transform(src)
23
+ end
24
+ assert_raise RuntimeError do
25
+ src = <<-SRC
26
+ <%
27
+ @write_cmd = 'cat'
28
+ %>
29
+ hello
30
+ SRC
31
+ Generator.new(nil, nil, nil).transform(src)
32
+ end
33
+ end
34
+
35
+ def test_simple_transform
36
+ out_file = Tempfile.new('testsimple')
37
+ src = <<-SRC
38
+ <%
39
+ @path = '#{out_file.path}'
40
+ %>
41
+ hello
42
+ SRC
43
+
44
+ Generator.new(nil, nil, nil).transform(src)
45
+ assert File.exists?(out_file.path), "transform did not generate an output file"
46
+ assert_equal "hello", File.read(out_file.path).strip, "transformed contents are incorrect"
47
+ end
48
+
49
+ def test_additive_transform
50
+ out_file = Tempfile.new('testadditive')
51
+ open(out_file.path, 'w') { |f| f.write("howdy\n")}
52
+ src = <<-SRC
53
+ <%
54
+ @path = '#{out_file.path}'
55
+ @additive = ['#start', '#end']
56
+ %>
57
+ hello
58
+ SRC
59
+
60
+ Generator.new(nil, nil, nil).transform(src)
61
+ assert File.exists?(out_file.path), "transform did not generate an output file"
62
+ assert_equal "howdy\n#start \n hello\n#end", File.read(out_file.path).strip, "transformed contents are incorrect"
63
+
64
+ src += "neato\n"
65
+ open(out_file.path, 'a') { |f| f.write("again\n")}
66
+ Generator.new(nil, nil, nil).transform(src)
67
+ assert File.exists?(out_file.path), "transform did not generate an output file"
68
+ assert_equal "howdy\n#start \n hello\nneato\n#end\nagain", File.read(out_file.path).strip, "transformed contents are incorrect"
69
+ end
70
+
71
+ def test_post_command
72
+ out_file = Tempfile.new('testpost')
73
+ post_file = out_file.path + '.post'
74
+ src = <<-SRC
75
+ <%
76
+ @path = '#{out_file.path}'
77
+ @post = 'touch #{post_file}'
78
+ %>
79
+ hello
80
+ SRC
81
+
82
+ Generator.new(nil, nil, nil).transform(src)
83
+ assert File.exists?(out_file.path), "transform did not generate an output file"
84
+ assert File.exists?(post_file), "transform did not execute post"
85
+ assert_equal "hello", File.read(out_file.path).strip, "transformed contents are incorrect"
86
+
87
+ FileUtils.rm_f(post_file)
88
+ Generator.new(nil, nil, nil).transform(src)
89
+ assert File.exists?(out_file.path), "transform did not generate an output file"
90
+ assert ! File.exists?(post_file), "post should not have been executed as dest file unchanged"
91
+ assert_equal "hello", File.read(out_file.path).strip, "transformed contents are incorrect"
92
+
93
+ FileUtils.rm_f(post_file)
94
+ gen = Generator.new(nil, nil, nil)
95
+ gen.no_post = true
96
+ gen.transform(src + "there")
97
+ assert File.exists?(out_file.path), "transform did not generate an output file"
98
+ assert ! File.exists?(post_file), "post should not have been generated for no_post option"
99
+ assert_equal "hello\nthere", File.read(out_file.path).strip, "transformed contents are incorrect"
100
+ end
101
+
102
+ def test_stop_on_error
103
+ out_file = Tempfile.new('teststoponerr')
104
+ post_file = out_file.path + '.post'
105
+ src = <<-SRC
106
+ <%
107
+ @path = '#{out_file.path}'
108
+ @post = 'badcommand; touch #{post_file}'
109
+ %>
110
+ hello
111
+ SRC
112
+
113
+ g = Generator.new(nil, nil, nil)
114
+ g.transform(src)
115
+ assert File.exists?(out_file.path), "transform did not generate an output file"
116
+ assert File.exists?(post_file), "transform did not execute post"
117
+ assert_equal "hello", File.read(out_file.path).strip, "transformed contents are incorrect"
118
+
119
+ FileUtils.rm_f(post_file)
120
+ g = Generator.new(nil, nil, nil)
121
+ g.stop_on_error_cmd = "function error_exit { exit 99; }; trap error_exit ERR"
122
+ g.transform(src)
123
+ assert File.exists?(out_file.path), "transform did not generate an output file"
124
+ assert ! File.exists?(post_file), "post should not have been executed as comes after error"
125
+ end
126
+
127
+ def test_pipe_command
128
+ out_file = Tempfile.new('testpipe')
129
+ src = <<-SRC
130
+ <%
131
+ @read_cmd = 'echo hi'
132
+ @write_cmd = 'cat > #{out_file.path}'
133
+ %>
134
+ hello
135
+ SRC
136
+
137
+ Generator.new(nil, nil, nil).transform(src)
138
+ assert File.exists?(out_file.path), "transform did not generate to write_cmd"
139
+ assert_equal "hello", File.read(out_file.path).strip, "transformed contents are incorrect"
140
+
141
+ FileUtils.rm_f(out_file.path)
142
+ src = <<-SRC
143
+ <%
144
+ @read_cmd = 'echo hi'
145
+ @write_cmd = 'cat > #{out_file.path}'
146
+ @additive = ['#start', '#end']
147
+ %>
148
+ hello
149
+ SRC
150
+
151
+ Generator.new(nil, nil, nil).transform(src)
152
+ assert File.exists?(out_file.path), "transform did not generate to write_cmd"
153
+ assert_equal "hi\n#start \n hello\n#end", File.read(out_file.path).strip, "transformed contents are incorrect"
154
+
155
+ FileUtils.rm_f(out_file.path)
156
+ src = <<-SRC
157
+ <%
158
+ @read_cmd = 'echo "#start\nthere\n#end\nhi\n"'
159
+ @write_cmd = 'cat > #{out_file.path}'
160
+ @additive = ['#start', '#end']
161
+ %>
162
+ hello
163
+ SRC
164
+
165
+ Generator.new(nil, nil, nil).transform(src)
166
+ assert File.exists?(out_file.path), "transform did not generate to write_cmd"
167
+ assert_equal "#start \n hello\n#end\nhi", File.read(out_file.path).strip, "transformed contents are incorrect"
168
+
169
+ FileUtils.rm_f(out_file.path)
170
+ src = <<-SRC
171
+ <%
172
+ @read_cmd = 'echo hi'
173
+ @write_cmd = 'badcommand'
174
+ %>
175
+ hello
176
+ SRC
177
+
178
+ assert_raises RuntimeError do
179
+ Generator.new(nil, nil, nil).transform(src)
180
+ end
181
+
182
+ end
183
+
184
+ def list_dir(dir)
185
+ l = Dir.entries(dir)
186
+ l.delete_if {|d| d =~ /(^\.+$)|\.bak$/}
187
+ end
188
+
189
+ def test_scoping
190
+ out_dir = "#{Dir::tmpdir}/test_rubber_scoping"
191
+ FileUtils.rm_rf(out_dir)
192
+ assert ! File.exists?(out_dir)
193
+
194
+ g = Generator.new("#{File.dirname(__FILE__)}/fixtures/basic", nil, nil, :out_dir => out_dir)
195
+ g.run()
196
+ assert File.directory?(out_dir), "scoped transform did not create dir"
197
+ assert_equal ['bar.conf', 'foo.conf'], list_dir(out_dir), "scoped transform did not create correct files"
198
+ assert_equal "common", File.read("#{out_dir}/foo.conf").strip, "transformed contents are incorrect"
199
+ assert_equal "common", File.read("#{out_dir}/bar.conf").strip, "transformed contents are incorrect"
200
+
201
+ FileUtils.rm_rf(out_dir)
202
+ assert ! File.exists?(out_dir)
203
+
204
+ g = Generator.new("#{File.dirname(__FILE__)}/fixtures/basic", ['role1'], nil, :out_dir => out_dir)
205
+ g.run()
206
+ assert File.directory?(out_dir), "scoped transform did not create dir"
207
+ assert_equal ['bar.conf', 'foo.conf'], list_dir(out_dir), "scoped transform did not create correct files"
208
+ assert_equal "role1", File.read("#{out_dir}/foo.conf").strip, "transformed contents are incorrect"
209
+ assert_equal "common", File.read("#{out_dir}/bar.conf").strip, "transformed contents are incorrect"
210
+
211
+ FileUtils.rm_rf(out_dir)
212
+ assert ! File.exists?(out_dir)
213
+
214
+ g = Generator.new("#{File.dirname(__FILE__)}/fixtures/basic", ['role2', 'role1'], nil, :out_dir => out_dir)
215
+ g.run()
216
+ assert File.directory?(out_dir), "scoped transform did not create dir"
217
+ assert_equal ['bar.conf', 'foo.conf'], list_dir(out_dir), "scoped transform did not create correct files"
218
+ assert_equal "role2", File.read("#{out_dir}/foo.conf").strip, "transformed contents are incorrect"
219
+ assert_equal "common", File.read("#{out_dir}/bar.conf").strip, "transformed contents are incorrect"
220
+
221
+ FileUtils.rm_rf(out_dir)
222
+ assert ! File.exists?(out_dir)
223
+
224
+ g = Generator.new("#{File.dirname(__FILE__)}/fixtures/basic", ['role1'], ['host1'], :out_dir => out_dir)
225
+ g.run()
226
+ assert File.directory?(out_dir), "scoped transform did not create dir"
227
+ assert_equal ['bar.conf', 'foo.conf'], list_dir(out_dir), "scoped transform did not create correct files"
228
+ assert_equal "host1", File.read("#{out_dir}/foo.conf").strip, "transformed contents are incorrect"
229
+ assert_equal "common", File.read("#{out_dir}/bar.conf").strip, "transformed contents are incorrect"
230
+
231
+
232
+ FileUtils.rm_rf(out_dir)
233
+ end
234
+
235
+ def test_ordering
236
+ out_dir = "#{Dir::tmpdir}/test_rubber_ordering"
237
+ FileUtils.rm_rf(out_dir)
238
+ assert ! File.exists?(out_dir)
239
+
240
+ g = Generator.new("#{File.dirname(__FILE__)}/fixtures/generator_order", ['role2', 'role1'], ['host1'], :out_dir => out_dir)
241
+ g.run()
242
+ assert File.directory?(out_dir), "transform did not create dir"
243
+ assert_equal ['out.conf'], list_dir(out_dir), "transform did not create correct file"
244
+ result = <<DATA
245
+ # common first start
246
+ first
247
+ # common first end
248
+ # common last start
249
+ last
250
+ # common last end
251
+ # role1 first start
252
+ role1 first
253
+ # role1 first end
254
+ # role1 last start
255
+ role1 last
256
+ # role1 last end
257
+ # role2 first start
258
+ role2 first
259
+ # role2 first end
260
+ # role2 last start
261
+ role2 last
262
+ # role2 last end
263
+ # host1 first start
264
+ host1 first
265
+ # host1 first end
266
+ # host1 last start
267
+ host1 last
268
+ # host1 last end
269
+ DATA
270
+ assert_equal result.strip, File.read("#{out_dir}/out.conf").strip, "transformed contents are incorrect"
271
+
272
+ FileUtils.rm_rf(out_dir)
273
+ end
274
+
275
+ def test_file_pattern
276
+ out_dir = "#{Dir::tmpdir}/test_rubber_scoping"
277
+ FileUtils.rm_rf(out_dir)
278
+ assert ! File.exists?(out_dir)
279
+
280
+ g = Generator.new("#{File.dirname(__FILE__)}/fixtures/basic", nil, nil, :out_dir => out_dir)
281
+ g.file_pattern = 'foo\.conf'
282
+ g.run()
283
+ assert File.directory?(out_dir), "scoped transform did not create dir"
284
+ assert_equal ['foo.conf'], list_dir(out_dir), "scoped transform did not create correct files"
285
+ assert_equal "common", File.read("#{out_dir}/foo.conf").strip, "transformed contents are incorrect"
286
+ end
287
+
288
+ def test_perms
289
+ user = ENV['USER']
290
+ group = `groups`.split.last
291
+ out_file = Tempfile.new('testperms')
292
+ src = <<-SRC
293
+ <%
294
+ @path = '#{out_file.path}'
295
+ @perms = 0777
296
+ @owner = '#{user}'
297
+ @group = '#{group}'
298
+ %>
299
+ hello
300
+ SRC
301
+
302
+ Generator.new(nil, nil, nil).transform(src)
303
+ assert File.exists?(out_file.path), "transform did not generate an output file"
304
+ assert_equal "100777", sprintf("%o", File.stat(out_file.path).mode), "transformed permissions are incorrect"
305
+ assert_equal Etc.getpwnam(user).uid, File.stat(out_file.path).uid, "transformed owner is incorrect"
306
+ assert_equal Etc.getgrnam(group).gid, File.stat(out_file.path).gid, "transformed group is incorrect"
307
+ end
308
+
309
+ def test_skip
310
+ out_file = Tempfile.new('testskip')
311
+ File.delete(out_file.path)
312
+ src = <<-SRC
313
+ <%
314
+ @skip = true
315
+ @path = '#{out_file.path}'
316
+ %>
317
+ hello
318
+ SRC
319
+
320
+ Generator.new(nil, nil, nil).transform(src)
321
+ assert ! File.exists?(out_file.path), "transform didn't skip generation of an output file"
322
+ end
323
+ end