rubber 1.15.0 → 2.0.0.pre1

data/{lib/generators/vulcanize/templates → templates}/postgresql/config/rubber/role/postgresql_slave/recovery.conf

@@ -6,8 +6,8 @@
 # -------------------------------
 #
 # Edit this file to provide the parameters that PostgreSQL needs to
-# perform an archive recovery of a database, or to act as a log-streaming
-# replication standby.
+# perform an archive recovery of a database, or to act as a replication
+# standby.
 #
 # If "recovery.conf" is present in the PostgreSQL data directory, it is
 # read on postmaster startup.  After successful recovery, it is renamed
@@ -16,9 +16,7 @@
 #
 # This file consists of lines of the form:
 #
-#   name = 'value'
-#
-# (The quotes around the value are NOT optional, but the "=" is.)
+#   name = value
 #
 # Comments are introduced with '#'.
 #
@@ -71,16 +69,19 @@
 # If you want to stop rollforward at a specific point, you
 # must set a recovery target.
 #
-# You may set a recovery target either by transactionId, or
-# by timestamp. Recovery may either include or exclude the
+# You may set a recovery target either by transactionId, by name,
+# or by timestamp. Recovery may either include or exclude the
 # transaction(s) with the recovery target value (ie, stop either
 # just after or just before the given target, respectively).
 #
+#
+#recovery_target_name = ''  # e.g. 'daily backup 2011-01-26'
+#
 #recovery_target_time = ''	# e.g. '2004-07-14 22:39:00 EST'
 #
 #recovery_target_xid = ''
 #
-#recovery_target_inclusive = 'true'
+#recovery_target_inclusive = true
 #
 #
 # If you want to recover into a timeline other than the "main line" shown in
@@ -89,25 +90,39 @@
 #
 #recovery_target_timeline = 'latest'
 #
+#
+# If pause_at_recovery_target is enabled, recovery will pause when
+# the recovery target is reached. The pause state will continue until
+# pg_xlog_replay_resume() is called. This setting has no effect if
+# hot standby is not enabled, or if no recovery target is set.
+#
+#pause_at_recovery_target = true
+#
 #---------------------------------------------------------------------------
 # STANDBY SERVER PARAMETERS
 #---------------------------------------------------------------------------
 #
-# When standby_mode is enabled, the PostgreSQL server will work as
-# a standby. It tries to connect to the primary according to the
-# connection settings primary_conninfo, and receives XLOG records
-# continuously.
+# standby_mode
+#
+# When standby_mode is enabled, the PostgreSQL server will work as a
+# standby. It will continuously wait for the additional XLOG records, using
+# restore_command and/or primary_conninfo.
+#
+standby_mode = on
+#
+# primary_conninfo
 #
-standby_mode = 'on'
+# If set, the PostgreSQL server will try to connect to the primary using this
+# connection string and receive XLOG records continuously.
 #
-primary_conninfo = 'host=<%= rubber_instances.for_role('postgresql_master').first.name %> port=5432 user=postgres'		# e.g. 'host=localhost port=5432'
+primary_conninfo = 'host=<%= rubber_instances.for_role('postgresql_master').first.name %> port=5432 user=<%= rubber_env.db_replication_user %> <%= rubber_env.db_replication_user_pass ? "password=#{rubber_env.db_replication_user}" : "" %> application_name=<%= rubber_env.app_name %>'		# e.g. 'host=localhost port=5432'
 #
 #
-# By default, a standby server keeps streaming XLOG records from the
-# primary indefinitely. If you want to stop streaming and finish recovery,
-# opening up the system in read/write mode, specify path to a trigger file.
-# Server will poll the trigger file path periodically and stop streaming
-# when it's found.
+# By default, a standby server keeps restoring XLOG records from the
+# primary indefinitely. If you want to stop the standby mode, finish recovery
+# and open the system in read/write mode, specify path to a trigger file.
+# The server will poll the trigger file path periodically and start as a
+# primary server when it's found.
 #
 trigger_file = '<%= rubber_env.postgresql_data_dir %>/trigger_file'
 #

data/{generators/vulcanize/templates → templates}/postgresql/config/rubber/rubber-postgresql.yml

@@ -1,24 +1,24 @@
-
 # REQUIRED: The credentials for creating/accessing your app's database
 #
 db_user: "#{app_name[0,16]}"
 db_pass:
-db_name: "#{app_name}_#{RUBBER_ENV}"
+db_name: "#{app_name}_#{Rubber.env}"
 
-db_slave_user: "#{('slave_' + db_user)[0,16]}"
-db_replicator_user: "#{('replicator_' + db_user)[0,16]}"
+db_replication_user: replication
+db_replication_pass:
+postgresql_synchronous_replication: false
 
 # REQUIRED: The command to use to do a full backup of your database using
-#           "rake rubber:backup_db"
+#           "rubber util:backup_db"
 db_backup_cmd: "nice sudo -u postgres pg_dumpall -U postgres | gzip -c > %backup_file%"
 
 # REQUIRED: The command to use to restore the database using
-#           "rake rubber:restore_db_s3"
+#           "rake util:restore_db"
 #           This command will receive the data generated by db_backup_cmd via
 #           its standard input
-db_restore_cmd: "zcat - | psql -U postgres -h %host% %name%"
+db_restore_cmd: "zcat - | psql -U %user% -h %host% %name%"
 
-postgresql_ver: 9.0
+postgresql_ver: 9.1
 postgresql_conf_dir: "/etc/postgresql/#{postgresql_ver}/main"
 postgresql_data_dir: "/mnt/postgresql/#{postgresql_ver}/data"
 postgresql_archive_dir: "/mnt/postgresql/#{postgresql_ver}/archive"
@@ -28,10 +28,10 @@ postgresql_ctl: "/usr/bin/env service postgresql"
 # Capistrano needs db:primary role for migrate to work,
 # so we might as well just make consistent across the board
 role_dependencies:
-  postgresql_master: ["db:primary=true"]
-  postgresql_slave: [db]
-  db:primary=true: [postgresql_master]
-  db: [postgresql_slave]
+  postgresql_master: [postgresql, "db:primary=true"]
+  postgresql_slave: [postgresql, db]
+  "db:primary=true": [postgresql, postgresql_master]
+  db: [postgresql, postgresql_slave]
 
 packages: [postgresql-client, libpq-dev]
 gems: [pg]
@@ -42,4 +42,4 @@ roles:
   postgresql_slave:
     db_server_id: 2
   db:
-    packages: [postgresql-9.0, libdbd-pg-perl]
+    packages: ["postgresql-#{postgresql_ver}"]

data/templates/redis/config/rubber/deploy-redis.rb

@@ -0,0 +1,79 @@
+namespace :rubber do
+
+  namespace :redis do
+
+    rubber.allow_optional_tasks(self)
+
+    after "rubber:install_packages", "rubber:redis:install"
+
+    task :install, :roles => :redis do
+      rubber.sudo_script 'install_redis', <<-ENDSCRIPT
+        if ! redis-server --version | grep "#{rubber_env.redis_server_version}" &> /dev/null; then
+          # Fetch the sources.
+          wget http://redis.googlecode.com/files/redis-#{rubber_env.redis_server_version}.tar.gz
+          tar -zxf redis-#{rubber_env.redis_server_version}.tar.gz
+
+          # Build the binaries.
+          cd redis-#{rubber_env.redis_server_version}
+          make
+
+          # Install the binaries.
+          make install
+
+          # create the user
+          if ! id redis &> /dev/null; then adduser --system --group redis; fi
+
+          # Clean up after ourselves.
+          cd ..
+          rm -rf redis-#{rubber_env.redis_server_version}
+          rm redis-#{rubber_env.redis_server_version}.tar.gz
+        fi
+      ENDSCRIPT
+    end
+
+    after "rubber:bootstrap", "rubber:redis:bootstrap"
+
+    task :bootstrap, :roles => :redis do
+      exists = capture("echo $(ls #{rubber_env.redis_db_dir} 2> /dev/null)")
+      if exists.strip.size == 0
+        
+        rubber.sudo_script 'bootstrap_redis', <<-ENDSCRIPT
+          mkdir -p #{rubber_env.redis_db_dir}
+          chown -R redis:redis #{rubber_env.redis_db_dir}
+        ENDSCRIPT
+  
+        # After everything installed on machines, we need the source tree
+        # on hosts in order to run rubber:config for bootstrapping the db
+        rubber.update_code_for_bootstrap
+  
+        # Gen just the conf for cassandra
+        rubber.run_config(:file => "role/redis", :force => true, :deploy_path => release_path)
+      
+        restart
+      end
+    end
+
+    desc "Stops the redis server"
+    task :stop, :roles => :redis, :on_error => :continue do
+      rsudo "service redis-server stop || true"
+    end
+
+    desc "Starts the redis server"
+    task :start, :roles => :redis do
+      rsudo "service redis-server start"
+    end
+
+    desc "Restarts the redis server"
+    task :restart, :roles => :redis do
+      stop
+      start
+    end
+
+    desc "Reloads the redis server"
+    task :reload, :roles => :redis do
+      restart
+    end
+
+  end
+
+end

data/{lib/generators/vulcanize/templates → templates}/redis/config/rubber/role/redis/crontab

@@ -10,6 +10,8 @@
   backup_redis_host ||= rubber_instances.for_role('redis_master').first
   if backup_redis_host.name == rubber_env.host
 %>
+
 # Backup redis DBs every hour.
-0 * * * * BACKUP_DIR=/mnt/redis_backups BACKUP_NAME=redis BACKUP_CMD="nice zip -r \%dir\%/redis_\%time_stamp\%.zip <%= rubber_env.redis_db_dir %> -x \*temp\*.rdb" <%= RUBBER_ROOT %>/script/cron-rake rubber:backup
+0 * * * * <%= Rubber.root %>/script/rubber cron --task util:backup --directory=/mnt/redis_backups --name=redis --command="nice zip -r \%dir\%/\%name\%_\%time_stamp\%.zip <%= rubber_env.redis_db_dir %> -x \*temp\*.rdb"
+
 <% end %>

data/templates/redis/config/rubber/role/redis/redis-upstart.conf

@@ -0,0 +1,23 @@
+<%
+  @path = "/etc/init/redis-server.conf"
+  @backup = false
+%>
+description "redis server"
+
+start on runlevel [2345]
+stop on runlevel [016]
+
+expect fork
+
+script
+  ulimit -n 65536
+  exec /usr/local/bin/redis-server <%= rubber_env.redis_server_conf_file %>
+end script
+
+post-start script
+  status redis-server | head -n1 | awk '{print $NF}' > <%= rubber_env.redis_server_pid_file %>
+end script
+
+post-stop script
+  rm -f <%= rubber_env.redis_server_pid_file %>
+end script

data/{generators/vulcanize/templates → templates}/redis/config/rubber/role/redis/redis.conf

@@ -1,9 +1,8 @@
 <%
-  @path = '/etc/redis/redis.conf'
+  @path = rubber_env.redis_server_conf_file
   @post = <<-SCRIPT
     mkdir -p #{rubber_env.redis_db_dir}
     chown -R redis:redis #{rubber_env.redis_db_dir}
-    /etc/init.d/redis-server restart
   SCRIPT
 %>
 
@@ -31,7 +30,7 @@ pidfile /var/run/redis.pid
 
 # Accept connections on the specified port, default is 6379.
 # If port 0 is specified Redis will not listen on a TCP socket.
-port 6379
+port <%= rubber_env.redis_server_port %>
 
 # If you want you can bind a single interface, if the bind option is not
 # specified all the interfaces will listen for incoming connections.
@@ -43,6 +42,7 @@ port 6379
 # on a unix socket when not specified.
 #
 # unixsocket /tmp/redis.sock
+# unixsocketperm 755
 
 # Close the connection after a client is idle for N seconds (0 to disable)
 timeout 300
@@ -58,7 +58,7 @@ loglevel notice
 # Specify the log file name. Also 'stdout' can be used to force
 # Redis to log on the standard output. Note that if you use standard
 # output for logging but daemonize, logs will be sent to /dev/null
-logfile /var/log/redis/redis-server.log
+logfile <%= rubber_env.redis_server_log_file %>
 
 # To enable logging to the system logger, just set 'syslog-enabled' to yes,
 # and optionally update the other syslog parameters to suit your needs.
@@ -137,7 +137,6 @@ slaveof <%= redis_master.name %> 6379
 #
 # masterauth <master-password>
 
-
 # When a slave lost the connection with the master, or when the replication
 # is still in progress, the slave can act in two different ways:
 #
@@ -214,14 +213,14 @@ slave-serve-stale-data yes
 
 # MAXMEMORY POLICY: how Redis will select what to remove when maxmemory
 # is reached? You can select among five behavior:
-# 
+#
 # volatile-lru -> remove the key with an expire set using an LRU algorithm
 # allkeys-lru -> remove any key accordingly to the LRU algorithm
 # volatile-random -> remove a random key with an expire set
 # allkeys->random -> remove a random key, any key
 # volatile-ttl -> remove the key with the nearest expire time (minor TTL)
 # noeviction -> don't expire at all, just return an error on write operations
-# 
+#
 # Note: with all the kind of policies, Redis will return an error on write
 #       operations, when there are not suitable keys for eviction.
 #
@@ -305,13 +304,60 @@ appendfsync everysec
 # the same as "appendfsync none", that in pratical terms means that it is
 # possible to lost up to 30 seconds of log in the worst scenario (with the
 # default Linux settings).
-# 
+#
 # If you have latency problems turn this to "yes". Otherwise leave it as
 # "no" that is the safest pick from the point of view of durability.
 no-appendfsync-on-rewrite no
 
+# Automatic rewrite of the append only file.
+# Redis is able to automatically rewrite the log file implicitly calling
+# BGREWRITEAOF when the AOF log size will growth by the specified percentage.
+#
+# This is how it works: Redis remembers the size of the AOF file after the
+# latest rewrite (or if no rewrite happened since the restart, the size of
+# the AOF at startup is used).
+#
+# This base size is compared to the current size. If the current size is
+# bigger than the specified percentage, the rewrite is triggered. Also
+# you need to specify a minimal size for the AOF file to be rewritten, this
+# is useful to avoid rewriting the AOF file even if the percentage increase
+# is reached but it is still pretty small.
+#
+# Specify a precentage of zero in order to disable the automatic AOF
+# rewrite feature.
+
+auto-aof-rewrite-percentage 100
+auto-aof-rewrite-min-size 64mb
+
+################################## SLOW LOG ###################################
+
+# The Redis Slow Log is a system to log queries that exceeded a specified
+# execution time. The execution time does not include the I/O operations
+# like talking with the client, sending the reply and so forth,
+# but just the time needed to actually execute the command (this is the only
+# stage of command execution where the thread is blocked and can not serve
+# other requests in the meantime).
+#
+# You can configure the slow log with two parameters: one tells Redis
+# what is the execution time, in microseconds, to exceed in order for the
+# command to get logged, and the other parameter is the length of the
+# slow log. When a new command is logged the oldest one is removed from the
+# queue of logged commands.
+
+# The following time is expressed in microseconds, so 1000000 is equivalent
+# to one second. Note that a negative number disables the slow log, while
+# a value of zero forces the logging of every command.
+slowlog-log-slower-than 10000
+
+# There is no limit to this length. Just be aware that it will consume memory.
+# You can reclaim memory used by the slow log with SLOWLOG RESET.
+slowlog-max-len 1024
+
 ################################ VIRTUAL MEMORY ###############################
 
+### WARNING! Virtual Memory is deprecated in Redis 2.4
+### The use of Virtual Memory is strongly discouraged.
+
 # Virtual Memory allows Redis to work with datasets bigger than the actual
 # amount of RAM needed to hold the whole dataset in memory.
 # In order to do so very used keys are taken in memory while the other keys
@@ -322,6 +368,7 @@ no-appendfsync-on-rewrite no
 # VM parameters accordingly to your needs.
 
 vm-enabled no
+# vm-enabled yes
 
 # This is the path of the Redis swap file. As you can guess, swap files
 # can't be shared by different Redis instances, so make sure to use a swap
@@ -403,6 +450,12 @@ list-max-ziplist-value 64
 # set in order to use this special memory saving encoding.
 set-max-intset-entries 512
 
+# Similarly to hashes and lists, sorted sets are also specially encoded in
+# order to save a lot of space. This encoding is only used when the length and
+# elements of a sorted set are below the following limits:
+zset-max-ziplist-entries 128
+zset-max-ziplist-value 64
+
 # Active rehashing uses 1 millisecond every 100 milliseconds of CPU time in
 # order to help rehashing the main Redis hash table (the one mapping top-level
 # keys to values). The hash table implementation redis uses (see dict.c)
@@ -431,4 +484,4 @@ activerehashing yes
 # other files, so use this wisely.
 #
 # include /path/to/local.conf
-# include /path/to/other.conf
+# include /path/to/other.conf

data/templates/redis/config/rubber/rubber-redis.yml

@@ -0,0 +1,17 @@
+redis_server_version: 2.4.6
+redis_server_pid_file: /var/run/redis-server.pid
+redis_server_conf_file: /etc/redis.conf
+redis_server_log_file: /var/log/redis-server.log
+redis_server_port: 6379
+
+redis_db_dir: /mnt/redis
+
+gems: [redis]
+
+role_dependencies:
+  redis_master: [redis]
+  redis_slave: [redis]
+
+roles:
+  redis:
+    packages: []

data/templates/resque/config/resque-pool.yml

@@ -0,0 +1,4 @@
+# Stub for running in development
+# edit config/rubber/role/resque_worker/resque-pool.yml for production/staging
+
+"*": 2

data/templates/resque/config/rubber/common/resque.yml

@@ -0,0 +1,5 @@
+<%
+  @path = "#{Rubber.root}/config/resque.yml"
+%>
+
+<%= Rubber.env %>: <%= rubber_instances.for_role('redis').first.full_name rescue 'localhost' %>:6379