sml-rubber 0.9.1

data/generators/vulcanize/templates/cruise/config/rubber/role/web_tools/cruise-nginx.conf

@@ -0,0 +1,11 @@
+<%
+  cruise_host = rubber_instances.for_role('cruise').first
+  @path = "/etc/nginx/rubber/cruise.conf"
+%>
+
+<% if cruise_host %>
+location /cruise
+{
+  proxy_pass http://<%= cruise_host.full_name %>:<%= rubber_env.cruise_port %>;
+}
+<% end %>

data/generators/vulcanize/templates/cruise/config/rubber/rubber-cruise.yml

@@ -0,0 +1,18 @@
+
+# These will probably need to be used by the rubber module used for the internal http role
+
+# REQUIRED: This determines where cruisecontrol gets installed
+cruise_dir: /mnt/cruisecontrolrb
+cruise_data_dir: /root/.cruise
+cruise_project_dir: "#{cruise_data_dir}/projects/#{app_name}/work"
+
+cruise_port: 3333
+
+# REQUIRED: The git repo to fetch your project from
+# cruise_repository: git@github.com:mylogin/myproject.git
+
+roles:
+  cruise:
+    gems: [grit, mime-types, mongrel]
+    packages: [mysql-server]
+

data/generators/vulcanize/templates/cruise/templates.yml

@@ -0,0 +1 @@
+description: The cruisecontrol.rb (CI server) module

data/generators/vulcanize/templates/haproxy/config/rubber/deploy-haproxy.rb

@@ -0,0 +1,45 @@
+
+namespace :rubber do
+
+  namespace :haproxy do
+  
+    rubber.allow_optional_tasks(self)
+  
+    # serial_task can only be called after roles defined - not normally a problem, but
+    # rubber auto-roles don't get defined till after all tasks are defined
+    on :load do
+      rubber.serial_task self, :serial_restart, :roles => :haproxy do
+        run "/etc/init.d/haproxy restart"
+      end
+      rubber.serial_task self, :serial_reload, :roles => :haproxy do
+        run "if ! ps ax | grep -v grep | grep -c haproxy &> /dev/null; then /etc/init.d/haproxy start; else /etc/init.d/haproxy reload; fi"
+      end
+    end
+    
+    before "deploy:stop", "rubber:haproxy:stop"
+    after "deploy:start", "rubber:haproxy:start"
+    after "deploy:restart", "rubber:haproxy:reload"
+    
+    desc "Stops the haproxy server"
+    task :stop, :roles => :haproxy, :on_error => :continue do
+      run "/etc/init.d/haproxy stop"
+    end
+    
+    desc "Starts the haproxy server"
+    task :start, :roles => :haproxy do
+      run "/etc/init.d/haproxy start"
+    end
+    
+    desc "Restarts the haproxy server"
+    task :restart, :roles => :haproxy do
+      serial_restart
+    end
+  
+    desc "Reloads the haproxy web server"
+    task :reload, :roles => :haproxy do
+      serial_reload
+    end
+  
+  end
+
+end

data/generators/vulcanize/templates/haproxy/config/rubber/role/haproxy/haproxy-default.conf

@@ -0,0 +1,8 @@
+<%
+  @path = '/etc/default/haproxy'
+%>
+
+# Set ENABLED to 1 if you want the init script to start haproxy.
+ENABLED=1
+# Add extra flags here.
+#EXTRAOPTS="-de -m 16"

data/generators/vulcanize/templates/haproxy/config/rubber/role/haproxy/haproxy.conf

@@ -0,0 +1,44 @@
+<%
+  @path = "/etc/haproxy/haproxy.cfg"
+%>
+global
+  log 127.0.0.1 local0 warning
+  maxconn 1024
+ 
+defaults
+  log global
+  mode http
+  retries 3
+  balance roundrobin
+  option abortonclose
+  option redispatch
+  contimeout 4000
+  clitimeout 150000
+  srvtimeout 30000
+  
+listen mongrel_proxy 0.0.0.0:<%= rubber_env.haproxy_listen_port %>
+  <% 
+    # Since we do "fair" load balancing to the mongrels, we don't want to share mongrels 
+    # between haproxy instances otherwise we still end up with requests getting queued behind 
+    # slow ones.
+    app_hosts = rubber_instances.for_role('app').collect { |i| i.name }
+    web_hosts = rubber_instances.for_role('web').collect { |i| i.name }
+    ports = (rubber_env.appserver_base_port.to_i ... rubber_env.appserver_base_port.to_i + rubber_env.appserver_count.to_i).to_a
+    host_count = web_hosts.size
+    host_mod = web_hosts.sort.index(rubber_env.host) rescue 0
+    ports = ports.find_all {|x| x % host_count == host_mod}
+  %>
+  <% app_hosts.each do |server| %>
+    <% ports.each do |port| %> 
+      server <%= server %>_<%= port %> <%= server %>:<%= port %> maxconn 1 check
+    <% end %> 
+  <% end %>
+
+# Admin interface for proxy stats
+listen admin 0.0.0.0:<%= rubber_env.haproxy_admin_port %>
+  stats uri /haproxy
+
+# So monit can check on haproxy
+# simple response : 'OK'
+listen health_check 0.0.0.0:<%= rubber_env.haproxy_monitor_port %>
+  mode health

data/generators/vulcanize/templates/haproxy/config/rubber/role/haproxy/monit-haproxy.conf

@@ -0,0 +1,9 @@
+<%
+  @path = '/etc/monit/monit.d/monit-haproxy.conf'
+%>
+<% PIDFILE = "/var/run/haproxy.pid" %>
+check process haproxy with pidfile <%= PIDFILE %>
+  group haproxy-<%= RUBBER_ENV %>
+  start program = "/etc/init.d/haproxy start"
+  stop program = "/etc/init.d/haproxy stop"
+  if failed host <%= rubber_env.host %> port <%= rubber_env.haproxy_monitor_port %> with timeout 10 seconds for 10 cycles then restart

data/generators/vulcanize/templates/haproxy/config/rubber/role/haproxy/syslog-haproxy.conf

@@ -0,0 +1,6 @@
+<%
+  @path = "/etc/syslog.conf"
+  @post = "/etc/init.d/sysklogd restart"
+  @additive = ["# rubber-haproxy-start", "# rubber-haproxy-end"]
+%>
+local0.* /var/log/haproxy.log

data/generators/vulcanize/templates/haproxy/config/rubber/role/haproxy/syslogd-default.conf

@@ -0,0 +1,17 @@
+<%
+  @path = "/etc/default/syslogd"
+  @post = "/etc/init.d/sysklogd restart"
+%>
+#
+# Top configuration file for syslogd
+#
+
+#
+# Full documentation of possible arguments are found in the manpage
+# syslogd(8).
+#
+
+#
+# For remote UDP logging use SYSLOGD="-r"
+#
+SYSLOGD="-r"

data/generators/vulcanize/templates/haproxy/config/rubber/role/web_tools/haproxy-nginx.conf

@@ -0,0 +1,10 @@
+<%
+  @path = "/etc/nginx/rubber/haproxy.conf"
+%>
+
+<% rubber_instances.for_role('haproxy').each do |ic| %>
+  location /haproxy_<%= ic.name %>
+  {
+    proxy_pass http://<%= ic.full_name %>:<%= rubber_env.haproxy_admin_port %>;
+  }
+<% end %>

data/generators/vulcanize/templates/haproxy/config/rubber/rubber-haproxy.yml

@@ -0,0 +1,12 @@
+
+# Uses from appserver (mongrel) rubber config 
+# appserver_base_port: 8000
+# appserver_count: 8
+
+haproxy_monitor_port: 60001
+haproxy_listen_port: 9000
+haproxy_admin_port: 9100
+
+roles:
+  haproxy:
+    packages: [haproxy]

data/generators/vulcanize/templates/haproxy/templates.yml

@@ -0,0 +1 @@
+description: The haproxy module

data/generators/vulcanize/templates/memcached/config/memcached.yml

@@ -0,0 +1,28 @@
+defaults:
+  ttl: 1800
+  readonly: false
+  urlencode: false
+  c_threshold: 10000
+  compression: true
+  debug: false
+  namespace: app
+  sessions: false
+  session_servers: false
+  fragments: false
+  memory: 64
+  servers: localhost:11211
+  benchmarking: true
+  raise_errors: true
+  fast_hash: false
+  fastest_hash: false
+
+development:
+  sessions: false
+  fragments: true
+  servers: localhost:11211
+
+# turn off caching
+test: 
+  disabled: true
+  fragments: true
+

data/generators/vulcanize/templates/memcached/config/rubber/common/memcached.yml

@@ -0,0 +1,14 @@
+<%
+  @path = "#{RUBBER_ROOT}/config/memcached.yml"
+  @additive =  ["# rubber-memcached-start", "# rubber-memcached-end"]
+%>
+
+
+<%= RUBBER_ENV %>:
+  memory: 256
+  fragments: true
+  benchmarking: false
+  servers:
+  <% rubber_instances.for_role('memcached').each do |ic| %>
+    - <%= ic.full_name %>:11211
+  <% end %>

data/generators/vulcanize/templates/memcached/config/rubber/role/memcached/memcached.conf

@@ -0,0 +1,52 @@
+<%
+  @path = "/etc/memcached.conf"
+  @post = "/etc/init.d/memcached restart"
+%>
+
+# memcached default config file edited for mobicious
+# 2003 - Jay Bonci <jaybonci@debian.org>
+# This configuration file is read by the start-memcached script provided as
+# part of the Debian GNU/Linux distribution. 
+
+# Run memcached as a daemon. This command is implied, and is not needed for the
+# daemon to run. See the README.Debian that comes with this package for more
+# information.
+-d
+
+# Log memcached's output to /var/log/memcached
+logfile /var/log/memcached.log
+
+# Be verbose
+ -v
+
+# Be even more verbose (print client commands as well)
+# -vv
+
+# Start with a cap of 64 megs of memory. It's reasonable, and the daemon default
+# Note that the daemon will grow to this size, but does not start out holding this much
+# memory
+-m 64
+
+# Default connection port is 11211
+-p 11211 
+
+# Run the daemon as root. The start-memcached will default to running as root if no
+# -u command is present in this config file
+-u nobody
+
+# Specify which IP address to listen on. The default is to listen on all IP addresses
+# This parameter is one of the only security measures that memcached has, so make sure
+# it's listening on a firewalled interface.
+#-l 127.0.0.1
+
+# Limit the number of simultaneous incoming connections. The daemon default is 1024
+# -c 1024
+
+# Lock down all paged memory. Consult with the README and homepage before you do this
+# -k
+
+# Return error when memory is exhausted (rather than removing items)
+# -M
+
+# Maximize core file limit
+# -r

data/generators/vulcanize/templates/memcached/config/rubber/role/memcached/memcached_munin_plugin

@@ -0,0 +1,249 @@
+<%
+  @path = "/usr/share/munin/plugins/memcached"
+  @perms = 0755
+  @post = <<-POST
+    ln -sf #{@path} /etc/munin/plugins/memcached_rates
+    ln -sf #{@path} /etc/munin/plugins/memcached_bytes
+    ln -sf #{@path} /etc/munin/plugins/memcached_counters
+    /etc/init.d/munin-node restart
+  POST
+%>#!/usr/bin/perl
+#
+# Plugin to monitor memcache statistics
+#
+# This module has 3 seperate graph datasets:
+# rates
+# bytes
+# counters
+#
+# For each of them, symlink the memcached_ script to memcached_FOO where foo is
+# the datset name.
+#
+# Parameters supported:
+#
+#  config
+#  autoconf
+#
+# Configurable variables
+#
+# host  Host of the memcache daemon
+# port  Port of the memcache daemon
+#
+# Author: Robin H. Johnson <robbat2@gentoo.org>
+# Slightly based on the original version by Joshua Thijssen
+# <jthijssen@noxlogic.nl>
+#
+# Magic markers:
+#%# family=auto
+#%# capabilities=autoconf
+use strict;
+my $ret = undef; 
+if (! eval "require Cache::Memcached;") {
+	$ret = "Cache::Memcached not found";
+} 
+my $HOST = exists $ENV{'host'} ? $ENV{'host'} : "127.0.0.1";
+my $PORT = exists $ENV{'port'} ? $ENV{'port'} : 11211; 
+if ( exists $ARGV[0] and $ARGV[0] eq "autoconf" ) {
+	if ($ret) {
+		print "no ($ret)\n";
+		exit 1;
+	} 
+# Todo: we can always connect to a memcache server without any errors so I cannot really
+# find a way to detect the presence of a memcache instance. Maybe a forced write/read/delete
+# but there should be a better way somewhere...
+	print "yes\n";
+	exit 0;
+}
+
+if($ret) {
+	print "no ($ret)\n";
+	exit 1;
+} 
+
+# We do everything by this array
+my %all_vars = (
+		rates => {
+			master => {
+				graph_title => 'Memcached - Commands',
+				graph_args => '--base 1000',
+				graph_vlabel => '/${graph_period}',
+				graph_category => 'memcache',
+			},
+			memcache_cache_hits => {
+				label => 'Cache hits',
+				type => 'DERIVE',
+				min => '0',
+				max => '5000',
+				draw => 'LINE2',
+				info => 'Number of cache hits',
+				stat_group => 'misc',
+				stat_name => 'get_hits'
+			},
+			memcache_cache_misses  => {
+				label => 'Cache misses',
+				type => 'DERIVE',
+				min => '0',
+				max => '5000',
+				draw => 'LINE2',
+				info => 'Number of cache misses',
+				stat_group => 'misc',
+				stat_name => 'get_misses'
+			},
+			memcache_cmd_get => {
+				label => 'GET requests',
+				type => 'DERIVE',
+				min => '0',
+				max => '5000',
+				draw => 'LINE2',
+				info => 'Number of GET commands seen',
+				stat_group => 'misc',
+				stat_name => 'cmd_get'
+			},
+			memcache_cmd_set => {
+				label => 'SET requests',
+				type => 'DERIVE',
+				min => '0',
+				max => '5000',
+				draw => 'LINE2',
+				info => 'Number of SET commands seen',
+				stat_group => 'misc',
+				stat_name => 'cmd_set'
+			},
+			memcache_total_items => {
+				label => 'New items*5',
+				type => 'DERIVE',
+				min => '0',
+				max => '5000',
+				draw => 'LINE2',
+				info => 'New items*5',
+				cdef => 'memcache_total_items,5,*',
+				stat_group => 'misc',
+				stat_name => 'total_items'
+			},
+			memcache_total_connections => {
+				label => 'New connections*100',
+				type => 'DERIVE',
+				min => '0',
+				max => '5000',
+				cdef => 'memcache_total_connections,100,*',
+				draw => 'LINE2',
+				info => 'New connections*100',
+				stat_group => 'misc',
+				stat_name => 'total_connections'
+			},
+		},
+		bytes => {
+			master => {
+				graph_title => 'Memcached - Network traffic',
+				graph_args => '--base 1000',
+				graph_vlabel => 'bytes in(-)/out(+) per ${graph_period}',
+				graph_category => 'memcache',
+			},
+			memcache_bytes_read => {
+				label => 'Bytes read',
+				type => 'COUNTER',
+				draw => 'LINE2',
+				max => '1000000',
+				info => 'Bytes read from network',
+				graph => 'no',
+				stat_group => 'misc',
+				stat_name => 'bytes_read'
+			},
+			memcache_bytes_written => {
+				label => 'Bytes written',
+				negative => 'memcache_bytes_read',
+				type => 'COUNTER',
+				max => '1000000',
+				draw => 'LINE2',
+				info => 'Bytes written to network',
+				stat_group => 'misc',
+				stat_name => 'bytes_written'
+			},
+		},
+		counters => {
+			master => {
+				graph_title => 'Memcached - Current values',
+				graph_args => '--base 1000',
+				#graph_args => '--base 1000 --loga',
+				graph_vlabel => 'Totals',
+				graph_category => 'memcache',
+				graph_scale => 'no',
+			},
+			memcache_curr_items => {
+				label => 'Current items',
+				type => 'GAUGE',
+				min => '0',
+				draw => 'LINE2',
+				info => 'Number of items in cache',
+				stat_group => 'misc',
+				stat_name => 'curr_items'
+			},
+			memcache_curr_connections => {
+				label => 'Current connections*100',
+				type => 'GAUGE',
+				min => '0',
+				draw => 'LINE2',
+				cdef => 'memcache_curr_connections,100,*',
+				info => 'Number of connections*100',
+				stat_group => 'misc',
+				stat_name => 'curr_connections'
+			},
+			memcache_bytes_allocated => {
+				label => 'Bytes allocated (KiB)',
+				type => 'GAUGE',
+				min => '0',
+				draw => 'LINE2',
+				cdef => 'memcache_bytes_allocated,1024,/',
+				info => 'Bytes allocated (KiB)',
+				stat_group => 'misc',
+				stat_name => 'bytes'
+			},
+		}
+		);
+
+$0 =~ /memcached_(.+)*$/;
+my $func = $1;
+exit 2 unless defined $func;
+my %vars = %{$all_vars{$func}};
+
+#	   STAT rusage_user 3941.052868
+#	   STAT rusage_system 18436.366246
+#	   STAT connection_structures 1112
+#	   STAT bytes 382985002
+#	   STAT limit_maxbytes 536870912
+
+if ( exists $ARGV[0] and $ARGV[0] eq "config" ) {
+	my %v = %{$vars{'master'}};
+	foreach my $k ( keys %v ) {
+		print "$k ".$v{$k}."\n"
+	}
+	print 'graph_order ';
+	foreach my $k ( sort(keys %vars) ) {
+		if($k eq 'master') { next; }
+		print $k." ";
+	}
+	print "\n";
+	foreach my $k ( sort(keys %vars) ) {
+		if($k eq 'master') { next; }
+		my %v = %{$vars{$k}};
+		foreach my $k2 (keys %v) {
+			if($k2 eq 'stat_group' or $k2 eq 'stat_name') { next; }
+			print "$k.$k2 ".$v{"$k2"}."\n";
+		}
+	}
+	exit 0;
+} 
+my $mc = new Cache::Memcached { 'servers' => [ "$HOST:$PORT" ] };
+my $stats = $mc->stats ('misc');
+
+foreach my $k ( sort(keys %vars) ) {
+	if($k eq 'master') { next; }
+	my %v = %{$vars{$k}};
+	if($v{type} eq 'COMPUTE') { next; }
+	my $sg = $v{stat_group};
+	my $sn = $v{stat_name};
+	my $value = $stats->{hosts}->{"$HOST:$PORT"}->{$sg}->{$sn};
+	defined($value) or $value = 'U';
+	print "$k.value ".$value."\n";
+}
+# vim:syntax=perl ts=4 sw=4: