rubber 1.15.0 → 2.0.0.pre1

data/{lib/generators/vulcanize/templates → templates}/base/config/rubber/rubber.yml

@@ -6,7 +6,7 @@ app_user: app
 
 # REQUIRED: Notification emails (e.g. monit) get sent to this address
 #
-admin_email: root@localhost
+admin_email: "root@#{full_host}"
 
 # OPTIONAL: If not set, you won't be able to access web_tools
 # server (munin stats, monit status, etc)
@@ -26,26 +26,27 @@ domain: foo.com
 # dns records (CNAME, MX, Round Robin DNS, etc)
 
 # OPTIONAL: Additional rubber file to pull config from if it exists.  This file will
-# also be pushed to remote host at RUBBER_ROOT/config/rubber/rubber-secret.yml
+# also be pushed to remote host at Rubber.root/config/rubber/rubber-secret.yml
 #
-# rubber_secret: "#{File.expand_path('~') + '/.ec2' + (RUBBER_ENV == 'production' ? '' : '_dev') + '/rubber-secret.yml' rescue ''}"
+# rubber_secret: "#{File.expand_path('~') + '/.ec2' + (Rubber.env == 'production' ? '' : '_dev') + '/rubber-secret.yml' rescue ''}"
 
 # REQUIRED All known cloud providers with the settings needed to configure them
-# There's only one coud provider right now - Amazon Web Services
-# To implement another, clone lib/rubber/cloud/aws.rb
+# There's only one working cloud provider right now - Amazon Web Services
+# To implement another, clone lib/rubber/cloud/aws.rb or make the fog provider 
+# work in a generic fashion
 #
 cloud_providers:
   aws:
-    # REQUIRED The endpoint for the region server that you want to use.
+    # REQUIRED The AWS region that you want to use.
     # 
     # Options include
-    # us-east-1.ec2.amazonaws.com
-    # eu-west-1.ec2.amazonaws.com
-    # ap-northeast-1.ec2.amazonaws.com
-    # ap-southeast-1.ec2.amazonaws.com
-    # ap-southeast-2.ec2.amazonaws.com
+    # us-east-1
+    # eu-west-1
+    # ap-northeast-1
+    # ap-southeast-1
+    # ap-southeast-2
     #
-    server_endpoint: us-east-1.ec2.amazonaws.com
+    region: us-east-1
     
     # REQUIRED The amazon keys and account ID (digits only, no dashes) used to access the AWS API
     #
@@ -74,12 +75,12 @@ cloud_providers:
 
     # REQUIRED: the ami and instance type for creating instances
     # The Ubuntu images at http://alestic.com/ work well
-    # Ubuntu 10.04 Lucid	base install	32-bit	ami-7000f019
-    # Ubuntu 10.04 Lucid	base install	64-bit	ami-fa01f193
+    # Ubuntu 10.04 Lucid	base install	32-bit	ami-4fd00726
+    # Ubuntu 10.04 Lucid	base install	64-bit	ami-35de095c
     #
     # m1.small or m1.large or m1.xlarge
     image_type: m1.small
-    image_id: "#{ %w{m1.small c1.medium}.include?(image_type) ? 'ami-7000f019' : 'ami-fa01f193' }"
+    image_id: "#{ %w{m1.small c1.medium}.include?(image_type) ? 'ami-4fd00726' : 'ami-35de095c' }"
 
     # OPTIONAL: EC2 spot instance request support.
     #
@@ -93,11 +94,44 @@ cloud_providers:
     # If a spot instance request can't be fulfilled in 3 minutes, fallback to on-demand instance creation.  If not set,
     # the default is infinite.
     # spot_instance_request_timeout: 180
+    
+  # Use an alternate cloud provider supported by fog.  This doesn't fully work
+  # yet due to differences in providers within fog, but gives you a starting
+  # point for contributing a new provider to rubber.  See rubber/lib/rubber/cloud(.rb)
+  fog:
+    credentials:
+      provider: rackspace
+      rackspace_api_key: 'XXX'
+      rackspace_username: 'YYY'
+    image_type: 123
+    image_id: 123
 
 # REQUIRED the cloud provider to use
 #
 cloud_provider: aws
 
+# OPTIONAL: Where to store instance data.
+# 
+# Allowed forms are:
+# filesystem: "file:#{Rubber.root}/config/rubber/instance-#{Rubber.env}.yml"
+# cloud storage (s3): "storage:#{cloud_provider.aws.backup_bucket}/RubberInstances_#{app_name}/instance-#{Rubber.env}.yml"
+# cloud table (simpledb): "table:RubberInstances_#{app_name}_#{Rubber.env}"
+#
+# If you need to port between forms, load the rails console then:
+# Rubber.instances.save(location)
+# where location is one of the allowed forms for this variable
+#
+# instance_storage: "file:#{Rubber.root}/config/rubber/instance-#{Rubber.env}.yml"
+
+# OPTIONAL: Where to store a backup of the instance data
+#
+# This is most useful when using a remote store in case you end up
+# wiping the single copy of your instance data.  When using the file
+# store, the instance file is typically under version control with
+# your project code, so that provides some safety.
+#
+# instance_storage_backup: "storage:#{cloud_provider.aws.backup_bucket}/RubberInstances_#{app_name}/instance-#{Rubber.env}-#{Time.now.strftime('%Y%m%d-%H%M%S')}.yml"
+
 # OPTIONAL: Define security groups
 # Each security group is a name associated with a sequence of maps where the
 # keys are the parameters to the ec2 AuthorizeSecurityGroupIngress API
@@ -136,7 +170,7 @@ prompt_for_security_group_sync: true
 # OPTIONAL: The packages to install on all instances
 # You can install a specific version of a package by using a sub-array of pkg, version
 # For example, packages: [[rake, 0.7.1], irb]
-packages: [postfix, build-essential, git-core, ec2-ami-tools]
+packages: [postfix, build-essential, git-core, ec2-ami-tools, libxslt-dev]
 
 # OPTIONAL: gem sources to setup for rubygems
 # gemsources: ["http://rubygems.org", "http://gems.github.com"]
@@ -154,10 +188,10 @@ stop_on_error_cmd: "function error_exit { exit 99; }; trap error_exit ERR"
 
 # OPTIONAL: The default set of roles to use when creating a staging instance
 # with "cap rubber:create_staging".  By default this uses all the known roles,
-# but this is not always desired for staging, so you can specify a different
-# set here
+# excluding slave roles, but this is not always desired for staging, so you can
+# specify a different set here
 #
-staging_roles: "web,app,db:primary=true"
+# staging_roles: "web,app,db:primary=true"
 
 
 # OPTIONAL: Lets one assign amazon elastic IPs (static IPs) to your instances
@@ -173,7 +207,7 @@ staging_roles: "web,app,db:primary=true"
 #           affect instances in another.  As such, it is good to specify these
 #           for instances that need to be redundant to reduce your chance of
 #           downtime. You should typically set this on the role/host level
-#           rather than globally.  Use rubber describe:zones to see the list
+#           rather than globally.  Use cap rubber:describe_zones to see the list
 #           of zones
 # availability_zone: us-east-1a
 

data/templates/base/script/rubber

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+export RUBBER_ROOT=`dirname $0`/..
+export BUNDLE_GEMFILE=$RUBBER_ROOT/Gemfile
+
+# run through "rvm do" so we work in crontab - default rvm ruby
+# version is set during bootstrap
+rvm default do bundle exec rubber "$@"

data/templates/base/templates.rb

@@ -0,0 +1,7 @@
+if Rubber::Util::is_bundler?
+  append_to_file "Gemfile", "gem 'rubber', '#{Rubber.version}'\n"
+  
+  # for cron-sh
+  append_to_file 'Gemfile', "gem 'open4'\n"
+  
+end

data/templates/base/templates.yml

@@ -0,0 +1 @@
+description: The core configuration needed by all others

data/{lib/generators/vulcanize/templates → templates}/cassandra/config/rubber/deploy-cassandra.rb

@@ -32,7 +32,7 @@ namespace :rubber do
             deploy.update_code
 
             # Gen just the conf for cassandra
-            rubber.run_config(:RUBBER_ENV => RUBBER_ENV, :FILE => "role/cassandra", :FORCE => true, :deploy_path => release_path)
+            rubber.run_config(:file => "role/cassandra", :force => true, :deploy_path => release_path)
 
             cassandra_start
             

data/templates/collectd/config/rubber/deploy-collectd.rb

@@ -0,0 +1,48 @@
+
+namespace :rubber do
+  
+  namespace :collectd do
+  
+    rubber.allow_optional_tasks(self)
+
+    after "rubber:bootstrap", "rubber:collectd:bootstrap"
+    after "deploy:restart", "rubber:collectd:restart_rubber_plugin"
+
+
+    task :bootstrap, :roles => :collectd do
+      exists = capture("echo $(grep Rubber /etc/collectd/collectd.conf 2> /dev/null)")
+      if exists.strip.size == 0
+        rubber.update_code_for_bootstrap
+
+        rubber.run_config(:file => "role/collectd", :force => true, :deploy_path => release_path)
+
+        restart
+      end
+    end
+
+    desc "Start collectd system monitoring"
+    task :start, :roles => :collectd do
+      rsudo "service collectd start"
+    end
+    
+    desc "Stop collectd system monitoring"
+    task :stop, :roles => :collectd do
+      rsudo "service collectd stop || true"
+    end
+    
+    desc "Restart collectd system monitoring"
+    task :restart, :roles => :collectd do
+      stop
+      start
+    end
+
+    desc "Restart collectd rubber plugin"
+    task :restart_rubber_plugin, :roles => :collectd do
+      # Need to kill rubber collectd runner script to force collectd to restart
+      # it after deploy so that the runner script gets the new paths
+      rsudo "pkill -fn #{rubber_env.rubber_collectd_runner.sub(/./, '[\0]')} ; exit 0"
+    end
+
+  end
+
+end

data/templates/collectd/config/rubber/role/collectd/collectd.conf

@@ -0,0 +1,69 @@
+<%
+  rubber_scripts = `#{Rubber.root}/#{rubber_env.rubber_collectd_runner} config`.split("\n") rescue []
+
+  @path = '/etc/collectd/collectd.conf'
+  @post = "mkdir -p /etc/collectd/conf.d/ /etc/collectd/thresholds.d/ /etc/collectd/filters.d/ /etc/collectd/plugins/ "
+%>
+# Config file for collectd(1). Generated by Rubber
+#
+# Some plugins need additional configuration and are disabled by default.
+# Please read collectd.conf(5) for details.
+#
+# You should also read /usr/share/doc/collectd-core/README.Debian.plugins
+# before enabling any more plugins.
+
+Hostname "<%= rubber_env.host %>"
+#BaseDir "/var/lib/collectd"
+#PluginDir "/usr/lib/collectd"
+TypesDB "/usr/share/collectd/types.db" "/etc/collectd/types.db"
+Interval 60
+#Timeout 2
+#ReadThreads 5
+
+LoadPlugin syslog
+
+<Plugin syslog>
+	LogLevel info
+</Plugin>
+
+LoadPlugin cpu
+LoadPlugin df
+LoadPlugin disk
+LoadPlugin exec
+LoadPlugin entropy
+LoadPlugin interface
+LoadPlugin irq
+LoadPlugin load
+LoadPlugin memory
+LoadPlugin processes
+#LoadPlugin python
+LoadPlugin rrdtool
+<% if `swapon -s | wc -l`.to_i > 1 %>
+LoadPlugin swap
+<% end %>
+#LoadPlugin table
+#LoadPlugin tail
+#LoadPlugin tcpconns
+LoadPlugin users
+
+LoadPlugin notify_email
+
+<Plugin "notify_email">
+  From "collectd@<%= rubber_env.full_host %>"
+  Recipient "<%= rubber_env.admin_email %>"
+  Subject "Collectd notify: %s@%s"
+</Plugin>
+
+<%
+  if rubber_scripts.size > 0
+%>
+
+<Plugin exec>
+	Exec <%= rubber_env.app_user %> "/bin/bash" "-l" "-c" "cd <%= "/mnt/#{rubber_env.app_name}-#{Rubber.env}/current" %> && bundle exec <%= rubber_env.rubber_collectd_runner %>"
+</Plugin>
+
+<% end %>
+
+Include "/etc/collectd/conf.d/*.conf"
+Include "/etc/collectd/thresholds.d/*.conf"
+Include "/etc/collectd/filters.d/*.conf"

data/templates/collectd/config/rubber/role/collectd/filters.conf

@@ -0,0 +1,35 @@
+<%
+  @path = '/etc/collectd/filters.d/filters.conf'
+%>
+# Filter configuration for collectd(1).
+#
+# See the section "FILTER CONFIGURATION" in collectd.conf(5) for details.
+
+#PreCacheChain "PreCache"
+#PostCacheChain "PostCache"
+
+#LoadPlugin match_empty_counter
+#LoadPlugin match_hashed
+#LoadPlugin match_regex
+#LoadPlugin match_timediff
+#LoadPlugin match_value
+
+#LoadPlugin target_notification
+#LoadPlugin target_replace
+#LoadPlugin target_scale
+#LoadPlugin target_set
+
+#<Chain "PreCache">
+#	<Rule "no_fqdn">
+#		<Match "regex">
+#			Host "^[^\.]*$"
+#			Invert false
+#		</Match>
+#		Target "stop"
+#	</Rule>
+#</Chain>
+
+#<Chain "PostCache">
+#	Target "write"
+#</Chain>
+

data/templates/collectd/config/rubber/role/collectd/thresholds.conf

@@ -0,0 +1,76 @@
+<%
+  is_worker_instance = rubber_instances[rubber_env.host].role_names.include?('resque_worker')
+
+  @path = '/etc/collectd/thresholds.d/thresholds.conf'
+%>
+# Threshold configuration for collectd(1).
+#
+# See the section "THRESHOLD CONFIGURATION" in collectd.conf(5) for details.
+
+<Threshold>
+
+  <Type "df">
+    DataSource "used"
+    WarningMax 80
+    FailureMax 95
+    Percentage true
+  </Type>
+
+  <Type "load">
+    DataSource "midterm"
+    WarningMax <%= is_worker_instance ? 100 : 20 %>
+    Hysteresis 0.3
+  </Type>
+
+</Threshold>
+
+#<Threshold>
+#	<Type "counter">
+#		WarningMin 0.00
+#		WarningMax 1000.00
+#		FailureMin 0
+#		FailureMax 1200.00
+#		Invert false
+#		Persist false
+#		Instance "some_instance"
+#	</Type>
+#
+#	<Type "df">
+#		WarningMax 90
+#		Percentage true
+#	</Type>
+#
+#	<Type "load">
+#		DataSource "midterm"
+#		WarningMax 1
+#		Hysteresis 0.3
+#	</Type>
+#
+#	<Type "cpu">
+#		Instance "user"
+#		WarningMax 85
+#		Hits 6
+#	</Type>
+#
+#	<Plugin "interface">
+#		Instance "eth0"
+#		<Type "if_octets">
+#			DataSource "rx"
+#			FailureMax 10000000
+#		</Type>
+#	</Plugin>
+#
+#	<Host "hostname">
+#		<Type "cpu">
+#			Instance "idle"
+#			FailureMin 10
+#		</Type>
+#
+#		<Plugin "memory">
+#			<Type "memory">
+#				Instance "cached"
+#				WarningMin 100000000
+#			</Type>
+#		</Plugin>
+#	</Host>
+#</Threshold>

data/templates/collectd/config/rubber/role/collectd/types.db

@@ -0,0 +1,5 @@
+<%
+  @path = '/etc/collectd/types.db'
+%>
+
+domains_by_segment    domain_count:GAUGE:0:U, avg_user_count:GAUGE:0:U, avg_storage:GAUGE:0:U

data/templates/collectd/config/rubber/role/passenger/collectd-passenger.conf

@@ -0,0 +1,7 @@
+<%
+  @path = "/etc/collectd/conf.d/passenger_info.conf"
+%>
+
+<Plugin exec>
+	Exec nobody "/bin/bash" "-l" "-c" "/etc/collectd/plugins/passenger_info.rb"
+</Plugin>

data/templates/collectd/config/rubber/role/passenger/collectd-passenger.rb.conf

@@ -0,0 +1,77 @@
+<%
+  @path = "/etc/collectd/plugins/passenger_info.rb"
+  @perms = 0755
+-%>
+#!/usr/bin/env ruby
+
+
+# Direct api access tricky to get working right with sudo, etc, so commenting out
+# and shelling out to passenger provided scripts for now
+#
+#  require 'phusion_passenger'
+#  require 'phusion_passenger/platform_info'
+#  require 'phusion_passenger/admin_tools/memory_stats'
+#  require 'phusion_passenger/admin_tools/server_instance'
+#
+#  procs = PhusionPassenger::AdminTools::MemoryStats.new.passenger_processes
+#  total_rss = procs.inject(0) {|sum, val| sum + val.rss }
+#  puts "PUTVAL #{HOSTNAME}/passenger/gauge-total_rss interval=#{INTERVAL} N:#{total_rss}"
+#
+#  server_instance = PhusionPassenger::AdminTools::ServerInstance.list.first
+#  server_instance.connect(:passenger_status) do
+#    stats = server_instance.stats
+#    puts "PUTVAL #{HOSTNAME}/passenger/gauge-instances_max interval=#{INTERVAL} N:#{stats.max}"
+#    puts "PUTVAL #{HOSTNAME}/passenger/gauge-instances_count interval=#{INTERVAL} N:#{stats.count}"
+#    puts "PUTVAL #{HOSTNAME}/passenger/gauge-instances_active interval=#{INTERVAL} N:#{stats.active}"
+#
+#    sessions = server_instance.groups.collect {|group| group.processes.collect {|process| process.sessions } }.flatten
+#    total_sessions = sessions.inject(0) {|sum, val| sum + val }
+#    puts "PUTVAL #{HOSTNAME}/passenger/gauge-total_sessions interval=#{INTERVAL} N:#{total_sessions}"
+#  end
+
+STDOUT.sync = STDERR.sync = true
+HOSTNAME = ENV["COLLECTD_HOSTNAME"] || `hostname -f`.chomp
+INTERVAL = (ENV["COLLECTD_INTERVAL"] || 60).to_i
+
+loop do
+  start_time = Time.now.to_i
+
+  status = `sudo <%= rubber_env.rvm_gem_home %>/bin/passenger-status`
+  unless $?.success?
+    $stderr.puts "failed executing passenger-status"
+    exit 1
+  end
+
+  if status =~ /max\s+=\s+(\d+)/
+    puts "PUTVAL #{HOSTNAME}/passenger/gauge-instances_max interval=#{INTERVAL} N:#{$1}"
+  end
+
+  if status =~ /count\s+=\s+(\d+)/
+    puts "PUTVAL #{HOSTNAME}/passenger/gauge-instances_count interval=#{INTERVAL} N:#{$1}"
+  end
+
+  if status =~ /active\s+=\s+(\d+)/
+    puts "PUTVAL #{HOSTNAME}/passenger/gauge-instances_active interval=#{INTERVAL} N:#{$1}"
+  end
+
+  total_sessions = 0
+  status.scan(/Sessions: (\d+)/).flatten.each { |count| total_sessions += count.to_i }
+  puts "PUTVAL #{HOSTNAME}/passenger/gauge-total_sessions interval=#{INTERVAL} N:#{total_sessions}"
+
+
+  status = `sudo <%= rubber_env.rvm_gem_home %>/bin/passenger-memory-stats | tail -1`
+  unless $?.success?
+    $stderr.puts "failed executing passenger-memory-stats"
+    exit 1
+  end
+
+  if status =~ /(\d+\.\d+)/
+    puts "PUTVAL #{HOSTNAME}/passenger/gauge-total_rss interval=#{INTERVAL} N:#{$1}"
+  end
+
+  run_time = Time.now.to_i - start_time
+  sleep_time = INTERVAL - run_time
+  sleep_time  = INTERVAL if sleep_time < 0
+
+  sleep sleep_time
+end