rubber 2.12.2 → 2.13.0

data/templates/graphite/config/rubber/role/graphite_server/graphite-carbon-default.conf

@@ -0,0 +1,8 @@
+<%
+  is_old_ubuntu = %w[10.04 12.04].include?(rubber_instance.os_version)
+
+  @path = '/etc/default/graphite-carbon'
+  @skip = is_old_ubuntu
+%>
+# Change to true, to enable carbon-cache on boot
+CARBON_CACHE_ENABLED=true

data/templates/graphite/config/rubber/role/graphite_server/graphite_server-upstart.conf

@@ -1,6 +1,9 @@
 <%
   @path = "/etc/init/graphite-server.conf"
   @backup = false
+
+  is_old_ubuntu = %w[10.04 12.04].include?(rubber_instance.os_version)
+  @skip = ! is_old_ubuntu
 %>
 description "graphite server"
 

data/templates/graphite/config/rubber/role/graphite_server/storage-schemas.conf

@@ -1,5 +1,6 @@
 <%
-  @path = '/opt/graphite/conf/storage-schemas.conf'
+  is_old_ubuntu = %w[10.04 12.04].include?(rubber_instance.os_version)
+  @path = is_old_ubuntu ? "#{rubber_env.graphite_dir}/conf/storage-schemas.conf" : '/etc/carbon/storage-schemas.conf'
 %>
 
 # Schema definitions for whisper files. Entries are scanned in order,

data/templates/graphite/config/rubber/role/graphite_web/crontab

@@ -5,4 +5,4 @@
 %>
 
 # Backup graphite web UI db every day
-0 1 * * * <%= Rubber.root %>/script/rubber cron --task util:backup --directory /mnt/graphite_backups --name graphite --command "nice zip \%dir\%/\%name\%_<%= rubber_env.host %>_\%time_stamp\%.zip /opt/graphite/storage/*.db"
+0 1 * * * <%= Rubber.root %>/script/rubber cron --task util:backup --directory /mnt/graphite_backups --name graphite --command "nice zip #{dir}/#{name}_<%= rubber_env.host %>_#{time_stamp}.zip <%= rubber_env.graphite_storage_dir %>/*.db"

data/templates/graphite/config/rubber/role/graphite_web/dashboard.conf

@@ -1,5 +1,5 @@
 <%
-  @path = '/opt/graphite/conf/dashboard.conf'
+  @path = "#{rubber_env.graphite_dir}/conf/dashboard.conf"
 %>
 # This configuration file controls the behavior of the Dashboard UI, available
 # at http://my-graphite-server/dashboard/.

data/templates/graphite/config/rubber/role/graphite_web/dashboard.html

@@ -1,6 +1,6 @@
 <!--
 <%
-  @path = "/opt/graphite/webapp/content/dashboard.html"
+  @path = "#{rubber_env.graphite_dir}/webapp/content/dashboard.html"
 %>
 
 Simple graphite dashboard cycler for use on a large display - e.g. googletv/chrome on a wall mounted TV

data/templates/graphite/config/rubber/role/graphite_web/graphite.wsgi

@@ -1,5 +1,8 @@
 <%
-  @path = '/opt/graphite/conf/graphite.wsgi'
+  is_old_ubuntu = %w[10.04 12.04].include?(rubber_instance.os_version)
+
+  @path = "#{rubber_env.graphite_dir}/conf/graphite.wsgi"
+  @skip = ! is_old_ubuntu
 %>
 import os, sys
 sys.path.append('/opt/graphite/webapp')

data/templates/graphite/config/rubber/role/graphite_web/local_settings.py

@@ -1,31 +1,128 @@
 <%
-  @path = '/opt/graphite/webapp/graphite/local_settings.py'
+  is_old_ubuntu = %w[10.04 12.04].include?(rubber_instance.os_version)
+
+  @path = is_old_ubuntu ? "#{rubber_env.graphite_dir}/webapp/graphite/local_settings.py" : '/etc/graphite/local_settings.py'
 %>
-# Edit this file to override the default graphite settings, do not edit settings.py!!!
+## Graphite local_settings.py
+# Edit this file to customize the default Graphite webapp settings
+#
+# Additional customizations to Django settings can be added to this file as well
+
+#####################################
+# General Configuration #
+#####################################
+<% unless is_old_ubuntu %>
+# Set this to a long, random unique string to use as a secret key for this
+# install. This key is used for salting of hashes used in auth tokens,
+# CRSF middleware, cookie storage, etc. This should be set identically among
+# instances if used behind a load balancer.
+SECRET_KEY = '<%= rubber_env.graphite_web_secret %>'
+<% end %>
 
-# Turn on debugging and restart apache if you ever see an "Internal Server Error" page
-#DEBUG = True
+# In Django 1.5+ set this to the list of hosts your graphite instances is
+# accessible as. See:
+# https://docs.djangoproject.com/en/dev/ref/settings/#std:setting-ALLOWED_HOSTS
+ALLOWED_HOSTS = [ '*' ]
 
-# Set your local timezone (django will *try* to figure this out automatically)
+# Set your local timezone (Django's default is America/Chicago)
 # If your graphs appear to be offset by a couple hours then this probably
 # needs to be explicitly set to your local timezone.
 TIME_ZONE = 'America/New_York'
 
-# Uncomment these to enable more performance-related logging
-#LOG_RENDERING_PERFORMANCE = True
-#LOG_CACHE_PERFORMANCE = True
+# Override this to provide documentation specific to your Graphite deployment
+#DOCUMENTATION_URL = "http://graphite.readthedocs.org/"
+
+# Logging
+# True see: https://answers.launchpad.net/graphite/+question/159731
+LOG_RENDERING_PERFORMANCE = True
+LOG_CACHE_PERFORMANCE = True
+LOG_METRIC_ACCESS = True
+
+# Enable full debug page display on exceptions (Internal Server Error pages)
+DEBUG = True
+
+# If using RRD files and rrdcached, set to the address or socket of the daemon
+#FLUSHRRDCACHED = 'unix:/var/run/rrdcached.sock'
+
+# This lists the memcached servers that will be used by this webapp.
+# If you have a cluster of webapps you should ensure all of them
+# have the *exact* same value for this setting. That will maximize cache
+# efficiency. Setting MEMCACHE_HOSTS to be empty will turn off use of
+# memcached entirely.
+#
+# You should not use the loopback address (127.0.0.1) here if using clustering
+# as every webapp in the cluster should use the exact same values to prevent
+# unneeded cache misses. Set to [] to disable caching of images and fetched data
+#MEMCACHE_HOSTS = ['10.10.10.10:11211', '10.10.10.11:11211', '10.10.10.12:11211']
+#DEFAULT_CACHE_DURATION = 60 # Cache images and data for 1 minute
+
+
+#####################################
+# Filesystem Paths #
+#####################################
+# Change only GRAPHITE_ROOT if your install is merely shifted from /opt/graphite
+# to somewhere else
+<% if is_old_ubuntu %>
+#GRAPHITE_ROOT = '/opt/graphite'
+<% else %>
+GRAPHITE_ROOT = '/usr/share/graphite-web'
+<% end %>
+
+# Most installs done outside of a separate tree such as /opt/graphite will only
+# need to change these three settings. Note that the default settings for each
+# of these is relative to GRAPHITE_ROOT
+<% if is_old_ubuntu %>
+#CONF_DIR = '/opt/graphite/conf'
+#STORAGE_DIR = '/opt/graphite/storage'
+#CONTENT_DIR = '/opt/graphite/webapp/content'
+<% else %>
+CONF_DIR = '/etc/graphite'
+STORAGE_DIR = '<%= rubber_env.graphite_storage_dir %>'
+CONTENT_DIR = '/usr/share/graphite-web/static'
+<% end %>
 
-# Override this if you need to provide documentation specific to your graphite deployment
-#DOCUMENTATION_URL = "http://wiki.mycompany.com/graphite"
+# To further or fully customize the paths, modify the following. Note that the
+# default settings for each of these are relative to CONF_DIR and STORAGE_DIR
+#
+## Webapp config files
+#DASHBOARD_CONF = '/opt/graphite/conf/dashboard.conf'
+#GRAPHTEMPLATES_CONF = '/opt/graphite/conf/graphTemplates.conf'
 
-# Enable email-related features
-#SMTP_SERVER = "mail.mycompany.com"
+## Data directories
+# NOTE: If any directory is unreadable in DATA_DIRS it will break metric browsing
+<% if is_old_ubuntu %>
+#WHISPER_DIR = '/opt/graphite/storage/whisper'
+#RRD_DIR = '/opt/graphite/storage/rrd'
+#DATA_DIRS = [WHISPER_DIR, RRD_DIR] # Default: set from the above variables
+#LOG_DIR = '/opt/graphite/storage/log/webapp'
+#INDEX_FILE = '/opt/graphite/storage/index'  # Search index file
+<% else %>
+#WHISPER_DIR = '/var/lib/graphite/whisper'
+#RRD_DIR = '/opt/graphite/storage/rrd'
+#DATA_DIRS = [WHISPER_DIR, RRD_DIR] # Default: set from the above variables
+LOG_DIR = '/var/log/graphite'
+#INDEX_FILE = '/var/lib/graphite/search_index'  # Search index file
+<% end %>
+
+#####################################
+# Email Configuration #
+#####################################
+# This is used for emailing rendered Graphs
+# Default backend is SMTP
+#EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'
+#EMAIL_HOST = 'localhost'
+#EMAIL_PORT = 25
+#EMAIL_HOST_USER = ''
+#EMAIL_HOST_PASSWORD = ''
+#EMAIL_USE_TLS = False
+# To drop emails on the floor, enable the Dummy backend:
+#EMAIL_BACKEND = 'django.core.mail.backends.dummy.EmailBackend'
 
 
 #####################################
-# LDAP Authentication Configuration #
+# Authentication Configuration #
 #####################################
-# LDAP / ActiveDirectory authentication setup
+## LDAP / ActiveDirectory authentication setup
 #USE_LDAP_AUTH = True
 #LDAP_SERVER = "ldap.mycompany.com"
 #LDAP_PORT = 389
@@ -47,24 +144,45 @@ TIME_ZONE = 'America/New_York'
 #ldap.set_option(ldap.OPT_X_TLS_KEYFILE, "/etc/ssl/mykey.pem")
 # See http://www.python-ldap.org/ for further details on these options.
 
+## REMOTE_USER authentication. See: https://docs.djangoproject.com/en/dev/howto/auth-remote-user/
+#USE_REMOTE_USER_AUTHENTICATION = True
+
+# Override the URL for the login link (e.g. for django_openid_auth)
+#LOGIN_URL = '/account/login'
+
 
 ##########################
 # Database Configuration #
 ##########################
 # By default sqlite is used. If you cluster multiple webapps you will need
-# to setup an external database (like mysql) and configure all the webapps
-# to use the same database. Note that this database is only used to store
-# django models like saved graphs, dashboards, user preferences, etc. Metric
-# data is not stored here.
+# to setup an external database (such as MySQL) and configure all of the webapp
+# instances to use the same database. Note that this database is only used to store
+# Django models such as saved graphs, dashboards, user preferences, etc.
+# Metric data is not stored here.
+#
+# DO NOT FORGET TO RUN 'manage.py syncdb' AFTER SETTING UP A NEW DATABASE
+#
+# The following built-in database engines are available:
+#  django.db.backends.postgresql          # Removed in Django 1.4
+#  django.db.backends.postgresql_psycopg2
+#  django.db.backends.mysql
+#  django.db.backends.sqlite3
+#  django.db.backends.oracle
 #
-# DON'T FORGET TO RUN 'manage.py syncdb' AFTER SETTING UP A NEW DB!
+# The default is 'django.db.backends.sqlite3' with file 'graphite.db'
+# located in STORAGE_DIR
 #
-#DATABASE_ENGINE = 'mysql' # or 'postgres'
-#DATABASE_NAME = 'graphite'
-#DATABASE_USER = 'graphite'
-#DATABASE_PASSWORD = 'graphite-is-awesome'
-#DATABASE_HOST = 'mysql.mycompany.com'
-#DATABASE_PORT = '3306'
+DATABASES = {
+    'default': {
+        'NAME': '<%= rubber_env.graphite_storage_dir %>/graphite.db',
+        'ENGINE': 'django.db.backends.sqlite3',
+        'USER': '',
+        'PASSWORD': '',
+        'HOST': '',
+        'PORT': ''
+    }
+}
+
 
 
 #########################
@@ -72,27 +190,39 @@ TIME_ZONE = 'America/New_York'
 #########################
 # (To avoid excessive DNS lookups you want to stick to using IP addresses only in this entire section)
 #
-# This should list the IP address (and optionally port) of each webapp in your cluster.
-# Strings are of the form "ip[:port]"
-# Usually this will be the same as MEMCACHE_HOSTS except for the port numbers.
-#
-#CLUSTER_SERVERS = []
+# This should list the IP address (and optionally port) of the webapp on each
+# remote server in the cluster. These servers must each have local access to
+# metric data. Note that the first server to return a match for a query will be
+# used.
+#CLUSTER_SERVERS = ["10.0.2.2:80", "10.0.2.3:80"]
 
-# This lists all the memcached servers that will be used by this webapp.
-# If you have a cluster of webapps you want to make sure all of them
-# have the *exact* same value for this setting. That will maximize cache
-# efficiency. Setting MEMCACHE_HOSTS to be empty will turn off use of
-# memcached entirely.
-#
-# You should not use the loopback address 127.0.0.1 here because every webapp in
-# the cluster should use the exact same value and should list every member in the
-# cluster.
-#MEMCACHE_HOSTS = ['10.10.10.10:11211', '10.10.10.11:11211', '10.10.10.12:11211']
+## These are timeout values (in seconds) for requests to remote webapps
+#REMOTE_STORE_FETCH_TIMEOUT = 6   # Timeout to fetch series data
+#REMOTE_STORE_FIND_TIMEOUT = 2.5  # Timeout for metric find requests
+#REMOTE_STORE_RETRY_DELAY = 60    # Time before retrying a failed remote webapp
+#REMOTE_FIND_CACHE_DURATION = 300 # Time to cache remote metric find results
+
+## Remote rendering settings
+# Set to True to enable rendering of Graphs on a remote webapp
+#REMOTE_RENDERING = True
+# List of IP (and optionally port) of the webapp on each remote server that
+# will be used for rendering. Note that each rendering host should have local
+# access to metric data or should have CLUSTER_SERVERS configured
+#RENDERING_HOSTS = []
+#REMOTE_RENDER_CONNECT_TIMEOUT = 1.0
 
 # If you are running multiple carbon-caches on this machine (typically behind a relay using
 # consistent hashing), you'll need to list the ip address, cache query port, and instance name of each carbon-cache
 # instance on the local machine (NOT every carbon-cache in the entire cluster). The default cache query port is 7002
 # and a common scheme is to use 7102 for instance b, 7202 for instance c, etc.
 #
-# You *should* use 127.0.0.1 here.
+# You *should* use 127.0.0.1 here in most cases
 #CARBONLINK_HOSTS = ["127.0.0.1:7002:a", "127.0.0.1:7102:b", "127.0.0.1:7202:c"]
+#CARBONLINK_TIMEOUT = 1.0
+
+#####################################
+# Additional Django Settings #
+#####################################
+# Uncomment the following line for direct access to Django settings such as
+# MIDDLEWARE_CLASSES or APPS
+#from graphite.app_settings import *

data/templates/graphite/config/rubber/role/graphite_web/uwsgi.ini

@@ -1,16 +1,19 @@
 <%
-  @path = '/opt/graphite/conf/uwsgi.ini'
+  @path = "#{rubber_env.graphite_dir}/conf/uwsgi.ini"
+
+  is_old_ubuntu = %w[10.04 12.04].include?(rubber_instance.os_version)
+  log_dir = is_old_ubuntu ? "#{rubber_env.graphite_storage_dir}/log/webapp" : '/var/log/graphite'
 %>
 
 [uwsgi]
 http=:<%= rubber_env.graphite_web_port %>
 chdir=/opt/graphite/webapp
-wsgi-file=/opt/graphite/conf/graphite.wsgi
+wsgi-file=<%= is_old_ubuntu ? '/opt/graphite/conf/graphite.wsgi' : '/usr/share/graphite-web/graphite.wsgi' %>
 master=True
 pidfile=/var/run/graphite-web.pid
 processes=4
 die-on-term=True
 vacuum=True
 max-requests=5000
-daemonize=/opt/graphite/storage/log/webapp/webapp.log
+daemonize=<%= log_dir %>/webapp.log
 plugins=python,http

data/templates/graphite/config/rubber/rubber-graphite.yml

@@ -20,6 +20,12 @@ web_tools_proxies:
 
 roles:
   graphite_web:
-    packages: [python-django, python-django-tagging, python-cairo, python-memcache, memcached, uwsgi, uwsgi-plugin-python, uwsgi-plugin-http, sqlite3, bzr, zip]
+    os_packages:
+      '10.04': [python-django, python-django-tagging, python-cairo, python-memcache, memcached, uwsgi, uwsgi-plugin-python, uwsgi-plugin-http, sqlite3, bzr, zip]
+      '12.04': [python-django, python-django-tagging, python-cairo, python-memcache, memcached, uwsgi, uwsgi-plugin-python, uwsgi-plugin-http, sqlite3, bzr, zip]
+      '14.04': [uwsgi, uwsgi-plugin-python, graphite-web]
+  graphite_server:
+    os_packages:
+      '14.04': [graphite-carbon]
   collectd:
     packages: [libperl-dev]

data/templates/graphite/templates.rb

@@ -0,0 +1,9 @@
+require 'securerandom'
+
+extra_config = <<-EXTRA
+# This was auto-generated by Rubber and is unique to your machine.  Feel free to regenerate if you'd like.
+# You may want to move this to your rubber-secret.yml to keep it out of version control.
+graphite_web_secret: #{SecureRandom.hex(96)}
+EXTRA
+
+append_to_file 'config/rubber/rubber-graphite.yml', "\n#{extra_config}"

data/templates/graylog/config/rubber/common/graylog-rsyslog.conf

@@ -3,7 +3,7 @@
   @post = "/usr/sbin/service rsyslog restart"
 
   graylog_server = rubber_instances.for_role("graylog_server").first
-  is_graylog_instance = rubber_instances[rubber_env.host].role_names.include?('graylog_server')
+  is_graylog_instance = rubber_instance.role_names.include?('graylog_server')
 %>
 
 <% if graylog_server %>

data/templates/graylog/config/rubber/role/graylog_server/graylog2.conf

@@ -34,12 +34,12 @@ root_password_sha2 = <%= root_password_sha2 %>
 plugin_dir = plugin
 
 # REST API listen URI. Must be reachable by other graylog2-server nodes if you run a cluster.
-rest_listen_uri = http://<%= rubber_instances[rubber_env.host].internal_ip %>:12900/
+rest_listen_uri = http://<%= rubber_instance.internal_ip %>:12900/
 
 # REST API transport address. Defaults to first non-loopback IPv4 system address and port 12900.
 # This will be promoted in the cluster discovery APIs and other nodes may try to connect on this
 # address. (see rest_listen_uri)
-rest_transport_uri = http://<%= rubber_instances[rubber_env.host].internal_ip %>:12900/
+rest_transport_uri = http://<%= rubber_instance.internal_ip %>:12900/
 
 # Enable CORS headers for REST api. This is necessary for JS-clients accessing the server directly.
 # If these are disabled, modern browsers will not be able to retrieve resources from the server.

data/templates/jenkins/config/rubber/role/jenkins/jenkins-apache-vhost.conf

@@ -1,5 +1,10 @@
 <%
+  is_old_ubuntu = %w[10.04 12.04].include?(rubber_instance.os_version)
+
+  # Apache 2.2 requires no extension while Apache 2.4 requires the '.conf' extension.
   @path = "/etc/apache2/sites-available/#{rubber_env.app_name}-jenkins"
+  @path << '.conf' unless is_old_ubuntu
+
   @post = "a2enmod rewrite && a2enmod ssl && a2enmod proxy_http && a2ensite #{rubber_env.app_name}-jenkins"
 %>
 

data/templates/memcached/config/rubber/role/memcached/memcached.conf

@@ -1,7 +1,7 @@
 <%
   @path = "/etc/memcached.conf"
   max_mem = rubber_env.memcached_max_mem
-  max_mem = max_mem[rubber_instances[rubber_env.host].image_type.split('.')[1]] if max_mem.is_a?(Hash)
+  max_mem = max_mem[rubber_instance.image_type.split('.')[1]] if max_mem.is_a?(Hash)
 %>
 
 # memcached default config file

data/templates/mysql/config/rubber/deploy-mysql.rb

@@ -29,7 +29,7 @@ namespace :rubber do
       
       master_instances = rubber_instances.for_role("mysql_master") & rubber_instances.filtered  
       master_instances.each do |ic|
-        task_name = "_bootstrap_mysql_master_#{ic.full_name}".to_sym()
+        task_name = "_bootstrap_mysql_master_#{ic.full_name}".to_sym
         task task_name, :hosts => ic.full_name do
           env = rubber_cfg.environment.bind("mysql_master", ic.name)
           exists = capture("echo $(ls #{env.db_data_dir}/ 2> /dev/null)")
@@ -52,7 +52,7 @@ namespace :rubber do
     
       slave_instances = rubber_instances.for_role("mysql_slave") & rubber_instances.filtered  
       slave_instances.each do |ic|
-        task_name = "_bootstrap_mysql_slave_#{ic.full_name}".to_sym()
+        task_name = "_bootstrap_mysql_slave_#{ic.full_name}".to_sym
         task task_name, :hosts => ic.full_name do
           env = rubber_cfg.environment.bind("mysql_slave", ic.name)
           exists = capture("echo $(ls #{env.db_data_dir}/ 2> /dev/null)")

data/templates/mysql/config/rubber/role/db/my.cnf

@@ -6,7 +6,7 @@
   @post = "mkdir -p #{log_dir}; chown mysql:adm #{log_dir}; chmod 2750 #{log_dir}"
 %>
 
-# Generated by rubber for host <%= rubber_instances[rubber_env.host].external_ip %>
+# Generated by rubber for host <%= rubber_instance.external_ip %>
 #
 # The MySQL database server configuration file.
 #

data/templates/mysql_cluster/config/rubber/deploy-mysql_cluster.rb

@@ -56,7 +56,7 @@ namespace :rubber do
       # been boostrapped for that role before
       
       rubber_instances.for_role("mysql_mgm").each do |ic|
-        task_name = "_bootstrap_mysql_mgm_#{ic.full_name}".to_sym()
+        task_name = "_bootstrap_mysql_mgm_#{ic.full_name}".to_sym
         task task_name, :hosts => ic.full_name do
           exists = capture("if grep -c rubber.*mysql_mgm /etc/mysql/ndb_mgmd.cnf &> /dev/null; then echo exists; fi")
           if exists.strip.size == 0
@@ -68,7 +68,7 @@ namespace :rubber do
       end
     
       rubber_instances.for_role("mysql_data").each do |ic|
-        task_name = "_bootstrap_mysql_data_#{ic.full_name}".to_sym()
+        task_name = "_bootstrap_mysql_data_#{ic.full_name}".to_sym
         task task_name, :hosts => ic.full_name do
           exists = capture("if grep -c rubber.*mysql_data /etc/mysql/my.cnf &> /dev/null; then echo exists; fi")
           if exists.strip.size == 0
@@ -80,7 +80,7 @@ namespace :rubber do
       end
       
       rubber_instances.for_role("mysql_sql").each do |ic|
-        task_name = "_bootstrap_mysql_sql_#{ic.full_name}".to_sym()
+        task_name = "_bootstrap_mysql_sql_#{ic.full_name}".to_sym
         task task_name, :hosts => ic.full_name do
           exists = capture("if grep -c rubber.*mysql_sql /etc/mysql/my.cnf &> /dev/null; then echo exists; fi")
           if exists.strip.size == 0