rubber 2.12.2 → 2.13.0

data/templates/apache/config/rubber/role/web_tools/tools-apache-vhost.conf

@@ -1,5 +1,10 @@
 <%
+  is_old_ubuntu = %w[10.04 12.04].include?(rubber_instance.os_version)
+
+  # Apache 2.2 requires no extension while Apache 2.4 requires the '.conf' extension.
   @path = "/etc/apache2/sites-available/#{rubber_env.app_name}-tools"
+  @path << '.conf' unless is_old_ubuntu
+
   @post = <<-EOS
     a2enmod rewrite
     a2enmod ssl
@@ -17,7 +22,10 @@ Listen <%= rubber_env.web_tools_port %>
   RewriteEngine   on
   RewriteCond     %{HTTPS} !=on
   RewriteRule     ^/(.*)$ https://%{SERVER_NAME}:<%= rubber_env.web_tools_ssl_port %>/$1 [L,R]
+
+  <%- if is_old_ubuntu %>
   RewriteLog      "/var/log/apache2/rewrite.log"
+  <% end %>
 </VirtualHost>
 
 Listen <%= rubber_env.web_tools_ssl_port %>

data/templates/apache/config/rubber/rubber-apache.yml

@@ -1,6 +1,6 @@
 
 roles:
   apache:
-    packages: [apache2, libapache2-mod-proxy-html, libcurl4-openssl-dev, libapache2-mod-xsendfile]
+    packages: [apache2, apache2-utils, libapache2-mod-proxy-html, libcurl4-openssl-dev, libapache2-mod-xsendfile]
   web_tools:
-    packages: [apache2, libapache2-mod-proxy-html, libcurl4-openssl-dev, libapache2-mod-xsendfile]
+    packages: [apache2, apache2-utils, libapache2-mod-proxy-html, libcurl4-openssl-dev, libapache2-mod-xsendfile]

data/templates/base/config/rubber/deploy-util.rb

@@ -13,7 +13,7 @@ namespace :rubber do
       # Select only one instance for backup.  Favor slave database.
       selected_db_instance = (slaves+master_instances).first
             
-      task_name = "_backup_db_#{selected_db_instance.full_name}".to_sym()
+      task_name = "_backup_db_#{selected_db_instance.full_name}".to_sym
       task task_name, :hosts => selected_db_instance.full_name do
         rsudo "cd #{current_path} && RUBBER_ENV=#{Rubber.env} ./script/rubber util:backup_db --directory=/mnt/db_backups --dbuser=#{rubber_env.db_user} --dbpass=#{rubber_env.db_pass} --dbname=#{rubber_env.db_name} --dbhost=#{selected_db_instance.full_name}"
       end
@@ -29,7 +29,7 @@ namespace :rubber do
       slaves = rubber_instances.for_role('db', {})
 
       for instance in master_instances+slaves
-        task_name = "_restore_db_cloud_#{instance.full_name}".to_sym()
+        task_name = "_restore_db_cloud_#{instance.full_name}".to_sym
         task task_name, :hosts => instance.full_name do
           rsudo "cd #{current_path} && RUBBER_ENV=#{Rubber.env} ./script/rubber util:restore_db --filename=#{filename} --dbuser=#{rubber_env.db_user} --dbpass=#{rubber_env.db_pass} --dbname=#{rubber_env.db_name} --dbhost=#{instance.full_name}"
         end
@@ -48,7 +48,7 @@ namespace :rubber do
       # Select only one instance for backup.  Favor slave database.
       selected_db_instance = (slaves+master_instances).first
             
-      task_name = "_load_local_to_#{selected_db_instance.full_name}".to_sym()
+      task_name = "_load_local_to_#{selected_db_instance.full_name}".to_sym
       task task_name, :hosts => selected_db_instance.full_name do
 
         # Dump Local to tmp folder

data/templates/base/config/rubber/rubber.yml

@@ -83,11 +83,11 @@ cloud_providers:
 
     # REQUIRED: the ami and instance type for creating instances
     # The Ubuntu images at http://alestic.com/ work well
-    # Ubuntu 12.04 Precise instance-store 64-bit: ami-eafa5883
+    # Ubuntu 14.04.1 Trusty instance-store 64-bit: ami-90c574f8
     #
     # m1.small or m1.large or m1.xlarge
     image_type: c1.medium
-    image_id: ami-b6089bdf
+    image_id: ami-90c574f8
 
     # OPTIONAL: EC2 spot instance request support.
     #
@@ -118,7 +118,7 @@ cloud_providers:
     region: New York 2
 
     # REQUIRED: The image name and type for creating instances.
-    image_id: Ubuntu 12.04.5 x64
+    image_id: Ubuntu 14.04 x64
     image_type: 512MB
 
     # Optionally enable private networking for your instances.
@@ -265,6 +265,11 @@ packages: [postfix, build-essential, git-core, libxslt-dev, ntp]
 # it, specify 'mirror://mirrors.ubuntu.com/mirrors.txt' as the value.
 # package_manager_mirror: 'mirror://mirrors.ubuntu.com/mirrors.txt'
 
+# OPTIONAL: The command used to identify your particular OS version.  This will be used for configurations
+# in Rubber templates that are parameterized by OS version (e.g., package lists).  If not specified, Ubuntu
+# will be assumed.
+os_version_cmd: 'lsb_release -sr'
+
 # OPTIONAL: gem sources to setup for rubygems
 # gemsources: ["https://rubygems.org"]
 

data/templates/cassandra/config/rubber/deploy-cassandra.rb

@@ -50,7 +50,7 @@ namespace :rubber do
     task :bootstrap, :roles => :cassandra do
       instances = rubber_instances.for_role("cassandra") & rubber_instances.filtered
       instances.each do |ic|
-        task_name = "_bootstrap_cassandra_#{ic.full_name}".to_sym()
+        task_name = "_bootstrap_cassandra_#{ic.full_name}".to_sym
         task task_name, :hosts => ic.full_name do
           env = rubber_cfg.environment.bind("cassandra", ic.name)
           exists = capture("echo $(ls #{env.cassandra_data_dirs.first}/ 2> /dev/null)")
@@ -86,7 +86,7 @@ namespace :rubber do
     def clustertool(*args)
       instances = rubber_instances.for_role("cassandra") & rubber_instances.filtered
       ic = instances.first
-      task_name = "_clustertool_#{ic.full_name}_#{args.join('_')}".to_sym()
+      task_name = "_clustertool_#{ic.full_name}_#{args.join('_')}".to_sym
       task task_name, :hosts => ic.full_name do
         rsudo "#{rubber_env.cassandra_dir}/bin/clustertool --host $CAPISTRANO:HOST$ --port #{rubber_env.cassandra_jmx_port} #{args.join(' ')}"
       end

data/templates/collectd/config/rubber/role/collectd/collectd-ping.conf

@@ -2,10 +2,10 @@
   @path = '/etc/collectd/conf.d/collectd-ping.conf'
 
   ping_hosts = []
-  if rubber_instances[rubber_env.host].role_names.include?('graphite_server')
+  if rubber_instance.role_names.include?('graphite_server')
     # monitor all servers from graphite_server
     ping_hosts = rubber_instances.collect {|ic| ic.full_name }.select {|h| h != rubber_env.full_host }
-  elsif rubber_instances[rubber_env.host].role_names.include?('web_tools')
+  elsif rubber_instance.role_names.include?('web_tools')
     # monitor just graphite_server from tools
     ping_hosts = Array((rubber_instances.for_role('graphite_server').first.full_name rescue nil))
   end
@@ -18,4 +18,13 @@ LoadPlugin ping
   <% ping_hosts.each do |h| %>
   Host "<%= h %>"
   <% end %>
+
+#	Host "host.foo.bar"
+#	Host "host.baz.qux"
+#	Interval 1.0
+#	Timeout 0.9
+#	TTL 255
+#	SourceAddress "1.2.3.4"
+#	Device "eth0"
+#	MaxMissed -1
 </Plugin>

data/templates/collectd/config/rubber/role/collectd/thresholds.conf

@@ -1,5 +1,5 @@
 <%
-  is_worker_instance = rubber_instances[rubber_env.host].role_names.include?('resque_worker')
+  is_worker_instance = rubber_instance.role_names.include?('resque_worker')
 
   @path = '/etc/collectd/thresholds.d/thresholds.conf'
 %>

data/templates/collectd/script/collectd/role/redis/redis_info.rb

@@ -10,7 +10,6 @@ items = {
   'blocked_clients' => 'gauge',
   'total_commands_processed' => 'derive',
   'total_connections_received' => 'derive',
-  'connected_clients' => 'gauge',
   'used_memory' => 'gauge',
   'changes_since_last_save' => 'gauge'
 }

data/templates/cruise/config/rubber/role/cruise/my.cnf

@@ -3,7 +3,7 @@
   @post = 'service mysql restart'
 %>
 
-# Generated by rubber for host <%= rubber_instances[rubber_env.host].external_ip %>
+# Generated by rubber for host <%= rubber_instance.external_ip %>
 #
 # The MySQL database server configuration file.
 #

data/templates/graphite/config/rubber/deploy-graphite.rb

@@ -22,33 +22,35 @@ namespace :rubber do
     end
 
     task :install_graphite_from_repo, :roles => [:graphite_server, :graphite_web] do
-      rubber.sudo_script 'install_graphite_from_repo', <<-ENDSCRIPT
-        if [[ ! -d "/opt/graphite" ]]; then
-          mkdir -p /tmp/graphite_install
-          cd /tmp/graphite_install
-          bzr branch lp:graphite -r #{rubber_env.graphite_repo_version}
-
-          pushd graphite/whisper
-          python setup.py install
-          popd
-
-          pushd graphite/carbon
-          python setup.py install
-          popd
-
-          pushd graphite
-          python setup.py install
-          popd
-        fi
-      ENDSCRIPT
+      if old_ubuntu?
+        rubber.sudo_script 'install_graphite_from_repo', <<-ENDSCRIPT
+          if [[ ! -d "/opt/graphite" ]]; then
+            mkdir -p /tmp/graphite_install
+            cd /tmp/graphite_install
+            bzr branch lp:graphite -r #{rubber_env.graphite_repo_version}
+
+            pushd graphite/whisper
+            python setup.py install
+            popd
+
+            pushd graphite/carbon
+            python setup.py install
+            popd
+
+            pushd graphite
+            python setup.py install
+            popd
+          fi
+        ENDSCRIPT
+      end
     end
 
     desc <<-DESC
-      Cleans out old whisper storage files for non-existant instances so they don't show in webui
+      Cleans out old whisper storage files for non-existent instances so they don't show in webui
     DESC
     task :clean_storage, :roles => [:graphite_server] do
       active_instances = rubber_instances.collect {|ic| ic.name }.sort
-      stored_instances = capture("ls /opt/graphite/storage/whisper/servers/").split.sort
+      stored_instances = capture("ls #{rubber_env.graphite_storage_dir}/whisper/servers/").split.sort
       cleaning_instances = stored_instances - active_instances
 
       if cleaning_instances.size > 0
@@ -61,7 +63,7 @@ namespace :rubber do
         end
 
         if do_clean
-          rsudo "rm -rf /opt/graphite/storage/whisper/servers/{#{cleaning_instances.join(',')}}"
+          rsudo "rm -rf #{rubber_env.graphite_storage_dir}/whisper/servers/{#{cleaning_instances.join(',')}}"
         end
 
       end
@@ -83,34 +85,38 @@ namespace :rubber do
         Installs graphite server components
       DESC
       task :install, :roles => :graphite_server do
-        rubber.sudo_script 'install_graphite_server', <<-ENDSCRIPT
-          if [[ ! -f "/opt/graphite/bin/carbon-cache.py" ]]; then
-            wget --content-disposition -qNP /tmp #{rubber_env.graphite_whisper_package_url}
-            tar -C /tmp -zxf /tmp/whisper-#{rubber_env.graphite_version}.tar.gz
-            cd /tmp/whisper-#{rubber_env.graphite_version}
-            python setup.py install
-            cd /tmp
-            rm -rf whisper-#{rubber_env.graphite_version}
-            rm whisper-#{rubber_env.graphite_version}.tar.gz
+        if old_ubuntu?
+          rubber.sudo_script 'install_graphite_server', <<-ENDSCRIPT
+            if [[ ! -f "/opt/graphite/bin/carbon-cache.py" ]]; then
+              wget --content-disposition -qNP /tmp #{rubber_env.graphite_whisper_package_url}
+              tar -C /tmp -zxf /tmp/whisper-#{rubber_env.graphite_version}.tar.gz
+              cd /tmp/whisper-#{rubber_env.graphite_version}
+              python setup.py install
+              cd /tmp
+              rm -rf whisper-#{rubber_env.graphite_version}
+              rm whisper-#{rubber_env.graphite_version}.tar.gz
+
+              wget --content-disposition -qNP /tmp #{rubber_env.graphite_carbon_package_url}
+              tar -C /tmp -zxf /tmp/carbon-#{rubber_env.graphite_version}.tar.gz
+              cd /tmp/carbon-#{rubber_env.graphite_version}
+              python setup.py install
+              cd /tmp
+              rm -r carbon-#{rubber_env.graphite_version}
+              rm carbon-#{rubber_env.graphite_version}.tar.gz
+
+              rm -rf #{rubber_env.graphite_storage_dir}
+              mkdir #{rubber_env.graphite_storage_dir}
+              chown www-data:www-data #{rubber_env.graphite_storage_dir}
+              ln -s #{rubber_env.graphite_storage_dir} /opt/graphite/storage
+            fi
+          ENDSCRIPT
+        end
 
-            wget --content-disposition -qNP /tmp #{rubber_env.graphite_carbon_package_url}
-            tar -C /tmp -zxf /tmp/carbon-#{rubber_env.graphite_version}.tar.gz
-            cd /tmp/carbon-#{rubber_env.graphite_version}
-            python setup.py install
-            cd /tmp
-            rm -r carbon-#{rubber_env.graphite_version}
-            rm carbon-#{rubber_env.graphite_version}.tar.gz
-
-            rm -rf /opt/graphite/storage
-            mkdir #{rubber_env.graphite_storage_dir}
-            chown www-data:www-data #{rubber_env.graphite_storage_dir}
-            ln -s #{rubber_env.graphite_storage_dir} /opt/graphite/storage
-          fi
-        ENDSCRIPT
+        create_storage_directory
       end
 
       task :bootstrap, :roles => :graphite_server do
-        exists = capture("echo $(ls /opt/graphite/storage/whisper/ 2> /dev/null)")
+        exists = capture("echo $(ls #{rubber_env.graphite_storage_dir}/whisper/ 2> /dev/null)")
         if exists.strip.size == 0
           rubber.update_code_for_bootstrap
 
@@ -122,12 +128,20 @@ namespace :rubber do
 
       desc "Start graphite system monitoring"
       task :start, :roles => :graphite_server do
-        rsudo "service graphite-server start"
+        if old_ubuntu?
+          rsudo 'service graphite-server start'
+        else
+          rsudo 'service carbon-cache start'
+        end
       end
 
       desc "Stop graphite system monitoring"
       task :stop, :roles => :graphite_server do
-        rsudo "service graphite-server stop || true"
+        if old_ubuntu?
+          rsudo 'service graphite-server stop || true'
+        else
+          rsudo 'service carbon-cache stop || true'
+        end
       end
 
       desc "Restart graphite system monitoring"
@@ -161,22 +175,26 @@ namespace :rubber do
         Installs graphite web components
       DESC
       task :install, :roles => :graphite_web do
-        rubber.sudo_script 'install_graphite_web', <<-ENDSCRIPT
-          if [[ ! -d "/opt/graphite/webapp" ]]; then
-            wget --content-disposition -qNP /tmp #{rubber_env.graphite_web_package_url}
-            tar -C /tmp -zxf /tmp/graphite-web-#{rubber_env.graphite_version}.tar.gz
-            cd /tmp/graphite-web-#{rubber_env.graphite_version}
-            # python check-dependencies.py
-            python setup.py install
-            cd /tmp
-            rm -r graphite-web-#{rubber_env.graphite_version}
-            rm graphite-web-#{rubber_env.graphite_version}.tar.gz
-          fi
-        ENDSCRIPT
+        if old_ubuntu?
+          rubber.sudo_script 'install_graphite_web', <<-ENDSCRIPT
+            if [[ ! -d "/opt/graphite/webapp" ]]; then
+              wget --content-disposition -qNP /tmp #{rubber_env.graphite_web_package_url}
+              tar -C /tmp -zxf /tmp/graphite-web-#{rubber_env.graphite_version}.tar.gz
+              cd /tmp/graphite-web-#{rubber_env.graphite_version}
+
+              # python check-dependencies.py
+              python setup.py install
+
+              cd /tmp
+              rm -r graphite-web-#{rubber_env.graphite_version}
+              rm graphite-web-#{rubber_env.graphite_version}.tar.gz
+            fi
+          ENDSCRIPT
+        end
       end
 
       task :bootstrap, :roles => :graphite_web do
-        exists = capture("echo $(ls /opt/graphite/storage/graphite.db 2> /dev/null)")
+        exists = capture("echo $(ls #{rubber_env.graphite_storage_dir}/graphite.db 2> /dev/null)")
         if exists.strip.size == 0
           rubber.update_code_for_bootstrap
 
@@ -188,19 +206,42 @@ namespace :rubber do
           env = rubber_cfg.environment.bind("graphite_web", web_instance.name)
           email = env.admin_email
 
-          rubber.sudo_script 'bootstrap_graphite_web', <<-ENDSCRIPT
-            cd /opt/graphite/webapp/graphite
-            python manage.py syncdb --noinput
-            python manage.py createsuperuser --username admin --email #{email} --noinput
-            python manage.py shell <<EOF
+          create_storage_directory
+
+          if old_ubuntu?
+            rubber.sudo_script 'bootstrap_graphite_web', <<-ENDSCRIPT
+              mkdir -p #{rubber_env.graphite_storage_dir}/log/webapp/
+              chown -R www-data:www-data #{rubber_env.graphite_storage_dir}/log/
+
+              cd /opt/graphite/webapp/graphite
+              python manage.py syncdb --noinput
+              python manage.py createsuperuser --username admin --email #{email} --noinput
+              python manage.py shell <<EOF
 from django.contrib.auth.models import User
 u = User.objects.get(username__exact='admin')
 u.set_password('admin1')
 u.save()
 EOF
-            mkdir -p /opt/graphite/storage/log/webapp/
-            chown -R www-data:www-data /opt/graphite/storage/
-          ENDSCRIPT
+            ENDSCRIPT
+          else
+            rubber.sudo_script 'bootstrap_graphite_web', <<-ENDSCRIPT
+              # Ubuntu 14.04 ships with a broken graphite-web package.  It renames the build-index.sh file to be a binary
+              # on the PATH, but it fails to update any of the code that references the original file location, causing
+              # graphite-web to fail on initial load.  We fix that here by setting up a symlink to the renamed binary.
+
+              mkdir -p /usr/share/graphite-web/bin/
+              ln -s /usr/bin/graphite-build-search-index /usr/share/graphite-web/bin/build-index.sh
+
+              graphite-manage syncdb --noinput
+              graphite-manage createsuperuser --username admin --email #{email} --noinput
+              graphite-manage shell <<EOF
+from django.contrib.auth.models import User
+u = User.objects.get(username__exact='admin')
+u.set_password('admin1')
+u.save()
+EOF
+            ENDSCRIPT
+          end
 
           restart
         end
@@ -231,6 +272,21 @@ EOF
 
     end
 
+    def create_storage_directory
+      owner = old_ubuntu? ? 'www-data' : '_graphite'
+
+      rubber.sudo_script 'create_graphite_storage_directory', <<-ENDSCRIPT
+        if [[ ! -e #{rubber_env.graphite_storage_dir} ]]; then
+          mkdir -p #{rubber_env.graphite_storage_dir}
+          chown -R #{owner}:www-data #{rubber_env.graphite_storage_dir}
+        fi
+      ENDSCRIPT
+    end
+
+    def old_ubuntu?
+      %w[10.04 12.04].include?(rubber_instance.os_version)
+    end
+
   end
 
 end

data/templates/graphite/config/rubber/role/graphite_server/carbon.conf

@@ -1,5 +1,7 @@
 <%
-  @path = '/opt/graphite/conf/carbon.conf'
+  is_old_ubuntu = %w[10.04 12.04].include?(rubber_instance.os_version)
+
+  @path = is_old_ubuntu ? "#{rubber_env.graphite_dir}/conf/carbon.conf" : '/etc/carbon/carbon.conf'
 %>
 [cache]
 # Configure carbon directories.
@@ -17,26 +19,24 @@
 # To change other directory paths, add settings to this file. The following
 # configuration variables are available with these default values:
 #
-#   STORAGE_DIR    = $GRAPHITE_STORAGE_DIR
-#   LOCAL_DATA_DIR = STORAGE_DIR/whisper/
-#   WHITELISTS_DIR = STORAGE_DIR/lists/
-#   CONF_DIR       = STORAGE_DIR/conf/
-#   LOG_DIR        = STORAGE_DIR/log/
-#   PID_DIR        = STORAGE_DIR/
-#
-# For FHS style directory structures, use:
-#
-#   STORAGE_DIR    = /var/lib/carbon/
-#   CONF_DIR       = /etc/carbon/
-#   LOG_DIR        = /var/log/carbon/
-#   PID_DIR        = /var/run/
-#
-LOCAL_DATA_DIR = /opt/graphite/storage/whisper/
+
+STORAGE_DIR    = <%= rubber_env.graphite_storage_dir %>/
+CONF_DIR       = <%= is_old_ubuntu ? "#{rubber_env.graphite_dir}/conf/" : '/etc/carbon/' %>
+LOG_DIR        = /var/log/carbon/
+PID_DIR        = /var/run/
+
+LOCAL_DATA_DIR = <%= rubber_env.graphite_storage_dir %>/whisper/
+
+# Enable daily log rotation. If disabled, a kill -HUP can be used after a manual rotate
+ENABLE_LOGROTATION = False
 
 # Specify the user to drop privileges to
 # If this is blank carbon runs as the user that invokes it
 # This user must have write access to the local data directory
-USER =
+USER = <%= is_old_ubuntu ? '' : '_graphite' %>
+#
+# NOTE: The above settings must be set under [relay] and [aggregator]
+#       to take effect for those daemons as well
 
 # Limit the size of the cache to avoid swapping or becoming CPU bound.
 # Sorts and serving cache queries gets more expensive as the cache grows.
@@ -50,6 +50,12 @@ MAX_CACHE_SIZE = inf
 # take effect and increase the overall throughput accordingly.
 MAX_UPDATES_PER_SECOND = 1000
 
+# If defined, this changes the MAX_UPDATES_PER_SECOND in Carbon when a
+# stop/shutdown is initiated.  This helps when MAX_UPDATES_PER_SECOND is
+# relatively low and carbon has cached a lot of updates; it enables the carbon
+# daemon to shutdown more quickly.
+# MAX_UPDATES_PER_SECOND_ON_SHUTDOWN = 1000
+
 # Softly limits the number of whisper files that get created each minute.
 # Setting this value low (like at 50) is a good way to ensure your graphite
 # system will not be adversely impacted when a bunch of new metrics are
@@ -72,6 +78,9 @@ UDP_RECEIVER_PORT = 2003
 PICKLE_RECEIVER_INTERFACE = 0.0.0.0
 PICKLE_RECEIVER_PORT = 2004
 
+# Set to false to disable logging of successful connections
+LOG_LISTENER_CONNECTIONS = True
+
 # Per security concerns outlined in Bug #817247 the pickle receiver
 # will use a more secure and slightly less efficient unpickler.
 # Set this to True to revert to the old-fashioned insecure unpickler.
@@ -86,15 +95,74 @@ CACHE_QUERY_PORT = <%= rubber_env.graphite_carbon_port %>
 # data until the cache size falls below 95% MAX_CACHE_SIZE.
 USE_FLOW_CONTROL = True
 
-# By default, carbon-cache will log every whisper update. This can be excessive and
+# By default, carbon-cache will log every whisper update and cache hit. This can be excessive and
 # degrade performance if logging on the same volume as the whisper data is stored.
 LOG_UPDATES = False
+LOG_CACHE_HITS = False
+LOG_CACHE_QUEUE_SORTS = True
+
+# The thread that writes metrics to disk can use on of the following strategies
+# determining the order in which metrics are removed from cache and flushed to
+# disk. The default option preserves the same behavior as has been historically
+# available in version 0.9.10.
+#
+# sorted - All metrics in the cache will be counted and an ordered list of
+# them will be sorted according to the number of datapoints in the cache at the
+# moment of the list's creation. Metrics will then be flushed from the cache to
+# disk in that order.
+#
+# max - The writer thread will always pop and flush the metric from cache
+# that has the most datapoints. This will give a strong flush preference to
+# frequently updated metrics and will also reduce random file-io. Infrequently
+# updated metrics may only ever be persisted to disk at daemon shutdown if
+# there are a large number of metrics which receive very frequent updates OR if
+# disk i/o is very slow.
+#
+# naive - Metrics will be flushed from the cache to disk in an unordered
+# fashion. This strategy may be desirable in situations where the storage for
+# whisper files is solid state, CPU resources are very limited or deference to
+# the OS's i/o scheduler is expected to compensate for the random write
+# pattern.
+#
+CACHE_WRITE_STRATEGY = sorted
 
 # On some systems it is desirable for whisper to write synchronously.
 # Set this option to True if you'd like to try this. Basically it will
 # shift the onus of buffering writes from the kernel into carbon's cache.
 WHISPER_AUTOFLUSH = False
 
+# By default new Whisper files are created pre-allocated with the data region
+# filled with zeros to prevent fragmentation and speed up contiguous reads and
+# writes (which are common). Enabling this option will cause Whisper to create
+# the file sparsely instead. Enabling this option may allow a large increase of
+# MAX_CREATES_PER_MINUTE but may have longer term performance implications
+# depending on the underlying storage configuration.
+# WHISPER_SPARSE_CREATE = False
+
+# Only beneficial on linux filesystems that support the fallocate system call.
+# It maintains the benefits of contiguous reads/writes, but with a potentially
+# much faster creation speed, by allowing the kernel to handle the block
+# allocation and zero-ing. Enabling this option may allow a large increase of
+# MAX_CREATES_PER_MINUTE. If enabled on an OS or filesystem that is unsupported
+# this option will gracefully fallback to standard POSIX file access methods.
+WHISPER_FALLOCATE_CREATE = True
+
+# Enabling this option will cause Whisper to lock each Whisper file it writes
+# to with an exclusive lock (LOCK_EX, see: man 2 flock). This is useful when
+# multiple carbon-cache daemons are writing to the same files
+# WHISPER_LOCK_WRITES = False
+
+# Set this to True to enable whitelisting and blacklisting of metrics in
+# CONF_DIR/whitelist and CONF_DIR/blacklist. If the whitelist is missing or
+# empty, all metrics will pass through
+# USE_WHITELIST = False
+
+# By default, carbon itself will log statistics (such as a count,
+# metricsReceived) with the top level prefix of 'carbon' at an interval of 60
+# seconds. Set CARBON_METRIC_INTERVAL to 0 to disable instrumentation
+# CARBON_METRIC_PREFIX = carbon
+# CARBON_METRIC_INTERVAL = 60
+
 # Enable AMQP if you want to receve metrics using an amqp broker
 # ENABLE_AMQP = False
 
@@ -132,13 +200,15 @@ WHISPER_AUTOFLUSH = False
 # Example: store everything
 # BIND_PATTERNS = #
 
-# To configure special settings for the 'foo' carbon-cache instance, uncomment this:
-#[cache:foo]
+# To configure special settings for the carbon-cache instance 'b', uncomment this:
+#[cache:b]
 #LINE_RECEIVER_PORT = 2103
 #PICKLE_RECEIVER_PORT = 2104
 #CACHE_QUERY_PORT = 7102
 # and any other settings you want to customize, defaults are inherited
 # from [carbon] section.
+# You can then specify the --instance=b option to manage this instance
+
 
 
 [relay]
@@ -147,14 +217,27 @@ LINE_RECEIVER_PORT = 2013
 PICKLE_RECEIVER_INTERFACE = 0.0.0.0
 PICKLE_RECEIVER_PORT = 2014
 
-# To use consistent hashing instead of the user defined relay-rules.conf,
-# change this to:
-# RELAY_METHOD = consistent-hashing
+# Set to false to disable logging of successful connections
+LOG_LISTENER_CONNECTIONS = True
+
+# Carbon-relay has several options for metric routing controlled by RELAY_METHOD
+#
+# Use relay-rules.conf to route metrics to destinations based on pattern rules
+#RELAY_METHOD = rules
+#
+# Use consistent-hashing for even distribution of metrics between destinations
+#RELAY_METHOD = consistent-hashing
+#
+# Use consistent-hashing but take into account an aggregation-rules.conf shared
+# by downstream carbon-aggregator daemons. This will ensure that all metrics
+# that map to a given aggregation rule are sent to the same carbon-aggregator
+# instance.
+# Enable this for carbon-relays that send to a group of carbon-aggregators
+#RELAY_METHOD = aggregated-consistent-hashing
 RELAY_METHOD = rules
 
-# If you use consistent-hashing you may want to add redundancy
-# of your data by replicating every datapoint to more than
-# one machine.
+# If you use consistent-hashing you can add redundancy by replicating every
+# datapoint to more than one machine.
 REPLICATION_FACTOR = 1
 
 # This is a list of carbon daemons we will send any relayed or
@@ -170,6 +253,9 @@ REPLICATION_FACTOR = 1
 # Note that if the destinations are all carbon-caches then this should
 # exactly match the webapp's CARBONLINK_HOSTS setting in terms of
 # instances listed (order matters!).
+#
+# If using RELAY_METHOD = rules, all destinations used in relay-rules.conf
+# must be defined in this list
 DESTINATIONS = 127.0.0.1:2004
 
 # This defines the maximum "message size" between carbon daemons.
@@ -183,6 +269,17 @@ MAX_QUEUE_SIZE = 10000
 # data until the send queues fall below 80% MAX_QUEUE_SIZE.
 USE_FLOW_CONTROL = True
 
+# Set this to True to enable whitelisting and blacklisting of metrics in
+# CONF_DIR/whitelist and CONF_DIR/blacklist. If the whitelist is missing or
+# empty, all metrics will pass through
+# USE_WHITELIST = False
+
+# By default, carbon itself will log statistics (such as a count,
+# metricsReceived) with the top level prefix of 'carbon' at an interval of 60
+# seconds. Set CARBON_METRIC_INTERVAL to 0 to disable instrumentation
+# CARBON_METRIC_PREFIX = carbon
+# CARBON_METRIC_INTERVAL = 60
+
 
 [aggregator]
 LINE_RECEIVER_INTERFACE = 0.0.0.0
@@ -191,6 +288,14 @@ LINE_RECEIVER_PORT = 2023
 PICKLE_RECEIVER_INTERFACE = 0.0.0.0
 PICKLE_RECEIVER_PORT = 2024
 
+# Set to false to disable logging of successful connections
+LOG_LISTENER_CONNECTIONS = True
+
+# If set true, metric received will be forwarded to DESTINATIONS in addition to
+# the output of the aggregation rules. If set false the carbon-aggregator will
+# only ever send the output of aggregation.
+FORWARD_ALL = True
+
 # This is a list of carbon daemons we will send any relayed or
 # generated metrics to. The default provided would send to a single
 # carbon-cache instance on the default port. However if you
@@ -230,3 +335,22 @@ MAX_DATAPOINTS_PER_MESSAGE = 500
 # each metric. Aggregation only happens for datapoints that fall in
 # the past MAX_AGGREGATION_INTERVALS * intervalSize seconds.
 MAX_AGGREGATION_INTERVALS = 5
+
+# By default (WRITE_BACK_FREQUENCY = 0), carbon-aggregator will write back
+# aggregated data points once every rule.frequency seconds, on a per-rule basis.
+# Set this (WRITE_BACK_FREQUENCY = N) to write back all aggregated data points
+# every N seconds, independent of rule frequency. This is useful, for example,
+# to be able to query partially aggregated metrics from carbon-cache without
+# having to first wait rule.frequency seconds.
+# WRITE_BACK_FREQUENCY = 0
+
+# Set this to True to enable whitelisting and blacklisting of metrics in
+# CONF_DIR/whitelist and CONF_DIR/blacklist. If the whitelist is missing or
+# empty, all metrics will pass through
+# USE_WHITELIST = False
+
+# By default, carbon itself will log statistics (such as a count,
+# metricsReceived) with the top level prefix of 'carbon' at an interval of 60
+# seconds. Set CARBON_METRIC_INTERVAL to 0 to disable instrumentation
+# CARBON_METRIC_PREFIX = carbon
+# CARBON_METRIC_INTERVAL = 60