rubber 1.15.0 → 2.0.0.pre1

data/templates/elasticsearch/config/rubber/role/elasticsearch/logging.yml

@@ -0,0 +1,48 @@
+<%
+  @path = "#{rubber_env.elasticsearch_dir}/config/logging.yml"
+%>
+
+rootLogger: INFO, console, file
+logger:
+  # log action execution errors for easier debugging
+  action: DEBUG
+  # reduce the logging for aws, too much is logged under the default INFO
+  com.amazonaws: WARN
+
+  # gateway
+  #gateway: DEBUG
+  #index.gateway: DEBUG
+
+  # peer shard recovery
+  #indices.recovery: DEBUG
+
+  # discovery
+  #discovery: TRACE
+
+  index.search.slowlog: TRACE, index_search_slow_log_file
+
+additivity:
+  index.search.slowlog: false
+
+appender:
+  console:
+    type: console
+    layout:
+      type: consolePattern
+      conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"
+
+  file:
+    type: dailyRollingFile
+    file: ${path.logs}/${cluster.name}.log
+    datePattern: "'.'yyyy-MM-dd"
+    layout:
+      type: pattern
+      conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"
+
+  index_search_slow_log_file:
+    type: dailyRollingFile
+    file: ${path.logs}/${cluster.name}_index_search_slowlog.log
+    datePattern: "'.'yyyy-MM-dd"
+    layout:
+      type: pattern
+      conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"

data/templates/elasticsearch/config/rubber/rubber-elasticsearch.yml

@@ -0,0 +1,12 @@
+elasticsearch_version: 0.18.7
+elasticsearch_prefix: /usr/local
+elasticsearch_dir: "#{elasticsearch_prefix}/elasticsearch-#{elasticsearch_version}"
+elasticsearch_data_dir: "/mnt/elasticsearch/data"
+elasticsearch_work_dir: "/mnt/elasticsearch/work"
+elasticsearch_log_dir: "/mnt/elasticsearch/logs"
+elasticsearch_pid_file: "/var/run/elasticsearch.pid"
+elasticsearch_http_port: 9200
+
+roles:
+  elasticsearch:
+    packages: [openjdk-6-jdk]

data/templates/elasticsearch/templates.yml

@@ -0,0 +1,2 @@
+description: The elasticsearch module
+  

data/templates/graphite/config/rubber/deploy-graphite.rb

@@ -0,0 +1,255 @@
+
+namespace :rubber do
+  
+  namespace :graphite do
+    
+    rubber.allow_optional_tasks(self)
+
+    after "rubber:install_packages", "rubber:graphite:install_collectd_graphite_plugin"
+
+    task :install_collectd_graphite_plugin, :roles => :collectd do
+      rubber.sudo_script 'install_collectd', <<-ENDSCRIPT
+        if [[ ! -f "/usr/local/share/perl/5.10.1/Collectd/Plugins/Graphite.pm" ]]; then
+          wget --no-check-certificate -qNO /tmp/joemiller-collectd-graphite.tar.gz https://github.com/joemiller/collectd-graphite/tarball/#{rubber_env.collectd_graphite_plugin_version}
+          tar -C /tmp -zxf /tmp/joemiller-collectd-graphite.tar.gz
+          cd /tmp/joemiller-collectd-graphite-*
+          perl Makefile.PL
+          make
+          make test
+          make install
+        fi
+      ENDSCRIPT
+    end
+
+    # graphite collectd plugin needs newer collectd (>= 4.9)
+    before "rubber:install_packages", "rubber:graphite:setup_apt_backport_of_collectd"
+
+    task :setup_apt_backport_of_collectd do
+      sources = <<-EOF
+        deb http://gb.archive.ubuntu.com/ubuntu/ maverick main universe
+        deb-src http://gb.archive.ubuntu.com/ubuntu/ maverick main universe
+      EOF
+      sources.gsub!(/^[ \t]*/, '')
+
+      prefs = <<-EOF
+        Package: *
+        Pin: release a=maverick
+        Pin-Priority: 400
+
+        Package: libxml2
+        Pin: release a=maverick
+        Pin-Priority: 900
+
+        Package: libxml2-dev
+        Pin: release a=maverick
+        Pin-Priority: 900
+
+        Package: collectd-core
+        Pin: release a=maverick
+        Pin-Priority: 900
+
+        Package: collectd-utils
+        Pin: release a=maverick
+        Pin-Priority: 900
+
+        Package: collectd
+        Pin: release a=maverick
+        Pin-Priority: 900
+      EOF
+      prefs.gsub!(/^[ \t]*/, '')
+
+      rubber.sudo_script 'setup_apt_backport_of_collectd', <<-ENDSCRIPT
+        release=`lsb_release -sr`
+        needs_backport=`echo "$release <= 10.04" | bc`
+        if [[ $needs_backport == 1 && ! -f /etc/apt/preferences.d/rubber-collectd ]]; then
+          echo -e #{sources.inspect} > /etc/apt/sources.list.d/rubber-collectd.list
+          echo -e #{prefs.inspect} > /etc/apt/preferences.d/rubber-collectd
+        fi
+      ENDSCRIPT
+    end
+    
+    task :install_graphite_from_repo, :roles => [:graphite_server, :graphite_web] do
+      rubber.sudo_script 'install_graphite_from_repo', <<-ENDSCRIPT
+        if [[ ! -d "/opt/graphite" ]]; then
+          mkdir -p /tmp/graphite_install
+          cd /tmp/graphite_install
+          bzr branch lp:graphite -r #{rubber_env.graphite_repo_version}
+
+          pushd graphite/whisper
+          python setup.py install
+          popd
+
+          pushd graphite/carbon
+          python setup.py install
+          popd
+
+          pushd graphite
+          python setup.py install
+          popd
+        fi
+      ENDSCRIPT
+    end
+
+    desc <<-DESC
+      Cleans out old whisper storage files for non-existant instances so they don't show in webui
+    DESC
+    task :clean_storage, :roles => [:graphite_server] do
+      active_instances = rubber_instances.collect {|ic| ic.name }.sort
+      stored_instances = capture("ls /opt/graphite/storage/whisper/servers/").split.sort
+      cleaning_instances = stored_instances - active_instances
+
+      if cleaning_instances.size > 0
+        logger.info "Cleaning dead instances from graphite storage: #{cleaning_instances.join(',')}"
+
+        do_clean = true
+        if (cleaning_instances.size.to_f / stored_instances.size) > 0.01
+          value = Capistrano::CLI.ui.ask("Graphite storage files to be cleaned exceeds threshold, proceed? [y/N]?: ")
+          do_clean = (value =~ /^y/)
+        end
+
+        if do_clean
+          rsudo "rm -rf /opt/graphite/storage/whisper/servers/{#{cleaning_instances.join(',')}}"
+        end
+
+      end
+    end
+
+    namespace :server do
+
+      rubber.allow_optional_tasks(self)
+
+      if Rubber::Configuration.rubber_env.graphite_repo_version
+        after "rubber:install_packages", "rubber:graphite:install_graphite_from_repo"
+      else
+        after "rubber:install_packages", "rubber:graphite:server:install"
+      end
+
+      after "rubber:bootstrap", "rubber:graphite:server:bootstrap"
+
+      desc <<-DESC
+        Installs graphite server components
+      DESC
+      task :install, :roles => :graphite_server do
+        rubber.sudo_script 'install_graphite_server', <<-ENDSCRIPT
+          if [[ ! -f "/opt/graphite/bin/carbon-cache.py" ]]; then
+            wget -qNP /tmp #{rubber_env.graphite_whisper_package_url}
+            tar -C /tmp -zxf /tmp/#{rubber_env.graphite_whisper_package_url.gsub(/.*\//, '')}
+            cd /tmp/#{rubber_env.graphite_whisper_package_url.gsub(/.*\//, '').gsub('.tar.gz', '')}
+            python setup.py install
+
+            wget -qNP /tmp #{rubber_env.graphite_carbon_package_url}
+            tar -C /tmp -zxf /tmp/#{rubber_env.graphite_carbon_package_url.gsub(/.*\//, '')}
+            cd /tmp/#{rubber_env.graphite_carbon_package_url.gsub(/.*\//, '').gsub('.tar.gz', '')}
+            python setup.py install
+
+          fi
+        ENDSCRIPT
+      end
+
+      task :bootstrap, :roles => :graphite_server do
+        exists = capture("echo $(ls /opt/graphite/storage/whisper/ 2> /dev/null)")
+        if exists.strip.size == 0
+          rubber.update_code_for_bootstrap
+
+          rubber.run_config(:file => "role/graphite_server", :force => true, :deploy_path => release_path)
+
+          restart
+        end
+      end
+
+      desc "Start graphite system monitoring"
+      task :start, :roles => :graphite_server do
+        rsudo "service graphite-server start"
+      end
+
+      desc "Stop graphite system monitoring"
+      task :stop, :roles => :graphite_server do
+        rsudo "service graphite-server stop || true"
+      end
+
+      desc "Restart graphite system monitoring"
+      task :restart, :roles => :graphite_server do
+        stop
+        start
+      end
+    end
+
+    namespace :web do
+
+      rubber.allow_optional_tasks(self)
+
+      if Rubber::Configuration.rubber_env.graphite_repo_version
+        after "rubber:graphite:server:install", "rubber:graphite:install_graphite_from_repo"
+      else
+        after "rubber:graphite:server:install", "rubber:graphite:web:install"
+      end
+      
+      after "rubber:graphite:server:bootstrap", "rubber:graphite:web:bootstrap"
+
+      desc <<-DESC
+        Installs graphite web components
+      DESC
+      task :install, :roles => :graphite_web do
+        rubber.sudo_script 'install_graphite_web', <<-ENDSCRIPT
+          if [[ ! -d "/opt/graphite/webapp" ]]; then
+            wget -qNP /tmp #{rubber_env.graphite_web_package_url}
+            tar -C /tmp -zxf /tmp/#{rubber_env.graphite_web_package_url.gsub(/.*\//, '')}
+            cd /tmp/#{rubber_env.graphite_web_package_url.gsub(/.*\//, '').gsub('.tar.gz', '')}
+            # python check-dependencies.py
+            python setup.py install
+          fi
+        ENDSCRIPT
+      end
+
+      task :bootstrap, :roles => :graphite_web do
+        exists = capture("echo $(ls /opt/graphite/storage/graphite.db 2> /dev/null)")
+        if exists.strip.size == 0
+          rubber.update_code_for_bootstrap
+
+          rubber.run_config(:file => "role/graphite_web", :force => true, :deploy_path => release_path)
+
+          # django email validation barfs on localhost, but for full_host to work
+          # in admin_email, we need a env with host defined
+          web_instance = rubber_instances.for_role("graphite_web").first
+          env = rubber_cfg.environment.bind("graphite_web", web_instance.name)
+          email = env.admin_email
+
+          rubber.sudo_script 'bootstrap_graphite_web', <<-ENDSCRIPT
+            cd /opt/graphite/webapp/graphite
+            python manage.py syncdb --noinput
+            python manage.py createsuperuser --username admin --email #{email} --noinput
+            python manage.py shell <<EOF
+from django.contrib.auth.models import User
+u = User.objects.get(username__exact='admin')
+u.set_password('admin1')
+u.save()
+EOF
+            mkdir -p /opt/graphite/storage/log/webapp/
+            chown -R www-data:www-data /opt/graphite/storage/
+          ENDSCRIPT
+
+          restart
+        end
+      end
+
+      desc "Start graphite system monitoring"
+      task :start, :roles => :graphite_web do
+        rsudo "service apache2 start"
+      end
+
+      desc "Stop graphite system monitoring"
+      task :stop, :roles => :graphite_web do
+        rsudo "service apache2 stop || true"
+      end
+
+      desc "Restart graphite system monitoring"
+      task :restart, :roles => :graphite_web do
+        stop
+        start
+      end
+
+    end
+
+  end
+
+end

data/templates/graphite/config/rubber/role/collectd/graphite-collectd.conf

@@ -0,0 +1,19 @@
+<%
+  @path = '/etc/collectd/conf.d/graphite.conf'
+%>
+
+<LoadPlugin "perl">
+    Globals true
+</LoadPlugin>
+
+<Plugin "perl">
+  BaseName "Collectd::Plugins"
+  LoadPlugin "Graphite"
+
+    <Plugin "Graphite">
+      Buffer "<%= Rubber.env == 'production' ? 1024 : 0 %>"
+      Prefix "servers"
+      Host   "<%= rubber_instances.for_role('graphite_server').first.full_name %>"
+      Port   "<%= rubber_env.graphite_server_port %>"
+    </Plugin>
+</Plugin>

data/templates/graphite/config/rubber/role/graphite_server/carbon.conf

@@ -0,0 +1,232 @@
+<%
+  @path = '/opt/graphite/conf/carbon.conf'
+%>
+[cache]
+# Configure carbon directories.
+#
+# OS environment variables can be used to tell carbon where graphite is
+# installed, where to read configuration from and where to write data.
+#
+#   GRAPHITE_ROOT        - Root directory of the graphite installation.
+#                          Defaults to ../
+#   GRAPHITE_CONF_DIR    - Configuration directory (where this file lives).
+#                          Defaults to $GRAPHITE_ROOT/conf/
+#   GRAPHITE_STORAGE_DIR - Storage directory for whipser/rrd/log/pid files.
+#                          Defaults to $GRAPHITE_ROOT/storage/
+#
+# To change other directory paths, add settings to this file. The following
+# configuration variables are available with these default values:
+#
+#   STORAGE_DIR    = $GRAPHITE_STORAGE_DIR
+#   LOCAL_DATA_DIR = STORAGE_DIR/whisper/
+#   WHITELISTS_DIR = STORAGE_DIR/lists/
+#   CONF_DIR       = STORAGE_DIR/conf/
+#   LOG_DIR        = STORAGE_DIR/log/
+#   PID_DIR        = STORAGE_DIR/
+#
+# For FHS style directory structures, use:
+#
+#   STORAGE_DIR    = /var/lib/carbon/
+#   CONF_DIR       = /etc/carbon/
+#   LOG_DIR        = /var/log/carbon/
+#   PID_DIR        = /var/run/
+#
+LOCAL_DATA_DIR = /opt/graphite/storage/whisper/
+
+# Specify the user to drop privileges to
+# If this is blank carbon runs as the user that invokes it
+# This user must have write access to the local data directory
+USER =
+
+# Limit the size of the cache to avoid swapping or becoming CPU bound.
+# Sorts and serving cache queries gets more expensive as the cache grows.
+# Use the value "inf" (infinity) for an unlimited cache size.
+MAX_CACHE_SIZE = inf
+
+# Limits the number of whisper update_many() calls per second, which effectively
+# means the number of write requests sent to the disk. This is intended to
+# prevent over-utilizing the disk and thus starving the rest of the system.
+# When the rate of required updates exceeds this, then carbon's caching will
+# take effect and increase the overall throughput accordingly.
+MAX_UPDATES_PER_SECOND = 1000
+
+# Softly limits the number of whisper files that get created each minute.
+# Setting this value low (like at 50) is a good way to ensure your graphite
+# system will not be adversely impacted when a bunch of new metrics are
+# sent to it. The trade off is that it will take much longer for those metrics'
+# database files to all get created and thus longer until the data becomes usable.
+# Setting this value high (like "inf" for infinity) will cause graphite to create
+# the files quickly but at the risk of slowing I/O down considerably for a while.
+MAX_CREATES_PER_MINUTE = 50
+
+LINE_RECEIVER_INTERFACE = 0.0.0.0
+LINE_RECEIVER_PORT = <%= rubber_env.graphite_server_port %>
+
+# Set this to True to enable the UDP listener. By default this is off
+# because it is very common to run multiple carbon daemons and managing
+# another (rarely used) port for every carbon instance is not fun.
+ENABLE_UDP_LISTENER = False
+UDP_RECEIVER_INTERFACE = 0.0.0.0
+UDP_RECEIVER_PORT = 2003
+
+PICKLE_RECEIVER_INTERFACE = 0.0.0.0
+PICKLE_RECEIVER_PORT = 2004
+
+# Per security concerns outlined in Bug #817247 the pickle receiver
+# will use a more secure and slightly less efficient unpickler.
+# Set this to True to revert to the old-fashioned insecure unpickler.
+USE_INSECURE_UNPICKLER = False
+
+CACHE_QUERY_INTERFACE = 0.0.0.0
+CACHE_QUERY_PORT = 7002
+
+# Set this to False to drop datapoints received after the cache
+# reaches MAX_CACHE_SIZE. If this is True (the default) then sockets
+# over which metrics are received will temporarily stop accepting
+# data until the cache size falls below 95% MAX_CACHE_SIZE.
+USE_FLOW_CONTROL = True
+
+# By default, carbon-cache will log every whisper update. This can be excessive and
+# degrade performance if logging on the same volume as the whisper data is stored.
+LOG_UPDATES = False
+
+# On some systems it is desirable for whisper to write synchronously.
+# Set this option to True if you'd like to try this. Basically it will
+# shift the onus of buffering writes from the kernel into carbon's cache.
+WHISPER_AUTOFLUSH = False
+
+# Enable AMQP if you want to receve metrics using an amqp broker
+# ENABLE_AMQP = False
+
+# Verbose means a line will be logged for every metric received
+# useful for testing
+# AMQP_VERBOSE = False
+
+# AMQP_HOST = localhost
+# AMQP_PORT = 5672
+# AMQP_VHOST = /
+# AMQP_USER = guest
+# AMQP_PASSWORD = guest
+# AMQP_EXCHANGE = graphite
+# AMQP_METRIC_NAME_IN_BODY = False
+
+# The manhole interface allows you to SSH into the carbon daemon
+# and get a python interpreter. BE CAREFUL WITH THIS! If you do
+# something like time.sleep() in the interpreter, the whole process
+# will sleep! This is *extremely* helpful in debugging, assuming
+# you are familiar with the code. If you are not, please don't
+# mess with this, you are asking for trouble :)
+#
+# ENABLE_MANHOLE = False
+# MANHOLE_INTERFACE = 127.0.0.1
+# MANHOLE_PORT = 7222
+# MANHOLE_USER = admin
+# MANHOLE_PUBLIC_KEY = ssh-rsa AAAAB3NzaC1yc2EAAAABiwAaAIEAoxN0sv/e4eZCPpi3N3KYvyzRaBaMeS2RsOQ/cDuKv11dlNzVeiyc3RFmCv5Rjwn/lQ79y0zyHxw67qLyhQ/kDzINc4cY41ivuQXm2tPmgvexdrBv5nsfEpjs3gLZfJnyvlcVyWK/lId8WUvEWSWHTzsbtmXAF2raJMdgLTbQ8wE=
+
+# Patterns for all of the metrics this machine will store. Read more at
+# http://en.wikipedia.org/wiki/Advanced_Message_Queuing_Protocol#Bindings
+#
+# Example: store all sales, linux servers, and utilization metrics
+# BIND_PATTERNS = sales.#, servers.linux.#, #.utilization
+#
+# Example: store everything
+# BIND_PATTERNS = #
+
+# To configure special settings for the 'foo' carbon-cache instance, uncomment this:
+#[cache:foo]
+#LINE_RECEIVER_PORT = 2103
+#PICKLE_RECEIVER_PORT = 2104
+#CACHE_QUERY_PORT = 7102
+# and any other settings you want to customize, defaults are inherited
+# from [carbon] section.
+
+
+[relay]
+LINE_RECEIVER_INTERFACE = 0.0.0.0
+LINE_RECEIVER_PORT = 2013
+PICKLE_RECEIVER_INTERFACE = 0.0.0.0
+PICKLE_RECEIVER_PORT = 2014
+
+# To use consistent hashing instead of the user defined relay-rules.conf,
+# change this to:
+# RELAY_METHOD = consistent-hashing
+RELAY_METHOD = rules
+
+# If you use consistent-hashing you may want to add redundancy
+# of your data by replicating every datapoint to more than
+# one machine.
+REPLICATION_FACTOR = 1
+
+# This is a list of carbon daemons we will send any relayed or
+# generated metrics to. The default provided would send to a single
+# carbon-cache instance on the default port. However if you
+# use multiple carbon-cache instances then it would look like this:
+#
+# DESTINATIONS = 127.0.0.1:2004:a, 127.0.0.1:2104:b
+#
+# The general form is IP:PORT:INSTANCE where the :INSTANCE part is
+# optional and refers to the "None" instance if omitted.
+#
+# Note that if the destinations are all carbon-caches then this should
+# exactly match the webapp's CARBONLINK_HOSTS setting in terms of
+# instances listed (order matters!).
+DESTINATIONS = 127.0.0.1:2004
+
+# This defines the maximum "message size" between carbon daemons.
+# You shouldn't need to tune this unless you really know what you're doing.
+MAX_DATAPOINTS_PER_MESSAGE = 500
+MAX_QUEUE_SIZE = 10000
+
+# Set this to False to drop datapoints when any send queue (sending datapoints
+# to a downstream carbon daemon) hits MAX_QUEUE_SIZE. If this is True (the
+# default) then sockets over which metrics are received will temporarily stop accepting
+# data until the send queues fall below 80% MAX_QUEUE_SIZE.
+USE_FLOW_CONTROL = True
+
+
+[aggregator]
+LINE_RECEIVER_INTERFACE = 0.0.0.0
+LINE_RECEIVER_PORT = 2023
+
+PICKLE_RECEIVER_INTERFACE = 0.0.0.0
+PICKLE_RECEIVER_PORT = 2024
+
+# This is a list of carbon daemons we will send any relayed or
+# generated metrics to. The default provided would send to a single
+# carbon-cache instance on the default port. However if you
+# use multiple carbon-cache instances then it would look like this:
+#
+# DESTINATIONS = 127.0.0.1:2004:a, 127.0.0.1:2104:b
+#
+# The format is comma-delimited IP:PORT:INSTANCE where the :INSTANCE part is
+# optional and refers to the "None" instance if omitted.
+#
+# Note that if the destinations are all carbon-caches then this should
+# exactly match the webapp's CARBONLINK_HOSTS setting in terms of
+# instances listed (order matters!).
+DESTINATIONS = 127.0.0.1:2004
+
+# If you want to add redundancy to your data by replicating every
+# datapoint to more than one machine, increase this.
+REPLICATION_FACTOR = 1
+
+# This is the maximum number of datapoints that can be queued up
+# for a single destination. Once this limit is hit, we will
+# stop accepting new data if USE_FLOW_CONTROL is True, otherwise
+# we will drop any subsequently received datapoints.
+MAX_QUEUE_SIZE = 10000
+
+# Set this to False to drop datapoints when any send queue (sending datapoints
+# to a downstream carbon daemon) hits MAX_QUEUE_SIZE. If this is True (the
+# default) then sockets over which metrics are received will temporarily stop accepting
+# data until the send queues fall below 80% MAX_QUEUE_SIZE.
+USE_FLOW_CONTROL = True
+
+# This defines the maximum "message size" between carbon daemons.
+# You shouldn't need to tune this unless you really know what you're doing.
+MAX_DATAPOINTS_PER_MESSAGE = 500
+
+# This defines how many datapoints the aggregator remembers for
+# each metric. Aggregation only happens for datapoints that fall in
+# the past MAX_AGGREGATION_INTERVALS * intervalSize seconds.
+MAX_AGGREGATION_INTERVALS = 5