rubber 2.6.5 → 2.7.0

data/templates/discourse/config/rubber/rubber-discourse.yml

@@ -0,0 +1,3 @@
+roles:
+  discourse:
+    packages: [libtool, libpq-dev, gawk, pngcrush, imagemagick, python-software-properties, postgresql-contrib]

data/templates/discourse/templates.yml

@@ -0,0 +1,5 @@
+description: A fairly complete and scalable discourse deployment setup using unicorn/nginx
+dependent_templates:
+  - complete_unicorn_nginx
+  - postgresql
+  - sidekiq

data/templates/elasticsearch/config/rubber/deploy-elasticsearch.rb

@@ -8,13 +8,13 @@ namespace :rubber do
 
     task :install, :roles => :elasticsearch do
       rubber.sudo_script 'install_elasticsearch', <<-ENDSCRIPT
-        if [[ ! -d "#{rubber_env.elasticsearch_dir}" ]]; then
-          wget --no-check-certificate -qNP /tmp http://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-#{rubber_env.elasticsearch_version}.zip
-          unzip -d #{rubber_env.elasticsearch_prefix} /tmp/elasticsearch-#{rubber_env.elasticsearch_version}.zip
-          rm /tmp/elasticsearch-#{rubber_env.elasticsearch_version}.zip
+        if [[ ! -f /usr/share/elasticsearch/lib/elasticsearch-#{rubber_env.elasticsearch_version}.jar ]]; then
+          wget -qNP /tmp https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-#{rubber_env.elasticsearch_version}.deb
+          dpkg -i /tmp/elasticsearch-#{rubber_env.elasticsearch_version}.deb
+          rm /tmp/elasticsearch-#{rubber_env.elasticsearch_version}.deb
 
-          #{rubber_env.elasticsearch_dir}/bin/plugin -install mobz/elasticsearch-head
-          
+          rm -rf /usr/share/elasticsearch/plugins/head
+          /usr/share/elasticsearch/bin/plugin -install mobz/elasticsearch-head
         fi
       ENDSCRIPT
     end
@@ -22,7 +22,7 @@ namespace :rubber do
     after "rubber:bootstrap", "rubber:elasticsearch:bootstrap"
 
     task :bootstrap, :roles => :elasticsearch do
-      exists = capture("echo $(ls /etc/init/elasticsearch.conf 2> /dev/null)")
+      exists = capture("echo $(ls #{rubber_env.elasticsearch_data_dir} 2> /dev/null)")
       if exists.strip.size == 0
         # After everything installed on machines, we need the source tree
         # on hosts in order to run rubber:config for bootstrapping the db

data/templates/elasticsearch/config/rubber/role/elasticsearch/elasticsearch-default.conf

@@ -0,0 +1,47 @@
+<%
+  @path = '/etc/default/elasticsearch'
+%>
+# Run ElasticSearch as this user ID and group ID
+#ES_USER=elasticsearch
+#ES_GROUP=elasticsearch
+
+# Heap Size (defaults to 256m min, 1g max)
+#ES_HEAP_SIZE=2g
+
+# Heap new generation
+#ES_HEAP_NEWSIZE=
+
+# max direct memory
+#ES_DIRECT_SIZE=
+
+# Maximum number of open files, defaults to 65535.
+#MAX_OPEN_FILES=65535
+
+# Maximum locked memory size. Set to "unlimited" if you use the
+# bootstrap.mlockall option in elasticsearch.yml. You must also set
+# ES_HEAP_SIZE.
+#MAX_LOCKED_MEMORY=unlimited
+
+# Maximum number of VMA (Virtual Memory Areas) a process can own
+#MAX_MAP_COUNT=262144
+
+# ElasticSearch log directory
+LOG_DIR=<%= rubber_env.elasticsearch_log_dir %>
+
+# ElasticSearch data directory
+DATA_DIR=<%= rubber_env.elasticsearch_data_dir %>
+
+# ElasticSearch work directory
+WORK_DIR=<%= rubber_env.elasticsearch_work_dir %>
+
+# ElasticSearch configuration directory
+#CONF_DIR=/etc/elasticsearch
+
+# ElasticSearch configuration file (elasticsearch.yml)
+#CONF_FILE=/etc/elasticsearch/elasticsearch.yml
+
+# Additional Java OPTS
+#ES_JAVA_OPTS=
+
+# Configure restart on package upgrade (true, every other setting will lead to not restarting)
+#RESTART_ON_UPGRADE=true

data/templates/elasticsearch/config/rubber/role/elasticsearch/elasticsearch.yml

@@ -1,12 +1,6 @@
 <%
-  @path = "#{rubber_env.elasticsearch_dir}/config/elasticsearch.yml"
-  @post = <<-POST
-    mkdir -p #{rubber_env.elasticsearch_work_dir}
-    mkdir -p #{rubber_env.elasticsearch_data_dir}
-    mkdir -p #{rubber_env.elasticsearch_log_dir}
-  POST
+  @path = '/etc/elasticsearch/elasticsearch.yml'
 -%>
-
 ##################### ElasticSearch Configuration Example #####################
 
 # This file contains an overview of various configuration settings,
@@ -14,7 +8,7 @@
 # consult the guide at <http://elasticsearch.org/guide>.
 #
 # The installation procedure is covered at
-# <http://elasticsearch.org/guide/reference/setup/installation.html>.
+# <http://elasticsearch.org/guide/en/elasticsearch/reference/current/setup.html>.
 #
 # ElasticSearch comes with reasonable defaults for most settings,
 # so you can try it out without bothering with configuration.
@@ -29,8 +23,8 @@
 #
 # node.rack: ${RACK_ENV_VAR}
 
-# See <http://elasticsearch.org/guide/reference/setup/configuration.html>
-# for information on supported formats and syntax for the configuration file.
+# For information on supported formats and syntax for the config file, see
+# <http://elasticsearch.org/guide/en/elasticsearch/reference/current/setup-configuration.html>
 
 
 ################################### Cluster ###################################
@@ -38,7 +32,7 @@
 # Cluster name identifies your cluster for auto-discovery. If you're running
 # multiple clusters on the same network, make sure you're using unique names.
 #
-cluster.name: <%= rubber_env.app_name %>
+cluster.name: <%= rubber_env.graylog_elasticsearch_index %>
 
 
 #################################### Node #####################################
@@ -91,6 +85,10 @@ cluster.name: <%= rubber_env.app_name %>
 #
 # node.rack: rack314
 
+# By default, multiple nodes are allowed to start from the same installation location
+# to disable it, set the following:
+# node.max_local_storage_nodes: 1
+
 
 #################################### Index ####################################
 
@@ -101,8 +99,8 @@ cluster.name: <%= rubber_env.app_name %>
 # Note, that it makes more sense to configure index settings specifically for
 # a certain index, either when creating it or by using the index templates API.
 #
-# See <http://elasticsearch.org/guide/reference/index-modules/> and
-# <http://elasticsearch.org/guide/reference/api/admin-indices-create-index.html>
+# See <http://elasticsearch.org/guide/en/elasticsearch/reference/current/index-modules.html> and
+# <http://elasticsearch.org/guide/en/elasticsearch/reference/current/indices-create-index.html>
 # for more information.
 
 # Set the number of shards (splits) of an index (5 by default):
@@ -150,10 +148,9 @@ cluster.name: <%= rubber_env.app_name %>
 # Path to directory where to store index data allocated for this node.
 #
 path.data: <%= rubber_env.elasticsearch_data_dir %>
-
 #
 # Can optionally include more than one location, causing data to be striped across
-# the locations (à la RAID 0) on a file level, favouring locations with most free
+# the locations (a la RAID 0) on a file level, favouring locations with most free
 # space on creation. For example:
 #
 # path.data: /path/to/data1,/path/to/data2
@@ -171,6 +168,13 @@ path.logs: <%= rubber_env.elasticsearch_log_dir %>
 # path.plugins: /path/to/plugins
 
 
+#################################### Plugin ###################################
+
+# If a plugin listed here is not installed for current node, the node will not start.
+#
+# plugin.mandatory: mapper-attachments,lang-groovy
+
+
 ################################### Memory ####################################
 
 # ElasticSearch performs poorly when JVM starts swapping: you should ensure that
@@ -236,15 +240,16 @@ http.port: <%= rubber_env.elasticsearch_http_port %>
 # in the gateway, and when the cluster starts up for the first time,
 # it will read its state from the gateway.
 
-# There are several types of gateway implementations. For more information,
-# see <http://elasticsearch.org/guide/reference/modules/gateway>.
+# There are several types of gateway implementations. For more information, see
+# <http://elasticsearch.org/guide/en/elasticsearch/reference/current/modules-gateway.html>.
 
 # The default gateway type is the "local" gateway (recommended):
 #
 # gateway.type: local
 
 # Settings below control how and when to start the initial recovery process on
-# a full cluster restart (to reuse as much local data as possible).
+# a full cluster restart (to reuse as much local data as possible when using shared
+# gateway).
 
 # Allow recovery process after N nodes in a cluster are up:
 #
@@ -256,7 +261,8 @@ http.port: <%= rubber_env.elasticsearch_http_port %>
 # gateway.recover_after_time: 5m
 
 # Set how many nodes are expected in this cluster. Once these N nodes
-# are up, begin recovery process immediately:
+# are up (and recover_after_nodes is met), begin recovery process immediately
+# (without waiting for recover_after_time to expire):
 #
 # gateway.expected_nodes: 2
 
@@ -277,9 +283,9 @@ http.port: <%= rubber_env.elasticsearch_http_port %>
 #
 # cluster.routing.allocation.node_concurrent_recoveries: 2
 
-# Set to throttle throughput when recovering (eg. 100mb, by default unlimited):
+# Set to throttle throughput when recovering (eg. 100mb, by default 20mb):
 #
-# indices.recovery.max_size_per_sec: 0
+# indices.recovery.max_bytes_per_sec: 20mb
 
 # Set to limit the number of open concurrent streams when
 # recovering a shard from a peer:
@@ -293,8 +299,8 @@ http.port: <%= rubber_env.elasticsearch_http_port %>
 # and master node is elected. Multicast discovery is the default.
 
 # Set to ensure a node sees N other master eligible nodes to be considered
-# operational within the cluster. Set this option to a higher value (2-4)
-# for large clusters:
+# operational within the cluster. Its recommended to set it to a higher value
+# than 1 when running more than 2 nodes in the cluster.
 #
 # discovery.zen.minimum_master_nodes: 1
 
@@ -304,8 +310,8 @@ http.port: <%= rubber_env.elasticsearch_http_port %>
 #
 # discovery.zen.ping.timeout: 3s
 
-# See <http://elasticsearch.org/guide/reference/modules/discovery/zen.html>
-# for more information.
+# For more information, see
+# <http://elasticsearch.org/guide/en/elasticsearch/reference/current/modules-discovery-zen.html>
 
 # Unicast discovery allows to explicitly control which nodes will be used
 # to discover the cluster. It can be used when multicast is not present,
@@ -318,16 +324,16 @@ http.port: <%= rubber_env.elasticsearch_http_port %>
 # 2. Configure an initial list of master nodes in the cluster
 #    to perform discovery when new nodes (master or data) are started:
 #
-# discovery.zen.ping.unicast.hosts: ["host1", "host2:port", "host3[portX-portY]"]
+# discovery.zen.ping.unicast.hosts: ["host1", "host2:port"]
 
 # EC2 discovery allows to use AWS EC2 API in order to perform discovery.
 #
 # You have to install the cloud-aws plugin for enabling the EC2 discovery.
 #
-# See <http://elasticsearch.org/guide/reference/modules/discovery/ec2.html>
-# for more information.
+# For more information, see
+# <http://elasticsearch.org/guide/en/elasticsearch/reference/current/modules-discovery-ec2.html>
 #
-# See <http://elasticsearch.org/tutorials/2011/08/22/elasticsearch-on-ec2.html>
+# See <http://elasticsearch.org/tutorials/elasticsearch-on-ec2/>
 # for a step-by-step tutorial.
 
 
@@ -335,7 +341,6 @@ http.port: <%= rubber_env.elasticsearch_http_port %>
 
 # Shard level query and fetch threshold logging.
 
-#index.search.slowlog.level: TRACE
 #index.search.slowlog.threshold.query.warn: 10s
 #index.search.slowlog.threshold.query.info: 5s
 #index.search.slowlog.threshold.query.debug: 2s
@@ -346,3 +351,17 @@ http.port: <%= rubber_env.elasticsearch_http_port %>
 #index.search.slowlog.threshold.fetch.debug: 500ms
 #index.search.slowlog.threshold.fetch.trace: 200ms
 
+#index.indexing.slowlog.threshold.index.warn: 10s
+#index.indexing.slowlog.threshold.index.info: 5s
+#index.indexing.slowlog.threshold.index.debug: 2s
+#index.indexing.slowlog.threshold.index.trace: 500ms
+
+################################## GC Logging ################################
+
+#monitor.jvm.gc.young.warn: 1000ms
+#monitor.jvm.gc.young.info: 700ms
+#monitor.jvm.gc.young.debug: 400ms
+
+#monitor.jvm.gc.old.warn: 10s
+#monitor.jvm.gc.old.info: 5s
+#monitor.jvm.gc.old.debug: 2s

data/templates/elasticsearch/config/rubber/role/elasticsearch/logging.yml

@@ -1,8 +1,9 @@
 <%
-  @path = "#{rubber_env.elasticsearch_dir}/config/logging.yml"
+  @path = '/etc/elasticsearch/logging.yml'
 %>
-
-rootLogger: INFO, console, file
+# you can override this using by setting a system property, for example -Des.logger.level=DEBUG
+es.logger.level: INFO
+rootLogger: ${es.logger.level}, console, file
 logger:
   # log action execution errors for easier debugging
   action: DEBUG
@@ -20,9 +21,11 @@ logger:
   #discovery: TRACE
 
   index.search.slowlog: TRACE, index_search_slow_log_file
+  index.indexing.slowlog: TRACE, index_indexing_slow_log_file
 
 additivity:
   index.search.slowlog: false
+  index.indexing.slowlog: false
 
 appender:
   console:
@@ -46,3 +49,11 @@ appender:
     layout:
       type: pattern
       conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"
+
+  index_indexing_slow_log_file:
+    type: dailyRollingFile
+    file: ${path.logs}/${cluster.name}_index_indexing_slowlog.log
+    datePattern: "'.'yyyy-MM-dd"
+    layout:
+      type: pattern
+      conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"

data/templates/elasticsearch/config/rubber/role/elasticsearch/monit-elasticsearch.conf

@@ -0,0 +1,8 @@
+<%
+  @path = '/etc/monit/monit.d/monit-elasticsearch.conf'
+%>
+check process elasticsearch with pidfile /var/run/elasticsearch.pid
+  group elasticsearch-<%= Rubber.env %>
+  start program = "/usr/bin/env service elasticsearch start"
+  stop program = "/usr/bin/env service elasticsearch stop"
+  if failed host <%= rubber_env.host %> port <%= rubber_env.elasticsearch_http_port %> with timeout 10 seconds for 10 cycles then restart

data/templates/elasticsearch/config/rubber/rubber-elasticsearch.yml

@@ -1,10 +1,7 @@
-elasticsearch_version: 0.19.12
-elasticsearch_prefix: /usr/local
-elasticsearch_dir: "#{elasticsearch_prefix}/elasticsearch-#{elasticsearch_version}"
+elasticsearch_version: 0.90.10
 elasticsearch_data_dir: "/mnt/elasticsearch/data"
 elasticsearch_work_dir: "/mnt/elasticsearch/work"
 elasticsearch_log_dir: "/mnt/elasticsearch/logs"
-elasticsearch_pid_file: "/var/run/elasticsearch.pid"
 elasticsearch_http_port: 9200
 
 roles:

data/templates/ffmpeg/config/rubber/deploy-ffmpeg.rb

@@ -0,0 +1,13 @@
+namespace :rubber do
+  namespace :ffmpeg do
+
+    rubber.allow_optional_tasks(self)
+
+    # Use Jon Severinsson's FFmpeg PPA so we can install the "real" ffmpeg (Ubuntu uses libav / avconv)
+    before "rubber:install_packages", "rubber:ffmpeg:setup_apt_sources"
+    task :setup_apt_sources do
+      run "add-apt-repository -y ppa:jon-severinsson/ffmpeg"
+    end
+  end
+end
+

data/templates/ffmpeg/config/rubber/rubber-ffmpeg.yml

@@ -0,0 +1,5 @@
+roles:
+  ffmpeg:
+    # NOTE: In order to get the correct ffmpeg package, we must use Jon Severinsson's FFmpeg PPA
+    # See deploy-ffmpeg.rb. libavcodec-extra-53 is needed to encode AAC audio
+    packages: [ffmpeg, libavcodec-extra-53]

data/templates/ffmpeg/templates.yml

@@ -0,0 +1 @@
+description: The FFmpeg module

data/templates/graylog/config/rubber/deploy-graylog.rb

@@ -11,8 +11,9 @@ namespace :rubber do
       task :install, :roles => :graylog_server do
         rubber.sudo_script 'install_graylog_server', <<-ENDSCRIPT
           if [[ ! -d "#{rubber_env.graylog_server_dir}" ]]; then
-            wget --no-check-certificate -qNP /tmp #{rubber_env.graylog_server_pkg_url}
-            tar -C #{rubber_env.graylog_server_prefix} -zxf /tmp/graylog2-server-#{rubber_env.graylog_server_version}.tar.gz
+            wget --no-check-certificate -qNP /tmp https://github.com/Graylog2/graylog2-server/releases/download/#{rubber_env.graylog_server_version}/graylog2-server-#{rubber_env.graylog_server_version}.tgz
+            tar -C #{rubber_env.graylog_server_prefix} -zxf /tmp/graylog2-server-#{rubber_env.graylog_server_version}.tgz
+            rm /tmp/graylog2-server-#{rubber_env.graylog_server_version}.tgz
           fi
         ENDSCRIPT
       end
@@ -27,6 +28,8 @@ namespace :rubber do
         end
       end
 
+      before "rubber:graylog:server:bootstrap", "rubber:mongodb:restart"
+
       task :bootstrap, :roles => :graylog_server do
         exists = capture("echo $(ls /etc/graylog2.conf 2> /dev/null)")
         if exists.strip.size == 0
@@ -64,11 +67,9 @@ namespace :rubber do
       task :install, :roles => :graylog_web do
         rubber.sudo_script 'install_graylog_web', <<-ENDSCRIPT
           if [[ ! -d "#{rubber_env.graylog_web_dir}" ]]; then
-            wget --no-check-certificate -qNP /tmp #{rubber_env.graylog_web_pkg_url}
-            tar -C #{rubber_env.graylog_web_prefix} -zxf /tmp/graylog2-web-interface-#{rubber_env.graylog_web_version}.tar.gz
-
-            mkdir #{rubber_env.graylog_web_dir}/log
-            mkdir #{rubber_env.graylog_web_dir}/tmp
+            wget --no-check-certificate -qNP /tmp https://github.com/Graylog2/graylog2-web-interface/releases/download/#{rubber_env.graylog_web_version}/graylog2-web-interface-#{rubber_env.graylog_web_version}.tgz
+            tar -C #{rubber_env.graylog_web_prefix} -zxf /tmp/graylog2-web-interface-#{rubber_env.graylog_web_version}.tgz
+            rm /tmp/graylog2-web-interface-#{rubber_env.graylog_web_version}.tgz
           fi
         ENDSCRIPT
       end
@@ -82,23 +83,19 @@ namespace :rubber do
 
           rubber.run_config(:file => "role/graylog_web/", :force => true, :deploy_path => release_path)
 
-          rubber.sudo_script 'bootstrap_graylog_web', <<-ENDSCRIPT
-            cd #{rubber_env.graylog_web_dir}
-
-            # Add puma to the Gemfile so we can run the server.
-            echo "gem 'puma'" >> Gemfile
-
-            export RAILS_ENV=production
-            bundle install
-
-            # Create the Graylog Web admin account.
-            ./script/rails runner "User.create(:login => '#{rubber_env.graylog_web_username}', :email => '#{rubber_env.graylog_web_email}', :password => '#{rubber_env.graylog_web_password}', :password_confirmation => '#{rubber_env.graylog_web_password}', :role => 'admin') if User.count == 0"
-          ENDSCRIPT
-
           restart
         end
       end
 
+      after "rubber:graylog:web:bootstrap", "rubber:graylog:web:create_inputs"
+
+      task :create_inputs, :roles => :graylog_web do
+        rubber.sudo_script 'create_inputs', <<-ENDSCRIPT
+          curl --user #{rubber_env.graylog_web_username}:#{rubber_env.graylog_web_password} -XPOST http://localhost:12900/system/inputs -H "Content-Type: application/json" -d '{"type": "org.graylog2.inputs.gelf.udp.GELFUDPInput", "creator_user_id": "admin", "title": "gelf-udp", "global": true, "configuration": { "port": #{rubber_env.graylog_server_port}, "bind_address": "0.0.0.0" } }'
+          curl --user #{rubber_env.graylog_web_username}:#{rubber_env.graylog_web_password} -XPOST http://localhost:12900/system/inputs -H "Content-Type: application/json" -d '{"type": "org.graylog2.inputs.syslog.udp.SyslogUDPInput", "creator_user_id": "admin", "title": "syslog-udp", "global": true, "configuration": { "port": #{rubber_env.graylog_server_syslog_port}, "bind_address": "0.0.0.0" } }'
+        ENDSCRIPT
+      end
+
       desc "Stops the graylog web"
       task :stop, :roles => :graylog_web, :on_error => :continue do
         rsudo "service graylog-web stop || true"

data/templates/graylog/config/rubber/role/graylog_server/graylog2.conf

@@ -1,42 +1,131 @@
 <%
   @path = "/etc/graylog2.conf"
+
+  require 'digest/sha2'
+  root_password_sha2 = (Digest::SHA2.new << rubber_env.graylog_web_password).to_s
+
+  es_servers = rubber_instances.for_role('graylog_elasticsearch').collect { |i| "#{i.internal_ip}:9300" }.join(',')
 %>
+# If you are running more than one instances of graylog2-server you have to select one of these
+# instances as master. The master will perform some periodical tasks that non-masters won't perform.
+is_master = true
+
+# The auto-generated node ID will be stored in this file and read after restarts. It is a good idea
+# to use an absolute file path here if you are starting graylog2-server from init scripts or similar.
+node_id_file = /etc/graylog2-server-node-id
+
+# You MUST set a secret to secure/pepper the stored user passwords here. Use at least 64 characters.
+# Generate one by using for example: pwgen -s 96
+password_secret = <%= rubber_env.graylog_server_secret %>
+
+# the default root user is named 'admin'
+root_username = <%= rubber_env.graylog_web_username %>
+
+# You MUST specify a hash password for the root user (which you only need to initially set up the
+# system and in case you lose connectivity to your authentication backend)
+# Create one by using for example: echo -n yourpassword | shasum -a 256
+# and put the resulting hash value into the following line
+root_password_sha2 = <%= root_password_sha2 %>
+
+# Set plugin directory here (relative or absolute)
+plugin_dir = plugin
+
+# REST API listen URI. Must be reachable by other graylog2-server nodes if you run a cluster.
+rest_listen_uri = http://<%= rubber_instances[rubber_env.host].internal_ip %>:12900/
+
+# REST API transport address. Defaults to first non-loopback IPv4 system address and port 12900.
+# This will be promoted in the cluster discovery APIs and other nodes may try to connect on this
+# address. (see rest_listen_uri)
+rest_transport_uri = http://<%= rubber_instances[rubber_env.host].internal_ip %>:12900/
+
+# Embedded elasticsearch configuration file
+# pay attention to the working directory of the server, maybe use an absolute path here
+elasticsearch_config_file = /etc/graylog2-elasticsearch.yml
+
+elasticsearch_max_docs_per_index = 20000000
+
+# How many indices do you want to keep?
+# elasticsearch_max_number_of_indices*elasticsearch_max_docs_per_index=total number of messages in your setup
+elasticsearch_max_number_of_indices = 20
+
+# Decide what happens with the oldest indices when the maximum number of indices is reached.
+# The following strategies are availble:
+#   - delete # Deletes the index completely (Default)
+#   - close # Closes the index and hides it from the system. Can be re-opened later.
+retention_strategy = delete
+
+# How many ElasticSearch shards and replicas should be used per index? Note that this only applies to newly created indices.
+elasticsearch_shards = <%= [rubber_instances.for_role('graylog_elasticsearch').size, 1].max %>
+elasticsearch_replicas = 0
 
-# On which port (UDP) should we listen for Syslog messages? (Standard: 514)
-syslog_listen_port = <%= rubber_env.graylog_server_syslog_port %>
-syslog_protocol = udp
+elasticsearch_index_prefix = graylog2
 
-# ElasticSearch URL (default: http://localhost:9200/)
-elasticsearch_url = http://<%= rubber_instances.for_role('graylog_elasticsearch').first.full_name %>:9200/
-elasticsearch_index_name = <%= rubber_env.graylog_elasticsearch_index %>
+# settings to be passed to elasticsearch's client (overriding those in the provided elasticsearch_config_file)
+# all these
+# this must be the same as for your elasticsearch cluster
+elasticsearch_cluster_name = <%= rubber_env.graylog_elasticsearch_index %>
 
-# Always try a reverse DNS lookup instead of parsing hostname from syslog message?
-force_syslog_rdns = false
+# you could also leave this out, but makes it easier to identify the graylog2 client instance
+elasticsearch_node_name = <%= rubber_env.host %>
+
+# we don't want the graylog2 server to store any data, or be master node
+#elasticsearch_node_master = false
+#elasticsearch_node_data = false
+
+# use a different port if you run multiple elasticsearch nodes on one machine
+#elasticsearch_transport_tcp_port = 9350
+# we don't need to run the embedded HTTP server here
+#elasticsearch_http_enabled = false
+
+elasticsearch_discovery_zen_ping_multicast_enabled = false
+elasticsearch_discovery_zen_ping_unicast_hosts = <%= es_servers %>
+
+
+# Analyzer (tokenizer) to use for message and full_message field. The "standard" filter usually is a good idea.
+# All supported analyzers are: standard, simple, whitespace, stop, keyword, pattern, language, snowball, custom
+# ElasticSearch documentation: http://www.elasticsearch.org/guide/reference/index-modules/analysis/
+# Note that this setting only takes effect on newly created indices.
+elasticsearch_analyzer = standard
+
+# Batch size for all outputs. This is the maximum (!) number of messages an output module will get at once.
+# For example, if this is set to 5000 (default), the ElasticSearch Output will not index more than 5000 messages
+# at once. After that index operation is performed, the next batch will be indexed. If there is only 1 message
+# waiting, it will only index that single message. It is important to raise this parameter if you send in so
+# many messages that it is not enough to index 5000 messages at once. (Only at *really* high message rates)
+output_batch_size = 5000
+
+# The number of parallel running processors.
+# Raise this number if your buffers are filling up.
+processbuffer_processors = 5
+outputbuffer_processors = 5
+
+# Wait strategy describing how buffer processors wait on a cursor sequence. (default: sleeping)
+# Possible types:
+#  - yielding
+#     Compromise between performance and CPU usage.
+#  - sleeping
+#     Compromise between performance and CPU usage. Latency spikes can occur after quiet periods.
+#  - blocking
+#     High throughput, low latency, higher CPU usage.
+#  - busy_spinning
+#     Avoids syscalls which could introduce latency jitter. Best when threads can be bound to specific CPU cores.
+processor_wait_strategy = blocking
+
+# Size of internal ring buffers. Raise this if raising outputbuffer_processors does not help anymore.
+# For optimum performance your LogMessage objects in the ring buffer should fit in your CPU L3 cache.
+# Start server with --statistics flag to see buffer utilization.
+# Must be a power of 2. (512, 1024, 2048, ...)
+ring_size = 1024
 
 # MongoDB Configuration
 mongodb_useauth = false
-mongodb_user = grayloguser
-mongodb_password = 123
+#mongodb_user = grayloguser
+#mongodb_password = 123
 mongodb_host = <%= rubber_instances.for_role('graylog_mongodb').first.full_name %>
 #mongodb_replica_set = localhost:27017,localhost:27018,localhost:27019
 mongodb_database = <%= rubber_env.graylog_mongo_database %>
 mongodb_port = 27017
 
-# Graylog2 uses an internal message queue that holds all received messages until they are indexed. The mq_batch_size parameter defines how many messages are sent
-# to ElasticSearch at once (using a _bulk update: http://www.elasticsearch.org/guide/reference/api/bulk.html). The mq_poll_freq parameter controls in which
-# interval (in seconds) the message batch is sent. Example: If you leave the standard values (mq_batch_size = 4000, mq_poll_freq = 1), Graylog2 will index 4000 messages
-# every second. If you have spikes with more than 4000 messages per second, the queue will start growing until you get under 4000 messages/second again. The queue is
-# FIFO and can grow until you run out of RAM. Note that the queue *only* resists in RAM, so if you set the mq_poll_freq to a high value, you may lose a lot of not yet
-# indexed messages when the server crashes. Run the server in debug mode (java -jar graylog2-server.jar --debug) with a |grep '^INFO' to see debug information about
-# the queue and it's size. (INFO : org.graylog2.periodical.BulkIndexerThread - About to index max 4000 messages. You have a total of 103 messages in the queue. [freq:1s])
-# You can also monitor the queue size in your graylog2-web-interface.
-mq_batch_size = 4000
-mq_poll_freq = 1
-
-# You can set a maximum size of the message queue. If this size is reached, all new messages will be rejected until messages are removed/indexed from the queue.
-# 0 = unlimited queue size (default)
-mq_max_size = 0
-
 # Raise this according to the maximum connections your MongoDB server can handle if you encounter MongoDB connection problems.
 mongodb_max_connections = 100
 
@@ -45,23 +134,48 @@ mongodb_max_connections = 100
 # http://api.mongodb.org/java/current/com/mongodb/MongoOptions.html#threadsAllowedToBlockForConnectionMultiplier
 mongodb_threads_allowed_to_block_multiplier = 5
 
-# Graylog Extended Log Format (GELF)
-use_gelf = true
-gelf_listen_address = 0.0.0.0
-gelf_listen_port = <%= rubber_env.graylog_server_port %>
 
 # Drools Rule File (Use to rewrite incoming log messages)
-# rules_file = /etc/graylog2.d/rules/graylog2.drl
-
-# AMQP
-amqp_enabled = false
-amqp_subscribed_queues = somequeue1:gelf,somequeue2:gelf,somequeue3:syslog
-amqp_host = localhost
-amqp_port = 5672
-amqp_username = guest
-amqp_password = guest
-amqp_virtualhost = /
-
-# Forwarders
-# Timeout in seconds for each connection and read of Logg.ly API when forwarding messages. Default: 3
-forwarder_loggly_timeout = 3
+# See: http://support.torch.sh/help/kb/graylog2-server/custom-message-rewritingprocessing
+# rules_file = /etc/graylog2.drl
+
+# Email transport
+transport_email_enabled = false
+transport_email_protocol = smtp
+transport_email_hostname = mail.example.com
+transport_email_port = 587
+transport_email_use_auth = true
+transport_email_use_tls = true
+transport_email_auth_username = you@example.com
+transport_email_auth_password = secret
+transport_email_subject_prefix = [graylog2]
+transport_email_from_email = graylog2@example.com
+transport_email_from_name = Graylog2
+transport_email_web_interface_url = http://your-graylog2.example.org
+
+# Jabber/XMPP transport
+transport_jabber_enabled = false
+transport_jabber_hostname = jabber.example.com
+transport_jabber_port = 5222
+transport_jabber_use_sasl_auth = true
+transport_jabber_allow_selfsigned_certs = false
+transport_jabber_auth_username = your_user
+transport_jabber_auth_password = secret
+transport_jabber_message_prefix = [graylog2]
+
+# Additional modules
+# Graphite
+#enable_graphite_output = false
+#graphite_carbon_host = 127.0.0.1
+#graphite_carbon_tcp_port = 2003
+#graphite_prefix = logs
+
+# Librato Metrics (http://support.torch.sh/help/kb/graylog2-server/using-librato-metrics-with-graylog2)
+#enable_libratometrics_output = false
+#enable_libratometrics_system_metrics = false
+#libratometrics_api_user = you@example.com
+#libratometrics_api_token = abcdefg12345
+#libratometrics_prefix = gl2-
+#libratometrics_interval = 60
+#libratometrics_stream_filter =
+#libratometrics_host_filter =